Record geo lookup results in auction request and context for subsequent usage (prebid#888)

* Divide privacy data extraction (from request) and usage into separate steps

Enrich bid request with country and possibly masked ip address coming from privacy enforcement`

Move request type metric detection to *RequestFactory classes

Uncomment test for further fixing

Fix broken tests

* Tests for new functionality

* Misc cleanup

Author: schernysh

docs/endpoints/event.md

@@ -19,4 +19,4 @@ This endpoint is used to notify about event and do request for tracking pixel if
 - `x` : Disables or enables analytics. Allowed values: `1` to enable analytics or `0` to disable. `1` is default.
 ### Sample request
 
-`GET http://prebid.site.com/event?type=win&bidid=12345&a=1111&bidder=rubicon&f=b`
+`GET http://prebid.site.com/event?type=win&bidid=12345&a=1111&bidder=rubicon&f=b`

src/main/java/com/iab/openrtb/request/Data.java

@@ -1,7 +1,7 @@
 package com.iab.openrtb.request;
 
 import com.fasterxml.jackson.databind.node.ObjectNode;
-import lombok.AllArgsConstructor;
+import lombok.Builder;
 import lombok.Value;
 
 import java.util.List;
@@ -14,14 +14,18 @@
  * The specific data providers in use should be published by the exchange
  * <em>a priori</em> to its bidders.
  */
+@Builder
 @Value
-@AllArgsConstructor(staticName = "of")
 public class Data {
 
-    /** Exchange-specific ID for the data provider. */
+    /**
+     * Exchange-specific ID for the data provider.
+     */
     String id;
 
-    /** Exchange-specific name for the data provider. */
+    /**
+     * Exchange-specific name for the data provider.
+     */
     String name;
 
     /**
@@ -30,6 +34,8 @@ public class Data {
      */
     List<Segment> segment;
 
-    /** Placeholder for exchange-specific extensions to OpenRTB. */
+    /**
+     * Placeholder for exchange-specific extensions to OpenRTB.
+     */
     ObjectNode ext;
 }

src/main/java/com/iab/openrtb/request/Deal.java

@@ -12,14 +12,18 @@
  * indicates that this impression is available under the terms of that deal.
  * Refer to Section 7.3 for more details.
  */
-@Builder
+@Builder(toBuilder = true)
 @Value
 public class Deal {
 
-    /** A unique identifier for the direct deal. (required) */
+    /**
+     * A unique identifier for the direct deal. (required)
+     */
     String id;
 
-    /** Minimum bid for this impression expressed in CPM. */
+    /**
+     * Minimum bid for this impression expressed in CPM.
+     */
     float bidfloor;
 
     /**
@@ -50,6 +54,8 @@ public class Deal {
      */
     List<String> wadomain;
 
-    /** Placeholder for exchange-specific extensions to OpenRTB. */
+    /**
+     * Placeholder for exchange-specific extensions to OpenRTB.
+     */
     ObjectNode ext;
 }

src/main/java/com/iab/openrtb/request/Geo.java

@@ -1,8 +1,8 @@
 package com.iab.openrtb.request;
 
-import com.fasterxml.jackson.databind.node.ObjectNode;
 import lombok.Builder;
 import lombok.Value;
+import org.prebid.server.proto.openrtb.ext.request.ExtGeo;
 
 /**
  * This object encapsulates various methods for specifying a geographic location.
@@ -98,5 +98,5 @@ public class Geo {
     /**
      * Placeholder for exchange-specific extensions to OpenRTB.
      */
-    ObjectNode ext;
+    ExtGeo ext;
 }

src/main/java/com/iab/openrtb/request/Segment.java

@@ -1,6 +1,7 @@
 package com.iab.openrtb.request;
 
 import com.fasterxml.jackson.databind.node.ObjectNode;
+import lombok.Builder;
 import lombok.Value;
 
 /**
@@ -10,6 +11,7 @@
  * The specific segment names and value options must be published by the
  * exchange <em>a priori</em> to its bidders.
  */
+@Builder
 @Value
 public class Segment {
 

src/main/java/com/iab/openrtb/response/BidResponse.java

@@ -6,6 +6,7 @@
 
 import java.util.Comparator;
 import java.util.List;
+import java.util.Objects;
 
 /**
  * This object is the top-level bid response object (i.e., the unnamed outer
@@ -63,22 +64,13 @@ public class BidResponse {
     ObjectNode ext;
 
     public static final Comparator<BidResponse> COMPARATOR = (left, right) -> {
-        if (left == null) {
+        if (Objects.isNull(left)
+                || left.getSeatbid().isEmpty()
+                || left.getSeatbid().get(0).getBid().isEmpty()
+                || Objects.isNull(right)) {
             return -1;
         }
-        if (left.getSeatbid().isEmpty()) {
-            return -1;
-        }
-        if (left.getSeatbid().get(0).getBid().isEmpty()) {
-            return -1;
-        }
-        if (right == null) {
-            return -1;
-        }
-        if (right.getSeatbid().isEmpty()) {
-            return 1;
-        }
-        if (right.getSeatbid().get(0).getBid().isEmpty()) {
+        if (right.getSeatbid().isEmpty() || right.getSeatbid().get(0).getBid().isEmpty()) {
             return 1;
         }
         return left.getSeatbid().get(0).getBid().get(0).getPrice()

src/main/java/org/prebid/server/auction/AmpRequestFactory.java

@@ -24,6 +24,7 @@
 import org.prebid.server.auction.model.Tuple2;
 import org.prebid.server.exception.InvalidRequestException;
 import org.prebid.server.json.JacksonMapper;
+import org.prebid.server.metric.MetricName;
 import org.prebid.server.privacy.ccpa.Ccpa;
 import org.prebid.server.privacy.gdpr.TcfDefinerService;
 import org.prebid.server.proto.openrtb.ext.request.ExtMediaTypePriceGranularity;
@@ -105,9 +106,13 @@ public Future<AuctionContext> fromRequest(RoutingContext routingContext, long st
             return Future.failedFuture(new InvalidRequestException("AMP requests require an AMP tag_id"));
         }
         return createBidRequest(routingContext, tagId)
-                .compose(bidRequestWithErrors ->
-                        auctionRequestFactory.toAuctionContext(routingContext, bidRequestWithErrors.getLeft(),
-                                bidRequestWithErrors.getRight(), startTime, timeoutResolver));
+                .compose(bidRequestWithErrors -> auctionRequestFactory.toAuctionContext(
+                        routingContext,
+                        bidRequestWithErrors.getLeft(),
+                        MetricName.amp,
+                        bidRequestWithErrors.getRight(),
+                        startTime,
+                        timeoutResolver));
     }
 
     /**
@@ -253,7 +258,7 @@ private BidRequest overrideParameters(BidRequest bidRequest, HttpServerRequest r
         String gdprConsent = null;
         String ccpaConsent = null;
         if (StringUtils.isNotBlank(consentString)) {
-            gdprConsent = TcfDefinerService.isGdprConsentValid(consentString) ? consentString : null;
+            gdprConsent = TcfDefinerService.isConsentStringValid(consentString) ? consentString : null;
             ccpaConsent = Ccpa.isValid(consentString) ? consentString : null;
 
             if (StringUtils.isAllBlank(gdprConsent, ccpaConsent)) {

src/main/java/org/prebid/server/auction/AuctionRequestFactory.java

@@ -6,6 +6,7 @@
 import com.iab.openrtb.request.App;
 import com.iab.openrtb.request.BidRequest;
 import com.iab.openrtb.request.Device;
+import com.iab.openrtb.request.Geo;
 import com.iab.openrtb.request.Imp;
 import com.iab.openrtb.request.Publisher;
 import com.iab.openrtb.request.Site;
@@ -32,9 +33,12 @@
 import org.prebid.server.exception.UnauthorizedAccountException;
 import org.prebid.server.execution.Timeout;
 import org.prebid.server.execution.TimeoutFactory;
+import org.prebid.server.geolocation.model.GeoInfo;
 import org.prebid.server.identity.IdGenerator;
 import org.prebid.server.json.JacksonMapper;
 import org.prebid.server.log.ConditionalLogger;
+import org.prebid.server.metric.MetricName;
+import org.prebid.server.privacy.model.PrivacyContext;
 import org.prebid.server.proto.openrtb.ext.request.ExtMediaTypePriceGranularity;
 import org.prebid.server.proto.openrtb.ext.request.ExtPriceGranularity;
 import org.prebid.server.proto.openrtb.ext.request.ExtPublisher;
@@ -98,6 +102,7 @@ public class AuctionRequestFactory {
     private final TimeoutFactory timeoutFactory;
     private final ApplicationSettings applicationSettings;
     private final IdGenerator idGenerator;
+    private final PrivacyEnforcementService privacyEnforcementService;
     private final JacksonMapper mapper;
     private final OrtbTypesResolver ortbTypesResolver;
 
@@ -119,6 +124,7 @@ public AuctionRequestFactory(long maxRequestSize,
                                  TimeoutFactory timeoutFactory,
                                  ApplicationSettings applicationSettings,
                                  IdGenerator idGenerator,
+                                 PrivacyEnforcementService privacyEnforcementService,
                                  JacksonMapper mapper) {
 
         this.maxRequestSize = maxRequestSize;
@@ -139,6 +145,7 @@ public AuctionRequestFactory(long maxRequestSize,
         this.timeoutFactory = Objects.requireNonNull(timeoutFactory);
         this.applicationSettings = Objects.requireNonNull(applicationSettings);
         this.idGenerator = Objects.requireNonNull(idGenerator);
+        this.privacyEnforcementService = Objects.requireNonNull(privacyEnforcementService);
         this.mapper = Objects.requireNonNull(mapper);
     }
 
@@ -171,9 +178,13 @@ public Future<AuctionContext> fromRequest(RoutingContext routingContext, long st
         }
 
         return updateBidRequest(routingContext, incomingBidRequest)
-                .compose(bidRequest ->
-                        toAuctionContext(routingContext, bidRequest, errors, startTime,
-                                timeoutResolver));
+                .compose(bidRequest -> toAuctionContext(
+                        routingContext,
+                        bidRequest,
+                        requestTypeMetric(bidRequest),
+                        errors,
+                        startTime,
+                        timeoutResolver));
     }
 
     /**
@@ -183,21 +194,28 @@ public Future<AuctionContext> fromRequest(RoutingContext routingContext, long st
      */
     Future<AuctionContext> toAuctionContext(RoutingContext routingContext,
                                             BidRequest bidRequest,
+                                            MetricName requestTypeMetric,
                                             List<String> errors,
                                             long startTime,
                                             TimeoutResolver timeoutResolver) {
 
         final Timeout timeout = timeout(bidRequest, startTime, timeoutResolver);
 
         return accountFrom(bidRequest, timeout, routingContext)
-                .map(account -> AuctionContext.builder()
-                        .routingContext(routingContext)
-                        .uidsCookie(uidsCookieService.parseFromRequest(routingContext))
-                        .bidRequest(enrichBidRequestWithAccountBasedData(bidRequest, account))
-                        .timeout(timeout)
-                        .account(account)
-                        .prebidErrors(errors)
-                        .build());
+                .compose(account -> privacyEnforcementService.contextFromBidRequest(
+                        bidRequest, account, requestTypeMetric, timeout)
+                        .map(privacyContext -> AuctionContext.builder()
+                                .routingContext(routingContext)
+                                .uidsCookie(uidsCookieService.parseFromRequest(routingContext))
+                                .bidRequest(enrichBidRequestWithAccountAndPrivacyData(
+                                        bidRequest, account, privacyContext))
+                                .requestTypeMetric(requestTypeMetric)
+                                .timeout(timeout)
+                                .account(account)
+                                .prebidErrors(errors)
+                                .privacyContext(privacyContext)
+                                .geoInfo(privacyContext.getTcfContext().getGeoInfo())
+                                .build()));
     }
 
     /**
@@ -430,14 +448,13 @@ private User populateUser(User user) {
         final ExtUser ext = userExtOrNull(user);
 
         if (ext != null) {
-            final User.UserBuilder builder = user == null ? User.builder() : user.toBuilder();
-            return builder.ext(ext).build();
+            return user.toBuilder().ext(ext).build();
         }
         return null;
     }
 
     /**
-     * Returns {@link ObjectNode} of updated {@link ExtUser} or null if no updates needed.
+     * Returns updated {@link ExtUser} or null if no updates needed.
      */
     private ExtUser userExtOrNull(User user) {
         final ExtUser extUser = user != null ? user.getExt() : null;
@@ -571,7 +588,7 @@ private ExtRequestTargeting targetingOrNull(ExtRequestPrebid prebid, Set<BidType
                     .pricegranularity(populatePriceGranularity(targeting, isPriceGranularityNull,
                             isPriceGranularityTextual, impMediaTypes))
                     .mediatypepricegranularity(targeting.getMediatypepricegranularity())
-                    .includewinners(isIncludeWinnersNull ? true : targeting.getIncludewinners())
+                    .includewinners(isIncludeWinnersNull || targeting.getIncludewinners())
                     .includebidderkeys(isIncludeBidderKeysNull
                             ? !isWinningOnly(prebid.getCache())
                             : targeting.getIncludebidderkeys())
@@ -838,13 +855,19 @@ private Future<Account> responseForUnknownAccount(String accountId) {
                 : Future.succeededFuture(Account.empty(accountId));
     }
 
-    private BidRequest enrichBidRequestWithAccountBasedData(BidRequest bidRequest, Account account) {
+    private BidRequest enrichBidRequestWithAccountAndPrivacyData(
+            BidRequest bidRequest, Account account, PrivacyContext privacyContext) {
+
         final ExtRequest requestExt = bidRequest.getExt();
         final ExtRequest enrichedRequestExt = enrichExtRequest(requestExt, account);
 
-        if (enrichedRequestExt != null) {
+        final Device device = bidRequest.getDevice();
+        final Device enrichedDevice = enrichDevice(device, privacyContext);
+
+        if (enrichedRequestExt != null || enrichedDevice != null) {
             return bidRequest.toBuilder()
-                    .ext(enrichedRequestExt)
+                    .ext(ObjectUtils.defaultIfNull(enrichedRequestExt, requestExt))
+                    .device(ObjectUtils.defaultIfNull(enrichedDevice, device))
                     .build();
         }
 
@@ -867,4 +890,39 @@ private ExtRequest enrichExtRequest(ExtRequest ext, Account account) {
 
         return null;
     }
+
+    private Device enrichDevice(Device device, PrivacyContext privacyContext) {
+        final String ipAddress = privacyContext.getIpAddress();
+        final String country = getIfNotNull(privacyContext.getTcfContext().getGeoInfo(), GeoInfo::getCountry);
+
+        final String ipAddressInRequest = getIfNotNull(device, Device::getIp);
+
+        final Geo geo = getIfNotNull(device, Device::getGeo);
+        final String countryFromRequest = getIfNotNull(geo, Geo::getCountry);
+
+        final boolean shouldUpdateIp = ipAddress != null && !Objects.equals(ipAddressInRequest, ipAddress);
+        final boolean shouldUpdateCountry = country != null && !Objects.equals(countryFromRequest, country);
+
+        if (shouldUpdateIp || shouldUpdateCountry) {
+            final Device.DeviceBuilder deviceBuilder = device != null ? device.toBuilder() : Device.builder();
+
+            if (shouldUpdateIp) {
+                deviceBuilder.ip(ipAddress);
+            }
+
+            if (shouldUpdateCountry) {
+                final Geo.GeoBuilder geoBuilder = geo != null ? geo.toBuilder() : Geo.builder();
+                geoBuilder.country(country);
+                deviceBuilder.geo(geoBuilder.build());
+            }
+
+            return deviceBuilder.build();
+        }
+
+        return null;
+    }
+
+    private static MetricName requestTypeMetric(BidRequest bidRequest) {
+        return bidRequest.getApp() != null ? MetricName.openrtb2app : MetricName.openrtb2web;
+    }
 }

src/main/java/org/prebid/server/auction/BidResponseCreator.java

@@ -73,6 +73,7 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
+import java.util.EnumMap;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
@@ -265,6 +266,7 @@ private ExtBidResponse toExtBidResponse(List<BidderResponse> bidderResponses,
                                             Map<String, List<ExtBidderError>> bidErrors) {
 
         final BidRequest bidRequest = auctionContext.getBidRequest();
+
         final ExtResponseDebug extResponseDebug = debugEnabled
                 ? ExtResponseDebug.of(toExtHttpCalls(bidderResponses, cacheResult), bidRequest)
                 : null;
@@ -985,7 +987,7 @@ private Map<BidType, TargetingKeywordsCreator> keywordsCreatorByBidType(ExtReque
             return Collections.emptyMap();
         }
 
-        final Map<BidType, TargetingKeywordsCreator> result = new HashMap<>();
+        final Map<BidType, TargetingKeywordsCreator> result = new EnumMap<>(BidType.class);
         final int resolvedTruncateAttrChars = resolveTruncateAttrChars(targeting, account);
 
         final ObjectNode banner = mediaTypePriceGranularity.getBanner();