Handle GVL deletions

docs/config-app.md

@@ -457,6 +457,8 @@ If not defined in config all other Health Checkers would be disabled and endpoin
 - `gdpr.special-features.sfN.vendor-exceptions[]` - bidder names that will be treated opposite to `sfN.enforce` value.
 - `gdpr.purpose-one-treatment-interpretation` - option that allows to skip the Purpose one enforcement workflow.
 - `gdpr.vendorlist.default-timeout-ms` - default operation timeout for obtaining new vendor list.
+- `gdpr.vendorlist.live-gvl-url` - URL of the latest TCF GVL used to detect vendors with a past `deletedDate`. Default `https://vendor-list.consensu.org/v3/vendor-list.json`.
+- `gdpr.vendorlist.live-gvl-refresh-period-ms` - how often to refresh the live GVL deleted-vendor set, in milliseconds. Default `86400000` (24 hours).
 - `gdpr.vendorlist.v2.http-endpoint-template` - template string for vendor list url version 2.
 - `gdpr.vendorlist.v2.refresh-missing-list-period-ms` - time to wait between attempts to fetch vendor list version that previously was reported to be missing by origin. Default `3600000` (one hour).
 - `gdpr.vendorlist.v2.fallback-vendor-list-path` - location on the file system of the fallback vendor list that will be used in place of missing vendor list versions. Optional.

docs/metrics.md

@@ -131,6 +131,7 @@ Following metrics are collected and submitted if account is configured with `det
 - `privacy.tcf.(v1,v2).in-geo` - number of requests received from TCF-concerned geo region with consent string of particular version 
 - `privacy.tcf.(v1,v2).out-geo` - number of requests received outside of TCF-concerned geo region with consent string of particular version
 - `privacy.tcf.(v1,v2).vendorlist.(missing|ok|err|fallback)` - number of processed vendor lists of particular version
+- `privacy.tcf.vendorlist.latest.(ok|err)` - number of successful or failed refreshes of the live GVL used for deleted-vendor detection
 - `privacy.usp.specified` - number of requests with a valid US Privacy string (CCPA)
 - `privacy.usp.opt-out` - number of requests that required privacy enforcement according to CCPA rules
 - `privacy.lmt` - number of requests that required privacy enforcement according to LMT flag

src/main/java/org/prebid/server/metric/Metrics.java

@@ -554,6 +554,14 @@ public void updatePrivacyTcfVendorListFallbackMetric(int version) {
         updatePrivacyTcfVendorListMetric(version, MetricName.fallback);
     }
 
+    public void updatePrivacyTcfVendorListLatestOkMetric() {
+        privacy().tcf().vendorListLatest().incCounter(MetricName.ok);
+    }
+
+    public void updatePrivacyTcfVendorListLatestErrorMetric() {
+        privacy().tcf().vendorListLatest().incCounter(MetricName.err);
+    }
+
     private void updatePrivacyTcfVendorListMetric(int version, MetricName metricName) {
         final TcfMetrics tcfMetrics = privacy().tcf();
         tcfMetrics.fromVersion(version).vendorList().incCounter(metricName);

src/main/java/org/prebid/server/metric/TcfMetrics.java

@@ -16,6 +16,7 @@ class TcfMetrics extends UpdatableMetrics {
 
     private final TcfVersionMetrics tcfVersion1Metrics;
     private final TcfVersionMetrics tcfVersion2Metrics;
+    private final VendorListLatestMetrics vendorListLatestMetrics;
 
     TcfMetrics(MetricRegistry metricRegistry, CounterType counterType, String prefix) {
         super(
@@ -25,6 +26,7 @@ class TcfMetrics extends UpdatableMetrics {
 
         tcfVersion1Metrics = new TcfVersionMetrics(metricRegistry, counterType, createTcfPrefix(prefix), "v1");
         tcfVersion2Metrics = new TcfVersionMetrics(metricRegistry, counterType, createTcfPrefix(prefix), "v2");
+        vendorListLatestMetrics = new VendorListLatestMetrics(metricRegistry, counterType, createTcfPrefix(prefix));
     }
 
     TcfVersionMetrics fromVersion(int version) {
@@ -35,6 +37,10 @@ TcfVersionMetrics fromVersion(int version) {
         };
     }
 
+    VendorListLatestMetrics vendorListLatest() {
+        return vendorListLatestMetrics;
+    }
+
     private static String createTcfPrefix(String prefix) {
         return prefix + ".tcf";
     }
@@ -87,4 +93,22 @@ private static Function<MetricName, String> nameCreator(String prefix) {
             return metricName -> "%s.%s".formatted(prefix, metricName);
         }
     }
+
+    static class VendorListLatestMetrics extends UpdatableMetrics {
+
+        VendorListLatestMetrics(MetricRegistry metricRegistry, CounterType counterType, String prefix) {
+            super(
+                    metricRegistry,
+                    counterType,
+                    nameCreator(createLatestPrefix(prefix)));
+        }
+
+        private static String createLatestPrefix(String prefix) {
+            return prefix + ".vendorlist.latest";
+        }
+
+        private static Function<MetricName, String> nameCreator(String prefix) {
+            return metricName -> "%s.%s".formatted(prefix, metricName);
+        }
+    }
 }

src/main/java/org/prebid/server/privacy/gdpr/vendorlist/LiveVendorListService.java

@@ -0,0 +1,131 @@
+package org.prebid.server.privacy.gdpr.vendorlist;
+
+import io.vertx.core.Promise;
+import io.vertx.core.Vertx;
+import org.prebid.server.exception.PreBidException;
+import org.prebid.server.json.JacksonMapper;
+import org.prebid.server.log.Logger;
+import org.prebid.server.log.LoggerFactory;
+import org.prebid.server.metric.Metrics;
+import org.prebid.server.privacy.gdpr.vendorlist.proto.Vendor;
+import org.prebid.server.privacy.gdpr.vendorlist.proto.VendorList;
+import org.prebid.server.util.HttpUtil;
+import org.prebid.server.vertx.Initializable;
+import org.prebid.server.vertx.httpclient.HttpClient;
+import org.prebid.server.vertx.httpclient.model.HttpClientResponse;
+
+import java.time.Clock;
+import java.time.Instant;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+public class LiveVendorListService implements Initializable {
+
+    private static final Logger logger = LoggerFactory.getLogger(LiveVendorListService.class);
+
+    private final String cacheDir;
+    private final String liveGvlUrl;
+    private final long refreshPeriodMs;
+    private final int defaultTimeoutMs;
+    private final Vertx vertx;
+    private final HttpClient httpClient;
+    private final VendorListFileStore vendorListFileStore;
+    private final Metrics metrics;
+    private final JacksonMapper mapper;
+    private final Clock clock;
+
+    private volatile Set<Integer> deletedVendorIds = Set.of();
+
+    public LiveVendorListService(String cacheDir,
+                                 String liveGvlUrl,
+                                 long refreshPeriodMs,
+                                 int defaultTimeoutMs,
+                                 Vertx vertx,
+                                 HttpClient httpClient,
+                                 VendorListFileStore vendorListFileStore,
+                                 Metrics metrics,
+                                 JacksonMapper mapper,
+                                 Clock clock) {
+
+        this.cacheDir = Objects.requireNonNull(cacheDir);
+        this.liveGvlUrl = HttpUtil.validateUrl(Objects.requireNonNull(liveGvlUrl));
+        this.refreshPeriodMs = refreshPeriodMs;
+        this.defaultTimeoutMs = defaultTimeoutMs;
+        this.vertx = Objects.requireNonNull(vertx);
+        this.httpClient = Objects.requireNonNull(httpClient);
+        this.vendorListFileStore = Objects.requireNonNull(vendorListFileStore);
+        this.metrics = Objects.requireNonNull(metrics);
+        this.mapper = Objects.requireNonNull(mapper);
+        this.clock = Objects.requireNonNull(clock);
+    }
+
+    public boolean isDeleted(Integer id) {
+        final Set<Integer> ids = deletedVendorIds;
+        return !ids.isEmpty() && ids.contains(id);
+    }
+
+    @Override
+    public void initialize(Promise<Void> initializePromise) {
+        initializeWithLatestCachedVersion();
+        vertx.setPeriodic(0, refreshPeriodMs, ignored -> refresh());
+
+        initializePromise.tryComplete();
+    }
+
+    private void initializeWithLatestCachedVersion() {
+        vendorListFileStore.getLatestVendorListFromCache(cacheDir).ifPresent(vendorList -> {
+            saveDeletedVendorsFromVendorList(vendorList);
+            logger.info("Initialized live GVL from cache with version %d".formatted(vendorList.getVendorListVersion()));
+        });
+    }
+
+    void refresh() {
+        httpClient.get(liveGvlUrl, defaultTimeoutMs)
+                .map(this::processResponse)
+                .map(this::saveDeletedVendorsFromVendorList)
+                .otherwise(this::handleError);
+    }
+
+    private Void saveDeletedVendorsFromVendorList(VendorList vendorList) {
+        updateDeletedVendorIds(extractDeletedVendorIds(vendorList));
+        return null;
+    }
+
+    private VendorList processResponse(HttpClientResponse response) {
+        final int statusCode = response.getStatusCode();
+        if (statusCode != 200) {
+            throw new PreBidException("HTTP status code " + statusCode);
+        }
+
+        final String body = response.getBody();
+        final VendorList vendorList = VendorListUtil.parseVendorList(body, mapper);
+
+        if (!VendorListUtil.vendorListIsValid(vendorList)) {
+            throw new PreBidException("Fetched vendor list parsed but has invalid data: " + body);
+        }
+
+        return vendorList;
+    }
+
+    Set<Integer> extractDeletedVendorIds(VendorList vendorList) {
+        final Instant now = clock.instant();
+        return vendorList.getVendors().values().stream()
+                .filter(vendor -> VendorListUtil.vendorIsDeletedAt(vendor, now))
+                .map(Vendor::getId)
+                .filter(Objects::nonNull)
+                .collect(Collectors.toUnmodifiableSet());
+    }
+
+    private Void updateDeletedVendorIds(Set<Integer> ids) {
+        deletedVendorIds = ids;
+        metrics.updatePrivacyTcfVendorListLatestOkMetric();
+        return null;
+    }
+
+    private Void handleError(Throwable exception) {
+        logger.warn("Error occurred while fetching live GVL", exception);
+        metrics.updatePrivacyTcfVendorListLatestErrorMetric();
+        return null;
+    }
+}

src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListFileStore.java

@@ -0,0 +1,134 @@
+package org.prebid.server.privacy.gdpr.vendorlist;
+
+import com.github.benmanes.caffeine.cache.Caffeine;
+import io.vertx.core.Future;
+import io.vertx.core.Promise;
+import io.vertx.core.buffer.Buffer;
+import io.vertx.core.file.FileProps;
+import io.vertx.core.file.FileSystem;
+import io.vertx.core.file.FileSystemException;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.exception.ExceptionUtils;
+import org.prebid.server.exception.PreBidException;
+import org.prebid.server.json.JacksonMapper;
+import org.prebid.server.log.ConditionalLogger;
+import org.prebid.server.log.Logger;
+import org.prebid.server.log.LoggerFactory;
+import org.prebid.server.privacy.gdpr.vendorlist.proto.Vendor;
+import org.prebid.server.privacy.gdpr.vendorlist.proto.VendorList;
+
+import java.io.File;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.Comparator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+public class VendorListFileStore {
+
+    private static final Logger logger = LoggerFactory.getLogger(VendorListFileStore.class);
+    private static final ConditionalLogger conditionalLogger = new ConditionalLogger(logger);
+
+    private static final String JSON_SUFFIX = ".json";
+
+    private final double logSamplingRate;
+    private final FileSystem fileSystem;
+    private final JacksonMapper mapper;
+
+    public VendorListFileStore(double logSamplingRate,
+                               FileSystem fileSystem,
+                               JacksonMapper mapper) {
+
+        this.logSamplingRate = logSamplingRate;
+        this.fileSystem = Objects.requireNonNull(fileSystem);
+        this.mapper = Objects.requireNonNull(mapper);
+    }
+
+    Map<Integer, Map<Integer, Vendor>> createCacheFromDisk(String cacheDir) {
+        createAndCheckWritePermissionsForCacheDir(cacheDir);
+        final Map<Integer, String> versionToFileContent = readFileSystemCache(cacheDir);
+
+        final Map<Integer, Map<Integer, Vendor>> cache = Caffeine.newBuilder()
+                .<Integer, Map<Integer, Vendor>>build()
+                .asMap();
+
+        for (Map.Entry<Integer, String> versionAndFileContent : versionToFileContent.entrySet()) {
+            final VendorList vendorList = VendorListUtil.parseVendorList(versionAndFileContent.getValue(), mapper);
+
+            cache.put(versionAndFileContent.getKey(), vendorList.getVendors());
+        }
+        return cache;
+    }
+
+    private void createAndCheckWritePermissionsForCacheDir(String cacheDir) {
+        final FileProps props = fileSystem.existsBlocking(cacheDir) ? fileSystem.propsBlocking(cacheDir) : null;
+        if (props == null || !props.isDirectory()) {
+            try {
+                fileSystem.mkdirsBlocking(cacheDir);
+            } catch (FileSystemException e) {
+                throw new PreBidException("Cannot create directory: " + cacheDir, e);
+            }
+        } else if (!Files.isWritable(Paths.get(cacheDir))) {
+            throw new PreBidException("No write permissions for directory: " + cacheDir);
+        }
+    }
+
+    private Map<Integer, String> readFileSystemCache(String cacheDir) {
+        return fileSystem.readDirBlocking(cacheDir).stream()
+                .filter(filepath -> filepath.endsWith(JSON_SUFFIX))
+                .collect(Collectors.toMap(VendorListFileStore::parseCachedFileVersion,
+                        filename -> fileSystem.readFileBlocking(filename).toString()));
+    }
+
+    Optional<VendorList> getLatestVendorListFromCache(String cacheDir) {
+        createAndCheckWritePermissionsForCacheDir(cacheDir);
+        return fileSystem.readDirBlocking(cacheDir).stream()
+                .filter(filepath -> filepath.endsWith(JSON_SUFFIX))
+                .max(Comparator.comparing(VendorListFileStore::parseCachedFileVersion))
+                .map(fileSystem::readFileBlocking)
+                .map(Buffer::toString)
+                .map(content -> VendorListUtil.parseVendorList(content, mapper));
+    }
+
+    private static Integer parseCachedFileVersion(String filepath) {
+        final String filename = new File(filepath).getName();
+        final String filenameWithoutExtension = StringUtils.removeEnd(filename, JSON_SUFFIX);
+        return Integer.valueOf(filenameWithoutExtension);
+    }
+
+    Future<VendorListResult> saveToFile(VendorListResult vendorListResult, String cacheDir, String generationVersion) {
+        final Promise<VendorListResult> promise = Promise.promise();
+        final int version = vendorListResult.getVersion();
+        final String filepath = new File(cacheDir, version + JSON_SUFFIX).getPath();
+
+        fileSystem.writeFile(filepath, Buffer.buffer(vendorListResult.getVendorListAsString()), result -> {
+            if (result.succeeded()) {
+                promise.complete(vendorListResult);
+            } else {
+                conditionalLogger.error(
+                        "Could not create new vendor list for version %s.%s, file: %s, trace: %s".formatted(
+                                generationVersion, version, filepath, ExceptionUtils.getStackTrace(result.cause())),
+                        logSamplingRate);
+                promise.fail(result.cause());
+            }
+        });
+
+        return promise.future();
+    }
+
+    Map<Integer, Vendor> readFallbackVendorList(String fallbackVendorListPath) {
+        if (StringUtils.isBlank(fallbackVendorListPath)) {
+            return null;
+        }
+
+        final String vendorListContent = fileSystem.readFileBlocking(fallbackVendorListPath).toString();
+        final VendorList vendorList = VendorListUtil.parseVendorList(vendorListContent, mapper);
+        if (!VendorListUtil.vendorListIsValid(vendorList)) {
+            throw new PreBidException("Fallback vendor list parsed but has invalid data: " + vendorListContent);
+        }
+
+        return vendorList.getVendors();
+    }
+}

src/main/java/org/prebid/server/privacy/gdpr/vendorlist/VendorListResult.java

@@ -0,0 +1,14 @@
+package org.prebid.server.privacy.gdpr.vendorlist;
+
+import lombok.Value;
+import org.prebid.server.privacy.gdpr.vendorlist.proto.VendorList;
+
+@Value(staticConstructor = "of")
+class VendorListResult {
+
+    int version;
+
+    String vendorListAsString;
+
+    VendorList vendorList;
+}