Upstream PR #4700: GVL: Periodically fetch latest valid IDs indicating deletions (#4700)

config/config.go

@@ -275,6 +275,7 @@ type GDPR struct {
 	NonStandardPublisherMap map[string]struct{}
 	TCF2                    TCF2 `mapstructure:"tcf2"`
 	AMPException            bool `mapstructure:"amp_exception"` // Deprecated: Use account-level GDPR settings (gdpr.integration_enabled.amp) instead
+	LiveGVLRefreshInterval  int  `mapstructure:"live_gvl_refresh_interval_seconds"`
 	// EEACountries (EEA = European Economic Area) are a list of countries where we should assume GDPR applies.
 	// If the gdpr flag is unset in a request, but geo.country is set, we will assume GDPR applies if and only
 	// if the country matches one on this list. If both the GDPR flag and country are not set, we default
@@ -1237,6 +1238,7 @@ func SetupViper(v *viper.Viper, filename string, bidderInfos BidderInfos) {
 	v.SetDefault("gdpr.timeouts_ms.init_vendorlist_fetches", 0)
 	v.SetDefault("gdpr.timeouts_ms.active_vendorlist_fetch", 0)
 	v.SetDefault("gdpr.non_standard_publishers", []string{""})
+	v.SetDefault("gdpr.live_gvl_refresh_interval_seconds", 86400) // 1 day
 	v.SetDefault("gdpr.tcf2.enabled", true)
 	v.SetDefault("gdpr.tcf2.purpose1.enforce_vendors", true)
 	v.SetDefault("gdpr.tcf2.purpose2.enforce_vendors", true)

config/config_test.go

@@ -254,6 +254,7 @@
 	cmpInts(t, "analytics.agma.buffers.count", 100, cfg.Analytics.Agma.Buffers.EventCount)
 	cmpStrings(t, "analytics.agma.buffers.timeout", "15m", cfg.Analytics.Agma.Buffers.Timeout)
 	cmpInts(t, "analytics.agma.accounts", 0, len(cfg.Analytics.Agma.Accounts))
+	cmpInts(t, "gdpr.live_gvl_refresh_interval_seconds", 86400, cfg.GDPR.LiveGVLRefreshInterval)
 	expectedTCF2 := TCF2{
 		Enabled: true,
 		Purpose1: TCF2Purpose{
@@ -386,6 +387,7 @@
   default_value: "1"
   non_standard_publishers: ["pub1", "pub2"]
   eea_countries: ["eea1", "eea2"]
+  live_gvl_refresh_interval_seconds: 3600
   tcf2:
     purpose1:
       enforce_vendors: false
@@ -686,6 +688,8 @@
 	cmpInts(t, "http_client_cache.idle_connection_timeout_seconds", 3, cfg.CacheClient.IdleConnTimeout)
 	cmpInts(t, "gdpr.host_vendor_id", 15, cfg.GDPR.HostVendorID)
 	cmpStrings(t, "gdpr.default_value", "1", cfg.GDPR.DefaultValue)
+	cmpInts(t, "gdpr.live_gvl_refresh_interval_seconds", 3600, cfg.GDPR.LiveGVLRefreshInterval)
+	cmpBools(t, "video.enable_deprecated_endpoint", true, cfg.Video.EnableDeprecatedEndpoint)
 	cmpStrings(t, "host_schain_node.asi", "pbshostcompany.com", cfg.HostSChainNode.ASI)
 	cmpStrings(t, "host_schain_node.sid", "00001", cfg.HostSChainNode.SID)
 	cmpStrings(t, "host_schain_node.rid", "BidRequest", cfg.HostSChainNode.RID)

gdpr/gdpr.go

@@ -35,16 +35,16 @@ type RequestInfo struct {
 }
 
 // NewPermissionsBuilder takes host config data used to configure the builder function it returns
-func NewPermissionsBuilder(cfg config.GDPR, gvlVendorIDs map[openrtb_ext.BidderName]uint16, vendorListFetcher VendorListFetcher, me metrics.MetricsEngine) PermissionsBuilder {
+func NewPermissionsBuilder(cfg config.GDPR, gvlVendorIDs map[openrtb_ext.BidderName]uint16, validGVLVendorIDs *LiveGVLVendorIDs, vendorListFetcher VendorListFetcher, me metrics.MetricsEngine) PermissionsBuilder {
 	return func(tcf2Cfg TCF2ConfigReader, requestInfo RequestInfo) Permissions {
 		purposeEnforcerBuilder := NewPurposeEnforcerBuilder(tcf2Cfg)
 
-		return NewPermissions(cfg, tcf2Cfg, gvlVendorIDs, vendorListFetcher, purposeEnforcerBuilder, requestInfo, me)
+		return NewPermissions(cfg, tcf2Cfg, gvlVendorIDs, validGVLVendorIDs, vendorListFetcher, purposeEnforcerBuilder, requestInfo, me)
 	}
 }
 
 // NewPermissions gets a per-request Permissions object that can then be used to check GDPR permissions for a given bidder.
-func NewPermissions(cfg config.GDPR, tcf2Config TCF2ConfigReader, vendorIDs map[openrtb_ext.BidderName]uint16, fetcher VendorListFetcher, purposeEnforcerBuilder PurposeEnforcerBuilder, requestInfo RequestInfo, metricsEngine metrics.MetricsEngine) Permissions {
+func NewPermissions(cfg config.GDPR, tcf2Config TCF2ConfigReader, vendorIDs map[openrtb_ext.BidderName]uint16, validGVLVendorIDs *LiveGVLVendorIDs, fetcher VendorListFetcher, purposeEnforcerBuilder PurposeEnforcerBuilder, requestInfo RequestInfo, metricsEngine metrics.MetricsEngine) Permissions {
 	if !cfg.Enabled {
 		return &AlwaysAllow{}
 	}
@@ -55,6 +55,7 @@ func NewPermissions(cfg config.GDPR, tcf2Config TCF2ConfigReader, vendorIDs map[
 		hostVendorID:           cfg.HostVendorID,
 		nonStandardPublishers:  cfg.NonStandardPublisherMap,
 		cfg:                    tcf2Config,
+		validGVLVendorIDs:      validGVLVendorIDs,
 		vendorIDs:              vendorIDs,
 		publisherID:            requestInfo.PublisherID,
 		gdprSignal:             SignalNormalize(requestInfo.GDPRSignal, cfg.DefaultValue),

gdpr/gdpr_test.go

@@ -51,7 +51,7 @@ func TestNewPermissions(t *testing.T) {
 		fakePurposeEnforcerBuilder := fakePurposeEnforcerBuilder{
 			purposeEnforcer: nil,
 		}.Builder
-		perms := NewPermissions(config, &tcf2Config{}, vendorIDs, vendorListFetcher, fakePurposeEnforcerBuilder, RequestInfo{}, &metrics.MetricsEngineMock{})
+		perms := NewPermissions(config, &tcf2Config{}, vendorIDs, nil, vendorListFetcher, fakePurposeEnforcerBuilder, RequestInfo{}, &metrics.MetricsEngineMock{})
 
 		assert.IsType(t, tt.wantType, perms, tt.description)
 	}

gdpr/gvl_vendor_ids.go

@@ -0,0 +1,119 @@
+package gdpr
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"sync/atomic"
+	"time"
+
+	"github.com/prebid/prebid-server/v3/logger"
+	"github.com/prebid/prebid-server/v3/metrics"
+	"github.com/prebid/prebid-server/v3/util/task"
+	"golang.org/x/net/context/ctxhttp"
+)
+
+// LiveGVLVendorIDs provides thread-safe access to the set of vendor IDs present in the latest
+// Global Vendor List. It uses atomic.Value so reads (on every request) are lock-free, while
+// writes (periodic refresh) are safe without mutexes.
+type LiveGVLVendorIDs struct {
+	ids atomic.Value // holds map[uint16]struct{}
+}
+
+// NewLiveGVLVendorIDs creates a LiveGVLVendorIDs initialized with an empty set.
+func NewLiveGVLVendorIDs() *LiveGVLVendorIDs {
+	l := &LiveGVLVendorIDs{}
+	l.ids.Store(make(map[uint16]struct{}))
+	return l
+}
+
+// Contains returns true if the given vendor ID is in the current valid set.
+// If the set is empty (e.g. the GVL fetch failed), all IDs are considered valid
+// as a safe fallback.
+func (l *LiveGVLVendorIDs) Contains(id uint16) bool {
+	m := l.ids.Load().(map[uint16]struct{})
+	if len(m) == 0 {
+		return true
+	}
+	_, ok := m[id]
+	return ok
+}
+
+// Update atomically replaces the valid vendor ID set. If newIDs is empty, the existing
+// set is retained.
+func (l *LiveGVLVendorIDs) Update(newIDs map[uint16]struct{}) {
+	if len(newIDs) > 0 {
+		l.ids.Store(newIDs)
+	}
+}
+
+// NewGVLVendorIDTickerTask creates a TickerTask that fetches the latest GVL vendor IDs and
+// updates the LiveGVLVendorIDs set. Calling Start on the returned task performs the initial
+// fetch immediately and then schedules periodic refreshes at the given interval.
+func NewGVLVendorIDTickerTask(interval time.Duration, client *http.Client, urlMaker func(uint16, uint16) string, live *LiveGVLVendorIDs, me metrics.MetricsEngine) *task.TickerTask {
+	return task.NewTickerTaskFromFunc(interval, func() error {
+		newIDs := FetchLatestGVLVendorIDs(context.Background(), client, urlMaker, me)
+		live.Update(newIDs)
+		return nil
+	})
+}
+
+// gvlVendorListContract is a lightweight contract for parsing only vendor IDs from GVL JSON
+type gvlVendorListContract struct {
+	Vendors map[string]struct {
+		ID uint16 `json:"id"`
+	} `json:"vendors"`
+}
+
+// FetchLatestGVLVendorIDs fetches the most recent Global Vendor List and returns a set of all
+// vendor IDs present in it. The returned map has vendor IDs as keys and empty structs as values.
+// If the fetch or parse fails, an empty map is returned.
+func FetchLatestGVLVendorIDs(ctx context.Context, client *http.Client, urlMaker func(uint16, uint16) string, me metrics.MetricsEngine) map[uint16]struct{} {
+	vendorIDs := make(map[uint16]struct{})
+
+	// Fetch latest GVL for the latest spec version (listVersion 0 means latest)
+	url := urlMaker(latestSpecVersion, 0)
+
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		logger.Errorf("Failed to build GET %s request for GVL vendor ID extraction: %v", url, err)
+		me.RecordLiveGVLFetch(false)
+		return vendorIDs
+	}
+
+	resp, err := ctxhttp.Do(ctx, client, req)
+	if err != nil {
+		logger.Errorf("Error calling GET %s for GVL vendor ID extraction: %v", url, err)
+		me.RecordLiveGVLFetch(false)
+		return vendorIDs
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		logger.Errorf("GET %s returned %d for GVL vendor ID extraction", url, resp.StatusCode)
+		me.RecordLiveGVLFetch(false)
+		return vendorIDs
+	}
+
+	respBody, err := io.ReadAll(resp.Body)
+	if err != nil {
+		logger.Errorf("Error reading response body from GET %s for GVL vendor ID extraction: %v", url, err)
+		me.RecordLiveGVLFetch(false)
+		return vendorIDs
+	}
+
+	var contract gvlVendorListContract
+	if err := json.Unmarshal(respBody, &contract); err != nil {
+		logger.Errorf("GET %s returned malformed JSON for GVL vendor ID extraction: %v", url, err)
+		me.RecordLiveGVLFetch(false)
+		return vendorIDs
+	}
+
+	for _, v := range contract.Vendors {
+		vendorIDs[v.ID] = struct{}{}
+	}
+
+	me.RecordLiveGVLFetch(true)
+	return vendorIDs
+}

gdpr/gvl_vendor_ids_test.go

@@ -0,0 +1,169 @@
+package gdpr
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/prebid/prebid-server/v3/metrics"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFetchLatestGVLVendorIDs(t *testing.T) {
+	tests := []struct {
+		name        string
+		settings    serverSettings
+		expectedIDs map[uint16]struct{}
+		expectError bool
+	}{
+		{
+			name: "fetch-with-multiple-vendors",
+			settings: serverSettings{
+				vendorListLatestVersion: 1,
+				vendorLists: map[int]map[int]string{
+					3: {
+						1: MarshalVendorList(vendorList{
+							GVLSpecificationVersion: 3,
+							VendorListVersion:       1,
+							Vendors: map[string]*vendor{
+								"10":  {ID: 10},
+								"20":  {ID: 20},
+								"100": {ID: 100},
+							},
+						}),
+					},
+				},
+			},
+			expectedIDs: map[uint16]struct{}{
+				10:  {},
+				20:  {},
+				100: {},
+			},
+		},
+		{
+			name: "fetch-with-no-vendors",
+			settings: serverSettings{
+				vendorListLatestVersion: 1,
+				vendorLists: map[int]map[int]string{
+					3: {
+						1: MarshalVendorList(vendorList{
+							GVLSpecificationVersion: 3,
+							VendorListVersion:       1,
+							Vendors:                 map[string]*vendor{},
+						}),
+					},
+				},
+			},
+			expectedIDs: map[uint16]struct{}{},
+		},
+		{
+			name: "error-spec-v3-not-available",
+			settings: serverSettings{
+				vendorListLatestVersion: 1,
+				vendorLists: map[int]map[int]string{
+					2: {
+						1: MarshalVendorList(vendorList{
+							GVLSpecificationVersion: 2,
+							VendorListVersion:       1,
+							Vendors:                 map[string]*vendor{"5": {ID: 5}},
+						}),
+					},
+				},
+			},
+			expectedIDs: map[uint16]struct{}{},
+			expectError: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			server := httptest.NewServer(http.HandlerFunc(mockServer(tt.settings)))
+			defer server.Close()
+
+			m := &metrics.MetricsEngineMock{}
+			if tt.expectError {
+				m.On("RecordLiveGVLFetch", false).Once()
+			} else {
+				m.On("RecordLiveGVLFetch", true).Once()
+			}
+			result := FetchLatestGVLVendorIDs(context.Background(), server.Client(), testURLMaker(server), m)
+			assert.Equal(t, tt.expectedIDs, result)
+			m.AssertExpectations(t)
+		})
+	}
+}
+
+func TestFetchLatestGVLVendorIDsServerError(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusInternalServerError)
+	}))
+	defer server.Close()
+
+	m := &metrics.MetricsEngineMock{}
+	m.On("RecordLiveGVLFetch", false).Once()
+	result := FetchLatestGVLVendorIDs(context.Background(), server.Client(), testURLMaker(server), m)
+	assert.Empty(t, result)
+	m.AssertExpectations(t)
+}
+
+func TestFetchLatestGVLVendorIDsMalformedJSON(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+		w.Write([]byte("not valid json"))
+	}))
+	defer server.Close()
+
+	m := &metrics.MetricsEngineMock{}
+	m.On("RecordLiveGVLFetch", false).Once()
+	result := FetchLatestGVLVendorIDs(context.Background(), server.Client(), testURLMaker(server), m)
+	assert.Empty(t, result)
+	m.AssertExpectations(t)
+}
+
+func TestLiveGVLVendorIDsContains(t *testing.T) {
+	tests := []struct {
+		name     string
+		ids      map[uint16]struct{}
+		checkID  uint16
+		expected bool
+	}{
+		{
+			name:     "empty-set-returns-true-for-any-id",
+			ids:      map[uint16]struct{}{},
+			checkID:  42,
+			expected: true,
+		},
+		{
+			name:     "id-present-in-set",
+			ids:      map[uint16]struct{}{10: {}, 20: {}},
+			checkID:  10,
+			expected: true,
+		},
+		{
+			name:     "id-not-present-in-set",
+			ids:      map[uint16]struct{}{10: {}, 20: {}},
+			checkID:  30,
+			expected: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			live := NewLiveGVLVendorIDs()
+			live.Update(tt.ids)
+			assert.Equal(t, tt.expected, live.Contains(tt.checkID))
+		})
+	}
+}
+
+func TestLiveGVLVendorIDsUpdateSkipsEmpty(t *testing.T) {
+	live := NewLiveGVLVendorIDs()
+	live.Update(map[uint16]struct{}{10: {}, 20: {}})
+
+	// Updating with empty should retain previous set
+	live.Update(map[uint16]struct{}{})
+	assert.True(t, live.Contains(10))
+	assert.True(t, live.Contains(20))
+	assert.False(t, live.Contains(30))
+}