diff --git a/config/config.go b/config/config.go index 02153c24..73ff0478 100644 --- a/config/config.go +++ b/config/config.go @@ -275,6 +275,7 @@ type GDPR struct { NonStandardPublisherMap map[string]struct{} TCF2 TCF2 `mapstructure:"tcf2"` AMPException bool `mapstructure:"amp_exception"` // Deprecated: Use account-level GDPR settings (gdpr.integration_enabled.amp) instead + LiveGVLRefreshInterval int `mapstructure:"live_gvl_refresh_interval_seconds"` // EEACountries (EEA = European Economic Area) are a list of countries where we should assume GDPR applies. // If the gdpr flag is unset in a request, but geo.country is set, we will assume GDPR applies if and only // if the country matches one on this list. If both the GDPR flag and country are not set, we default @@ -1237,6 +1238,7 @@ func SetupViper(v *viper.Viper, filename string, bidderInfos BidderInfos) { v.SetDefault("gdpr.timeouts_ms.init_vendorlist_fetches", 0) v.SetDefault("gdpr.timeouts_ms.active_vendorlist_fetch", 0) v.SetDefault("gdpr.non_standard_publishers", []string{""}) + v.SetDefault("gdpr.live_gvl_refresh_interval_seconds", 86400) // 1 day v.SetDefault("gdpr.tcf2.enabled", true) v.SetDefault("gdpr.tcf2.purpose1.enforce_vendors", true) v.SetDefault("gdpr.tcf2.purpose2.enforce_vendors", true) diff --git a/config/config_test.go b/config/config_test.go index 7d7ebe91..19ba4759 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -254,6 +254,7 @@ func TestDefaults(t *testing.T) { cmpInts(t, "analytics.agma.buffers.count", 100, cfg.Analytics.Agma.Buffers.EventCount) cmpStrings(t, "analytics.agma.buffers.timeout", "15m", cfg.Analytics.Agma.Buffers.Timeout) cmpInts(t, "analytics.agma.accounts", 0, len(cfg.Analytics.Agma.Accounts)) + cmpInts(t, "gdpr.live_gvl_refresh_interval_seconds", 86400, cfg.GDPR.LiveGVLRefreshInterval) expectedTCF2 := TCF2{ Enabled: true, Purpose1: TCF2Purpose{ @@ -386,6 +387,7 @@ gdpr: default_value: "1" non_standard_publishers: ["pub1", "pub2"] eea_countries: ["eea1", "eea2"] + live_gvl_refresh_interval_seconds: 3600 tcf2: purpose1: enforce_vendors: false @@ -686,6 +688,8 @@ func TestFullConfig(t *testing.T) { cmpInts(t, "http_client_cache.idle_connection_timeout_seconds", 3, cfg.CacheClient.IdleConnTimeout) cmpInts(t, "gdpr.host_vendor_id", 15, cfg.GDPR.HostVendorID) cmpStrings(t, "gdpr.default_value", "1", cfg.GDPR.DefaultValue) + cmpInts(t, "gdpr.live_gvl_refresh_interval_seconds", 3600, cfg.GDPR.LiveGVLRefreshInterval) + cmpBools(t, "video.enable_deprecated_endpoint", true, cfg.Video.EnableDeprecatedEndpoint) cmpStrings(t, "host_schain_node.asi", "pbshostcompany.com", cfg.HostSChainNode.ASI) cmpStrings(t, "host_schain_node.sid", "00001", cfg.HostSChainNode.SID) cmpStrings(t, "host_schain_node.rid", "BidRequest", cfg.HostSChainNode.RID) diff --git a/gdpr/gdpr.go b/gdpr/gdpr.go index 26e04e0b..d8cd56fe 100644 --- a/gdpr/gdpr.go +++ b/gdpr/gdpr.go @@ -35,16 +35,16 @@ type RequestInfo struct { } // NewPermissionsBuilder takes host config data used to configure the builder function it returns -func NewPermissionsBuilder(cfg config.GDPR, gvlVendorIDs map[openrtb_ext.BidderName]uint16, vendorListFetcher VendorListFetcher, me metrics.MetricsEngine) PermissionsBuilder { +func NewPermissionsBuilder(cfg config.GDPR, gvlVendorIDs map[openrtb_ext.BidderName]uint16, validGVLVendorIDs *LiveGVLVendorIDs, vendorListFetcher VendorListFetcher, me metrics.MetricsEngine) PermissionsBuilder { return func(tcf2Cfg TCF2ConfigReader, requestInfo RequestInfo) Permissions { purposeEnforcerBuilder := NewPurposeEnforcerBuilder(tcf2Cfg) - return NewPermissions(cfg, tcf2Cfg, gvlVendorIDs, vendorListFetcher, purposeEnforcerBuilder, requestInfo, me) + return NewPermissions(cfg, tcf2Cfg, gvlVendorIDs, validGVLVendorIDs, vendorListFetcher, purposeEnforcerBuilder, requestInfo, me) } } // NewPermissions gets a per-request Permissions object that can then be used to check GDPR permissions for a given bidder. -func NewPermissions(cfg config.GDPR, tcf2Config TCF2ConfigReader, vendorIDs map[openrtb_ext.BidderName]uint16, fetcher VendorListFetcher, purposeEnforcerBuilder PurposeEnforcerBuilder, requestInfo RequestInfo, metricsEngine metrics.MetricsEngine) Permissions { +func NewPermissions(cfg config.GDPR, tcf2Config TCF2ConfigReader, vendorIDs map[openrtb_ext.BidderName]uint16, validGVLVendorIDs *LiveGVLVendorIDs, fetcher VendorListFetcher, purposeEnforcerBuilder PurposeEnforcerBuilder, requestInfo RequestInfo, metricsEngine metrics.MetricsEngine) Permissions { if !cfg.Enabled { return &AlwaysAllow{} } @@ -55,6 +55,7 @@ func NewPermissions(cfg config.GDPR, tcf2Config TCF2ConfigReader, vendorIDs map[ hostVendorID: cfg.HostVendorID, nonStandardPublishers: cfg.NonStandardPublisherMap, cfg: tcf2Config, + validGVLVendorIDs: validGVLVendorIDs, vendorIDs: vendorIDs, publisherID: requestInfo.PublisherID, gdprSignal: SignalNormalize(requestInfo.GDPRSignal, cfg.DefaultValue), diff --git a/gdpr/gdpr_test.go b/gdpr/gdpr_test.go index 6329dad8..01e829eb 100644 --- a/gdpr/gdpr_test.go +++ b/gdpr/gdpr_test.go @@ -51,7 +51,7 @@ func TestNewPermissions(t *testing.T) { fakePurposeEnforcerBuilder := fakePurposeEnforcerBuilder{ purposeEnforcer: nil, }.Builder - perms := NewPermissions(config, &tcf2Config{}, vendorIDs, vendorListFetcher, fakePurposeEnforcerBuilder, RequestInfo{}, &metrics.MetricsEngineMock{}) + perms := NewPermissions(config, &tcf2Config{}, vendorIDs, nil, vendorListFetcher, fakePurposeEnforcerBuilder, RequestInfo{}, &metrics.MetricsEngineMock{}) assert.IsType(t, tt.wantType, perms, tt.description) } diff --git a/gdpr/gvl_vendor_ids.go b/gdpr/gvl_vendor_ids.go new file mode 100644 index 00000000..401642c5 --- /dev/null +++ b/gdpr/gvl_vendor_ids.go @@ -0,0 +1,119 @@ +package gdpr + +import ( + "context" + "encoding/json" + "io" + "net/http" + "sync/atomic" + "time" + + "github.com/prebid/prebid-server/v3/logger" + "github.com/prebid/prebid-server/v3/metrics" + "github.com/prebid/prebid-server/v3/util/task" + "golang.org/x/net/context/ctxhttp" +) + +// LiveGVLVendorIDs provides thread-safe access to the set of vendor IDs present in the latest +// Global Vendor List. It uses atomic.Value so reads (on every request) are lock-free, while +// writes (periodic refresh) are safe without mutexes. +type LiveGVLVendorIDs struct { + ids atomic.Value // holds map[uint16]struct{} +} + +// NewLiveGVLVendorIDs creates a LiveGVLVendorIDs initialized with an empty set. +func NewLiveGVLVendorIDs() *LiveGVLVendorIDs { + l := &LiveGVLVendorIDs{} + l.ids.Store(make(map[uint16]struct{})) + return l +} + +// Contains returns true if the given vendor ID is in the current valid set. +// If the set is empty (e.g. the GVL fetch failed), all IDs are considered valid +// as a safe fallback. +func (l *LiveGVLVendorIDs) Contains(id uint16) bool { + m := l.ids.Load().(map[uint16]struct{}) + if len(m) == 0 { + return true + } + _, ok := m[id] + return ok +} + +// Update atomically replaces the valid vendor ID set. If newIDs is empty, the existing +// set is retained. +func (l *LiveGVLVendorIDs) Update(newIDs map[uint16]struct{}) { + if len(newIDs) > 0 { + l.ids.Store(newIDs) + } +} + +// NewGVLVendorIDTickerTask creates a TickerTask that fetches the latest GVL vendor IDs and +// updates the LiveGVLVendorIDs set. Calling Start on the returned task performs the initial +// fetch immediately and then schedules periodic refreshes at the given interval. +func NewGVLVendorIDTickerTask(interval time.Duration, client *http.Client, urlMaker func(uint16, uint16) string, live *LiveGVLVendorIDs, me metrics.MetricsEngine) *task.TickerTask { + return task.NewTickerTaskFromFunc(interval, func() error { + newIDs := FetchLatestGVLVendorIDs(context.Background(), client, urlMaker, me) + live.Update(newIDs) + return nil + }) +} + +// gvlVendorListContract is a lightweight contract for parsing only vendor IDs from GVL JSON +type gvlVendorListContract struct { + Vendors map[string]struct { + ID uint16 `json:"id"` + } `json:"vendors"` +} + +// FetchLatestGVLVendorIDs fetches the most recent Global Vendor List and returns a set of all +// vendor IDs present in it. The returned map has vendor IDs as keys and empty structs as values. +// If the fetch or parse fails, an empty map is returned. +func FetchLatestGVLVendorIDs(ctx context.Context, client *http.Client, urlMaker func(uint16, uint16) string, me metrics.MetricsEngine) map[uint16]struct{} { + vendorIDs := make(map[uint16]struct{}) + + // Fetch latest GVL for the latest spec version (listVersion 0 means latest) + url := urlMaker(latestSpecVersion, 0) + + req, err := http.NewRequest("GET", url, nil) + if err != nil { + logger.Errorf("Failed to build GET %s request for GVL vendor ID extraction: %v", url, err) + me.RecordLiveGVLFetch(false) + return vendorIDs + } + + resp, err := ctxhttp.Do(ctx, client, req) + if err != nil { + logger.Errorf("Error calling GET %s for GVL vendor ID extraction: %v", url, err) + me.RecordLiveGVLFetch(false) + return vendorIDs + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + logger.Errorf("GET %s returned %d for GVL vendor ID extraction", url, resp.StatusCode) + me.RecordLiveGVLFetch(false) + return vendorIDs + } + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + logger.Errorf("Error reading response body from GET %s for GVL vendor ID extraction: %v", url, err) + me.RecordLiveGVLFetch(false) + return vendorIDs + } + + var contract gvlVendorListContract + if err := json.Unmarshal(respBody, &contract); err != nil { + logger.Errorf("GET %s returned malformed JSON for GVL vendor ID extraction: %v", url, err) + me.RecordLiveGVLFetch(false) + return vendorIDs + } + + for _, v := range contract.Vendors { + vendorIDs[v.ID] = struct{}{} + } + + me.RecordLiveGVLFetch(true) + return vendorIDs +} diff --git a/gdpr/gvl_vendor_ids_test.go b/gdpr/gvl_vendor_ids_test.go new file mode 100644 index 00000000..d9974176 --- /dev/null +++ b/gdpr/gvl_vendor_ids_test.go @@ -0,0 +1,169 @@ +package gdpr + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + + "github.com/prebid/prebid-server/v3/metrics" + "github.com/stretchr/testify/assert" +) + +func TestFetchLatestGVLVendorIDs(t *testing.T) { + tests := []struct { + name string + settings serverSettings + expectedIDs map[uint16]struct{} + expectError bool + }{ + { + name: "fetch-with-multiple-vendors", + settings: serverSettings{ + vendorListLatestVersion: 1, + vendorLists: map[int]map[int]string{ + 3: { + 1: MarshalVendorList(vendorList{ + GVLSpecificationVersion: 3, + VendorListVersion: 1, + Vendors: map[string]*vendor{ + "10": {ID: 10}, + "20": {ID: 20}, + "100": {ID: 100}, + }, + }), + }, + }, + }, + expectedIDs: map[uint16]struct{}{ + 10: {}, + 20: {}, + 100: {}, + }, + }, + { + name: "fetch-with-no-vendors", + settings: serverSettings{ + vendorListLatestVersion: 1, + vendorLists: map[int]map[int]string{ + 3: { + 1: MarshalVendorList(vendorList{ + GVLSpecificationVersion: 3, + VendorListVersion: 1, + Vendors: map[string]*vendor{}, + }), + }, + }, + }, + expectedIDs: map[uint16]struct{}{}, + }, + { + name: "error-spec-v3-not-available", + settings: serverSettings{ + vendorListLatestVersion: 1, + vendorLists: map[int]map[int]string{ + 2: { + 1: MarshalVendorList(vendorList{ + GVLSpecificationVersion: 2, + VendorListVersion: 1, + Vendors: map[string]*vendor{"5": {ID: 5}}, + }), + }, + }, + }, + expectedIDs: map[uint16]struct{}{}, + expectError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(mockServer(tt.settings))) + defer server.Close() + + m := &metrics.MetricsEngineMock{} + if tt.expectError { + m.On("RecordLiveGVLFetch", false).Once() + } else { + m.On("RecordLiveGVLFetch", true).Once() + } + result := FetchLatestGVLVendorIDs(context.Background(), server.Client(), testURLMaker(server), m) + assert.Equal(t, tt.expectedIDs, result) + m.AssertExpectations(t) + }) + } +} + +func TestFetchLatestGVLVendorIDsServerError(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + defer server.Close() + + m := &metrics.MetricsEngineMock{} + m.On("RecordLiveGVLFetch", false).Once() + result := FetchLatestGVLVendorIDs(context.Background(), server.Client(), testURLMaker(server), m) + assert.Empty(t, result) + m.AssertExpectations(t) +} + +func TestFetchLatestGVLVendorIDsMalformedJSON(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte("not valid json")) + })) + defer server.Close() + + m := &metrics.MetricsEngineMock{} + m.On("RecordLiveGVLFetch", false).Once() + result := FetchLatestGVLVendorIDs(context.Background(), server.Client(), testURLMaker(server), m) + assert.Empty(t, result) + m.AssertExpectations(t) +} + +func TestLiveGVLVendorIDsContains(t *testing.T) { + tests := []struct { + name string + ids map[uint16]struct{} + checkID uint16 + expected bool + }{ + { + name: "empty-set-returns-true-for-any-id", + ids: map[uint16]struct{}{}, + checkID: 42, + expected: true, + }, + { + name: "id-present-in-set", + ids: map[uint16]struct{}{10: {}, 20: {}}, + checkID: 10, + expected: true, + }, + { + name: "id-not-present-in-set", + ids: map[uint16]struct{}{10: {}, 20: {}}, + checkID: 30, + expected: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + live := NewLiveGVLVendorIDs() + live.Update(tt.ids) + assert.Equal(t, tt.expected, live.Contains(tt.checkID)) + }) + } +} + +func TestLiveGVLVendorIDsUpdateSkipsEmpty(t *testing.T) { + live := NewLiveGVLVendorIDs() + live.Update(map[uint16]struct{}{10: {}, 20: {}}) + + // Updating with empty should retain previous set + live.Update(map[uint16]struct{}{}) + assert.True(t, live.Contains(10)) + assert.True(t, live.Contains(20)) + assert.False(t, live.Contains(30)) +} diff --git a/gdpr/impl.go b/gdpr/impl.go index 24c5b301..386ce351 100644 --- a/gdpr/impl.go +++ b/gdpr/impl.go @@ -23,6 +23,7 @@ type permissionsImpl struct { metrics metrics.MetricsEngine nonStandardPublishers map[string]struct{} purposeEnforcerBuilder PurposeEnforcerBuilder + validGVLVendorIDs *LiveGVLVendorIDs vendorIDs map[openrtb_ext.BidderName]uint16 // request-specific aliasGVLIDs map[string]uint16 @@ -48,7 +49,7 @@ func (p *permissionsImpl) BidderSyncAllowed(ctx context.Context, bidder openrtb_ } id, ok := p.vendorIDs[bidder] - if ok { + if ok && p.isValidVendorID(id) { vendorExceptions := p.cfg.PurposeVendorExceptions(consentconstants.Purpose(1)) _, vendorException := vendorExceptions[string(bidder)] return p.allowSync(ctx, id, bidder, vendorException) @@ -107,17 +108,34 @@ func (p *permissionsImpl) defaultPermissions() AuctionPermissions { } // resolveVendorID gets the vendor ID for the specified bidder from either the alias GVL IDs -// provided in the request or from the bidder configs loaded at startup +// provided in the request or from the bidder configs loaded at startup. If the resolved ID +// is not present in the latest Global Vendor List, it returns 0 and false. func (p *permissionsImpl) resolveVendorID(bidderCoreName openrtb_ext.BidderName, bidder openrtb_ext.BidderName) (id uint16, ok bool) { if id, ok = p.aliasGVLIDs[string(bidder)]; ok { + if !p.isValidVendorID(id) { + return 0, false + } return id, ok } id, ok = p.vendorIDs[bidderCoreName] + if ok && !p.isValidVendorID(id) { + return 0, false + } return id, ok } +// isValidVendorID checks whether the given vendor ID exists in the latest Global Vendor List. +// If the valid set is nil (e.g. the GVL fetch failed), all IDs are considered valid +// as a safe fallback. +func (p *permissionsImpl) isValidVendorID(id uint16) bool { + if p.validGVLVendorIDs == nil { + return true + } + return p.validGVLVendorIDs.Contains(id) +} + // allowSync computes cookie sync activity legal basis for a given bidder using the enforcement // algorithms selected by the purpose enforcer builder func (p *permissionsImpl) allowSync(ctx context.Context, vendorID uint16, bidder openrtb_ext.BidderName, vendorException bool) (bool, error) { diff --git a/gdpr/impl_test.go b/gdpr/impl_test.go index 60a688fc..ac993f25 100644 --- a/gdpr/impl_test.go +++ b/gdpr/impl_test.go @@ -1321,6 +1321,94 @@ func TestDefaultPermissions(t *testing.T) { } } +func TestIsValidVendorID(t *testing.T) { + vendor2AndPurpose1Consent := "CPGWbY_PGWbY_GYAAAENABCAAIAAAAAAAAAAACEAAAAA" + vendor2AndPurpose2Consent := "CPGWbY_PGWbY_GYAAAENABCAAEAAAAAAAAAAACEAAAAA" + vendorListData := MarshalVendorList(vendorList{ + VendorListVersion: 2, + Vendors: map[string]*vendor{ + "2": { + ID: 2, + Purposes: []int{1, 2}, + }, + }, + }) + + tcf2AggConfig := allPurposesEnabledTCF2Config() + + tests := []struct { + name string + validGVLVendorIDs *LiveGVLVendorIDs + consent string + expectedAllowSync bool + expectedAllowBidReq bool + expectedPassID bool + }{ + { + name: "nil-live-gvl-vendor-ids", + validGVLVendorIDs: nil, + consent: vendor2AndPurpose1Consent, + expectedAllowSync: true, + expectedAllowBidReq: false, + expectedPassID: false, + }, + { + name: "vendor-id-in-live-gvl", + validGVLVendorIDs: func() *LiveGVLVendorIDs { + l := NewLiveGVLVendorIDs() + l.Update(map[uint16]struct{}{2: {}}) + return l + }(), + consent: vendor2AndPurpose2Consent, + expectedAllowSync: false, + expectedAllowBidReq: true, + expectedPassID: true, + }, + { + name: "vendor-id-not-in-live-gvl", + validGVLVendorIDs: func() *LiveGVLVendorIDs { + l := NewLiveGVLVendorIDs() + l.Update(map[uint16]struct{}{99: {}}) + return l + }(), + consent: vendor2AndPurpose2Consent, + expectedAllowSync: false, + expectedAllowBidReq: false, + expectedPassID: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + perms := permissionsImpl{ + cfg: &tcf2AggConfig, + hostVendorID: 2, + vendorIDs: map[openrtb_ext.BidderName]uint16{ + openrtb_ext.BidderAppnexus: 2, + }, + fetchVendorList: listFetcher(map[uint16]map[uint16]vendorlist.VendorList{ + 2: { + 1: parseVendorListDataV2(t, vendorListData), + }, + }), + purposeEnforcerBuilder: NewPurposeEnforcerBuilder(&tcf2AggConfig), + nonStandardPublishers: map[string]struct{}{}, + gdprSignal: SignalYes, + consent: tt.consent, + validGVLVendorIDs: tt.validGVLVendorIDs, + } + + allowSync, err := perms.BidderSyncAllowed(context.Background(), openrtb_ext.BidderAppnexus) + assert.NoError(t, err) + assert.Equal(t, tt.expectedAllowSync, allowSync, "BidderSyncAllowed") + + auctionPerms := perms.AuctionActivitiesAllowed(context.Background(), openrtb_ext.BidderAppnexus, openrtb_ext.BidderAppnexus) + assert.Equal(t, tt.expectedAllowBidReq, auctionPerms.AllowBidRequest, "AllowBidRequest") + assert.Equal(t, tt.expectedPassID, auctionPerms.PassID, "PassID") + }) + } +} + func TestVendorListSelection(t *testing.T) { policyVersion3WithVendor2AndPurpose1Consent := "CPGWbY_PGWbY_GYAAAENABDAAIAAAAAAAAAAACEAAAAA" policyVersion4WithVendor2AndPurpose1Consent := "CPGWbY_PGWbY_GYAAAENABEAAIAAAAAAAAAAACEAAAAA" diff --git a/gdpr/vendorlist-fetching.go b/gdpr/vendorlist-fetching.go index 2a3be433..c97c2bdb 100644 --- a/gdpr/vendorlist-fetching.go +++ b/gdpr/vendorlist-fetching.go @@ -22,6 +22,10 @@ import ( type saveVendors func(uint16, uint16, api.VendorList) type VendorListFetcher func(ctx context.Context, specVersion uint16, listVersion uint16, metricsEngine metrics.MetricsEngine) (vendorlist.VendorList, error) +// latestSpecVersion is the highest GVL specification version supported. Update this value when +// a new spec version is introduced by the IAB. +const latestSpecVersion = 3 + // This file provides the vendorlist-fetching function for Prebid Server. // // For more info, see https://github.com/prebid/prebid-server/issues/504 @@ -72,7 +76,7 @@ func preloadCache(ctx context.Context, client *http.Client, urlMaker func(uint16 firstListVersion: 2, // The GVL for TCF2 has no vendors defined in its first version. It's very unlikely to be used, so don't preload it. }, { - specVersion: 3, + specVersion: latestSpecVersion, firstListVersion: 1, }, } diff --git a/metrics/config/metrics.go b/metrics/config/metrics.go index 8e6c3947..03247f9f 100644 --- a/metrics/config/metrics.go +++ b/metrics/config/metrics.go @@ -299,6 +299,12 @@ func (me *MultiMetricsEngine) RecordGvlListRequest() { } } +func (me *MultiMetricsEngine) RecordLiveGVLFetch(success bool) { + for _, thisME := range *me { + thisME.RecordLiveGVLFetch(success) + } +} + func (me *MultiMetricsEngine) RecordAdsCertReq(success bool) { for _, thisME := range *me { thisME.RecordAdsCertReq(success) @@ -558,6 +564,9 @@ func (me *NilMetricsEngine) RecordStoredResponse(pubId string) { func (me *NilMetricsEngine) RecordGvlListRequest() { } +func (me *NilMetricsEngine) RecordLiveGVLFetch(success bool) { +} + func (me *NilMetricsEngine) RecordAdsCertReq(success bool) { } diff --git a/metrics/go_metrics.go b/metrics/go_metrics.go index fbf89052..6a4d96d0 100644 --- a/metrics/go_metrics.go +++ b/metrics/go_metrics.go @@ -36,6 +36,8 @@ type Metrics struct { BidderServerResponseTimer metrics.Timer StoredResponsesMeter metrics.Meter GvlListRequestsMeter metrics.Meter + LiveGVLFetchSuccess metrics.Meter + LiveGVLFetchFailure metrics.Meter // Metrics for OpenRTB requests specifically RequestStatuses map[RequestType]map[RequestStatus]metrics.Meter @@ -195,6 +197,8 @@ func NewBlankMetrics(registry metrics.Registry, exchanges []string, disabledMetr SyncerSetsMeter: make(map[string]map[SyncerSetUidStatus]metrics.Meter), StoredResponsesMeter: blankMeter, GvlListRequestsMeter: blankMeter, + LiveGVLFetchSuccess: blankMeter, + LiveGVLFetchFailure: blankMeter, ImpsTypeBanner: blankMeter, ImpsTypeVideo: blankMeter, @@ -327,6 +331,8 @@ func NewMetrics(registry metrics.Registry, exchanges []openrtb_ext.BidderName, d newMetrics.PrebidCacheRequestTimerError = metrics.GetOrRegisterTimer("prebid_cache_request_time.err", registry) newMetrics.StoredResponsesMeter = metrics.GetOrRegisterMeter("stored_responses", registry) newMetrics.GvlListRequestsMeter = metrics.GetOrRegisterMeter("gvl_requests", registry) + newMetrics.LiveGVLFetchSuccess = metrics.GetOrRegisterMeter("live_gvl_fetch.ok", registry) + newMetrics.LiveGVLFetchFailure = metrics.GetOrRegisterMeter("live_gvl_fetch.failed", registry) newMetrics.OverheadTimer = makeOverheadTimerMetrics(registry) newMetrics.BidderServerResponseTimer = metrics.GetOrRegisterTimer("bidder_server_response_time_seconds", registry) @@ -677,6 +683,14 @@ func (me *Metrics) RecordGvlListRequest() { me.GvlListRequestsMeter.Mark(1) } +func (me *Metrics) RecordLiveGVLFetch(success bool) { + if success { + me.LiveGVLFetchSuccess.Mark(1) + } else { + me.LiveGVLFetchFailure.Mark(1) + } +} + func (me *Metrics) RecordImps(labels ImpLabels) { me.ImpMeter.Mark(int64(1)) if labels.BannerImps { diff --git a/metrics/go_metrics_test.go b/metrics/go_metrics_test.go index 2fcec9cb..65d8399c 100644 --- a/metrics/go_metrics_test.go +++ b/metrics/go_metrics_test.go @@ -37,6 +37,8 @@ func TestNewMetrics(t *testing.T) { ensureContains(t, registry, "setuid_requests.syncer_unknown", m.SetUidStatusMeter[SetUidSyncerUnknown]) ensureContains(t, registry, "stored_responses", m.StoredResponsesMeter) ensureContains(t, registry, "gvl_requests", m.GvlListRequestsMeter) + ensureContains(t, registry, "live_gvl_fetch.ok", m.LiveGVLFetchSuccess) + ensureContains(t, registry, "live_gvl_fetch.failed", m.LiveGVLFetchFailure) ensureContains(t, registry, "prebid_cache_request_time.ok", m.PrebidCacheRequestTimerSuccess) ensureContains(t, registry, "prebid_cache_request_time.err", m.PrebidCacheRequestTimerError) diff --git a/metrics/metrics.go b/metrics/metrics.go index cfbd0303..a521896f 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -571,6 +571,7 @@ type MetricsEngine interface { RecordDebugRequest(debugEnabled bool, pubId string) RecordStoredResponse(pubId string) RecordGvlListRequest() + RecordLiveGVLFetch(success bool) RecordAdsCertReq(success bool) RecordAdsCertSignTime(adsCertSignTime time.Duration) RecordBidValidationCreativeSizeError(adapter openrtb_ext.BidderName, account string) diff --git a/metrics/metrics_mock.go b/metrics/metrics_mock.go index 432e74f5..aa8a64ba 100644 --- a/metrics/metrics_mock.go +++ b/metrics/metrics_mock.go @@ -179,6 +179,10 @@ func (me *MetricsEngineMock) RecordGvlListRequest() { me.Called() } +func (me *MetricsEngineMock) RecordLiveGVLFetch(success bool) { + me.Called(success) +} + func (me *MetricsEngineMock) RecordAdsCertReq(success bool) { me.Called(success) } diff --git a/metrics/prometheus/prometheus.go b/metrics/prometheus/prometheus.go index 7174e102..a19a70e0 100644 --- a/metrics/prometheus/prometheus.go +++ b/metrics/prometheus/prometheus.go @@ -55,6 +55,7 @@ type Metrics struct { privacyTCF *prometheus.CounterVec storedResponses prometheus.Counter gvlListRequests prometheus.Counter + liveGVLFetch *prometheus.CounterVec storedResponsesFetchTimer *prometheus.HistogramVec storedResponsesErrors *prometheus.CounterVec adsCertRequests *prometheus.CounterVec @@ -389,6 +390,11 @@ func NewMetrics(cfg config.PrometheusMetrics, disabledMetrics config.DisabledMet "gvl_requests", "Count number of times GVL list is fetched") + metrics.liveGVLFetch = newCounter(cfg, reg, + "live_gvl_fetch", + "Count of live GVL vendor ID fetches labeled by success or failure.", + []string{successLabel}) + metrics.adapterBids = newCounter(cfg, reg, "adapter_bids", "Count of bids labeled by adapter and markup delivery type (adm or nurl).", @@ -780,6 +786,18 @@ func (m *Metrics) RecordGvlListRequest() { m.gvlListRequests.Inc() } +func (m *Metrics) RecordLiveGVLFetch(success bool) { + if success { + m.liveGVLFetch.With(prometheus.Labels{ + successLabel: requestSuccessful, + }).Inc() + } else { + m.liveGVLFetch.With(prometheus.Labels{ + successLabel: requestFailed, + }).Inc() + } +} + func (m *Metrics) RecordImps(labels metrics.ImpLabels) { m.impressions.With(prometheus.Labels{ isBannerLabel: strconv.FormatBool(labels.BannerImps), diff --git a/metrics/prometheus/prometheus_test.go b/metrics/prometheus/prometheus_test.go index d4edca88..971bc74f 100644 --- a/metrics/prometheus/prometheus_test.go +++ b/metrics/prometheus/prometheus_test.go @@ -1953,6 +1953,26 @@ func TestRecordGvlListRequest(t *testing.T) { assertCounterValue(t, "Record instance of fetched GVL list", "success", m.gvlListRequests, 1.00) } +func TestRecordLiveGVLFetch(t *testing.T) { + m := createMetricsForTesting() + + m.RecordLiveGVLFetch(true) + m.RecordLiveGVLFetch(true) + m.RecordLiveGVLFetch(false) + + assertCounterVecValue(t, "", "live_gvl_fetch:ok", m.liveGVLFetch, + float64(2), + prometheus.Labels{ + successLabel: requestSuccessful, + }) + + assertCounterVecValue(t, "", "live_gvl_fetch:fail", m.liveGVLFetch, + float64(1), + prometheus.Labels{ + successLabel: requestFailed, + }) +} + func TestRecordAdsCertReqMetric(t *testing.T) { testCases := []struct { description string diff --git a/router/router.go b/router/router.go index 45099477..64e5f7ac 100644 --- a/router/router.go +++ b/router/router.go @@ -221,9 +221,6 @@ func New(cfg *config.Configuration, rateConvertor *currency.RateConverter) (r *R analyticsRunner := analyticsBuild.New(&cfg.Analytics, r.MetricsEngine) - // register the analytics runner for shutdown - r.shutdowns = append(r.shutdowns, shutdown, analyticsRunner.Shutdown, shutdownModules.Shutdown) - paramsValidator, err := openrtb_ext.NewBidderParamsValidator(schemaDirectory) if err != nil { logger.Fatalf("Failed to create the bidder params validator. %v", err) @@ -236,9 +233,16 @@ func New(cfg *config.Configuration, rateConvertor *currency.RateConverter) (r *R gvlVendorIDs := cfg.BidderInfos.ToGVLVendorIDMap() vendorListFetcher := gdpr.NewVendorListFetcher(context.Background(), cfg.GDPR, generalHttpClient, r.MetricsEngine, gdpr.VendorListURLMaker) - gdprPermsBuilder := gdpr.NewPermissionsBuilder(cfg.GDPR, gvlVendorIDs, vendorListFetcher, r.MetricsEngine) + liveGVLVendorIDs := gdpr.NewLiveGVLVendorIDs() + refreshInterval := time.Duration(cfg.GDPR.LiveGVLRefreshInterval) * time.Second + gvlVendorIDTask := gdpr.NewGVLVendorIDTickerTask(refreshInterval, generalHttpClient, gdpr.VendorListURLMaker, liveGVLVendorIDs, r.MetricsEngine) + gvlVendorIDTask.Start() + gdprPermsBuilder := gdpr.NewPermissionsBuilder(cfg.GDPR, gvlVendorIDs, liveGVLVendorIDs, vendorListFetcher, r.MetricsEngine) tcf2CfgBuilder := gdpr.NewTCF2Config + // register the analytics runner, modules and live GVL Vendor ID ticker task for shutdown + r.shutdowns = append(r.shutdowns, shutdown, analyticsRunner.Shutdown, shutdownModules.Shutdown, gvlVendorIDTask.Stop) + cacheClient := pbc.NewClient(cacheHttpClient, &cfg.CacheURL, &cfg.ExtCacheURL, r.MetricsEngine) adapters, singleFormatAdapters, adaptersErrs := exchange.BuildAdapters(generalHttpClient, cfg, cfg.BidderInfos, r.MetricsEngine)