Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Don't Load GVL v1 for TCF2 (+ TCF1 Cleanup) #1693

Merged
merged 4 commits into from
Feb 16, 2021
Merged

Don't Load GVL v1 for TCF2 (+ TCF1 Cleanup) #1693

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
fd2d9f2
Initial Commit
SyntaxNode Feb 4, 2021
4538502
Remove Fallback URL For TCF2 Tests
SyntaxNode Feb 4, 2021
46a25f7
Fixed Comment
SyntaxNode Feb 9, 2021
2e053fb
Naming Feedback
SyntaxNode Feb 16, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions gdpr/gdpr.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,8 @@ func NewPermissions(ctx context.Context, cfg config.GDPR, vendorIDs map[openrtb_
cfg: cfg,
vendorIDs: vendorIDs,
fetchVendorList: map[uint8]func(ctx context.Context, id uint16) (vendorlist.VendorList, error){
tcf1SpecVersion: newVendorListFetcher(ctx, cfg, client, vendorListURLMaker, tcf1SpecVersion),
tcf2SpecVersion: newVendorListFetcher(ctx, cfg, client, vendorListURLMaker, tcf2SpecVersion)},
tcf1SpecVersion: newVendorListFetcherTCF1(cfg),
tcf2SpecVersion: newVendorListFetcherTCF2(ctx, cfg, client, vendorListURLMaker)},
}

if cfg.HostVendorID == 0 {
Expand Down
91 changes: 37 additions & 54 deletions gdpr/vendorlist-fetching.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,30 +26,40 @@ type saveVendors func(uint16, api.VendorList)
//
// Nothing in this file is exported. Public APIs can be found in gdpr.go

func newVendorListFetcher(initCtx context.Context, cfg config.GDPR, client *http.Client, urlMaker func(uint16, uint8) string, tcfSpecVersion uint8) func(ctx context.Context, id uint16) (vendorlist.VendorList, error) {
var fallback api.VendorList

if tcfSpecVersion == tcf1SpecVersion && len(cfg.TCF1.FallbackGVLPath) == 0 {
return func(ctx context.Context, vendorListVersion uint16) (vendorlist.VendorList, error) {
func newVendorListFetcherTCF1(cfg config.GDPR) func(ctx context.Context, id uint16) (vendorlist.VendorList, error) {
if len(cfg.TCF1.FallbackGVLPath) == 0 {
return func(_ context.Context, vendorListVersion uint16) (vendorlist.VendorList, error) {
return nil, makeVendorListNotFoundError(vendorListVersion)
}
}

if tcfSpecVersion == tcf1SpecVersion {
fallback = loadFallbackGVL(cfg.TCF1.FallbackGVLPath)
fallback := loadFallbackGVLForTCF1(cfg.TCF1.FallbackGVLPath)
return func(_ context.Context, _ uint16) (vendorlist.VendorList, error) {
return fallback, nil
}
}

return func(ctx context.Context, vendorListVersion uint16) (vendorlist.VendorList, error) {
return fallback, nil
}
func loadFallbackGVLForTCF1(fallbackGVLPath string) vendorlist.VendorList {
fallbackContents, err := ioutil.ReadFile(fallbackGVLPath)
if err != nil {
glog.Fatalf("Error reading from file %s: %v", fallbackGVLPath, err)
}

fallback, err := vendorlist.ParseEagerly(fallbackContents)
if err != nil {
glog.Fatalf("Error processing default GVL from %s: %v", fallbackGVLPath, err)
}
return fallback
}

cacheSave, cacheLoad := newVendorListCache(fallback)
func newVendorListFetcherTCF2(initCtx context.Context, cfg config.GDPR, client *http.Client, urlMaker func(uint16) string) func(ctx context.Context, id uint16) (vendorlist.VendorList, error) {
cacheSave, cacheLoad := newVendorListCache()

preloadContext, cancel := context.WithTimeout(initCtx, cfg.Timeouts.InitTimeout())
defer cancel()
preloadCache(preloadContext, client, urlMaker, cacheSave, tcfSpecVersion)
preloadCache(preloadContext, client, urlMaker, cacheSave)

saveOneRateLimited := newOccasionalSaver(cfg.Timeouts.ActiveTimeout(), tcfSpecVersion)
saveOneRateLimited := newOccasionalSaver(cfg.Timeouts.ActiveTimeout())
return func(ctx context.Context, vendorListVersion uint16) (vendorlist.VendorList, error) {
// Attempt To Load From Cache
if list := cacheLoad(vendorListVersion); list != nil {
Expand All @@ -58,19 +68,14 @@ func newVendorListFetcher(initCtx context.Context, cfg config.GDPR, client *http

// Attempt To Download
// - May not add to cache immediately.
saveOneRateLimited(ctx, client, urlMaker(vendorListVersion, tcfSpecVersion), cacheSave)
saveOneRateLimited(ctx, client, urlMaker(vendorListVersion), cacheSave)

// Attempt To Load From Cache Again
// - May have been added by the call to saveOneRateLimited.
if list := cacheLoad(vendorListVersion); list != nil {
return list, nil
}

// Attempt To Use Hardcoded Fallback
if fallback != nil {
return fallback, nil
}

// Give Up
return nil, makeVendorListNotFoundError(vendorListVersion)
}
Expand All @@ -81,35 +86,30 @@ func makeVendorListNotFoundError(vendorListVersion uint16) error {
}

// preloadCache saves all the known versions of the vendor list for future use.
func preloadCache(ctx context.Context, client *http.Client, urlMaker func(uint16, uint8) string, saver saveVendors, tcfSpecVersion uint8) {
latestVersion := saveOne(ctx, client, urlMaker(0, tcfSpecVersion), saver, tcfSpecVersion)
func preloadCache(ctx context.Context, client *http.Client, urlMaker func(uint16) string, saver saveVendors) {
latestVersion := saveOne(ctx, client, urlMaker(0), saver)
firstVersion := uint16(2) // The first version of the GVL for TCF2 has no vendors defined and is very unlikely to be used. Don't preload it.
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thoughts on calling this secondVersion instead of firstVersion and moving this comment to just above the for loop that follows this?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How about firstVersionToLoad? That's a more a accurate description IMHO.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed.


for i := uint16(1); i < latestVersion; i++ {
saveOne(ctx, client, urlMaker(i, tcfSpecVersion), saver, tcfSpecVersion)
for i := firstVersion; i < latestVersion; i++ {
saveOne(ctx, client, urlMaker(i), saver)
}
}

// Make a URL which can be used to fetch a given version of the Global Vendor List. If the version is 0,
// this will fetch the latest version.
func vendorListURLMaker(vendorListVersion uint16, tcfSpecVersion uint8) string {
if tcfSpecVersion == tcf2SpecVersion {
if vendorListVersion == 0 {
return "https://vendor-list.consensu.org/v2/vendor-list.json"
}
return "https://vendor-list.consensu.org/v2/archives/vendor-list-v" + strconv.Itoa(int(vendorListVersion)) + ".json"
}
func vendorListURLMaker(vendorListVersion uint16) string {
if vendorListVersion == 0 {
return "https://vendor-list.consensu.org/vendorlist.json"
return "https://vendor-list.consensu.org/v2/vendor-list.json"
}
return "https://vendor-list.consensu.org/v-" + strconv.Itoa(int(vendorListVersion)) + "/vendorlist.json"
return "https://vendor-list.consensu.org/v2/archives/vendor-list-v" + strconv.Itoa(int(vendorListVersion)) + ".json"
}

// newOccasionalSaver returns a wrapped version of saveOne() which only activates every few minutes.
//
// The goal here is to update quickly when new versions of the VendorList are released, but not wreck
// server performance if a bad CMP starts sending us malformed consent strings that advertize a version
// that doesn't exist yet.
func newOccasionalSaver(timeout time.Duration, tcfSpecVersion uint8) func(ctx context.Context, client *http.Client, url string, saver saveVendors) {
func newOccasionalSaver(timeout time.Duration) func(ctx context.Context, client *http.Client, url string, saver saveVendors) {
lastSaved := &atomic.Value{}
lastSaved.Store(time.Time{})

Expand All @@ -120,13 +120,13 @@ func newOccasionalSaver(timeout time.Duration, tcfSpecVersion uint8) func(ctx co
if timeSinceLastSave.Minutes() > 10 {
withTimeout, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
saveOne(withTimeout, client, url, saver, tcfSpecVersion)
saveOne(withTimeout, client, url, saver)
lastSaved.Store(now)
}
}
}

func saveOne(ctx context.Context, client *http.Client, url string, saver saveVendors, tcfSpecVersion uint8) uint16 {
func saveOne(ctx context.Context, client *http.Client, url string, saver saveVendors) uint16 {
req, err := http.NewRequest("GET", url, nil)
if err != nil {
glog.Errorf("Failed to build GET %s request. Cookie syncs may be affected: %v", url, err)
Expand All @@ -150,11 +150,7 @@ func saveOne(ctx context.Context, client *http.Client, url string, saver saveVen
return 0
}
var newList api.VendorList
if tcfSpecVersion == tcf2SpecVersion {
newList, err = vendorlist2.ParseEagerly(respBody)
} else {
newList, err = vendorlist.ParseEagerly(respBody)
}
newList, err = vendorlist2.ParseEagerly(respBody)
if err != nil {
glog.Errorf("GET %s returned malformed JSON. Cookie syncs may be affected. Error was %v. Body was %s", url, err, string(respBody))
return 0
Expand All @@ -164,7 +160,7 @@ func saveOne(ctx context.Context, client *http.Client, url string, saver saveVen
return newList.Version()
}

func newVendorListCache(fallbackVL api.VendorList) (save func(vendorListVersion uint16, list api.VendorList), load func(vendorListVersion uint16) api.VendorList) {
func newVendorListCache() (save func(vendorListVersion uint16, list api.VendorList), load func(vendorListVersion uint16) api.VendorList) {
cache := &sync.Map{}

save = func(vendorListVersion uint16, list api.VendorList) {
Expand All @@ -180,16 +176,3 @@ func newVendorListCache(fallbackVL api.VendorList) (save func(vendorListVersion
}
return
}

func loadFallbackGVL(fallbackGVLPath string) vendorlist.VendorList {
fallbackContents, err := ioutil.ReadFile(fallbackGVLPath)
if err != nil {
glog.Fatalf("Error reading from file %s: %v", fallbackGVLPath, err)
}

fallback, err := vendorlist.ParseEagerly(fallbackContents)
if err != nil {
glog.Fatalf("Error processing default GVL from %s: %v", fallbackGVLPath, err)
}
return fallback
}
Loading