From 12507e813cdded63e66b20796125bcc758185305 Mon Sep 17 00:00:00 2001 From: Yash Vekaria <30694521+Yash-Vekaria@users.noreply.github.com> Date: Sat, 18 May 2024 18:30:04 +0000 Subject: [PATCH 01/21] added support for confidential entries --- dist/ads.js | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/dist/ads.js b/dist/ads.js index 320ff7e5..a953b5f5 100644 --- a/dist/ads.js +++ b/dist/ads.js @@ -11,7 +11,7 @@ const fetchAndParse = async (url, parser) => { setTimeout(() => controller.abort(), 5000); try { - const response = await fetch(url, { signal }); + const response = await fetch(url, { signal, redirect: 'follow' }); return parser(response); } catch (error) { return { @@ -84,7 +84,7 @@ const parseAdsTxt = async (response) => { // Count unique and remove domain Sets for now for (let accountType of Object.values(result.account_types)) { accountType.domain_count = accountType.domains.size; - delete accountType.domains // Keeping a list of domains may be valuable for further research, e.g. accountType.domains = [...accountType.domains]; + accountType.domains = [...accountType.domains]; // delete accountType.domains } result.variables = [...result.variables]; } @@ -127,7 +127,8 @@ const parseSellersJSON = async (response) => { seller_count: 0, } }, - passthrough_count: 0 + passthrough_count: 0, + confidential_count: 0 } }; @@ -135,6 +136,17 @@ const parseSellersJSON = async (response) => { result.seller_count = content.sellers.length; for (let seller of content.sellers) { + // Validating records + if (!seller.seller_type && !seller.domain) { + continue; + } + if (!seller.seller_type) { + seller.seller_type = ""; + } + if (!seller.domain) { + seller.domain = ""; + } + // Seller records let type = seller.seller_type.trim().toLowerCase(), domain = seller.domain.trim().toLowerCase(); @@ -147,12 +159,17 @@ const parseSellersJSON = async (response) => { if (seller.is_passthrough) { result.passthrough_count += 1; } + + // Confidential + if (seller.is_confidential) { + result.confidential_count += 1; + } } // Count unique and remove domain Sets for now for (let seller_type of Object.values(result.seller_types)) { seller_type.domain_count = seller_type.domains.size; - delete seller_type.domains //seller_type.domains = [...seller_type.domains]; + seller_type.domains = [...seller_type.domains]; // delete seller_type.domains; } }; @@ -173,4 +190,4 @@ return Promise.all([ return JSON.stringify({ error: error }); -}); +}); \ No newline at end of file From dbdd617041749df3eb65d729ce660501e0c86156 Mon Sep 17 00:00:00 2001 From: Yash Vekaria <30694521+Yash-Vekaria@users.noreply.github.com> Date: Sat, 18 May 2024 20:43:21 +0000 Subject: [PATCH 02/21] added support to fetch Google's sellers.json --- dist/ads.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dist/ads.js b/dist/ads.js index a953b5f5..25640a9f 100644 --- a/dist/ads.js +++ b/dist/ads.js @@ -10,6 +10,11 @@ const fetchAndParse = async (url, parser) => { const { signal } = controller; setTimeout(() => controller.abort(), 5000); + // Google being popular ad-exchange hosting sellers.json at custom location, added its support + if (url.endsWith("google.com/sellers.json")) { + url = "https://storage.googleapis.com/adx-rtb-dictionaries/sellers.json" + } + try { const response = await fetch(url, { signal, redirect: 'follow' }); return parser(response); From 522dc2b584d146db7f15171131e8d0d28f99daa5 Mon Sep 17 00:00:00 2001 From: Yash Vekaria Date: Mon, 20 May 2024 02:45:00 -0700 Subject: [PATCH 03/21] testing returning null --- dist/privacy-sandbox.js | 91 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 dist/privacy-sandbox.js diff --git a/dist/privacy-sandbox.js b/dist/privacy-sandbox.js new file mode 100644 index 00000000..ca1b0684 --- /dev/null +++ b/dist/privacy-sandbox.js @@ -0,0 +1,91 @@ +[privacy-sandbox] +// Topics API Usage Reference: https://developers.google.com/privacy-sandbox/relevance/topics/demo#the-topics-api-demo +// Header Usage Reference: https://developers.google.com/privacy-sandbox/relevance/topics/demo#the-topics-api-demo + +let requests = $WPT_BODIES; +const testURL = new URL($WPT_URL); +let firstPartyDomain = testURL.hostname; + +result = { + topicsAvailable: document.featurePolicy.allowsFeature('browsing-topics'), + thirdPartiesUsingBrowsingTopics: {} +} + +function fetchAndCheckAttestation(url) { + const controller = new AbortController(); + const { signal } = controller; + setTimeout(() => controller.abort(), 5000); + + try { + const response = fetch(url, { signal }); + if (response.ok && response.headers.get('Content-Type').includes('application/json')) { + return true; + } + else { + return false; + } + } catch (error) { + return false; + } +} + +// Checks if the request corresponds to a JavaScript file +function isJavaScript(requestUrl, trimmedURL, contentType) { + return (requestUrl.endsWith('.js') || trimmedURL.endsWith('.js') || contentType.includes('javascript')); +} + +requests.forEach(request => { + const url = new URL(request.url); + const trimmedURL = url.origin + url.pathname; + const isScript = request.type === 'Script'; + const isDocument = request.type === 'Document'; + + let thirdPartyDomain = ''; + if (url.hostname !== firstPartyDomain) { + thirdPartyDomain = url.hostname; + } + + let jsUsage = false; + let headerUsage = false; + let receiverObserving = false; + let attestation = false; + let tldPlus1Attestation = false; + + // If the request is to fetch a javascript file or HTML document, perform string search in the returned file content for usage of Topics API + // document.browsingTopics() [JS] or fetch() request call includes: {browsingTopics: true} or XHR request call includes: {deprecatedBrowsingTopics: true} + if (isScript || isJavaScript(request.url, trimmedURL, request.response_headers['Content-Type']) || isDocument) { + if (request.response_body && (request.response_body.includes('browsingTopics') || request.response_body.includes('BrowsingTopics'))) { + jsUsage = true; + } + } + + // Checking request header usage of Topics: header: 'Sec-Browsing-Topics: true' + if (request.request_headers.some(header => header.name === 'Sec-Browsing-Topics')) { + headerUsage = true; + // Checking is sent Topics are observed by the receiver using response header 'Observe-Browsing-Topics' + // If value is ?1 then they are observed else they are not observed + if (request.response_headers.some(header => header.name === 'Observe-Browsing-Topics' && header.value === '?1')) { + receiverObserving = true; + } + } + + // Checking if the third party has published an attestation for Privacy Sandbox + if (thirdPartyDomain) { + attestation = fetchAndCheckAttestation(`${url.origin}/.well-known/privacy-sandbox-attestations.json`); + tldPlus1Origin = 'https://' + url.hostname.split('.').slice(-2).join('.'); + tldPlus1Attestation = fetchAndCheckAttestation(`${tldPlus1Origin}/.well-known/privacy-sandbox-attestations.json`); + } + + if (thirdPartyDomain && (jsUsage || headerUsage || attestationPublished)) { + result.thirdPartiesUsingBrowsingTopics[thirdPartyDomain] = { + topicsAccessJs: jsUsage, + topicsAccessHeader: headerUsage, + observingTopics: receiverObserving, + attestationPublished: attestation, + tldPlus1AttestationPublished: tldPlus1Attestation + }; + } + +}); + +return result; \ No newline at end of file From 9e37e905c6961e33b99926828810d717b71285b6 Mon Sep 17 00:00:00 2001 From: Yash Vekaria <30694521+Yash-Vekaria@users.noreply.github.com> Date: Wed, 22 May 2024 08:32:40 -0700 Subject: [PATCH 04/21] Update dist/ads.js Co-authored-by: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> --- dist/ads.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dist/ads.js b/dist/ads.js index 25640a9f..b3d55ec7 100644 --- a/dist/ads.js +++ b/dist/ads.js @@ -142,7 +142,7 @@ const parseSellersJSON = async (response) => { for (let seller of content.sellers) { // Validating records - if (!seller.seller_type && !seller.domain) { + if (!seller.seller_type || !seller.id) { continue; } if (!seller.seller_type) { From 290d00e33a92fc2822127e0997ffb52c40898d5e Mon Sep 17 00:00:00 2001 From: Yash Vekaria <30694521+Yash-Vekaria@users.noreply.github.com> Date: Wed, 22 May 2024 08:37:36 -0700 Subject: [PATCH 05/21] Update dist/ads.js Co-authored-by: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> --- dist/ads.js | 3 --- 1 file changed, 3 deletions(-) diff --git a/dist/ads.js b/dist/ads.js index b3d55ec7..a593f54e 100644 --- a/dist/ads.js +++ b/dist/ads.js @@ -145,9 +145,6 @@ const parseSellersJSON = async (response) => { if (!seller.seller_type || !seller.id) { continue; } - if (!seller.seller_type) { - seller.seller_type = ""; - } if (!seller.domain) { seller.domain = ""; } From 2871a050c9c215dede537622d5fc4416bfd7d56d Mon Sep 17 00:00:00 2001 From: Yash Vekaria Date: Wed, 22 May 2024 16:53:13 -0700 Subject: [PATCH 06/21] Bug fixes --- dist/ads.js | 44 +++++++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/dist/ads.js b/dist/ads.js index a593f54e..65bc941a 100644 --- a/dist/ads.js +++ b/dist/ads.js @@ -6,15 +6,16 @@ const SELLER_TYPES = ['publisher', 'intermediary', 'both']; const isPresent = (response, endings) => response.ok && endings.some(ending => response.url.endsWith(ending)); const fetchAndParse = async (url, parser) => { - const controller = new AbortController(); - const { signal } = controller; - setTimeout(() => controller.abort(), 5000); - + let timeout = 5000; // Google being popular ad-exchange hosting sellers.json at custom location, added its support - if (url.endsWith("google.com/sellers.json")) { + if (document.location.origin.toLowerCase().includes("google") || url.toLowerCase().includes("sellers.json")) { + timeout = 10000; url = "https://storage.googleapis.com/adx-rtb-dictionaries/sellers.json" } - + const controller = new AbortController(); + const { signal } = controller; + setTimeout(() => controller.abort(), timeout); + try { const response = await fetch(url, { signal, redirect: 'follow' }); return parser(response); @@ -34,6 +35,7 @@ const parseAdsTxt = async (response) => { let result = { present: isPresent(response, ['/ads.txt', '/app-ads.txt']), status: response.status, + redirected: response.redirected, }; if (result.present && content) { @@ -53,8 +55,7 @@ const parseAdsTxt = async (response) => { }, line_count: 0, variables: new Set(), - variable_count: 0, - redirected: response.redirected, + variable_count: 0 } }; @@ -142,20 +143,9 @@ const parseSellersJSON = async (response) => { for (let seller of content.sellers) { // Validating records - if (!seller.seller_type || !seller.id) { + if (!seller.seller_type && !seller.id) { continue; } - if (!seller.domain) { - seller.domain = ""; - } - - // Seller records - let type = seller.seller_type.trim().toLowerCase(), - domain = seller.domain.trim().toLowerCase(); - if (Object.keys(result.seller_types).includes(type)) { - result.seller_types[type].domains.add(domain); - result.seller_types[type].seller_count += 1; - } // Passthrough if (seller.is_passthrough) { @@ -166,6 +156,18 @@ const parseSellersJSON = async (response) => { if (seller.is_confidential) { result.confidential_count += 1; } + + if (!seller.domain) { + continue; + } + + // Seller records + let type = seller.seller_type.trim().toLowerCase(), + domain = seller.domain.trim().toLowerCase(); + if (Object.keys(result.seller_types).includes(type)) { + result.seller_types[type].domains.add(domain); + result.seller_types[type].seller_count += 1; + } } // Count unique and remove domain Sets for now @@ -181,7 +183,7 @@ const parseSellersJSON = async (response) => { return Promise.all([ fetchAndParse("/ads.txt", parseAdsTxt).catch(e => e), fetchAndParse("/app-ads.txt", parseAdsTxt).catch(e => e), - fetchAndParse("/sellers.json", parseSellersJSON).catch(e => e), + fetchAndParse("sellers.json", parseSellersJSON).catch(e => e), ]).then((all_data) => { return JSON.stringify({ ads: all_data[0], From ed247467a6ba7fb72e283b64d0160512590e4271 Mon Sep 17 00:00:00 2001 From: Yash Vekaria Date: Thu, 23 May 2024 08:45:41 -0700 Subject: [PATCH 07/21] Removing Google's sellers.json parsing --- dist/ads.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dist/ads.js b/dist/ads.js index 65bc941a..48b0065d 100644 --- a/dist/ads.js +++ b/dist/ads.js @@ -8,10 +8,12 @@ const isPresent = (response, endings) => response.ok && endings.some(ending => r const fetchAndParse = async (url, parser) => { let timeout = 5000; // Google being popular ad-exchange hosting sellers.json at custom location, added its support + /* Removing because Google's sellers.json is huge with >1M entries. if (document.location.origin.toLowerCase().includes("google") || url.toLowerCase().includes("sellers.json")) { timeout = 10000; url = "https://storage.googleapis.com/adx-rtb-dictionaries/sellers.json" } + */ const controller = new AbortController(); const { signal } = controller; setTimeout(() => controller.abort(), timeout); @@ -183,7 +185,7 @@ const parseSellersJSON = async (response) => { return Promise.all([ fetchAndParse("/ads.txt", parseAdsTxt).catch(e => e), fetchAndParse("/app-ads.txt", parseAdsTxt).catch(e => e), - fetchAndParse("sellers.json", parseSellersJSON).catch(e => e), + fetchAndParse("/sellers.json", parseSellersJSON).catch(e => e), ]).then((all_data) => { return JSON.stringify({ ads: all_data[0], From 664f9a75d4e02d7b1a72fb36bab9ee6e28fa776d Mon Sep 17 00:00:00 2001 From: Yash Vekaria Date: Thu, 23 May 2024 08:48:57 -0700 Subject: [PATCH 08/21] Removed redirect: follow --- dist/ads.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dist/ads.js b/dist/ads.js index 48b0065d..ae771a3a 100644 --- a/dist/ads.js +++ b/dist/ads.js @@ -19,7 +19,7 @@ const fetchAndParse = async (url, parser) => { setTimeout(() => controller.abort(), timeout); try { - const response = await fetch(url, { signal, redirect: 'follow' }); + const response = await fetch(url, { signal }); return parser(response); } catch (error) { return { From 198f718ec5e30f2ef2f22996a423cc8e35e4cc14 Mon Sep 17 00:00:00 2001 From: Yash Vekaria Date: Thu, 23 May 2024 13:48:54 -0700 Subject: [PATCH 09/21] Adding suggested changes --- dist/ads.js | 36 +++++++++++++++--------------------- 1 file changed, 15 insertions(+), 21 deletions(-) diff --git a/dist/ads.js b/dist/ads.js index ae771a3a..ef665af3 100644 --- a/dist/ads.js +++ b/dist/ads.js @@ -6,13 +6,11 @@ const SELLER_TYPES = ['publisher', 'intermediary', 'both']; const isPresent = (response, endings) => response.ok && endings.some(ending => response.url.endsWith(ending)); const fetchAndParse = async (url, parser) => { - let timeout = 5000; - // Google being popular ad-exchange hosting sellers.json at custom location, added its support - /* Removing because Google's sellers.json is huge with >1M entries. - if (document.location.origin.toLowerCase().includes("google") || url.toLowerCase().includes("sellers.json")) { - timeout = 10000; - url = "https://storage.googleapis.com/adx-rtb-dictionaries/sellers.json" - } + const timeout = 5000; + /* + Google's sellers.json size is 120Mb as of May 2024 - too big for custom metrics. + It's available at realtimebidding.google.com/sellers.json, so not part of crawled pages list. + More details: https://support.google.com/authorizedbuyers/answer/9895942 */ const controller = new AbortController(); const { signal } = controller; @@ -92,9 +90,9 @@ const parseAdsTxt = async (response) => { // Count unique and remove domain Sets for now for (let accountType of Object.values(result.account_types)) { accountType.domain_count = accountType.domains.size; - accountType.domains = [...accountType.domains]; // delete accountType.domains + accountType.domains = Array.from(accountType.domains); // delete accountType.domains } - result.variables = [...result.variables]; + result.variables = Array.from(result.variables); } return result; @@ -144,8 +142,9 @@ const parseSellersJSON = async (response) => { result.seller_count = content.sellers.length; for (let seller of content.sellers) { + const stype = seller.seller_type.trim().toLowerCase(); // Validating records - if (!seller.seller_type && !seller.id) { + if (!SELLER_TYPES.includes(stype) || !seller.seller_id) { continue; } @@ -159,23 +158,18 @@ const parseSellersJSON = async (response) => { result.confidential_count += 1; } - if (!seller.domain) { - continue; - } - // Seller records - let type = seller.seller_type.trim().toLowerCase(), - domain = seller.domain.trim().toLowerCase(); - if (Object.keys(result.seller_types).includes(type)) { - result.seller_types[type].domains.add(domain); - result.seller_types[type].seller_count += 1; + if (seller.domain) { + const domain = seller.domain.trim().toLowerCase(); + result.seller_types[stype].domains.add(domain); + result.seller_types[stype].seller_count += 1; } } // Count unique and remove domain Sets for now for (let seller_type of Object.values(result.seller_types)) { seller_type.domain_count = seller_type.domains.size; - seller_type.domains = [...seller_type.domains]; // delete seller_type.domains; + seller_type.domains = Array.from(seller_type.domains); // delete seller_type.domains; } }; @@ -194,6 +188,6 @@ return Promise.all([ }); }).catch(error => { return JSON.stringify({ - error: error + error: error.message }); }); \ No newline at end of file From adfa8a6529477ee67ca6644fcb7859ff0b28a387 Mon Sep 17 00:00:00 2001 From: Yash Vekaria Date: Thu, 23 May 2024 13:59:41 -0700 Subject: [PATCH 10/21] Fixed Lint Code Base formatting issues --- dist/ads.js | 338 ++++++++++++++++++++-------------------- dist/privacy-sandbox.js | 128 +++++++-------- 2 files changed, 233 insertions(+), 233 deletions(-) diff --git a/dist/ads.js b/dist/ads.js index ef665af3..845dbd4a 100644 --- a/dist/ads.js +++ b/dist/ads.js @@ -6,188 +6,188 @@ const SELLER_TYPES = ['publisher', 'intermediary', 'both']; const isPresent = (response, endings) => response.ok && endings.some(ending => response.url.endsWith(ending)); const fetchAndParse = async (url, parser) => { - const timeout = 5000; - /* - Google's sellers.json size is 120Mb as of May 2024 - too big for custom metrics. - It's available at realtimebidding.google.com/sellers.json, so not part of crawled pages list. - More details: https://support.google.com/authorizedbuyers/answer/9895942 - */ - const controller = new AbortController(); - const { signal } = controller; - setTimeout(() => controller.abort(), timeout); - - try { - const response = await fetch(url, { signal }); - return parser(response); - } catch (error) { - return { - status: -1, - present: false, - error: error.message - }; - } + const timeout = 5000; + /* + Google's sellers.json size is 120Mb as of May 2024 - too big for custom metrics. + It's available at realtimebidding.google.com/sellers.json, so not part of crawled pages list. + More details: https://support.google.com/authorizedbuyers/answer/9895942 + */ + const controller = new AbortController(); + const { signal } = controller; + setTimeout(() => controller.abort(), timeout); + + try { + const response = await fetch(url, { signal }); + return parser(response); + } catch (error) { + return { + status: -1, + present: false, + error: error.message + }; + } }; // https://iabtechlab.com/wp-content/uploads/2022/04/Ads.txt-1.1.pdf const parseAdsTxt = async (response) => { - let content = await response.text(); - - let result = { - present: isPresent(response, ['/ads.txt', '/app-ads.txt']), - status: response.status, - redirected: response.redirected, - }; - - if (result.present && content) { - result = { - ...result, - ...{ - account_count: 0, - account_types: { - direct: { - domains: new Set(), - account_count: 0, - }, - reseller: { - domains: new Set(), - account_count: 0, - } - }, - line_count: 0, - variables: new Set(), - variable_count: 0 - } - }; - - // Clean up file content - content = content.replace(/#.*$/gm, ''); - content = content.replace(/\r/g, ''); - - let lines = content.split('\n'); - result.line_count = lines.length; - - for (let line of lines) { - // Variables - let variables = line.split('='); - if (variables.length == 2) { - result.variables.add(variables[0].trim().toLowerCase()); - result.variable_count += 1; - continue; - } - - // Account records - let relation_parts = line.split(','); - if (relation_parts.length >= 3) { - let type = relation_parts[2].trim().toLowerCase(); - if (['direct', 'reseller'].includes(type)) { - result.account_types[type].domains.add(relation_parts[0].trim()); - result.account_types[type].account_count += 1; - } - result.account_count += 1; - } - }; - - // Count unique and remove domain Sets for now - for (let accountType of Object.values(result.account_types)) { - accountType.domain_count = accountType.domains.size; - accountType.domains = Array.from(accountType.domains); // delete accountType.domains - } - result.variables = Array.from(result.variables); - } - - return result; + let content = await response.text(); + + let result = { + present: isPresent(response, ['/ads.txt', '/app-ads.txt']), + status: response.status, + redirected: response.redirected, + }; + + if (result.present && content) { + result = { + ...result, + ...{ + account_count: 0, + account_types: { + direct: { + domains: new Set(), + account_count: 0, + }, + reseller: { + domains: new Set(), + account_count: 0, + } + }, + line_count: 0, + variables: new Set(), + variable_count: 0 + } + }; + + // Clean up file content + content = content.replace(/#.*$/gm, ''); + content = content.replace(/\r/g, ''); + + let lines = content.split('\n'); + result.line_count = lines.length; + + for (let line of lines) { + // Variables + let variables = line.split('='); + if (variables.length == 2) { + result.variables.add(variables[0].trim().toLowerCase()); + result.variable_count += 1; + continue; + } + + // Account records + let relation_parts = line.split(','); + if (relation_parts.length >= 3) { + let type = relation_parts[2].trim().toLowerCase(); + if (['direct', 'reseller'].includes(type)) { + result.account_types[type].domains.add(relation_parts[0].trim()); + result.account_types[type].account_count += 1; + } + result.account_count += 1; + } + }; + + // Count unique and remove domain Sets for now + for (let accountType of Object.values(result.account_types)) { + accountType.domain_count = accountType.domains.size; + accountType.domains = Array.from(accountType.domains); // delete accountType.domains + } + result.variables = Array.from(result.variables); + } + + return result; } // https://iabtechlab.com/wp-content/uploads/2019/07/Sellers.json_Final.pdf const parseSellersJSON = async (response) => { - let content; - try { - content = JSON.parse(await response.text()); - } catch { - content = null; - } - let result = { - present: isPresent(response, ['/sellers.json']), - redirected: response.redirected, - status: response.status, - }; - - if (result.present && content) { - result = { - ...result, - ...{ - seller_count: 0, - seller_types: { - publisher: { - domains: new Set(), - seller_count: 0, - }, - intermediary: { - domains: new Set(), - seller_count: 0, - }, - both: { - domains: new Set(), - seller_count: 0, - } - }, - passthrough_count: 0, - confidential_count: 0 - } - }; - - // Clean up file content - result.seller_count = content.sellers.length; - - for (let seller of content.sellers) { - const stype = seller.seller_type.trim().toLowerCase(); - // Validating records - if (!SELLER_TYPES.includes(stype) || !seller.seller_id) { - continue; - } - - // Passthrough - if (seller.is_passthrough) { - result.passthrough_count += 1; - } - - // Confidential - if (seller.is_confidential) { - result.confidential_count += 1; - } - - // Seller records - if (seller.domain) { - const domain = seller.domain.trim().toLowerCase(); - result.seller_types[stype].domains.add(domain); - result.seller_types[stype].seller_count += 1; - } - } - - // Count unique and remove domain Sets for now - for (let seller_type of Object.values(result.seller_types)) { - seller_type.domain_count = seller_type.domains.size; - seller_type.domains = Array.from(seller_type.domains); // delete seller_type.domains; - } - }; - - return result; + let content; + try { + content = JSON.parse(await response.text()); + } catch { + content = null; + } + let result = { + present: isPresent(response, ['/sellers.json']), + redirected: response.redirected, + status: response.status, + }; + + if (result.present && content) { + result = { + ...result, + ...{ + seller_count: 0, + seller_types: { + publisher: { + domains: new Set(), + seller_count: 0, + }, + intermediary: { + domains: new Set(), + seller_count: 0, + }, + both: { + domains: new Set(), + seller_count: 0, + } + }, + passthrough_count: 0, + confidential_count: 0 + } + }; + + // Clean up file content + result.seller_count = content.sellers.length; + + for (let seller of content.sellers) { + const stype = seller.seller_type.trim().toLowerCase(); + // Validating records + if (!SELLER_TYPES.includes(stype) || !seller.seller_id) { + continue; + } + + // Passthrough + if (seller.is_passthrough) { + result.passthrough_count += 1; + } + + // Confidential + if (seller.is_confidential) { + result.confidential_count += 1; + } + + // Seller records + if (seller.domain) { + const domain = seller.domain.trim().toLowerCase(); + result.seller_types[stype].domains.add(domain); + result.seller_types[stype].seller_count += 1; + } + } + + // Count unique and remove domain Sets for now + for (let seller_type of Object.values(result.seller_types)) { + seller_type.domain_count = seller_type.domains.size; + seller_type.domains = Array.from(seller_type.domains); // delete seller_type.domains; + } + }; + + return result; } return Promise.all([ - fetchAndParse("/ads.txt", parseAdsTxt).catch(e => e), - fetchAndParse("/app-ads.txt", parseAdsTxt).catch(e => e), - fetchAndParse("/sellers.json", parseSellersJSON).catch(e => e), + fetchAndParse("/ads.txt", parseAdsTxt).catch(e => e), + fetchAndParse("/app-ads.txt", parseAdsTxt).catch(e => e), + fetchAndParse("/sellers.json", parseSellersJSON).catch(e => e), ]).then((all_data) => { - return JSON.stringify({ - ads: all_data[0], - app_ads: all_data[1], - sellers: all_data[2] - }); + return JSON.stringify({ + ads: all_data[0], + app_ads: all_data[1], + sellers: all_data[2] + }); }).catch(error => { - return JSON.stringify({ - error: error.message - }); + return JSON.stringify({ + error: error.message + }); }); \ No newline at end of file diff --git a/dist/privacy-sandbox.js b/dist/privacy-sandbox.js index ca1b0684..be52f486 100644 --- a/dist/privacy-sandbox.js +++ b/dist/privacy-sandbox.js @@ -1,4 +1,4 @@ -[privacy-sandbox] +//[privacy-sandbox] // Topics API Usage Reference: https://developers.google.com/privacy-sandbox/relevance/topics/demo#the-topics-api-demo // Header Usage Reference: https://developers.google.com/privacy-sandbox/relevance/topics/demo#the-topics-api-demo @@ -7,84 +7,84 @@ const testURL = new URL($WPT_URL); let firstPartyDomain = testURL.hostname; result = { - topicsAvailable: document.featurePolicy.allowsFeature('browsing-topics'), - thirdPartiesUsingBrowsingTopics: {} + topicsAvailable: document.featurePolicy.allowsFeature('browsing-topics'), + thirdPartiesUsingBrowsingTopics: {} } function fetchAndCheckAttestation(url) { - const controller = new AbortController(); - const { signal } = controller; - setTimeout(() => controller.abort(), 5000); + const controller = new AbortController(); + const { signal } = controller; + setTimeout(() => controller.abort(), 5000); - try { - const response = fetch(url, { signal }); - if (response.ok && response.headers.get('Content-Type').includes('application/json')) { - return true; - } - else { - return false; - } - } catch (error) { - return false; - } + try { + const response = fetch(url, { signal }); + if (response.ok && response.headers.get('Content-Type').includes('application/json')) { + return true; + } + else { + return false; + } + } catch (error) { + return false; + } } // Checks if the request corresponds to a JavaScript file function isJavaScript(requestUrl, trimmedURL, contentType) { - return (requestUrl.endsWith('.js') || trimmedURL.endsWith('.js') || contentType.includes('javascript')); + return (requestUrl.endsWith('.js') || trimmedURL.endsWith('.js') || contentType.includes('javascript')); } requests.forEach(request => { - const url = new URL(request.url); - const trimmedURL = url.origin + url.pathname; - const isScript = request.type === 'Script'; - const isDocument = request.type === 'Document'; - - let thirdPartyDomain = ''; - if (url.hostname !== firstPartyDomain) { - thirdPartyDomain = url.hostname; - } + const url = new URL(request.url); + const trimmedURL = url.origin + url.pathname; + const isScript = request.type === 'Script'; + const isDocument = request.type === 'Document'; - let jsUsage = false; - let headerUsage = false; - let receiverObserving = false; - let attestation = false; - let tldPlus1Attestation = false; + let thirdPartyDomain = ''; + if (url.hostname !== firstPartyDomain) { + thirdPartyDomain = url.hostname; + } - // If the request is to fetch a javascript file or HTML document, perform string search in the returned file content for usage of Topics API - // document.browsingTopics() [JS] or fetch() request call includes: {browsingTopics: true} or XHR request call includes: {deprecatedBrowsingTopics: true} - if (isScript || isJavaScript(request.url, trimmedURL, request.response_headers['Content-Type']) || isDocument) { - if (request.response_body && (request.response_body.includes('browsingTopics') || request.response_body.includes('BrowsingTopics'))) { - jsUsage = true; - } - } - - // Checking request header usage of Topics: header: 'Sec-Browsing-Topics: true' - if (request.request_headers.some(header => header.name === 'Sec-Browsing-Topics')) { - headerUsage = true; - // Checking is sent Topics are observed by the receiver using response header 'Observe-Browsing-Topics' - // If value is ?1 then they are observed else they are not observed - if (request.response_headers.some(header => header.name === 'Observe-Browsing-Topics' && header.value === '?1')) { - receiverObserving = true; - } - } + let jsUsage = false; + let headerUsage = false; + let receiverObserving = false; + let attestation = false; + let tldPlus1Attestation = false; - // Checking if the third party has published an attestation for Privacy Sandbox - if (thirdPartyDomain) { - attestation = fetchAndCheckAttestation(`${url.origin}/.well-known/privacy-sandbox-attestations.json`); - tldPlus1Origin = 'https://' + url.hostname.split('.').slice(-2).join('.'); - tldPlus1Attestation = fetchAndCheckAttestation(`${tldPlus1Origin}/.well-known/privacy-sandbox-attestations.json`); - } + // If the request is to fetch a javascript file or HTML document, perform string search in the returned file content for usage of Topics API + // document.browsingTopics() [JS] or fetch() request call includes: {browsingTopics: true} or XHR request call includes: {deprecatedBrowsingTopics: true} + if (isScript || isJavaScript(request.url, trimmedURL, request.response_headers['Content-Type']) || isDocument) { + if (request.response_body && (request.response_body.includes('browsingTopics') || request.response_body.includes('BrowsingTopics'))) { + jsUsage = true; + } + } - if (thirdPartyDomain && (jsUsage || headerUsage || attestationPublished)) { - result.thirdPartiesUsingBrowsingTopics[thirdPartyDomain] = { - topicsAccessJs: jsUsage, - topicsAccessHeader: headerUsage, - observingTopics: receiverObserving, - attestationPublished: attestation, - tldPlus1AttestationPublished: tldPlus1Attestation - }; - } + // Checking request header usage of Topics: header: 'Sec-Browsing-Topics: true' + if (request.request_headers.some(header => header.name === 'Sec-Browsing-Topics')) { + headerUsage = true; + // Checking is sent Topics are observed by the receiver using response header 'Observe-Browsing-Topics' + // If value is ?1 then they are observed else they are not observed + if (request.response_headers.some(header => header.name === 'Observe-Browsing-Topics' && header.value === '?1')) { + receiverObserving = true; + } + } + + // Checking if the third party has published an attestation for Privacy Sandbox + if (thirdPartyDomain) { + attestation = fetchAndCheckAttestation(`${url.origin}/.well-known/privacy-sandbox-attestations.json`); + tldPlus1Origin = 'https://' + url.hostname.split('.').slice(-2).join('.'); + tldPlus1Attestation = fetchAndCheckAttestation(`${tldPlus1Origin}/.well-known/privacy-sandbox-attestations.json`); + } + + if (thirdPartyDomain && (jsUsage || headerUsage || attestationPublished)) { + result.thirdPartiesUsingBrowsingTopics[thirdPartyDomain] = { + topicsAccessJs: jsUsage, + topicsAccessHeader: headerUsage, + observingTopics: receiverObserving, + attestationPublished: attestation, + tldPlus1AttestationPublished: tldPlus1Attestation + }; + } }); From 8d7c12b7a449a22071b0d411aba99c4e9ede4f2f Mon Sep 17 00:00:00 2001 From: Yash Vekaria Date: Thu, 23 May 2024 14:03:31 -0700 Subject: [PATCH 11/21] Adjusting left spacing --- dist/ads.js | 338 ++++++++++++++++++++-------------------- dist/privacy-sandbox.js | 122 +++++++-------- 2 files changed, 230 insertions(+), 230 deletions(-) diff --git a/dist/ads.js b/dist/ads.js index 845dbd4a..80aeb96f 100644 --- a/dist/ads.js +++ b/dist/ads.js @@ -6,188 +6,188 @@ const SELLER_TYPES = ['publisher', 'intermediary', 'both']; const isPresent = (response, endings) => response.ok && endings.some(ending => response.url.endsWith(ending)); const fetchAndParse = async (url, parser) => { - const timeout = 5000; - /* - Google's sellers.json size is 120Mb as of May 2024 - too big for custom metrics. - It's available at realtimebidding.google.com/sellers.json, so not part of crawled pages list. - More details: https://support.google.com/authorizedbuyers/answer/9895942 - */ - const controller = new AbortController(); - const { signal } = controller; - setTimeout(() => controller.abort(), timeout); - - try { - const response = await fetch(url, { signal }); - return parser(response); - } catch (error) { - return { - status: -1, - present: false, - error: error.message - }; - } + const timeout = 5000; + /* + Google's sellers.json size is 120Mb as of May 2024 - too big for custom metrics. + It's available at realtimebidding.google.com/sellers.json, so not part of crawled pages list. + More details: https://support.google.com/authorizedbuyers/answer/9895942 + */ + const controller = new AbortController(); + const { signal } = controller; + setTimeout(() => controller.abort(), timeout); + + try { + const response = await fetch(url, { signal }); + return parser(response); + } catch (error) { + return { + status: -1, + present: false, + error: error.message + }; + } }; // https://iabtechlab.com/wp-content/uploads/2022/04/Ads.txt-1.1.pdf const parseAdsTxt = async (response) => { - let content = await response.text(); - - let result = { - present: isPresent(response, ['/ads.txt', '/app-ads.txt']), - status: response.status, - redirected: response.redirected, - }; - - if (result.present && content) { - result = { - ...result, - ...{ - account_count: 0, - account_types: { - direct: { - domains: new Set(), - account_count: 0, - }, - reseller: { - domains: new Set(), - account_count: 0, - } - }, - line_count: 0, - variables: new Set(), - variable_count: 0 - } - }; - - // Clean up file content - content = content.replace(/#.*$/gm, ''); - content = content.replace(/\r/g, ''); - - let lines = content.split('\n'); - result.line_count = lines.length; - - for (let line of lines) { - // Variables - let variables = line.split('='); - if (variables.length == 2) { - result.variables.add(variables[0].trim().toLowerCase()); - result.variable_count += 1; - continue; - } - - // Account records - let relation_parts = line.split(','); - if (relation_parts.length >= 3) { - let type = relation_parts[2].trim().toLowerCase(); - if (['direct', 'reseller'].includes(type)) { - result.account_types[type].domains.add(relation_parts[0].trim()); - result.account_types[type].account_count += 1; - } - result.account_count += 1; - } - }; - - // Count unique and remove domain Sets for now - for (let accountType of Object.values(result.account_types)) { - accountType.domain_count = accountType.domains.size; - accountType.domains = Array.from(accountType.domains); // delete accountType.domains - } - result.variables = Array.from(result.variables); - } - - return result; + let content = await response.text(); + + let result = { + present: isPresent(response, ['/ads.txt', '/app-ads.txt']), + status: response.status, + redirected: response.redirected, + }; + + if (result.present && content) { + result = { + ...result, + ...{ + account_count: 0, + account_types: { + direct: { + domains: new Set(), + account_count: 0, + }, + reseller: { + domains: new Set(), + account_count: 0, + } + }, + line_count: 0, + variables: new Set(), + variable_count: 0 + } + }; + + // Clean up file content + content = content.replace(/#.*$/gm, ''); + content = content.replace(/\r/g, ''); + + let lines = content.split('\n'); + result.line_count = lines.length; + + for (let line of lines) { + // Variables + let variables = line.split('='); + if (variables.length == 2) { + result.variables.add(variables[0].trim().toLowerCase()); + result.variable_count += 1; + continue; + } + + // Account records + let relation_parts = line.split(','); + if (relation_parts.length >= 3) { + let type = relation_parts[2].trim().toLowerCase(); + if (['direct', 'reseller'].includes(type)) { + result.account_types[type].domains.add(relation_parts[0].trim()); + result.account_types[type].account_count += 1; + } + result.account_count += 1; + } + }; + + // Count unique and remove domain Sets for now + for (let accountType of Object.values(result.account_types)) { + accountType.domain_count = accountType.domains.size; + accountType.domains = Array.from(accountType.domains); // delete accountType.domains + } + result.variables = Array.from(result.variables); + } + + return result; } // https://iabtechlab.com/wp-content/uploads/2019/07/Sellers.json_Final.pdf const parseSellersJSON = async (response) => { - let content; - try { - content = JSON.parse(await response.text()); - } catch { - content = null; - } - let result = { - present: isPresent(response, ['/sellers.json']), - redirected: response.redirected, - status: response.status, - }; - - if (result.present && content) { - result = { - ...result, - ...{ - seller_count: 0, - seller_types: { - publisher: { - domains: new Set(), - seller_count: 0, - }, - intermediary: { - domains: new Set(), - seller_count: 0, - }, - both: { - domains: new Set(), - seller_count: 0, - } - }, - passthrough_count: 0, - confidential_count: 0 - } - }; - - // Clean up file content - result.seller_count = content.sellers.length; - - for (let seller of content.sellers) { - const stype = seller.seller_type.trim().toLowerCase(); - // Validating records - if (!SELLER_TYPES.includes(stype) || !seller.seller_id) { - continue; - } - - // Passthrough - if (seller.is_passthrough) { - result.passthrough_count += 1; - } - - // Confidential - if (seller.is_confidential) { - result.confidential_count += 1; - } - - // Seller records - if (seller.domain) { - const domain = seller.domain.trim().toLowerCase(); - result.seller_types[stype].domains.add(domain); - result.seller_types[stype].seller_count += 1; - } - } - - // Count unique and remove domain Sets for now - for (let seller_type of Object.values(result.seller_types)) { - seller_type.domain_count = seller_type.domains.size; - seller_type.domains = Array.from(seller_type.domains); // delete seller_type.domains; - } - }; - - return result; + let content; + try { + content = JSON.parse(await response.text()); + } catch { + content = null; + } + let result = { + present: isPresent(response, ['/sellers.json']), + redirected: response.redirected, + status: response.status, + }; + + if (result.present && content) { + result = { + ...result, + ...{ + seller_count: 0, + seller_types: { + publisher: { + domains: new Set(), + seller_count: 0, + }, + intermediary: { + domains: new Set(), + seller_count: 0, + }, + both: { + domains: new Set(), + seller_count: 0, + } + }, + passthrough_count: 0, + confidential_count: 0 + } + }; + + // Clean up file content + result.seller_count = content.sellers.length; + + for (let seller of content.sellers) { + const stype = seller.seller_type.trim().toLowerCase(); + // Validating records + if (!SELLER_TYPES.includes(stype) || !seller.seller_id) { + continue; + } + + // Passthrough + if (seller.is_passthrough) { + result.passthrough_count += 1; + } + + // Confidential + if (seller.is_confidential) { + result.confidential_count += 1; + } + + // Seller records + if (seller.domain) { + const domain = seller.domain.trim().toLowerCase(); + result.seller_types[stype].domains.add(domain); + result.seller_types[stype].seller_count += 1; + } + } + + // Count unique and remove domain Sets for now + for (let seller_type of Object.values(result.seller_types)) { + seller_type.domain_count = seller_type.domains.size; + seller_type.domains = Array.from(seller_type.domains); // delete seller_type.domains; + } + }; + + return result; } return Promise.all([ - fetchAndParse("/ads.txt", parseAdsTxt).catch(e => e), - fetchAndParse("/app-ads.txt", parseAdsTxt).catch(e => e), - fetchAndParse("/sellers.json", parseSellersJSON).catch(e => e), + fetchAndParse("/ads.txt", parseAdsTxt).catch(e => e), + fetchAndParse("/app-ads.txt", parseAdsTxt).catch(e => e), + fetchAndParse("/sellers.json", parseSellersJSON).catch(e => e), ]).then((all_data) => { - return JSON.stringify({ - ads: all_data[0], - app_ads: all_data[1], - sellers: all_data[2] - }); + return JSON.stringify({ + ads: all_data[0], + app_ads: all_data[1], + sellers: all_data[2] + }); }).catch(error => { - return JSON.stringify({ - error: error.message - }); + return JSON.stringify({ + error: error.message + }); }); \ No newline at end of file diff --git a/dist/privacy-sandbox.js b/dist/privacy-sandbox.js index be52f486..075ea491 100644 --- a/dist/privacy-sandbox.js +++ b/dist/privacy-sandbox.js @@ -7,84 +7,84 @@ const testURL = new URL($WPT_URL); let firstPartyDomain = testURL.hostname; result = { - topicsAvailable: document.featurePolicy.allowsFeature('browsing-topics'), - thirdPartiesUsingBrowsingTopics: {} + topicsAvailable: document.featurePolicy.allowsFeature('browsing-topics'), + thirdPartiesUsingBrowsingTopics: {} } function fetchAndCheckAttestation(url) { - const controller = new AbortController(); - const { signal } = controller; - setTimeout(() => controller.abort(), 5000); + const controller = new AbortController(); + const { signal } = controller; + setTimeout(() => controller.abort(), 5000); - try { - const response = fetch(url, { signal }); - if (response.ok && response.headers.get('Content-Type').includes('application/json')) { - return true; - } - else { - return false; - } - } catch (error) { - return false; - } + try { + const response = fetch(url, { signal }); + if (response.ok && response.headers.get('Content-Type').includes('application/json')) { + return true; + } + else { + return false; + } + } catch (error) { + return false; + } } // Checks if the request corresponds to a JavaScript file function isJavaScript(requestUrl, trimmedURL, contentType) { - return (requestUrl.endsWith('.js') || trimmedURL.endsWith('.js') || contentType.includes('javascript')); + return (requestUrl.endsWith('.js') || trimmedURL.endsWith('.js') || contentType.includes('javascript')); } requests.forEach(request => { - const url = new URL(request.url); - const trimmedURL = url.origin + url.pathname; - const isScript = request.type === 'Script'; - const isDocument = request.type === 'Document'; + const url = new URL(request.url); + const trimmedURL = url.origin + url.pathname; + const isScript = request.type === 'Script'; + const isDocument = request.type === 'Document'; - let thirdPartyDomain = ''; - if (url.hostname !== firstPartyDomain) { - thirdPartyDomain = url.hostname; - } + let thirdPartyDomain = ''; + if (url.hostname !== firstPartyDomain) { + thirdPartyDomain = url.hostname; + } - let jsUsage = false; - let headerUsage = false; - let receiverObserving = false; - let attestation = false; - let tldPlus1Attestation = false; + let jsUsage = false; + let headerUsage = false; + let receiverObserving = false; + let attestation = false; + let tldPlus1Attestation = false; - // If the request is to fetch a javascript file or HTML document, perform string search in the returned file content for usage of Topics API - // document.browsingTopics() [JS] or fetch() request call includes: {browsingTopics: true} or XHR request call includes: {deprecatedBrowsingTopics: true} - if (isScript || isJavaScript(request.url, trimmedURL, request.response_headers['Content-Type']) || isDocument) { - if (request.response_body && (request.response_body.includes('browsingTopics') || request.response_body.includes('BrowsingTopics'))) { - jsUsage = true; - } - } + // If the request is to fetch a javascript file or HTML document, perform string search in the returned file content for usage of Topics API + // document.browsingTopics() [JS] or fetch() request call includes: {browsingTopics: true} or XHR request call includes: {deprecatedBrowsingTopics: true} + if (isScript || isJavaScript(request.url, trimmedURL, request.response_headers['Content-Type']) || isDocument) { + if (request.response_body && (request.response_body.includes('browsingTopics') || request.response_body.includes('BrowsingTopics'))) { + jsUsage = true; + } + } - // Checking request header usage of Topics: header: 'Sec-Browsing-Topics: true' - if (request.request_headers.some(header => header.name === 'Sec-Browsing-Topics')) { - headerUsage = true; - // Checking is sent Topics are observed by the receiver using response header 'Observe-Browsing-Topics' - // If value is ?1 then they are observed else they are not observed - if (request.response_headers.some(header => header.name === 'Observe-Browsing-Topics' && header.value === '?1')) { - receiverObserving = true; - } - } + // Checking request header usage of Topics: header: 'Sec-Browsing-Topics: true' + if (request.request_headers.some(header => header.name === 'Sec-Browsing-Topics')) { + headerUsage = true; + // Checking is sent Topics are observed by the receiver using response header 'Observe-Browsing-Topics' + // If value is ?1 then they are observed else they are not observed + if (request.response_headers.some(header => header.name === 'Observe-Browsing-Topics' && header.value === '?1')) { + receiverObserving = true; + } + } - // Checking if the third party has published an attestation for Privacy Sandbox - if (thirdPartyDomain) { - attestation = fetchAndCheckAttestation(`${url.origin}/.well-known/privacy-sandbox-attestations.json`); - tldPlus1Origin = 'https://' + url.hostname.split('.').slice(-2).join('.'); - tldPlus1Attestation = fetchAndCheckAttestation(`${tldPlus1Origin}/.well-known/privacy-sandbox-attestations.json`); - } + // Checking if the third party has published an attestation for Privacy Sandbox + if (thirdPartyDomain) { + attestation = fetchAndCheckAttestation(`${url.origin}/.well-known/privacy-sandbox-attestations.json`); + tldPlus1Origin = 'https://' + url.hostname.split('.').slice(-2).join('.'); + tldPlus1Attestation = fetchAndCheckAttestation(`${tldPlus1Origin}/.well-known/privacy-sandbox-attestations.json`); + } - if (thirdPartyDomain && (jsUsage || headerUsage || attestationPublished)) { - result.thirdPartiesUsingBrowsingTopics[thirdPartyDomain] = { - topicsAccessJs: jsUsage, - topicsAccessHeader: headerUsage, - observingTopics: receiverObserving, - attestationPublished: attestation, - tldPlus1AttestationPublished: tldPlus1Attestation - }; - } + if (thirdPartyDomain && (jsUsage || headerUsage || attestationPublished)) { + result.thirdPartiesUsingBrowsingTopics[thirdPartyDomain] = { + topicsAccessJs: jsUsage, + topicsAccessHeader: headerUsage, + observingTopics: receiverObserving, + attestationPublished: attestation, + tldPlus1AttestationPublished: tldPlus1Attestation + }; + } }); From 490d0c534ca49d82f514b92dd27bc45e1c4ed049 Mon Sep 17 00:00:00 2001 From: Yash Vekaria Date: Thu, 23 May 2024 14:06:44 -0700 Subject: [PATCH 12/21] Ading line endings to the files --- dist/ads.js | 2 +- dist/privacy-sandbox.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dist/ads.js b/dist/ads.js index 80aeb96f..6af647aa 100644 --- a/dist/ads.js +++ b/dist/ads.js @@ -190,4 +190,4 @@ return Promise.all([ return JSON.stringify({ error: error.message }); -}); \ No newline at end of file +}); diff --git a/dist/privacy-sandbox.js b/dist/privacy-sandbox.js index 075ea491..da4a8d94 100644 --- a/dist/privacy-sandbox.js +++ b/dist/privacy-sandbox.js @@ -88,4 +88,4 @@ requests.forEach(request => { }); -return result; \ No newline at end of file +return result; From d56f339ee03ecf98321a14d1031fe5b2c997c369 Mon Sep 17 00:00:00 2001 From: Yash Vekaria Date: Thu, 23 May 2024 18:09:48 -0700 Subject: [PATCH 13/21] Simplified the implementation --- dist/privacy-sandbox.js | 111 ++++++++++++++++++---------------------- 1 file changed, 49 insertions(+), 62 deletions(-) diff --git a/dist/privacy-sandbox.js b/dist/privacy-sandbox.js index da4a8d94..c9dea8b1 100644 --- a/dist/privacy-sandbox.js +++ b/dist/privacy-sandbox.js @@ -3,89 +3,76 @@ // Header Usage Reference: https://developers.google.com/privacy-sandbox/relevance/topics/demo#the-topics-api-demo let requests = $WPT_BODIES; -const testURL = new URL($WPT_URL); -let firstPartyDomain = testURL.hostname; +const firstPartyDomain = document.location.hostname; -result = { +let result = { topicsAvailable: document.featurePolicy.allowsFeature('browsing-topics'), thirdPartiesUsingBrowsingTopics: {} } -function fetchAndCheckAttestation(url) { +async function fetchAndCheckAttestation(url) { const controller = new AbortController(); const { signal } = controller; setTimeout(() => controller.abort(), 5000); try { - const response = fetch(url, { signal }); - if (response.ok && response.headers.get('Content-Type').includes('application/json')) { - return true; - } - else { - return false; - } + const response = await fetch(url, { signal, mode: 'no-cors' }); + return response ? true : false; } catch (error) { return false; } } -// Checks if the request corresponds to a JavaScript file -function isJavaScript(requestUrl, trimmedURL, contentType) { - return (requestUrl.endsWith('.js') || trimmedURL.endsWith('.js') || contentType.includes('javascript')); -} - -requests.forEach(request => { - const url = new URL(request.url); - const trimmedURL = url.origin + url.pathname; - const isScript = request.type === 'Script'; - const isDocument = request.type === 'Document'; +let seenThirdParties = []; +(async () => { + for (const request of requests) { + const url = new URL(request.url); + const isScript = request.type === 'Script'; + const isDocument = request.type === 'Document'; - let thirdPartyDomain = ''; - if (url.hostname !== firstPartyDomain) { - thirdPartyDomain = url.hostname; - } + let thirdPartyDomain = ''; + if (url.hostname !== firstPartyDomain) { + thirdPartyDomain = url.hostname; + } + if (thirdPartyDomain && seenThirdParties.includes(thirdPartyDomain)) { + continue; + } + seenThirdParties.push(thirdPartyDomain); - let jsUsage = false; - let headerUsage = false; - let receiverObserving = false; - let attestation = false; - let tldPlus1Attestation = false; + let jsUsage = false; + let headerUsage = false; + let receiverObserving = false; - // If the request is to fetch a javascript file or HTML document, perform string search in the returned file content for usage of Topics API - // document.browsingTopics() [JS] or fetch() request call includes: {browsingTopics: true} or XHR request call includes: {deprecatedBrowsingTopics: true} - if (isScript || isJavaScript(request.url, trimmedURL, request.response_headers['Content-Type']) || isDocument) { - if (request.response_body && (request.response_body.includes('browsingTopics') || request.response_body.includes('BrowsingTopics'))) { - jsUsage = true; + // If the request is to fetch a javascript file or HTML document, perform string search in the returned file content for usage of Topics API + // document.browsingTopics() [JS] or fetch() request call includes: {browsingTopics: true} or XHR request call includes: {deprecatedBrowsingTopics: true} + if (isScript || isDocument) { + if (request.response_body && (request.response_body.includes('browsingTopics') || request.response_body.includes('BrowsingTopics'))) { + jsUsage = true; + } } - } - // Checking request header usage of Topics: header: 'Sec-Browsing-Topics: true' - if (request.request_headers.some(header => header.name === 'Sec-Browsing-Topics')) { - headerUsage = true; - // Checking is sent Topics are observed by the receiver using response header 'Observe-Browsing-Topics' - // If value is ?1 then they are observed else they are not observed - if (request.response_headers.some(header => header.name === 'Observe-Browsing-Topics' && header.value === '?1')) { - receiverObserving = true; + // Checking request header usage of Topics: header: 'Sec-Browsing-Topics: true' + if ('Sec-Browsing-Topics' in request.request_headers) { + headerUsage = true; + // Checking is sent Topics are observed by the receiver using response header 'Observe-Browsing-Topics' + // If value is ?1 then they are observed else they are not observed + if (request.response_headers['Observe-Browsing-Topics'] === '?1') { + receiverObserving = true; + } } - } - - // Checking if the third party has published an attestation for Privacy Sandbox - if (thirdPartyDomain) { - attestation = fetchAndCheckAttestation(`${url.origin}/.well-known/privacy-sandbox-attestations.json`); - tldPlus1Origin = 'https://' + url.hostname.split('.').slice(-2).join('.'); - tldPlus1Attestation = fetchAndCheckAttestation(`${tldPlus1Origin}/.well-known/privacy-sandbox-attestations.json`); - } - if (thirdPartyDomain && (jsUsage || headerUsage || attestationPublished)) { - result.thirdPartiesUsingBrowsingTopics[thirdPartyDomain] = { - topicsAccessJs: jsUsage, - topicsAccessHeader: headerUsage, - observingTopics: receiverObserving, - attestationPublished: attestation, - tldPlus1AttestationPublished: tldPlus1Attestation - }; - } - -}); + if (thirdPartyDomain && (jsUsage || headerUsage)) { + // Checking if the third party has published an attestation for Privacy Sandbox + const tldPlus1Origin = 'https://' + url.hostname.split('.').slice(-2).join('.'); + result.thirdPartiesUsingBrowsingTopics[thirdPartyDomain] = { + topicsAccessJs: jsUsage, + topicsAccessHeader: headerUsage, + observingTopics: receiverObserving, + attestationPublished: await fetchAndCheckAttestation(`${url.origin}/.well-known/privacy-sandbox-attestations.json`), + tldPlus1AttestationPublished: await fetchAndCheckAttestation(`${tldPlus1Origin}/.well-known/privacy-sandbox-attestations.json`) + }; + } + }; +})(); return result; From a4cfe2bb5978661719202ad90fbe82cd51919143 Mon Sep 17 00:00:00 2001 From: Yash Vekaria Date: Fri, 24 May 2024 01:53:00 -0700 Subject: [PATCH 14/21] Added deprecatedBrowsingTopics --- dist/privacy-sandbox.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dist/privacy-sandbox.js b/dist/privacy-sandbox.js index c9dea8b1..3a3972ad 100644 --- a/dist/privacy-sandbox.js +++ b/dist/privacy-sandbox.js @@ -44,9 +44,9 @@ let seenThirdParties = []; let receiverObserving = false; // If the request is to fetch a javascript file or HTML document, perform string search in the returned file content for usage of Topics API - // document.browsingTopics() [JS] or fetch() request call includes: {browsingTopics: true} or XHR request call includes: {deprecatedBrowsingTopics: true} + // document.browsingTopics() [JS] or fetch() request call includes: {browsingTopics: true} or XHR request call includes: {deprecatedBrowsingTopics: true} (to be deprecated) if (isScript || isDocument) { - if (request.response_body && (request.response_body.includes('browsingTopics') || request.response_body.includes('BrowsingTopics'))) { + if (request.response_body && (request.response_body.includes('browsingTopics') || request.response_body.includes('deprecatedBrowsingTopics'))) { jsUsage = true; } } From 022a08505a6da6975302ce06aa136682cb2c0cf3 Mon Sep 17 00:00:00 2001 From: Yash Vekaria Date: Fri, 24 May 2024 14:09:09 -0700 Subject: [PATCH 15/21] Added cannonical domain based third-party detection --- dist/privacy-sandbox.js | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/dist/privacy-sandbox.js b/dist/privacy-sandbox.js index 3a3972ad..785320c5 100644 --- a/dist/privacy-sandbox.js +++ b/dist/privacy-sandbox.js @@ -3,13 +3,22 @@ // Header Usage Reference: https://developers.google.com/privacy-sandbox/relevance/topics/demo#the-topics-api-demo let requests = $WPT_BODIES; -const firstPartyDomain = document.location.hostname; +const cannonicalFirstPartyDomain = getCanonicalDomain(document.location.hostname); let result = { topicsAvailable: document.featurePolicy.allowsFeature('browsing-topics'), thirdPartiesUsingBrowsingTopics: {} } +function getCanonicalDomain(hostname) { + const parts = hostname.split('.').reverse(); + if (parts.length >= 2) { + return `${parts[1]}.${parts[0]}`; + } else { + return hostname; + } +} + async function fetchAndCheckAttestation(url) { const controller = new AbortController(); const { signal } = controller; @@ -31,7 +40,8 @@ let seenThirdParties = []; const isDocument = request.type === 'Document'; let thirdPartyDomain = ''; - if (url.hostname !== firstPartyDomain) { + // Maps all first party hostnames to the corresponding cannonical first party domain to truly consider only third-parties + if (getCanonicalDomain(url.hostname) !== cannonicalFirstPartyDomain) { thirdPartyDomain = url.hostname; } if (thirdPartyDomain && seenThirdParties.includes(thirdPartyDomain)) { From d8639e5c08d059879dd7aecfa133f16afa875633 Mon Sep 17 00:00:00 2001 From: Yash Vekaria Date: Fri, 24 May 2024 16:50:45 -0700 Subject: [PATCH 16/21] Added attribution-specific extendable structure --- dist/privacy-sandbox.js | 52 ++++++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 22 deletions(-) diff --git a/dist/privacy-sandbox.js b/dist/privacy-sandbox.js index 785320c5..bcd45588 100644 --- a/dist/privacy-sandbox.js +++ b/dist/privacy-sandbox.js @@ -6,8 +6,17 @@ let requests = $WPT_BODIES; const cannonicalFirstPartyDomain = getCanonicalDomain(document.location.hostname); let result = { - topicsAvailable: document.featurePolicy.allowsFeature('browsing-topics'), - thirdPartiesUsingBrowsingTopics: {} + 'topicsAPI': { + 'topicsAvailable': document.featurePolicy.allowsFeature('browsing-topics'), + 'usingBrowsingTopics': [], + 'topicsAccessJs': [], + 'topicsAccessHeader': [], + 'observingTopics': [] + }, + 'protected_audience': { + // Protected Audience API metrics + }, + attestationPublished: [] } function getCanonicalDomain(hostname) { @@ -38,10 +47,11 @@ let seenThirdParties = []; const url = new URL(request.url); const isScript = request.type === 'Script'; const isDocument = request.type === 'Document'; + const cannonicalRequestDomain = getCanonicalDomain(url.hostname); let thirdPartyDomain = ''; - // Maps all first party hostnames to the corresponding cannonical first party domain to truly consider only third-parties - if (getCanonicalDomain(url.hostname) !== cannonicalFirstPartyDomain) { + // Maps all first party hostnames to the corresponding cannonical first party domain to truly consider only third-parties + if (cannonicalRequestDomain !== cannonicalFirstPartyDomain) { thirdPartyDomain = url.hostname; } if (thirdPartyDomain && seenThirdParties.includes(thirdPartyDomain)) { @@ -49,40 +59,38 @@ let seenThirdParties = []; } seenThirdParties.push(thirdPartyDomain); - let jsUsage = false; - let headerUsage = false; - let receiverObserving = false; - // If the request is to fetch a javascript file or HTML document, perform string search in the returned file content for usage of Topics API // document.browsingTopics() [JS] or fetch() request call includes: {browsingTopics: true} or XHR request call includes: {deprecatedBrowsingTopics: true} (to be deprecated) if (isScript || isDocument) { if (request.response_body && (request.response_body.includes('browsingTopics') || request.response_body.includes('deprecatedBrowsingTopics'))) { - jsUsage = true; + result['topicsAPI']['topicsAccessJs'].push(thirdPartyDomain); } } // Checking request header usage of Topics: header: 'Sec-Browsing-Topics: true' if ('Sec-Browsing-Topics' in request.request_headers) { - headerUsage = true; + result['topicsAPI']['topicsAccessHeader'].push(thirdPartyDomain); // Checking is sent Topics are observed by the receiver using response header 'Observe-Browsing-Topics' // If value is ?1 then they are observed else they are not observed if (request.response_headers['Observe-Browsing-Topics'] === '?1') { - receiverObserving = true; + result['topicsAPI']['observingTopics'].push(thirdPartyDomain); } } - if (thirdPartyDomain && (jsUsage || headerUsage)) { - // Checking if the third party has published an attestation for Privacy Sandbox - const tldPlus1Origin = 'https://' + url.hostname.split('.').slice(-2).join('.'); - result.thirdPartiesUsingBrowsingTopics[thirdPartyDomain] = { - topicsAccessJs: jsUsage, - topicsAccessHeader: headerUsage, - observingTopics: receiverObserving, - attestationPublished: await fetchAndCheckAttestation(`${url.origin}/.well-known/privacy-sandbox-attestations.json`), - tldPlus1AttestationPublished: await fetchAndCheckAttestation(`${tldPlus1Origin}/.well-known/privacy-sandbox-attestations.json`) - }; + const attestationPublished = await fetchAndCheckAttestation(`${url.origin}/.well-known/privacy-sandbox-attestations.json`); + if (attestationPublished) { + result.attestationPublished.push(thirdPartyDomain); + } + const tldPlus1AttestationPublished = await fetchAndCheckAttestation(`${'https://'+cannonicalRequestDomain}/.well-known/privacy-sandbox-attestations.json`); + if (tldPlus1AttestationPublished) { + result.attestationPublished.push(cannonicalRequestDomain); } - }; + } + + result['topicsAPI']['usingBrowsingTopics'] = Array.from(new Set([ + ...result['topicsAPI']['topicsAccessJs'], + ...result['topicsAPI']['topicsAccessHeader'] + ])); })(); return result; From 9def5653861166fc1488316b2a9519a99d3634f8 Mon Sep 17 00:00:00 2001 From: Yash Vekaria Date: Wed, 5 Jun 2024 10:20:58 -0700 Subject: [PATCH 17/21] Fixed bugs --- dist/privacy-sandbox.js | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/dist/privacy-sandbox.js b/dist/privacy-sandbox.js index bcd45588..d9b145e4 100644 --- a/dist/privacy-sandbox.js +++ b/dist/privacy-sandbox.js @@ -1,6 +1,11 @@ //[privacy-sandbox] -// Topics API Usage Reference: https://developers.google.com/privacy-sandbox/relevance/topics/demo#the-topics-api-demo -// Header Usage Reference: https://developers.google.com/privacy-sandbox/relevance/topics/demo#the-topics-api-demo +/* +Topics API Usage Reference: https://developers.google.com/privacy-sandbox/relevance/topics/demo#the-topics-api-demo + +Header Usage Reference: https://developers.google.com/privacy-sandbox/relevance/topics/demo#the-topics-api-demo + +Required command line flags: --enable-features=BrowsingTopics,InterestGroupStorage,PrivacySandboxAdsAPIsOverride +*/ let requests = $WPT_BODIES; const cannonicalFirstPartyDomain = getCanonicalDomain(document.location.hostname); @@ -13,9 +18,7 @@ let result = { 'topicsAccessHeader': [], 'observingTopics': [] }, - 'protected_audience': { - // Protected Audience API metrics - }, + 'protected_audience': {}, attestationPublished: [] } @@ -44,9 +47,10 @@ async function fetchAndCheckAttestation(url) { let seenThirdParties = []; (async () => { for (const request of requests) { - const url = new URL(request.url); - const isScript = request.type === 'Script'; - const isDocument = request.type === 'Document'; + const url = new URL(request.full_url); + // FIX THIS --- IS IT request.request_type OR request.type? + const isScript = request.request_type === 'Script'; + const isDocument = request.request_type === 'Document'; const cannonicalRequestDomain = getCanonicalDomain(url.hostname); let thirdPartyDomain = ''; @@ -68,23 +72,21 @@ let seenThirdParties = []; } // Checking request header usage of Topics: header: 'Sec-Browsing-Topics: true' - if ('Sec-Browsing-Topics' in request.request_headers) { + if ('sec-browsing-topics' in request.request_headers.toLowerCase()) { result['topicsAPI']['topicsAccessHeader'].push(thirdPartyDomain); - // Checking is sent Topics are observed by the receiver using response header 'Observe-Browsing-Topics' - // If value is ?1 then they are observed else they are not observed - if (request.response_headers['Observe-Browsing-Topics'] === '?1') { - result['topicsAPI']['observingTopics'].push(thirdPartyDomain); - } + } + + // Checking if sent Topics in the request (either via JS or headers) are observed by the receiver using response header 'Observe-Browsing-Topics' + // If value is ?1 then they are observed else they are not observed + let respHeaders = new Map(Object.entries(request.response_headers).map(([key, value]) => [key.toLowerCase(), value])); + if (respHeaders.has('observe-browsing-topics') && respHeaders.get('observe-browsing-topics') === "?1") { + result['topicsAPI']['observingTopics'].push(thirdPartyDomain); } const attestationPublished = await fetchAndCheckAttestation(`${url.origin}/.well-known/privacy-sandbox-attestations.json`); if (attestationPublished) { result.attestationPublished.push(thirdPartyDomain); } - const tldPlus1AttestationPublished = await fetchAndCheckAttestation(`${'https://'+cannonicalRequestDomain}/.well-known/privacy-sandbox-attestations.json`); - if (tldPlus1AttestationPublished) { - result.attestationPublished.push(cannonicalRequestDomain); - } } result['topicsAPI']['usingBrowsingTopics'] = Array.from(new Set([ From cf4eb5fb4120333ff3558205651520ee1bdb7da0 Mon Sep 17 00:00:00 2001 From: Yash Vekaria Date: Wed, 5 Jun 2024 14:40:12 -0700 Subject: [PATCH 18/21] Added Attribution Reporting API custom metric --- dist/privacy-sandbox.js | 66 ++++++++++++++++++++++++++++++++++------- 1 file changed, 56 insertions(+), 10 deletions(-) diff --git a/dist/privacy-sandbox.js b/dist/privacy-sandbox.js index d9b145e4..a1e97362 100644 --- a/dist/privacy-sandbox.js +++ b/dist/privacy-sandbox.js @@ -1,10 +1,9 @@ //[privacy-sandbox] -/* -Topics API Usage Reference: https://developers.google.com/privacy-sandbox/relevance/topics/demo#the-topics-api-demo - -Header Usage Reference: https://developers.google.com/privacy-sandbox/relevance/topics/demo#the-topics-api-demo - -Required command line flags: --enable-features=BrowsingTopics,InterestGroupStorage,PrivacySandboxAdsAPIsOverride +/** + * Topics API + * Protected Audience API + * Attribution Reporting API + * Required command line flags: --enable-features=BrowsingTopics,InterestGroupStorage,PrivacySandboxAdsAPIsOverride */ let requests = $WPT_BODIES; @@ -18,7 +17,18 @@ let result = { 'topicsAccessHeader': [], 'observingTopics': [] }, - 'protected_audience': {}, + 'protectedAudienceAPI': {}, + 'attributionReportingAPI': { + 'attributionReportingAvailable': document.featurePolicy.allowsFeature('attribution-reporting'), + 'attributionReportingEligibleHeader': { + 'sentByBrowser': false, + 'sentTo': [], + }, + 'completedRegistrations': { + 'AttributionReportingRegisterSourceHeader': {}, + 'AttributionReportingRegisterTriggerHeader': [] + }, + }, attestationPublished: [] } @@ -48,9 +58,8 @@ let seenThirdParties = []; (async () => { for (const request of requests) { const url = new URL(request.full_url); - // FIX THIS --- IS IT request.request_type OR request.type? - const isScript = request.request_type === 'Script'; - const isDocument = request.request_type === 'Document'; + const isScript = request.type === 'Script'; + const isDocument = request.type === 'Document'; const cannonicalRequestDomain = getCanonicalDomain(url.hostname); let thirdPartyDomain = ''; @@ -63,6 +72,12 @@ let seenThirdParties = []; } seenThirdParties.push(thirdPartyDomain); + /** + * Topics API + * API Usage Reference: https://developers.google.com/privacy-sandbox/relevance/topics/demo#the-topics-api-demo + * Header Usage Reference: https://developers.google.com/privacy-sandbox/relevance/topics/demo#the-topics-api-demo + */ + // If the request is to fetch a javascript file or HTML document, perform string search in the returned file content for usage of Topics API // document.browsingTopics() [JS] or fetch() request call includes: {browsingTopics: true} or XHR request call includes: {deprecatedBrowsingTopics: true} (to be deprecated) if (isScript || isDocument) { @@ -87,12 +102,43 @@ let seenThirdParties = []; if (attestationPublished) { result.attestationPublished.push(thirdPartyDomain); } + + /** + * Attribution Reporting API + * https://developer.mozilla.org/en-US/docs/Web/API/Attribution_Reporting_API + */ + + // Checking if the request header includes 'Attribution-Reporting-Eligible' to initiate the registration of source or trigger + if ('attribution-reporting-eligible' in request.request_headers.toLowerCase()) { + result['attributionReportingAPI']['attributionReportingEligibleHeader']['sentTo'].push(cannonicalRequestDomain); + } + + // Checking if the response header includes 'Attribution-Reporting-Register-Source' or 'Attribution-Reporting-Register-Trigger' to complete registration of source or trigger + // Source registration happens on seller (e.g., publisher) website where impression is registered and trigger registration happens on buyer (e.g., advertiser) website where conversion completes. + // Each entry in result['attributionReportingAPI']['completedRegistrations']['AttributionReportingRegisterSourceHeader'] is represented as {cannonicalRequestDomain: {"destination": "", "eventEpsilon": 0}} + // Higher the epsilon, the more the privacy protection + if ('attribution-reporting-register-source' in respHeaders) { + jsonString = respHeaders.get('attribution-reporting-register-source'); + const { destination, event_level_epsilon } = JSON.parse(jsonString); + if (!result['attributionReportingAPI']['completedRegistrations']['AttributionReportingRegisterSourceHeader'][cannonicalRequestDomain]) { + result['attributionReportingAPI']['completedRegistrations']['AttributionReportingRegisterSourceHeader'][cannonicalRequestDomain] = []; + } + result['attributionReportingAPI']['completedRegistrations']['AttributionReportingRegisterSourceHeader'][cannonicalRequestDomain].push({"destination": destination, "eventEpsilon": event_level_epsilon}); + } else if ('attribution-reporting-register-trigger' in respHeaders) { + result['attributionReportingAPI']['completedRegistrations']['AttributionReportingRegisterTriggerHeader'].push(cannonicalRequestDomain); + } } result['topicsAPI']['usingBrowsingTopics'] = Array.from(new Set([ ...result['topicsAPI']['topicsAccessJs'], ...result['topicsAPI']['topicsAccessHeader'] ])); + + // if "Attribution-Reporting-Eligible" request header is sent to more than one domains, set sentByBrowser to true + if (result['attributionReportingAPI']['attributionReportingEligibleHeader']['sentTo'].length > 0) { + result['attributionReportingAPI']['attributionReportingEligibleHeader']['sentByBrowser'] = true; + } + })(); return result; From 3ad1a7bca574d6f22bd8d7e7b2ef98539dad3526 Mon Sep 17 00:00:00 2001 From: Yash Vekaria Date: Wed, 5 Jun 2024 14:43:54 -0700 Subject: [PATCH 19/21] Added Attribution Reporting API custom metric --- dist/privacy-sandbox.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dist/privacy-sandbox.js b/dist/privacy-sandbox.js index a1e97362..75a4abdb 100644 --- a/dist/privacy-sandbox.js +++ b/dist/privacy-sandbox.js @@ -73,7 +73,7 @@ let seenThirdParties = []; seenThirdParties.push(thirdPartyDomain); /** - * Topics API + * Topics API * API Usage Reference: https://developers.google.com/privacy-sandbox/relevance/topics/demo#the-topics-api-demo * Header Usage Reference: https://developers.google.com/privacy-sandbox/relevance/topics/demo#the-topics-api-demo */ @@ -104,10 +104,10 @@ let seenThirdParties = []; } /** - * Attribution Reporting API + * Attribution Reporting API * https://developer.mozilla.org/en-US/docs/Web/API/Attribution_Reporting_API */ - + // Checking if the request header includes 'Attribution-Reporting-Eligible' to initiate the registration of source or trigger if ('attribution-reporting-eligible' in request.request_headers.toLowerCase()) { result['attributionReportingAPI']['attributionReportingEligibleHeader']['sentTo'].push(cannonicalRequestDomain); From 6a9c9b7c5fd1c3ef4fc0be070c755adf5365f2ba Mon Sep 17 00:00:00 2001 From: Yash Vekaria Date: Wed, 5 Jun 2024 19:54:07 -0700 Subject: [PATCH 20/21] Added Protected Audience API custom metric --- dist/privacy-sandbox.js | 52 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/dist/privacy-sandbox.js b/dist/privacy-sandbox.js index 75a4abdb..82dfeec6 100644 --- a/dist/privacy-sandbox.js +++ b/dist/privacy-sandbox.js @@ -17,7 +17,20 @@ let result = { 'topicsAccessHeader': [], 'observingTopics': [] }, - 'protectedAudienceAPI': {}, + 'protectedAudienceAPI': { + 'protectedAudienceAvailable': document.featurePolicy.allowsFeature('join-ad-interest-group'), + 'interestGroups': { + 'joinAdInterestGroup': [], + 'leaveAdInterestGroup': [], + 'updateAdInterestGroups': [], + 'clearOriginJoinedAdInterestGroups': [] + }, + 'runAdAuction': [], + 'generateBid': [], + 'scoreAd': [], + 'reportWin': [], + 'reportResult': [] + }, 'attributionReportingAPI': { 'attributionReportingAvailable': document.featurePolicy.allowsFeature('attribution-reporting'), 'attributionReportingEligibleHeader': { @@ -103,6 +116,43 @@ let seenThirdParties = []; result.attestationPublished.push(thirdPartyDomain); } + /** + * Protected Audience API + * https://github.com/WICG/turtledove/blob/main/FLEDGE.md + * https://developers.google.com/privacy-sandbox/relevance/protected-audience + */ + + // Checking if the request header includes 'Sec-Interest-Group-Storage' to join, leave or update an interest group + if (isScript || isDocument) { + if (request.response_body && (request.response_body.includes('joinAdInterestGroup('))) { + result['protectedAudienceAPI']['interestGroups']['joinAdInterestGroup'].push(cannonicalRequestDomain); + } + if (request.response_body && (request.response_body.includes('leaveAdInterestGroup('))) { + result['protectedAudienceAPI']['interestGroups']['leaveAdInterestGroup'].push(cannonicalRequestDomain); + } + if (request.response_body && (request.response_body.includes('updateAdInterestGroups('))) { + result['protectedAudienceAPI']['interestGroups']['updateAdInterestGroups'].push(cannonicalRequestDomain); + } + if (request.response_body && (request.response_body.includes('clearOriginJoinedAdInterestGroups('))) { + result['protectedAudienceAPI']['interestGroups']['clearOriginJoinedAdInterestGroups'].push(cannonicalRequestDomain); + } + if (request.response_body && (request.response_body.includes('runAdAuction('))) { + result['protectedAudienceAPI']['runAdAuction'].push(cannonicalRequestDomain); + } + if (request.response_body && (request.response_body.includes('generateBid('))) { + result['protectedAudienceAPI']['generateBid'].push(cannonicalRequestDomain); + } + if (request.response_body && (request.response_body.includes('scoreAd('))) { + result['protectedAudienceAPI']['scoreAd'].push(cannonicalRequestDomain); + } + if (request.response_body && (request.response_body.includes('reportWin('))) { + result['protectedAudienceAPI']['reportWin'].push(cannonicalRequestDomain); + } + if (request.response_body && (request.response_body.includes('reportResult(') || request.response_body.includes('sendReportTo('))) { + result['protectedAudienceAPI']['reportResult'].push(cannonicalRequestDomain); + } + } + /** * Attribution Reporting API * https://developer.mozilla.org/en-US/docs/Web/API/Attribution_Reporting_API From 0c791cd7702670ce9aa212323e3e62367f5ee07c Mon Sep 17 00:00:00 2001 From: Yash Vekaria Date: Thu, 6 Jun 2024 14:50:55 -0700 Subject: [PATCH 21/21] Renaming branch to retain only ads.js related commits --- dist/privacy-sandbox.js | 194 ---------------------------------------- 1 file changed, 194 deletions(-) delete mode 100644 dist/privacy-sandbox.js diff --git a/dist/privacy-sandbox.js b/dist/privacy-sandbox.js deleted file mode 100644 index 82dfeec6..00000000 --- a/dist/privacy-sandbox.js +++ /dev/null @@ -1,194 +0,0 @@ -//[privacy-sandbox] -/** - * Topics API - * Protected Audience API - * Attribution Reporting API - * Required command line flags: --enable-features=BrowsingTopics,InterestGroupStorage,PrivacySandboxAdsAPIsOverride -*/ - -let requests = $WPT_BODIES; -const cannonicalFirstPartyDomain = getCanonicalDomain(document.location.hostname); - -let result = { - 'topicsAPI': { - 'topicsAvailable': document.featurePolicy.allowsFeature('browsing-topics'), - 'usingBrowsingTopics': [], - 'topicsAccessJs': [], - 'topicsAccessHeader': [], - 'observingTopics': [] - }, - 'protectedAudienceAPI': { - 'protectedAudienceAvailable': document.featurePolicy.allowsFeature('join-ad-interest-group'), - 'interestGroups': { - 'joinAdInterestGroup': [], - 'leaveAdInterestGroup': [], - 'updateAdInterestGroups': [], - 'clearOriginJoinedAdInterestGroups': [] - }, - 'runAdAuction': [], - 'generateBid': [], - 'scoreAd': [], - 'reportWin': [], - 'reportResult': [] - }, - 'attributionReportingAPI': { - 'attributionReportingAvailable': document.featurePolicy.allowsFeature('attribution-reporting'), - 'attributionReportingEligibleHeader': { - 'sentByBrowser': false, - 'sentTo': [], - }, - 'completedRegistrations': { - 'AttributionReportingRegisterSourceHeader': {}, - 'AttributionReportingRegisterTriggerHeader': [] - }, - }, - attestationPublished: [] -} - -function getCanonicalDomain(hostname) { - const parts = hostname.split('.').reverse(); - if (parts.length >= 2) { - return `${parts[1]}.${parts[0]}`; - } else { - return hostname; - } -} - -async function fetchAndCheckAttestation(url) { - const controller = new AbortController(); - const { signal } = controller; - setTimeout(() => controller.abort(), 5000); - - try { - const response = await fetch(url, { signal, mode: 'no-cors' }); - return response ? true : false; - } catch (error) { - return false; - } -} - -let seenThirdParties = []; -(async () => { - for (const request of requests) { - const url = new URL(request.full_url); - const isScript = request.type === 'Script'; - const isDocument = request.type === 'Document'; - const cannonicalRequestDomain = getCanonicalDomain(url.hostname); - - let thirdPartyDomain = ''; - // Maps all first party hostnames to the corresponding cannonical first party domain to truly consider only third-parties - if (cannonicalRequestDomain !== cannonicalFirstPartyDomain) { - thirdPartyDomain = url.hostname; - } - if (thirdPartyDomain && seenThirdParties.includes(thirdPartyDomain)) { - continue; - } - seenThirdParties.push(thirdPartyDomain); - - /** - * Topics API - * API Usage Reference: https://developers.google.com/privacy-sandbox/relevance/topics/demo#the-topics-api-demo - * Header Usage Reference: https://developers.google.com/privacy-sandbox/relevance/topics/demo#the-topics-api-demo - */ - - // If the request is to fetch a javascript file or HTML document, perform string search in the returned file content for usage of Topics API - // document.browsingTopics() [JS] or fetch() request call includes: {browsingTopics: true} or XHR request call includes: {deprecatedBrowsingTopics: true} (to be deprecated) - if (isScript || isDocument) { - if (request.response_body && (request.response_body.includes('browsingTopics') || request.response_body.includes('deprecatedBrowsingTopics'))) { - result['topicsAPI']['topicsAccessJs'].push(thirdPartyDomain); - } - } - - // Checking request header usage of Topics: header: 'Sec-Browsing-Topics: true' - if ('sec-browsing-topics' in request.request_headers.toLowerCase()) { - result['topicsAPI']['topicsAccessHeader'].push(thirdPartyDomain); - } - - // Checking if sent Topics in the request (either via JS or headers) are observed by the receiver using response header 'Observe-Browsing-Topics' - // If value is ?1 then they are observed else they are not observed - let respHeaders = new Map(Object.entries(request.response_headers).map(([key, value]) => [key.toLowerCase(), value])); - if (respHeaders.has('observe-browsing-topics') && respHeaders.get('observe-browsing-topics') === "?1") { - result['topicsAPI']['observingTopics'].push(thirdPartyDomain); - } - - const attestationPublished = await fetchAndCheckAttestation(`${url.origin}/.well-known/privacy-sandbox-attestations.json`); - if (attestationPublished) { - result.attestationPublished.push(thirdPartyDomain); - } - - /** - * Protected Audience API - * https://github.com/WICG/turtledove/blob/main/FLEDGE.md - * https://developers.google.com/privacy-sandbox/relevance/protected-audience - */ - - // Checking if the request header includes 'Sec-Interest-Group-Storage' to join, leave or update an interest group - if (isScript || isDocument) { - if (request.response_body && (request.response_body.includes('joinAdInterestGroup('))) { - result['protectedAudienceAPI']['interestGroups']['joinAdInterestGroup'].push(cannonicalRequestDomain); - } - if (request.response_body && (request.response_body.includes('leaveAdInterestGroup('))) { - result['protectedAudienceAPI']['interestGroups']['leaveAdInterestGroup'].push(cannonicalRequestDomain); - } - if (request.response_body && (request.response_body.includes('updateAdInterestGroups('))) { - result['protectedAudienceAPI']['interestGroups']['updateAdInterestGroups'].push(cannonicalRequestDomain); - } - if (request.response_body && (request.response_body.includes('clearOriginJoinedAdInterestGroups('))) { - result['protectedAudienceAPI']['interestGroups']['clearOriginJoinedAdInterestGroups'].push(cannonicalRequestDomain); - } - if (request.response_body && (request.response_body.includes('runAdAuction('))) { - result['protectedAudienceAPI']['runAdAuction'].push(cannonicalRequestDomain); - } - if (request.response_body && (request.response_body.includes('generateBid('))) { - result['protectedAudienceAPI']['generateBid'].push(cannonicalRequestDomain); - } - if (request.response_body && (request.response_body.includes('scoreAd('))) { - result['protectedAudienceAPI']['scoreAd'].push(cannonicalRequestDomain); - } - if (request.response_body && (request.response_body.includes('reportWin('))) { - result['protectedAudienceAPI']['reportWin'].push(cannonicalRequestDomain); - } - if (request.response_body && (request.response_body.includes('reportResult(') || request.response_body.includes('sendReportTo('))) { - result['protectedAudienceAPI']['reportResult'].push(cannonicalRequestDomain); - } - } - - /** - * Attribution Reporting API - * https://developer.mozilla.org/en-US/docs/Web/API/Attribution_Reporting_API - */ - - // Checking if the request header includes 'Attribution-Reporting-Eligible' to initiate the registration of source or trigger - if ('attribution-reporting-eligible' in request.request_headers.toLowerCase()) { - result['attributionReportingAPI']['attributionReportingEligibleHeader']['sentTo'].push(cannonicalRequestDomain); - } - - // Checking if the response header includes 'Attribution-Reporting-Register-Source' or 'Attribution-Reporting-Register-Trigger' to complete registration of source or trigger - // Source registration happens on seller (e.g., publisher) website where impression is registered and trigger registration happens on buyer (e.g., advertiser) website where conversion completes. - // Each entry in result['attributionReportingAPI']['completedRegistrations']['AttributionReportingRegisterSourceHeader'] is represented as {cannonicalRequestDomain: {"destination": "", "eventEpsilon": 0}} - // Higher the epsilon, the more the privacy protection - if ('attribution-reporting-register-source' in respHeaders) { - jsonString = respHeaders.get('attribution-reporting-register-source'); - const { destination, event_level_epsilon } = JSON.parse(jsonString); - if (!result['attributionReportingAPI']['completedRegistrations']['AttributionReportingRegisterSourceHeader'][cannonicalRequestDomain]) { - result['attributionReportingAPI']['completedRegistrations']['AttributionReportingRegisterSourceHeader'][cannonicalRequestDomain] = []; - } - result['attributionReportingAPI']['completedRegistrations']['AttributionReportingRegisterSourceHeader'][cannonicalRequestDomain].push({"destination": destination, "eventEpsilon": event_level_epsilon}); - } else if ('attribution-reporting-register-trigger' in respHeaders) { - result['attributionReportingAPI']['completedRegistrations']['AttributionReportingRegisterTriggerHeader'].push(cannonicalRequestDomain); - } - } - - result['topicsAPI']['usingBrowsingTopics'] = Array.from(new Set([ - ...result['topicsAPI']['topicsAccessJs'], - ...result['topicsAPI']['topicsAccessHeader'] - ])); - - // if "Attribution-Reporting-Eligible" request header is sent to more than one domains, set sentByBrowser to true - if (result['attributionReportingAPI']['attributionReportingEligibleHeader']['sentTo'].length > 0) { - result['attributionReportingAPI']['attributionReportingEligibleHeader']['sentByBrowser'] = true; - } - -})(); - -return result;