Skip to content

Commit

Permalink
[mv3] Add support for converting ^responseheader() filters to DNR
Browse files Browse the repository at this point in the history 
Additionally, finalize versioning scheme for uBOL. Since most updates
will be simply related to update rulesets, the version will from now
on reflects the date at which the extension package was created:

  year.month.day.minutes

So for example:

  2023.8.19.690
  • Loading branch information
@gorhill
gorhill committed Aug 19, 2023
1 parent eb23540 commit 857abb3
Show file tree
Hide file tree
Showing 3 changed files with 208 additions and 170 deletions.
57 changes: 26 additions & 31 deletions platform/mv3/make-rulesets.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -320,18 +320,19 @@ async function processNetworkFilters(assetDetails, network) {
log(`Output rule count: ${rules.length}`);

// https://developer.mozilla.org/en-US/docs/Mozilla/Add-ons/WebExtensions/API/declarativeNetRequest/RuleCondition#browser_compatibility
// isUrlFilterCaseSensitive is false by default in Firefox. It will be
// isUrlFilterCaseSensitive is true by default in Chromium. It will be
// false by default in Chromium 118+.
if ( platform === 'firefox' ) {
if ( platform !== 'firefox' ) {
for ( const rule of rules ) {
if ( rule.condition === undefined ) { continue; }
if ( rule.condition.urlFilter === undefined ) {
if ( rule.condition.regexFilter === undefined ) { continue; }
const { condition } = rule;
if ( condition === undefined ) { continue; }
if ( condition.urlFilter === undefined ) {
if ( condition.regexFilter === undefined ) { continue; }
}
if ( rule.condition.isUrlFilterCaseSensitive === undefined ) {
rule.condition.isUrlFilterCaseSensitive = true;
} else if ( rule.condition.isUrlFilterCaseSensitive === false ) {
rule.condition.isUrlFilterCaseSensitive = undefined;
if ( condition.isUrlFilterCaseSensitive === undefined ) {
condition.isUrlFilterCaseSensitive = false;
} else if ( condition.isUrlFilterCaseSensitive === true ) {
condition.isUrlFilterCaseSensitive = undefined;
}
}
}
Expand Down Expand Up @@ -1098,23 +1099,15 @@ async function rulesetFromURLs(assetDetails) {

async function main() {

// Get manifest content
const manifest = await fs.readFile(
`${outputDir}/manifest.json`,
{ encoding: 'utf8' }
).then(text =>
JSON.parse(text)
);

// Create unique version number according to build time
let version = manifest.version;
let version = '';
{
const now = new Date();
const yearPart = now.getUTCFullYear() - 2000;
const monthPart = (now.getUTCMonth() + 1) * 1000;
const dayPart = now.getUTCDate() * 10;
const hourPart = Math.floor(now.getUTCHours() / 3) + 1;
version += `.${yearPart}.${monthPart + dayPart + hourPart}`;
const yearPart = now.getUTCFullYear();
const monthPart = now.getUTCMonth() + 1;
const dayPart = now.getUTCDate();
const hourPart = Math.floor(now.getUTCHours());
const minutePart = Math.floor(now.getUTCMinutes());
version = `${yearPart}.${monthPart}.${dayPart}.${hourPart * 60 + minutePart}`;
}
log(`Version: ${version}`);

Expand Down Expand Up @@ -1300,6 +1293,13 @@ async function main() {
await Promise.all(writeOps);

// Patch manifest
// Get manifest content
const manifest = await fs.readFile(
`${outputDir}/manifest.json`,
{ encoding: 'utf8' }
).then(text =>
JSON.parse(text)
);
// Patch declarative_net_request key
manifest.declarative_net_request = { rule_resources: ruleResources };
// Patch web_accessible_resources key
Expand All @@ -1312,13 +1312,8 @@ async function main() {
}
manifest.web_accessible_resources = [ web_accessible_resources ];

// Patch version key
const now = new Date();
const yearPart = now.getUTCFullYear() - 2000;
const monthPart = (now.getUTCMonth() + 1) * 1000;
const dayPart = now.getUTCDate() * 10;
const hourPart = Math.floor(now.getUTCHours() / 3) + 1;
manifest.version = manifest.version + `.${yearPart}.${monthPart + dayPart + hourPart}`;
// Patch manifest version property
manifest.version = version;
// Commit changes
await fs.writeFile(
`${outputDir}/manifest.json`,
Expand Down
180 changes: 177 additions & 3 deletions src/js/static-dnr-filtering.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,54 @@ function addExtendedToDNR(context, parser) {
}

// Response header filtering
if ( (parser.flavorBits & parser.BITFlavorExtResponseHeader) !== 0 ) {
if ( parser.isResponseheaderFilter() ) {
if ( parser.hasError() ) { return; }
if ( parser.hasOptions() === false ) { return; }
if ( parser.isException() ) { return; }
const node = parser.getBranchFromType(sfp.NODE_TYPE_EXT_PATTERN_RESPONSEHEADER);
if ( node === 0 ) { return; }
const header = parser.getNodeString(node);
if ( context.responseHeaderRules === undefined ) {
context.responseHeaderRules = [];
}
const rule = {
action: {
responseHeaders: [
{
header,
operation: 'remove',
}
],
type: 'modifyHeaders'
},
condition: {
resourceTypes: [
'main_frame',
'sub_frame'
]
},
};
for ( const { hn, not, bad } of parser.getExtFilterDomainIterator() ) {
if ( bad ) { continue; }
if ( not ) {
if ( rule.condition.excludedInitiatorDomains === undefined ) {
rule.condition.excludedInitiatorDomains = [];
}
rule.condition.excludedInitiatorDomains.push(hn);
continue;
}
if ( hn === '*' ) {
if ( rule.condition.initiatorDomains !== undefined ) {
rule.condition.initiatorDomains = undefined;
}
continue;
}
if ( rule.condition.initiatorDomains === undefined ) {
rule.condition.initiatorDomains = [];
}
rule.condition.initiatorDomains.push(hn);
}
context.responseHeaderRules.push(rule);
return;
}

Expand Down Expand Up @@ -286,6 +333,129 @@ function addToDNR(context, list) {

/******************************************************************************/

function finalizeRuleset(context, network) {
const ruleset = network.ruleset;

// Assign rule ids
const rulesetMap = new Map();
{
let ruleId = 1;
for ( const rule of ruleset ) {
rulesetMap.set(ruleId++, rule);
}
}
// Merge rules where possible by merging arrays of a specific property.
//
// https://github.com/uBlockOrigin/uBOL-issues/issues/10#issuecomment-1304822579
// Do not merge rules which have errors.
const mergeRules = (rulesetMap, mergeTarget) => {
const mergeMap = new Map();
const sorter = (_, v) => {
if ( Array.isArray(v) ) {
return typeof v[0] === 'string' ? v.sort() : v;
}
if ( v instanceof Object ) {
const sorted = {};
for ( const kk of Object.keys(v).sort() ) {
sorted[kk] = v[kk];
}
return sorted;
}
return v;
};
const ruleHasher = (rule, target) => {
return JSON.stringify(rule, (k, v) => {
if ( k.startsWith('_') ) { return; }
if ( k === target ) { return; }
return sorter(k, v);
});
};
const extractTargetValue = (obj, target) => {
for ( const [ k, v ] of Object.entries(obj) ) {
if ( Array.isArray(v) && k === target ) { return v; }
if ( v instanceof Object ) {
const r = extractTargetValue(v, target);
if ( r !== undefined ) { return r; }
}
}
};
const extractTargetOwner = (obj, target) => {
for ( const [ k, v ] of Object.entries(obj) ) {
if ( Array.isArray(v) && k === target ) { return obj; }
if ( v instanceof Object ) {
const r = extractTargetOwner(v, target);
if ( r !== undefined ) { return r; }
}
}
};
for ( const [ id, rule ] of rulesetMap ) {
if ( rule._error !== undefined ) { continue; }
const hash = ruleHasher(rule, mergeTarget);
if ( mergeMap.has(hash) === false ) {
mergeMap.set(hash, []);
}
mergeMap.get(hash).push(id);
}
for ( const ids of mergeMap.values() ) {
if ( ids.length === 1 ) { continue; }
const leftHand = rulesetMap.get(ids[0]);
const leftHandSet = new Set(
extractTargetValue(leftHand, mergeTarget) || []
);
for ( let i = 1; i < ids.length; i++ ) {
const rightHandId = ids[i];
const rightHand = rulesetMap.get(rightHandId);
const rightHandArray = extractTargetValue(rightHand, mergeTarget);
if ( rightHandArray !== undefined ) {
if ( leftHandSet.size !== 0 ) {
for ( const item of rightHandArray ) {
leftHandSet.add(item);
}
}
} else {
leftHandSet.clear();
}
rulesetMap.delete(rightHandId);
}
const leftHandOwner = extractTargetOwner(leftHand, mergeTarget);
if ( leftHandSet.size > 1 ) {
//if ( leftHandOwner === undefined ) { debugger; }
leftHandOwner[mergeTarget] = Array.from(leftHandSet).sort();
} else if ( leftHandSet.size === 0 ) {
if ( leftHandOwner !== undefined ) {
leftHandOwner[mergeTarget] = undefined;
}
}
}
};
mergeRules(rulesetMap, 'resourceTypes');
mergeRules(rulesetMap, 'initiatorDomains');
mergeRules(rulesetMap, 'requestDomains');
mergeRules(rulesetMap, 'removeParams');
mergeRules(rulesetMap, 'responseHeaders');

// Patch id
const rulesetFinal = [];
{
let ruleId = 1;
for ( const rule of rulesetMap.values() ) {
if ( rule._error === undefined ) {
rule.id = ruleId++;
} else {
rule.id = 0;
}
rulesetFinal.push(rule);
}
for ( const invalid of context.invalid ) {
rulesetFinal.push({ _error: [ invalid ] });
}
}

network.ruleset = rulesetFinal;
}

/******************************************************************************/

async function dnrRulesetFromRawLists(lists, options = {}) {
const context = Object.assign({}, options);
staticNetFilteringEngine.dnrFromCompiled('begin', context);
Expand All @@ -300,15 +470,19 @@ async function dnrRulesetFromRawLists(lists, options = {}) {
}
}
await Promise.all(toLoad);

return {
const result = {
network: staticNetFilteringEngine.dnrFromCompiled('end', context),
genericCosmetic: context.genericCosmeticFilters,
genericHighCosmetic: context.genericHighCosmeticFilters,
genericCosmeticExceptions: context.genericCosmeticExceptions,
specificCosmetic: context.specificCosmeticFilters,
scriptlet: context.scriptletFilters,
};
if ( context.responseHeaderRules ) {
result.network.ruleset.push(...context.responseHeaderRules);
}
finalizeRuleset(context, result.network);
return result;
}

/******************************************************************************/
Expand Down
Loading

0 comments on commit 857abb3

Please sign in to comment.