Skip to content

Commit

Permalink
feat: Changement de la méthode d'analyse en recréant le DOM de la pag…
Browse files Browse the repository at this point in the history
…e en mémoire.

Cela évite de charge le code de l'analyseur dans la page elle même.
Cela permet d'analyser des sites ayant des Content Security Policy de configurés.

Ces modifications sont issues de la proposition de @hayaofr (via #16), j'ai simplement ici réduit au maximum le nombre de modification pour faciliter la relecture de la PR

Fix: #19
  • Loading branch information
hayaofr authored and jycr committed Feb 15, 2022
1 parent 2f7dc76 commit 3c345ef
Show file tree
Hide file tree
Showing 35 changed files with 1,310 additions and 794 deletions.
33 changes: 12 additions & 21 deletions cli-core/analysis.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,9 @@ const fs = require('fs')
const path = require('path');
const ProgressBar = require('progress');
const sizes = require('../sizes.js');
const translator = require('./translator.js').translator;
const jsdom = require("jsdom");
const {JSDOM} = jsdom;
const launchAnalyse = require('../greenit-core/greenpanel.js').launchAnalyse;

//Path to the url file
const SUBRESULTS_DIRECTORY = path.join(__dirname,'../results');
Expand Down Expand Up @@ -65,26 +67,15 @@ async function analyseURL(browser, pageInformations, options) {
const client = await page.target().createCDPSession();
let ressourceTree = await client.send('Page.getResourceTree');
await client.detach()

// replace chrome.i18n.getMessage call by i18n custom implementation working in page
// fr is default catalog
await page.evaluate(language_array =>(chrome = { "i18n" : {"getMessage" : function (message, parameters = []) {
return language_array[message].replace(/%s/g, function() {
// parameters is string or array
return Array.isArray(parameters) ? parameters.shift() : parameters;
});
}}}), translator.getCatalog());

//add script, get run, then remove it to not interfere with the analysis
let script = await page.addScriptTag({ path: path.join(__dirname,'../dist/bundle.js')});
await script.evaluate(x=>(x.remove()));

//pass node object to browser
await page.evaluate(x=>(har = x), harObj.log);
await page.evaluate(x=>(resources = x), ressourceTree.frameTree.resources);


//retrieve page content
const pageContent = await page.content();

// Building in memory Page DOM
const dom = new JSDOM(pageContent);

//launch analyse
result = await page.evaluate(()=>(launchAnalyse()));
result = await launchAnalyse(dom.window.document, harObj.log, ressourceTree.frameTree.resources);

page.close();
result.success = true;
Expand Down Expand Up @@ -323,4 +314,4 @@ async function createJsonReports(browser, pagesInformations, options, proxy, hea
module.exports = {
createJsonReports,
login
}
};
4 changes: 2 additions & 2 deletions cli-core/reportHtml.js
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,7 @@ function extractBestPractices(bestPracticesFromReport) {
}

function writeGlobalReport(templateEngine, globalReportVariables, outputFile) {
templateEngine.processFile('cli-core/template/global.html', globalReportVariables)
templateEngine.processFile(__dirname + '/template/global.html', globalReportVariables)
.then(globalReportHtml => {
fs.writeFileSync(outputFile, globalReportHtml);
})
Expand All @@ -173,7 +173,7 @@ function writeGlobalReport(templateEngine, globalReportVariables, outputFile) {

function writeAllReports(templateEngine, allReportsVariables, outputFolder) {
allReportsVariables.forEach(reportVariables => {
templateEngine.processFile('cli-core/template/page.html', reportVariables)
templateEngine.processFile(__dirname + '/template/page.html', reportVariables)
.then(singleReportHtml => {
fs.writeFileSync(`${outputFolder}/${reportVariables.filename}`, singleReportHtml);
})
Expand Down
Empty file modified greenit
100644 → 100755
Empty file.
4 changes: 4 additions & 0 deletions greenit-core/analyseFrameCore.js
Original file line number Diff line number Diff line change
Expand Up @@ -132,3 +132,7 @@ function getImagesResizedInBrowser() {
});
return imagesResized;
}

module.exports = {
start_analyse_core
};
16 changes: 16 additions & 0 deletions greenit-core/chrome.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
const {translator} = require("../cli-core/translator");

chrome = {
"i18n": {
"getMessage": function (message, parameters = []) {
return translator.getCatalog()[message].replace(/%s/g, function () {
// parameters is string or array
return Array.isArray(parameters) ? parameters.shift() : parameters;
});
}
}
};

module.exports = {
chrome
};
7 changes: 7 additions & 0 deletions greenit-core/ecoIndex.js
Original file line number Diff line number Diff line change
Expand Up @@ -65,3 +65,10 @@ function computeWaterConsumptionfromEcoIndex(ecoIndex)
return (Math.round(100 * (3 + 3 * (50 - ecoIndex) / 100)) / 100);
}

module.exports = {
computeEcoIndex,
computeWaterConsumptionfromEcoIndex,
computeGreenhouseGasesEmissionfromEcoIndex,
computeQuantile,
getEcoIndexGrade
};
102 changes: 79 additions & 23 deletions greenit-core/greenpanel.js
Original file line number Diff line number Diff line change
Expand Up @@ -16,17 +16,48 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/

const utils = require('./utils.js');
const {start_analyse_core} = require("./analyseFrameCore");
const {
computeEcoIndex, computeWaterConsumptionfromEcoIndex, computeGreenhouseGasesEmissionfromEcoIndex,
getEcoIndexGrade
} = require("./ecoIndex");
const {isNetworkResource, isDataResource} = require("./utils");
const {registerAddExpiresOrCacheControlHeaders} = require("./rules/AddExpiresOrCacheControlHeaders");
const {registerCompressHttp} = require("./rules/CompressHttp");
const {registerDontResizeImageInBrowser} = require("./rules/DontResizeImageInBrowser");
const {registerDomainsNumber} = require("./rules/DomainsNumber");
const {registerEmptySrcTag} = require("./rules/EmptySrcTag");
const {registerExternalizeCss} = require("./rules/ExternalizeCss");
const {registerExternalizeJs} = require("./rules/ExternalizeJs");
const {registerHttpError} = require("./rules/HttpError");
const {registerHttpRequests} = require("./rules/HttpRequests");
const {registerImageDownloadedNotDisplayed} = require("./rules/ImageDownloadedNotDisplayed");
const {registerJsValidate} = require("./rules/JsValidate");
const {registerMaxCookiesLength} = require("./rules/MaxCookiesLength");
const {registerMinifiedCss} = require("./rules/MinifiedCss");
const {registerMinifiedJs} = require("./rules/MinifiedJs");
const {registerNoCookieForStaticRessources} = require("./rules/NoCookieForStaticRessources");
const {registerNoRedirect} = require("./rules/NoRedirect");
const {registerOptimizeBitmapImages} = require("./rules/OptimizeBitmapImages");
const {registerOptimizeSvg} = require("./rules/OptimizeSvg");
const {registerPlugins} = require("./rules/Plugins");
const {registerPrintStyleSheet} = require("./rules/PrintStyleSheet");
const {registerSocialNetworkButton} = require("./rules/SocialNetworkButton");
const {registerStyleSheets} = require("./rules/StyleSheets");
const {registerUseETags} = require("./rules/UseETags");
const {registerUseStandardTypefaces} = require("./rules/UseStandardTypefaces");
const RulesManager = require('./rulesManager.js').RulesManager

let backgroundPageConnection;
let currentRulesChecker;
let lastAnalyseStartingTime = 0;
let measuresAcquisition;
let analyseBestPractices = true;
let har;
let resources;

function handleResponseFromBackground(frameMeasures) {
if (isOldAnalyse(frameMeasures.analyseStartingTime)) {
debug(() => `Analyse is too old for url ${frameMeasures.url} , time = ${frameMeasures.analyseStartingTime}`);
utils.debug(() => `Analyse is too old for url ${frameMeasures.url} , time = ${frameMeasures.analyseStartingTime}`);
return;
}
measuresAcquisition.aggregateFrameMeasures(frameMeasures);
Expand All @@ -41,21 +72,42 @@ function computeEcoIndexMeasures(measures) {
measures.grade = getEcoIndexGrade(measures.ecoIndex);
}

function registerRules(rulesManager) {
registerAddExpiresOrCacheControlHeaders(rulesManager)
registerCompressHttp(rulesManager)
registerDontResizeImageInBrowser(rulesManager)
registerDomainsNumber(rulesManager)
registerEmptySrcTag(rulesManager)
registerExternalizeCss(rulesManager)
registerExternalizeJs(rulesManager)
registerHttpError(rulesManager)
registerHttpRequests(rulesManager)
registerImageDownloadedNotDisplayed(rulesManager)
registerJsValidate(rulesManager)
registerMaxCookiesLength(rulesManager)
registerMinifiedCss(rulesManager)
registerMinifiedJs(rulesManager)
registerNoCookieForStaticRessources(rulesManager)
registerNoRedirect(rulesManager)
registerOptimizeBitmapImages(rulesManager)
registerOptimizeSvg(rulesManager)
registerPlugins(rulesManager)
registerPrintStyleSheet(rulesManager)
registerSocialNetworkButton(rulesManager)
registerStyleSheets(rulesManager)
registerUseETags(rulesManager)
registerUseStandardTypefaces(rulesManager)
}

function launchAnalyse() {
let now = Date.now();

// To avoid parallel analyse , force 1 secondes between analysis
if (now - lastAnalyseStartingTime < 1000) {
debug(() => "Ignore click");
return;
}
lastAnalyseStartingTime = now;
async function launchAnalyse(document, har, resources) {
lastAnalyseStartingTime = Date.now();
let rulesManager = new RulesManager()
registerRules(rulesManager);
currentRulesChecker = rulesManager.getNewRulesChecker();
measuresAcquisition = new MeasuresAcquisition(currentRulesChecker);
measuresAcquisition.initializeMeasures();
measuresAcquisition.aggregateFrameMeasures(start_analyse_core())
measuresAcquisition.startMeasuring();
measuresAcquisition.aggregateFrameMeasures(start_analyse_core(document, analyseBestPractices))
measuresAcquisition.startMeasuring(har, resources);
let returnObj = measuresAcquisition.getMeasures();
returnObj.bestPractices = measuresAcquisition.getBestPractices()
return returnObj;
Expand Down Expand Up @@ -88,9 +140,9 @@ function MeasuresAcquisition(rules) {
};
}

this.startMeasuring = function () {
getNetworkMeasure();
if (analyseBestPractices) getResourcesMeasure();
this.startMeasuring = function (har, resources) {
getNetworkMeasure(har);
if (analyseBestPractices) getResourcesMeasure(resources);
}

this.getMeasures = () => measures;
Expand Down Expand Up @@ -126,18 +178,18 @@ function MeasuresAcquisition(rules) {



const getNetworkMeasure = () => {
const getNetworkMeasure = (har) => {

console.log("Start network measure...");
// only account for network traffic, filtering resources embedded through data urls
let entries = har.entries.filter(entry => isNetworkResource(entry));

// Get the "mother" url
// Get the "mother" url
if (entries.length > 0) measures.url = entries[0].request.url;
else {
// Bug with firefox when we first get har.entries when starting the plugin , we need to ask again to have it
// Bug with firefox when we first get har.entries when starting the plugin , we need to ask again to have it
if (nbGetHarTry < 1) {
debug(() => 'No entries, try again to get HAR in 1s');
utils.debug(() => 'No entries, try again to get HAR in 1s');
nbGetHarTry++;
setTimeout(getNetworkMeasure, 1000);
}
Expand Down Expand Up @@ -170,7 +222,7 @@ function MeasuresAcquisition(rules) {
}
}

function getResourcesMeasure() {
function getResourcesMeasure(resources) {
resources.forEach(resource => {
if (resource.url.startsWith("file") || resource.url.startsWith("http")) {
if ((resource.type === 'script') || (resource.type === 'stylesheet') || (resource.type === 'image')) {
Expand Down Expand Up @@ -234,3 +286,7 @@ function storeAnalysisInHistory() {

localStorage.setItem("analyse_history", JSON.stringify(analyse_history));
}

module.exports = {
launchAnalyse
};
59 changes: 34 additions & 25 deletions greenit-core/rules/AddExpiresOrCacheControlHeaders.js
Original file line number Diff line number Diff line change
@@ -1,31 +1,40 @@
rulesManager.registerRule({
complianceLevel: 'A',
id: "AddExpiresOrCacheControlHeaders",
comment: "",
detailComment: "",
const { chrome } = require("../chrome");
const {isStaticRessource, hasValidCacheHeaders} = require("../utils");

check: function (measures) {
let staticResourcesSize = 0;
let staticResourcesWithCache = 0;
function registerAddExpiresOrCacheControlHeaders(rulesManager) {
rulesManager.registerRule({
complianceLevel: 'A',
id: "AddExpiresOrCacheControlHeaders",
comment: "",
detailComment: "",

if (measures.entries.length) measures.entries.forEach(entry => {
if (isStaticRessource(entry)) {
staticResourcesSize += entry.response.content.size;
if (hasValidCacheHeaders(entry)) {
staticResourcesWithCache += entry.response.content.size;
check: function (measures) {
let staticResourcesSize = 0;
let staticResourcesWithCache = 0;

if (measures.entries.length) measures.entries.forEach(entry => {
if (isStaticRessource(entry)) {
staticResourcesSize += entry.response.content.size;
if (hasValidCacheHeaders(entry)) {
staticResourcesWithCache += entry.response.content.size;
}
else this.detailComment += chrome.i18n.getMessage("rule_AddExpiresOrCacheControlHeaders_DetailComment", `${entry.request.url} ${Math.round(entry.response.content.size / 100) / 10}`) + '<br>';
}
else this.detailComment += chrome.i18n.getMessage("rule_AddExpiresOrCacheControlHeaders_DetailComment",`${entry.request.url} ${Math.round(entry.response.content.size / 100) / 10}`) + '<br>';
}
});
});

if (staticResourcesSize > 0) {
const cacheHeaderRatio = staticResourcesWithCache / staticResourcesSize * 100;
if (cacheHeaderRatio < 95) {
if (cacheHeaderRatio < 90) this.complianceLevel = 'C'
else this.complianceLevel = 'B';
if (staticResourcesSize > 0) {
const cacheHeaderRatio = staticResourcesWithCache / staticResourcesSize * 100;
if (cacheHeaderRatio < 95) {
if (cacheHeaderRatio < 90) this.complianceLevel = 'C'
else this.complianceLevel = 'B';
}
else this.complianceLevel = 'A';
this.comment = chrome.i18n.getMessage("rule_AddExpiresOrCacheControlHeaders_Comment", String(Math.round(cacheHeaderRatio * 10) / 10) + "%");
}
else this.complianceLevel = 'A';
this.comment = chrome.i18n.getMessage("rule_AddExpiresOrCacheControlHeaders_Comment", String(Math.round(cacheHeaderRatio * 10) / 10) + "%");
}
}
}, "harReceived");
}, "harReceived");
}

module.exports = {
registerAddExpiresOrCacheControlHeaders
};
59 changes: 34 additions & 25 deletions greenit-core/rules/CompressHttp.js
Original file line number Diff line number Diff line change
@@ -1,29 +1,38 @@
rulesManager.registerRule({
complianceLevel: 'A',
id: "CompressHttp",
comment: "",
detailComment: "",
const { chrome } = require("../chrome");
const {isResourceCompressed, isCompressibleResource} = require("../utils");

check: function (measures) {
let compressibleResourcesSize = 0;
let compressibleResourcesCompressedSize = 0;
if (measures.entries.length) measures.entries.forEach(entry => {
if (isCompressibleResource(entry)) {
compressibleResourcesSize += entry.response.content.size;
if (isResourceCompressed(entry)) {
compressibleResourcesCompressedSize += entry.response.content.size;
function registerCompressHttp(rulesManager) {
rulesManager.registerRule({
complianceLevel: 'A',
id: "CompressHttp",
comment: "",
detailComment: "",

check: function (measures) {
let compressibleResourcesSize = 0;
let compressibleResourcesCompressedSize = 0;
if (measures.entries.length) measures.entries.forEach(entry => {
if (isCompressibleResource(entry)) {
compressibleResourcesSize += entry.response.content.size;
if (isResourceCompressed(entry)) {
compressibleResourcesCompressedSize += entry.response.content.size;
}
else this.detailComment += chrome.i18n.getMessage("rule_CompressHttp_DetailComment", `${entry.request.url} ${Math.round(entry.response.content.size / 100) / 10}`) + '<br>';
}
else this.detailComment += chrome.i18n.getMessage("rule_CompressHttp_DetailComment",`${entry.request.url} ${Math.round(entry.response.content.size / 100) / 10}`) + '<br>';
}
});
if (compressibleResourcesSize > 0) {
const compressRatio = compressibleResourcesCompressedSize / compressibleResourcesSize * 100;
if (compressRatio < 95) {
if (compressRatio < 90) this.complianceLevel = 'C'
else this.complianceLevel = 'B';
});
if (compressibleResourcesSize > 0) {
const compressRatio = compressibleResourcesCompressedSize / compressibleResourcesSize * 100;
if (compressRatio < 95) {
if (compressRatio < 90) this.complianceLevel = 'C'
else this.complianceLevel = 'B';
}
else this.complianceLevel = 'A';
this.comment = chrome.i18n.getMessage("rule_CompressHttp_Comment", String(Math.round(compressRatio * 10) / 10) + "%");
}
else this.complianceLevel = 'A';
this.comment = chrome.i18n.getMessage("rule_CompressHttp_Comment", String(Math.round(compressRatio * 10) / 10) + "%");
}
}
}, "harReceived");
}, "harReceived");
}

module.exports = {
registerCompressHttp
};
Loading

0 comments on commit 3c345ef

Please sign in to comment.