diff --git a/.github/workflows/publish-docker-image.yml b/.github/workflows/publish-docker-image.yml index 085555c..2808f5f 100644 --- a/.github/workflows/publish-docker-image.yml +++ b/.github/workflows/publish-docker-image.yml @@ -36,7 +36,7 @@ jobs: uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - tags: type=raw,value=3.4 + tags: type=raw,value=3.5 # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages. # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository. # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step. diff --git a/EcoSonar-API/Dockerfile b/EcoSonar-API/Dockerfile index 933253e..9983018 100644 --- a/EcoSonar-API/Dockerfile +++ b/EcoSonar-API/Dockerfile @@ -1,7 +1,7 @@ # See : https://github.com/puppeteer/puppeteer/blob/main/docs/troubleshooting.md#running-puppeteer-in-docker -FROM node:16-slim +FROM node:18-slim -# Uncomment if you need to configure proxy. +# Uncomment if you need to configure proxy. # You can init these variables by using --build-args during docker build # Example : docker build [...] --build-args http_proxy=http://:@: #ENV HTTP_PROXY=$http_proxy @@ -13,28 +13,17 @@ RUN apt-get update \ && wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \ && sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' \ && apt-get update \ - && apt-get install -y google-chrome-stable fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg \ - fonts-kacst fonts-freefont-ttf libxss1 gconf-service libasound2 libatk1.0-0 libc6 libcairo2 libcups2 \ - libdbus-1-3 libexpat1 libfontconfig1 libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 libgtk-3-0 \ - libnspr4 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 \ - libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6 ca-certificates \ - fonts-liberation libappindicator1 libnss3 lsb-release xdg-utils wget\ + && apt-get install -y google-chrome-stable fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst fonts-freefont-ttf libxss1 libx11-xcb1 libnss3-tools curl \ --no-install-recommends \ && rm -rf /var/lib/apt/lists/* - -# Create app directory WORKDIR /app - -# Bundle app source COPY . . -# Uncomment if you need to configure proxy. -#RUN npm config set proxy $HTTP_PROXY +# Uncomment if you need to configure proxy. +#RUN npm config set proxy $HTTP_PROXY -# If you are building your code for production -# RUN npm ci --only=production -# otherwise run npm install -RUN npm install \ +RUN npm i \ + && npm link \ && groupadd -r pptruser && useradd -r -g pptruser -G audio,video pptruser \ && mkdir -p /home/pptruser/Downloads \ && chown -R pptruser:pptruser /home/pptruser \ @@ -42,8 +31,9 @@ RUN npm install \ USER pptruser -# To avoid "Error: ENOENT: no such file or directory, open '/app/dist/bundle.js'" -RUN npm i +# To avoid "Error: ENOENT: no such file or directory, open '/app/dist/greenItBundle.js'" +RUN npm i \ + && node node_modules/puppeteer/install.mjs ENV PORT=3000 EXPOSE 3000 diff --git a/EcoSonar-API/builder.js b/EcoSonar-API/builder.js index 8875a91..d915e2f 100644 --- a/EcoSonar-API/builder.js +++ b/EcoSonar-API/builder.js @@ -1,6 +1,6 @@ -const concat = require('concat-files') -const glob = require('glob') -const fs = require('fs') +import concat from 'concat-files' +import { glob } from 'glob' +import fs from 'fs' const DIR = './services/greenit-analysis/dist' diff --git a/EcoSonar-API/configuration/database.js b/EcoSonar-API/configuration/database.js index 5a98727..082d110 100644 --- a/EcoSonar-API/configuration/database.js +++ b/EcoSonar-API/configuration/database.js @@ -1,5 +1,5 @@ -const mongoose = require('mongoose') -const { retrievePassword } = require('./retrieveDatabasePasswordFromCloudProvider') +import mongoose from 'mongoose' +import retrievePassword from './retrieveDatabasePasswordFromCloudProvider.js' mongoose.set('strictQuery', false) @@ -18,9 +18,8 @@ Database.prototype.connection = async function () { const port = process.env.ECOSONAR_ENV_DB_PORT || 27017 dbName = process.env.ECOSONAR_ENV_DB_NAME || '' connectionString = `mongodb://${cluster}:${port}/${dbName}` - mongoose.connect(connectionString, - { useNewUrlParser: true, useUnifiedTopology: true } - ).then(() => console.log('Connection to MongoDB successful')) + mongoose.connect(connectionString) + .then(() => console.log('Connection to MongoDB successful')) .catch((reason) => console.error('\x1b[31m%s\x1b[0m', 'Unable to connect to the mongodb instance. Error: ', reason)) } else if (mongoDBType === 'MongoDB_Atlas') { // connection to dataBase MongoDB Atlas for MongoDB API @@ -29,9 +28,8 @@ Database.prototype.connection = async function () { cluster = process.env.ECOSONAR_ENV_CLUSTER || '' dbName = process.env.ECOSONAR_ENV_DB_NAME || '' connectionString = `mongodb+srv://${user}:${password}@${cluster}/${dbName}?retryWrites=true&w=majority` - mongoose.connect(connectionString, - { useNewUrlParser: true, useUnifiedTopology: true } - ).then(() => console.log('Connection to MongoDB Atlas successful')) + mongoose.connect(connectionString) + .then(() => console.log('Connection to MongoDB Atlas successful')) .catch((reason) => console.error('\x1b[31m%s\x1b[0m', 'Unable to connect to the mongodb instance. Error: ', reason)) } else if (mongoDBType === 'CosmosDB') { // connection to dataBase Azure CosmosDB for MongoDB API @@ -46,8 +44,6 @@ Database.prototype.connection = async function () { username: user, password }, - useNewUrlParser: true, - useUnifiedTopology: true, retryWrites: false }) .then(() => console.log('Connection to CosmosDB successful')) @@ -58,4 +54,4 @@ Database.prototype.connection = async function () { } const database = new Database() -module.exports = database +export default database diff --git a/EcoSonar-API/configuration/retrieveDatabasePasswordFromCloudProvider.js b/EcoSonar-API/configuration/retrieveDatabasePasswordFromCloudProvider.js index 5393513..e4a0cb2 100644 --- a/EcoSonar-API/configuration/retrieveDatabasePasswordFromCloudProvider.js +++ b/EcoSonar-API/configuration/retrieveDatabasePasswordFromCloudProvider.js @@ -1,7 +1,7 @@ -const { DefaultAzureCredential } = require('@azure/identity') -const { SecretClient } = require('@azure/keyvault-secrets') +import { DefaultAzureCredential } from '@azure/identity' +import { SecretClient } from '@azure/keyvault-secrets' -async function retrievePassword () { +export default async function retrievePassword () { if (process.env.ECOSONAR_ENV_CLOUD_PROVIDER === 'AZURE') { return retrievePasswordFromAzure() } else if (process.env.ECOSONAR_ENV_CLOUD_PROVIDER === 'local') { @@ -23,5 +23,3 @@ async function retrievePasswordFromAzure () { const retrievedSecret = await client.getSecret(secretName) return retrievedSecret.value } - -module.exports = { retrievePassword } diff --git a/EcoSonar-API/dataBase/bestPracticesRepository.js b/EcoSonar-API/dataBase/bestPracticesRepository.js index 603f62a..67d4758 100644 --- a/EcoSonar-API/dataBase/bestPracticesRepository.js +++ b/EcoSonar-API/dataBase/bestPracticesRepository.js @@ -1,5 +1,5 @@ -const bestpractices = require('./models/bestpractices') -const SystemError = require('../utils/SystemError') +import bestpractices from './models/bestpractices.js' +import SystemError from '../utils/SystemError.js' const BestPracticesRepository = function () { /** @@ -151,4 +151,4 @@ function replaceDotWithUnderscoreInKeys (obj) { } const bestPracticesRepository = new BestPracticesRepository() -module.exports = bestPracticesRepository +export default bestPracticesRepository diff --git a/EcoSonar-API/dataBase/greenItRepository.js b/EcoSonar-API/dataBase/greenItRepository.js index 87b8756..440a2c4 100644 --- a/EcoSonar-API/dataBase/greenItRepository.js +++ b/EcoSonar-API/dataBase/greenItRepository.js @@ -1,6 +1,6 @@ -const greenits = require('./models/greenits') -const urlsprojects = require('./models/urlsprojects') -const SystemError = require('../utils/SystemError') +import greenits from './models/greenits.js' +import urlsprojects from './models/urlsprojects.js' +import SystemError from '../utils/SystemError.js' const GreenItRepository = function () { /** @@ -162,4 +162,4 @@ async function checkValues (arrayToInsert) { } const greenItRepository = new GreenItRepository() -module.exports = greenItRepository +export default greenItRepository diff --git a/EcoSonar-API/dataBase/lighthouseRepository.js b/EcoSonar-API/dataBase/lighthouseRepository.js index 57946e3..20edb8c 100644 --- a/EcoSonar-API/dataBase/lighthouseRepository.js +++ b/EcoSonar-API/dataBase/lighthouseRepository.js @@ -1,5 +1,5 @@ -const lighthouses = require('./models/lighthouses') -const SystemError = require('../utils/SystemError') +import lighthouses from './models/lighthouses.js' +import SystemError from '../utils/SystemError.js' const LighthouseRepository = function () { /** @@ -176,4 +176,4 @@ const LighthouseRepository = function () { } const lighthouseRepository = new LighthouseRepository() -module.exports = lighthouseRepository +export default lighthouseRepository diff --git a/EcoSonar-API/dataBase/models/bestpractices.js b/EcoSonar-API/dataBase/models/bestpractices.js index 9be018a..908009f 100644 --- a/EcoSonar-API/dataBase/models/bestpractices.js +++ b/EcoSonar-API/dataBase/models/bestpractices.js @@ -1,4 +1,4 @@ -const mongoose = require('mongoose') +import mongoose from 'mongoose' const Schema = mongoose.Schema const bestPracticesSchema = new Schema({ @@ -226,12 +226,6 @@ const bestPracticesSchema = new Schema({ description: { type: Array }, auditedMetric: { type: Number } }, - preloadLcpImage: { - score: { type: Number }, - scoreDisplayMode: { type: String }, - description: { type: Array }, - auditedMetric: { type: Number } - }, domSize: { score: { type: Number }, scoreDisplayMode: { type: String }, @@ -627,4 +621,4 @@ const bestPracticesSchema = new Schema({ }) const bestPractices = mongoose.model('bestpractices', bestPracticesSchema) -module.exports = bestPractices +export default bestPractices diff --git a/EcoSonar-API/dataBase/models/greenits.js b/EcoSonar-API/dataBase/models/greenits.js index f02ea86..66e9b8a 100644 --- a/EcoSonar-API/dataBase/models/greenits.js +++ b/EcoSonar-API/dataBase/models/greenits.js @@ -1,4 +1,4 @@ -const mongoose = require('mongoose') +import mongoose from 'mongoose' const Schema = mongoose.Schema const greenItSchema = new Schema({ @@ -42,4 +42,4 @@ const greenItSchema = new Schema({ }) const greenIt = mongoose.model('greenits', greenItSchema) -module.exports = greenIt +export default greenIt diff --git a/EcoSonar-API/dataBase/models/lighthouses.js b/EcoSonar-API/dataBase/models/lighthouses.js index 9a2b290..e05fb3b 100644 --- a/EcoSonar-API/dataBase/models/lighthouses.js +++ b/EcoSonar-API/dataBase/models/lighthouses.js @@ -1,4 +1,4 @@ -const mongoose = require('mongoose') +import mongoose from 'mongoose' const Schema = mongoose.Schema const lighthouseSchema = new Schema({ @@ -122,4 +122,4 @@ const lighthouseSchema = new Schema({ }) const lighthouse = mongoose.model('lighthouses', lighthouseSchema) -module.exports = lighthouse +export default lighthouse diff --git a/EcoSonar-API/dataBase/models/projects.js b/EcoSonar-API/dataBase/models/projects.js index e7caef8..8aa34ce 100644 --- a/EcoSonar-API/dataBase/models/projects.js +++ b/EcoSonar-API/dataBase/models/projects.js @@ -1,4 +1,4 @@ -const mongoose = require('mongoose') +import mongoose from 'mongoose' const Schema = mongoose.Schema const projectsSchema = new Schema({ @@ -24,4 +24,4 @@ const projectsSchema = new Schema({ }) const project = mongoose.model('projects', projectsSchema) -module.exports = project +export default project diff --git a/EcoSonar-API/dataBase/models/tempurlsproject.js b/EcoSonar-API/dataBase/models/tempurlsproject.js index 079a84a..a09ad69 100644 --- a/EcoSonar-API/dataBase/models/tempurlsproject.js +++ b/EcoSonar-API/dataBase/models/tempurlsproject.js @@ -1,4 +1,4 @@ -const mongoose = require('mongoose') +import mongoose from 'mongoose' const Schema = mongoose.Schema const tempUrlsProjectSchema = new Schema({ @@ -16,4 +16,4 @@ const tempUrlsProjectSchema = new Schema({ }) const tempurlsProject = mongoose.model('tempurlsprojects', tempUrlsProjectSchema) -module.exports = tempurlsProject +export default tempurlsProject diff --git a/EcoSonar-API/dataBase/models/urlsprojects.js b/EcoSonar-API/dataBase/models/urlsprojects.js index 35fb1f5..75fff21 100644 --- a/EcoSonar-API/dataBase/models/urlsprojects.js +++ b/EcoSonar-API/dataBase/models/urlsprojects.js @@ -1,4 +1,4 @@ -const mongoose = require('mongoose') +import mongoose from 'mongoose' const Schema = mongoose.Schema const urlsProjectSchema = new Schema({ @@ -22,4 +22,4 @@ const urlsProjectSchema = new Schema({ }) const urlsProject = mongoose.model('urlsprojects', urlsProjectSchema) -module.exports = urlsProject +export default urlsProject diff --git a/EcoSonar-API/dataBase/models/w3cs.js b/EcoSonar-API/dataBase/models/w3cs.js index 4238f9e..9aae3ba 100644 --- a/EcoSonar-API/dataBase/models/w3cs.js +++ b/EcoSonar-API/dataBase/models/w3cs.js @@ -1,4 +1,4 @@ -const mongoose = require('mongoose') +import mongoose from 'mongoose' const Schema = mongoose.Schema const w3cSchema = new Schema({ @@ -14,4 +14,4 @@ const w3cSchema = new Schema({ }) const w3cs = mongoose.model('w3cs', w3cSchema) -module.exports = w3cs +export default w3cs diff --git a/EcoSonar-API/dataBase/projectsRepository.js b/EcoSonar-API/dataBase/projectsRepository.js index 7faeb83..67cd10c 100644 --- a/EcoSonar-API/dataBase/projectsRepository.js +++ b/EcoSonar-API/dataBase/projectsRepository.js @@ -1,5 +1,5 @@ -const projects = require('./models/projects') -const SystemError = require('../utils/SystemError') +import projects from './models/projects.js' +import SystemError from '../utils/SystemError.js' const ProjectsRepository = function () { /** @@ -9,7 +9,7 @@ const ProjectsRepository = function () { */ this.createProcedure = function (projectName, procedure) { return new Promise((resolve, reject) => { - projects.create({ projectName, procedure }) + projects.create({ projectName: { $eq: projectName }, procedure }) .then(() => { resolve() }) .catch((error) => { console.error('\x1b[31m%s\x1b[0m', error) @@ -28,7 +28,7 @@ const ProjectsRepository = function () { */ this.updateProjectProcedure = async function (projectName, selectedProcedure) { return new Promise((resolve, reject) => { - projects.updateOne({ projectName }, { procedure: selectedProcedure }) + projects.updateOne({ projectName: { $eq: projectName } }, selectedProcedure) .then(() => { resolve() }) @@ -50,7 +50,7 @@ const ProjectsRepository = function () { this.createLoginConfiguration = async function (projectName, loginCredentials) { const loginMap = (loginCredentials !== undefined && loginCredentials !== null) ? new Map(Object.entries(loginCredentials)) : {} return new Promise((resolve, reject) => { - projects.create({ projectName, login: loginMap }) + projects.create({ projectName: { $eq: projectName }, login: loginMap }) .then(() => { resolve() }) .catch((error) => { console.error('\x1b[31m%s\x1b[0m', error) @@ -66,7 +66,7 @@ const ProjectsRepository = function () { */ this.createProxyConfiguration = async function (projectName, proxy) { return new Promise((resolve, reject) => { - projects.create({ projectName, proxy }) + projects.create({ projectName: { $eq: projectName }, proxy }) .then(() => { resolve() }) .catch((error) => { console.error('\x1b[31m%s\x1b[0m', error) @@ -83,7 +83,7 @@ const ProjectsRepository = function () { this.updateLoginConfiguration = async function (projectName, loginCredentials) { const loginMap = new Map(Object.entries(loginCredentials)) return new Promise((resolve, reject) => { - projects.updateOne({ projectName }, { login: loginMap }) + projects.updateOne({ projectName: { $eq: projectName } }, { login: loginMap }) .then(() => { resolve() }) .catch((error) => { console.error('\x1b[31m%s\x1b[0m', error) @@ -99,7 +99,7 @@ const ProjectsRepository = function () { */ this.updateProxyConfiguration = async function (projectName, proxy) { return new Promise((resolve, reject) => { - projects.updateOne({ projectName }, { proxy }) + projects.updateOne({ projectName: { $eq: projectName } }, { proxy }) .then(() => { resolve() }) .catch((error) => { console.error('\x1b[31m%s\x1b[0m', error) @@ -115,7 +115,7 @@ const ProjectsRepository = function () { */ this.getProjectSettings = async function (projectName) { return new Promise((resolve, reject) => { - projects.findOne({ projectName }, { login: 1, procedure: 1, proxy: 1 }) + projects.findOne({ projectName: { $eq: projectName } }, { login: 1, procedure: 1, proxy: 1 }) .then((result) => { resolve(result) }) @@ -136,9 +136,9 @@ const ProjectsRepository = function () { let systemError = false try { if (procedureRegistered !== undefined || proxyRegistered.ipAddress !== undefined || proxyRegistered.port !== undefined) { - await projects.updateOne({ projectName: projectNameReq }, { $unset: { login: '' } }) + await projects.updateOne({ projectName: { $eq: projectNameReq } }, { $unset: { login: '' } }) } else { - await projects.deleteOne({ projectName: projectNameReq }) + await projects.deleteOne({ projectName: { $eq: projectNameReq } }) } } catch (error) { console.error('\x1b[31m%s\x1b[0m', error) @@ -163,9 +163,9 @@ const ProjectsRepository = function () { let systemError = false try { if (procedureRegistered !== undefined || loginRegistered !== undefined) { - await projects.updateOne({ projectName: projectNameReq }, { $unset: { proxy: '', ipAddress: '', port: '' } }) + await projects.updateOne({ projectName: { $eq: projectNameReq } }, { $unset: { proxy: '', ipAddress: '', port: '' } }) } else { - await projects.deleteOne({ projectName: projectNameReq }) + await projects.deleteOne({ projectName: { $eq: projectNameReq } }) } } catch (error) { console.error('\x1b[31m%s\x1b[0m', error) @@ -186,7 +186,7 @@ const ProjectsRepository = function () { */ this.deleteProjectPerProjectName = async function (projectNameReq) { return new Promise((resolve, reject) => { - projects.deleteOne({ projectName: projectNameReq }) + projects.deleteOne({ projectName: { $eq: projectNameReq } }) .then(() => { console.log(`DELETE URLS PROJECT - project ${projectNameReq} deleted`) resolve() @@ -199,4 +199,4 @@ const ProjectsRepository = function () { } const projectsRepository = new ProjectsRepository() -module.exports = projectsRepository +export default projectsRepository diff --git a/EcoSonar-API/dataBase/tempurlsProjectRepository.js b/EcoSonar-API/dataBase/tempurlsProjectRepository.js index 6e693bd..f121f0f 100644 --- a/EcoSonar-API/dataBase/tempurlsProjectRepository.js +++ b/EcoSonar-API/dataBase/tempurlsProjectRepository.js @@ -1,6 +1,6 @@ -const uniqid = require('uniqid') -const tempurlsproject = require('./models/tempurlsproject') -const SystemError = require('../utils/SystemError') +import uniqid from 'uniqid' +import tempurlsproject from './models/tempurlsproject.js' +import SystemError from '../utils/SystemError.js' const TempUrlsProjectRepository = function () { /** @@ -30,7 +30,7 @@ const TempUrlsProjectRepository = function () { */ this.updateUrls = async function (projectName, urls) { return new Promise((resolve, reject) => { - tempurlsproject.updateOne({ projectName }, { urlsList: urls }) + tempurlsproject.updateOne({ projectName: { $eq: projectName } }, { urlsList: urls }) .then(() => { resolve() }) .catch((err) => { console.error('\x1b[31m%s\x1b[0m', err) @@ -46,7 +46,7 @@ const TempUrlsProjectRepository = function () { */ this.findUrls = async function (name) { return new Promise((resolve, reject) => { - tempurlsproject.findOne({ projectName: name }) + tempurlsproject.findOne({ projectName: { $eq: name } }) .then((result) => { resolve(result) }) .catch((err) => { console.error('\x1b[31m%s\x1b[0m', err) @@ -61,7 +61,7 @@ const TempUrlsProjectRepository = function () { */ this.deleteProject = async function (projectName) { return new Promise((resolve, reject) => { - tempurlsproject.deleteOne({ projectName }) + tempurlsproject.deleteOne({ projectName: { $eq: projectName } }) .then((result) => { console.log(`DELETE URLS PROJECT - On tempurlsproject project ${projectName} removed`) resolve() @@ -75,4 +75,4 @@ const TempUrlsProjectRepository = function () { } const tempurlsProjectRepository = new TempUrlsProjectRepository() -module.exports = tempurlsProjectRepository +export default tempurlsProjectRepository diff --git a/EcoSonar-API/dataBase/urlsProjectRepository.js b/EcoSonar-API/dataBase/urlsProjectRepository.js index f1dd3de..f956833 100644 --- a/EcoSonar-API/dataBase/urlsProjectRepository.js +++ b/EcoSonar-API/dataBase/urlsProjectRepository.js @@ -1,6 +1,7 @@ -const uniqid = require('uniqid') -const urlsprojects = require('./models/urlsprojects') -const SystemError = require('../utils/SystemError') +import uniqid from 'uniqid' +import escapeRegExp from 'lodash/escapeRegExp.js' +import urlsprojects from './models/urlsprojects.js' +import SystemError from '../utils/SystemError.js' const UrlsProjectRepository = function () { /** @@ -84,7 +85,7 @@ const UrlsProjectRepository = function () { */ this.findAll = async function (projectNameReq) { return new Promise((resolve, reject) => { - urlsprojects.find({ projectName: projectNameReq }) + urlsprojects.find({ projectName: { $eq: projectNameReq } }) .then((result) => { resolve(result) }) @@ -120,7 +121,7 @@ const UrlsProjectRepository = function () { */ this.getUserFlow = async function (projectName, urlName) { return new Promise((resolve, reject) => { - urlsprojects.findOne({ projectName, urlName }, { idKey: 1, projectName: 1, urlName: 1, userFlow: 1 }) + urlsprojects.findOne({ projectName: { $eq: projectName }, urlName: { $eq: urlName } }, { idKey: 1, projectName: 1, urlName: 1, userFlow: 1 }) .then((result) => { resolve(result) }) @@ -138,7 +139,7 @@ const UrlsProjectRepository = function () { */ this.deleteUserFlow = async function (projectName, urlName) { return new Promise((resolve, reject) => { - urlsprojects.updateOne({ projectName, urlName }, { $unset: { userFlow: '' } }) + urlsprojects.updateOne({ projectName: { $eq: projectName }, urlName: { $eq: urlName } }, { $unset: { userFlow: '' } }) .then(() => { resolve() }) .catch((error) => { console.error('\x1b[31m%s\x1b[0m', error) @@ -149,7 +150,7 @@ const UrlsProjectRepository = function () { this.findUrl = async function (projectName, urlName) { return new Promise((resolve, reject) => { - urlsprojects.find({ projectName, urlName }, { idKey: 1 }) + urlsprojects.find({ projectName: { $eq: projectName }, urlName: { $eq: urlName } }, { idKey: 1 }) .then((result) => { resolve(result) }) .catch((error) => { console.error('\x1b[31m%s\x1b[0m', error) @@ -165,7 +166,7 @@ const UrlsProjectRepository = function () { this.findAllProjectsNames = async function (filterName) { let query = {} if (filterName !== null) { - query = { projectName: { $regex: new RegExp(filterName, 'i') } } + query = { projectName: { $regex: new RegExp(escapeRegExp(filterName), 'i') } } } return new Promise((resolve, reject) => { urlsprojects.find(query) @@ -181,4 +182,4 @@ const UrlsProjectRepository = function () { } const urlsProjectRepository = new UrlsProjectRepository() -module.exports = urlsProjectRepository +export default urlsProjectRepository diff --git a/EcoSonar-API/dataBase/w3cRepository.js b/EcoSonar-API/dataBase/w3cRepository.js index 5713358..6c86d4e 100644 --- a/EcoSonar-API/dataBase/w3cRepository.js +++ b/EcoSonar-API/dataBase/w3cRepository.js @@ -1,6 +1,5 @@ - -const SystemError = require('../utils/SystemError') -const w3cs = require('./models/w3cs') +import SystemError from '../utils/SystemError.js' +import w3cs from './models/w3cs.js' const W3cRepository = function () { /** @@ -140,4 +139,4 @@ const W3cRepository = function () { } const w3cRepository = new W3cRepository() -module.exports = w3cRepository +export default w3cRepository diff --git a/EcoSonar-API/package.json b/EcoSonar-API/package.json index e75d895..d14a459 100644 --- a/EcoSonar-API/package.json +++ b/EcoSonar-API/package.json @@ -1,12 +1,13 @@ { "name": "ecosonar-api", - "version": "3.4", + "version": "3.5", "description": "Ecodesign and accessibility tool to help developpers minimize carbon footprint of their web-application", "repository": { "type": "git", "url": "https://github.com/Accenture/EcoSonar" }, "main": "app.js", + "type": "module", "scripts": { "postinstall": "node builder.js", "test": "echo \"Error: no test specified\" && exit 1", @@ -17,34 +18,37 @@ "author": "", "license": "GNU", "dependencies": { - "@azure/identity": "^3.3.0", - "@azure/keyvault-secrets": "^4.6.0", + "@azure/identity": "4.0.1", + "@azure/keyvault-secrets": "^4.8.0", "cheerio": "^1.0.0-rc.12", + "chrome-har": "^0.13.2", "concat-files": "^0.1.1", "cors": "^2.8.5", - "dotenv": "^16.0.3", - "exceljs": "^4.3.0", + "dotenv": "^16.3.2", + "exceljs": "^4.4.0", "express": "^4.18.2", - "glob": "^8.1.0", + "express-rate-limit": "^7.2.0", + "glob": "^10.3.10", + "helmet": "^7.1.0", "html-validator": "^6.0.1", "js-yaml": "^4.1.0", - "lighthouse": "^9.6.8", - "mongodb": "^5.0.1", - "mongoose": " ^7.6.2", - "puppeteer": "^18.2.1", - "puppeteer-har": "^1.1.2", + "lighthouse": "^11.6.0", + "lodash": "^4.17.21", + "mongodb": "^6.3.0", + "mongoose": " ^8.2.0", + "puppeteer": "^22.4.1", "swagger-jsdoc": "^6.2.8", "swagger-ui-express": "^5.0.0", "uniqid": "^5.4.0" }, "devDependencies": { - "@ecocode/eslint-plugin": "^0.2.0", - "eslint": "^8.39.0", - "eslint-config-standard": "^17.0.0", - "eslint-plugin-import": "^2.27.5", - "eslint-plugin-n": "^15.6.1", + "@ecocode/eslint-plugin": "^1.4.0", + "eslint": "^8.57.0", + "eslint-config-standard": "^17.1.0", + "eslint-plugin-import": "^2.29.1", + "eslint-plugin-n": "^16.6.2", "eslint-plugin-node": "^11.1.0", "eslint-plugin-promise": "^6.1.1", - "nodemon": "^3.0.1" + "nodemon": "^3.1.0" } } diff --git a/EcoSonar-API/routes/app.js b/EcoSonar-API/routes/app.js index e77bc8b..e137fe3 100644 --- a/EcoSonar-API/routes/app.js +++ b/EcoSonar-API/routes/app.js @@ -1,22 +1,26 @@ -const express = require('express') -const cors = require('cors') -const dotenv = require('dotenv') -const urlConfigurationService = require('../services/urlConfigurationService') -const analysisService = require('../services/analysisService') -const retrieveAnalysisService = require('../services/retrieveAnalysisService') -const retrieveBestPracticesService = require('../services/retrieveBestPracticesService') -const crawlerService = require('../services/crawler/crawlerService') -const procedureService = require('../services/procedureService') -const loginProxyConfigurationService = require('../services/loginProxyConfigurationService') -const userJourneyService = require('../services/userJourneyService') -const exportAuditService = require('../services/exportAuditService') -const SystemError = require('../utils/SystemError') -const asyncMiddleware = require('../utils/AsyncMiddleware') -const swaggerUi = require('swagger-ui-express') -const swaggerSpec = require('../swagger') -const projectService = require('../services/projectService') +import express from 'express' +import cors from 'cors' +import dotenv from 'dotenv' +import swaggerUi from 'swagger-ui-express' +import helmet from 'helmet' +import swaggerSpec from '../swagger.js' +import RateLimit from 'express-rate-limit' +import urlConfigurationService from '../services/urlConfigurationService.js' +import analysisService from '../services/analysisService.js' +import retrieveAnalysisService from '../services/retrieveAnalysisService.js' +import retrieveBestPracticesService from '../services/retrieveBestPracticesService.js' +import crawlerService from '../services/crawler/crawlerService.js' +import procedureService from '../services/procedureService.js' +import loginProxyConfigurationService from '../services/loginProxyConfigurationService.js' +import userJourneyService from '../services/userJourneyService.js' +import exportAuditService from '../services/exportAuditService.js' +import SystemError from '../utils/SystemError.js' +import asyncMiddleware from '../utils/AsyncMiddleware.js' +import projectService from '../services/projectService.js' +import bestPracticesServices from '../services/bestPracticesService.js' +import { createRequire } from 'node:module' +const require = createRequire(import.meta.url) const packageJson = require('../package.json') -const bestPracticesServices = require('../services/bestPracticesService') dotenv.config() @@ -25,6 +29,7 @@ app.disable('x-powered-by') app.use(express.urlencoded({ extended: true })) app.use(express.json()) +app.use(helmet()) app.use('/swagger', swaggerUi.serve, swaggerUi.setup(swaggerSpec)) const PORT = process.env.SWAGGER_PORT || 3002 @@ -33,11 +38,9 @@ app.listen(PORT, () => console.log(`Swagger in progress on port ${PORT}`)) const sonarqubeServerUrl = process.env.ECOSONAR_ENV_SONARQUBE_SERVER_URL || '' const whitelist = [sonarqubeServerUrl] -if (process.env.ECOSONAR_ENV_CLOUD_PROVIDER === 'local') { - const localServers = process.env.ECOSONAR_ENV_LOCAL_DEV_SERVER_URL?.split(';') || [] - for (const localServer of localServers) { - whitelist.push(localServer) - } +const localServers = process.env.ECOSONAR_ENV_LOCAL_DEV_SERVER_URL?.split(';') || [] +for (const localServer of localServers) { + whitelist.push(localServer) } const corsOptions = { @@ -51,6 +54,13 @@ const corsOptions = { } app.use(cors(corsOptions)) +const limiter = RateLimit({ + windowMs: 15 * 60 * 1000, // 15 minutes + max: 300 // max 300 requests per windowMs +}) + +app.use(limiter) + app.use((_req, res, next) => { res.setHeader('Access-Control-Allow-Headers', 'Origin, X-Requested-With, Content, Accept, Content-Type, Authorization') res.setHeader('Access-Control-Allow-Methods', 'GET, POST, PUT, DELETE, PATCH, OPTIONS') @@ -629,6 +639,10 @@ app.delete('/api/user-flow', asyncMiddleware(async (req, res, _next) => { * properties: * projectName: * type: string + * username: + * type: string + * password: + * type: string * responses: * 202: * description: Analysis launched @@ -636,8 +650,10 @@ app.delete('/api/user-flow', asyncMiddleware(async (req, res, _next) => { */ app.post('/api/greenit/insert', asyncMiddleware(async (req, res, _next) => { const projectName = req.body.projectName + const username = req.body.username + const password = req.body.password console.log('INSERT ANALYSIS - Launch analysis for project ' + projectName) - analysisService.insert(projectName) + analysisService.insert(projectName, username, password) res.status(202).send() })) @@ -990,6 +1006,10 @@ app.post('/api/bestPractices/url', asyncMiddleware(async (req, res, _next) => { * type: string * saveUrls: * type: boolean + * username: + * type: string + * password: + * type: string * responses: * 202: * description: Crawler started. @@ -998,9 +1018,10 @@ app.post('/api/crawl', asyncMiddleware(async (req, res, _next) => { const projectName = req.body.projectName const mainUrl = req.body.mainUrl const saveUrls = req.body.saveUrls + const username = req.body.username + const password = req.body.password console.log(`CRAWLER - Running crawler from ${mainUrl}`) - crawlerService.launchCrawl(projectName, mainUrl, saveUrls) - console.log('CRAWLER - Crawler started') + crawlerService.launchCrawl(projectName, mainUrl, saveUrls, username, password) return res.status(202).send() })) @@ -1228,4 +1249,4 @@ app.delete('/api/project', asyncMiddleware(async (req, res, _next) => { }) })) -module.exports = app +export default app diff --git a/EcoSonar-API/server.js b/EcoSonar-API/server.js index aee006b..3b99e07 100644 --- a/EcoSonar-API/server.js +++ b/EcoSonar-API/server.js @@ -1,6 +1,6 @@ -const http = require('http') -const app = require('./routes/app') -const database = require('./configuration/database') +import http from 'http' +import app from './routes/app.js' +import database from './configuration/database.js' // connection BDD database.connection() diff --git a/EcoSonar-API/services/W3C/w3cAnalysis.js b/EcoSonar-API/services/W3C/w3cAnalysis.js index f02178c..028e98f 100644 --- a/EcoSonar-API/services/W3C/w3cAnalysis.js +++ b/EcoSonar-API/services/W3C/w3cAnalysis.js @@ -1,6 +1,6 @@ -const validator = require('html-validator') -const puppeteer = require('puppeteer') -const authenticationService = require('../authenticationService') +import validator from 'html-validator' +import puppeteer from 'puppeteer' +import authenticationService from '../authenticationService.js' class W3cAnalysis {} @@ -39,7 +39,7 @@ W3cAnalysis.prototype.w3cAnalysisWithAPI = async function (urlsList) { * @param {Array} urlsList is a list of urls that needs to be analysed by W3C HTML Validator * @returns a list of HTML issues */ -W3cAnalysis.prototype.w3cAnalysisLocal = async function (urlsList, projectName) { +W3cAnalysis.prototype.w3cAnalysisLocal = async function (urlsList, projectName, username, password) { // Initializing variables const resultForUrlsList = [] const htmlResults = [] @@ -60,7 +60,7 @@ W3cAnalysis.prototype.w3cAnalysisLocal = async function (urlsList, projectName) // Starting the browser const browser = await puppeteer.launch({ - headless: true, + headless: 'new', args: browserArgs, ignoreHTTPSErrors: true, // Keep gpu horsepower in headless @@ -71,7 +71,7 @@ W3cAnalysis.prototype.w3cAnalysisLocal = async function (urlsList, projectName) try { // Extracting the HTML content of each url with pupeteer - const loginSucceeded = await authenticationService.loginIfNeeded(browser) + const loginSucceeded = await authenticationService.loginIfNeeded(browser, projectName, username, password) if (loginSucceeded) { // analyse each page @@ -122,4 +122,4 @@ W3cAnalysis.prototype.w3cAnalysisLocal = async function (urlsList, projectName) } } const w3cAnalysis = new W3cAnalysis() -module.exports = w3cAnalysis +export default w3cAnalysis diff --git a/EcoSonar-API/services/analysisService.js b/EcoSonar-API/services/analysisService.js index dd1901a..14bbad8 100644 --- a/EcoSonar-API/services/analysisService.js +++ b/EcoSonar-API/services/analysisService.js @@ -1,16 +1,16 @@ -const uniqid = require('uniqid') -const greenItRepository = require('../dataBase/greenItRepository') -const bestPracticesRepository = require('../dataBase/bestPracticesRepository') -const urlsProjectRepository = require('../dataBase/urlsProjectRepository') -const lighthouseRepository = require('../dataBase/lighthouseRepository') -const lighthouseAnalysis = require('./lighthouse/lighthouse') -const formatLighthouseMetrics = require('./format/formatLighthouseMetrics') -const greenItAnalysis = require('./greenit-analysis/analyseService') -const formatLighthouseBestPractices = require('./format/formatLighthouseBestPractices') -const w3cAnalysis = require('../services/W3C/w3cAnalysis') -const w3cRepository = require('../dataBase/w3cRepository') -const formatW3cBestPractices = require('./format/formatW3cBestPractices') -const formatW3cAnalysis = require('./format/formatW3cAnalysis') +import uniqid from 'uniqid' +import greenItRepository from '../dataBase/greenItRepository.js' +import bestPracticesRepository from '../dataBase/bestPracticesRepository.js' +import urlsProjectRepository from '../dataBase/urlsProjectRepository.js' +import lighthouseRepository from '../dataBase/lighthouseRepository.js' +import lighthouseAnalysis from './lighthouse/lighthouse.js' +import formatLighthouseMetrics from './format/formatLighthouseMetrics.js' +import analyse from './greenit-analysis/analyseService.js' +import formatLighthouseBestPractices from './format/formatLighthouseBestPractices.js' +import w3cAnalysis from '../services/W3C/w3cAnalysis.js' +import w3cRepository from '../dataBase/w3cRepository.js' +import formatW3cBestPractices from './format/formatW3cBestPractices.js' +import formatW3cAnalysis from './format/formatW3cAnalysis.js' class AnalysisService {} @@ -19,7 +19,7 @@ class AnalysisService {} * @param {string} projectName * @param {boolean} autoscroll is used to enable autoscrolling for each tab opened during analysis */ -AnalysisService.prototype.insert = async function (projectName, autoscroll) { +AnalysisService.prototype.insert = async function (projectName, username, password, autoscroll) { const allowExternalAPI = process.env.ECOSONAR_ENV_ALLOW_EXTERNAL_API || 'false' let urlProjectList = [] let reports = [] @@ -34,7 +34,7 @@ AnalysisService.prototype.insert = async function (projectName, autoscroll) { if (systemError || urlProjectList.length === 0) { console.warn('GREENIT INSERT - project has no url to do the audit. Audit stopped') } else { - reports = await launchAuditsToUrlList(urlProjectList, autoscroll, projectName, allowExternalAPI) + reports = await launchAuditsToUrlList(urlProjectList, projectName, allowExternalAPI, username, password, autoscroll) const reportsFormatted = formatAuditsToBeSaved(reports, urlProjectList) greenItRepository @@ -76,19 +76,19 @@ AnalysisService.prototype.insert = async function (projectName, autoscroll) { } } -async function launchAuditsToUrlList (urlProjectList, autoscroll, projectName, allowExternalAPI) { +async function launchAuditsToUrlList (urlProjectList, projectName, allowExternalAPI, username, password, autoscroll) { let reportsGreenit = [] let reportsLighthouse = [] let reportsW3c = [] const urlList = urlProjectList.map((url) => url.urlName) try { - reportsGreenit = await greenItAnalysis.analyse(urlList, autoscroll, projectName) + reportsGreenit = await analyse(urlList, projectName, username, password, autoscroll) } catch (error) { console.error(error) } try { - reportsLighthouse = await lighthouseAnalysis.lighthouseAnalysis(urlList, projectName) + reportsLighthouse = await lighthouseAnalysis(urlList, projectName, username, password) } catch (error) { console.error(error) } @@ -208,4 +208,4 @@ function formatAuditsToBeSaved (reports, urlProjectList) { } const analysisService = new AnalysisService() -module.exports = analysisService +export default analysisService diff --git a/EcoSonar-API/services/authenticationService.js b/EcoSonar-API/services/authenticationService.js index e097fa7..2f80a23 100644 --- a/EcoSonar-API/services/authenticationService.js +++ b/EcoSonar-API/services/authenticationService.js @@ -1,12 +1,12 @@ -const { waitForSelectors, applyChange } = require('../utils/playSelectors') -const loginProxyConfigurationService = require('./loginProxyConfigurationService') -const viewPortParams = require('../utils/viewportParams') +import { waitForSelectors, applyChange } from '../utils/playSelectors.js' +import loginProxyConfigurationService from './loginProxyConfigurationService.js' +import viewPortParams from '../utils/viewportParams.js' class AuthenticationService { } -AuthenticationService.prototype.loginIfNeeded = async function (browser, projectName) { +AuthenticationService.prototype.loginIfNeeded = async function (browser, projectName, username, password) { let loginInformations - await loginProxyConfigurationService.getLoginCredentials(projectName) + await loginProxyConfigurationService.getLoginCredentials(projectName, username, password) .then((login) => { loginInformations = login }) @@ -17,12 +17,12 @@ AuthenticationService.prototype.loginIfNeeded = async function (browser, project // Go to login url const [page] = await browser.pages() - await page.setViewport(viewPortParams.viewPortParams) + await page.setViewport(viewPortParams) await page.goto(loginInformations.authentication_url, { timeout: 0, waitUntil: 'networkidle2' }) // Fill login fields if (loginInformations.steps) { - return loginOnMultiPages(page, loginInformations.steps) + return loginOnMultiPages(page, loginInformations) } else { return loginOnOnePage(page, loginInformations) } @@ -59,18 +59,24 @@ async function loginOnOnePage (page, loginInformations) { } } -async function loginOnMultiPages (page, steps) { +async function loginOnMultiPages (page, loginInformations) { const timeout = 10000 let step; let element try { - for (step of steps) { + for (step of loginInformations.steps) { element = await waitForSelectors(step.selectors, page, { timeout, visible: true }) switch (step.type) { case 'click': await element.click() break case 'change': - await applyChange(step.value, element) + if (step.value === '%USERNAME%') { + await applyChange(loginInformations.username, element) + } else if (step.value === '%PASSWORD%') { + await applyChange(loginInformations.password, element) + } else { + await applyChange(step.value, element) + } break default: break @@ -101,4 +107,4 @@ AuthenticationService.prototype.useProxyIfNeeded = async function (projectName) } const authenticationService = new AuthenticationService() -module.exports = authenticationService +export default authenticationService diff --git a/EcoSonar-API/services/bestPracticesService.js b/EcoSonar-API/services/bestPracticesService.js index 3c6ddf7..d98d030 100644 --- a/EcoSonar-API/services/bestPracticesService.js +++ b/EcoSonar-API/services/bestPracticesService.js @@ -1,3 +1,5 @@ +import { createRequire } from 'node:module' +const require = createRequire(import.meta.url) const greenItData = require('../utils/bestPractices/greenItData.json') const lighthouseAccessibilityData = require('../utils/bestPractices/lighthouseAccessibilityData.json') const lighthousePerformanceData = require('../utils/bestPractices/lighthousePerformanceData.json') @@ -13,4 +15,4 @@ BestPracticesServices.prototype.getAllBestPracticesRules = function () { } const bestPracticesServices = new BestPracticesServices() -module.exports = bestPracticesServices +export default bestPracticesServices diff --git a/EcoSonar-API/services/crawler/crawlerService.js b/EcoSonar-API/services/crawler/crawlerService.js index 714b4d9..e2850fc 100644 --- a/EcoSonar-API/services/crawler/crawlerService.js +++ b/EcoSonar-API/services/crawler/crawlerService.js @@ -1,8 +1,8 @@ -const cheerio = require('cheerio') -const puppeteer = require('puppeteer') -const authenticationService = require('../authenticationService') -const urlConfigurationService = require('../urlConfigurationService') -const tempUrlsProjectRepository = require('../../dataBase/tempurlsProjectRepository') +import cheerio from 'cheerio' +import puppeteer from 'puppeteer' +import authenticationService from '../authenticationService.js' +import urlConfigurationService from '../urlConfigurationService.js' +import tempUrlsProjectRepository from '../../dataBase/tempurlsProjectRepository.js' class CrawlerService { } @@ -13,59 +13,73 @@ class CrawlerService { } * @param {*} savedAsPermanent boolean value that mentions if urls crawled should be saved permanently or temporary * launch crawling of the website and save the urls crawled according to context */ -CrawlerService.prototype.launchCrawl = async function (projectName, mainUrl, savedAsPermanent) { +CrawlerService.prototype.launchCrawl = async function (projectName, mainUrl, savedAsPermanent, username, password) { let crawledUrls = [] let projectUrls = [] const seenUrls = [] - const browserArgs = [ - '--no-sandbox', // can't run inside docker without - '--disable-setuid-sandbox', // but security issues - '--ignore-certificate-errors', - '--window-size=1920,1080' - ] + const isUrlValid = verifyUrl(mainUrl) + if (!isUrlValid) { + console.error('URL set is not valid and can not be crawled') + } else { + const browserArgs = [ + '--no-sandbox', // can't run inside docker without + '--disable-setuid-sandbox', // but security issues + '--ignore-certificate-errors', + '--window-size=1920,1080' + ] - const proxyConfiguration = await authenticationService.useProxyIfNeeded(projectName) - if (proxyConfiguration) { - browserArgs.push(proxyConfiguration) - } + const proxyConfiguration = await authenticationService.useProxyIfNeeded(projectName) + if (proxyConfiguration) { + browserArgs.push(proxyConfiguration) + } - // start browser and authenticate - const browser = await puppeteer.launch({ - headless: true, - args: browserArgs, - ignoreHTTPSErrors: true, - ignoreDefaultArgs: ['--disable-gpu'] - }) + // start browser and authenticate + const browser = await puppeteer.launch({ + headless: 'new', + args: browserArgs, + ignoreHTTPSErrors: true, + ignoreDefaultArgs: ['--disable-gpu'] + }) - try { - const websiteDetails = getWebsiteProtocolAndPrefix(mainUrl) - await authenticationService.loginIfNeeded(browser) - await recursiveCrawl(mainUrl, browser, crawledUrls, seenUrls, websiteDetails) - } catch (error) { - console.error(error) - } finally { browser.close() } + try { + const loginSucceeded = await authenticationService.loginIfNeeded(browser, projectName, username, password) + if (loginSucceeded) { + const websiteDetails = getWebsiteProtocolAndPrefix(mainUrl) + await recursiveCrawl(mainUrl, browser, crawledUrls, seenUrls, websiteDetails) + } else { + console.warn('Could not log in, crawled is skipped') + } + } catch (error) { + console.error(error) + } finally { browser.close() } - // Get all the URL already registered in the project to avoid crawling them again - await urlConfigurationService.getAll(projectName) - .then((result) => { - projectUrls = result - }).catch(() => { - console.log('An error occured when retrieving urls saved to be audited for the project or no urls were saved') - }) - // Removing mainUrl and aliases from return list if already exist in the project - if (projectUrls.includes(mainUrl) || projectUrls.includes(mainUrl + '/') || projectUrls.includes(mainUrl.slice(0, -1)) || crawledUrls.includes(mainUrl + '/')) { - crawledUrls = crawledUrls.filter((url) => (url !== mainUrl)) - } + // Get all the URL already registered in the project to avoid crawling them again + await urlConfigurationService.getAll(projectName) + .then((result) => { + projectUrls = result + }).catch(() => { + console.log('An error occured when retrieving urls saved to be audited for the project or no urls were saved') + }) + // Removing mainUrl and aliases from return list if already exist in the project + if (projectUrls.includes(mainUrl) || projectUrls.includes(mainUrl + '/') || projectUrls.includes(mainUrl.slice(0, -1)) || crawledUrls.includes(mainUrl + '/')) { + crawledUrls = crawledUrls.filter((url) => (url !== mainUrl)) + } - // Remove URLS that are being crawler twice due to ending slash in it (www.myurl.com/ vs www.myurl.com) - for (const crawledUrl of crawledUrls) { - crawledUrls = crawledUrls.filter((url) => (url !== crawledUrl.slice(0, -1))) - } + // Remove URLS that are being crawler twice due to ending slash in it (www.myurl.com/ vs www.myurl.com) + for (const crawledUrl of crawledUrls) { + crawledUrls = crawledUrls.filter((url) => (url !== crawledUrl.slice(0, -1))) + } + + crawledUrls = crawledUrls.filter((url) => (!projectUrls.includes(url) && !projectUrls.includes(url + '/') && !projectUrls.includes(url.slice(0, -1)))) - crawledUrls = crawledUrls.filter((url) => (!projectUrls.includes(url) && !projectUrls.includes(url + '/') && !projectUrls.includes(url.slice(0, -1)))) + saveUrlsCrawled(projectName, crawledUrls, savedAsPermanent) + } +} - saveUrlsCrawled(projectName, crawledUrls, savedAsPermanent) +function verifyUrl (homepageUrl) { + const urlPattern = /^(https?|):\/\/[^\s/$.?#].[^\s]*$/i + return urlPattern.test(homepageUrl) } /** @@ -258,4 +272,4 @@ CrawlerService.prototype.retrieveCrawledUrl = async function (projectName) { } const crawlerService = new CrawlerService() -module.exports = crawlerService +export default crawlerService diff --git a/EcoSonar-API/services/ecoIndexCalculationService.js b/EcoSonar-API/services/ecoIndexCalculationService.js index 5f1bd1d..2980e75 100644 --- a/EcoSonar-API/services/ecoIndexCalculationService.js +++ b/EcoSonar-API/services/ecoIndexCalculationService.js @@ -55,4 +55,4 @@ EcoIndexCalculationService.prototype.setScoreLetter = function (metric, value) { } else return 'G' } const ecoIndexCalculationService = new EcoIndexCalculationService() -module.exports = ecoIndexCalculationService +export default ecoIndexCalculationService diff --git a/EcoSonar-API/services/exportAuditService.js b/EcoSonar-API/services/exportAuditService.js index 03761eb..ec9d63a 100644 --- a/EcoSonar-API/services/exportAuditService.js +++ b/EcoSonar-API/services/exportAuditService.js @@ -1,7 +1,7 @@ -const exceljs = require('exceljs') -const retrieveAnalysisService = require('./retrieveAnalysisService') -const urlConfigurationService = require('./urlConfigurationService') -const SystemError = require('../utils/SystemError') +import exceljs from 'exceljs' +import retrieveAnalysisService from './retrieveAnalysisService.js' +import urlConfigurationService from './urlConfigurationService.js' +import SystemError from '../utils/SystemError.js' class ExportAuditService { } @@ -762,4 +762,4 @@ function checkAnalysisIsEmpty (analysis) { } const exportAuditService = new ExportAuditService() -module.exports = exportAuditService +export default exportAuditService diff --git a/EcoSonar-API/services/format/bestPracticesSorting.js b/EcoSonar-API/services/format/bestPracticesSorting.js index ccd97bc..f8238d3 100644 --- a/EcoSonar-API/services/format/bestPracticesSorting.js +++ b/EcoSonar-API/services/format/bestPracticesSorting.js @@ -1,3 +1,5 @@ +import { createRequire } from 'node:module' +const require = createRequire(import.meta.url) const ecodesignMetric = require('../../utils/feature_importance_model.json') const quickWinsConfig = require('../../utils/quick_wins_configuration.json') @@ -198,4 +200,4 @@ function addBestPracticesMissingFromHighestImpactModel (list, newList, complianc } const bestPracticesSorting = new BestPracticesSorting() -module.exports = bestPracticesSorting +export default bestPracticesSorting diff --git a/EcoSonar-API/services/format/formatBestPracticesForProject.js b/EcoSonar-API/services/format/formatBestPracticesForProject.js index e6f0522..9de4d29 100644 --- a/EcoSonar-API/services/format/formatBestPracticesForProject.js +++ b/EcoSonar-API/services/format/formatBestPracticesForProject.js @@ -1,5 +1,5 @@ +import formatCompliance from '../../services/format/formatCompliance.js' -const formatCompliance = require('../../services/format/formatCompliance') class FormatBestPracticesForProject { } FormatBestPracticesForProject.prototype.addScores = function (totalScore, score, numberOfValues) { @@ -95,4 +95,4 @@ FormatBestPracticesForProject.prototype.formatResponse = function (auditedMetric } const formatBestPracticesForProject = new FormatBestPracticesForProject() -module.exports = formatBestPracticesForProject +export default formatBestPracticesForProject diff --git a/EcoSonar-API/services/format/formatCompliance.js b/EcoSonar-API/services/format/formatCompliance.js index 0c22f9e..19ae3ed 100644 --- a/EcoSonar-API/services/format/formatCompliance.js +++ b/EcoSonar-API/services/format/formatCompliance.js @@ -52,4 +52,4 @@ FormatCompliance.prototype.complianceLevelW3c = function (type) { } const formatCompliance = new FormatCompliance() -module.exports = formatCompliance +export default formatCompliance diff --git a/EcoSonar-API/services/format/formatGreenItAnalysis.js b/EcoSonar-API/services/format/formatGreenItAnalysis.js index 2f7a98d..9604a87 100644 --- a/EcoSonar-API/services/format/formatGreenItAnalysis.js +++ b/EcoSonar-API/services/format/formatGreenItAnalysis.js @@ -1,5 +1,5 @@ -const ecoIndexCalculationService = require('../ecoIndexCalculationService') -const formatCompliance = require('./formatCompliance') +import ecoIndexCalculationService from '../ecoIndexCalculationService.js' +import formatCompliance from './formatCompliance.js' class FormatGreenItAnalysis {} @@ -115,4 +115,4 @@ function compareFullDate (firstDate, secondDate) { } const formatGreenItAnalysis = new FormatGreenItAnalysis() -module.exports = formatGreenItAnalysis +export default formatGreenItAnalysis diff --git a/EcoSonar-API/services/format/formatGreenItBestPractices.js b/EcoSonar-API/services/format/formatGreenItBestPractices.js index 36fa860..aeaa46d 100644 --- a/EcoSonar-API/services/format/formatGreenItBestPractices.js +++ b/EcoSonar-API/services/format/formatGreenItBestPractices.js @@ -1,4 +1,4 @@ -const formatBestPracticesForProject = require('../format/formatBestPracticesForProject') +import formatBestPracticesForProject from '../format/formatBestPracticesForProject.js' class FormatGreenItBestPractices {} @@ -567,4 +567,4 @@ FormatGreenItBestPractices.prototype.useStandardTypefaces = function (reports) { } const formatGreenItBestPractices = new FormatGreenItBestPractices() -module.exports = formatGreenItBestPractices +export default formatGreenItBestPractices diff --git a/EcoSonar-API/services/format/formatGreenItReports.js b/EcoSonar-API/services/format/formatGreenItReports.js index b02b1b8..89f511c 100644 --- a/EcoSonar-API/services/format/formatGreenItReports.js +++ b/EcoSonar-API/services/format/formatGreenItReports.js @@ -1,6 +1,5 @@ - -const formatGreenItBestPractices = require('./formatGreenItBestPractices') -const formatBestPracticesForProject = require('../format/formatBestPracticesForProject') +import formatGreenItBestPractices from './formatGreenItBestPractices.js' +import formatBestPracticesForProject from '../format/formatBestPracticesForProject.js' class FormatGreenItReports {} @@ -39,4 +38,4 @@ FormatGreenItReports.prototype.returnFormattedGreenIt = function (reports) { return formatBestPracticesForProject.sortByScore(greenItBestPractices) } const formatGreenItReports = new FormatGreenItReports() -module.exports = formatGreenItReports +export default formatGreenItReports diff --git a/EcoSonar-API/services/format/formatLighthouseAnalysis.js b/EcoSonar-API/services/format/formatLighthouseAnalysis.js index 2dd5f6f..c3076ff 100644 --- a/EcoSonar-API/services/format/formatLighthouseAnalysis.js +++ b/EcoSonar-API/services/format/formatLighthouseAnalysis.js @@ -1,4 +1,4 @@ -const formatCompliance = require('./formatCompliance') +import formatCompliance from './formatCompliance.js' class FormatLighthouseAnalysis {} @@ -251,4 +251,4 @@ function compareFullDate (firstDate, secondDate) { } const formatLighthouseAnalysis = new FormatLighthouseAnalysis() -module.exports = formatLighthouseAnalysis +export default formatLighthouseAnalysis diff --git a/EcoSonar-API/services/format/formatLighthouseBestPractices.js b/EcoSonar-API/services/format/formatLighthouseBestPractices.js index a4cc4ea..115dadd 100644 --- a/EcoSonar-API/services/format/formatLighthouseBestPractices.js +++ b/EcoSonar-API/services/format/formatLighthouseBestPractices.js @@ -1,6 +1,6 @@ -const enumAudits = require('../../utils/enumAudits') -const formatBestPracticesForProject = require('../format/formatBestPracticesForProject') -const metricsCalculate = require('../../utils/metricsCalculate') +import enumAudits from '../../utils/enumAudits.js' +import formatBestPracticesForProject from '../format/formatBestPracticesForProject.js' +import metricsCalculate from '../../utils/metricsCalculate.js' class FormatLighthouseBestPractices { } @@ -47,16 +47,21 @@ FormatLighthouseBestPractices.prototype.formatPerformance = function (report) { const performanceBestPractices = enumAudits.performanceNamesToSave() const formattedReports = {} for (const element in performanceBestPractices) { - const score = report.audits[performanceBestPractices[element]].score - const scoreDisplayMode = report.audits[performanceBestPractices[element]].scoreDisplayMode - const items = (report.audits[performanceBestPractices[element]].details !== undefined) ? report.audits[performanceBestPractices[element]].details.items : [] - let displayValue = report.audits[performanceBestPractices[element]].displayValue - if (displayValue) { - displayValue = Number(displayValue.replace(',', '').match(reg)[0]) - } else { - displayValue = 0 + try { + const score = report.audits[performanceBestPractices[element]].score ?? 0 + const scoreDisplayMode = report.audits[performanceBestPractices[element]].scoreDisplayMode ?? null + const items = (report.audits[performanceBestPractices[element]].details !== undefined) ? report.audits[performanceBestPractices[element]].details.items : [] + let displayValue = report.audits[performanceBestPractices[element]].displayValue ?? null + if (displayValue) { + displayValue = Number(displayValue.replace(',', '').match(reg)[0]) + } else { + displayValue = 0 + } + formattedReports[element] = { score: score * 100, scoreDisplayMode, description: items, auditedMetric: displayValue } + } catch (error) { + console.error('error for url ' + report.url + ' on element ' + element) + console.error(error) } - formattedReports[element] = { score: score * 100, scoreDisplayMode, description: items, auditedMetric: displayValue } } return { ...formattedReports, url: report.url } } @@ -222,4 +227,4 @@ FormatLighthouseBestPractices.prototype.returnFormattedAccessibility = function } const formatLighthouseBestPractices = new FormatLighthouseBestPractices() -module.exports = formatLighthouseBestPractices +export default formatLighthouseBestPractices diff --git a/EcoSonar-API/services/format/formatLighthouseMetrics.js b/EcoSonar-API/services/format/formatLighthouseMetrics.js index 013eaa5..548d1a0 100644 --- a/EcoSonar-API/services/format/formatLighthouseMetrics.js +++ b/EcoSonar-API/services/format/formatLighthouseMetrics.js @@ -1,5 +1,5 @@ -const enumAudits = require('../../utils/enumAudits') -const formatCompliance = require('./formatCompliance') +import enumAudits from '../../utils/enumAudits.js' +import formatCompliance from './formatCompliance.js' class FormatLighthouseMetrics {} @@ -8,21 +8,25 @@ FormatLighthouseMetrics.prototype.formatLighthouseMetrics = function (reports) { const reg = /\d+\.*\d*/g const formattedReports = {} for (const element in lighthouseMetrics) { - let score, complianceLevel, displayValue - if (element === 'performance' || element === 'accessibility') { - score = Math.trunc(reports.categories[lighthouseMetrics[element]].score * 100) - complianceLevel = element === 'performance' ? formatCompliance.getEcodesignGrade(Math.trunc(reports.categories[lighthouseMetrics[element]].score * 100)) : formatCompliance.getAccessibilityGrade(Math.trunc(reports.categories[lighthouseMetrics[element]].score * 100)) - formattedReports[element] = { score, complianceLevel } - } else { - score = Math.trunc(reports.audits[lighthouseMetrics[element]].score * 100) - displayValue = reports.audits[lighthouseMetrics[element]].displayValue - displayValue = displayValue ? Number(displayValue.replace(',', '').match(reg)[0]) : 0 - complianceLevel = formatCompliance.getEcodesignGrade(Math.trunc(reports.audits[lighthouseMetrics[element]].score * 100)) - formattedReports[element] = { score, displayValue, complianceLevel } + try { + let score, complianceLevel, displayValue + if (element === 'performance' || element === 'accessibility') { + score = Math.trunc(reports.categories[lighthouseMetrics[element]].score * 100) + complianceLevel = element === 'performance' ? formatCompliance.getEcodesignGrade(Math.trunc(reports.categories[lighthouseMetrics[element]].score * 100)) : formatCompliance.getAccessibilityGrade(Math.trunc(reports.categories[lighthouseMetrics[element]].score * 100)) + formattedReports[element] = { score, complianceLevel } + } else { + score = Math.trunc(reports.audits[lighthouseMetrics[element]].score * 100) + displayValue = reports.audits[lighthouseMetrics[element]].displayValue + displayValue = displayValue ? Number(displayValue.replace(',', '').match(reg)[0]) : 0 + complianceLevel = formatCompliance.getEcodesignGrade(Math.trunc(reports.audits[lighthouseMetrics[element]].score * 100)) + formattedReports[element] = { score, displayValue, complianceLevel } + } + } catch (error) { + console.error(error) } } return formattedReports } const formatLighthouseMetrics = new FormatLighthouseMetrics() -module.exports = formatLighthouseMetrics +export default formatLighthouseMetrics diff --git a/EcoSonar-API/services/format/formatW3cAnalysis.js b/EcoSonar-API/services/format/formatW3cAnalysis.js index c2b2ce2..7229cff 100644 --- a/EcoSonar-API/services/format/formatW3cAnalysis.js +++ b/EcoSonar-API/services/format/formatW3cAnalysis.js @@ -1,4 +1,6 @@ -const formatCompliance = require('./formatCompliance') +import formatCompliance from './formatCompliance.js' +import { createRequire } from 'node:module' +const require = createRequire(import.meta.url) const metricsW3c = require('../../utils/metricsW3c.json') class FormatW3cAnalysis {} @@ -187,4 +189,4 @@ function formatW3c (errorsList) { } } const formatW3cAnalysis = new FormatW3cAnalysis() -module.exports = formatW3cAnalysis +export default formatW3cAnalysis diff --git a/EcoSonar-API/services/format/formatW3cBestPractices.js b/EcoSonar-API/services/format/formatW3cBestPractices.js index fb6cb35..3ddfa88 100644 --- a/EcoSonar-API/services/format/formatW3cBestPractices.js +++ b/EcoSonar-API/services/format/formatW3cBestPractices.js @@ -1,4 +1,4 @@ -const formatCompliance = require('./formatCompliance') +import formatCompliance from './formatCompliance.js' class FormatW3cBestPractices {} @@ -89,4 +89,4 @@ FormatW3cBestPractices.prototype.returnFormattedW3c = function (latestW3cAnalysi } const formatW3cBestPractices = new FormatW3cBestPractices() -module.exports = formatW3cBestPractices +export default formatW3cBestPractices diff --git a/EcoSonar-API/services/greenit-analysis/analyseService.js b/EcoSonar-API/services/greenit-analysis/analyseService.js index ab559e9..be5178a 100644 --- a/EcoSonar-API/services/greenit-analysis/analyseService.js +++ b/EcoSonar-API/services/greenit-analysis/analyseService.js @@ -1,9 +1,9 @@ -const puppeteer = require('puppeteer') -const createGreenITReports = require('./greenit-analysis.js').createGreenITReports -const authenticationService = require('../authenticationService') -const viewPortParams = require('../../utils/viewportParams') +import puppeteer from 'puppeteer' +import createGreenITReports from './greenit-analysis.js' +import authenticationService from '../authenticationService.js' +import viewPortParams from '../../utils/viewportParams.js' -async function analyse (urlList, autoscroll, projectName) { +export default async function analyse (urlList, projectName, username, password, autoscroll) { let reports = [] const browserArgs = [ '--no-sandbox', // can't run inside docker without @@ -21,29 +21,29 @@ async function analyse (urlList, autoscroll, projectName) { const userJourneyEnabled = process.env.ECOSONAR_ENV_USER_JOURNEY_ENABLED || 'false' if (userJourneyEnabled === 'true') { console.log('Your EcoSonar project is using user journey to audit your website, GreenIT analysis will be made into different Chromium browser for right cookies configuration') - reports = await launchAllAnalysisOnDifferentBrowser(browserArgs, urlList, projectName, autoscroll) + reports = await launchAllAnalysisOnDifferentBrowser(browserArgs, urlList, projectName, username, password, autoscroll) } else { - reports = await launchAllAnalysisOnSameBrowser(browserArgs, urlList, projectName, autoscroll) + reports = await launchAllAnalysisOnSameBrowser(browserArgs, urlList, projectName, username, password, autoscroll) } return reports } -async function launchAllAnalysisOnDifferentBrowser (browserArgs, urlList, projectName, autoscroll) { +async function launchAllAnalysisOnDifferentBrowser (browserArgs, urlList, projectName, username, password, autoscroll) { let reports = [] let report for (const url of urlList) { // start browser const browser = await puppeteer.launch({ - headless: true, + headless: 'new', args: browserArgs, ignoreHTTPSErrors: true, // Keep gpu horsepower in headless ignoreDefaultArgs: ['--disable-gpu'], - defaultViewport: viewPortParams.viewPortParams + defaultViewport: viewPortParams }) try { - const loginSucceeded = await authenticationService.loginIfNeeded(browser, projectName) + const loginSucceeded = await authenticationService.loginIfNeeded(browser, projectName, username, password) if (loginSucceeded) { report = await createGreenITReports(browser, projectName, [url], autoscroll) reports = reports.concat(report) @@ -60,20 +60,20 @@ async function launchAllAnalysisOnDifferentBrowser (browserArgs, urlList, projec return reports } -async function launchAllAnalysisOnSameBrowser (browserArgs, urlList, projectName, autoscroll) { +async function launchAllAnalysisOnSameBrowser (browserArgs, urlList, projectName, username, password, autoscroll) { // start browser const browser = await puppeteer.launch({ - headless: true, + headless: 'new', args: browserArgs, ignoreHTTPSErrors: true, // Keep gpu horsepower in headless ignoreDefaultArgs: ['--disable-gpu'], - defaultViewport: viewPortParams.viewPortParams + defaultViewport: viewPortParams }) let reports = [] try { - const loginSucceeded = await authenticationService.loginIfNeeded(browser, projectName) + const loginSucceeded = await authenticationService.loginIfNeeded(browser, projectName, username, password) if (loginSucceeded) { // analyse each page reports = await createGreenITReports(browser, projectName, urlList, autoscroll) @@ -99,7 +99,3 @@ async function closeBrowser (browser) { })).catch((error) => console.error('\x1b[31m%s\x1b[0m', error)) await browser.close() } - -module.exports = { - analyse -} diff --git a/EcoSonar-API/services/greenit-analysis/greenit-analysis.js b/EcoSonar-API/services/greenit-analysis/greenit-analysis.js index 64eedc8..5b89b69 100644 --- a/EcoSonar-API/services/greenit-analysis/greenit-analysis.js +++ b/EcoSonar-API/services/greenit-analysis/greenit-analysis.js @@ -1,87 +1,85 @@ /* eslint-disable no-undef */ -const PuppeteerHar = require('puppeteer-har') -const path = require('path') -const userJourneyService = require('../userJourneyService') -const viewPortParams = require('../../utils/viewportParams') - -module.exports = { - async createGreenITReports (browser, projectName, urlList, autoscroll) { - // Concurent tab - const MAX_TAB = 40 - // Nb of retry before dropping analysis - const RETRY = 2 - - const asyncFunctions = [] - let results - let index = 0 - const reports = [] - - const convert = [] - - for (let i = 0; i < MAX_TAB; i++) { - convert[i] = i - } +import path from 'path' +import PuppeteerHar from '../../utils/PuppeteerHar.js' +import userJourneyService from '../userJourneyService.js' +import viewPortParams from '../../utils/viewportParams.js' + +export default async function createGreenITReports (browser, projectName, urlList, autoscroll) { + // Concurent tab + const MAX_TAB = 40 + // Nb of retry before dropping analysis + const RETRY = 2 + + const asyncFunctions = [] + let results + let index = 0 + const reports = [] + + const convert = [] + + for (let i = 0; i < MAX_TAB; i++) { + convert[i] = i + } - // Asynchronous analysis with MAX_TAB open simultaneously to json - for (let i = 0; i < MAX_TAB && index < urlList.length; i++) { - asyncFunctions.push( + // Asynchronous analysis with MAX_TAB open simultaneously to json + for (let i = 0; i < MAX_TAB && index < urlList.length; i++) { + asyncFunctions.push( + analyseURL( + browser, + projectName, + urlList[index], + { + tabId: i + }, + autoscroll + ) + ) + index++ + } + + while (asyncFunctions.length !== 0) { + results = await Promise.race(asyncFunctions) + if (!results.success && results.tryNb <= RETRY) { + asyncFunctions.splice( + convert[results.tabId], + 1, analyseURL( browser, projectName, - urlList[index], + results.url, { - tabId: i + tabId: results.tabId, + tryNb: results.tryNb + 1 }, autoscroll ) - ) - index++ - } - - while (asyncFunctions.length !== 0) { - results = await Promise.race(asyncFunctions) - if (!results.success && results.tryNb <= RETRY) { + ) // convert is NEEDED, variable size array + } else { + reports.push(results) + if (index === urlList.length) { + asyncFunctions.splice(convert[results.tabId], 1) // convert is NEEDED, variable size array + for (let i = results.tabId + 1; i < convert.length; i++) { + convert[i] = convert[i] - 1 + } + } else { asyncFunctions.splice( - convert[results.tabId], + results.tabId, 1, analyseURL( browser, projectName, - results.url, + urlList[index], { - tabId: results.tabId, - tryNb: results.tryNb + 1 + tabId: results.tabId }, autoscroll ) - ) // convert is NEEDED, variable size array - } else { - reports.push(results) - if (index === urlList.length) { - asyncFunctions.splice(convert[results.tabId], 1) // convert is NEEDED, variable size array - for (let i = results.tabId + 1; i < convert.length; i++) { - convert[i] = convert[i] - 1 - } - } else { - asyncFunctions.splice( - results.tabId, - 1, - analyseURL( - browser, - projectName, - urlList[index], - { - tabId: results.tabId - }, - autoscroll - ) - ) // No need for convert, fixed size array - index++ - } + ) // No need for convert, fixed size array + index++ } } - return reports } + return reports } // Analyse a webpage @@ -141,7 +139,8 @@ async function analyseURL (browser, projectName, url, options, autoscroll) { }) ) // add script, get run, then remove it to not interfere with the analysis - const script = await page.addScriptTag({ path: path.join(__dirname, './dist/bundle.js') }) + const __dirname = path.resolve(path.dirname('')) + const script = await page.addScriptTag({ path: path.join(__dirname, './services/greenit-analysis/dist/bundle.js') }) await script.evaluate((x) => x.remove()) // pass node object to browser @@ -172,7 +171,7 @@ async function analyseURL (browser, projectName, url, options, autoscroll) { async function launchPageWithoutUserJourney (browser, url, autoscroll) { const page = await browser.newPage() - await page.setViewport(viewPortParams.viewPortParams) + await page.setViewport(viewPortParams) // disabling cache await page.setCacheEnabled(false) diff --git a/EcoSonar-API/services/lighthouse/config.js b/EcoSonar-API/services/lighthouse/config.js index d74c555..d5032c2 100644 --- a/EcoSonar-API/services/lighthouse/config.js +++ b/EcoSonar-API/services/lighthouse/config.js @@ -1,15 +1,18 @@ -module.exports = { +import viewPortParams from '../../utils/viewportParams.js' + +export default { extends: 'lighthouse:default', settings: { - onlyAudits: ['metrics'] + onlyCategories: ['performance', 'accessibility'], + formFactor: 'desktop', + screenEmulation: { + mobile: false, + width: viewPortParams.width, + height: viewPortParams.height, + disabled: false + }, + emulatedUserAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4695.0 Safari/537.36 Chrome-Lighthouse' }, port: 36951, - screenEmulation: { - mobile: false, - width: 1920, - height: 1080, - deviceScaleFactor: 1, - disabled: false - }, - emulatedUserAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4695.0 Safari/537.36 Chrome-Lighthouse' + logLevel: 'error' } diff --git a/EcoSonar-API/services/lighthouse/lighthouse.js b/EcoSonar-API/services/lighthouse/lighthouse.js index e31c48c..3b559c7 100644 --- a/EcoSonar-API/services/lighthouse/lighthouse.js +++ b/EcoSonar-API/services/lighthouse/lighthouse.js @@ -1,75 +1,79 @@ -const lighthouse = require('lighthouse') -const config = require('./config.js') -const puppeteer = require('puppeteer') -const authenticationService = require('../authenticationService') -const userJourneyService = require('../userJourneyService') -const viewPortParams = require('../../utils/viewportParams') +import lighthouse from 'lighthouse' +import puppeteer from 'puppeteer' +import config from './config.js' +import authenticationService from '../authenticationService.js' +import userJourneyService from '../userJourneyService.js' +import viewPortParams from '../../utils/viewportParams.js' -module.exports = { - lighthouseAnalysis: async function (urlList, projectName) { - const browserArgs = [ - '--no-sandbox', // can't run inside docker without - '--disable-setuid-sandbox', // but security issues - '--ignore-certificate-errors', - '--window-size=1920,1080', - '--start-maximized' - ] +export default async function lighthouseAnalysis (urlList, projectName, username, password) { + const browserArgs = [ + '--no-sandbox', // can't run inside docker without + '--disable-setuid-sandbox', // but security issues + '--ignore-certificate-errors', + '--window-size=1920,1080', + '--start-maximized' + ] - const proxyConfiguration = await authenticationService.useProxyIfNeeded(projectName) - if (proxyConfiguration) { - browserArgs.push(proxyConfiguration) - } + const proxyConfiguration = await authenticationService.useProxyIfNeeded(projectName) + if (proxyConfiguration) { + browserArgs.push(proxyConfiguration) + } - // start browser - const browser = await puppeteer.launch({ - headless: true, - args: browserArgs, - ignoreHTTPSErrors: true, - // Keep gpu horsepower in headless - ignoreDefaultArgs: [ - '--disable-gpu' - ], - defaultViewport: viewPortParams.viewPortParams - }) + // start browser + const browser = await puppeteer.launch({ + headless: 'new', + args: browserArgs, + ignoreHTTPSErrors: true, + // Keep gpu horsepower in headless + ignoreDefaultArgs: [ + '--disable-gpu', + '--enable-automation' + ], + defaultViewport: viewPortParams + }) - const results = [] - const options = { logLevel: 'error', output: 'json', onlyCategories: ['performance', 'accessibility'], port: (new URL(browser.wsEndpoint())).port, disableStorageReset: true } + const results = [] - try { - let lighthouseResults - let userJourney - const loginSucceeded = await authenticationService.loginIfNeeded(browser, projectName) - if (loginSucceeded) { - for (const [index, url] of urlList.entries()) { - await browser.newPage() // prevent browser to close before ending all pages analysis - try { - console.log('Lighthouse Analysis launched for url ' + url) - await userJourneyService.getUserFlow(projectName, url) - .then((result) => { - userJourney = result - }).catch((error) => { - console.log(error.message) - }) - if (userJourney) { - lighthouseResults = await userJourneyService.playUserFlowLighthouse(url, browser, userJourney) - } else { - lighthouseResults = await lighthouse(url, options, config) - } - console.log('Lighthouse Analysis ended for url ' + url) - results[index] = { ...lighthouseResults.lhr, url } - } catch (error) { - console.error('LIGHTHOUSE ANALYSIS - An error occured when auditing ' + url) - console.error('\x1b[31m%s\x1b[0m', error) + try { + let lighthouseResults + let userJourney + const loginSucceeded = await authenticationService.loginIfNeeded(browser, projectName, username, password) + if (loginSucceeded) { + for (const [index, url] of urlList.entries()) { + const page = await browser.newPage() // prevent browser to close before ending all pages analysis + try { + console.log('Lighthouse Analysis launched for url ' + url) + await userJourneyService.getUserFlow(projectName, url) + .then((result) => { + userJourney = result + }).catch((error) => { + console.log(error.message) + }) + if (userJourney) { + lighthouseResults = await userJourneyService.playUserFlowLighthouse(url, browser, userJourney) + } else { + // Wait for Lighthouse to open url, then inject our stylesheet. + browser.on('targetchanged', async target => { + if (page && page.url() === url) { + await page.addStyleTag({ content: '* {color: red}' }) + } + }) + lighthouseResults = await lighthouse(url, { disableStorageReset: true }, config, page) } + console.log('Lighthouse Analysis ended for url ' + url) + results[index] = { ...lighthouseResults.lhr, url } + } catch (error) { + console.error('LIGHTHOUSE ANALYSIS - An error occured when auditing ' + url) + console.error('\x1b[31m%s\x1b[0m', error) } - } else { - console.warn('Could not log in, audit for lighthouse analysis is skipped') } - } catch (error) { - console.error('\x1b[31m%s\x1b[0m', error) - } finally { - await browser.close() + } else { + console.warn('Could not log in, audit for lighthouse analysis is skipped') } - return results + } catch (error) { + console.error('\x1b[31m%s\x1b[0m', error) + } finally { + await browser.close() } + return results } diff --git a/EcoSonar-API/services/loginProxyConfigurationService.js b/EcoSonar-API/services/loginProxyConfigurationService.js index 538c05c..028920e 100644 --- a/EcoSonar-API/services/loginProxyConfigurationService.js +++ b/EcoSonar-API/services/loginProxyConfigurationService.js @@ -1,6 +1,6 @@ -const projectsRepository = require('../dataBase/projectsRepository') -const SystemError = require('../utils/SystemError') -const retrieveLoginProxyYamlConfigurationService = require('./yamlConfiguration/retrieveLoginProxyYamlConfiguration') +import projectsRepository from '../dataBase/projectsRepository.js' +import SystemError from '../utils/SystemError.js' +import retrieveLoginProxyYamlConfigurationService from './yamlConfiguration/retrieveLoginProxyYamlConfiguration.js' class LoginProxyConfigurationService {} @@ -64,45 +64,30 @@ LoginProxyConfigurationService.prototype.insertProxyConfiguration = async functi }) } -LoginProxyConfigurationService.prototype.getLoginCredentials = async function (projectName) { - const configSettings = process.env.ECOSONAR_ENV_AUTHENTICATION_CONFIGURATION || 'database' - const authenticationUrl = process.env.ECOSONAR_ENV_AUTHENTICATION_URL || '' - const username = process.env.ECOSONAR_ENV_AUTHENTICATION_USERNAME || '' - const password = process.env.ECOSONAR_ENV_AUTHENTICATION_PASSWORD || '' +LoginProxyConfigurationService.prototype.getLoginCredentials = async function (projectName, username, password) { let loginInformations = {} let error = false - if (configSettings === 'env' && authenticationUrl !== '' && username !== '' && password !== '') { - loginInformations = { - authentication_url: authenticationUrl, - username, - password - } - } else if (configSettings === 'yaml') { - const login = await retrieveLoginProxyYamlConfigurationService.getLoginInformations() - if (login !== false && (login.projectName === undefined || login.projectName.includes(projectName)) && login.authentication_url !== undefined && login.username !== undefined && login.password !== undefined) { - loginInformations = { - authentication_url: login.authentication_url, - username: login.username, - password: login.password - } - } - } else { - await projectsRepository.getProjectSettings(projectName) - .then((result) => { - if (!(result === null || result.login === undefined)) { - loginInformations = { - authentication_url: result.login.get('authentication_url'), - username: result.login.get('username'), - password: result.login.get('password'), - steps: result.login.get('steps') - } + await projectsRepository.getProjectSettings(projectName) + .then((result) => { + if (!(result === null || result.login === undefined)) { + const usernameSaved = username !== undefined ? username : result.login.get('username') + const passwordSaved = password !== undefined ? password : result.login.get('password') + loginInformations = { + authentication_url: result.login.get('authentication_url'), + loginButtonSelector: result.login.get('loginButtonSelector'), + usernameSelector: result.login.get('usernameSelector'), + passwordSelector: result.login.get('passwordSelector'), + steps: result.login.get('steps'), + username: usernameSaved, + password: passwordSaved } - }) - .catch(() => { - error = true - }) - } + } + }) + .catch(() => { + error = true + }) + return new Promise((resolve, reject) => { if (error) { reject(new SystemError()) @@ -213,4 +198,4 @@ LoginProxyConfigurationService.prototype.deleteProxyConfiguration = async functi } const loginproxyConfigurationService = new LoginProxyConfigurationService() -module.exports = loginproxyConfigurationService +export default loginproxyConfigurationService diff --git a/EcoSonar-API/services/procedureService.js b/EcoSonar-API/services/procedureService.js index 6d10045..386a96e 100644 --- a/EcoSonar-API/services/procedureService.js +++ b/EcoSonar-API/services/procedureService.js @@ -1,5 +1,5 @@ -const projectsRepository = require('../dataBase/projectsRepository') -const SystemError = require('../utils/SystemError') +import projectsRepository from '../dataBase/projectsRepository.js' +import SystemError from '../utils/SystemError.js' class ProcedureService { } @@ -41,4 +41,4 @@ ProcedureService.prototype.getProcedure = async function (projectName) { } const procedureService = new ProcedureService() -module.exports = procedureService +export default procedureService diff --git a/EcoSonar-API/services/projectService.js b/EcoSonar-API/services/projectService.js index 4875861..4aad2f4 100644 --- a/EcoSonar-API/services/projectService.js +++ b/EcoSonar-API/services/projectService.js @@ -1,12 +1,12 @@ -const SystemError = require('../utils/SystemError') -const retrieveAnalysisService = require('../services/retrieveAnalysisService') -const projectsRepository = require('../dataBase/projectsRepository') -const w3cRepository = require('../dataBase/w3cRepository') -const lighthouseRepository = require('../dataBase/lighthouseRepository') -const greenItRepository = require('../dataBase/greenItRepository') -const bestPracticesRepository = require('../dataBase/bestPracticesRepository') -const urlsProjectRepository = require('../dataBase/urlsProjectRepository') -const tempurlsProjectRepository = require('../dataBase/tempurlsProjectRepository') +import SystemError from '../utils/SystemError.js' +import retrieveAnalysisService from '../services/retrieveAnalysisService.js' +import projectsRepository from '../dataBase/projectsRepository.js' +import w3cRepository from '../dataBase/w3cRepository.js' +import lighthouseRepository from '../dataBase/lighthouseRepository.js' +import greenItRepository from '../dataBase/greenItRepository.js' +import bestPracticesRepository from '../dataBase/bestPracticesRepository.js' +import urlsProjectRepository from '../dataBase/urlsProjectRepository.js' +import tempurlsProjectRepository from '../dataBase/tempurlsProjectRepository.js' class ProjectService { } @@ -299,4 +299,4 @@ ProjectService.prototype.deleteProject = async function (projectName) { } const projectService = new ProjectService() -module.exports = projectService +export default projectService diff --git a/EcoSonar-API/services/retrieveAnalysisService.js b/EcoSonar-API/services/retrieveAnalysisService.js index 7a035e5..c2fc2e5 100644 --- a/EcoSonar-API/services/retrieveAnalysisService.js +++ b/EcoSonar-API/services/retrieveAnalysisService.js @@ -1,11 +1,11 @@ -const greenItRepository = require('../dataBase/greenItRepository') -const lighthouseRepository = require('../dataBase/lighthouseRepository') -const w3cRepository = require('../dataBase/w3cRepository') -const urlsProjectRepository = require('../dataBase/urlsProjectRepository') -const formatLighthouseAnalysis = require('./format/formatLighthouseAnalysis') -const SystemError = require('../utils/SystemError') -const formatGreenItAnalysis = require('./format/formatGreenItAnalysis') -const formatW3cAnalysis = require('./format/formatW3cAnalysis') +import greenItRepository from '../dataBase/greenItRepository.js' +import lighthouseRepository from '../dataBase/lighthouseRepository.js' +import w3cRepository from '../dataBase/w3cRepository.js' +import urlsProjectRepository from '../dataBase/urlsProjectRepository.js' +import formatLighthouseAnalysis from './format/formatLighthouseAnalysis.js' +import SystemError from '../utils/SystemError.js' +import formatGreenItAnalysis from './format/formatGreenItAnalysis.js' +import formatW3cAnalysis from './format/formatW3cAnalysis.js' class RetrieveAnalysisService { } @@ -391,4 +391,4 @@ RetrieveAnalysisService.prototype.getProjectScores = async function (projectName } const retrieveAnalysisService = new RetrieveAnalysisService() -module.exports = retrieveAnalysisService +export default retrieveAnalysisService diff --git a/EcoSonar-API/services/retrieveBestPracticesService.js b/EcoSonar-API/services/retrieveBestPracticesService.js index aa07b19..574a55b 100644 --- a/EcoSonar-API/services/retrieveBestPracticesService.js +++ b/EcoSonar-API/services/retrieveBestPracticesService.js @@ -1,12 +1,12 @@ -const bestPracticesRepository = require('../dataBase/bestPracticesRepository') -const formatGreenItReports = require('./format/formatGreenItReports') -const formatLighthouseBestPractices = require('./format/formatLighthouseBestPractices') -const w3cRepository = require('../dataBase/w3cRepository') -const formatW3cBestPractices = require('./format/formatW3cBestPractices') -const bestPracticesSorting = require('./format/bestPracticesSorting') -const projectsRepository = require('../dataBase/projectsRepository') -const urlsProjectRepository = require('../dataBase/urlsProjectRepository') -const SystemError = require('../utils/SystemError') +import bestPracticesRepository from '../dataBase/bestPracticesRepository.js' +import formatGreenItReports from './format/formatGreenItReports.js' +import formatLighthouseBestPractices from './format/formatLighthouseBestPractices.js' +import w3cRepository from '../dataBase/w3cRepository.js' +import formatW3cBestPractices from './format/formatW3cBestPractices.js' +import bestPracticesSorting from './format/bestPracticesSorting.js' +import projectsRepository from '../dataBase/projectsRepository.js' +import urlsProjectRepository from '../dataBase/urlsProjectRepository.js' +import SystemError from '../utils/SystemError.js' class RetrieveBestPracticesService { } @@ -200,4 +200,4 @@ RetrieveBestPracticesService.prototype.getUrlBestPractices = async function (pro } const retrieveBestPracticesService = new RetrieveBestPracticesService() -module.exports = retrieveBestPracticesService +export default retrieveBestPracticesService diff --git a/EcoSonar-API/services/urlConfigurationService.js b/EcoSonar-API/services/urlConfigurationService.js index 343f08d..4cda201 100644 --- a/EcoSonar-API/services/urlConfigurationService.js +++ b/EcoSonar-API/services/urlConfigurationService.js @@ -1,10 +1,10 @@ -const tempUrlsProjectRepository = require('../dataBase/tempurlsProjectRepository') -const urlsProjectRepository = require('../dataBase/urlsProjectRepository') -const greenItRepository = require('../dataBase/greenItRepository') -const lighthouseRepository = require('../dataBase/lighthouseRepository') -const bestPracticesRepository = require('../dataBase/bestPracticesRepository') -const w3cRepository = require('../dataBase/w3cRepository') -const SystemError = require('../utils/SystemError') +import tempUrlsProjectRepository from '../dataBase/tempurlsProjectRepository.js' +import urlsProjectRepository from '../dataBase/urlsProjectRepository.js' +import greenItRepository from '../dataBase/greenItRepository.js' +import lighthouseRepository from '../dataBase/lighthouseRepository.js' +import bestPracticesRepository from '../dataBase/bestPracticesRepository.js' +import w3cRepository from '../dataBase/w3cRepository.js' +import SystemError from '../utils/SystemError.js' class UrlConfigurationService { } @@ -161,4 +161,4 @@ UrlConfigurationService.prototype.delete = async function (projectName, urlName) } const urlConfigurationService = new UrlConfigurationService() -module.exports = urlConfigurationService +export default urlConfigurationService diff --git a/EcoSonar-API/services/userJourneyService.js b/EcoSonar-API/services/userJourneyService.js index 13f8184..264060f 100644 --- a/EcoSonar-API/services/userJourneyService.js +++ b/EcoSonar-API/services/userJourneyService.js @@ -1,15 +1,15 @@ -const lighthouseUserFlow = require('lighthouse/lighthouse-core/fraggle-rock/api.js') -const PuppeteerHar = require('puppeteer-har') -const { clickOnElement, waitForSelectors, applyChange } = require('../utils/playSelectors') -const urlsProjectRepository = require('../dataBase/urlsProjectRepository') -const viewPortParams = require('../utils/viewportParams') -const SystemError = require('../utils/SystemError') +import { startFlow } from 'lighthouse' +import PuppeteerHar from '../utils/PuppeteerHar.js' +import { clickOnElement, waitForSelectors, applyChange } from '../utils/playSelectors.js' +import urlsProjectRepository from '../dataBase/urlsProjectRepository.js' +import viewPortParams from '../utils/viewportParams.js' +import SystemError from '../utils/SystemError.js' class UserJourneyService { } UserJourneyService.prototype.playUserJourney = async function (url, browser, userJourney) { const page = await browser.newPage() - await page.setViewport(viewPortParams.viewPortParams) + await page.setViewport(viewPortParams) // disabling cache await page.setCacheEnabled(false) @@ -61,7 +61,7 @@ UserJourneyService.prototype.playUserJourney = async function (url, browser, use console.error(error) } } - await page.waitForNavigation() + // await page.waitForNavigation() const harObj = await pptrHar.stop() return { page, @@ -72,8 +72,8 @@ UserJourneyService.prototype.playUserJourney = async function (url, browser, use UserJourneyService.prototype.playUserFlowLighthouse = async function (url, browser, userJourney) { const timeout = 10000 const targetPage = await browser.newPage() - await targetPage.setViewport(viewPortParams.viewPortParams) - const flow = await lighthouseUserFlow.startFlow(targetPage, { name: url }) + await targetPage.setViewport(viewPortParams) + const flow = await startFlow(targetPage, { name: url }) const steps = userJourney.steps let step; let element for (step of steps) { @@ -82,7 +82,7 @@ UserJourneyService.prototype.playUserFlowLighthouse = async function (url, brows await flow.navigate(step.url, { stepName: step.url }) - await targetPage.setViewport(viewPortParams.viewPortParams) + await targetPage.setViewport(viewPortParams) break case 'click': element = await waitForSelectors(step.selectors, targetPage, { timeout, visible: true }) @@ -99,7 +99,7 @@ UserJourneyService.prototype.playUserFlowLighthouse = async function (url, brows break } } - await targetPage.waitForNavigation() + // await targetPage.waitForNavigation() targetPage.close() const lighthouseResults = await flow.createFlowResult() return lighthouseResults.steps[0] @@ -207,4 +207,4 @@ UserJourneyService.prototype.scrollUntil = async function (page, distancePercent } const userJourneyService = new UserJourneyService() -module.exports = userJourneyService +export default userJourneyService diff --git a/EcoSonar-API/services/yamlConfiguration/retrieveLoginProxyYamlConfiguration.js b/EcoSonar-API/services/yamlConfiguration/retrieveLoginProxyYamlConfiguration.js index 7a74805..b378ce0 100644 --- a/EcoSonar-API/services/yamlConfiguration/retrieveLoginProxyYamlConfiguration.js +++ b/EcoSonar-API/services/yamlConfiguration/retrieveLoginProxyYamlConfiguration.js @@ -1,13 +1,14 @@ -const path = require('path') -const fs = require('fs') -const YAML = require('js-yaml') +import path from 'path' +import fs from 'fs' +import YAML from 'js-yaml' class RetrieveLoginProxyYamlConfiguration { } RetrieveLoginProxyYamlConfiguration.prototype.getLoginInformations = async function () { try { - const ymlFile = fs.readFileSync(path.join(__dirname, './login.yaml'), 'utf8') + const __dirname = path.resolve(path.dirname('')) + const ymlFile = fs.readFileSync(path.join(__dirname, './services/yamlConfiguration/login.yaml'), 'utf8') return YAML.load(ymlFile) } catch (e) { return false @@ -16,26 +17,13 @@ RetrieveLoginProxyYamlConfiguration.prototype.getLoginInformations = async funct RetrieveLoginProxyYamlConfiguration.prototype.getProxyInformations = async function () { try { - const ymlFile = fs.readFileSync(path.join(__dirname, './proxy.yaml'), 'utf8') + const __dirname = path.resolve(path.dirname('')) + const ymlFile = fs.readFileSync(path.join(__dirname, './services/yamlConfiguration/proxy.yaml'), 'utf8') return YAML.load(ymlFile) } catch { return false } } -RetrieveLoginProxyYamlConfiguration.prototype.retrieveUserJourneyInformation = async function (url) { - try { - const files = fs.readdirSync(path.join(__dirname, './userJourney')) - const urlWithoutSpecialCharacters = url.replace(/[:?/]/g, '') + '.json' - if (files.includes(urlWithoutSpecialCharacters)) { - return JSON.parse(fs.readFileSync(path.join(__dirname, './userJourney/', urlWithoutSpecialCharacters))) - } else { - return false - } - } catch (error) { - return false - } -} - const retrieveLoginProxyYamlConfigurationService = new RetrieveLoginProxyYamlConfiguration() -module.exports = retrieveLoginProxyYamlConfigurationService +export default retrieveLoginProxyYamlConfigurationService diff --git a/EcoSonar-API/swagger.js b/EcoSonar-API/swagger.js index 6cb3a18..390e7b4 100644 --- a/EcoSonar-API/swagger.js +++ b/EcoSonar-API/swagger.js @@ -1,10 +1,10 @@ -const swaggerJSDoc = require('swagger-jsdoc') +import swaggerJSDoc from 'swagger-jsdoc' const swagger = { swaggerDefinition: { info: { title: 'API EcoSonar', - version: '3.2', + version: '3.5', description: 'Swagger UI of EcoSonar API' } }, @@ -12,5 +12,4 @@ const swagger = { } const swaggerSpec = swaggerJSDoc(swagger) - -module.exports = swaggerSpec +export default swaggerSpec diff --git a/EcoSonar-API/utils/AsyncMiddleware.js b/EcoSonar-API/utils/AsyncMiddleware.js index d7aaaa3..e855b2e 100644 --- a/EcoSonar-API/utils/AsyncMiddleware.js +++ b/EcoSonar-API/utils/AsyncMiddleware.js @@ -1,7 +1,6 @@ // with promises, asyncrone function are necessary -function asyncMiddleware (fn) { +export default function asyncMiddleware (fn) { return (req, res, next) => { Promise.resolve(fn(req, res, next)).catch(next) } } -module.exports = asyncMiddleware diff --git a/EcoSonar-API/utils/PuppeteerHar.js b/EcoSonar-API/utils/PuppeteerHar.js new file mode 100644 index 0000000..2b000d3 --- /dev/null +++ b/EcoSonar-API/utils/PuppeteerHar.js @@ -0,0 +1,129 @@ +import fs from 'fs' +import { promisify } from 'util' +import { harFromMessages } from 'chrome-har' + +// event types to observe +const pageObserve = [ + 'Page.loadEventFired', + 'Page.domContentEventFired', + 'Page.frameStartedLoading', + 'Page.frameAttached', + 'Page.frameScheduledNavigation' +] + +const networkObserve = [ + 'Network.requestWillBeSent', + 'Network.requestServedFromCache', + 'Network.dataReceived', + 'Network.responseReceived', + 'Network.resourceChangedPriority', + 'Network.loadingFinished', + 'Network.loadingFailed' +] + +class PuppeteerHar { + /** + * @param {object} page + */ + constructor (page) { + this.page = page + this.mainFrame = this.page.mainFrame() + this.inProgress = false + this.cleanUp() + } + + /** + * @returns {void} + */ + cleanUp () { + this.network_events = [] + this.page_events = [] + this.response_body_promises = [] + } + + /** + * @param {{path: string}=} options + * @return {Promise} + */ + async start ({ path, saveResponse, captureMimeTypes } = {}) { + this.inProgress = true + this.saveResponse = saveResponse || false + this.captureMimeTypes = captureMimeTypes || [ + 'text/html', + 'application/json' + ] + this.path = path + this.client = await this.page.target().createCDPSession() + await this.client.send('Page.enable') + await this.client.send('Network.enable') + pageObserve.forEach((method) => { + this.client.on(method, (params) => { + if (!this.inProgress) { + return + } + this.page_events.push({ method, params }) + }) + }) + networkObserve.forEach((method) => { + this.client.on(method, (params) => { + if (!this.inProgress) { + return + } + this.network_events.push({ method, params }) + + if (this.saveResponse && method === 'Network.responseReceived') { + const response = params.response + const requestId = params.requestId + + // Response body is unavailable for redirects, no-content, image, audio and video responses + if ( + response.status !== 204 && + response.headers.location == null && + this.captureMimeTypes.includes(response.mimeType) + ) { + const promise = this.client + .send('Network.getResponseBody', { requestId }) + .then( + (responseBody) => { + // Set the response so `chrome-har` can add it to the HAR + // eslint-disable-next-line new-cap + params.response.body = new Buffer.from( + + responseBody.body, + responseBody.base64Encoded ? 'base64' : undefined + ).toString() + }, + (reason) => { + // Resources (i.e. response bodies) are flushed after page commits + // navigation and we are no longer able to retrieve them. In this + // case, fail soft so we still add the rest of the response to the + // HAR. Possible option would be force wait before navigation... + } + ) + this.response_body_promises.push(promise) + } + } + }) + }) + } + + /** + * @returns {Promise} + */ + async stop () { + this.inProgress = false + await Promise.all(this.response_body_promises) + await this.client.detach() + const har = harFromMessages(this.page_events.concat(this.network_events), { + includeTextFromResponseBody: this.saveResponse + }) + this.cleanUp() + if (this.path) { + await promisify(fs.writeFile)(this.path, JSON.stringify(har)) + } else { + return har + } + } +} + +export default PuppeteerHar diff --git a/EcoSonar-API/utils/SystemError.js b/EcoSonar-API/utils/SystemError.js index ea89bab..5ce45d6 100644 --- a/EcoSonar-API/utils/SystemError.js +++ b/EcoSonar-API/utils/SystemError.js @@ -1,4 +1,2 @@ -class SystemError { +export default class SystemError { } - -module.exports = SystemError diff --git a/EcoSonar-API/utils/enumAudits.js b/EcoSonar-API/utils/enumAudits.js index 80a7e36..0fb195a 100644 --- a/EcoSonar-API/utils/enumAudits.js +++ b/EcoSonar-API/utils/enumAudits.js @@ -36,7 +36,6 @@ EnumAudits.prototype.performanceNamesToSave = function () { lcpLazyLoaded: 'lcp-lazy-loaded', longTasks: 'long-tasks', nonCompositedAnimations: 'non-composited-animations', - preloadLcpImage: 'preload-lcp-image', domSize: 'dom-size', usesLongCacheTtl: 'uses-long-cache-ttl', usesResponsiveImages: 'uses-responsive-images', @@ -143,4 +142,4 @@ EnumAudits.prototype.accessibilityNames = function () { } const enumAudits = new EnumAudits() -module.exports = enumAudits +export default enumAudits diff --git a/EcoSonar-API/utils/metricsCalculate.js b/EcoSonar-API/utils/metricsCalculate.js index 95135e2..6867416 100644 --- a/EcoSonar-API/utils/metricsCalculate.js +++ b/EcoSonar-API/utils/metricsCalculate.js @@ -97,4 +97,4 @@ MetricsCalculate.prototype.isPresentForAccessibilityMetrics = function () { } const metricsCalculate = new MetricsCalculate() -module.exports = metricsCalculate +export default metricsCalculate diff --git a/EcoSonar-API/utils/playSelectors.js b/EcoSonar-API/utils/playSelectors.js index 768fb39..27e5ace 100644 --- a/EcoSonar-API/utils/playSelectors.js +++ b/EcoSonar-API/utils/playSelectors.js @@ -75,4 +75,4 @@ async function applyChange (valueToChange, element) { } } -module.exports = { clickOnElement, waitForSelectors, applyChange } +export { clickOnElement, waitForSelectors, applyChange } diff --git a/EcoSonar-API/utils/viewportParams.js b/EcoSonar-API/utils/viewportParams.js index 18ec263..745e420 100644 --- a/EcoSonar-API/utils/viewportParams.js +++ b/EcoSonar-API/utils/viewportParams.js @@ -1,6 +1,4 @@ -const viewPortParams = { +export default { width: 1920, height: 1080 } - -module.exports = { viewPortParams } diff --git a/EcoSonar-SonarQube/ecocode/ecocode-android-1.0.1.jar b/EcoSonar-SonarQube/ecocode/ecocode-android-1.1.0.jar similarity index 92% rename from EcoSonar-SonarQube/ecocode/ecocode-android-1.0.1.jar rename to EcoSonar-SonarQube/ecocode/ecocode-android-1.1.0.jar index 5fb6c73..abe1af4 100644 Binary files a/EcoSonar-SonarQube/ecocode/ecocode-android-1.0.1.jar and b/EcoSonar-SonarQube/ecocode/ecocode-android-1.1.0.jar differ diff --git a/EcoSonar-SonarQube/ecocode/ecocode-java-plugin-1.4.0.jar b/EcoSonar-SonarQube/ecocode/ecocode-java-plugin-1.6.0.jar similarity index 52% rename from EcoSonar-SonarQube/ecocode/ecocode-java-plugin-1.4.0.jar rename to EcoSonar-SonarQube/ecocode/ecocode-java-plugin-1.6.0.jar index 7ac8078..f84ed49 100644 Binary files a/EcoSonar-SonarQube/ecocode/ecocode-java-plugin-1.4.0.jar and b/EcoSonar-SonarQube/ecocode/ecocode-java-plugin-1.6.0.jar differ diff --git a/EcoSonar-SonarQube/ecocode/ecocode-javascript-plugin-1.3.1-SNAPSHOT.jar b/EcoSonar-SonarQube/ecocode/ecocode-javascript-plugin-1.3.1-SNAPSHOT.jar deleted file mode 100644 index c9ff09f..0000000 Binary files a/EcoSonar-SonarQube/ecocode/ecocode-javascript-plugin-1.3.1-SNAPSHOT.jar and /dev/null differ diff --git a/EcoSonar-SonarQube/ecocode/ecocode-javascript-plugin-1.4.0.jar b/EcoSonar-SonarQube/ecocode/ecocode-javascript-plugin-1.4.0.jar new file mode 100644 index 0000000..09c87e0 Binary files /dev/null and b/EcoSonar-SonarQube/ecocode/ecocode-javascript-plugin-1.4.0.jar differ diff --git a/EcoSonar-SonarQube/ecocode/ecocode-php-plugin-1.4.0.jar b/EcoSonar-SonarQube/ecocode/ecocode-php-plugin-1.4.0.jar deleted file mode 100644 index f4160f9..0000000 Binary files a/EcoSonar-SonarQube/ecocode/ecocode-php-plugin-1.4.0.jar and /dev/null differ diff --git a/EcoSonar-SonarQube/ecocode/ecocode-php-plugin-1.4.2.jar b/EcoSonar-SonarQube/ecocode/ecocode-php-plugin-1.4.2.jar new file mode 100644 index 0000000..84cd001 Binary files /dev/null and b/EcoSonar-SonarQube/ecocode/ecocode-php-plugin-1.4.2.jar differ diff --git a/EcoSonar-SonarQube/ecocode/ecocode-python-plugin-1.4.0.jar b/EcoSonar-SonarQube/ecocode/ecocode-python-plugin-1.4.0.jar deleted file mode 100644 index 52410c0..0000000 Binary files a/EcoSonar-SonarQube/ecocode/ecocode-python-plugin-1.4.0.jar and /dev/null differ diff --git a/EcoSonar-SonarQube/ecocode/ecocode-python-plugin-1.4.2.jar b/EcoSonar-SonarQube/ecocode/ecocode-python-plugin-1.4.2.jar new file mode 100644 index 0000000..e8c00c3 Binary files /dev/null and b/EcoSonar-SonarQube/ecocode/ecocode-python-plugin-1.4.2.jar differ diff --git a/EcoSonar-SonarQube/package.json b/EcoSonar-SonarQube/package.json index 88257b1..b971300 100644 --- a/EcoSonar-SonarQube/package.json +++ b/EcoSonar-SonarQube/package.json @@ -1,6 +1,6 @@ { "name": "ecosonar-plugin", - "version": "3.4", + "version": "3.5", "description": "Ecodesign and accessibility tool to help developpers minimize carbon footprint of their web-application", "main": "index.js", "scripts": { diff --git a/EcoSonar-SonarQube/pom.xml b/EcoSonar-SonarQube/pom.xml index c39ca7a..a746db6 100644 --- a/EcoSonar-SonarQube/pom.xml +++ b/EcoSonar-SonarQube/pom.xml @@ -6,7 +6,7 @@ com.ls ecosonar - 3.4 + 3.5 sonar-plugin diff --git a/EcoSonar-SonarQube/src/main/java/com/ls/api/GreenITAnalysis.java b/EcoSonar-SonarQube/src/main/java/com/ls/api/GreenITAnalysis.java index a967141..1e3a9a4 100644 --- a/EcoSonar-SonarQube/src/main/java/com/ls/api/GreenITAnalysis.java +++ b/EcoSonar-SonarQube/src/main/java/com/ls/api/GreenITAnalysis.java @@ -34,7 +34,11 @@ public void execute(SensorContext context) { String route = "/api/greenit/insert"; String uri = baseUrlHosted + route; String projectKey = context.config().get("sonar.projectKey").orElse(""); - String body = "{\"projectName\":\"" + projectKey + "\"}"; + String username = context.config().get("sonar.analysis.username").orElse(""); + String password = context.config().get("sonar.analysis.password").orElse(""); + String body = "{\"projectName\":\"" + projectKey + "\""+ ", " + + "\"username\":\"" + username + "\""+ ", " + + "\"password\":\"" + password + "\"}"; HttpClient client = HttpClient.newHttpClient(); HttpRequest request = HttpRequest.newBuilder() diff --git a/docker-compose.yml b/docker-compose.yml index 8814e6d..de34cd1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -51,12 +51,12 @@ services: SONAR_ES_BOOTSTRAP_CHECKS_DISABLE: 'true' volumes: - ./EcoSonar-SonarQube/target/:/opt/sonarqube/extensions/plugins/ - - ./EcoSonar-SonarQube/ecocode/ecocode-android-1.0.1.jar:/opt/sonarqube/extensions/plugins/ecocode-android-1.0.1.jar + - ./EcoSonar-SonarQube/ecocode/ecocode-android-1.1.0.jar:/opt/sonarqube/extensions/plugins/ecocode-android-1.1.0.jar - ./EcoSonar-SonarQube/ecocode/ecocode-ios-1.1.0.jar:/opt/sonarqube/extensions/plugins/ecocode-ios-1.1.0.jar:ro - - ./EcoSonar-SonarQube/ecocode/ecocode-java-plugin-1.4.0.jar:/opt/sonarqube/extensions/plugins/ecocode-java-plugin-1.4.0.jar:ro - - ./EcoSonar-SonarQube/ecocode/ecocode-javascript-plugin-1.3.1-SNAPSHOT.jar:/opt/sonarqube/extensions/plugins/ecocode-javascript-plugin-1.3.1-SNAPSHOT.jar:ro - - ./EcoSonar-SonarQube/ecocode/ecocode-php-plugin-1.4.0.jar:/opt/sonarqube/extensions/plugins/ecocode-php-plugin-1.4.0.jar:ro - - ./EcoSonar-SonarQube/ecocode/ecocode-python-plugin-1.4.0.jar:/opt/sonarqube/extensions/plugins/ecocode-python-plugin-1.4.0.jar:ro + - ./EcoSonar-SonarQube/ecocode/ecocode-java-plugin-1.6.0.jar:/opt/sonarqube/extensions/plugins/ecocode-java-plugin-1.6.0.jar:ro + - ./EcoSonar-SonarQube/ecocode/ecocode-javascript-plugin-1.4.0.jar:/opt/sonarqube/extensions/plugins/ecocode-javascript-plugin-1.4.0.jar:ro + - ./EcoSonar-SonarQube/ecocode/ecocode-php-plugin-1.4.2.jar:/opt/sonarqube/extensions/plugins/ecocode-php-plugin-1.4.2.jar:ro + - ./EcoSonar-SonarQube/ecocode/ecocode-python-plugin-1.4.2.jar:/opt/sonarqube/extensions/plugins/ecocode-python-plugin-1.4.2.jar:ro - sonarqube_extensions:/opt/sonarqube/extensions - sonarqube_logs:/opt/sonarqube/logs - sonarqube_data:/opt/sonarqube/data