From 1fab553d960db0b3830b89b515e67875f1c32796 Mon Sep 17 00:00:00 2001 From: Bastien Caudan Date: Fri, 7 Feb 2020 15:49:48 +0100 Subject: [PATCH 1/6] :page_facing_up: add check-licenses script --- scripts/check-licenses.js | 71 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 scripts/check-licenses.js diff --git a/scripts/check-licenses.js b/scripts/check-licenses.js new file mode 100644 index 0000000000..26faedbff0 --- /dev/null +++ b/scripts/check-licenses.js @@ -0,0 +1,71 @@ +'use strict' + +const fs = require('fs') +const path = require('path') +const readline = require('readline') +const util = require('util') +const exec = util.promisify(require('child_process').exec) + +const LICENSE_FILE = 'LICENSE-3rdparty.csv' + +async function main() { + const packageJsonPaths = await findPackageJsonPaths() + + console.log(`Look for dependencies in:\n`, packageJsonPaths) + const declaredDependencies = packageJsonPaths + .map(retrievePackageJsonDependencies) + .reduce(withoutDuplicates) + .sort() + + const declaredLicenses = (await retrieveLicenses()).sort() + + if (JSON.stringify(declaredDependencies) !== JSON.stringify(declaredLicenses)) { + console.error(`\n❌ package.json dependencies and ${LICENSE_FILE} mismatch`) + console.error( + `\nIn package.json but not in ${LICENSE_FILE}:\n`, + declaredDependencies.filter((d) => !declaredLicenses.includes(d)) + ) + console.error( + `\nIn ${LICENSE_FILE} but not in package.json:\n`, + declaredLicenses.filter((d) => !declaredDependencies.includes(d)) + ) + throw new Error() + } + console.log(`\n✅ All dependencies listed in ${LICENSE_FILE}`) +} + +async function findPackageJsonPaths() { + const { stdout } = await exec('find . -name "package.json" | grep -v node_modules') + return stdout.trim().split('\n') +} + +function retrievePackageJsonDependencies(packageJsonPath) { + const packageJson = require(path.join(__dirname, '..', packageJsonPath)) + + return Object.keys(packageJson.dependencies || {}) + .concat(Object.keys(packageJson.devDependencies || {})) + .filter((dependency) => !dependency.includes('@datadog')) +} + +function withoutDuplicates(a, b) { + return [...new Set([...a, ...b])] +} + +async function retrieveLicenses() { + const fileStream = fs.createReadStream(path.join(__dirname, '..', LICENSE_FILE)) + const rl = readline.createInterface({ input: fileStream }) + const licenses = [] + let header = true + for await (const line of rl) { + const csvColumns = line.split(',') + if (!header && csvColumns[0] !== 'file') { + licenses.push(csvColumns[1]) + } + header = false + } + return licenses +} + +main().catch(() => { + process.exit(1) +}) From 5cf626fce386d9aee04ddd8fc5103a90834502e5 Mon Sep 17 00:00:00 2001 From: Bastien Caudan Date: Fri, 7 Feb 2020 15:58:25 +0100 Subject: [PATCH 2/6] :page_facing_up: update LICENSE-3rdparty.csv --- LICENSE-3rdparty.csv | 3 +++ 1 file changed, 3 insertions(+) diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 179567e4cd..b20c77b3ec 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -5,6 +5,7 @@ require,tslib,Apache-2.0,Copyright Microsoft Corporation require,url-polyfill,MIT,Copyright 2017 Valentin Richard file,tracekit,MIT,Copyright 2013 Onur Can Cakmak and all TraceKit contributors dev,@types/jasmine,MIT,Copyright Microsoft Corporation +dev,@types/lodash.assign,MIT,Copyright Microsoft Corporation dev,@types/lodash.merge,MIT,Copyright Microsoft Corporation dev,@types/request,MIT,Copyright Microsoft Corporation dev,@types/sinon,MIT,Copyright Microsoft Corporation @@ -16,6 +17,7 @@ dev,@wdio/selenium-standalone-service,MIT,Copyright JS Foundation and other cont dev,@wdio/spec-reporter,MIT,Copyright JS Foundation and other contributors dev,@wdio/sync,MIT,Copyright JS Foundation and other contributors dev,body-parser,MIT,Copyright 2014 Jonathan Ong 2014-2015 Douglas Christopher Wilson +dev,codecov,MIT,Copyright 2014 Gregg Caines dev,cors,MIT,Copyright 2013 Troy Goode dev,express,MIT,Copyright 2009-2014 TJ Holowaychuk 2013-2014 Roman Shtylman 2014-2015 Douglas Christopher Wilson dev,istanbul-instrumenter-loader,MIT,Copyright JS Foundation and other contributors @@ -30,6 +32,7 @@ dev,karma-spec-reporter,MIT,Copyright 2015 Michael Lex dev,karma-typescript-preprocessor2,MIT,Copyright 2015 Cleiton Gomes Loiola dev,karma-webpack,MIT,Copyright JS Foundation and other contributors dev,lerna,MIT,Copyright 2015-present Lerna Contributors +dev,morgan,MIT,Copyright 2014 Jonathan Ong 2014-2017 Douglas Christopher Wilson dev,npm-run-all,MIT,Copyright 2015 Toru Nagashima dev,prettier,MIT,Copyright James Long and contributors dev,replace-in-file,MIT,Copyright 2015-2019 Adam Reis From a8708d466cf683267787f202b244f8e54d6f453e Mon Sep 17 00:00:00 2001 From: Bastien Caudan Date: Fri, 7 Feb 2020 16:08:02 +0100 Subject: [PATCH 3/6] :wrench: add check licenses gitlab stage --- .gitlab-ci.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 29bf8453fd..5e9c8ac614 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -81,6 +81,13 @@ npm-e2e: - yarn - yarn test:e2e:npm +check-licenses: + stage: test + tags: ['runner:main', 'size:large'] + image: $CI_IMAGE + script: + - node --no-warnings scripts/check-licenses.js + unit-cbt: except: refs: From ebee74c512037542554916c5cb5a70071120ead4 Mon Sep 17 00:00:00 2001 From: Bastien Caudan Date: Mon, 10 Feb 2020 13:14:42 +0100 Subject: [PATCH 4/6] :ok_hand: Improve find package.json MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Benoît Zugmeyer --- scripts/check-licenses.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/check-licenses.js b/scripts/check-licenses.js index 26faedbff0..5cbf5eaf15 100644 --- a/scripts/check-licenses.js +++ b/scripts/check-licenses.js @@ -35,7 +35,7 @@ async function main() { } async function findPackageJsonPaths() { - const { stdout } = await exec('find . -name "package.json" | grep -v node_modules') + const { stdout } = await exec('find . -path "*/node_modules/*" -prune -o -name "package.json" -print') return stdout.trim().split('\n') } From d29c03b42c7ccf18a215b82f77cd12b25604f806 Mon Sep 17 00:00:00 2001 From: Bastien Caudan Date: Mon, 10 Feb 2020 13:28:02 +0100 Subject: [PATCH 5/6] :ok_hand: improve retrieve dependencies --- scripts/check-licenses.js | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/scripts/check-licenses.js b/scripts/check-licenses.js index 5cbf5eaf15..b9dfaac1d8 100644 --- a/scripts/check-licenses.js +++ b/scripts/check-licenses.js @@ -12,10 +12,7 @@ async function main() { const packageJsonPaths = await findPackageJsonPaths() console.log(`Look for dependencies in:\n`, packageJsonPaths) - const declaredDependencies = packageJsonPaths - .map(retrievePackageJsonDependencies) - .reduce(withoutDuplicates) - .sort() + const declaredDependencies = withoutDuplicates(packageJsonPaths.flatMap(retrievePackageJsonDependencies)).sort() const declaredLicenses = (await retrieveLicenses()).sort() @@ -47,8 +44,8 @@ function retrievePackageJsonDependencies(packageJsonPath) { .filter((dependency) => !dependency.includes('@datadog')) } -function withoutDuplicates(a, b) { - return [...new Set([...a, ...b])] +function withoutDuplicates(a) { + return [...new Set(a)] } async function retrieveLicenses() { From d45a1bf99231a6831889abd72b833924fb613459 Mon Sep 17 00:00:00 2001 From: Bastien Caudan Date: Mon, 10 Feb 2020 13:31:33 +0100 Subject: [PATCH 6/6] :ok_hand: do not swallow errors --- scripts/check-licenses.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/check-licenses.js b/scripts/check-licenses.js index b9dfaac1d8..3974ae2aa6 100644 --- a/scripts/check-licenses.js +++ b/scripts/check-licenses.js @@ -26,7 +26,7 @@ async function main() { `\nIn ${LICENSE_FILE} but not in package.json:\n`, declaredLicenses.filter((d) => !declaredDependencies.includes(d)) ) - throw new Error() + throw new Error('dependencies mismatch') } console.log(`\n✅ All dependencies listed in ${LICENSE_FILE}`) } @@ -63,6 +63,7 @@ async function retrieveLicenses() { return licenses } -main().catch(() => { +main().catch((e) => { + console.error('\nStacktrace:\n', e) process.exit(1) })