From b3a6a6a3b99df29a6d91ae783bc71a96e0b4de84 Mon Sep 17 00:00:00 2001 From: Dominique Hazael-Massieux Date: Fri, 12 Jun 2020 18:22:55 +0200 Subject: [PATCH 1/7] Report specs found as candidates as github issue per #71 --- .github/candidate-specs.md | 10 ++++ .github/workflows/report-new-specs.yml | 22 +++++++ package-lock.json | 5 ++ package.json | 1 + src/find-specs.js | 81 +++++++++++++------------- 5 files changed, 77 insertions(+), 42 deletions(-) create mode 100644 .github/candidate-specs.md create mode 100644 .github/workflows/report-new-specs.yml diff --git a/.github/candidate-specs.md b/.github/candidate-specs.md new file mode 100644 index 00000000..a6ab9f9c --- /dev/null +++ b/.github/candidate-specs.md @@ -0,0 +1,10 @@ +--- +title: New specs for review +assigneeds: tidoust, dontcallmedom +labels: enhancement +--- +[find-specs](src/find-specs.js) has identified the following candidates as potential new specs to consider: + +{{ env.candidate_list }} + +Please review if they match the inclusion criteria. Those that don't and never will should be added to [[src/data/ignore.json]], those that don't match yet but may in the future can be added to [[src/data/monitor-repos.json]], and those that do match should be brought as a pull request on [[specs.json]]. \ No newline at end of file diff --git a/.github/workflows/report-new-specs.yml b/.github/workflows/report-new-specs.yml new file mode 100644 index 00000000..fe5af17c --- /dev/null +++ b/.github/workflows/report-new-specs.yml @@ -0,0 +1,22 @@ +name: Report new specs + +on: + schedule: + - cron: '0 0 0 * *' +jobs: + find-specs: + name: Find potential new specs + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Use Node.js 12.x + uses: actions/setup-node@v1 + with: + node-version: 12.x + - run: npm install + - run: node src/find-specs.js # sets candidate_list env variable + - uses: JasonEtco/create-an-issue@v2 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + filename: .github/candidate-specs.md \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 2dccc641..dbeb57dc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4,6 +4,11 @@ "lockfileVersion": 1, "requires": true, "dependencies": { + "@actions/core": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.2.4.tgz", + "integrity": "sha512-YJCEq8BE3CdN8+7HPZ/4DxJjk/OkZV2FFIf+DlZTC/4iBlzYCD5yjRR6eiOS5llO11zbRltIRuKAjMKaWTE6cg==" + }, "@babel/code-frame": { "version": "7.8.3", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.8.3.tgz", diff --git a/package.json b/package.json index 28babcc5..871e085d 100644 --- a/package.json +++ b/package.json @@ -25,6 +25,7 @@ "release-it": "^13.5.1" }, "dependencies": { + "@actions/core": "^1.2.4", "node-fetch": "^2.6.0" } } diff --git a/src/find-specs.js b/src/find-specs.js index 02a32dae..9a200521 100644 --- a/src/find-specs.js +++ b/src/find-specs.js @@ -1,4 +1,5 @@ 'use strict'; +const core = require('@actions/core'); const fetch = require("node-fetch"); @@ -34,7 +35,7 @@ function canonicalizeGhUrl(r) { if (url.pathname.lastIndexOf('/') === 0 && url.pathname.length > 1) { url.pathname += '/'; } - return url.toString(); + return {repo: r.owner.login + '/' + r.name, spec: url.toString()}; } function canonicalizeTRUrl(url) { @@ -43,23 +44,25 @@ function canonicalizeTRUrl(url) { return url.toString(); } -const toGhUrl = repo => `https://${repo.owner.login.toLowerCase()}.github.io/${repo.name}/` +const toGhUrl = repo => { return {repo: `${repo.owner.login}/${repo.name}`, spec: `https://${repo.owner.login.toLowerCase()}.github.io/${repo.name}/`}; }; const matchRepoName = fullName => r => fullName === r.owner.login + '/' + r.name; const isRelevantRepo = fullName => !Object.keys(ignorable.repos).includes(fullName) && !Object.keys(temporarilyIgnorableRepos).includes(fullName); -const isRelevantSpec = url => !Object.keys(ignorable.specs).includes(url); -const isUnknownSpec = url => !specs.find(s => s.nightly.url.startsWith(url) +const hasRelevantSpec = ({spec: url}) => !Object.keys(ignorable.specs).includes(url); +const hasUnknownSpec = ({spec: url}) => !specs.find(s => s.nightly.url.startsWith(url) || (s.release && s.release.url === url)) const hasRepoType = type => r => r.w3c && r.w3c["repo-type"] && (r.w3c["repo-type"] === type || r.w3c["repo-type"].includes(type)); -const urlIfExists = u => fetch(u).then(({ok, url}) => { - if (ok) return url; +const hasExistingSpec = (candidate) => fetch(candidate.spec).then(({ok, url}) => { + if (ok) return {...candidate, spec: url}; }); (async function() { + let candidates = []; + const {groups, repos} = await fetch("https://w3c.github.io/validate-repos/report.json").then(r => r.json()); const specRepos = await fetch("https://w3c.github.io/spec-dashboard/repo-map.json").then(r => r.json()); const whatwgSpecs = await fetch("https://raw.githubusercontent.com/whatwg/sg/master/db.json").then(r => r.json()) - .then(d => d.workstreams.map(w => w.standards).flat()); + .then(d => d.workstreams.map(w => { return {...w.standards[0], id: w.id}; })); const wgs = Object.values(groups).filter(g => g.type === "working group" && !nonBrowserSpecWgs.includes(g.name)); const cgs = Object.values(groups).filter(g => g.type === "community group" && watchedBrowserCgs.includes(g.name)); @@ -72,30 +75,25 @@ const urlIfExists = u => fetch(u).then(({ok, url}) => { const recTrackRepos = wgRepos.filter(hasRepoType('rec-track')); // * look if those with homepage URLs have a match in the list of specs - console.log("URLs from a repo of a browser-spec producing WG with no matching URL in spec list") - console.log(recTrackRepos.filter(r => r.homepageUrl) - .map(canonicalizeGhUrl) - .filter(isUnknownSpec) - .filter(isRelevantSpec) - ); + candidates = recTrackRepos.filter(r => r.homepageUrl) + .map(canonicalizeGhUrl) + .filter(hasUnknownSpec) + .filter(hasRelevantSpec); // * look if those without a homepage URL have a match with their generated URL - const wgUrls = (await Promise.all(recTrackRepos.filter(r => !r.homepageUrl) + candidates = candidates.concat((await Promise.all(recTrackRepos.filter(r => !r.homepageUrl) .map(toGhUrl) - .filter(isUnknownSpec) - .filter(isRelevantSpec) - .map(urlIfExists))).filter(x => x); - console.log("Unadvertized URLs from a repo of a browser-spec producing WG with no matching URL in spec list") - console.log(wgUrls); + .filter(hasUnknownSpec) + .filter(hasRelevantSpec) + .map(hasExistingSpec))).filter(x => x)); // Look which of the specRepos on recTrack from a browser-producing WG have no match - console.log("TR specs from browser-producing WGs") - console.log( + candidates = candidates.concat( Object.keys(specRepos).map( - r => specRepos[r].filter(s => s.recTrack && wgs.find(g => g.id === s.group)).map(s => canonicalizeTRUrl(s.url))) + r => specRepos[r].filter(s => s.recTrack && wgs.find(g => g.id === s.group)).map(s => { return {repo: r, spec: canonicalizeTRUrl(s.url)};})) .flat() - .filter(isUnknownSpec) - .filter(isRelevantSpec) + .filter(hasUnknownSpec) + .filter(hasRelevantSpec) ); // CGs @@ -107,27 +105,26 @@ const urlIfExists = u => fetch(u).then(({ok, url}) => { const cgSpecRepos = cgRepos.filter(r => !r.w3c || hasRepoType('cg-report')(r)); // * look if those with homepage URLs have a match in the list of specs - console.log("URLs from a repo of a browser-spec producing CG with no matching URL in spec list") - console.log(cgSpecRepos.filter(r => r.homepageUrl) + candidates = candidates.concat(cgSpecRepos.filter(r => r.homepageUrl) .map(canonicalizeGhUrl) - .filter(isUnknownSpec) - .filter(isRelevantSpec) + .filter(hasUnknownSpec) + .filter(hasRelevantSpec) ); // * look if those without a homepage URL have a match with their generated URL - const cgUrls = (await Promise.all(cgSpecRepos.filter(r => !r.homepageUrl) - .map(toGhUrl) - .filter(isUnknownSpec) - .filter(isRelevantSpec) - .map(urlIfExists))).filter(x => x); - console.log("Unadvertized URLs from a repo of a browser-spec producing CG with no matching URL in spec list") - console.log(cgUrls); - - - const whatwgUrls = whatwgSpecs.map(s => s.href) - .filter(isUnknownSpec) - .filter(isRelevantSpec); - console.log("URLs from WHATWG with no matching URL in spec list") - console.log(whatwgUrls); + candidates = candidates.concat((await Promise.all(cgSpecRepos.filter(r => !r.homepageUrl) + .map(toGhUrl) + .filter(hasUnknownSpec) + .filter(hasRelevantSpec) + .map(hasExistingSpec)) + ).filter(x => x)); + + candidates = candidates.concat(whatwgSpecs.map(s => { return {repo: `whatwg/${s.id}`, spec: s.href};}) + .filter(hasUnknownSpec) + .filter(hasRelevantSpec)); + const candidate_list = candidates.sort((c1, c2) => c1.spec.localeCompare(c2.spec)) + .map(c => `- [ ] ${c.spec} from ${c.repo}`).join("\n"); + core.exportVariable("candidate_list", candidate_list); + console.log(candidate_list); })().catch(e => { console.error(e); process.exit(1); From 09bcd282601ef17e1dcf051ddfcd3ba8768c7b07 Mon Sep 17 00:00:00 2001 From: Dominique Hazael-Massieux Date: Sun, 14 Jun 2020 21:13:29 +0200 Subject: [PATCH 2/7] Fix cron syntax --- .github/workflows/report-new-specs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/report-new-specs.yml b/.github/workflows/report-new-specs.yml index fe5af17c..8bdf28ec 100644 --- a/.github/workflows/report-new-specs.yml +++ b/.github/workflows/report-new-specs.yml @@ -2,7 +2,7 @@ name: Report new specs on: schedule: - - cron: '0 0 0 * *' + - cron: '0 0 * * 1' jobs: find-specs: name: Find potential new specs From 668324ff5a26d51f95e21f2692ca2860565cdbd5 Mon Sep 17 00:00:00 2001 From: Dominique Hazael-Massieux Date: Mon, 15 Jun 2020 08:44:41 +0200 Subject: [PATCH 3/7] Activate workflow on pushes to its development branch --- .github/workflows/report-new-specs.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/report-new-specs.yml b/.github/workflows/report-new-specs.yml index 8bdf28ec..79161d81 100644 --- a/.github/workflows/report-new-specs.yml +++ b/.github/workflows/report-new-specs.yml @@ -1,6 +1,8 @@ name: Report new specs on: + push: + branches: auto-report-new-specs schedule: - cron: '0 0 * * 1' jobs: From 15b449dc9a5468af82fc1b744d22f1bf5639843f Mon Sep 17 00:00:00 2001 From: Dominique Hazael-Massieux Date: Mon, 15 Jun 2020 08:53:09 +0200 Subject: [PATCH 4/7] Fix links to local files --- .github/candidate-specs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/candidate-specs.md b/.github/candidate-specs.md index a6ab9f9c..06e1a4a4 100644 --- a/.github/candidate-specs.md +++ b/.github/candidate-specs.md @@ -7,4 +7,4 @@ labels: enhancement {{ env.candidate_list }} -Please review if they match the inclusion criteria. Those that don't and never will should be added to [[src/data/ignore.json]], those that don't match yet but may in the future can be added to [[src/data/monitor-repos.json]], and those that do match should be brought as a pull request on [[specs.json]]. \ No newline at end of file +Please review if they match the inclusion criteria. Those that don't and never will should be added to [ignore.json](src/data/ignore.json), those that don't match yet but may in the future can be added to [monitor-repo.json](src/data/monitor-repos.json), and those that do match should be brought as a pull request on [specs.json](specs.json). \ No newline at end of file From 91d169a93826708cb2011ef8ec38dfde700d6b75 Mon Sep 17 00:00:00 2001 From: Dominique Hazael-Massieux Date: Mon, 15 Jun 2020 08:53:28 +0200 Subject: [PATCH 5/7] Remove activation on push --- .github/workflows/report-new-specs.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/report-new-specs.yml b/.github/workflows/report-new-specs.yml index 79161d81..8bdf28ec 100644 --- a/.github/workflows/report-new-specs.yml +++ b/.github/workflows/report-new-specs.yml @@ -1,8 +1,6 @@ name: Report new specs on: - push: - branches: auto-report-new-specs schedule: - cron: '0 0 * * 1' jobs: From fd434078058743c1178b5221b6404242b19f29b1 Mon Sep 17 00:00:00 2001 From: Dominique Hazael-Massieux Date: Mon, 15 Jun 2020 08:53:46 +0200 Subject: [PATCH 6/7] Add links to source github repositories in report --- src/find-specs.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/find-specs.js b/src/find-specs.js index 9a200521..e7b4bc7a 100644 --- a/src/find-specs.js +++ b/src/find-specs.js @@ -122,7 +122,7 @@ const hasExistingSpec = (candidate) => fetch(candidate.spec).then(({ok, url}) => .filter(hasUnknownSpec) .filter(hasRelevantSpec)); const candidate_list = candidates.sort((c1, c2) => c1.spec.localeCompare(c2.spec)) - .map(c => `- [ ] ${c.spec} from ${c.repo}`).join("\n"); + .map(c => `- [ ] ${c.spec} from [${c.repo}](https://github.com/${c.repo})`).join("\n"); core.exportVariable("candidate_list", candidate_list); console.log(candidate_list); })().catch(e => { From 45cb9fa3d47efc0db4a472086924c83f8037766e Mon Sep 17 00:00:00 2001 From: Dominique Hazael-Massieux Date: Mon, 15 Jun 2020 09:46:25 +0200 Subject: [PATCH 7/7] Fix issue assignees syntax --- .github/candidate-specs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/candidate-specs.md b/.github/candidate-specs.md index 06e1a4a4..fd601431 100644 --- a/.github/candidate-specs.md +++ b/.github/candidate-specs.md @@ -1,6 +1,6 @@ --- title: New specs for review -assigneeds: tidoust, dontcallmedom +assignees: tidoust, dontcallmedom labels: enhancement --- [find-specs](src/find-specs.js) has identified the following candidates as potential new specs to consider: