Skip to content

Commit

Permalink
Throttle filename check (#1136)
Browse files Browse the repository at this point in the history
GitHub did not like the switch to Node.js v20 in jobs. Build fails at the
determine filename step, without much info on top of a "fetch failed".
This update forces throttling on the fetch requests in the hope that this
can improve things.

It also logs specs being processed to provide more clues about what the problem
might be.
  • Loading branch information
tidoust authored Nov 22, 2023
1 parent 02047d6 commit 92e8917
Showing 1 changed file with 17 additions and 13 deletions.
30 changes: 17 additions & 13 deletions src/build-index.js
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ const determineTestPath = require("./determine-testpath.js");
const extractPages = require("./extract-pages.js");
const fetchInfo = require("./fetch-info.js");
const fetchGroups = require("./fetch-groups.js");
const throttle = require("./throttle");
const githubToken = (_ => {
try {
return require("../config.json").GH_TOKEN;
Expand Down Expand Up @@ -306,7 +307,8 @@ async function runPages(index) {
}


async function runFilename(index, { previousIndex }) {
async function runFilename(index, { previousIndex, log }) {

// Use previous filename info when it cannot be determined (this usually means
// that there was a transient network error)
async function determineSpecFilename(spec, type) {
Expand All @@ -319,20 +321,22 @@ async function runFilename(index, { previousIndex }) {
return previous ? previous[type].filename : null;
}

return Promise.all(
index.map(async spec => {
spec.nightly.filename = await determineSpecFilename(spec, "nightly");
if (spec.release) {
spec.release.filename = await determineSpecFilename(spec, "release");
}
async function checkSpec(spec) {
log(`- find filenames for ${spec.shortname}`);
spec.nightly.filename = await determineSpecFilename(spec, "nightly");
if (spec.release) {
spec.release.filename = await determineSpecFilename(spec, "release");
}

// Sleep a bit as draft CSS WG server does not seem to like receiving too
// many requests in a row.
await sleep(50);
// Sleep a bit as draft CSS WG server does not seem to like receiving too
// many requests in a row.
await sleep(50);

return spec;
})
);
return spec;
}

const throttledCheck = throttle(checkSpec, 2);
return Promise.all(index.map(throttledCheck));
}


Expand Down

0 comments on commit 92e8917

Please sign in to comment.