diff --git a/package.json b/package.json index 0c747922..a355a3e0 100644 --- a/package.json +++ b/package.json @@ -41,7 +41,8 @@ "dependencies": { "ajv": "^7.0.3", "ajv-formats": "^1.5.1", - "node-fetch": "^2.6.1" + "node-fetch": "^2.6.1", + "p-queue": "^6.6.2" }, "devDependencies": { "@semantic-release/changelog": "5.0.1", diff --git a/src/lib/options.ts b/src/lib/options.ts index e1cff6f3..1823830e 100644 --- a/src/lib/options.ts +++ b/src/lib/options.ts @@ -1,4 +1,9 @@ export default { + queue: { + // See https://github.com/sindresorhus/p-queue for all options + concurrency: 8, // Min: 1, Max: Infinity + timeout: 60, + }, validation: { logErrors: true, logOptionsErrors: true, diff --git a/src/lib/yahooFinanceFetch.js b/src/lib/yahooFinanceFetch.js index fcabdcee..5ed10578 100644 --- a/src/lib/yahooFinanceFetch.js +++ b/src/lib/yahooFinanceFetch.js @@ -1,3 +1,5 @@ +const { default: PQueue } = require("p-queue"); + const errors = require("./errors"); const pkg = require("../../package.json"); @@ -14,6 +16,14 @@ async function yahooFinanceFetch( "yahooFinanceFetch called without this._env set" ); + const qOpts = this._opts.queue; + let queue = yahooFinanceFetch.queue; + + if (!queue) queue = yahooFinanceFetch.queue = new PQueue(this._opts.queue); + if (queue.concurrency !== qOpts.concurrency) + queue.concurrency = qOpts.concurrency; + if (queue.timeout !== qOpts.timeout) queue.timeout = qOpts.timeout; + const { URLSearchParams, fetch, fetchDevel } = this._env; const urlSearchParams = new URLSearchParams(params); @@ -32,7 +42,7 @@ async function yahooFinanceFetch( // used in moduleExec.ts if (func === "csv") func = "text"; - const res = await fetchFunc(url, fetchOptions); + const res = await queue.add(() => fetchFunc(url, fetchOptions)); const result = await res[func](); /* @@ -69,4 +79,6 @@ async function yahooFinanceFetch( return result; } +yahooFinanceFetch.queue = null; + module.exports = yahooFinanceFetch; diff --git a/yarn.lock b/yarn.lock index 019ec5e4..7c9f3305 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2493,6 +2493,11 @@ esutils@^2.0.2: resolved "https://registry.yarnpkg.com/esutils/-/esutils-2.0.3.tgz#74d2eb4de0b8da1293711910d50775b9b710ef64" integrity sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g== +eventemitter3@^4.0.4: + version "4.0.7" + resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-4.0.7.tgz#2de9b68f6528d5644ef5c59526a1b4a07306169f" + integrity sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw== + exec-sh@^0.3.2: version "0.3.4" resolved "https://registry.yarnpkg.com/exec-sh/-/exec-sh-0.3.4.tgz#3a018ceb526cc6f6df2bb504b2bfe8e3a4934ec5" @@ -5560,6 +5565,14 @@ p-map@^4.0.0: dependencies: aggregate-error "^3.0.0" +p-queue@^6.6.2: + version "6.6.2" + resolved "https://registry.yarnpkg.com/p-queue/-/p-queue-6.6.2.tgz#2068a9dcf8e67dd0ec3e7a2bcb76810faa85e426" + integrity sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ== + dependencies: + eventemitter3 "^4.0.4" + p-timeout "^3.2.0" + p-reduce@^2.0.0: version "2.1.0" resolved "https://registry.yarnpkg.com/p-reduce/-/p-reduce-2.1.0.tgz#09408da49507c6c274faa31f28df334bc712b64a" @@ -5573,6 +5586,13 @@ p-retry@^4.0.0: "@types/retry" "^0.12.0" retry "^0.12.0" +p-timeout@^3.2.0: + version "3.2.0" + resolved "https://registry.yarnpkg.com/p-timeout/-/p-timeout-3.2.0.tgz#c7e17abc971d2a7962ef83626b35d635acf23dfe" + integrity sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg== + dependencies: + p-finally "^1.0.0" + p-try@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/p-try/-/p-try-1.0.0.tgz#cbc79cdbaf8fd4228e13f621f2b1a237c1b207b3"