From 2e1a696369183620340d74697d2902860c687c51 Mon Sep 17 00:00:00 2001
From: Gadi Cohen <dragon@wastelands.net>
Date: Wed, 24 Feb 2021 10:45:53 +0200
Subject: [PATCH] chore(concurrency): early support

chore(concurrency): cleanup
---
 package.json                 |  3 ++-
 src/lib/options.ts           |  5 +++++
 src/lib/yahooFinanceFetch.js | 14 +++++++++++++-
 yarn.lock                    | 20 ++++++++++++++++++++
 4 files changed, 40 insertions(+), 2 deletions(-)

diff --git a/package.json b/package.json
index 0c747922..a355a3e0 100644
--- a/package.json
+++ b/package.json
@@ -41,7 +41,8 @@
   "dependencies": {
     "ajv": "^7.0.3",
     "ajv-formats": "^1.5.1",
-    "node-fetch": "^2.6.1"
+    "node-fetch": "^2.6.1",
+    "p-queue": "^6.6.2"
   },
   "devDependencies": {
     "@semantic-release/changelog": "5.0.1",
diff --git a/src/lib/options.ts b/src/lib/options.ts
index e1cff6f3..1823830e 100644
--- a/src/lib/options.ts
+++ b/src/lib/options.ts
@@ -1,4 +1,9 @@
 export default {
+  queue: {
+    // See https://github.com/sindresorhus/p-queue for all options
+    concurrency: 8, // Min: 1, Max: Infinity
+    timeout: 60,
+  },
   validation: {
     logErrors: true,
     logOptionsErrors: true,
diff --git a/src/lib/yahooFinanceFetch.js b/src/lib/yahooFinanceFetch.js
index fcabdcee..5ed10578 100644
--- a/src/lib/yahooFinanceFetch.js
+++ b/src/lib/yahooFinanceFetch.js
@@ -1,3 +1,5 @@
+const { default: PQueue } = require("p-queue");
+
 const errors = require("./errors");
 const pkg = require("../../package.json");
 
@@ -14,6 +16,14 @@ async function yahooFinanceFetch(
       "yahooFinanceFetch called without this._env set"
     );
 
+  const qOpts = this._opts.queue;
+  let queue = yahooFinanceFetch.queue;
+
+  if (!queue) queue = yahooFinanceFetch.queue = new PQueue(this._opts.queue);
+  if (queue.concurrency !== qOpts.concurrency)
+    queue.concurrency = qOpts.concurrency;
+  if (queue.timeout !== qOpts.timeout) queue.timeout = qOpts.timeout;
+
   const { URLSearchParams, fetch, fetchDevel } = this._env;
 
   const urlSearchParams = new URLSearchParams(params);
@@ -32,7 +42,7 @@ async function yahooFinanceFetch(
   // used in moduleExec.ts
   if (func === "csv") func = "text";
 
-  const res = await fetchFunc(url, fetchOptions);
+  const res = await queue.add(() => fetchFunc(url, fetchOptions));
   const result = await res[func]();
 
   /*
@@ -69,4 +79,6 @@ async function yahooFinanceFetch(
   return result;
 }
 
+yahooFinanceFetch.queue = null;
+
 module.exports = yahooFinanceFetch;
diff --git a/yarn.lock b/yarn.lock
index 019ec5e4..7c9f3305 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -2493,6 +2493,11 @@ esutils@^2.0.2:
   resolved "https://registry.yarnpkg.com/esutils/-/esutils-2.0.3.tgz#74d2eb4de0b8da1293711910d50775b9b710ef64"
   integrity sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==
 
+eventemitter3@^4.0.4:
+  version "4.0.7"
+  resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-4.0.7.tgz#2de9b68f6528d5644ef5c59526a1b4a07306169f"
+  integrity sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==
+
 exec-sh@^0.3.2:
   version "0.3.4"
   resolved "https://registry.yarnpkg.com/exec-sh/-/exec-sh-0.3.4.tgz#3a018ceb526cc6f6df2bb504b2bfe8e3a4934ec5"
@@ -5560,6 +5565,14 @@ p-map@^4.0.0:
   dependencies:
     aggregate-error "^3.0.0"
 
+p-queue@^6.6.2:
+  version "6.6.2"
+  resolved "https://registry.yarnpkg.com/p-queue/-/p-queue-6.6.2.tgz#2068a9dcf8e67dd0ec3e7a2bcb76810faa85e426"
+  integrity sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ==
+  dependencies:
+    eventemitter3 "^4.0.4"
+    p-timeout "^3.2.0"
+
 p-reduce@^2.0.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/p-reduce/-/p-reduce-2.1.0.tgz#09408da49507c6c274faa31f28df334bc712b64a"
@@ -5573,6 +5586,13 @@ p-retry@^4.0.0:
     "@types/retry" "^0.12.0"
     retry "^0.12.0"
 
+p-timeout@^3.2.0:
+  version "3.2.0"
+  resolved "https://registry.yarnpkg.com/p-timeout/-/p-timeout-3.2.0.tgz#c7e17abc971d2a7962ef83626b35d635acf23dfe"
+  integrity sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==
+  dependencies:
+    p-finally "^1.0.0"
+
 p-try@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/p-try/-/p-try-1.0.0.tgz#cbc79cdbaf8fd4228e13f621f2b1a237c1b207b3"