From b298844f2fdd848e18473261e4d9cf4666ee5f30 Mon Sep 17 00:00:00 2001 From: Gadi Cohen Date: Fri, 15 Sep 2023 16:05:13 +0100 Subject: [PATCH] feat(getCrumb): ability to specify custom cookie jar / store (#670) --- src/lib/cookieJar.ts | 6 +---- src/lib/getCrumb.spec.ts | 43 +++++++++++++++++++------------ src/lib/getCrumb.ts | 17 ++++++------ src/lib/options.ts | 3 +++ src/lib/setGlobalConfig.ts | 1 + src/lib/yahooFinanceFetch.spec.ts | 17 +++++++----- src/lib/yahooFinanceFetch.ts | 17 +++++++----- 7 files changed, 63 insertions(+), 41 deletions(-) diff --git a/src/lib/cookieJar.ts b/src/lib/cookieJar.ts index 3290243d..fbfa6fe7 100644 --- a/src/lib/cookieJar.ts +++ b/src/lib/cookieJar.ts @@ -1,6 +1,6 @@ import { Cookie, CookieJar } from "tough-cookie"; -export class MyCookieJar extends CookieJar { +export class ExtendedCookieJar extends CookieJar { async setFromSetCookieHeaders( setCookieHeader: string | Array, url: string @@ -24,7 +24,3 @@ export class MyCookieJar extends CookieJar { } } } - -const cookiejar = new MyCookieJar(); - -export default cookiejar; diff --git a/src/lib/getCrumb.spec.ts b/src/lib/getCrumb.spec.ts index 2dba6f7d..2e30a268 100644 --- a/src/lib/getCrumb.spec.ts +++ b/src/lib/getCrumb.spec.ts @@ -1,11 +1,16 @@ import env from "../env-node"; import getCrumb, { _getCrumb, getCrumbClear } from "./getCrumb"; -import { MyCookieJar } from "./cookieJar.js"; import { jest } from "@jest/globals"; import { consoleSilent, consoleRestore } from "../../tests/console.js"; +import { ExtendedCookieJar } from "./cookieJar.js"; + describe("getCrumb", () => { - beforeAll(consoleSilent); + let cookieJar: ExtendedCookieJar; + beforeAll(() => { + consoleSilent(); + cookieJar = new ExtendedCookieJar(); + }); afterAll(consoleRestore); describe("_getCrumb", () => { @@ -13,13 +18,13 @@ describe("getCrumb", () => { const fetch = await env.fetchDevel(); const crumb = await _getCrumb( + new ExtendedCookieJar(), fetch, // @ts-expect-error: fetchDevel still has no types (yet) { devel: true }, "https://finance.yahoo.com/quote/AAPL", "getCrumb-quote-AAPL.json", - true, - new MyCookieJar() + true ); expect(crumb).toBe("mloUP8q7ZPH"); }); @@ -28,6 +33,7 @@ describe("getCrumb", () => { const fetch = await env.fetchDevel(); const crumb = await _getCrumb( + cookieJar, fetch, // @ts-expect-error: fetchDevel still has no types (yet) { devel: true } @@ -39,6 +45,7 @@ describe("getCrumb", () => { const fetch = await env.fetchDevel(); let crumb = await _getCrumb( + cookieJar, fetch, // @ts-expect-error: fetchDevel still has no types (yet) { devel: true }, @@ -49,6 +56,7 @@ describe("getCrumb", () => { // TODO, at tests to see how many times fetch was called, etc. crumb = await _getCrumb( + cookieJar, fetch, // @ts-expect-error: fetchDevel still has no types (yet) { devel: true }, @@ -62,13 +70,13 @@ describe("getCrumb", () => { await expect(() => _getCrumb( + new ExtendedCookieJar(), fetch, // @ts-expect-error: fetchDevel still has no types (yet) { devel: true }, "https://finance.yahoo.com/quote/AAPL", "getCrumb-quote-AAPL-no-cookies.fake.json", - true, - new MyCookieJar() + true ) ).rejects.toThrowError(/No set-cookie/); }); @@ -78,13 +86,13 @@ describe("getCrumb", () => { await expect(() => _getCrumb( + new ExtendedCookieJar(), fetch, // @ts-expect-error: fetchDevel still has no types (yet) { devel: true }, "https://finance.yahoo.com/quote/AAPL", "getCrumb-quote-AAPL-no-context.fake.json", - true, - new MyCookieJar() + true ) ).rejects.toThrowError(/Could not find window.YAHOO.context/); }); @@ -94,13 +102,13 @@ describe("getCrumb", () => { await expect(() => _getCrumb( + new ExtendedCookieJar(), fetch, // @ts-expect-error: fetchDevel still has no types (yet) { devel: true }, "https://finance.yahoo.com/quote/AAPL", "getCrumb-quote-AAPL-invalid-json.fake.json", - true, - new MyCookieJar() + true ) ).rejects.toThrowError(/Could not parse window.YAHOO.context/); }); @@ -110,13 +118,13 @@ describe("getCrumb", () => { await expect(() => _getCrumb( + new ExtendedCookieJar(), fetch, // @ts-expect-error: fetchDevel still has no types (yet) { devel: true }, "https://finance.yahoo.com/quote/AAPL", "getCrumb-quote-AAPL-no-crumb.fake.json", - true, - new MyCookieJar() + true ) ).rejects.toThrowError(/Could not find crumb/); }); @@ -126,13 +134,13 @@ describe("getCrumb", () => { const fetch = await env.fetchDevel(); const crumb = await _getCrumb( + new ExtendedCookieJar(), fetch, // @ts-expect-error: fetchDevel still has no types (yet) { devel: true }, "https://finance.yahoo.com/quote/AAPL", "getCrumb-quote-AAPL-pre-consent-VPN-UK.json", - true, - new MyCookieJar() + true ); expect(crumb).toBe("Ky3Po5TGQRZ"); // consoleSilent(); @@ -141,9 +149,10 @@ describe("getCrumb", () => { describe("getCrumb", () => { it("works", async () => { - await getCrumbClear(); + await getCrumbClear(cookieJar); const fetch = await env.fetchDevel(); const crumb = await getCrumb( + cookieJar, fetch, // @ts-expect-error: fetchDevel still has no types (yet) { devel: true } @@ -154,9 +163,10 @@ describe("getCrumb", () => { it("only calls getCrumb once", async () => { const fetch = await env.fetchDevel(); const _getCrumb = jest.fn(() => "crumb"); - await getCrumbClear(); + await getCrumbClear(cookieJar); getCrumb( + cookieJar, fetch, // @ts-expect-error: fetchDevel still has no types (yet) { devel: true }, @@ -165,6 +175,7 @@ describe("getCrumb", () => { ); getCrumb( + cookieJar, fetch, // @ts-expect-error: fetchDevel still has no types (yet) { devel: true }, diff --git a/src/lib/getCrumb.ts b/src/lib/getCrumb.ts index 2c039e32..80607ac1 100644 --- a/src/lib/getCrumb.ts +++ b/src/lib/getCrumb.ts @@ -1,5 +1,5 @@ import type { RequestInfo, RequestInit, Response } from "node-fetch"; -import defaultCookieJar from "./cookieJar.js"; +import type { ExtendedCookieJar } from "./cookieJar"; let crumb: string | null = null; // let crumbFetchTime = 0; @@ -11,12 +11,12 @@ const parseHtmlEntities = (str: string) => ); export async function _getCrumb( + cookieJar: ExtendedCookieJar, fetch: (url: RequestInfo, init?: RequestInit) => Promise, fetchOptionsBase: RequestInit, url = "https://finance.yahoo.com/quote/AAPL", develOverride = "getCrumb-quote-AAPL.json", - noCache = false, - cookieJar = defaultCookieJar + noCache = false ): Promise { // if (crumb && crumbFetchTime + MAX_CRUMB_CACHE_TIME > Date.now()) return crumb; @@ -207,12 +207,12 @@ export async function _getCrumb( */ return await _getCrumb( + cookieJar, fetch, finalResponseFetchOptions, copyConsentResponseLocation, "getCrumb-quote-AAPL-consent-final-redirect.html", - noCache, - cookieJar + noCache ); } } else { @@ -272,14 +272,15 @@ export async function _getCrumb( let promise: Promise | null = null; let promiseTime = 0; -export async function getCrumbClear() { +export async function getCrumbClear(cookieJar: ExtendedCookieJar) { crumb = null; promise = null; promiseTime = 0; - await defaultCookieJar.removeAllCookies(); + await cookieJar.removeAllCookies(); } export default function getCrumb( + cookieJar: ExtendedCookieJar, fetch: (url: RequestInfo, init?: RequestInit) => Promise, fetchOptionsBase: RequestInit, url = "https://finance.yahoo.com/quote/AAPL", @@ -288,7 +289,7 @@ export default function getCrumb( // TODO, rather do this with cookie expire time somehow const now = Date.now(); if (!promise || now - promiseTime > 60_000) { - promise = __getCrumb(fetch, fetchOptionsBase, url); + promise = __getCrumb(cookieJar, fetch, fetchOptionsBase, url); promiseTime = now; } diff --git a/src/lib/options.ts b/src/lib/options.ts index 36a07541..2c06967b 100644 --- a/src/lib/options.ts +++ b/src/lib/options.ts @@ -1,15 +1,18 @@ // TODO, keep defaults there too? import type { ValidationOptions } from "./validateAndCoerceTypes.js"; import type { QueueOptions } from "./queue.js"; +import { ExtendedCookieJar } from "./cookieJar.js"; export interface YahooFinanceOptions { YF_QUERY_HOST?: string; + cookieJar: ExtendedCookieJar; queue?: QueueOptions; validation?: ValidationOptions; } const options: YahooFinanceOptions = { YF_QUERY_HOST: process.env.YF_QUERY_HOST || "query2.finance.yahoo.com", + cookieJar: new ExtendedCookieJar(), queue: { concurrency: 4, // Min: 1, Max: Infinity timeout: 60, diff --git a/src/lib/setGlobalConfig.ts b/src/lib/setGlobalConfig.ts index c21484f3..35698629 100644 --- a/src/lib/setGlobalConfig.ts +++ b/src/lib/setGlobalConfig.ts @@ -13,6 +13,7 @@ export default function setGlobalConfig( options: this._opts.validation, schemaKey: "#/definitions/YahooFinanceOptions", }); + // @ts-expect-error: TODO mergeObjects(this._opts, config as Obj); } diff --git a/src/lib/yahooFinanceFetch.spec.ts b/src/lib/yahooFinanceFetch.spec.ts index 72f8c9e5..68e1704d 100644 --- a/src/lib/yahooFinanceFetch.spec.ts +++ b/src/lib/yahooFinanceFetch.spec.ts @@ -120,21 +120,23 @@ describe("yahooFinanceFetch", () => { expect(queue.concurrency).toBe(5); }); - it("yahooFinanceFetch branch check for alternate queue", () => { + it("yahooFinanceFetch branch check for alternate queue", async () => { const promises = [ yahooFinanceFetch("", {}), yahooFinanceFetch("", {}, {}), yahooFinanceFetch("", {}, { queue: {} }), ]; + await immediate(); + env.fetch.fetches[0].resolveWith({ ok: true }); env.fetch.fetches[1].resolveWith({ ok: true }); env.fetch.fetches[2].resolveWith({ ok: true }); - return Promise.all(promises); + await Promise.all(promises); }); - it("assert defualts to {} for empty queue opts", () => { + it("assert defualts to {} for empty queue opts", async () => { moduleOpts.queue.concurrency = 1; const opts = { ..._opts }; // @ts-ignore: intentional to test runtime failures @@ -142,16 +144,18 @@ describe("yahooFinanceFetch", () => { const yahooFinanceFetch = _yahooFinanceFetch.bind({ _env: env, _opts }); const promise = yahooFinanceFetch("", {}, moduleOpts); + await immediate(); env.fetch.fetches[0].resolveWith({ ok: true }); - return expect(promise).resolves.toMatchObject({ ok: true }); + await expect(promise).resolves.toMatchObject({ ok: true }); }); - it("single item in queue", () => { + it("single item in queue", async () => { moduleOpts.queue.concurrency = 1; const promise = yahooFinanceFetch("", {}, moduleOpts); + await immediate(); env.fetch.fetches[0].resolveWith({ ok: true }); - return expect(promise).resolves.toMatchObject({ ok: true }); + await expect(promise).resolves.toMatchObject({ ok: true }); }); it("waits if exceeding concurrency max", async () => { @@ -161,6 +165,7 @@ describe("yahooFinanceFetch", () => { yahooFinanceFetch("", {}, moduleOpts), yahooFinanceFetch("", {}, moduleOpts), ]; + await immediate(); // Second func should not be called until 1st reoslves (limit 1) expect(env.fetch.fetches.length).toBe(1); diff --git a/src/lib/yahooFinanceFetch.ts b/src/lib/yahooFinanceFetch.ts index 6610e432..9ab77039 100644 --- a/src/lib/yahooFinanceFetch.ts +++ b/src/lib/yahooFinanceFetch.ts @@ -7,7 +7,6 @@ import type { QueueOptions } from "./queue.js"; import errors from "./errors.js"; import pkg from "../../package.json"; import getCrumb from "./getCrumb.js"; -import cookieJar from "./cookieJar.js"; const userAgent = `${pkg.name}/${pkg.version} (+${pkg.repository})`; @@ -62,7 +61,7 @@ function substituteVariables(this: YahooFinanceFetchThis, urlBase: string) { async function yahooFinanceFetch( this: YahooFinanceFetchThis, urlBase: string, - params = {}, + params: Record = {}, moduleOpts: YahooFinanceFetchModuleOptions = {}, func = "json", needsCrumb = false @@ -93,8 +92,12 @@ async function yahooFinanceFetch( }; if (needsCrumb) { - // @ts-expect-error: TODO, crumb string type for partial params - params.crumb = await getCrumb(fetchFunc, fetchOptionsBase); + const crumb = await getCrumb( + this._opts.cookieJar, + fetchFunc, + fetchOptionsBase + ); + if (crumb) params.crumb = crumb; } // @ts-expect-error: TODO copy interface? @types lib? @@ -109,7 +112,9 @@ async function yahooFinanceFetch( ...fetchOptionsBase, headers: { ...fetchOptionsBase.headers, - cookie: cookieJar.getCookieStringSync(url, { allPaths: true }), + cookie: await this._opts.cookieJar.getCookieString(url, { + allPaths: true, + }), }, }; @@ -122,7 +127,7 @@ async function yahooFinanceFetch( const setCookieHeaders = response.headers.raw()["set-cookie"]; if (setCookieHeaders) - cookieJar.setFromSetCookieHeaders(setCookieHeaders, url); + this._opts.cookieJar.setFromSetCookieHeaders(setCookieHeaders, url); const result = await response[func]();