Skip to content

Commit

Permalink
feat(getCrumb): ability to specify custom cookie jar / store (#670)
Browse files Browse the repository at this point in the history
  • Loading branch information
gadicc committed Sep 15, 2023
1 parent 37f51d9 commit b298844
Show file tree
Hide file tree
Showing 7 changed files with 63 additions and 41 deletions.
6 changes: 1 addition & 5 deletions src/lib/cookieJar.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { Cookie, CookieJar } from "tough-cookie";

export class MyCookieJar extends CookieJar {
export class ExtendedCookieJar extends CookieJar {
async setFromSetCookieHeaders(
setCookieHeader: string | Array<string>,
url: string
Expand All @@ -24,7 +24,3 @@ export class MyCookieJar extends CookieJar {
}
}
}

const cookiejar = new MyCookieJar();

export default cookiejar;
43 changes: 27 additions & 16 deletions src/lib/getCrumb.spec.ts
Original file line number Diff line number Diff line change
@@ -1,25 +1,30 @@
import env from "../env-node";
import getCrumb, { _getCrumb, getCrumbClear } from "./getCrumb";
import { MyCookieJar } from "./cookieJar.js";
import { jest } from "@jest/globals";
import { consoleSilent, consoleRestore } from "../../tests/console.js";

import { ExtendedCookieJar } from "./cookieJar.js";

describe("getCrumb", () => {
beforeAll(consoleSilent);
let cookieJar: ExtendedCookieJar;
beforeAll(() => {
consoleSilent();
cookieJar = new ExtendedCookieJar();
});
afterAll(consoleRestore);

describe("_getCrumb", () => {
it("finds crumb in context", async () => {
const fetch = await env.fetchDevel();

const crumb = await _getCrumb(
new ExtendedCookieJar(),
fetch,
// @ts-expect-error: fetchDevel still has no types (yet)
{ devel: true },
"https://finance.yahoo.com/quote/AAPL",
"getCrumb-quote-AAPL.json",
true,
new MyCookieJar()
true
);
expect(crumb).toBe("mloUP8q7ZPH");
});
Expand All @@ -28,6 +33,7 @@ describe("getCrumb", () => {
const fetch = await env.fetchDevel();

const crumb = await _getCrumb(
cookieJar,
fetch,
// @ts-expect-error: fetchDevel still has no types (yet)
{ devel: true }
Expand All @@ -39,6 +45,7 @@ describe("getCrumb", () => {
const fetch = await env.fetchDevel();

let crumb = await _getCrumb(
cookieJar,
fetch,
// @ts-expect-error: fetchDevel still has no types (yet)
{ devel: true },
Expand All @@ -49,6 +56,7 @@ describe("getCrumb", () => {
// TODO, at tests to see how many times fetch was called, etc.

crumb = await _getCrumb(
cookieJar,
fetch,
// @ts-expect-error: fetchDevel still has no types (yet)
{ devel: true },
Expand All @@ -62,13 +70,13 @@ describe("getCrumb", () => {

await expect(() =>
_getCrumb(
new ExtendedCookieJar(),
fetch,
// @ts-expect-error: fetchDevel still has no types (yet)
{ devel: true },
"https://finance.yahoo.com/quote/AAPL",
"getCrumb-quote-AAPL-no-cookies.fake.json",
true,
new MyCookieJar()
true
)
).rejects.toThrowError(/No set-cookie/);
});
Expand All @@ -78,13 +86,13 @@ describe("getCrumb", () => {

await expect(() =>
_getCrumb(
new ExtendedCookieJar(),
fetch,
// @ts-expect-error: fetchDevel still has no types (yet)
{ devel: true },
"https://finance.yahoo.com/quote/AAPL",
"getCrumb-quote-AAPL-no-context.fake.json",
true,
new MyCookieJar()
true
)
).rejects.toThrowError(/Could not find window.YAHOO.context/);
});
Expand All @@ -94,13 +102,13 @@ describe("getCrumb", () => {

await expect(() =>
_getCrumb(
new ExtendedCookieJar(),
fetch,
// @ts-expect-error: fetchDevel still has no types (yet)
{ devel: true },
"https://finance.yahoo.com/quote/AAPL",
"getCrumb-quote-AAPL-invalid-json.fake.json",
true,
new MyCookieJar()
true
)
).rejects.toThrowError(/Could not parse window.YAHOO.context/);
});
Expand All @@ -110,13 +118,13 @@ describe("getCrumb", () => {

await expect(() =>
_getCrumb(
new ExtendedCookieJar(),
fetch,
// @ts-expect-error: fetchDevel still has no types (yet)
{ devel: true },
"https://finance.yahoo.com/quote/AAPL",
"getCrumb-quote-AAPL-no-crumb.fake.json",
true,
new MyCookieJar()
true
)
).rejects.toThrowError(/Could not find crumb/);
});
Expand All @@ -126,13 +134,13 @@ describe("getCrumb", () => {
const fetch = await env.fetchDevel();

const crumb = await _getCrumb(
new ExtendedCookieJar(),
fetch,
// @ts-expect-error: fetchDevel still has no types (yet)
{ devel: true },
"https://finance.yahoo.com/quote/AAPL",
"getCrumb-quote-AAPL-pre-consent-VPN-UK.json",
true,
new MyCookieJar()
true
);
expect(crumb).toBe("Ky3Po5TGQRZ");
// consoleSilent();
Expand All @@ -141,9 +149,10 @@ describe("getCrumb", () => {

describe("getCrumb", () => {
it("works", async () => {
await getCrumbClear();
await getCrumbClear(cookieJar);
const fetch = await env.fetchDevel();
const crumb = await getCrumb(
cookieJar,
fetch,
// @ts-expect-error: fetchDevel still has no types (yet)
{ devel: true }
Expand All @@ -154,9 +163,10 @@ describe("getCrumb", () => {
it("only calls getCrumb once", async () => {
const fetch = await env.fetchDevel();
const _getCrumb = jest.fn(() => "crumb");
await getCrumbClear();
await getCrumbClear(cookieJar);

getCrumb(
cookieJar,
fetch,
// @ts-expect-error: fetchDevel still has no types (yet)
{ devel: true },
Expand All @@ -165,6 +175,7 @@ describe("getCrumb", () => {
);

getCrumb(
cookieJar,
fetch,
// @ts-expect-error: fetchDevel still has no types (yet)
{ devel: true },
Expand Down
17 changes: 9 additions & 8 deletions src/lib/getCrumb.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import type { RequestInfo, RequestInit, Response } from "node-fetch";
import defaultCookieJar from "./cookieJar.js";
import type { ExtendedCookieJar } from "./cookieJar";

let crumb: string | null = null;
// let crumbFetchTime = 0;
Expand All @@ -11,12 +11,12 @@ const parseHtmlEntities = (str: string) =>
);

export async function _getCrumb(
cookieJar: ExtendedCookieJar,
fetch: (url: RequestInfo, init?: RequestInit) => Promise<Response>,
fetchOptionsBase: RequestInit,
url = "https://finance.yahoo.com/quote/AAPL",
develOverride = "getCrumb-quote-AAPL.json",
noCache = false,
cookieJar = defaultCookieJar
noCache = false
): Promise<string | null> {
// if (crumb && crumbFetchTime + MAX_CRUMB_CACHE_TIME > Date.now()) return crumb;

Expand Down Expand Up @@ -207,12 +207,12 @@ export async function _getCrumb(
*/

return await _getCrumb(
cookieJar,
fetch,
finalResponseFetchOptions,
copyConsentResponseLocation,
"getCrumb-quote-AAPL-consent-final-redirect.html",
noCache,
cookieJar
noCache
);
}
} else {
Expand Down Expand Up @@ -272,14 +272,15 @@ export async function _getCrumb(
let promise: Promise<string | null> | null = null;
let promiseTime = 0;

export async function getCrumbClear() {
export async function getCrumbClear(cookieJar: ExtendedCookieJar) {
crumb = null;
promise = null;
promiseTime = 0;
await defaultCookieJar.removeAllCookies();
await cookieJar.removeAllCookies();
}

export default function getCrumb(
cookieJar: ExtendedCookieJar,
fetch: (url: RequestInfo, init?: RequestInit) => Promise<Response>,
fetchOptionsBase: RequestInit,
url = "https://finance.yahoo.com/quote/AAPL",
Expand All @@ -288,7 +289,7 @@ export default function getCrumb(
// TODO, rather do this with cookie expire time somehow
const now = Date.now();
if (!promise || now - promiseTime > 60_000) {
promise = __getCrumb(fetch, fetchOptionsBase, url);
promise = __getCrumb(cookieJar, fetch, fetchOptionsBase, url);
promiseTime = now;
}

Expand Down
3 changes: 3 additions & 0 deletions src/lib/options.ts
Original file line number Diff line number Diff line change
@@ -1,15 +1,18 @@
// TODO, keep defaults there too?
import type { ValidationOptions } from "./validateAndCoerceTypes.js";
import type { QueueOptions } from "./queue.js";
import { ExtendedCookieJar } from "./cookieJar.js";

export interface YahooFinanceOptions {
YF_QUERY_HOST?: string;
cookieJar: ExtendedCookieJar;
queue?: QueueOptions;
validation?: ValidationOptions;
}

const options: YahooFinanceOptions = {
YF_QUERY_HOST: process.env.YF_QUERY_HOST || "query2.finance.yahoo.com",
cookieJar: new ExtendedCookieJar(),
queue: {
concurrency: 4, // Min: 1, Max: Infinity
timeout: 60,
Expand Down
1 change: 1 addition & 0 deletions src/lib/setGlobalConfig.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ export default function setGlobalConfig(
options: this._opts.validation,
schemaKey: "#/definitions/YahooFinanceOptions",
});
// @ts-expect-error: TODO
mergeObjects(this._opts, config as Obj);
}

Expand Down
17 changes: 11 additions & 6 deletions src/lib/yahooFinanceFetch.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -120,38 +120,42 @@ describe("yahooFinanceFetch", () => {
expect(queue.concurrency).toBe(5);
});

it("yahooFinanceFetch branch check for alternate queue", () => {
it("yahooFinanceFetch branch check for alternate queue", async () => {
const promises = [
yahooFinanceFetch("", {}),
yahooFinanceFetch("", {}, {}),
yahooFinanceFetch("", {}, { queue: {} }),
];

await immediate();

env.fetch.fetches[0].resolveWith({ ok: true });
env.fetch.fetches[1].resolveWith({ ok: true });
env.fetch.fetches[2].resolveWith({ ok: true });

return Promise.all(promises);
await Promise.all(promises);
});

it("assert defualts to {} for empty queue opts", () => {
it("assert defualts to {} for empty queue opts", async () => {
moduleOpts.queue.concurrency = 1;
const opts = { ..._opts };
// @ts-ignore: intentional to test runtime failures
delete opts.queue;
const yahooFinanceFetch = _yahooFinanceFetch.bind({ _env: env, _opts });

const promise = yahooFinanceFetch("", {}, moduleOpts);
await immediate();
env.fetch.fetches[0].resolveWith({ ok: true });
return expect(promise).resolves.toMatchObject({ ok: true });
await expect(promise).resolves.toMatchObject({ ok: true });
});

it("single item in queue", () => {
it("single item in queue", async () => {
moduleOpts.queue.concurrency = 1;

const promise = yahooFinanceFetch("", {}, moduleOpts);
await immediate();
env.fetch.fetches[0].resolveWith({ ok: true });
return expect(promise).resolves.toMatchObject({ ok: true });
await expect(promise).resolves.toMatchObject({ ok: true });
});

it("waits if exceeding concurrency max", async () => {
Expand All @@ -161,6 +165,7 @@ describe("yahooFinanceFetch", () => {
yahooFinanceFetch("", {}, moduleOpts),
yahooFinanceFetch("", {}, moduleOpts),
];
await immediate();

// Second func should not be called until 1st reoslves (limit 1)
expect(env.fetch.fetches.length).toBe(1);
Expand Down
17 changes: 11 additions & 6 deletions src/lib/yahooFinanceFetch.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ import type { QueueOptions } from "./queue.js";
import errors from "./errors.js";
import pkg from "../../package.json";
import getCrumb from "./getCrumb.js";
import cookieJar from "./cookieJar.js";

const userAgent = `${pkg.name}/${pkg.version} (+${pkg.repository})`;

Expand Down Expand Up @@ -62,7 +61,7 @@ function substituteVariables(this: YahooFinanceFetchThis, urlBase: string) {
async function yahooFinanceFetch(
this: YahooFinanceFetchThis,
urlBase: string,
params = {},
params: Record<string, string> = {},
moduleOpts: YahooFinanceFetchModuleOptions = {},
func = "json",
needsCrumb = false
Expand Down Expand Up @@ -93,8 +92,12 @@ async function yahooFinanceFetch(
};

if (needsCrumb) {
// @ts-expect-error: TODO, crumb string type for partial params
params.crumb = await getCrumb(fetchFunc, fetchOptionsBase);
const crumb = await getCrumb(
this._opts.cookieJar,
fetchFunc,
fetchOptionsBase
);
if (crumb) params.crumb = crumb;
}

// @ts-expect-error: TODO copy interface? @types lib?
Expand All @@ -109,7 +112,9 @@ async function yahooFinanceFetch(
...fetchOptionsBase,
headers: {
...fetchOptionsBase.headers,
cookie: cookieJar.getCookieStringSync(url, { allPaths: true }),
cookie: await this._opts.cookieJar.getCookieString(url, {
allPaths: true,
}),
},
};

Expand All @@ -122,7 +127,7 @@ async function yahooFinanceFetch(

const setCookieHeaders = response.headers.raw()["set-cookie"];
if (setCookieHeaders)
cookieJar.setFromSetCookieHeaders(setCookieHeaders, url);
this._opts.cookieJar.setFromSetCookieHeaders(setCookieHeaders, url);

const result = await response[func]();

Expand Down

0 comments on commit b298844

Please sign in to comment.