From 04797f3df1a701d4a66690487db00ed124123407 Mon Sep 17 00:00:00 2001 From: Gadi Cohen Date: Fri, 15 Sep 2023 17:21:05 +0100 Subject: [PATCH] initial work to use file cookie store in cli (#670) --- bin/yahoo-finance.js | 13 +++++++++++++ package.json | 3 ++- schema.json | 3 --- src/lib/options.ts | 2 +- src/lib/setGlobalConfig.ts | 14 ++++++++++++-- src/lib/yahooFinanceFetch.ts | 8 +++++++- yarn.lock | 9 ++++++++- 7 files changed, 43 insertions(+), 9 deletions(-) diff --git a/bin/yahoo-finance.js b/bin/yahoo-finance.js index 488fff6b..8b1fae05 100755 --- a/bin/yahoo-finance.js +++ b/bin/yahoo-finance.js @@ -1,6 +1,15 @@ #!/usr/bin/env -S node --experimental-json-modules --experimental-vm-modules +import os from "os"; +import path from "path"; +import { FileCookieStore } from "tough-cookie-file-store"; import yahooFinance from "../dist/esm/src/index-node.js"; +import { ExtendedCookieJar } from "../dist/esm/src/lib/cookieJar.js"; + +const cookiePath = path.join(os.homedir(), ".yf2-cookies.json"); +const cookieJar = new ExtendedCookieJar(new FileCookieStore(cookiePath)); +// yahooFinance.setGlobalConfig({ cookieJar }); + const moduleNames = Object.keys(yahooFinance).filter((n) => !n.startsWith("_")); // moduleNames.push("_chart"); // modules in development @@ -32,6 +41,8 @@ if (!moduleNames.includes(moduleName)) { process.exit(); } +console.log("Storing cookies in " + cookiePath); + function decodeArgs(stringArgs) { return stringArgs.map((arg) => { if (arg[0] === "{") return JSON.parse(arg); @@ -56,4 +67,6 @@ function decodeArgs(stringArgs) { if (process.stdout.isTTY) console.dir(result, { depth: null, colors: true }); else console.log(JSON.stringify(result, null, 2)); + + console.dir(await yahooFinance[moduleName](...args)); })(); diff --git a/package.json b/package.json index faea7044..1c4d6f2a 100644 --- a/package.json +++ b/package.json @@ -63,7 +63,8 @@ "ajv": "8.10.0", "ajv-formats": "2.1.1", "node-fetch": "^2.6.1", - "tough-cookie": "^4.1.2" + "tough-cookie": "^4.1.2", + "tough-cookie-file-store": "^2.0.3" }, "devDependencies": { "@semantic-release/changelog": "6.0.3", diff --git a/schema.json b/schema.json index 3eedec10..3aa90c15 100644 --- a/schema.json +++ b/schema.json @@ -18,9 +18,6 @@ "$ref": "#/definitions/ValidationOptions" } }, - "required": [ - "cookieJar" - ], "additionalProperties": false }, "ExtendedCookieJar": { diff --git a/src/lib/options.ts b/src/lib/options.ts index 2c06967b..fb5a3b76 100644 --- a/src/lib/options.ts +++ b/src/lib/options.ts @@ -5,7 +5,7 @@ import { ExtendedCookieJar } from "./cookieJar.js"; export interface YahooFinanceOptions { YF_QUERY_HOST?: string; - cookieJar: ExtendedCookieJar; + cookieJar?: ExtendedCookieJar; queue?: QueueOptions; validation?: ValidationOptions; } diff --git a/src/lib/setGlobalConfig.ts b/src/lib/setGlobalConfig.ts index 35698629..10a737d6 100644 --- a/src/lib/setGlobalConfig.ts +++ b/src/lib/setGlobalConfig.ts @@ -1,11 +1,15 @@ import type { YahooFinanceOptions } from "./options.js"; import type { ModuleThis } from "./moduleCommon.js"; import validateAndCoerceTypes from "./validateAndCoerceTypes.js"; +import { ExtendedCookieJar } from "./cookieJar.js"; export default function setGlobalConfig( this: ModuleThis, - config: YahooFinanceOptions + _config: YahooFinanceOptions ): void { + // Instances (e.g. cookieJar) don't validate well :) + const { cookieJar, ...config } = _config; + validateAndCoerceTypes({ object: config, source: "setGlobalConfig", @@ -13,8 +17,14 @@ export default function setGlobalConfig( options: this._opts.validation, schemaKey: "#/definitions/YahooFinanceOptions", }); - // @ts-expect-error: TODO + mergeObjects(this._opts, config as Obj); + + if (cookieJar) { + if (!(cookieJar instanceof ExtendedCookieJar)) + throw new Error("cookieJar must be an instance of ExtendedCookieJar"); + this._opts.cookieJar = cookieJar; + } } type Obj = Record; diff --git a/src/lib/yahooFinanceFetch.ts b/src/lib/yahooFinanceFetch.ts index 9ab77039..ff5016f6 100644 --- a/src/lib/yahooFinanceFetch.ts +++ b/src/lib/yahooFinanceFetch.ts @@ -92,6 +92,8 @@ async function yahooFinanceFetch( }; if (needsCrumb) { + if (!this._opts.cookieJar) throw new Error("No cookieJar set"); + const crumb = await getCrumb( this._opts.cookieJar, fetchFunc, @@ -108,6 +110,8 @@ async function yahooFinanceFetch( // console.log(cookieJar.serializeSync()); + if (!this._opts.cookieJar) throw new Error("No cookieJar set"); + const fetchOptions = { ...fetchOptionsBase, headers: { @@ -126,8 +130,10 @@ async function yahooFinanceFetch( const response = (await queue.add(() => fetchFunc(url, fetchOptions))) as any; const setCookieHeaders = response.headers.raw()["set-cookie"]; - if (setCookieHeaders) + if (setCookieHeaders) { + if (!this._opts.cookieJar) throw new Error("No cookieJar set"); this._opts.cookieJar.setFromSetCookieHeaders(setCookieHeaders, url); + } const result = await response[func](); diff --git a/yarn.lock b/yarn.lock index 30eb1d2d..86095399 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6307,7 +6307,14 @@ to-regex-range@^5.0.1: dependencies: is-number "^7.0.0" -tough-cookie@^4.1.2: +tough-cookie-file-store@^2.0.3: + version "2.0.3" + resolved "https://registry.yarnpkg.com/tough-cookie-file-store/-/tough-cookie-file-store-2.0.3.tgz#788f7a6fe5cd8f61a1afb71b2f0b964ebf914b80" + integrity sha512-sMpZVcmFf6EYFHFFl+SYH4W1/OnXBYMGDsv2IlbQ2caHyFElW/UR/gpj/KYU1JwmP4dE9xqwv2+vWcmlXHojSw== + dependencies: + tough-cookie "^4.0.0" + +tough-cookie@^4.0.0, tough-cookie@^4.1.2: version "4.1.3" resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-4.1.3.tgz#97b9adb0728b42280aa3d814b6b999b2ff0318bf" integrity sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw==