diff --git a/jest.config.ts b/jest.config.ts index af2bf3dc..61c8f80e 100644 --- a/jest.config.ts +++ b/jest.config.ts @@ -12,7 +12,8 @@ const config: Config.InitialOptions = { modulePathIgnorePatterns: ["/tests-modules"], extensionsToTreatAsEsm: [".ts"], transform: { - "^.+\\.tsx?$": [ + // to process js/ts with `ts-jest` + "^.+\\.[tj]sx?$": [ "ts-jest", { useESM: true, diff --git a/package.json b/package.json index a184bc96..b18cadef 100644 --- a/package.json +++ b/package.json @@ -61,7 +61,6 @@ "@types/tough-cookie": "^4.0.2", "ajv": "8.10.0", "ajv-formats": "2.1.1", - "node-fetch": "^2.6.1", "tough-cookie": "^4.1.2", "tough-cookie-file-store": "^2.0.3" }, @@ -82,6 +81,8 @@ "eslint-config-prettier": "8.10.0", "globby": "13.2.2", "jest": "29.7.0", + "jest-fetch-mock": "^3.0.3", + "jest-fetch-mock-cache": "^1.5.0", "jest-tobetype": "1.2.3", "oas-schema-walker": "1.1.5", "prettier": "2.8.8", diff --git a/src/env-node.ts b/src/env-node.ts index 681b11d9..944e024c 100644 --- a/src/env-node.ts +++ b/src/env-node.ts @@ -1,5 +1,4 @@ import { URLSearchParams } from "url"; -import fetch from "node-fetch"; export default { fetch, diff --git a/src/env-test.ts b/src/env-test.ts index 1f7c04a2..9e10f060 100644 --- a/src/env-test.ts +++ b/src/env-test.ts @@ -1,6 +1,4 @@ import { URLSearchParams } from "url"; -import fetch from "node-fetch"; -import type { RequestInfo, RequestInit, Response } from "node-fetch"; // This let's us still only require the file if we need it, at runtime. let fetchDevelFunc: (url: RequestInfo, init?: RequestInit) => Promise; diff --git a/src/lib/__mocks__/cookieJar.ts b/src/lib/__mocks__/cookieJar.ts new file mode 100644 index 00000000..63e9e91a --- /dev/null +++ b/src/lib/__mocks__/cookieJar.ts @@ -0,0 +1,10 @@ +import crypto from "crypto"; + +const getCookies = jest.fn(() => { + return [crypto.randomBytes(32).toString("hex")]; +}); + +// ES6 class mock. +export const ExtendedCookieJar = jest.fn().mockImplementation(() => { + return { getCookies }; +}); diff --git a/src/lib/cookieJar.ts b/src/lib/cookieJar.ts index fbfa6fe7..0c3822c3 100644 --- a/src/lib/cookieJar.ts +++ b/src/lib/cookieJar.ts @@ -1,26 +1,23 @@ import { Cookie, CookieJar } from "tough-cookie"; export class ExtendedCookieJar extends CookieJar { - async setFromSetCookieHeaders( - setCookieHeader: string | Array, - url: string - ) { - let cookies; - // console.log("setFromSetCookieHeaders", setCookieHeader); + async setFromHeaders(headers: Headers, url: string): Promise { + const setCookieHeader = headers.get("set-cookie"); + let cookies: (Cookie | undefined)[]; - if (typeof setCookieHeader === "undefined") { - // no-op - } else if (setCookieHeader instanceof Array) { + if (Array.isArray(setCookieHeader)) { cookies = setCookieHeader.map((header) => Cookie.parse(header)); } else if (typeof setCookieHeader === "string") { cookies = [Cookie.parse(setCookieHeader)]; - } + } else return false; - if (cookies) - for (const cookie of cookies) + if (cookies) { + for (const cookie of cookies) { if (cookie instanceof Cookie) { - // console.log("setCookieSync", cookie, url); await this.setCookie(cookie, url); } + } + return true; + } else return false; } } diff --git a/src/lib/fetchDevel.js b/src/lib/fetchDevel.js index d244d39b..700fed2f 100644 --- a/src/lib/fetchDevel.js +++ b/src/lib/fetchDevel.js @@ -1,5 +1,4 @@ /* istanbul ignore file */ -import nodeFetch, { Headers } from "node-fetch"; import fs from "fs"; import crypto from "crypto"; @@ -11,8 +10,6 @@ class FakeResponse { Object.keys(props).forEach((key) => (this[key] = props[key])); const rawHeaders = this.headers; this.headers = new Headers(rawHeaders); - // node-fetch extension, needed to handle multiple set-cookie headers - this.headers.raw = () => rawHeaders; } async json() { @@ -34,7 +31,7 @@ const cache = {}; async function fetchDevel(url, fetchOptions) { if (process.env.FETCH_DEVEL === "nocache") - return await nodeFetch(url, fetchOptions); + return await fetch(url, fetchOptions); // Use query2 for all our tests / fixtures / cache url = url.replace( @@ -70,7 +67,7 @@ async function fetchDevel(url, fetchOptions) { contentObj = JSON.parse(contentJson); } catch (error) { if (error.code === "ENOENT") { - const res = await nodeFetch(origUrl, fetchOptions); + const res = await fetch(origUrl, fetchOptions); contentObj = { request: { diff --git a/src/lib/getCookies.spec.ts b/src/lib/getCookies.spec.ts new file mode 100644 index 00000000..e1322291 --- /dev/null +++ b/src/lib/getCookies.spec.ts @@ -0,0 +1,40 @@ +import crypto from "crypto"; +import { expect, jest } from "@jest/globals"; +import fetchMock from "jest-fetch-mock"; +import { createCachingMock, NodeFSStore } from "jest-fetch-mock-cache"; +import { ExtendedCookieJar } from "./cookieJar"; +import { getCookies } from "./getCookies"; +import options from "./options"; +fetchMock.enableMocks(); + +//jest.mock("./cookieJar"); + +const BASE_URL = "https://finance.yahoo.com"; +const cachingMock = createCachingMock({ store: new NodeFSStore() }); + +describe("getCookies", () => { + const { logger } = options; + let cookieJar: ExtendedCookieJar; + + beforeEach(() => { + fetchMock.mockImplementation(cachingMock); + cookieJar = new ExtendedCookieJar(); + }); + + afterAll(() => { + jest.restoreAllMocks(); + }); + + it("cookieJar with random cookie string", async () => { + jest.spyOn(cookieJar, "getCookies").mockImplementation(() => { + return [crypto.randomBytes(32).toString("hex")]; + }); + const cookies = await getCookies(cookieJar, {}, logger, BASE_URL); + expect(cookieJar.getCookies).toHaveBeenCalledTimes(1); + expect(cookies).toBeTruthy(); + }); + + it("cookieJar without cookies", async () => { + const cookies = await getCookies(cookieJar, {}, logger, BASE_URL); + }); +}); diff --git a/src/lib/getCookies.ts b/src/lib/getCookies.ts new file mode 100644 index 00000000..ad80831b --- /dev/null +++ b/src/lib/getCookies.ts @@ -0,0 +1,58 @@ +import type { ExtendedCookieJar } from "./cookieJar"; +import { Logger } from "./options.js"; + +// Prepare cookies for the API connection. +export async function getCookies( + cookieJar: ExtendedCookieJar, + fetchOptionsBase: RequestInit, + logger: Logger, + url: string +) { + const cookies = await cookieJar.getCookies(url, { + expire: true, + }); + if (cookies.length > 0) return true; + + // This request will get our first cookies, so nothing to send. + logger.debug("Fetching cookies from " + url + "..."); + const fetchOptions: RequestInit = { + ...fetchOptionsBase, + headers: { + ...fetchOptionsBase.headers, + // NB, we won't get a set-cookie header back without this: + accept: "text/html,application/xhtml+xml,application/xml", + }, + redirect: "manual", + }; + + const response = await fetch(url, fetchOptions); + await cookieJar.setFromHeaders(response.headers, url); + + // Redirects when consent is missing. + const location = response.headers.get("location"); + if (location) { + if (location.match(/guce.yahoo/)) { + const consentFetchOptions: RequestInit = { + ...fetchOptions, + headers: { + ...fetchOptions.headers, + // GUCS=XXXXXXXX; Max-Age=1800; Domain=.yahoo.com; Path=/; Secure + cookie: await cookieJar.getCookieString(location), + }, + }; + + // Returns 302 to collectConsent?sessionId=XXX + logger.debug("Fetch", location); + const consentResponse = await fetch(location, consentFetchOptions); + const consentLocation = consentResponse.headers.get("location"); + } else { + throw new Error( + "Unsupported redirect to " + location + ", please report." + ); + } + } else { + logger.debug("Consent is given."); + } + + return true; +} diff --git a/src/lib/getCrumb.ts b/src/lib/getCrumb.ts index f1f1e702..abe142ad 100644 --- a/src/lib/getCrumb.ts +++ b/src/lib/getCrumb.ts @@ -1,4 +1,3 @@ -import type { RequestInfo, RequestInit, Response } from "node-fetch"; import type { ExtendedCookieJar } from "./cookieJar"; import pkg from "../../package.json"; import { Logger } from "./options.js"; @@ -40,17 +39,6 @@ export async function _getCrumb( if (existingCookies.length) return crumb; } - async function processSetCookieHeader( - header: string[] | undefined, - url: string - ) { - if (header) { - await cookieJar.setFromSetCookieHeaders(header, url); - return true; - } - return false; - } - logger.debug("Fetching crumb and cookies from " + url + "..."); const fetchOptions: CrumbOptions = { @@ -67,7 +55,7 @@ export async function _getCrumb( }; const response = await fetch(url, fetchOptions); - await processSetCookieHeader(response.headers.raw()["set-cookie"], url); + await cookieJar.setFromHeaders(response.headers, url); // logger.debug(response.headers.raw()); // logger.debug(cookieJar); @@ -147,8 +135,8 @@ export async function _getCrumb( // Set-Cookie: CFC=AQABCAFkWkdkjEMdLwQ9&s=AQAAAClxdtC-&g=ZFj24w; Expires=Wed, 8 May 2024 01:18:54 GMT; Domain=consent.yahoo.com; Path=/; Secure if ( - !(await processSetCookieHeader( - collectConsentSubmitResponse.headers.raw()["set-cookie"], + !(await cookieJar.setFromHeaders( + collectConsentSubmitResponse.headers, consentLocation )) ) @@ -185,8 +173,8 @@ export async function _getCrumb( ); if ( - !(await processSetCookieHeader( - copyConsentResponse.headers.raw()["set-cookie"], + !(await cookieJar.setFromHeaders( + copyConsentResponse.headers, collectConsentSubmitResponseLocation )) ) diff --git a/src/lib/yahooFinanceFetch.spec.ts b/src/lib/yahooFinanceFetch.spec.ts index 6b6671af..40f2e83d 100644 --- a/src/lib/yahooFinanceFetch.spec.ts +++ b/src/lib/yahooFinanceFetch.spec.ts @@ -1,6 +1,5 @@ import * as util from "util"; import { jest } from "@jest/globals"; -import { Headers } from "node-fetch"; import Queue from "./queue.js"; import _yahooFinanceFetch, { diff --git a/src/lib/yahooFinanceFetch.ts b/src/lib/yahooFinanceFetch.ts index 56aa3126..b4040221 100644 --- a/src/lib/yahooFinanceFetch.ts +++ b/src/lib/yahooFinanceFetch.ts @@ -1,4 +1,3 @@ -import type { RequestInfo, RequestInit, Response } from "node-fetch"; import Queue from "./queue.js"; import type { YahooFinanceOptions } from "./options.js"; @@ -132,11 +131,9 @@ async function yahooFinanceFetch( const response = (await queue.add(() => fetchFunc(url, fetchOptions))) as any; - const setCookieHeaders = response.headers.raw()["set-cookie"]; - if (setCookieHeaders) { - if (!this._opts.cookieJar) throw new Error("No cookieJar set"); - this._opts.cookieJar.setFromSetCookieHeaders(setCookieHeaders, url); - } + if (this._opts.cookieJar) { + await this._opts.cookieJar.setFromHeaders(response.headers, url); + } else throw new Error("No cookieJar set"); const result = await response[func](); diff --git a/tests/__cache__/finance.yahoo.com[headers=93267ff].json b/tests/__cache__/finance.yahoo.com[headers=93267ff].json new file mode 100644 index 00000000..05c980ee --- /dev/null +++ b/tests/__cache__/finance.yahoo.com[headers=93267ff].json @@ -0,0 +1,31 @@ +{ + "request": { + "url": "https://finance.yahoo.com/", + "headers": { + "accept": "text/html,application/xhtml+xml,application/xml" + } + }, + "response": { + "ok": false, + "status": 307, + "statusText": "Temporary Redirect", + "headers": { + "cache-control": "no-store", + "connection": "close", + "content-language": "en", + "content-length": "0", + "content-type": "text/html; charset=utf-8", + "date": "Thu, 18 Apr 2024 13:26:03 GMT", + "expect-ct": "max-age=31536000, report-uri=\"http://csp.yahoo.com/beacon/csp?src=yahoocom-expect-ct-report-only\"", + "location": "https://guce.yahoo.com/consent?brandType=nonEu&gcrumb=S9YkmNI&done=https%3A%2F%2Ffinance.yahoo.com%2F", + "server": "ATS", + "set-cookie": [ + "GUCS=AUvWJJjS; Max-Age=1800; Domain=.yahoo.com; Path=/; Secure" + ], + "strict-transport-security": "max-age=31536000", + "x-content-type-options": "nosniff", + "x-xss-protection": "1; mode=block" + }, + "bodyText": "" + } +} \ No newline at end of file diff --git a/tests/__cache__/guce.yahoo.com!consent!brandType=nonEu&done=https!finance.yahoo.com!&gcrumb=S9YkmNI[headers=a32e5d6].json b/tests/__cache__/guce.yahoo.com!consent!brandType=nonEu&done=https!finance.yahoo.com!&gcrumb=S9YkmNI[headers=a32e5d6].json new file mode 100644 index 00000000..ea8e9127 --- /dev/null +++ b/tests/__cache__/guce.yahoo.com!consent!brandType=nonEu&done=https!finance.yahoo.com!&gcrumb=S9YkmNI[headers=a32e5d6].json @@ -0,0 +1,23 @@ +{ + "request": { + "url": "https://guce.yahoo.com/consent?brandType=nonEu&gcrumb=S9YkmNI&done=https%3A%2F%2Ffinance.yahoo.com%2F", + "headers": { + "accept": "text/html,application/xhtml+xml,application/xml", + "cookie": "GUCS=AUvWJJjS" + } + }, + "response": { + "ok": false, + "status": 302, + "statusText": "Found", + "headers": { + "connection": "close", + "content-length": "0", + "date": "Thu, 18 Apr 2024 13:33:09 GMT", + "location": "https://consent.yahoo.com/v2/collectConsent?sessionId=3_cc-session_de69d6c2-87e4-4586-a751-dd7523728b87", + "server": "guce", + "strict-transport-security": "max-age=31536000; includeSubDomains" + }, + "bodyText": "" + } +} \ No newline at end of file diff --git a/tests/http/getCrumb-getcrumb b/tests/http/getCrumb-getcrumb new file mode 100644 index 00000000..6e1d548d --- /dev/null +++ b/tests/http/getCrumb-getcrumb @@ -0,0 +1,64 @@ +{ + "request": { + "url": "https://query2.finance.yahoo.com/v1/test/getcrumb" + }, + "response": { + "ok": true, + "status": 200, + "statusText": "OK", + "headers": { + "content-type": [ + "text/plain;charset=utf-8" + ], + "access-control-allow-origin": [ + "https://finance.yahoo.com" + ], + "access-control-allow-credentials": [ + "true" + ], + "cache-control": [ + "private, max-age=60, stale-while-revalidate=30" + ], + "vary": [ + "Origin,Accept-Encoding" + ], + "content-length": [ + "11" + ], + "x-envoy-upstream-service-time": [ + "0" + ], + "date": [ + "Sat, 20 Apr 2024 06:39:43 GMT" + ], + "server": [ + "ATS" + ], + "x-envoy-decorator-operation": [ + "finance-external-services-api--mtls-production-ir2.finance-k8s.svc.yahoo.local:4080/*" + ], + "age": [ + "0" + ], + "strict-transport-security": [ + "max-age=31536000" + ], + "referrer-policy": [ + "no-referrer-when-downgrade" + ], + "connection": [ + "close" + ], + "expect-ct": [ + "max-age=31536000, report-uri=\"http://csp.yahoo.com/beacon/csp?src=yahoocom-expect-ct-report-only\"" + ], + "x-xss-protection": [ + "1; mode=block" + ], + "x-content-type-options": [ + "nosniff" + ] + }, + "body": "mloUP8q7ZPH" + } +} \ No newline at end of file