Update test framework

jest.config.ts

@@ -12,7 +12,8 @@ const config: Config.InitialOptions = {
   modulePathIgnorePatterns: ["<rootDir>/tests-modules"],
   extensionsToTreatAsEsm: [".ts"],
   transform: {
-    "^.+\\.tsx?$": [
+    // to process js/ts with `ts-jest`
+    "^.+\\.[tj]sx?$": [
       "ts-jest",
       {
         useESM: true,

package.json

@@ -61,7 +61,6 @@
     "@types/tough-cookie": "^4.0.2",
     "ajv": "8.10.0",
     "ajv-formats": "2.1.1",
-    "node-fetch": "^2.6.1",
     "tough-cookie": "^4.1.2",
     "tough-cookie-file-store": "^2.0.3"
   },
@@ -82,6 +81,8 @@
     "eslint-config-prettier": "8.10.0",
     "globby": "13.2.2",
     "jest": "29.7.0",
+    "jest-fetch-mock": "^3.0.3",
+    "jest-fetch-mock-cache": "^1.5.0",
     "jest-tobetype": "1.2.3",
     "oas-schema-walker": "1.1.5",
     "prettier": "2.8.8",

src/env-node.ts

@@ -1,5 +1,4 @@
 import { URLSearchParams } from "url";
-import fetch from "node-fetch";
 
 export default {
   fetch,

src/env-test.ts

@@ -1,6 +1,4 @@
 import { URLSearchParams } from "url";
-import fetch from "node-fetch";
-import type { RequestInfo, RequestInit, Response } from "node-fetch";
 
 // This let's us still only require the file if we need it, at runtime.
 let fetchDevelFunc: (url: RequestInfo, init?: RequestInit) => Promise<Response>;

src/lib/__mocks__/cookieJar.ts

@@ -0,0 +1,10 @@
+import crypto from "crypto";
+
+const getCookies = jest.fn(() => {
+  return [crypto.randomBytes(32).toString("hex")];
+});
+
+// ES6 class mock.
+export const ExtendedCookieJar = jest.fn().mockImplementation(() => {
+  return { getCookies };
+});

src/lib/cookieJar.ts

@@ -1,26 +1,23 @@
 import { Cookie, CookieJar } from "tough-cookie";
 
 export class ExtendedCookieJar extends CookieJar {
-  async setFromSetCookieHeaders(
-    setCookieHeader: string | Array<string>,
-    url: string
-  ) {
-    let cookies;
-    // console.log("setFromSetCookieHeaders", setCookieHeader);
+  async setFromHeaders(headers: Headers, url: string): Promise<boolean> {
+    const setCookieHeader = headers.get("set-cookie");
+    let cookies: (Cookie | undefined)[];
 
-    if (typeof setCookieHeader === "undefined") {
-      // no-op
-    } else if (setCookieHeader instanceof Array) {
+    if (Array.isArray(setCookieHeader)) {
       cookies = setCookieHeader.map((header) => Cookie.parse(header));
     } else if (typeof setCookieHeader === "string") {
       cookies = [Cookie.parse(setCookieHeader)];
-    }
+    } else return false;
 
-    if (cookies)
-      for (const cookie of cookies)
+    if (cookies) {
+      for (const cookie of cookies) {
         if (cookie instanceof Cookie) {
-          // console.log("setCookieSync", cookie, url);
           await this.setCookie(cookie, url);
         }
+      }
+      return true;
+    } else return false;
   }
 }

src/lib/fetchDevel.js

@@ -1,5 +1,4 @@
 /* istanbul ignore file */
-import nodeFetch, { Headers } from "node-fetch";
 import fs from "fs";
 import crypto from "crypto";
 
@@ -11,8 +10,6 @@ class FakeResponse {
     Object.keys(props).forEach((key) => (this[key] = props[key]));
     const rawHeaders = this.headers;
     this.headers = new Headers(rawHeaders);
-    // node-fetch extension, needed to handle multiple set-cookie headers
-    this.headers.raw = () => rawHeaders;
   }
 
   async json() {
@@ -34,7 +31,7 @@ const cache = {};
 
 async function fetchDevel(url, fetchOptions) {
   if (process.env.FETCH_DEVEL === "nocache")
-    return await nodeFetch(url, fetchOptions);
+    return await fetch(url, fetchOptions);
 
   // Use query2 for all our tests / fixtures / cache
   url = url.replace(
@@ -70,7 +67,7 @@ async function fetchDevel(url, fetchOptions) {
     contentObj = JSON.parse(contentJson);
   } catch (error) {
     if (error.code === "ENOENT") {
-      const res = await nodeFetch(origUrl, fetchOptions);
+      const res = await fetch(origUrl, fetchOptions);
 
       contentObj = {
         request: {

src/lib/getCookies.spec.ts

@@ -0,0 +1,40 @@
+import crypto from "crypto";
+import { expect, jest } from "@jest/globals";
+import fetchMock from "jest-fetch-mock";
+import { createCachingMock, NodeFSStore } from "jest-fetch-mock-cache";
+import { ExtendedCookieJar } from "./cookieJar";
+import { getCookies } from "./getCookies";
+import options from "./options";
+fetchMock.enableMocks();
+
+//jest.mock("./cookieJar");
+
+const BASE_URL = "https://finance.yahoo.com";
+const cachingMock = createCachingMock({ store: new NodeFSStore() });
+
+describe("getCookies", () => {
+  const { logger } = options;
+  let cookieJar: ExtendedCookieJar;
+
+  beforeEach(() => {
+    fetchMock.mockImplementation(cachingMock);
+    cookieJar = new ExtendedCookieJar();
+  });
+
+  afterAll(() => {
+    jest.restoreAllMocks();
+  });
+
+  it("cookieJar with random cookie string", async () => {
+    jest.spyOn(cookieJar, "getCookies").mockImplementation(() => {
+      return [crypto.randomBytes(32).toString("hex")];
+    });
+    const cookies = await getCookies(cookieJar, {}, logger, BASE_URL);
+    expect(cookieJar.getCookies).toHaveBeenCalledTimes(1);
+    expect(cookies).toBeTruthy();
+  });
+
+  it("cookieJar without cookies", async () => {
+    const cookies = await getCookies(cookieJar, {}, logger, BASE_URL);
+  });
+});

src/lib/getCookies.ts

@@ -0,0 +1,58 @@
+import type { ExtendedCookieJar } from "./cookieJar";
+import { Logger } from "./options.js";
+
+// Prepare cookies for the API connection.
+export async function getCookies(
+  cookieJar: ExtendedCookieJar,
+  fetchOptionsBase: RequestInit,
+  logger: Logger,
+  url: string
+) {
+  const cookies = await cookieJar.getCookies(url, {
+    expire: true,
+  });
+  if (cookies.length > 0) return true;
+
+  // This request will get our first cookies, so nothing to send.
+  logger.debug("Fetching cookies from " + url + "...");
+  const fetchOptions: RequestInit = {
+    ...fetchOptionsBase,
+    headers: {
+      ...fetchOptionsBase.headers,
+      // NB, we won't get a set-cookie header back without this:
+      accept: "text/html,application/xhtml+xml,application/xml",
+    },
+    redirect: "manual",
+  };
+
+  const response = await fetch(url, fetchOptions);
+  await cookieJar.setFromHeaders(response.headers, url);
+
+  // Redirects when consent is missing.
+  const location = response.headers.get("location");
+  if (location) {
+    if (location.match(/guce.yahoo/)) {
+      const consentFetchOptions: RequestInit = {
+        ...fetchOptions,
+        headers: {
+          ...fetchOptions.headers,
+          // GUCS=XXXXXXXX; Max-Age=1800; Domain=.yahoo.com; Path=/; Secure
+          cookie: await cookieJar.getCookieString(location),
+        },
+      };
+
+      // Returns 302 to collectConsent?sessionId=XXX
+      logger.debug("Fetch", location);
+      const consentResponse = await fetch(location, consentFetchOptions);
+      const consentLocation = consentResponse.headers.get("location");
+    } else {
+      throw new Error(
+        "Unsupported redirect to " + location + ", please report."
+      );
+    }
+  } else {
+    logger.debug("Consent is given.");
+  }
+
+  return true;
+}

src/lib/getCrumb.ts

@@ -1,4 +1,3 @@
-import type { RequestInfo, RequestInit, Response } from "node-fetch";
 import type { ExtendedCookieJar } from "./cookieJar";
 import pkg from "../../package.json";
 import { Logger } from "./options.js";
@@ -40,17 +39,6 @@ export async function _getCrumb(
     if (existingCookies.length) return crumb;
   }
 
-  async function processSetCookieHeader(
-    header: string[] | undefined,
-    url: string
-  ) {
-    if (header) {
-      await cookieJar.setFromSetCookieHeaders(header, url);
-      return true;
-    }
-    return false;
-  }
-
   logger.debug("Fetching crumb and cookies from " + url + "...");
 
   const fetchOptions: CrumbOptions = {
@@ -67,7 +55,7 @@ export async function _getCrumb(
   };
 
   const response = await fetch(url, fetchOptions);
-  await processSetCookieHeader(response.headers.raw()["set-cookie"], url);
+  await cookieJar.setFromHeaders(response.headers, url);
 
   // logger.debug(response.headers.raw());
   // logger.debug(cookieJar);
@@ -147,8 +135,8 @@ export async function _getCrumb(
 
         // Set-Cookie: CFC=AQABCAFkWkdkjEMdLwQ9&s=AQAAAClxdtC-&g=ZFj24w; Expires=Wed, 8 May 2024 01:18:54 GMT; Domain=consent.yahoo.com; Path=/; Secure
         if (
-          !(await processSetCookieHeader(
-            collectConsentSubmitResponse.headers.raw()["set-cookie"],
+          !(await cookieJar.setFromHeaders(
+            collectConsentSubmitResponse.headers,
             consentLocation
           ))
         )
@@ -185,8 +173,8 @@ export async function _getCrumb(
         );
 
         if (
-          !(await processSetCookieHeader(
-            copyConsentResponse.headers.raw()["set-cookie"],
+          !(await cookieJar.setFromHeaders(
+            copyConsentResponse.headers,
             collectConsentSubmitResponseLocation
           ))
         )

src/lib/yahooFinanceFetch.spec.ts

@@ -1,6 +1,5 @@
 import * as util from "util";
 import { jest } from "@jest/globals";
-import { Headers } from "node-fetch";
 
 import Queue from "./queue.js";
 import _yahooFinanceFetch, {

src/lib/yahooFinanceFetch.ts

@@ -1,4 +1,3 @@
-import type { RequestInfo, RequestInit, Response } from "node-fetch";
 import Queue from "./queue.js";
 
 import type { YahooFinanceOptions } from "./options.js";
@@ -132,11 +131,9 @@ async function yahooFinanceFetch(
 
   const response = (await queue.add(() => fetchFunc(url, fetchOptions))) as any;
 
-  const setCookieHeaders = response.headers.raw()["set-cookie"];
-  if (setCookieHeaders) {
-    if (!this._opts.cookieJar) throw new Error("No cookieJar set");
-    this._opts.cookieJar.setFromSetCookieHeaders(setCookieHeaders, url);
-  }
+  if (this._opts.cookieJar) {
+    await this._opts.cookieJar.setFromHeaders(response.headers, url);
+  } else throw new Error("No cookieJar set");
 
   const result = await response[func]();
 

tests/__cache__/finance.yahoo.com[headers=93267ff].json

@@ -0,0 +1,31 @@
+{
+  "request": {
+    "url": "https://finance.yahoo.com/",
+    "headers": {
+      "accept": "text/html,application/xhtml+xml,application/xml"
+    }
+  },
+  "response": {
+    "ok": false,
+    "status": 307,
+    "statusText": "Temporary Redirect",
+    "headers": {
+      "cache-control": "no-store",
+      "connection": "close",
+      "content-language": "en",
+      "content-length": "0",
+      "content-type": "text/html; charset=utf-8",
+      "date": "Thu, 18 Apr 2024 13:26:03 GMT",
+      "expect-ct": "max-age=31536000, report-uri=\"http://csp.yahoo.com/beacon/csp?src=yahoocom-expect-ct-report-only\"",
+      "location": "https://guce.yahoo.com/consent?brandType=nonEu&gcrumb=S9YkmNI&done=https%3A%2F%2Ffinance.yahoo.com%2F",
+      "server": "ATS",
+      "set-cookie": [
+        "GUCS=AUvWJJjS; Max-Age=1800; Domain=.yahoo.com; Path=/; Secure"
+      ],
+      "strict-transport-security": "max-age=31536000",
+      "x-content-type-options": "nosniff",
+      "x-xss-protection": "1; mode=block"
+    },
+    "bodyText": ""
+  }
+}

tests/__cache__/guce.yahoo.com!consent!brandType=nonEu&done=https!finance.yahoo.com!&gcrumb=S9YkmNI[headers=a32e5d6].json

@@ -0,0 +1,23 @@
+{
+  "request": {
+    "url": "https://guce.yahoo.com/consent?brandType=nonEu&gcrumb=S9YkmNI&done=https%3A%2F%2Ffinance.yahoo.com%2F",
+    "headers": {
+      "accept": "text/html,application/xhtml+xml,application/xml",
+      "cookie": "GUCS=AUvWJJjS"
+    }
+  },
+  "response": {
+    "ok": false,
+    "status": 302,
+    "statusText": "Found",
+    "headers": {
+      "connection": "close",
+      "content-length": "0",
+      "date": "Thu, 18 Apr 2024 13:33:09 GMT",
+      "location": "https://consent.yahoo.com/v2/collectConsent?sessionId=3_cc-session_de69d6c2-87e4-4586-a751-dd7523728b87",
+      "server": "guce",
+      "strict-transport-security": "max-age=31536000; includeSubDomains"
+    },
+    "bodyText": ""
+  }
+}