From 03803e4fc49480b2d898e926af01ff272a8995ed Mon Sep 17 00:00:00 2001 From: Eddie Atkinson <45678264+eddie-atkinson@users.noreply.github.com> Date: Sat, 17 Aug 2024 14:14:29 +0800 Subject: [PATCH] remove node-fetch --- package.json | 3 +- src/env-test.ts | 2 -- src/lib/fetchDevel.js | 20 ++++++-------- src/lib/getCrumb.ts | 15 +++++----- src/lib/yahooFinanceFetch.spec.ts | 1 - src/lib/yahooFinanceFetch.ts | 4 +-- yarn.lock | 46 ------------------------------- 7 files changed, 18 insertions(+), 73 deletions(-) diff --git a/package.json b/package.json index ec85fd80..2e793097 100644 --- a/package.json +++ b/package.json @@ -54,7 +54,6 @@ ], "dependencies": { "@sinclair/typebox": "^0.32.27", - "@types/tough-cookie": "^4.0.2", "tough-cookie": "^4.1.2", "tough-cookie-file-store": "^2.0.3" }, @@ -68,7 +67,7 @@ "@tsconfig/node12": "12.1.3", "@types/har-format": "^1.2.15", "@types/jest": "29.5.12", - "@types/node-fetch": "2.6.11", + "@types/tough-cookie": "^4.0.2", "@typescript-eslint/eslint-plugin": "5.62.0", "@typescript-eslint/parser": "5.62.0", "eslint": "8.57.0", diff --git a/src/env-test.ts b/src/env-test.ts index 1f7c04a2..9e10f060 100644 --- a/src/env-test.ts +++ b/src/env-test.ts @@ -1,6 +1,4 @@ import { URLSearchParams } from "url"; -import fetch from "node-fetch"; -import type { RequestInfo, RequestInit, Response } from "node-fetch"; // This let's us still only require the file if we need it, at runtime. let fetchDevelFunc: (url: RequestInfo, init?: RequestInit) => Promise; diff --git a/src/lib/fetchDevel.js b/src/lib/fetchDevel.js index b7e1687a..f22f9ddb 100644 --- a/src/lib/fetchDevel.js +++ b/src/lib/fetchDevel.js @@ -1,5 +1,4 @@ /* istanbul ignore file */ -import nodeFetch, { Headers } from "node-fetch"; import fs from "fs"; import crypto from "crypto"; @@ -11,8 +10,7 @@ class FakeResponse { Object.keys(props).forEach((key) => (this[key] = props[key])); const rawHeaders = this.headers; this.headers = new Headers(rawHeaders); - // node-fetch extension, needed to handle multiple set-cookie headers - this.headers.raw = () => rawHeaders; + this.headers.getSetCookie = () => rawHeaders['set-cookie']; } async json() { @@ -34,7 +32,7 @@ const cache = {}; async function fetchDevel(url, fetchOptions) { if (process.env.FETCH_DEVEL === "nocache") - return await nodeFetch(url, fetchOptions); + return await fetch(url, fetchOptions); // Use query2 for all our tests / fixtures / cache url = url.replace( @@ -70,18 +68,18 @@ async function fetchDevel(url, fetchOptions) { contentObj = JSON.parse(contentJson); } catch (error) { if (error.code === "ENOENT") { - const res = await nodeFetch(origUrl, fetchOptions); - + const res = await fetch(origUrl, fetchOptions); + const {ok, status, statusText, headers} = res + contentObj = { request: { url: url, }, response: { - ok: res.ok, - status: res.status, - statusText: res.statusText, - headers: res.headers.raw(), - // body: await res.text(), + ok, + status, + statusText, + headers, }, }; diff --git a/src/lib/getCrumb.ts b/src/lib/getCrumb.ts index 28ef57ea..b5b54bee 100644 --- a/src/lib/getCrumb.ts +++ b/src/lib/getCrumb.ts @@ -1,4 +1,3 @@ -import type { RequestInfo, RequestInit, Response } from "node-fetch"; import type { ExtendedCookieJar } from "./cookieJar"; // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore: we have to ignore this for csm output. @@ -26,6 +25,7 @@ export async function _getCrumb( develOverride = "getCrumb-quote-AAPL.json", noCache = false, ): Promise { + if (!crumb) { const cookies = await cookieJar.getCookies(CONFIG_FAKE_URL); for (const cookie of cookies) { @@ -69,13 +69,11 @@ export async function _getCrumb( devel: fetchOptionsBase.devel && develOverride, }; - const response = await fetch(url, fetchOptions); - await processSetCookieHeader(response.headers.raw()["set-cookie"], url); - - // logger.debug(response.headers.raw()); - // logger.debug(cookieJar); + const response = await fetch(url, fetchOptions); + await processSetCookieHeader(response.headers.getSetCookie(), url); const location = response.headers.get("location"); + if (location) { if (location.match(/guce.yahoo/)) { const consentFetchOptions: typeof fetchOptions = { @@ -151,7 +149,7 @@ export async function _getCrumb( // Set-Cookie: CFC=AQABCAFkWkdkjEMdLwQ9&s=AQAAAClxdtC-&g=ZFj24w; Expires=Wed, 8 May 2024 01:18:54 GMT; Domain=consent.yahoo.com; Path=/; Secure if ( !(await processSetCookieHeader( - collectConsentSubmitResponse.headers.raw()["set-cookie"], + collectConsentSubmitResponse.headers.getSetCookie(), consentLocation, )) ) @@ -189,7 +187,7 @@ export async function _getCrumb( if ( !(await processSetCookieHeader( - copyConsentResponse.headers.raw()["set-cookie"], + copyConsentResponse.headers.getSetCookie(), collectConsentSubmitResponseLocation, )) ) @@ -238,6 +236,7 @@ export async function _getCrumb( } } + const cookie = (await cookieJar.getCookies(url, { expire: true }))[0]; if (cookie) { logger.debug("Success. Cookie expires on " + cookie.expires); diff --git a/src/lib/yahooFinanceFetch.spec.ts b/src/lib/yahooFinanceFetch.spec.ts index dc07de78..1823b0f3 100644 --- a/src/lib/yahooFinanceFetch.spec.ts +++ b/src/lib/yahooFinanceFetch.spec.ts @@ -1,6 +1,5 @@ import * as util from "util"; import { jest } from "@jest/globals"; -import { Headers } from "node-fetch"; import Queue from "./queue.js"; import _yahooFinanceFetch, { diff --git a/src/lib/yahooFinanceFetch.ts b/src/lib/yahooFinanceFetch.ts index 10020a75..8bd18a89 100644 --- a/src/lib/yahooFinanceFetch.ts +++ b/src/lib/yahooFinanceFetch.ts @@ -1,4 +1,3 @@ -import type { RequestInfo, RequestInit, Response } from "node-fetch"; import Queue from "./queue.js"; import type { YahooFinanceOptions } from "./options.js"; @@ -127,14 +126,13 @@ async function yahooFinanceFetch( }, }; - // console.log("fetch", url, fetchOptions); // used in moduleExec.ts if (func === "csv") func = "text"; const response = (await queue.add(() => fetchFunc(url, fetchOptions))) as any; - const setCookieHeaders = response.headers.raw()["set-cookie"]; + const setCookieHeaders = response.headers.getSetCookie(); if (setCookieHeaders) { if (!this._opts.cookieJar) throw new Error("No cookieJar set"); this._opts.cookieJar.setFromSetCookieHeaders(setCookieHeaders, url); diff --git a/yarn.lock b/yarn.lock index 57f16dd1..61f73a54 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1654,14 +1654,6 @@ resolved "https://registry.yarnpkg.com/@types/minimist/-/minimist-1.2.1.tgz#283f669ff76d7b8260df8ab7a4262cc83d988256" integrity sha512-fZQQafSREFyuZcdWFAExYjBiCL7AUCdgsk80iO0q4yihYYdcIiH28CcuPTGFgLOCC8RlW49GSQxdHwZP+I7CNg== -"@types/node-fetch@2.6.11": - version "2.6.11" - resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.11.tgz#9b39b78665dae0e82a08f02f4967d62c66f95d24" - integrity sha512-24xFj9R5+rfQJLRyM56qh+wnVSYhyXC2tkoBndtY0U+vubqNsYXGjufB2nn8Q6gt0LrARwL6UBtMCSVCwl4B1g== - dependencies: - "@types/node" "*" - form-data "^4.0.0" - "@types/node@*", "@types/node@>= 8": version "14.14.22" resolved "https://registry.yarnpkg.com/@types/node/-/node-14.14.22.tgz#0d29f382472c4ccf3bd96ff0ce47daf5b7b84b18" @@ -2019,11 +2011,6 @@ asap@^2.0.0: resolved "https://registry.yarnpkg.com/asap/-/asap-2.0.6.tgz#e50347611d7e690943208bbdafebcbc2fb866d46" integrity sha1-5QNHYR1+aQlDIIu9r+vLwvuGbUY= -asynckit@^0.4.0: - version "0.4.0" - resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79" - integrity sha1-x57Zf380y48robyXkLzDZkdLS3k= - babel-jest@^29.7.0: version "29.7.0" resolved "https://registry.yarnpkg.com/babel-jest/-/babel-jest-29.7.0.tgz#f4369919225b684c56085998ac63dbd05be020d5" @@ -2431,13 +2418,6 @@ columnify@~1.5.4: strip-ansi "^3.0.0" wcwidth "^1.0.0" -combined-stream@^1.0.8: - version "1.0.8" - resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.8.tgz#c3d45a8b34fd730631a110a8a2520682b31d5a7f" - integrity sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg== - dependencies: - delayed-stream "~1.0.0" - common-ancestor-path@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/common-ancestor-path/-/common-ancestor-path-1.0.1.tgz#4f7d2d1394d91b7abdf51871c62f71eadb0182a7" @@ -2664,11 +2644,6 @@ del@^6.0.0: rimraf "^3.0.2" slash "^3.0.0" -delayed-stream@~1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619" - integrity sha1-3zrhmayt+31ECqrgsp4icrJOxhk= - delegates@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/delegates/-/delegates-1.0.0.tgz#84c6e159b81904fdca59a0ef44cd870d31250f9a" @@ -3132,15 +3107,6 @@ flatted@^3.1.0: resolved "https://registry.yarnpkg.com/flatted/-/flatted-3.1.1.tgz#c4b489e80096d9df1dfc97c79871aea7c617c469" integrity sha512-zAoAQiudy+r5SvnSw3KJy5os/oRJYHzrzja/tBDqrZtNhUw8bt6y8OBzMWcjWr+8liV8Eb6yOhw8WZ7VFZ5ZzA== -form-data@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/form-data/-/form-data-4.0.0.tgz#93919daeaf361ee529584b9b31664dc12c9fa452" - integrity sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww== - dependencies: - asynckit "^0.4.0" - combined-stream "^1.0.8" - mime-types "^2.1.12" - from2@^2.3.0: version "2.3.0" resolved "https://registry.yarnpkg.com/from2/-/from2-2.3.0.tgz#8bfb5502bde4a4d36cfdeea007fcca21d7e382af" @@ -4677,18 +4643,6 @@ micromatch@^4.0.4: braces "^3.0.1" picomatch "^2.2.3" -mime-db@1.45.0: - version "1.45.0" - resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.45.0.tgz#cceeda21ccd7c3a745eba2decd55d4b73e7879ea" - integrity sha512-CkqLUxUk15hofLoLyljJSrukZi8mAtgd+yE5uO4tqRZsdsAJKv0O+rFMhVDRJgozy+yG6md5KwuXhD4ocIoP+w== - -mime-types@^2.1.12: - version "2.1.28" - resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.28.tgz#1160c4757eab2c5363888e005273ecf79d2a0ecd" - integrity sha512-0TO2yJ5YHYr7M2zzT7gDU1tbwHxEUWBCLt0lscSNpcdAfFyJOVEpRYNS7EXVcTLNj/25QO8gulHC5JtTzSE2UQ== - dependencies: - mime-db "1.45.0" - mime@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/mime/-/mime-3.0.0.tgz#b374550dca3a0c18443b0c950a6a58f1931cf7a7"