diff --git a/src/modules/historical.spec.ts b/src/modules/historical.spec.ts index b474ce67..94e948ae 100644 --- a/src/modules/historical.spec.ts +++ b/src/modules/historical.spec.ts @@ -79,6 +79,30 @@ describe("historical", () => { ); }); + /* + * { + * "date": "2024-09-13T13:30:00.000Z", + * "high": null, + * "volume": null, + * "open": null, + * "low": null, + * "close": null, + * "adjclose": null + * } + */ + it("filters out null rows", async () => { + await yf.historical( + "0P0000XTS7", + { + period1: "2024-09-12", + period2: "2024-09-16", + }, + { + devel: "historical-via-chart-0P0000XTS7-2024-09-12-to-2024-09-16.json", + }, + ); + }); + /* // Note: module no longer moduleExec, instead calls chart() describe("transformWith", () => { diff --git a/src/modules/historical.ts b/src/modules/historical.ts index 0b185969..e68e9309 100644 --- a/src/modules/historical.ts +++ b/src/modules/historical.ts @@ -132,6 +132,16 @@ const queryOptionsDefaults: Omit = { includeAdjustedClose: true, }; +// Count number of null values in object (1-level deep) +function nullFieldCount(object: unknown) { + if (object == null) { + return; + } + let nullCount = 0; + for (const val of Object.values(object)) if (val === null) nullCount++; + return nullCount; +} + export default function historical( this: ModuleThis, symbol: string, @@ -228,7 +238,27 @@ export default async function historical( stockSplits: s.splitRatio, })); } else { - out = result.quotes; + out = (result.quotes ?? []).filter((quote) => { + const fieldCount = Object.keys(quote).length; + const nullCount = nullFieldCount(quote); + + if (nullCount === 0) { + // No nulls is a legit (regular) result + return true; + } else if (nullCount !== fieldCount - 1 /* skip "date" */) { + // Unhandled case: some but not all values are null. + // Note: no need to check for null "date", validation does it for us + console.error(nullCount, quote); + throw new Error( + "Historical returned a result with SOME (but not " + + "all) null values. Please report this, and provide the " + + "query that caused it.", + ); + } else { + // All fields (except "date") are null + return false; + } + }); } const validateResult = diff --git a/tests/http/historical-via-chart-0P0000XTS7-2024-09-12-to-2024-09-16.json b/tests/http/historical-via-chart-0P0000XTS7-2024-09-12-to-2024-09-16.json new file mode 100644 index 00000000..e0f12357 --- /dev/null +++ b/tests/http/historical-via-chart-0P0000XTS7-2024-09-12-to-2024-09-16.json @@ -0,0 +1,64 @@ +{ + "request": { + "url": "https://query2.finance.yahoo.com/v8/finance/chart/0P0000XTS7?useYfid=true&interval=1d&includePrePost=true&events=&lang=en-US&period1=1726099200&period2=1726444800" + }, + "response": { + "ok": true, + "status": 200, + "statusText": "OK", + "headers": { + "content-type": [ + "application/json;charset=utf-8" + ], + "y-rid": [ + "6cp5sj5jegm24" + ], + "vary": [ + "Origin,Accept-Encoding" + ], + "cache-control": [ + "public, max-age=10, stale-while-revalidate=20" + ], + "content-length": [ + "1122" + ], + "x-envoy-upstream-service-time": [ + "4" + ], + "date": [ + "Mon, 16 Sep 2024 16:09:40 GMT" + ], + "server": [ + "ATS" + ], + "x-envoy-decorator-operation": [ + "finance-chart-api--mtls-production-ir2.finance-k8s.svc.yahoo.local:4080/*" + ], + "age": [ + "0" + ], + "strict-transport-security": [ + "max-age=31536000" + ], + "referrer-policy": [ + "no-referrer-when-downgrade" + ], + "x-frame-options": [ + "SAMEORIGIN" + ], + "connection": [ + "close" + ], + "expect-ct": [ + "max-age=31536000, report-uri=\"http://csp.yahoo.com/beacon/csp?src=yahoocom-expect-ct-report-only\"" + ], + "x-xss-protection": [ + "1; mode=block" + ], + "x-content-type-options": [ + "nosniff" + ] + }, + "body": "{\"chart\":{\"result\":[{\"meta\":{\"currency\":\"USD\",\"symbol\":\"0P0000XTS7\",\"exchangeName\":\"PNK\",\"fullExchangeName\":\"OTC Markets OTCPK\",\"instrumentType\":\"MUTUALFUND\",\"firstTradeDate\":1546353000,\"regularMarketTime\":1721419200,\"hasPrePostMarketData\":false,\"gmtoffset\":-14400,\"timezone\":\"EDT\",\"exchangeTimezoneName\":\"America/New_York\",\"regularMarketPrice\":11796.7,\"longName\":\"GS USD Liquid Reserve R Acc\",\"shortName\":\"Goldman Sachs US$ Liquid Reserv\",\"chartPreviousClose\":11885.41,\"priceHint\":2,\"currentTradingPeriod\":{\"pre\":{\"timezone\":\"EDT\",\"start\":1726473600,\"end\":1726493400,\"gmtoffset\":-14400},\"regular\":{\"timezone\":\"EDT\",\"start\":1726493400,\"end\":1726516800,\"gmtoffset\":-14400},\"post\":{\"timezone\":\"EDT\",\"start\":1726516800,\"end\":1726531200,\"gmtoffset\":-14400}},\"dataGranularity\":\"1d\",\"range\":\"\",\"validRanges\":[\"1mo\",\"3mo\",\"6mo\",\"ytd\",\"1y\",\"2y\",\"5y\",\"10y\",\"max\"]},\"timestamp\":[1726147800,1726234200],\"indicators\":{\"quote\":[{\"close\":[11887.099609375,null],\"open\":[11887.099609375,null],\"low\":[11887.099609375,null],\"volume\":[0,null],\"high\":[11887.099609375,null]}],\"adjclose\":[{\"adjclose\":[11887.099609375,null]}]}}],\"error\":null}}" + } +} \ No newline at end of file