diff --git a/docs/modules/historical.md b/docs/modules/historical.md index b54d8902..fab1bf0c 100644 --- a/docs/modules/historical.md +++ b/docs/modules/historical.md @@ -59,3 +59,21 @@ Dates* can be: ### Module Options See [Common Options](../README.md#common-options). + +## Quirks + +### Null rows + +Yahoo occasionally provides data like this: + +```csv +Date,Open,High,Low,Close,Adj Close,Volume +2019-10-08,0.892830,0.899880,0.892200,0.892800,0.892800,0 +2019-10-09,null,null,null,null,null,null +``` + +`node-yahoo-finance2` will silently skip these null rows, so as usual, you +can rely on received input to be well validated. Note: this only happens +when *all* fields are `null`. If you come across a case where only some +fields are null, an error will be thrown - in that case, please let us know +what symbol and date it happened on. diff --git a/src/modules/historical.spec.ts b/src/modules/historical.spec.ts index bb8f5c26..97f52c3d 100644 --- a/src/modules/historical.spec.ts +++ b/src/modules/historical.spec.ts @@ -4,6 +4,7 @@ import historical from "./historical.js"; import { testSymbols } from "../../tests/symbols.js"; import testYf from "../../tests/testYf.js"; +import { consoleSilent, consoleRestore } from "../../tests/console.js"; const yf = testYf({ historical }); @@ -37,4 +38,42 @@ describe("historical", () => { expect(options.period2).toBe(Math.floor(now.getTime() / 1000)); }); }); + + // #208 + describe("null values", () => { + it("strips all-null rows", async () => { + const createHistoricalPromise = () => + yf.historical( + "EURGBP=X", + { + period1: 1567728000, + period2: 1570665600, + }, + { devel: "historical-EURGBP-nulls.json" } + ); + + await expect(createHistoricalPromise()).resolves.toBeDefined(); + + const result = await createHistoricalPromise(); + + // Without stripping, it's about 25 rows. + expect(result.length).toBe(5); + + // No need to really check there are no nulls in the data, as + // validation handles that for us automatically. + }); + + it("throws on a row with some nulls", () => { + consoleSilent(); + return expect( + yf + .historical( + "EURGBP=X", + { period1: 1567728000, period2: 1570665600 }, + { devel: "historical-EURGBP-nulls.fake.json" } + ) + .finally(consoleRestore) + ).rejects.toThrow("SOME (but not all) null values"); + }); + }); }); diff --git a/src/modules/historical.ts b/src/modules/historical.ts index b7fb65f9..553d104b 100644 --- a/src/modules/historical.ts +++ b/src/modules/historical.ts @@ -64,7 +64,7 @@ export default function historical( if (!queryOptions.period2) queryOptions.period2 = new Date(); const dates = ["period1", "period2"] as const; - for (let fieldName of dates) { + for (const fieldName of dates) { const value = queryOptions[fieldName]; if (value instanceof Date) queryOptions[fieldName] = Math.floor(value.getTime() / 1000); @@ -80,6 +80,45 @@ export default function historical( result: { schemaKey: "#/definitions/HistoricalResult", + transformWith(result: any) { + const filteredResults = []; + const fieldCount = Object.keys(result[0]).length; + + // Count number of null values in object (1-level deep) + function nullFieldCount(object: Object) { + let nullCount = 0; + for (const val of Object.values(object)) + if (val === null) nullCount++; + return nullCount; + } + + for (const row of result) { + const nullCount = nullFieldCount(row); + + if (nullCount === 0) { + // No nulls is a legit (regular) result + filteredResults.push(row); + } else if (nullCount !== fieldCount - 1 /* skip "date" */) { + // Unhandled case: some but not all values are null. + // Note: no need to check for null "date", validation does it for us + console.error(nullCount, row); + throw new Error( + "Historical returned a result with SOME (but not " + + "all) null values. Please report this, and provide the " + + "query that caused it." + ); + } else { + // All fields (except "date") are null: silently skip (no-op) + } + } + + /* + * We may consider, for future optimization, to count rows and create + * new array in advance, and skip consecutive blocks of null results. + * Of doubtful utility. + */ + return filteredResults; + }, }, moduleOptions, diff --git a/tests/http/historical-EURGBP-nulls.fake.json b/tests/http/historical-EURGBP-nulls.fake.json new file mode 100644 index 00000000..508d28f8 --- /dev/null +++ b/tests/http/historical-EURGBP-nulls.fake.json @@ -0,0 +1,73 @@ +{ + "request": { + "url": "https://query1.finance.yahoo.com/v7/finance/download/EURGBP=X?interval=1d&events=history&includeAdjustedClose=true&period1=1567728000&period2=1570665600" + }, + "response": { + "ok": true, + "status": 200, + "statusText": "OK", + "headers": { + "content-disposition": [ + "attachment; filename=EURGBP=X.csv" + ], + "content-type": [ + "text/csv;charset=utf-8" + ], + "vary": [ + "Origin" + ], + "cache-control": [ + "private, max-age=10, stale-while-revalidate=20" + ], + "y-rid": [ + "9bb0rftgc6iqg" + ], + "x-yahoo-request-id": [ + "9bb0rftgc6iqg" + ], + "x-request-id": [ + "f02f2aa9-70d6-4739-b832-45c66461f72f" + ], + "content-length": [ + "1151" + ], + "x-envoy-upstream-service-time": [ + "3" + ], + "date": [ + "Fri, 11 Jun 2021 11:38:56 GMT" + ], + "server": [ + "ATS" + ], + "x-envoy-decorator-operation": [ + "finance-chart-api--mtls-baseline-production-ir2.finance-k8s.svc.yahoo.local:4080/*" + ], + "age": [ + "0" + ], + "strict-transport-security": [ + "max-age=15552000" + ], + "referrer-policy": [ + "no-referrer-when-downgrade" + ], + "x-frame-options": [ + "SAMEORIGIN" + ], + "connection": [ + "close" + ], + "expect-ct": [ + "max-age=31536000, report-uri=\"http://csp.yahoo.com/beacon/csp?src=yahoocom-expect-ct-report-only\"" + ], + "x-xss-protection": [ + "1; mode=block" + ], + "x-content-type-options": [ + "nosniff" + ] + }, + "body": "Date,Open,High,Low,Close,Adj Close,Volume\n2019-09-06,0.895210,0.898500,0.894400,0.895200,0.895200,0\n2019-09-09,0.896900,0.901240,0.890670,0.897090,0.897090,0\n2019-09-10,0.894700,0.897260,0.892090,0.894890,0.894890,0\n2019-09-11,null,null,null,null,null,null\n2019-09-12,null,null,null,null,null,null\n2019-09-13,null,null,null,null,null,null\n2019-09-16,null,null,null,null,null,null\n2019-09-17,null,null,null,null,null,null\n2019-09-18,null,null,null,null,null,null\n2019-09-19,null,null,null,0,null,null\n2019-09-20,null,null,null,null,null,null\n2019-09-23,null,null,null,null,null,null\n2019-09-24,null,null,null,null,null,null\n2019-09-25,null,null,null,null,null,null\n2019-09-26,null,null,null,null,null,null\n2019-09-27,null,null,null,null,null,null\n2019-09-30,null,null,null,null,null,null\n2019-10-01,null,null,null,null,null,null\n2019-10-02,null,null,null,null,null,null\n2019-10-03,null,null,null,null,null,null\n2019-10-04,null,null,null,null,null,null\n2019-10-07,null,null,null,null,null,null\n2019-10-08,0.892830,0.899880,0.892200,0.892800,0.892800,0\n2019-10-09,null,null,null,null,null,null\n2019-10-10,0.899390,0.901830,0.889430,0.899470,0.899470,0" + } +} diff --git a/tests/http/historical-EURGBP-nulls.json b/tests/http/historical-EURGBP-nulls.json new file mode 100644 index 00000000..c08d9221 --- /dev/null +++ b/tests/http/historical-EURGBP-nulls.json @@ -0,0 +1,73 @@ +{ + "request": { + "url": "https://query1.finance.yahoo.com/v7/finance/download/EURGBP=X?interval=1d&events=history&includeAdjustedClose=true&period1=1567728000&period2=1570665600" + }, + "response": { + "ok": true, + "status": 200, + "statusText": "OK", + "headers": { + "content-disposition": [ + "attachment; filename=EURGBP=X.csv" + ], + "content-type": [ + "text/csv;charset=utf-8" + ], + "vary": [ + "Origin" + ], + "cache-control": [ + "private, max-age=10, stale-while-revalidate=20" + ], + "y-rid": [ + "45u8milgc6iqg" + ], + "x-yahoo-request-id": [ + "45u8milgc6iqg" + ], + "x-request-id": [ + "57b4fbfd-abde-4555-90fc-034f9f82a043" + ], + "content-length": [ + "1151" + ], + "x-envoy-upstream-service-time": [ + "4" + ], + "date": [ + "Fri, 11 Jun 2021 11:38:56 GMT" + ], + "server": [ + "ATS" + ], + "x-envoy-decorator-operation": [ + "finance-chart-api--mtls-production-ir2.finance-k8s.svc.yahoo.local:4080/*" + ], + "age": [ + "0" + ], + "strict-transport-security": [ + "max-age=15552000" + ], + "referrer-policy": [ + "no-referrer-when-downgrade" + ], + "x-frame-options": [ + "SAMEORIGIN" + ], + "connection": [ + "close" + ], + "expect-ct": [ + "max-age=31536000, report-uri=\"http://csp.yahoo.com/beacon/csp?src=yahoocom-expect-ct-report-only\"" + ], + "x-xss-protection": [ + "1; mode=block" + ], + "x-content-type-options": [ + "nosniff" + ] + }, + "body": "Date,Open,High,Low,Close,Adj Close,Volume\n2019-09-06,0.895210,0.898500,0.894400,0.895200,0.895200,0\n2019-09-09,0.896900,0.901240,0.890670,0.897090,0.897090,0\n2019-09-10,0.894700,0.897260,0.892090,0.894890,0.894890,0\n2019-09-11,null,null,null,null,null,null\n2019-09-12,null,null,null,null,null,null\n2019-09-13,null,null,null,null,null,null\n2019-09-16,null,null,null,null,null,null\n2019-09-17,null,null,null,null,null,null\n2019-09-18,null,null,null,null,null,null\n2019-09-19,null,null,null,null,null,null\n2019-09-20,null,null,null,null,null,null\n2019-09-23,null,null,null,null,null,null\n2019-09-24,null,null,null,null,null,null\n2019-09-25,null,null,null,null,null,null\n2019-09-26,null,null,null,null,null,null\n2019-09-27,null,null,null,null,null,null\n2019-09-30,null,null,null,null,null,null\n2019-10-01,null,null,null,null,null,null\n2019-10-02,null,null,null,null,null,null\n2019-10-03,null,null,null,null,null,null\n2019-10-04,null,null,null,null,null,null\n2019-10-07,null,null,null,null,null,null\n2019-10-08,0.892830,0.899880,0.892200,0.892800,0.892800,0\n2019-10-09,null,null,null,null,null,null\n2019-10-10,0.899390,0.901830,0.889430,0.899470,0.899470,0" + } +} \ No newline at end of file