Skip to content

Commit

Permalink
fix(historical): skip null rows
Browse files Browse the repository at this point in the history 
chore(tests): add devel field, complete coverage

chore(historical): update docs to reflect skipping of null rows

chore(historical): use const (not let) in for loops
  • Loading branch information
@gadicc
gadicc committed Jun 14, 2021
1 parent d077680 commit e98e51e
Show file tree
Hide file tree
Showing 5 changed files with 243 additions and 1 deletion.
18 changes: 18 additions & 0 deletions docs/modules/historical.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,3 +59,21 @@ Dates* can be:
### Module Options

See [Common Options](../README.md#common-options).

## Quirks

### Null rows

Yahoo occasionally provides data like this:

```csv
Date,Open,High,Low,Close,Adj Close,Volume
2019-10-08,0.892830,0.899880,0.892200,0.892800,0.892800,0
2019-10-09,null,null,null,null,null,null
```

`node-yahoo-finance2` will silently skip these null rows, so as usual, you
can rely on received input to be well validated. Note: this only happens
when *all* fields are `null`. If you come across a case where only some
fields are null, an error will be thrown - in that case, please let us know
what symbol and date it happened on.
39 changes: 39 additions & 0 deletions src/modules/historical.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import historical from "./historical.js";
import { testSymbols } from "../../tests/symbols.js";

import testYf from "../../tests/testYf.js";
import { consoleSilent, consoleRestore } from "../../tests/console.js";

const yf = testYf({ historical });

Expand Down Expand Up @@ -37,4 +38,42 @@ describe("historical", () => {
expect(options.period2).toBe(Math.floor(now.getTime() / 1000));
});
});

// #208
describe("null values", () => {
it("strips all-null rows", async () => {
const createHistoricalPromise = () =>
yf.historical(
"EURGBP=X",
{
period1: 1567728000,
period2: 1570665600,
},
{ devel: "historical-EURGBP-nulls.json" }
);

await expect(createHistoricalPromise()).resolves.toBeDefined();

const result = await createHistoricalPromise();

// Without stripping, it's about 25 rows.
expect(result.length).toBe(5);

// No need to really check there are no nulls in the data, as
// validation handles that for us automatically.
});

it("throws on a row with some nulls", () => {
consoleSilent();
return expect(
yf
.historical(
"EURGBP=X",
{ period1: 1567728000, period2: 1570665600 },
{ devel: "historical-EURGBP-nulls.fake.json" }
)
.finally(consoleRestore)
).rejects.toThrow("SOME (but not all) null values");
});
});
});
41 changes: 40 additions & 1 deletion src/modules/historical.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ export default function historical(
if (!queryOptions.period2) queryOptions.period2 = new Date();

const dates = ["period1", "period2"] as const;
for (let fieldName of dates) {
for (const fieldName of dates) {
const value = queryOptions[fieldName];
if (value instanceof Date)
queryOptions[fieldName] = Math.floor(value.getTime() / 1000);
Expand All @@ -80,6 +80,45 @@ export default function historical(

result: {
schemaKey: "#/definitions/HistoricalResult",
transformWith(result: any) {
const filteredResults = [];
const fieldCount = Object.keys(result[0]).length;

// Count number of null values in object (1-level deep)
function nullFieldCount(object: Object) {
let nullCount = 0;
for (const val of Object.values(object))
if (val === null) nullCount++;
return nullCount;
}

for (const row of result) {
const nullCount = nullFieldCount(row);

if (nullCount === 0) {
// No nulls is a legit (regular) result
filteredResults.push(row);
} else if (nullCount !== fieldCount - 1 /* skip "date" */) {
// Unhandled case: some but not all values are null.
// Note: no need to check for null "date", validation does it for us
console.error(nullCount, row);
throw new Error(
"Historical returned a result with SOME (but not " +
"all) null values. Please report this, and provide the " +
"query that caused it."
);
} else {
// All fields (except "date") are null: silently skip (no-op)
}
}

/*
* We may consider, for future optimization, to count rows and create
* new array in advance, and skip consecutive blocks of null results.
* Of doubtful utility.
*/
return filteredResults;
},
},

moduleOptions,
Expand Down
73 changes: 73 additions & 0 deletions tests/http/historical-EURGBP-nulls.fake.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
{
"request": {
"url": "https://query1.finance.yahoo.com/v7/finance/download/EURGBP=X?interval=1d&events=history&includeAdjustedClose=true&period1=1567728000&period2=1570665600"
},
"response": {
"ok": true,
"status": 200,
"statusText": "OK",
"headers": {
"content-disposition": [
"attachment; filename=EURGBP=X.csv"
],
"content-type": [
"text/csv;charset=utf-8"
],
"vary": [
"Origin"
],
"cache-control": [
"private, max-age=10, stale-while-revalidate=20"
],
"y-rid": [
"9bb0rftgc6iqg"
],
"x-yahoo-request-id": [
"9bb0rftgc6iqg"
],
"x-request-id": [
"f02f2aa9-70d6-4739-b832-45c66461f72f"
],
"content-length": [
"1151"
],
"x-envoy-upstream-service-time": [
"3"
],
"date": [
"Fri, 11 Jun 2021 11:38:56 GMT"
],
"server": [
"ATS"
],
"x-envoy-decorator-operation": [
"finance-chart-api--mtls-baseline-production-ir2.finance-k8s.svc.yahoo.local:4080/*"
],
"age": [
"0"
],
"strict-transport-security": [
"max-age=15552000"
],
"referrer-policy": [
"no-referrer-when-downgrade"
],
"x-frame-options": [
"SAMEORIGIN"
],
"connection": [
"close"
],
"expect-ct": [
"max-age=31536000, report-uri=\"http://csp.yahoo.com/beacon/csp?src=yahoocom-expect-ct-report-only\""
],
"x-xss-protection": [
"1; mode=block"
],
"x-content-type-options": [
"nosniff"
]
},
"body": "Date,Open,High,Low,Close,Adj Close,Volume\n2019-09-06,0.895210,0.898500,0.894400,0.895200,0.895200,0\n2019-09-09,0.896900,0.901240,0.890670,0.897090,0.897090,0\n2019-09-10,0.894700,0.897260,0.892090,0.894890,0.894890,0\n2019-09-11,null,null,null,null,null,null\n2019-09-12,null,null,null,null,null,null\n2019-09-13,null,null,null,null,null,null\n2019-09-16,null,null,null,null,null,null\n2019-09-17,null,null,null,null,null,null\n2019-09-18,null,null,null,null,null,null\n2019-09-19,null,null,null,0,null,null\n2019-09-20,null,null,null,null,null,null\n2019-09-23,null,null,null,null,null,null\n2019-09-24,null,null,null,null,null,null\n2019-09-25,null,null,null,null,null,null\n2019-09-26,null,null,null,null,null,null\n2019-09-27,null,null,null,null,null,null\n2019-09-30,null,null,null,null,null,null\n2019-10-01,null,null,null,null,null,null\n2019-10-02,null,null,null,null,null,null\n2019-10-03,null,null,null,null,null,null\n2019-10-04,null,null,null,null,null,null\n2019-10-07,null,null,null,null,null,null\n2019-10-08,0.892830,0.899880,0.892200,0.892800,0.892800,0\n2019-10-09,null,null,null,null,null,null\n2019-10-10,0.899390,0.901830,0.889430,0.899470,0.899470,0"
}
}
73 changes: 73 additions & 0 deletions tests/http/historical-EURGBP-nulls.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
{
"request": {
"url": "https://query1.finance.yahoo.com/v7/finance/download/EURGBP=X?interval=1d&events=history&includeAdjustedClose=true&period1=1567728000&period2=1570665600"
},
"response": {
"ok": true,
"status": 200,
"statusText": "OK",
"headers": {
"content-disposition": [
"attachment; filename=EURGBP=X.csv"
],
"content-type": [
"text/csv;charset=utf-8"
],
"vary": [
"Origin"
],
"cache-control": [
"private, max-age=10, stale-while-revalidate=20"
],
"y-rid": [
"45u8milgc6iqg"
],
"x-yahoo-request-id": [
"45u8milgc6iqg"
],
"x-request-id": [
"57b4fbfd-abde-4555-90fc-034f9f82a043"
],
"content-length": [
"1151"
],
"x-envoy-upstream-service-time": [
"4"
],
"date": [
"Fri, 11 Jun 2021 11:38:56 GMT"
],
"server": [
"ATS"
],
"x-envoy-decorator-operation": [
"finance-chart-api--mtls-production-ir2.finance-k8s.svc.yahoo.local:4080/*"
],
"age": [
"0"
],
"strict-transport-security": [
"max-age=15552000"
],
"referrer-policy": [
"no-referrer-when-downgrade"
],
"x-frame-options": [
"SAMEORIGIN"
],
"connection": [
"close"
],
"expect-ct": [
"max-age=31536000, report-uri=\"http://csp.yahoo.com/beacon/csp?src=yahoocom-expect-ct-report-only\""
],
"x-xss-protection": [
"1; mode=block"
],
"x-content-type-options": [
"nosniff"
]
},
"body": "Date,Open,High,Low,Close,Adj Close,Volume\n2019-09-06,0.895210,0.898500,0.894400,0.895200,0.895200,0\n2019-09-09,0.896900,0.901240,0.890670,0.897090,0.897090,0\n2019-09-10,0.894700,0.897260,0.892090,0.894890,0.894890,0\n2019-09-11,null,null,null,null,null,null\n2019-09-12,null,null,null,null,null,null\n2019-09-13,null,null,null,null,null,null\n2019-09-16,null,null,null,null,null,null\n2019-09-17,null,null,null,null,null,null\n2019-09-18,null,null,null,null,null,null\n2019-09-19,null,null,null,null,null,null\n2019-09-20,null,null,null,null,null,null\n2019-09-23,null,null,null,null,null,null\n2019-09-24,null,null,null,null,null,null\n2019-09-25,null,null,null,null,null,null\n2019-09-26,null,null,null,null,null,null\n2019-09-27,null,null,null,null,null,null\n2019-09-30,null,null,null,null,null,null\n2019-10-01,null,null,null,null,null,null\n2019-10-02,null,null,null,null,null,null\n2019-10-03,null,null,null,null,null,null\n2019-10-04,null,null,null,null,null,null\n2019-10-07,null,null,null,null,null,null\n2019-10-08,0.892830,0.899880,0.892200,0.892800,0.892800,0\n2019-10-09,null,null,null,null,null,null\n2019-10-10,0.899390,0.901830,0.889430,0.899470,0.899470,0"
}
}

0 comments on commit e98e51e

Please sign in to comment.