fix(historical): re-add / adapt old null row filter

gadicc · Sep 16, 2024 · 63c4498 · 63c4498
1 parent 4693dbb
commit 63c4498
Show file tree

Hide file tree

Showing 3 changed files with 119 additions and 1 deletion.
diff --git a/src/modules/historical.spec.ts b/src/modules/historical.spec.ts
@@ -79,6 +79,30 @@ describe("historical", () => {
     );
   });
 
+  /*
+   * {
+   *    "date": "2024-09-13T13:30:00.000Z",
+   *    "high": null,
+   *    "volume": null,
+   *    "open": null,
+   *    "low": null,
+   *    "close": null,
+   *    "adjclose": null
+   * }
+   */
+  it("filters out null rows", async () => {
+    await yf.historical(
+      "0P0000XTS7",
+      {
+        period1: "2024-09-12",
+        period2: "2024-09-16",
+      },
+      {
+        devel: "historical-via-chart-0P0000XTS7-2024-09-12-to-2024-09-16.json",
+      },
+    );
+  });
+
   /*
   // Note: module no longer moduleExec, instead calls chart()
   describe("transformWith", () => {

diff --git a/src/modules/historical.ts b/src/modules/historical.ts
@@ -132,6 +132,16 @@ const queryOptionsDefaults: Omit<HistoricalOptions, "period1"> = {
   includeAdjustedClose: true,
 };
 
+// Count number of null values in object (1-level deep)
+function nullFieldCount(object: unknown) {
+  if (object == null) {
+    return;
+  }
+  let nullCount = 0;
+  for (const val of Object.values(object)) if (val === null) nullCount++;
+  return nullCount;
+}
+
 export default function historical(
   this: ModuleThis,
   symbol: string,
@@ -228,7 +238,27 @@ export default async function historical(
       stockSplits: s.splitRatio,
     }));
   } else {
-    out = result.quotes;
+    out = (result.quotes ?? []).filter((quote) => {
+      const fieldCount = Object.keys(quote).length;
+      const nullCount = nullFieldCount(quote);
+
+      if (nullCount === 0) {
+        // No nulls is a legit (regular) result
+        return true;
+      } else if (nullCount !== fieldCount - 1 /* skip "date" */) {
+        // Unhandled case: some but not all values are null.
+        // Note: no need to check for null "date", validation does it for us
+        console.error(nullCount, quote);
+        throw new Error(
+          "Historical returned a result with SOME (but not " +
+            "all) null values.  Please report this, and provide the " +
+            "query that caused it.",
+        );
+      } else {
+        // All fields (except "date") are null
+        return false;
+      }
+    });
   }
 
   const validateResult =

diff --git a/tests/http/historical-via-chart-0P0000XTS7-2024-09-12-to-2024-09-16.json b/tests/http/historical-via-chart-0P0000XTS7-2024-09-12-to-2024-09-16.json
@@ -0,0 +1,64 @@
+{
+  "request": {
+    "url": "https://query2.finance.yahoo.com/v8/finance/chart/0P0000XTS7?useYfid=true&interval=1d&includePrePost=true&events=&lang=en-US&period1=1726099200&period2=1726444800"
+  },
+  "response": {
+    "ok": true,
+    "status": 200,
+    "statusText": "OK",
+    "headers": {
+      "content-type": [
+        "application/json;charset=utf-8"
+      ],
+      "y-rid": [
+        "6cp5sj5jegm24"
+      ],
+      "vary": [
+        "Origin,Accept-Encoding"
+      ],
+      "cache-control": [
+        "public, max-age=10, stale-while-revalidate=20"
+      ],
+      "content-length": [
+        "1122"
+      ],
+      "x-envoy-upstream-service-time": [
+        "4"
+      ],
+      "date": [
+        "Mon, 16 Sep 2024 16:09:40 GMT"
+      ],
+      "server": [
+        "ATS"
+      ],
+      "x-envoy-decorator-operation": [
+        "finance-chart-api--mtls-production-ir2.finance-k8s.svc.yahoo.local:4080/*"
+      ],
+      "age": [
+        "0"
+      ],
+      "strict-transport-security": [
+        "max-age=31536000"
+      ],
+      "referrer-policy": [
+        "no-referrer-when-downgrade"
+      ],
+      "x-frame-options": [
+        "SAMEORIGIN"
+      ],
+      "connection": [
+        "close"
+      ],
+      "expect-ct": [
+        "max-age=31536000, report-uri=\"http://csp.yahoo.com/beacon/csp?src=yahoocom-expect-ct-report-only\""
+      ],
+      "x-xss-protection": [
+        "1; mode=block"
+      ],
+      "x-content-type-options": [
+        "nosniff"
+      ]
+    },
+    "body": "{\"chart\":{\"result\":[{\"meta\":{\"currency\":\"USD\",\"symbol\":\"0P0000XTS7\",\"exchangeName\":\"PNK\",\"fullExchangeName\":\"OTC Markets OTCPK\",\"instrumentType\":\"MUTUALFUND\",\"firstTradeDate\":1546353000,\"regularMarketTime\":1721419200,\"hasPrePostMarketData\":false,\"gmtoffset\":-14400,\"timezone\":\"EDT\",\"exchangeTimezoneName\":\"America/New_York\",\"regularMarketPrice\":11796.7,\"longName\":\"GS USD Liquid Reserve R Acc\",\"shortName\":\"Goldman Sachs US$ Liquid Reserv\",\"chartPreviousClose\":11885.41,\"priceHint\":2,\"currentTradingPeriod\":{\"pre\":{\"timezone\":\"EDT\",\"start\":1726473600,\"end\":1726493400,\"gmtoffset\":-14400},\"regular\":{\"timezone\":\"EDT\",\"start\":1726493400,\"end\":1726516800,\"gmtoffset\":-14400},\"post\":{\"timezone\":\"EDT\",\"start\":1726516800,\"end\":1726531200,\"gmtoffset\":-14400}},\"dataGranularity\":\"1d\",\"range\":\"\",\"validRanges\":[\"1mo\",\"3mo\",\"6mo\",\"ytd\",\"1y\",\"2y\",\"5y\",\"10y\",\"max\"]},\"timestamp\":[1726147800,1726234200],\"indicators\":{\"quote\":[{\"close\":[11887.099609375,null],\"open\":[11887.099609375,null],\"low\":[11887.099609375,null],\"volume\":[0,null],\"high\":[11887.099609375,null]}],\"adjclose\":[{\"adjclose\":[11887.099609375,null]}]}}],\"error\":null}}"
+  }
+}