diff --git a/docs/README.md b/docs/README.md index 5fd2441d..8d25d87c 100644 --- a/docs/README.md +++ b/docs/README.md @@ -17,7 +17,21 @@ ## Common Options -Coming soon. +Coming soon. Briefly: + +```js +const queryOpts = {}; // query options specific to the module + +const moduleOpts = { + devel: boolean|string, // see the main README + fetchOptions: {}, // options to pass to fetch + validateResult:boolean, // READ SUPER NB VALIDATION DOC BEFORE TURNING THIS OFF +} + +const result = await yahooFinance.module(query, queryOpts, moduleOpts); +``` + + ## Error Handling and Validation. diff --git a/docs/validation.md b/docs/validation.md index 03b04b9c..2c55bd3c 100644 --- a/docs/validation.md +++ b/docs/validation.md @@ -6,6 +6,7 @@ side-step this if you understand the risks. 1. [Why Validate](#why-validate) 1. [Using Unvalidated Data](#using-unvalidated-data) +1. [Skip Validation Completely](#using-unvalidated-data) 1. [Don't Log Validation Fails](#dont-log-validation-fails) @@ -72,12 +73,33 @@ try { result = error.result; } // and will do my own validation -if (result && isArray(result.Result) && result.Result[0] && typeof result.Result[0].name === 'string') +if (result + && isArray(result.Result) + && result.Result[0] + && typeof result.Result[0].name === 'string') $('input').value(result.Result[0].name); ``` You also have access to `error.errors` which is an array of schema validation errors. You could decide that some errors are ok to ignore but others not. + +## Skip Validation Completely + +Following on from the above, if you really don't care for validation, you can +use the `{ validateResult: false }` module option to prevent throwing errors +altogether on results that don't pass validation. + +```js +const result = await yahooFinance.search('gold', {}, { validateResult: false }); + +if (result + && isArray(result.Result) + && result.Result[0] + && typeof result.Result[0].name === 'string') + $('input').value(result.Result[0].name); + +``` + ## Don't Log Validation Fails diff --git a/src/lib/moduleCommon.ts b/src/lib/moduleCommon.ts new file mode 100644 index 00000000..63cd4839 --- /dev/null +++ b/src/lib/moduleCommon.ts @@ -0,0 +1,15 @@ +export interface ModuleOptions { + validateResult?: boolean; + devel?: boolean | string; + fetchOptions?: object; +} + +export interface ModuleOptionsWithValidateFalse extends ModuleOptions { + validateResult: false; +} + +export interface ModuleOptionsWithValidateTrue extends ModuleOptions { + validateResult?: true; +} + +export type ModuleThis = { [key:string]: any, _moduleExec: Function }; diff --git a/src/lib/moduleExec.ts b/src/lib/moduleExec.ts index 405106a1..efaa91c6 100644 --- a/src/lib/moduleExec.ts +++ b/src/lib/moduleExec.ts @@ -56,10 +56,6 @@ interface ModuleExecOptions { * runtime params. Will be validated with schemaKey. */ overrides: any; - /** - * Any options to pass to fetch() just for this request. - */ - fetchOptions?: any; /** * Called with the merged (defaults,runtime,overrides) before running * the query. Useful to transform options we allow but not Yahoo, e.g. @@ -83,12 +79,24 @@ interface ModuleExecOptions { */ transformWith?: TransformFunc; }; + + moduleOptions?: { + /** + * Allow validation failures to pass if false; + */ + validateResult?: boolean; + /** + * Any options to pass to fetch() just for this request. + */ + fetchOptions?: any; + } } type ThisWithFetch = { [key: string]: any; _moduleExec: Function }; export default async function moduleExec(this: ThisWithFetch, opts: ModuleExecOptions) { const query = opts.query; + const moduleOpts = opts.moduleOptions; // Check that query options passed by the user are valid for this module validateAndCoerceTypes(query.overrides, query.schemaKey, opts.moduleName); @@ -108,7 +116,7 @@ export default async function moduleExec(this: ThisWithFetch, opts: ModuleExecOp queryOptions = query.transformWith(queryOptions); // this._fetch is lib/yahooFinanceFetch - let result = await this._fetch(query.url, queryOptions, query.fetchOptions, query.fetchType); + let result = await this._fetch(query.url, queryOptions, moduleOpts, query.fetchType); if (query.fetchType === 'csv') result = csv2json(result); @@ -137,7 +145,12 @@ export default async function moduleExec(this: ThisWithFetch, opts: ModuleExecOp * The idea is that if you receive a result, it's safe to use / store in * database, etc. Otherwise you'll receive an error. */ - validateAndCoerceTypes(result, opts.result.schemaKey, undefined, this._options?.validation); + try { + validateAndCoerceTypes(result, opts.result.schemaKey, undefined, this._options?.validation); + } catch (error) { + if (!moduleOpts || moduleOpts.validateResult === undefined || moduleOpts.validateResult === true) + throw error; + } return result; } diff --git a/src/lib/yahooFinanceFetch.js b/src/lib/yahooFinanceFetch.js index 8ff54608..ed19e82c 100644 --- a/src/lib/yahooFinanceFetch.js +++ b/src/lib/yahooFinanceFetch.js @@ -3,7 +3,7 @@ const pkg = require('../../package.json'); const userAgent = `${pkg.name}/${pkg.version} (+${pkg.repository})`; -async function yahooFinanceFetch(urlBase, params={}, fetchOptionsOverrides={}, func='json') { +async function yahooFinanceFetch(urlBase, params={}, moduleOpts={}, func='json') { if (!this._env) throw new errors.NoEnvironmentError("yahooFinanceFetch called without this._env set"); @@ -13,13 +13,14 @@ async function yahooFinanceFetch(urlBase, params={}, fetchOptionsOverrides={}, f const url = urlBase + '?' + urlSearchParams.toString(); /* istanbul ignore next */ - const fetchFunc = fetchOptionsOverrides.devel + const fetchFunc = moduleOpts.devel ? require('./fetchDevel') : fetch; // no need to force coverage on real network request. const fetchOptions = { "User-Agent": userAgent, - ...fetchOptionsOverrides + ...moduleOpts.fetchOptions, + devel: moduleOpts.devel, }; // used in moduleExec.ts diff --git a/src/modules/autoc.ts b/src/modules/autoc.ts index 0690d196..27670f85 100644 --- a/src/modules/autoc.ts +++ b/src/modules/autoc.ts @@ -1,3 +1,10 @@ +import type { + ModuleOptions, + ModuleOptionsWithValidateTrue, + ModuleOptionsWithValidateFalse, + ModuleThis, +} from '../lib/moduleCommon'; + export interface AutocResultSet { Query: string; Result: Array @@ -23,11 +30,25 @@ const queryOptionsDefaults = { }; export default function autoc( - this: { [key:string]: any, _moduleExec: Function }, + this: ModuleThis, + query: string, + queryOptionsOverrides?: AutocOptions, + moduleOptions?: ModuleOptionsWithValidateFalse +): Promise; + +export default function autoc( + this: ModuleThis, query: string, queryOptionsOverrides?: AutocOptions, - fetchOptions?: object -): Promise { + moduleOptions?: ModuleOptionsWithValidateTrue +): Promise; + +export default function autoc( + this: ModuleThis, + query: string, + queryOptionsOverrides?: AutocOptions, + moduleOptions?: ModuleOptions +): Promise { return this._moduleExec({ moduleName: "autoc", @@ -38,7 +59,6 @@ export default function autoc( defaults: queryOptionsDefaults, runtime: { query }, overrides: queryOptionsOverrides, - fetchOptions, }, result: { @@ -48,7 +68,9 @@ export default function autoc( throw new Error("Unexpected result: " + JSON.stringify(result)); return result.ResultSet; } - } + }, + + moduleOptions, }); } diff --git a/src/modules/historical.ts b/src/modules/historical.ts index 6dc0cc2a..4940a8cf 100644 --- a/src/modules/historical.ts +++ b/src/modules/historical.ts @@ -1,9 +1,9 @@ -import validateAndCoerceTypes from '../lib/validateAndCoerceTypes'; -import csv2json from '../lib/csv2json'; - -const QUERY_URL = 'https://query1.finance.yahoo.com/v7/finance/download'; -const QUERY_RESULT_SCHEMA_KEY = "#/definitions/HistoricalResult"; -const QUERY_OPTIONS_SCHEMA_KEY = '#/definitions/HistoricalOptions'; +import type { + ModuleOptions, + ModuleOptionsWithValidateTrue, + ModuleOptionsWithValidateFalse, + ModuleThis, +} from '../lib/moduleCommon'; export type HistoricalResult = Array; @@ -32,11 +32,25 @@ const queryOptionsDefaults: Omit = { }; export default function historical( - this: { [key:string]: any, _moduleExec: Function }, + this: ModuleThis, + symbol: string, + queryOptionsOverrides: HistoricalOptions, + moduleOptions?: ModuleOptionsWithValidateFalse +): Promise; + +export default function historical( + this: ModuleThis, symbol: string, queryOptionsOverrides: HistoricalOptions, - fetchOptions?: object -): Promise { + moduleOptions?: ModuleOptionsWithValidateTrue +): Promise; + +export default function historical( + this: ModuleThis, + symbol: string, + queryOptionsOverrides: HistoricalOptions, + moduleOptions?: ModuleOptions +): Promise { return this._moduleExec({ moduleName: "historical", @@ -46,7 +60,6 @@ export default function historical( schemaKey: "#/definitions/HistoricalOptions", defaults: queryOptionsDefaults, overrides: queryOptionsOverrides, - fetchOptions, fetchType: 'csv', transformWith(queryOptions: HistoricalOptions) { if (!queryOptions.period2) @@ -67,6 +80,8 @@ export default function historical( result: { schemaKey: "#/definitions/HistoricalResult", - } + }, + + moduleOptions, }); } diff --git a/src/modules/quoteSummary.ts b/src/modules/quoteSummary.ts index 6fe48caf..7c848441 100644 --- a/src/modules/quoteSummary.ts +++ b/src/modules/quoteSummary.ts @@ -2,6 +2,13 @@ // import QuoteSummaryResult from "QuoteSummaryIfaces"; import { QuoteSummaryResult } from './quoteSummary-iface'; +import type { + ModuleOptions, + ModuleOptionsWithValidateTrue, + ModuleOptionsWithValidateFalse, + ModuleThis, +} from '../lib/moduleCommon'; + export const quoteSummary_modules = [ 'assetProfile', 'balanceSheetHistory', @@ -85,10 +92,24 @@ const queryOptionsDefaults = { }; export default function quoteSummary( - this: { [key:string]: any, _moduleExec: Function }, + this: ModuleThis, + symbol: string, + queryOptionsOverrides?: QuoteSummaryOptions, + moduleOptions?: ModuleOptionsWithValidateFalse +): Promise; + +export default function quoteSummary( + this: ModuleThis, symbol: string, queryOptionsOverrides?: QuoteSummaryOptions, - fetchOptions?: object + moduleOptions?: ModuleOptionsWithValidateTrue +): Promise; + +export default function quoteSummary( + this: ModuleThis, + symbol: string, + queryOptionsOverrides?: QuoteSummaryOptions, + moduleOptions?: ModuleOptions ): Promise { return this._moduleExec({ @@ -99,7 +120,6 @@ export default function quoteSummary( schemaKey: "#/definitions/QuoteSummaryOptions", defaults: queryOptionsDefaults, overrides: queryOptionsOverrides, - fetchOptions, transformWith(options: QuoteSummaryOptions) { if (options.modules === 'all') options.modules = quoteSummary_modules as Array; @@ -115,7 +135,9 @@ export default function quoteSummary( return result.quoteSummary.result[0]; } - } + }, + + moduleOptions, }); } diff --git a/src/modules/search.spec.ts b/src/modules/search.spec.ts index f5197cdb..e42103bf 100644 --- a/src/modules/search.spec.ts +++ b/src/modules/search.spec.ts @@ -35,4 +35,16 @@ describe('search', () => { await expect(rwo({ invalid: true })).rejects.toThrow(InvalidOptionsError) }); + it('throws on unexpected input', async () => { + await expect(yf.search('AAPL', {}, { devel: 'search-fakeBadResult.json' })) + .rejects.toThrow(/Failed Yahoo Schema/) + }); + + it('does not throw on unexpected input if called with {validateResult: false}', async () => { + await expect(yf.search('AAPL', {}, { + devel: 'search-fakeBadResult.json', + validateResult: false + })).resolves.toBeDefined(); + }); + }); diff --git a/src/modules/search.ts b/src/modules/search.ts index 8cca8faa..173a2867 100644 --- a/src/modules/search.ts +++ b/src/modules/search.ts @@ -1,3 +1,10 @@ +import type { + ModuleOptions, + ModuleOptionsWithValidateTrue, + ModuleOptionsWithValidateFalse, + ModuleThis, +} from '../lib/moduleCommon'; + export interface SearchQuoteYahooEquity { exchange: string; // "NYQ" shortname: string; // "Alibaba Group Holding Limited" @@ -82,11 +89,25 @@ const queryOptionsDefaults = { }; export default function search( - this: { [key:string]: any, _moduleExec: Function }, + this: ModuleThis, + query: string, + queryOptionsOverrides?: SearchOptions, + moduleOptions?: ModuleOptionsWithValidateFalse, +): Promise; + +export default function search( + this: ModuleThis, query: string, queryOptionsOverrides?: SearchOptions, - fetchOptions?: object -): Promise { + moduleOptions?: ModuleOptionsWithValidateTrue, +): Promise; + +export default function search( + this: ModuleThis, + query: string, + queryOptionsOverrides?: SearchOptions, + moduleOptions?: ModuleOptions +): Promise { return this._moduleExec({ moduleName: "search", @@ -97,12 +118,13 @@ export default function search( defaults: queryOptionsDefaults, runtime: { q: query }, overrides: queryOptionsOverrides, - fetchOptions, }, result: { schemaKey: "#/definitions/SearchResult", - } + }, + + moduleOptions, }); } diff --git a/tests/http/search-fakeBadResult.json b/tests/http/search-fakeBadResult.json new file mode 100644 index 00000000..ecaf6f41 --- /dev/null +++ b/tests/http/search-fakeBadResult.json @@ -0,0 +1,76 @@ +{ + "request": { + "url": "https://query2.finance.yahoo.com/v1/finance/search?q=AAPL&lang=en-US®ion=US"esCount=6&newsCount=4&enableFuzzyQuery=false"esQueryId=tss_match_phrase_query&multiQuoteQueryId=multi_quote_single_token_query&newsQueryId=news_cie_vespa&enableCb=true&enableNavLinks=true&enableEnhancedTrivialQuery=true" + }, + "response": { + "ok": true, + "status": 200, + "statusText": "OK", + "headers": { + "content-type": [ + "application/json" + ], + "vary": [ + "Origin,Origin,Accept-Encoding" + ], + "cache-control": [ + "public, max-age=120, stale-while-revalidate=180" + ], + "y-rid": [ + "5gn8qn9g1n85v" + ], + "x-yahoo-request-id": [ + "5gn8qn9g1n85v" + ], + "x-request-id": [ + "f8939a5c-6623-4368-8038-ab15baf22750" + ], + "content-encoding": [ + "gzip" + ], + "content-length": [ + "956" + ], + "x-envoy-upstream-service-time": [ + "29" + ], + "date": [ + "Thu, 04 Feb 2021 07:22:39 GMT" + ], + "server": [ + "ATS" + ], + "x-envoy-decorator-operation": [ + "finance-search--mtls-production-ir2.finance-k8s.svc.yahoo.local:4080/*" + ], + "age": [ + "227" + ], + "strict-transport-security": [ + "max-age=15552000" + ], + "warning": [ + "110 Response is stale" + ], + "referrer-policy": [ + "no-referrer-when-downgrade" + ], + "x-frame-options": [ + "SAMEORIGIN" + ], + "connection": [ + "close" + ], + "expect-ct": [ + "max-age=31536000, report-uri=\"http://csp.yahoo.com/beacon/csp?src=yahoocom-expect-ct-report-only\"" + ], + "x-xss-protection": [ + "1; mode=block" + ], + "x-content-type-options": [ + "nosniff" + ] + }, + "body": "{\"explains\":\"SHOULD_NOT_BE_A_STRING\",\"count\":11,\"quotes\":[{\"exchange\":\"NMS\",\"shortname\":\"Apple Inc.\",\"quoteType\":\"EQUITY\",\"symbol\":\"AAPL\",\"index\":\"quotes\",\"score\":2.057644E8,\"typeDisp\":\"Equity\",\"longname\":\"Apple Inc.\",\"isYahooFinance\":true},{\"exchange\":\"MEX\",\"shortname\":\"APPLE INC\",\"quoteType\":\"EQUITY\",\"symbol\":\"AAPL.MX\",\"index\":\"quotes\",\"score\":21483.0,\"typeDisp\":\"Equity\",\"longname\":\"Apple Inc.\",\"isYahooFinance\":true},{\"exchange\":\"OPR\",\"shortname\":\"AAPL Feb 2021 65.000 call\",\"quoteType\":\"OPTION\",\"symbol\":\"AAPL210205C00065000\",\"index\":\"quotes\",\"score\":20467.0,\"typeDisp\":\"Option\",\"isYahooFinance\":true},{\"exchange\":\"OPR\",\"shortname\":\"AAPL Feb 2021 135.000 call\",\"quoteType\":\"OPTION\",\"symbol\":\"AAPL210205C00135000\",\"index\":\"quotes\",\"score\":20346.0,\"typeDisp\":\"Option\",\"isYahooFinance\":true},{\"exchange\":\"BUE\",\"shortname\":\"APPLE INC\",\"quoteType\":\"EQUITY\",\"symbol\":\"AAPL.BA\",\"index\":\"quotes\",\"score\":20209.0,\"typeDisp\":\"Equity\",\"longname\":\"Apple Inc.\",\"isYahooFinance\":true},{\"exchange\":\"OPR\",\"shortname\":\"AAPL Feb 2021 140.000 call\",\"quoteType\":\"OPTION\",\"symbol\":\"AAPL210205C00140000\",\"index\":\"quotes\",\"score\":20172.0,\"typeDisp\":\"Option\",\"isYahooFinance\":true},{\"index\":\"78ddc07626ff4bbcae663e88514c23a0\",\"name\":\"AAPlasma\",\"permalink\":\"aaplasma\",\"isYahooFinance\":false}],\"news\":[{\"uuid\":\"970d97ea-0296-33b5-adf4-04247e2d9ce1\",\"title\":\"Dow Jones Futures: AAPL Rises On Apple Car Buzz, But iPhone Chipmakers Tumble; eBay, PayPal Jump\",\"publisher\":\"Investor's Business Daily\",\"link\":\"https://finance.yahoo.com/m/970d97ea-0296-33b5-adf4-04247e2d9ce1/dow-jones-futures%3A-aapl-rises.html\",\"providerPublishTime\":1612410257,\"type\":\"STORY\"},{\"uuid\":\"d1a1aed0-b2b2-33f0-8514-7126f09fd48b\",\"title\":\"Apple (AAPL) Reducing iPad, iPhone Manufacturing in China\",\"publisher\":\"Investopedia\",\"link\":\"https://finance.yahoo.com/m/d1a1aed0-b2b2-33f0-8514-7126f09fd48b/apple-%28aapl%29-reducing-ipad%2C.html\",\"providerPublishTime\":1612363530,\"type\":\"STORY\"},{\"uuid\":\"03b450c5-7d55-373e-b893-d31bca71750e\",\"title\":\"Apple (AAPL) Sells $14B of Bonds\",\"publisher\":\"Investopedia\",\"link\":\"https://finance.yahoo.com/m/03b450c5-7d55-373e-b893-d31bca71750e/apple-%28aapl%29-sells-%2414b-of.html\",\"providerPublishTime\":1612360942,\"type\":\"STORY\"},{\"uuid\":\"f103ee09-cd0c-309a-84c9-1d62e4209f84\",\"title\":\"Apple (AAPL) Q1 2021 Earnings Call Transcript\",\"publisher\":\"Motley Fool\",\"link\":\"https://finance.yahoo.com/m/f103ee09-cd0c-309a-84c9-1d62e4209f84/apple-%28aapl%29-q1-2021-earnings.html\",\"providerPublishTime\":1611817319,\"type\":\"STORY\"}],\"nav\":[],\"lists\":[],\"researchReports\":[],\"totalTime\":27,\"timeTakenForQuotes\":421,\"timeTakenForNews\":700,\"timeTakenForAlgowatchlist\":400,\"timeTakenForPredefinedScreener\":400,\"timeTakenForCrunchbase\":400,\"timeTakenForNav\":400,\"timeTakenForResearchReports\":0}" + } +}