Abstract fetchJson for data.

ethers-io · Jul 30, 2020 · e2d6f28 · e2d6f28
1 parent ba404ff
commit e2d6f28
Show file tree

Hide file tree

Showing 4 changed files with 101 additions and 55 deletions.
diff --git a/packages/web/src.ts/browser-geturl.ts b/packages/web/src.ts/browser-geturl.ts
@@ -1,17 +1,10 @@
 "use strict";
 
-export type GetUrlResponse = {
-    statusCode: number,
-    statusMessage: string;
-    headers: { [ key: string] : string };
-    body: string;
-};
-
-export type Options = {
-    method?: string,
-    body?: string
-    headers?: { [ key: string] : string },
-};
+import { arrayify } from "@ethersproject/bytes";
+
+import type { GetUrlResponse, Options } from "./types";
+
+export { GetUrlResponse, Options };
 
 export async function getUrl(href: string, options?: Options): Promise<GetUrlResponse> {
     if (options == null) { options = { }; }
@@ -29,7 +22,7 @@ export async function getUrl(href: string, options?: Options): Promise<GetUrlRes
     };
 
     const response = await fetch(href, request);
-    const body = await response.text();
+    const body = await response.arrayBuffer();
 
     const headers: { [ name: string ]: string } = { };
     if (response.headers.forEach) {
@@ -46,6 +39,6 @@ export async function getUrl(href: string, options?: Options): Promise<GetUrlRes
         headers: headers,
         statusCode: response.status,
         statusMessage: response.statusText,
-        body: body,
+        body: arrayify(new Uint8Array(body)),
     }
 }
diff --git a/packages/web/src.ts/geturl.ts b/packages/web/src.ts/geturl.ts
@@ -4,22 +4,15 @@ import http from "http";
 import https from "https";
 import { parse } from "url"
 
+import { concat } from "@ethersproject/bytes";
+
+import type { GetUrlResponse, Options } from "./types";
+
 import { Logger } from "@ethersproject/logger";
 import { version } from "./_version";
 const logger = new Logger(version);
 
-export type GetUrlResponse = {
-    statusCode: number,
-    statusMessage: string;
-    headers: { [ key: string] : string };
-    body: string;
-};
-
-export type Options = {
-    method?: string,
-    body?: string
-    headers?: { [ key: string] : string },
-};
+export { GetUrlResponse, Options };
 
 function getResponse(request: http.ClientRequest): Promise<GetUrlResponse> {
     return new Promise((resolve, reject) => {
@@ -37,11 +30,11 @@ function getResponse(request: http.ClientRequest): Promise<GetUrlResponse> {
                 }, <{ [ name: string ]: string }>{ }),
                 body: null
             };
-            resp.setEncoding("utf8");
+            //resp.setEncoding("utf8");
 
-            resp.on("data", (chunk: string) => {
-                if (response.body == null) { response.body = ""; }
-                response.body += chunk;
+            resp.on("data", (chunk: Uint8Array) => {
+                if (response.body == null) { response.body = new Uint8Array(0); }
+                response.body = concat([ response.body, chunk ]);
             });
 
             resp.on("end", () => {
@@ -100,7 +93,7 @@ export async function getUrl(href: string, options?: Options): Promise<GetUrlRes
     }
 
     if (options.body) {
-        req.write(options.body);
+        req.write(Buffer.from(options.body));
     }
     req.end();
 

diff --git a/packages/web/src.ts/index.ts b/packages/web/src.ts/index.ts
@@ -2,7 +2,7 @@
 
 import { encode as base64Encode } from "@ethersproject/base64";
 import { shallowCopy } from "@ethersproject/properties";
-import { toUtf8Bytes } from "@ethersproject/strings";
+import { toUtf8Bytes, toUtf8String } from "@ethersproject/strings";
 
 import { Logger } from "@ethersproject/logger";
 import { version } from "./_version";
@@ -59,7 +59,7 @@ export type FetchJsonResponse = {
 
 type Header = { key: string, value: string };
 
-export function fetchJson(connection: string | ConnectionInfo, json?: string, processFunc?: (value: any, response: FetchJsonResponse) => any): Promise<any> {
+export function fetchData<T = Uint8Array>(connection: string | ConnectionInfo, body?: Uint8Array, processFunc?: (value: Uint8Array, response: FetchJsonResponse) => T): Promise<T> {
 
     // How many times to retry in the event of a throttle
     const attemptLimit = (typeof(connection) === "object" && connection.throttleLimit != null) ? connection.throttleLimit: 12;
@@ -124,10 +124,12 @@ export function fetchJson(connection: string | ConnectionInfo, json?: string, pr
         }
     }
 
-    if (json) {
+    if (body) {
         options.method = "POST";
-        options.body = json;
-        headers["content-type"] = { key: "Content-Type", value: "application/json" };
+        options.body = body;
+        if (headers["content-type"] == null) {
+            headers["content-type"] = { key: "Content-Type", value: "application/octet-stream" };
+        }
     }
 
     const flatHeaders: { [ key: string ]: string } = { };
@@ -139,7 +141,7 @@ export function fetchJson(connection: string | ConnectionInfo, json?: string, pr
 
     const runningTimeout = (function() {
         let timer: NodeJS.Timer = null;
-        const promise = new Promise(function(resolve, reject) {
+        const promise: Promise<never> = new Promise(function(resolve, reject) {
             if (timeout) {
                 timer = setTimeout(() => {
                     if (timer == null) { return; }
@@ -189,6 +191,7 @@ export function fetchJson(connection: string | ConnectionInfo, json?: string, pr
                             stall = throttleSlotInterval * parseInt(String(Math.random() * Math.pow(2, attempt)));
                         }
 
+                        //console.log("Stalling 429");
                         await staller(stall);
                         continue;
                     }
@@ -225,25 +228,12 @@ export function fetchJson(connection: string | ConnectionInfo, json?: string, pr
                 });
             }
 
-            let json: any = null;
-            if (body != null) {
+            if (processFunc) {
                 try {
-                    json = JSON.parse(body);
-                } catch (error) {
+                    const result = await processFunc(body, response);
                     runningTimeout.cancel();
-                    logger.throwError("invalid JSON", Logger.errors.SERVER_ERROR, {
-                        body: body,
-                        error: error,
-                        requestBody: (options.body || null),
-                        requestMethod: options.method,
-                        url: url
-                    });
-                }
-            }
+                    return result;
 
-            if (processFunc) {
-                try {
-                    json = await processFunc(json, response);
                 } catch (error) {
                     // Allow the processFunc to trigger a throttle
                     if (error.throttleRetry && attempt < attemptLimit) {
@@ -254,14 +244,15 @@ export function fetchJson(connection: string | ConnectionInfo, json?: string, pr
 
                         if (tryAgain) {
                             const timeout = throttleSlotInterval * parseInt(String(Math.random() * Math.pow(2, attempt)));
+                            //console.log("Stalling callback");
                             await staller(timeout);
                             continue;
                         }
                     }
 
                     runningTimeout.cancel();
                     logger.throwError("processing response error", Logger.errors.SERVER_ERROR, {
-                        body: json,
+                        body: body,
                         error: error,
                         requestBody: (options.body || null),
                         requestMethod: options.method,
@@ -271,13 +262,67 @@ export function fetchJson(connection: string | ConnectionInfo, json?: string, pr
             }
 
             runningTimeout.cancel();
-            return json;
+
+            // If we had a processFunc, it eitehr returned a T or threw above.
+            // The "body" is now a Uint8Array.
+            return <T>(<unknown>body);
         }
+
+        return logger.throwError("failed response", Logger.errors.SERVER_ERROR, {
+            requestBody: (options.body || null),
+            requestMethod: options.method,
+            url: url
+        });
     })();
 
     return Promise.race([ runningTimeout.promise, runningFetch ]);
 }
 
+export function fetchJson(connection: string | ConnectionInfo, json?: string, processFunc?: (value: any, response: FetchJsonResponse) => any): Promise<any> {
+    let processJsonFunc = (value: Uint8Array, response: FetchJsonResponse) => {
+        let result: any = null;
+        if (value != null) {
+            try {
+                result = JSON.parse(toUtf8String(value));
+            } catch (error) {
+                logger.throwError("invalid JSON", Logger.errors.SERVER_ERROR, {
+                    body: value,
+                    error: error
+                });
+            }
+        }
+
+        if (processFunc) {
+            result = processFunc(result, response);
+        }
+
+        return result;
+    }
+
+    // If we have json to send, we must
+    // - add content-type of application/json (unless already overridden)
+    // - convert the json to bytes
+    let body: Uint8Array = null;
+    if (json != null) {
+        body = toUtf8Bytes(json);
+
+        // Create a connection with the content-type set for JSON
+        const updated: ConnectionInfo = (typeof(connection) === "string") ? ({ url: connection }): connection;
+        if (updated.headers) {
+            const hasContentType = (Object.keys(updated.headers).filter((k) => (k.toLowerCase() === "content-type")).length) !== 0;
+            if (!hasContentType) {
+                updated.headers = shallowCopy(updated.headers);
+                updated.headers["content-type"] = "application/json";
+            }
+        } else {
+            updated.headers = { "content-type": "application/json" };
+        }
+        connection = updated;
+    }
+
+    return fetchData<any>(connection, body, processJsonFunc);
+}
+
 export function poll<T>(func: () => Promise<T>, options?: PollOptions): Promise<T> {
     if (!options) { options = {}; }
     options = shallowCopy(options);

diff --git a/packages/web/src.ts/types.ts b/packages/web/src.ts/types.ts
@@ -0,0 +1,15 @@
+"use strict";
+
+export type GetUrlResponse = {
+    statusCode: number,
+    statusMessage: string;
+    headers: { [ key: string] : string };
+    body: Uint8Array;
+};
+
+export type Options = {
+    method?: string,
+    body?: Uint8Array
+    headers?: { [ key: string] : string },
+};
+