From d11a6edecc82e697875fcbe1c29cd376c71542e8 Mon Sep 17 00:00:00 2001 From: Jacob Ebey Date: Mon, 11 Apr 2022 14:08:10 -0700 Subject: [PATCH 01/47] feat: replace node-fetch with @web-std/fetch --- integration/file-uploads-test.ts | 1 - integration/multiple-cookies-test.ts | 12 +- .../remix-architect/__tests__/server-test.ts | 145 ++++++++---------- .../remix-express/__tests__/server-test.ts | 122 +++++++-------- packages/remix-express/server.ts | 50 ++++-- .../remix-netlify/__tests__/server-test.ts | 136 ++++++++-------- packages/remix-netlify/server.ts | 52 +++++-- .../__tests__/parseMultipartFormData-test.ts | 7 +- packages/remix-node/fetch.ts | 125 +++++---------- packages/remix-node/index.ts | 6 +- packages/remix-node/package.json | 2 +- packages/remix-node/parseMultipartFormData.ts | 41 +++-- .../remix-node/upload/fileUploadHandler.ts | 4 + packages/remix-server-runtime/data.ts | 2 +- .../remix-vercel/__tests__/server-test.ts | 127 +++++++-------- packages/remix-vercel/server.ts | 46 ++++-- yarn.lock | 41 +++++ 17 files changed, 473 insertions(+), 446 deletions(-) diff --git a/integration/file-uploads-test.ts b/integration/file-uploads-test.ts index f5352cf9cb8..591b1b97960 100644 --- a/integration/file-uploads-test.ts +++ b/integration/file-uploads-test.ts @@ -37,7 +37,6 @@ describe("file-uploads", () => { let formData = await parseMultipartFormData(request, uploadHandler); let file = formData.get("file"); - if (typeof file === "string" || !file) { throw new Error("invalid file type"); } diff --git a/integration/multiple-cookies-test.ts b/integration/multiple-cookies-test.ts index ba0554dbc39..49c768b7568 100644 --- a/integration/multiple-cookies-test.ts +++ b/integration/multiple-cookies-test.ts @@ -64,12 +64,7 @@ describe("pathless layout routes", () => { it("should get multiple cookies from the loader", async () => { await app.goto("/"); - expect(responses[0].headers()["set-cookie"]).toBe( - ` -foo=bar -bar=baz - `.trim() - ); + expect(responses[0].headers()["set-cookie"]).toBe(`foo=bar, bar=baz`); expect(responses).toHaveLength(1); }); @@ -79,10 +74,7 @@ bar=baz await app.page.click("button[type=submit]"); await app.page.waitForSelector(`[data-testid="action-success"]`); expect(responses[0].headers()["set-cookie"]).toBe( - ` -another=one -how-about=two - `.trim() + `another=one, how-about=two` ); // one for the POST and one for the GET expect(responses).toHaveLength(2); diff --git a/packages/remix-architect/__tests__/server-test.ts b/packages/remix-architect/__tests__/server-test.ts index 09b98ad0762..7de8c519003 100644 --- a/packages/remix-architect/__tests__/server-test.ts +++ b/packages/remix-architect/__tests__/server-test.ts @@ -159,8 +159,9 @@ describe("architect createRemixHeaders", () => { describe("creates fetch headers from architect headers", () => { it("handles empty headers", () => { expect(createRemixHeaders({}, undefined)).toMatchInlineSnapshot(` - Headers { - Symbol(map): Object {}, + Headers$1 { + Symbol(query): Array [], + Symbol(context): null, } `); }); @@ -168,12 +169,12 @@ describe("architect createRemixHeaders", () => { it("handles simple headers", () => { expect(createRemixHeaders({ "x-foo": "bar" }, undefined)) .toMatchInlineSnapshot(` - Headers { - Symbol(map): Object { - "x-foo": Array [ - "bar", - ], - }, + Headers$1 { + Symbol(query): Array [ + "x-foo", + "bar", + ], + Symbol(context): null, } `); }); @@ -181,15 +182,14 @@ describe("architect createRemixHeaders", () => { it("handles multiple headers", () => { expect(createRemixHeaders({ "x-foo": "bar", "x-bar": "baz" }, undefined)) .toMatchInlineSnapshot(` - Headers { - Symbol(map): Object { - "x-bar": Array [ - "baz", - ], - "x-foo": Array [ - "bar", - ], - }, + Headers$1 { + Symbol(query): Array [ + "x-foo", + "bar", + "x-bar", + "baz", + ], + Symbol(context): null, } `); }); @@ -197,12 +197,12 @@ describe("architect createRemixHeaders", () => { it("handles headers with multiple values", () => { expect(createRemixHeaders({ "x-foo": "bar, baz" }, undefined)) .toMatchInlineSnapshot(` - Headers { - Symbol(map): Object { - "x-foo": Array [ - "bar, baz", - ], - }, + Headers$1 { + Symbol(query): Array [ + "x-foo", + "bar, baz", + ], + Symbol(context): null, } `); }); @@ -211,15 +211,14 @@ describe("architect createRemixHeaders", () => { expect( createRemixHeaders({ "x-foo": "bar, baz", "x-bar": "baz" }, undefined) ).toMatchInlineSnapshot(` - Headers { - Symbol(map): Object { - "x-bar": Array [ - "baz", - ], - "x-foo": Array [ - "bar, baz", - ], - }, + Headers$1 { + Symbol(query): Array [ + "x-foo", + "bar, baz", + "x-bar", + "baz", + ], + Symbol(context): null, } `); }); @@ -231,15 +230,14 @@ describe("architect createRemixHeaders", () => { "__other=some_other_value", ]) ).toMatchInlineSnapshot(` - Headers { - Symbol(map): Object { - "Cookie": Array [ - "__session=some_value; __other=some_other_value", - ], - "x-something-else": Array [ - "true", - ], - }, + Headers$1 { + Symbol(query): Array [ + "x-something-else", + "true", + "cookie", + "__session=some_value; __other=some_other_value", + ], + Symbol(context): null, } `); }); @@ -261,56 +259,41 @@ describe("architect createRemixRequest", () => { "compress": true, "counter": 0, "follow": 20, + "highWaterMark": 16384, + "insecureHTTPParser": false, "size": 0, - "timeout": 0, Symbol(Body internals): Object { "body": null, + "boundary": null, "disturbed": false, "error": null, + "size": 0, + "type": null, }, Symbol(Request internals): Object { - "headers": Headers { - Symbol(map): Object { - "Cookie": Array [ - "__session=value", - ], - "accept": Array [ - "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", - ], - "accept-encoding": Array [ - "gzip, deflate", - ], - "accept-language": Array [ - "en-US,en;q=0.9", - ], - "host": Array [ - "localhost:3333", - ], - "upgrade-insecure-requests": Array [ - "1", - ], - "user-agent": Array [ - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.0 Safari/605.1.15", - ], - }, + "headers": Headers$1 { + Symbol(query): Array [ + "accept", + "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "accept-encoding", + "gzip, deflate", + "accept-language", + "en-US,en;q=0.9", + "cookie", + "__session=value", + "host", + "localhost:3333", + "upgrade-insecure-requests", + "1", + "user-agent", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.0 Safari/605.1.15", + ], + Symbol(context): null, }, "method": "GET", - "parsedURL": Url { - "auth": null, - "hash": null, - "host": "localhost:3333", - "hostname": "localhost", - "href": "https://localhost:3333/", - "path": "/", - "pathname": "/", - "port": "3333", - "protocol": "https:", - "query": null, - "search": null, - "slashes": true, - }, + "parsedURL": "https://localhost:3333/", "redirect": "follow", - "signal": undefined, + "signal": null, }, } `); diff --git a/packages/remix-express/__tests__/server-test.ts b/packages/remix-express/__tests__/server-test.ts index 3ff8b5733b2..6309c4b52b4 100644 --- a/packages/remix-express/__tests__/server-test.ts +++ b/packages/remix-express/__tests__/server-test.ts @@ -125,9 +125,7 @@ describe("express createRequestHandler", () => { expect(res.headers["x-time-of-year"]).toBe("most wonderful"); expect(res.headers["set-cookie"]).toEqual([ - "first=one; Expires=0; Path=/; HttpOnly; Secure; SameSite=Lax", - "second=two; MaxAge=1209600; Path=/; HttpOnly; Secure; SameSite=Lax", - "third=three; Expires=Wed, 21 Oct 2015 07:28:00 GMT; Path=/; HttpOnly; Secure; SameSite=Lax", + "first=one; Expires=0; Path=/; HttpOnly; Secure; SameSite=Lax, second=two; MaxAge=1209600; Path=/; HttpOnly; Secure; SameSite=Lax, third=three; Expires=Wed, 21 Oct 2015 07:28:00 GMT; Path=/; HttpOnly; Secure; SameSite=Lax", ]); }); }); @@ -137,20 +135,21 @@ describe("express createRemixHeaders", () => { describe("creates fetch headers from express headers", () => { it("handles empty headers", () => { expect(createRemixHeaders({})).toMatchInlineSnapshot(` - Headers { - Symbol(map): Object {}, + Headers$1 { + Symbol(query): Array [], + Symbol(context): null, } `); }); it("handles simple headers", () => { expect(createRemixHeaders({ "x-foo": "bar" })).toMatchInlineSnapshot(` - Headers { - Symbol(map): Object { - "x-foo": Array [ - "bar", - ], - }, + Headers$1 { + Symbol(query): Array [ + "x-foo", + "bar", + ], + Symbol(context): null, } `); }); @@ -158,15 +157,14 @@ describe("express createRemixHeaders", () => { it("handles multiple headers", () => { expect(createRemixHeaders({ "x-foo": "bar", "x-bar": "baz" })) .toMatchInlineSnapshot(` - Headers { - Symbol(map): Object { - "x-bar": Array [ - "baz", - ], - "x-foo": Array [ - "bar", - ], - }, + Headers$1 { + Symbol(query): Array [ + "x-foo", + "bar", + "x-bar", + "baz", + ], + Symbol(context): null, } `); }); @@ -174,12 +172,12 @@ describe("express createRemixHeaders", () => { it("handles headers with multiple values", () => { expect(createRemixHeaders({ "x-foo": "bar, baz" })) .toMatchInlineSnapshot(` - Headers { - Symbol(map): Object { - "x-foo": Array [ - "bar, baz", - ], - }, + Headers$1 { + Symbol(query): Array [ + "x-foo", + "bar, baz", + ], + Symbol(context): null, } `); }); @@ -187,15 +185,14 @@ describe("express createRemixHeaders", () => { it("handles headers with multiple values and multiple headers", () => { expect(createRemixHeaders({ "x-foo": "bar, baz", "x-bar": "baz" })) .toMatchInlineSnapshot(` - Headers { - Symbol(map): Object { - "x-bar": Array [ - "baz", - ], - "x-foo": Array [ - "bar, baz", - ], - }, + Headers$1 { + Symbol(query): Array [ + "x-foo", + "bar, baz", + "x-bar", + "baz", + ], + Symbol(context): null, } `); }); @@ -209,13 +206,14 @@ describe("express createRemixHeaders", () => { ], }) ).toMatchInlineSnapshot(` - Headers { - Symbol(map): Object { - "set-cookie": Array [ - "__session=some_value; Path=/; Secure; HttpOnly; MaxAge=7200; SameSite=Lax", - "__other=some_other_value; Path=/; Secure; HttpOnly; MaxAge=3600; SameSite=Lax", - ], - }, + Headers$1 { + Symbol(query): Array [ + "set-cookie", + "__session=some_value; Path=/; Secure; HttpOnly; MaxAge=7200; SameSite=Lax", + "set-cookie", + "__other=some_other_value; Path=/; Secure; HttpOnly; MaxAge=3600; SameSite=Lax", + ], + Symbol(context): null, } `); }); @@ -242,41 +240,31 @@ describe("express createRemixRequest", () => { "compress": true, "counter": 0, "follow": 20, + "highWaterMark": 16384, + "insecureHTTPParser": false, "size": 0, - "timeout": 0, Symbol(Body internals): Object { "body": null, + "boundary": null, "disturbed": false, "error": null, + "size": 0, + "type": null, }, Symbol(Request internals): Object { - "headers": Headers { - Symbol(map): Object { - "cache-control": Array [ - "max-age=300, s-maxage=3600", - ], - "host": Array [ - "localhost:3000", - ], - }, + "headers": Headers$1 { + Symbol(query): Array [ + "cache-control", + "max-age=300, s-maxage=3600", + "host", + "localhost:3000", + ], + Symbol(context): null, }, "method": "GET", - "parsedURL": Url { - "auth": null, - "hash": null, - "host": "localhost:3000", - "hostname": "localhost", - "href": "http://localhost:3000/foo/bar", - "path": "/foo/bar", - "pathname": "/foo/bar", - "port": "3000", - "protocol": "http:", - "query": null, - "search": null, - "slashes": true, - }, + "parsedURL": "http://localhost:3000/foo/bar", "redirect": "follow", - "signal": undefined, + "signal": null, }, } `); diff --git a/packages/remix-express/server.ts b/packages/remix-express/server.ts index 0f66d0fd434..50b17bca05b 100644 --- a/packages/remix-express/server.ts +++ b/packages/remix-express/server.ts @@ -6,6 +6,7 @@ import type { RequestInit as NodeRequestInit, Response as NodeResponse, } from "@remix-run/node"; +import { ReadableStream } from "@web-std/stream"; import { // This has been added as a global in node 15+ AbortController, @@ -60,10 +61,10 @@ export function createRequestHandler({ ? getLoadContext(req, res) : undefined; - let response = (await handleRequest( + let response = await handleRequest( request as unknown as Request, loadContext - )) as unknown as NodeResponse; + ); sendRemixResponse(res, response, abortController); } catch (error) { @@ -76,7 +77,7 @@ export function createRequestHandler({ export function createRemixHeaders( requestHeaders: express.Request["headers"] -): NodeHeaders { +): Headers { let headers = new NodeHeaders(); for (let [key, values] of Object.entries(requestHeaders)) { @@ -109,7 +110,16 @@ export function createRemixRequest( }; if (req.method !== "GET" && req.method !== "HEAD") { - init.body = req.pipe(new PassThrough({ highWaterMark: 16384 })); + init.body = new ReadableStream({ + start(controller) { + req.on("data", (chunk) => { + controller.enqueue(chunk); + }); + req.on("end", () => { + controller.close(); + }); + }, + }); } return new NodeRequest(url.href, init); @@ -117,27 +127,41 @@ export function createRemixRequest( export function sendRemixResponse( res: express.Response, - nodeResponse: NodeResponse, + nodeResponse: Response, abortController: AbortController ): void { res.statusMessage = nodeResponse.statusText; res.status(nodeResponse.status); - for (let [key, values] of Object.entries(nodeResponse.headers.raw())) { - for (let value of values) { - res.append(key, value); - } + for (let [key, value] of nodeResponse.headers.entries()) { + res.append(key, value); } if (abortController.signal.aborted) { res.set("Connection", "close"); } - if (Buffer.isBuffer(nodeResponse.body)) { - res.end(nodeResponse.body); - } else if (nodeResponse.body?.pipe) { - nodeResponse.body.pipe(res); + if (nodeResponse.body) { + let reader = nodeResponse.body.getReader(); + async function read() { + let { done, value } = await reader.read(); + if (done) { + res.end(value); + return; + } + + res.write(value); + read(); + } + read(); } else { res.end(); } + // if (Buffer.isBuffer(nodeResponse.body)) { + // res.end(nodeResponse.body); + // } else if (nodeResponse.body?.pipe) { + // nodeResponse.body.pipe(res); + // } else { + // res.end(); + // } } diff --git a/packages/remix-netlify/__tests__/server-test.ts b/packages/remix-netlify/__tests__/server-test.ts index f47b1984813..aa2eca3b9f8 100644 --- a/packages/remix-netlify/__tests__/server-test.ts +++ b/packages/remix-netlify/__tests__/server-test.ts @@ -123,13 +123,11 @@ describe("netlify createRequestHandler", () => { await lambdaTester(createRequestHandler({ build: undefined })) .event(createMockEvent({ rawUrl: "http://localhost:3000" })) .expectResolve((res) => { - expect(res.multiValueHeaders["X-Time-Of-Year"]).toEqual([ + expect(res.multiValueHeaders["x-time-of-year"]).toEqual([ "most wonderful", ]); - expect(res.multiValueHeaders["Set-Cookie"]).toEqual([ - "first=one; Expires=0; Path=/; HttpOnly; Secure; SameSite=Lax", - "second=two; MaxAge=1209600; Path=/; HttpOnly; Secure; SameSite=Lax", - "third=three; Expires=Wed, 21 Oct 2015 07:28:00 GMT; Path=/; HttpOnly; Secure; SameSite=Lax", + expect(res.multiValueHeaders["set-cookie"]).toEqual([ + "first=one; Expires=0; Path=/; HttpOnly; Secure; SameSite=Lax, second=two; MaxAge=1209600; Path=/; HttpOnly; Secure; SameSite=Lax, third=three; Expires=Wed, 21 Oct 2015 07:28:00 GMT; Path=/; HttpOnly; Secure; SameSite=Lax", ]); }); }); @@ -140,20 +138,21 @@ describe("netlify createRemixHeaders", () => { describe("creates fetch headers from netlify headers", () => { it("handles empty headers", () => { expect(createRemixHeaders({})).toMatchInlineSnapshot(` - Headers { - Symbol(map): Object {}, + Headers$1 { + Symbol(query): Array [], + Symbol(context): null, } `); }); it("handles simple headers", () => { expect(createRemixHeaders({ "x-foo": ["bar"] })).toMatchInlineSnapshot(` - Headers { - Symbol(map): Object { - "x-foo": Array [ - "bar", - ], - }, + Headers$1 { + Symbol(query): Array [ + "x-foo", + "bar", + ], + Symbol(context): null, } `); }); @@ -161,15 +160,14 @@ describe("netlify createRemixHeaders", () => { it("handles multiple headers", () => { expect(createRemixHeaders({ "x-foo": ["bar"], "x-bar": ["baz"] })) .toMatchInlineSnapshot(` - Headers { - Symbol(map): Object { - "x-bar": Array [ - "baz", - ], - "x-foo": Array [ - "bar", - ], - }, + Headers$1 { + Symbol(query): Array [ + "x-foo", + "bar", + "x-bar", + "baz", + ], + Symbol(context): null, } `); }); @@ -177,13 +175,14 @@ describe("netlify createRemixHeaders", () => { it("handles headers with multiple values", () => { expect(createRemixHeaders({ "x-foo": ["bar", "baz"] })) .toMatchInlineSnapshot(` - Headers { - Symbol(map): Object { - "x-foo": Array [ - "bar", - "baz", - ], - }, + Headers$1 { + Symbol(query): Array [ + "x-foo", + "bar", + "x-foo", + "baz", + ], + Symbol(context): null, } `); }); @@ -191,16 +190,16 @@ describe("netlify createRemixHeaders", () => { it("handles headers with multiple values and multiple headers", () => { expect(createRemixHeaders({ "x-foo": ["bar", "baz"], "x-bar": ["baz"] })) .toMatchInlineSnapshot(` - Headers { - Symbol(map): Object { - "x-bar": Array [ - "baz", - ], - "x-foo": Array [ - "bar", - "baz", - ], - }, + Headers$1 { + Symbol(query): Array [ + "x-foo", + "bar", + "x-foo", + "baz", + "x-bar", + "baz", + ], + Symbol(context): null, } `); }); @@ -212,19 +211,20 @@ describe("netlify createRemixHeaders", () => { "__session=some_value; Path=/; Secure; HttpOnly; MaxAge=7200; SameSite=Lax", "__other=some_other_value; Path=/; Secure; HttpOnly; Expires=Wed, 21 Oct 2015 07:28:00 GMT; SameSite=Lax", ], + "x-something-else": ["true"], }) ).toMatchInlineSnapshot(` - Headers { - Symbol(map): Object { - "Cookie": Array [ - "__session=some_value; Path=/; Secure; HttpOnly; MaxAge=7200; SameSite=Lax", - "__other=some_other_value; Path=/; Secure; HttpOnly; Expires=Wed, 21 Oct 2015 07:28:00 GMT; SameSite=Lax", - ], - "x-something-else": Array [ - "true", - ], - }, + Headers$1 { + Symbol(query): Array [ + "cookie", + "__session=some_value; Path=/; Secure; HttpOnly; MaxAge=7200; SameSite=Lax", + "cookie", + "__other=some_other_value; Path=/; Secure; HttpOnly; Expires=Wed, 21 Oct 2015 07:28:00 GMT; SameSite=Lax", + "x-something-else", + "true", + ], + Symbol(context): null, } `); }); @@ -248,39 +248,31 @@ describe("netlify createRemixRequest", () => { "compress": true, "counter": 0, "follow": 20, + "highWaterMark": 16384, + "insecureHTTPParser": false, "size": 0, - "timeout": 0, Symbol(Body internals): Object { "body": null, + "boundary": null, "disturbed": false, "error": null, + "size": 0, + "type": null, }, Symbol(Request internals): Object { - "headers": Headers { - Symbol(map): Object { - "Cookie": Array [ - "__session=value", - "__other=value", - ], - }, + "headers": Headers$1 { + Symbol(query): Array [ + "cookie", + "__session=value", + "cookie", + "__other=value", + ], + Symbol(context): null, }, "method": "GET", - "parsedURL": Url { - "auth": null, - "hash": null, - "host": "localhost:3000", - "hostname": "localhost", - "href": "http://localhost:3000/", - "path": "/", - "pathname": "/", - "port": "3000", - "protocol": "http:", - "query": null, - "search": null, - "slashes": true, - }, + "parsedURL": "http://localhost:3000/", "redirect": "follow", - "signal": undefined, + "signal": null, }, } `); diff --git a/packages/remix-netlify/server.ts b/packages/remix-netlify/server.ts index f1a48aaa319..1e0f6f89e81 100644 --- a/packages/remix-netlify/server.ts +++ b/packages/remix-netlify/server.ts @@ -53,10 +53,10 @@ export function createRequestHandler({ ? getLoadContext(event, context) : undefined; - let response = (await handleRequest( + let response = await handleRequest( request as unknown as Request, loadContext - )) as unknown as NodeResponse; + ); return sendRemixResponse(response, abortController); }; @@ -99,7 +99,7 @@ export function createRemixRequest( export function createRemixHeaders( requestHeaders: HandlerEvent["multiValueHeaders"] -): NodeHeaders { +): Headers { let headers = new NodeHeaders(); for (let [key, values] of Object.entries(requestHeaders)) { @@ -139,7 +139,7 @@ function getRawPath(event: HandlerEvent): string { } export async function sendRemixResponse( - nodeResponse: NodeResponse, + nodeResponse: Response, abortController: AbortController ): Promise { if (abortController.signal.aborted) { @@ -147,21 +147,43 @@ export async function sendRemixResponse( } let contentType = nodeResponse.headers.get("Content-Type"); - let isBinary = isBinaryType(contentType); - let body; - let isBase64Encoded = false; - - if (isBinary) { - let blob = await nodeResponse.arrayBuffer(); - body = Buffer.from(blob).toString("base64"); - isBase64Encoded = true; - } else { - body = await nodeResponse.text(); + let body: string | undefined; + let isBase64Encoded = isBinaryType(contentType); + + if (nodeResponse.body) { + if (isBase64Encoded) { + let reader = nodeResponse.body.getReader(); + body = ""; + async function read() { + let { done, value } = await reader.read(); + if (done) { + return; + } else if (value) { + body += Buffer.from(value).toString("base64"); + } + await read(); + } + + await read(); + } else { + body = await nodeResponse.text(); + } } + let multiValueHeaders: Record = {}; + for (let [key, value] of nodeResponse.headers) { + if (typeof multiValueHeaders[key] === "undefined") { + multiValueHeaders[key] = [value]; + } else { + (multiValueHeaders[key] as string[]).push(value); + } + } + + console.log({ multiValueHeaders }); + return { statusCode: nodeResponse.status, - multiValueHeaders: nodeResponse.headers.raw(), + multiValueHeaders, body, isBase64Encoded, }; diff --git a/packages/remix-node/__tests__/parseMultipartFormData-test.ts b/packages/remix-node/__tests__/parseMultipartFormData-test.ts index 7cd02866d1f..4ec7f8939ec 100644 --- a/packages/remix-node/__tests__/parseMultipartFormData-test.ts +++ b/packages/remix-node/__tests__/parseMultipartFormData-test.ts @@ -1,7 +1,8 @@ +import { FormData as NodeFormData } from "@web-std/fetch"; import { Blob, File } from "@web-std/file"; import { Request as NodeRequest } from "../fetch"; -import { FormData as NodeFormData } from "../formData"; +// import { FormData as NodeFormData } from "../formData"; import { internalParseFormData } from "../parseMultipartFormData"; import { createMemoryUploadHandler } from "../upload/memoryUploadHandler"; @@ -19,8 +20,8 @@ describe("internalParseFormData", () => { let uploadHandler = createMemoryUploadHandler({}); let parsedFormData = await internalParseFormData( - req.headers.get("Content-Type"), - req.body as any, + req, + req.formData, undefined, uploadHandler ); diff --git a/packages/remix-node/fetch.ts b/packages/remix-node/fetch.ts index d6f2dc7419c..cb95d6ce396 100644 --- a/packages/remix-node/fetch.ts +++ b/packages/remix-node/fetch.ts @@ -1,67 +1,19 @@ import type { Readable } from "stream"; import { PassThrough } from "stream"; import type AbortController from "abort-controller"; -import FormStream from "form-data"; -import type { RequestInfo, RequestInit, Response } from "node-fetch"; -import nodeFetch, { Request as BaseNodeRequest } from "node-fetch"; +// import FormStream from "form-data"; +// import type { RequestInfo, RequestInit, Response } from "node-fetch"; +// import nodeFetch, { Request as BaseNodeRequest } from "node-fetch"; +import { Request as BaseNodeRequest } from "@web-std/fetch"; -import { FormData as NodeFormData, isFile } from "./formData"; +// import { FormData as NodeFormData, isFile } from "./formData"; import type { UploadHandler } from "./formData"; import { internalParseFormData } from "./parseMultipartFormData"; -export type { HeadersInit, RequestInfo, ResponseInit } from "node-fetch"; -export { Headers, Response } from "node-fetch"; +// export type { HeadersInit, RequestInfo, ResponseInit } from "node-fetch"; +// export { Headers, Response } from "node-fetch"; -function formDataToStream(formData: NodeFormData): FormStream { - let formStream = new FormStream(); - - function toNodeStream(input: any) { - // The input is either a Node stream or a web stream, if it has - // a `on` method it's a node stream so we can just return it - if (typeof input?.on === "function") { - return input; - } - - let passthrough = new PassThrough(); - let stream = input as ReadableStream; - let reader = stream.getReader(); - reader - .read() - .then(async ({ done, value }) => { - while (!done) { - passthrough.push(value); - ({ done, value } = await reader.read()); - } - passthrough.push(null); - }) - .catch((error) => { - passthrough.emit("error", error); - }); - - return passthrough; - } - - for (let [key, value] of formData.entries()) { - if (typeof value === "string") { - formStream.append(key, value); - } else if (isFile(value)) { - let stream = toNodeStream(value.stream()); - formStream.append(key, stream, { - filename: value.name, - contentType: value.type, - knownLength: value.size, - }); - } else { - let file = value as File; - let stream = toNodeStream(file.stream()); - formStream.append(key, stream, { - filename: "unknown", - }); - } - } - - return formStream; -} +export { fetch, Headers, Response } from "@web-std/fetch"; interface NodeRequestInit extends RequestInit { abortController?: AbortController; @@ -71,13 +23,6 @@ class NodeRequest extends BaseNodeRequest { private abortController?: AbortController; constructor(input: RequestInfo, init?: NodeRequestInit | undefined) { - if (init?.body instanceof NodeFormData) { - init = { - ...init, - body: formDataToStream(init.body), - }; - } - super(input, init); let anyInput = input as any; @@ -95,8 +40,8 @@ class NodeRequest extends BaseNodeRequest { /multipart\/form-data/.test(contentType)) ) { return await internalParseFormData( - contentType, - this.body as Readable, + this, + super.formData, this.abortController, uploadHandler ); @@ -112,28 +57,28 @@ class NodeRequest extends BaseNodeRequest { export { NodeRequest as Request, NodeRequestInit as RequestInit }; -/** - * A `fetch` function for node that matches the web Fetch API. Based on - * `node-fetch`. - * - * @see https://github.com/node-fetch/node-fetch - * @see https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API - */ -export function fetch( - input: RequestInfo, - init?: RequestInit -): Promise { - init = { compress: false, ...init }; - - if (init?.body instanceof NodeFormData) { - init = { - ...init, - body: formDataToStream(init.body), - }; - } - - // Default to { compress: false } so responses can be proxied through more - // easily in loaders. Otherwise the response stream encoding will not match - // the Content-Encoding response header. - return nodeFetch(input, init); -} +// /** +// * A `fetch` function for node that matches the web Fetch API. Based on +// * `node-fetch`. +// * +// * @see https://github.com/node-fetch/node-fetch +// * @see https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API +// */ +// export function fetch( +// input: RequestInfo, +// init?: RequestInit +// ): Promise { +// init = { compress: false, ...init }; + +// if (init?.body instanceof NodeFormData) { +// init = { +// ...init, +// body: formDataToStream(init.body), +// }; +// } + +// // Default to { compress: false } so responses can be proxied through more +// // easily in loaders. Otherwise the response stream encoding will not match +// // the Content-Encoding response header. +// return nodeFetch(input, init); +// } diff --git a/packages/remix-node/index.ts b/packages/remix-node/index.ts index 5af593cc30d..9c1a2fc9921 100644 --- a/packages/remix-node/index.ts +++ b/packages/remix-node/index.ts @@ -5,10 +5,10 @@ sourceMapSupport.install(); export { AbortController } from "abort-controller"; export type { - HeadersInit, - RequestInfo, + // HeadersInit, + // RequestInfo, RequestInit, - ResponseInit, + // ResponseInit, } from "./fetch"; export { Headers, Request, Response, fetch } from "./fetch"; diff --git a/packages/remix-node/package.json b/packages/remix-node/package.json index bc29a009823..f7e2ebfeb58 100644 --- a/packages/remix-node/package.json +++ b/packages/remix-node/package.json @@ -15,13 +15,13 @@ "@remix-run/server-runtime": "1.3.5", "@types/busboy": "^0.3.1", "@types/node-fetch": "^2.5.12", + "@web-std/fetch": "^4.0.0", "@web-std/file": "^3.0.0", "abort-controller": "^3.0.0", "blob-stream": "^0.1.3", "busboy": "^0.3.1", "cookie-signature": "^1.1.0", "form-data": "^4.0.0", - "node-fetch": "^2.6.1", "source-map-support": "^0.5.21" }, "devDependencies": { diff --git a/packages/remix-node/parseMultipartFormData.ts b/packages/remix-node/parseMultipartFormData.ts index cbddb069cd7..3dff80549b5 100644 --- a/packages/remix-node/parseMultipartFormData.ts +++ b/packages/remix-node/parseMultipartFormData.ts @@ -1,9 +1,9 @@ -import { Readable } from "stream"; +import { PassThrough } from "stream"; import Busboy from "busboy"; +import { FormData } from "@web-std/fetch"; import type { Request as NodeRequest } from "./fetch"; import type { UploadHandler } from "./formData"; -import { FormData as NodeFormData } from "./formData"; /** * Allows you to handle multipart forms (file uploads) for your app. @@ -18,19 +18,42 @@ export function parseMultipartFormData( } export async function internalParseFormData( - contentType: string, - body: string | Buffer | Readable, + request: Request, + internalFormData: any, abortController?: AbortController, uploadHandler?: UploadHandler ) { - let formData = new NodeFormData(); + let formData = new FormData(); + let contentType = request.headers.get("Content-Type") || ""; + if (/application\/x-www-form-urlencoded/.test(contentType)) { + let searchParams = new URLSearchParams(await request.text()); + for (let [key, value] of searchParams.entries()) { + formData.append(key, value); + } + return formData; + } + + if (!uploadHandler) { + return internalFormData(); + } + let fileWorkQueue: Promise[] = []; - let stream: Readable; - if (typeof body === "string" || Buffer.isBuffer(body)) { - stream = Readable.from(body); + let stream: PassThrough = new PassThrough(); + if (request.body) { + let reader = request.body.getReader(); + async function read() { + let { done, value } = await reader.read(); + if (done) { + stream.end(value); + return; + } + stream.write(value); + read(); + } + read(); } else { - stream = body; + stream.end(); } await new Promise(async (resolve, reject) => { diff --git a/packages/remix-node/upload/fileUploadHandler.ts b/packages/remix-node/upload/fileUploadHandler.ts index 55d9068fb57..7593d4893e5 100644 --- a/packages/remix-node/upload/fileUploadHandler.ts +++ b/packages/remix-node/upload/fileUploadHandler.ts @@ -187,4 +187,8 @@ export class NodeOnDiskFile implements File { text(): Promise { return readFile(this.filepath, "utf-8"); } + + get [Symbol.toStringTag]() { + return "File" + } } diff --git a/packages/remix-server-runtime/data.ts b/packages/remix-server-runtime/data.ts index 674f3dbc1f3..7800cfab220 100644 --- a/packages/remix-server-runtime/data.ts +++ b/packages/remix-server-runtime/data.ts @@ -33,7 +33,7 @@ export async function callRouteAction({ let result; try { result = await action({ - request: stripDataParam(stripIndexParam(request)), + request: stripDataParam(stripIndexParam(request.clone())), context: loadContext, params: match.params, }); diff --git a/packages/remix-vercel/__tests__/server-test.ts b/packages/remix-vercel/__tests__/server-test.ts index 1b0501b6235..2c7f327256c 100644 --- a/packages/remix-vercel/__tests__/server-test.ts +++ b/packages/remix-vercel/__tests__/server-test.ts @@ -130,9 +130,7 @@ describe("vercel createRequestHandler", () => { expect(res.headers["x-time-of-year"]).toBe("most wonderful"); expect(res.headers["set-cookie"]).toEqual([ - "first=one; Expires=0; Path=/; HttpOnly; Secure; SameSite=Lax", - "second=two; MaxAge=1209600; Path=/; HttpOnly; Secure; SameSite=Lax", - "third=three; Expires=Wed, 21 Oct 2015 07:28:00 GMT; Path=/; HttpOnly; Secure; SameSite=Lax", + "first=one; Expires=0; Path=/; HttpOnly; Secure; SameSite=Lax, second=two; MaxAge=1209600; Path=/; HttpOnly; Secure; SameSite=Lax, third=three; Expires=Wed, 21 Oct 2015 07:28:00 GMT; Path=/; HttpOnly; Secure; SameSite=Lax", ]); }); }); @@ -142,20 +140,21 @@ describe("vercel createRemixHeaders", () => { describe("creates fetch headers from vercel headers", () => { it("handles empty headers", () => { expect(createRemixHeaders({})).toMatchInlineSnapshot(` - Headers { - Symbol(map): Object {}, + Headers$1 { + Symbol(query): Array [], + Symbol(context): null, } `); }); it("handles simple headers", () => { expect(createRemixHeaders({ "x-foo": "bar" })).toMatchInlineSnapshot(` - Headers { - Symbol(map): Object { - "x-foo": Array [ - "bar", - ], - }, + Headers$1 { + Symbol(query): Array [ + "x-foo", + "bar", + ], + Symbol(context): null, } `); }); @@ -163,15 +162,14 @@ describe("vercel createRemixHeaders", () => { it("handles multiple headers", () => { expect(createRemixHeaders({ "x-foo": "bar", "x-bar": "baz" })) .toMatchInlineSnapshot(` - Headers { - Symbol(map): Object { - "x-bar": Array [ - "baz", - ], - "x-foo": Array [ - "bar", - ], - }, + Headers$1 { + Symbol(query): Array [ + "x-foo", + "bar", + "x-bar", + "baz", + ], + Symbol(context): null, } `); }); @@ -179,12 +177,12 @@ describe("vercel createRemixHeaders", () => { it("handles headers with multiple values", () => { expect(createRemixHeaders({ "x-foo": "bar, baz" })) .toMatchInlineSnapshot(` - Headers { - Symbol(map): Object { - "x-foo": Array [ - "bar, baz", - ], - }, + Headers$1 { + Symbol(query): Array [ + "x-foo", + "bar, baz", + ], + Symbol(context): null, } `); }); @@ -192,15 +190,14 @@ describe("vercel createRemixHeaders", () => { it("handles headers with multiple values and multiple headers", () => { expect(createRemixHeaders({ "x-foo": "bar, baz", "x-bar": "baz" })) .toMatchInlineSnapshot(` - Headers { - Symbol(map): Object { - "x-bar": Array [ - "baz", - ], - "x-foo": Array [ - "bar, baz", - ], - }, + Headers$1 { + Symbol(query): Array [ + "x-foo", + "bar, baz", + "x-bar", + "baz", + ], + Symbol(context): null, } `); }); @@ -214,13 +211,14 @@ describe("vercel createRemixHeaders", () => { ], }) ).toMatchInlineSnapshot(` - Headers { - Symbol(map): Object { - "set-cookie": Array [ - "__session=some_value; Path=/; Secure; HttpOnly; MaxAge=7200; SameSite=Lax", - "__other=some_other_value; Path=/; Secure; HttpOnly; MaxAge=3600; SameSite=Lax", - ], - }, + Headers$1 { + Symbol(query): Array [ + "set-cookie", + "__session=some_value; Path=/; Secure; HttpOnly; MaxAge=7200; SameSite=Lax", + "set-cookie", + "__other=some_other_value; Path=/; Secure; HttpOnly; MaxAge=3600; SameSite=Lax", + ], + Symbol(context): null, } `); }); @@ -246,44 +244,33 @@ describe("vercel createRemixRequest", () => { "compress": true, "counter": 0, "follow": 20, + "highWaterMark": 16384, + "insecureHTTPParser": false, "size": 0, - "timeout": 0, Symbol(Body internals): Object { "body": null, + "boundary": null, "disturbed": false, "error": null, + "size": 0, + "type": null, }, Symbol(Request internals): Object { - "headers": Headers { - Symbol(map): Object { - "cache-control": Array [ - "max-age=300, s-maxage=3600", - ], - "x-forwarded-host": Array [ - "localhost:3000", - ], - "x-forwarded-proto": Array [ - "http", - ], - }, + "headers": Headers$1 { + Symbol(query): Array [ + "cache-control", + "max-age=300, s-maxage=3600", + "x-forwarded-host", + "localhost:3000", + "x-forwarded-proto", + "http", + ], + Symbol(context): null, }, "method": "GET", - "parsedURL": Url { - "auth": null, - "hash": null, - "host": "localhost:3000", - "hostname": "localhost", - "href": "http://localhost:3000/foo/bar", - "path": "/foo/bar", - "pathname": "/foo/bar", - "port": "3000", - "protocol": "http:", - "query": null, - "search": null, - "slashes": true, - }, + "parsedURL": "http://localhost:3000/foo/bar", "redirect": "follow", - "signal": undefined, + "signal": null, }, } `); diff --git a/packages/remix-vercel/server.ts b/packages/remix-vercel/server.ts index ab100de7db5..8d6371e5a7f 100644 --- a/packages/remix-vercel/server.ts +++ b/packages/remix-vercel/server.ts @@ -53,10 +53,10 @@ export function createRequestHandler({ ? getLoadContext(req, res) : undefined; - let response = (await handleRequest( + let response = await handleRequest( request as unknown as Request, loadContext - )) as unknown as NodeResponse; + ); if (abortController.signal.aborted) { response.headers.set("Connection", "close"); @@ -68,7 +68,7 @@ export function createRequestHandler({ export function createRemixHeaders( requestHeaders: VercelRequest["headers"] -): NodeHeaders { +): Headers { let headers = new NodeHeaders(); for (let key in requestHeaders) { let header = requestHeaders[key]!; @@ -102,7 +102,16 @@ export function createRemixRequest( }; if (req.method !== "GET" && req.method !== "HEAD") { - init.body = req; + init.body = new ReadableStream({ + start(controller) { + req.on("data", (chunk) => { + controller.enqueue(chunk); + }); + req.on("end", () => { + controller.close(); + }); + }, + }); } return new NodeRequest(url.href, init); @@ -110,7 +119,7 @@ export function createRemixRequest( export function sendRemixResponse( res: VercelResponse, - nodeResponse: NodeResponse + nodeResponse: Response ): void { let arrays = new Map(); for (let [key, value] of nodeResponse.headers.entries()) { @@ -125,12 +134,29 @@ export function sendRemixResponse( } res.statusMessage = nodeResponse.statusText; - res.writeHead(nodeResponse.status, nodeResponse.headers.raw()); + let multiValueHeaders: Record = {}; + for (let [key, value] of nodeResponse.headers) { + if (typeof multiValueHeaders[key] === "undefined") { + multiValueHeaders[key] = [value]; + } else { + (multiValueHeaders[key] as string[]).push(value); + } + } + res.writeHead(nodeResponse.status, multiValueHeaders); + + if (nodeResponse.body) { + let reader = nodeResponse.body.getReader(); + async function read() { + let { done, value } = await reader.read(); + if (done) { + res.end(value); + return; + } - if (Buffer.isBuffer(nodeResponse.body)) { - res.end(nodeResponse.body); - } else if (nodeResponse.body?.pipe) { - nodeResponse.body.pipe(res); + res.write(value); + read(); + } + read(); } else { res.end(); } diff --git a/yarn.lock b/yarn.lock index 587a58335eb..15a6a04cf88 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2407,6 +2407,25 @@ "@web-std/stream" "1.0.0" web-encoding "1.1.5" +"@web-std/blob@^3.0.3": + version "3.0.4" + resolved "https://registry.npmjs.org/@web-std/blob/-/blob-3.0.4.tgz#dd67a685547331915428d69e723c7da2015c3fc5" + integrity sha512-+dibyiw+uHYK4dX5cJ7HA+gtDAaUUe6JsOryp2ZpAC7h4ICsh49E34JwHoEKPlPvP0llCrNzz45vvD+xX5QDBg== + dependencies: + "@web-std/stream" "1.0.0" + web-encoding "1.1.5" + +"@web-std/fetch@^4.0.0": + version "4.0.0" + resolved "https://registry.npmjs.org/@web-std/fetch/-/fetch-4.0.0.tgz#32cb02e38c25518599843a51c4518b7ccb108d2c" + integrity sha512-RZUY1m7WoSsNGLfBeef3oAsmskU/IrlDSCMEakQZjD1csC91bdq3MJG7GiKijKJNg/PKRC45YxOo5yeSrAz5mA== + dependencies: + "@web-std/blob" "^3.0.3" + "@web-std/form-data" "^3.0.2" + "@web3-storage/multipart-parser" "^1.0.0" + data-uri-to-buffer "^3.0.1" + mrmime "^1.0.0" + "@web-std/file@^3.0.0": version "3.0.0" resolved "https://registry.npmjs.org/@web-std/file/-/file-3.0.0.tgz" @@ -2414,6 +2433,13 @@ dependencies: "@web-std/blob" "^3.0.0" +"@web-std/form-data@^3.0.2": + version "3.0.2" + resolved "https://registry.npmjs.org/@web-std/form-data/-/form-data-3.0.2.tgz#c71d9def6a593138ea92fe3d1ffbce19f43e869c" + integrity sha512-rhc8IRw66sJ0FHcnC84kT3mTN6eACTuNftkt1XSl1Ef6WRKq4Pz65xixxqZymAZl1K3USpwhLci4SKNn4PYxWQ== + dependencies: + web-encoding "1.1.5" + "@web-std/stream@1.0.0": version "1.0.0" resolved "https://registry.npmjs.org/@web-std/stream/-/stream-1.0.0.tgz" @@ -2421,6 +2447,11 @@ dependencies: web-streams-polyfill "^3.1.1" +"@web3-storage/multipart-parser@^1.0.0": + version "1.0.0" + resolved "https://registry.npmjs.org/@web3-storage/multipart-parser/-/multipart-parser-1.0.0.tgz#6b69dc2a32a5b207ba43e556c25cc136a56659c4" + integrity sha512-BEO6al7BYqcnfX15W2cnGR+Q566ACXAT9UQykORCWW80lmkpWsnEob6zJS1ZVBKsSJC8+7vJkHwlp+lXG1UCdw== + "@xmldom/xmldom@^0.7.5": version "0.7.5" resolved "https://registry.yarnpkg.com/@xmldom/xmldom/-/xmldom-0.7.5.tgz#09fa51e356d07d0be200642b0e4f91d8e6dd408d" @@ -3911,6 +3942,11 @@ damerau-levenshtein@^1.0.7: resolved "https://registry.npmjs.org/damerau-levenshtein/-/damerau-levenshtein-1.0.7.tgz" integrity sha512-VvdQIPGdWP0SqFXghj79Wf/5LArmreyMsGLa6FG6iC4t3j7j5s71TrwWmT/4akbDQIqjfACkLZmjXhA7g2oUZw== +data-uri-to-buffer@^3.0.1: + version "3.0.1" + resolved "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-3.0.1.tgz#594b8973938c5bc2c33046535785341abc4f3636" + integrity sha512-WboRycPNsVw3B3TL559F7kuBUM4d8CgMEvk6xEJlOp7OBPjt6G7z8WMWlD2rOFZLk6OYfFIUGsCOWzcQH9K2og== + data-urls@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/data-urls/-/data-urls-2.0.0.tgz" @@ -7890,6 +7926,11 @@ morgan@^1.10.0: on-finished "~2.3.0" on-headers "~1.0.2" +mrmime@^1.0.0: + version "1.0.0" + resolved "https://registry.npmjs.org/mrmime/-/mrmime-1.0.0.tgz#14d387f0585a5233d291baba339b063752a2398b" + integrity sha512-a70zx7zFfVO7XpnQ2IX1Myh9yY4UYvfld/dikWRnsXxbyvMcfz+u6UfgNAtH+k2QqtJuzVpv6eLTx1G2+WKZbQ== + ms@2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz" From 86c63d5cf730da984e4da4d87225a45f4b194d0b Mon Sep 17 00:00:00 2001 From: Jacob Ebey Date: Mon, 11 Apr 2022 15:03:40 -0700 Subject: [PATCH 02/47] added stream to express peer deps --- packages/remix-express/package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/remix-express/package.json b/packages/remix-express/package.json index a52a1cb45d6..85c823db133 100644 --- a/packages/remix-express/package.json +++ b/packages/remix-express/package.json @@ -15,6 +15,7 @@ "@remix-run/node": "1.3.5" }, "peerDependencies": { + "@web-std/stream": "^1.0.1", "express": "^4.17.1" }, "devDependencies": { From 7610062416f02d913f6d2bfd386f171443d8989a Mon Sep 17 00:00:00 2001 From: Jacob Ebey Date: Mon, 11 Apr 2022 15:05:57 -0700 Subject: [PATCH 03/47] chore: lint issues --- packages/remix-netlify/server.ts | 1 - packages/remix-node/fetch.ts | 28 ---------------------------- packages/remix-vercel/server.ts | 1 - 3 files changed, 30 deletions(-) diff --git a/packages/remix-netlify/server.ts b/packages/remix-netlify/server.ts index 1e0f6f89e81..38b746e8603 100644 --- a/packages/remix-netlify/server.ts +++ b/packages/remix-netlify/server.ts @@ -14,7 +14,6 @@ import type { import type { AppLoadContext, ServerBuild, - Response as NodeResponse, RequestInit as NodeRequestInit, } from "@remix-run/node"; diff --git a/packages/remix-node/fetch.ts b/packages/remix-node/fetch.ts index cb95d6ce396..2b0912f9089 100644 --- a/packages/remix-node/fetch.ts +++ b/packages/remix-node/fetch.ts @@ -1,5 +1,3 @@ -import type { Readable } from "stream"; -import { PassThrough } from "stream"; import type AbortController from "abort-controller"; // import FormStream from "form-data"; // import type { RequestInfo, RequestInit, Response } from "node-fetch"; @@ -56,29 +54,3 @@ class NodeRequest extends BaseNodeRequest { } export { NodeRequest as Request, NodeRequestInit as RequestInit }; - -// /** -// * A `fetch` function for node that matches the web Fetch API. Based on -// * `node-fetch`. -// * -// * @see https://github.com/node-fetch/node-fetch -// * @see https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API -// */ -// export function fetch( -// input: RequestInfo, -// init?: RequestInit -// ): Promise { -// init = { compress: false, ...init }; - -// if (init?.body instanceof NodeFormData) { -// init = { -// ...init, -// body: formDataToStream(init.body), -// }; -// } - -// // Default to { compress: false } so responses can be proxied through more -// // easily in loaders. Otherwise the response stream encoding will not match -// // the Content-Encoding response header. -// return nodeFetch(input, init); -// } diff --git a/packages/remix-vercel/server.ts b/packages/remix-vercel/server.ts index 8d6371e5a7f..c1f43710f80 100644 --- a/packages/remix-vercel/server.ts +++ b/packages/remix-vercel/server.ts @@ -3,7 +3,6 @@ import type { AppLoadContext, ServerBuild, RequestInit as NodeRequestInit, - Response as NodeResponse, } from "@remix-run/node"; import { // This has been added as a global in node 15+ From fd3c35154ebfff6e8b02aef71dbb13bb772e2927 Mon Sep 17 00:00:00 2001 From: Jacob Ebey Date: Mon, 11 Apr 2022 16:26:28 -0700 Subject: [PATCH 04/47] updated dep --- packages/remix-node/package.json | 2 +- yarn.lock | 18 +++++------------- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/packages/remix-node/package.json b/packages/remix-node/package.json index f7e2ebfeb58..422eb279da4 100644 --- a/packages/remix-node/package.json +++ b/packages/remix-node/package.json @@ -16,7 +16,7 @@ "@types/busboy": "^0.3.1", "@types/node-fetch": "^2.5.12", "@web-std/fetch": "^4.0.0", - "@web-std/file": "^3.0.0", + "@web-std/file": "^3.0.2", "abort-controller": "^3.0.0", "blob-stream": "^0.1.3", "busboy": "^0.3.1", diff --git a/yarn.lock b/yarn.lock index 4fe27f1d930..55f8c3b76ec 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2417,14 +2417,6 @@ ts-node "8.9.1" typescript "4.3.4" -"@web-std/blob@^3.0.0": - version "3.0.1" - resolved "https://registry.npmjs.org/@web-std/blob/-/blob-3.0.1.tgz" - integrity sha512-opuhO8ZGGUj2jdFwfgMjWjVdKaHlQanGWXxj5wV2YQ1uGTuL/SADnsDitpMfRb+lSpmQyzpwZFfj4CNKQuwSKQ== - dependencies: - "@web-std/stream" "1.0.0" - web-encoding "1.1.5" - "@web-std/blob@^3.0.3": version "3.0.4" resolved "https://registry.npmjs.org/@web-std/blob/-/blob-3.0.4.tgz#dd67a685547331915428d69e723c7da2015c3fc5" @@ -2444,12 +2436,12 @@ data-uri-to-buffer "^3.0.1" mrmime "^1.0.0" -"@web-std/file@^3.0.0": - version "3.0.0" - resolved "https://registry.npmjs.org/@web-std/file/-/file-3.0.0.tgz" - integrity sha512-ac2H3IUOky3GRJdbdJYgVvH+OApzpr0KX0t9p6Nj9AuHxKudc0pD7mVruekCW4CZv6DbOReDukwwskJN1bSCzA== +"@web-std/file@^3.0.2": + version "3.0.2" + resolved "https://registry.npmjs.org/@web-std/file/-/file-3.0.2.tgz#b84cc9ed754608b18dcf78ac62c40dbcc6a94692" + integrity sha512-pIH0uuZsmY8YFvSHP1NsBIiMT/1ce0suPrX74fEeO3Wbr1+rW0fUGEe4d0R99iLwXtyCwyserqCFI4BJkJlkRA== dependencies: - "@web-std/blob" "^3.0.0" + "@web-std/blob" "^3.0.3" "@web-std/form-data@^3.0.2": version "3.0.2" From 9b598f5887a7e7908b00726bb2e7699a482a3a7c Mon Sep 17 00:00:00 2001 From: Jacob Ebey Date: Thu, 28 Apr 2022 15:49:48 -0700 Subject: [PATCH 05/47] updated deps to use our fork --- packages/remix-express/package.json | 2 +- packages/remix-express/server.ts | 2 +- .../remix-node/__tests__/formData-test.ts | 2 +- .../__tests__/parseMultipartFormData-test.ts | 4 +- packages/remix-node/fetch.ts | 13 +-- packages/remix-node/globals.ts | 2 +- packages/remix-node/package.json | 5 +- packages/remix-node/parseMultipartFormData.ts | 2 +- .../remix-node/upload/memoryUploadHandler.ts | 2 +- yarn.lock | 83 ++++++++++--------- 10 files changed, 55 insertions(+), 62 deletions(-) diff --git a/packages/remix-express/package.json b/packages/remix-express/package.json index 85c823db133..5535585de23 100644 --- a/packages/remix-express/package.json +++ b/packages/remix-express/package.json @@ -15,7 +15,7 @@ "@remix-run/node": "1.3.5" }, "peerDependencies": { - "@web-std/stream": "^1.0.1", + "@remix-run/web-stream": "^1.0.2", "express": "^4.17.1" }, "devDependencies": { diff --git a/packages/remix-express/server.ts b/packages/remix-express/server.ts index 50b17bca05b..c69ea5accb6 100644 --- a/packages/remix-express/server.ts +++ b/packages/remix-express/server.ts @@ -6,7 +6,7 @@ import type { RequestInit as NodeRequestInit, Response as NodeResponse, } from "@remix-run/node"; -import { ReadableStream } from "@web-std/stream"; +import { ReadableStream } from "@remix-run/web-stream"; import { // This has been added as a global in node 15+ AbortController, diff --git a/packages/remix-node/__tests__/formData-test.ts b/packages/remix-node/__tests__/formData-test.ts index 1f483ef44fb..562c1d08368 100644 --- a/packages/remix-node/__tests__/formData-test.ts +++ b/packages/remix-node/__tests__/formData-test.ts @@ -1,4 +1,4 @@ -import { Blob, File } from "@web-std/file"; +import { Blob, File } from "@remix-run/web-file"; import { FormData as NodeFormData } from "../formData"; diff --git a/packages/remix-node/__tests__/parseMultipartFormData-test.ts b/packages/remix-node/__tests__/parseMultipartFormData-test.ts index 4ec7f8939ec..73ea0ae8303 100644 --- a/packages/remix-node/__tests__/parseMultipartFormData-test.ts +++ b/packages/remix-node/__tests__/parseMultipartFormData-test.ts @@ -1,5 +1,5 @@ -import { FormData as NodeFormData } from "@web-std/fetch"; -import { Blob, File } from "@web-std/file"; +import { FormData as NodeFormData } from "@remix-run/web-fetch"; +import { Blob, File } from "@remix-run/web-file"; import { Request as NodeRequest } from "../fetch"; // import { FormData as NodeFormData } from "../formData"; diff --git a/packages/remix-node/fetch.ts b/packages/remix-node/fetch.ts index 2b0912f9089..549bf191e5b 100644 --- a/packages/remix-node/fetch.ts +++ b/packages/remix-node/fetch.ts @@ -1,17 +1,10 @@ import type AbortController from "abort-controller"; -// import FormStream from "form-data"; -// import type { RequestInfo, RequestInit, Response } from "node-fetch"; -// import nodeFetch, { Request as BaseNodeRequest } from "node-fetch"; -import { Request as BaseNodeRequest } from "@web-std/fetch"; +import { Request as BaseNodeRequest } from "@remix-run/web-fetch"; -// import { FormData as NodeFormData, isFile } from "./formData"; import type { UploadHandler } from "./formData"; import { internalParseFormData } from "./parseMultipartFormData"; -// export type { HeadersInit, RequestInfo, ResponseInit } from "node-fetch"; -// export { Headers, Response } from "node-fetch"; - -export { fetch, Headers, Response } from "@web-std/fetch"; +export { fetch, Headers, Response } from "@remix-run/web-fetch"; interface NodeRequestInit extends RequestInit { abortController?: AbortController; @@ -21,7 +14,7 @@ class NodeRequest extends BaseNodeRequest { private abortController?: AbortController; constructor(input: RequestInfo, init?: NodeRequestInit | undefined) { - super(input, init); + super(input as any, init); let anyInput = input as any; let anyInit = init as any; diff --git a/packages/remix-node/globals.ts b/packages/remix-node/globals.ts index dbb00e88be1..24669c07e0c 100644 --- a/packages/remix-node/globals.ts +++ b/packages/remix-node/globals.ts @@ -1,4 +1,4 @@ -import { Blob as NodeBlob, File as NodeFile } from "@web-std/file"; +import { Blob as NodeBlob, File as NodeFile } from "@remix-run/web-file"; import { atob, btoa } from "./base64"; import { diff --git a/packages/remix-node/package.json b/packages/remix-node/package.json index 422eb279da4..67c70fa731c 100644 --- a/packages/remix-node/package.json +++ b/packages/remix-node/package.json @@ -14,9 +14,8 @@ "dependencies": { "@remix-run/server-runtime": "1.3.5", "@types/busboy": "^0.3.1", - "@types/node-fetch": "^2.5.12", - "@web-std/fetch": "^4.0.0", - "@web-std/file": "^3.0.2", + "@remix-run/web-fetch": "^4.1.0", + "@remix-run/web-file": "^3.0.2", "abort-controller": "^3.0.0", "blob-stream": "^0.1.3", "busboy": "^0.3.1", diff --git a/packages/remix-node/parseMultipartFormData.ts b/packages/remix-node/parseMultipartFormData.ts index 3dff80549b5..1b8c9997e8d 100644 --- a/packages/remix-node/parseMultipartFormData.ts +++ b/packages/remix-node/parseMultipartFormData.ts @@ -1,6 +1,6 @@ import { PassThrough } from "stream"; import Busboy from "busboy"; -import { FormData } from "@web-std/fetch"; +import { FormData } from "@remix-run/web-fetch"; import type { Request as NodeRequest } from "./fetch"; import type { UploadHandler } from "./formData"; diff --git a/packages/remix-node/upload/memoryUploadHandler.ts b/packages/remix-node/upload/memoryUploadHandler.ts index 5dde5b803f0..ed671540aac 100644 --- a/packages/remix-node/upload/memoryUploadHandler.ts +++ b/packages/remix-node/upload/memoryUploadHandler.ts @@ -1,6 +1,6 @@ import type { TransformCallback } from "stream"; import { Transform } from "stream"; -import { File as BufferFile } from "@web-std/file"; +import { File as BufferFile } from "@remix-run/web-file"; import { Meter } from "./meter"; import type { UploadHandler } from "../formData"; diff --git a/yarn.lock b/yarn.lock index 01b0082a1c3..912b5033889 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1561,6 +1561,47 @@ stack-utils "2.0.5" yazl "2.5.1" +"@remix-run/web-blob@^3.0.3": + version "3.0.4" + resolved "https://registry.npmjs.org/@remix-run/web-blob/-/web-blob-3.0.4.tgz#99c67b9d0fb641bd0c07d267fd218ae5aa4ae5ed" + integrity sha512-AfegzZvSSDc+LwnXV+SwROTrDtoLiPxeFW+jxgvtDAnkuCX1rrzmVJ6CzqZ1Ai0bVfmJadkG5GxtAfYclpPmgw== + dependencies: + "@remix-run/web-stream" "^1.0.0" + web-encoding "1.1.5" + +"@remix-run/web-fetch@^4.1.0": + version "4.1.0" + resolved "https://registry.npmjs.org/@remix-run/web-fetch/-/web-fetch-4.1.0.tgz#f4e7d31863add71c1bf759fbde7c9750d0c57313" + integrity sha512-V1q8ZnViqePZ44OCuHZbEbuof3jnru5L9ZxOgO2zjrbryIxc3J/ZgyZxIV0HeviD3vUWNdz283FNl2mUOZ5+ZA== + dependencies: + "@remix-run/web-blob" "^3.0.3" + "@remix-run/web-form-data" "^3.0.2" + "@remix-run/web-stream" "^1.0.1" + "@web3-storage/multipart-parser" "^1.0.0" + data-uri-to-buffer "^3.0.1" + mrmime "^1.0.0" + +"@remix-run/web-file@^3.0.2": + version "3.0.2" + resolved "https://registry.npmjs.org/@remix-run/web-file/-/web-file-3.0.2.tgz#1a6cc0900a1310ede4bc96abad77ac6eb27a2131" + integrity sha512-eFC93Onh/rZ5kUNpCQersmBtxedGpaXK2/gsUl49BYSGK/DvuPu3l06vmquEDdcPaEuXcsdGP0L7zrmUqrqo4A== + dependencies: + "@remix-run/web-blob" "^3.0.3" + +"@remix-run/web-form-data@^3.0.2": + version "3.0.2" + resolved "https://registry.npmjs.org/@remix-run/web-form-data/-/web-form-data-3.0.2.tgz#733a4c8f8176523b7b60a8bd0dc6704fd4d498f3" + integrity sha512-F8tm3iB1sPxMpysK6Js7lV3gvLfTNKGmIW38t/e6dtPEB5L1WdbRG1cmLyhsonFc7rT1x1JKdz+2jCtoSdnIUw== + dependencies: + web-encoding "1.1.5" + +"@remix-run/web-stream@^1.0.0", "@remix-run/web-stream@^1.0.1": + version "1.0.2" + resolved "https://registry.npmjs.org/@remix-run/web-stream/-/web-stream-1.0.2.tgz#f07dc9cf6db02507ea71a234bc8e06103a2207b4" + integrity sha512-FO4om5mrwMs5bi7L5hbLMP1hm+flAS2oYRptfNPkK2u0Hhv0crS9GiE9/MsVvY53tTAxVkzUG/m+9ET1mTjEnw== + dependencies: + web-streams-polyfill "^3.1.1" + "@rollup/plugin-babel@^5.2.2": version "5.3.0" resolved "https://registry.npmjs.org/@rollup/plugin-babel/-/plugin-babel-5.3.0.tgz" @@ -2056,7 +2097,7 @@ resolved "https://registry.npmjs.org/@types/ms/-/ms-0.7.31.tgz" integrity sha512-iiUgKzV9AuaEkZqkOLDIvlQiL6ltuZd9tGcW3gwpnX8JbuiuhFlEGmmFXEXkN50Cvq7Os88IY2v0dkDqXYWVgA== -"@types/node-fetch@^2.5.12", "@types/node-fetch@^2.5.7": +"@types/node-fetch@^2.5.7": version "2.5.12" resolved "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.5.12.tgz" integrity sha512-MKgC4dlq4kKNa/mYrwpKfzQMB5X3ee5U6fSprkKpToBqBmX4nFZL9cW5jl6sWn+xpRJ7ypWh2yyqqr8UUCstSw== @@ -2354,46 +2395,6 @@ ts-node "8.9.1" typescript "4.3.4" -"@web-std/blob@^3.0.3": - version "3.0.4" - resolved "https://registry.npmjs.org/@web-std/blob/-/blob-3.0.4.tgz#dd67a685547331915428d69e723c7da2015c3fc5" - integrity sha512-+dibyiw+uHYK4dX5cJ7HA+gtDAaUUe6JsOryp2ZpAC7h4ICsh49E34JwHoEKPlPvP0llCrNzz45vvD+xX5QDBg== - dependencies: - "@web-std/stream" "1.0.0" - web-encoding "1.1.5" - -"@web-std/fetch@^4.0.0": - version "4.0.0" - resolved "https://registry.npmjs.org/@web-std/fetch/-/fetch-4.0.0.tgz#32cb02e38c25518599843a51c4518b7ccb108d2c" - integrity sha512-RZUY1m7WoSsNGLfBeef3oAsmskU/IrlDSCMEakQZjD1csC91bdq3MJG7GiKijKJNg/PKRC45YxOo5yeSrAz5mA== - dependencies: - "@web-std/blob" "^3.0.3" - "@web-std/form-data" "^3.0.2" - "@web3-storage/multipart-parser" "^1.0.0" - data-uri-to-buffer "^3.0.1" - mrmime "^1.0.0" - -"@web-std/file@^3.0.2": - version "3.0.2" - resolved "https://registry.npmjs.org/@web-std/file/-/file-3.0.2.tgz#b84cc9ed754608b18dcf78ac62c40dbcc6a94692" - integrity sha512-pIH0uuZsmY8YFvSHP1NsBIiMT/1ce0suPrX74fEeO3Wbr1+rW0fUGEe4d0R99iLwXtyCwyserqCFI4BJkJlkRA== - dependencies: - "@web-std/blob" "^3.0.3" - -"@web-std/form-data@^3.0.2": - version "3.0.2" - resolved "https://registry.npmjs.org/@web-std/form-data/-/form-data-3.0.2.tgz#c71d9def6a593138ea92fe3d1ffbce19f43e869c" - integrity sha512-rhc8IRw66sJ0FHcnC84kT3mTN6eACTuNftkt1XSl1Ef6WRKq4Pz65xixxqZymAZl1K3USpwhLci4SKNn4PYxWQ== - dependencies: - web-encoding "1.1.5" - -"@web-std/stream@1.0.0": - version "1.0.0" - resolved "https://registry.npmjs.org/@web-std/stream/-/stream-1.0.0.tgz" - integrity sha512-jyIbdVl+0ZJyKGTV0Ohb9E6UnxP+t7ZzX4Do3AHjZKxUXKMs9EmqnBDQgHF7bEw0EzbQygOjtt/7gvtmi//iCQ== - dependencies: - web-streams-polyfill "^3.1.1" - "@web3-storage/multipart-parser@^1.0.0": version "1.0.0" resolved "https://registry.npmjs.org/@web3-storage/multipart-parser/-/multipart-parser-1.0.0.tgz#6b69dc2a32a5b207ba43e556c25cc136a56659c4" From 1abd9bf0f2425b0b65c6120dd276ae6b52e2f81f Mon Sep 17 00:00:00 2001 From: Jacob Ebey Date: Thu, 28 Apr 2022 16:05:28 -0700 Subject: [PATCH 06/47] remove content type check as it was merged in the fetch fork --- packages/remix-node/parseMultipartFormData.ts | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/packages/remix-node/parseMultipartFormData.ts b/packages/remix-node/parseMultipartFormData.ts index 1b8c9997e8d..270ff91c572 100644 --- a/packages/remix-node/parseMultipartFormData.ts +++ b/packages/remix-node/parseMultipartFormData.ts @@ -23,20 +23,12 @@ export async function internalParseFormData( abortController?: AbortController, uploadHandler?: UploadHandler ) { - let formData = new FormData(); - let contentType = request.headers.get("Content-Type") || ""; - if (/application\/x-www-form-urlencoded/.test(contentType)) { - let searchParams = new URLSearchParams(await request.text()); - for (let [key, value] of searchParams.entries()) { - formData.append(key, value); - } - return formData; - } - if (!uploadHandler) { return internalFormData(); } - + + let formData = new FormData(); + let contentType = request.headers.get("Content-Type") || ""; let fileWorkQueue: Promise[] = []; let stream: PassThrough = new PassThrough(); From 7da9ec7eda04c3b87a21c8f298818bd48178a3ac Mon Sep 17 00:00:00 2001 From: Jacob Ebey Date: Thu, 28 Apr 2022 16:12:34 -0700 Subject: [PATCH 07/47] updated architect and express adapters --- packages/remix-architect/server.ts | 20 +++++++------------- packages/remix-express/server.ts | 2 +- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/packages/remix-architect/server.ts b/packages/remix-architect/server.ts index 066a396f44b..262808e30b0 100644 --- a/packages/remix-architect/server.ts +++ b/packages/remix-architect/server.ts @@ -11,11 +11,7 @@ import type { APIGatewayProxyHandlerV2, APIGatewayProxyStructuredResultV2, } from "aws-lambda"; -import type { - AppLoadContext, - ServerBuild, - Response as NodeResponse, -} from "@remix-run/node"; +import type { AppLoadContext, ServerBuild } from "@remix-run/node"; import { isBinaryType } from "./binaryTypes"; @@ -53,10 +49,10 @@ export function createRequestHandler({ let loadContext = typeof getLoadContext === "function" ? getLoadContext(event) : undefined; - let response = (await handleRequest( + let response = await handleRequest( request as unknown as Request, loadContext - )) as unknown as NodeResponse; + ); return sendRemixResponse(response, abortController); }; @@ -91,7 +87,7 @@ export function createRemixRequest( export function createRemixHeaders( requestHeaders: APIGatewayProxyEventHeaders, requestCookies?: string[] -): NodeHeaders { +): Headers { let headers = new NodeHeaders(); for (let [header, value] of Object.entries(requestHeaders)) { @@ -108,17 +104,15 @@ export function createRemixHeaders( } export async function sendRemixResponse( - nodeResponse: NodeResponse, + nodeResponse: Response, abortController: AbortController ): Promise { let cookies: string[] = []; // Arc/AWS API Gateway will send back set-cookies outside of response headers. - for (let [key, values] of Object.entries(nodeResponse.headers.raw())) { + for (let [key, value] of nodeResponse.headers) { if (key.toLowerCase() === "set-cookie") { - for (let value of values) { - cookies.push(value); - } + cookies.push(value); } } diff --git a/packages/remix-express/server.ts b/packages/remix-express/server.ts index c69ea5accb6..249e10f50ab 100644 --- a/packages/remix-express/server.ts +++ b/packages/remix-express/server.ts @@ -133,7 +133,7 @@ export function sendRemixResponse( res.statusMessage = nodeResponse.statusText; res.status(nodeResponse.status); - for (let [key, value] of nodeResponse.headers.entries()) { + for (let [key, value] of nodeResponse.headers) { res.append(key, value); } From fa6cd4c59b16a97501a3b0aaca1911bbbea2d20e Mon Sep 17 00:00:00 2001 From: Jacob Ebey Date: Thu, 28 Apr 2022 16:15:03 -0700 Subject: [PATCH 08/47] updated architect and express adapters --- packages/remix-architect/server.ts | 2 +- packages/remix-express/server.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/remix-architect/server.ts b/packages/remix-architect/server.ts index 262808e30b0..ef979c0dad5 100644 --- a/packages/remix-architect/server.ts +++ b/packages/remix-architect/server.ts @@ -110,7 +110,7 @@ export async function sendRemixResponse( let cookies: string[] = []; // Arc/AWS API Gateway will send back set-cookies outside of response headers. - for (let [key, value] of nodeResponse.headers) { + for (let [key, value] of Object.entries(nodeResponse.headers)) { if (key.toLowerCase() === "set-cookie") { cookies.push(value); } diff --git a/packages/remix-express/server.ts b/packages/remix-express/server.ts index 249e10f50ab..61376820ce5 100644 --- a/packages/remix-express/server.ts +++ b/packages/remix-express/server.ts @@ -133,7 +133,7 @@ export function sendRemixResponse( res.statusMessage = nodeResponse.statusText; res.status(nodeResponse.status); - for (let [key, value] of nodeResponse.headers) { + for (let [key, value] of Object.entries(nodeResponse.headers)) { res.append(key, value); } From 008ec1f2b67823b9ea9c0b8a1b5555dfabb5a956 Mon Sep 17 00:00:00 2001 From: Jacob Ebey Date: Thu, 28 Apr 2022 16:36:29 -0700 Subject: [PATCH 09/47] reverted cookie changes bind formData to request --- integration/multiple-cookies-test.ts | 4 ++-- packages/remix-architect/server.ts | 8 ++++++-- packages/remix-express/__tests__/server-test.ts | 4 +++- packages/remix-express/server.ts | 8 ++++++-- packages/remix-netlify/__tests__/server-test.ts | 4 +++- packages/remix-netlify/server.ts | 9 +-------- packages/remix-node/fetch.ts | 2 +- packages/remix-vercel/server.ts | 12 +++++++----- 8 files changed, 29 insertions(+), 22 deletions(-) diff --git a/integration/multiple-cookies-test.ts b/integration/multiple-cookies-test.ts index 063b6f61a33..719212beffa 100644 --- a/integration/multiple-cookies-test.ts +++ b/integration/multiple-cookies-test.ts @@ -56,7 +56,7 @@ test.describe("pathless layout routes", () => { let responses = app.collectResponses((url) => url.pathname === "/"); await app.goto("/"); let setCookies = await responses[0].headerValues("set-cookie"); - expect(setCookies).toEqual(["foo=bar, bar=baz"]); + expect(setCookies).toEqual(["foo=bar", "bar=baz"]); expect(responses).toHaveLength(1); }); @@ -68,7 +68,7 @@ test.describe("pathless layout routes", () => { await page.click("button[type=submit]"); await page.waitForSelector(`[data-testid="action-success"]`); let setCookies = await responses[0].headerValues("set-cookie"); - expect(setCookies).toEqual([`another=one, how-about=two`]); + expect(setCookies).toEqual(["another=one", "how-about=two"]); // one for the POST and one for the GET expect(responses).toHaveLength(2); }); diff --git a/packages/remix-architect/server.ts b/packages/remix-architect/server.ts index ef979c0dad5..42d39c716e7 100644 --- a/packages/remix-architect/server.ts +++ b/packages/remix-architect/server.ts @@ -110,9 +110,13 @@ export async function sendRemixResponse( let cookies: string[] = []; // Arc/AWS API Gateway will send back set-cookies outside of response headers. - for (let [key, value] of Object.entries(nodeResponse.headers)) { + for (let [key, values] of Object.entries( + (nodeResponse.headers as any).raw() as Record + )) { if (key.toLowerCase() === "set-cookie") { - cookies.push(value); + for (let value of values) { + cookies.push(value); + } } } diff --git a/packages/remix-express/__tests__/server-test.ts b/packages/remix-express/__tests__/server-test.ts index 6309c4b52b4..a4937d862f7 100644 --- a/packages/remix-express/__tests__/server-test.ts +++ b/packages/remix-express/__tests__/server-test.ts @@ -125,7 +125,9 @@ describe("express createRequestHandler", () => { expect(res.headers["x-time-of-year"]).toBe("most wonderful"); expect(res.headers["set-cookie"]).toEqual([ - "first=one; Expires=0; Path=/; HttpOnly; Secure; SameSite=Lax, second=two; MaxAge=1209600; Path=/; HttpOnly; Secure; SameSite=Lax, third=three; Expires=Wed, 21 Oct 2015 07:28:00 GMT; Path=/; HttpOnly; Secure; SameSite=Lax", + "first=one; Expires=0; Path=/; HttpOnly; Secure; SameSite=Lax", + "second=two; MaxAge=1209600; Path=/; HttpOnly; Secure; SameSite=Lax", + "third=three; Expires=Wed, 21 Oct 2015 07:28:00 GMT; Path=/; HttpOnly; Secure; SameSite=Lax", ]); }); }); diff --git a/packages/remix-express/server.ts b/packages/remix-express/server.ts index 61376820ce5..ffe5009681d 100644 --- a/packages/remix-express/server.ts +++ b/packages/remix-express/server.ts @@ -133,8 +133,12 @@ export function sendRemixResponse( res.statusMessage = nodeResponse.statusText; res.status(nodeResponse.status); - for (let [key, value] of Object.entries(nodeResponse.headers)) { - res.append(key, value); + for (let [key, values] of Object.entries( + (nodeResponse.headers as any).raw() as Record + )) { + for (let value of values) { + res.append(key, value); + } } if (abortController.signal.aborted) { diff --git a/packages/remix-netlify/__tests__/server-test.ts b/packages/remix-netlify/__tests__/server-test.ts index aa2eca3b9f8..410d6cb466d 100644 --- a/packages/remix-netlify/__tests__/server-test.ts +++ b/packages/remix-netlify/__tests__/server-test.ts @@ -127,7 +127,9 @@ describe("netlify createRequestHandler", () => { "most wonderful", ]); expect(res.multiValueHeaders["set-cookie"]).toEqual([ - "first=one; Expires=0; Path=/; HttpOnly; Secure; SameSite=Lax, second=two; MaxAge=1209600; Path=/; HttpOnly; Secure; SameSite=Lax, third=three; Expires=Wed, 21 Oct 2015 07:28:00 GMT; Path=/; HttpOnly; Secure; SameSite=Lax", + "first=one; Expires=0; Path=/; HttpOnly; Secure; SameSite=Lax", + "second=two; MaxAge=1209600; Path=/; HttpOnly; Secure; SameSite=Lax", + "third=three; Expires=Wed, 21 Oct 2015 07:28:00 GMT; Path=/; HttpOnly; Secure; SameSite=Lax", ]); }); }); diff --git a/packages/remix-netlify/server.ts b/packages/remix-netlify/server.ts index 38b746e8603..d952cd42f3a 100644 --- a/packages/remix-netlify/server.ts +++ b/packages/remix-netlify/server.ts @@ -169,14 +169,7 @@ export async function sendRemixResponse( } } - let multiValueHeaders: Record = {}; - for (let [key, value] of nodeResponse.headers) { - if (typeof multiValueHeaders[key] === "undefined") { - multiValueHeaders[key] = [value]; - } else { - (multiValueHeaders[key] as string[]).push(value); - } - } + let multiValueHeaders: Record = (nodeResponse.headers as any).raw(); console.log({ multiValueHeaders }); diff --git a/packages/remix-node/fetch.ts b/packages/remix-node/fetch.ts index 549bf191e5b..4a8f7a30ead 100644 --- a/packages/remix-node/fetch.ts +++ b/packages/remix-node/fetch.ts @@ -32,7 +32,7 @@ class NodeRequest extends BaseNodeRequest { ) { return await internalParseFormData( this, - super.formData, + super.formData.bind(this), this.abortController, uploadHandler ); diff --git a/packages/remix-vercel/server.ts b/packages/remix-vercel/server.ts index c1f43710f80..38c9cc3b793 100644 --- a/packages/remix-vercel/server.ts +++ b/packages/remix-vercel/server.ts @@ -120,15 +120,17 @@ export function sendRemixResponse( res: VercelResponse, nodeResponse: Response ): void { - let arrays = new Map(); - for (let [key, value] of nodeResponse.headers.entries()) { + let arrays = new Map(); + for (let [key, values] of Object.entries( + (nodeResponse.headers as any).raw() as Record + )) { if (arrays.has(key)) { - let newValue = arrays.get(key).concat(value); + let newValue = arrays.get(key)!.concat(...values); res.setHeader(key, newValue); arrays.set(key, newValue); } else { - res.setHeader(key, value); - arrays.set(key, [value]); + res.setHeader(key, values); + arrays.set(key, values); } } From fb8851edf91ba28bb43661143c7337d74939ffa0 Mon Sep 17 00:00:00 2001 From: Jacob Ebey Date: Thu, 28 Apr 2022 16:47:53 -0700 Subject: [PATCH 10/47] bind in test --- packages/remix-node/__tests__/parseMultipartFormData-test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/remix-node/__tests__/parseMultipartFormData-test.ts b/packages/remix-node/__tests__/parseMultipartFormData-test.ts index 73ea0ae8303..6d3ee6e89aa 100644 --- a/packages/remix-node/__tests__/parseMultipartFormData-test.ts +++ b/packages/remix-node/__tests__/parseMultipartFormData-test.ts @@ -21,7 +21,7 @@ describe("internalParseFormData", () => { let uploadHandler = createMemoryUploadHandler({}); let parsedFormData = await internalParseFormData( req, - req.formData, + req.formData.bind(req), undefined, uploadHandler ); From 1b0606c22eb0d6dc9086a363f9880746698e1091 Mon Sep 17 00:00:00 2001 From: Jacob Ebey Date: Thu, 28 Apr 2022 16:48:43 -0700 Subject: [PATCH 11/47] remove console.log --- packages/remix-netlify/server.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/remix-netlify/server.ts b/packages/remix-netlify/server.ts index d952cd42f3a..2dccf375a60 100644 --- a/packages/remix-netlify/server.ts +++ b/packages/remix-netlify/server.ts @@ -169,9 +169,9 @@ export async function sendRemixResponse( } } - let multiValueHeaders: Record = (nodeResponse.headers as any).raw(); - - console.log({ multiValueHeaders }); + let multiValueHeaders: Record = ( + nodeResponse.headers as any + ).raw(); return { statusCode: nodeResponse.status, From 0a7493c58eda13621d7ae0257a5d90e178a7f847 Mon Sep 17 00:00:00 2001 From: Jacob Ebey Date: Thu, 28 Apr 2022 16:52:58 -0700 Subject: [PATCH 12/47] updated vercel adapter --- .../remix-vercel/__tests__/server-test.ts | 4 ++- packages/remix-vercel/server.ts | 26 +++++++------------ 2 files changed, 12 insertions(+), 18 deletions(-) diff --git a/packages/remix-vercel/__tests__/server-test.ts b/packages/remix-vercel/__tests__/server-test.ts index 2c7f327256c..c367eb4ed6f 100644 --- a/packages/remix-vercel/__tests__/server-test.ts +++ b/packages/remix-vercel/__tests__/server-test.ts @@ -130,7 +130,9 @@ describe("vercel createRequestHandler", () => { expect(res.headers["x-time-of-year"]).toBe("most wonderful"); expect(res.headers["set-cookie"]).toEqual([ - "first=one; Expires=0; Path=/; HttpOnly; Secure; SameSite=Lax, second=two; MaxAge=1209600; Path=/; HttpOnly; Secure; SameSite=Lax, third=three; Expires=Wed, 21 Oct 2015 07:28:00 GMT; Path=/; HttpOnly; Secure; SameSite=Lax", + "first=one; Expires=0; Path=/; HttpOnly; Secure; SameSite=Lax", + "second=two; MaxAge=1209600; Path=/; HttpOnly; Secure; SameSite=Lax", + "third=three; Expires=Wed, 21 Oct 2015 07:28:00 GMT; Path=/; HttpOnly; Secure; SameSite=Lax", ]); }); }); diff --git a/packages/remix-vercel/server.ts b/packages/remix-vercel/server.ts index 38c9cc3b793..dff20849c60 100644 --- a/packages/remix-vercel/server.ts +++ b/packages/remix-vercel/server.ts @@ -120,30 +120,22 @@ export function sendRemixResponse( res: VercelResponse, nodeResponse: Response ): void { - let arrays = new Map(); + res.statusMessage = nodeResponse.statusText; + let multiValueHeaders: Record = {}; for (let [key, values] of Object.entries( (nodeResponse.headers as any).raw() as Record )) { - if (arrays.has(key)) { - let newValue = arrays.get(key)!.concat(...values); - res.setHeader(key, newValue); - arrays.set(key, newValue); - } else { - res.setHeader(key, values); - arrays.set(key, values); - } - } - - res.statusMessage = nodeResponse.statusText; - let multiValueHeaders: Record = {}; - for (let [key, value] of nodeResponse.headers) { if (typeof multiValueHeaders[key] === "undefined") { - multiValueHeaders[key] = [value]; + multiValueHeaders[key] = [...values]; } else { - (multiValueHeaders[key] as string[]).push(value); + (multiValueHeaders[key] as string[]).push(...values); } } - res.writeHead(nodeResponse.status, multiValueHeaders); + res.writeHead( + nodeResponse.status, + nodeResponse.statusText, + multiValueHeaders + ); if (nodeResponse.body) { let reader = nodeResponse.body.getReader(); From 269fea50f1e65faea09b2b90efab703dd45efb8a Mon Sep 17 00:00:00 2001 From: Jacob Ebey Date: Wed, 4 May 2022 11:12:19 -0700 Subject: [PATCH 13/47] feat(remix-node): replaced busboy feat: updated upload handlers to use new API fix: clean up adapters --- .gitignore | 1 + integration/action-test.ts | 204 ---------------- integration/upload-test.ts | 220 ++++++++++++++++++ packages/remix-express/server.ts | 47 +--- packages/remix-netlify/server.ts | 29 +-- packages/remix-node/__tests__/fetch-test.ts | 1 + .../remix-node/__tests__/formData-test.ts | 36 --- .../__tests__/parseMultipartFormData-test.ts | 55 ++++- packages/remix-node/fetch.ts | 64 +++-- packages/remix-node/formData.ts | 119 +--------- packages/remix-node/globals.ts | 4 +- packages/remix-node/index.ts | 9 +- packages/remix-node/package.json | 4 +- packages/remix-node/parseMultipartFormData.ts | 142 ++++------- packages/remix-node/stream.ts | 39 ++++ packages/remix-node/tsconfig.json | 2 +- .../remix-node/upload/fileUploadHandler.ts | 69 +++--- .../remix-node/upload/memoryUploadHandler.ts | 83 ++++--- packages/remix-node/upload/meter.ts | 23 -- packages/remix-vercel/server.ts | 34 +-- yarn.lock | 2 +- 21 files changed, 513 insertions(+), 674 deletions(-) create mode 100644 integration/upload-test.ts delete mode 100644 packages/remix-node/__tests__/formData-test.ts create mode 100644 packages/remix-node/stream.ts diff --git a/.gitignore b/.gitignore index ecdbc4d89c5..1ded76d7f95 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ yarn-error.log /fixtures/deno-app /playwright-report /test-results +/uploads .eslintcache .tmp diff --git a/integration/action-test.ts b/integration/action-test.ts index 80b3078de2d..9e92dc1b681 100644 --- a/integration/action-test.ts +++ b/integration/action-test.ts @@ -1,5 +1,4 @@ import { test, expect } from "@playwright/test"; -import path from "path"; import { createFixture, createAppFixture, js } from "./helpers/create-fixture"; import type { Fixture, AppFixture } from "./helpers/create-fixture"; @@ -11,12 +10,9 @@ test.describe("actions", () => { let FIELD_NAME = "message"; let WAITING_VALUE = "Waiting..."; - let ACTION_DATA_VALUE = "heyooo, data from the action:"; let SUBMITTED_VALUE = "Submission"; let THROWS_REDIRECT = "redirect-throw"; let REDIRECT_TARGET = "page"; - let HAS_FILE_ACTIONS = "file-actions"; - let MAX_FILE_UPLOAD_SIZE = 1234; let PAGE_TEXT = "PAGE_TEXT"; test.beforeAll(async () => { @@ -69,94 +65,6 @@ test.describe("actions", () => { return
${PAGE_TEXT}
} `, - - [`app/routes/${HAS_FILE_ACTIONS}.jsx`]: js` - import { - json, - unstable_parseMultipartFormData as parseMultipartFormData, - unstable_createFileUploadHandler as createFileUploadHandler, - } from "@remix-run/node"; - import { Form, useActionData } from "@remix-run/react"; - - export async function action({ request }) { - const uploadHandler = createFileUploadHandler({ - directory: ".tmp/uploads", - maxFileSize: ${MAX_FILE_UPLOAD_SIZE}, - // You probably do *not* want to do this in prod. - // We passthrough the name and allow conflicts for test fixutres. - avoidFileConflicts: false, - file: ({ filename }) => filename, - }); - - let files = []; - let formData = await parseMultipartFormData(request, uploadHandler); - - let file = formData.get("file"); - if (file && typeof file !== "string") { - files.push({ name: file.name, size: file.size }); - } - - return json( - { - files, - message: "${ACTION_DATA_VALUE} " + formData.get("field1"), - }, - { - headers: { - "x-test": "works", - }, - } - ); - }; - - export function headers({ actionHeaders }) { - return { - "x-test": actionHeaders.get("x-test"), - }; - }; - - export function ErrorBoundary({ error }) { - return ( -
-

Actions Error Boundary

-

{error.message}

-
- ); - } - - export default function Actions() { - let { files, message } = useActionData() || {}; - - return ( -
-

- {message ? {message} : "${WAITING_VALUE}"} -

- {files ? ( -
    - {files.map((file) => ( -
  • - 
    -                          {JSON.stringify(file, null, 2)}
-
    -
    • - ))} -
- ) : null} -

- - -

-

- - -

-
- ); - } - `, }, }); @@ -229,116 +137,4 @@ test.describe("actions", () => { expect(new URL(page.url()).pathname).toBe(`/${REDIRECT_TARGET}`); expect(await app.getHtml()).toMatch(PAGE_TEXT); }); - - test("can upload file with JavaScript", async ({ page }) => { - let app = new PlaywrightFixture(appFixture, page); - await app.goto(`/${HAS_FILE_ACTIONS}`); - - let html = await app.getHtml("#action-text"); - expect(html).toMatch(WAITING_VALUE); - - await app.uploadFile( - "#file", - path.resolve(__dirname, "assets/toupload.txt") - ); - - await page.click("button[type=submit]"); - await page.waitForSelector("#action-data"); - - html = await app.getHtml("#action-text"); - expect(html).toMatch(ACTION_DATA_VALUE + " stuff"); - }); - - // TODO: figure out what the heck is wrong with this test... - // For some reason the error message is "Unexpected Server Error" in the test - // but if you try the app in the browser it works as expected. - test.skip("rejects too big of an upload with JavaScript", async ({ - page, - }) => { - let app = new PlaywrightFixture(appFixture, page); - await app.goto(`/${HAS_FILE_ACTIONS}`); - - let html = await app.getHtml("#action-text"); - expect(html).toMatch(WAITING_VALUE); - - await app.uploadFile( - "#file", - path.resolve(__dirname, "assets/touploadtoobig.txt") - ); - - await page.click("button[type=submit]"); - await page.waitForSelector("#actions-error-boundary"); - - let text = await app.getHtml("#actions-error-text"); - expect(text).toMatch( - `Field "file" exceeded upload size of ${MAX_FILE_UPLOAD_SIZE} bytes` - ); - - let logs: string[] = []; - page.on("console", (msg) => { - logs.push(msg.text()); - }); - expect(logs).toHaveLength(1); - expect(logs[0]).toMatch(/exceeded upload size/i); - }); - - test.describe("without JavaScript", () => { - test.use({ javaScriptEnabled: false }); - - test("can upload file", async ({ page }) => { - let app = new PlaywrightFixture(appFixture, page); - await app.goto(`/${HAS_FILE_ACTIONS}`); - - let html = await app.getHtml("#action-text"); - expect(html).toMatch(WAITING_VALUE); - - await app.uploadFile( - "#file", - path.resolve(__dirname, "assets/toupload.txt") - ); - - let [response] = await Promise.all([ - page.waitForNavigation(), - page.click("#submit"), - ]); - - expect(response!.status()).toBe(200); - expect(response!.headers()["x-test"]).toBe("works"); - - html = await app.getHtml("#action-text"); - expect(html).toMatch(ACTION_DATA_VALUE + " stuff"); - }); - - // TODO: figure out what the heck is wrong with this test... - // "Failed to load resource: the server responded with a status of 500 (Internal Server Error)" - test.skip("rejects too big of an upload", async ({ page }) => { - let app = new PlaywrightFixture(appFixture, page); - let logs: string[] = []; - page.on("console", (msg) => { - logs.push(msg.text()); - }); - - await app.goto(`/${HAS_FILE_ACTIONS}`); - - let html = await app.getHtml("#action-text"); - expect(html).toMatch(WAITING_VALUE); - - await app.uploadFile( - "#file", - path.resolve(__dirname, "assets/touploadtoobig.txt") - ); - - let [response] = await Promise.all([ - page.waitForNavigation(), - page.click("#submit"), - ]); - expect(response!.status()).toBe(500); - let text = await app.getHtml("#actions-error-text"); - let errorMessage = `Field "file" exceeded upload size of ${MAX_FILE_UPLOAD_SIZE} bytes`; - expect(text).toMatch(errorMessage); - - expect(logs).toHaveLength(1); - expect(logs[0]).toMatch(/error running.*action.*routes\/file-actions/i); - }); - }); }); diff --git a/integration/upload-test.ts b/integration/upload-test.ts new file mode 100644 index 00000000000..6407d1609a3 --- /dev/null +++ b/integration/upload-test.ts @@ -0,0 +1,220 @@ +import * as path from "path"; +import { test, expect } from "@playwright/test"; + +import { PlaywrightFixture } from "./helpers/playwright-fixture"; +import type { Fixture, AppFixture } from "./helpers/create-fixture"; +import { createAppFixture, createFixture, js } from "./helpers/create-fixture"; + +let fixture: Fixture; +let appFixture: AppFixture; + +test.beforeAll(async () => { + fixture = await createFixture({ + files: { + "app/routes/file-upload-handler.jsx": js` + import { + json, + unstable_createFileUploadHandler as createFileUploadHandler, + unstable_parseMultipartFormData as parseMultipartFormData, + MeterError, + } from "@remix-run/node"; + import { Form, useActionData } from "@remix-run/react"; + + export let action = async ({ request }) => { + let uploadHandler = createFileUploadHandler({ + directory: "./uploads", + maxFileSize: 15, + avoidFileConflicts: false, + file: ({ filename }) => filename, + }); + + try { + let formData = await parseMultipartFormData(request, uploadHandler); + let file = formData.get("file"); + let size = typeof file !== "string" && file ? file.size : 0; + + return json({ message: "SUCCESS", size }); + } catch (error) { + if (error instanceof MeterError) { + return json({ message: "FILE_TOO_LARGE", size: error.maxBytes }); + } + return json({ message: "ERROR" }, 500); + } + }; + + export default function FileUpload() { + let { message, size } = useActionData() || {}; + return ( +
+
+ +
+ +
+ + {message &&

{message}

} + {size &&

{size}

} +
+
+ ); + } + `, + + "app/routes/memory-upload-handler.jsx": js` + import { + json, + unstable_createMemoryUploadHandler as createMemoryUploadHandler, + unstable_parseMultipartFormData as parseMultipartFormData, + MeterError, + } from "@remix-run/node"; + import { Form, useActionData } from "@remix-run/react"; + + export let action = async ({ request }) => { + let uploadHandler = createMemoryUploadHandler({ + maxFileSize: 15, + }); + + try { + let formData = await parseMultipartFormData(request, uploadHandler); + let file = formData.get("file"); + let size = typeof file !== "string" && file ? file.size : 0; + + return json({ message: "SUCCESS", size }); + } catch (error) { + if (error instanceof MeterError) { + return json({ message: "FILE_TOO_LARGE", size: error.maxBytes }); + } + return json({ message: "ERROR" }, 500); + } + }; + + export default function MemoryUpload() { + let { message, size } = useActionData() || {}; + return ( +
+
+ +
+ +
+ + {message &&

{message}

} + {size &&

{size}

} +
+
+ ); + } + `, + }, + }); + + appFixture = await createAppFixture(fixture); +}); + +test.afterAll(async () => appFixture.close()); + +test("can upload a file with createFileUploadHandler", async ({ page }) => { + let app = new PlaywrightFixture(appFixture, page); + await app.goto("/file-upload-handler"); + await app.uploadFile("#file", path.resolve(__dirname, "assets/toupload.txt")); + await app.clickSubmitButton("/file-upload-handler"); + + expect(await app.getHtml("#message")).toMatch(">SUCCESS<"); + expect(await app.getHtml("#size")).toMatch(">14<"); +}); + +test("can catch MeterError when file is too big with createFileUploadHandler", async ({ page }) => { + let app = new PlaywrightFixture(appFixture, page); + await app.goto("/file-upload-handler"); + await app.uploadFile( + "#file", + path.resolve(__dirname, "assets/touploadtoobig.txt") + ); + await app.clickSubmitButton("/file-upload-handler"); + + expect(await app.getHtml("#message")).toMatch(">FILE_TOO_LARGE<"); + expect(await app.getHtml("#size")).toMatch(">15<"); +}); + +test("can upload a file with createMemoryUploadHandler", async ({ page }) => { + let app = new PlaywrightFixture(appFixture, page); + await app.goto("/memory-upload-handler"); + await app.uploadFile("#file", path.resolve(__dirname, "assets/toupload.txt")); + await app.clickSubmitButton("/memory-upload-handler"); + + expect(await app.getHtml("#message")).toMatch(">SUCCESS<"); + expect(await app.getHtml("#size")).toMatch(">14<"); +}); + +test("can catch MeterError when file is too big with createMemoryUploadHandler", async ({ page }) => { + let app = new PlaywrightFixture(appFixture, page); + await app.goto("/memory-upload-handler"); + await app.uploadFile( + "#file", + path.resolve(__dirname, "assets/touploadtoobig.txt") + ); + await app.clickSubmitButton("/memory-upload-handler"); + + expect(await app.getHtml("#message")).toMatch(">FILE_TOO_LARGE<"); + expect(await app.getHtml("#size")).toMatch(">15<"); +}); + +test.describe("without javascript", () => { + test.use({ javaScriptEnabled: false }); + + test("can upload a file with createFileUploadHandler", async ({ page }) => { + let app = new PlaywrightFixture(appFixture, page); + await app.goto("/file-upload-handler"); + await app.uploadFile( + "#file", + path.resolve(__dirname, "assets/toupload.txt") + ); + + await Promise.all([page.click("#submit"), page.waitForNavigation()]); + + expect(await app.getHtml("#message")).toMatch(">SUCCESS<"); + expect(await app.getHtml("#size")).toMatch(">14<"); + }); + + test("can catch MeterError when file is too big with createFileUploadHandler", async ({ page }) => { + let app = new PlaywrightFixture(appFixture, page); + await app.goto("/file-upload-handler"); + await app.uploadFile( + "#file", + path.resolve(__dirname, "assets/touploadtoobig.txt") + ); + + await Promise.all([page.click("#submit"), page.waitForNavigation()]); + + expect(await app.getHtml("#message")).toMatch(">FILE_TOO_LARGE<"); + expect(await app.getHtml("#size")).toMatch(">15<"); + }); + + test("can upload a file with createMemoryUploadHandler", async ({ page }) => { + let app = new PlaywrightFixture(appFixture, page); + await app.goto("/memory-upload-handler"); + await app.uploadFile( + "#file", + path.resolve(__dirname, "assets/toupload.txt") + ); + + await Promise.all([page.click("#submit"), page.waitForNavigation()]); + + expect(await app.getHtml("#message")).toMatch(">SUCCESS<"); + expect(await app.getHtml("#size")).toMatch(">14<"); + }); + + test("can catch MeterError when file is too big with createMemoryUploadHandler", async ({ page }) => { + let app = new PlaywrightFixture(appFixture, page); + await app.goto("/memory-upload-handler"); + await app.uploadFile( + "#file", + path.resolve(__dirname, "assets/touploadtoobig.txt") + ); + + await Promise.all([page.click("#submit"), page.waitForNavigation()]); + + expect(await app.getHtml("#message")).toMatch(">FILE_TOO_LARGE<"); + expect(await app.getHtml("#size")).toMatch(">15<"); + }); +}); diff --git a/packages/remix-express/server.ts b/packages/remix-express/server.ts index ffe5009681d..30af6f9d7bf 100644 --- a/packages/remix-express/server.ts +++ b/packages/remix-express/server.ts @@ -1,4 +1,3 @@ -import { PassThrough } from "stream"; import type * as express from "express"; import type { AppLoadContext, @@ -6,13 +5,13 @@ import type { RequestInit as NodeRequestInit, Response as NodeResponse, } from "@remix-run/node"; -import { ReadableStream } from "@remix-run/web-stream"; import { // This has been added as a global in node 15+ AbortController, createRequestHandler as createRemixRequestHandler, Headers as NodeHeaders, Request as NodeRequest, + pipeReadableStreamToWritable, } from "@remix-run/node"; /** @@ -61,12 +60,9 @@ export function createRequestHandler({ ? getLoadContext(req, res) : undefined; - let response = await handleRequest( - request as unknown as Request, - loadContext - ); + let response = await handleRequest(request, loadContext); - sendRemixResponse(res, response, abortController); + sendRemixResponse(res, response as NodeResponse, abortController); } catch (error) { // Express doesn't support async functions, so we have to pass along the // error manually using next(). @@ -77,7 +73,7 @@ export function createRequestHandler({ export function createRemixHeaders( requestHeaders: express.Request["headers"] -): Headers { +): NodeHeaders { let headers = new NodeHeaders(); for (let [key, values] of Object.entries(requestHeaders)) { @@ -110,16 +106,7 @@ export function createRemixRequest( }; if (req.method !== "GET" && req.method !== "HEAD") { - init.body = new ReadableStream({ - start(controller) { - req.on("data", (chunk) => { - controller.enqueue(chunk); - }); - req.on("end", () => { - controller.close(); - }); - }, - }); + init.body = req; } return new NodeRequest(url.href, init); @@ -127,14 +114,14 @@ export function createRemixRequest( export function sendRemixResponse( res: express.Response, - nodeResponse: Response, + nodeResponse: NodeResponse, abortController: AbortController ): void { res.statusMessage = nodeResponse.statusText; res.status(nodeResponse.status); for (let [key, values] of Object.entries( - (nodeResponse.headers as any).raw() as Record + (nodeResponse.headers as NodeHeaders).raw() )) { for (let value of values) { res.append(key, value); @@ -146,26 +133,8 @@ export function sendRemixResponse( } if (nodeResponse.body) { - let reader = nodeResponse.body.getReader(); - async function read() { - let { done, value } = await reader.read(); - if (done) { - res.end(value); - return; - } - - res.write(value); - read(); - } - read(); + pipeReadableStreamToWritable(nodeResponse.body, res); } else { res.end(); } - // if (Buffer.isBuffer(nodeResponse.body)) { - // res.end(nodeResponse.body); - // } else if (nodeResponse.body?.pipe) { - // nodeResponse.body.pipe(res); - // } else { - // res.end(); - // } } diff --git a/packages/remix-netlify/server.ts b/packages/remix-netlify/server.ts index 2dccf375a60..720e0c4a7f1 100644 --- a/packages/remix-netlify/server.ts +++ b/packages/remix-netlify/server.ts @@ -4,6 +4,7 @@ import { createRequestHandler as createRemixRequestHandler, Headers as NodeHeaders, Request as NodeRequest, + readableStreamToBase64String, } from "@remix-run/node"; import type { Handler, @@ -15,6 +16,7 @@ import type { AppLoadContext, ServerBuild, RequestInit as NodeRequestInit, + Response as NodeResponse, } from "@remix-run/node"; import { isBinaryType } from "./binaryTypes"; @@ -52,12 +54,9 @@ export function createRequestHandler({ ? getLoadContext(event, context) : undefined; - let response = await handleRequest( - request as unknown as Request, - loadContext - ); + let response = await handleRequest(request, loadContext); - return sendRemixResponse(response, abortController); + return sendRemixResponse(response as NodeResponse, abortController); }; } @@ -98,7 +97,7 @@ export function createRemixRequest( export function createRemixHeaders( requestHeaders: HandlerEvent["multiValueHeaders"] -): Headers { +): NodeHeaders { let headers = new NodeHeaders(); for (let [key, values] of Object.entries(requestHeaders)) { @@ -138,7 +137,7 @@ function getRawPath(event: HandlerEvent): string { } export async function sendRemixResponse( - nodeResponse: Response, + nodeResponse: NodeResponse, abortController: AbortController ): Promise { if (abortController.signal.aborted) { @@ -151,26 +150,14 @@ export async function sendRemixResponse( if (nodeResponse.body) { if (isBase64Encoded) { - let reader = nodeResponse.body.getReader(); - body = ""; - async function read() { - let { done, value } = await reader.read(); - if (done) { - return; - } else if (value) { - body += Buffer.from(value).toString("base64"); - } - await read(); - } - - await read(); + body = await readableStreamToBase64String(nodeResponse.body); } else { body = await nodeResponse.text(); } } let multiValueHeaders: Record = ( - nodeResponse.headers as any + nodeResponse.headers as NodeHeaders ).raw(); return { diff --git a/packages/remix-node/__tests__/fetch-test.ts b/packages/remix-node/__tests__/fetch-test.ts index e1dda7584d7..954d48da3e2 100644 --- a/packages/remix-node/__tests__/fetch-test.ts +++ b/packages/remix-node/__tests__/fetch-test.ts @@ -75,6 +75,7 @@ describe("Request", () => { it("clones", async () => { let body = new PassThrough(); test.source.forEach((chunk) => body.write(chunk)); + body.end(); let req = new Request("http://test.com", { method: "post", diff --git a/packages/remix-node/__tests__/formData-test.ts b/packages/remix-node/__tests__/formData-test.ts deleted file mode 100644 index 562c1d08368..00000000000 --- a/packages/remix-node/__tests__/formData-test.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { Blob, File } from "@remix-run/web-file"; - -import { FormData as NodeFormData } from "../formData"; - -describe("FormData", () => { - it("allows for mix of set and append", () => { - let formData = new NodeFormData(); - formData.set("single", "heyo"); - formData.append("multi", "one"); - formData.append("multi", "two"); - - let results = []; - for (let [k, v] of formData) results.push([k, v]); - expect(results).toEqual([ - ["single", "heyo"], - ["multi", "one"], - ["multi", "two"], - ]); - }); - - it("restores correctly empty string values with get method", () => { - let formData = new NodeFormData(); - formData.set("single", ""); - expect(formData.get("single")).toBe(""); - }); - - it("allows for mix of set and append with blobs and files", () => { - let formData = new NodeFormData(); - formData.set("single", new Blob([])); - formData.append("multi", new Blob([])); - formData.append("multi", new File([], "test.txt")); - - expect(formData.getAll("single")).toHaveLength(1); - expect(formData.getAll("multi")).toHaveLength(2); - }); -}); diff --git a/packages/remix-node/__tests__/parseMultipartFormData-test.ts b/packages/remix-node/__tests__/parseMultipartFormData-test.ts index 6d3ee6e89aa..4f210ada051 100644 --- a/packages/remix-node/__tests__/parseMultipartFormData-test.ts +++ b/packages/remix-node/__tests__/parseMultipartFormData-test.ts @@ -1,29 +1,37 @@ -import { FormData as NodeFormData } from "@remix-run/web-fetch"; import { Blob, File } from "@remix-run/web-file"; import { Request as NodeRequest } from "../fetch"; -// import { FormData as NodeFormData } from "../formData"; +import { FormData as NodeFormData } from "../formData"; import { internalParseFormData } from "../parseMultipartFormData"; -import { createMemoryUploadHandler } from "../upload/memoryUploadHandler"; describe("internalParseFormData", () => { - it("plays nice with node-fetch", async () => { + it("can use a custom upload handler", async () => { let formData = new NodeFormData(); formData.set("a", "value"); formData.set("blob", new Blob(["blob"]), "blob.txt"); formData.set("file", new File(["file"], "file.txt")); + // TODO: Figure out why the stream is failing when formData is passed directly as body let req = new NodeRequest("https://test.com", { method: "post", - body: formData as any, + body: formData, + }); + req = new NodeRequest("https://test.com", { + method: "post", + headers: req.headers, + body: await req.text(), }); - let uploadHandler = createMemoryUploadHandler({}); let parsedFormData = await internalParseFormData( req, - req.formData.bind(req), - undefined, - uploadHandler + async ({ filename, data, contentType }) => { + let chunks = []; + for await (let chunk of data) { + chunks.push(chunk); + } + return new File(chunks, filename, { type: contentType }); + }, + undefined ); expect(parsedFormData.get("a")).toBe("value"); @@ -33,4 +41,33 @@ describe("internalParseFormData", () => { expect(file.name).toBe("file.txt"); expect(await file.text()).toBe("file"); }); + + it("can throw errors in upload handlers", async () => { + let formData = new NodeFormData(); + formData.set("blob", new Blob(["blob"]), "blob.txt"); + + // TODO: Figure out why the stream is failing when formData is passed directly as body + let req = new NodeRequest("https://test.com", { + method: "post", + body: formData, + }); + req = new NodeRequest("https://test.com", { + method: "post", + headers: req.headers, + body: await req.text(), + }); + + try { + await internalParseFormData( + req, + async () => { + throw new Error("test error"); + }, + undefined + ); + throw new Error("should have thrown"); + } catch (err) { + expect(err.message).toBe("test error"); + } + }); }); diff --git a/packages/remix-node/fetch.ts b/packages/remix-node/fetch.ts index 4a8f7a30ead..c416be74433 100644 --- a/packages/remix-node/fetch.ts +++ b/packages/remix-node/fetch.ts @@ -1,20 +1,41 @@ +import type { Readable } from "stream"; + import type AbortController from "abort-controller"; import { Request as BaseNodeRequest } from "@remix-run/web-fetch"; import type { UploadHandler } from "./formData"; import { internalParseFormData } from "./parseMultipartFormData"; -export { fetch, Headers, Response } from "@remix-run/web-fetch"; +import type { Headers as NodeHeaders } from "@remix-run/web-fetch"; +import { fetch as nodeFetch } from "@remix-run/web-fetch"; + +export { File, Blob } from "@remix-run/web-file"; +export { Headers, Response } from "@remix-run/web-fetch"; + +type NodeHeadersInit = ConstructorParameters[0]; +type NodeRequestInfo = ConstructorParameters[0]; +type BaseNodeRequestInit = NonNullable< + ConstructorParameters[1] +>; -interface NodeRequestInit extends RequestInit { +type NodeResponseInit = Omit< + NonNullable[1]>, + "body" +> & { + body?: + | NonNullable[1]>["body"] + | Readable; +}; +interface NodeRequestInit extends Omit { abortController?: AbortController; + body?: BaseNodeRequestInit["body"] | Readable; } class NodeRequest extends BaseNodeRequest { private abortController?: AbortController; - constructor(input: RequestInfo, init?: NodeRequestInit | undefined) { - super(input as any, init); + constructor(input: NodeRequestInfo, init?: NodeRequestInit | undefined) { + super(input as any, init as BaseNodeRequestInit); let anyInput = input as any; let anyInit = init as any; @@ -23,22 +44,17 @@ class NodeRequest extends BaseNodeRequest { anyInput?.abortController || anyInit?.abortController; } - async formData(uploadHandler?: UploadHandler): Promise { + formData(uploadHandler?: UploadHandler): Promise { let contentType = this.headers.get("Content-Type"); if ( + uploadHandler && contentType && - (/application\/x-www-form-urlencoded/.test(contentType) || - /multipart\/form-data/.test(contentType)) + /multipart\/form-data/.test(contentType) ) { - return await internalParseFormData( - this, - super.formData.bind(this), - this.abortController, - uploadHandler - ); + return internalParseFormData(this, uploadHandler, this.abortController); } - throw new Error("Invalid MIME type"); + return super.formData(); } clone(): NodeRequest { @@ -46,4 +62,22 @@ class NodeRequest extends BaseNodeRequest { } } -export { NodeRequest as Request, NodeRequestInit as RequestInit }; +export const fetch: typeof nodeFetch = ( + input: NodeRequestInfo, + init?: NodeRequestInit +) => { + init = { + compress: false, + ...init, + }; + + return nodeFetch(input, init as BaseNodeRequestInit); +}; + +export type { + NodeHeadersInit as HeadersInit, + NodeRequestInfo as RequestInfo, + NodeRequestInit as RequestInit, + NodeResponseInit as ResponseInit, +}; +export { NodeRequest as Request }; diff --git a/packages/remix-node/formData.ts b/packages/remix-node/formData.ts index 2e4cb8ceb09..81e9bf0f984 100644 --- a/packages/remix-node/formData.ts +++ b/packages/remix-node/formData.ts @@ -1,125 +1,12 @@ -import type { Readable } from "stream"; +export { FormData } from "@remix-run/web-fetch"; export type UploadHandlerArgs = { name: string; - stream: Readable; filename: string; - encoding: string; - mimetype: string; + contentType: string; + data: AsyncIterable; }; export type UploadHandler = ( args: UploadHandlerArgs ) => Promise; - -function isBlob(value: any): value is Blob { - return ( - typeof value === "object" && - (typeof value.arrayBuffer === "function" || - typeof value.size === "number" || - typeof value.slice === "function" || - typeof value.stream === "function" || - typeof value.text === "function" || - typeof value.type === "string") - ); -} - -export function isFile(blob: Blob): blob is File { - let file = blob as File; - return typeof file.name === "string"; -} - -class NodeFormData implements FormData { - private _fields: Record; - - constructor(form?: any) { - if (typeof form !== "undefined") { - throw new Error("Form data on the server is not supported."); - } - this._fields = {}; - } - - append(name: string, value: string | Blob, fileName?: string): void { - if (typeof value !== "string" && !isBlob(value)) { - throw new Error("formData.append can only accept a string or Blob"); - } - - this._fields[name] = this._fields[name] || []; - if (typeof value === "string" || isFile(value)) { - this._fields[name].push(value); - } else { - this._fields[name].push(new File([value], fileName || "unknown")); - } - } - - delete(name: string): void { - delete this._fields[name]; - } - - get(name: string): FormDataEntryValue | null { - let arr = this._fields[name]; - return arr?.slice(-1)[0] ?? null; - } - - getAll(name: string): FormDataEntryValue[] { - let arr = this._fields[name]; - return arr || []; - } - - has(name: string): boolean { - return name in this._fields; - } - - set(name: string, value: string | Blob, fileName?: string): void { - if (typeof value !== "string" && !isBlob(value)) { - throw new Error("formData.set can only accept a string or Blob"); - } - - if (typeof value === "string" || isFile(value)) { - this._fields[name] = [value]; - } else { - this._fields[name] = [new File([value], fileName || "unknown")]; - } - } - - forEach( - callbackfn: ( - value: FormDataEntryValue, - key: string, - parent: FormData - ) => void, - thisArg?: any - ): void { - Object.entries(this._fields).forEach(([name, values]) => { - values.forEach((value) => callbackfn(value, name, thisArg), thisArg); - }); - } - - entries(): IterableIterator<[string, FormDataEntryValue]> { - return Object.entries(this._fields) - .reduce((entries, [name, values]) => { - values.forEach((value) => entries.push([name, value])); - return entries; - }, [] as [string, FormDataEntryValue][]) - .values(); - } - - keys(): IterableIterator { - return Object.keys(this._fields).values(); - } - - values(): IterableIterator { - return Object.entries(this._fields) - .reduce((results, [name, values]) => { - values.forEach((value) => results.push(value)); - return results; - }, [] as FormDataEntryValue[]) - .values(); - } - - *[Symbol.iterator](): IterableIterator<[string, FormDataEntryValue]> { - yield* this.entries(); - } -} - -export { NodeFormData as FormData }; diff --git a/packages/remix-node/globals.ts b/packages/remix-node/globals.ts index 24669c07e0c..e1c74ea1b80 100644 --- a/packages/remix-node/globals.ts +++ b/packages/remix-node/globals.ts @@ -1,7 +1,7 @@ -import { Blob as NodeBlob, File as NodeFile } from "@remix-run/web-file"; - import { atob, btoa } from "./base64"; import { + Blob as NodeBlob, + File as NodeFile, Headers as NodeHeaders, Request as NodeRequest, Response as NodeResponse, diff --git a/packages/remix-node/index.ts b/packages/remix-node/index.ts index 9c1a2fc9921..0b305188882 100644 --- a/packages/remix-node/index.ts +++ b/packages/remix-node/index.ts @@ -7,8 +7,7 @@ export { AbortController } from "abort-controller"; export type { // HeadersInit, // RequestInfo, - RequestInit, - // ResponseInit, + RequestInit, // ResponseInit, } from "./fetch"; export { Headers, Request, Response, fetch } from "./fetch"; @@ -26,6 +25,7 @@ export { NodeOnDiskFile, } from "./upload/fileUploadHandler"; export { createMemoryUploadHandler as unstable_createMemoryUploadHandler } from "./upload/memoryUploadHandler"; +export { MeterError } from "./upload/meter"; export { createCookie, @@ -34,6 +34,11 @@ export { createSessionStorage, } from "./implementations"; +export { + pipeReadableStreamToWritable, + readableStreamToBase64String, +} from "./stream"; + export { createRequestHandler, createSession, diff --git a/packages/remix-node/package.json b/packages/remix-node/package.json index c2f453124bc..91f02f1e026 100644 --- a/packages/remix-node/package.json +++ b/packages/remix-node/package.json @@ -13,12 +13,12 @@ }, "dependencies": { "@remix-run/server-runtime": "1.4.2", - "@types/busboy": "^0.3.1", "@remix-run/web-fetch": "^4.1.0", "@remix-run/web-file": "^3.0.2", + "@remix-run/web-stream": "^1.0.2", + "@web3-storage/multipart-parser": "^1.0.0", "abort-controller": "^3.0.0", "blob-stream": "^0.1.3", - "busboy": "^0.3.1", "cookie-signature": "^1.1.0", "form-data": "^4.0.0", "source-map-support": "^0.5.21" diff --git a/packages/remix-node/parseMultipartFormData.ts b/packages/remix-node/parseMultipartFormData.ts index 270ff91c572..e62b0ff807d 100644 --- a/packages/remix-node/parseMultipartFormData.ts +++ b/packages/remix-node/parseMultipartFormData.ts @@ -1,6 +1,5 @@ -import { PassThrough } from "stream"; -import Busboy from "busboy"; import { FormData } from "@remix-run/web-fetch"; +import { streamMultipart } from "@web3-storage/multipart-parser"; import type { Request as NodeRequest } from "./fetch"; import type { UploadHandler } from "./formData"; @@ -11,121 +10,60 @@ import type { UploadHandler } from "./formData"; * @see https://remix.run/api/remix#parsemultipartformdata-node */ export function parseMultipartFormData( - request: Request, + request: Request | NodeRequest, uploadHandler: UploadHandler ) { return (request as unknown as NodeRequest).formData(uploadHandler); } -export async function internalParseFormData( - request: Request, - internalFormData: any, - abortController?: AbortController, - uploadHandler?: UploadHandler -) { - if (!uploadHandler) { - return internalFormData(); - } - - let formData = new FormData(); +export const internalParseFormData = async ( + request: NodeRequest, + uploadHandler: UploadHandler, + abortController: AbortController | undefined +) => { let contentType = request.headers.get("Content-Type") || ""; - let fileWorkQueue: Promise[] = []; + let [type, boundary] = contentType.split(/\s*;\s*boundary=/); - let stream: PassThrough = new PassThrough(); - if (request.body) { - let reader = request.body.getReader(); - async function read() { - let { done, value } = await reader.read(); - if (done) { - stream.end(value); - return; - } - stream.write(value); - read(); - } - read(); - } else { - stream.end(); + if (!request.body || !boundary || type !== "multipart/form-data") { + throw new TypeError("Could not parse content as FormData."); } - await new Promise(async (resolve, reject) => { - try { - let busboy = new Busboy({ - highWaterMark: 2 * 1024 * 1024, - headers: { - "content-type": contentType, - }, - }); + let formData = new FormData(); - let aborted = false; - function abort(error?: Error) { - if (aborted) return; - aborted = true; + let parts = streamMultipart(request.clone().body, boundary); - stream.unpipe(); - stream.removeAllListeners(); - busboy.removeAllListeners(); + for await (let part of parts) { + if (part.done) break; - abortController?.abort(); - reject(error || new Error("failed to parse form data")); + if (!part.filename) { + let chunks = []; + for await (let chunk of part.data) { + chunks.push(chunk); } - busboy.on("field", (name, value) => { - formData.append(name, value); - }); - - busboy.on("file", (name, filestream, filename, encoding, mimetype) => { - if (uploadHandler) { - fileWorkQueue.push( - (async () => { - try { - let value = await uploadHandler({ - name, - stream: filestream, - filename, - encoding, - mimetype, - }); - - if (typeof value !== "undefined") { - formData.append(name, value); - } - } catch (error: any) { - // Emit error to busboy to bail early if possible - busboy.emit("error", error); - // It's possible that the handler is doing stuff and fails - // *after* busboy has finished. Rethrow the error for surfacing - // in the Promise.all(fileWorkQueue) below. - throw error; - } finally { - filestream.resume(); - } - })() - ); - } else { - filestream.resume(); - } - - if (!uploadHandler) { - console.warn( - `Tried to parse multipart file upload for field "${name}" but no uploadHandler was provided.` + - " Read more here: https://remix.run/api/remix#parseMultipartFormData-node" - ); - } - }); - - stream.on("error", abort); - stream.on("aborted", abort); - busboy.on("error", abort); - busboy.on("finish", resolve); - - stream.pipe(busboy); - } catch (err) { - reject(err); + formData.append( + part.name, + new TextDecoder().decode(mergeArrays(...chunks)) + ); + } else { + let file = await uploadHandler(part); + if (typeof file !== "undefined") { + formData.append(part.name, file); + } } - }); - - await Promise.all(fileWorkQueue); + } return formData; +}; + +export function mergeArrays(...arrays: Uint8Array[]) { + const out = new Uint8Array( + arrays.reduce((total, arr) => total + arr.length, 0) + ); + let offset = 0; + for (const arr of arrays) { + out.set(arr, offset); + offset += arr.length; + } + return out; } diff --git a/packages/remix-node/stream.ts b/packages/remix-node/stream.ts new file mode 100644 index 00000000000..2779c922c36 --- /dev/null +++ b/packages/remix-node/stream.ts @@ -0,0 +1,39 @@ +import type { Writable } from "stream"; + +export function pipeReadableStreamToWritable( + stream: ReadableStream, + writable: Writable +) { + const reader = stream.getReader(); + + async function read() { + const { done, value } = await reader.read(); + if (done) { + writable.end(); + return; + } + + writable.write(value); + read(); + } + + read(); +} + +export async function readableStreamToBase64String(stream: ReadableStream) { + let reader = stream.getReader(); + let body = ""; + async function read() { + let { done, value } = await reader.read(); + if (done) { + return; + } else if (value) { + body += Buffer.from(value).toString("base64"); + } + await read(); + } + + await read(); + + return body; +} diff --git a/packages/remix-node/tsconfig.json b/packages/remix-node/tsconfig.json index 33578ea0891..7b510fdf637 100644 --- a/packages/remix-node/tsconfig.json +++ b/packages/remix-node/tsconfig.json @@ -1,7 +1,7 @@ { "exclude": ["__tests__"], "compilerOptions": { - "lib": ["ES2019", "DOM.Iterable"], + "lib": ["ES2019", "DOM", "DOM.Iterable"], "target": "ES2019", "moduleResolution": "node", diff --git a/packages/remix-node/upload/fileUploadHandler.ts b/packages/remix-node/upload/fileUploadHandler.ts index 7593d4893e5..52c5c0ca868 100644 --- a/packages/remix-node/upload/fileUploadHandler.ts +++ b/packages/remix-node/upload/fileUploadHandler.ts @@ -4,19 +4,19 @@ import { rm, mkdir, readFile, stat } from "fs/promises"; import { tmpdir } from "os"; import { basename, dirname, extname, resolve as resolvePath } from "path"; -import { Meter } from "./meter"; +import { MeterError } from "./meter"; import type { UploadHandler } from "../formData"; export type FileUploadHandlerFilterArgs = { filename: string; - encoding: string; - mimetype: string; + contentType: string; + name: string; }; export type FileUploadHandlerPathResolverArgs = { filename: string; - encoding: string; - mimetype: string; + contentType: string; + name: string; }; /** @@ -87,29 +87,26 @@ export function createFileUploadHandler({ filter, maxFileSize = 3000000, }: FileUploadHandlerOptions): UploadHandler { - return async ({ name, stream, filename, encoding, mimetype }) => { - if (filter && !(await filter({ filename, encoding, mimetype }))) { - stream.resume(); - return; + return async ({ name, filename, contentType, data }) => { + if (filter && !(await filter({ name, filename, contentType }))) { + return undefined; } let dir = typeof directory === "string" ? directory - : directory({ filename, encoding, mimetype }); + : directory({ name, filename, contentType }); if (!dir) { - stream.resume(); - return; + return undefined; } let filedir = resolvePath(dir); let path = - typeof file === "string" ? file : file({ filename, encoding, mimetype }); + typeof file === "string" ? file : file({ name, filename, contentType }); if (!path) { - stream.resume(); - return; + return undefined; } let filepath = resolvePath(filedir, path); @@ -120,35 +117,21 @@ export function createFileUploadHandler({ await mkdir(dirname(filepath), { recursive: true }).catch(() => {}); - let meter = new Meter(name, maxFileSize); - await new Promise((resolve, reject) => { - let writeFileStream = createWriteStream(filepath); - - let aborted = false; - async function abort(error: Error) { - if (aborted) return; - aborted = true; - - stream.unpipe(); - meter.unpipe(); - stream.removeAllListeners(); - meter.removeAllListeners(); - writeFileStream.removeAllListeners(); - - await rm(filepath, { force: true }).catch(() => {}); - - reject(error); + let writeFileStream = createWriteStream(filepath); + let size = 0; + try { + for await (let chunk of data) { + size += chunk.length; + if (size > maxFileSize) { + throw new MeterError(name, maxFileSize); + } + writeFileStream.write(chunk); } + } finally { + writeFileStream.close(); + } - stream.on("error", abort); - meter.on("error", abort); - writeFileStream.on("error", abort); - writeFileStream.on("finish", resolve); - - stream.pipe(meter).pipe(writeFileStream); - }); - - return new NodeOnDiskFile(filepath, meter.bytes, mimetype); + return new NodeOnDiskFile(filepath, size, contentType); }; } @@ -189,6 +172,6 @@ export class NodeOnDiskFile implements File { } get [Symbol.toStringTag]() { - return "File" + return "File"; } } diff --git a/packages/remix-node/upload/memoryUploadHandler.ts b/packages/remix-node/upload/memoryUploadHandler.ts index ed671540aac..8e393b69042 100644 --- a/packages/remix-node/upload/memoryUploadHandler.ts +++ b/packages/remix-node/upload/memoryUploadHandler.ts @@ -1,14 +1,14 @@ import type { TransformCallback } from "stream"; import { Transform } from "stream"; -import { File as BufferFile } from "@remix-run/web-file"; +import { File } from "../fetch"; -import { Meter } from "./meter"; +import { MeterError } from "./meter"; import type { UploadHandler } from "../formData"; export type MemoryUploadHandlerFilterArgs = { filename: string; - encoding: string; - mimetype: string; + contentType: string; + name: string; }; export type MemoryUploadHandlerOptions = { @@ -30,42 +30,61 @@ export function createMemoryUploadHandler({ filter, maxFileSize = 3000000, }: MemoryUploadHandlerOptions): UploadHandler { - return async ({ name, stream, filename, encoding, mimetype }) => { - if (filter && !(await filter({ filename, encoding, mimetype }))) { - stream.resume(); - return; + return async ({ filename, contentType, name, data }) => { + if (filter && !(await filter({ filename, contentType, name }))) { + return undefined; } - let bufferStream = new BufferStream(); - await new Promise((resolve, reject) => { - let meter = new Meter(name, maxFileSize); + let size = 0; + const chunks = []; + for await (let chunk of data) { + chunks.push(chunk); + size += chunk.length; + if (size > maxFileSize) { + throw new MeterError(name, maxFileSize); + } + } - let aborted = false; - async function abort(error: Error) { - if (aborted) return; - aborted = true; + const file = new File(chunks, filename, { type: contentType }); - stream.unpipe(); - meter.unpipe(); - stream.removeAllListeners(); - meter.removeAllListeners(); - bufferStream.removeAllListeners(); + return file; + }; + // return async ({ name, stream, filename, encoding, mimetype }) => { + // if (filter && !(await filter({ filename, encoding, mimetype }))) { + // stream.resume(); + // return; + // } - reject(error); - } + // let bufferStream = new BufferStream(); + // await new Promise((resolve, reject) => { + // let meter = new Meter(name, maxFileSize); - stream.on("error", abort); - meter.on("error", abort); - bufferStream.on("error", abort); - bufferStream.on("finish", resolve); + // let aborted = false; + // async function abort(error: Error) { + // if (aborted) return; + // aborted = true; - stream.pipe(meter).pipe(bufferStream); - }); + // stream.unpipe(); + // meter.unpipe(); + // stream.removeAllListeners(); + // meter.removeAllListeners(); + // bufferStream.removeAllListeners(); - return new BufferFile(bufferStream.data, filename, { - type: mimetype, - }); - }; + // reject(error); + // } + + // stream.on("error", abort); + // meter.on("error", abort); + // bufferStream.on("error", abort); + // bufferStream.on("finish", resolve); + + // stream.pipe(meter).pipe(bufferStream); + // }); + + // return new BufferFile(bufferStream.data, filename, { + // type: mimetype, + // }); + // }; } class BufferStream extends Transform { diff --git a/packages/remix-node/upload/meter.ts b/packages/remix-node/upload/meter.ts index 01e64b96410..10300610994 100644 --- a/packages/remix-node/upload/meter.ts +++ b/packages/remix-node/upload/meter.ts @@ -1,26 +1,3 @@ -import type { TransformCallback } from "stream"; -import { Transform } from "stream"; - -export class Meter extends Transform { - public bytes: number; - - constructor(public field: string, public maxBytes: number | undefined) { - super(); - this.bytes = 0; - } - - _transform(chunk: any, _: BufferEncoding, callback: TransformCallback) { - this.bytes += chunk.length; - this.push(chunk); - - if (typeof this.maxBytes === "number" && this.bytes > this.maxBytes) { - return callback(new MeterError(this.field, this.maxBytes)); - } - - callback(); - } -} - export class MeterError extends Error { constructor(public field: string, public maxBytes: number) { super(`Field "${field}" exceeded upload size of ${maxBytes} bytes.`); diff --git a/packages/remix-vercel/server.ts b/packages/remix-vercel/server.ts index dff20849c60..fc31669f855 100644 --- a/packages/remix-vercel/server.ts +++ b/packages/remix-vercel/server.ts @@ -3,6 +3,7 @@ import type { AppLoadContext, ServerBuild, RequestInit as NodeRequestInit, + Response as NodeResponse, } from "@remix-run/node"; import { // This has been added as a global in node 15+ @@ -10,6 +11,7 @@ import { createRequestHandler as createRemixRequestHandler, Headers as NodeHeaders, Request as NodeRequest, + pipeReadableStreamToWritable, } from "@remix-run/node"; /** @@ -61,13 +63,13 @@ export function createRequestHandler({ response.headers.set("Connection", "close"); } - sendRemixResponse(res, response); + sendRemixResponse(res, response as NodeResponse); }; } export function createRemixHeaders( requestHeaders: VercelRequest["headers"] -): Headers { +): NodeHeaders { let headers = new NodeHeaders(); for (let key in requestHeaders) { let header = requestHeaders[key]!; @@ -101,16 +103,7 @@ export function createRemixRequest( }; if (req.method !== "GET" && req.method !== "HEAD") { - init.body = new ReadableStream({ - start(controller) { - req.on("data", (chunk) => { - controller.enqueue(chunk); - }); - req.on("end", () => { - controller.close(); - }); - }, - }); + init.body = req; } return new NodeRequest(url.href, init); @@ -118,12 +111,12 @@ export function createRemixRequest( export function sendRemixResponse( res: VercelResponse, - nodeResponse: Response + nodeResponse: NodeResponse ): void { res.statusMessage = nodeResponse.statusText; let multiValueHeaders: Record = {}; for (let [key, values] of Object.entries( - (nodeResponse.headers as any).raw() as Record + (nodeResponse.headers as NodeHeaders).raw() )) { if (typeof multiValueHeaders[key] === "undefined") { multiValueHeaders[key] = [...values]; @@ -138,18 +131,7 @@ export function sendRemixResponse( ); if (nodeResponse.body) { - let reader = nodeResponse.body.getReader(); - async function read() { - let { done, value } = await reader.read(); - if (done) { - res.end(value); - return; - } - - res.write(value); - read(); - } - read(); + pipeReadableStreamToWritable(nodeResponse.body, res); } else { res.end(); } diff --git a/yarn.lock b/yarn.lock index 25939a3ab45..35f9f6e58f5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1602,7 +1602,7 @@ dependencies: web-encoding "1.1.5" -"@remix-run/web-stream@^1.0.0", "@remix-run/web-stream@^1.0.1": +"@remix-run/web-stream@^1.0.0", "@remix-run/web-stream@^1.0.1", "@remix-run/web-stream@^1.0.2": version "1.0.2" resolved "https://registry.npmjs.org/@remix-run/web-stream/-/web-stream-1.0.2.tgz#f07dc9cf6db02507ea71a234bc8e06103a2207b4" integrity sha512-FO4om5mrwMs5bi7L5hbLMP1hm+flAS2oYRptfNPkK2u0Hhv0crS9GiE9/MsVvY53tTAxVkzUG/m+9ET1mTjEnw== From ff89d630f7c9edd4d4e05953cdbf7f7ab799bea1 Mon Sep 17 00:00:00 2001 From: Jacob Ebey Date: Wed, 4 May 2022 17:48:05 -0700 Subject: [PATCH 14/47] chore: make types proper for node fetch feat: remove abort controller from adapters and request --- .../remix-architect/__tests__/server-test.ts | 15 ++-- packages/remix-architect/server.ts | 26 ++---- .../remix-express/__tests__/server-test.ts | 15 ++-- packages/remix-express/package.json | 1 - packages/remix-express/server.ts | 23 ++--- .../remix-netlify/__tests__/server-test.ts | 15 ++-- packages/remix-netlify/server.ts | 23 ++--- packages/remix-node/__tests__/fetch-test.ts | 6 +- .../__tests__/parseMultipartFormData-test.ts | 13 +-- packages/remix-node/fetch.ts | 89 +++++++++---------- packages/remix-node/package.json | 2 +- packages/remix-node/parseMultipartFormData.ts | 7 +- .../remix-vercel/__tests__/server-test.ts | 15 ++-- packages/remix-vercel/server.ts | 30 +------ yarn.lock | 34 +------ 15 files changed, 102 insertions(+), 212 deletions(-) diff --git a/packages/remix-architect/__tests__/server-test.ts b/packages/remix-architect/__tests__/server-test.ts index 7de8c519003..0dc6bca7a42 100644 --- a/packages/remix-architect/__tests__/server-test.ts +++ b/packages/remix-architect/__tests__/server-test.ts @@ -159,7 +159,7 @@ describe("architect createRemixHeaders", () => { describe("creates fetch headers from architect headers", () => { it("handles empty headers", () => { expect(createRemixHeaders({}, undefined)).toMatchInlineSnapshot(` - Headers$1 { + Headers { Symbol(query): Array [], Symbol(context): null, } @@ -169,7 +169,7 @@ describe("architect createRemixHeaders", () => { it("handles simple headers", () => { expect(createRemixHeaders({ "x-foo": "bar" }, undefined)) .toMatchInlineSnapshot(` - Headers$1 { + Headers { Symbol(query): Array [ "x-foo", "bar", @@ -182,7 +182,7 @@ describe("architect createRemixHeaders", () => { it("handles multiple headers", () => { expect(createRemixHeaders({ "x-foo": "bar", "x-bar": "baz" }, undefined)) .toMatchInlineSnapshot(` - Headers$1 { + Headers { Symbol(query): Array [ "x-foo", "bar", @@ -197,7 +197,7 @@ describe("architect createRemixHeaders", () => { it("handles headers with multiple values", () => { expect(createRemixHeaders({ "x-foo": "bar, baz" }, undefined)) .toMatchInlineSnapshot(` - Headers$1 { + Headers { Symbol(query): Array [ "x-foo", "bar, baz", @@ -211,7 +211,7 @@ describe("architect createRemixHeaders", () => { expect( createRemixHeaders({ "x-foo": "bar, baz", "x-bar": "baz" }, undefined) ).toMatchInlineSnapshot(` - Headers$1 { + Headers { Symbol(query): Array [ "x-foo", "bar, baz", @@ -230,7 +230,7 @@ describe("architect createRemixHeaders", () => { "__other=some_other_value", ]) ).toMatchInlineSnapshot(` - Headers$1 { + Headers { Symbol(query): Array [ "x-something-else", "true", @@ -254,7 +254,6 @@ describe("architect createRemixRequest", () => { ) ).toMatchInlineSnapshot(` NodeRequest { - "abortController": undefined, "agent": undefined, "compress": true, "counter": 0, @@ -271,7 +270,7 @@ describe("architect createRemixRequest", () => { "type": null, }, Symbol(Request internals): Object { - "headers": Headers$1 { + "headers": Headers { Symbol(query): Array [ "accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", diff --git a/packages/remix-architect/server.ts b/packages/remix-architect/server.ts index 42d39c716e7..f137516059c 100644 --- a/packages/remix-architect/server.ts +++ b/packages/remix-architect/server.ts @@ -44,24 +44,17 @@ export function createRequestHandler({ let handleRequest = createRemixRequestHandler(build, mode); return async (event /*, context*/) => { - let abortController = new AbortController(); - let request = createRemixRequest(event, abortController); + let request = createRemixRequest(event); let loadContext = typeof getLoadContext === "function" ? getLoadContext(event) : undefined; - let response = await handleRequest( - request as unknown as Request, - loadContext - ); + let response = await handleRequest(request, loadContext); - return sendRemixResponse(response, abortController); + return sendRemixResponse(response); }; } -export function createRemixRequest( - event: APIGatewayProxyEventV2, - abortController?: AbortController -): NodeRequest { +export function createRemixRequest(event: APIGatewayProxyEventV2): NodeRequest { let host = event.headers["x-forwarded-host"] || event.headers.host; let search = event.rawQueryString.length ? `?${event.rawQueryString}` : ""; let scheme = process.env.ARC_SANDBOX ? "http" : "https"; @@ -79,8 +72,6 @@ export function createRemixRequest( ? Buffer.from(event.body, "base64") : Buffer.from(event.body, "base64").toString() : event.body, - abortController, - signal: abortController?.signal, }); } @@ -104,14 +95,13 @@ export function createRemixHeaders( } export async function sendRemixResponse( - nodeResponse: Response, - abortController: AbortController + nodeResponse: Response ): Promise { let cookies: string[] = []; // Arc/AWS API Gateway will send back set-cookies outside of response headers. for (let [key, values] of Object.entries( - (nodeResponse.headers as any).raw() as Record + (nodeResponse.headers as NodeHeaders).raw() )) { if (key.toLowerCase() === "set-cookie") { for (let value of values) { @@ -124,10 +114,6 @@ export async function sendRemixResponse( nodeResponse.headers.delete("Set-Cookie"); } - if (abortController.signal.aborted) { - nodeResponse.headers.set("Connection", "close"); - } - let contentType = nodeResponse.headers.get("Content-Type"); let isBinary = isBinaryType(contentType); let body; diff --git a/packages/remix-express/__tests__/server-test.ts b/packages/remix-express/__tests__/server-test.ts index a4937d862f7..4532360eb07 100644 --- a/packages/remix-express/__tests__/server-test.ts +++ b/packages/remix-express/__tests__/server-test.ts @@ -137,7 +137,7 @@ describe("express createRemixHeaders", () => { describe("creates fetch headers from express headers", () => { it("handles empty headers", () => { expect(createRemixHeaders({})).toMatchInlineSnapshot(` - Headers$1 { + Headers { Symbol(query): Array [], Symbol(context): null, } @@ -146,7 +146,7 @@ describe("express createRemixHeaders", () => { it("handles simple headers", () => { expect(createRemixHeaders({ "x-foo": "bar" })).toMatchInlineSnapshot(` - Headers$1 { + Headers { Symbol(query): Array [ "x-foo", "bar", @@ -159,7 +159,7 @@ describe("express createRemixHeaders", () => { it("handles multiple headers", () => { expect(createRemixHeaders({ "x-foo": "bar", "x-bar": "baz" })) .toMatchInlineSnapshot(` - Headers$1 { + Headers { Symbol(query): Array [ "x-foo", "bar", @@ -174,7 +174,7 @@ describe("express createRemixHeaders", () => { it("handles headers with multiple values", () => { expect(createRemixHeaders({ "x-foo": "bar, baz" })) .toMatchInlineSnapshot(` - Headers$1 { + Headers { Symbol(query): Array [ "x-foo", "bar, baz", @@ -187,7 +187,7 @@ describe("express createRemixHeaders", () => { it("handles headers with multiple values and multiple headers", () => { expect(createRemixHeaders({ "x-foo": "bar, baz", "x-bar": "baz" })) .toMatchInlineSnapshot(` - Headers$1 { + Headers { Symbol(query): Array [ "x-foo", "bar, baz", @@ -208,7 +208,7 @@ describe("express createRemixHeaders", () => { ], }) ).toMatchInlineSnapshot(` - Headers$1 { + Headers { Symbol(query): Array [ "set-cookie", "__session=some_value; Path=/; Secure; HttpOnly; MaxAge=7200; SameSite=Lax", @@ -237,7 +237,6 @@ describe("express createRemixRequest", () => { expect(createRemixRequest(expressRequest)).toMatchInlineSnapshot(` NodeRequest { - "abortController": undefined, "agent": undefined, "compress": true, "counter": 0, @@ -254,7 +253,7 @@ describe("express createRemixRequest", () => { "type": null, }, Symbol(Request internals): Object { - "headers": Headers$1 { + "headers": Headers { Symbol(query): Array [ "cache-control", "max-age=300, s-maxage=3600", diff --git a/packages/remix-express/package.json b/packages/remix-express/package.json index f4e948200e4..a454d6d2279 100644 --- a/packages/remix-express/package.json +++ b/packages/remix-express/package.json @@ -15,7 +15,6 @@ "@remix-run/node": "1.4.2" }, "peerDependencies": { - "@remix-run/web-stream": "^1.0.2", "express": "^4.17.1" }, "devDependencies": { diff --git a/packages/remix-express/server.ts b/packages/remix-express/server.ts index 30af6f9d7bf..c58a774b4a8 100644 --- a/packages/remix-express/server.ts +++ b/packages/remix-express/server.ts @@ -53,8 +53,7 @@ export function createRequestHandler({ next: express.NextFunction ) => { try { - let abortController = new AbortController(); - let request = createRemixRequest(req, abortController); + let request = createRemixRequest(req); let loadContext = typeof getLoadContext === "function" ? getLoadContext(req, res) @@ -62,7 +61,7 @@ export function createRequestHandler({ let response = await handleRequest(request, loadContext); - sendRemixResponse(res, response as NodeResponse, abortController); + sendRemixResponse(res, response as NodeResponse); } catch (error) { // Express doesn't support async functions, so we have to pass along the // error manually using next(). @@ -91,18 +90,13 @@ export function createRemixHeaders( return headers; } -export function createRemixRequest( - req: express.Request, - abortController?: AbortController -): NodeRequest { +export function createRemixRequest(req: express.Request): NodeRequest { let origin = `${req.protocol}://${req.get("host")}`; let url = new URL(req.url, origin); let init: NodeRequestInit = { method: req.method, headers: createRemixHeaders(req.headers), - signal: abortController?.signal, - abortController, }; if (req.method !== "GET" && req.method !== "HEAD") { @@ -114,24 +108,17 @@ export function createRemixRequest( export function sendRemixResponse( res: express.Response, - nodeResponse: NodeResponse, - abortController: AbortController + nodeResponse: NodeResponse ): void { res.statusMessage = nodeResponse.statusText; res.status(nodeResponse.status); - for (let [key, values] of Object.entries( - (nodeResponse.headers as NodeHeaders).raw() - )) { + for (let [key, values] of Object.entries(nodeResponse.headers.raw())) { for (let value of values) { res.append(key, value); } } - if (abortController.signal.aborted) { - res.set("Connection", "close"); - } - if (nodeResponse.body) { pipeReadableStreamToWritable(nodeResponse.body, res); } else { diff --git a/packages/remix-netlify/__tests__/server-test.ts b/packages/remix-netlify/__tests__/server-test.ts index 410d6cb466d..2f344a77a0a 100644 --- a/packages/remix-netlify/__tests__/server-test.ts +++ b/packages/remix-netlify/__tests__/server-test.ts @@ -140,7 +140,7 @@ describe("netlify createRemixHeaders", () => { describe("creates fetch headers from netlify headers", () => { it("handles empty headers", () => { expect(createRemixHeaders({})).toMatchInlineSnapshot(` - Headers$1 { + Headers { Symbol(query): Array [], Symbol(context): null, } @@ -149,7 +149,7 @@ describe("netlify createRemixHeaders", () => { it("handles simple headers", () => { expect(createRemixHeaders({ "x-foo": ["bar"] })).toMatchInlineSnapshot(` - Headers$1 { + Headers { Symbol(query): Array [ "x-foo", "bar", @@ -162,7 +162,7 @@ describe("netlify createRemixHeaders", () => { it("handles multiple headers", () => { expect(createRemixHeaders({ "x-foo": ["bar"], "x-bar": ["baz"] })) .toMatchInlineSnapshot(` - Headers$1 { + Headers { Symbol(query): Array [ "x-foo", "bar", @@ -177,7 +177,7 @@ describe("netlify createRemixHeaders", () => { it("handles headers with multiple values", () => { expect(createRemixHeaders({ "x-foo": ["bar", "baz"] })) .toMatchInlineSnapshot(` - Headers$1 { + Headers { Symbol(query): Array [ "x-foo", "bar", @@ -192,7 +192,7 @@ describe("netlify createRemixHeaders", () => { it("handles headers with multiple values and multiple headers", () => { expect(createRemixHeaders({ "x-foo": ["bar", "baz"], "x-bar": ["baz"] })) .toMatchInlineSnapshot(` - Headers$1 { + Headers { Symbol(query): Array [ "x-foo", "bar", @@ -217,7 +217,7 @@ describe("netlify createRemixHeaders", () => { "x-something-else": ["true"], }) ).toMatchInlineSnapshot(` - Headers$1 { + Headers { Symbol(query): Array [ "cookie", "__session=some_value; Path=/; Secure; HttpOnly; MaxAge=7200; SameSite=Lax", @@ -245,7 +245,6 @@ describe("netlify createRemixRequest", () => { ) ).toMatchInlineSnapshot(` NodeRequest { - "abortController": undefined, "agent": undefined, "compress": true, "counter": 0, @@ -262,7 +261,7 @@ describe("netlify createRemixRequest", () => { "type": null, }, Symbol(Request internals): Object { - "headers": Headers$1 { + "headers": Headers { Symbol(query): Array [ "cookie", "__session=value", diff --git a/packages/remix-netlify/server.ts b/packages/remix-netlify/server.ts index 720e0c4a7f1..534c2dab0e0 100644 --- a/packages/remix-netlify/server.ts +++ b/packages/remix-netlify/server.ts @@ -47,8 +47,7 @@ export function createRequestHandler({ let handleRequest = createRemixRequestHandler(build, mode); return async (event, context) => { - let abortController = new AbortController(); - let request = createRemixRequest(event, abortController); + let request = createRemixRequest(event); let loadContext = typeof getLoadContext === "function" ? getLoadContext(event, context) @@ -56,14 +55,11 @@ export function createRequestHandler({ let response = await handleRequest(request, loadContext); - return sendRemixResponse(response as NodeResponse, abortController); + return sendRemixResponse(response as NodeResponse); }; } -export function createRemixRequest( - event: HandlerEvent, - abortController?: AbortController -): NodeRequest { +export function createRemixRequest(event: HandlerEvent): NodeRequest { let url: URL; if (process.env.NODE_ENV !== "development") { @@ -77,8 +73,6 @@ export function createRemixRequest( let init: NodeRequestInit = { method: event.httpMethod, headers: createRemixHeaders(event.multiValueHeaders), - abortController, - signal: abortController?.signal, }; if (event.httpMethod !== "GET" && event.httpMethod !== "HEAD" && event.body) { @@ -137,13 +131,8 @@ function getRawPath(event: HandlerEvent): string { } export async function sendRemixResponse( - nodeResponse: NodeResponse, - abortController: AbortController + nodeResponse: NodeResponse ): Promise { - if (abortController.signal.aborted) { - nodeResponse.headers.set("Connection", "close"); - } - let contentType = nodeResponse.headers.get("Content-Type"); let body: string | undefined; let isBase64Encoded = isBinaryType(contentType); @@ -156,9 +145,7 @@ export async function sendRemixResponse( } } - let multiValueHeaders: Record = ( - nodeResponse.headers as NodeHeaders - ).raw(); + let multiValueHeaders = nodeResponse.headers.raw(); return { statusCode: nodeResponse.status, diff --git a/packages/remix-node/__tests__/fetch-test.ts b/packages/remix-node/__tests__/fetch-test.ts index 954d48da3e2..7ddc72f9f0c 100644 --- a/packages/remix-node/__tests__/fetch-test.ts +++ b/packages/remix-node/__tests__/fetch-test.ts @@ -70,8 +70,6 @@ let test = { }; describe("Request", () => { - let uploadHandler = createMemoryUploadHandler({}); - it("clones", async () => { let body = new PassThrough(); test.source.forEach((chunk) => body.write(chunk)); @@ -88,8 +86,8 @@ describe("Request", () => { let cloned = req.clone(); expect(Object.getPrototypeOf(req)).toBe(Object.getPrototypeOf(cloned)); - let formData = await req.formData(uploadHandler); - let clonedFormData = await cloned.formData(uploadHandler); + let formData = await req.formData(); + let clonedFormData = await cloned.formData(); expect(formData.get("file_name_0")).toBe("super alpha file"); expect(clonedFormData.get("file_name_0")).toBe("super alpha file"); diff --git a/packages/remix-node/__tests__/parseMultipartFormData-test.ts b/packages/remix-node/__tests__/parseMultipartFormData-test.ts index 4f210ada051..cd1f1ff353d 100644 --- a/packages/remix-node/__tests__/parseMultipartFormData-test.ts +++ b/packages/remix-node/__tests__/parseMultipartFormData-test.ts @@ -30,8 +30,7 @@ describe("internalParseFormData", () => { chunks.push(chunk); } return new File(chunks, filename, { type: contentType }); - }, - undefined + } ); expect(parsedFormData.get("a")).toBe("value"); @@ -58,13 +57,9 @@ describe("internalParseFormData", () => { }); try { - await internalParseFormData( - req, - async () => { - throw new Error("test error"); - }, - undefined - ); + await internalParseFormData(req, async () => { + throw new Error("test error"); + }); throw new Error("should have thrown"); } catch (err) { expect(err.message).toBe("test error"); diff --git a/packages/remix-node/fetch.ts b/packages/remix-node/fetch.ts index c416be74433..66627ff2d95 100644 --- a/packages/remix-node/fetch.ts +++ b/packages/remix-node/fetch.ts @@ -1,24 +1,21 @@ import type { Readable } from "stream"; -import type AbortController from "abort-controller"; -import { Request as BaseNodeRequest } from "@remix-run/web-fetch"; - -import type { UploadHandler } from "./formData"; -import { internalParseFormData } from "./parseMultipartFormData"; - -import type { Headers as NodeHeaders } from "@remix-run/web-fetch"; -import { fetch as nodeFetch } from "@remix-run/web-fetch"; +import { + fetch as nodeFetch, + Headers as BaseNodeHeaders, + Request as BaseNodeRequest, + Response as BaseNodeResponse, +} from "@remix-run/web-fetch"; export { File, Blob } from "@remix-run/web-file"; -export { Headers, Response } from "@remix-run/web-fetch"; -type NodeHeadersInit = ConstructorParameters[0]; -type NodeRequestInfo = ConstructorParameters[0]; -type BaseNodeRequestInit = NonNullable< - ConstructorParameters[1] +type NodeHeadersInit = ConstructorParameters[0]; +type NodeResponseBody = ConstructorParameters[0]; +type NodeResponseInit = NonNullable< + ConstructorParameters[1] >; - -type NodeResponseInit = Omit< +type NodeRequestInfo = ConstructorParameters[0]; +type NodeRequestInit = Omit< NonNullable[1]>, "body" > & { @@ -26,42 +23,44 @@ type NodeResponseInit = Omit< | NonNullable[1]>["body"] | Readable; }; -interface NodeRequestInit extends Omit { - abortController?: AbortController; - body?: BaseNodeRequestInit["body"] | Readable; -} - -class NodeRequest extends BaseNodeRequest { - private abortController?: AbortController; - constructor(input: NodeRequestInfo, init?: NodeRequestInit | undefined) { - super(input as any, init as BaseNodeRequestInit); +export type { + NodeHeadersInit as HeadersInit, + NodeRequestInfo as RequestInfo, + NodeRequestInit as RequestInit, + NodeResponseInit as ResponseInit, +}; - let anyInput = input as any; - let anyInit = init as any; +class NodeRequest extends BaseNodeRequest { + constructor(input: NodeRequestInfo, init?: NodeRequestInit) { + super(input, init as RequestInit); + } - this.abortController = - anyInput?.abortController || anyInit?.abortController; + public get headers(): BaseNodeHeaders { + return super.headers as BaseNodeHeaders; } - formData(uploadHandler?: UploadHandler): Promise { - let contentType = this.headers.get("Content-Type"); - if ( - uploadHandler && - contentType && - /multipart\/form-data/.test(contentType) - ) { - return internalParseFormData(this, uploadHandler, this.abortController); - } + public clone(): NodeRequest { + return new NodeRequest(this); + } +} - return super.formData(); +class NodeResponse extends BaseNodeResponse { + constructor(input: NodeResponseBody, init?: NodeResponseInit) { + super(input, init); } - clone(): NodeRequest { - return new NodeRequest(this); + public get headers(): BaseNodeHeaders { + return super.headers as BaseNodeHeaders; } } +export { + BaseNodeHeaders as Headers, + NodeRequest as Request, + NodeResponse as Response, +}; + export const fetch: typeof nodeFetch = ( input: NodeRequestInfo, init?: NodeRequestInit @@ -71,13 +70,5 @@ export const fetch: typeof nodeFetch = ( ...init, }; - return nodeFetch(input, init as BaseNodeRequestInit); -}; - -export type { - NodeHeadersInit as HeadersInit, - NodeRequestInfo as RequestInfo, - NodeRequestInit as RequestInit, - NodeResponseInit as ResponseInit, + return nodeFetch(input, init as RequestInit); }; -export { NodeRequest as Request }; diff --git a/packages/remix-node/package.json b/packages/remix-node/package.json index 91f02f1e026..63cd627ed83 100644 --- a/packages/remix-node/package.json +++ b/packages/remix-node/package.json @@ -13,7 +13,7 @@ }, "dependencies": { "@remix-run/server-runtime": "1.4.2", - "@remix-run/web-fetch": "^4.1.0", + "@remix-run/web-fetch": "^4.1.1", "@remix-run/web-file": "^3.0.2", "@remix-run/web-stream": "^1.0.2", "@web3-storage/multipart-parser": "^1.0.0", diff --git a/packages/remix-node/parseMultipartFormData.ts b/packages/remix-node/parseMultipartFormData.ts index e62b0ff807d..d087d2d8b9f 100644 --- a/packages/remix-node/parseMultipartFormData.ts +++ b/packages/remix-node/parseMultipartFormData.ts @@ -13,13 +13,12 @@ export function parseMultipartFormData( request: Request | NodeRequest, uploadHandler: UploadHandler ) { - return (request as unknown as NodeRequest).formData(uploadHandler); + return internalParseFormData(request, uploadHandler); } export const internalParseFormData = async ( - request: NodeRequest, - uploadHandler: UploadHandler, - abortController: AbortController | undefined + request: Request, + uploadHandler: UploadHandler ) => { let contentType = request.headers.get("Content-Type") || ""; let [type, boundary] = contentType.split(/\s*;\s*boundary=/); diff --git a/packages/remix-vercel/__tests__/server-test.ts b/packages/remix-vercel/__tests__/server-test.ts index c367eb4ed6f..53d6aa25027 100644 --- a/packages/remix-vercel/__tests__/server-test.ts +++ b/packages/remix-vercel/__tests__/server-test.ts @@ -142,7 +142,7 @@ describe("vercel createRemixHeaders", () => { describe("creates fetch headers from vercel headers", () => { it("handles empty headers", () => { expect(createRemixHeaders({})).toMatchInlineSnapshot(` - Headers$1 { + Headers { Symbol(query): Array [], Symbol(context): null, } @@ -151,7 +151,7 @@ describe("vercel createRemixHeaders", () => { it("handles simple headers", () => { expect(createRemixHeaders({ "x-foo": "bar" })).toMatchInlineSnapshot(` - Headers$1 { + Headers { Symbol(query): Array [ "x-foo", "bar", @@ -164,7 +164,7 @@ describe("vercel createRemixHeaders", () => { it("handles multiple headers", () => { expect(createRemixHeaders({ "x-foo": "bar", "x-bar": "baz" })) .toMatchInlineSnapshot(` - Headers$1 { + Headers { Symbol(query): Array [ "x-foo", "bar", @@ -179,7 +179,7 @@ describe("vercel createRemixHeaders", () => { it("handles headers with multiple values", () => { expect(createRemixHeaders({ "x-foo": "bar, baz" })) .toMatchInlineSnapshot(` - Headers$1 { + Headers { Symbol(query): Array [ "x-foo", "bar, baz", @@ -192,7 +192,7 @@ describe("vercel createRemixHeaders", () => { it("handles headers with multiple values and multiple headers", () => { expect(createRemixHeaders({ "x-foo": "bar, baz", "x-bar": "baz" })) .toMatchInlineSnapshot(` - Headers$1 { + Headers { Symbol(query): Array [ "x-foo", "bar, baz", @@ -213,7 +213,7 @@ describe("vercel createRemixHeaders", () => { ], }) ).toMatchInlineSnapshot(` - Headers$1 { + Headers { Symbol(query): Array [ "set-cookie", "__session=some_value; Path=/; Secure; HttpOnly; MaxAge=7200; SameSite=Lax", @@ -241,7 +241,6 @@ describe("vercel createRemixRequest", () => { expect(createRemixRequest(request)).toMatchInlineSnapshot(` NodeRequest { - "abortController": undefined, "agent": undefined, "compress": true, "counter": 0, @@ -258,7 +257,7 @@ describe("vercel createRemixRequest", () => { "type": null, }, Symbol(Request internals): Object { - "headers": Headers$1 { + "headers": Headers { Symbol(query): Array [ "cache-control", "max-age=300, s-maxage=3600", diff --git a/packages/remix-vercel/server.ts b/packages/remix-vercel/server.ts index fc31669f855..f6db2893fd6 100644 --- a/packages/remix-vercel/server.ts +++ b/packages/remix-vercel/server.ts @@ -47,21 +47,13 @@ export function createRequestHandler({ let handleRequest = createRemixRequestHandler(build, mode); return async (req, res) => { - let abortController = new AbortController(); - let request = createRemixRequest(req, abortController); + let request = createRemixRequest(req); let loadContext = typeof getLoadContext === "function" ? getLoadContext(req, res) : undefined; - let response = await handleRequest( - request as unknown as Request, - loadContext - ); - - if (abortController.signal.aborted) { - response.headers.set("Connection", "close"); - } + let response = await handleRequest(request, loadContext); sendRemixResponse(res, response as NodeResponse); }; @@ -86,10 +78,7 @@ export function createRemixHeaders( return headers; } -export function createRemixRequest( - req: VercelRequest, - abortController?: AbortController -): NodeRequest { +export function createRemixRequest(req: VercelRequest): NodeRequest { let host = req.headers["x-forwarded-host"] || req.headers["host"]; // doesn't seem to be available on their req object! let protocol = req.headers["x-forwarded-proto"] || "https"; @@ -98,8 +87,6 @@ export function createRemixRequest( let init: NodeRequestInit = { method: req.method, headers: createRemixHeaders(req.headers), - abortController, - signal: abortController?.signal, }; if (req.method !== "GET" && req.method !== "HEAD") { @@ -114,16 +101,7 @@ export function sendRemixResponse( nodeResponse: NodeResponse ): void { res.statusMessage = nodeResponse.statusText; - let multiValueHeaders: Record = {}; - for (let [key, values] of Object.entries( - (nodeResponse.headers as NodeHeaders).raw() - )) { - if (typeof multiValueHeaders[key] === "undefined") { - multiValueHeaders[key] = [...values]; - } else { - (multiValueHeaders[key] as string[]).push(...values); - } - } + let multiValueHeaders = nodeResponse.headers.raw(); res.writeHead( nodeResponse.status, nodeResponse.statusText, diff --git a/yarn.lock b/yarn.lock index 35f9f6e58f5..dabb0256e8e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1576,10 +1576,10 @@ "@remix-run/web-stream" "^1.0.0" web-encoding "1.1.5" -"@remix-run/web-fetch@^4.1.0": - version "4.1.0" - resolved "https://registry.npmjs.org/@remix-run/web-fetch/-/web-fetch-4.1.0.tgz#f4e7d31863add71c1bf759fbde7c9750d0c57313" - integrity sha512-V1q8ZnViqePZ44OCuHZbEbuof3jnru5L9ZxOgO2zjrbryIxc3J/ZgyZxIV0HeviD3vUWNdz283FNl2mUOZ5+ZA== +"@remix-run/web-fetch@^4.1.1": + version "4.1.1" + resolved "https://registry.npmjs.org/@remix-run/web-fetch/-/web-fetch-4.1.1.tgz#2b7ab898599ea1a273a31b357e7a19148e6cec26" + integrity sha512-rsGqRERL+aCYWgtHaZyEy4Xzic4IHS4A2AzWndtoDy+8mUqtBe95QtUoxX5J9jBMs1yl4A/YjXD6HwWP1yyLrw== dependencies: "@remix-run/web-blob" "^3.0.3" "@remix-run/web-form-data" "^3.0.2" @@ -1802,13 +1802,6 @@ "@types/connect" "*" "@types/node" "*" -"@types/busboy@^0.3.1": - version "0.3.1" - resolved "https://registry.npmjs.org/@types/busboy/-/busboy-0.3.1.tgz" - integrity sha512-8BPLNy4x+7lbTOGkAyUIZrrPEZ7WzbO7YlVGMf9EZi9J9mqILEkYbt/kgVWQ7fizOISo1hM/7cAsWVTa7EhQDg== - dependencies: - "@types/node" "*" - "@types/cacache@^15.0.0": version "15.0.1" resolved "https://registry.npmjs.org/@types/cacache/-/cacache-15.0.1.tgz" @@ -3229,13 +3222,6 @@ builtin-modules@^3.1.0: resolved "https://registry.npmjs.org/builtin-modules/-/builtin-modules-3.2.0.tgz" integrity sha512-lGzLKcioL90C7wMczpkY0n/oART3MbBa8R9OFGE1rJxoVI86u4WAGfEk8Wjv10eKSyTHVGkSo3bvBylCEtk7LA== -busboy@^0.3.1: - version "0.3.1" - resolved "https://registry.npmjs.org/busboy/-/busboy-0.3.1.tgz" - integrity sha512-y7tTxhGKXcyBxRKAni+awqx8uqaJKrSFSNFSeRG5CsWNdmy2BIK+6VGWEW7TZnIO/533mtMEA4rOevQV815YJw== - dependencies: - dicer "0.3.0" - bytes@3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz" @@ -4027,13 +4013,6 @@ detect-newline@3.1.0, detect-newline@^3.0.0: resolved "https://registry.npmjs.org/detect-newline/-/detect-newline-3.1.0.tgz" integrity sha512-TLz+x/vEXm/Y7P7wn1EJFNLxYpUD4TgMosxY6fAVJUnJMbupHBOncxyWUG9OpTaH9EBD7uFI5LfEgmMOc54DsA== -dicer@0.3.0: - version "0.3.0" - resolved "https://registry.npmjs.org/dicer/-/dicer-0.3.0.tgz" - integrity sha512-MdceRRWqltEG2dZqO769g27N/3PXfcKl04VhYnBlo2YhH7zPi88VebsjTKclaOyiuMaGU72hTfw3VkUitGcVCA== - dependencies: - streamsearch "0.1.2" - diff-sequences@^27.5.1: version "27.5.1" resolved "https://registry.npmjs.org/diff-sequences/-/diff-sequences-27.5.1.tgz" @@ -9461,11 +9440,6 @@ stream-shift@^1.0.0: resolved "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.1.tgz" integrity sha512-AiisoFqQ0vbGcZgQPY1cdP2I76glaVA/RauYR4G4thNFgkTqr90yXTo4LYX60Jl+sIlPNHHdGSwo01AvbKUSVQ== -streamsearch@0.1.2: - version "0.1.2" - resolved "https://registry.npmjs.org/streamsearch/-/streamsearch-0.1.2.tgz" - integrity sha1-gIudDlb8Jz2Am6VzOOkpkZoanxo= - strict-event-emitter@^0.2.0: version "0.2.0" resolved "https://registry.npmjs.org/strict-event-emitter/-/strict-event-emitter-0.2.0.tgz" From 567a11b0013a292f90abcc8e6a8fb8eab8c5776b Mon Sep 17 00:00:00 2001 From: Jacob Ebey Date: Wed, 4 May 2022 18:15:27 -0700 Subject: [PATCH 15/47] remove form-data dep --- packages/remix-node/package.json | 1 - yarn.lock | 9 --------- 2 files changed, 10 deletions(-) diff --git a/packages/remix-node/package.json b/packages/remix-node/package.json index 63cd627ed83..c252883e29a 100644 --- a/packages/remix-node/package.json +++ b/packages/remix-node/package.json @@ -20,7 +20,6 @@ "abort-controller": "^3.0.0", "blob-stream": "^0.1.3", "cookie-signature": "^1.1.0", - "form-data": "^4.0.0", "source-map-support": "^0.5.21" }, "devDependencies": { diff --git a/yarn.lock b/yarn.lock index dabb0256e8e..e76c06059bb 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5084,15 +5084,6 @@ form-data@^3.0.0: combined-stream "^1.0.8" mime-types "^2.1.12" -form-data@^4.0.0: - version "4.0.0" - resolved "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz" - integrity sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww== - dependencies: - asynckit "^0.4.0" - combined-stream "^1.0.8" - mime-types "^2.1.12" - format@^0.2.0: version "0.2.2" resolved "https://registry.npmjs.org/format/-/format-0.2.2.tgz" From 2c8b53e8e0fbd311518cf6ed739dff719cb6bd66 Mon Sep 17 00:00:00 2001 From: Jacob Ebey Date: Wed, 4 May 2022 19:22:00 -0700 Subject: [PATCH 16/47] chore: fix lint issues --- packages/remix-architect/server.ts | 2 - packages/remix-express/server.ts | 2 - packages/remix-netlify/server.ts | 2 - packages/remix-node/__tests__/fetch-test.ts | 1 - packages/remix-node/fetch.ts | 3 +- packages/remix-node/parseMultipartFormData.ts | 4 +- packages/remix-node/stream.ts | 4 +- .../remix-node/upload/fileUploadHandler.ts | 5 ++ .../remix-node/upload/memoryUploadHandler.ts | 61 +------------------ packages/remix-vercel/server.ts | 2 - 10 files changed, 13 insertions(+), 73 deletions(-) diff --git a/packages/remix-architect/server.ts b/packages/remix-architect/server.ts index f137516059c..67da96753e1 100644 --- a/packages/remix-architect/server.ts +++ b/packages/remix-architect/server.ts @@ -1,6 +1,4 @@ import { - // This has been added as a global in node 15+ - AbortController, Headers as NodeHeaders, Request as NodeRequest, createRequestHandler as createRemixRequestHandler, diff --git a/packages/remix-express/server.ts b/packages/remix-express/server.ts index c58a774b4a8..79c999ad5cb 100644 --- a/packages/remix-express/server.ts +++ b/packages/remix-express/server.ts @@ -6,8 +6,6 @@ import type { Response as NodeResponse, } from "@remix-run/node"; import { - // This has been added as a global in node 15+ - AbortController, createRequestHandler as createRemixRequestHandler, Headers as NodeHeaders, Request as NodeRequest, diff --git a/packages/remix-netlify/server.ts b/packages/remix-netlify/server.ts index 534c2dab0e0..9c1488fa5c1 100644 --- a/packages/remix-netlify/server.ts +++ b/packages/remix-netlify/server.ts @@ -1,6 +1,4 @@ import { - // This has been added as a global in node 15+ - AbortController, createRequestHandler as createRemixRequestHandler, Headers as NodeHeaders, Request as NodeRequest, diff --git a/packages/remix-node/__tests__/fetch-test.ts b/packages/remix-node/__tests__/fetch-test.ts index 7ddc72f9f0c..5a4ea67227e 100644 --- a/packages/remix-node/__tests__/fetch-test.ts +++ b/packages/remix-node/__tests__/fetch-test.ts @@ -1,7 +1,6 @@ import { PassThrough } from "stream"; import { Request } from "../fetch"; -import { createMemoryUploadHandler } from "../upload/memoryUploadHandler"; let test = { source: [ diff --git a/packages/remix-node/fetch.ts b/packages/remix-node/fetch.ts index 66627ff2d95..60b094e08fc 100644 --- a/packages/remix-node/fetch.ts +++ b/packages/remix-node/fetch.ts @@ -1,12 +1,10 @@ import type { Readable } from "stream"; - import { fetch as nodeFetch, Headers as BaseNodeHeaders, Request as BaseNodeRequest, Response as BaseNodeResponse, } from "@remix-run/web-fetch"; - export { File, Blob } from "@remix-run/web-file"; type NodeHeadersInit = ConstructorParameters[0]; @@ -46,6 +44,7 @@ class NodeRequest extends BaseNodeRequest { } class NodeResponse extends BaseNodeResponse { + // eslint-disable-next-line @typescript-eslint/no-useless-constructor constructor(input: NodeResponseBody, init?: NodeResponseInit) { super(input, init); } diff --git a/packages/remix-node/parseMultipartFormData.ts b/packages/remix-node/parseMultipartFormData.ts index d087d2d8b9f..c081f0d12af 100644 --- a/packages/remix-node/parseMultipartFormData.ts +++ b/packages/remix-node/parseMultipartFormData.ts @@ -56,11 +56,11 @@ export const internalParseFormData = async ( }; export function mergeArrays(...arrays: Uint8Array[]) { - const out = new Uint8Array( + let out = new Uint8Array( arrays.reduce((total, arr) => total + arr.length, 0) ); let offset = 0; - for (const arr of arrays) { + for (let arr of arrays) { out.set(arr, offset); offset += arr.length; } diff --git a/packages/remix-node/stream.ts b/packages/remix-node/stream.ts index 2779c922c36..6f8bd6f70a5 100644 --- a/packages/remix-node/stream.ts +++ b/packages/remix-node/stream.ts @@ -4,10 +4,10 @@ export function pipeReadableStreamToWritable( stream: ReadableStream, writable: Writable ) { - const reader = stream.getReader(); + let reader = stream.getReader(); async function read() { - const { done, value } = await reader.read(); + let { done, value } = await reader.read(); if (done) { writable.end(); return; diff --git a/packages/remix-node/upload/fileUploadHandler.ts b/packages/remix-node/upload/fileUploadHandler.ts index 52c5c0ca868..a91bf11b9d7 100644 --- a/packages/remix-node/upload/fileUploadHandler.ts +++ b/packages/remix-node/upload/fileUploadHandler.ts @@ -119,16 +119,21 @@ export function createFileUploadHandler({ let writeFileStream = createWriteStream(filepath); let size = 0; + let deleteFile = false; try { for await (let chunk of data) { size += chunk.length; if (size > maxFileSize) { + deleteFile = true; throw new MeterError(name, maxFileSize); } writeFileStream.write(chunk); } } finally { writeFileStream.close(); + if (deleteFile) { + await rm(filepath).catch(() => {}); + } } return new NodeOnDiskFile(filepath, size, contentType); diff --git a/packages/remix-node/upload/memoryUploadHandler.ts b/packages/remix-node/upload/memoryUploadHandler.ts index 8e393b69042..69f6d665e4b 100644 --- a/packages/remix-node/upload/memoryUploadHandler.ts +++ b/packages/remix-node/upload/memoryUploadHandler.ts @@ -1,9 +1,6 @@ -import type { TransformCallback } from "stream"; -import { Transform } from "stream"; import { File } from "../fetch"; - -import { MeterError } from "./meter"; import type { UploadHandler } from "../formData"; +import { MeterError } from "./meter"; export type MemoryUploadHandlerFilterArgs = { filename: string; @@ -36,7 +33,7 @@ export function createMemoryUploadHandler({ } let size = 0; - const chunks = []; + let chunks = []; for await (let chunk of data) { chunks.push(chunk); size += chunk.length; @@ -45,58 +42,6 @@ export function createMemoryUploadHandler({ } } - const file = new File(chunks, filename, { type: contentType }); - - return file; + return new File(chunks, filename, { type: contentType }); }; - // return async ({ name, stream, filename, encoding, mimetype }) => { - // if (filter && !(await filter({ filename, encoding, mimetype }))) { - // stream.resume(); - // return; - // } - - // let bufferStream = new BufferStream(); - // await new Promise((resolve, reject) => { - // let meter = new Meter(name, maxFileSize); - - // let aborted = false; - // async function abort(error: Error) { - // if (aborted) return; - // aborted = true; - - // stream.unpipe(); - // meter.unpipe(); - // stream.removeAllListeners(); - // meter.removeAllListeners(); - // bufferStream.removeAllListeners(); - - // reject(error); - // } - - // stream.on("error", abort); - // meter.on("error", abort); - // bufferStream.on("error", abort); - // bufferStream.on("finish", resolve); - - // stream.pipe(meter).pipe(bufferStream); - // }); - - // return new BufferFile(bufferStream.data, filename, { - // type: mimetype, - // }); - // }; -} - -class BufferStream extends Transform { - public data: any[]; - - constructor() { - super(); - this.data = []; - } - - _transform(chunk: any, _: BufferEncoding, callback: TransformCallback) { - this.data.push(chunk); - callback(); - } } diff --git a/packages/remix-vercel/server.ts b/packages/remix-vercel/server.ts index f6db2893fd6..da0c07a8fcc 100644 --- a/packages/remix-vercel/server.ts +++ b/packages/remix-vercel/server.ts @@ -6,8 +6,6 @@ import type { Response as NodeResponse, } from "@remix-run/node"; import { - // This has been added as a global in node 15+ - AbortController, createRequestHandler as createRemixRequestHandler, Headers as NodeHeaders, Request as NodeRequest, From 3fc5f04ecfeaf747bc1db4c3b67116f621140714 Mon Sep 17 00:00:00 2001 From: Jacob Ebey Date: Thu, 5 May 2022 16:16:15 -0700 Subject: [PATCH 17/47] updated types --- packages/remix-node/fetch.ts | 18 +++++++++++++++--- packages/remix-node/globals.ts | 12 ++++++------ 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/packages/remix-node/fetch.ts b/packages/remix-node/fetch.ts index 60b094e08fc..05c45ac31ce 100644 --- a/packages/remix-node/fetch.ts +++ b/packages/remix-node/fetch.ts @@ -12,7 +12,9 @@ type NodeResponseBody = ConstructorParameters[0]; type NodeResponseInit = NonNullable< ConstructorParameters[1] >; -type NodeRequestInfo = ConstructorParameters[0]; +type NodeRequestInfo = + | ConstructorParameters[0] + | NodeRequest; type NodeRequestInit = Omit< NonNullable[1]>, "body" @@ -30,8 +32,8 @@ export type { }; class NodeRequest extends BaseNodeRequest { - constructor(input: NodeRequestInfo, init?: NodeRequestInit) { - super(input, init as RequestInit); + constructor(info: NodeRequestInfo, init?: NodeRequestInit) { + super(info, init as RequestInit); } public get headers(): BaseNodeHeaders { @@ -52,6 +54,16 @@ class NodeResponse extends BaseNodeResponse { public get headers(): BaseNodeHeaders { return super.headers as BaseNodeHeaders; } + + public clone(): NodeResponse { + return new NodeResponse(super.clone().body, { + url: this.url, + status: this.status, + statusText: this.statusText, + headers: this.headers, + size: this.size, + }); + } } export { diff --git a/packages/remix-node/globals.ts b/packages/remix-node/globals.ts index e1c74ea1b80..c263bd49686 100644 --- a/packages/remix-node/globals.ts +++ b/packages/remix-node/globals.ts @@ -35,12 +35,12 @@ export function installGlobals() { global.atob = atob; global.btoa = btoa; - global.Blob = NodeBlob as unknown as typeof Blob; - global.File = NodeFile as unknown as typeof File; + global.Blob = NodeBlob; + global.File = NodeFile; - global.Headers = NodeHeaders as unknown as typeof Headers; - global.Request = NodeRequest as unknown as typeof Request; + global.Headers = NodeHeaders; + global.Request = NodeRequest as typeof Request; global.Response = NodeResponse as unknown as typeof Response; - global.fetch = nodeFetch as unknown as typeof fetch; - global.FormData = NodeFormData as unknown as typeof FormData; + global.fetch = nodeFetch as typeof fetch; + global.FormData = NodeFormData; } From 4b56405e1fbec4c6d6f235f5b7ad514fc8c211f8 Mon Sep 17 00:00:00 2001 From: Jacob Ebey Date: Thu, 5 May 2022 18:51:42 -0700 Subject: [PATCH 18/47] update @remix-run/web-stream to v1.0.3 --- packages/remix-node/package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/remix-node/package.json b/packages/remix-node/package.json index e0ead26c85b..07f4dff0214 100644 --- a/packages/remix-node/package.json +++ b/packages/remix-node/package.json @@ -15,7 +15,7 @@ "@remix-run/server-runtime": "1.4.3", "@remix-run/web-fetch": "^4.1.1", "@remix-run/web-file": "^3.0.2", - "@remix-run/web-stream": "^1.0.2", + "@remix-run/web-stream": "^1.0.3", "@web3-storage/multipart-parser": "^1.0.0", "abort-controller": "^3.0.0", "blob-stream": "^0.1.3", diff --git a/yarn.lock b/yarn.lock index 4001264b1c9..124eb01f595 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1602,10 +1602,10 @@ dependencies: web-encoding "1.1.5" -"@remix-run/web-stream@^1.0.0", "@remix-run/web-stream@^1.0.1", "@remix-run/web-stream@^1.0.2": - version "1.0.2" - resolved "https://registry.npmjs.org/@remix-run/web-stream/-/web-stream-1.0.2.tgz#f07dc9cf6db02507ea71a234bc8e06103a2207b4" - integrity sha512-FO4om5mrwMs5bi7L5hbLMP1hm+flAS2oYRptfNPkK2u0Hhv0crS9GiE9/MsVvY53tTAxVkzUG/m+9ET1mTjEnw== +"@remix-run/web-stream@^1.0.0", "@remix-run/web-stream@^1.0.1", "@remix-run/web-stream@^1.0.3": + version "1.0.3" + resolved "https://registry.yarnpkg.com/@remix-run/web-stream/-/web-stream-1.0.3.tgz#3284a6a45675d1455c4d9c8f31b89225c9006438" + integrity sha512-wlezlJaA5NF6SsNMiwQnnAW6tnPzQ5I8qk0Y0pSohm0eHKa2FQ1QhEKLVVcDDu02TmkfHgnux0igNfeYhDOXiA== dependencies: web-streams-polyfill "^3.1.1" From 5c910f2410879c39c051abf1c66b4db00237292b Mon Sep 17 00:00:00 2001 From: Jacob Ebey Date: Thu, 5 May 2022 18:58:44 -0700 Subject: [PATCH 19/47] chore: speed up build with ts config change fix types --- packages/remix-node/globals.ts | 2 +- tsconfig.json | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/remix-node/globals.ts b/packages/remix-node/globals.ts index c263bd49686..1e28ca5816e 100644 --- a/packages/remix-node/globals.ts +++ b/packages/remix-node/globals.ts @@ -38,7 +38,7 @@ export function installGlobals() { global.Blob = NodeBlob; global.File = NodeFile; - global.Headers = NodeHeaders; + global.Headers = NodeHeaders as typeof Headers; global.Request = NodeRequest as typeof Request; global.Response = NodeResponse as unknown as typeof Response; global.fetch = nodeFetch as typeof fetch; diff --git a/tsconfig.json b/tsconfig.json index 8ccfcf30261..222ff9cbbce 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,5 +1,8 @@ { "files": [], + "exclude": [ + "node_modules" + ], "references": [ { "path": "packages/create-remix" }, { "path": "packages/remix" }, From 0ef2b13c38e8e71c489c982fe9d1956f70488b5c Mon Sep 17 00:00:00 2001 From: Michael Jackson Date: Fri, 6 May 2022 11:43:09 -0700 Subject: [PATCH 20/47] Clean up some types Also moved upload handling into server runtime! --- packages/remix-architect/server.ts | 38 +++++++------ packages/remix-cloudflare/index.ts | 3 + packages/remix-express/server.ts | 13 +++-- packages/remix-netlify/server.ts | 4 +- packages/remix-node/__tests__/fetch-test.ts | 8 ++- packages/remix-node/fetch.ts | 57 ++++++++----------- packages/remix-node/formData.ts | 12 ---- packages/remix-node/globals.ts | 2 +- packages/remix-node/index.ts | 17 +++--- packages/remix-node/stream.ts | 17 ++++-- .../remix-node/upload/fileUploadHandler.ts | 2 +- .../remix-node/upload/memoryUploadHandler.ts | 3 +- .../__tests__/parseMultipartFormData-test.ts | 20 +++---- .../formData.ts} | 55 +++++++++--------- packages/remix-server-runtime/index.ts | 3 + packages/remix-server-runtime/package.json | 1 + packages/remix-server-runtime/reexport.ts | 2 + packages/remix-vercel/server.ts | 11 ++-- 18 files changed, 135 insertions(+), 133 deletions(-) delete mode 100644 packages/remix-node/formData.ts rename packages/{remix-node => remix-server-runtime}/__tests__/parseMultipartFormData-test.ts (74%) rename packages/{remix-node/parseMultipartFormData.ts => remix-server-runtime/formData.ts} (55%) diff --git a/packages/remix-architect/server.ts b/packages/remix-architect/server.ts index 67da96753e1..0884ee0adcd 100644 --- a/packages/remix-architect/server.ts +++ b/packages/remix-architect/server.ts @@ -1,7 +1,13 @@ +import type { + AppLoadContext, + ServerBuild, + Response as NodeResponse, +} from "@remix-run/node"; import { Headers as NodeHeaders, Request as NodeRequest, createRequestHandler as createRemixRequestHandler, + readableStreamToBase64String, } from "@remix-run/node"; import type { APIGatewayProxyEventHeaders, @@ -9,7 +15,6 @@ import type { APIGatewayProxyHandlerV2, APIGatewayProxyStructuredResultV2, } from "aws-lambda"; -import type { AppLoadContext, ServerBuild } from "@remix-run/node"; import { isBinaryType } from "./binaryTypes"; @@ -46,7 +51,7 @@ export function createRequestHandler({ let loadContext = typeof getLoadContext === "function" ? getLoadContext(event) : undefined; - let response = await handleRequest(request, loadContext); + let response = (await handleRequest(request, loadContext)) as NodeResponse; return sendRemixResponse(response); }; @@ -76,7 +81,7 @@ export function createRemixRequest(event: APIGatewayProxyEventV2): NodeRequest { export function createRemixHeaders( requestHeaders: APIGatewayProxyEventHeaders, requestCookies?: string[] -): Headers { +): NodeHeaders { let headers = new NodeHeaders(); for (let [header, value] of Object.entries(requestHeaders)) { @@ -93,14 +98,12 @@ export function createRemixHeaders( } export async function sendRemixResponse( - nodeResponse: Response + nodeResponse: NodeResponse ): Promise { let cookies: string[] = []; // Arc/AWS API Gateway will send back set-cookies outside of response headers. - for (let [key, values] of Object.entries( - (nodeResponse.headers as NodeHeaders).raw() - )) { + for (let [key, values] of Object.entries(nodeResponse.headers.raw())) { if (key.toLowerCase() === "set-cookie") { for (let value of values) { cookies.push(value); @@ -113,21 +116,20 @@ export async function sendRemixResponse( } let contentType = nodeResponse.headers.get("Content-Type"); - let isBinary = isBinaryType(contentType); - let body; - let isBase64Encoded = false; - - if (isBinary) { - let blob = await nodeResponse.arrayBuffer(); - body = Buffer.from(blob).toString("base64"); - isBase64Encoded = true; - } else { - body = await nodeResponse.text(); + let isBase64Encoded = isBinaryType(contentType); + let body: string | undefined; + + if (nodeResponse.body) { + if (isBase64Encoded) { + body = await readableStreamToBase64String(nodeResponse.body); + } else { + body = await nodeResponse.text(); + } } return { statusCode: nodeResponse.status, - headers: Object.fromEntries(nodeResponse.headers), + headers: Object.fromEntries(nodeResponse.headers.entries()), cookies, body, isBase64Encoded, diff --git a/packages/remix-cloudflare/index.ts b/packages/remix-cloudflare/index.ts index 05f62f1236e..8352d1200eb 100644 --- a/packages/remix-cloudflare/index.ts +++ b/packages/remix-cloudflare/index.ts @@ -15,6 +15,7 @@ export { isCookie, isSession, json, + parseMultipartFormData, redirect, } from "@remix-run/server-runtime"; @@ -51,4 +52,6 @@ export type { SessionData, SessionIdStorageStrategy, SessionStorage, + UploadHandlerPart, + UploadHandler, } from "@remix-run/server-runtime"; diff --git a/packages/remix-express/server.ts b/packages/remix-express/server.ts index 79c999ad5cb..ad63e37a27b 100644 --- a/packages/remix-express/server.ts +++ b/packages/remix-express/server.ts @@ -57,9 +57,12 @@ export function createRequestHandler({ ? getLoadContext(req, res) : undefined; - let response = await handleRequest(request, loadContext); + let response = (await handleRequest( + request, + loadContext + )) as NodeResponse; - sendRemixResponse(res, response as NodeResponse); + await sendRemixResponse(res, response); } catch (error) { // Express doesn't support async functions, so we have to pass along the // error manually using next(). @@ -104,10 +107,10 @@ export function createRemixRequest(req: express.Request): NodeRequest { return new NodeRequest(url.href, init); } -export function sendRemixResponse( +export async function sendRemixResponse( res: express.Response, nodeResponse: NodeResponse -): void { +): Promise { res.statusMessage = nodeResponse.statusText; res.status(nodeResponse.status); @@ -118,7 +121,7 @@ export function sendRemixResponse( } if (nodeResponse.body) { - pipeReadableStreamToWritable(nodeResponse.body, res); + await pipeReadableStreamToWritable(nodeResponse.body, res); } else { res.end(); } diff --git a/packages/remix-netlify/server.ts b/packages/remix-netlify/server.ts index 9c1488fa5c1..9e4aa447ba5 100644 --- a/packages/remix-netlify/server.ts +++ b/packages/remix-netlify/server.ts @@ -51,9 +51,9 @@ export function createRequestHandler({ ? getLoadContext(event, context) : undefined; - let response = await handleRequest(request, loadContext); + let response = (await handleRequest(request, loadContext)) as NodeResponse; - return sendRemixResponse(response as NodeResponse); + return sendRemixResponse(response); }; } diff --git a/packages/remix-node/__tests__/fetch-test.ts b/packages/remix-node/__tests__/fetch-test.ts index 5a4ea67227e..9468160d65a 100644 --- a/packages/remix-node/__tests__/fetch-test.ts +++ b/packages/remix-node/__tests__/fetch-test.ts @@ -83,8 +83,6 @@ describe("Request", () => { }); let cloned = req.clone(); - expect(Object.getPrototypeOf(req)).toBe(Object.getPrototypeOf(cloned)); - let formData = await req.formData(); let clonedFormData = await cloned.formData(); @@ -107,3 +105,9 @@ describe("Request", () => { expect(file.size).toBe(1023); }); }); + +describe("fetch", () => { + // fetch a gzip-encoded json blob + // call res.json() and make sure it's decoded properly + it.todo("decodes gzip encoded body"); +}); diff --git a/packages/remix-node/fetch.ts b/packages/remix-node/fetch.ts index 05c45ac31ce..ce42c4ed8d3 100644 --- a/packages/remix-node/fetch.ts +++ b/packages/remix-node/fetch.ts @@ -1,26 +1,26 @@ import type { Readable } from "stream"; import { - fetch as nodeFetch, - Headers as BaseNodeHeaders, - Request as BaseNodeRequest, - Response as BaseNodeResponse, + fetch as webFetch, + Headers as WebHeaders, + Request as WebRequest, + Response as WebResponse, } from "@remix-run/web-fetch"; +export { FormData } from "@remix-run/web-fetch"; export { File, Blob } from "@remix-run/web-file"; -type NodeHeadersInit = ConstructorParameters[0]; -type NodeResponseBody = ConstructorParameters[0]; +type NodeHeadersInit = ConstructorParameters[0]; type NodeResponseInit = NonNullable< - ConstructorParameters[1] + ConstructorParameters[1] >; type NodeRequestInfo = - | ConstructorParameters[0] + | ConstructorParameters[0] | NodeRequest; type NodeRequestInit = Omit< - NonNullable[1]>, + NonNullable[1]>, "body" > & { body?: - | NonNullable[1]>["body"] + | NonNullable[1]>["body"] | Readable; }; @@ -31,55 +31,46 @@ export type { NodeResponseInit as ResponseInit, }; -class NodeRequest extends BaseNodeRequest { +class NodeRequest extends WebRequest { constructor(info: NodeRequestInfo, init?: NodeRequestInit) { super(info, init as RequestInit); } - public get headers(): BaseNodeHeaders { - return super.headers as BaseNodeHeaders; + public get headers(): WebHeaders { + return super.headers as WebHeaders; } public clone(): NodeRequest { - return new NodeRequest(this); + return super.clone() as NodeRequest; } } -class NodeResponse extends BaseNodeResponse { - // eslint-disable-next-line @typescript-eslint/no-useless-constructor - constructor(input: NodeResponseBody, init?: NodeResponseInit) { - super(input, init); - } - - public get headers(): BaseNodeHeaders { - return super.headers as BaseNodeHeaders; +class NodeResponse extends WebResponse { + public get headers(): WebHeaders { + return super.headers as WebHeaders; } public clone(): NodeResponse { - return new NodeResponse(super.clone().body, { - url: this.url, - status: this.status, - statusText: this.statusText, - headers: this.headers, - size: this.size, - }); + return super.clone() as NodeResponse; } } export { - BaseNodeHeaders as Headers, + WebHeaders as Headers, NodeRequest as Request, NodeResponse as Response, }; -export const fetch: typeof nodeFetch = ( - input: NodeRequestInfo, +export const fetch: typeof webFetch = ( + info: NodeRequestInfo, init?: NodeRequestInit ) => { init = { + // Disable compression handling so people can return the result of a fetch + // directly in the loader without messing with the Content-Encoding header. compress: false, ...init, }; - return nodeFetch(input, init as RequestInit); + return webFetch(info, init as RequestInit); }; diff --git a/packages/remix-node/formData.ts b/packages/remix-node/formData.ts deleted file mode 100644 index 81e9bf0f984..00000000000 --- a/packages/remix-node/formData.ts +++ /dev/null @@ -1,12 +0,0 @@ -export { FormData } from "@remix-run/web-fetch"; - -export type UploadHandlerArgs = { - name: string; - filename: string; - contentType: string; - data: AsyncIterable; -}; - -export type UploadHandler = ( - args: UploadHandlerArgs -) => Promise; diff --git a/packages/remix-node/globals.ts b/packages/remix-node/globals.ts index 1e28ca5816e..3159d9a1d1a 100644 --- a/packages/remix-node/globals.ts +++ b/packages/remix-node/globals.ts @@ -2,12 +2,12 @@ import { atob, btoa } from "./base64"; import { Blob as NodeBlob, File as NodeFile, + FormData as NodeFormData, Headers as NodeHeaders, Request as NodeRequest, Response as NodeResponse, fetch as nodeFetch, } from "./fetch"; -import { FormData as NodeFormData } from "./formData"; declare global { namespace NodeJS { diff --git a/packages/remix-node/index.ts b/packages/remix-node/index.ts index 0b305188882..49f24a65abf 100644 --- a/packages/remix-node/index.ts +++ b/packages/remix-node/index.ts @@ -5,19 +5,15 @@ sourceMapSupport.install(); export { AbortController } from "abort-controller"; export type { - // HeadersInit, - // RequestInfo, - RequestInit, // ResponseInit, + HeadersInit, + RequestInfo, + RequestInit, + ResponseInit, } from "./fetch"; -export { Headers, Request, Response, fetch } from "./fetch"; - -export { FormData } from "./formData"; -export type { UploadHandler, UploadHandlerArgs } from "./formData"; +export { fetch, Headers, Request, Response, FormData } from "./fetch"; export { installGlobals } from "./globals"; -export { parseMultipartFormData as unstable_parseMultipartFormData } from "./parseMultipartFormData"; - export { createFileSessionStorage } from "./sessions/fileStorage"; export { @@ -45,6 +41,7 @@ export { isCookie, isSession, json, + parseMultipartFormData, redirect, } from "@remix-run/server-runtime"; @@ -81,4 +78,6 @@ export type { SessionData, SessionIdStorageStrategy, SessionStorage, + UploadHandlerPart, + UploadHandler, } from "@remix-run/server-runtime"; diff --git a/packages/remix-node/stream.ts b/packages/remix-node/stream.ts index 6f8bd6f70a5..992040b0629 100644 --- a/packages/remix-node/stream.ts +++ b/packages/remix-node/stream.ts @@ -1,6 +1,6 @@ import type { Writable } from "stream"; -export function pipeReadableStreamToWritable( +export async function pipeReadableStreamToWritable( stream: ReadableStream, writable: Writable ) { @@ -8,32 +8,37 @@ export function pipeReadableStreamToWritable( async function read() { let { done, value } = await reader.read(); + if (done) { writable.end(); return; } writable.write(value); - read(); + + await read(); } - read(); + await read(); } export async function readableStreamToBase64String(stream: ReadableStream) { let reader = stream.getReader(); - let body = ""; + let chunks: Uint8Array[] = []; + async function read() { let { done, value } = await reader.read(); + if (done) { return; } else if (value) { - body += Buffer.from(value).toString("base64"); + chunks.push(value); } + await read(); } await read(); - return body; + return Buffer.concat(chunks).toString("base64"); } diff --git a/packages/remix-node/upload/fileUploadHandler.ts b/packages/remix-node/upload/fileUploadHandler.ts index a91bf11b9d7..c204ee2e329 100644 --- a/packages/remix-node/upload/fileUploadHandler.ts +++ b/packages/remix-node/upload/fileUploadHandler.ts @@ -3,9 +3,9 @@ import { createReadStream, createWriteStream } from "fs"; import { rm, mkdir, readFile, stat } from "fs/promises"; import { tmpdir } from "os"; import { basename, dirname, extname, resolve as resolvePath } from "path"; +import type { UploadHandler } from "@remix-run/server-runtime"; import { MeterError } from "./meter"; -import type { UploadHandler } from "../formData"; export type FileUploadHandlerFilterArgs = { filename: string; diff --git a/packages/remix-node/upload/memoryUploadHandler.ts b/packages/remix-node/upload/memoryUploadHandler.ts index 69f6d665e4b..edb673249fb 100644 --- a/packages/remix-node/upload/memoryUploadHandler.ts +++ b/packages/remix-node/upload/memoryUploadHandler.ts @@ -1,5 +1,6 @@ +import type { UploadHandler } from "@remix-run/server-runtime"; + import { File } from "../fetch"; -import type { UploadHandler } from "../formData"; import { MeterError } from "./meter"; export type MemoryUploadHandlerFilterArgs = { diff --git a/packages/remix-node/__tests__/parseMultipartFormData-test.ts b/packages/remix-server-runtime/__tests__/parseMultipartFormData-test.ts similarity index 74% rename from packages/remix-node/__tests__/parseMultipartFormData-test.ts rename to packages/remix-server-runtime/__tests__/parseMultipartFormData-test.ts index cd1f1ff353d..5c1ac194b3d 100644 --- a/packages/remix-node/__tests__/parseMultipartFormData-test.ts +++ b/packages/remix-server-runtime/__tests__/parseMultipartFormData-test.ts @@ -1,28 +1,24 @@ +import { + Request as NodeRequest, + FormData as NodeFormData, +} from "@remix-run/web-fetch"; import { Blob, File } from "@remix-run/web-file"; -import { Request as NodeRequest } from "../fetch"; -import { FormData as NodeFormData } from "../formData"; -import { internalParseFormData } from "../parseMultipartFormData"; +import { parseMultipartFormData } from "../formData"; -describe("internalParseFormData", () => { +describe("parseMultipartFormData", () => { it("can use a custom upload handler", async () => { let formData = new NodeFormData(); formData.set("a", "value"); formData.set("blob", new Blob(["blob"]), "blob.txt"); formData.set("file", new File(["file"], "file.txt")); - // TODO: Figure out why the stream is failing when formData is passed directly as body let req = new NodeRequest("https://test.com", { method: "post", body: formData, }); - req = new NodeRequest("https://test.com", { - method: "post", - headers: req.headers, - body: await req.text(), - }); - let parsedFormData = await internalParseFormData( + let parsedFormData = await parseMultipartFormData( req, async ({ filename, data, contentType }) => { let chunks = []; @@ -57,7 +53,7 @@ describe("internalParseFormData", () => { }); try { - await internalParseFormData(req, async () => { + await parseMultipartFormData(req, async () => { throw new Error("test error"); }); throw new Error("should have thrown"); diff --git a/packages/remix-node/parseMultipartFormData.ts b/packages/remix-server-runtime/formData.ts similarity index 55% rename from packages/remix-node/parseMultipartFormData.ts rename to packages/remix-server-runtime/formData.ts index c081f0d12af..22cdc044932 100644 --- a/packages/remix-node/parseMultipartFormData.ts +++ b/packages/remix-server-runtime/formData.ts @@ -1,25 +1,26 @@ -import { FormData } from "@remix-run/web-fetch"; import { streamMultipart } from "@web3-storage/multipart-parser"; -import type { Request as NodeRequest } from "./fetch"; -import type { UploadHandler } from "./formData"; +export type UploadHandlerPart = { + name: string; + filename: string; + contentType: string; + data: AsyncIterable; +}; + +export type UploadHandler = ( + part: UploadHandlerPart +) => Promise; /** * Allows you to handle multipart forms (file uploads) for your app. * + * TODO: Update this comment * @see https://remix.run/api/remix#parsemultipartformdata-node */ -export function parseMultipartFormData( - request: Request | NodeRequest, - uploadHandler: UploadHandler -) { - return internalParseFormData(request, uploadHandler); -} - -export const internalParseFormData = async ( +export async function parseMultipartFormData( request: Request, uploadHandler: UploadHandler -) => { +): Promise { let contentType = request.headers.get("Content-Type") || ""; let [type, boundary] = contentType.split(/\s*;\s*boundary=/); @@ -28,22 +29,14 @@ export const internalParseFormData = async ( } let formData = new FormData(); - - let parts = streamMultipart(request.clone().body, boundary); + let parts: AsyncIterable = + streamMultipart(request.body, boundary); for await (let part of parts) { if (part.done) break; - if (!part.filename) { - let chunks = []; - for await (let chunk of part.data) { - chunks.push(chunk); - } - - formData.append( - part.name, - new TextDecoder().decode(mergeArrays(...chunks)) - ); + if (!part.contentType || part.contentType.startsWith("text/")) { + formData.append(part.name, await bufferPart(part)); } else { let file = await uploadHandler(part); if (typeof file !== "undefined") { @@ -53,9 +46,19 @@ export const internalParseFormData = async ( } return formData; -}; +} + +async function bufferPart(part: UploadHandlerPart): Promise { + let chunks = []; + + for await (let chunk of part.data) { + chunks.push(chunk); + } + + return new TextDecoder().decode(mergeArrays(...chunks)); +} -export function mergeArrays(...arrays: Uint8Array[]) { +function mergeArrays(...arrays: Uint8Array[]) { let out = new Uint8Array( arrays.reduce((total, arr) => total + arr.length, 0) ); diff --git a/packages/remix-server-runtime/index.ts b/packages/remix-server-runtime/index.ts index 7d19aea99fe..9b7cd91f194 100644 --- a/packages/remix-server-runtime/index.ts +++ b/packages/remix-server-runtime/index.ts @@ -1,5 +1,6 @@ // Default implementations for the Remix server runtime interface export { createCookieFactory, isCookie } from "./cookies"; +export { parseMultipartFormData } from "./formData"; export { json, redirect } from "./responses"; export { createRequestHandler } from "./server"; export { @@ -59,4 +60,6 @@ export type { SessionStorage, SignFunction, UnsignFunction, + UploadHandlerPart, + UploadHandler, } from "./reexport"; diff --git a/packages/remix-server-runtime/package.json b/packages/remix-server-runtime/package.json index e45fe1aa2d3..0ea0df135c6 100644 --- a/packages/remix-server-runtime/package.json +++ b/packages/remix-server-runtime/package.json @@ -15,6 +15,7 @@ }, "dependencies": { "@types/cookie": "^0.4.0", + "@web3-storage/multipart-parser": "^1.0.0", "cookie": "^0.4.1", "jsesc": "^3.0.1", "react-router-dom": "^6.2.2", diff --git a/packages/remix-server-runtime/reexport.ts b/packages/remix-server-runtime/reexport.ts index 3747a38bf48..b8525981da9 100644 --- a/packages/remix-server-runtime/reexport.ts +++ b/packages/remix-server-runtime/reexport.ts @@ -5,6 +5,8 @@ export type { ServerEntryModule, } from "./build"; +export type { UploadHandlerPart, UploadHandler } from "./formData"; + export type { Cookie, CookieOptions, diff --git a/packages/remix-vercel/server.ts b/packages/remix-vercel/server.ts index da0c07a8fcc..96a0cf14d11 100644 --- a/packages/remix-vercel/server.ts +++ b/packages/remix-vercel/server.ts @@ -51,9 +51,9 @@ export function createRequestHandler({ ? getLoadContext(req, res) : undefined; - let response = await handleRequest(request, loadContext); + let response = (await handleRequest(request, loadContext)) as NodeResponse; - sendRemixResponse(res, response as NodeResponse); + await sendRemixResponse(res, response); }; } @@ -61,6 +61,7 @@ export function createRemixHeaders( requestHeaders: VercelRequest["headers"] ): NodeHeaders { let headers = new NodeHeaders(); + for (let key in requestHeaders) { let header = requestHeaders[key]!; // set-cookie is an array (maybe others) @@ -94,10 +95,10 @@ export function createRemixRequest(req: VercelRequest): NodeRequest { return new NodeRequest(url.href, init); } -export function sendRemixResponse( +export async function sendRemixResponse( res: VercelResponse, nodeResponse: NodeResponse -): void { +): Promise { res.statusMessage = nodeResponse.statusText; let multiValueHeaders = nodeResponse.headers.raw(); res.writeHead( @@ -107,7 +108,7 @@ export function sendRemixResponse( ); if (nodeResponse.body) { - pipeReadableStreamToWritable(nodeResponse.body, res); + await pipeReadableStreamToWritable(nodeResponse.body, res); } else { res.end(); } From ab3491b861c0a23997f8cf8c8dcf4721553f3caa Mon Sep 17 00:00:00 2001 From: Jacob Ebey Date: Sat, 7 May 2022 15:18:17 -0700 Subject: [PATCH 21/47] feat: updated node file upload handler to support slice feat: moved form data tests to server-runtime chore: added tests --- packages/remix-cloudflare/index.ts | 6 +- packages/remix-node/__tests__/assets/test.txt | 1 + .../__tests__/fileUploadHandler-test.ts | 53 ++++++ packages/remix-node/index.ts | 8 +- packages/remix-node/magicExports/remix.ts | 2 +- packages/remix-node/package.json | 5 +- packages/remix-node/stream.ts | 152 ++++++++++++++++++ .../remix-node/upload/fileUploadHandler.ts | 44 +++-- ...ipartFormData-test.ts => formData-test.ts} | 7 +- packages/remix-server-runtime/formData.ts | 4 +- packages/remix-server-runtime/index.ts | 6 +- packages/remix-server-runtime/package.json | 1 + packages/remix-server-runtime/reexport.ts | 4 + .../upload/memoryUploadHandler.ts | 3 +- .../upload/meter.ts | 0 rollup.config.js | 56 +++---- yarn.lock | 21 ++- 17 files changed, 310 insertions(+), 63 deletions(-) create mode 100644 packages/remix-node/__tests__/assets/test.txt create mode 100644 packages/remix-node/__tests__/fileUploadHandler-test.ts rename packages/remix-server-runtime/__tests__/{parseMultipartFormData-test.ts => formData-test.ts} (88%) rename packages/{remix-node => remix-server-runtime}/upload/memoryUploadHandler.ts (97%) rename packages/{remix-node => remix-server-runtime}/upload/meter.ts (100%) diff --git a/packages/remix-cloudflare/index.ts b/packages/remix-cloudflare/index.ts index 8352d1200eb..805785b00df 100644 --- a/packages/remix-cloudflare/index.ts +++ b/packages/remix-cloudflare/index.ts @@ -15,8 +15,10 @@ export { isCookie, isSession, json, - parseMultipartFormData, redirect, + unstable_parseMultipartFormData, + unstable_createMemoryUploadHandler, + MeterError } from "@remix-run/server-runtime"; export type { @@ -54,4 +56,6 @@ export type { SessionStorage, UploadHandlerPart, UploadHandler, + MemoryUploadHandlerOptions, + MemoryUploadHandlerFilterArgs, } from "@remix-run/server-runtime"; diff --git a/packages/remix-node/__tests__/assets/test.txt b/packages/remix-node/__tests__/assets/test.txt new file mode 100644 index 00000000000..30f51a3fba5 --- /dev/null +++ b/packages/remix-node/__tests__/assets/test.txt @@ -0,0 +1 @@ +hello, world! \ No newline at end of file diff --git a/packages/remix-node/__tests__/fileUploadHandler-test.ts b/packages/remix-node/__tests__/fileUploadHandler-test.ts new file mode 100644 index 00000000000..47e864ea807 --- /dev/null +++ b/packages/remix-node/__tests__/fileUploadHandler-test.ts @@ -0,0 +1,53 @@ +import * as fs from "fs"; +import * as path from "path"; +import { ReadableStream } from "@remix-run/web-stream"; + +import { NodeOnDiskFile } from "../upload/fileUploadHandler"; +import { readableStreamToString } from "../stream"; + +beforeAll(() => { + global.ReadableStream = ReadableStream; +}); + +describe("NodeOnDiskFile", () => { + let filepath = path.resolve(__dirname, "assets/test.txt"); + let size = fs.statSync(filepath).size; + let contents = fs.readFileSync(filepath, "utf-8"); + let file: NodeOnDiskFile; + beforeEach(() => { + file = new NodeOnDiskFile(filepath, size, "text/plain"); + }); + + it("can read file as text", async () => { + expect(await file.text()).toBe(contents); + }); + + it("can get an arrayBuffer", async () => { + let buffer = await file.arrayBuffer(); + expect(buffer.byteLength).toBe(size); + expect(buffer).toEqual(Buffer.from(contents)); + }); + + it("can use stream", async () => { + expect(await readableStreamToString(file.stream() as any)).toBe(contents); + }); + + it("can slice file and get text", async () => { + let sliced = await file.slice(1, 5); + expect(await sliced.text()).toBe(contents.slice(1, 5)); + }); + + it("can sice file and get an arrayBuffer", async () => { + let sliced = await file.slice(1, 5); + let buffer = await sliced.arrayBuffer(); + expect(buffer.byteLength).toBe(4); + expect(buffer).toEqual(Buffer.from(contents.slice(1, 5))); + }); + + it("can slice file and use stream", async () => { + let sliced = await file.slice(1, 5); + expect(await readableStreamToString(sliced.stream() as any)).toBe( + contents.slice(1, 5) + ); + }); +}); diff --git a/packages/remix-node/index.ts b/packages/remix-node/index.ts index 49f24a65abf..e4c2621ceb9 100644 --- a/packages/remix-node/index.ts +++ b/packages/remix-node/index.ts @@ -20,8 +20,6 @@ export { createFileUploadHandler as unstable_createFileUploadHandler, NodeOnDiskFile, } from "./upload/fileUploadHandler"; -export { createMemoryUploadHandler as unstable_createMemoryUploadHandler } from "./upload/memoryUploadHandler"; -export { MeterError } from "./upload/meter"; export { createCookie, @@ -41,8 +39,10 @@ export { isCookie, isSession, json, - parseMultipartFormData, redirect, + unstable_parseMultipartFormData, + unstable_createMemoryUploadHandler, + MeterError, } from "@remix-run/server-runtime"; export type { @@ -80,4 +80,6 @@ export type { SessionStorage, UploadHandlerPart, UploadHandler, + MemoryUploadHandlerOptions, + MemoryUploadHandlerFilterArgs, } from "@remix-run/server-runtime"; diff --git a/packages/remix-node/magicExports/remix.ts b/packages/remix-node/magicExports/remix.ts index 8f637aad858..c20a24a48df 100644 --- a/packages/remix-node/magicExports/remix.ts +++ b/packages/remix-node/magicExports/remix.ts @@ -13,4 +13,4 @@ export { unstable_parseMultipartFormData, } from "@remix-run/node"; -export type { UploadHandler, UploadHandlerArgs } from "@remix-run/node"; +export type { UploadHandler, UploadHandlerPart } from "@remix-run/node"; diff --git a/packages/remix-node/package.json b/packages/remix-node/package.json index 07f4dff0214..f6e0cda9d70 100644 --- a/packages/remix-node/package.json +++ b/packages/remix-node/package.json @@ -13,14 +13,15 @@ }, "dependencies": { "@remix-run/server-runtime": "1.4.3", - "@remix-run/web-fetch": "^4.1.1", + "@remix-run/web-fetch": "^4.1.2", "@remix-run/web-file": "^3.0.2", "@remix-run/web-stream": "^1.0.3", "@web3-storage/multipart-parser": "^1.0.0", "abort-controller": "^3.0.0", "blob-stream": "^0.1.3", "cookie-signature": "^1.1.0", - "source-map-support": "^0.5.21" + "source-map-support": "^0.5.21", + "stream-slice": "^0.1.2" }, "devDependencies": { "@types/blob-stream": "^0.1.30", diff --git a/packages/remix-node/stream.ts b/packages/remix-node/stream.ts index 992040b0629..20e65afc9b0 100644 --- a/packages/remix-node/stream.ts +++ b/packages/remix-node/stream.ts @@ -1,4 +1,7 @@ import type { Writable } from "stream"; +import { Stream } from "stream"; + +const { readableHighWaterMark } = new Stream.Readable(); export async function pipeReadableStreamToWritable( stream: ReadableStream, @@ -42,3 +45,152 @@ export async function readableStreamToBase64String(stream: ReadableStream) { return Buffer.concat(chunks).toString("base64"); } + +export async function readableStreamToString(stream: ReadableStream) { + let reader = stream.getReader(); + let chunks: Uint8Array[] = []; + + async function read() { + let { done, value } = await reader.read(); + + if (done) { + return; + } else if (value) { + chunks.push(value); + } + + await read(); + } + + await read(); + + return Buffer.concat(chunks).toString(); +} + +export const readableStreamFromStream = ( + source: Stream & { readableHighWaterMark?: number } +) => { + let pump = new StreamPump(source); + let stream = new ReadableStream(pump, pump); + return stream; +}; + +class StreamPump { + public highWaterMark: number; + public accumalatedSize: number; + private stream: Stream & { + readableHighWaterMark?: number; + readable?: boolean; + resume?: () => void; + pause?: () => void; + destroy?: (error?: Error) => void; + }; + private controller?: ReadableStreamController; + + /** + * @param {Stream & { + * readableHighWaterMark?: number + * readable?:boolean, + * resume?: () => void, + * pause?: () => void + * destroy?: (error?:Error) => void + * }} stream + */ + constructor( + stream: Stream & { + readableHighWaterMark?: number; + readable?: boolean; + resume?: () => void; + pause?: () => void; + destroy?: (error?: Error) => void; + } + ) { + this.highWaterMark = stream.readableHighWaterMark || readableHighWaterMark; + this.accumalatedSize = 0; + this.stream = stream; + this.enqueue = this.enqueue.bind(this); + this.error = this.error.bind(this); + this.close = this.close.bind(this); + } + + /** + * @param {Uint8Array} [chunk] + */ + size(chunk: Uint8Array) { + return chunk?.byteLength || 0; + } + + /** + * @param {ReadableStreamController} controller + */ + start(controller: ReadableStreamController) { + this.controller = controller; + this.stream.on("data", this.enqueue); + this.stream.once("error", this.error); + this.stream.once("end", this.close); + this.stream.once("close", this.close); + } + + pull() { + this.resume(); + } + + cancel(reason: Error) { + if (this.stream.destroy) { + this.stream.destroy(reason); + } + + this.stream.off("data", this.enqueue); + this.stream.off("error", this.error); + this.stream.off("end", this.close); + this.stream.off("close", this.close); + } + + enqueue(chunk: Uint8Array | string) { + if (this.controller) { + try { + let bytes = chunk instanceof Uint8Array ? chunk : Buffer.from(chunk); + + let available = (this.controller.desiredSize || 0) - bytes.byteLength; + this.controller.enqueue(bytes); + if (available <= 0) { + this.pause(); + } + } catch { + this.controller.error( + new Error( + "Could not create Buffer, chunk must be of type string or an instance of Buffer, ArrayBuffer, or Array or an Array-like Object" + ) + ); + // @ts-expect-error + this.cancel(); + } + } + } + + pause() { + if (this.stream.pause) { + this.stream.pause(); + } + } + + resume() { + if (this.stream.readable && this.stream.resume) { + this.stream.resume(); + } + } + + close() { + if (this.controller) { + this.controller.close(); + delete this.controller; + } + } + + error(error: Error) { + if (this.controller) { + this.controller.error(error); + delete this.controller; + } + } +} diff --git a/packages/remix-node/upload/fileUploadHandler.ts b/packages/remix-node/upload/fileUploadHandler.ts index c204ee2e329..659606baacd 100644 --- a/packages/remix-node/upload/fileUploadHandler.ts +++ b/packages/remix-node/upload/fileUploadHandler.ts @@ -1,11 +1,15 @@ import { randomBytes } from "crypto"; import { createReadStream, createWriteStream } from "fs"; -import { rm, mkdir, readFile, stat } from "fs/promises"; +import { rm, mkdir, stat } from "fs/promises"; import { tmpdir } from "os"; import { basename, dirname, extname, resolve as resolvePath } from "path"; +import type { Readable, Writable } from "stream"; +import { MeterError } from "@remix-run/server-runtime"; import type { UploadHandler } from "@remix-run/server-runtime"; +// @ts-expect-error +import * as streamSlice from "stream-slice"; -import { MeterError } from "./meter"; +import { readableStreamFromStream, readableStreamToString } from "../stream"; export type FileUploadHandlerFilterArgs = { filename: string; @@ -148,13 +152,28 @@ export class NodeOnDiskFile implements File { constructor( private filepath: string, public size: number, - public type: string + public type: string, + private slicer?: Writable & Readable ) { this.name = basename(filepath); } + slice(start?: number, end?: number, contentType?: string): Blob { + start = typeof start !== "undefined" ? start : 0; + end = typeof end !== "undefined" ? end : this.size; + return new NodeOnDiskFile( + this.filepath, + end - start, + this.type, + streamSlice.slice(start, end) + ); + } + async arrayBuffer(): Promise { - let stream = createReadStream(this.filepath); + let stream: Readable = createReadStream(this.filepath); + if (this.slicer) { + stream = stream.pipe(this.slicer); + } return new Promise((resolve, reject) => { let buf: any[] = []; @@ -164,16 +183,23 @@ export class NodeOnDiskFile implements File { }); } - slice(start?: any, end?: any, contentType?: any): Blob { - throw new Error("Method not implemented."); - } stream(): ReadableStream; stream(): NodeJS.ReadableStream; stream(): ReadableStream | NodeJS.ReadableStream { - return createReadStream(this.filepath); + let stream: Readable = createReadStream(this.filepath); + if (this.slicer) { + stream = stream.pipe(this.slicer); + } + return readableStreamFromStream(stream); } + text(): Promise { - return readFile(this.filepath, "utf-8"); + let stream: Readable = createReadStream(this.filepath); + if (this.slicer) { + stream = stream.pipe(this.slicer); + } + + return readableStreamToString(readableStreamFromStream(stream)); } get [Symbol.toStringTag]() { diff --git a/packages/remix-server-runtime/__tests__/parseMultipartFormData-test.ts b/packages/remix-server-runtime/__tests__/formData-test.ts similarity index 88% rename from packages/remix-server-runtime/__tests__/parseMultipartFormData-test.ts rename to packages/remix-server-runtime/__tests__/formData-test.ts index 5c1ac194b3d..ac78ccb8e86 100644 --- a/packages/remix-server-runtime/__tests__/parseMultipartFormData-test.ts +++ b/packages/remix-server-runtime/__tests__/formData-test.ts @@ -25,6 +25,7 @@ describe("parseMultipartFormData", () => { for await (let chunk of data) { chunks.push(chunk); } + console.log(chunks); return new File(chunks, filename, { type: contentType }); } ); @@ -41,16 +42,10 @@ describe("parseMultipartFormData", () => { let formData = new NodeFormData(); formData.set("blob", new Blob(["blob"]), "blob.txt"); - // TODO: Figure out why the stream is failing when formData is passed directly as body let req = new NodeRequest("https://test.com", { method: "post", body: formData, }); - req = new NodeRequest("https://test.com", { - method: "post", - headers: req.headers, - body: await req.text(), - }); try { await parseMultipartFormData(req, async () => { diff --git a/packages/remix-server-runtime/formData.ts b/packages/remix-server-runtime/formData.ts index 22cdc044932..0a8929d2b02 100644 --- a/packages/remix-server-runtime/formData.ts +++ b/packages/remix-server-runtime/formData.ts @@ -35,12 +35,12 @@ export async function parseMultipartFormData( for await (let part of parts) { if (part.done) break; - if (!part.contentType || part.contentType.startsWith("text/")) { + if (!part.filename) { formData.append(part.name, await bufferPart(part)); } else { let file = await uploadHandler(part); if (typeof file !== "undefined") { - formData.append(part.name, file); + formData.append(part.name, file as Blob); } } } diff --git a/packages/remix-server-runtime/index.ts b/packages/remix-server-runtime/index.ts index 9b7cd91f194..2a2890f1778 100644 --- a/packages/remix-server-runtime/index.ts +++ b/packages/remix-server-runtime/index.ts @@ -1,6 +1,6 @@ // Default implementations for the Remix server runtime interface export { createCookieFactory, isCookie } from "./cookies"; -export { parseMultipartFormData } from "./formData"; +export { parseMultipartFormData as unstable_parseMultipartFormData } from "./formData"; export { json, redirect } from "./responses"; export { createRequestHandler } from "./server"; export { @@ -10,6 +10,8 @@ export { } from "./sessions"; export { createCookieSessionStorageFactory } from "./sessions/cookieStorage"; export { createMemorySessionStorageFactory } from "./sessions/memoryStorage"; +export { createMemoryUploadHandler as unstable_createMemoryUploadHandler } from "./upload/memoryUploadHandler"; +export { MeterError } from "./upload/meter"; // Types for the Remix server runtime interface export type { @@ -62,4 +64,6 @@ export type { UnsignFunction, UploadHandlerPart, UploadHandler, + MemoryUploadHandlerOptions, + MemoryUploadHandlerFilterArgs, } from "./reexport"; diff --git a/packages/remix-server-runtime/package.json b/packages/remix-server-runtime/package.json index 0ea0df135c6..9044b8671df 100644 --- a/packages/remix-server-runtime/package.json +++ b/packages/remix-server-runtime/package.json @@ -27,6 +27,7 @@ "react-dom": ">=16.8" }, "devDependencies": { + "@remix-run/web-file": "^3.0.2", "@types/jsesc": "^2.5.1", "@types/set-cookie-parser": "^2.4.1" }, diff --git a/packages/remix-server-runtime/reexport.ts b/packages/remix-server-runtime/reexport.ts index b8525981da9..add615c4b6c 100644 --- a/packages/remix-server-runtime/reexport.ts +++ b/packages/remix-server-runtime/reexport.ts @@ -6,6 +6,10 @@ export type { } from "./build"; export type { UploadHandlerPart, UploadHandler } from "./formData"; +export type { + MemoryUploadHandlerOptions, + MemoryUploadHandlerFilterArgs, +} from "./upload/memoryUploadHandler"; export type { Cookie, diff --git a/packages/remix-node/upload/memoryUploadHandler.ts b/packages/remix-server-runtime/upload/memoryUploadHandler.ts similarity index 97% rename from packages/remix-node/upload/memoryUploadHandler.ts rename to packages/remix-server-runtime/upload/memoryUploadHandler.ts index edb673249fb..f2c8c3f5fdc 100644 --- a/packages/remix-node/upload/memoryUploadHandler.ts +++ b/packages/remix-server-runtime/upload/memoryUploadHandler.ts @@ -1,6 +1,5 @@ import type { UploadHandler } from "@remix-run/server-runtime"; -import { File } from "../fetch"; import { MeterError } from "./meter"; export type MemoryUploadHandlerFilterArgs = { @@ -36,11 +35,11 @@ export function createMemoryUploadHandler({ let size = 0; let chunks = []; for await (let chunk of data) { - chunks.push(chunk); size += chunk.length; if (size > maxFileSize) { throw new MeterError(name, maxFileSize); } + chunks.push(chunk); } return new File(chunks, filename, { type: contentType }); diff --git a/packages/remix-node/upload/meter.ts b/packages/remix-server-runtime/upload/meter.ts similarity index 100% rename from packages/remix-node/upload/meter.ts rename to packages/remix-server-runtime/upload/meter.ts diff --git a/rollup.config.js b/rollup.config.js index 145fedd3139..47b4acfb207 100644 --- a/rollup.config.js +++ b/rollup.config.js @@ -872,34 +872,34 @@ function copyToPlaygrounds() { return { name: "copy-to-remix-playground", async writeBundle(options, bundle) { - let playgroundsDir = path.join(__dirname, "playground"); - let playgrounds = await fs.promises.readdir(playgroundsDir); - let writtenDir = path.join(__dirname, options.dir); - for (let playground of playgrounds) { - let playgroundDir = path.join(playgroundsDir, playground); - if (!fse.statSync(playgroundDir).isDirectory()) { - continue; - } - let destDir = writtenDir.replace( - path.join(__dirname, "build"), - playgroundDir - ); - await fse.copy(writtenDir, destDir); - - // tickle live reload by touching the server entry - let serverEntry = ["tsx", "js", "jsx"].find((entryPathExtension) => - fse.existsSync( - path.join( - playgroundDir, - "app", - `entry.server.${entryPathExtension}` - ) - ) - ); - let serverEntryPath = path.join(playgroundDir, "app", serverEntry); - let serverEntryContent = await fse.readFile(serverEntryPath); - await fse.writeFile(serverEntryPath, serverEntryContent); - } + // let playgroundsDir = path.join(__dirname, "playground"); + // let playgrounds = await fs.promises.readdir(playgroundsDir); + // let writtenDir = path.join(__dirname, options.dir); + // for (let playground of playgrounds) { + // let playgroundDir = path.join(playgroundsDir, playground); + // if (!fse.statSync(playgroundDir).isDirectory()) { + // continue; + // } + // let destDir = writtenDir.replace( + // path.join(__dirname, "build"), + // playgroundDir + // ); + // await fse.copy(writtenDir, destDir); + + // // tickle live reload by touching the server entry + // let serverEntry = ["tsx", "js", "jsx"].find((entryPathExtension) => + // fse.existsSync( + // path.join( + // playgroundDir, + // "app", + // `entry.server.${entryPathExtension}` + // ) + // ) + // ); + // let serverEntryPath = path.join(playgroundDir, "app", serverEntry); + // let serverEntryContent = await fse.readFile(serverEntryPath); + // await fse.writeFile(serverEntryPath, serverEntryContent); + // } }, }; } diff --git a/yarn.lock b/yarn.lock index 124eb01f595..c08d55a7660 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1568,7 +1568,7 @@ stack-utils "2.0.5" yazl "2.5.1" -"@remix-run/web-blob@^3.0.3": +"@remix-run/web-blob@^3.0.3", "@remix-run/web-blob@^3.0.4": version "3.0.4" resolved "https://registry.npmjs.org/@remix-run/web-blob/-/web-blob-3.0.4.tgz#99c67b9d0fb641bd0c07d267fd218ae5aa4ae5ed" integrity sha512-AfegzZvSSDc+LwnXV+SwROTrDtoLiPxeFW+jxgvtDAnkuCX1rrzmVJ6CzqZ1Ai0bVfmJadkG5GxtAfYclpPmgw== @@ -1576,14 +1576,14 @@ "@remix-run/web-stream" "^1.0.0" web-encoding "1.1.5" -"@remix-run/web-fetch@^4.1.1": - version "4.1.1" - resolved "https://registry.npmjs.org/@remix-run/web-fetch/-/web-fetch-4.1.1.tgz#2b7ab898599ea1a273a31b357e7a19148e6cec26" - integrity sha512-rsGqRERL+aCYWgtHaZyEy4Xzic4IHS4A2AzWndtoDy+8mUqtBe95QtUoxX5J9jBMs1yl4A/YjXD6HwWP1yyLrw== +"@remix-run/web-fetch@^4.1.2": + version "4.1.2" + resolved "https://registry.npmjs.org/@remix-run/web-fetch/-/web-fetch-4.1.2.tgz#3a95be3c683f2d3d36fc9376199fdc7060b15ee6" + integrity sha512-3XQZnhZhhXw6W8B66rOFo9ZLXRXlfU608kni+Mz8Zi0tihF5M0Ar1Tb83aCjldetepyXtTcCO1V2li6QtqEqlw== dependencies: - "@remix-run/web-blob" "^3.0.3" + "@remix-run/web-blob" "^3.0.4" "@remix-run/web-form-data" "^3.0.2" - "@remix-run/web-stream" "^1.0.1" + "@remix-run/web-stream" "^1.0.3" "@web3-storage/multipart-parser" "^1.0.0" data-uri-to-buffer "^3.0.1" mrmime "^1.0.0" @@ -1602,7 +1602,7 @@ dependencies: web-encoding "1.1.5" -"@remix-run/web-stream@^1.0.0", "@remix-run/web-stream@^1.0.1", "@remix-run/web-stream@^1.0.3": +"@remix-run/web-stream@^1.0.0", "@remix-run/web-stream@^1.0.3": version "1.0.3" resolved "https://registry.yarnpkg.com/@remix-run/web-stream/-/web-stream-1.0.3.tgz#3284a6a45675d1455c4d9c8f31b89225c9006438" integrity sha512-wlezlJaA5NF6SsNMiwQnnAW6tnPzQ5I8qk0Y0pSohm0eHKa2FQ1QhEKLVVcDDu02TmkfHgnux0igNfeYhDOXiA== @@ -9420,6 +9420,11 @@ stream-shift@^1.0.0: resolved "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.1.tgz" integrity sha512-AiisoFqQ0vbGcZgQPY1cdP2I76glaVA/RauYR4G4thNFgkTqr90yXTo4LYX60Jl+sIlPNHHdGSwo01AvbKUSVQ== +stream-slice@^0.1.2: + version "0.1.2" + resolved "https://registry.npmjs.org/stream-slice/-/stream-slice-0.1.2.tgz#2dc4f4e1b936fb13f3eb39a2def1932798d07a4b" + integrity sha1-LcT04bk2+xPz6zmi3vGTJ5jQeks= + strict-event-emitter@^0.2.0: version "0.2.0" resolved "https://registry.npmjs.org/strict-event-emitter/-/strict-event-emitter-0.2.0.tgz" From 07b84a6f8496ef5602542225d2a0e93e192d66c6 Mon Sep 17 00:00:00 2001 From: Jacob Ebey Date: Sat, 7 May 2022 15:22:32 -0700 Subject: [PATCH 22/47] updated test to check error type --- .../remix-server-runtime/__tests__/formData-test.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/remix-server-runtime/__tests__/formData-test.ts b/packages/remix-server-runtime/__tests__/formData-test.ts index ac78ccb8e86..f08aeb7b4d0 100644 --- a/packages/remix-server-runtime/__tests__/formData-test.ts +++ b/packages/remix-server-runtime/__tests__/formData-test.ts @@ -25,7 +25,6 @@ describe("parseMultipartFormData", () => { for await (let chunk of data) { chunks.push(chunk); } - console.log(chunks); return new File(chunks, filename, { type: contentType }); } ); @@ -39,6 +38,12 @@ describe("parseMultipartFormData", () => { }); it("can throw errors in upload handlers", async () => { + class CustomError extends Error { + constructor() { + super("test error"); + } + } + let formData = new NodeFormData(); formData.set("blob", new Blob(["blob"]), "blob.txt"); @@ -49,11 +54,12 @@ describe("parseMultipartFormData", () => { try { await parseMultipartFormData(req, async () => { - throw new Error("test error"); + throw new CustomError(); }); throw new Error("should have thrown"); } catch (err) { expect(err.message).toBe("test error"); + expect(err).toBeInstanceOf(CustomError); } }); }); From c935a41cf62e355db4ecf3b4b0982d6a70c515c2 Mon Sep 17 00:00:00 2001 From: Jacob Ebey Date: Sat, 7 May 2022 15:53:51 -0700 Subject: [PATCH 23/47] fix: allow multiple slice of file fix: read size from disk for NodeOnDiskFile --- .../__tests__/fileUploadHandler-test.ts | 13 ++++- .../remix-node/upload/fileUploadHandler.ts | 52 ++++++++++++------- 2 files changed, 45 insertions(+), 20 deletions(-) diff --git a/packages/remix-node/__tests__/fileUploadHandler-test.ts b/packages/remix-node/__tests__/fileUploadHandler-test.ts index 47e864ea807..e73f8b0a4c5 100644 --- a/packages/remix-node/__tests__/fileUploadHandler-test.ts +++ b/packages/remix-node/__tests__/fileUploadHandler-test.ts @@ -15,7 +15,7 @@ describe("NodeOnDiskFile", () => { let contents = fs.readFileSync(filepath, "utf-8"); let file: NodeOnDiskFile; beforeEach(() => { - file = new NodeOnDiskFile(filepath, size, "text/plain"); + file = new NodeOnDiskFile(filepath, "text/plain"); }); it("can read file as text", async () => { @@ -32,11 +32,22 @@ describe("NodeOnDiskFile", () => { expect(await readableStreamToString(file.stream() as any)).toBe(contents); }); + it("can slice file and change type", async () => { + let sliced = await file.slice(1, 5, "text/rofl"); + expect(sliced.type).toBe("text/rofl"); + expect(await sliced.text()).toBe(contents.slice(1, 5)); + }); + it("can slice file and get text", async () => { let sliced = await file.slice(1, 5); expect(await sliced.text()).toBe(contents.slice(1, 5)); }); + it("can slice file twice and get text", async () => { + let sliced = (await file.slice(1, 5)).slice(1, 2); + expect(await sliced.text()).toBe(contents.slice(1, 5).slice(1, 2)); + }); + it("can sice file and get an arrayBuffer", async () => { let sliced = await file.slice(1, 5); let buffer = await sliced.arrayBuffer(); diff --git a/packages/remix-node/upload/fileUploadHandler.ts b/packages/remix-node/upload/fileUploadHandler.ts index 659606baacd..6b2a1bc09ae 100644 --- a/packages/remix-node/upload/fileUploadHandler.ts +++ b/packages/remix-node/upload/fileUploadHandler.ts @@ -1,9 +1,9 @@ import { randomBytes } from "crypto"; -import { createReadStream, createWriteStream } from "fs"; -import { rm, mkdir, stat } from "fs/promises"; +import { createReadStream, createWriteStream, statSync } from "fs"; +import { rm, mkdir, stat as statAsync } from "fs/promises"; import { tmpdir } from "os"; import { basename, dirname, extname, resolve as resolvePath } from "path"; -import type { Readable, Writable } from "stream"; +import type { Readable } from "stream"; import { MeterError } from "@remix-run/server-runtime"; import type { UploadHandler } from "@remix-run/server-runtime"; // @ts-expect-error @@ -71,7 +71,7 @@ async function uniqueFile(filepath: string) { for ( let i = 1; - await stat(uniqueFilepath) + await statAsync(uniqueFilepath) .then(() => true) .catch(() => false); i++ @@ -140,7 +140,7 @@ export function createFileUploadHandler({ } } - return new NodeOnDiskFile(filepath, size, contentType); + return new NodeOnDiskFile(filepath, contentType); }; } @@ -151,28 +151,38 @@ export class NodeOnDiskFile implements File { constructor( private filepath: string, - public size: number, public type: string, - private slicer?: Writable & Readable + private slicer?: { start: number; end: number } ) { this.name = basename(filepath); } - slice(start?: number, end?: number, contentType?: string): Blob { - start = typeof start !== "undefined" ? start : 0; - end = typeof end !== "undefined" ? end : this.size; - return new NodeOnDiskFile( - this.filepath, - end - start, - this.type, - streamSlice.slice(start, end) - ); + public get size(): number { + if (this.slicer) { + return this.slicer.end - this.slicer.start; + } + + let stats = statSync(this.filepath); + return stats.size; + } + + slice(start?: number, end?: number, type?: string): Blob { + let startOffset = this.slicer?.start || 0; + + start = startOffset + (start || 0); + end = startOffset + (end || this.size); + return new NodeOnDiskFile(this.filepath, type || this.type, { + start, + end, + }); } async arrayBuffer(): Promise { let stream: Readable = createReadStream(this.filepath); if (this.slicer) { - stream = stream.pipe(this.slicer); + stream = stream.pipe( + streamSlice.slice(this.slicer.start, this.slicer.end) + ); } return new Promise((resolve, reject) => { @@ -188,7 +198,9 @@ export class NodeOnDiskFile implements File { stream(): ReadableStream | NodeJS.ReadableStream { let stream: Readable = createReadStream(this.filepath); if (this.slicer) { - stream = stream.pipe(this.slicer); + stream = stream.pipe( + streamSlice.slice(this.slicer.start, this.slicer.end) + ); } return readableStreamFromStream(stream); } @@ -196,7 +208,9 @@ export class NodeOnDiskFile implements File { text(): Promise { let stream: Readable = createReadStream(this.filepath); if (this.slicer) { - stream = stream.pipe(this.slicer); + stream = stream.pipe( + streamSlice.slice(this.slicer.start, this.slicer.end) + ); } return readableStreamToString(readableStreamFromStream(stream)); From af1b64af10fdca8d36c3159aaa0df0423fdd1c9e Mon Sep 17 00:00:00 2001 From: Jacob Ebey Date: Sat, 7 May 2022 15:54:46 -0700 Subject: [PATCH 24/47] do typecheck --- packages/remix-node/upload/fileUploadHandler.ts | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/remix-node/upload/fileUploadHandler.ts b/packages/remix-node/upload/fileUploadHandler.ts index 6b2a1bc09ae..a123e451f42 100644 --- a/packages/remix-node/upload/fileUploadHandler.ts +++ b/packages/remix-node/upload/fileUploadHandler.ts @@ -171,10 +171,14 @@ export class NodeOnDiskFile implements File { start = startOffset + (start || 0); end = startOffset + (end || this.size); - return new NodeOnDiskFile(this.filepath, type || this.type, { - start, - end, - }); + return new NodeOnDiskFile( + this.filepath, + typeof type === "string" ? type : this.type, + { + start, + end, + } + ); } async arrayBuffer(): Promise { From 09006c2814533d3e93ac6b1fa1436bffc8070db9 Mon Sep 17 00:00:00 2001 From: Jacob Ebey Date: Sun, 8 May 2022 14:15:22 -0700 Subject: [PATCH 25/47] feat: add Readable and WritableStream to globals chore: added signal back to express and vercel --- packages/remix-express/__tests__/server-test.ts | 2 +- packages/remix-express/server.ts | 8 ++++++++ packages/remix-node/globals.ts | 11 +++++++++++ packages/remix-vercel/__tests__/server-test.ts | 2 +- packages/remix-vercel/server.ts | 8 ++++++++ 5 files changed, 29 insertions(+), 2 deletions(-) diff --git a/packages/remix-express/__tests__/server-test.ts b/packages/remix-express/__tests__/server-test.ts index 4532360eb07..3d3b02b9c37 100644 --- a/packages/remix-express/__tests__/server-test.ts +++ b/packages/remix-express/__tests__/server-test.ts @@ -265,7 +265,7 @@ describe("express createRemixRequest", () => { "method": "GET", "parsedURL": "http://localhost:3000/foo/bar", "redirect": "follow", - "signal": null, + "signal": AbortSignal {}, }, } `); diff --git a/packages/remix-express/server.ts b/packages/remix-express/server.ts index ad63e37a27b..7ec529418a2 100644 --- a/packages/remix-express/server.ts +++ b/packages/remix-express/server.ts @@ -6,6 +6,7 @@ import type { Response as NodeResponse, } from "@remix-run/node"; import { + AbortController, createRequestHandler as createRemixRequestHandler, Headers as NodeHeaders, Request as NodeRequest, @@ -95,9 +96,16 @@ export function createRemixRequest(req: express.Request): NodeRequest { let origin = `${req.protocol}://${req.get("host")}`; let url = new URL(req.url, origin); + let controller = new AbortController(); + + req.on("close", () => { + controller.abort(); + }); + let init: NodeRequestInit = { method: req.method, headers: createRemixHeaders(req.headers), + signal: controller.signal, }; if (req.method !== "GET" && req.method !== "HEAD") { diff --git a/packages/remix-node/globals.ts b/packages/remix-node/globals.ts index 3159d9a1d1a..2732cbe17c4 100644 --- a/packages/remix-node/globals.ts +++ b/packages/remix-node/globals.ts @@ -1,3 +1,8 @@ +import { + ReadableStream as NodeReadableStream, + WritableStream as NodeWritableStream, +} from "@remix-run/web-stream"; + import { atob, btoa } from "./base64"; import { Blob as NodeBlob, @@ -27,6 +32,9 @@ declare global { Response: typeof Response; fetch: typeof fetch; FormData: typeof FormData; + + ReadableStream: typeof ReadableStream; + WritableStream: typeof WritableStream; } } } @@ -43,4 +51,7 @@ export function installGlobals() { global.Response = NodeResponse as unknown as typeof Response; global.fetch = nodeFetch as typeof fetch; global.FormData = NodeFormData; + + global.ReadableStream = NodeReadableStream; + global.WritableStream = NodeWritableStream; } diff --git a/packages/remix-vercel/__tests__/server-test.ts b/packages/remix-vercel/__tests__/server-test.ts index 53d6aa25027..7b2f138b16c 100644 --- a/packages/remix-vercel/__tests__/server-test.ts +++ b/packages/remix-vercel/__tests__/server-test.ts @@ -271,7 +271,7 @@ describe("vercel createRemixRequest", () => { "method": "GET", "parsedURL": "http://localhost:3000/foo/bar", "redirect": "follow", - "signal": null, + "signal": AbortSignal {}, }, } `); diff --git a/packages/remix-vercel/server.ts b/packages/remix-vercel/server.ts index 96a0cf14d11..eaef8f180ba 100644 --- a/packages/remix-vercel/server.ts +++ b/packages/remix-vercel/server.ts @@ -6,6 +6,7 @@ import type { Response as NodeResponse, } from "@remix-run/node"; import { + AbortController, createRequestHandler as createRemixRequestHandler, Headers as NodeHeaders, Request as NodeRequest, @@ -83,9 +84,16 @@ export function createRemixRequest(req: VercelRequest): NodeRequest { let protocol = req.headers["x-forwarded-proto"] || "https"; let url = new URL(req.url!, `${protocol}://${host}`); + let controller = new AbortController(); + + req.on("close", () => { + controller.abort(); + }); + let init: NodeRequestInit = { method: req.method, headers: createRemixHeaders(req.headers), + signal: controller.signal, }; if (req.method !== "GET" && req.method !== "HEAD") { From dc70911a660e0d0dabc62935303c142c2e87d9ec Mon Sep 17 00:00:00 2001 From: Jacob Ebey Date: Tue, 10 May 2022 12:03:31 -0700 Subject: [PATCH 26/47] updated fetch dep --- packages/remix-node/package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/remix-node/package.json b/packages/remix-node/package.json index f6e0cda9d70..fe1ff08825a 100644 --- a/packages/remix-node/package.json +++ b/packages/remix-node/package.json @@ -13,7 +13,7 @@ }, "dependencies": { "@remix-run/server-runtime": "1.4.3", - "@remix-run/web-fetch": "^4.1.2", + "@remix-run/web-fetch": "^4.1.3", "@remix-run/web-file": "^3.0.2", "@remix-run/web-stream": "^1.0.3", "@web3-storage/multipart-parser": "^1.0.0", diff --git a/yarn.lock b/yarn.lock index 2e08419cd59..8ae7779d3bd 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1576,10 +1576,10 @@ "@remix-run/web-stream" "^1.0.0" web-encoding "1.1.5" -"@remix-run/web-fetch@^4.1.2": - version "4.1.2" - resolved "https://registry.npmjs.org/@remix-run/web-fetch/-/web-fetch-4.1.2.tgz#3a95be3c683f2d3d36fc9376199fdc7060b15ee6" - integrity sha512-3XQZnhZhhXw6W8B66rOFo9ZLXRXlfU608kni+Mz8Zi0tihF5M0Ar1Tb83aCjldetepyXtTcCO1V2li6QtqEqlw== +"@remix-run/web-fetch@^4.1.3": + version "4.1.3" + resolved "https://registry.npmjs.org/@remix-run/web-fetch/-/web-fetch-4.1.3.tgz#8ad3077c1b5bd9fe2a8813d0ad3c84970a495c04" + integrity sha512-D3KXAEkzhR248mu7wCHReQrMrIo3Y9pDDa7TrlISnsOEvqkfWkJJF+PQWmOIKpOSHAhDg7TCb2tzvW8lc/MfHw== dependencies: "@remix-run/web-blob" "^3.0.4" "@remix-run/web-form-data" "^3.0.2" From dc8ee67bfd2ef6ecf6b9faeabd5ddfd130a1d4d9 Mon Sep 17 00:00:00 2001 From: Jacob Ebey Date: Tue, 10 May 2022 13:11:42 -0700 Subject: [PATCH 27/47] wait for file to finish writing before finishing --- packages/remix-node/upload/fileUploadHandler.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/remix-node/upload/fileUploadHandler.ts b/packages/remix-node/upload/fileUploadHandler.ts index a123e451f42..13fcc78efec 100644 --- a/packages/remix-node/upload/fileUploadHandler.ts +++ b/packages/remix-node/upload/fileUploadHandler.ts @@ -4,6 +4,8 @@ import { rm, mkdir, stat as statAsync } from "fs/promises"; import { tmpdir } from "os"; import { basename, dirname, extname, resolve as resolvePath } from "path"; import type { Readable } from "stream"; +import { finished } from "stream"; +import { promisify } from "util"; import { MeterError } from "@remix-run/server-runtime"; import type { UploadHandler } from "@remix-run/server-runtime"; // @ts-expect-error @@ -126,7 +128,7 @@ export function createFileUploadHandler({ let deleteFile = false; try { for await (let chunk of data) { - size += chunk.length; + size += chunk.byteLength; if (size > maxFileSize) { deleteFile = true; throw new MeterError(name, maxFileSize); @@ -134,7 +136,9 @@ export function createFileUploadHandler({ writeFileStream.write(chunk); } } finally { - writeFileStream.close(); + writeFileStream.end(); + await promisify(finished)(writeFileStream); + if (deleteFile) { await rm(filepath).catch(() => {}); } From c7bc9077c76f24f1a4608b0c57dea28cd60c5710 Mon Sep 17 00:00:00 2001 From: Jacob Ebey Date: Tue, 10 May 2022 13:13:21 -0700 Subject: [PATCH 28/47] use byteLength in memory upload handler --- packages/remix-server-runtime/upload/memoryUploadHandler.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/remix-server-runtime/upload/memoryUploadHandler.ts b/packages/remix-server-runtime/upload/memoryUploadHandler.ts index f2c8c3f5fdc..d0f8aec5b2d 100644 --- a/packages/remix-server-runtime/upload/memoryUploadHandler.ts +++ b/packages/remix-server-runtime/upload/memoryUploadHandler.ts @@ -35,7 +35,7 @@ export function createMemoryUploadHandler({ let size = 0; let chunks = []; for await (let chunk of data) { - size += chunk.length; + size += chunk.byteLength; if (size > maxFileSize) { throw new MeterError(name, maxFileSize); } From 9e549a4882c8da4bcd0a7ae8f92de2cecc32b85b Mon Sep 17 00:00:00 2001 From: Jacob Ebey Date: Tue, 10 May 2022 14:15:07 -0700 Subject: [PATCH 29/47] feat: updated parseMultipartFormData API feat: introduced composeUploadHandlers Each upload handler is now responsible for handling both files and fields. --- integration/file-uploads-test.ts | 36 +++++++++----- integration/upload-test.ts | 45 +++++++++++++---- packages/remix-cloudflare/index.ts | 1 + packages/remix-node/index.ts | 1 + .../remix-node/upload/fileUploadHandler.ts | 7 ++- .../__tests__/formData-test.ts | 6 ++- packages/remix-server-runtime/formData.ts | 49 +++++++------------ packages/remix-server-runtime/index.ts | 5 +- .../upload/memoryUploadHandler.ts | 10 ++-- 9 files changed, 101 insertions(+), 59 deletions(-) diff --git a/integration/file-uploads-test.ts b/integration/file-uploads-test.ts index 1e6cb60dcc3..70a115427b9 100644 --- a/integration/file-uploads-test.ts +++ b/integration/file-uploads-test.ts @@ -15,17 +15,24 @@ test.describe("file-uploads", () => { files: { "app/fileUploadHandler.js": js` import * as path from "path"; - import { unstable_createFileUploadHandler as createFileUploadHandler } from "@remix-run/node"; - - export let uploadHandler = createFileUploadHandler({ - directory: path.resolve(__dirname, "..", "uploads"), - maxFileSize: 10_000, // 10kb - // you probably want to avoid conflicts in production - // do not set to false or passthrough filename in real - // applications. - avoidFileConflicts: false, - file: ({ filename }) => filename - }); + import { + unstable_composeUploadHandlers as composeUploadHandlers, + unstable_createFileUploadHandler as createFileUploadHandler, + unstable_createMemoryUploadHandler as createMemoryUploadHandler, + } from "@remix-run/node"; + + export let uploadHandler = composeUploadHandlers( + createFileUploadHandler({ + directory: path.resolve(__dirname, "..", "uploads"), + maxFileSize: 10_000, // 10kb + // you probably want to avoid conflicts in production + // do not set to false or passthrough filename in real + // applications. + avoidFileConflicts: false, + file: ({ filename }) => filename + }), + createMemoryUploadHandler(), + ); `, "app/routes/file-upload.jsx": js` import { @@ -38,9 +45,13 @@ test.describe("file-uploads", () => { try { let formData = await parseMultipartFormData(request, uploadHandler); + if (formData.get("test") !== "hidden") { + return { errorMessage: "hidden field not in form data" }; + } + let file = formData.get("file"); if (typeof file === "string" || !file) { - throw new Error("invalid file type"); + return { errorMessage: "invalid file type" }; } return { name: file.name, size: file.size }; @@ -55,6 +66,7 @@ test.describe("file-uploads", () => {
+ 
{JSON.stringify(useActionData(), null, 2)}
diff --git a/integration/upload-test.ts b/integration/upload-test.ts index 6407d1609a3..1951ab3fe8e 100644 --- a/integration/upload-test.ts +++ b/integration/upload-test.ts @@ -14,22 +14,32 @@ test.beforeAll(async () => { "app/routes/file-upload-handler.jsx": js` import { json, + unstable_composeUploadHandlers as composeUploadHandlers, unstable_createFileUploadHandler as createFileUploadHandler, + unstable_createMemoryUploadHandler as createMemoryUploadHandler, unstable_parseMultipartFormData as parseMultipartFormData, MeterError, } from "@remix-run/node"; import { Form, useActionData } from "@remix-run/react"; export let action = async ({ request }) => { - let uploadHandler = createFileUploadHandler({ - directory: "./uploads", - maxFileSize: 15, - avoidFileConflicts: false, - file: ({ filename }) => filename, - }); + let uploadHandler = composeUploadHandlers( + createFileUploadHandler({ + directory: "./uploads", + maxFileSize: 15, + avoidFileConflicts: false, + file: ({ filename }) => filename, + }), + createMemoryUploadHandler(), + ); try { let formData = await parseMultipartFormData(request, uploadHandler); + + if (formData.get("test") !== "hidden") { + return { message: "hidden field not in form data" }; + } + let file = formData.get("file"); let size = typeof file !== "string" && file ? file.size : 0; @@ -47,6 +57,7 @@ test.beforeAll(async () => { return (
+
@@ -76,6 +87,11 @@ test.beforeAll(async () => { try { let formData = await parseMultipartFormData(request, uploadHandler); + + if (formData.get("test") !== "hidden") { + return { message: "hidden field not in form data" }; + } + let file = formData.get("file"); let size = typeof file !== "string" && file ? file.size : 0; @@ -93,6 +109,7 @@ test.beforeAll(async () => { return (
+
@@ -123,7 +140,9 @@ test("can upload a file with createFileUploadHandler", async ({ page }) => { expect(await app.getHtml("#size")).toMatch(">14<"); }); -test("can catch MeterError when file is too big with createFileUploadHandler", async ({ page }) => { +test("can catch MeterError when file is too big with createFileUploadHandler", async ({ + page, +}) => { let app = new PlaywrightFixture(appFixture, page); await app.goto("/file-upload-handler"); await app.uploadFile( @@ -146,7 +165,9 @@ test("can upload a file with createMemoryUploadHandler", async ({ page }) => { expect(await app.getHtml("#size")).toMatch(">14<"); }); -test("can catch MeterError when file is too big with createMemoryUploadHandler", async ({ page }) => { +test("can catch MeterError when file is too big with createMemoryUploadHandler", async ({ + page, +}) => { let app = new PlaywrightFixture(appFixture, page); await app.goto("/memory-upload-handler"); await app.uploadFile( @@ -176,7 +197,9 @@ test.describe("without javascript", () => { expect(await app.getHtml("#size")).toMatch(">14<"); }); - test("can catch MeterError when file is too big with createFileUploadHandler", async ({ page }) => { + test("can catch MeterError when file is too big with createFileUploadHandler", async ({ + page, + }) => { let app = new PlaywrightFixture(appFixture, page); await app.goto("/file-upload-handler"); await app.uploadFile( @@ -204,7 +227,9 @@ test.describe("without javascript", () => { expect(await app.getHtml("#size")).toMatch(">14<"); }); - test("can catch MeterError when file is too big with createMemoryUploadHandler", async ({ page }) => { + test("can catch MeterError when file is too big with createMemoryUploadHandler", async ({ + page, + }) => { let app = new PlaywrightFixture(appFixture, page); await app.goto("/memory-upload-handler"); await app.uploadFile( diff --git a/packages/remix-cloudflare/index.ts b/packages/remix-cloudflare/index.ts index 805785b00df..aedb73b08e8 100644 --- a/packages/remix-cloudflare/index.ts +++ b/packages/remix-cloudflare/index.ts @@ -16,6 +16,7 @@ export { isSession, json, redirect, + unstable_composeUploadHandlers, unstable_parseMultipartFormData, unstable_createMemoryUploadHandler, MeterError diff --git a/packages/remix-node/index.ts b/packages/remix-node/index.ts index e4c2621ceb9..6a3d67e74da 100644 --- a/packages/remix-node/index.ts +++ b/packages/remix-node/index.ts @@ -40,6 +40,7 @@ export { isSession, json, redirect, + unstable_composeUploadHandlers, unstable_parseMultipartFormData, unstable_createMemoryUploadHandler, MeterError, diff --git a/packages/remix-node/upload/fileUploadHandler.ts b/packages/remix-node/upload/fileUploadHandler.ts index 13fcc78efec..712bb441bbf 100644 --- a/packages/remix-node/upload/fileUploadHandler.ts +++ b/packages/remix-node/upload/fileUploadHandler.ts @@ -92,9 +92,12 @@ export function createFileUploadHandler({ file = defaultFilePathResolver, filter, maxFileSize = 3000000, -}: FileUploadHandlerOptions): UploadHandler { +}: FileUploadHandlerOptions = {}): UploadHandler { return async ({ name, filename, contentType, data }) => { - if (filter && !(await filter({ name, filename, contentType }))) { + if ( + !filename || + (filter && !(await filter({ name, filename, contentType }))) + ) { return undefined; } diff --git a/packages/remix-server-runtime/__tests__/formData-test.ts b/packages/remix-server-runtime/__tests__/formData-test.ts index f08aeb7b4d0..ca5f25e6512 100644 --- a/packages/remix-server-runtime/__tests__/formData-test.ts +++ b/packages/remix-server-runtime/__tests__/formData-test.ts @@ -25,7 +25,11 @@ describe("parseMultipartFormData", () => { for await (let chunk of data) { chunks.push(chunk); } - return new File(chunks, filename, { type: contentType }); + if (filename) { + return new File(chunks, filename, { type: contentType }); + } + + return await new Blob(chunks, { type: contentType }).text(); } ); diff --git a/packages/remix-server-runtime/formData.ts b/packages/remix-server-runtime/formData.ts index 0a8929d2b02..8ada4064d44 100644 --- a/packages/remix-server-runtime/formData.ts +++ b/packages/remix-server-runtime/formData.ts @@ -9,7 +9,22 @@ export type UploadHandlerPart = { export type UploadHandler = ( part: UploadHandlerPart -) => Promise; +) => Promise; + +export function composeUploadHandlers( + ...handlers: UploadHandler[] +): UploadHandler { + return async (part) => { + for (let handler of handlers) { + let value = await handler(part); + if (typeof value !== "undefined" && value !== null) { + return value; + } + } + + return undefined; + }; +} /** * Allows you to handle multipart forms (file uploads) for your app. @@ -35,37 +50,11 @@ export async function parseMultipartFormData( for await (let part of parts) { if (part.done) break; - if (!part.filename) { - formData.append(part.name, await bufferPart(part)); - } else { - let file = await uploadHandler(part); - if (typeof file !== "undefined") { - formData.append(part.name, file as Blob); - } + let value = await uploadHandler(part); + if (typeof value !== "undefined" && value !== null) { + formData.append(part.name, value); } } return formData; } - -async function bufferPart(part: UploadHandlerPart): Promise { - let chunks = []; - - for await (let chunk of part.data) { - chunks.push(chunk); - } - - return new TextDecoder().decode(mergeArrays(...chunks)); -} - -function mergeArrays(...arrays: Uint8Array[]) { - let out = new Uint8Array( - arrays.reduce((total, arr) => total + arr.length, 0) - ); - let offset = 0; - for (let arr of arrays) { - out.set(arr, offset); - offset += arr.length; - } - return out; -} diff --git a/packages/remix-server-runtime/index.ts b/packages/remix-server-runtime/index.ts index 2a2890f1778..597e946a9c7 100644 --- a/packages/remix-server-runtime/index.ts +++ b/packages/remix-server-runtime/index.ts @@ -1,6 +1,9 @@ // Default implementations for the Remix server runtime interface export { createCookieFactory, isCookie } from "./cookies"; -export { parseMultipartFormData as unstable_parseMultipartFormData } from "./formData"; +export { + composeUploadHandlers as unstable_composeUploadHandlers, + parseMultipartFormData as unstable_parseMultipartFormData, +} from "./formData"; export { json, redirect } from "./responses"; export { createRequestHandler } from "./server"; export { diff --git a/packages/remix-server-runtime/upload/memoryUploadHandler.ts b/packages/remix-server-runtime/upload/memoryUploadHandler.ts index d0f8aec5b2d..24111df8733 100644 --- a/packages/remix-server-runtime/upload/memoryUploadHandler.ts +++ b/packages/remix-server-runtime/upload/memoryUploadHandler.ts @@ -3,7 +3,7 @@ import type { UploadHandler } from "@remix-run/server-runtime"; import { MeterError } from "./meter"; export type MemoryUploadHandlerFilterArgs = { - filename: string; + filename?: string; contentType: string; name: string; }; @@ -26,7 +26,7 @@ export type MemoryUploadHandlerOptions = { export function createMemoryUploadHandler({ filter, maxFileSize = 3000000, -}: MemoryUploadHandlerOptions): UploadHandler { +}: MemoryUploadHandlerOptions = {}): UploadHandler { return async ({ filename, contentType, name, data }) => { if (filter && !(await filter({ filename, contentType, name }))) { return undefined; @@ -42,6 +42,10 @@ export function createMemoryUploadHandler({ chunks.push(chunk); } - return new File(chunks, filename, { type: contentType }); + if (typeof filename === "string") { + return new File(chunks, filename, { type: contentType }); + } + + return await new Blob(chunks, { type: contentType }).text(); }; } From 45d072e188443890991c6750f48494933ad5b365 Mon Sep 17 00:00:00 2001 From: Jacob Ebey Date: Tue, 10 May 2022 14:39:55 -0700 Subject: [PATCH 30/47] fix build --- packages/remix-server-runtime/formData.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/remix-server-runtime/formData.ts b/packages/remix-server-runtime/formData.ts index 8ada4064d44..d43e8f60cea 100644 --- a/packages/remix-server-runtime/formData.ts +++ b/packages/remix-server-runtime/formData.ts @@ -9,7 +9,7 @@ export type UploadHandlerPart = { export type UploadHandler = ( part: UploadHandlerPart -) => Promise; +) => Promise; export function composeUploadHandlers( ...handlers: UploadHandler[] @@ -52,7 +52,7 @@ export async function parseMultipartFormData( let value = await uploadHandler(part); if (typeof value !== "undefined" && value !== null) { - formData.append(part.name, value); + formData.append(part.name, value as any); } } From 1f259c4c93b3a73c0f113a72a10e66f4c38fa1e1 Mon Sep 17 00:00:00 2001 From: Jacob Ebey Date: Tue, 10 May 2022 17:22:49 -0700 Subject: [PATCH 31/47] feat: renamed MeterError to MaxPartSizeExceededError fix: NodeOnDiskFile now reflects sliced size --- docs/api/remix.md | 8 +-- docs/pages/gotchas.md | 4 +- .../app/routes/local-upload.tsx | 2 +- integration/file-uploads-test.ts | 2 +- integration/upload-test.ts | 20 +++---- packages/remix-cloudflare/index.ts | 2 +- .../__tests__/fileUploadHandler-test.ts | 53 +++++++++++++++++-- packages/remix-node/index.ts | 2 +- packages/remix-node/package.json | 2 - .../remix-node/upload/fileUploadHandler.ts | 34 ++++++------ packages/remix-server-runtime/index.ts | 2 +- .../upload/{meter.ts => errors.ts} | 2 +- .../upload/memoryUploadHandler.ts | 10 ++-- yarn.lock | 19 ------- 14 files changed, 91 insertions(+), 71 deletions(-) rename packages/remix-server-runtime/upload/{meter.ts => errors.ts} (72%) diff --git a/docs/api/remix.md b/docs/api/remix.md index 525a27da54b..2ff8748a25e 100644 --- a/docs/api/remix.md +++ b/docs/api/remix.md @@ -1590,7 +1590,7 @@ export const action: ActionFunction = async ({ request, }) => { const uploadHandler = unstable_createFileUploadHandler({ - maxFileSize: 5_000_000, + maxPartSize: 5_000_000, file: ({ filename }) => filename, }); const formData = await unstable_parseMultipartFormData( @@ -1612,7 +1612,7 @@ export const action: ActionFunction = async ({ | avoidFileConflicts | boolean | true | Avoid file conflicts by appending a timestamp on the end of the filename if it already exists on disk | | directory | string \| Function | os.tmpdir() | The directory to write the upload. | | file | Function | () => `upload_${random}.${ext}` | The name of the file in the directory. Can be a relative path, the directory structure will be created if it does not exist. | -| maxFileSize | number | 3000000 | The maximum upload size allowed (in bytes). If the size is exceeded an error will be thrown. | +| maxPartSize | number | 3000000 | The maximum upload size allowed (in bytes). If the size is exceeded an error will be thrown. | | filter | Function | OPTIONAL | A function you can write to prevent a file upload from being saved based on filename, mimetype, or encoding. Return `false` and the file will be ignored. | The function API for `file` and `directory` are the same. They accept an `object` and return a `string`. The object it accepts has `filename`, `encoding`, and `mimetype` (all strings).The `string` returned is the path. @@ -1628,7 +1628,7 @@ export const action: ActionFunction = async ({ request, }) => { const uploadHandler = unstable_createMemoryUploadHandler({ - maxFileSize: 500_000, + maxPartSize: 500_000, }); const formData = await unstable_parseMultipartFormData( request, @@ -1642,7 +1642,7 @@ export const action: ActionFunction = async ({ }; ``` -**Options:** The only options supported are `maxFileSize` and `filter` which work the same as in `unstable_createFileUploadHandler` above. This API is not recommended for anything at scale, but is a convenient utility for simple use cases. +**Options:** The only options supported are `maxPartSize` and `filter` which work the same as in `unstable_createFileUploadHandler` above. This API is not recommended for anything at scale, but is a convenient utility for simple use cases. ### Custom `uploadHandler` diff --git a/docs/pages/gotchas.md b/docs/pages/gotchas.md index d75d974d3b5..76c1b5fd7be 100644 --- a/docs/pages/gotchas.md +++ b/docs/pages/gotchas.md @@ -65,7 +65,7 @@ So instead of doing: import { unstable_createFileUploadHandler } from "@remix-run/{runtime}"; const uploadHandler = unstable_createFileUploadHandler({ - maxFileSize: 5_000_000, + maxPartSize: 5_000_000, file: ({ filename }) => filename, }); @@ -81,7 +81,7 @@ import { unstable_createFileUploadHandler } from "@remix-run/{runtime}"; export async function action() { const uploadHandler = unstable_createFileUploadHandler({ - maxFileSize: 5_000_000, + maxPartSize: 5_000_000, file: ({ filename }) => filename, }); diff --git a/examples/file-and-cloudinary-upload/app/routes/local-upload.tsx b/examples/file-and-cloudinary-upload/app/routes/local-upload.tsx index 0db294ca324..19a2996151b 100644 --- a/examples/file-and-cloudinary-upload/app/routes/local-upload.tsx +++ b/examples/file-and-cloudinary-upload/app/routes/local-upload.tsx @@ -14,7 +14,7 @@ type ActionData = { export const action: ActionFunction = async ({ request }) => { const uploadHandler = unstable_createFileUploadHandler({ directory: "public", - maxFileSize: 30000, + maxPartSize: 30000, }); const formData = await unstable_parseMultipartFormData( request, diff --git a/integration/file-uploads-test.ts b/integration/file-uploads-test.ts index 70a115427b9..189bf24308c 100644 --- a/integration/file-uploads-test.ts +++ b/integration/file-uploads-test.ts @@ -24,7 +24,7 @@ test.describe("file-uploads", () => { export let uploadHandler = composeUploadHandlers( createFileUploadHandler({ directory: path.resolve(__dirname, "..", "uploads"), - maxFileSize: 10_000, // 10kb + maxPartSize: 10_000, // 10kb // you probably want to avoid conflicts in production // do not set to false or passthrough filename in real // applications. diff --git a/integration/upload-test.ts b/integration/upload-test.ts index 1951ab3fe8e..c11881f950a 100644 --- a/integration/upload-test.ts +++ b/integration/upload-test.ts @@ -18,7 +18,7 @@ test.beforeAll(async () => { unstable_createFileUploadHandler as createFileUploadHandler, unstable_createMemoryUploadHandler as createMemoryUploadHandler, unstable_parseMultipartFormData as parseMultipartFormData, - MeterError, + MaxPartSizeExceededError, } from "@remix-run/node"; import { Form, useActionData } from "@remix-run/react"; @@ -26,7 +26,7 @@ test.beforeAll(async () => { let uploadHandler = composeUploadHandlers( createFileUploadHandler({ directory: "./uploads", - maxFileSize: 15, + maxPartSize: 15, avoidFileConflicts: false, file: ({ filename }) => filename, }), @@ -45,7 +45,7 @@ test.beforeAll(async () => { return json({ message: "SUCCESS", size }); } catch (error) { - if (error instanceof MeterError) { + if (error instanceof MaxPartSizeExceededError) { return json({ message: "FILE_TOO_LARGE", size: error.maxBytes }); } return json({ message: "ERROR" }, 500); @@ -76,13 +76,13 @@ test.beforeAll(async () => { json, unstable_createMemoryUploadHandler as createMemoryUploadHandler, unstable_parseMultipartFormData as parseMultipartFormData, - MeterError, + MaxPartSizeExceededError, } from "@remix-run/node"; import { Form, useActionData } from "@remix-run/react"; export let action = async ({ request }) => { let uploadHandler = createMemoryUploadHandler({ - maxFileSize: 15, + maxPartSize: 15, }); try { @@ -97,7 +97,7 @@ test.beforeAll(async () => { return json({ message: "SUCCESS", size }); } catch (error) { - if (error instanceof MeterError) { + if (error instanceof MaxPartSizeExceededError) { return json({ message: "FILE_TOO_LARGE", size: error.maxBytes }); } return json({ message: "ERROR" }, 500); @@ -140,7 +140,7 @@ test("can upload a file with createFileUploadHandler", async ({ page }) => { expect(await app.getHtml("#size")).toMatch(">14<"); }); -test("can catch MeterError when file is too big with createFileUploadHandler", async ({ +test("can catch MaxPartSizeExceededError when file is too big with createFileUploadHandler", async ({ page, }) => { let app = new PlaywrightFixture(appFixture, page); @@ -165,7 +165,7 @@ test("can upload a file with createMemoryUploadHandler", async ({ page }) => { expect(await app.getHtml("#size")).toMatch(">14<"); }); -test("can catch MeterError when file is too big with createMemoryUploadHandler", async ({ +test("can catch MaxPartSizeExceededError when file is too big with createMemoryUploadHandler", async ({ page, }) => { let app = new PlaywrightFixture(appFixture, page); @@ -197,7 +197,7 @@ test.describe("without javascript", () => { expect(await app.getHtml("#size")).toMatch(">14<"); }); - test("can catch MeterError when file is too big with createFileUploadHandler", async ({ + test("can catch MaxPartSizeExceededError when file is too big with createFileUploadHandler", async ({ page, }) => { let app = new PlaywrightFixture(appFixture, page); @@ -227,7 +227,7 @@ test.describe("without javascript", () => { expect(await app.getHtml("#size")).toMatch(">14<"); }); - test("can catch MeterError when file is too big with createMemoryUploadHandler", async ({ + test("can catch MaxPartSizeExceededError when file is too big with createMemoryUploadHandler", async ({ page, }) => { let app = new PlaywrightFixture(appFixture, page); diff --git a/packages/remix-cloudflare/index.ts b/packages/remix-cloudflare/index.ts index aedb73b08e8..51429f1ff44 100644 --- a/packages/remix-cloudflare/index.ts +++ b/packages/remix-cloudflare/index.ts @@ -19,7 +19,7 @@ export { unstable_composeUploadHandlers, unstable_parseMultipartFormData, unstable_createMemoryUploadHandler, - MeterError + MaxPartSizeExceededError } from "@remix-run/server-runtime"; export type { diff --git a/packages/remix-node/__tests__/fileUploadHandler-test.ts b/packages/remix-node/__tests__/fileUploadHandler-test.ts index e73f8b0a4c5..0d52c6eae7e 100644 --- a/packages/remix-node/__tests__/fileUploadHandler-test.ts +++ b/packages/remix-node/__tests__/fileUploadHandler-test.ts @@ -50,15 +50,58 @@ describe("NodeOnDiskFile", () => { it("can sice file and get an arrayBuffer", async () => { let sliced = await file.slice(1, 5); + let slicedRes = contents.slice(1, 5); let buffer = await sliced.arrayBuffer(); - expect(buffer.byteLength).toBe(4); - expect(buffer).toEqual(Buffer.from(contents.slice(1, 5))); + expect(buffer.byteLength).toBe(slicedRes.length); + expect(buffer).toEqual(Buffer.from(slicedRes)); }); it("can slice file and use stream", async () => { let sliced = await file.slice(1, 5); - expect(await readableStreamToString(sliced.stream() as any)).toBe( - contents.slice(1, 5) - ); + let slicedRes = contents.slice(1, 5); + expect(sliced.size).toBe(slicedRes.length); + expect(await sliced.text()).toBe(slicedRes); + }); + + it("can slice file with negative start and no end", async () => { + let sliced = await file.slice(-2); + let slicedRes = contents.slice(-2); + expect(sliced.size).toBe(slicedRes.length); + expect(await sliced.text()).toBe(slicedRes); + }); + + it("can slice file with negative start and negative end", async () => { + let sliced = await file.slice(-3, -1); + let slicedRes = contents.slice(-3, -1); + expect(sliced.size).toBe(slicedRes.length); + expect(await sliced.text()).toBe(slicedRes); + }); + + it("can slice file with negative start and negative end twice", async () => { + let sliced = await file.slice(-3, -1).slice(1, -1); + let slicedRes = contents.slice(-3, -1).slice(1, -1); + expect(sliced.size).toBe(slicedRes.length); + expect(await sliced.text()).toBe(slicedRes); + }); + + it("can slice file with start and negative end", async () => { + let sliced = await file.slice(1, -2); + let slicedRes = contents.slice(1, -2); + expect(sliced.size).toBe(slicedRes.length); + expect(await sliced.text()).toBe(slicedRes); + }); + + it("can slice file with negaive start and end", async () => { + let sliced = await file.slice(-3, 1); + let slicedRes = contents.slice(-3, 1); + expect(sliced.size).toBe(slicedRes.length); + expect(await sliced.text()).toBe(slicedRes); + }); + + it("can slice oob", async () => { + let sliced = await file.slice(0, 10000); + let slicedRes = contents.slice(0, 10000); + expect(sliced.size).toBe(slicedRes.length); + expect(await sliced.text()).toBe(slicedRes); }); }); diff --git a/packages/remix-node/index.ts b/packages/remix-node/index.ts index 6a3d67e74da..d756c676963 100644 --- a/packages/remix-node/index.ts +++ b/packages/remix-node/index.ts @@ -43,7 +43,7 @@ export { unstable_composeUploadHandlers, unstable_parseMultipartFormData, unstable_createMemoryUploadHandler, - MeterError, + MaxPartSizeExceededError, } from "@remix-run/server-runtime"; export type { diff --git a/packages/remix-node/package.json b/packages/remix-node/package.json index fe1ff08825a..81d36d9761d 100644 --- a/packages/remix-node/package.json +++ b/packages/remix-node/package.json @@ -18,13 +18,11 @@ "@remix-run/web-stream": "^1.0.3", "@web3-storage/multipart-parser": "^1.0.0", "abort-controller": "^3.0.0", - "blob-stream": "^0.1.3", "cookie-signature": "^1.1.0", "source-map-support": "^0.5.21", "stream-slice": "^0.1.2" }, "devDependencies": { - "@types/blob-stream": "^0.1.30", "@types/cookie-signature": "^1.0.3", "@types/source-map-support": "^0.5.4" }, diff --git a/packages/remix-node/upload/fileUploadHandler.ts b/packages/remix-node/upload/fileUploadHandler.ts index 712bb441bbf..a85c57a86c1 100644 --- a/packages/remix-node/upload/fileUploadHandler.ts +++ b/packages/remix-node/upload/fileUploadHandler.ts @@ -6,7 +6,7 @@ import { basename, dirname, extname, resolve as resolvePath } from "path"; import type { Readable } from "stream"; import { finished } from "stream"; import { promisify } from "util"; -import { MeterError } from "@remix-run/server-runtime"; +import { MaxPartSizeExceededError } from "@remix-run/server-runtime"; import type { UploadHandler } from "@remix-run/server-runtime"; // @ts-expect-error import * as streamSlice from "stream-slice"; @@ -52,7 +52,7 @@ export type FileUploadHandlerOptions = { * The maximum upload size allowed. If the size is exceeded an error will be thrown. * Defaults to 3000000B (3MB). */ - maxFileSize?: number; + maxPartSize?: number; /** * * @param filename @@ -91,7 +91,7 @@ export function createFileUploadHandler({ avoidFileConflicts = true, file = defaultFilePathResolver, filter, - maxFileSize = 3000000, + maxPartSize = 3000000, }: FileUploadHandlerOptions = {}): UploadHandler { return async ({ name, filename, contentType, data }) => { if ( @@ -132,9 +132,9 @@ export function createFileUploadHandler({ try { for await (let chunk of data) { size += chunk.byteLength; - if (size > maxFileSize) { + if (size > maxPartSize) { deleteFile = true; - throw new MeterError(name, maxFileSize); + throw new MaxPartSizeExceededError(name, maxPartSize); } writeFileStream.write(chunk); } @@ -164,16 +164,21 @@ export class NodeOnDiskFile implements File { this.name = basename(filepath); } - public get size(): number { + get size(): number { + let stats = statSync(this.filepath); + if (this.slicer) { - return this.slicer.end - this.slicer.start; + let slice = this.slicer.end - this.slicer.start; + return slice < 0 ? 0 : slice > stats.size ? stats.size : slice; } - let stats = statSync(this.filepath); return stats.size; } slice(start?: number, end?: number, type?: string): Blob { + if (typeof start === "number" && start < 0) start = this.size + start; + if (typeof end === "number" && end < 0) end = this.size + end; + let startOffset = this.slicer?.start || 0; start = startOffset + (start || 0); @@ -216,18 +221,11 @@ export class NodeOnDiskFile implements File { return readableStreamFromStream(stream); } - text(): Promise { - let stream: Readable = createReadStream(this.filepath); - if (this.slicer) { - stream = stream.pipe( - streamSlice.slice(this.slicer.start, this.slicer.end) - ); - } - - return readableStreamToString(readableStreamFromStream(stream)); + async text(): Promise { + return readableStreamToString(this.stream()); } - get [Symbol.toStringTag]() { + public get [Symbol.toStringTag]() { return "File"; } } diff --git a/packages/remix-server-runtime/index.ts b/packages/remix-server-runtime/index.ts index 597e946a9c7..6af6134390b 100644 --- a/packages/remix-server-runtime/index.ts +++ b/packages/remix-server-runtime/index.ts @@ -14,7 +14,7 @@ export { export { createCookieSessionStorageFactory } from "./sessions/cookieStorage"; export { createMemorySessionStorageFactory } from "./sessions/memoryStorage"; export { createMemoryUploadHandler as unstable_createMemoryUploadHandler } from "./upload/memoryUploadHandler"; -export { MeterError } from "./upload/meter"; +export { MaxPartSizeExceededError } from "./upload/errors"; // Types for the Remix server runtime interface export type { diff --git a/packages/remix-server-runtime/upload/meter.ts b/packages/remix-server-runtime/upload/errors.ts similarity index 72% rename from packages/remix-server-runtime/upload/meter.ts rename to packages/remix-server-runtime/upload/errors.ts index 10300610994..e5ed3d42f6b 100644 --- a/packages/remix-server-runtime/upload/meter.ts +++ b/packages/remix-server-runtime/upload/errors.ts @@ -1,4 +1,4 @@ -export class MeterError extends Error { +export class MaxPartSizeExceededError extends Error { constructor(public field: string, public maxBytes: number) { super(`Field "${field}" exceeded upload size of ${maxBytes} bytes.`); } diff --git a/packages/remix-server-runtime/upload/memoryUploadHandler.ts b/packages/remix-server-runtime/upload/memoryUploadHandler.ts index 24111df8733..51331faa9be 100644 --- a/packages/remix-server-runtime/upload/memoryUploadHandler.ts +++ b/packages/remix-server-runtime/upload/memoryUploadHandler.ts @@ -1,6 +1,6 @@ import type { UploadHandler } from "@remix-run/server-runtime"; -import { MeterError } from "./meter"; +import { MaxPartSizeExceededError } from "./errors"; export type MemoryUploadHandlerFilterArgs = { filename?: string; @@ -13,7 +13,7 @@ export type MemoryUploadHandlerOptions = { * The maximum upload size allowed. If the size is exceeded an error will be thrown. * Defaults to 3000000B (3MB). */ - maxFileSize?: number; + maxPartSize?: number; /** * * @param filename @@ -25,7 +25,7 @@ export type MemoryUploadHandlerOptions = { export function createMemoryUploadHandler({ filter, - maxFileSize = 3000000, + maxPartSize = 3000000, }: MemoryUploadHandlerOptions = {}): UploadHandler { return async ({ filename, contentType, name, data }) => { if (filter && !(await filter({ filename, contentType, name }))) { @@ -36,8 +36,8 @@ export function createMemoryUploadHandler({ let chunks = []; for await (let chunk of data) { size += chunk.byteLength; - if (size > maxFileSize) { - throw new MeterError(name, maxFileSize); + if (size > maxPartSize) { + throw new MaxPartSizeExceededError(name, maxPartSize); } chunks.push(chunk); } diff --git a/yarn.lock b/yarn.lock index e7f8281d33c..2a3e850612c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1824,13 +1824,6 @@ dependencies: "@babel/types" "^7.3.0" -"@types/blob-stream@^0.1.30": - version "0.1.30" - resolved "https://registry.npmjs.org/@types/blob-stream/-/blob-stream-0.1.30.tgz" - integrity sha512-Cyp7/3KZfpQXcUPhcb/+VPubLQE8YzFXbUh1/KNVzBH6sykr0AJohdIzX8YWSy0YZIg1yI75DULDeEfr7lESSg== - dependencies: - "@types/node" "*" - "@types/body-parser@*": version "1.19.1" resolved "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.1.tgz" @@ -3192,23 +3185,11 @@ bl@^4.0.3, bl@^4.1.0: inherits "^2.0.4" readable-stream "^3.4.0" -blob-stream@^0.1.3: - version "0.1.3" - resolved "https://registry.npmjs.org/blob-stream/-/blob-stream-0.1.3.tgz" - integrity sha1-mNZor2mW4PMu9mbQbiFczH13aGw= - dependencies: - blob "0.0.4" - blob-util@^2.0.2: version "2.0.2" resolved "https://registry.npmjs.org/blob-util/-/blob-util-2.0.2.tgz" integrity sha512-T7JQa+zsXXEa6/8ZhHcQEW1UFfVM49Ts65uBkFL6fz2QmrElqmbajIDJvuA0tEhRe5eIjpV9ZF+0RfZR9voJFQ== -blob@0.0.4: - version "0.0.4" - resolved "https://registry.npmjs.org/blob/-/blob-0.0.4.tgz" - integrity sha1-vPEwUspURj8w+fx+lbmkdjCpSSE= - bluebird@^3.7.2: version "3.7.2" resolved "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz" From f2f6f00e23cfa28d49f183e3bdd50aefb6543858 Mon Sep 17 00:00:00 2001 From: Jacob Ebey Date: Tue, 10 May 2022 17:29:04 -0700 Subject: [PATCH 32/47] only pass basename to handlers --- packages/remix-server-runtime/formData.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/remix-server-runtime/formData.ts b/packages/remix-server-runtime/formData.ts index d43e8f60cea..7df1770d3d0 100644 --- a/packages/remix-server-runtime/formData.ts +++ b/packages/remix-server-runtime/formData.ts @@ -2,7 +2,7 @@ import { streamMultipart } from "@web3-storage/multipart-parser"; export type UploadHandlerPart = { name: string; - filename: string; + filename?: string; contentType: string; data: AsyncIterable; }; @@ -50,6 +50,11 @@ export async function parseMultipartFormData( for await (let part of parts) { if (part.done) break; + if (typeof part.filename === "string") { + // only pass basename as the multipart/form-data spec recommends + part.filename = part.filename.split(/[/\\]/).pop(); + } + let value = await uploadHandler(part); if (typeof value !== "undefined" && value !== null) { formData.append(part.name, value as any); From 37576e4e26f1be6c2b0198c2a3796da9ee994778 Mon Sep 17 00:00:00 2001 From: Jacob Ebey Date: Tue, 10 May 2022 17:51:42 -0700 Subject: [PATCH 33/47] added another test for upload handlers --- .../__tests__/formData-test.ts | 35 +++++++++++++++---- 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/packages/remix-server-runtime/__tests__/formData-test.ts b/packages/remix-server-runtime/__tests__/formData-test.ts index ca5f25e6512..444b9df5e1a 100644 --- a/packages/remix-server-runtime/__tests__/formData-test.ts +++ b/packages/remix-server-runtime/__tests__/formData-test.ts @@ -10,8 +10,8 @@ describe("parseMultipartFormData", () => { it("can use a custom upload handler", async () => { let formData = new NodeFormData(); formData.set("a", "value"); - formData.set("blob", new Blob(["blob"]), "blob.txt"); - formData.set("file", new File(["file"], "file.txt")); + formData.set("blob", new Blob(["blob".repeat(1000)]), "blob.txt"); + formData.set("file", new File(["file".repeat(1000)], "file.txt")); let req = new NodeRequest("https://test.com", { method: "post", @@ -35,10 +35,31 @@ describe("parseMultipartFormData", () => { expect(parsedFormData.get("a")).toBe("value"); let blob = parsedFormData.get("blob") as Blob; - expect(await blob.text()).toBe("blob"); + expect(await blob.text()).toBe("blob".repeat(1000)); let file = parsedFormData.get("file") as File; expect(file.name).toBe("file.txt"); - expect(await file.text()).toBe("file"); + expect(await file.text()).toBe("file".repeat(1000)); + }); + + it("can return undefined", async () => { + let formData = new NodeFormData(); + formData.set("a", "value"); + formData.set("blob", new Blob(["blob".repeat(1000)]), "blob.txt"); + formData.set("file", new File(["file".repeat(1000)], "file.txt")); + + let req = new NodeRequest("https://test.com", { + method: "post", + body: formData, + }); + + let parsedFormData = await parseMultipartFormData( + req, + async () => undefined + ); + + expect(parsedFormData.get("a")).toBe(null); + expect(parsedFormData.get("blob")).toBe(null); + expect(parsedFormData.get("file")).toBe(null); }); it("can throw errors in upload handlers", async () => { @@ -56,14 +77,16 @@ describe("parseMultipartFormData", () => { body: formData, }); + let error: Error; try { await parseMultipartFormData(req, async () => { throw new CustomError(); }); throw new Error("should have thrown"); } catch (err) { - expect(err.message).toBe("test error"); - expect(err).toBeInstanceOf(CustomError); + error = err; } + expect(error).toBeInstanceOf(CustomError); + expect(error.message).toBe("test error"); }); }); From 5792bb145b90668bc1f7fbf61dc8b90d8654b853 Mon Sep 17 00:00:00 2001 From: Jacob Ebey Date: Tue, 10 May 2022 18:23:30 -0700 Subject: [PATCH 34/47] added more tests --- integration/upload-test.ts | 73 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 71 insertions(+), 2 deletions(-) diff --git a/integration/upload-test.ts b/integration/upload-test.ts index c11881f950a..69e28ddaa9e 100644 --- a/integration/upload-test.ts +++ b/integration/upload-test.ts @@ -46,7 +46,10 @@ test.beforeAll(async () => { return json({ message: "SUCCESS", size }); } catch (error) { if (error instanceof MaxPartSizeExceededError) { - return json({ message: "FILE_TOO_LARGE", size: error.maxBytes }); + return json( + { message: "FILE_TOO_LARGE", size: error.maxBytes }, + { status: 413, headers: { "Connection": "close" } } + ); } return json({ message: "ERROR" }, 500); } @@ -98,7 +101,10 @@ test.beforeAll(async () => { return json({ message: "SUCCESS", size }); } catch (error) { if (error instanceof MaxPartSizeExceededError) { - return json({ message: "FILE_TOO_LARGE", size: error.maxBytes }); + return json( + { message: "FILE_TOO_LARGE", size: error.maxBytes }, + { status: 413, headers: { "Connection": "close" } } + ); } return json({ message: "ERROR" }, 500); } @@ -122,6 +128,47 @@ test.beforeAll(async () => { ); } `, + + "app/routes/passthrough-upload-handler.jsx": js` + import { + json, + unstable_parseMultipartFormData as parseMultipartFormData, + } from "@remix-run/node"; + import { Form, useActionData } from "@remix-run/react"; + + export let action = async ({ request }) => { + try { + let formData = await parseMultipartFormData(request, () => undefined); + + return json( + { message: "SUCCESS", size: 0 }, + ); + } catch (error) { + return json( + { message: "ERROR" }, + { status: 500, headers: { "Connection": "close" } } + ); + } + }; + + export default function PassthroughUpload() { + let { message, size } = useActionData() || {}; + return ( +
+ + + +
+ +
+ + {message &&

{message}

} + {size &&

{size}

} + +
+ ); + } + `, }, }); @@ -165,6 +212,15 @@ test("can upload a file with createMemoryUploadHandler", async ({ page }) => { expect(await app.getHtml("#size")).toMatch(">14<"); }); +test("can upload a file with a passthrough handler", async ({ page }) => { + let app = new PlaywrightFixture(appFixture, page); + await app.goto("/passthrough-upload-handler"); + await app.uploadFile("#file", path.resolve(__dirname, "assets/toupload.txt")); + await app.clickSubmitButton("/passthrough-upload-handler"); + + expect(await app.getHtml("#message")).toMatch(">SUCCESS<"); +}); + test("can catch MaxPartSizeExceededError when file is too big with createMemoryUploadHandler", async ({ page, }) => { @@ -227,6 +283,19 @@ test.describe("without javascript", () => { expect(await app.getHtml("#size")).toMatch(">14<"); }); + test("can upload a file with passthrough handler", async ({ page }) => { + let app = new PlaywrightFixture(appFixture, page); + await app.goto("/passthrough-upload-handler"); + await app.uploadFile( + "#file", + path.resolve(__dirname, "assets/toupload.txt") + ); + + await Promise.all([page.click("#submit"), page.waitForNavigation()]); + + expect(await app.getHtml("#message")).toMatch(">SUCCESS<"); + }); + test("can catch MaxPartSizeExceededError when file is too big with createMemoryUploadHandler", async ({ page, }) => { From 44f10e6b3167f664a9662796902d1ef0c4e983a9 Mon Sep 17 00:00:00 2001 From: Jacob Ebey Date: Tue, 10 May 2022 19:19:04 -0700 Subject: [PATCH 35/47] updated docs and example --- docs/api/remix.md | 26 +++++++++++------- .../app/routes/cloudinary-upload.tsx | 27 ++++++++++++------- .../app/routes/local-upload.tsx | 22 ++++++++------- .../app/utils/utils.server.ts | 25 ++++++++++++++--- .../rules/packageExports.js | 5 ++-- 5 files changed, 69 insertions(+), 36 deletions(-) diff --git a/docs/api/remix.md b/docs/api/remix.md index 2ff8748a25e..9cc717b55a2 100644 --- a/docs/api/remix.md +++ b/docs/api/remix.md @@ -1523,7 +1523,7 @@ return new Response(null, { }); ``` -## `unstable_parseMultipartFormData` (node) +## `unstable_parseMultipartFormData` Allows you to handle multipart forms (file uploads) for your app. @@ -1572,7 +1572,7 @@ export default function AvatarUploadRoute() { ### `uploadHandler` -The `uploadHandler` is the key to the whole thing. It's responsible for what happens to the file as it's being streamed from the client. You can save it to disk, store it in memory, or act as a proxy to send it somewhere else (like a file storage provider). +The `uploadHandler` is the key to the whole thing. It's responsible for what happens to the multipart/form-data parts as they are being streamed from the client. You can save it to disk, store it in memory, or act as a proxy to send it somewhere else (like a file storage provider). Remix has two utilities to create `uploadHandler`s for you: @@ -1581,7 +1581,9 @@ Remix has two utilities to create `uploadHandler`s for you: These are fully featured utilities for handling fairly simple use cases. It's not recommended to load anything but quite small files into memory. Saving files to disk is a reasonable solution for many use cases. But if you want to upload the file to a file hosting provider, then you'll need to write your own. -#### `unstable_createFileUploadHandler` +#### `unstable_createFileUploadHandler (node)` + +An upload handler that will write parts with a filename to disk to keep them out of memory, parts without a filename will not be parsed. Should be composed with another upload handler. **Example:** @@ -1589,10 +1591,14 @@ These are fully featured utilities for handling fairly simple use cases. It's no export const action: ActionFunction = async ({ request, }) => { - const uploadHandler = unstable_createFileUploadHandler({ - maxPartSize: 5_000_000, - file: ({ filename }) => filename, - }); + const uploadHandler = composeUploadHandlers( + unstable_createFileUploadHandler({ + maxPartSize: 5_000_000, + file: ({ filename }) => filename, + }), + // parse everything else into memory + unstable_createMemoryUploadHandler() + ); const formData = await unstable_parseMultipartFormData( request, uploadHandler @@ -1600,7 +1606,7 @@ export const action: ActionFunction = async ({ const file = formData.get("avatar"); - // file is a "NodeFile" which has a similar API to "File" + // file is a "NodeOnDiskFile" which implements the "File" API // ... etc }; ``` @@ -1612,7 +1618,7 @@ export const action: ActionFunction = async ({ | avoidFileConflicts | boolean | true | Avoid file conflicts by appending a timestamp on the end of the filename if it already exists on disk | | directory | string \| Function | os.tmpdir() | The directory to write the upload. | | file | Function | () => `upload_${random}.${ext}` | The name of the file in the directory. Can be a relative path, the directory structure will be created if it does not exist. | -| maxPartSize | number | 3000000 | The maximum upload size allowed (in bytes). If the size is exceeded an error will be thrown. | +| maxPartSize | number | 3000000 | The maximum upload size allowed (in bytes). If the size is exceeded a MaxPartSizeExceededError will be thrown. | | filter | Function | OPTIONAL | A function you can write to prevent a file upload from being saved based on filename, mimetype, or encoding. Return `false` and the file will be ignored. | The function API for `file` and `directory` are the same. They accept an `object` and return a `string`. The object it accepts has `filename`, `encoding`, and `mimetype` (all strings).The `string` returned is the path. @@ -1642,7 +1648,7 @@ export const action: ActionFunction = async ({ }; ``` -**Options:** The only options supported are `maxPartSize` and `filter` which work the same as in `unstable_createFileUploadHandler` above. This API is not recommended for anything at scale, but is a convenient utility for simple use cases. +**Options:** The only options supported are `maxPartSize` and `filter` which work the same as in `unstable_createFileUploadHandler` above. This API is not recommended for anything at scale, but is a convenient utility for simple use cases and as a fallback for another handler. ### Custom `uploadHandler` diff --git a/examples/file-and-cloudinary-upload/app/routes/cloudinary-upload.tsx b/examples/file-and-cloudinary-upload/app/routes/cloudinary-upload.tsx index 3a4a297e819..2cd45032f11 100644 --- a/examples/file-and-cloudinary-upload/app/routes/cloudinary-upload.tsx +++ b/examples/file-and-cloudinary-upload/app/routes/cloudinary-upload.tsx @@ -1,5 +1,10 @@ import type { ActionFunction, UploadHandler } from "@remix-run/node"; -import { json, unstable_parseMultipartFormData } from "@remix-run/node"; +import { + json, + unstable_composeUploadHandlers as composeUploadHandlers, + unstable_createMemoryUploadHandler as createMemoryUploadHandler, + unstable_parseMultipartFormData as parseMultipartFormData +} from "@remix-run/node"; import { Form, useActionData } from "@remix-run/react"; import { uploadImage } from "~/utils/utils.server"; @@ -11,16 +16,18 @@ type ActionData = { }; export const action: ActionFunction = async ({ request }) => { - const uploadHandler: UploadHandler = async ({ name, stream }) => { - if (name !== "img") { - stream.resume(); - return; - } - const uploadedImage = await uploadImage(stream); - return uploadedImage.secure_url; - }; + const uploadHandler: UploadHandler = composeUploadHandlers( + async ({ name, contentType, data, filename }) => { + if (name !== "img") { + return undefined; + } + const uploadedImage = await uploadImage(data); + return uploadedImage.secure_url; + }, + createMemoryUploadHandler() + ); - const formData = await unstable_parseMultipartFormData( + const formData = await parseMultipartFormData( request, uploadHandler ); diff --git a/examples/file-and-cloudinary-upload/app/routes/local-upload.tsx b/examples/file-and-cloudinary-upload/app/routes/local-upload.tsx index 19a2996151b..1775fd8be80 100644 --- a/examples/file-and-cloudinary-upload/app/routes/local-upload.tsx +++ b/examples/file-and-cloudinary-upload/app/routes/local-upload.tsx @@ -1,8 +1,10 @@ import type { ActionFunction } from "@remix-run/node"; import { json, - unstable_createFileUploadHandler, - unstable_parseMultipartFormData, + unstable_composeUploadHandlers as composeUploadHandlers, + unstable_createFileUploadHandler as createFileUploadHandler, + unstable_createMemoryUploadHandler as createMemoryUploadHandler, + unstable_parseMultipartFormData as parseMultipartFormData, } from "@remix-run/node"; import { Form, useActionData } from "@remix-run/react"; @@ -12,16 +14,16 @@ type ActionData = { }; export const action: ActionFunction = async ({ request }) => { - const uploadHandler = unstable_createFileUploadHandler({ - directory: "public", - maxPartSize: 30000, - }); - const formData = await unstable_parseMultipartFormData( - request, - uploadHandler + const uploadHandler = composeUploadHandlers( + createFileUploadHandler({ + directory: "public/uploads", + maxPartSize: 30000, + }), + createMemoryUploadHandler() ); + const formData = await parseMultipartFormData(request, uploadHandler); const image = formData.get("img"); - if (!image) { + if (!image || typeof image === "string") { return json({ error: "something wrong", }); diff --git a/examples/file-and-cloudinary-upload/app/utils/utils.server.ts b/examples/file-and-cloudinary-upload/app/utils/utils.server.ts index c0e1b34829c..569acfe59f4 100644 --- a/examples/file-and-cloudinary-upload/app/utils/utils.server.ts +++ b/examples/file-and-cloudinary-upload/app/utils/utils.server.ts @@ -1,5 +1,5 @@ +import { PassThrough } from "node:stream"; import cloudinary from "cloudinary"; -import type { Stream } from "stream"; cloudinary.v2.config({ cloud_name: process.env.CLOUD_NAME, @@ -7,8 +7,9 @@ cloudinary.v2.config({ api_secret: process.env.API_SECRET, }); -async function uploadImage(fileStream: Stream) { - return new Promise((resolve, reject) => { +async function uploadImage(data: AsyncIterable) { + const dataStream = new PassThrough(); + const uploadPromise = new Promise((resolve, reject) => { const uploadStream = cloudinary.v2.uploader.upload_stream( { folder: "remix", @@ -16,12 +17,28 @@ async function uploadImage(fileStream: Stream) { (error, result) => { if (error) { reject(error); + return; } resolve(result); } ); - fileStream.pipe(uploadStream); + dataStream.pipe(uploadStream); }); + + let errored = false; + try { + for await (let chunk of data) { + dataStream.write(chunk); + } + } catch (error: any) { + errored = true; + dataStream.destroy(error); + } + if (!errored) { + dataStream.end(); + } + + return uploadPromise; } console.log("configs", cloudinary.v2.config()); diff --git a/packages/remix-eslint-config/rules/packageExports.js b/packages/remix-eslint-config/rules/packageExports.js index 11b82a42f65..5054e93b331 100644 --- a/packages/remix-eslint-config/rules/packageExports.js +++ b/packages/remix-eslint-config/rules/packageExports.js @@ -15,6 +15,9 @@ const defaultRuntimeExports = { "isSession", "json", "redirect", + "unstable_composeUploadHandlers", + "unstable_createMemoryUploadHandler", + "unstable_parseMultipartFormData", ], type: [ "ActionFunction", @@ -83,8 +86,6 @@ const nodeSpecificExports = { "Request", "Response", "unstable_createFileUploadHandler", - "unstable_createMemoryUploadHandler", - "unstable_parseMultipartFormData", ], type: [ "HeadersInit", From 3f0654213f4702e5b22be828c2b9e28a66153802 Mon Sep 17 00:00:00 2001 From: Jacob Ebey Date: Tue, 10 May 2022 19:52:17 -0700 Subject: [PATCH 36/47] fix: add arch to the playwright cache key --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5477ab5d0f7..28918bbcacc 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -151,7 +151,7 @@ jobs: id: playwright-cache with: path: ${{ matrix.playwright_binary_path }} - key: ${{ runner.os }}-cache-playwright-${{ steps.playwright-version.outputs.version }} + key: ${{ runner.os }}-${{ runner.arch }}-cache-playwright-${{ steps.playwright-version.outputs.version }} - name: 🖨️ Playwright info shell: bash @@ -159,7 +159,7 @@ jobs: echo "OS: ${{ matrix.os }}" echo "Playwright version: ${{ steps.playwright-version.outputs.version }}" echo "Playwright install dir: ${{ matrix.playwright_binary_path }}" - echo "Cache key: ${{ runner.os }}-cache-playwright-${{ steps.playwright-version.outputs.version }}" + echo "Cache key: ${{ runner.os }}-${{ runner.arch }}-cache-playwright-${{ steps.playwright-version.outputs.version }}" echo "Cache hit: ${{ steps.playwright-cache.outputs.cache-hit == 'true' }}" - name: 📥 Install Playwright From cc0497e9cf84bede56dd9f283f7dc38521dcf796 Mon Sep 17 00:00:00 2001 From: Jacob Ebey Date: Tue, 10 May 2022 20:35:34 -0700 Subject: [PATCH 37/47] updated file upload tests to work on windows The issue was relying on a file that is managed via git with a newline char. These are converted based on the platform and result in a byteLength change. --- integration/assets/toupload.txt | 2 +- integration/upload-test.ts | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/integration/assets/toupload.txt b/integration/assets/toupload.txt index 8ab686eafeb..b45ef6fec89 100644 --- a/integration/assets/toupload.txt +++ b/integration/assets/toupload.txt @@ -1 +1 @@ -Hello, World! +Hello, World! \ No newline at end of file diff --git a/integration/upload-test.ts b/integration/upload-test.ts index 69e28ddaa9e..b3b88909a7d 100644 --- a/integration/upload-test.ts +++ b/integration/upload-test.ts @@ -26,7 +26,7 @@ test.beforeAll(async () => { let uploadHandler = composeUploadHandlers( createFileUploadHandler({ directory: "./uploads", - maxPartSize: 15, + maxPartSize: 13, avoidFileConflicts: false, file: ({ filename }) => filename, }), @@ -85,7 +85,7 @@ test.beforeAll(async () => { export let action = async ({ request }) => { let uploadHandler = createMemoryUploadHandler({ - maxPartSize: 15, + maxPartSize: 13, }); try { @@ -184,7 +184,7 @@ test("can upload a file with createFileUploadHandler", async ({ page }) => { await app.clickSubmitButton("/file-upload-handler"); expect(await app.getHtml("#message")).toMatch(">SUCCESS<"); - expect(await app.getHtml("#size")).toMatch(">14<"); + expect(await app.getHtml("#size")).toMatch(">13<"); }); test("can catch MaxPartSizeExceededError when file is too big with createFileUploadHandler", async ({ @@ -199,7 +199,7 @@ test("can catch MaxPartSizeExceededError when file is too big with createFileUpl await app.clickSubmitButton("/file-upload-handler"); expect(await app.getHtml("#message")).toMatch(">FILE_TOO_LARGE<"); - expect(await app.getHtml("#size")).toMatch(">15<"); + expect(await app.getHtml("#size")).toMatch(">13<"); }); test("can upload a file with createMemoryUploadHandler", async ({ page }) => { @@ -209,7 +209,7 @@ test("can upload a file with createMemoryUploadHandler", async ({ page }) => { await app.clickSubmitButton("/memory-upload-handler"); expect(await app.getHtml("#message")).toMatch(">SUCCESS<"); - expect(await app.getHtml("#size")).toMatch(">14<"); + expect(await app.getHtml("#size")).toMatch(">13<"); }); test("can upload a file with a passthrough handler", async ({ page }) => { @@ -233,7 +233,7 @@ test("can catch MaxPartSizeExceededError when file is too big with createMemoryU await app.clickSubmitButton("/memory-upload-handler"); expect(await app.getHtml("#message")).toMatch(">FILE_TOO_LARGE<"); - expect(await app.getHtml("#size")).toMatch(">15<"); + expect(await app.getHtml("#size")).toMatch(">13<"); }); test.describe("without javascript", () => { @@ -250,7 +250,7 @@ test.describe("without javascript", () => { await Promise.all([page.click("#submit"), page.waitForNavigation()]); expect(await app.getHtml("#message")).toMatch(">SUCCESS<"); - expect(await app.getHtml("#size")).toMatch(">14<"); + expect(await app.getHtml("#size")).toMatch(">13<"); }); test("can catch MaxPartSizeExceededError when file is too big with createFileUploadHandler", async ({ @@ -266,7 +266,7 @@ test.describe("without javascript", () => { await Promise.all([page.click("#submit"), page.waitForNavigation()]); expect(await app.getHtml("#message")).toMatch(">FILE_TOO_LARGE<"); - expect(await app.getHtml("#size")).toMatch(">15<"); + expect(await app.getHtml("#size")).toMatch(">13<"); }); test("can upload a file with createMemoryUploadHandler", async ({ page }) => { @@ -280,7 +280,7 @@ test.describe("without javascript", () => { await Promise.all([page.click("#submit"), page.waitForNavigation()]); expect(await app.getHtml("#message")).toMatch(">SUCCESS<"); - expect(await app.getHtml("#size")).toMatch(">14<"); + expect(await app.getHtml("#size")).toMatch(">13<"); }); test("can upload a file with passthrough handler", async ({ page }) => { @@ -309,6 +309,6 @@ test.describe("without javascript", () => { await Promise.all([page.click("#submit"), page.waitForNavigation()]); expect(await app.getHtml("#message")).toMatch(">FILE_TOO_LARGE<"); - expect(await app.getHtml("#size")).toMatch(">15<"); + expect(await app.getHtml("#size")).toMatch(">13<"); }); }); From c1734337e970548d26e76cf36ed283f31fcd6992 Mon Sep 17 00:00:00 2001 From: Jacob Ebey Date: Thu, 12 May 2022 09:52:14 -0700 Subject: [PATCH 38/47] added link to spec Co-authored-by: Michael Jackson --- packages/remix-server-runtime/formData.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/remix-server-runtime/formData.ts b/packages/remix-server-runtime/formData.ts index 7df1770d3d0..8d6d4095575 100644 --- a/packages/remix-server-runtime/formData.ts +++ b/packages/remix-server-runtime/formData.ts @@ -52,6 +52,7 @@ export async function parseMultipartFormData( if (typeof part.filename === "string") { // only pass basename as the multipart/form-data spec recommends + // https://datatracker.ietf.org/doc/html/rfc7578#section-4.2 part.filename = part.filename.split(/[/\\]/).pop(); } From 36fd7b0732ce2163b4caa8d07650c9e984e0493a Mon Sep 17 00:00:00 2001 From: Jacob Ebey Date: Thu, 12 May 2022 10:09:05 -0700 Subject: [PATCH 39/47] updated docs added more formdata tests --- docs/api/remix.md | 2 +- .../__tests__/formData-test.ts | 95 +++++++++++++++++-- 2 files changed, 90 insertions(+), 7 deletions(-) diff --git a/docs/api/remix.md b/docs/api/remix.md index 9cc717b55a2..916860f749a 100644 --- a/docs/api/remix.md +++ b/docs/api/remix.md @@ -1591,7 +1591,7 @@ An upload handler that will write parts with a filename to disk to keep them out export const action: ActionFunction = async ({ request, }) => { - const uploadHandler = composeUploadHandlers( + const uploadHandler = unstable_composeUploadHandlers( unstable_createFileUploadHandler({ maxPartSize: 5_000_000, file: ({ filename }) => filename, diff --git a/packages/remix-server-runtime/__tests__/formData-test.ts b/packages/remix-server-runtime/__tests__/formData-test.ts index 444b9df5e1a..3dd974c4efb 100644 --- a/packages/remix-server-runtime/__tests__/formData-test.ts +++ b/packages/remix-server-runtime/__tests__/formData-test.ts @@ -6,6 +6,12 @@ import { Blob, File } from "@remix-run/web-file"; import { parseMultipartFormData } from "../formData"; +class CustomError extends Error { + constructor() { + super("test error"); + } +} + describe("parseMultipartFormData", () => { it("can use a custom upload handler", async () => { let formData = new NodeFormData(); @@ -63,12 +69,6 @@ describe("parseMultipartFormData", () => { }); it("can throw errors in upload handlers", async () => { - class CustomError extends Error { - constructor() { - super("test error"); - } - } - let formData = new NodeFormData(); formData.set("blob", new Blob(["blob"]), "blob.txt"); @@ -89,4 +89,87 @@ describe("parseMultipartFormData", () => { expect(error).toBeInstanceOf(CustomError); expect(error.message).toBe("test error"); }); + + describe("stream should propagate events", () => { + it("when controller errors", async () => { + let formData = new NodeFormData(); + formData.set("a", "value"); + formData.set("blob", new Blob(["blob".repeat(1000)]), "blob.txt"); + formData.set("file", new File(["file".repeat(1000)], "file.txt")); + + let underlyingRequest = new NodeRequest("https://test.com", { + method: "post", + body: formData, + }); + let underlyingBody = await underlyingRequest.text(); + + let encoder = new TextEncoder(); + let body = new ReadableStream({ + start(controller) { + controller.enqueue( + encoder.encode(underlyingBody.slice(0, underlyingBody.length / 2)) + ); + controller.error(new CustomError()); + }, + }); + + let req = new NodeRequest("https://test.com", { + method: "post", + body, + headers: underlyingRequest.headers, + }); + + let error: Error; + try { + await parseMultipartFormData(req, async () => undefined); + throw new Error("should have thrown"); + } catch (err) { + error = err; + } + + expect(error).toBeInstanceOf(CustomError); + expect(error.message).toBe("test error"); + }); + + it("when controller is closed", async () => { + let formData = new NodeFormData(); + formData.set("a", "value"); + formData.set("blob", new Blob(["blob".repeat(1000)]), "blob.txt"); + formData.set("file", new File(["file".repeat(1000)], "file.txt")); + + let underlyingRequest = new NodeRequest("https://test.com", { + method: "post", + body: formData, + }); + let underlyingBody = await underlyingRequest.text(); + + let encoder = new TextEncoder(); + let body = new ReadableStream({ + start(controller) { + controller.enqueue( + encoder.encode(underlyingBody.slice(0, underlyingBody.length / 2)) + ); + controller.close(); + }, + }); + + let req = new NodeRequest("https://test.com", { + method: "post", + body, + headers: underlyingRequest.headers, + }); + + let error: Error; + try { + let formData = await parseMultipartFormData(req, async () => undefined); + console.log(formData); + throw new Error("should have thrown"); + } catch (err) { + error = err; + } + + expect(error).toBeInstanceOf(Error); + expect(error.message).toMatch("malformed multipart-form data"); + }); + }); }); From c23b3ac4c739f8569979e728119c0f7b213219b1 Mon Sep 17 00:00:00 2001 From: Jacob Ebey Date: Thu, 12 May 2022 10:22:21 -0700 Subject: [PATCH 40/47] renamed stream util and added error handling --- packages/remix-architect/server.ts | 4 ++-- packages/remix-netlify/server.ts | 4 ++-- packages/remix-node/index.ts | 4 ++-- packages/remix-node/stream.ts | 33 +++++++++--------------------- 4 files changed, 16 insertions(+), 29 deletions(-) diff --git a/packages/remix-architect/server.ts b/packages/remix-architect/server.ts index 0884ee0adcd..08cc52f59b0 100644 --- a/packages/remix-architect/server.ts +++ b/packages/remix-architect/server.ts @@ -7,7 +7,7 @@ import { Headers as NodeHeaders, Request as NodeRequest, createRequestHandler as createRemixRequestHandler, - readableStreamToBase64String, + readableStreamToString, } from "@remix-run/node"; import type { APIGatewayProxyEventHeaders, @@ -121,7 +121,7 @@ export async function sendRemixResponse( if (nodeResponse.body) { if (isBase64Encoded) { - body = await readableStreamToBase64String(nodeResponse.body); + body = await readableStreamToString(nodeResponse.body, "base64"); } else { body = await nodeResponse.text(); } diff --git a/packages/remix-netlify/server.ts b/packages/remix-netlify/server.ts index 9e4aa447ba5..97eb953a714 100644 --- a/packages/remix-netlify/server.ts +++ b/packages/remix-netlify/server.ts @@ -2,7 +2,7 @@ import { createRequestHandler as createRemixRequestHandler, Headers as NodeHeaders, Request as NodeRequest, - readableStreamToBase64String, + readableStreamToString, } from "@remix-run/node"; import type { Handler, @@ -137,7 +137,7 @@ export async function sendRemixResponse( if (nodeResponse.body) { if (isBase64Encoded) { - body = await readableStreamToBase64String(nodeResponse.body); + body = await readableStreamToString(nodeResponse.body, "base64"); } else { body = await nodeResponse.text(); } diff --git a/packages/remix-node/index.ts b/packages/remix-node/index.ts index d756c676963..5d4bbc8d563 100644 --- a/packages/remix-node/index.ts +++ b/packages/remix-node/index.ts @@ -29,8 +29,8 @@ export { } from "./implementations"; export { - pipeReadableStreamToWritable, - readableStreamToBase64String, + writeReadableStreamToWritable, + readableStreamToString, } from "./stream"; export { diff --git a/packages/remix-node/stream.ts b/packages/remix-node/stream.ts index 20e65afc9b0..a932d8cee6c 100644 --- a/packages/remix-node/stream.ts +++ b/packages/remix-node/stream.ts @@ -3,7 +3,7 @@ import { Stream } from "stream"; const { readableHighWaterMark } = new Stream.Readable(); -export async function pipeReadableStreamToWritable( +export async function writeReadableStreamToWritable( stream: ReadableStream, writable: Writable ) { @@ -22,31 +22,18 @@ export async function pipeReadableStreamToWritable( await read(); } - await read(); -} - -export async function readableStreamToBase64String(stream: ReadableStream) { - let reader = stream.getReader(); - let chunks: Uint8Array[] = []; - - async function read() { - let { done, value } = await reader.read(); - - if (done) { - return; - } else if (value) { - chunks.push(value); - } - + try { await read(); + } catch (error: any) { + writable.destroy(error); + throw error; } - - await read(); - - return Buffer.concat(chunks).toString("base64"); } -export async function readableStreamToString(stream: ReadableStream) { +export async function readableStreamToString( + stream: ReadableStream, + encoding?: BufferEncoding +) { let reader = stream.getReader(); let chunks: Uint8Array[] = []; @@ -64,7 +51,7 @@ export async function readableStreamToString(stream: ReadableStream) { await read(); - return Buffer.concat(chunks).toString(); + return Buffer.concat(chunks).toString(encoding); } export const readableStreamFromStream = ( From 593e166fec31c99266808c989fa4a1b7f4b14eb3 Mon Sep 17 00:00:00 2001 From: Jacob Ebey Date: Thu, 12 May 2022 10:27:55 -0700 Subject: [PATCH 41/47] update references from pipeReadableStreamToWritable -> writeReadableStreamToWritable --- packages/remix-express/server.ts | 4 ++-- packages/remix-node/stream.ts | 12 ++++++------ packages/remix-vercel/server.ts | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/remix-express/server.ts b/packages/remix-express/server.ts index 7ec529418a2..568e28f8614 100644 --- a/packages/remix-express/server.ts +++ b/packages/remix-express/server.ts @@ -10,7 +10,7 @@ import { createRequestHandler as createRemixRequestHandler, Headers as NodeHeaders, Request as NodeRequest, - pipeReadableStreamToWritable, + writeReadableStreamToWritable, } from "@remix-run/node"; /** @@ -129,7 +129,7 @@ export async function sendRemixResponse( } if (nodeResponse.body) { - await pipeReadableStreamToWritable(nodeResponse.body, res); + await writeReadableStreamToWritable(nodeResponse.body, res); } else { res.end(); } diff --git a/packages/remix-node/stream.ts b/packages/remix-node/stream.ts index a932d8cee6c..5430f00078d 100644 --- a/packages/remix-node/stream.ts +++ b/packages/remix-node/stream.ts @@ -1,8 +1,6 @@ -import type { Writable } from "stream"; +import type { Readable, Writable } from "stream"; import { Stream } from "stream"; -const { readableHighWaterMark } = new Stream.Readable(); - export async function writeReadableStreamToWritable( stream: ReadableStream, writable: Writable @@ -55,7 +53,7 @@ export async function readableStreamToString( } export const readableStreamFromStream = ( - source: Stream & { readableHighWaterMark?: number } + source: Readable & { readableHighWaterMark?: number } ) => { let pump = new StreamPump(source); let stream = new ReadableStream(pump, pump); @@ -75,7 +73,7 @@ class StreamPump { private controller?: ReadableStreamController; /** - * @param {Stream & { + * @param {Readable & { * readableHighWaterMark?: number * readable?:boolean, * resume?: () => void, @@ -92,7 +90,9 @@ class StreamPump { destroy?: (error?: Error) => void; } ) { - this.highWaterMark = stream.readableHighWaterMark || readableHighWaterMark; + this.highWaterMark = + stream.readableHighWaterMark || + new Stream.Readable().readableHighWaterMark; this.accumalatedSize = 0; this.stream = stream; this.enqueue = this.enqueue.bind(this); diff --git a/packages/remix-vercel/server.ts b/packages/remix-vercel/server.ts index eaef8f180ba..994b7632bcb 100644 --- a/packages/remix-vercel/server.ts +++ b/packages/remix-vercel/server.ts @@ -10,7 +10,7 @@ import { createRequestHandler as createRemixRequestHandler, Headers as NodeHeaders, Request as NodeRequest, - pipeReadableStreamToWritable, + writeReadableStreamToWritable, } from "@remix-run/node"; /** @@ -116,7 +116,7 @@ export async function sendRemixResponse( ); if (nodeResponse.body) { - await pipeReadableStreamToWritable(nodeResponse.body, res); + await writeReadableStreamToWritable(nodeResponse.body, res); } else { res.end(); } From b8797f7b6dcaa7c496b29067950943c619509536 Mon Sep 17 00:00:00 2001 From: Jacob Ebey Date: Thu, 12 May 2022 10:33:53 -0700 Subject: [PATCH 42/47] feat: added writeAsyncIterableToWritable chore: updated cloudinary upload example --- .../app/utils/utils.server.ts | 20 +++---------------- packages/remix-node/index.ts | 3 ++- packages/remix-node/stream.ts | 17 +++++++++++++++- 3 files changed, 21 insertions(+), 19 deletions(-) diff --git a/examples/file-and-cloudinary-upload/app/utils/utils.server.ts b/examples/file-and-cloudinary-upload/app/utils/utils.server.ts index 569acfe59f4..0cc958dfcc5 100644 --- a/examples/file-and-cloudinary-upload/app/utils/utils.server.ts +++ b/examples/file-and-cloudinary-upload/app/utils/utils.server.ts @@ -1,5 +1,5 @@ -import { PassThrough } from "node:stream"; import cloudinary from "cloudinary"; +import { writeAsyncIterableToWritable } from "@remix-run/node"; cloudinary.v2.config({ cloud_name: process.env.CLOUD_NAME, @@ -8,8 +8,7 @@ cloudinary.v2.config({ }); async function uploadImage(data: AsyncIterable) { - const dataStream = new PassThrough(); - const uploadPromise = new Promise((resolve, reject) => { + const uploadPromise = new Promise(async (resolve, reject) => { const uploadStream = cloudinary.v2.uploader.upload_stream( { folder: "remix", @@ -22,22 +21,9 @@ async function uploadImage(data: AsyncIterable) { resolve(result); } ); - dataStream.pipe(uploadStream); + await writeAsyncIterableToWritable(data, uploadStream); }); - let errored = false; - try { - for await (let chunk of data) { - dataStream.write(chunk); - } - } catch (error: any) { - errored = true; - dataStream.destroy(error); - } - if (!errored) { - dataStream.end(); - } - return uploadPromise; } diff --git a/packages/remix-node/index.ts b/packages/remix-node/index.ts index 5d4bbc8d563..0257e713f93 100644 --- a/packages/remix-node/index.ts +++ b/packages/remix-node/index.ts @@ -29,8 +29,9 @@ export { } from "./implementations"; export { - writeReadableStreamToWritable, readableStreamToString, + writeAsyncIterableToWritable, + writeReadableStreamToWritable, } from "./stream"; export { diff --git a/packages/remix-node/stream.ts b/packages/remix-node/stream.ts index 5430f00078d..cdaca6f36a1 100644 --- a/packages/remix-node/stream.ts +++ b/packages/remix-node/stream.ts @@ -28,8 +28,23 @@ export async function writeReadableStreamToWritable( } } +export async function writeAsyncIterableToWritable( + iterable: AsyncIterable, + writable: Writable +) { + try { + for await (let chunk of iterable) { + writable.write(chunk); + } + writable.end(); + } catch (error: any) { + writable.destroy(error); + throw error; + } +} + export async function readableStreamToString( - stream: ReadableStream, + stream: ReadableStream, encoding?: BufferEncoding ) { let reader = stream.getReader(); From 83b9e16ec6e898dfc7e501efe314a0ac3414b0f5 Mon Sep 17 00:00:00 2001 From: Jacob Ebey Date: Thu, 12 May 2022 10:37:21 -0700 Subject: [PATCH 43/47] rename readableStreamFromStream -> createReadableStreamFromReadable and export it for use --- packages/remix-node/index.ts | 1 + packages/remix-node/stream.ts | 2 +- packages/remix-node/upload/fileUploadHandler.ts | 7 +++++-- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/remix-node/index.ts b/packages/remix-node/index.ts index 0257e713f93..5c3186554af 100644 --- a/packages/remix-node/index.ts +++ b/packages/remix-node/index.ts @@ -29,6 +29,7 @@ export { } from "./implementations"; export { + createReadableStreamFromReadable, readableStreamToString, writeAsyncIterableToWritable, writeReadableStreamToWritable, diff --git a/packages/remix-node/stream.ts b/packages/remix-node/stream.ts index cdaca6f36a1..603d4453015 100644 --- a/packages/remix-node/stream.ts +++ b/packages/remix-node/stream.ts @@ -67,7 +67,7 @@ export async function readableStreamToString( return Buffer.concat(chunks).toString(encoding); } -export const readableStreamFromStream = ( +export const createReadableStreamFromReadable = ( source: Readable & { readableHighWaterMark?: number } ) => { let pump = new StreamPump(source); diff --git a/packages/remix-node/upload/fileUploadHandler.ts b/packages/remix-node/upload/fileUploadHandler.ts index a85c57a86c1..ad2836ab9c2 100644 --- a/packages/remix-node/upload/fileUploadHandler.ts +++ b/packages/remix-node/upload/fileUploadHandler.ts @@ -11,7 +11,10 @@ import type { UploadHandler } from "@remix-run/server-runtime"; // @ts-expect-error import * as streamSlice from "stream-slice"; -import { readableStreamFromStream, readableStreamToString } from "../stream"; +import { + createReadableStreamFromReadable, + readableStreamToString, +} from "../stream"; export type FileUploadHandlerFilterArgs = { filename: string; @@ -218,7 +221,7 @@ export class NodeOnDiskFile implements File { streamSlice.slice(this.slicer.start, this.slicer.end) ); } - return readableStreamFromStream(stream); + return createReadableStreamFromReadable(stream); } async text(): Promise { From 518f8b53c7b6261069f145d60ac3ed7b552da17e Mon Sep 17 00:00:00 2001 From: Jacob Ebey Date: Thu, 12 May 2022 18:21:08 -0700 Subject: [PATCH 44/47] revert tsconfig lib change removed jsdocs --- packages/remix-node/stream.ts | 20 ++------------------ packages/remix-node/tsconfig.json | 2 +- 2 files changed, 3 insertions(+), 19 deletions(-) diff --git a/packages/remix-node/stream.ts b/packages/remix-node/stream.ts index 603d4453015..6335cf92d0d 100644 --- a/packages/remix-node/stream.ts +++ b/packages/remix-node/stream.ts @@ -87,15 +87,6 @@ class StreamPump { }; private controller?: ReadableStreamController; - /** - * @param {Readable & { - * readableHighWaterMark?: number - * readable?:boolean, - * resume?: () => void, - * pause?: () => void - * destroy?: (error?:Error) => void - * }} stream - */ constructor( stream: Stream & { readableHighWaterMark?: number; @@ -115,16 +106,10 @@ class StreamPump { this.close = this.close.bind(this); } - /** - * @param {Uint8Array} [chunk] - */ size(chunk: Uint8Array) { return chunk?.byteLength || 0; } - /** - * @param {ReadableStreamController} controller - */ start(controller: ReadableStreamController) { this.controller = controller; this.stream.on("data", this.enqueue); @@ -137,7 +122,7 @@ class StreamPump { this.resume(); } - cancel(reason: Error) { + cancel(reason?: Error) { if (this.stream.destroy) { this.stream.destroy(reason); } @@ -158,13 +143,12 @@ class StreamPump { if (available <= 0) { this.pause(); } - } catch { + } catch (error: any) { this.controller.error( new Error( "Could not create Buffer, chunk must be of type string or an instance of Buffer, ArrayBuffer, or Array or an Array-like Object" ) ); - // @ts-expect-error this.cancel(); } } diff --git a/packages/remix-node/tsconfig.json b/packages/remix-node/tsconfig.json index 7b510fdf637..33578ea0891 100644 --- a/packages/remix-node/tsconfig.json +++ b/packages/remix-node/tsconfig.json @@ -1,7 +1,7 @@ { "exclude": ["__tests__"], "compilerOptions": { - "lib": ["ES2019", "DOM", "DOM.Iterable"], + "lib": ["ES2019", "DOM.Iterable"], "target": "ES2019", "moduleResolution": "node", From 48538d086eadf5ef8a3f699d62dcb07196181f46 Mon Sep 17 00:00:00 2001 From: Jacob Ebey Date: Fri, 13 May 2022 16:03:35 -0700 Subject: [PATCH 45/47] fix: pass through request without clone docs: updated docs to include note about not reading body in loader --- docs/api/conventions.md | 2 +- docs/decisions/0002-do-not-clone-request.md | 19 +++++++++++++++++++ docs/decisions/template.md | 11 +++++++++++ packages/remix-react/routes.tsx | 4 ++-- packages/remix-server-runtime/data.ts | 4 ++-- packages/remix-server-runtime/server.ts | 8 ++++---- 6 files changed, 39 insertions(+), 9 deletions(-) create mode 100644 docs/decisions/0002-do-not-clone-request.md create mode 100644 docs/decisions/template.md diff --git a/docs/api/conventions.md b/docs/api/conventions.md index 638f69cf9bd..f2ade008d3e 100644 --- a/docs/api/conventions.md +++ b/docs/api/conventions.md @@ -546,7 +546,7 @@ export default function SomeRouteComponent() { Watch the 📼 Remix Single: Loading data into components -Each route can define a "loader" function that will be called on the server before rendering to provide data to the route. +Each route can define a "loader" function that will be called on the server before rendering to provide data to the route. You may think of this as a "GET" request handler in that you should not be reading the body of the request; that is the job of an [`action`](#action). ```js import { json } from "@remix-run/{runtime}"; diff --git a/docs/decisions/0002-do-not-clone-request.md b/docs/decisions/0002-do-not-clone-request.md new file mode 100644 index 00000000000..6fef84ca87c --- /dev/null +++ b/docs/decisions/0002-do-not-clone-request.md @@ -0,0 +1,19 @@ +# Do not clone request + + Date: 2022-05-13 + + Status: accepted + + ## Context + + To allow multiple loaders / actions to read the body of a request, we have been cloning the request before forwarding it to user-code. This is not the best thing to do as some runtimes will begin buffering the body to allow for multiple consumers. It is also goes against "the platform" that states a request body should only be consumed once. + + ## Decision + + Do not clone requests before they are passed to user-code (loaders, actions, handleDocumentRequest, handleDataRequest, etc.). + + ## Consequences + +If you are reading the request body in both an action and a loader this will now fail. Loaders should be thought of as a "GET" / "HEAD" request handler. These request methods are not allowed to have a body, therefore you should not be reading it in your Remix loader function. + +If you wish to continue reading the request body in multiple places for a single request against recommendations, consider using `.clone()` before reading it. diff --git a/docs/decisions/template.md b/docs/decisions/template.md new file mode 100644 index 00000000000..6453f5276d2 --- /dev/null +++ b/docs/decisions/template.md @@ -0,0 +1,11 @@ +# Title + + Date: YYYY-MM-DD + + Status: proposed | rejected | accepted | deprecated | … | superseded by [0005](0005-example.md) + + ## Context + + ## Decision + + ## Consequences \ No newline at end of file diff --git a/packages/remix-react/routes.tsx b/packages/remix-react/routes.tsx index bcb6e3e8616..5873e13bf17 100644 --- a/packages/remix-react/routes.tsx +++ b/packages/remix-react/routes.tsx @@ -162,7 +162,7 @@ function createLoader(route: EntryRoute, routeModules: RouteModules) { throw new CatchValue( result.status, result.statusText, - await extractData(result.clone()) + await extractData(result) ); } @@ -199,7 +199,7 @@ function createAction(route: EntryRoute, routeModules: RouteModules) { throw new CatchValue( result.status, result.statusText, - await extractData(result.clone()) + await extractData(result) ); } diff --git a/packages/remix-server-runtime/data.ts b/packages/remix-server-runtime/data.ts index 7800cfab220..4f909630bbc 100644 --- a/packages/remix-server-runtime/data.ts +++ b/packages/remix-server-runtime/data.ts @@ -33,7 +33,7 @@ export async function callRouteAction({ let result; try { result = await action({ - request: stripDataParam(stripIndexParam(request.clone())), + request: stripDataParam(stripIndexParam(request)), context: loadContext, params: match.params, }); @@ -80,7 +80,7 @@ export async function callRouteLoader({ let result; try { result = await loader({ - request: stripDataParam(stripIndexParam(request.clone())), + request: stripDataParam(stripIndexParam(request)), context: loadContext, params: match.params, }); diff --git a/packages/remix-server-runtime/server.ts b/packages/remix-server-runtime/server.ts index 2da6cd3a2ae..35bdc2593dc 100644 --- a/packages/remix-server-runtime/server.ts +++ b/packages/remix-server-runtime/server.ts @@ -150,10 +150,10 @@ async function handleDataRequest({ } if (handleDataRequest) { - response = await handleDataRequest(response.clone(), { + response = await handleDataRequest(response, { context: loadContext, params: match.params, - request: request.clone(), + request, }); } @@ -457,7 +457,7 @@ async function handleDocumentRequest({ let handleDocumentRequest = build.entry.module.default; try { return await handleDocumentRequest( - request.clone(), + request, responseStatusCode, responseHeaders, entryContext @@ -477,7 +477,7 @@ async function handleDocumentRequest({ try { return await handleDocumentRequest( - request.clone(), + request, responseStatusCode, responseHeaders, entryContext From 28a748168e78811cfd6340c6b276ee9fd85eaa03 Mon Sep 17 00:00:00 2001 From: Jacob Ebey Date: Fri, 13 May 2022 16:06:43 -0700 Subject: [PATCH 46/47] updated doc --- docs/decisions/0002-do-not-clone-request.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/decisions/0002-do-not-clone-request.md b/docs/decisions/0002-do-not-clone-request.md index 6fef84ca87c..a37a799969f 100644 --- a/docs/decisions/0002-do-not-clone-request.md +++ b/docs/decisions/0002-do-not-clone-request.md @@ -16,4 +16,4 @@ If you are reading the request body in both an action and a loader this will now fail. Loaders should be thought of as a "GET" / "HEAD" request handler. These request methods are not allowed to have a body, therefore you should not be reading it in your Remix loader function. -If you wish to continue reading the request body in multiple places for a single request against recommendations, consider using `.clone()` before reading it. +If you wish to continue reading the request body in multiple places for a single request against recommendations, consider using `.clone()` before reading it; just know this comes with tradeoffs. From a57d90b950268c2303a81cd486415412dd6983a5 Mon Sep 17 00:00:00 2001 From: Jacob Ebey Date: Fri, 13 May 2022 16:50:29 -0700 Subject: [PATCH 47/47] updated docs --- docs/api/remix.md | 82 ++++++++++++++++++++++------------------------- 1 file changed, 38 insertions(+), 44 deletions(-) diff --git a/docs/api/remix.md b/docs/api/remix.md index 916860f749a..d5d3c0efafb 100644 --- a/docs/api/remix.md +++ b/docs/api/remix.md @@ -1618,7 +1618,7 @@ export const action: ActionFunction = async ({ | avoidFileConflicts | boolean | true | Avoid file conflicts by appending a timestamp on the end of the filename if it already exists on disk | | directory | string \| Function | os.tmpdir() | The directory to write the upload. | | file | Function | () => `upload_${random}.${ext}` | The name of the file in the directory. Can be a relative path, the directory structure will be created if it does not exist. | -| maxPartSize | number | 3000000 | The maximum upload size allowed (in bytes). If the size is exceeded a MaxPartSizeExceededError will be thrown. | +| maxPartSize | number | 3000000 | The maximum upload size allowed (in bytes). If the size is exceeded a MaxPartSizeExceededError will be thrown. | | filter | Function | OPTIONAL | A function you can write to prevent a file upload from being saved based on filename, mimetype, or encoding. Return `false` and the file will be ignored. | The function API for `file` and `directory` are the same. They accept an `object` and return a `string`. The object it accepts has `filename`, `encoding`, and `mimetype` (all strings).The `string` returned is the path. @@ -1658,64 +1658,58 @@ Most of the time, you'll probably want to proxy the file stream to a file host. ```tsx import type { UploadHandler } from "@remix-run/{runtime}"; +import { + unstable_composeUploadHandlers, + unstable_createMemoryUploadHandler, +} from "@remix-run/{runtime}"; +// writeAsyncIterableToWritable is a node only utility +import { writeAsyncIterableToWritable } from "@remix-run/node"; import type { - UploadApiErrorResponse, UploadApiOptions, UploadApiResponse, UploadStream, } from "cloudinary"; import cloudinary from "cloudinary"; +async function uploadImageToCloudinary(data: AsyncIterable) { + const uploadPromise = new Promise(async (resolve, reject) => { + const uploadStream = cloudinary.v2.uploader.upload_stream( + { + folder: "remix", + }, + (error, result) => { + if (error) { + reject(error); + return; + } + resolve(result); + } + ); + await writeAsyncIterableToWritable(data, uploadStream); + }); + + return uploadPromise; +} + export const action: ActionFunction = async ({ request, }) => { const userId = getUserId(request); - function uploadStreamToCloudinary( - stream: Readable, - options?: UploadApiOptions - ): Promise { - return new Promise((resolve, reject) => { - const uploader = cloudinary.v2.uploader.upload_stream( - options, - (error, result) => { - if (result) { - resolve(result); - } else { - reject(error); - } + const uploadHandler = + unstable_composeUploadHandlers( + // our custom upload handler + async ({ name, contentType, data, filename }) => { + if (name !== "img") { + return undefined; } - ); - - stream.pipe(uploader); - }); - } - - const uploadHandler: UploadHandler = async ({ - name, - stream, - }) => { - // we only care about the file form field called "avatar" - // so we'll ignore anything else - // NOTE: the way our form is set up, we shouldn't get any other fields, - // but this is good defensive programming in case someone tries to hit our - // action directly via curl or something weird like that. - if (name !== "avatar") { - stream.resume(); - return; - } - - const uploadedImage = await uploadStreamToCloudinary( - stream, - { - public_id: userId, - folder: "/my-site/avatars", - } + const uploadedImage = await uploadImageToCloudinary(data); + return uploadedImage.secure_url; + }, + // fallback to memory for everything else + unstable_createMemoryUploadHandler() ); - return uploadedImage.secure_url; - }; - const formData = await unstable_parseMultipartFormData( request, uploadHandler