From 6c006e12eaa6705cdf20b7b43cccc44a1f7ea185 Mon Sep 17 00:00:00 2001 From: Arda TANRIKULU Date: Fri, 8 Nov 2024 06:48:14 -0500 Subject: [PATCH] New `TextDecoderStream` and `TextEncoderStream` implementations (#1782) --- .changeset/flat-dragons-press.md | 6 +++ packages/fetch/dist/create-node-ponyfill.js | 4 ++ packages/fetch/dist/esm-ponyfill.js | 4 ++ packages/fetch/dist/global-ponyfill.js | 4 +- packages/fetch/dist/index.d.ts | 4 ++ packages/fetch/dist/node-ponyfill.js | 2 + packages/fetch/dist/shouldSkipPonyfill.js | 3 +- packages/node-fetch/src/Blob.ts | 21 ++++++++ .../src/TextEncoderDecoderStream.ts | 52 ++++++++++++++++++ packages/node-fetch/src/index.ts | 4 ++ .../tests/TextEncoderDecoderStream.spec.ts | 53 +++++++++++++++++++ yarn.lock | 1 - 12 files changed, 154 insertions(+), 4 deletions(-) create mode 100644 .changeset/flat-dragons-press.md create mode 100644 packages/node-fetch/src/TextEncoderDecoderStream.ts create mode 100644 packages/node-fetch/tests/TextEncoderDecoderStream.spec.ts diff --git a/.changeset/flat-dragons-press.md b/.changeset/flat-dragons-press.md new file mode 100644 index 00000000000..e634906d576 --- /dev/null +++ b/.changeset/flat-dragons-press.md @@ -0,0 +1,6 @@ +--- +'@whatwg-node/node-fetch': minor +'@whatwg-node/fetch': minor +--- + +\`TextDecoderStream\` and \`TextEncoderStream\` diff --git a/packages/fetch/dist/create-node-ponyfill.js b/packages/fetch/dist/create-node-ponyfill.js index 9009bd37550..e0d68cddec9 100644 --- a/packages/fetch/dist/create-node-ponyfill.js +++ b/packages/fetch/dist/create-node-ponyfill.js @@ -23,6 +23,8 @@ module.exports = function createNodePonyfill(opts = {}) { TransformStream: globalThis.TransformStream, CompressionStream: globalThis.CompressionStream, DecompressionStream: globalThis.DecompressionStream, + TextDecoderStream: globalThis.TextDecoderStream, + TextEncoderStream: globalThis.TextEncoderStream, Blob: globalThis.Blob, File: globalThis.File, crypto: globalThis.crypto, @@ -51,6 +53,8 @@ module.exports = function createNodePonyfill(opts = {}) { ponyfills.TransformStream = newNodeFetch.TransformStream; ponyfills.CompressionStream = newNodeFetch.CompressionStream; ponyfills.DecompressionStream = newNodeFetch.DecompressionStream; + ponyfills.TextDecoderStream = newNodeFetch.TextDecoderStream; + ponyfills.TextEncoderStream = newNodeFetch.TextEncoderStream; ponyfills.Blob = newNodeFetch.Blob; ponyfills.File = newNodeFetch.File; diff --git a/packages/fetch/dist/esm-ponyfill.js b/packages/fetch/dist/esm-ponyfill.js index b267b70399e..02bf81c1e42 100644 --- a/packages/fetch/dist/esm-ponyfill.js +++ b/packages/fetch/dist/esm-ponyfill.js @@ -8,6 +8,8 @@ const WritableStream = globalThis.WritableStream; const TransformStream = globalThis.TransformStream; const CompressionStream = globalThis.CompressionStream; const DecompressionStream = globalThis.DecompressionStream; +const TextDecoderStream = globalThis.TextDecoderStream; +const TextEncoderStream = globalThis.TextEncoderStream; const Blob = globalThis.Blob; const File = globalThis.File; const crypto = globalThis.crypto; @@ -29,6 +31,8 @@ export { TransformStream, CompressionStream, DecompressionStream, + TextDecoderStream, + TextEncoderStream, Blob, File, crypto, diff --git a/packages/fetch/dist/global-ponyfill.js b/packages/fetch/dist/global-ponyfill.js index 80b2ac974e2..321ae1971a4 100644 --- a/packages/fetch/dist/global-ponyfill.js +++ b/packages/fetch/dist/global-ponyfill.js @@ -1,4 +1,4 @@ -module.exports.fetch = globalThis.fetch; // To enable: import {fetch} from 'cross-fetch' +module.exports.fetch = globalThis.fetch; module.exports.Headers = globalThis.Headers; module.exports.Request = globalThis.Request; module.exports.Response = globalThis.Response; @@ -8,6 +8,8 @@ module.exports.WritableStream = globalThis.WritableStream; module.exports.TransformStream = globalThis.TransformStream; module.exports.CompressionStream = globalThis.CompressionStream; module.exports.DecompressionStream = globalThis.DecompressionStream; +module.exports.TextDecoderStream = globalThis.TextDecoderStream; +module.exports.TextEncoderStream = globalThis.TextEncoderStream; module.exports.Blob = globalThis.Blob; module.exports.File = globalThis.File; module.exports.crypto = globalThis.crypto; diff --git a/packages/fetch/dist/index.d.ts b/packages/fetch/dist/index.d.ts index d16ddf2f4e3..f55c0838ebf 100644 --- a/packages/fetch/dist/index.d.ts +++ b/packages/fetch/dist/index.d.ts @@ -17,6 +17,8 @@ declare module '@whatwg-node/fetch' { export const TransformStream: typeof globalThis.TransformStream; export const CompressionStream: typeof globalThis.CompressionStream; export const DecompressionStream: typeof globalThis.DecompressionStream; + export const TextDecoderStream: typeof globalThis.TextDecoderStream; + export const TextEncoderStream: typeof globalThis.TextEncoderStream; export const Blob: typeof globalThis.Blob; export const File: typeof globalThis.File; export const crypto: typeof globalThis.crypto; @@ -57,6 +59,8 @@ declare module '@whatwg-node/fetch' { TransformStream: typeof TransformStream; CompressionStream: typeof CompressionStream; DecompressionStream: typeof DecompressionStream; + TextDecoderStream: typeof TextDecoderStream; + TextEncoderStream: typeof TextEncoderStream; Blob: typeof Blob; File: typeof File; crypto: typeof crypto; diff --git a/packages/fetch/dist/node-ponyfill.js b/packages/fetch/dist/node-ponyfill.js index fc41a0c43ab..c5bf8f05efc 100644 --- a/packages/fetch/dist/node-ponyfill.js +++ b/packages/fetch/dist/node-ponyfill.js @@ -20,6 +20,8 @@ module.exports.WritableStream = ponyfills.WritableStream; module.exports.TransformStream = ponyfills.TransformStream; module.exports.CompressionStream = ponyfills.CompressionStream; module.exports.DecompressionStream = ponyfills.DecompressionStream; +module.exports.TextDecoderStream = ponyfills.TextDecoderStream; +module.exports.TextEncoderStream = ponyfills.TextEncoderStream; module.exports.Blob = ponyfills.Blob; module.exports.File = ponyfills.File; module.exports.crypto = ponyfills.crypto; diff --git a/packages/fetch/dist/shouldSkipPonyfill.js b/packages/fetch/dist/shouldSkipPonyfill.js index 3c5a56a2d35..eeda068ff45 100644 --- a/packages/fetch/dist/shouldSkipPonyfill.js +++ b/packages/fetch/dist/shouldSkipPonyfill.js @@ -4,11 +4,10 @@ function isNextJs() { } module.exports = function shouldSkipPonyfill() { - // Bun and Deno already have a Fetch API if (globalThis.Deno) { return true } - if (process.versions.bun) { + if (globalThis.Bun) { return true } if (isNextJs()) { diff --git a/packages/node-fetch/src/Blob.ts b/packages/node-fetch/src/Blob.ts index 6e19f857e24..26462063fe3 100644 --- a/packages/node-fetch/src/Blob.ts +++ b/packages/node-fetch/src/Blob.ts @@ -210,6 +210,27 @@ export class PonyfillBlob implements Blob { }); } + _json: any = null; + + json() { + if (this._json) { + return fakePromise(this._json); + } + return this.text().then(text => { + this._json = JSON.parse(text); + return this._json; + }); + } + + _formData: FormData | null = null; + + formData() { + if (this._formData) { + return fakePromise(this._formData); + } + throw new Error('Not implemented'); + } + get size() { if (this._size == null) { this._size = 0; diff --git a/packages/node-fetch/src/TextEncoderDecoderStream.ts b/packages/node-fetch/src/TextEncoderDecoderStream.ts new file mode 100644 index 00000000000..43b99858481 --- /dev/null +++ b/packages/node-fetch/src/TextEncoderDecoderStream.ts @@ -0,0 +1,52 @@ +import { PonyfillTextDecoder, PonyfillTextEncoder } from './TextEncoderDecoder.js'; +import { PonyfillTransformStream } from './TransformStream.js'; + +export class PonyfillTextDecoderStream + extends PonyfillTransformStream + implements TextDecoderStream +{ + private textDecoder: TextDecoder; + constructor(encoding?: BufferEncoding, options?: TextDecoderOptions) { + super({ + transform: (chunk, controller) => { + controller.enqueue(this.textDecoder.decode(chunk, { stream: true })); + }, + }); + this.textDecoder = new PonyfillTextDecoder(encoding, options); + } + + get encoding(): string { + return this.textDecoder.encoding; + } + + get fatal(): boolean { + return this.textDecoder.fatal; + } + + get ignoreBOM(): boolean { + return this.textDecoder.ignoreBOM; + } +} + +export class PonyfillTextEncoderStream + extends PonyfillTransformStream + implements TextEncoderStream +{ + private textEncoder: TextEncoder; + constructor(encoding?: BufferEncoding) { + super({ + transform: (chunk, controller) => { + controller.enqueue(this.textEncoder.encode(chunk)); + }, + }); + this.textEncoder = new PonyfillTextEncoder(encoding); + } + + get encoding(): string { + return this.textEncoder.encoding; + } + + encode(input: string): Uint8Array { + return this.textEncoder.encode(input); + } +} diff --git a/packages/node-fetch/src/index.ts b/packages/node-fetch/src/index.ts index a7c2f522eef..8549eabbc9a 100644 --- a/packages/node-fetch/src/index.ts +++ b/packages/node-fetch/src/index.ts @@ -19,3 +19,7 @@ export { PonyfillTransformStream as TransformStream } from './TransformStream.js export { PonyfillCompressionStream as CompressionStream } from './CompressionStream.js'; export { PonyfillDecompressionStream as DecompressionStream } from './DecompressionStream.js'; export { PonyfillIteratorObject as IteratorObject } from './IteratorObject.js'; +export { + PonyfillTextDecoderStream as TextDecoderStream, + PonyfillTextEncoderStream as TextEncoderStream, +} from './TextEncoderDecoderStream.js'; diff --git a/packages/node-fetch/tests/TextEncoderDecoderStream.spec.ts b/packages/node-fetch/tests/TextEncoderDecoderStream.spec.ts new file mode 100644 index 00000000000..f49311fc6fb --- /dev/null +++ b/packages/node-fetch/tests/TextEncoderDecoderStream.spec.ts @@ -0,0 +1,53 @@ +import { PonyfillReadableStream } from '../src/ReadableStream'; +import { PonyfillTextEncoder } from '../src/TextEncoderDecoder'; +import { + PonyfillTextDecoderStream, + PonyfillTextEncoderStream, +} from '../src/TextEncoderDecoderStream'; + +describe('TextEncoderDecoderStream', () => { + it('TextEncoderStream', async () => { + const readableStream = new PonyfillReadableStream({ + start(controller) { + controller.enqueue(Buffer.from('Hello, ')); + controller.enqueue(Buffer.from('world!')); + controller.close(); + }, + }); + const pipedStream = readableStream.pipeThrough(new PonyfillTextEncoderStream()); + const reader = pipedStream.getReader(); + const chunks: Uint8Array[] = []; + while (true) { + const { done, value } = await reader.read(); + if (done) { + break; + } + chunks.push(value); + } + const encoded = Buffer.concat(chunks); + expect(encoded.toString('utf-8')).toBe('Hello, world!'); + }); + it('TextDecoderStream', async () => { + const textEncoder = new PonyfillTextEncoder(); + const decodedHello = textEncoder.encode('Hello, '); + const decodedWorld = textEncoder.encode('world!'); + const readableStream = new PonyfillReadableStream({ + start(controller) { + controller.enqueue(decodedHello); + controller.enqueue(decodedWorld); + controller.close(); + }, + }); + const chunks: string[] = []; + const pipedStream = readableStream.pipeThrough(new PonyfillTextDecoderStream()); + const reader = pipedStream.getReader(); + while (true) { + const { done, value } = await reader.read(); + if (done) { + break; + } + chunks.push(value); + } + expect(chunks.join('')).toBe('Hello, world!'); + }); +}); diff --git a/yarn.lock b/yarn.lock index 9a4e88928e1..a35a1036f8e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -9499,7 +9499,6 @@ typescript@5.6.3: uWebSockets.js@uNetworking/uWebSockets.js#v20.49.0: version "20.49.0" - uid "442087c0a01bf146acb7386910739ec81df06700" resolved "https://codeload.github.com/uNetworking/uWebSockets.js/tar.gz/442087c0a01bf146acb7386910739ec81df06700" ufo@^1.5.4: