From 3512c71df7643f89d18db87d962284c6689a4905 Mon Sep 17 00:00:00 2001 From: HiroyukiYagihashi Date: Sun, 21 Feb 2021 22:14:21 +0900 Subject: [PATCH] fs: `writeFile` support `AsyncIterable`, `Iterable` & `Stream` as `data` argument Fixes: https://github.com/nodejs/node/issues/37391 --- doc/api/fs.md | 6 +- lib/internal/fs/promises.js | 28 ++++++-- test/parallel/test-fs-append-file.js | 2 +- test/parallel/test-fs-promises-writefile.js | 74 +++++++++++++++++++-- 4 files changed, 95 insertions(+), 15 deletions(-) diff --git a/doc/api/fs.md b/doc/api/fs.md index 5ffd9df27451a4..0262176f6f6be7 100644 --- a/doc/api/fs.md +++ b/doc/api/fs.md @@ -1247,6 +1247,9 @@ All the [caveats][] for `fs.watch()` also apply to `fsPromises.watch()`. * `file` {string|Buffer|URL|FileHandle} filename or `FileHandle` -* `data` {string|Buffer|Uint8Array|Object} +* `data` {string|Buffer|Uint8Array|Object|AsyncIterable|Iterable + |Stream} * `options` {Object|string} * `encoding` {string|null} **Default:** `'utf8'` * `mode` {integer} **Default:** `0o666` diff --git a/lib/internal/fs/promises.js b/lib/internal/fs/promises.js index 26e4fd8f73be98..bedfb6034fa570 100644 --- a/lib/internal/fs/promises.js +++ b/lib/internal/fs/promises.js @@ -42,7 +42,7 @@ const { }, AbortError, } = require('internal/errors'); -const { isArrayBufferView } = require('internal/util/types'); +const { isArrayBuffer, isArrayBufferView } = require('internal/util/types'); const { rimrafPromises } = require('internal/fs/rimraf'); const { copyObject, @@ -78,6 +78,7 @@ const pathModule = require('path'); const { promisify } = require('internal/util'); const { EventEmitterMixin } = require('internal/event_target'); const { watch } = require('internal/fs/watchers'); +const { isIterable } = require('internal/streams/utils'); const kHandle = Symbol('kHandle'); const kFd = Symbol('kFd'); @@ -273,8 +274,17 @@ function checkAborted(signal) { throw new AbortError(); } -async function writeFileHandle(filehandle, data, signal) { - // `data` could be any kind of typed array. +async function writeFileHandle(filehandle, data, signal, encoding) { + checkAborted(signal); + if (isCustomIterable(data)) { + for await (const buf of data) { + checkAborted(signal); + await write( + filehandle, buf, undefined, isArrayBuffer(buf) ? buf.length : encoding); + checkAborted(signal); + } + return; + } data = new Uint8Array(data.buffer, data.byteOffset, data.byteLength); let remaining = data.length; if (remaining === 0) return; @@ -679,20 +689,24 @@ async function writeFile(path, data, options) { options = getOptions(options, { encoding: 'utf8', mode: 0o666, flag: 'w' }); const flag = options.flag || 'w'; - if (!isArrayBufferView(data)) { + if (!isArrayBufferView(data) && !isCustomIterable(data)) { validateStringAfterArrayBufferView(data, 'data'); data = Buffer.from(data, options.encoding || 'utf8'); } validateAbortSignal(options.signal); if (path instanceof FileHandle) - return writeFileHandle(path, data, options.signal); + return writeFileHandle(path, data, options.signal, options.encoding); checkAborted(options.signal); const fd = await open(path, flag, options.mode); - const { signal } = options; - return PromisePrototypeFinally(writeFileHandle(fd, data, signal), fd.close); + return PromisePrototypeFinally( + writeFileHandle(fd, data, options.signal, options.encoding), fd.close); +} + +function isCustomIterable(obj) { + return isIterable(obj) && !isArrayBufferView(obj) && typeof obj !== 'string'; } async function appendFile(path, data, options) { diff --git a/test/parallel/test-fs-append-file.js b/test/parallel/test-fs-append-file.js index a191f8b20693c1..70919830f6cb0b 100644 --- a/test/parallel/test-fs-append-file.js +++ b/test/parallel/test-fs-append-file.js @@ -121,7 +121,7 @@ const throwNextTick = (e) => { process.nextTick(() => { throw e; }); }; } // Test that appendFile does not accept invalid data type (callback API). -[false, 5, {}, [], null, undefined].forEach(async (data) => { +[false, 5, {}, null, undefined].forEach(async (data) => { const errObj = { code: 'ERR_INVALID_ARG_TYPE', message: /"data"|"buffer"/ diff --git a/test/parallel/test-fs-promises-writefile.js b/test/parallel/test-fs-promises-writefile.js index 7fbe12dda4dc2d..0f1cf7be33f55d 100644 --- a/test/parallel/test-fs-promises-writefile.js +++ b/test/parallel/test-fs-promises-writefile.js @@ -7,6 +7,7 @@ const path = require('path'); const tmpdir = require('../common/tmpdir'); const assert = require('assert'); const tmpDir = tmpdir.path; +const { Readable } = require('stream'); tmpdir.refresh(); @@ -14,6 +15,23 @@ const dest = path.resolve(tmpDir, 'tmp.txt'); const otherDest = path.resolve(tmpDir, 'tmp-2.txt'); const buffer = Buffer.from('abc'.repeat(1000)); const buffer2 = Buffer.from('xyz'.repeat(1000)); +const stream = Readable.from(['a', 'b', 'c']); +const iterable = { + expected: 'abc', + *[Symbol.iterator]() { + yield 'a'; + yield 'b'; + yield 'c'; + } +}; +const asyncIterable = { + expected: 'abc', + async* [Symbol.asyncIterator]() { + yield 'a'; + yield 'b'; + yield 'c'; + } +}; async function doWrite() { await fsPromises.writeFile(dest, buffer); @@ -21,6 +39,44 @@ async function doWrite() { assert.deepStrictEqual(data, buffer); } +async function doWriteStream() { + await fsPromises.writeFile(dest, stream); + const expected = 'abc'; + const data = fs.readFileSync(dest, 'utf-8'); + assert.deepStrictEqual(data, expected); +} + +async function doWriteStreamWithCancel() { + const controller = new AbortController(); + const { signal } = controller; + process.nextTick(() => controller.abort()); + assert.rejects(fsPromises.writeFile(otherDest, stream, { signal }), { + name: 'AbortError' + }); +} + +async function doWriteIterable() { + await fsPromises.writeFile(dest, iterable); + const data = fs.readFileSync(dest, 'utf-8'); + assert.deepStrictEqual(data, iterable.expected); +} + +async function doWriteAsyncIterable() { + await fsPromises.writeFile(dest, asyncIterable); + const data = fs.readFileSync(dest, 'utf-8'); + assert.deepStrictEqual(data, asyncIterable.expected); +} + +async function doWriteInvalidValues() { + await Promise.all( + [42, 42n, {}, Symbol('42'), true, undefined, null, NaN].map((value) => + assert.rejects(fsPromises.writeFile(dest, value), { + code: 'ERR_INVALID_ARG_TYPE', + }) + ) + ); +} + async function doWriteWithCancel() { const controller = new AbortController(); const { signal } = controller; @@ -50,9 +106,15 @@ async function doReadWithEncoding() { assert.deepStrictEqual(data, syncData); } -doWrite() - .then(doWriteWithCancel) - .then(doAppend) - .then(doRead) - .then(doReadWithEncoding) - .then(common.mustCall()); +(async () => { + await doWrite(); + await doWriteWithCancel(); + await doAppend(); + await doRead(); + await doReadWithEncoding(); + await doWriteStream(); + await doWriteStreamWithCancel(); + await doWriteIterable(); + await doWriteAsyncIterable(); + await doWriteInvalidValues(); +})().then(common.mustCall());