From b5aa0b3b39e387fde919fa0a28148d34130b24c5 Mon Sep 17 00:00:00 2001 From: cjihrig Date: Wed, 29 Dec 2021 14:12:45 -0500 Subject: [PATCH 1/2] fs: use async directory processing in cp() The readdir() functions do not scale well, which is why opendir(), etc. were introduced. This is exacerbated in the current cp() implementation, which calls readdir() recursively. This commit updates cp() to use the opendir() style iteration. --- lib/internal/fs/cp/cp-sync.js | 24 ++++++++++++++++-------- lib/internal/fs/cp/cp.js | 13 +++++++------ 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/lib/internal/fs/cp/cp-sync.js b/lib/internal/fs/cp/cp-sync.js index 3ee6a70f959582..497c8c57c317dd 100644 --- a/lib/internal/fs/cp/cp-sync.js +++ b/lib/internal/fs/cp/cp-sync.js @@ -32,7 +32,7 @@ const { existsSync, lstatSync, mkdirSync, - readdirSync, + opendirSync, readlinkSync, statSync, symlinkSync, @@ -275,14 +275,22 @@ function mkDirAndCopy(srcMode, src, dest, opts) { } function copyDir(src, dest, opts) { - readdirSync(src).forEach((item) => copyDirItem(item, src, dest, opts)); -} + const dir = opendirSync(src); + + try { + let dirent; -function copyDirItem(item, src, dest, opts) { - const srcItem = join(src, item); - const destItem = join(dest, item); - const { destStat } = checkPathsSync(srcItem, destItem, opts); - return startCopy(destStat, srcItem, destItem, opts); + while ((dirent = dir.readSync()) !== null) { + const { name } = dirent; + const srcItem = join(src, name); + const destItem = join(dest, name); + const { destStat } = checkPathsSync(srcItem, destItem, opts); + + startCopy(destStat, srcItem, destItem, opts); + } + } finally { + dir.closeSync(); + } } function onLink(destStat, src, dest) { diff --git a/lib/internal/fs/cp/cp.js b/lib/internal/fs/cp/cp.js index 090a4023df14e8..69d4c2c4bdb7fe 100644 --- a/lib/internal/fs/cp/cp.js +++ b/lib/internal/fs/cp/cp.js @@ -41,7 +41,7 @@ const { copyFile, lstat, mkdir, - readdir, + opendir, readlink, stat, symlink, @@ -325,11 +325,12 @@ async function mkDirAndCopy(srcMode, src, dest, opts) { } async function copyDir(src, dest, opts) { - const dir = await readdir(src); - for (let i = 0; i < dir.length; i++) { - const item = dir[i]; - const srcItem = join(src, item); - const destItem = join(dest, item); + const dir = await opendir(src); + + for await (const dirent of dir) { + const { name } = dirent; + const srcItem = join(src, name); + const destItem = join(dest, name); const { destStat } = await checkPaths(srcItem, destItem, opts); await startCopy(destStat, srcItem, destItem, opts); } From 625c12ff183b2d1a49990a8cf259b346515c10d0 Mon Sep 17 00:00:00 2001 From: Colin Ihrig Date: Wed, 29 Dec 2021 14:51:10 -0500 Subject: [PATCH 2/2] Update lib/internal/fs/cp/cp.js Co-authored-by: Luigi Pinca --- lib/internal/fs/cp/cp.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/internal/fs/cp/cp.js b/lib/internal/fs/cp/cp.js index 69d4c2c4bdb7fe..6dc212f2f6a5fc 100644 --- a/lib/internal/fs/cp/cp.js +++ b/lib/internal/fs/cp/cp.js @@ -327,8 +327,7 @@ async function mkDirAndCopy(srcMode, src, dest, opts) { async function copyDir(src, dest, opts) { const dir = await opendir(src); - for await (const dirent of dir) { - const { name } = dirent; + for await (const { name } of dir) { const srcItem = join(src, name); const destItem = join(dest, name); const { destStat } = await checkPaths(srcItem, destItem, opts);