From 410fb0e2e1ad2a48c8b8f5a3f8f68685d6e12c2a Mon Sep 17 00:00:00 2001 From: Yixin Sun <43978285+piccaSun@users.noreply.github.com> Date: Sat, 24 Feb 2024 23:11:09 +0800 Subject: [PATCH] =?UTF-8?q?feat(portal):=20=E5=8F=AA=E5=9C=A8=E4=BD=BF?= =?UTF-8?q?=E7=94=A8=E6=96=87=E4=BB=B6=E4=BC=A0=E8=BE=93=E5=8A=9F=E8=83=BD?= =?UTF-8?q?=E6=97=B6=E5=88=A9=E7=94=A8touch=20-a=E6=9B=B4=E6=96=B0?= =?UTF-8?q?=E6=97=B6=E9=97=B4=E6=88=B3=20(#1133)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### 问题 因为文件传输功能需要使用 `touch -a` 更新时间戳来触发 nfs 缓存机制 目前在没有使用文件传输功能时也会在访问文件路径时对其下所有文件使用 `touch -a` ，给后台维护带来不便 此PR修复为只在文件传输功能中对文件执行`touch -a`，同时修复文件名中出现中文字符或特殊字符如";"时 `touch -a` 无法正常执行会在后台打印logger.error的问题 ### 修复前 在只进行文件管理下的路径访问时，后台也会出现部分 `touch -a` 的报错 ![image](https://github.com/PKUHPC/SCOW/assets/43978285/5bd3e70f-da35-4728-9cb5-5bacbc77b9d6) ![image](https://github.com/PKUHPC/SCOW/assets/43978285/78ea6889-63d0-4e31-817c-c38bdd26dba6) ### 修复后 **1.只在文件传输时执行`touch -a`更新访问时间戳** 访问/data/home/demo_admin/前 /data/home/demo_admin/MNIST.ipynb的时间戳 ![image](https://github.com/PKUHPC/SCOW/assets/43978285/4b64dd39-f8ed-4368-8245-398bc017c83c) 文件管理及提交作业页面选择文件路径时不执行touch -a，不出现touch -a相关报错 ![image](https://github.com/PKUHPC/SCOW/assets/43978285/6f3b4172-664e-4aee-b527-532054516631) /data/home/demo_admin/MNIST.ipynb的时间戳无变化 ![image](https://github.com/PKUHPC/SCOW/assets/43978285/82b39cbf-d437-486a-bd68-28fa4c0a87bc) 在文件传输功能时因为/data/home/demo_admin/下某文件路径权限问题touch -a报错 ![image](https://github.com/PKUHPC/SCOW/assets/43978285/3f823a01-e74a-4ad0-b6bf-316f2bbae4d6) /data/home/demo_admin/MNIST.ipynb的时间戳更新 ![image](https://github.com/PKUHPC/SCOW/assets/43978285/5ce31248-08c4-4dba-b3fc-4b20494d3b86) 开发环境文件传输相关功能无异常 **2.特殊字符的情况可以正常执行 `touch -a` 不会在后台留下错误** ![image](https://github.com/PKUHPC/SCOW/assets/43978285/14edd2af-0c7d-4482-bad6-0a691ff5b450) 时间戳正常更新 ![image](https://github.com/PKUHPC/SCOW/assets/43978285/dba27f86-4206-4dfe-ab5c-f6775dcfca2d) ### 同时此PR修复了下面有关`touch -a`执行的问题 **对 #1103 中下述问题进行修复：** 1. 异步会导致ssh提前关闭，无法正常执行`touch -a`， 会一直报错 `unable to exec` ![image](https://github.com/PKUHPC/SCOW/assets/43978285/5f680c39-f144-4115-b5a5-83b896080a03) 2. 文件切分时切分的是字符串而不是文件数组，导致执行路径出现错误 **针对文件名长度可能引发的错误问题下调每次执行文件数量安全值：** 按一般系统最大命令字节数2097152字节，文件名最长4096字节，下调每一次touch -a执行的文件数量为安全值500 --- .changeset/great-jeans-obey.md | 5 ++++ .changeset/late-islands-mate.md | 6 ++++ apps/portal-server/src/services/file.ts | 28 ++++++++++++------- .../filemanager/ClusterFileTable.tsx | 2 +- apps/portal-web/src/pages/api/file/list.ts | 5 ++-- protos/portal/file.proto | 4 +++ 6 files changed, 37 insertions(+), 13 deletions(-) create mode 100644 .changeset/great-jeans-obey.md create mode 100644 .changeset/late-islands-mate.md diff --git a/.changeset/great-jeans-obey.md b/.changeset/great-jeans-obey.md new file mode 100644 index 0000000000..b22f87ae33 --- /dev/null +++ b/.changeset/great-jeans-obey.md @@ -0,0 +1,5 @@ +--- +"@scow/grpc-api": patch +--- + +在文件管理的 readDirectory 接口下增加可选参数 updateAccessTime，只在文件传输功能时更新时间戳 diff --git a/.changeset/late-islands-mate.md b/.changeset/late-islands-mate.md new file mode 100644 index 0000000000..265ac20480 --- /dev/null +++ b/.changeset/late-islands-mate.md @@ -0,0 +1,6 @@ +--- +"@scow/portal-server": patch +"@scow/portal-web": patch +--- + +修复只需在文件传输时使用 touch -a 来更新时间戳，修复 touch -a 执行时 ssh 关闭报错，文件名特殊字符报错等问题 diff --git a/apps/portal-server/src/services/file.ts b/apps/portal-server/src/services/file.ts index 37b90ed89d..287c7e0225 100644 --- a/apps/portal-server/src/services/file.ts +++ b/apps/portal-server/src/services/file.ts @@ -160,7 +160,7 @@ export const fileServiceServer = plugin((server) => { }, readDirectory: async ({ request, logger }) => { - const { userId, cluster, path } = request; + const { userId, cluster, path, updateAccessTime } = request; const host = getClusterLoginNode(cluster); @@ -188,18 +188,26 @@ export const fileServiceServer = plugin((server) => { // 通过touch -a命令实现共享文件系统的缓存刷新 const pureFiles = files.filter((file) => !file.longname.startsWith("d")); - if (pureFiles.length > 0) { - const filePaths = pureFiles.map((file) => join(path, file.filename)).join(" "); + if (pureFiles.length > 0 && updateAccessTime) { // 避免目录下文件过多导致 touch -a 命令报错，采用分批异步执行的方式 - // 一次执行 1000 个文件是根据经验设置的安全值，可修改 - for (let i = 0; i < filePaths.length; i += 1000) { - const execFilePaths = filePaths.slice(i, i + 1000); - const fileSyncCmd = `touch -a ${execFilePaths}`; - loggedExec(ssh, logger, false, fileSyncCmd, []).catch((err) => { - logger.error(err, "touch -a failed", userId, execFilePaths); - }); + // 一次执行 500 个文件是根据经验设置的安全值，可修改 + // 根据一般系统 getconf ARG_MAX 的值为 2097152 字节，linux 下带有文件路径的文件名最长 4096 字节 设置安全值为500 + const TOUCH_FILES_COUNT = 500; + const execFilePathsList: string[][] = []; + + for (let i = 0; i < pureFiles.length; i += TOUCH_FILES_COUNT) { + const slicedExecFiles = pureFiles.slice(i, i + TOUCH_FILES_COUNT); + const slicedExecFilesPaths = slicedExecFiles.map((file) => join(path, file.filename)); + execFilePathsList.push(slicedExecFilesPaths); } + + await Promise.allSettled(execFilePathsList.map(async (execFilePaths) => { + return loggedExec(ssh, logger, false, "touch -a", execFilePaths).catch((err) => { + logger.error(err, "touch -a %s failed as %s", execFilePaths, userId); + }); + })); + } for (const file of files) { diff --git a/apps/portal-web/src/pageComponents/filemanager/ClusterFileTable.tsx b/apps/portal-web/src/pageComponents/filemanager/ClusterFileTable.tsx index 7a79ac70c4..190a268e7d 100644 --- a/apps/portal-web/src/pageComponents/filemanager/ClusterFileTable.tsx +++ b/apps/portal-web/src/pageComponents/filemanager/ClusterFileTable.tsx @@ -72,7 +72,7 @@ export const ClusterFileTable: React.FC = ({ // 清空SelectedKeys setSelectedKeys([]); selectedCluster ? ( - await api.listFile({ query: { cluster: selectedCluster.id, path: path } }) + await api.listFile({ query: { cluster: selectedCluster.id, path: path, updateAccessTime: true } }) .then((d) => { setFiles(d.items); }) diff --git a/apps/portal-web/src/pages/api/file/list.ts b/apps/portal-web/src/pages/api/file/list.ts index fc4ee0c8d7..2257375d20 100644 --- a/apps/portal-web/src/pages/api/file/list.ts +++ b/apps/portal-web/src/pages/api/file/list.ts @@ -42,6 +42,7 @@ export const ListFileSchema = typeboxRouteSchema({ query: Type.Object({ cluster: Type.String(), path: Type.String(), + updateAccessTime: Type.Optional(Type.Boolean()), }), responses: { @@ -66,12 +67,12 @@ export default route(ListFileSchema, async (req, res) => { if (!info) { return; } - const { cluster, path } = req.query; + const { cluster, path, updateAccessTime } = req.query; const client = getClient(FileServiceClient); return asyncUnaryCall(client, "readDirectory", { - cluster, userId: info.identityId, path, + cluster, userId: info.identityId, path, updateAccessTime, }).then(({ results }) => ({ 200: { items: results.map(({ mode, mtime, name, size, type }) => ({ mode, mtime, name, size, type: mapType[type], diff --git a/protos/portal/file.proto b/protos/portal/file.proto index 2a7d1b4127..b27d5c3c83 100644 --- a/protos/portal/file.proto +++ b/protos/portal/file.proto @@ -81,6 +81,10 @@ message ReadDirectoryRequest { string user_id = 1; string cluster = 2; string path = 3; + // update_access_time 用于指示是否需要更新目录下全部文件的访问时间戳，主要在 SCOW 文件传输功能中使用 + // 为 true 时表示需要更新文件的访问时间戳，从而确保文件的访问时间与传输时间同步 + // 为 false 时表示不需要更新文件的访问时间戳 + optional bool update_access_time = 4; } message FileInfo {