fix: fix import issue (#522)

vesoft-inc · Apr 4, 2023 · 302af55 · 302af55
1 parent 7b5be9b
commit 302af55
Show file tree

Hide file tree

Showing 15 changed files with 35 additions and 16 deletions.
diff --git a/app/config/locale/en-US.json b/app/config/locale/en-US.json
@@ -220,7 +220,7 @@
     "importYaml": "Import the YAML file",
     "templateMatchError": "The user in the configuration does not match the current login account",
     "uploadSuccessfully": "Upload files successfully.",
-    "fileSizeLimit": "The file is too large and exceeds the upload limit({size}), please upload the file to the data/upload directory under the installation directory via scp",
+    "fileSizeLimit": "{name} is too large and exceeds the upload limit({size}), please upload the file to the data/upload directory under the installation directory via scp",
     "noHttp": "The address in the configuration file does not support http protocol, please remove http(s)://",
     "addressMatch": "The address in the configuration file must contain the Graph address of current login connection. Separate multiple addresses with ",
     "dataSourceFile": "Data source file",

diff --git a/app/config/locale/zh-CN.json b/app/config/locale/zh-CN.json
@@ -220,7 +220,7 @@
     "importYaml": "导入 YAML 文件",
     "templateMatchError": "配置中的用户姓名与当前登录账号不一致",
     "uploadSuccessfully": "上传文件成功",
-    "fileSizeLimit": "文件过大，超过上传限制({size})，请将文件通过 scp 的方式上传到安装目录下的 data/upload 目录",
+    "fileSizeLimit": "{name}文件过大，超过上传限制({size})，请将文件通过 scp 的方式上传到安装目录下的 data/upload 目录",
     "noHttp": "配置文件中的 address 不支持携带 http 协议，请去除 http(s)://",
     "addressMatch": "配置文件中的 address 字段必须包含当前登录的 Graph 地址。多个地址用“,”隔开。",
     "dataSourceFile": "文件源",

diff --git a/app/pages/Import/DatasourceList/DatasourceConfig/FileUploadBtn/index.tsx b/app/pages/Import/DatasourceList/DatasourceConfig/FileUploadBtn/index.tsx
@@ -21,9 +21,9 @@ const UploadBtn = (props: IUploadBtnProps) => {
   const { fileList, uploadFile } = files;
   const [visible, setVisible] = useState(false);
   const transformFile = async (_file: StudioFile, fileList: StudioFile[]) => {
-    const size = fileList.reduce((acc, cur) => acc + cur.size, 0);
-    if(size > SizeLimit) {
-      message.error(intl.get('import.fileSizeLimit', { size: getFileSize(SizeLimit) }));
+    const bigFiles = fileList.filter(file => file.size > SizeLimit);
+    if(bigFiles.length > 0) {
+      message.error(intl.get('import.fileSizeLimit', { name: bigFiles.map(i => i.name).join(', '), size: getFileSize(SizeLimit) }));
       return false;
     }
     fileList.forEach(file => {

diff --git a/app/pages/Import/TaskCreate/SchemaConfig/FileMapping/FileSelect.tsx b/app/pages/Import/TaskCreate/SchemaConfig/FileMapping/FileSelect.tsx
@@ -93,8 +93,9 @@ const FileSelect = observer((props: IFileSelect) => {
     }
   }, []);
   const handleRefresh = useCallback(async () => {
+    if (!activeId) return;
     setState({ loading: true });
-    if (!activeId || activeId === IDatasourceType.Local) {
+    if (activeId === IDatasourceType.Local) {
       getLocalFiles();
       return;
     }

diff --git a/app/pages/Import/TaskCreate/SchemaConfig/index.module.less b/app/pages/Import/TaskCreate/SchemaConfig/index.module.less
@@ -152,13 +152,15 @@
     }
     .concatItems {
       background: #F3F6F9;
-      padding: 8px;
+      padding: 8px 8px 0;
       flex: 1;
       .tagItem {
         background: #8697B0;
         border-radius: 3px;
         padding: 6px;
         color: #fff;
+        margin-bottom: 10px;
+        display: inline-block;
         &:not(:last-child) {
           margin-right: 10px;
         }

diff --git a/app/pages/Import/TaskList/TemplateModal/index.tsx b/app/pages/Import/TaskList/TemplateModal/index.tsx
@@ -176,7 +176,7 @@ const TemplateModal = (props: IProps) => {
 
   const handleImport = async (values) => {
     const code = await importTask({
-      config: JSON.parse(values.content), 
+      config: values.content, 
       name: values.name
     });
     if(code === 0) {

diff --git a/app/utils/import.ts b/app/utils/import.ts
@@ -53,7 +53,7 @@ export function configToJson(payload: IConfig) {
     },
     sources
   };
-  return configJson;
+  return JSON.stringify(configJson);
 }
 
 const getIdConfig = (payload: {
@@ -69,7 +69,9 @@ const getIdConfig = (payload: {
     function: vidFunction,
   } as any;
   if(indexes.length > 1 || !!prefix || !!suffix) {
-    id.concatItems = [prefix, ...indexes, suffix];
+    id.concatItems = [...indexes];
+    prefix && id.concatItems.unshift(prefix);
+    suffix && id.concatItems.push(suffix);
   } else {
     id.index = indexes[0];
   }

diff --git a/server/api/studio/internal/service/datasource.go b/server/api/studio/internal/service/datasource.go
@@ -208,6 +208,7 @@ func (d *datasourceService) ListContents(request types.DatasourceListContentsReq
 		return nil, err
 	}
 	fileList, err := store.ListFiles(request.Path)
+	defer store.Close()
 	list := make([]types.FileConfig, 0)
 	for _, item := range fileList {
 		list = append(list, types.FileConfig{
@@ -235,6 +236,7 @@ func (d *datasourceService) PreviewFile(request types.DatasourcePreviewFileReque
 	}
 	// read three lines
 	contents, err := store.ReadFile(request.Path, 0, 4)
+	defer store.Close()
 	if err != nil {
 		return nil, ecode.WithErrorMessage(ecode.ErrBadRequest, err, "readFiles failed")
 	}

diff --git a/server/api/studio/internal/service/import.go b/server/api/studio/internal/service/import.go
@@ -68,7 +68,11 @@ func NewImportService(ctx context.Context, svcCtx *svc.ServiceContext) ImportSer
 }
 
 func (i *importService) updateDatasourceConfig(conf *types.CreateImportTaskRequest) (*types.ImportTaskConfig, error) {
-	config := conf.Config
+	// config := conf.Config
+	var config types.ImportTaskConfig
+	if err := json.Unmarshal([]byte(conf.Config), &config); err != nil {
+		return nil, ecode.WithErrorMessage(ecode.ErrBadRequest, err)
+	}
 	for _, source := range config.Sources {
 		if source.DatasourceId != nil {
 			var dbs db.Datasource

diff --git a/server/api/studio/internal/types/types.go b/server/api/studio/internal/types/types.go
diff --git a/server/api/studio/pkg/filestore/filestore.go b/server/api/studio/pkg/filestore/filestore.go
@@ -9,6 +9,7 @@ type (
 	FileStore interface {
 		ReadFile(path string, startLine ...int) ([]string, error)
 		ListFiles(dir string) ([]FileConfig, error)
+		Close() error
 	}
 
 	FileConfig struct {

diff --git a/server/api/studio/pkg/filestore/s3store.go b/server/api/studio/pkg/filestore/s3store.go
@@ -156,3 +156,7 @@ func (s *S3Store) ListBuckets() ([]string, error) {
 
 	return buckets, nil
 }
+
+func (s *S3Store) Close() error {
+	return nil
+}
diff --git a/server/api/studio/pkg/filestore/sftpstore.go b/server/api/studio/pkg/filestore/sftpstore.go
@@ -125,3 +125,7 @@ func (s *SftpStore) ListFiles(dir string) ([]FileConfig, error) {
 	}
 	return files, nil
 }
+
+func (s *SftpStore) Close() error {
+	return s.SftpClient.Close()
+}
diff --git a/server/api/studio/pkg/utils/datasourceParse.go b/server/api/studio/pkg/utils/datasourceParse.go
@@ -22,7 +22,6 @@ func ParseEndpoint(platform, rawEndpoint string) (string, string, error) {
 	}
 	host := u.Hostname()
 	parts := strings.SplitN(host, ".", 2)
-	fmt.Println("host", host, parts)
 	var (
 		bucket   string
 		endpoint string

diff --git a/server/api/studio/restapi/import.api b/server/api/studio/restapi/import.api
@@ -132,8 +132,8 @@ type (
 	}
 
 	CreateImportTaskRequest {
-		Name   string           `json:"name" validate:"required"`
-		Config ImportTaskConfig `json:"config" validate:"required"`
+		Name   string `json:"name" validate:"required"`
+		Config string `json:"config" validate:"required"`
 	}
 
 	CreateImportTaskData {