perf: http body;perf: create by json;perf: create by curl (#3570)

* perf: http body * feat: create app by json (#3557) * feat: create app by json * fix build * perf: create by json;perf: create by curl * fix: ts --------- Co-authored-by: heheer <[email protected]>
labring · Jan 12, 2025 · d0d1a2c · d0d1a2c
1 parent f1f0ae2
commit d0d1a2c
Show file tree

Hide file tree

Showing 34 changed files with 1,276 additions and 516 deletions.
diff --git a/docSite/content/zh-cn/docs/agreement/open-source.md b/docSite/content/zh-cn/docs/agreement/open-source.md
@@ -7,7 +7,7 @@ toc: true
 weight: 1210
 ---
 
-FastGPT 项目在 Apache License 2.0 许可下开源，同时包含以下附加条件：
+FastGPT 项目在 Apache License 2.0 许可下开源，但包含以下附加条件：
 
 + FastGPT 允许被用于商业化，例如作为其他应用的“后端即服务”使用，或者作为应用开发平台提供给企业。然而，当满足以下条件时，必须联系作者获得商业许可：
 

diff --git a/docSite/content/zh-cn/docs/development/upgrading/4818.md b/docSite/content/zh-cn/docs/development/upgrading/4818.md
@@ -24,14 +24,17 @@ curl --location --request POST 'https://{{host}}/api/admin/initv4818' \
 
 ## 完整更新内容
 
-1. 新增 - 支持部门架构权限模式。
-2. 新增 - 支持配置自定跨域安全策略，默认全开。
-3. 优化 - 分享链接随机生成用户头像。
-4. 优化 - 图片上传安全校验。并增加头像图片唯一存储，确保不会累计存储。
-5. 优化 - Mongo 全文索引表分离。
-6. 优化 - 知识库检索查询语句合并，同时减少查库数量。
-7. 优化 - 文件编码检测，减少 CSV 文件乱码概率。
-8. 优化 - 异步读取文件内容，减少进程阻塞。
-9. 优化 - 文件阅读，HTML 直接下载，不允许在线阅读。
-10. 修复 - HTML 文件上传，base64 图片无法自动转图片链接。
-11. 修复 - 插件计费错误。
+1. 新增 - 支持通过 JSON 配置直接创建应用。
+2. 新增 - 支持通过 CURL 脚本快速创建 HTTP 插件。
+3. 新增 - 商业版支持部门架构权限模式。
+4. 新增 - 支持配置自定跨域安全策略，默认全开。
+5. 优化 - HTTP Body 增加特殊处理，解决字符串变量带换行时无法解析问题。
+6. 优化 - 分享链接随机生成用户头像。
+7. 优化 - 图片上传安全校验。并增加头像图片唯一存储，确保不会累计存储。
+8. 优化 - Mongo 全文索引表分离。
+9. 优化 - 知识库检索查询语句合并，同时减少查库数量。
+10. 优化 - 文件编码检测，减少 CSV 文件乱码概率。
+11. 优化 - 异步读取文件内容，减少进程阻塞。
+12. 优化 - 文件阅读，HTML 直接下载，不允许在线阅读。
+13. 修复 - HTML 文件上传，base64 图片无法自动转图片链接。
+14. 修复 - 插件计费错误。
diff --git a/docSite/content/zh-cn/docs/intro.md b/docSite/content/zh-cn/docs/intro.md
@@ -54,7 +54,7 @@ FastGPT 对外的 API 接口对齐了 OpenAI 官方接口，可以直接接入
 
 1. **项目开源**
 
-   FastGPT 遵循附加条件 Apache License 2.0 开源协议，你可以 [Fork](https://github.com/labring/FastGPT/fork) 之后进行二次开发和发布。FastGPT 社区版将保留核心功能，商业版仅在社区版基础上使用 API 的形式进行扩展，不影响学习使用。
+   FastGPT 遵循**附加条件 Apache License 2.0 开源协议**，你可以 [Fork](https://github.com/labring/FastGPT/fork) 之后进行二次开发和发布。FastGPT 社区版将保留核心功能，商业版仅在社区版基础上使用 API 的形式进行扩展，不影响学习使用。
 
 2. **独特的 QA 结构**
 

diff --git a/packages/global/common/error/utils.ts b/packages/global/common/error/utils.ts
@@ -5,6 +5,6 @@ export const getErrText = (err: any, def = ''): any => {
     typeof err === 'string'
       ? err
       : err?.response?.data?.message || err?.response?.message || err?.message || def;
-  msg && console.log('error =>', msg);
+  // msg && console.log('error =>', msg);
   return replaceSensitiveText(msg);
 };
diff --git a/packages/global/common/string/http.ts b/packages/global/common/string/http.ts
@@ -0,0 +1,38 @@
+import parse from '@bany/curl-to-json';
+
+type RequestMethod = 'get' | 'post' | 'put' | 'delete' | 'patch';
+const methodMap: { [K in RequestMethod]: string } = {
+  get: 'GET',
+  post: 'POST',
+  put: 'PUT',
+  delete: 'DELETE',
+  patch: 'PATCH'
+};
+
+export const parseCurl = (curlContent: string) => {
+  const parsed = parse(curlContent);
+
+  if (!parsed.url) {
+    throw new Error('url not found');
+  }
+
+  const newParams = Object.keys(parsed.params || {}).map((key) => ({
+    key,
+    value: parsed.params?.[key],
+    type: 'string'
+  }));
+  const newHeaders = Object.keys(parsed.header || {}).map((key) => ({
+    key,
+    value: parsed.header?.[key],
+    type: 'string'
+  }));
+  const newBody = JSON.stringify(parsed.data, null, 2);
+
+  return {
+    url: parsed.url,
+    method: methodMap[parsed.method?.toLowerCase() as RequestMethod] || 'GET',
+    params: newParams,
+    headers: newHeaders,
+    body: newBody
+  };
+};
diff --git a/packages/global/core/app/utils.ts b/packages/global/core/app/utils.ts
@@ -5,6 +5,8 @@ import type { FlowNodeInputItemType } from '../workflow/type/io.d';
 import { getAppChatConfig } from '../workflow/utils';
 import { StoreNodeItemType } from '../workflow/type/node';
 import { DatasetSearchModeEnum } from '../dataset/constants';
+import { WorkflowTemplateBasicType } from '../workflow/type';
+import { AppTypeEnum } from './constants';
 
 export const getDefaultAppForm = (): AppSimpleEditFormType => {
   return {
@@ -127,3 +129,20 @@ export const appWorkflow2Form = ({
 
   return defaultAppForm;
 };
+
+export const getAppType = (config?: WorkflowTemplateBasicType | AppSimpleEditFormType) => {
+  if (!config) return '';
+
+  if ('aiSettings' in config) {
+    return AppTypeEnum.simple;
+  }
+
+  if (!('nodes' in config)) return '';
+  if (config.nodes.some((node) => node.flowNodeType === 'workflowStart')) {
+    return AppTypeEnum.workflow;
+  }
+  if (config.nodes.some((node) => node.flowNodeType === 'pluginInput')) {
+    return AppTypeEnum.plugin;
+  }
+  return '';
+};
diff --git a/packages/global/package.json b/packages/global/package.json
@@ -14,7 +14,8 @@
     "openai": "4.61.0",
     "openapi-types": "^12.1.3",
     "json5": "^2.2.3",
-    "timezones-list": "^3.0.2"
+    "timezones-list": "^3.0.2",
+    "@bany/curl-to-json": "^1.2.8"
   },
   "devDependencies": {
     "@types/js-yaml": "^4.0.9",

diff --git a/packages/service/core/workflow/dispatch/index.ts b/packages/service/core/workflow/dispatch/index.ts
@@ -487,16 +487,16 @@ export async function dispatchWorkFlow(data: Props): Promise<DispatchFlowRespons
       if (input.key === dynamicInput?.key) return;
 
       // Skip some special key
-      if (input.key === NodeInputKeyEnum.childrenNodeIdList) {
+      if (
+        [NodeInputKeyEnum.childrenNodeIdList, NodeInputKeyEnum.httpJsonBody].includes(
+          input.key as any
+        )
+      ) {
         params[input.key] = input.value;
-
         return;
       }
 
-      // replace {{xx}} variables
-      // let value = replaceVariable(input.value, variables);
-
-      // replace {{$xx.xx$}} variables
+      // replace {{$xx.xx$}} and {{xx}} variables
       let value = replaceEditorVariable({
         text: input.value,
         nodes: runtimeNodes,
@@ -606,6 +606,11 @@ export async function dispatchWorkFlow(data: Props): Promise<DispatchFlowRespons
       };
     }
 
+    // Error
+    if (dispatchRes?.responseData?.error) {
+      addLog.warn('workflow error', dispatchRes.responseData.error);
+    }
+
     return {
       node,
       runStatus: 'run',

diff --git a/packages/service/core/workflow/dispatch/tools/http468.ts b/packages/service/core/workflow/dispatch/tools/http468.ts
@@ -2,6 +2,7 @@ import type { ModuleDispatchProps } from '@fastgpt/global/core/workflow/runtime/
 import {
   NodeInputKeyEnum,
   NodeOutputKeyEnum,
+  VARIABLE_NODE_ID,
   WorkflowIOValueTypeEnum
 } from '@fastgpt/global/core/workflow/constants';
 import {
@@ -16,7 +17,9 @@ import { DispatchNodeResultType } from '@fastgpt/global/core/workflow/runtime/ty
 import { getErrText } from '@fastgpt/global/common/error/utils';
 import {
   textAdaptGptResponse,
-  replaceEditorVariable
+  replaceEditorVariable,
+  formatVariableValByType,
+  getReferenceVariableValue
 } from '@fastgpt/global/core/workflow/runtime/utils';
 import { ContentTypes } from '@fastgpt/global/core/workflow/constants';
 import { uploadFileFromBase64Img } from '../../../../common/file/gridfs/controller';
@@ -104,15 +107,73 @@ export const dispatchHttp468Request = async (props: HttpRequestProps): Promise<H
     [NodeInputKeyEnum.addInputParam]: concatVariables,
     ...concatVariables
   };
+
+  // General data for variable substitution（Exclude: json body)
   const replaceStringVariables = (text: string) => {
-    return replaceVariable(
-      replaceEditorVariable({
-        text,
-        nodes: runtimeNodes,
-        variables: allVariables
-      }),
-      allVariables
-    );
+    return replaceEditorVariable({
+      text,
+      nodes: runtimeNodes,
+      variables: allVariables
+    });
+  };
+  /* 特殊处理 JSON 的字符串，减少解码错误
+    1. 找不到的值，替换成 null
+    2. 有换行字符串
+  */
+  const replaceJsonBodyString = (text: string) => {
+    const valToStr = (val: any) => {
+      if (val === undefined) return 'null';
+      if (val === null) return 'null';
+
+      if (typeof val === 'object') return JSON.stringify(val);
+
+      if (typeof val === 'string') {
+        const str = JSON.stringify(val);
+        return str.startsWith('"') && str.endsWith('"') ? str.slice(1, -1) : str;
+      }
+
+      return String(val);
+    };
+
+    // 1. Replace {{key}} variables
+    const regex1 = /{{([^}]+)}}/g;
+    const matches1 = text.match(regex1) || [];
+    const uniqueKeys1 = [...new Set(matches1.map((match) => match.slice(2, -2)))];
+    for (const key of uniqueKeys1) {
+      text = text.replace(new RegExp(`{{(${key})}}`, 'g'), () => valToStr(variables[key]));
+    }
+
+    // 2. Replace {{key.key}} variables
+    const regex2 = /\{\{\$([^.]+)\.([^$]+)\$\}\}/g;
+    const matches2 = [...text.matchAll(regex2)];
+    if (matches2.length === 0) return text;
+    matches2.forEach((match) => {
+      const nodeId = match[1];
+      const id = match[2];
+
+      const variableVal = (() => {
+        if (nodeId === VARIABLE_NODE_ID) {
+          return variables[id];
+        }
+        // Find upstream node input/output
+        const node = runtimeNodes.find((node) => node.nodeId === nodeId);
+        if (!node) return;
+
+        const output = node.outputs.find((output) => output.id === id);
+        if (output) return formatVariableValByType(output.value, output.valueType);
+
+        const input = node.inputs.find((input) => input.key === id);
+        if (input)
+          return getReferenceVariableValue({ value: input.value, nodes: runtimeNodes, variables });
+      })();
+
+      const formatVal = valToStr(variableVal);
+
+      const regex = new RegExp(`\\{\\{\\$(${nodeId}\\.${id})\\$\\}\\}`, 'g');
+      text = text.replace(regex, () => formatVal);
+    });
+
+    return text.replace(/(".*?")\s*:\s*undefined\b/g, '$1: null');
   };
 
   httpReqUrl = replaceStringVariables(httpReqUrl);
@@ -176,15 +237,10 @@ export const dispatchHttp468Request = async (props: HttpRequestProps): Promise<H
       }
       if (!httpJsonBody) return {};
       if (httpContentType === ContentTypes.json) {
-        httpJsonBody = replaceStringVariables(httpJsonBody);
-
-        const replaceJsonBody = httpJsonBody.replace(/(".*?")\s*:\s*undefined\b/g, '$1: null');
-
-        // Json body, parse and return
-        const jsonParse = json5.parse(replaceJsonBody);
-        const removeSignJson = removeUndefinedSign(jsonParse);
-        return removeSignJson;
+        return json5.parse(replaceJsonBodyString(httpJsonBody));
       }
+
+      // Raw text, xml
       httpJsonBody = replaceStringVariables(httpJsonBody);
       return httpJsonBody.replaceAll(UNDEFINED_SIGN, 'null');
     } catch (error) {
@@ -335,40 +391,40 @@ async function fetchData({
   };
 }
 
-function replaceVariable(text: string, obj: Record<string, any>) {
-  for (const [key, value] of Object.entries(obj)) {
-    if (value === undefined) {
-      text = text.replace(new RegExp(`{{(${key})}}`, 'g'), UNDEFINED_SIGN);
-    } else {
-      const replacement = JSON.stringify(value);
-      const unquotedReplacement =
-        replacement.startsWith('"') && replacement.endsWith('"')
-          ? replacement.slice(1, -1)
-          : replacement;
-      text = text.replace(new RegExp(`{{(${key})}}`, 'g'), () => unquotedReplacement);
-    }
-  }
-  return text || '';
-}
-function removeUndefinedSign(obj: Record<string, any>) {
-  for (const key in obj) {
-    if (obj[key] === UNDEFINED_SIGN) {
-      obj[key] = undefined;
-    } else if (Array.isArray(obj[key])) {
-      obj[key] = obj[key].map((item: any) => {
-        if (item === UNDEFINED_SIGN) {
-          return undefined;
-        } else if (typeof item === 'object') {
-          removeUndefinedSign(item);
-        }
-        return item;
-      });
-    } else if (typeof obj[key] === 'object') {
-      removeUndefinedSign(obj[key]);
-    }
-  }
-  return obj;
-}
+// function replaceVariable(text: string, obj: Record<string, any>) {
+//   for (const [key, value] of Object.entries(obj)) {
+//     if (value === undefined) {
+//       text = text.replace(new RegExp(`{{(${key})}}`, 'g'), UNDEFINED_SIGN);
+//     } else {
+//       const replacement = JSON.stringify(value);
+//       const unquotedReplacement =
+//         replacement.startsWith('"') && replacement.endsWith('"')
+//           ? replacement.slice(1, -1)
+//           : replacement;
+//       text = text.replace(new RegExp(`{{(${key})}}`, 'g'), () => unquotedReplacement);
+//     }
+//   }
+//   return text || '';
+// }
+// function removeUndefinedSign(obj: Record<string, any>) {
+//   for (const key in obj) {
+//     if (obj[key] === UNDEFINED_SIGN) {
+//       obj[key] = undefined;
+//     } else if (Array.isArray(obj[key])) {
+//       obj[key] = obj[key].map((item: any) => {
+//         if (item === UNDEFINED_SIGN) {
+//           return undefined;
+//         } else if (typeof item === 'object') {
+//           removeUndefinedSign(item);
+//         }
+//         return item;
+//       });
+//     } else if (typeof obj[key] === 'object') {
+//       removeUndefinedSign(obj[key]);
+//     }
+//   }
+//   return obj;
+// }
 
 // Replace some special response from system plugin
 async function replaceSystemPluginResponse({

diff --git a/packages/web/components/common/Icon/constants.ts b/packages/web/components/common/Icon/constants.ts
@@ -142,6 +142,7 @@ export const iconPaths = {
   'core/app/ttsFill': () => import('./icons/core/app/ttsFill.svg'),
   'core/app/type/httpPlugin': () => import('./icons/core/app/type/httpPlugin.svg'),
   'core/app/type/httpPluginFill': () => import('./icons/core/app/type/httpPluginFill.svg'),
+  'core/app/type/jsonImport': () => import('./icons/core/app/type/jsonImport.svg'),
   'core/app/type/plugin': () => import('./icons/core/app/type/plugin.svg'),
   'core/app/type/pluginFill': () => import('./icons/core/app/type/pluginFill.svg'),
   'core/app/type/pluginLight': () => import('./icons/core/app/type/pluginLight.svg'),

diff --git a/packages/web/components/common/Icon/icons/core/app/type/jsonImport.svg b/packages/web/components/common/Icon/icons/core/app/type/jsonImport.svg