perf: http body parse (#3357)

* perf: http body parse

* perf: http body parse

docSite/content/zh-cn/docs/development/upgrading/4815.md

@@ -41,9 +41,10 @@ weight: 809
 9. 优化 - 支持 Markdown 文本分割时，只有标题，无内容。
 10. 优化 - 字符串变量替换，未赋值的变量会转成 undefined，而不是保留原来 id 串。
 11. 优化 - 全局变量默认值在 API 生效，并且自定义变量支持默认值。
-12. 修复 - 分享链接点赞鉴权问题。
-13. 修复 - 对话页面切换自动执行应用时，会误触发非自动执行应用。
-14. 修复 - 语言播放鉴权问题。
-15. 修复 - 插件应用知识库引用上限始终为 3000
-16. 修复 - 工作流编辑记录存储上限，去掉本地存储，增加异常离开时，强制自动保存。
-17. 修复 - 工作流特殊变量替换问题。（$开头的字符串无法替换）
+12. 优化 - 增加 HTTP Body 的 JSON 解析，正则将 undefined 转 null，减少 Body 解析错误。
+13. 修复 - 分享链接点赞鉴权问题。
+14. 修复 - 对话页面切换自动执行应用时，会误触发非自动执行应用。
+15. 修复 - 语言播放鉴权问题。
+16. 修复 - 插件应用知识库引用上限始终为 3000
+17. 修复 - 工作流编辑记录存储上限，去掉本地存储，增加异常离开时，强制自动保存。
+18. 修复 - 工作流特殊变量替换问题。（$开头的字符串无法替换）

packages/global/core/workflow/runtime/utils.ts

@@ -1,5 +1,5 @@
 import { ChatCompletionRequestMessageRoleEnum } from '../../ai/constants';
-import { NodeInputKeyEnum, NodeOutputKeyEnum } from '../constants';
+import { NodeInputKeyEnum, NodeOutputKeyEnum, WorkflowIOValueTypeEnum } from '../constants';
 import { FlowNodeTypeEnum } from '../node/constant';
 import { StoreNodeItemType } from '../type/node';
 import { StoreEdgeItemType } from '../type/edge';
@@ -279,6 +279,27 @@ export const getReferenceVariableValue = ({
   return value;
 };
 
+export const formatVariableValByType = (val: any, valueType?: WorkflowIOValueTypeEnum) => {
+  if (!valueType) return val;
+  // Value type check, If valueType invalid, return undefined
+  if (valueType.startsWith('array') && !Array.isArray(val)) return undefined;
+  if (valueType === WorkflowIOValueTypeEnum.boolean && typeof val !== 'boolean') return undefined;
+  if (valueType === WorkflowIOValueTypeEnum.number && typeof val !== 'number') return undefined;
+  if (valueType === WorkflowIOValueTypeEnum.string && typeof val !== 'string') return undefined;
+  if (
+    [
+      WorkflowIOValueTypeEnum.object,
+      WorkflowIOValueTypeEnum.chatHistory,
+      WorkflowIOValueTypeEnum.datasetQuote,
+      WorkflowIOValueTypeEnum.selectApp,
+      WorkflowIOValueTypeEnum.selectDataset
+    ].includes(valueType) &&
+    typeof val !== 'object'
+  )
+    return undefined;
+
+  return val;
+};
 // replace {{$xx.xx$}} variables for text
 export function replaceEditorVariable({
   text,
@@ -308,7 +329,7 @@ export function replaceEditorVariable({
       if (!node) return;
 
       const output = node.outputs.find((output) => output.id === id);
-      if (output) return output.value;
+      if (output) return formatVariableValByType(output.value, output.valueType);
 
       const input = node.inputs.find((input) => input.key === id);
       if (input) return getReferenceVariableValue({ value: input.value, nodes, variables });

packages/service/core/workflow/dispatch/index.ts

@@ -42,7 +42,8 @@ import {
   filterWorkflowEdges,
   checkNodeRunStatus,
   textAdaptGptResponse,
-  replaceEditorVariable
+  replaceEditorVariable,
+  formatVariableValByType
 } from '@fastgpt/global/core/workflow/runtime/utils';
 import { ChatNodeUsageType } from '@fastgpt/global/support/wallet/bill/type';
 import { dispatchRunTools } from './agent/runTool/index';
@@ -72,6 +73,7 @@ import { dispatchLoopEnd } from './loop/runLoopEnd';
 import { dispatchLoopStart } from './loop/runLoopStart';
 import { dispatchFormInput } from './interactive/formInput';
 import { dispatchToolParams } from './agent/runTool/toolParams';
+import { AppChatConfigType } from '@fastgpt/global/core/app/type';
 
 const callbackMap: Record<FlowNodeTypeEnum, Function> = {
   [FlowNodeTypeEnum.workflowStart]: dispatchWorkflowStart,
@@ -685,15 +687,15 @@ export async function dispatchWorkFlow(data: Props): Promise<DispatchFlowRespons
 }
 
 /* get system variable */
-export function getSystemVariable({
+const getSystemVariable = ({
   user,
   runningAppInfo,
   chatId,
   responseChatItemId,
   histories = [],
   uid,
   chatConfig
-}: Props): SystemVariablesType {
+}: Props): SystemVariablesType => {
   const variables = chatConfig?.variables || [];
   const variablesMap = variables.reduce<Record<string, any>>((acc, item) => {
     acc[item.key] = valueTypeFormat(item.defaultValue, item.valueType);
@@ -709,10 +711,10 @@ export function getSystemVariable({
     histories,
     cTime: getSystemTime(user.timezone)
   };
-}
+};
 
 /* Merge consecutive text messages into one */
-export const mergeAssistantResponseAnswerText = (response: AIChatItemValueItemType[]) => {
+const mergeAssistantResponseAnswerText = (response: AIChatItemValueItemType[]) => {
   const result: AIChatItemValueItemType[] = [];
   // 合并连续的text
   for (let i = 0; i < response.length; i++) {

packages/service/core/workflow/dispatch/tools/http468.ts

@@ -24,6 +24,7 @@ import { ReadFileBaseUrl } from '@fastgpt/global/common/file/constants';
 import { createFileToken } from '../../../../support/permission/controller';
 import { JSONPath } from 'jsonpath-plus';
 import type { SystemPluginSpecialResponse } from '../../../../../plugins/type';
+import json5 from 'json5';
 
 type PropsArrType = {
   key: string;
@@ -103,8 +104,6 @@ export const dispatchHttp468Request = async (props: HttpRequestProps): Promise<H
     [NodeInputKeyEnum.addInputParam]: concatVariables,
     ...concatVariables
   };
-  httpReqUrl = replaceVariable(httpReqUrl, allVariables);
-
   const replaceStringVariables = (text: string) => {
     return replaceVariable(
       replaceEditorVariable({
@@ -116,6 +115,8 @@ export const dispatchHttp468Request = async (props: HttpRequestProps): Promise<H
     );
   };
 
+  httpReqUrl = replaceStringVariables(httpReqUrl);
+
   // parse header
   const headers = await (() => {
     try {
@@ -175,9 +176,11 @@ export const dispatchHttp468Request = async (props: HttpRequestProps): Promise<H
       }
       if (!httpJsonBody) return {};
       if (httpContentType === ContentTypes.json) {
-        httpJsonBody = replaceVariable(httpJsonBody, allVariables);
+        httpJsonBody = replaceStringVariables(httpJsonBody);
         // Json body, parse and return
-        const jsonParse = JSON.parse(httpJsonBody);
+        const jsonParse = json5.parse(
+          httpJsonBody.replace(/(".*?")\s*:\s*undefined\b/g, '$1: null')
+        );
         const removeSignJson = removeUndefinedSign(jsonParse);
         return removeSignJson;
       }
@@ -195,7 +198,7 @@ export const dispatchHttp468Request = async (props: HttpRequestProps): Promise<H
       return Object.fromEntries(requestBody);
     } else if (typeof requestBody === 'string') {
       try {
-        return JSON.parse(requestBody);
+        return json5.parse(requestBody);
       } catch {
         return { content: requestBody };
       }

packages/web/components/common/Icon/constants.ts

@@ -7,6 +7,7 @@ export const iconPaths = {
   check: () => import('./icons/check.svg'),
   checkCircle: () => import('./icons/checkCircle.svg'),
   closeSolid: () => import('./icons/closeSolid.svg'),
+  code: () => import('./icons/code.svg'),
   collectionLight: () => import('./icons/collectionLight.svg'),
   collectionSolid: () => import('./icons/collectionSolid.svg'),
   comment: () => import('./icons/comment.svg'),
@@ -58,8 +59,6 @@ export const iconPaths = {
   'common/monitor': () => import('./icons/common/monitor.svg'),
   'common/more': () => import('./icons/common/more.svg'),
   'common/moreFill': () => import('./icons/common/moreFill.svg'),
-  'common/navbar/pluginFill': () => import('./icons/common/navbar/pluginFill.svg'),
-  'common/navbar/pluginLight': () => import('./icons/common/navbar/pluginLight.svg'),
   'common/openai': () => import('./icons/common/openai.svg'),
   'common/overviewLight': () => import('./icons/common/overviewLight.svg'),
   'common/paramsLight': () => import('./icons/common/paramsLight.svg'),
@@ -94,9 +93,6 @@ export const iconPaths = {
   'common/wechatFill': () => import('./icons/common/wechatFill.svg'),
   configmap: () => import('./icons/configmap.svg'),
   copy: () => import('./icons/copy.svg'),
-  code: () => import('./icons/code.svg'),
-  preview: () => import('./icons/preview.svg'),
-  fullScreen: () => import('./icons/fullScreen.svg'),
   'core/app/aiFill': () => import('./icons/core/app/aiFill.svg'),
   'core/app/aiLight': () => import('./icons/core/app/aiLight.svg'),
   'core/app/aiLightSmall': () => import('./icons/core/app/aiLightSmall.svg'),
@@ -129,6 +125,7 @@ export const iconPaths = {
   'core/app/type/plugin': () => import('./icons/core/app/type/plugin.svg'),
   'core/app/type/pluginFill': () => import('./icons/core/app/type/pluginFill.svg'),
   'core/app/type/simple': () => import('./icons/core/app/type/simple.svg'),
+  'core/app/type/simpleFill': () => import('./icons/core/app/type/simpleFill.svg'),
   'core/app/type/workflow': () => import('./icons/core/app/type/workflow.svg'),
   'core/app/type/workflowFill': () => import('./icons/core/app/type/workflowFill.svg'),
   'core/app/variable/external': () => import('./icons/core/app/variable/external.svg'),
@@ -320,6 +317,7 @@ export const iconPaths = {
   'file/pdf': () => import('./icons/file/pdf.svg'),
   'file/qaImport': () => import('./icons/file/qaImport.svg'),
   'file/uploadFile': () => import('./icons/file/uploadFile.svg'),
+  fullScreen: () => import('./icons/fullScreen.svg'),
   help: () => import('./icons/help.svg'),
   history: () => import('./icons/history.svg'),
   infoRounded: () => import('./icons/infoRounded.svg'),
@@ -345,6 +343,7 @@ export const iconPaths = {
   'plugins/doc2x': () => import('./icons/plugins/doc2x.svg'),
   'plugins/textEditor': () => import('./icons/plugins/textEditor.svg'),
   point: () => import('./icons/point.svg'),
+  preview: () => import('./icons/preview.svg'),
   'price/bg': () => import('./icons/price/bg.svg'),
   'price/right': () => import('./icons/price/right.svg'),
   save: () => import('./icons/save.svg'),

packages/web/i18n/en/common.json

@@ -751,7 +751,6 @@
   "core.module.template.config_params": "Can configure application system parameters",
   "core.module.template.empty_plugin": "Blank plugin",
   "core.module.template.empty_workflow": "Blank workflow",
-  "core.module.template.http body placeholder": "Same syntax as Apifox",
   "core.module.template.self_input": "Plug-in input",
   "core.module.template.self_output": "Custom plug-in output",
   "core.module.template.system_config": "System configuration",

packages/web/i18n/en/workflow.json

@@ -64,6 +64,7 @@
   "full_response_data": "Full Response Data",
   "greater_than": "Greater Than",
   "greater_than_or_equal_to": "Greater Than or Equal To",
+  "http_body_placeholder": "Similar syntax to APIFox, variable selection can be activated via /. \nString variables need to be enclosed in double quotes, other types of variables do not need to be enclosed in double quotes. \nPlease strictly check whether it conforms to JSON format.",
   "http_extract_output": "Output field extraction",
   "http_extract_output_description": "Specified fields in the response value can be extracted through JSONPath syntax",
   "http_raw_response_description": "Raw HTTP response. Only accepts string or JSON type response data.",
@@ -193,4 +194,4 @@
   "workflow.Switch_success": "Switch Successful",
   "workflow.Team cloud": "Team Cloud",
   "workflow.exit_tips": "Your changes have not been saved. 'Exit directly' will not save your edits."
-}
+}

packages/web/i18n/zh-CN/common.json

@@ -750,7 +750,6 @@
   "core.module.template.config_params": "可以配置应用的系统参数",
   "core.module.template.empty_plugin": "空白插件",
   "core.module.template.empty_workflow": "空白工作流",
-  "core.module.template.http body placeholder": "与 Apifox 相同的语法",
   "core.module.template.self_input": "插件输入",
   "core.module.template.self_output": "插件输出",
   "core.module.template.system_config": "系统配置",

packages/web/i18n/zh-CN/workflow.json

@@ -10,6 +10,7 @@
   "Variable_name": "变量名",
   "add_new_input": "新增输入",
   "add_new_output": "新增输出",
+  "http_body_placeholder": "与 APIFox 类似的语法，可通过 / 来激活变量选择。字符串变量均需加双引号，其他类型变量无需加双引号。请严格检查是否符合 JSON 格式。",
   "append_application_reply_to_history_as_new_context": "将该应用回复内容拼接到历史记录中，作为新的上下文返回",
   "application_call": "应用调用",
   "assigned_reply": "指定回复",
@@ -193,4 +194,4 @@
   "workflow.Switch_success": "切换成功",
   "workflow.Team cloud": "团队云端",
   "workflow.exit_tips": "您的更改尚未保存，「直接退出」将不会保存您的编辑记录。"
-}
+}