From c8763ad868c484a4784ff551e87702314479455d Mon Sep 17 00:00:00 2001
From: bill <yibie_jingnian@163.com>
Date: Mon, 12 Aug 2024 15:44:01 +0800
Subject: [PATCH] feat: Add delimiter field to naive parsing method #1909

---
 .../components/chunk-method-modal/index.tsx   |  8 ++++-
 web/src/components/delimiter.tsx              | 34 +++++++++++++++++++
 web/src/locales/en.ts                         |  1 +
 web/src/locales/zh-traditional.ts             |  1 +
 web/src/locales/zh.ts                         |  1 +
 .../knowledge-setting/configuration.tsx       |  2 ++
 6 files changed, 46 insertions(+), 1 deletion(-)
 create mode 100644 web/src/components/delimiter.tsx

diff --git a/web/src/components/chunk-method-modal/index.tsx b/web/src/components/chunk-method-modal/index.tsx
index 5deca9f40ae..ff4b4a06d53 100644
--- a/web/src/components/chunk-method-modal/index.tsx
+++ b/web/src/components/chunk-method-modal/index.tsx
@@ -22,6 +22,7 @@ import React, { useEffect, useMemo } from 'react';
 import { useFetchParserListOnMount } from './hooks';
 
 import { useTranslate } from '@/hooks/common-hooks';
+import Delimiter from '../delimiter';
 import EntityTypesItem from '../entity-types-item';
 import LayoutRecognize from '../layout-recognize';
 import ParseConfiguration, {
@@ -268,7 +269,12 @@ const ChunkMethodModal: React.FC<IProps> = ({
             }
           </Form.Item>
         )}
-        {showMaxTokenNumber && <MaxTokenNumber></MaxTokenNumber>}
+        {showMaxTokenNumber && (
+          <>
+            <MaxTokenNumber></MaxTokenNumber>
+            <Delimiter></Delimiter>
+          </>
+        )}
         {showRaptorParseConfiguration(selectedTag) && (
           <ParseConfiguration></ParseConfiguration>
         )}
diff --git a/web/src/components/delimiter.tsx b/web/src/components/delimiter.tsx
new file mode 100644
index 00000000000..5605763de5f
--- /dev/null
+++ b/web/src/components/delimiter.tsx
@@ -0,0 +1,34 @@
+import { Form, Input } from 'antd';
+import { useTranslation } from 'react-i18next';
+
+interface IProps {
+  value?: string | undefined;
+  onChange?: (val: string | undefined) => void;
+}
+
+const DelimiterInput = ({ value, onChange }: IProps) => {
+  const nextValue = value?.replaceAll('\n', '\\n');
+  const handleInputChange = (e: React.ChangeEvent<HTMLInputElement>) => {
+    const val = e.target.value;
+    const nextValue = val.replaceAll('\\n', '\n');
+    onChange?.(nextValue);
+  };
+  return <Input value={nextValue} onChange={handleInputChange}></Input>;
+};
+
+const Delimiter = () => {
+  const { t } = useTranslation();
+
+  return (
+    <Form.Item
+      name={['parser_config', 'delimiter']}
+      label={t('knowledgeDetails.delimiter')}
+      initialValue={`\\n!?;。；！？`}
+      rules={[{ required: true }]}
+    >
+      <DelimiterInput />
+    </Form.Item>
+  );
+};
+
+export default Delimiter;
diff --git a/web/src/locales/en.ts b/web/src/locales/en.ts
index c1ae7a44efd..c5c4cf57d11 100644
--- a/web/src/locales/en.ts
+++ b/web/src/locales/en.ts
@@ -148,6 +148,7 @@ export default {
       rerankTip: `If it's empty. It uses embeddings of query and chunks to compuste vector cosine similarity. Otherwise, it uses rerank score in place of  vector cosine similarity.`,
       topK: 'Top-K',
       topKTip: `K chunks will be fed into rerank models.`,
+      delimiter: `Delimiter`,
     },
     knowledgeConfiguration: {
       titleDescription:
diff --git a/web/src/locales/zh-traditional.ts b/web/src/locales/zh-traditional.ts
index 8a160a42389..200b0b231dc 100644
--- a/web/src/locales/zh-traditional.ts
+++ b/web/src/locales/zh-traditional.ts
@@ -144,6 +144,7 @@ export default {
       rerankTip: `如果是空的。它使用查詢和塊的嵌入來構成矢量餘弦相似性。否則，它使用rerank評分代替矢量餘弦相似性。`,
       topK: 'Top-K',
       topKTip: `K塊將被送入Rerank型號。`,
+      delimiter: `分段標識符`,
     },
     knowledgeConfiguration: {
       titleDescription: '在這裡更新您的知識庫詳細信息，尤其是解析方法。',
diff --git a/web/src/locales/zh.ts b/web/src/locales/zh.ts
index ec2a8f02683..730e9038e00 100644
--- a/web/src/locales/zh.ts
+++ b/web/src/locales/zh.ts
@@ -145,6 +145,7 @@ export default {
       rerankTip: `如果是空的。它使用查询和块的嵌入来构成矢量余弦相似性。否则，它使用rerank评分代替矢量余弦相似性。`,
       topK: 'Top-K',
       topKTip: `K块将被送入Rerank型号。`,
+      delimiter: `分段标识符`,
     },
     knowledgeConfiguration: {
       titleDescription: '在这里更新您的知识库详细信息，尤其是解析方法。',
diff --git a/web/src/pages/add-knowledge/components/knowledge-setting/configuration.tsx b/web/src/pages/add-knowledge/components/knowledge-setting/configuration.tsx
index 82e03c3a1ff..c251ecd752e 100644
--- a/web/src/pages/add-knowledge/components/knowledge-setting/configuration.tsx
+++ b/web/src/pages/add-knowledge/components/knowledge-setting/configuration.tsx
@@ -1,3 +1,4 @@
+import Delimiter from '@/components/delimiter';
 import EntityTypesItem from '@/components/entity-types-item';
 import LayoutRecognize from '@/components/layout-recognize';
 import MaxTokenNumber from '@/components/max-token-number';
@@ -111,6 +112,7 @@ const ConfigurationForm = ({ form }: { form: FormInstance }) => {
               {parserId === 'naive' && (
                 <>
                   <MaxTokenNumber></MaxTokenNumber>
+                  <Delimiter></Delimiter>
                   <LayoutRecognize></LayoutRecognize>
                 </>
               )}