From 03f00c9e6f98583dd0e9b68a122e735c0c716ddc Mon Sep 17 00:00:00 2001 From: Zhichang Yu Date: Tue, 10 Dec 2024 16:32:58 +0800 Subject: [PATCH 1/2] Rename page_num_list, top_list, position_list (#3940) ### What problem does this PR solve? Rename page_num_list, top_list, position_list to page_num_int, top_int, position_int ### Type of change - [x] Refactoring --- Dockerfile | 16 +++++++++++--- api/apps/chunk_app.py | 2 +- api/apps/sdk/doc.py | 2 +- conf/infinity_mapping.json | 6 ++--- graphrag/search.py | 2 +- rag/app/presentation.py | 14 +++++------- rag/nlp/__init__.py | 19 ++++++++-------- rag/nlp/search.py | 9 ++++---- rag/svr/task_executor.py | 6 ++--- rag/utils/es_conn.py | 10 +++++++-- rag/utils/infinity_conn.py | 45 +++++++++++++++++++++++++++++++++----- 11 files changed, 89 insertions(+), 42 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3e316e87a1..c8f3e7f01a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,9 +34,15 @@ RUN --mount=type=bind,from=infiniflow/ragflow_deps:latest,source=/,target=/deps cp /deps/cl100k_base.tiktoken /ragflow/9b5ad71b2ce5302211f9c61530b329a4922fc6a4 ENV TIKA_SERVER_JAR="file:///ragflow/tika-server-standard-3.0.0.jar" +ENV DEBIAN_FRONTEND=noninteractive # Setup apt -# cv2 requires libGL.so.1 +# Python package and implicit dependencies: +# opencv-python: libglib2.0-0 libglx-mesa0 libgl1 +# aspose-slides: pkg-config libicu-dev libgdiplus libssl1.1_1.1.1f-1ubuntu2_amd64.deb +# python-pptx: default-jdk tika-server-standard-3.0.0.jar +# selenium: libatk-bridge2.0-0 chrome-linux64-121-0-6167-85 +# Building C extensions: libpython3-dev libgtk-4-1 libnss3 xdg-utils libgbm-dev RUN --mount=type=cache,id=ragflow_apt,target=/var/cache/apt,sharing=locked \ if [ "$NEED_MIRROR" == "1" ]; then \ sed -i 's|http://archive.ubuntu.com|https://mirrors.tuna.tsinghua.edu.cn|g' /etc/apt/sources.list; \ @@ -47,8 +53,12 @@ RUN --mount=type=cache,id=ragflow_apt,target=/var/cache/apt,sharing=locked \ apt update && \ apt --no-install-recommends install -y ca-certificates && \ apt update && \ - DEBIAN_FRONTEND=noninteractive apt install -y curl libpython3-dev nginx libglib2.0-0 libglx-mesa0 pkg-config libicu-dev libgdiplus default-jdk python3-pip pipx \ - libatk-bridge2.0-0 libgtk-4-1 libnss3 xdg-utils unzip libgbm-dev wget git nginx libgl1 vim less + apt install -y libglib2.0-0 libglx-mesa0 libgl1 && \ + apt install -y pkg-config libicu-dev libgdiplus && \ + apt install -y default-jdk && \ + apt install -y libatk-bridge2.0-0 && \ + apt install -y libpython3-dev libgtk-4-1 libnss3 xdg-utils libgbm-dev && \ + apt install -y python3-pip pipx nginx unzip curl wget git vim less RUN if [ "$NEED_MIRROR" == "1" ]; then \ pip3 config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple && \ diff --git a/api/apps/chunk_app.py b/api/apps/chunk_app.py index d60483ad44..6dc1f9b646 100644 --- a/api/apps/chunk_app.py +++ b/api/apps/chunk_app.py @@ -71,7 +71,7 @@ def list_chunk(): "question_kwd": sres.field[id].get("question_kwd", []), "image_id": sres.field[id].get("img_id", ""), "available_int": int(sres.field[id].get("available_int", 1)), - "positions": json.loads(sres.field[id].get("position_list", "[]")), + "positions": sres.field[id].get("position_int", []), } assert isinstance(d["positions"], list) assert len(d["positions"]) == 0 or (isinstance(d["positions"][0], list) and len(d["positions"][0]) == 5) diff --git a/api/apps/sdk/doc.py b/api/apps/sdk/doc.py index 6bcbe58e72..7fa6458461 100644 --- a/api/apps/sdk/doc.py +++ b/api/apps/sdk/doc.py @@ -846,7 +846,7 @@ def list_chunks(tenant_id, dataset_id, document_id): "question_kwd": sres.field[id].get("question_kwd", []), "img_id": sres.field[id].get("img_id", ""), "available_int": sres.field[id].get("available_int", 1), - "positions": sres.field[id].get("position_int", "").split("\t"), + "positions": sres.field[id].get("position_int", []), } if len(d["positions"]) % 5 == 0: poss = [] diff --git a/conf/infinity_mapping.json b/conf/infinity_mapping.json index a9d1d4f0ee..118f205f99 100644 --- a/conf/infinity_mapping.json +++ b/conf/infinity_mapping.json @@ -16,9 +16,9 @@ "content_with_weight": {"type": "varchar", "default": ""}, "content_ltks": {"type": "varchar", "default": ""}, "content_sm_ltks": {"type": "varchar", "default": ""}, - "page_num_list": {"type": "varchar", "default": ""}, - "top_list": {"type": "varchar", "default": ""}, - "position_list": {"type": "varchar", "default": ""}, + "page_num_int": {"type": "varchar", "default": ""}, + "top_int": {"type": "varchar", "default": ""}, + "position_int": {"type": "varchar", "default": ""}, "weight_int": {"type": "integer", "default": 0}, "weight_flt": {"type": "float", "default": 0.0}, "rank_int": {"type": "integer", "default": 0}, diff --git a/graphrag/search.py b/graphrag/search.py index 4e9f444817..a803494bbc 100644 --- a/graphrag/search.py +++ b/graphrag/search.py @@ -58,7 +58,7 @@ def merge_into_first(sres, title="") -> dict[str, str]: matchDense = self.get_vector(qst, emb_mdl, 1024, req.get("similarity", 0.1)) q_vec = matchDense.embedding_data src = req.get("fields", ["docnm_kwd", "content_ltks", "kb_id", "img_id", "title_tks", "important_kwd", - "doc_id", f"q_{len(q_vec)}_vec", "position_list", "name_kwd", + "doc_id", f"q_{len(q_vec)}_vec", "position_int", "name_kwd", "available_int", "content_with_weight", "weight_int", "weight_flt" ]) diff --git a/rag/app/presentation.py b/rag/app/presentation.py index bebf64bff9..a3305bdc11 100644 --- a/rag/app/presentation.py +++ b/rag/app/presentation.py @@ -20,7 +20,6 @@ from rag.nlp import rag_tokenizer from deepdoc.parser import PdfParser, PptParser, PlainParser from PyPDF2 import PdfReader as pdf2_read -import json class Ppt(PptParser): @@ -109,9 +108,9 @@ def chunk(filename, binary=None, from_page=0, to_page=100000, d = copy.deepcopy(doc) pn += from_page d["image"] = img - d["page_num_list"] = json.dumps([pn + 1]) - d["top_list"] = json.dumps([0]) - d["position_list"] = json.dumps([(pn + 1, 0, img.size[0], 0, img.size[1])]) + d["page_num_int"] = [pn + 1] + d["top_int"] = [0] + d["position_int"] = [(pn + 1, 0, img.size[0], 0, img.size[1])] tokenize(d, txt, eng) res.append(d) return res @@ -125,10 +124,9 @@ def chunk(filename, binary=None, from_page=0, to_page=100000, pn += from_page if img: d["image"] = img - d["page_num_list"] = json.dumps([pn + 1]) - d["top_list"] = json.dumps([0]) - d["position_list"] = json.dumps([ - (pn + 1, 0, img.size[0] if img else 0, 0, img.size[1] if img else 0)]) + d["page_num_int"] = [pn + 1] + d["top_int"] = [0] + d["position_int"] = [(pn + 1, 0, img.size[0] if img else 0, 0, img.size[1] if img else 0)] tokenize(d, txt, eng) res.append(d) return res diff --git a/rag/nlp/__init__.py b/rag/nlp/__init__.py index 69573375a8..2aaf98aab0 100644 --- a/rag/nlp/__init__.py +++ b/rag/nlp/__init__.py @@ -22,7 +22,6 @@ from . import rag_tokenizer import re import copy -import json import roman_numbers as r from word2number import w2n from cn2an import cn2an @@ -311,16 +310,16 @@ def tokenize_table(tbls, doc, eng, batch_size=10): def add_positions(d, poss): if not poss: return - page_num_list = [] - position_list = [] - top_list = [] + page_num_int = [] + position_int = [] + top_int = [] for pn, left, right, top, bottom in poss: - page_num_list.append(int(pn + 1)) - top_list.append(int(top)) - position_list.append((int(pn + 1), int(left), int(right), int(top), int(bottom))) - d["page_num_list"] = json.dumps(page_num_list) - d["position_list"] = json.dumps(position_list) - d["top_list"] = json.dumps(top_list) + page_num_int.append(int(pn + 1)) + top_int.append(int(top)) + position_int.append((int(pn + 1), int(left), int(right), int(top), int(bottom))) + d["page_num_int"] = page_num_int + d["position_int"] = position_int + d["top_int"] = top_int def remove_contents_table(sections, eng=False): diff --git a/rag/nlp/search.py b/rag/nlp/search.py index c8de4df056..9f877ef4fb 100644 --- a/rag/nlp/search.py +++ b/rag/nlp/search.py @@ -16,7 +16,6 @@ import logging import re -import json from dataclasses import dataclass from rag.utils import rmSpace @@ -74,7 +73,7 @@ def search(self, req, idx_names: str | list[str], kb_ids: list[str], emb_mdl=Non offset, limit = pg * ps, (pg + 1) * ps src = req.get("fields", ["docnm_kwd", "content_ltks", "kb_id", "img_id", "title_tks", "important_kwd", - "doc_id", "position_list", "knowledge_graph_kwd", "question_kwd", "question_tks", + "doc_id", "page_num_int", "top_int", "create_timestamp_flt", "knowledge_graph_kwd", "question_kwd", "question_tks", "available_int", "content_with_weight", "pagerank_fea"]) kwds = set([]) @@ -82,6 +81,8 @@ def search(self, req, idx_names: str | list[str], kb_ids: list[str], emb_mdl=Non q_vec = [] if not qst: if req.get("sort"): + orderBy.asc("page_num_int") + orderBy.asc("top_int") orderBy.desc("create_timestamp_flt") res = self.dataStore.search(src, [], filters, [], orderBy, offset, limit, idx_names, kb_ids) total=self.dataStore.getTotal(res) @@ -340,7 +341,7 @@ def floor_sim(score): chunk = sres.field[id] dnm = chunk["docnm_kwd"] did = chunk["doc_id"] - position_list = chunk.get("position_list", "[]") + position_int = chunk.get("position_int", []) d = { "chunk_id": id, "content_ltks": chunk["content_ltks"], @@ -354,7 +355,7 @@ def floor_sim(score): "vector_similarity": vsim[i], "term_similarity": tsim[i], "vector": chunk.get(vector_column, zero_vector), - "positions": json.loads(position_list) + "positions": position_int, } if highlight and sres.highlight: if id in sres.highlight: diff --git a/rag/svr/task_executor.py b/rag/svr/task_executor.py index 606bb07571..902c1e31ae 100644 --- a/rag/svr/task_executor.py +++ b/rag/svr/task_executor.py @@ -211,9 +211,9 @@ def build_chunks(task, progress_callback): if not d.get("image"): _ = d.pop("image", None) d["img_id"] = "" - d["page_num_list"] = json.dumps([]) - d["position_list"] = json.dumps([]) - d["top_list"] = json.dumps([]) + d["page_num_int"] = [] + d["position_int"] = [] + d["top_int"] = [] docs.append(d) continue diff --git a/rag/utils/es_conn.py b/rag/utils/es_conn.py index a473833f45..35d6428695 100644 --- a/rag/utils/es_conn.py +++ b/rag/utils/es_conn.py @@ -185,8 +185,14 @@ def search(self, selectFields: list[str], highlightFields: list[str], condition: orders = list() for field, order in orderBy.fields: order = "asc" if order == 0 else "desc" - orders.append({field: {"order": order, "unmapped_type": "float", - "mode": "avg", "numeric_type": "double"}}) + if field in ["page_num_int", "top_int"]: + order_info = {"order": order, "unmapped_type": "float", + "mode": "avg", "numeric_type": "double"} + elif field.endswith("_int") or field.endswith("_flt"): + order_info = {"order": order, "unmapped_type": "float"} + else: + order_info = {"order": order, "unmapped_type": "text"} + orders.append({field: order_info}) s = s.sort(*orders) if limit > 0: diff --git a/rag/utils/infinity_conn.py b/rag/utils/infinity_conn.py index 5c0e2ef695..227c0bc6a4 100644 --- a/rag/utils/infinity_conn.py +++ b/rag/utils/infinity_conn.py @@ -297,7 +297,7 @@ def search( df_list.append(kb_res) self.connPool.release_conn(inf_conn) res = concat_dataframes(df_list, selectFields) - logger.debug("INFINITY search tables: " + str(table_list)) + logger.debug(f"INFINITY search tables: {str(table_list)}, result: {str(res)}") return res def get( @@ -307,8 +307,10 @@ def get( db_instance = inf_conn.get_database(self.dbName) df_list = list() assert isinstance(knowledgebaseIds, list) + table_list = list() for knowledgebaseId in knowledgebaseIds: table_name = f"{indexName}_{knowledgebaseId}" + table_list.append(table_name) table_instance = db_instance.get_table(table_name) kb_res = table_instance.output(["*"]).filter(f"id = '{chunkId}'").to_pl() if len(kb_res) != 0 and kb_res.shape[0] > 0: @@ -316,6 +318,7 @@ def get( self.connPool.release_conn(inf_conn) res = concat_dataframes(df_list, ["id"]) + logger.debug(f"INFINITY get tables: {str(table_list)}, result: {str(res)}") res_fields = self.getFields(res, res.columns) return res_fields.get(chunkId, None) @@ -349,15 +352,22 @@ def insert( for k, v in d.items(): if k.endswith("_kwd") and isinstance(v, list): d[k] = " ".join(v) - if k == 'kb_id': + elif k == 'kb_id': if isinstance(d[k], list): d[k] = d[k][0] # since d[k] is a list, but we need a str + elif k == "position_int": + assert isinstance(v, list) + arr = [num for row in v for num in row] + d[k] = "_".join(f"{num:08x}" for num in arr) + elif k in ["page_num_int", "top_int", "position_int"]: + assert isinstance(v, list) + d[k] = "_".join(f"{num:08x}" for num in v) ids = ["'{}'".format(d["id"]) for d in documents] str_ids = ", ".join(ids) str_filter = f"id IN ({str_ids})" table_instance.delete(str_filter) # for doc in documents: - # logger.info(f"insert position_list: {doc['position_list']}") + # logger.info(f"insert position_int: {doc['position_int']}") # logger.info(f"InfinityConnection.insert {json.dumps(documents)}") table_instance.insert(documents) self.connPool.release_conn(inf_conn) @@ -367,8 +377,8 @@ def insert( def update( self, condition: dict, newValue: dict, indexName: str, knowledgebaseId: str ) -> bool: - # if 'position_list' in newValue: - # logger.info(f"upsert position_list: {newValue['position_list']}") + # if 'position_int' in newValue: + # logger.info(f"update position_int: {newValue['position_int']}") inf_conn = self.connPool.get_conn() db_instance = inf_conn.get_database(self.dbName) table_name = f"{indexName}_{knowledgebaseId}" @@ -377,6 +387,16 @@ def update( for k, v in newValue.items(): if k.endswith("_kwd") and isinstance(v, list): newValue[k] = " ".join(v) + elif k == 'kb_id': + if isinstance(newValue[k], list): + newValue[k] = newValue[k][0] # since d[k] is a list, but we need a str + elif k == "position_int": + assert isinstance(v, list) + arr = [num for row in v for num in row] + newValue[k] = "_".join(f"{num:08x}" for num in arr) + elif k in ["page_num_int", "top_int"]: + assert isinstance(v, list) + newValue[k] = "_".join(f"{num:08x}" for num in v) table_instance.update(filter, newValue) self.connPool.release_conn(inf_conn) return True @@ -423,9 +443,22 @@ def getFields(self, res, fields: list[str]) -> list[str, dict]: v = res[fieldnm][i] if isinstance(v, Series): v = list(v) - elif fieldnm == "important_kwd": + elif fieldnm.endswith("_kwd"): assert isinstance(v, str) v = v.split() + elif fieldnm == "position_int": + assert isinstance(v, str) + if v: + arr = [int(hex_val, 16) for hex_val in v.split('_')] + v = [arr[i:i + 4] for i in range(0, len(arr), 4)] + else: + v = [] + elif fieldnm in ["page_num_int", "top_int"]: + assert isinstance(v, str) + if v: + v = [int(hex_val, 16) for hex_val in v.split('_')] + else: + v = [] else: if not isinstance(v, str): v = str(v) From fc4e644e5f174d0364eb3137fe31be0c78d91ee0 Mon Sep 17 00:00:00 2001 From: balibabu Date: Tue, 10 Dec 2024 16:36:16 +0800 Subject: [PATCH 2/2] Feat: Modify the data structure of the chunk in the conversation #3909 (#3955) ### What problem does this PR solve? Feat: Modify the data structure of the chunk in the conversation #3909 ### Type of change - [x] New Feature (non-breaking change which adds functionality) --- .../api-service/chat-api-key-modal/index.tsx | 2 +- .../chat-overview-modal/api-content.tsx | 17 +-- .../api-service/embed-modal/index.less | 4 + .../api-service/embed-modal/index.tsx | 35 +++++- web/src/components/api-service/hooks.ts | 55 ++++++--- web/src/components/message-item/index.tsx | 5 +- web/src/components/pdf-drawer/hooks.ts | 8 +- web/src/components/pdf-drawer/index.tsx | 3 +- web/src/components/pdf-previewer/index.tsx | 7 +- web/src/hooks/chat-hooks.ts | 8 +- web/src/hooks/document-hooks.ts | 5 +- web/src/hooks/logic-hooks.ts | 2 + web/src/hooks/user-setting-hooks.tsx | 24 +++- web/src/interfaces/database/chat.ts | 19 +++- web/src/locales/config.ts | 2 +- web/src/locales/en.ts | 1 + web/src/locales/ja.ts | 33 +++--- web/src/locales/zh-traditional.ts | 1 + web/src/locales/zh.ts | 1 + web/src/pages/chat/index.tsx | 35 +++--- web/src/pages/chat/markdown-content/index.tsx | 18 +-- web/src/pages/chat/share/large.tsx | 31 +++-- web/src/pages/chat/shared-hooks.ts | 107 ++++++------------ web/src/pages/flow/header/index.tsx | 47 ++++---- web/src/pages/flow/hooks.tsx | 14 +++ web/src/utils/authorization-util.ts | 6 +- web/src/utils/document-util.ts | 10 +- 27 files changed, 301 insertions(+), 199 deletions(-) diff --git a/web/src/components/api-service/chat-api-key-modal/index.tsx b/web/src/components/api-service/chat-api-key-modal/index.tsx index e586d37273..2497f0fa26 100644 --- a/web/src/components/api-service/chat-api-key-modal/index.tsx +++ b/web/src/components/api-service/chat-api-key-modal/index.tsx @@ -63,7 +63,7 @@ const ChatApiKeyModal = ({ diff --git a/web/src/components/api-service/chat-overview-modal/api-content.tsx b/web/src/components/api-service/chat-overview-modal/api-content.tsx index 033e28e8f9..cd00e69fe8 100644 --- a/web/src/components/api-service/chat-overview-modal/api-content.tsx +++ b/web/src/components/api-service/chat-overview-modal/api-content.tsx @@ -3,8 +3,7 @@ import apiDoc from '@parent/docs/references/http_api_reference.md'; import MarkdownPreview from '@uiw/react-markdown-preview'; import { Button, Card, Flex, Space } from 'antd'; import ChatApiKeyModal from '../chat-api-key-modal'; -import EmbedModal from '../embed-modal'; -import { usePreviewChat, useShowEmbedModal } from '../hooks'; +import { usePreviewChat } from '../hooks'; import BackendServiceApi from './backend-service-api'; const ApiContent = ({ @@ -22,10 +21,10 @@ const ApiContent = ({ hideModal: hideApiKeyModal, showModal: showApiKeyModal, } = useSetModalState(); - const { embedVisible, hideEmbedModal, showEmbedModal, embedToken } = - useShowEmbedModal(idKey, id); + // const { embedVisible, hideEmbedModal, showEmbedModal, embedToken } = + // useShowEmbedModal(idKey); - const { handlePreview } = usePreviewChat(idKey, id); + const { handlePreview } = usePreviewChat(idKey); return (
@@ -36,7 +35,9 @@ const ApiContent = ({ - + {/* */} @@ -50,13 +51,13 @@ const ApiContent = ({ idKey={idKey} > )} - {embedVisible && ( + {/* {embedVisible && ( - )} + )} */}
); }; diff --git a/web/src/components/api-service/embed-modal/index.less b/web/src/components/api-service/embed-modal/index.less index b2c0cfbba8..cf39347008 100644 --- a/web/src/components/api-service/embed-modal/index.less +++ b/web/src/components/api-service/embed-modal/index.less @@ -6,3 +6,7 @@ padding: 10px; background-color: #ffffff09; } + +.id { + .linkText(); +} diff --git a/web/src/components/api-service/embed-modal/index.tsx b/web/src/components/api-service/embed-modal/index.tsx index 7c142b7205..1caee0a644 100644 --- a/web/src/components/api-service/embed-modal/index.tsx +++ b/web/src/components/api-service/embed-modal/index.tsx @@ -1,21 +1,33 @@ import CopyToClipboard from '@/components/copy-to-clipboard'; import HightLightMarkdown from '@/components/highlight-markdown'; +import { SharedFrom } from '@/constants/chat'; import { useTranslate } from '@/hooks/common-hooks'; import { IModalProps } from '@/interfaces/common'; -import { Card, Modal, Tabs, TabsProps } from 'antd'; +import { Card, Modal, Tabs, TabsProps, Typography } from 'antd'; + import styles from './index.less'; +const { Paragraph, Link } = Typography; + const EmbedModal = ({ visible, hideModal, token = '', -}: IModalProps & { token: string }) => { + form, + beta = '', + isAgent, +}: IModalProps & { + token: string; + form: SharedFrom; + beta: string; + isAgent: boolean; +}) => { const { t } = useTranslate('chat'); const text = ` ~~~ html