add file management feature

README.md

@@ -58,6 +58,7 @@
 
 ## 📌 Latest Features
 
+- 2024-04-26 Add file management.
 - 2024-04-19 Support conversation API ([detail](./docs/conversation_api.md)).
 - 2024-04-16 Add an embedding model 'bce-embedding-base_v1' from [BCEmbedding](https://github.com/netease-youdao/BCEmbedding).
 - 2024-04-16 Add [FastEmbed](https://github.com/qdrant/fastembed), which is designed specifically for light and speedy embedding.

README_ja.md

@@ -58,6 +58,7 @@
 
 ## 📌 最新の機能
 
+- 2024-04-26 「ファイル管理」機能を追加しました。
 - 2024-04-19 会話 API をサポートします ([詳細](./docs/conversation_api.md))。
 - 2024-04-16 [BCEmbedding](https://github.com/netease-youdao/BCEmbedding) から埋め込みモデル「bce-embedding-base_v1」を追加します。
 - 2024-04-16 [FastEmbed](https://github.com/qdrant/fastembed) は、軽量かつ高速な埋め込み用に設計されています。

README_zh.md

@@ -58,6 +58,7 @@
 
 ## 📌 新增功能
 
+- 2024-04-26 增添了'文件管理'功能.
 - 2024-04-19 支持对话 API ([更多](./docs/conversation_api.md)).
 - 2024-04-16 添加嵌入模型 [BCEmbedding](https://github.com/netease-youdao/BCEmbedding) 。
 - 2024-04-16 添加 [FastEmbed](https://github.com/qdrant/fastembed) 专为轻型和高速嵌入而设计。

api/apps/document_app.py

@@ -23,6 +23,9 @@
 from elasticsearch_dsl import Q
 from flask import request
 from flask_login import login_required, current_user
+
+from api.db.services.file2document_service import File2DocumentService
+from api.db.services.file_service import FileService
 from rag.nlp import search
 from rag.utils import ELASTICSEARCH
 from api.db.services import duplicate_name
@@ -68,7 +71,7 @@ def upload():
             name=file.filename,
             kb_id=kb.id)
         filetype = filename_type(filename)
-        if not filetype:
+        if filetype == FileType.OTHER.value:
             return get_data_error_result(
                 retmsg="This type of file has not been supported yet!")
 
@@ -218,26 +221,39 @@ def change_status():
 @validate_request("doc_id")
 def rm():
     req = request.json
-    try:
-        e, doc = DocumentService.get_by_id(req["doc_id"])
-        if not e:
-            return get_data_error_result(retmsg="Document not found!")
-        tenant_id = DocumentService.get_tenant_id(req["doc_id"])
-        if not tenant_id:
-            return get_data_error_result(retmsg="Tenant not found!")
-        ELASTICSEARCH.deleteByQuery(
-            Q("match", doc_id=doc.id), idxnm=search.index_name(tenant_id))
-
-        DocumentService.increment_chunk_num(
-            doc.id, doc.kb_id, doc.token_num * -1, doc.chunk_num * -1, 0)
-        if not DocumentService.delete(doc):
-            return get_data_error_result(
-                retmsg="Database error (Document removal)!")
+    doc_ids = req["doc_id"]
+    if isinstance(doc_ids, str): doc_ids = [doc_ids]
+    errors = ""
+    for doc_id in doc_ids:
+        try:
+            e, doc = DocumentService.get_by_id(doc_id)
+
+            if not e:
+                return get_data_error_result(retmsg="Document not found!")
+            tenant_id = DocumentService.get_tenant_id(doc_id)
+            if not tenant_id:
+                return get_data_error_result(retmsg="Tenant not found!")
+
+            ELASTICSEARCH.deleteByQuery(
+                Q("match", doc_id=doc.id), idxnm=search.index_name(tenant_id))
+            DocumentService.increment_chunk_num(
+                doc.id, doc.kb_id, doc.token_num * -1, doc.chunk_num * -1, 0)
+            if not DocumentService.delete(doc):
+                return get_data_error_result(
+                    retmsg="Database error (Document removal)!")
+
+            informs = File2DocumentService.get_by_document_id(doc_id)
+            if not informs:
+                MINIO.rm(doc.kb_id, doc.location)
+            else:
+                File2DocumentService.delete_by_document_id(doc_id)
+        except Exception as e:
+            errors += str(e)
+
+
+    if errors: return server_error_response(e)
+    return get_json_result(data=True)
 
-        MINIO.rm(doc.kb_id, doc.location)
-        return get_json_result(data=True)
-    except Exception as e:
-        return server_error_response(e)
 
 
 @manager.route('/run', methods=['POST'])
@@ -302,7 +318,13 @@ def get(doc_id):
         if not e:
             return get_data_error_result(retmsg="Document not found!")
 
-        response = flask.make_response(MINIO.get(doc.kb_id, doc.location))
+        informs = File2DocumentService.get_by_document_id(doc_id)
+        if not informs:
+            response = flask.make_response(MINIO.get(doc.kb_id, doc.location))
+        else:
+            e, file = FileService.get_by_id(informs[0].file_id)
+            response = flask.make_response(MINIO.get(file.parent_id, doc.location))
+
         ext = re.search(r"\.([^.]+)$", doc.name)
         if ext:
             if doc.type == FileType.VISUAL.value:

api/apps/file2document_app.py

@@ -0,0 +1,137 @@
+#
+#  Copyright 2024 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License
+#
+from elasticsearch_dsl import Q
+
+from api.db.db_models import File2Document
+from api.db.services.file2document_service import File2DocumentService
+from api.db.services.file_service import FileService
+
+from flask import request
+from flask_login import login_required, current_user
+from api.db.services.knowledgebase_service import KnowledgebaseService
+from api.utils.api_utils import server_error_response, get_data_error_result, validate_request
+from api.utils import get_uuid
+from api.db import FileType
+from api.db.services.document_service import DocumentService
+from api.settings import RetCode
+from api.utils.api_utils import get_json_result
+from rag.nlp import search
+from rag.utils import ELASTICSEARCH
+
+
+@manager.route('/convert', methods=['POST'])
+@login_required
+@validate_request("file_ids", "kb_ids")
+def convert():
+    req = request.json
+    kb_ids = req["kb_ids"]
+    file_ids = req["file_ids"]
+    file2documents = []
+
+    try:
+        for file_id in file_ids:
+            e, file = FileService.get_by_id(file_id)
+            file_ids_list = [file_id]
+            if file.type == FileType.FOLDER:
+                file_ids_list = FileService.get_all_innermost_file_ids(file_id, [])
+            for id in file_ids_list:
+                informs = File2DocumentService.get_by_file_id(id)
+                # delete
+                for inform in informs:
+                    doc_id = inform.document_id
+                    e, doc = DocumentService.get_by_id(doc_id)
+                    if not e:
+                        return get_data_error_result(retmsg="Document not found!")
+                    tenant_id = DocumentService.get_tenant_id(doc_id)
+                    if not tenant_id:
+                        return get_data_error_result(retmsg="Tenant not found!")
+                    ELASTICSEARCH.deleteByQuery(
+                        Q("match", doc_id=doc.id), idxnm=search.index_name(tenant_id))
+                    DocumentService.increment_chunk_num(
+                        doc.id, doc.kb_id, doc.token_num * -1, doc.chunk_num * -1, 0)
+                    if not DocumentService.delete(doc):
+                        return get_data_error_result(
+                            retmsg="Database error (Document removal)!")
+                File2DocumentService.delete_by_file_id(id)
+
+                # insert
+                for kb_id in kb_ids:
+                    e, kb = KnowledgebaseService.get_by_id(kb_id)
+                    if not e:
+                        return get_data_error_result(
+                            retmsg="Can't find this knowledgebase!")
+                    e, file = FileService.get_by_id(id)
+                    if not e:
+                        return get_data_error_result(
+                            retmsg="Can't find this file!")
+
+                    doc = DocumentService.insert({
+                        "id": get_uuid(),
+                        "kb_id": kb.id,
+                        "parser_id": kb.parser_id,
+                        "parser_config": kb.parser_config,
+                        "created_by": current_user.id,
+                        "type": file.type,
+                        "name": file.name,
+                        "location": file.location,
+                        "size": file.size
+                    })
+                    file2document = File2DocumentService.insert({
+                        "id": get_uuid(),
+                        "file_id": id,
+                        "document_id": doc.id,
+                    })
+                    file2documents.append(file2document.to_json())
+        return get_json_result(data=file2documents)
+    except Exception as e:
+        return server_error_response(e)
+
+
+@manager.route('/rm', methods=['POST'])
+@login_required
+@validate_request("file_ids")
+def rm():
+    req = request.json
+    file_ids = req["file_ids"]
+    if not file_ids:
+        return get_json_result(
+            data=False, retmsg='Lack of "Files ID"', retcode=RetCode.ARGUMENT_ERROR)
+    try:
+        for file_id in file_ids:
+            informs = File2DocumentService.get_by_file_id(file_id)
+            if not informs:
+                return get_data_error_result(retmsg="Inform not found!")
+            for inform in informs:
+                if not inform:
+                    return get_data_error_result(retmsg="Inform not found!")
+                File2DocumentService.delete_by_file_id(file_id)
+                doc_id = inform.document_id
+                e, doc = DocumentService.get_by_id(doc_id)
+                if not e:
+                    return get_data_error_result(retmsg="Document not found!")
+                tenant_id = DocumentService.get_tenant_id(doc_id)
+                if not tenant_id:
+                    return get_data_error_result(retmsg="Tenant not found!")
+                ELASTICSEARCH.deleteByQuery(
+                    Q("match", doc_id=doc.id), idxnm=search.index_name(tenant_id))
+                DocumentService.increment_chunk_num(
+                    doc.id, doc.kb_id, doc.token_num * -1, doc.chunk_num * -1, 0)
+                if not DocumentService.delete(doc):
+                    return get_data_error_result(
+                        retmsg="Database error (Document removal)!")
+        return get_json_result(data=True)
+    except Exception as e:
+        return server_error_response(e)