Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Rag/improve file manage #393

Open
wants to merge 9 commits into
base: main
Choose a base branch
from
+184 −74
Open

Rag/improve file manage #393

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
075c658
1. Support delete and reupload. 2. Implement safe delete
Dec 16, 2024
7bdeed6
delete document record in case of unique constraint failing
Dec 17, 2024
d932ad2
stash current modification
Dec 19, 2024
7640161
1. doc_manager only set deleting. Document will de deleted in worker
Dec 19, 2024
bf77195
fix lint error
Dec 19, 2024
f44c3bd
restore old one-line style
Dec 19, 2024
134c9d6
remove function comment
Dec 19, 2024
db63373
remove print
Dec 19, 2024
7447213
restore to old-one-line style
Dec 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 13 additions & 2 deletions lazyllm/tools/rag/doc_manager.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,10 @@ def upload_files(self, files: List[UploadFile], override: bool = False, # noqa
return BaseResponse(code=400, msg=f'file [{files[idx].filename}]: {err_msg}', data=None)

file_paths = [os.path.join(self._manager._path, user_path or '', file.filename) for file in files]
ids = self._manager.add_files(file_paths, metadatas=metadatas, status=DocListManager.Status.working)
ids, file_paths = self._manager.add_files(
file_paths, metadatas=metadatas, status=DocListManager.Status.working
)
assert len(files) == len(ids), "len(files) uploaded vs len(ids) recorede"
results = []
for file, path in zip(files, file_paths):
if os.path.exists(path):
Expand Down Expand Up @@ -116,7 +119,9 @@ def add_files(self, files: List[str] = Body(...),
else:
id_mapping[file] = None

new_ids = self._manager.add_files(new_files, metadatas=new_metadatas, status=DocListManager.Status.success)
new_ids, _ = self._manager.add_files(
new_files, metadatas=new_metadatas, status=DocListManager.Status.success
)
if group_name:
self._manager.add_files_to_kb_group(new_ids + exist_ids, group=group_name)

Expand Down Expand Up @@ -178,6 +183,12 @@ def delete_files(self, request: FileGroupRequest):
if request.group_name:
return self.delete_files_from_group(request)
else:
safe_delete_ids = set(self._manager.get_safe_delete_files())
SuperEver marked this conversation as resolved.
Show resolved Hide resolved
tmp = request.file_ids
request.file_ids = []
for file_id in tmp:
if file_id in safe_delete_ids:
request.file_ids.append(file_id)
self._manager.update_kb_group_file_status(
file_ids=request.file_ids, status=DocListManager.Status.deleting)
docs = self._manager.update_file_status(file_ids=request.file_ids, status=DocListManager.Status.deleting)
Expand Down
Loading
Loading