Skip to content

Commit

Permalink
feat(llm): modify clear buttons (apache#156)
Browse files Browse the repository at this point in the history 
3 buttons:
1. Clear Chunks Vector Index
2. Clear Graph Vid Vector Index (clear text2gql vector at the same time)
3. Clear Graph Data
4. hide abs path in UI

---------

Co-authored-by: imbajin <[email protected]>
  • Loading branch information
@MrJs133 @imbajin
MrJs133 and imbajin authored Jan 9, 2025
1 parent 9239799 commit 84c4d7e
Show file tree
Hide file tree
Showing 3 changed files with 27 additions and 15 deletions.
18 changes: 12 additions & 6 deletions hugegraph-llm/src/hugegraph_llm/demo/rag_demo/vector_graph_block.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
from hugegraph_llm.utils.graph_index_utils import (
get_graph_index_info,
clean_all_graph_index,
clean_all_graph_data,
update_vid_embedding,
extract_graph,
import_graph_data,
Expand Down Expand Up @@ -63,7 +64,7 @@ def create_vector_graph_block():
with gr.Tab("text") as tab_upload_text:
input_text = gr.Textbox(
value=prompt.doc_input_text,
label="Doc(s)",
label="Input Doc(s)",
lines=20,
show_copy_button=True
)
Expand All @@ -73,21 +74,22 @@ def create_vector_graph_block():
label="Docs (multi-files can be selected together)",
file_count="multiple",
)
input_schema = gr.Textbox(value=prompt.graph_schema, label="Schema", lines=15, show_copy_button=True)
input_schema = gr.Textbox(value=prompt.graph_schema, label="Graph Schema", lines=15, show_copy_button=True)
info_extract_template = gr.Textbox(
value=prompt.extract_graph_prompt, label="Graph Extract Prompt Header", lines=15, show_copy_button=True
)
out = gr.Code(label="Output", language="json", elem_classes="code-container-edit")
out = gr.Code(label="Output Info", language="json", elem_classes="code-container-edit")

with gr.Row():
with gr.Accordion("Get RAG Info", open=False):
with gr.Column():
vector_index_btn0 = gr.Button("Get Vector Index Info", size="sm")
graph_index_btn0 = gr.Button("Get Graph Index Info", size="sm")
with gr.Accordion("Clear RAG Info", open=False):
with gr.Accordion("Clear RAG Data", open=False):
with gr.Column():
vector_index_btn1 = gr.Button("Clear Vector Index", size="sm")
graph_index_btn1 = gr.Button("Clear Graph Data & Index", size="sm")
vector_index_btn1 = gr.Button("Clear Chunks Vector Index", size="sm")
graph_index_btn1 = gr.Button("Clear Graph Vid Vector Index", size="sm")
graph_data_btn0 = gr.Button("Clear Graph Data", size="sm")

vector_import_bt = gr.Button("Import into Vector", variant="primary")
graph_extract_bt = gr.Button("Extract Graph Data (1)", variant="primary")
Expand All @@ -114,6 +116,10 @@ def create_vector_graph_block():
store_prompt,
inputs=[input_text, input_schema, info_extract_template],
)
graph_data_btn0.click(clean_all_graph_data).then(
store_prompt,
inputs=[input_text, input_schema, info_extract_template],
)
graph_index_rebuild_bt.click(update_vid_embedding, outputs=out).then(
store_prompt,
inputs=[input_text, input_schema, info_extract_template],
Expand Down
14 changes: 10 additions & 4 deletions hugegraph-llm/src/hugegraph_llm/utils/graph_index_utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,9 +46,16 @@ def get_graph_index_info():


def clean_all_graph_index():
clean_hg_data()
VectorIndex.clean(str(os.path.join(resource_path, huge_settings.graph_name, "graph_vids")))
gr.Info("Clean graph index successfully!")
VectorIndex.clean(str(os.path.join(resource_path, "gremlin_examples")))
log.warning("Clear graph index and text2gql index successfully!")
gr.Info("Clear graph index and text2gql index successfully!")


def clean_all_graph_data():
clean_hg_data()
log.warning("Clear graph data successfully!")
gr.Info("Clear graph data successfully!")


def parse_schema(schema: str, builder: KgBuilder) -> Optional[str]:
Expand All @@ -67,7 +74,6 @@ def parse_schema(schema: str, builder: KgBuilder) -> Optional[str]:


def extract_graph(input_file, input_text, schema, example_prompt) -> str:

texts = read_documents(input_file, input_text)
builder = KgBuilder(LLMs().get_chat_llm(), Embeddings().get_embedding(), get_hg_client())
if not schema:
Expand All @@ -87,7 +93,7 @@ def extract_graph(input_file, input_text, schema, example_prompt) -> str:
"vertices": context["vertices"],
"edges": context["edges"],
"warning": "The schema may not match the Doc"
},
},
ensure_ascii=False,
indent=2
)
Expand Down
10 changes: 5 additions & 5 deletions hugegraph-llm/src/hugegraph_llm/utils/hugegraph_utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,14 +101,12 @@ def create_dir_safely(path):
def backup_data():
try:
client = get_hg_client()

create_dir_safely(BACKUP_DIR)

date_str = datetime.now().strftime("%Y%m%d")
backup_subdir = os.path.join(BACKUP_DIR, f"{date_str}")
create_dir_safely(backup_subdir)


files = {
"vertices.json": f"g.V().limit({MAX_VERTICES})"
f".aggregate('vertices').count().as('count').select('count','vertices')",
Expand All @@ -121,9 +119,10 @@ def backup_data():
data = client.gremlin().exec(query)["data"] if "schema" not in filename else query
json.dump(data, f, ensure_ascii=False)

log.info("Backup completed successfully in %s.", backup_subdir)
log.info("Backup successfully in %s.", backup_subdir)
relative_backup_subdir = os.path.relpath(backup_subdir, start=resource_path)
del_info = manage_backup_retention()
return f"Backup completed successfully in {backup_subdir} \n{del_info}"
return f"Backup successfully in '{relative_backup_subdir}' \n{del_info}"
except Exception as e: # pylint: disable=W0718
log.critical("Backup failed: %s", e, exc_info=True)
raise Exception("Failed to execute backup") from e
Expand All @@ -141,7 +140,8 @@ def manage_backup_retention():
old_backup = backup_dirs.pop(0)
shutil.rmtree(old_backup)
log.info("Deleted old backup: %s", old_backup)
return f"Deleted old backup: {old_backup}"
relative_old_backup = os.path.relpath(old_backup, start=resource_path)
return f"Deleted old backup: {relative_old_backup}"
return f"The current number of backup files <= {MAX_BACKUP_DIRS}, so no files are deleted"
except Exception as e: # pylint: disable=W0718
log.error("Failed to manage backup retention: %s", e, exc_info=True)
Expand Down

0 comments on commit 84c4d7e

Please sign in to comment.