Fix rag ignore suffix case

nesis/api/alembic/versions/0fc14df78ac3_add_app_entity.py

@@ -54,7 +54,7 @@ def upgrade() -> None:
     )
 
     # ### end Alembic commands ###
-    op.execute("ALTER TYPE role_action_resource_type ADD VALUE 'APP';")
+    op.execute("ALTER TYPE role_action_resource_type ADD VALUE IF NOT EXISTS 'APP';")
 
 
 def downgrade() -> None:

nesis/api/alembic/versions/50bebf15de30_add_tasks_to_role_action_resource_type.py

@@ -21,8 +21,8 @@
 
 def upgrade() -> None:
     # ### commands auto generated by Alembic - please adjust! ###
-    op.execute("ALTER TYPE role_action_resource_type ADD VALUE 'APPS';")
-    op.execute("ALTER TYPE role_action_resource_type ADD VALUE 'TASKS';")
+    op.execute("ALTER TYPE role_action_resource_type ADD VALUE IF NOT EXISTS 'APPS';")
+    op.execute("ALTER TYPE role_action_resource_type ADD VALUE IF NOT EXISTS 'TASKS';")
     # ### end Alembic commands ###
 
 

nesis/api/alembic/versions/64a6f5fbfc82_migrations_0_0_3_rc1.py

@@ -52,7 +52,7 @@ def upgrade() -> None:
     # ### end Alembic commands ###
 
     # Upgrade the document_status type
-    op.execute("ALTER TYPE datasource_status ADD VALUE 'INGESTING';")
+    op.execute("ALTER TYPE datasource_status ADD VALUE IF NOT EXISTS 'INGESTING';")
 
 
 def downgrade() -> None:

nesis/api/alembic/versions/7cfa662dff86_fix_document_unique_key.py

@@ -0,0 +1,39 @@
+"""fix document unique key
+
+Revision ID: 7cfa662dff86
+Revises: 9f486f3bf6ac
+Create Date: 2024-06-24 14:36:18.455799
+
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = "7cfa662dff86"
+down_revision: Union[str, None] = "9f486f3bf6ac"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_constraint("uq_document_uuid_base_url", "document", type_="unique")
+    op.create_unique_constraint(
+        "uq_document_uuid_base_url_filename",
+        "document",
+        ["uuid", "base_uri", "filename"],
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_constraint("uq_document_uuid_base_url_filename", "document", type_="unique")
+    op.create_unique_constraint(
+        "uq_document_uuid_base_url", "document", ["uuid", "base_uri"]
+    )
+    # ### end Alembic commands ###

nesis/api/alembic/versions/fbf94c515e04_migrations_0_0_3_rc2.py

@@ -22,7 +22,7 @@
 def upgrade() -> None:
     # ### commands auto generated by Alembic - please adjust! ###
     # Upgrade the document_status type
-    op.execute("ALTER TYPE datasource_type ADD VALUE 'S3';")
+    op.execute("ALTER TYPE datasource_type ADD VALUE IF NOT EXISTS 'S3';")
     # ### end Alembic commands ###
 
 

nesis/api/core/document_loaders/minio.py

@@ -178,14 +178,14 @@ def _sync_document(
                             doc_id=document_data["doc_id"],
                         )
                     except:
-                        _LOG.warn(
+                        _LOG.warning(
                             f"Failed to delete document {document_data['doc_id']}"
                         )
 
                 try:
                     delete_document(document_id=item.etag)
                 except:
-                    _LOG.warn(
+                    _LOG.warning(
                         f"Failed to delete document {item.object_name}'s record. Continuing anyway..."
                     )
 
@@ -207,7 +207,7 @@ def _sync_document(
 
         save_document(
             document_id=item.etag,
-            filename=item.object_name,
+            filename=f"{item.bucket_name}/{item.object_name}",
             base_uri=endpoint,
             rag_metadata=response_json,
             store_metadata={

nesis/api/core/document_loaders/samba.py

@@ -259,7 +259,7 @@ def _process_file(
 
         save_document(
             document_id=file_unique_id,
-            filename=file_name,
+            filename=file_share.path,
             base_uri=endpoint,
             rag_metadata=response_json,
             store_metadata=file_metadata,

nesis/api/core/document_loaders/sharepoint.py

@@ -257,7 +257,7 @@ def _sync_document(
 
             save_document(
                 document_id=file.unique_id,
-                filename=file.name,
+                filename=file.serverRelativeUrl,
                 base_uri=site_url,
                 rag_metadata=response_json,
                 store_metadata={

nesis/api/core/models/entities.py

@@ -171,7 +171,9 @@ class Document(Base):
     store_metadata = Column(JSONB)
 
     __table_args__ = (
-        UniqueConstraint("uuid", "base_uri", name="uq_document_uuid_base_url"),
+        UniqueConstraint(
+            "uuid", "base_uri", "filename", name="uq_document_uuid_base_url_filename"
+        ),
     )
 
     def __init__(

nesis/rag/core/components/ingest/ingest_helper.py

@@ -77,7 +77,7 @@ def transform_file_into_documents(
     def _load_file_to_documents(file_name: str, file_data: Path) -> list[Document]:
         logger.debug("Transforming file_name=%s into documents", file_name)
         extension = Path(file_name).suffix
-        reader_cls = FILE_READER_CLS.get(extension)
+        reader_cls = FILE_READER_CLS.get(extension.lower())
         if reader_cls is None:
             logger.debug(
                 "No reader found for extension=%s, using default string reader",

package.sh

@@ -1,4 +1,4 @@
 version=$1
-docker build --build-arg NESIS_VERSION="$version" -t ametnes/nesis:"$version"-rag . -f nesis/rag/Dockerfile
 docker build --build-arg NESIS_VERSION="$version" -t ametnes/nesis:"$version"-api . -f nesis/api/Dockerfile
 docker build --build-arg NESIS_VERSION="$version" --build-arg PUBLIC_URL=/ --build-arg PROFILE=PROD -t ametnes/nesis:"$version"-frontend . -f nesis/frontend/Dockerfile
+docker build --build-arg NESIS_VERSION="$version" -t ametnes/nesis:"$version"-rag . -f nesis/rag/Dockerfile

version.txt

@@ -1 +1 @@
-0.1.2
+0.1.2.1