Feature/basic retrieval

app/domain/data/image_message_content_dto.py

@@ -1,7 +1,8 @@
-from pydantic import BaseModel
+from pydantic import BaseModel, Field, ConfigDict
 from typing import Optional
 
 
 class ImageMessageContentDTO(BaseModel):
-    base64: str
-    prompt: Optional[str]
+    base64: str = Field(..., alias="pdfFile")
+    prompt: Optional[str] = None
+    model_config = ConfigDict(populate_by_name=True)

app/domain/data/lecture_unit_dto.py

@@ -3,11 +3,12 @@
 
 class LectureUnitDTO(BaseModel):
     to_update: bool = Field(alias="toUpdate")
-    pdf_file_base64: str = Field(alias="pdfFile")
+    base_url: str = Field(alias="artemisBaseUrl")
+    pdf_file_base64: str = Field(default="", alias="pdfFile")
     lecture_unit_id: int = Field(alias="lectureUnitId")
-    lecture_unit_name: str = Field(alias="lectureUnitName")
+    lecture_unit_name: str = Field(default="", alias="lectureUnitName")
     lecture_id: int = Field(alias="lectureId")
-    lecture_name: str = Field(alias="lectureName")
+    lecture_name: str = Field(default="", alias="lectureName")
     course_id: int = Field(alias="courseId")
-    course_name: str = Field(alias="courseName")
-    course_description: str = Field(alias="courseDescription")
+    course_name: str = Field(default="", alias="courseName")
+    course_description: str = Field(default="", alias="courseDescription")

app/domain/pipeline_execution_settings_dto.py

@@ -5,5 +5,8 @@
 
 class PipelineExecutionSettingsDTO(BaseModel):
     authentication_token: str = Field(alias="authenticationToken")
-    allowed_model_identifiers: List[str] = Field(alias="allowedModelIdentifiers")
+    Field(alias="authenticationToken")
+    allowed_model_identifiers: List[str] = Field(
+        default=[], alias="allowedModelIdentifiers"
+    )
     artemis_base_url: str = Field(alias="artemisBaseUrl")

app/pipeline/chat/lecture_chat_pipeline.py

@@ -98,6 +98,7 @@ def __call__(self, dto: LectureChatPipelineExecutionDTO):
             student_query=query.contents[0].text_content,
             result_limit=10,
             course_name=dto.course.name,
+            course_id=dto.course.id,
         )
 
         self._add_relevant_chunks_to_prompt(retrieved_lecture_chunks)

app/pipeline/chat/tutor_chat_pipeline.py

@@ -214,6 +214,7 @@ def _run_tutor_chat_pipeline(self, dto: TutorChatPipelineExecutionDTO):
             student_query=query.contents[0].text_content,
             result_limit=10,
             course_name=dto.course.name,
+            course_id=dto.course.id,
             problem_statement=problem_statement,
             exercise_title=exercise_title,
         )

app/retrieval/lecture_retrieval.py

@@ -104,6 +104,7 @@ def __call__(
         result_limit: int,
         course_name: str = None,
         course_id: int = None,
+        base_url: str = None,
         problem_statement: str = None,
         exercise_title: str = None,
     ) -> List[dict]:
@@ -126,6 +127,7 @@ def __call__(
             course_language=course_language,
             course_name=course_name,
             course_id=course_id,
+            base_url=base_url,
             problem_statement=problem_statement,
             exercise_title=exercise_title,
         )
@@ -141,11 +143,41 @@ def __call__(
         merged_chunks = merge_retrieved_chunks(
             basic_retrieved_lecture_chunks, hyde_retrieved_lecture_chunks
         )
+        if len(merged_chunks) != 0:
+            selected_chunks_index = self.reranker_pipeline(
+                paragraphs=merged_chunks, query=student_query, chat_history=chat_history
+            )
+            return [merged_chunks[int(i)] for i in selected_chunks_index]
+        return []
 
-        selected_chunks_index = self.reranker_pipeline(
-            paragraphs=merged_chunks, query=student_query, chat_history=chat_history
+    def basic_lecture_retrieval(
+        self,
+        chat_history: list[PyrisMessage],
+        student_query: str,
+        result_limit: int,
+        course_name: str = None,
+        course_id: int = None,
+        base_url: str = None,
+    ) -> list[dict[str, dict]]:
+        """
+        Basic retrieval for pipelines thaat need performance and fast answers.
+        """
+        rewritten_query = self.rewrite_student_query(
+            chat_history, student_query, "course_language", course_name
+        )
+        response = self.search_in_db(
+            query=rewritten_query,
+            hybrid_factor=0.9,
+            result_limit=result_limit,
+            course_id=course_id,
+            base_url=base_url,
         )
-        return [merged_chunks[int(i)] for i in selected_chunks_index]
+
+        basic_retrieved_lecture_chunks: list[dict[str, dict]] = [
+            {"id": obj.uuid.int, "properties": obj.properties}
+            for obj in response.objects
+        ]
+        return basic_retrieved_lecture_chunks
 
     def rewrite_student_query(
         self,
@@ -290,29 +322,51 @@ def rewrite_elaborated_query_with_exercise_context(
             raise e
 
     def search_in_db(
-        self, query: str, hybrid_factor: float, result_limit: int, course_id: int = None
+        self,
+        query: str,
+        hybrid_factor: float,
+        result_limit: int,
+        course_id: int = None,
+        base_url: str = None,
     ):
         """
-        Search the query in the database and return the results.
+        Search the database for the given query.
         """
-        return self.collection.query.hybrid(
+        # Initialize filter to None by default
+        filter_weaviate = None
+
+        # Check if course_id is provided
+        if course_id:
+            # Create a filter for course_id
+            filter_weaviate = Filter.by_property(LectureSchema.COURSE_ID.value).equal(
+                course_id
+            )
+
+            # Extend the filter based on the presence of base_url
+            if base_url:
+                filter_weaviate &= Filter.by_property(
+                    LectureSchema.BASE_URL.value
+                ).equal(base_url)
+            else:
+                filter_weaviate = Filter.by_property(
+                    LectureSchema.BASE_URL.value
+                ).equal(base_url)
+
+        return_value = self.collection.query.hybrid(
             query=query,
-            filters=(
-                Filter.by_property(LectureSchema.COURSE_ID.value).equal(course_id)
-                if course_id
-                else None
-            ),
             alpha=hybrid_factor,
             vector=self.llm_embedding.embed(query),
             return_properties=[
                 LectureSchema.PAGE_TEXT_CONTENT.value,
-                LectureSchema.PAGE_IMAGE_DESCRIPTION.value,
                 LectureSchema.COURSE_NAME.value,
                 LectureSchema.LECTURE_NAME.value,
                 LectureSchema.PAGE_NUMBER.value,
+                LectureSchema.COURSE_ID.value,
             ],
             limit=result_limit,
+            filters=filter_weaviate,
         )
+        return return_value
 
     def run_parallel_rewrite_tasks(
         self,
@@ -322,6 +376,7 @@ def run_parallel_rewrite_tasks(
         course_language: str,
         course_name: str = None,
         course_id: int = None,
+        base_url: str = None,
         problem_statement: str = None,
         exercise_title: str = None,
     ):
@@ -351,8 +406,10 @@ def run_parallel_rewrite_tasks(
                 )
 
                 # Get the results once both tasks are complete
-                rewritten_query = rewritten_query_future.result()
-                hypothetical_answer_query = hypothetical_answer_query_future.result()
+                rewritten_query: str = rewritten_query_future.result()
+                hypothetical_answer_query: str = (
+                    hypothetical_answer_query_future.result()
+                )
         else:
             with concurrent.futures.ThreadPoolExecutor() as executor:
                 # Schedule the rewrite tasks to run in parallel
@@ -378,10 +435,20 @@ def run_parallel_rewrite_tasks(
             # Execute the database search tasks
         with concurrent.futures.ThreadPoolExecutor() as executor:
             response_future = executor.submit(
-                self.search_in_db, rewritten_query, 1, result_limit, course_id
+                self.search_in_db,
+                query=rewritten_query,
+                hybrid_factor=0.7,
+                result_limit=result_limit,
+                course_id=course_id,
+                base_url=base_url,
             )
             response_hyde_future = executor.submit(
-                self.search_in_db, hypothetical_answer_query, 1, result_limit, course_id
+                self.search_in_db,
+                query=hypothetical_answer_query,
+                hybrid_factor=0.9,
+                result_limit=result_limit,
+                course_id=course_id,
+                base_url=base_url,
             )
 
             # Get the results once both tasks are complete