Fix more tests to deal with answer many-to-many with theme

consultation_analyser/consultations/download_consultation.py

@@ -16,7 +16,8 @@ def select_keys_from_model(model, keys):
 
 def consultation_to_json(consultation):
     """
-    Return the consultation in a format compliant with the public schema
+    Return the consultation in a format compliant with the public schema.
+    Return the latest theme for each answer.
 
     Raises:
         pydantic.ValidationError: if the generated JSON is not compliant
@@ -62,7 +63,9 @@ def consultation_to_json(consultation):
                 answer, ["question", "multiple_choice", "free_text"]
             )
 
-            answer_attrs["theme_id"] = str(answer.theme.id) if answer.theme else None
+            answer_attrs["theme_id"] = (
+                str(answer.theme_from_latest_run.id) if answer.theme_from_latest_run else None
+            )
             answer_attrs["question_id"] = str(answer_attrs.pop("question"))
             answers.append(answer_attrs)
 
@@ -73,6 +76,7 @@ def consultation_to_json(consultation):
     if themes:
         attrs["themes"] = themes
         ConsultationWithResponsesAndThemes(**attrs)
+        pass
     else:
         ConsultationWithResponses(**attrs)
 

consultation_analyser/consultations/dummy_data.py

@@ -6,11 +6,11 @@
     ConsultationFactory,
     ConsultationResponseFactory,
     FakeConsultationData,
+    ProcessingRunFactory,
     QuestionFactory,
     SectionFactory,
     ThemeFactory,
-    ProcessingRunFactory,
-    TopicModelMetadataFactory
+    TopicModelMetadataFactory,
 )
 from consultation_analyser.hosting_environment import HostingEnvironment
 
@@ -60,15 +60,18 @@ def create_dummy_data(responses=10, include_themes=True, number_questions=10, **
                     )
                 )
             else:
-                answers.append(
-                    AnswerFactory(question=q, consultation_response=response)
-                )
+                answers.append(AnswerFactory(question=q, consultation_response=response))
         if include_themes:
             processing_run = ProcessingRunFactory(consultation=consultation)
             # Set themes per question, multiple answers with the same theme
             for q in questions:
-                topic_model_metadata = TopicModelMetadataFactory(processing_run=processing_run, question=q)
-                themes = [ThemeFactory(topic_id=i, topic_model_metadata=topic_model_metadata) for i in range(-1, 4)]
+                topic_model_metadata = TopicModelMetadataFactory(
+                    processing_run=processing_run, question=q
+                )
+                themes = [
+                    ThemeFactory(topic_id=i, topic_model_metadata=topic_model_metadata)
+                    for i in range(-1, 4)
+                ]
                 for a in answers:
                     random_theme = random.choice(themes)
                     a.theme.add(random_theme)

consultation_analyser/consultations/models.py

@@ -186,8 +186,13 @@ def save_theme_to_answer(
             topic_keywords=topic_keywords,
             topic_id=topic_id,
         )
-        self.theme = theme
-        self.save()
-
-    def get_theme_from_latest_run(self):
-        self.theme.all().order_by("topic_model_metadata__processing_run__created_at")
+        self.theme.add(theme)
+
+    @property
+    def theme_from_latest_run(self):
+        if not self.theme:
+            return None
+        lastest = (
+            self.theme.all().order_by("topic_model_metadata__processing_run__created_at").last()
+        )
+        return lastest

consultation_analyser/pipeline/llm_summariser.py

@@ -3,7 +3,6 @@
 from consultation_analyser.consultations.models import Theme
 from consultation_analyser.pipeline.backends.llm_backend import LLMBackend
 
-from .backends.types import ThemeSummary
 
 logger = logging.getLogger("pipeline")
 
@@ -12,11 +11,10 @@ def create_llm_summaries_for_consultation(consultation, llm_backend: LLMBackend)
     logger.info(
         f"Starting LLM summarisation for consultation: {consultation.name} with backend {llm_backend.__class__.__name__}"
     )
-    themes = Theme.objects.filter(question__section__consultation=consultation).filter(
-        question__has_free_text=True
-    )
+    themes = Theme.objects.filter(
+        topic_model_metadata__processing_run__consultation=consultation
+    ).filter(topic_model_metadata__question__has_free_text=True)
 
-    theme: ThemeSummary
     for theme in themes:
         logger.info(f"Starting LLM summarisation for theme with keywords: {theme.topic_keywords}")
         theme_summary_data = llm_backend.summarise_theme(theme)

consultation_analyser/pipeline/processing.py

@@ -52,8 +52,9 @@ def process_consultation_themes(consultation, topic_backend=None, llm_backend=No
         llm_backend = get_llm_backend(llm_backend)
 
     # TODO - add more metadata to processing run
-    ProcessingRun(consultation=consultation).save()
-    save_themes_for_consultation(consultation.id, topic_backend)
+    processing_run = ProcessingRun(consultation=consultation)
+    processing_run.save()
+    save_themes_for_consultation(consultation.id, topic_backend, processing_run)
     create_llm_summaries_for_consultation(consultation, llm_backend)