Merge branch 'main' into rc-merge-main-20242025

bigbang/tasks_scheduler.py

@@ -107,6 +107,8 @@ def _schedule_migration_and_publication(username, enabled):
             collection_acron=None,
             publication_year=None,
             force_update=False,
+            force_import_acron_id_file=False,
+            force_migrate_document_records=False,
         ),
         description=_("Migra e publica"),
         priority=MIGRATION_PRIORITY,
@@ -129,7 +131,9 @@ def _schedule_migrate_and_publish_journals(username, enabled):
         kwargs=dict(
             username=None,
             collection_acron=None,
+            journal_acron=None,
             force_update=False,
+            force_import_acron_id_file=False,
         ),
         description=_("Migra e publica os periódicos"),
         priority=TITLE_DB_MIGRATION_PRIORITY,
@@ -152,7 +156,11 @@ def _schedule_migrate_and_publish_issues(username, enabled):
         kwargs=dict(
             username=None,
             collection_acron=None,
+            journal_acron=None,
+            publication_year=None,
+            issue_folder=None,
             force_update=False,
+            force_migrate_document_records=False,
         ),
         description=_("Migra e publica os fascículos"),
         priority=ISSUE_DB_MIGRATION_PRIORITY,
@@ -176,7 +184,7 @@ def _schedule_migrate_and_publish_articles(username, enabled):
             issue_folder=None,
             force_update=False,
         ),
-        description=_("Obtém as base de dados 'article'"),
+        description=_("Migra e publica artigos"),
         priority=ARTICLE_DB_MIGRATION_PRIORITY,
         enabled=enabled,
         run_once=False,

core/utils/requester.py

@@ -83,6 +83,10 @@ def post_data(
     try:
         response.raise_for_status()
     except requests.HTTPError as exc:
+        try:
+            return response.json()
+        except Exception as json_error:
+            pass
         if 400 <= exc.response.status_code < 500:
             raise NonRetryableError(exc) from exc
         elif 500 <= exc.response.status_code < 600:

journal/migrations/0005_officialjournal_next_journal_title_and_more.py

@@ -0,0 +1,29 @@
+# Generated by Django 5.0.3 on 2024-10-12 22:53
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("journal", "0004_journal_contact_address_journal_contact_location_and_more"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="officialjournal",
+            name="next_journal_title",
+            field=models.CharField(blank=True, max_length=128, null=True),
+        ),
+        migrations.AddField(
+            model_name="officialjournal",
+            name="previous_journal_title",
+            field=models.CharField(blank=True, max_length=128, null=True),
+        ),
+        migrations.AlterField(
+            model_name="journal",
+            name="journal_acron",
+            field=models.CharField(
+                blank=True, max_length=16, null=True, verbose_name="Journal Acronym"
+            ),
+        ),
+    ]

journal/models.py

@@ -148,6 +148,8 @@ class OfficialJournal(CommonControlField):
         _("ISSN Eletronic"), max_length=9, null=True, blank=True
     )
     issnl = models.CharField(_("ISSNL"), max_length=9, null=True, blank=True)
+    previous_journal_title = models.CharField(max_length=128, null=True, blank=True)
+    next_journal_title = models.CharField(max_length=128, null=True, blank=True)
 
     base_form_class = OfficialJournalForm
 
@@ -195,14 +197,14 @@ def data(self):
         return d
 
     @classmethod
-    def get(cls, issn_print=None, issn_electronic=None, issnl=None):
-        logging.info(f"OfficialJournal.get({issn_print}, {issn_electronic}, {issnl})")
+    def get(cls, issn_print=None, issn_electronic=None, issnl=None, title=None):
         if issn_electronic:
             return cls.objects.get(issn_electronic=issn_electronic)
         if issn_print:
             return cls.objects.get(issn_print=issn_print)
         if issnl:
             return cls.objects.get(issnl=issnl)
+        raise ValueError(f"{title} - OfficialJournal.get requires issn_print, issn_electronic")
 
     @classmethod
     def create_or_update(
@@ -215,9 +217,6 @@ def create_or_update(
         title_iso=None,
         foundation_year=None,
     ):
-        logging.info(
-            f"OfficialJournal.create_or_update({issn_print}, {issn_electronic}, {issnl})"
-        )
         try:
             obj = cls.get(issn_print, issn_electronic, issnl)
             obj.updated_by = user
@@ -233,9 +232,13 @@ def create_or_update(
         obj.issn_electronic = issn_electronic or obj.issn_electronic
         obj.foundation_year = foundation_year or obj.foundation_year
         obj.save()
-        logging.info(f"return {obj}")
         return obj
 
+    def add_related_journal(self, previous_journal_title, next_journal_title):
+        self.previous_journal_title = previous_journal_title
+        self.next_journal_title = next_journal_title
+        self.save()
+
 
 class Journal(CommonControlField, ClusterableModel):
     """
@@ -246,7 +249,7 @@ class Journal(CommonControlField, ClusterableModel):
         _("Short Title"), max_length=100, null=True, blank=True
     )
     title = models.CharField(_("Title"), max_length=265, null=True, blank=True)
-    journal_acron = models.CharField(_("Journal Acronym"), max_length=8, null=True, blank=True)
+    journal_acron = models.CharField(_("Journal Acronym"), max_length=16, null=True, blank=True)
     official_journal = models.ForeignKey(
         "OfficialJournal",
         null=True,
@@ -282,7 +285,7 @@ def __str__(self):
     panels_identification = [
         AutocompletePanel("official_journal"),
         FieldPanel("short_title"),
-        FieldPanel("journal_acron")
+        FieldPanel("journal_acron"),
     ]
 
     panels_owner = [
@@ -306,6 +309,14 @@ def __str__(self):
         ]
     )
 
+    @property
+    def issn_print(self):
+        return self.official_journal.issn_print
+
+    @property
+    def issn_electronic(self):
+        return self.official_journal.issn_electronic
+
     @property
     def first_letters(self):
         return "".join(
@@ -389,10 +400,8 @@ def create_or_update(
         short_title=None,
         journal_acron=None,
     ):
-        logging.info(f"Journal.create_or_update({official_journal}")
         try:
             obj = cls.get(official_journal=official_journal)
-            logging.info("update {}".format(obj))
             obj.updated_by = user
             obj.updated = datetime.utcnow()
             obj.official_journal = official_journal or obj.official_journal
@@ -510,7 +519,6 @@ def create_or_update(
         journal=None,
         email=None,
     ):
-        logging.info(f"Journal.create_or_update({journal})")
         try:
             obj = cls.get(journal, email)
             obj.journal = journal
@@ -889,13 +897,14 @@ def data(self):
             "year": self.year,
             "month": self.month,
             "day": self.day,
+            "date": self.date,
         }
 
         return d
 
     @property
     def date(self):
-        return f"{self.year}-{str(self.month).zfill(2)}-{str(self.day).zfill(2)}"
+        return "-".join((str(i).zfill(2) for i in [self.year, self.month, self.day] if i))
 
     @property
     def opac_event_type(self):

journal/wagtail_hooks.py

@@ -63,11 +63,13 @@ class JournalAdmin(ModelAdmin):
     add_to_settings_menu = False
     exclude_from_explorer = False
 
-    list_display = ("title", "short_title")
+    list_display = ("title", "journal_acron", "issn_electronic", "issn_print")
     search_fields = (
         "official_journal__issn_electronic",
         "official_journal__issn_print",
-        "short_title",
+        "official_journal__title",
+        "title",
+        "journal_acron",
     )
 
 

migration/controller.py

@@ -56,14 +56,37 @@ def create_or_update_journal(
     classic_website_journal = classic_ws.Journal(journal_data)
 
     year, month, day = parse_yyyymmdd(classic_website_journal.first_year)
-    official_journal = OfficialJournal.create_or_update(
-        user=user,
-        issn_electronic=classic_website_journal.electronic_issn,
-        issn_print=classic_website_journal.print_issn,
-        title=classic_website_journal.title,
-        title_iso=classic_website_journal.title_iso,
-        foundation_year=year,
-    )
+    try:
+        eissn = classic_website_journal.electronic_issn
+        pissn = classic_website_journal.print_issn
+        if not eissn and not pissn:
+            issn = classic_website_journal.get_field_content(
+                "v935", subfields={"_": "issn"}, single=True, simple=True)
+            issn_type = classic_website_journal.get_field_content(
+                "v035", subfields={"_": "issn_type"}, single=True, simple=True)
+            if issn and issn_type:
+                if issn_type["issn_type"] == "PRINT":
+                    pissn = issn["issn"]
+                elif issn_type["issn_type"] == "ONLIN":
+                    eissn = issn["issn"]
+
+        if not eissn and not pissn:
+            raise ValueError(f"Missing ISSN for {journal_data}")
+        official_journal = OfficialJournal.create_or_update(
+            user=user,
+            issn_electronic=eissn,
+            issn_print=pissn,
+            title=classic_website_journal.title,
+            title_iso=classic_website_journal.title_iso,
+            foundation_year=year,
+        )
+        official_journal.add_related_journal(
+            classic_website_journal.previous_title,
+            classic_website_journal.next_title,
+        )
+    except ValueError as e:
+        logging.exception(f"{journal_proc} {journal_data} {e}")
+        raise e
     journal = Journal.create_or_update(
         user=user,
         official_journal=official_journal,
@@ -85,12 +108,42 @@ def create_or_update_journal(
     journal.add_email(classic_website_journal.publisher_email)
     journal.save()
 
+    journal_proc.update(
+        user=user,
+        journal=journal,
+        acron=classic_website_journal.acronym,
+        title=classic_website_journal.title,
+        availability_status=classic_website_journal.current_status,
+        migration_status=tracker_choices.PROGRESS_STATUS_DONE,
+        force_update=force_update,
+    )
+
+    missions = {}
     for item in classic_website_journal.mission:
+        try:
+            text = item["text"]
+        except KeyError as exc:
+            text = ''
+            logging.exception(f"{journal} - Missing mission data {item}")
+            continue
+        try:
+            lang = item["language"]
+        except KeyError as exc:
+            lang = None
+            logging.info(f"Mission no lang (1). {journal_proc} {item}")
+
+        if not lang or not text:
+            logging.info(f"Mission no lang or no text (2). {journal_proc} {item}")
+            continue
+        missions.setdefault(lang, [])
+        missions[lang].append(text)
+
+    for lang, text in missions.items():
         language = Language.get_or_create(
-            name=None, code2=item["language"], creator=user
+            name=None, code2=lang, creator=user
         )
         journal.mission.add(
-            Mission.create_or_update(user, journal, language, item["text"])
+            Mission.create_or_update(user, journal, language, "\n".join(text))
         )
 
     for code in classic_website_journal.subject_areas:
@@ -109,16 +162,6 @@ def create_or_update_journal(
         journal.owner.add(Owner.create_or_update(user, journal, institution))
         journal.publisher.add(Publisher.create_or_update(user, journal, institution))
 
-    journal_proc.update(
-        user=user,
-        journal=journal,
-        acron=classic_website_journal.acronym,
-        title=classic_website_journal.title,
-        availability_status=classic_website_journal.current_status,
-        migration_status=tracker_choices.PROGRESS_STATUS_DONE,
-        force_update=force_update,
-    )
-
     jc = JournalCollection.create_or_update(user, collection, journal)
 
     create_journal_history(user, jc, classic_website_journal)
@@ -795,9 +838,11 @@ def register_acron_id_file_content(
             )
 
             completed = True
+            total = journal_id_file.id_file_records.filter().count()
+            detail = {"total": total}
             if force_update or start < journal_id_file.updated.isoformat():
                 completed = False
-                total = 0
+                done = 0
                 changed = 0
 
                 logging.info(f"Reading {source_path}")
@@ -815,19 +860,15 @@ def register_acron_id_file_content(
                     )
                     if force_update or start < rec.updated.isoformat():
                         changed += 1
-                    total += 1
-                completed = (
-                    journal_id_file.id_file_records.filter().count() == total
-                )
+                    done += 1
+                completed = total == done
 
                 detail.update({
                     "force_update": force_update,
-                    "total": total,
+                    "done": done,
                     "changed": changed,
                     "updated": journal_id_file.updated.isoformat(),
                 })
-                logging.info(f"detail: {detail}")
-
             operation.finish(
                 user,
                 completed=completed,