Melhora o registro das operações das tarefas relacionadas à migração …

…e publicação (scieloorg#422) * Melhora os rótulos, deixa todos os campos não editáveis, apresenta os eventos do mais recente para o mais antigo * Adiciona Article.data, Issue.data, Journal.data * Adiciona retorno às função que criam instâncias de Article, Issue e Journal * Adiciona Article.data, Issue.data, Journal.data nos detalhes das operações de entrada de dados * Aplica black * Adiciona * Adiciona mais detalhes ao registro da tarefa de gerar o XML a partir do HTML * Adiciona mais detalhes ao registro da tarefa de gerar o pacote SPS * Corrige o valor de 'completed' dos resultados das operações de solicitação de pid v3 * Adiciona o parâmetro compression em ZipFile * Modifica o sps_pkg_status para PENDING se o pacote não tem todos os texts * Modifica o sps_pkg_status para DONE se o pacote não tem todos os texts * Modifica o sps_pkg_status para PENDING se o pacote não tem todos os texts * Corrige ausência de importação de ZIP_DEFLATED * Adiciona o atributo order para a listagem dos itens na área administrativa * Adiciona as migrações de banco de dados * Adiciona detalhes do processamento da adição de arquivos no minio
robertatakenaka · Mar 29, 2024 · ebb61b5 · ebb61b5
1 parent cb75b16
commit ebb61b5
Show file tree

Hide file tree

Showing 13 changed files with 216 additions and 59 deletions.
diff --git a/article/models.py b/article/models.py
@@ -117,13 +117,25 @@ class Meta:
 
     base_form_class = ArticleForm
 
-    autocomplete_search_field = "pid_v3"
+    autocomplete_search_field = "sps_pkg__sps_pkg_name"
 
     def autocomplete_label(self):
-        return self.pid_v3
+        return self.sps_pkg.sps_pkg_name
 
     def __str__(self):
-        return f"{self.pid_v3}"
+        return f"{self.sps_pkg.sps_pkg_name}"
+
+    @property
+    def data(self):
+        # TODO completar com itens que identifique o artigo
+        return dict(
+            xml=self.sps_pkg and self.sps_pkg.xml_uri,
+            issue=self.issue.data,
+            journal=self.journal.data,
+            pid_v3=self.pid_v3,
+            created=created.isoformat(),
+            updated=updated.isoformat(),
+        )
 
     @classmethod
     def get(cls, pid_v3):

diff --git a/htmlxml/migrations/0002_alter_htmlxml_options.py b/htmlxml/migrations/0002_alter_htmlxml_options.py
@@ -0,0 +1,16 @@
+# Generated by Django 5.0.3 on 2024-03-29 17:32
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("htmlxml", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name="htmlxml",
+            options={"ordering": ["-updated"]},
+        ),
+    ]
diff --git a/htmlxml/models.py b/htmlxml/models.py
@@ -23,6 +23,7 @@
 from core.models import CommonControlField
 from package.models import BasicXMLFile
 from migration.models import MigratedArticle
+
 # from tracker.models import EventLogger
 from tracker import choices as tracker_choices
 
@@ -84,6 +85,7 @@ class BodyAndBackFile(BasicXMLFile, Orderable):
     ]
 
     class Meta:
+
         indexes = [
             models.Index(fields=["version"]),
         ]
@@ -133,9 +135,9 @@ def create_or_update(cls, user, bb_parent, version, file_content, pkg_name):
             return obj
         except Exception as e:
             raise exceptions.CreateOrUpdateBodyAndBackFileError(
-                _(
-                    "Unable to create_or_update_body and back file {} {} {} {}"
-                ).format(bb_parent, version, type(e), e)
+                _("Unable to create_or_update_body and back file {} {} {} {}").format(
+                    bb_parent, version, type(e), e
+                )
             )
 
 
@@ -214,6 +216,7 @@ def data(self):
     ]
 
     class Meta:
+
         indexes = [
             models.Index(fields=["attention_demands"]),
         ]
@@ -491,6 +494,8 @@ def autocomplete_label(self):
         return self.migrated_article
 
     class Meta:
+        ordering = ['-updated']
+
         indexes = [
             models.Index(fields=["html2xml_status"]),
             models.Index(fields=["quality"]),
@@ -561,14 +566,25 @@ def html_to_xml(
     ):
         try:
             self.html2xml_status = tracker_choices.PROGRESS_STATUS_DOING
-            self.html_translation_langs = "-".join(sorted(article_proc.translations.keys()))
-            self.pdf_langs = "-".join(sorted([item.lang or article_proc.main_lang for item in article_proc.renditions]))
+            self.html_translation_langs = "-".join(
+                sorted(article_proc.translations.keys())
+            )
+            self.pdf_langs = "-".join(
+                sorted(
+                    [
+                        item.lang or article_proc.main_lang
+                        for item in article_proc.renditions
+                    ]
+                )
+            )
             self.save()
 
             document = Document(article_proc.migrated_data.data)
             document._translated_html_by_lang = article_proc.translations
 
-            body_and_back = self._generate_xml_body_and_back(user, article_proc, document)
+            body_and_back = self._generate_xml_body_and_back(
+                user, article_proc, document
+            )
             xml_content = self._generate_xml_from_html(user, article_proc, document)
 
             if xml_content and body_and_back:
@@ -615,7 +631,14 @@ def generate_report(self, user, article_proc):
             else:
                 self.quality = choices.HTML2XML_QA_NOT_EVALUATED
             self.save()
-            op.finish(user, completed=True)
+            op.finish(
+                user,
+                completed=True,
+                detail={
+                    "attention_demands": self.attention_demands,
+                    "quality": self.quality,
+                },
+            )
         except Exception as e:
             op.finish(user, completed=False, detail={"error": str(e)})
 
@@ -625,9 +648,12 @@ def _generate_xml_body_and_back(self, user, article_proc, document):
         """
         done = False
         operation = article_proc.start(user, "generate xml body and back")
+
+        languages = document._translated_html_by_lang
         detail = {}
+        detail.update(languages)
         try:
-            document.generate_body_and_back_from_html(document._translated_html_by_lang)
+            document.generate_body_and_back_from_html(languages)
             done = True
         except GenerateBodyAndBackFromHTMLError as e:
             # cria xml_body_and_back padrão
@@ -645,7 +671,7 @@ def _generate_xml_body_and_back(self, user, article_proc, document):
                     file_content=xml_body_and_back,
                     pkg_name=article_proc.pkg_name,
                 )
-
+                detail["xml_to_html_steps"] = i
         operation.finish(user, done, detail=detail)
         return done
 
@@ -655,7 +681,9 @@ def _generate_xml_from_html(self, user, article_proc, document):
         detail = {}
         try:
             xml_content = document.generate_full_xml(None).decode("utf-8")
-            self.save_file(article_proc.pkg_name + ".xml", xml_content)
+            xml_file = article_proc.pkg_name + ".xml"
+            self.save_file(xml_file, xml_content)
+            detail["xml"] = xml_file
         except Exception as e:
             detail = {"error": str(e)}
         operation.finish(user, bool(xml_content), detail=detail)

diff --git a/issue/models.py b/issue/models.py
@@ -47,6 +47,18 @@ def __str__(self):
     supplement = models.CharField(_("Supplement"), max_length=16, null=True, blank=True)
     publication_year = models.CharField(_("Year"), max_length=4, null=True, blank=True)
 
+    @property
+    def data(self):
+        return dict(
+            journal=self.journal.data,
+            volume=self.volume,
+            number=self.number,
+            supplement=self.supplement,
+            publication_year=self.publication_year,
+            created=created.isoformat(),
+            updated=updated.isoformat(),
+        )
+
     @staticmethod
     def autocomplete_custom_queryset_filter(search_term):
         parts = search_term.split()
@@ -60,7 +72,12 @@ def autocomplete_custom_queryset_filter(search_term):
         )
 
     def autocomplete_label(self):
-        return f"{self.journal.title} {self.volume or self.number}"
+        return "%s %s%s%s" % (
+            self.journal,
+            self.volume and f"v{self.volume}",
+            self.number and f"n{self.number}",
+            self.supplement and f"s{self.supplement}",
+        )
 
     panels = [
         AutocompletePanel("journal"),

diff --git a/journal/models.py b/journal/models.py
@@ -167,6 +167,17 @@ def __str__(self):
         ]
     )
 
+    @property
+    def data(self):
+        return dict(
+            title=self.title,
+            issn_print=self.official_journal.issn_print,
+            issn_electronic=self.official_journal.issn_electronic,
+            foundation_year=self.official_journal.foundation_year,
+            created=created.isoformat(),
+            updated=updated.isoformat(),
+        )
+
     def autocomplete_label(self):
         return self.title or self.official_journal.title
 

diff --git a/migration/controller.py b/migration/controller.py
@@ -3,7 +3,7 @@
 import sys
 from copy import deepcopy
 from datetime import datetime
-from zipfile import ZipFile
+from zipfile import ZipFile, ZIP_DEFLATED
 
 from django.utils.translation import gettext_lazy as _
 from scielo_classic_website import classic_ws
@@ -327,7 +327,7 @@ def build_sps_package(
         sps_pkg_zip_path = os.path.join(output_folder, f"{self.sps_pkg_name}.zip")
 
         # cria pacote zip
-        with ZipFile(sps_pkg_zip_path, "w") as zf:
+        with ZipFile(sps_pkg_zip_path, "w", compression=ZIP_DEFLATED) as zf:
 
             # A partir do XML, obtém os nomes dos arquivos dos ativos digitais
             self._build_sps_package_add_assets(zf, issue_proc)

diff --git a/package/migrations/0002_alter_spspkg_options.py b/package/migrations/0002_alter_spspkg_options.py
@@ -0,0 +1,16 @@
+# Generated by Django 5.0.3 on 2024-03-29 17:32
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("package", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name="spspkg",
+            options={"ordering": ["-updated"]},
+        ),
+    ]
diff --git a/package/models.py b/package/models.py
@@ -4,7 +4,7 @@
 import sys
 from datetime import datetime
 from tempfile import TemporaryDirectory
-from zipfile import ZipFile
+from zipfile import ZipFile, ZIP_DEFLATED
 
 from django.core.files.base import ContentFile
 from django.db.models import Q
@@ -392,6 +392,8 @@ def __str__(self):
     )
 
     class Meta:
+        ordering = ['-updated']
+
         indexes = [
             models.Index(fields=["pid_v3"]),
             models.Index(fields=["sps_pkg_name"]),
@@ -482,7 +484,6 @@ def create_or_update(
 
             obj.validate(True)
 
-            logging.info(f"Depois de criar sps_pkg.pid_v3: {obj.pid_v3}")
             article_proc.update_sps_pkg_status()
             operation.finish(user, completed=obj.is_complete, detail=obj.data)
 
@@ -570,7 +571,7 @@ def add_pid_v3_to_zip(cls, user, zip_xml_file_path, is_public, article_proc):
 
                 if response.get("xml_changed"):
                     # atualiza conteúdo de zip
-                    with ZipFile(zip_xml_file_path, "a") as zf:
+                    with ZipFile(zip_xml_file_path, "a", compression=ZIP_DEFLATED) as zf:
                         zf.writestr(
                             response["filename"],
                             xml_with_pre.tostring(pretty_print=True),
@@ -687,7 +688,12 @@ def _save_components_in_cloud(self, user, original_pkg_components, article_proc)
                     component_data,
                     failures,
                 )
-        op.finish(user, completed=not failures, detail=failures)
+        items = [
+            dict(basename=c.basename, uri=c.uri)
+            for c in self.components.all()
+        ]
+        detail = {"items": items, "failures": failures}
+        op.finish(user, completed=not failures, detail=detail)
         return xml_with_pre
 
     def _save_component_in_cloud(
@@ -704,8 +710,8 @@ def _save_component_in_cloud(
             uri = None
             failures.append(
                 dict(
-                    item_id=item,
-                    response=response,
+                    basename=item,
+                    error=str(e),
                 )
             )
         self.components.add(
@@ -741,7 +747,6 @@ def _save_xml_in_cloud(self, user, xml_with_pre, article_proc):
             uri = response["uri"]
         except Exception as e:
             uri = None
-            op.finish(user, completed=False, detail=response)
         self.xml_uri = uri
         self.save()
         self.components.add(
@@ -755,7 +760,7 @@ def _save_xml_in_cloud(self, user, xml_with_pre, article_proc):
                 legacy_uri=None,
             )
         )
-        op.finish(user, completed=True)
+        op.finish(user, completed=bool(uri), detail=response)
 
     def synchronize(self, user, article_proc):
         zip_xml_file_path = self.file.path
@@ -769,7 +774,7 @@ def synchronize(self, user, article_proc):
 
             if response.get("v3") and self.pid_v3 != response.get("v3"):
                 # atualiza conteúdo de zip
-                with ZipFile(zip_xml_file_path, "a") as zf:
+                with ZipFile(zip_xml_file_path, "a", compression=ZIP_DEFLATED) as zf:
                     zf.writestr(
                         response["filename"],
                         response["xml_with_pre"].tostring(pretty_print=True),

diff --git a/pid_provider/requester.py b/pid_provider/requester.py
@@ -86,6 +86,7 @@ def request_pid_for_xml_with_pre(
         )
 
         if registered.get("error_type"):
+            main_op.finish(user, completed=False, detail=registered)
             return registered
 
         # Solicita pid para Core
@@ -114,7 +115,7 @@ def request_pid_for_xml_with_pre(
             registered["registered_in_upload"] = bool(resp.get("v3"))
             op.finish(
                 user,
-                completed=True,
+                completed=registered["registered_in_upload"],
                 detail={"registered": registered, "response": resp},
             )
 
@@ -127,7 +128,7 @@ def request_pid_for_xml_with_pre(
 
         detail = registered.copy()
         detail["xml_with_pre"] = xml_with_pre.data
-        main_op.finish(user, completed=True, detail={"registered": detail})
+        main_op.finish(user, completed=registered["synchronized"], detail=detail)
         return registered
 
     @staticmethod
@@ -144,6 +145,7 @@ def get_registration_demand(xml_with_pre, article_proc, user):
 
         registered = PidProviderXML.is_registered(xml_with_pre)
         if registered.get("error_type"):
+            op.finish(user, completed=False, detail=registered)
             return registered
 
         if registered.get("is_equal"):
@@ -155,7 +157,7 @@ def get_registration_demand(xml_with_pre, article_proc, user):
             registered["do_core_registration"] = True
             registered["do_upload_registration"] = True
 
-        op.finish(user, completed=True, detail={"registered": registered})
+        op.finish(user, completed=True, detail=registered)
 
         return registered
 
@@ -187,7 +189,7 @@ def core_registration(self, xml_with_pre, registered, article_proc, user):
             registered["registered_in_core"] = bool(response.get("v3"))
             op.finish(
                 user,
-                completed=True,
+                completed=registered["registered_in_core"],
                 detail={"registered": registered, "response": response},
             )