diff --git a/article/models.py b/article/models.py index 19a6ef48..04440dec 100644 --- a/article/models.py +++ b/article/models.py @@ -117,13 +117,25 @@ class Meta: base_form_class = ArticleForm - autocomplete_search_field = "pid_v3" + autocomplete_search_field = "sps_pkg__sps_pkg_name" def autocomplete_label(self): - return self.pid_v3 + return self.sps_pkg.sps_pkg_name def __str__(self): - return f"{self.pid_v3}" + return f"{self.sps_pkg.sps_pkg_name}" + + @property + def data(self): + # TODO completar com itens que identifique o artigo + return dict( + xml=self.sps_pkg and self.sps_pkg.xml_uri, + issue=self.issue.data, + journal=self.journal.data, + pid_v3=self.pid_v3, + created=created.isoformat(), + updated=updated.isoformat(), + ) @classmethod def get(cls, pid_v3): diff --git a/htmlxml/migrations/0002_alter_htmlxml_options.py b/htmlxml/migrations/0002_alter_htmlxml_options.py new file mode 100644 index 00000000..95b3d558 --- /dev/null +++ b/htmlxml/migrations/0002_alter_htmlxml_options.py @@ -0,0 +1,16 @@ +# Generated by Django 5.0.3 on 2024-03-29 17:32 + +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("htmlxml", "0001_initial"), + ] + + operations = [ + migrations.AlterModelOptions( + name="htmlxml", + options={"ordering": ["-updated"]}, + ), + ] diff --git a/htmlxml/models.py b/htmlxml/models.py index c2fdd342..40813083 100644 --- a/htmlxml/models.py +++ b/htmlxml/models.py @@ -23,6 +23,7 @@ from core.models import CommonControlField from package.models import BasicXMLFile from migration.models import MigratedArticle + # from tracker.models import EventLogger from tracker import choices as tracker_choices @@ -84,6 +85,7 @@ class BodyAndBackFile(BasicXMLFile, Orderable): ] class Meta: + indexes = [ models.Index(fields=["version"]), ] @@ -133,9 +135,9 @@ def create_or_update(cls, user, bb_parent, version, file_content, pkg_name): return obj except Exception as e: raise exceptions.CreateOrUpdateBodyAndBackFileError( - _( - "Unable to create_or_update_body and back file {} {} {} {}" - ).format(bb_parent, version, type(e), e) + _("Unable to create_or_update_body and back file {} {} {} {}").format( + bb_parent, version, type(e), e + ) ) @@ -214,6 +216,7 @@ def data(self): ] class Meta: + indexes = [ models.Index(fields=["attention_demands"]), ] @@ -491,6 +494,8 @@ def autocomplete_label(self): return self.migrated_article class Meta: + ordering = ['-updated'] + indexes = [ models.Index(fields=["html2xml_status"]), models.Index(fields=["quality"]), @@ -561,14 +566,25 @@ def html_to_xml( ): try: self.html2xml_status = tracker_choices.PROGRESS_STATUS_DOING - self.html_translation_langs = "-".join(sorted(article_proc.translations.keys())) - self.pdf_langs = "-".join(sorted([item.lang or article_proc.main_lang for item in article_proc.renditions])) + self.html_translation_langs = "-".join( + sorted(article_proc.translations.keys()) + ) + self.pdf_langs = "-".join( + sorted( + [ + item.lang or article_proc.main_lang + for item in article_proc.renditions + ] + ) + ) self.save() document = Document(article_proc.migrated_data.data) document._translated_html_by_lang = article_proc.translations - body_and_back = self._generate_xml_body_and_back(user, article_proc, document) + body_and_back = self._generate_xml_body_and_back( + user, article_proc, document + ) xml_content = self._generate_xml_from_html(user, article_proc, document) if xml_content and body_and_back: @@ -615,7 +631,14 @@ def generate_report(self, user, article_proc): else: self.quality = choices.HTML2XML_QA_NOT_EVALUATED self.save() - op.finish(user, completed=True) + op.finish( + user, + completed=True, + detail={ + "attention_demands": self.attention_demands, + "quality": self.quality, + }, + ) except Exception as e: op.finish(user, completed=False, detail={"error": str(e)}) @@ -625,9 +648,12 @@ def _generate_xml_body_and_back(self, user, article_proc, document): """ done = False operation = article_proc.start(user, "generate xml body and back") + + languages = document._translated_html_by_lang detail = {} + detail.update(languages) try: - document.generate_body_and_back_from_html(document._translated_html_by_lang) + document.generate_body_and_back_from_html(languages) done = True except GenerateBodyAndBackFromHTMLError as e: # cria xml_body_and_back padrão @@ -645,7 +671,7 @@ def _generate_xml_body_and_back(self, user, article_proc, document): file_content=xml_body_and_back, pkg_name=article_proc.pkg_name, ) - + detail["xml_to_html_steps"] = i operation.finish(user, done, detail=detail) return done @@ -655,7 +681,9 @@ def _generate_xml_from_html(self, user, article_proc, document): detail = {} try: xml_content = document.generate_full_xml(None).decode("utf-8") - self.save_file(article_proc.pkg_name + ".xml", xml_content) + xml_file = article_proc.pkg_name + ".xml" + self.save_file(xml_file, xml_content) + detail["xml"] = xml_file except Exception as e: detail = {"error": str(e)} operation.finish(user, bool(xml_content), detail=detail) diff --git a/issue/models.py b/issue/models.py index 860b0012..af270dfd 100644 --- a/issue/models.py +++ b/issue/models.py @@ -47,6 +47,18 @@ def __str__(self): supplement = models.CharField(_("Supplement"), max_length=16, null=True, blank=True) publication_year = models.CharField(_("Year"), max_length=4, null=True, blank=True) + @property + def data(self): + return dict( + journal=self.journal.data, + volume=self.volume, + number=self.number, + supplement=self.supplement, + publication_year=self.publication_year, + created=created.isoformat(), + updated=updated.isoformat(), + ) + @staticmethod def autocomplete_custom_queryset_filter(search_term): parts = search_term.split() @@ -60,7 +72,12 @@ def autocomplete_custom_queryset_filter(search_term): ) def autocomplete_label(self): - return f"{self.journal.title} {self.volume or self.number}" + return "%s %s%s%s" % ( + self.journal, + self.volume and f"v{self.volume}", + self.number and f"n{self.number}", + self.supplement and f"s{self.supplement}", + ) panels = [ AutocompletePanel("journal"), diff --git a/journal/models.py b/journal/models.py index e932d752..5cfbc484 100644 --- a/journal/models.py +++ b/journal/models.py @@ -167,6 +167,17 @@ def __str__(self): ] ) + @property + def data(self): + return dict( + title=self.title, + issn_print=self.official_journal.issn_print, + issn_electronic=self.official_journal.issn_electronic, + foundation_year=self.official_journal.foundation_year, + created=created.isoformat(), + updated=updated.isoformat(), + ) + def autocomplete_label(self): return self.title or self.official_journal.title diff --git a/migration/controller.py b/migration/controller.py index c8ccbd7c..3045e35e 100644 --- a/migration/controller.py +++ b/migration/controller.py @@ -3,7 +3,7 @@ import sys from copy import deepcopy from datetime import datetime -from zipfile import ZipFile +from zipfile import ZipFile, ZIP_DEFLATED from django.utils.translation import gettext_lazy as _ from scielo_classic_website import classic_ws @@ -327,7 +327,7 @@ def build_sps_package( sps_pkg_zip_path = os.path.join(output_folder, f"{self.sps_pkg_name}.zip") # cria pacote zip - with ZipFile(sps_pkg_zip_path, "w") as zf: + with ZipFile(sps_pkg_zip_path, "w", compression=ZIP_DEFLATED) as zf: # A partir do XML, obtém os nomes dos arquivos dos ativos digitais self._build_sps_package_add_assets(zf, issue_proc) diff --git a/package/migrations/0002_alter_spspkg_options.py b/package/migrations/0002_alter_spspkg_options.py new file mode 100644 index 00000000..a6171c44 --- /dev/null +++ b/package/migrations/0002_alter_spspkg_options.py @@ -0,0 +1,16 @@ +# Generated by Django 5.0.3 on 2024-03-29 17:32 + +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("package", "0001_initial"), + ] + + operations = [ + migrations.AlterModelOptions( + name="spspkg", + options={"ordering": ["-updated"]}, + ), + ] diff --git a/package/models.py b/package/models.py index b0f78059..3a746304 100644 --- a/package/models.py +++ b/package/models.py @@ -4,7 +4,7 @@ import sys from datetime import datetime from tempfile import TemporaryDirectory -from zipfile import ZipFile +from zipfile import ZipFile, ZIP_DEFLATED from django.core.files.base import ContentFile from django.db.models import Q @@ -392,6 +392,8 @@ def __str__(self): ) class Meta: + ordering = ['-updated'] + indexes = [ models.Index(fields=["pid_v3"]), models.Index(fields=["sps_pkg_name"]), @@ -482,7 +484,6 @@ def create_or_update( obj.validate(True) - logging.info(f"Depois de criar sps_pkg.pid_v3: {obj.pid_v3}") article_proc.update_sps_pkg_status() operation.finish(user, completed=obj.is_complete, detail=obj.data) @@ -570,7 +571,7 @@ def add_pid_v3_to_zip(cls, user, zip_xml_file_path, is_public, article_proc): if response.get("xml_changed"): # atualiza conteúdo de zip - with ZipFile(zip_xml_file_path, "a") as zf: + with ZipFile(zip_xml_file_path, "a", compression=ZIP_DEFLATED) as zf: zf.writestr( response["filename"], xml_with_pre.tostring(pretty_print=True), @@ -687,7 +688,12 @@ def _save_components_in_cloud(self, user, original_pkg_components, article_proc) component_data, failures, ) - op.finish(user, completed=not failures, detail=failures) + items = [ + dict(basename=c.basename, uri=c.uri) + for c in self.components.all() + ] + detail = {"items": items, "failures": failures} + op.finish(user, completed=not failures, detail=detail) return xml_with_pre def _save_component_in_cloud( @@ -704,8 +710,8 @@ def _save_component_in_cloud( uri = None failures.append( dict( - item_id=item, - response=response, + basename=item, + error=str(e), ) ) self.components.add( @@ -741,7 +747,6 @@ def _save_xml_in_cloud(self, user, xml_with_pre, article_proc): uri = response["uri"] except Exception as e: uri = None - op.finish(user, completed=False, detail=response) self.xml_uri = uri self.save() self.components.add( @@ -755,7 +760,7 @@ def _save_xml_in_cloud(self, user, xml_with_pre, article_proc): legacy_uri=None, ) ) - op.finish(user, completed=True) + op.finish(user, completed=bool(uri), detail=response) def synchronize(self, user, article_proc): zip_xml_file_path = self.file.path @@ -769,7 +774,7 @@ def synchronize(self, user, article_proc): if response.get("v3") and self.pid_v3 != response.get("v3"): # atualiza conteúdo de zip - with ZipFile(zip_xml_file_path, "a") as zf: + with ZipFile(zip_xml_file_path, "a", compression=ZIP_DEFLATED) as zf: zf.writestr( response["filename"], response["xml_with_pre"].tostring(pretty_print=True), diff --git a/pid_provider/requester.py b/pid_provider/requester.py index 6eee1a15..32dfc9ea 100644 --- a/pid_provider/requester.py +++ b/pid_provider/requester.py @@ -86,6 +86,7 @@ def request_pid_for_xml_with_pre( ) if registered.get("error_type"): + main_op.finish(user, completed=False, detail=registered) return registered # Solicita pid para Core @@ -114,7 +115,7 @@ def request_pid_for_xml_with_pre( registered["registered_in_upload"] = bool(resp.get("v3")) op.finish( user, - completed=True, + completed=registered["registered_in_upload"], detail={"registered": registered, "response": resp}, ) @@ -127,7 +128,7 @@ def request_pid_for_xml_with_pre( detail = registered.copy() detail["xml_with_pre"] = xml_with_pre.data - main_op.finish(user, completed=True, detail={"registered": detail}) + main_op.finish(user, completed=registered["synchronized"], detail=detail) return registered @staticmethod @@ -144,6 +145,7 @@ def get_registration_demand(xml_with_pre, article_proc, user): registered = PidProviderXML.is_registered(xml_with_pre) if registered.get("error_type"): + op.finish(user, completed=False, detail=registered) return registered if registered.get("is_equal"): @@ -155,7 +157,7 @@ def get_registration_demand(xml_with_pre, article_proc, user): registered["do_core_registration"] = True registered["do_upload_registration"] = True - op.finish(user, completed=True, detail={"registered": registered}) + op.finish(user, completed=True, detail=registered) return registered @@ -187,7 +189,7 @@ def core_registration(self, xml_with_pre, registered, article_proc, user): registered["registered_in_core"] = bool(response.get("v3")) op.finish( user, - completed=True, + completed=registered["registered_in_core"], detail={"registered": registered, "response": response}, ) diff --git a/proc/controller.py b/proc/controller.py index e2fe69ad..daa504d7 100644 --- a/proc/controller.py +++ b/proc/controller.py @@ -23,7 +23,7 @@ def create_or_update_journal( journal_proc.migration_status != tracker_choices.PROGRESS_STATUS_TODO and not force_update ): - return + return journal_proc.journal collection = journal_proc.collection journal_data = journal_proc.migrated_data.data @@ -56,6 +56,7 @@ def create_or_update_journal( migration_status=tracker_choices.PROGRESS_STATUS_DONE, force_update=force_update, ) + return journal def create_or_update_issue( @@ -70,7 +71,7 @@ def create_or_update_issue( issue_proc.migration_status != tracker_choices.PROGRESS_STATUS_TODO and not force_update ): - return + return issue_proc.issue classic_website_issue = classic_ws.Issue(issue_proc.migrated_data.data) journal_proc = JournalProc.get( @@ -96,6 +97,7 @@ def create_or_update_issue( migration_status=tracker_choices.PROGRESS_STATUS_DONE, force_update=force_update, ) + return issue def create_or_update_article( @@ -110,9 +112,10 @@ def create_or_update_article( article_proc.migration_status != tracker_choices.PROGRESS_STATUS_TODO and not force_update ): - return + return article_proc.article - create_article(article_proc.sps_pkg, user, force_update) + article = create_article(article_proc.sps_pkg, user, force_update) article_proc.migration_status = tracker_choices.PROGRESS_STATUS_DONE article_proc.updated_by = user article_proc.save() + return article["article"] diff --git a/proc/migrations/0004_alter_articleproc_options_alter_issueproc_options_and_more.py b/proc/migrations/0004_alter_articleproc_options_alter_issueproc_options_and_more.py new file mode 100644 index 00000000..fa7cbe2b --- /dev/null +++ b/proc/migrations/0004_alter_articleproc_options_alter_issueproc_options_and_more.py @@ -0,0 +1,36 @@ +# Generated by Django 5.0.3 on 2024-03-29 17:32 + +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("proc", "0003_procreport_item_type_and_more"), + ] + + operations = [ + migrations.AlterModelOptions( + name="articleproc", + options={"ordering": ["-updated"]}, + ), + migrations.AlterModelOptions( + name="issueproc", + options={"ordering": ["-updated"]}, + ), + migrations.AlterModelOptions( + name="journalproc", + options={"ordering": ["-updated"]}, + ), + migrations.AlterModelOptions( + name="operation", + options={"ordering": ["-created"]}, + ), + migrations.AlterModelOptions( + name="procreport", + options={ + "ordering": ["-created"], + "verbose_name": "Processing report", + "verbose_name_plural": "Processing reports", + }, + ), + ] diff --git a/proc/models.py b/proc/models.py index 3e7b8a4a..45c6dce1 100644 --- a/proc/models.py +++ b/proc/models.py @@ -4,7 +4,6 @@ import json from datetime import datetime from tempfile import TemporaryDirectory -from zipfile import ZipFile from django.core.files.base import ContentFile from django.db import models @@ -81,14 +80,16 @@ class Operation(CommonControlField): base_form_class = ProcAdminModelForm panels = [ - FieldPanel("name"), + FieldPanel("name", read_only=True), FieldPanel("created", read_only=True), FieldPanel("updated", read_only=True), - FieldPanel("completed"), - FieldPanel("detail"), + FieldPanel("completed", read_only=True), + FieldPanel("detail", read_only=True), ] class Meta: + # isso faz com que em InlinePanel mostre do mais recente para o mais antigo + ordering = ['-created'] indexes = [ models.Index(fields=["name"]), ] @@ -121,7 +122,7 @@ def create(cls, user, proc, name): # obtém todos os ítens criados após este evento rows = [] - for row in cls.objects.filter(proc=proc, created__gte=item.created).iterator(): + for row in cls.objects.filter(proc=proc, created__gte=item.created).order_by('created').iterator(): rows.append(row.data) try: @@ -254,15 +255,17 @@ class ProcReport(CommonControlField): item_type = models.CharField(_("Item type"), max_length=16, null=True, blank=True) panel_files = [ - FieldPanel("task_name"), - FieldPanel("report_date"), - FieldPanel("file"), + FieldPanel("task_name", read_only=True), + FieldPanel("report_date", read_only=True), + FieldPanel("file", read_only=True), ] def __str__(self): return f"{self.collection.acron} {self.pid} {self.task_name} {self.report_date}" class Meta: + ordering = ['-created'] + verbose_name = _("Processing report") verbose_name_plural = _("Processing reports") indexes = [ @@ -410,6 +413,8 @@ class BaseProc(CommonControlField): class Meta: abstract = True + ordering = ['-updated'] + indexes = [ models.Index(fields=["pid"]), ] @@ -431,7 +436,7 @@ class Meta: edit_handler = TabbedInterface( [ ObjectList(panel_status, heading=_("Status")), - ObjectList(panel_proc_result, heading=_("Result")), + ObjectList(panel_proc_result, heading=_("Events newest to oldest")), ] ) @@ -504,6 +509,7 @@ def register_classic_website_data( obj.migration_status == tracker_choices.PROGRESS_STATUS_TODO ), message=None, + detail=obj.migrated_data, ) return obj except Exception as e: @@ -571,13 +577,13 @@ def create_or_update_item( operation = self.start(user, f"create or update {item_name}") - callable_register_data(user, self, force_update) - + registered = callable_register_data(user, self, force_update) operation.finish( user, completed=( self.migration_status == tracker_choices.PROGRESS_STATUS_DONE ), + detail=registered and registered.data, ) except Exception as e: exc_type, exc_value, exc_traceback = sys.exc_info() @@ -710,18 +716,19 @@ class JournalProc(BaseProc, ClusterableModel): base_form_class = ProcAdminModelForm panel_proc_result = [ - InlinePanel("journal_proc_result", label=_("Proc result")), + InlinePanel("journal_proc_result", label=_("Event")), ] MigratedDataClass = MigratedJournal edit_handler = TabbedInterface( [ ObjectList(BaseProc.panel_status, heading=_("Status")), - ObjectList(panel_proc_result, heading=_("Result")), + ObjectList(panel_proc_result, heading=_("Events newest to oldest")), ] ) class Meta: + ordering = ['-updated'] indexes = [ models.Index(fields=["acron"]), ] @@ -843,12 +850,12 @@ def __str__(self): AutocompletePanel("issue_files"), ] panel_proc_result = [ - InlinePanel("issue_proc_result"), + InlinePanel("issue_proc_result", label=_("Event")), ] edit_handler = TabbedInterface( [ ObjectList(panel_status, heading=_("Status")), - ObjectList(panel_proc_result, heading=_("Result")), + ObjectList(panel_proc_result, heading=_("Events newest to oldest")), ] ) @@ -873,6 +880,7 @@ def status(self): ) class Meta: + ordering = ['-updated'] indexes = [ models.Index(fields=["issue_folder"]), models.Index(fields=["docs_status"]), @@ -1125,8 +1133,8 @@ class ArticleProc(BaseProc, ClusterableModel): ProcResult = ArticleProcResult panel_files = [ - FieldPanel("pkg_name"), - AutocompletePanel("sps_pkg"), + FieldPanel("pkg_name", read_only=True), + AutocompletePanel("sps_pkg", read_only=True), ] panel_status = [ FieldPanel("xml_status"), @@ -1139,19 +1147,20 @@ class ArticleProc(BaseProc, ClusterableModel): # AutocompletePanel("events"), # ] panel_proc_result = [ - InlinePanel("article_proc_result"), + InlinePanel("article_proc_result", label=_("Event")), ] edit_handler = TabbedInterface( [ ObjectList(panel_status, heading=_("Status")), ObjectList(panel_files, heading=_("Files")), - ObjectList(panel_proc_result, heading=_("Result")), + ObjectList(panel_proc_result, heading=_("Events newest to oldest")), ] ) MigratedDataClass = MigratedArticle class Meta: + ordering = ['-updated'] indexes = [ models.Index(fields=["pkg_name"]), models.Index(fields=["xml_status"]), @@ -1199,9 +1208,9 @@ def get_xml(self, user, htmlxml, body_and_back_xml): self.save() if htmlxml: - xml = htmlxml.html_to_xml(user, self, body_and_back_xml) - else: - xml = get_migrated_xml_with_pre(self) + htmlxml.html_to_xml(user, self, body_and_back_xml) + + xml = get_migrated_xml_with_pre(self) if xml: self.xml_status = tracker_choices.PROGRESS_STATUS_DONE @@ -1212,6 +1221,7 @@ def get_xml(self, user, htmlxml, body_and_back_xml): operation.finish( user, completed=self.xml_status == tracker_choices.PROGRESS_STATUS_DONE, + detail=xml and xml.data, ) except Exception as e: exc_type, exc_value, exc_traceback = sys.exc_info() @@ -1394,7 +1404,7 @@ def generate_sps_package( operation.finish( user, completed=bool(self.sps_pkg and self.sps_pkg.is_complete), - detail=self.sps_pkg and self.sps_pkg.data or None, + detail=self.sps_pkg and self.sps_pkg.data, ) except Exception as e: @@ -1405,7 +1415,7 @@ def generate_sps_package( user, exc_traceback=exc_traceback, exception=e, - detail=self.sps_pkg and self.sps_pkg.data or None, + detail=self.sps_pkg and self.sps_pkg.data, ) def fix_pid_v2(self, user): @@ -1422,7 +1432,7 @@ def update_sps_pkg_status(self): elif not self.sps_pkg.valid_components: self.sps_pkg_status = tracker_choices.PROGRESS_STATUS_REPROC else: - self.sps_pkg_status = tracker_choices.PROGRESS_STATUS_REPROC + self.sps_pkg_status = tracker_choices.PROGRESS_STATUS_PENDING self.save() @property diff --git a/tracker/models.py b/tracker/models.py index 0d7dbe37..474e009d 100644 --- a/tracker/models.py +++ b/tracker/models.py @@ -98,7 +98,8 @@ def create( try: json.dumps(detail) obj.detail = detail - except: + except Exception as json_e: + logging.exception(json_e) obj.detail = str(detail) if exc_traceback: obj.traceback = traceback.format_tb(exc_traceback)