From cf8a89d9039526596861e0c7746c82741f175210 Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Wed, 27 Mar 2024 17:02:53 -0300 Subject: [PATCH 1/3] Cria o modelo ArticleProcReport e ArticleProcReportModelAdmin --- proc/models.py | 141 ++++++++++++++++++++++++++++++++++++++---- proc/wagtail_hooks.py | 33 +++++++++- 2 files changed, 161 insertions(+), 13 deletions(-) diff --git a/proc/models.py b/proc/models.py index 9f363414..25793e6b 100644 --- a/proc/models.py +++ b/proc/models.py @@ -37,7 +37,11 @@ MigratedIssue, MigratedJournal, ) -from migration.controller import PkgZipBuilder, get_migrated_xml_with_pre, XMLVersionXmlWithPreError +from migration.controller import ( + PkgZipBuilder, + get_migrated_xml_with_pre, + XMLVersionXmlWithPreError, +) from package import choices as package_choices from package.models import SPSPkg from proc import exceptions @@ -179,6 +183,120 @@ def finish( ) +def proc_report_directory_path(instance, filename): + try: + subdir = instance.sps_pkg_name.replace("-", "/") + year = instance.created.isoformat()[:8] + return f"sps_pkg/{subdir}/reports/{year}/{filename}" + except AttributeError: + return f"reports/{filename}" + + +class ArticleProcReport(CommonControlField): + article_proc = models.ForeignKey( + "ArticleProc", on_delete=models.SET_NULL, null=True, blank=True + ) + sps_pkg_name = models.CharField( + _("SPS package name"), max_length=32, null=True, blank=True + ) + task_name = models.CharField( + _("Procedure name"), max_length=32, null=True, blank=True + ) + file = models.FileField(upload_to=proc_report_directory_path, null=True, blank=True) + report_date = models.CharField( + _("Identification"), max_length=34, null=True, blank=True + ) + + panel_files = [ + FieldPanel("sps_pkg_name"), + FieldPanel("task_name"), + FieldPanel("report_date"), + FieldPanel("file"), + ] + + def __str__(self): + return f"{self.sps_pkg_name} {self.task_name} {self.report_date}" + + class Meta: + verbose_name = _("Article processing report") + verbose_name_plural = _("Article processing reports") + indexes = [ + models.Index(fields=["sps_pkg_name"]), + models.Index(fields=["task_name"]), + models.Index(fields=["report_date"]), + ] + + @staticmethod + def autocomplete_custom_queryset_filter(search_term): + return State.objects.filter( + Q(sps_pkg_name__icontains=search_term) + | Q(task_name__icontains=search_term) + | Q(report_date__icontains=search_term) + ) + + def autocomplete_label(self): + return str(self) + + def save_file(self, name, content): + try: + self.file.delete(save=True) + except Exception as e: + pass + try: + self.file.save(name, ContentFile(content)) + except Exception as e: + raise Exception(f"Unable to save {name}. Exception: {e}") + + @classmethod + def get(cls, article_proc=None, task_name=None, report_date=None): + if article_proc and task_name and report_date: + try: + return cls.objects.get( + article_proc=article_proc, + task_name=task_name, + report_date=report_date, + ) + except cls.MultipleObjectsReturned: + return cls.objects.filter( + article_proc=article_proc, + task_name=task_name, + report_date=report_date, + ).first() + raise ValueError( + "ArticleProcReport.get requires article_proc and task_name and report_date" + ) + + @classmethod + def create(cls, user, article_proc, task_name, report_date, file_content): + if article_proc and task_name and report_date: + try: + obj = cls() + obj.task_name = task_name + obj.acronym = acronym + obj.creator = user + obj.save() + return obj + except IntegrityError: + return cls.get(article_proc, task_name, report_date) + raise ValueError( + "ArticleProcReport.create requires article_proc and task_name and report_date" + ) + + @classmethod + def create_or_update(cls, user, article_proc, task_name, report_date, file_content): + try: + obj = cls.get( + article_proc=article_proc, task_name=task_name, report_date=report_date + ) + obj.updated_by = user + obj.task_name = task_name or obj.task_name + obj.report_date = report_date or obj.report_date + obj.save() + except cls.DoesNotExist: + obj = cls.create(user, article_proc, task_name, report_date, file_content) + return obj + + class JournalProcResult(Operation, Orderable): proc = ParentalKey("JournalProc", related_name="journal_proc_result") @@ -716,7 +834,9 @@ def update( ) @classmethod - def files_to_migrate(cls, collection, journal_acron, publication_year, force_update): + def files_to_migrate( + cls, collection, journal_acron, publication_year, force_update + ): """ Muda o status de PROGRESS_STATUS_REPROC para PROGRESS_STATUS_TODO E se force_update = True, muda o status de PROGRESS_STATUS_DONE para PROGRESS_STATUS_TODO @@ -737,9 +857,9 @@ def files_to_migrate(cls, collection, journal_acron, publication_year, force_upd params = {} if publication_year: - params['issue__publication_year'] = publication_year + params["issue__publication_year"] = publication_year if journal_acron: - params['journal_proc__acron'] = journal_acron + params["journal_proc__acron"] = journal_acron return cls.objects.filter( files_status=tracker_choices.PROGRESS_STATUS_TODO, @@ -818,9 +938,9 @@ def docs_to_migrate(cls, collection, journal_acron, publication_year, force_upda params = {} if publication_year: - params['issue__publication_year'] = publication_year + params["issue__publication_year"] = publication_year if journal_acron: - params['journal_proc__acron'] = journal_acron + params["journal_proc__acron"] = journal_acron return cls.objects.filter( docs_status=tracker_choices.PROGRESS_STATUS_TODO, @@ -1020,7 +1140,7 @@ def get_xml(self, user, htmlxml, body_and_back_xml): operation.finish( user, - completed=self.xml_status==tracker_choices.PROGRESS_STATUS_DONE, + completed=self.xml_status == tracker_choices.PROGRESS_STATUS_DONE, ) except Exception as e: exc_type, exc_value, exc_traceback = sys.exc_info() @@ -1219,8 +1339,7 @@ def generate_sps_package( def fix_pid_v2(self, user): if self.sps_pkg: - self.sps_pkg.fix_pid_v2( - user, correct_pid_v2=self.migrated_data.pid) + self.sps_pkg.fix_pid_v2(user, correct_pid_v2=self.migrated_data.pid) def update_sps_pkg_status(self): if not self.sps_pkg: @@ -1251,9 +1370,7 @@ def synchronize(self, user): operation = self.start(user, "synchronize to core") self.sps_pkg.synchronize(user, self) - operation.finish( - user, completed=self.sps_pkg.registered_in_core - ) + operation.finish(user, completed=self.sps_pkg.registered_in_core) except Exception as e: exc_type, exc_value, exc_traceback = sys.exc_info() diff --git a/proc/wagtail_hooks.py b/proc/wagtail_hooks.py index 04541f23..2c9e8b31 100644 --- a/proc/wagtail_hooks.py +++ b/proc/wagtail_hooks.py @@ -15,7 +15,7 @@ from package.models import SPSPkg from htmlxml.models import HTMLXML -from .models import ArticleProc, IssueProc, JournalProc +from .models import ArticleProc, IssueProc, JournalProc, ArticleProcReport class ProcCreateView(CreateView): @@ -210,6 +210,36 @@ class ArticleProcModelAdmin(ModelAdmin): ) +class ArticleProcReportModelAdmin(ModelAdmin): + model = ArticleProcReport + menu_label = _("Article Processing Report") + inspect_view_enabled = True + menu_icon = "doc-full" + menu_order = 200 + add_to_settings_menu = False + exclude_from_explorer = False + + list_display = ( + "article_proc", + "sps_pkg_name", + "task_name", + "file", + "report_date", + "updated", + "created", + ) + list_filter = ( + "task_name", + ) + search_fields = ( + "article_proc__pid", + "article_proc__collection__name", + "sps_pkg_name", + "task_name", + "report_date", + ) + + class ProcessModelAdminGroup(ModelAdminGroup): menu_label = _("Processing") menu_icon = "folder-open-inverse" @@ -221,6 +251,7 @@ class ProcessModelAdminGroup(ModelAdminGroup): HTMLXMLModelAdmin, SPSPkgModelAdmin, ArticleProcModelAdmin, + ArticleProcReportModelAdmin, ) From d7f1cedba6e07693b7997eb17fd636889c113ea8 Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Thu, 28 Mar 2024 08:07:05 -0300 Subject: [PATCH 2/3] Cria o modelo ProcReport para armazenar processamentos anteriores, mantendo apenas o vigente nos respectivos ArticleProc, IssueProc, JournalProc --- proc/models.py | 143 +++++++++++++++++++++++++++++++----------- proc/wagtail_hooks.py | 24 +++---- 2 files changed, 120 insertions(+), 47 deletions(-) diff --git a/proc/models.py b/proc/models.py index 25793e6b..3e7b8a4a 100644 --- a/proc/models.py +++ b/proc/models.py @@ -96,6 +96,16 @@ class Meta: def __str__(self): return f"{self.name} {self.started} {self.finished} {self.completed}" + @property + def data(self): + return dict( + name=self.name, + completed=self.completed, + event=self.event and self.event.data, + detail=self.detail, + created=self.created.isoformat(), + ) + @property def started(self): return self.created and self.created.isoformat() or "" @@ -104,6 +114,47 @@ def started(self): def finished(self): return self.updated and self.updated.isoformat() or "" + @classmethod + def create(cls, user, proc, name): + for item in cls.objects.filter(proc=proc, name=name).order_by('created'): + # obtém o primeiro ocorrência de proc e name + + # obtém todos os ítens criados após este evento + rows = [] + for row in cls.objects.filter(proc=proc, created__gte=item.created).iterator(): + rows.append(row.data) + + try: + # converte para json + file_content = json.dumps(rows) + file_extension = ".json" + except Exception as e: + # caso não seja serializável, converte para str + file_content = str(rows) + file_extension = ".txt" + logging.info(proc.pid) + logging.exception(e) + + try: + report_date = item.created.isoformat() + # cria um arquivo com o conteúdo + ProcReport.create_or_update( + user, proc, name, report_date, file_content, file_extension, + ) + # apaga todas as ocorrências que foram armazenadas no arquivo + cls.objects.filter(proc=proc, created__gte=item.created).delete() + except Exception as e: + logging.info(proc.pid) + logging.exception(e) + break + + obj = cls() + obj.proc = proc + obj.name = name + obj.creator = user + obj.save() + return obj + @classmethod def start( cls, @@ -112,12 +163,7 @@ def start( name=None, ): try: - obj = cls() - obj.proc = proc - obj.name = name - obj.creator = user - obj.save() - return obj + return cls.create(user, proc, name) except Exception as exc: raise OperationStartError( f"Unable to create Operation ({name}). EXCEPTION: {type(exc)} {exc}" @@ -185,20 +231,19 @@ def finish( def proc_report_directory_path(instance, filename): try: - subdir = instance.sps_pkg_name.replace("-", "/") - year = instance.created.isoformat()[:8] - return f"sps_pkg/{subdir}/reports/{year}/{filename}" + subdir = instance.directory_path + YYYY = instance.report_date[:4] + return f"archive/{subdir}/proc/{YYYY}/{filename}" except AttributeError: - return f"reports/{filename}" + return f"archive/{filename}" -class ArticleProcReport(CommonControlField): - article_proc = models.ForeignKey( - "ArticleProc", on_delete=models.SET_NULL, null=True, blank=True - ) - sps_pkg_name = models.CharField( - _("SPS package name"), max_length=32, null=True, blank=True +class ProcReport(CommonControlField): + collection = models.ForeignKey( + Collection, on_delete=models.SET_NULL, null=True, blank=True ) + + pid = models.CharField(_("PID"), max_length=23, null=True, blank=True) task_name = models.CharField( _("Procedure name"), max_length=32, null=True, blank=True ) @@ -206,30 +251,33 @@ class ArticleProcReport(CommonControlField): report_date = models.CharField( _("Identification"), max_length=34, null=True, blank=True ) + item_type = models.CharField(_("Item type"), max_length=16, null=True, blank=True) panel_files = [ - FieldPanel("sps_pkg_name"), FieldPanel("task_name"), FieldPanel("report_date"), FieldPanel("file"), ] def __str__(self): - return f"{self.sps_pkg_name} {self.task_name} {self.report_date}" + return f"{self.collection.acron} {self.pid} {self.task_name} {self.report_date}" class Meta: - verbose_name = _("Article processing report") - verbose_name_plural = _("Article processing reports") + verbose_name = _("Processing report") + verbose_name_plural = _("Processing reports") indexes = [ - models.Index(fields=["sps_pkg_name"]), + models.Index(fields=["item_type"]), + models.Index(fields=["pid"]), models.Index(fields=["task_name"]), models.Index(fields=["report_date"]), ] @staticmethod def autocomplete_custom_queryset_filter(search_term): - return State.objects.filter( - Q(sps_pkg_name__icontains=search_term) + return ProcReport.objects.filter( + Q(pid__icontains=search_term) + | Q(collection__acron__icontains=search_term) + | Q(collection__name__icontains=search_term) | Q(task_name__icontains=search_term) | Q(report_date__icontains=search_term) ) @@ -248,54 +296,76 @@ def save_file(self, name, content): raise Exception(f"Unable to save {name}. Exception: {e}") @classmethod - def get(cls, article_proc=None, task_name=None, report_date=None): - if article_proc and task_name and report_date: + def get(cls, proc=None, task_name=None, report_date=None): + if proc and task_name and report_date: try: return cls.objects.get( - article_proc=article_proc, + collection=proc.collection, pid=proc.pid, task_name=task_name, report_date=report_date, ) except cls.MultipleObjectsReturned: return cls.objects.filter( - article_proc=article_proc, + collection=proc.collection, pid=proc.pid, task_name=task_name, report_date=report_date, ).first() raise ValueError( - "ArticleProcReport.get requires article_proc and task_name and report_date" + "ProcReport.get requires proc and task_name and report_date" ) + @staticmethod + def get_item_type(pid): + if len(pid) == 23: + return "article" + if len(pid) == 9: + return "journal" + return "issue" + @classmethod - def create(cls, user, article_proc, task_name, report_date, file_content): - if article_proc and task_name and report_date: + def create(cls, user, proc, task_name, report_date, file_content, file_extension): + if proc and task_name and report_date and file_content and file_extension: try: obj = cls() + obj.collection = proc.collection + obj.pid = proc.pid obj.task_name = task_name - obj.acronym = acronym + obj.item_type = ProcReport.get_item_type(proc.pid) + obj.report_date = report_date obj.creator = user obj.save() + obj.save_file(f"{task_name}{file_extension}", file_content) return obj except IntegrityError: - return cls.get(article_proc, task_name, report_date) + return cls.get(proc, task_name, report_date) raise ValueError( - "ArticleProcReport.create requires article_proc and task_name and report_date" + "ProcReport.create requires proc and task_name and report_date and file_content and file_extension" ) @classmethod - def create_or_update(cls, user, article_proc, task_name, report_date, file_content): + def create_or_update(cls, user, proc, task_name, report_date, file_content, file_extension): try: obj = cls.get( - article_proc=article_proc, task_name=task_name, report_date=report_date + proc=proc, task_name=task_name, report_date=report_date ) obj.updated_by = user obj.task_name = task_name or obj.task_name obj.report_date = report_date or obj.report_date obj.save() + obj.save_file(f"{task_name}{file_extension}", file_content) except cls.DoesNotExist: - obj = cls.create(user, article_proc, task_name, report_date, file_content) + obj = cls.create(user, proc, task_name, report_date, file_content, file_extension) return obj + @property + def directory_path(self): + pid = self.pid + if len(self.pid) == 23: + pid = self.pid[1:] + paths = [self.collection.acron, pid[:9], pid[9:13], pid[13:17], pid[17:]] + paths = [path for path in paths if path] + return os.path.join(*paths) + class JournalProcResult(Operation, Orderable): proc = ParentalKey("JournalProc", related_name="journal_proc_result") @@ -1053,6 +1123,7 @@ class ArticleProc(BaseProc, ClusterableModel): ) ProcResult = ArticleProcResult + panel_files = [ FieldPanel("pkg_name"), AutocompletePanel("sps_pkg"), diff --git a/proc/wagtail_hooks.py b/proc/wagtail_hooks.py index 2c9e8b31..a8d4a008 100644 --- a/proc/wagtail_hooks.py +++ b/proc/wagtail_hooks.py @@ -15,7 +15,7 @@ from package.models import SPSPkg from htmlxml.models import HTMLXML -from .models import ArticleProc, IssueProc, JournalProc, ArticleProcReport +from .models import ArticleProc, IssueProc, JournalProc, ProcReport class ProcCreateView(CreateView): @@ -210,31 +210,33 @@ class ArticleProcModelAdmin(ModelAdmin): ) -class ArticleProcReportModelAdmin(ModelAdmin): - model = ArticleProcReport - menu_label = _("Article Processing Report") +class ProcReportModelAdmin(ModelAdmin): + model = ProcReport + menu_label = _("Processing Report") inspect_view_enabled = True menu_icon = "doc-full" menu_order = 200 add_to_settings_menu = False exclude_from_explorer = False + list_per_page = 50 + list_display = ( - "article_proc", - "sps_pkg_name", + "pid", + "collection", "task_name", - "file", "report_date", "updated", "created", ) list_filter = ( "task_name", + "collection", + "item_type", ) search_fields = ( - "article_proc__pid", - "article_proc__collection__name", - "sps_pkg_name", + "pid", + "collection__name", "task_name", "report_date", ) @@ -251,7 +253,7 @@ class ProcessModelAdminGroup(ModelAdminGroup): HTMLXMLModelAdmin, SPSPkgModelAdmin, ArticleProcModelAdmin, - ArticleProcReportModelAdmin, + ProcReportModelAdmin, ) From 125b4d0a9ed355a7f5d8019c6757ea8e1e231b81 Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Thu, 28 Mar 2024 10:05:45 -0300 Subject: [PATCH 3/3] =?UTF-8?q?Adiciona=20as=20migra=C3=A7=C3=B5es=20de=20?= =?UTF-8?q?banco=20de=20dados?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- proc/migrations/0002_procreport.py | 119 ++++++++++++++++++ .../0003_procreport_item_type_and_more.py | 28 +++++ 2 files changed, 147 insertions(+) create mode 100644 proc/migrations/0002_procreport.py create mode 100644 proc/migrations/0003_procreport_item_type_and_more.py diff --git a/proc/migrations/0002_procreport.py b/proc/migrations/0002_procreport.py new file mode 100644 index 00000000..129c8c12 --- /dev/null +++ b/proc/migrations/0002_procreport.py @@ -0,0 +1,119 @@ +# Generated by Django 5.0.3 on 2024-03-28 11:53 + +import django.db.models.deletion +import proc.models +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("collection", "0002_remove_websiteconfiguration_api_token_and_more"), + ("proc", "0001_initial"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name="ProcReport", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "created", + models.DateTimeField( + auto_now_add=True, verbose_name="Creation date" + ), + ), + ( + "updated", + models.DateTimeField( + auto_now=True, verbose_name="Last update date" + ), + ), + ( + "pid", + models.CharField( + blank=True, max_length=23, null=True, verbose_name="PID" + ), + ), + ( + "task_name", + models.CharField( + blank=True, + max_length=32, + null=True, + verbose_name="Procedure name", + ), + ), + ( + "file", + models.FileField( + blank=True, + null=True, + upload_to=proc.models.proc_report_directory_path, + ), + ), + ( + "report_date", + models.CharField( + blank=True, + max_length=34, + null=True, + verbose_name="Identification", + ), + ), + ( + "collection", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to="collection.collection", + ), + ), + ( + "creator", + models.ForeignKey( + editable=False, + on_delete=django.db.models.deletion.CASCADE, + related_name="%(class)s_creator", + to=settings.AUTH_USER_MODEL, + verbose_name="Creator", + ), + ), + ( + "updated_by", + models.ForeignKey( + blank=True, + editable=False, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="%(class)s_last_mod_user", + to=settings.AUTH_USER_MODEL, + verbose_name="Updater", + ), + ), + ], + options={ + "verbose_name": "Processing report", + "verbose_name_plural": "Processing reports", + "indexes": [ + models.Index(fields=["pid"], name="proc_procre_pid_2ea179_idx"), + models.Index( + fields=["task_name"], name="proc_procre_task_na_33520a_idx" + ), + models.Index( + fields=["report_date"], name="proc_procre_report__370dc9_idx" + ), + ], + }, + ), + ] diff --git a/proc/migrations/0003_procreport_item_type_and_more.py b/proc/migrations/0003_procreport_item_type_and_more.py new file mode 100644 index 00000000..4dce6efe --- /dev/null +++ b/proc/migrations/0003_procreport_item_type_and_more.py @@ -0,0 +1,28 @@ +# Generated by Django 5.0.3 on 2024-03-28 12:58 + +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("collection", "0002_remove_websiteconfiguration_api_token_and_more"), + ("proc", "0002_procreport"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.AddField( + model_name="procreport", + name="item_type", + field=models.CharField( + blank=True, max_length=16, null=True, verbose_name="Item type" + ), + ), + migrations.AddIndex( + model_name="procreport", + index=models.Index( + fields=["item_type"], name="proc_procre_item_ty_0b33db_idx" + ), + ), + ]