diff --git a/proc/migrations/0002_procreport.py b/proc/migrations/0002_procreport.py new file mode 100644 index 00000000..129c8c12 --- /dev/null +++ b/proc/migrations/0002_procreport.py @@ -0,0 +1,119 @@ +# Generated by Django 5.0.3 on 2024-03-28 11:53 + +import django.db.models.deletion +import proc.models +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("collection", "0002_remove_websiteconfiguration_api_token_and_more"), + ("proc", "0001_initial"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name="ProcReport", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "created", + models.DateTimeField( + auto_now_add=True, verbose_name="Creation date" + ), + ), + ( + "updated", + models.DateTimeField( + auto_now=True, verbose_name="Last update date" + ), + ), + ( + "pid", + models.CharField( + blank=True, max_length=23, null=True, verbose_name="PID" + ), + ), + ( + "task_name", + models.CharField( + blank=True, + max_length=32, + null=True, + verbose_name="Procedure name", + ), + ), + ( + "file", + models.FileField( + blank=True, + null=True, + upload_to=proc.models.proc_report_directory_path, + ), + ), + ( + "report_date", + models.CharField( + blank=True, + max_length=34, + null=True, + verbose_name="Identification", + ), + ), + ( + "collection", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to="collection.collection", + ), + ), + ( + "creator", + models.ForeignKey( + editable=False, + on_delete=django.db.models.deletion.CASCADE, + related_name="%(class)s_creator", + to=settings.AUTH_USER_MODEL, + verbose_name="Creator", + ), + ), + ( + "updated_by", + models.ForeignKey( + blank=True, + editable=False, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="%(class)s_last_mod_user", + to=settings.AUTH_USER_MODEL, + verbose_name="Updater", + ), + ), + ], + options={ + "verbose_name": "Processing report", + "verbose_name_plural": "Processing reports", + "indexes": [ + models.Index(fields=["pid"], name="proc_procre_pid_2ea179_idx"), + models.Index( + fields=["task_name"], name="proc_procre_task_na_33520a_idx" + ), + models.Index( + fields=["report_date"], name="proc_procre_report__370dc9_idx" + ), + ], + }, + ), + ] diff --git a/proc/migrations/0003_procreport_item_type_and_more.py b/proc/migrations/0003_procreport_item_type_and_more.py new file mode 100644 index 00000000..4dce6efe --- /dev/null +++ b/proc/migrations/0003_procreport_item_type_and_more.py @@ -0,0 +1,28 @@ +# Generated by Django 5.0.3 on 2024-03-28 12:58 + +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("collection", "0002_remove_websiteconfiguration_api_token_and_more"), + ("proc", "0002_procreport"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.AddField( + model_name="procreport", + name="item_type", + field=models.CharField( + blank=True, max_length=16, null=True, verbose_name="Item type" + ), + ), + migrations.AddIndex( + model_name="procreport", + index=models.Index( + fields=["item_type"], name="proc_procre_item_ty_0b33db_idx" + ), + ), + ] diff --git a/proc/models.py b/proc/models.py index 9f363414..3e7b8a4a 100644 --- a/proc/models.py +++ b/proc/models.py @@ -37,7 +37,11 @@ MigratedIssue, MigratedJournal, ) -from migration.controller import PkgZipBuilder, get_migrated_xml_with_pre, XMLVersionXmlWithPreError +from migration.controller import ( + PkgZipBuilder, + get_migrated_xml_with_pre, + XMLVersionXmlWithPreError, +) from package import choices as package_choices from package.models import SPSPkg from proc import exceptions @@ -92,6 +96,16 @@ class Meta: def __str__(self): return f"{self.name} {self.started} {self.finished} {self.completed}" + @property + def data(self): + return dict( + name=self.name, + completed=self.completed, + event=self.event and self.event.data, + detail=self.detail, + created=self.created.isoformat(), + ) + @property def started(self): return self.created and self.created.isoformat() or "" @@ -100,6 +114,47 @@ def started(self): def finished(self): return self.updated and self.updated.isoformat() or "" + @classmethod + def create(cls, user, proc, name): + for item in cls.objects.filter(proc=proc, name=name).order_by('created'): + # obtém o primeiro ocorrência de proc e name + + # obtém todos os ítens criados após este evento + rows = [] + for row in cls.objects.filter(proc=proc, created__gte=item.created).iterator(): + rows.append(row.data) + + try: + # converte para json + file_content = json.dumps(rows) + file_extension = ".json" + except Exception as e: + # caso não seja serializável, converte para str + file_content = str(rows) + file_extension = ".txt" + logging.info(proc.pid) + logging.exception(e) + + try: + report_date = item.created.isoformat() + # cria um arquivo com o conteúdo + ProcReport.create_or_update( + user, proc, name, report_date, file_content, file_extension, + ) + # apaga todas as ocorrências que foram armazenadas no arquivo + cls.objects.filter(proc=proc, created__gte=item.created).delete() + except Exception as e: + logging.info(proc.pid) + logging.exception(e) + break + + obj = cls() + obj.proc = proc + obj.name = name + obj.creator = user + obj.save() + return obj + @classmethod def start( cls, @@ -108,12 +163,7 @@ def start( name=None, ): try: - obj = cls() - obj.proc = proc - obj.name = name - obj.creator = user - obj.save() - return obj + return cls.create(user, proc, name) except Exception as exc: raise OperationStartError( f"Unable to create Operation ({name}). EXCEPTION: {type(exc)} {exc}" @@ -179,6 +229,144 @@ def finish( ) +def proc_report_directory_path(instance, filename): + try: + subdir = instance.directory_path + YYYY = instance.report_date[:4] + return f"archive/{subdir}/proc/{YYYY}/{filename}" + except AttributeError: + return f"archive/{filename}" + + +class ProcReport(CommonControlField): + collection = models.ForeignKey( + Collection, on_delete=models.SET_NULL, null=True, blank=True + ) + + pid = models.CharField(_("PID"), max_length=23, null=True, blank=True) + task_name = models.CharField( + _("Procedure name"), max_length=32, null=True, blank=True + ) + file = models.FileField(upload_to=proc_report_directory_path, null=True, blank=True) + report_date = models.CharField( + _("Identification"), max_length=34, null=True, blank=True + ) + item_type = models.CharField(_("Item type"), max_length=16, null=True, blank=True) + + panel_files = [ + FieldPanel("task_name"), + FieldPanel("report_date"), + FieldPanel("file"), + ] + + def __str__(self): + return f"{self.collection.acron} {self.pid} {self.task_name} {self.report_date}" + + class Meta: + verbose_name = _("Processing report") + verbose_name_plural = _("Processing reports") + indexes = [ + models.Index(fields=["item_type"]), + models.Index(fields=["pid"]), + models.Index(fields=["task_name"]), + models.Index(fields=["report_date"]), + ] + + @staticmethod + def autocomplete_custom_queryset_filter(search_term): + return ProcReport.objects.filter( + Q(pid__icontains=search_term) + | Q(collection__acron__icontains=search_term) + | Q(collection__name__icontains=search_term) + | Q(task_name__icontains=search_term) + | Q(report_date__icontains=search_term) + ) + + def autocomplete_label(self): + return str(self) + + def save_file(self, name, content): + try: + self.file.delete(save=True) + except Exception as e: + pass + try: + self.file.save(name, ContentFile(content)) + except Exception as e: + raise Exception(f"Unable to save {name}. Exception: {e}") + + @classmethod + def get(cls, proc=None, task_name=None, report_date=None): + if proc and task_name and report_date: + try: + return cls.objects.get( + collection=proc.collection, pid=proc.pid, + task_name=task_name, + report_date=report_date, + ) + except cls.MultipleObjectsReturned: + return cls.objects.filter( + collection=proc.collection, pid=proc.pid, + task_name=task_name, + report_date=report_date, + ).first() + raise ValueError( + "ProcReport.get requires proc and task_name and report_date" + ) + + @staticmethod + def get_item_type(pid): + if len(pid) == 23: + return "article" + if len(pid) == 9: + return "journal" + return "issue" + + @classmethod + def create(cls, user, proc, task_name, report_date, file_content, file_extension): + if proc and task_name and report_date and file_content and file_extension: + try: + obj = cls() + obj.collection = proc.collection + obj.pid = proc.pid + obj.task_name = task_name + obj.item_type = ProcReport.get_item_type(proc.pid) + obj.report_date = report_date + obj.creator = user + obj.save() + obj.save_file(f"{task_name}{file_extension}", file_content) + return obj + except IntegrityError: + return cls.get(proc, task_name, report_date) + raise ValueError( + "ProcReport.create requires proc and task_name and report_date and file_content and file_extension" + ) + + @classmethod + def create_or_update(cls, user, proc, task_name, report_date, file_content, file_extension): + try: + obj = cls.get( + proc=proc, task_name=task_name, report_date=report_date + ) + obj.updated_by = user + obj.task_name = task_name or obj.task_name + obj.report_date = report_date or obj.report_date + obj.save() + obj.save_file(f"{task_name}{file_extension}", file_content) + except cls.DoesNotExist: + obj = cls.create(user, proc, task_name, report_date, file_content, file_extension) + return obj + + @property + def directory_path(self): + pid = self.pid + if len(self.pid) == 23: + pid = self.pid[1:] + paths = [self.collection.acron, pid[:9], pid[9:13], pid[13:17], pid[17:]] + paths = [path for path in paths if path] + return os.path.join(*paths) + + class JournalProcResult(Operation, Orderable): proc = ParentalKey("JournalProc", related_name="journal_proc_result") @@ -716,7 +904,9 @@ def update( ) @classmethod - def files_to_migrate(cls, collection, journal_acron, publication_year, force_update): + def files_to_migrate( + cls, collection, journal_acron, publication_year, force_update + ): """ Muda o status de PROGRESS_STATUS_REPROC para PROGRESS_STATUS_TODO E se force_update = True, muda o status de PROGRESS_STATUS_DONE para PROGRESS_STATUS_TODO @@ -737,9 +927,9 @@ def files_to_migrate(cls, collection, journal_acron, publication_year, force_upd params = {} if publication_year: - params['issue__publication_year'] = publication_year + params["issue__publication_year"] = publication_year if journal_acron: - params['journal_proc__acron'] = journal_acron + params["journal_proc__acron"] = journal_acron return cls.objects.filter( files_status=tracker_choices.PROGRESS_STATUS_TODO, @@ -818,9 +1008,9 @@ def docs_to_migrate(cls, collection, journal_acron, publication_year, force_upda params = {} if publication_year: - params['issue__publication_year'] = publication_year + params["issue__publication_year"] = publication_year if journal_acron: - params['journal_proc__acron'] = journal_acron + params["journal_proc__acron"] = journal_acron return cls.objects.filter( docs_status=tracker_choices.PROGRESS_STATUS_TODO, @@ -933,6 +1123,7 @@ class ArticleProc(BaseProc, ClusterableModel): ) ProcResult = ArticleProcResult + panel_files = [ FieldPanel("pkg_name"), AutocompletePanel("sps_pkg"), @@ -1020,7 +1211,7 @@ def get_xml(self, user, htmlxml, body_and_back_xml): operation.finish( user, - completed=self.xml_status==tracker_choices.PROGRESS_STATUS_DONE, + completed=self.xml_status == tracker_choices.PROGRESS_STATUS_DONE, ) except Exception as e: exc_type, exc_value, exc_traceback = sys.exc_info() @@ -1219,8 +1410,7 @@ def generate_sps_package( def fix_pid_v2(self, user): if self.sps_pkg: - self.sps_pkg.fix_pid_v2( - user, correct_pid_v2=self.migrated_data.pid) + self.sps_pkg.fix_pid_v2(user, correct_pid_v2=self.migrated_data.pid) def update_sps_pkg_status(self): if not self.sps_pkg: @@ -1251,9 +1441,7 @@ def synchronize(self, user): operation = self.start(user, "synchronize to core") self.sps_pkg.synchronize(user, self) - operation.finish( - user, completed=self.sps_pkg.registered_in_core - ) + operation.finish(user, completed=self.sps_pkg.registered_in_core) except Exception as e: exc_type, exc_value, exc_traceback = sys.exc_info() diff --git a/proc/wagtail_hooks.py b/proc/wagtail_hooks.py index 04541f23..a8d4a008 100644 --- a/proc/wagtail_hooks.py +++ b/proc/wagtail_hooks.py @@ -15,7 +15,7 @@ from package.models import SPSPkg from htmlxml.models import HTMLXML -from .models import ArticleProc, IssueProc, JournalProc +from .models import ArticleProc, IssueProc, JournalProc, ProcReport class ProcCreateView(CreateView): @@ -210,6 +210,38 @@ class ArticleProcModelAdmin(ModelAdmin): ) +class ProcReportModelAdmin(ModelAdmin): + model = ProcReport + menu_label = _("Processing Report") + inspect_view_enabled = True + menu_icon = "doc-full" + menu_order = 200 + add_to_settings_menu = False + exclude_from_explorer = False + + list_per_page = 50 + + list_display = ( + "pid", + "collection", + "task_name", + "report_date", + "updated", + "created", + ) + list_filter = ( + "task_name", + "collection", + "item_type", + ) + search_fields = ( + "pid", + "collection__name", + "task_name", + "report_date", + ) + + class ProcessModelAdminGroup(ModelAdminGroup): menu_label = _("Processing") menu_icon = "folder-open-inverse" @@ -221,6 +253,7 @@ class ProcessModelAdminGroup(ModelAdminGroup): HTMLXMLModelAdmin, SPSPkgModelAdmin, ArticleProcModelAdmin, + ProcReportModelAdmin, )