From d0fc2cc120a82618ee92f3eb8b71a2efda67e031 Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Tue, 19 Nov 2024 00:58:25 -0300 Subject: [PATCH 01/22] =?UTF-8?q?Omite=20(temporariamente)=20os=20bot?= =?UTF-8?q?=C3=B5es=20de=20consultar=20solicita=C3=A7=C3=A3o=20de=20altera?= =?UTF-8?q?=C3=A7=C3=A3o=20e=20de=20submeter=20altera=C3=A7=C3=A3o?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- article/button_helper.py | 47 +++++++++---------- ...-v3.0.0rc2.yml => production-v3.0.0rc4.yml | 0 2 files changed, 22 insertions(+), 25 deletions(-) rename production-v3.0.0rc2.yml => production-v3.0.0rc4.yml (100%) diff --git a/article/button_helper.py b/article/button_helper.py index 1756b4df..611cd38c 100644 --- a/article/button_helper.py +++ b/article/button_helper.py @@ -1,10 +1,7 @@ from django.utils.translation import gettext as _ from wagtail.contrib.modeladmin.helpers import ButtonHelper -# FIXME -# from upload.choices import PC_ERRATUM, PC_UPDATE - -from .choices import AS_REQUIRE_ERRATUM, AS_REQUIRE_UPDATE +from article.choices import AS_REQUIRE_ERRATUM, AS_REQUIRE_UPDATE class RequestArticleChangeButtonHelper(ButtonHelper): @@ -60,13 +57,13 @@ def get_buttons_for_obj( if url_name.endswith("_modeladmin_index"): classnames.extend(ArticleButtonHelper.index_button_classnames) - if ph.user_can_make_article_change(usr, obj.article) and obj.article.status in ( - AS_REQUIRE_ERRATUM, - AS_REQUIRE_UPDATE, - ): - if obj.demanded_user == usr: - btns.append(self.submit_change(obj, classnames)) - btns.append(self.see_instructions(obj, classnames)) + # if ph.user_can_make_article_change(usr, obj.article) and obj.article.status in ( + # AS_REQUIRE_ERRATUM, + # AS_REQUIRE_UPDATE, + # ): + # if obj.demanded_user == usr: + # btns.append(self.submit_change(obj, classnames)) + # btns.append(self.see_instructions(obj, classnames)) return btns @@ -123,19 +120,19 @@ def get_buttons_for_obj( if url_name == "article_article_modeladmin_index": classnames.extend(ArticleButtonHelper.index_button_classnames) - if ph.user_can_request_article_change(usr, obj) and obj.status not in ( - AS_REQUIRE_UPDATE, - AS_REQUIRE_ERRATUM, - ): - btns.append(self.request_change(obj, classnames)) - - if ph.user_can_make_article_change(usr, obj) and obj.status in ( - AS_REQUIRE_ERRATUM, - AS_REQUIRE_UPDATE, - ): - for rac in obj.requestarticlechange_set.all(): - if rac.demanded_user == usr: - btns.append(self.submit_change(obj, classnames)) - break + # if ph.user_can_request_article_change(usr, obj) and obj.status not in ( + # AS_REQUIRE_UPDATE, + # AS_REQUIRE_ERRATUM, + # ): + # btns.append(self.request_change(obj, classnames)) + + # if ph.user_can_make_article_change(usr, obj) and obj.status in ( + # AS_REQUIRE_ERRATUM, + # AS_REQUIRE_UPDATE, + # ): + # for rac in obj.requestarticlechange_set.all(): + # if rac.demanded_user == usr: + # btns.append(self.submit_change(obj, classnames)) + # break return btns diff --git a/production-v3.0.0rc2.yml b/production-v3.0.0rc4.yml similarity index 100% rename from production-v3.0.0rc2.yml rename to production-v3.0.0rc4.yml From 5e3bbaac4cac4abe7800aa45460a616fc883bda6 Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Tue, 19 Nov 2024 00:58:43 -0300 Subject: [PATCH 02/22] =?UTF-8?q?Corrige=20a=20atribui=C3=A7=C3=A3o=20de?= =?UTF-8?q?=20article.status=20ap=C3=B3s=20solicitar=20altera=C3=A7=C3=A3o?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- article/forms.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/article/forms.py b/article/forms.py index eb4d73d2..4c44d74d 100644 --- a/article/forms.py +++ b/article/forms.py @@ -49,7 +49,7 @@ def clean(self): def save_all(self, user): request_article_change = super().save_all(user) - article = request_article_change + article = request_article_change.article if request_article_change.change_type == choices.RCT_ERRATUM: article.status = choices.AS_REQUIRE_ERRATUM elif request_article_change.change_type == choices.RCT_UPDATE: From aff75e526fa12cc66ef05765e69f525736ba2b7b Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Tue, 19 Nov 2024 00:58:58 -0300 Subject: [PATCH 03/22] Apresenta os campos de Article como read_only e completa Article.update_status --- article/models.py | 46 +++++++++++++++++++++++++++++++++------------- 1 file changed, 33 insertions(+), 13 deletions(-) diff --git a/article/models.py b/article/models.py index 4983a138..333a311e 100644 --- a/article/models.py +++ b/article/models.py @@ -88,7 +88,7 @@ class Article(ClusterableModel, CommonControlField): ) panel_article_ids.children = [ # FieldPanel("pid_v2"), - FieldPanel("pid_v3"), + FieldPanel("pid_v3", read_only=True), # FieldPanel("aop_pid"), ] @@ -96,19 +96,19 @@ class Article(ClusterableModel, CommonControlField): heading="Article details", classname="collapsible" ) panel_article_details.children = [ - FieldPanel("first_publication_date"), - FieldPanel("article_type"), - FieldPanel("status"), + FieldPanel("first_publication_date", read_only=True), + FieldPanel("article_type", read_only=True), + FieldPanel("status", read_only=True), InlinePanel(relation_name="title_with_lang", label="Title with Language"), - FieldPanel("elocation_id"), - FieldPanel("fpage"), - FieldPanel("lpage"), + FieldPanel("elocation_id", read_only=True), + FieldPanel("fpage", read_only=True), + FieldPanel("lpage", read_only=True), ] panels = [ panel_article_ids, panel_article_details, - FieldPanel("issue", classname="collapsible"), + FieldPanel("issue", classname="collapsible", read_only=True), ] base_form_class = ArticleForm @@ -347,8 +347,18 @@ def display_sections(self): return str(self.multilingual_sections) def update_status(self, new_status=None): + # AS_UPDATE_SUBMITTED = "update-submitted" + # AS_ERRATUM_SUBMITTED = "erratum-submitted" + # AS_REQUIRE_UPDATE = "required-update" + # AS_REQUIRE_ERRATUM = "required-erratum" + # AS_PREPARE_TO_PUBLISH = "prepare-to-publish" + # AS_READY_TO_PUBLISH = "ready-to-publish" + # AS_SCHEDULED_TO_PUBLISH = "scheduled-to-publish" + # AS_PUBLISHED = "published" + # TODO create PublicationEvent - if self.status == choices.AS_UPDATE_SUBMITTED: + + if self.status in (choices.AS_UPDATE_SUBMITTED, choices.AS_READY_TO_PUBLISH, choices.AS_PUBLISHED): self.status = choices.AS_REQUIRE_UPDATE self.save() return @@ -358,10 +368,16 @@ def update_status(self, new_status=None): self.save() return - # AS_PREPARE_TO_PUBLISH = "prepare-to-publish" - # AS_READY_TO_PUBLISH = "ready-to-publish" - # AS_SCHEDULED_TO_PUBLISH = "scheduled-to-publish" - # AS_PUBLISHED = "published" + if self.status == choices.AS_REQUIRE_UPDATE: + self.status = choices.AS_UPDATE_SUBMITTED + self.save() + return + + if self.status == choices.AS_REQUIRE_ERRATUM: + self.status = choices.AS_ERRATUM_SUBMITTED + self.save() + return + self.status = new_status or choices.AS_PUBLISHED self.save() @@ -380,6 +396,10 @@ class ArticleTitle(HTMLTextModel, CommonControlField): "Article", on_delete=models.CASCADE, related_name="title_with_lang" ) + panels = [ + FieldPanel("text", read_only=True), + FieldPanel("language", read_only=True), + ] class RelatedItem(CommonControlField): item_type = models.CharField( From 4ae6574f06475fd245d028960c5011ea3cda1b6f Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Tue, 19 Nov 2024 00:59:08 -0300 Subject: [PATCH 04/22] =?UTF-8?q?Omite=20(temporariamente)=20a=20op=C3=A7?= =?UTF-8?q?=C3=A3o=20de=20solicitar=20altera=C3=A7=C3=A3o=20de=20Article?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- article/wagtail_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/article/wagtail_hooks.py b/article/wagtail_hooks.py index 9b4383f6..1313ee2d 100644 --- a/article/wagtail_hooks.py +++ b/article/wagtail_hooks.py @@ -154,7 +154,7 @@ class ArticleModelAdminGroup(ModelAdminGroup): items = ( ArticleModelAdmin, # RelatedItemModelAdmin, - RequestArticleChangeModelAdmin, + # omitir temporariamente RequestArticleChangeModelAdmin, # ApprovedArticleModelAdmin, ) From 73d88f923e3761f84081afe55a00597a068f1176 Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Tue, 19 Nov 2024 00:59:15 -0300 Subject: [PATCH 05/22] =?UTF-8?q?Ao=20criar=20SPSPkg,=20no=20lugar=20de=20?= =?UTF-8?q?guardar=20o=20zip=20otimizado,=20guarda=20o=20zip=20sem=20otimi?= =?UTF-8?q?za=C3=A7=C3=A3o,=20para=20ser=20mais=20similar=20ao=20pacote=20?= =?UTF-8?q?submetido?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- package/models.py | 77 ++++++++++++++++++++++++----------------------- 1 file changed, 39 insertions(+), 38 deletions(-) diff --git a/package/models.py b/package/models.py index d54ce23a..d0802ea7 100644 --- a/package/models.py +++ b/package/models.py @@ -158,7 +158,7 @@ def save_file(self, name, content): try: self.file.delete(save=True) except Exception as e: - pass + logging.exception(f"Unable to delete {self.file.path} {e} {type(e)}") try: self.file.save(name, ContentFile(content)) except Exception as e: @@ -340,7 +340,7 @@ def save_file(self, name, content): try: self.file.delete(save=True) except Exception as e: - pass + logging.exception(f"Unable to delete {self.file.path} {e} {type(e)}") try: self.file.save(name, ContentFile(content)) except Exception as e: @@ -628,45 +628,29 @@ def add_pid_v3_to_zip(cls, user, zip_xml_file_path, is_public, article_proc): def save_pkg_zip_file(self, user, zip_file_path, article_proc): operation = article_proc.start(user, "save_pkg_zip_file") filename = self.sps_pkg_name + ".zip" + # saved original try: - with TemporaryDirectory() as targetdir: - with TemporaryDirectory() as workdir: - target = os.path.join(targetdir, filename) - package = SPPackage.from_file(zip_file_path, workdir) - package.optimise(new_package_file_path=target, preserve_files=False) - - # saved optimised - with open(target, "rb") as fp: - self.save_file(filename, fp.read()) - operation.finish( - user, - completed=True, - detail={"source": target, "optimized": True}, - ) + with open(zip_file_path, "rb") as fp: + self.save_file(filename, fp.read()) + operation.finish( + user, + completed=True, + detail={"source": zip_file_path, "optimized": False}, + ) except Exception as e: - # saved original - try: - with open(zip_file_path, "rb") as fp: - self.save_file(filename, fp.read()) - operation.finish( - user, - completed=True, - detail={"source": zip_file_path, "optimized": False, "message": str(e)}, - ) - except Exception as e: - exc_type, exc_value, exc_traceback = sys.exc_info() - operation.finish( - user, - exc_traceback=exc_traceback, - exception=e, - detail={"source": zip_file_path, "optimized": False}, - ) + exc_type, exc_value, exc_traceback = sys.exc_info() + operation.finish( + user, + exc_traceback=exc_traceback, + exception=e, + detail={"source": zip_file_path, "optimized": False}, + ) def save_file(self, name, content): try: self.file.delete(save=True) except Exception as e: - pass + logging.exception(f"Unable to delete {self.file.path} {e} {type(e)}") try: self.file.save(name, ContentFile(content)) except Exception as e: @@ -688,8 +672,8 @@ def upload_package_to_the_cloud(self, user, original_pkg_components, article_pro xml_with_pre = self.upload_items_to_the_cloud( user, article_proc, - "upload_assets_to_the_cloud", - self.upload_assets_to_the_cloud, + "upload_zip_content_to_the_cloud", + self.upload_zip_content_to_the_cloud, **{"original_pkg_components": original_pkg_components}, ) self.upload_items_to_the_cloud( @@ -776,10 +760,27 @@ def upload_to_the_cloud( response["filename"] = filename return response - def upload_assets_to_the_cloud(self, user, original_pkg_components): + def upload_zip_content_to_the_cloud(self, user, original_pkg_components): + filename = self.sps_pkg_name + ".zip" + try: + result = {} + with TemporaryDirectory() as targetdir: + with TemporaryDirectory() as workdir: + zip_file_path = os.path.join(targetdir, filename) + package = SPPackage.from_file(zip_file_path, workdir) + package.optimise(new_package_file_path=target, preserve_files=False) + result = self.upload_components_to_the_cloud(user, original_pkg_components, zip_file_path) + return result + except Exception as e: + zip_file_path = self.file.path + return self.upload_components_to_the_cloud(user, original_pkg_components, zip_file_path) + + + def upload_components_to_the_cloud(self, user, original_pkg_components, zip_file_path): xml_with_pre = None items = [] - with ZipFile(self.file.path) as optimised_fp: + + with ZipFile(zip_file_path) as optimised_fp: for item in set(optimised_fp.namelist()): name, ext = os.path.splitext(item) content = optimised_fp.read(item) From c6f6b39b9d02167e36b432af39b365682e569024 Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Tue, 19 Nov 2024 00:59:21 -0300 Subject: [PATCH 06/22] =?UTF-8?q?Adiciona=20o=20status=20PS=5FARCHIVED=20p?= =?UTF-8?q?ara=20pacotes=20e=20a=20categoria=20VAL=5FCAT=5FQA=5FCONCLUSION?= =?UTF-8?q?=20para=20relat=C3=B3rio=20de=20erro?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- upload/choices.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/upload/choices.py b/upload/choices.py index be4cbb82..565d8810 100644 --- a/upload/choices.py +++ b/upload/choices.py @@ -56,6 +56,21 @@ PS_REQUIRED_ERRATUM = "required-erratum" PS_REQUIRED_UPDATE = "required-update" PS_DEPUBLISHED = "depublished" +PS_ARCHIVED = "archived" + +PS_WIP = ( + PS_SUBMITTED, + PS_ENQUEUED_FOR_VALIDATION, + PS_VALIDATED_WITH_ERRORS, + PS_PENDING_QA_DECISION, + # PS_UNEXPECTED, + PS_READY_TO_PREVIEW, + PS_PREVIEW, + PS_READY_TO_PUBLISH, + # PS_PUBLISHED, + # PS_DEPUBLISHED, + # PS_ARCHIVED, +) PACKAGE_STATUS = ( (PS_REQUIRED_ERRATUM, _("Required erratum")), @@ -70,6 +85,7 @@ (PS_PREVIEW, _("Preview")), (PS_READY_TO_PUBLISH, _("Ready to publish on public website")), (PS_PUBLISHED, _("Published")), + (PS_ARCHIVED, _("Archived")), ) CRITICAL_ERROR_DECISION = ( @@ -122,6 +138,8 @@ VAL_CAT_RENDITION_CONTENT = "rendition-content" VAL_CAT_WEB_PAGE_CONTENT = "web-page-content" VAL_CAT_GROUP_DATA = "group" +VAL_CAT_QA_CONCLUSION = "qa-conclusion" + VALIDATION_CATEGORY = ( (VAL_CAT_ARTICLE_JOURNAL_COMPATIBILITY, "ARTICLE_JOURNAL_COMPATIBILITY"), @@ -137,6 +155,7 @@ (VAL_CAT_ASSET, "ASSET"), (VAL_CAT_RENDITION, "RENDITION"), (VAL_CAT_PACKAGE_FILE, "PACKAGE_FILE"), + (VAL_CAT_QA_CONCLUSION, "QA conclusion"), ) # Model ValidationResult, Field status From 79168eb53e85e2f18a828dcaf6d70ec21e5c7528 Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Tue, 19 Nov 2024 00:59:28 -0300 Subject: [PATCH 07/22] Cria a rota archive --- upload/urls.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/upload/urls.py b/upload/urls.py index 91db35e9..855604bd 100644 --- a/upload/urls.py +++ b/upload/urls.py @@ -1,6 +1,6 @@ from django.urls import path -from .views import assign, download_errors, display_xml, finish_deposit, preview_document +from .views import assign, download_errors, display_xml, finish_deposit, preview_document, archive_package app_name = "upload" @@ -10,4 +10,5 @@ path("finish", view=finish_deposit, name="finish_deposit"), path("download-errors", view=download_errors, name="download_errors"), path("xml", view=display_xml, name="display_xml"), + path("archive", view=archive_package, name="archive_package") ] From e9f456a65ea161a4b9d06ae473342483ee42a357 Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Tue, 19 Nov 2024 00:59:35 -0300 Subject: [PATCH 08/22] =?UTF-8?q?Cria=20a=20fun=C3=A7=C3=A3o=20archive=5Fp?= =?UTF-8?q?ackage=20para=20package.status=20=3D=20PS=5FARCHIVED?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- upload/views.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/upload/views.py b/upload/views.py index 67771de5..1150dcdf 100644 --- a/upload/views.py +++ b/upload/views.py @@ -317,3 +317,21 @@ def assign(request): # messages.warning(request, _("Package has been reassigned with success.")) return redirect(f"/admin/upload/qapackage/edit/{package_id}") + + +def archive_package(request): + package_id = request.GET.get("package_id") + user = request.user + + if not user.has_perm("upload.user_can_packagezip_create"): + messages.error(request, _("You do not have permission to archive packages.")) + elif package_id: + package = get_object_or_404(Package, pk=package_id) + + if package.status == choices.PS_UNEXPECTED: + package.status = choices.PS_ARCHIVED + package.save() + messages.success(request, _("Package was archived.")) + else: + messages.warning(request, _("Unable to archive package which status = {}.").format(package.status)) + return redirect(f"/admin/upload/package/") From 6da678077939f152a69f9b838e0d12bb81200cc4 Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Tue, 19 Nov 2024 00:59:41 -0300 Subject: [PATCH 09/22] =?UTF-8?q?Corrige=20PackageZip.split=20que=20gerava?= =?UTF-8?q?=20pacotes=20incompletos;=20Cria=20Package.register=5Fqa=5Fcomm?= =?UTF-8?q?ent=5Fas=5Ferror=20para=20registrar=20coment=C3=A1rios=20do=20a?= =?UTF-8?q?nalista=20em=20relat=C3=B3rio=20de=20problemas;=20Cria=20Archiv?= =?UTF-8?q?edPackage=20para=20filtrar=20os=20pacotes=20com=20status=20'ter?= =?UTF-8?q?minado'?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- upload/models.py | 56 ++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 47 insertions(+), 9 deletions(-) diff --git a/upload/models.py b/upload/models.py index b1bc7438..d74ed2ce 100644 --- a/upload/models.py +++ b/upload/models.py @@ -139,7 +139,7 @@ def save_file(self, filename=None, content=None): try: self.file.delete(save=True) except Exception as e: - pass + logging.exception(f"Unable to delete {self.file.path} {e} {type(e)}") self.file.save(filename, ContentFile(content)) @property @@ -153,7 +153,7 @@ def split(self, user): with ZipFile(self.file.path) as zf: # obtém os components do zip file_paths = set(zf.namelist() or []) - logging.info(file_paths) + logging.info(f"file_paths: {file_paths}") other_files = {} xmls = {} @@ -172,21 +172,23 @@ def split(self, user): other_files.setdefault(name, []) other_files[name].append(data) - logging.info(xmls) - logging.info(other_files) - + logging.info(f"xmls: {xmls}") + logging.info(f"other_files: {other_files}") + for key in xmls.keys(): other_files_copy = other_files.copy() for other_files_key in other_files_copy.keys(): + logging.info(f"xml: {key}, other_files_key: {other_files_key}") if other_files_key.startswith(key + "."): xmls[key].extend(other_files.pop(other_files_key)) - break + logging.info(f"files ({key}): {xmls[key]}") elif other_files_key.startswith(key + "-"): xmls[key].extend(other_files.pop(other_files_key)) - break + logging.info(f"files ({key}): {xmls[key]}") logging.info(xmls) for key, files in xmls.items(): + logging.info(f"{key} {files}") try: with TemporaryDirectory() as tmpdirname: zfile = os.path.join(tmpdirname, f"{key}.zip") @@ -559,7 +561,7 @@ def is_validation_finished(self): return False return True - def finish_validations(self, task_process_qa_decision=None): + def finish_validations(self, task_process_qa_decision=None, blocking_error_status=None): """ 1. Verifica se as validações que executaram em paralelo, finalizaram 2. Calcula os números de problemas do pacote @@ -588,7 +590,8 @@ def finish_validations(self, task_process_qa_decision=None): # verifica status a partir destes números if metrics["total_blocking"]: # pacote tem erros indiscutíveis - self.status = choices.PS_PENDING_CORRECTION + # choices.PS_PENDING_CORRECTION | choices.PS_UNEXPECTED + self.status = blocking_error_status or choices.PS_PENDING_CORRECTION elif ( metrics["total_validations"] > 0 and (metrics["total_xml_issues"] + metrics["total_pkg_issues"]) == 0 @@ -891,6 +894,10 @@ def get_errors_report_content(self): def process_qa_decision(self, user): operation = self.start(user, "process_qa_decision") + if self.qa_decision == choices.PS_PENDING_QA_DECISION: + self.finish_qa_decision(user, operation, websites=None, result=None, rule=None) + return [] + if self.qa_decision == choices.PS_PENDING_CORRECTION: self.finish_qa_decision(user, operation, websites=None, result=None, rule=None) return [] @@ -991,6 +998,9 @@ def finish_qa_decision(self, user, operation, websites, result, rule): except (KeyError, AttributeError, TypeError, ValueError): exception = None + if self.qa_comment: + self.register_qa_comment_as_error(user) + if result and result.get("request_correction"): self.qa_decision = choices.PS_PENDING_CORRECTION self.assignee = self.creator @@ -1179,6 +1189,27 @@ def save_file(self, filename=None, content=None): pass self.file.save(filename, ContentFile(content)) + def register_qa_comment_as_error(self, user, data=None): + data = data or {} + data.update({"qa_decision": self.qa_decision}) + report_title = _("QA decision report") + category = choices.VAL_CAT_QA_CONCLUSION + report = ValidationReport.create_or_update( + user, + self, + report_title, + category, + reset_validations=False, + ) + validation_result = report.add_validation_result( + status=choices.VALIDATION_RESULT_FAILURE, + message=self.qa_comment, + data=data, + subject="qa decision", + ) + report.creation = choices.REPORT_CREATION_DONE + report.save() + class QAPackage(Package): """ @@ -1955,3 +1986,10 @@ def check_xml_warnings_percentage(self, value): def check_impossible_to_fix_percentage(self, value): return value <= self.max_impossible_to_fix_percentage + + +class ArchivedPackage(Package): + + class Meta: + proxy = True + From 0ef1bfc9e8992784487fb233e4ee195a1845026f Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Tue, 19 Nov 2024 00:59:48 -0300 Subject: [PATCH 10/22] =?UTF-8?q?Para=20Package.name,=20usa=20o=20nome=20d?= =?UTF-8?q?o=20arquivo=20XML=20no=20lugar=20do=20sps=5Fpkg=5Fname;=20Alter?= =?UTF-8?q?a=20a=20l=C3=B3gica=20de=20=5Fcheck=5Fpackage=5Fis=5Fexpected;?= =?UTF-8?q?=20Cria=20=5Farchive=5Fpending=5Fcorrection=5Fpackage;=20Corrig?= =?UTF-8?q?e=20=5Fcheck=5Fxml=5Fand=5Fregistered=5Fdata=5Fcompability=20qu?= =?UTF-8?q?e=20erroneamente=20informava=20que=20os=20dados=20eram=20incomp?= =?UTF-8?q?at=C3=ADveis?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- upload/controller.py | 112 +++++++++++++++++++++---------------------- upload/tasks.py | 2 - 2 files changed, 56 insertions(+), 58 deletions(-) diff --git a/upload/controller.py b/upload/controller.py index 57b35a33..8d119293 100644 --- a/upload/controller.py +++ b/upload/controller.py @@ -38,12 +38,10 @@ pp = PidRequester() -class UnexpectedPackageError(Exception): - ... +class UnexpectedPackageError(Exception): ... -class PackageDataError(Exception): - ... +class PackageDataError(Exception): ... def get_last_package(article_id, **kwargs): @@ -70,7 +68,6 @@ def receive_package(user, package): if item is not package: package.linked.add(item) package.main_doi = xml_with_pre.main_doi - package.name = xml_with_pre.sps_pkg_name package.add_order(xml_with_pre.order, xml_with_pre.fpage) package.save() @@ -87,7 +84,7 @@ def receive_package(user, package): ) package.category = response.get("package_category") package.status = response.get("package_status") - package.expiration_date = response.get("previous_package") + package.expiration_date = response.get("expiration_date") package.save() error = ( @@ -111,7 +108,7 @@ def receive_package(user, package): ) # falhou, retorna response report.finish_validations() - package.finish_validations() + package.finish_validations(blocking_error_status=response.get("package_status")) return response if package.article: @@ -168,7 +165,8 @@ def _check_article_and_journal(package, xml_with_pre, user): response = pp.is_registered_xml_with_pre(xml_with_pre, xml_with_pre.filename) logging.info(f"is_registered_xml_with_pre: {response}") # verifica se o XML é esperado (novo, requer correção, requer atualização) - _check_package_is_expected(response, package, xml_with_pre.sps_pkg_name) + name, ext = os.path.splitext(xml_with_pre.filename) + _check_package_is_expected(response, package, name) logging.info(f"_check_package_is_expected: {response}") # verifica se journal e issue estão registrados @@ -185,6 +183,8 @@ def _check_article_and_journal(package, xml_with_pre, user): logging.info(f"_check_xml_and_registered_data_compability: {response}") response["package_status"] = choices.PS_ENQUEUED_FOR_VALIDATION + + _archive_pending_correction_package(response, name) return response except UnexpectedPackageError as e: response["package_status"] = choices.PS_UNEXPECTED @@ -212,6 +212,7 @@ def _check_package_is_expected(response, package, sps_pkg_name): article = None params = {"name": sps_pkg_name} + logging.info(f"_check_package_is_expected: {params}") try: # se o pacote anterior está pendente de correção, então é aceitável previous_package = Package.objects.filter(**params).order_by("-created")[1] @@ -220,48 +221,40 @@ def _check_package_is_expected(response, package, sps_pkg_name): response["article"] = article - if article: - if previous_package: - if previous_package.status == choices.PS_PENDING_CORRECTION: - response["previous_package"] = previous_package.expiration_date - if previous_package.expiration_date < datetime.utcnow(): - raise UnexpectedPackageError( - _("The package is late. It was expected until {}").format( - previous_package.expiration_date.isoformat()) - ) - elif previous_package.status not in ( - choices.PS_REQUIRED_ERRATUM, - choices.PS_REQUIRED_UPDATE, - ): - raise UnexpectedPackageError( - _("There is a previous package in progress ({}) for {}").format( - previous_package.status, article) - ) + if previous_package and previous_package.status in choices.PS_WIP: + raise UnexpectedPackageError( + _("Not allowed to accept new package because there is a package already being processed ({}). To force updating, package status must be changed to '{}'").format( + previous_package.status, choices.PS_PENDING_CORRECTION + ) + ) + if article: if article.status == article_choices.AS_REQUIRE_UPDATE: response["package_category"] = choices.PC_UPDATE - article.status = article_choices.AS_UPDATE_SUBMITTED - article.save() elif article.status == article_choices.AS_REQUIRE_ERRATUM: response["package_category"] = choices.PC_ERRATUM - article.status = article_choices.AS_ERRATUM_SUBMITTED - article.save() else: response["package_category"] = choices.PC_UPDATE + raise UnexpectedPackageError( - _("Package is rejected because the article status is: {}").format(article.status) + _("Not allowed to accept new package because there is a package already being processed ({}). To force updating, package status must be changed to '{}'").format( + article.status, choices.PS_PENDING_CORRECTION + ) ) + else: + response["package_category"] = choices.PC_NEW_DOCUMENT + return + +def _archive_pending_correction_package(response, name): + params = {} + if response.get("article"): + params["article"] = response.get("article") else: - if ( - not previous_package - or previous_package.status == choices.PS_PENDING_CORRECTION - ): - response["package_category"] = choices.PC_NEW_DOCUMENT - return - raise UnexpectedPackageError( - _("Unexpected package {} (status={})").format(sps_pkg_name, previous_package.status) - ) + params["name"] = name + Package.objects.filter( + **params, status=choices.PS_PENDING_CORRECTION + ).update(status=choices.PS_ARCHIVED) def _check_journal(response, xmltree, user): @@ -284,15 +277,19 @@ def _check_journal(response, xmltree, user): } similar_journals = [] for j in Journal.get_similar_items(journal_title, issn_electronic, issn_print): - similar_journals.append({ - "journal_title": j.title, - "issn_electronic": j.official_journal.issn_electronic, - "issn_print": j.official_journal.issn_print, - }) + similar_journals.append( + { + "journal_title": j.title, + "issn_electronic": j.official_journal.issn_electronic, + "issn_print": j.official_journal.issn_print, + } + ) if similar_journals: similar_journals = _("Expected journal data: {}. ").format(similar_journals) else: - similar_journals = _("No journal with similar data is registered in the system") + similar_journals = _( + "No journal with similar data is registered in the system" + ) raise PackageDataError( _( "Journal data in XML: {}. {}Check and fix the journal data in XML" @@ -328,22 +325,25 @@ def _check_issue(response, xmltree, user): if publication_year and xml.volume: items = Issue.objects.filter( Q(publication_year=publication_year) | Q(volume=xml.volume), - journal=response["journal"]) + journal=response["journal"], + ) elif publication_year: items = Issue.objects.filter( - Q(publication_year=publication_year), - journal=response["journal"]) + Q(publication_year=publication_year), journal=response["journal"] + ) if not items or not items.count(): items = Issue.objects.filter(journal=response["journal"]) issues = [] for item in items.order_by("-publication_year"): - issues.append({ - "publication_year": publication_year, - "volume": item.volume, - "number": item.number, - "supplement": item.supplement, - }) + issues.append( + { + "publication_year": publication_year, + "volume": item.volume, + "number": item.number, + "supplement": item.supplement, + } + ) if issues: issues = _("Expected issue data: {}. ").format(issues) else: @@ -360,7 +360,7 @@ def _check_xml_and_registered_data_compability(response): if article: journal = response["journal"] - if journal is not article.journal: + if not journal == article.journal: raise PackageDataError( _("{} (registered, {}) differs from {} (XML, {})").format( article.journal, article.journal.id, journal, journal.id @@ -368,7 +368,7 @@ def _check_xml_and_registered_data_compability(response): ) issue = response["issue"] - if issue is not article.issue: + if not issue == article.issue: raise PackageDataError( _("{} (registered, {}) differs from {} (XML, {})").format( article.issue, article.issue.id, issue, issue.id diff --git a/upload/tasks.py b/upload/tasks.py index e1df49af..4126487b 100644 --- a/upload/tasks.py +++ b/upload/tasks.py @@ -303,8 +303,6 @@ def task_validate_original_zip_file( name, ext = os.path.splitext(xml_path) logging.info(f"xmlpre: {xml_with_pre.xmlpre}") package = Package.objects.get(pk=package_id) - package.name = xml_with_pre.sps_pkg_name - package.save() # FIXME nao usar o otimizado neste momento optimised_filepath = task_optimise_package(file_path) From 6d99f2992f94a0c87dd77246fda9b466a34e7d1e Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Tue, 19 Nov 2024 00:59:56 -0300 Subject: [PATCH 11/22] Criar UploadPermissionHelper.user_is_analyst_team_member e UploadPermissionHelper.user_is_xml_producer --- upload/permission_helper.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/upload/permission_helper.py b/upload/permission_helper.py index 12dcab31..a9918498 100644 --- a/upload/permission_helper.py +++ b/upload/permission_helper.py @@ -1,6 +1,6 @@ from wagtail.contrib.modeladmin.helpers import PermissionHelper -from team.models import has_permission +from team.models import has_permission, CollectionTeamMember ACCESS_ALL_PACKAGES = "access_all_packages" @@ -33,3 +33,11 @@ def user_can_analyse_error_validation_resolution(self, user, obj): return self.user_has_specific_permission( user, ANALYSE_VALIDATION_ERROR_RESOLUTION ) + + def user_is_analyst_team_member(self, user, obj): + if self.user_can_use_upload_module(user, obj): + return CollectionTeamMember.objects.filter(user=user).exists() + + def user_is_xml_producer(self, user, obj): + if self.user_can_use_upload_module(user, obj): + return not self.user_is_analyst_team_member(user, obj) From d15d611b85ec0ff4138a2f2afc3f4e3326767d69 Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Tue, 19 Nov 2024 01:00:02 -0300 Subject: [PATCH 12/22] =?UTF-8?q?Substituir=20os=20m=C3=A9todos=20de=20per?= =?UTF-8?q?mission=5Fhelper?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- upload/button_helper.py | 113 ++++++++++++++++++++++++---------------- 1 file changed, 67 insertions(+), 46 deletions(-) diff --git a/upload/button_helper.py b/upload/button_helper.py index 2276105e..3d9988c9 100644 --- a/upload/button_helper.py +++ b/upload/button_helper.py @@ -12,34 +12,6 @@ class UploadButtonHelper(ButtonHelper): "icon", ] - def assign(self, obj, classnames, label=None): - text = label or _("Accept / Reject the package or delegate it") - return { - "url": reverse("upload:assign") + "?package_id=%s" % str(obj.id), - "label": text, - "classname": self.finalise_classname(classnames), - "title": text, - } - - def finish_deposit_button(self, obj, classnames): - text = _("Finish deposit") - return { - "url": reverse("upload:finish_deposit") + "?package_id=%s" % str(obj.id), - "label": text, - "classname": self.finalise_classname(classnames), - "title": text, - } - - def view_published_document(self, obj, classnames): - # TODO: A URL deve ser configurada para ver o documento publicado - text = _("View document") - return { - "url": "", - "label": text, - "classname": self.finalise_classname(classnames), - "title": text, - } - def get_buttons_for_obj( self, obj, exclude=None, classnames_add=None, classnames_exclude=None ): @@ -51,9 +23,30 @@ def get_buttons_for_obj( usr = self.request.user url_name = self.request.resolver_match.url_name - if not ph.user_can_analyse_error_validation_resolution(usr, obj): + print(f"urlname: {url_name}") + + analyst_team_member = ph.user_is_analyst_team_member(usr, obj) + + exclude = ["delete"] + if analyst_team_member: + if obj.status in ( + choices.PS_SUBMITTED, + choices.PS_ENQUEUED_FOR_VALIDATION, + choices.PS_VALIDATED_WITH_ERRORS, + choices.PS_PENDING_QA_DECISION, + choices.PS_PENDING_CORRECTION, + choices.PS_UNEXPECTED, + choices.PS_PUBLISHED, + choices.PS_REQUIRED_ERRATUM, + choices.PS_REQUIRED_UPDATE, + choices.PS_ARCHIVED, + ): + exclude.append("edit") + else: # usuário sem poder de análise - exclude = ["edit", "delete"] + exclude.append("edit") + if url_name.endswith("_modeladmin_inspect"): + exclude.append("inspect") btns = super().get_buttons_for_obj( obj, exclude, classnames_add, classnames_exclude @@ -68,21 +61,49 @@ def get_buttons_for_obj( if url_name.endswith("_modeladmin_index"): classnames.extend(UploadButtonHelper.index_button_classnames) - if ( - obj.status == choices.PS_VALIDATED_WITH_ERRORS - and ph.user_can_finish_deposit(usr, obj) - and url_name == "upload_package_modeladmin_inspect" - ): - btns.append(self.finish_deposit_button(obj, classnames)) - - if ( - obj.status - in ( - choices.PS_PENDING_QA_DECISION, - choices.PS_VALIDATED_WITH_ERRORS, - ) - and ph.user_can_assign_package(usr, obj) - ): - btns.append(self.assign(obj, classnames)) + self.add_finish_deposit_button(btns, obj, classnames, url_name) + if analyst_team_member: + self.add_assign_button(btns, obj, classnames) + self.add_archive_button(btns, obj, classnames) return btns + + def add_assign_button(self, btns, obj, classnames, label=None): + status = ( + choices.PS_PENDING_QA_DECISION, + choices.PS_VALIDATED_WITH_ERRORS, + ) + if obj.status in status: + text = label or _("Accept / Reject the package or delegate it") + btns.append({ + "url": reverse("upload:assign") + "?package_id=%s" % str(obj.id), + "label": text, + "classname": self.finalise_classname(classnames), + "title": text, + }) + + def add_finish_deposit_button(self, btns, obj, classnames, url_name): + status = ( + choices.PS_VALIDATED_WITH_ERRORS, + ) + if obj.status in status and url_name == "upload_package_modeladmin_inspect": + text = _("Finish deposit") + btns.append({ + "url": reverse("upload:finish_deposit") + "?package_id=%s" % str(obj.id), + "label": text, + "classname": self.finalise_classname(classnames), + "title": text, + }) + + def add_archive_button(self, btns, obj, classnames, label=None): + status = ( + choices.PS_UNEXPECTED, + ) + if obj.status in status: + text = label or _("Archive") + btns.append({ + "url": reverse("upload:archive_package") + "?package_id=%s" % str(obj.id), + "label": text, + "classname": self.finalise_classname(classnames), + "title": text, + }) From 458d2e109bf5d0b4ae1537601efe61eb83c5c744 Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Tue, 19 Nov 2024 01:00:10 -0300 Subject: [PATCH 13/22] =?UTF-8?q?Cria=20ArchivedPackageAdmin=20para=20todo?= =?UTF-8?q?s=20os=20pacotes=20'terminados'=20e=20apresenta=20os=20artigos?= =?UTF-8?q?=20cujo=20status=3DPS=5FPUBLISHED=20na=20=C3=A1rea=20administra?= =?UTF-8?q?tiva=20Upload=20>=20Package=20publication;=20Substituir=20os=20?= =?UTF-8?q?m=C3=A9todos=20de=20permission=5Fhelper?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- upload/wagtail_hooks.py | 133 +++++++++++++++++++++++++++------------- 1 file changed, 92 insertions(+), 41 deletions(-) diff --git a/upload/wagtail_hooks.py b/upload/wagtail_hooks.py index 0c6a2c28..5643ec04 100644 --- a/upload/wagtail_hooks.py +++ b/upload/wagtail_hooks.py @@ -35,6 +35,7 @@ XMLInfoReport, choices, PackageZip, + ArchivedPackage, ) from .permission_helper import UploadPermissionHelper from team.models import has_permission @@ -73,9 +74,9 @@ class PackageZipAdmin(ModelAdmin): def get_queryset(self, request): if not self.permission_helper.user_can_use_upload_module(request.user, None): return super().get_queryset(request).none() - if self.permission_helper.user_can_access_all_packages(request.user, None): - params = {} - elif self.permission_helper.user_can_finish_deposit(request.user, None): + + params = {} + if not self.permission_helper.user_is_analyst_team_member(request.user, None): params = {"creator": request.user} return super().get_queryset(request).filter(**params) @@ -89,6 +90,7 @@ class PackageAdmin(ModelAdmin): # create_view_class = PackageCreateView inspect_view_enabled = True inspect_view_class = PackageAdminInspectView + inspect_template_name = "modeladmin/upload/package/inspect.html" menu_label = _("Package Validation") menu_icon = "folder" menu_order = 200 @@ -135,15 +137,14 @@ class PackageAdmin(ModelAdmin): def get_queryset(self, request): if not self.permission_helper.user_can_use_upload_module(request.user, None): return super().get_queryset(request).none() - params = {} + params = {} try: params["pkg_zip__id"] = request.GET["pkg_zip_id"] except KeyError: logging.info(request.GET) - if self.permission_helper.user_can_access_all_packages(request.user, None): - params = {} + if self.permission_helper.user_is_analyst_team_member(request.user, None): waiting_status = [ choices.PS_ENQUEUED_FOR_VALIDATION, choices.PS_PENDING_CORRECTION, @@ -174,7 +175,7 @@ def get_queryset(self, request): + action_required_qa_menu + action_required_publication_menu ) - elif self.permission_helper.user_can_finish_deposit(request.user, None): + else: params["creator"] = request.user action_required = [ choices.PS_VALIDATED_WITH_ERRORS, @@ -260,12 +261,12 @@ def get_queryset(self, request): choices.PS_PENDING_CORRECTION, choices.PS_PENDING_QA_DECISION, ] - if self.permission_helper.user_can_access_all_packages(request.user, None): + if self.permission_helper.user_is_analyst_team_member(request.user, None): params = {} return super().get_queryset(request).filter( Q(assignee__isnull=True) | Q(assignee=request.user), status__in=status, **params) - elif self.permission_helper.user_can_finish_deposit(request.user, None): + else: params = {"creator": request.user} return super().get_queryset(request).filter(status__in=status, **params) @@ -322,13 +323,13 @@ def get_queryset(self, request): choices.PS_READY_TO_PREVIEW, choices.PS_PREVIEW, choices.PS_READY_TO_PUBLISH, + choices.PS_PUBLISHED, # choices.PS_SCHEDULED_PUBLICATION, ] - if self.permission_helper.user_can_access_all_packages(request.user, None): + if self.permission_helper.user_is_analyst_team_member(request.user, None): params = {} - return super().get_queryset(request).filter( - status__in=status, **params) - elif self.permission_helper.user_can_finish_deposit(request.user, None): + return super().get_queryset(request).filter(status__in=status, **params) + else: params = {"creator": request.user} return super().get_queryset(request).filter(status__in=status, **params) @@ -364,10 +365,8 @@ class XMLErrorReportAdmin(ModelAdmin): def get_queryset(self, request): if not self.permission_helper.user_can_use_upload_module(request.user, None): return super().get_queryset(request).none() - if ( - request.user.is_superuser - or self.permission_helper.user_can_access_all_packages(request.user, None) - ): + + if self.permission_helper.user_is_analyst_team_member(request.user, None): return super().get_queryset(request) return super().get_queryset(request).filter(package__creator=request.user) @@ -408,10 +407,8 @@ class XMLErrorAdmin(ModelAdmin): def get_queryset(self, request): if not self.permission_helper.user_can_use_upload_module(request.user, None): return super().get_queryset(request).none() - if ( - request.user.is_superuser - or self.permission_helper.user_can_access_all_packages(request.user, None) - ): + + if self.permission_helper.user_is_analyst_team_member(request.user, None): return super().get_queryset(request) return super().get_queryset(request).filter(package__creator=request.user) @@ -447,10 +444,8 @@ class XMLInfoReportAdmin(ModelAdmin): def get_queryset(self, request): if not self.permission_helper.user_can_use_upload_module(request.user, None): return super().get_queryset(request).none() - if ( - request.user.is_superuser - or self.permission_helper.user_can_access_all_packages(request.user, None) - ): + + if self.permission_helper.user_is_analyst_team_member(request.user, None): return super().get_queryset(request) return super().get_queryset(request).filter(package__creator=request.user) @@ -491,10 +486,8 @@ class XMLInfoAdmin(ModelAdmin): def get_queryset(self, request): if not self.permission_helper.user_can_use_upload_module(request.user, None): return super().get_queryset(request).none() - if ( - request.user.is_superuser - or self.permission_helper.user_can_access_all_packages(request.user, None) - ): + + if self.permission_helper.user_is_analyst_team_member(request.user, None): return super().get_queryset(request) return super().get_queryset(request).filter(package__creator=request.user) @@ -531,10 +524,8 @@ class ValidationReportAdmin(ModelAdmin): def get_queryset(self, request): if not self.permission_helper.user_can_use_upload_module(request.user, None): return super().get_queryset(request).none() - if ( - request.user.is_superuser - or self.permission_helper.user_can_access_all_packages(request.user, None) - ): + + if self.permission_helper.user_is_analyst_team_member(request.user, None): return super().get_queryset(request) return super().get_queryset(request).filter(package__creator=request.user) @@ -566,10 +557,8 @@ class ValidationAdmin(ModelAdmin): def get_queryset(self, request): if not self.permission_helper.user_can_use_upload_module(request.user, None): return super().get_queryset(request).none() - if ( - request.user.is_superuser - or self.permission_helper.user_can_access_all_packages(request.user, None) - ): + + if self.permission_helper.user_is_analyst_team_member(request.user, None): return super().get_queryset(request) return super().get_queryset(request).filter(package__creator=request.user) @@ -601,15 +590,76 @@ class UploadValidatorAdmin(ModelAdmin): def get_queryset(self, request): if not self.permission_helper.user_can_use_upload_module(request.user, None): return super().get_queryset(request).none() - if ( - request.user.is_superuser - or self.permission_helper.user_can_access_all_packages(request.user, None) - ): + + if self.permission_helper.user_is_analyst_team_member(request.user, None): return super().get_queryset(request) return super().get_queryset(request).none() +class ArchivedPackageAdmin(ModelAdmin): + model = ArchivedPackage + button_helper_class = UploadButtonHelper + permission_helper_class = UploadPermissionHelper + create_view_enabled = False + # create_view_class = PackageCreateView + inspect_view_enabled = True + inspect_view_class = PackageAdminInspectView + inspect_template_name = "modeladmin/upload/package/inspect.html" + menu_label = _("Archived Packages") + menu_icon = "folder" + menu_order = 200 + add_to_settings_menu = False + exclude_from_explorer = False + list_per_page = 20 + + list_display = ( + "__str__", + "critical_errors", + "xml_errors_percentage", + "category", + "status", + "creator", + "updated", + "expiration_date", + ) + list_filter = ( + "creator", + "category", + "status", + ) + search_fields = ( + "name", + "journal__official_journal__title", + "issue__journal__official_journal__title", + "article__pid_v3", + "creator__username", + "updated_by__username", + "pkg_zip__file", + ) + inspect_view_fields = ( + "article", + "issue", + "category", + "status", + "file", + "created", + "updated", + "expiration_date", + "files_list", + ) + + def get_queryset(self, request): + if not self.permission_helper.user_can_use_upload_module(request.user, None): + return super().get_queryset(request).none() + + params = {} + if not self.permission_helper.user_is_analyst_team_member(request.user, None): + params = {"creator": request.user} + + return super().get_queryset(request).filter(~Q(status__in=choices.PS_WIP), **params) + + class UploadModelAdminGroup(ModelAdminGroup): menu_icon = "folder" menu_label = "Upload" @@ -618,6 +668,7 @@ class UploadModelAdminGroup(ModelAdminGroup): PackageAdmin, QualityAnalysisPackageAdmin, ReadyToPublishPackageAdmin, + ArchivedPackageAdmin, ) menu_order = get_menu_order("upload") From 77198316be3411b7bc81ce6790f348f9f6af7c43 Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Tue, 19 Nov 2024 01:00:18 -0300 Subject: [PATCH 14/22] =?UTF-8?q?Adiciona=20merge=20de=20migra=C3=A7=C3=B5?= =?UTF-8?q?es=20de=20banco=20de=20dados=20de=20journal.models?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- journal/migrations/0007_merge_20241113_1945.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 journal/migrations/0007_merge_20241113_1945.py diff --git a/journal/migrations/0007_merge_20241113_1945.py b/journal/migrations/0007_merge_20241113_1945.py new file mode 100644 index 00000000..2abdf6ad --- /dev/null +++ b/journal/migrations/0007_merge_20241113_1945.py @@ -0,0 +1,12 @@ +# Generated by Django 5.0.3 on 2024-11-13 19:45 + +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("journal", "0005_journaltoc_alter_journal_official_journal_and_more"), + ("journal", "0006_alter_journalsection_text"), + ] + + operations = [] From ad75dbc1e159bbfb938b729684e021743d52f551 Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Tue, 19 Nov 2024 01:00:18 -0300 Subject: [PATCH 15/22] =?UTF-8?q?Adiciona=20migra=C3=A7=C3=B5es=20de=20ban?= =?UTF-8?q?co=20de=20dados=20de=20upload.models?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...edpackage_alter_package_status_and_more.py | 118 ++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 upload/migrations/0004_archivedpackage_alter_package_status_and_more.py diff --git a/upload/migrations/0004_archivedpackage_alter_package_status_and_more.py b/upload/migrations/0004_archivedpackage_alter_package_status_and_more.py new file mode 100644 index 00000000..fd82919a --- /dev/null +++ b/upload/migrations/0004_archivedpackage_alter_package_status_and_more.py @@ -0,0 +1,118 @@ +# Generated by Django 5.0.3 on 2024-11-18 14:45 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("upload", "0003_alter_package_options_alter_packagezip_options_and_more"), + ] + + operations = [ + migrations.CreateModel( + name="ArchivedPackage", + fields=[], + options={ + "proxy": True, + "indexes": [], + "constraints": [], + }, + bases=("upload.package",), + ), + migrations.AlterField( + model_name="package", + name="status", + field=models.CharField( + choices=[ + ("required-erratum", "Required erratum"), + ("required-update", "Required update"), + ("submitted", "Submetido"), + ("enqueued-for-validation", "Enqueued for validation"), + ("validated-with-errors", "Validated with errors"), + ("pending-qa-decision", "Pending quality analysis decision"), + ("pending-correction", "Pending for correction"), + ("unexpected", "Unexpected"), + ("ready-to-preview", "Ready to preview on QA website"), + ("preview", "Pré-visualizar"), + ("ready-to-publish", "Ready to publish on public website"), + ("published", "Publicado"), + ("archived", "Archived"), + ], + default="enqueued-for-validation", + max_length=32, + verbose_name="Status", + ), + ), + migrations.AlterField( + model_name="validationreport", + name="category", + field=models.CharField( + choices=[ + ("journal-incompatibility", "ARTICLE_JOURNAL_COMPATIBILITY"), + ("article-is-not-new", "ARTICLE_IS_NOT_NEW"), + ("xml-format", "XML_FORMAT"), + ("xml-style", "XML_STYLE"), + ("xml-content", "VAL_CAT_XML_CONTENT"), + ("group", "VAL_CAT_GROUP_DATA"), + ("bibliometrics-data", "BIBLIOMETRICS_DATA"), + ("services-data", "SERVICES_DATA"), + ("data-consistency", "DATA_CONSISTENCY"), + ("criteria-issues", "CRITERIA_ISSUES"), + ("asset", "ASSET"), + ("rendition", "RENDITION"), + ("package-file", "PACKAGE_FILE"), + ("qa-conclusion", "QA conclusion"), + ], + max_length=32, + verbose_name="Validation category", + ), + ), + migrations.AlterField( + model_name="xmlerrorreport", + name="category", + field=models.CharField( + choices=[ + ("journal-incompatibility", "ARTICLE_JOURNAL_COMPATIBILITY"), + ("article-is-not-new", "ARTICLE_IS_NOT_NEW"), + ("xml-format", "XML_FORMAT"), + ("xml-style", "XML_STYLE"), + ("xml-content", "VAL_CAT_XML_CONTENT"), + ("group", "VAL_CAT_GROUP_DATA"), + ("bibliometrics-data", "BIBLIOMETRICS_DATA"), + ("services-data", "SERVICES_DATA"), + ("data-consistency", "DATA_CONSISTENCY"), + ("criteria-issues", "CRITERIA_ISSUES"), + ("asset", "ASSET"), + ("rendition", "RENDITION"), + ("package-file", "PACKAGE_FILE"), + ("qa-conclusion", "QA conclusion"), + ], + max_length=32, + verbose_name="Validation category", + ), + ), + migrations.AlterField( + model_name="xmlinforeport", + name="category", + field=models.CharField( + choices=[ + ("journal-incompatibility", "ARTICLE_JOURNAL_COMPATIBILITY"), + ("article-is-not-new", "ARTICLE_IS_NOT_NEW"), + ("xml-format", "XML_FORMAT"), + ("xml-style", "XML_STYLE"), + ("xml-content", "VAL_CAT_XML_CONTENT"), + ("group", "VAL_CAT_GROUP_DATA"), + ("bibliometrics-data", "BIBLIOMETRICS_DATA"), + ("services-data", "SERVICES_DATA"), + ("data-consistency", "DATA_CONSISTENCY"), + ("criteria-issues", "CRITERIA_ISSUES"), + ("asset", "ASSET"), + ("rendition", "RENDITION"), + ("package-file", "PACKAGE_FILE"), + ("qa-conclusion", "QA conclusion"), + ], + max_length=32, + verbose_name="Validation category", + ), + ), + ] From 981836ca24158f1ab6148342bf96ffb7c53f30ef Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Tue, 19 Nov 2024 01:00:29 -0300 Subject: [PATCH 16/22] =?UTF-8?q?Apresenta=20a=20vers=C3=A3o=20v3.0.0rc4?= =?UTF-8?q?=20abaixo=20do=20logo=20da=20aplica=C3=A7=C3=A3o?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/templates/wagtailadmin/base.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/templates/wagtailadmin/base.html b/core/templates/wagtailadmin/base.html index a14a66d7..a5d451c3 100644 --- a/core/templates/wagtailadmin/base.html +++ b/core/templates/wagtailadmin/base.html @@ -12,5 +12,5 @@ {% else %} SciELO logo {% endif %} -
Upload v3.0.0rc3 -{% endblock %} \ No newline at end of file +
Upload v3.0.0rc4 +{% endblock %} From 5ecbdf278bed79efa60237c740a48d3954185fcf Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Tue, 19 Nov 2024 01:00:43 -0300 Subject: [PATCH 17/22] Atualiza production-*.yml com a imagem v3.0.0rc4 --- production-v3.0.0rc4.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/production-v3.0.0rc4.yml b/production-v3.0.0rc4.yml index 8f6af04d..d86c143b 100644 --- a/production-v3.0.0rc4.yml +++ b/production-v3.0.0rc4.yml @@ -2,7 +2,7 @@ version: '3' services: django: &django - image: infrascielo/upload + image: infrascielo/upload:v3.0.0rc4 container_name: upload_production_django platform: linux/x86_64 depends_on: From 2686c479bb752683211978b6cd41edf62ce96b3e Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Tue, 19 Nov 2024 16:18:17 -0300 Subject: [PATCH 18/22] =?UTF-8?q?Atualiza=20Article.update=5Fstatus=20cons?= =?UTF-8?q?iderando=20que=20o=20status=20do=20artigo=20pode=20voltar=20ao?= =?UTF-8?q?=20status=20anterior=20ou=20avan=C3=A7ar=20para=20o=20pr=C3=B3x?= =?UTF-8?q?imo=20status?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- article/models.py | 42 ++++++++++++++++++++---------------------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/article/models.py b/article/models.py index 333a311e..d89cf648 100644 --- a/article/models.py +++ b/article/models.py @@ -346,7 +346,7 @@ def multilingual_sections(self): def display_sections(self): return str(self.multilingual_sections) - def update_status(self, new_status=None): + def update_status(self, new_status=None, rollback=False): # AS_UPDATE_SUBMITTED = "update-submitted" # AS_ERRATUM_SUBMITTED = "erratum-submitted" # AS_REQUIRE_UPDATE = "required-update" @@ -358,28 +358,26 @@ def update_status(self, new_status=None): # TODO create PublicationEvent - if self.status in (choices.AS_UPDATE_SUBMITTED, choices.AS_READY_TO_PUBLISH, choices.AS_PUBLISHED): - self.status = choices.AS_REQUIRE_UPDATE - self.save() - return - - if self.status == choices.AS_ERRATUM_SUBMITTED: - self.status = choices.AS_REQUIRE_ERRATUM - self.save() - return - - if self.status == choices.AS_REQUIRE_UPDATE: - self.status = choices.AS_UPDATE_SUBMITTED - self.save() - return - - if self.status == choices.AS_REQUIRE_ERRATUM: - self.status = choices.AS_ERRATUM_SUBMITTED + if rollback: + if self.status == choices.AS_ERRATUM_SUBMITTED: + self.status = choices.AS_REQUIRE_ERRATUM + self.save() + return + elif self.status not in (choices.AS_REQUIRE_UPDATE, choices.AS_REQUIRE_ERRATUM): + self.status = choices.AS_REQUIRE_UPDATE + self.save() + return + else: + if self.status == choices.AS_REQUIRE_UPDATE: + self.status = choices.AS_UPDATE_SUBMITTED + self.save() + return + if self.status == choices.AS_REQUIRE_ERRATUM: + self.status = choices.AS_ERRATUM_SUBMITTED + self.save() + return + self.status = new_status or choices.AS_PUBLISHED self.save() - return - - self.status = new_status or choices.AS_PUBLISHED - self.save() def get_zip_filename_and_content(self): return self.sps_pkg.get_zip_filename_and_content() From 61ce33cdcbfd59551080f93b85a004d9725c98bd Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Tue, 19 Nov 2024 16:19:04 -0300 Subject: [PATCH 19/22] Ajusta a chamada ao Article.update_status --- upload/models.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/upload/models.py b/upload/models.py index d74ed2ce..c737e53b 100644 --- a/upload/models.py +++ b/upload/models.py @@ -432,7 +432,7 @@ def save(self, *args, **kwargs): if self.status == choices.PS_PENDING_CORRECTION and self.article: # volta article.status para o status antes da submissão do pacote - self.article.update_status() + self.article.update_status(rollback=True) super(Package, self).save(*args, **kwargs) @@ -1166,11 +1166,13 @@ def update_publication_stage(self, website_kind, completed): self.qa_ws_pubdate = datetime.utcnow() self.status = choices.PS_PREVIEW self.save() + self.article.update_status(new_status=article_choices.AS_PREPARE_TO_PUBLISH) elif website_kind == collection_choices.PUBLIC: self.public_ws_status = status self.public_ws_pubdate = datetime.utcnow() self.status = choices.PS_PUBLISHED self.save() + self.article.update_status(new_status=article_choices.PS_PUBLISHED) @property def toc_sections(self): From a679e2161be7c880cffba7898c7e7401642ec878 Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Tue, 19 Nov 2024 16:21:17 -0300 Subject: [PATCH 20/22] =?UTF-8?q?Adiciona=20ao=20log=20de=20eventos=20a=20?= =?UTF-8?q?cria=C3=A7=C3=A3o=20de=20journal=20e=20issue,=20caso=20eles=20s?= =?UTF-8?q?ejam=20criados=20no=20momento=20da=20publica=C3=A7=C3=A3o?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- publication/tasks.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/publication/tasks.py b/publication/tasks.py index 12232d62..1f1656c3 100644 --- a/publication/tasks.py +++ b/publication/tasks.py @@ -20,8 +20,6 @@ from publication.api.publication import PublicationAPI from tracker.models import UnexpectedEvent -# FIXME -# from upload.models import Package User = get_user_model() @@ -355,33 +353,34 @@ def task_publish_article( """ try: user = _get_user(user_id, username) - manager = None op_main = None - + manager = None if upload_package_id: manager = Package.objects.get(pk=upload_package_id) issue = manager.article.issue elif article_proc_id: manager = ArticleProc.objects.get(pk=article_proc_id) issue = manager.issue_proc.issue + op_main = manager.start(user, f"publish on {website_kind}") article = manager.article journal = article.journal issue = article.issue - op_main = manager.start(user, f"publish on {website_kind}") - if not JournalProc.objects.filter(journal=journal).exists(): - create_or_update_journal( + op_journal_proc = manager.start(user, f"publish on {website_kind} - create_or_update_journal") + created = create_or_update_journal( journal_title=journal.title, issn_electronic=journal.official_journal.issn_electronic, issn_print=journal.official_journal.issn_print, user=user, force_update=True, ) + op_journal_proc.finish(user, completed=bool(created)) if not IssueProc.objects.filter(issue=issue).exists(): - create_or_update_issue( + op_issue_proc = manager.start(user, f"publish on {website_kind} - create_or_update_issue") + created = create_or_update_issue( journal=journal, pub_year=issue.publication_year, volume=issue.volume, @@ -390,6 +389,7 @@ def task_publish_article( user=user, force_update=True, ) + op_issue_proc.finish(user, completed=bool(created)) for journal_proc in JournalProc.objects.filter(journal=journal): From d613b989193901876c0063ae7392f26e4da70c04 Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Wed, 20 Nov 2024 06:39:12 -0300 Subject: [PATCH 21/22] Corrige self duplicado: self.self --- proc/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proc/models.py b/proc/models.py index 8f0f3769..f7d6bf9b 100644 --- a/proc/models.py +++ b/proc/models.py @@ -660,7 +660,7 @@ def publish( self.migrated_data.content_type == "article" and (not self.sps_pkg or not self.sps_pkg.registered_in_core) ): - detail["registered_in_core"] = self.self.sps_pkg.registered_in_core + detail["registered_in_core"] = self.sps_pkg.registered_in_core doit = False else: doit = tracker_choices.allowed_to_run( From ef0c8863c234819f32965fc30cc9c89fb0762463 Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Wed, 20 Nov 2024 07:15:21 -0300 Subject: [PATCH 22/22] =?UTF-8?q?Corrige=20erro=20em=20m=C3=A9todo=20que?= =?UTF-8?q?=20cria=20partes=20do=20payload:=20related=5Farticles=20e=20aut?= =?UTF-8?q?ores?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- publication/api/document.py | 25 ++++++++++---- publication/utils/document.py | 61 +++++++++++++++++++++++------------ 2 files changed, 60 insertions(+), 26 deletions(-) diff --git a/publication/api/document.py b/publication/api/document.py index 8da45628..a8c6fbde 100644 --- a/publication/api/document.py +++ b/publication/api/document.py @@ -139,6 +139,11 @@ def add_author(self, surname, given_names, suffix, affiliation, orcid): # ) # self.data["authors"].append(_author) + def add_collab(self, name): + # collab + self.data.setdefault("collabs", []) + self.data["collabs"].append({"name": name}) + def add_translated_title(self, language, text): # translated_titles"] = EmbeddedDocumentListField(TranslatedTitle)) if self.data["translated_titles"] is None: @@ -186,16 +191,24 @@ def add_doi_with_lang(self, language, doi): _doi_with_lang_item["language"] = language self.data["doi_with_lang"].append(_doi_with_lang_item) - def add_related_article(self, doi, ref_id, related_type): + def add_related_article(self, ref_id, related_type, ext_link_type, href): # related_article"] = EmbeddedDocumentListField(RelatedArticle)) if self.data["related_articles"] is None: self.data["related_articles"] = [] _related_article = {} - _related_article["doi"] = doi - _related_article["ref_id"] = ref_id - _related_article["related_type"] = related_type - self.data["related_articles"].append(_related_article) - + if ext_link_type == "doi": + _related_article["doi"] = href + _related_article["ref_id"] = ref_id + _related_article["related_type"] = related_type + self.data["related_articles"].append(_related_article) + else: + pass + # TODO depende de resolver https://github.com/scieloorg/opac_5/issues/212 + # _related_article["href"] = href + # _related_article["ref_id"] = ref_id + # _related_article["related_type"] = related_type + # self.data["related_articles"].append(_related_article) + def add_xml(self, xml): self.data["xml"] = xml diff --git a/publication/utils/document.py b/publication/utils/document.py index d6ca1fe1..fe448863 100644 --- a/publication/utils/document.py +++ b/publication/utils/document.py @@ -3,7 +3,7 @@ from lxml import etree from packtools.sps.models.article_abstract import Abstract from packtools.sps.models.article_and_subarticles import ArticleAndSubArticles -from packtools.sps.models.article_authors import Authors +from packtools.sps.models.article_contribs import ArticleContribs from packtools.sps.models.article_doi_with_lang import DoiWithLang from packtools.sps.models.article_ids import ArticleIds from packtools.sps.models.article_renditions import ArticleRenditions @@ -75,9 +75,15 @@ def build_article(builder, article, journal_id, order, pub_date): builder.add_publication_date(**article_xml.get_publication_date(pub_date)) - for item in article_xml.get_authors(): + contribs = article_xml.get_contribs() + for item in contribs.get("names") or []: builder.add_author(**item) + # depende de https://github.com/scieloorg/opac_5/issues/214 + # TODO + # for item in contribs.get("collabs") or []: + # builder.add_collab(**item) + for item in article_xml.get_translated_title(): builder.add_translated_title(**item) @@ -111,30 +117,45 @@ def get_publication_date(self, pub_date): "day": str(pub_date.day).zfill(2), } - def get_authors(self): - for item in Authors(self.xmltree).contribs_with_affs: - try: - affiliation = ", ".join( - [a.get("original") or a.get("orgname") for a in item["affs"]] + def get_contribs(self): + authors = ArticleContribs(self.xmltree) + + names = [] + collabs = [] + for item in authors.contribs: + if item.get("parent") != "article": + break + + if contrib := item.get("contrib_name"): + try: + affs = item.get("affs") or [] + affiliation = ", ".join( + [a.get("original") or a.get("orgname") for a in affs] + ) + except KeyError: + affiliation = None + contrib_ids = item.get("contrib_ids") or {} + names.append( + dict( + surname=contrib.get("surname"), + given_names=contrib.get("given-names"), + suffix=contrib.get("suffix"), + affiliation=affiliation, + orcid=contrib_ids.get("orcid"), + ) ) - except KeyError: - affiliation = None - yield dict( - surname=item["surname"], - given_names=item["given_names"], - suffix=item.get("suffix"), - affiliation=affiliation, - orcid=item.get("orcid"), - ) + if contrib := item.get("collab"): + collabs.append({"name": collab}) + return {"names": names, "collabs": collabs} def get_related_articles(self): items = RelatedItems(self.xmltree) for item in items.related_articles: yield { - "ext-link-type": item["ext-link-type"], - "ref_id": item["id"], - "href": item["href"], - "related_type": item["related-article-type"], + "ext_link_type": item.get("ext-link-type"), + "ref_id": item.get("id"), + "href": item.get("href"), + "related_type": item.get("related-article-type"), } def get_translated_title(self):