From 6e5b67568668e0a90b3f25ed30e9e157ead988a6 Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Sat, 9 Nov 2024 11:46:08 -0300 Subject: [PATCH 1/5] =?UTF-8?q?Cria=20core.utils.requester.is=5Fhttp=5Ferr?= =?UTF-8?q?or=5Fjson=5Fresponse=20para=20retornar=20HTTPError=20em=20forma?= =?UTF-8?q?to=20JSON,=20caso=20contr=C3=A1rio=20levanta=20exce=C3=A7=C3=A3?= =?UTF-8?q?o?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/utils/requester.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/core/utils/requester.py b/core/utils/requester.py index 0cda5cc1..a114e1c2 100644 --- a/core/utils/requester.py +++ b/core/utils/requester.py @@ -83,10 +83,8 @@ def post_data( try: response.raise_for_status() except requests.HTTPError as exc: - try: - return response.json() - except Exception as json_error: - pass + if response := is_http_error_json_response(json, response): + return response if 400 <= exc.response.status_code < 500: raise NonRetryableError(exc) from exc elif 500 <= exc.response.status_code < 600: @@ -100,6 +98,22 @@ def post_data( return response.content if not json else response.json() +def is_http_error_json_response(json, response): + """ + Algumas API, por exemplo, opac_5, retornam a mensagem de erro em formato + JSON + """ + if not json: + return + + try: + data = response.json() + if isinstance(data, dict): + return data + except Exception as json_error: + return + + @retry( retry=retry_if_exception_type(RetryableError), wait=wait_exponential(multiplier=1, min=1, max=5), From 925f152ac1f5560ad9dcd2cb5adb022c36fd34d0 Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Sat, 9 Nov 2024 11:46:08 -0300 Subject: [PATCH 2/5] Faz ajustes para registrar o defeito de post_data retornar str no lugar de dict --- pid_provider/client.py | 123 ++++++++++++++++++++++------------------- 1 file changed, 67 insertions(+), 56 deletions(-) diff --git a/pid_provider/client.py b/pid_provider/client.py index 90b8b4e6..ba0ff8ed 100644 --- a/pid_provider/client.py +++ b/pid_provider/client.py @@ -154,7 +154,7 @@ def _get_token(self, username, password, timeout): timeout=timeout, json=True, ) - return resp.get("access") + return resp["access"] except Exception as e: # TODO tratar as exceções logging.exception(e) @@ -179,16 +179,19 @@ def _prepare_and_post_xml(self, xml_with_pre, name, token): create_xml_zip_file( zip_xml_file_path, xml_with_pre.tostring(pretty_print=True) ) - try: - return self._post_xml(zip_xml_file_path, self.token, self.timeout) + response = self._post_xml(zip_xml_file_path, self.token, self.timeout) + if isinstance(response, list): + return response except Exception as e: - self.token = self._get_token( - username=self.api_username, - password=self.api_password, - timeout=self.timeout, - ) - return self._post_xml(zip_xml_file_path, self.token, self.timeout) + logging.exception(e) + + self.token = self._get_token( + username=self.api_username, + password=self.api_password, + timeout=self.timeout, + ) + return self._post_xml(zip_xml_file_path, self.token, self.timeout) def _post_xml(self, zip_xml_file_path, token, timeout): """ @@ -198,21 +201,21 @@ def _post_xml(self, zip_xml_file_path, token, timeout): -H 'Authorization: Bearer eyJ0b2tlb' \ http://localhost:8000/api/v2/pid/pid_provider/ --output output.json """ - basename = os.path.basename(zip_xml_file_path) - - files = { - "file": ( - basename, - open(zip_xml_file_path, "rb"), - "application/zip", - ) - } - header = { - "Authorization": "Bearer " + token, - "content-type": "multi-part/form-data", - "Content-Disposition": "attachment; filename=%s" % basename, - } try: + basename = os.path.basename(zip_xml_file_path) + + files = { + "file": ( + basename, + open(zip_xml_file_path, "rb"), + "application/zip", + ) + } + header = { + "Authorization": "Bearer " + token, + "content-type": "multi-part/form-data", + "Content-Disposition": "attachment; filename=%s" % basename, + } return post_data( self.pid_provider_api_post_xml, files=files, @@ -238,42 +241,50 @@ def _process_post_xml_response(self, response, xml_with_pre, created=None): logging.info(f"Pid Provider Response: none") return for item in response: - logging.info(f"Pid Provider Response: {item}") + try: + self._process_item_response(item, xml_with_pre, created) + except AttributeError: + raise ValueError( + f"Unexpected pid provider response: {response}" + ) - if not item.get("xml_changed"): - # pids do xml_with_pre não mudaram - logging.info("No xml changes") - return + def _process_item_response(self, item, xml_with_pre, created=None): + logging.info(f"Pid Provider Response: {item}") - try: - # atualiza xml_with_pre com valor do XML registrado no Core - if not item.get("apply_xml_changes"): - # exceto 'apply_xml_changes=True' ou - # exceto se o registro do Core foi criado posteriormente - if created and created < item["created"]: - # não atualizar com os dados do Core - logging.info( - { - "created_at_upload": created, - "created_at_core": item["created"], - } - ) - return + if not item.get("xml_changed"): + # pids do xml_with_pre não mudaram + logging.info("No xml changes") + return - for pid_type, pid_value in item["xml_changed"].items(): - try: - if pid_type == "pid_v3": - xml_with_pre.v3 = pid_value - elif pid_type == "pid_v2": - xml_with_pre.v2 = pid_value - elif pid_type == "aop_pid": - xml_with_pre.aop_pid = pid_value - logging.info("XML changed") - except Exception as e: - pass - return - except KeyError: - pass + try: + # atualiza xml_with_pre com valor do XML registrado no Core + if not item.get("apply_xml_changes"): + # exceto 'apply_xml_changes=True' ou + # exceto se o registro do Core foi criado posteriormente + if created and created < item["created"]: + # não atualizar com os dados do Core + logging.info( + { + "created_at_upload": created, + "created_at_core": item["created"], + } + ) + return + + for pid_type, pid_value in item["xml_changed"].items(): + try: + if pid_type == "pid_v3": + xml_with_pre.v3 = pid_value + elif pid_type == "pid_v2": + xml_with_pre.v2 = pid_value + elif pid_type == "aop_pid": + xml_with_pre.aop_pid = pid_value + logging.info("XML changed") + except Exception as e: + pass + return + except KeyError: + pass def fix_pid_v2(self, pid_v3, correct_pid_v2): """ From 43ef9530126381689fc705efa4637181fe3a6ac7 Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Sat, 9 Nov 2024 11:46:09 -0300 Subject: [PATCH 3/5] =?UTF-8?q?Corre=C3=A7=C3=B5es=20e=20melhorias=20em=20?= =?UTF-8?q?proc.models=20-=20Melhora=20o=20registro=20de=20eventos=20assoc?= =?UTF-8?q?iados=20com=20*Proc;=20-=20Modifica=20ArticleProc.update=5Fsps?= =?UTF-8?q?=5Fpkg=5Fstatus=20para=20que=20mude=20o=20status=20para=20DONE?= =?UTF-8?q?=20mesmo=20que=20sps=5Fpkg.is=5Fcomplete=20=3D=3D=20False=20-?= =?UTF-8?q?=20Modifica=20ArticleProc.migrate=5Farticle=20para=20que=20n?= =?UTF-8?q?=C3=A3o=20se=20interrompa=20porque=20sps=5Fpkg.is=5Fcomplete=20?= =?UTF-8?q?=3D=3D=20False=20-=20Modifica=20BaseProc.publish=20para=20adici?= =?UTF-8?q?onar=20mais=20detalhes=20no=20registro=20do=20evento=20de=20pub?= =?UTF-8?q?licar=20e=20restringe=20a=20publica=C3=A7=C3=A3o=20de=20artigos?= =?UTF-8?q?=20em=20PUBLIC=20se=20n=C3=A3o=20estiver=20registrado=20no=20Co?= =?UTF-8?q?re?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- proc/models.py | 114 +++++++++++++++++++++++++------------------------ 1 file changed, 58 insertions(+), 56 deletions(-) diff --git a/proc/models.py b/proc/models.py index 5386c209..df74485a 100644 --- a/proc/models.py +++ b/proc/models.py @@ -63,7 +63,6 @@ class Operation(CommonControlField): blank=True, ) completed = models.BooleanField(null=True, blank=True, default=False) - event = models.ForeignKey(Event, on_delete=models.SET_NULL, null=True, blank=True) detail = models.JSONField(null=True, blank=True) base_form_class = ProcAdminModelForm @@ -91,7 +90,6 @@ def data(self): return dict( name=self.name, completed=self.completed, - event=self.event and self.event.data, detail=self.detail, created=self.created.isoformat(), ) @@ -153,32 +151,21 @@ def finish( exc_traceback=None, detail=None, ): - if not message_type: - if not completed: - message_type = "ERROR" + detail = detail or {} + if exception: + detail["exception_message"] = str(exception) + detail["exception_type"] = str(type(exception)) + if exc_traceback: + detail["traceback"] = str(format_traceback(exc_traceback)) + if message_type: + detail["message_type"] = message_type + if message: + detail["message"] = message - if detail: - try: - json.dumps(detail) - except Exception as exc_detail: - detail = str(detail) - - if message_type or exception or exc_traceback: - event = Event.create( - user=user, - message_type=message_type, - message=message, - e=exception, - exc_traceback=exc_traceback, - detail=detail, - ) - detail = event.data - - if detail: - try: - json.dumps(detail) - except Exception as exc_detail: - detail = str(detail) + try: + json.dumps(detail) + except Exception as exc_detail: + detail = str(detail) self.detail = detail self.completed = completed @@ -646,20 +633,37 @@ def publish( force_update=None, ): website_kind = website_kind or collection_choices.QA - + detail = { + "website_kind": website_kind, + "force_update": force_update, + } doit = False if website_kind == collection_choices.QA: + detail["qa_ws_status"] = self.qa_ws_status doit = tracker_choices.allowed_to_run(self.qa_ws_status, force_update) else: - doit = tracker_choices.allowed_to_run(self.public_ws_status, force_update) + detail["public_ws_status"] = self.public_ws_status + if ( + self.migrated_data.content_type == "article" and + (not self.sps_pkg or not self.sps_pkg.registered_in_core) + ): + detail["registered_in_core"] = self.self.sps_pkg.registered_in_core + doit = False + else: + doit = tracker_choices.allowed_to_run( + self.public_ws_status, force_update + ) + + detail["doit"] = doit + operation = self.start( + user, f"publish {self.migrated_data.content_type} {self} on {website_kind}" + ) if not doit: # logging.info(f"Skip publish on {website_kind} {self.pid}") + operation.finish(user, completed=True, detail=detail) return - operation = self.start( - user, f"publish {self.migrated_data.content_type} {self} on {website_kind}" - ) logging.info(f"publish {self} on {website_kind}") if website_kind == collection_choices.QA: self.qa_ws_status = tracker_choices.PROGRESS_STATUS_DOING @@ -677,7 +681,8 @@ def publish( response = callable_publish(self, api_data) completed = bool(response.get("result") == "OK") self.update_publication_stage(website_kind, completed) - operation.finish(user, completed=completed, detail=response) + detail.update(response) + operation.finish(user, completed=completed, detail=detail) return completed def update_publication_stage(self, website_kind, completed): @@ -1186,7 +1191,13 @@ def migrate_document_records(self, user, force_update=None): done += 1 except Exception as e: exc_type, exc_value, exc_traceback = sys.exc_info() - errors.append({"pid": record.item_pid, "error_type": str(exc_type), "error_message": str(exc_value)}) + errors.append( + { + "pid": record.item_pid, + "error_type": str(exc_type), + "error_message": str(exc_value), + } + ) UnexpectedEvent.create( e=e, exc_traceback=exc_traceback, @@ -1201,12 +1212,14 @@ def migrate_document_records(self, user, force_update=None): ) got = id_file_records.count() detail = params - detail.update({ - "total issue documents": self.issue.total_documents, - "total records": got, - "total done": done, - "errors": errors, - }) + detail.update( + { + "total issue documents": self.issue.total_documents, + "total records": got, + "total done": done, + "errors": errors, + } + ) completed = got == done operation.finish( user, @@ -1680,16 +1693,10 @@ def fix_pid_v2(self, user): self.sps_pkg.fix_pid_v2(user, correct_pid_v2=self.migrated_data.pid) def update_sps_pkg_status(self): - if not self.sps_pkg: - self.sps_pkg_status = tracker_choices.PROGRESS_STATUS_REPROC - elif self.sps_pkg.is_complete: - self.sps_pkg_status = tracker_choices.PROGRESS_STATUS_DONE - elif not self.sps_pkg.registered_in_core: - self.sps_pkg_status = tracker_choices.PROGRESS_STATUS_REPROC - elif not self.sps_pkg.valid_components: + if not self.sps_pkg or not self.sps_pkg.xml_with_pre: self.sps_pkg_status = tracker_choices.PROGRESS_STATUS_REPROC else: - self.sps_pkg_status = tracker_choices.PROGRESS_STATUS_PENDING + self.sps_pkg_status = tracker_choices.PROGRESS_STATUS_DONE self.save() @property @@ -1710,18 +1717,13 @@ def migrate_article(self, user, force_update): if not self.get_xml(user, body_and_back_xml): return None - if not self.generate_sps_package( + self.generate_sps_package( user, body_and_back_xml, html_to_xml, - ): - return None + ) - if self.sps_pkg: - article = self.create_or_update_item( - user, force_update, create_or_update_article - ) - return article + return self.create_or_update_item(user, force_update, create_or_update_article) def synchronize(self, user): try: From ade79341a218984baf3ff46a851f92b3fd964271 Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Sat, 9 Nov 2024 11:48:26 -0300 Subject: [PATCH 4/5] Adiciona migracao de banco de dados de proc.models --- proc/migrations/0007_remove_operation_event.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 proc/migrations/0007_remove_operation_event.py diff --git a/proc/migrations/0007_remove_operation_event.py b/proc/migrations/0007_remove_operation_event.py new file mode 100644 index 00000000..969c35e9 --- /dev/null +++ b/proc/migrations/0007_remove_operation_event.py @@ -0,0 +1,16 @@ +# Generated by Django 5.0.3 on 2024-11-09 14:19 + +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("proc", "0006_issueproc_resumption_date"), + ] + + operations = [ + migrations.RemoveField( + model_name="operation", + name="event", + ), + ] From cd623dbb28c075874590599cd007a0a4e711a07d Mon Sep 17 00:00:00 2001 From: Roberta Takenaka Date: Mon, 11 Nov 2024 07:19:05 -0300 Subject: [PATCH 5/5] =?UTF-8?q?Atualiza=20a=20apresenta=C3=A7=C3=A3o=20da?= =?UTF-8?q?=20vers=C3=A3o=20v2.6.12=20na=20=C3=A1rea=20administrativa?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/templates/wagtailadmin/base.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/templates/wagtailadmin/base.html b/core/templates/wagtailadmin/base.html index 7e2d5a13..e7da00d8 100644 --- a/core/templates/wagtailadmin/base.html +++ b/core/templates/wagtailadmin/base.html @@ -12,5 +12,5 @@ {% else %} SciELO logo {% endif %} -
Upload v2.6.9 +
Upload v2.6.12 {% endblock %} \ No newline at end of file