Skip to content

Commit

Permalink
Merge pull request #567 from robertatakenaka/rc-main_2024-11-11
Browse files Browse the repository at this point in the history 
Atualiza o branch rc com main (2024-11-11)
  • Loading branch information
@robertatakenaka
robertatakenaka authored Nov 11, 2024
2 parents c3c6976 + 96b1edc commit ced41ea
Show file tree
Hide file tree
Showing 4 changed files with 130 additions and 89 deletions.
22 changes: 18 additions & 4 deletions core/utils/requester.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,10 +83,8 @@ def post_data(
try:
response.raise_for_status()
except requests.HTTPError as exc:
try:
return response.json()
except Exception as json_error:
pass
if response := is_http_error_json_response(json, response):
return response
if 400 <= exc.response.status_code < 500:
raise NonRetryableError(exc) from exc
elif 500 <= exc.response.status_code < 600:
Expand All @@ -100,6 +98,22 @@ def post_data(
return response.content if not json else response.json()


def is_http_error_json_response(json, response):
"""
Algumas API, por exemplo, opac_5, retornam a mensagem de erro em formato
JSON
"""
if not json:
return

try:
data = response.json()
if isinstance(data, dict):
return data
except Exception as json_error:
return


@retry(
retry=retry_if_exception_type(RetryableError),
wait=wait_exponential(multiplier=1, min=1, max=5),
Expand Down
123 changes: 67 additions & 56 deletions pid_provider/client.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,7 @@ def _get_token(self, username, password, timeout):
timeout=timeout,
json=True,
)
return resp.get("access")
return resp["access"]
except Exception as e:
# TODO tratar as exceções
logging.exception(e)
Expand All @@ -179,16 +179,19 @@ def _prepare_and_post_xml(self, xml_with_pre, name, token):
create_xml_zip_file(
zip_xml_file_path, xml_with_pre.tostring(pretty_print=True)
)

try:
return self._post_xml(zip_xml_file_path, self.token, self.timeout)
response = self._post_xml(zip_xml_file_path, self.token, self.timeout)
if isinstance(response, list):
return response
except Exception as e:
self.token = self._get_token(
username=self.api_username,
password=self.api_password,
timeout=self.timeout,
)
return self._post_xml(zip_xml_file_path, self.token, self.timeout)
logging.exception(e)

self.token = self._get_token(
username=self.api_username,
password=self.api_password,
timeout=self.timeout,
)
return self._post_xml(zip_xml_file_path, self.token, self.timeout)

def _post_xml(self, zip_xml_file_path, token, timeout):
"""
Expand All @@ -198,21 +201,21 @@ def _post_xml(self, zip_xml_file_path, token, timeout):
-H 'Authorization: Bearer eyJ0b2tlb' \
http://localhost:8000/api/v2/pid/pid_provider/ --output output.json
"""
basename = os.path.basename(zip_xml_file_path)

files = {
"file": (
basename,
open(zip_xml_file_path, "rb"),
"application/zip",
)
}
header = {
"Authorization": "Bearer " + token,
"content-type": "multi-part/form-data",
"Content-Disposition": "attachment; filename=%s" % basename,
}
try:
basename = os.path.basename(zip_xml_file_path)

files = {
"file": (
basename,
open(zip_xml_file_path, "rb"),
"application/zip",
)
}
header = {
"Authorization": "Bearer " + token,
"content-type": "multi-part/form-data",
"Content-Disposition": "attachment; filename=%s" % basename,
}
return post_data(
self.pid_provider_api_post_xml,
files=files,
Expand All @@ -238,42 +241,50 @@ def _process_post_xml_response(self, response, xml_with_pre, created=None):
logging.info(f"Pid Provider Response: none")
return
for item in response:
logging.info(f"Pid Provider Response: {item}")
try:
self._process_item_response(item, xml_with_pre, created)
except AttributeError:
raise ValueError(
f"Unexpected pid provider response: {response}"
)

if not item.get("xml_changed"):
# pids do xml_with_pre não mudaram
logging.info("No xml changes")
return
def _process_item_response(self, item, xml_with_pre, created=None):
logging.info(f"Pid Provider Response: {item}")

try:
# atualiza xml_with_pre com valor do XML registrado no Core
if not item.get("apply_xml_changes"):
# exceto 'apply_xml_changes=True' ou
# exceto se o registro do Core foi criado posteriormente
if created and created < item["created"]:
# não atualizar com os dados do Core
logging.info(
{
"created_at_upload": created,
"created_at_core": item["created"],
}
)
return
if not item.get("xml_changed"):
# pids do xml_with_pre não mudaram
logging.info("No xml changes")
return

for pid_type, pid_value in item["xml_changed"].items():
try:
if pid_type == "pid_v3":
xml_with_pre.v3 = pid_value
elif pid_type == "pid_v2":
xml_with_pre.v2 = pid_value
elif pid_type == "aop_pid":
xml_with_pre.aop_pid = pid_value
logging.info("XML changed")
except Exception as e:
pass
return
except KeyError:
pass
try:
# atualiza xml_with_pre com valor do XML registrado no Core
if not item.get("apply_xml_changes"):
# exceto 'apply_xml_changes=True' ou
# exceto se o registro do Core foi criado posteriormente
if created and created < item["created"]:
# não atualizar com os dados do Core
logging.info(
{
"created_at_upload": created,
"created_at_core": item["created"],
}
)
return

for pid_type, pid_value in item["xml_changed"].items():
try:
if pid_type == "pid_v3":
xml_with_pre.v3 = pid_value
elif pid_type == "pid_v2":
xml_with_pre.v2 = pid_value
elif pid_type == "aop_pid":
xml_with_pre.aop_pid = pid_value
logging.info("XML changed")
except Exception as e:
pass
return
except KeyError:
pass

def fix_pid_v2(self, pid_v3, correct_pid_v2):
"""
Expand Down
2 changes: 1 addition & 1 deletion proc/migrations/0007_remove_operation_event.py
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Generated by Django 5.0.3 on 2024-10-24 12:54
# Generated by Django 5.0.3 on 2024-11-09 14:19

from django.db import migrations

Expand Down
72 changes: 44 additions & 28 deletions proc/models.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -646,19 +646,37 @@ def publish(
except AttributeError:
raise Exception("*Proc.publish requires content_type parameter")

detail = {
"website_kind": website_kind,
"force_update": force_update,
}
doit = False
if website_kind == collection_choices.QA:
detail["qa_ws_status"] = self.qa_ws_status
doit = tracker_choices.allowed_to_run(self.qa_ws_status, force_update)
else:
doit = tracker_choices.allowed_to_run(self.public_ws_status, force_update)
detail["public_ws_status"] = self.public_ws_status
if (
self.migrated_data.content_type == "article" and
(not self.sps_pkg or not self.sps_pkg.registered_in_core)
):
detail["registered_in_core"] = self.self.sps_pkg.registered_in_core
doit = False
else:
doit = tracker_choices.allowed_to_run(
self.public_ws_status, force_update
)

detail["doit"] = doit
operation = self.start(
user, f"publish {content_type} {self} on {website_kind}"
)

if not doit:
# logging.info(f"Skip publish on {website_kind} {self.pid}")
operation.finish(user, completed=True, detail=detail)
return

operation = self.start(
user, f"publish {content_type} {self} on {website_kind}"
)
logging.info(f"publish {self} on {website_kind}")
if website_kind == collection_choices.QA:
self.qa_ws_status = tracker_choices.PROGRESS_STATUS_DOING
Expand All @@ -676,7 +694,8 @@ def publish(
response = callable_publish(self, api_data)
completed = bool(response.get("result") == "OK")
self.update_publication_stage(website_kind, completed)
operation.finish(user, completed=completed, detail=response)
detail.update(response)
operation.finish(user, completed=completed, detail=detail)
return completed

def update_publication_stage(self, website_kind, completed):
Expand Down Expand Up @@ -1192,7 +1211,13 @@ def migrate_document_records(self, user, force_update=None):
done += 1
except Exception as e:
exc_type, exc_value, exc_traceback = sys.exc_info()
errors.append({"pid": record.item_pid, "error_type": str(exc_type), "error_message": str(exc_value)})
errors.append(
{
"pid": record.item_pid,
"error_type": str(exc_type),
"error_message": str(exc_value),
}
)
UnexpectedEvent.create(
e=e,
exc_traceback=exc_traceback,
Expand All @@ -1207,12 +1232,14 @@ def migrate_document_records(self, user, force_update=None):
)
got = id_file_records.count()
detail = params
detail.update({
"total issue documents": self.issue.total_documents,
"total records": got,
"total done": done,
"errors": errors,
})
detail.update(
{
"total issue documents": self.issue.total_documents,
"total records": got,
"total done": done,
"errors": errors,
}
)
completed = got == done
operation.finish(
user,
Expand Down Expand Up @@ -1686,16 +1713,10 @@ def fix_pid_v2(self, user):
self.sps_pkg.fix_pid_v2(user, correct_pid_v2=self.migrated_data.pid)

def update_sps_pkg_status(self):
if not self.sps_pkg:
self.sps_pkg_status = tracker_choices.PROGRESS_STATUS_REPROC
elif self.sps_pkg.is_complete:
self.sps_pkg_status = tracker_choices.PROGRESS_STATUS_DONE
elif not self.sps_pkg.registered_in_core:
self.sps_pkg_status = tracker_choices.PROGRESS_STATUS_REPROC
elif not self.sps_pkg.valid_components:
if not self.sps_pkg or not self.sps_pkg.xml_with_pre:
self.sps_pkg_status = tracker_choices.PROGRESS_STATUS_REPROC
else:
self.sps_pkg_status = tracker_choices.PROGRESS_STATUS_PENDING
self.sps_pkg_status = tracker_choices.PROGRESS_STATUS_DONE
self.save()

@property
Expand All @@ -1716,18 +1737,13 @@ def migrate_article(self, user, force_update):
if not self.get_xml(user, body_and_back_xml):
return None

if not self.generate_sps_package(
self.generate_sps_package(
user,
body_and_back_xml,
html_to_xml,
):
return None
)

if self.sps_pkg:
article = self.create_or_update_item(
user, force_update, create_or_update_article
)
return article
return self.create_or_update_item(user, force_update, create_or_update_article)

def synchronize(self, user):
try:
Expand Down

0 comments on commit ced41ea

Please sign in to comment.