From 04b174fff2eed04a7ba54e5694235cda832f5660 Mon Sep 17 00:00:00 2001 From: Eliton Jorge Date: Fri, 12 Apr 2024 15:34:10 -0300 Subject: [PATCH 01/15] Creates unit tests for WebhookEventProcessor (#472) --- marketplace/wpp_templates/tests/test_utils.py | 352 ++++++++++++++++++ 1 file changed, 352 insertions(+) create mode 100644 marketplace/wpp_templates/tests/test_utils.py diff --git a/marketplace/wpp_templates/tests/test_utils.py b/marketplace/wpp_templates/tests/test_utils.py new file mode 100644 index 00000000..38193663 --- /dev/null +++ b/marketplace/wpp_templates/tests/test_utils.py @@ -0,0 +1,352 @@ +from unittest.mock import patch, MagicMock +from datetime import datetime + +from django.test import TestCase +from django.db.models.query import QuerySet + +from marketplace.wpp_templates.utils import WebhookEventProcessor +from marketplace.wpp_templates.utils import extract_template_data +from marketplace.wpp_templates.utils import handle_error_and_update_config +from marketplace.applications.models import App + + +class TestWebhookEventProcessor(TestCase): + @patch("marketplace.applications.models.App.objects.filter") + def test_process_template_status_update_no_apps(self, mock_filter): + mock_query_set = MagicMock(spec=QuerySet) + mock_query_set.exists.return_value = False + mock_filter.return_value = mock_query_set + + WebhookEventProcessor.process_template_status_update("123", {}, {}) + mock_filter.assert_called_once_with(config__wa_waba_id="123") + + @patch( + "marketplace.services.flows.service.FlowsService.update_facebook_templates_webhook" + ) + @patch("marketplace.wpp_templates.utils.extract_template_data") + @patch("marketplace.wpp_templates.models.TemplateMessage.objects.filter") + @patch("marketplace.applications.models.App.objects.filter") + def test_process_template_status_update_with_apps( + self, mock_filter, mock_template_filter, mock_extract_data, mock_flows_service + ): + mock_query_set = MagicMock(spec=QuerySet) + mock_query_set.exists.return_value = True + mock_app = MagicMock() + mock_app.flow_object_uuid = "uuid123" + mock_query_set.__iter__.return_value = iter([mock_app]) + mock_filter.return_value = mock_query_set + + mock_template = MagicMock() + mock_translation = MagicMock() + mock_translation.status = "pending" + mock_template.translations.filter.return_value = [mock_translation] + mock_template_filter.return_value.first.return_value = mock_template + + mock_extract_data.return_value = {"data": "value"} + + WebhookEventProcessor.process_template_status_update( + "123", + {"event": "approved", "message_template_name": "template1"}, + "webhook_data", + ) + self.assertEqual(mock_translation.status, "approved") + mock_flows_service.assert_called_once() + + @patch("marketplace.wpp_templates.utils.logger") + @patch("marketplace.wpp_templates.utils.WebhookEventProcessor.get_apps_by_waba_id") + def test_status_already_updated(self, mock_get_apps_by_waba_id, mock_logger): + mock_query_set = MagicMock() + mock_query_set.exists.return_value = True + app = MagicMock() + app.uuid = "123" + mock_query_set.__iter__.return_value = iter([app]) + mock_get_apps_by_waba_id.return_value = mock_query_set + + translation = MagicMock() + translation.status = "updated" + mock_translation_query = MagicMock() + mock_translation_query.filter.return_value = [translation] + + # Mock TemplateMessage.objects.filter + with patch( + "marketplace.wpp_templates.models.TemplateMessage.objects.filter", + return_value=MagicMock( + first=MagicMock( + return_value=MagicMock(translations=mock_translation_query) + ) + ), + ): + value = { + "event": "updated", + "message_template_name": "test", + "message_template_language": "pt_BR", + "message_template_id": "123456789", + } + WebhookEventProcessor.process_template_status_update( + "123456789", + value, + {"webhook": "webhook_info"}, + ) + mock_logger.info.assert_called_with( + "The template status: updated is already updated for this App: 123" + ) + + @patch("marketplace.wpp_templates.utils.logger") + @patch("marketplace.wpp_templates.utils.WebhookEventProcessor.get_apps_by_waba_id") + def test_unexpected_error_during_processing( + self, mock_get_apps_by_waba_id, mock_logger + ): + mock_app = MagicMock() + mock_app.uuid = "123" + mock_query_set = MagicMock(spec=QuerySet) + mock_query_set.exists.return_value = True + mock_query_set.__iter__.return_value = iter([mock_app]) + mock_get_apps_by_waba_id.return_value = mock_query_set + + with patch( + "marketplace.wpp_templates.models.TemplateMessage.objects.filter", + side_effect=Exception("Test error"), + ): + value = { + "event": "APPROVE", + "message_template_name": "test", + "message_template_language": "pt_BR", + "message_template_id": "123456789", + } + WebhookEventProcessor.process_template_status_update( + "123456789", value, {"webhook": "webhook_info"} + ) + + mock_logger.error.assert_called_with( + f"Unexpected error processing template status update by webhook for App {mock_app.uuid}: Test error" + ) + + @patch( + "marketplace.wpp_templates.utils.WebhookEventProcessor.process_template_status_update" + ) + def test_process_event_calls_correct_method( + self, mock_process_template_status_update + ): + WebhookEventProcessor.process_event( + "waba_id", {"some": "data"}, "message_template_status_update", "webhook" + ) + mock_process_template_status_update.assert_called_once_with( + "waba_id", {"some": "data"}, "webhook" + ) + WebhookEventProcessor.process_event( + "waba_id", {"some": "data"}, "template_category_update", "webhook" + ) + WebhookEventProcessor.process_event( + "waba_id", {"some": "data"}, "message_template_quality_update", "webhook" + ) + mock_process_template_status_update.assert_called_once() + + @patch("marketplace.wpp_templates.utils.logger") + @patch( + "marketplace.wpp_templates.utils.FlowsService.update_facebook_templates_webhook" + ) + @patch("marketplace.wpp_templates.utils.WebhookEventProcessor.get_apps_by_waba_id") + def test_error_sending_template_update( + self, + mock_get_apps_by_waba_id, + mock_update_facebook_templates_webhook, + mock_logger, + ): + mock_query_set = MagicMock() + mock_query_set.exists.return_value = True + app = MagicMock() + app.uuid = "123" + app.flow_object_uuid = "flow_uuid" + mock_query_set.__iter__.return_value = iter([app]) + mock_get_apps_by_waba_id.return_value = mock_query_set + + translation = MagicMock() + translation.status = "pending" + translation.language = "pt_BR" + translation.message_template_id = "123456789" + + template = MagicMock() + template.name = "test_template" + translation.template = template + + mock_translation_queryset = MagicMock() + mock_translation_queryset.__iter__.return_value = iter([translation]) + template.translations.filter.return_value = mock_translation_queryset + + with patch( + "marketplace.wpp_templates.models.TemplateMessage.objects.filter", + return_value=MagicMock(first=lambda: template), + ): + mock_update_facebook_templates_webhook.side_effect = Exception( + "Test flow service failure" + ) + + value = { + "event": "APPROVE", + "message_template_name": "test_template", + "message_template_language": "pt_BR", + "message_template_id": "123456789", + } + WebhookEventProcessor.process_template_status_update( + "123456789", value, {"webhook": "webhook_info"} + ) + expected_message = ( + "Fail to sends template update: test_template, translation: pt_BR," + "translation ID: 123456789. Error: Test flow service failure" + ) + mock_logger.error.assert_called_with(expected_message) + + +class TestExtractTemplateData(TestCase): + def setUp(self): + self.translation = MagicMock() + self.translation.template.name = "Template Name" + self.translation.language = "en" + self.translation.status = "active" + self.translation.category = "generic" + self.translation.message_template_id = 123 + + def test_extract_template_data_all_components(self): + # Mock Headers + header_mock = MagicMock() + header_mock.header_type = "IMAGE" + header_mock.example = "['example1', 'example2']" + header_mock.text = "Header Text" + self.translation.headers.all.return_value = [header_mock] + + # Mock Body + self.translation.body = "Sample body text" + + # Mock Footer + self.translation.footer = "Sample footer text" + + # Mock Buttons + button_mock = MagicMock() + button_mock.button_type = "URL" + button_mock.text = "Visit" + button_mock.url = "http://example.com" + button_mock.phone_number = None + self.translation.buttons.all.return_value = [button_mock] + + result = extract_template_data(self.translation) + + self.assertIn( + "HEADER", [component["type"] for component in result["components"]] + ) + self.assertIn("BODY", [component["type"] for component in result["components"]]) + self.assertIn( + "FOOTER", [component["type"] for component in result["components"]] + ) + self.assertIn( + "BUTTONS", [component["type"] for component in result["components"]] + ) + self.assertEqual(result["name"], "Template Name") + self.assertEqual(result["language"], "en") + + def test_extract_template_data_with_invalid_header_example(self): + header_mock = MagicMock() + header_mock.header_type = "IMAGE" + header_mock.example = "not a list" + header_mock.text = "Header Text" + self.translation.headers.all.return_value = [header_mock] + + result = extract_template_data(self.translation) + + self.assertIn("not a list", result["components"][0]["example"]["header_handle"]) + + def test_header_with_text_type(self): + header_mock = MagicMock() + header_mock.header_type = "TEXT" + header_mock.text = "Header Text" + header_mock.example = None + self.translation.headers.all.return_value = [header_mock] + + result = extract_template_data(self.translation) + self.assertEqual(result["components"][0]["text"], "Header Text") + + def test_header_with_example(self): + header_mock = MagicMock() + header_mock.header_type = "TEXT" + header_mock.text = "Header Text" + header_mock.example = "Example Text" + self.translation.headers.all.return_value = [header_mock] + + result = extract_template_data(self.translation) + self.assertIn("Example Text", result["components"][0]["example"]["header_text"]) + + def test_button_with_phone_number(self): + button_mock = MagicMock() + button_mock.button_type = "CALL" + button_mock.text = "Call Us" + button_mock.url = None + button_mock.phone_number = "1234567890" + button_mock.country_code = "1" + self.translation.buttons.all.return_value = [button_mock] + + result = extract_template_data(self.translation) + + buttons_index = next( + i + for i, comp in enumerate(result["components"]) + if comp["type"] == "BUTTONS" + ) + self.assertEqual( + result["components"][buttons_index]["buttons"][0]["phone_number"], + "+1 1234567890", + ) + + def test_header_example_not_list(self): + header_mock = MagicMock() + header_mock.header_type = "IMAGE" + header_mock.example = "'single element'" + self.translation.headers.all.return_value = [header_mock] + + result = extract_template_data(self.translation) + + self.assertIn( + "'single element'", result["components"][0]["example"]["header_handle"] + ) + + +class TestHandleErrorAndUpdateConfig(TestCase): + @patch("marketplace.wpp_templates.utils.logger") + def setUp(self, mock_logger): + self.app = MagicMock(spec=App) + self.app.config = {} + self.app.uuid = "123-abc" + + self.error_data = { + "code": 100, + "error_subcode": 33, + "message": "Error occurred", + } + + @patch("marketplace.applications.models.App.save") + @patch("marketplace.wpp_templates.utils.datetime") + def test_handle_error_and_update_config_correct_condition( + self, mock_datetime, mock_save + ): + mock_datetime.now.return_value = datetime(2024, 1, 1) + iso_date = mock_datetime.now.return_value.isoformat() + + handle_error_and_update_config(self.app, self.error_data) + + expected_config = { + "ignores_meta_sync": { + "last_error_date": iso_date, + "last_error_message": "Error occurred", + "code": 100, + "error_subcode": 33, + } + } + + self.assertEqual(self.app.config, expected_config) + + @patch("marketplace.applications.models.App.save") + def test_handle_error_and_update_config_incorrect_condition(self, mock_save): + self.error_data["code"] = 101 + + handle_error_and_update_config(self.app, self.error_data) + + self.assertEqual(self.app.config, {}) + mock_save.assert_not_called() From e67488d4c703adf4397229ac5d775a1833c433f8 Mon Sep 17 00:00:00 2001 From: Eliton Jorge Date: Thu, 18 Apr 2024 18:00:37 -0300 Subject: [PATCH 02/15] Check if there is a feed upload in progress before uploading a new (#473) * Check if there is a feed upload in progress before uploading a new --- marketplace/clients/facebook/client.py | 18 ++++++++++++++++++ marketplace/services/facebook/service.py | 3 +++ marketplace/services/vtex/generic_service.py | 18 +++++++++++++++++- 3 files changed, 38 insertions(+), 1 deletion(-) diff --git a/marketplace/clients/facebook/client.py b/marketplace/clients/facebook/client.py index e52fb250..e6dbd178 100644 --- a/marketplace/clients/facebook/client.py +++ b/marketplace/clients/facebook/client.py @@ -267,3 +267,21 @@ def get_upload_status_by_feed(self, feed_id, upload_id): return "end_time" in upload return False + + def get_uploads_in_progress_by_feed(self, feed_id): + url = self.get_url + f"{feed_id}/uploads" + + headers = self._get_headers() + response = self.make_request(url, method="GET", headers=headers) + data = response.json() + upload = next( + ( + upload + for upload in data.get("data", []) + if upload.get("end_time") is None + ), + None, + ) + if upload: + if "end_time" not in upload: + return upload.get("id") diff --git a/marketplace/services/facebook/service.py b/marketplace/services/facebook/service.py index a809d078..27c946b8 100644 --- a/marketplace/services/facebook/service.py +++ b/marketplace/services/facebook/service.py @@ -86,6 +86,9 @@ def upload_product_feed( def get_upload_status_by_feed(self, feed_id, upload_id) -> bool: return self.client.get_upload_status_by_feed(feed_id, upload_id) + def get_in_process_uploads_by_feed(self, feed_id) -> str: + return self.client.get_uploads_in_progress_by_feed(feed_id) + # ================================ # Private Methods # ================================ diff --git a/marketplace/services/vtex/generic_service.py b/marketplace/services/vtex/generic_service.py index 26fd10f0..80873647 100644 --- a/marketplace/services/vtex/generic_service.py +++ b/marketplace/services/vtex/generic_service.py @@ -226,6 +226,12 @@ def webhook_product_insert(self): return products_dto def _webhook_update_products_on_facebook(self, products_csv) -> bool: + upload_id_in_process = self._uploads_in_progress() + + if upload_id_in_process: + print("There is already a feed upload in progress, waiting for completion.") + self._wait_for_upload_completion(upload_id_in_process) + current_time = datetime.now().strftime("%Y-%m-%d_%H-%M") file_name = f"update_{current_time}_{self.product_feed.name}" upload_id = self._webhook_upload_product_feed( @@ -254,6 +260,7 @@ def _wait_for_upload_completion(self, upload_id) -> bool: wait_time = 5 max_wait_time = 20 * 60 total_wait_time = 0 + attempt = 1 while total_wait_time < max_wait_time: upload_complete = self.fba_service.get_upload_status_by_feed( @@ -263,11 +270,20 @@ def _wait_for_upload_completion(self, upload_id) -> bool: return True print( - f"Waiting {wait_time} seconds to get feed: {self.feed_id} upload {upload_id} status." + f"Attempt {attempt}: Waiting {wait_time} seconds " + f"to get feed: {self.feed_id} upload {upload_id} status." ) time.sleep(wait_time) total_wait_time += wait_time wait_time = min(wait_time * 2, 160) + attempt += 1 + + return False + + def _uploads_in_progress(self): + upload_id = self.fba_service.get_in_process_uploads_by_feed(self.feed_id) + if upload_id: + return upload_id return False From 5752039ee2ea24b0ad23a12c75a9d7fa16137cc1 Mon Sep 17 00:00:00 2001 From: Lucas Date: Thu, 18 Apr 2024 18:31:25 -0300 Subject: [PATCH 03/15] Permission update consumer (#471) --- marketplace/accounts/consumers/__init__.py | 1 + .../accounts/consumers/update_permissions.py | 27 ++++++++ marketplace/accounts/handle.py | 9 +++ marketplace/accounts/usecases/__init__.py | 1 + .../accounts/usecases/permission_update.py | 69 +++++++++++++++++++ marketplace/event_driven/handle.py | 2 + 6 files changed, 109 insertions(+) create mode 100644 marketplace/accounts/consumers/__init__.py create mode 100644 marketplace/accounts/consumers/update_permissions.py create mode 100644 marketplace/accounts/handle.py create mode 100644 marketplace/accounts/usecases/__init__.py create mode 100644 marketplace/accounts/usecases/permission_update.py diff --git a/marketplace/accounts/consumers/__init__.py b/marketplace/accounts/consumers/__init__.py new file mode 100644 index 00000000..da950859 --- /dev/null +++ b/marketplace/accounts/consumers/__init__.py @@ -0,0 +1 @@ +from .update_permissions import UpdatePermissionConsumer diff --git a/marketplace/accounts/consumers/update_permissions.py b/marketplace/accounts/consumers/update_permissions.py new file mode 100644 index 00000000..26e1e2c3 --- /dev/null +++ b/marketplace/accounts/consumers/update_permissions.py @@ -0,0 +1,27 @@ +import amqp +from sentry_sdk import capture_exception + +from ..usecases import update_permission +from marketplace.event_driven.parsers import JSONParser +from marketplace.event_driven.consumers import EDAConsumer + + +class UpdatePermissionConsumer(EDAConsumer): + def consume(self, message: amqp.Message): # pragma: no cover + print(f"[UpdatePermission] - Consuming a message. Body: {message.body}") + try: + body = JSONParser.parse(message.body) + + update_permission( + project_uuid=body.get("project"), + action=body.get("action"), + user_email=body.get("user"), + role=body.get("role"), + ) + + message.channel.basic_ack(message.delivery_tag) + + except Exception as exception: + capture_exception(exception) + message.channel.basic_reject(message.delivery_tag, requeue=False) + print(f"[UpdatePermission] - Message rejected by: {exception}") diff --git a/marketplace/accounts/handle.py b/marketplace/accounts/handle.py new file mode 100644 index 00000000..f3770b47 --- /dev/null +++ b/marketplace/accounts/handle.py @@ -0,0 +1,9 @@ +from amqp.channel import Channel + +from .consumers import UpdatePermissionConsumer + + +def handle_consumers(channel: Channel) -> None: + channel.basic_consume( + "integrations.update-permission", callback=UpdatePermissionConsumer().handle + ) diff --git a/marketplace/accounts/usecases/__init__.py b/marketplace/accounts/usecases/__init__.py new file mode 100644 index 00000000..992aa77f --- /dev/null +++ b/marketplace/accounts/usecases/__init__.py @@ -0,0 +1 @@ +from .permission_update import update_permission diff --git a/marketplace/accounts/usecases/permission_update.py b/marketplace/accounts/usecases/permission_update.py new file mode 100644 index 00000000..761aa2cf --- /dev/null +++ b/marketplace/accounts/usecases/permission_update.py @@ -0,0 +1,69 @@ +from marketplace.accounts.models import ProjectAuthorization + + +from django.contrib.auth import get_user_model + +from marketplace.projects.models import Project + +User = get_user_model() + + +def get_or_create_user_by_email(email: str) -> tuple: # pragma: no cover + return User.objects.get_or_create(email=email) + + +def set_user_project_authorization_role( + user: User, project: Project, role: int +): # pragma: no cover + + project_authorization, created = ProjectAuthorization.objects.get_or_create( + user=user, project_uuid=project.uuid + ) + + project_authorization.role = role + project_authorization.save(update_fields=["role"]) + + +def update_user_permission(role: int, project: Project, user: User): # pragma: no cover + if role == 1: + set_user_project_authorization_role( + user=user, project=project, role=ProjectAuthorization.ROLE_VIEWER + ) + + elif role == 2 or role == 5: + set_user_project_authorization_role( + user=user, project=project, role=ProjectAuthorization.ROLE_CONTRIBUTOR + ) + + elif role == 3 or role == 4: + set_user_project_authorization_role( + user=user, project=project, role=ProjectAuthorization.ROLE_ADMIN + ) + + else: + set_user_project_authorization_role( + user=user, project=project, role=ProjectAuthorization.ROLE_NOT_SETTED + ) + + +def delete_permisison(role, project, user): # pragma: no cover + project_authorization = ProjectAuthorization.objects.get( + user=user, project_uuid=project.uuid, role=role + ) + + project_authorization.delete() + + +def update_permission( + project_uuid: Project, action: str, user_email: str, role: int +) -> Project: # pragma: no cover + project = Project.objects.get(uuid=project_uuid) + user, _ = get_or_create_user_by_email(user_email) + + if action == "create" or action == "update": + update_user_permission(role, project, user) + + if action == "delete": + delete_permisison(role, project, user) + + return project diff --git a/marketplace/event_driven/handle.py b/marketplace/event_driven/handle.py index 8ba70da5..bf725237 100644 --- a/marketplace/event_driven/handle.py +++ b/marketplace/event_driven/handle.py @@ -1,7 +1,9 @@ from amqp.channel import Channel from marketplace.projects.handle import handle_consumers as project_handle_consumers +from marketplace.accounts.handle import handle_consumers as update_permission_consumers def handle_consumers(channel: Channel) -> None: project_handle_consumers(channel) + update_permission_consumers(channel) From 53ddf68521ddf6e78a8ef704bcdf29a73d2db8b6 Mon Sep 17 00:00:00 2001 From: Eliton Jorge Date: Thu, 18 Apr 2024 18:35:42 -0300 Subject: [PATCH 04/15] Update from v3.5.0 to v3.6.0 (#474) --- CHANGELOG.md | 5 +++++ marketplace/swagger.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b4cd82cf..367f29bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +v3.6.0 +---------- +* Permission update consumer +* Check if there is a feed upload in progress before uploading a new + v3.5.0 ---------- * Sends webhook and template data to flows diff --git a/marketplace/swagger.py b/marketplace/swagger.py index 459007b7..56c0f20c 100644 --- a/marketplace/swagger.py +++ b/marketplace/swagger.py @@ -6,7 +6,7 @@ view = get_schema_view( openapi.Info( title="Integrations API Documentation", - default_version="v3.5.0", + default_version="v3.6.0", desccription="Documentation of the Integrations APIs", ), public=True, From 7c48ee816ce80493f081af82769bd71eb4c54180 Mon Sep 17 00:00:00 2001 From: Eliton Jorge Date: Tue, 23 Apr 2024 16:25:06 -0300 Subject: [PATCH 05/15] Add vtex webhook log and add retry in facebook methods (#476) --- marketplace/clients/facebook/client.py | 4 +++ marketplace/clients/vtex/client.py | 2 ++ .../services/webhook/vtex/webhook_manager.py | 35 +++++++++++++++++++ 3 files changed, 41 insertions(+) diff --git a/marketplace/clients/facebook/client.py b/marketplace/clients/facebook/client.py index e6dbd178..04d15edc 100644 --- a/marketplace/clients/facebook/client.py +++ b/marketplace/clients/facebook/client.py @@ -4,6 +4,8 @@ from django.conf import settings from marketplace.clients.base import RequestClient +from marketplace.clients.decorators import retry_on_exception + WHATSAPP_VERSION = settings.WHATSAPP_VERSION @@ -43,6 +45,7 @@ def destroy_catalog(self, catalog_id): return response.json().get("success") + @retry_on_exception() def create_product_feed(self, product_catalog_id, name): url = self.get_url + f"{product_catalog_id}/product_feeds" @@ -52,6 +55,7 @@ def create_product_feed(self, product_catalog_id, name): return response.json() + @retry_on_exception() def upload_product_feed( self, feed_id, file, file_name, file_content_type, update_only=False ): diff --git a/marketplace/clients/vtex/client.py b/marketplace/clients/vtex/client.py index 5aa1aea5..2a22d683 100644 --- a/marketplace/clients/vtex/client.py +++ b/marketplace/clients/vtex/client.py @@ -18,6 +18,7 @@ def _get_headers(self): class VtexCommonClient(RequestClient): + @retry_on_exception() def check_domain(self, domain): try: url = f"https://{domain}/api/catalog_system/pub/products/search/" @@ -35,6 +36,7 @@ def search_product_by_sku_id(self, skuid, domain, sellerid=1): class VtexPrivateClient(VtexAuthorization, VtexCommonClient): + @retry_on_exception() def is_valid_credentials(self, domain): try: url = ( diff --git a/marketplace/services/webhook/vtex/webhook_manager.py b/marketplace/services/webhook/vtex/webhook_manager.py index 8038435c..cb3fd691 100644 --- a/marketplace/services/webhook/vtex/webhook_manager.py +++ b/marketplace/services/webhook/vtex/webhook_manager.py @@ -5,6 +5,8 @@ from django_redis import get_redis_connection +from marketplace.applications.models import App + logger = logging.getLogger(__name__) @@ -14,6 +16,9 @@ def __init__(self, app_uuid): self.app_uuid = app_uuid self.redis_client = get_redis_connection() + def get_webhooks_key(self): + return f"vtex:product-uploading:{self.app_uuid}" + def get_sku_list_key(self): return f"vtex:skus-list:{self.app_uuid}" @@ -21,6 +26,7 @@ def get_lock_key(self): return f"vtex:processing-lock:{self.app_uuid}" def enqueue_webhook_data(self, sku_id, webhook): + webhooks_key = self.get_webhooks_key() skus_list_key = self.get_sku_list_key() skus_in_processing = cache.get(skus_list_key) or [] @@ -28,6 +34,15 @@ def enqueue_webhook_data(self, sku_id, webhook): skus_in_processing.append(sku_id) cache.set(skus_list_key, skus_in_processing, timeout=7200) + # Update webhooks log + webhooks_log = cache.get(webhooks_key) or {} + if not webhooks_log: + # Sets the log lifetime to 24 hours if it is the first entry + cache.set(webhooks_key, {sku_id: webhook}, timeout=86400) + else: + webhooks_log[sku_id] = webhook + cache.set(webhooks_key, webhooks_log) + def dequeue_webhook_data(self): batch_size = settings.VTEX_UPDATE_BATCH_SIZE skus_list_key = self.get_sku_list_key() @@ -65,3 +80,23 @@ def dequeue_webhook_data(self): def is_processing_locked(self): return bool(self.redis_client.get(self.get_lock_key())) + + +class WebhookMultQueueManager: + def __init__(self): + self.redis_client = get_redis_connection() + + def reset_in_processing_keys(self): + apps = App.objects.filter(code="vtex", config__initial_sync_completed=True) + for app in apps: + key = f"vtex:processing-lock:{str(app.uuid)}" + if self.redis_client.get(key): + self.redis_client.delete(key) + print(f"{key} has deleted.") + + def list_in_processing_keys(self): + apps = App.objects.filter(code="vtex", config__initial_sync_completed=True) + for app in apps: + key = f"vtex:processing-lock:{str(app.uuid)}" + if self.redis_client.get(key): + print(f"{key}") From db2d7450363a0eab2ebe2721cfd9bab454a493d5 Mon Sep 17 00:00:00 2001 From: Eliton Jorge Date: Tue, 23 Apr 2024 16:30:39 -0300 Subject: [PATCH 06/15] Update from v3.6.0 to v3.6.1 (#478) --- CHANGELOG.md | 4 ++++ marketplace/swagger.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 367f29bc..08684385 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +v3.6.1 +---------- +* Add vtex webhook log and add retry in facebook methods + v3.6.0 ---------- * Permission update consumer diff --git a/marketplace/swagger.py b/marketplace/swagger.py index 56c0f20c..cf21189a 100644 --- a/marketplace/swagger.py +++ b/marketplace/swagger.py @@ -6,7 +6,7 @@ view = get_schema_view( openapi.Info( title="Integrations API Documentation", - default_version="v3.6.0", + default_version="v3.6.1", desccription="Documentation of the Integrations APIs", ), public=True, From bd34ffa61aa947520d86642f9df6eded2b21f7e5 Mon Sep 17 00:00:00 2001 From: Eliton Jorge Date: Wed, 24 Apr 2024 15:39:53 -0300 Subject: [PATCH 07/15] Remove vtex webhook log (#479) --- marketplace/services/webhook/vtex/webhook_manager.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/marketplace/services/webhook/vtex/webhook_manager.py b/marketplace/services/webhook/vtex/webhook_manager.py index cb3fd691..0168d399 100644 --- a/marketplace/services/webhook/vtex/webhook_manager.py +++ b/marketplace/services/webhook/vtex/webhook_manager.py @@ -26,7 +26,6 @@ def get_lock_key(self): return f"vtex:processing-lock:{self.app_uuid}" def enqueue_webhook_data(self, sku_id, webhook): - webhooks_key = self.get_webhooks_key() skus_list_key = self.get_sku_list_key() skus_in_processing = cache.get(skus_list_key) or [] @@ -34,15 +33,6 @@ def enqueue_webhook_data(self, sku_id, webhook): skus_in_processing.append(sku_id) cache.set(skus_list_key, skus_in_processing, timeout=7200) - # Update webhooks log - webhooks_log = cache.get(webhooks_key) or {} - if not webhooks_log: - # Sets the log lifetime to 24 hours if it is the first entry - cache.set(webhooks_key, {sku_id: webhook}, timeout=86400) - else: - webhooks_log[sku_id] = webhook - cache.set(webhooks_key, webhooks_log) - def dequeue_webhook_data(self): batch_size = settings.VTEX_UPDATE_BATCH_SIZE skus_list_key = self.get_sku_list_key() From 69beaee372b40ade4e5ee9b576a9d4b91a6c337b Mon Sep 17 00:00:00 2001 From: Eliton Jorge Date: Wed, 24 Apr 2024 16:01:58 -0300 Subject: [PATCH 08/15] Update from v3.6.1 to v3.6.2 (#480) --- CHANGELOG.md | 4 ++++ marketplace/swagger.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 08684385..9f4dbef9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +v3.6.2 +---------- +* Remove vtex webhook log + v3.6.1 ---------- * Add vtex webhook log and add retry in facebook methods diff --git a/marketplace/swagger.py b/marketplace/swagger.py index cf21189a..f0a8b0d8 100644 --- a/marketplace/swagger.py +++ b/marketplace/swagger.py @@ -6,7 +6,7 @@ view = get_schema_view( openapi.Info( title="Integrations API Documentation", - default_version="v3.6.1", + default_version="v3.6.2", desccription="Documentation of the Integrations APIs", ), public=True, From 7bb70683b9e9265c1a931081d70c741d81e8a216 Mon Sep 17 00:00:00 2001 From: Eliton Jorge Date: Tue, 7 May 2024 13:20:22 -0300 Subject: [PATCH 09/15] [FLOWS-1453] Ignore 404 error when fetching vtex product details (#477) * Ignore 404 error when fetching vtex product details * Displays error reason after reaching maximum attempts --- marketplace/clients/decorators.py | 55 ++++++++++++++----- .../services/vtex/utils/data_processor.py | 11 +++- 2 files changed, 51 insertions(+), 15 deletions(-) diff --git a/marketplace/clients/decorators.py b/marketplace/clients/decorators.py index ed6b36c7..cc737e46 100644 --- a/marketplace/clients/decorators.py +++ b/marketplace/clients/decorators.py @@ -1,28 +1,55 @@ import time import functools +import requests -def retry_on_exception(max_attempts=10, start_sleep_time=1, factor=2): +def retry_on_exception(max_attempts=8, start_sleep_time=1, factor=2): def decorator_retry(func): @functools.wraps(func) def wrapper(*args, **kwargs): attempts, sleep_time = 0, start_sleep_time + last_exception = "" while attempts < max_attempts: try: return func(*args, **kwargs) - except ( - Exception - ) as e: # TODO: Map only timeout errors or errors from many requests - if attempts > 5: - print( - f"Retrying... Attempt {attempts + 1} after {sleep_time} seconds, {str(e)}" - ) - time.sleep(sleep_time) - attempts += 1 - sleep_time *= factor - - print("Max retry attempts reached. Raising exception.") - raise Exception("Rate limit exceeded, max retry attempts reached.") + except requests.exceptions.Timeout as e: + last_exception = e + if attempts >= 5: + print(f"Timeout error: {str(e)}. Retrying...") + except requests.exceptions.HTTPError as e: + last_exception = e + if attempts >= 5: + if e.response.status_code == 429: + print(f"Too many requests: {str(e)}. Retrying...") + elif e.response.status_code == 500: + print(f"A 500 error occurred: {str(e)}. Retrying...") + else: + raise + except Exception as e: + last_exception = e + if hasattr(e, "status_code") and e.status_code == 404: + print(f"Not Found: {str(e)}. Not retrying this.") + raise + + if attempts >= 5: + print(f"An unexpected error has occurred: {e}") + + if attempts >= 5: + print( + f"Retrying... Attempt {attempts + 1} after {sleep_time} seconds" + ) + + time.sleep(sleep_time) + attempts += 1 + sleep_time *= factor + + message = ( + f"Rate limit exceeded, max retry attempts reached. Last error in {func.__name__}:" + f"Last error:{last_exception}, after {attempts} attempts." + ) + + print(message) + raise Exception(message) return wrapper diff --git a/marketplace/services/vtex/utils/data_processor.py b/marketplace/services/vtex/utils/data_processor.py index ab5a05d4..8f12dd7d 100644 --- a/marketplace/services/vtex/utils/data_processor.py +++ b/marketplace/services/vtex/utils/data_processor.py @@ -11,6 +11,8 @@ from tqdm import tqdm from queue import Queue +from marketplace.clients.exceptions import CustomAPIException + @dataclass class FacebookProductDTO: @@ -162,7 +164,14 @@ def worker(self): def process_single_sku(self, sku_id): facebook_products = [] - product_details = self.service.get_product_details(sku_id, self.domain) + + try: + product_details = self.service.get_product_details(sku_id, self.domain) + except CustomAPIException as e: + if e.status_code == 404: + print(f"SKU {sku_id} not found. Skipping...") + return [] + is_active = product_details.get("IsActive") if not is_active and not self.update_product: return facebook_products From 10451e823333eff004d8bdc77a883ef676752efe Mon Sep 17 00:00:00 2001 From: Eliton Jorge Date: Tue, 7 May 2024 17:37:48 -0300 Subject: [PATCH 10/15] Round up prices for weight-calculated products (#487) * Round up prices for weight-calculated products --- .../rules/round_up_calculate_by_weight.py | 108 ++++++++++ .../vtex/business/rules/rule_mappings.py | 2 + .../vtex/business/rules/tests/__init__.py | 0 .../vtex/business/rules/tests/tests_rules.py | 189 ++++++++++++++++++ 4 files changed, 299 insertions(+) create mode 100644 marketplace/services/vtex/business/rules/round_up_calculate_by_weight.py create mode 100644 marketplace/services/vtex/business/rules/tests/__init__.py create mode 100644 marketplace/services/vtex/business/rules/tests/tests_rules.py diff --git a/marketplace/services/vtex/business/rules/round_up_calculate_by_weight.py b/marketplace/services/vtex/business/rules/round_up_calculate_by_weight.py new file mode 100644 index 00000000..13aa633b --- /dev/null +++ b/marketplace/services/vtex/business/rules/round_up_calculate_by_weight.py @@ -0,0 +1,108 @@ +import math +from typing import Tuple + +from .interface import Rule +from marketplace.services.vtex.utils.data_processor import FacebookProductDTO + + +class RoundUpCalculateByWeight(Rule): + def apply(self, product: FacebookProductDTO, **kwargs) -> bool: + if self._calculates_by_weight(product): + unit_multiplier, weight = self._get_product_measurements(product) + + product.price *= unit_multiplier + product.sale_price *= unit_multiplier + + # Price rounding up + product.price = math.ceil(product.price) + product.sale_price = math.ceil(product.sale_price) + + price_per_kg = 0 + if weight > 0: + formatted_price = float(f"{product.sale_price / 100:.2f}") + price_per_kg = (formatted_price / unit_multiplier) * 100 + + price_per_kg = math.ceil(price_per_kg) + formatted_price_per_kg = f"{price_per_kg / 100:.2f}" + + product.description = ( + f"{product.title} - Aprox. {self._format_grams(weight)}, " + f"Preço do KG: R$ {formatted_price_per_kg}" + ) + product.title = f"{product.title} Unidade" + + return True + + def _get_multiplier(self, product: FacebookProductDTO) -> float: + return product.product_details.get("UnitMultiplier", 1.0) + + def _get_product_measurements( + self, product: FacebookProductDTO + ) -> Tuple[float, float]: + unit_multiplier = self._get_multiplier(product) + weight = self._get_weight(product) * unit_multiplier + return unit_multiplier, weight + + def _get_weight(self, product: FacebookProductDTO) -> float: + return product.product_details["Dimension"]["weight"] + + def _calculates_by_weight(self, product: FacebookProductDTO) -> bool: + """ + Determines if the weight calculation should be applied to a product based + on its categories and description. + + The method checks if the product title or description ends with a unit + indicator such as 'kg', 'g', or 'ml', which suggests that the product + is sold by unit and not by weight. If any such indicators are found, the + product is excluded from weight-based pricing calculations. + + Additionally, the product is excluded if 'iogurte' is among its categories, + as these are typically sold by volume. + + Finally, the method checks if the product's categories intersect with a + predefined set of categories known to require weight-based calculations + ('hortifruti', 'carnes e aves', 'frios e laticínios', 'padaria'). If there + is no intersection, the product does not qualify for weight-based calculations. + + Returns: + bool: True if the product should be calculated by weight, False otherwise. + """ + title_lower = product.title.lower() + description_lower = product.description.lower() + + if any(title_lower.endswith(ending) for ending in ["kg", "g", "ml"]): + return False + if any( + description_lower.endswith(ending) + for ending in ["kg", "g", "unid", "unidade", "ml"] + ): + return False + + product_categories = { + value.lower() + for value in product.product_details["ProductCategories"].values() + } + categories_to_calculate = { + "hortifruti", + "carnes e aves", + "frios e laticínios", + "padaria", + } + + if "iogurte" in product_categories: + return False + + return not categories_to_calculate.isdisjoint(product_categories) + + def _format_grams(self, value: float) -> str: + if 0 < value < 1: + grams = int(value * 1000) + else: + grams = int(value) + + if grams > 999: + formatted = f"{grams:,}".replace(",", ".") + else: + formatted = str(grams) + + return f"{formatted}g" diff --git a/marketplace/services/vtex/business/rules/rule_mappings.py b/marketplace/services/vtex/business/rules/rule_mappings.py index 8b4e2146..814fd482 100644 --- a/marketplace/services/vtex/business/rules/rule_mappings.py +++ b/marketplace/services/vtex/business/rules/rule_mappings.py @@ -3,6 +3,7 @@ from .calculate_by_weight import CalculateByWeight from .exclude_alcoholic_drinks import ExcludeAlcoholicDrinks from .unifies_id_with_seller import UnifiesIdWithSeller +from .round_up_calculate_by_weight import RoundUpCalculateByWeight RULE_MAPPINGS = { @@ -10,4 +11,5 @@ "calculate_by_weight": CalculateByWeight, "exclude_alcoholic_drinks": ExcludeAlcoholicDrinks, "unifies_id_with_seller": UnifiesIdWithSeller, + "round_up_calculate_by_weight": RoundUpCalculateByWeight, } diff --git a/marketplace/services/vtex/business/rules/tests/__init__.py b/marketplace/services/vtex/business/rules/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/marketplace/services/vtex/business/rules/tests/tests_rules.py b/marketplace/services/vtex/business/rules/tests/tests_rules.py new file mode 100644 index 00000000..19cc04c2 --- /dev/null +++ b/marketplace/services/vtex/business/rules/tests/tests_rules.py @@ -0,0 +1,189 @@ +import unittest + +from marketplace.services.vtex.business.rules.round_up_calculate_by_weight import ( + RoundUpCalculateByWeight, +) +from marketplace.services.vtex.utils.data_processor import FacebookProductDTO + + +class MockProductsDTO(unittest.TestCase): + def setUp(self): + self.products = [ + FacebookProductDTO( + id="test1", + title="Batata Doce Yakon Embalada", + description="Batata Doce Yakon Embalada", + availability="in stock", + status="active", + condition="new", + price=2439, + link="http://example.com/product", + image_link="http://example.com/product.jpg", + brand="ExampleBrand", + sale_price=2439, + product_details={ + "UnitMultiplier": 0.6, + "Dimension": { + "cubicweight": 0.3063, + "height": 5.0, + "length": 21.0, + "weight": 1000.0, + "width": 14.0, + }, + "ProductCategories": {"1": "carnes e aves"}, + }, + ), + FacebookProductDTO( + id="test2", + title="Iogurte Natural", + description="Iogurte Natural 1L", + availability="in stock", + status="active", + condition="new", + price=450, + link="http://example.com/yogurt", + image_link="http://example.com/yogurt.jpg", + brand="DairyBrand", + sale_price=450, + product_details={ + "UnitMultiplier": 1.0, + "Dimension": {"weight": 1000}, + "ProductCategories": { + "557": "Leite Fermentado", + "290": "Iogurte", + "220": "Frios e Laticínios", + }, + }, + ), + FacebookProductDTO( + id="test3", + title="Produto 1000kg", + description="Descrição do Produto", + availability="in stock", + status="active", + condition="new", + price=1000, + link="http://example.com/product1", + image_link="http://example.com/image1.jpg", + brand="TestBrand", + sale_price=1000, + product_details={ + "UnitMultiplier": 1.0, + "Dimension": {"weight": 1}, + "ProductCategories": {"category": "padaria"}, + }, + ), + FacebookProductDTO( + id="test4", + title="Produto", + description="Descrição 1000ml", + availability="in stock", + status="active", + condition="new", + price=200, + link="http://example.com/product2", + image_link="http://example.com/image2.jpg", + brand="TestBrand2", + sale_price=200, + product_details={ + "UnitMultiplier": 1.0, + "Dimension": {"weight": 1}, + "ProductCategories": {"category": "padaria"}, + }, + ), + FacebookProductDTO( + id="test5", + title="Pequeno Produto", + description="Peso menor que um kg", + availability="in stock", + status="active", + condition="new", + price=50, + link="http://example.com/product3", + image_link="http://example.com/image3.jpg", + brand="TestBrand3", + sale_price=50, + product_details={ + "UnitMultiplier": 1.0, + "Dimension": {"weight": 0.5}, + "ProductCategories": {"category": "hortifruti"}, + }, + ), + FacebookProductDTO( + id="test6", + title="Grande Produto", + description="Peso acima de um milhar", + availability="in stock", + status="active", + condition="new", + price=1050, + link="http://example.com/product4", + image_link="http://example.com/image4.jpg", + brand="TestBrand4", + sale_price=1050, + product_details={ + "UnitMultiplier": 1.0, + "Dimension": {"weight": 1050}, + "ProductCategories": {"category": "hortifruti"}, + }, + ), + ] + + +class TestRoundUpCalculateByWeight(MockProductsDTO): + def setUp(self): + super().setUp() # Call setUp of MockProductsDTO + self.rule = RoundUpCalculateByWeight() + + def test_apply_rounding_up_product_1(self): + product = self.products[0] + + before_title = product.title + expected_title = f"{before_title} Unidade" + expected_price_per_kg = "24.40" + + _, weight = self.rule._get_product_measurements(product) + grams = self.rule._format_grams(weight) + expected_description = ( + f"{before_title} - Aprox. {grams}, Preço do KG: R$ {expected_price_per_kg}" + ) + + self.rule.apply(product) + + self.assertEqual(product.title, expected_title) + self.assertEqual(product.description, expected_description) + self.assertEqual(product.price, 1464) + self.assertEqual(product.sale_price, 1464) + + def test_not_apply_rule(self): + product = self.products[1] + + before_title = product.title + expected_title = f"{before_title}" + before_description = product.description + expected_description = before_description + + self.rule.apply(product) + + self.assertEqual(product.title, expected_title) + self.assertEqual(product.description, expected_description) + self.assertEqual(product.price, 450) + self.assertEqual(product.sale_price, 450) + + def test_title_ends_with_unit_indicator(self): + product = self.products[2] + self.assertFalse(self.rule._calculates_by_weight(product)) + + def test_description_ends_with_unit_indicator(self): + product = self.products[3] + self.assertFalse(self.rule._calculates_by_weight(product)) + + def test_weight_less_than_one_kg(self): + product = self.products[4] + weight = self.rule._get_weight(product) + self.assertEqual(self.rule._format_grams(weight), "500g") + + def test_format_grams_above_thousand(self): + product = self.products[5] + weight = self.rule._get_weight(product) + self.assertEqual(self.rule._format_grams(weight), "1.050g") From e7f270a6b20adde522561c00b6f6ab769e405318 Mon Sep 17 00:00:00 2001 From: Eliton Jorge Date: Tue, 7 May 2024 18:07:24 -0300 Subject: [PATCH 11/15] First sync vtex products for specific sellers (#488) --- marketplace/services/vtex/generic_service.py | 27 ++++++++++++++----- .../services/vtex/private/products/service.py | 21 ++++++++++++--- marketplace/wpp_products/tasks.py | 3 ++- 3 files changed, 40 insertions(+), 11 deletions(-) diff --git a/marketplace/services/vtex/generic_service.py b/marketplace/services/vtex/generic_service.py index 80873647..6d4ff5d8 100644 --- a/marketplace/services/vtex/generic_service.py +++ b/marketplace/services/vtex/generic_service.py @@ -6,6 +6,8 @@ from datetime import datetime +from typing import Optional, List + from django.db import close_old_connections from django_redis import get_redis_connection @@ -110,12 +112,17 @@ def get_vtex_credentials_or_raise(self, app): class ProductInsertionService(VtexServiceBase): - def first_product_insert(self, credentials: APICredentials, catalog: Catalog): + def first_product_insert( + self, + credentials: APICredentials, + catalog: Catalog, + sellers: Optional[List[str]] = None, + ): pvt_service = self.get_private_service( credentials.app_key, credentials.app_token ) products = pvt_service.list_all_products( - credentials.domain, catalog.vtex_app.config + credentials.domain, catalog.vtex_app.config, sellers ) if not products: return None @@ -290,7 +297,9 @@ def _uploads_in_progress(self): class CatalogProductInsertion: @classmethod - def first_product_insert_with_catalog(cls, vtex_app: App, catalog_id: str): + def first_product_insert_with_catalog( + cls, vtex_app: App, catalog_id: str, sellers: Optional[List[str]] = None + ): """Inserts the first product with the given catalog.""" wpp_cloud_uuid = cls._get_wpp_cloud_uuid(vtex_app) credentials = cls._get_credentials(vtex_app) @@ -301,7 +310,7 @@ def first_product_insert_with_catalog(cls, vtex_app: App, catalog_id: str): cls._update_app_connected_catalog_flag(wpp_cloud) cls._link_catalog_to_vtex_app_if_needed(catalog, vtex_app) - cls._send_insert_task(credentials, catalog) + cls._send_insert_task(credentials, catalog, sellers) @staticmethod def _get_wpp_cloud_uuid(vtex_app) -> str: @@ -392,13 +401,19 @@ def _update_app_connected_catalog_flag(app) -> None: print("Changed connected_catalog to True") @staticmethod - def _send_insert_task(credentials, catalog) -> None: + def _send_insert_task( + credentials, catalog, sellers: Optional[List[str]] = None + ) -> None: from marketplace.celery import app as celery_app """Sends the insert task to the task queue.""" celery_app.send_task( name="task_insert_vtex_products", - kwargs={"credentials": credentials, "catalog_uuid": str(catalog.uuid)}, + kwargs={ + "credentials": credentials, + "catalog_uuid": str(catalog.uuid), + "sellers": sellers, + }, queue="product_first_synchronization", ) print( diff --git a/marketplace/services/vtex/private/products/service.py b/marketplace/services/vtex/private/products/service.py index 3d9ee6f0..ac4e0049 100644 --- a/marketplace/services/vtex/private/products/service.py +++ b/marketplace/services/vtex/private/products/service.py @@ -31,7 +31,7 @@ # Use products data as needed """ -from typing import List +from typing import List, Optional from marketplace.services.vtex.exceptions import CredentialsValidationError from marketplace.services.vtex.utils.data_processor import DataProcessor @@ -60,13 +60,26 @@ def validate_private_credentials(self, domain): self.check_is_valid_domain(domain) return self.client.is_valid_credentials(domain) - def list_all_products(self, domain, config) -> List[FacebookProductDTO]: - active_sellers = self.client.list_active_sellers(domain) + def list_all_products( + self, domain: str, config: dict, sellers: Optional[List[str]] = None + ) -> List[FacebookProductDTO]: + active_sellers = set(self.client.list_active_sellers(domain)) + if sellers is not None: + valid_sellers = [seller for seller in sellers if seller in active_sellers] + invalid_sellers = set(sellers) - active_sellers + if invalid_sellers: + print( + f"Warning: Sellers IDs {invalid_sellers} are not active and will be ignored." + ) + sellers_ids = valid_sellers + else: + sellers_ids = list(active_sellers) + skus_ids = self.client.list_all_products_sku_ids(domain) rules = self._load_rules(config.get("rules", [])) store_domain = config.get("store_domain") products_dto = self.data_processor.process_product_data( - skus_ids, active_sellers, self, domain, store_domain, rules + skus_ids, sellers_ids, self, domain, store_domain, rules ) return products_dto diff --git a/marketplace/wpp_products/tasks.py b/marketplace/wpp_products/tasks.py index 343ac45c..c1671976 100644 --- a/marketplace/wpp_products/tasks.py +++ b/marketplace/wpp_products/tasks.py @@ -124,6 +124,7 @@ def task_insert_vtex_products(**kwargs): credentials = kwargs.get("credentials") catalog_uuid = kwargs.get("catalog_uuid") + sellers = kwargs.get("sellers") if not all([credentials, catalog_uuid]): logger.error( @@ -145,7 +146,7 @@ def task_insert_vtex_products(**kwargs): domain=credentials["domain"], ) print(f"Starting first product insert for catalog: {str(catalog.name)}") - products = vtex_service.first_product_insert(api_credentials, catalog) + products = vtex_service.first_product_insert(api_credentials, catalog, sellers) if products is None: print("There are no products to be shipped after processing the rules") return From 7ca72b134099f4da142319760a3ae056dd66fc44 Mon Sep 17 00:00:00 2001 From: Eliton Jorge Date: Tue, 7 May 2024 18:15:36 -0300 Subject: [PATCH 12/15] Vtex rules for colombia (#483) --- .../business/rules/calculate_by_weight_co.py | 88 +++++++++++++++++++ .../vtex/business/rules/currency_co.py | 15 ++++ .../vtex/business/rules/rule_mappings.py | 4 + 3 files changed, 107 insertions(+) create mode 100644 marketplace/services/vtex/business/rules/calculate_by_weight_co.py create mode 100644 marketplace/services/vtex/business/rules/currency_co.py diff --git a/marketplace/services/vtex/business/rules/calculate_by_weight_co.py b/marketplace/services/vtex/business/rules/calculate_by_weight_co.py new file mode 100644 index 00000000..fe42d580 --- /dev/null +++ b/marketplace/services/vtex/business/rules/calculate_by_weight_co.py @@ -0,0 +1,88 @@ +from .interface import Rule +from marketplace.services.vtex.utils.data_processor import FacebookProductDTO +from typing import Union + + +class CalculateByWeightCO(Rule): + def apply(self, product: FacebookProductDTO, **kwargs) -> bool: + if self._calculates_by_weight(product): + unit_multiplier = self._get_multiplier(product) + weight = self._get_weight(product) * unit_multiplier + + # 10% increase for specific categories + if self._is_increased_price_category(product) and weight >= 500: + increase_factor = 1.10 # 10% + product.price *= unit_multiplier * increase_factor + product.sale_price *= unit_multiplier * increase_factor + else: + product.price *= unit_multiplier + product.sale_price *= unit_multiplier + + price_per_kg = 0 + if weight > 0: + formatted_price = float(f"{product.sale_price / 100:.2f}") + price_per_kg = formatted_price / unit_multiplier + + product.description = ( + f"{product.title} - Aprox. {self._format_grams(weight)}, " + f"Precio por KG: COP {self._format_price(price_per_kg)}" + ) + product.title = f"{product.title} Unidad" + + return True + + def _is_increased_price_category(self, product: FacebookProductDTO) -> bool: + categories = [ + "carne y pollo", + "carne res", + "pescados y mariscos", + "pescado congelado", + ] + product_categories = { + k: v.lower() + for k, v in product.product_details["ProductCategories"].items() + } + + return any(category in product_categories.values() for category in categories) + + def _get_multiplier(self, product: FacebookProductDTO) -> float: + return product.product_details.get("UnitMultiplier", 1.0) + + def _get_weight(self, product: FacebookProductDTO) -> float: + return product.product_details["Dimension"]["weight"] + + def _calculates_by_weight(self, product: FacebookProductDTO) -> bool: + all_categories = [ + "carne y pollo", + "carne res", + "pescados y mariscos", + "pescado congelado", + "verduras", + "frutas", + ] + product_categories = { + k: v.lower() + for k, v in product.product_details["ProductCategories"].items() + } + + for category in all_categories: + if category in product_categories.values(): + return True + + return False + + def _format_price(self, price: Union[int, float]) -> str: + return f"{price:.2f}" + + def _format_grams(self, value: float) -> str: + if 0 < value < 1: + grams = int(value * 1000) + else: + grams = int(value) + + if grams > 999: + formatted = f"{grams:,}".replace(",", ".") + else: + formatted = str(grams) + + return f"{formatted}g" diff --git a/marketplace/services/vtex/business/rules/currency_co.py b/marketplace/services/vtex/business/rules/currency_co.py new file mode 100644 index 00000000..91aa5a71 --- /dev/null +++ b/marketplace/services/vtex/business/rules/currency_co.py @@ -0,0 +1,15 @@ +from .interface import Rule +from typing import Union +from marketplace.services.vtex.utils.data_processor import FacebookProductDTO + + +class CurrencyCOP(Rule): + def apply(self, product: FacebookProductDTO, **kwargs) -> bool: + product.price = self.format_price(product.price) + product.sale_price = self.format_price(product.sale_price) + return True + + @staticmethod + def format_price(price: Union[int, float]) -> str: + formatted_price = f"{price / 100:.2f} COP" + return formatted_price diff --git a/marketplace/services/vtex/business/rules/rule_mappings.py b/marketplace/services/vtex/business/rules/rule_mappings.py index 814fd482..31b70a2d 100644 --- a/marketplace/services/vtex/business/rules/rule_mappings.py +++ b/marketplace/services/vtex/business/rules/rule_mappings.py @@ -3,6 +3,8 @@ from .calculate_by_weight import CalculateByWeight from .exclude_alcoholic_drinks import ExcludeAlcoholicDrinks from .unifies_id_with_seller import UnifiesIdWithSeller +from .calculate_by_weight_co import CalculateByWeightCO +from .currency_co import CurrencyCOP from .round_up_calculate_by_weight import RoundUpCalculateByWeight @@ -11,5 +13,7 @@ "calculate_by_weight": CalculateByWeight, "exclude_alcoholic_drinks": ExcludeAlcoholicDrinks, "unifies_id_with_seller": UnifiesIdWithSeller, + "calculate_by_weight_co": CalculateByWeightCO, + "currency_co": CurrencyCOP, "round_up_calculate_by_weight": RoundUpCalculateByWeight, } From 4f0f4dc5066f2ae32c77650b83d50c817c061f3c Mon Sep 17 00:00:00 2001 From: Eliton Jorge Date: Tue, 7 May 2024 18:39:57 -0300 Subject: [PATCH 13/15] Add method to list only active skus IDs (#486) * Add method to list only active skus IDs * Extract only sku_ids --- marketplace/clients/vtex/client.py | 25 +++++++++++++++++++ .../services/vtex/private/products/service.py | 2 +- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/marketplace/clients/vtex/client.py b/marketplace/clients/vtex/client.py index 2a22d683..fe20a0fb 100644 --- a/marketplace/clients/vtex/client.py +++ b/marketplace/clients/vtex/client.py @@ -105,3 +105,28 @@ def pub_simulate_cart_for_seller(self, sku_id, seller_id, domain): "price": 0, "list_price": 0, } + + @retry_on_exception() + def list_all_active_products(self, domain): + unique_skus = set() + step = 250 + current_from = 1 + + while True: + current_to = current_from + step - 1 + url = ( + f"https://{domain}/api/catalog_system/pvt/products/" + f"GetProductAndSkuIds?_from={current_from}&_to={current_to}&status=1" + ) + headers = self._get_headers() + response = self.make_request(url, method="GET", headers=headers) + + data = response.json().get("data", {}) + if not data: + break + + for _, skus in data.items(): + unique_skus.update(skus) + current_from += step + + return list(unique_skus) diff --git a/marketplace/services/vtex/private/products/service.py b/marketplace/services/vtex/private/products/service.py index ac4e0049..021f085d 100644 --- a/marketplace/services/vtex/private/products/service.py +++ b/marketplace/services/vtex/private/products/service.py @@ -75,7 +75,7 @@ def list_all_products( else: sellers_ids = list(active_sellers) - skus_ids = self.client.list_all_products_sku_ids(domain) + skus_ids = self.client.list_all_active_products(domain) rules = self._load_rules(config.get("rules", [])) store_domain = config.get("store_domain") products_dto = self.data_processor.process_product_data( From e4360eaa2a855e827f087309c98df31e211fe179 Mon Sep 17 00:00:00 2001 From: Eliton Jorge Date: Tue, 7 May 2024 18:52:31 -0300 Subject: [PATCH 14/15] Update from v3.6.2 to v3.6.3 (#489) --- CHANGELOG.md | 8 ++++++++ marketplace/swagger.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f4dbef9..347bd8cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +v3.6.3 +---------- +* Ignore 404 error when fetching vtex product details +* Round up prices for weight-calculated products +* First sync vtex products for specific sellers +* Vtex rules for colombia +* Add method to list only active skus IDs + v3.6.2 ---------- * Remove vtex webhook log diff --git a/marketplace/swagger.py b/marketplace/swagger.py index f0a8b0d8..2d48877a 100644 --- a/marketplace/swagger.py +++ b/marketplace/swagger.py @@ -6,7 +6,7 @@ view = get_schema_view( openapi.Info( title="Integrations API Documentation", - default_version="v3.6.2", + default_version="v3.6.3", desccription="Documentation of the Integrations APIs", ), public=True, From db6dd0fb5c65d02adf22e224b6773bd14c6b53b6 Mon Sep 17 00:00:00 2001 From: lucaslinhares Date: Wed, 15 May 2024 16:49:13 -0300 Subject: [PATCH 15/15] add showAttachmentsButton in WWC config --- marketplace/core/types/channels/weni_web_chat/serializers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/marketplace/core/types/channels/weni_web_chat/serializers.py b/marketplace/core/types/channels/weni_web_chat/serializers.py index da1895ce..f4777343 100644 --- a/marketplace/core/types/channels/weni_web_chat/serializers.py +++ b/marketplace/core/types/channels/weni_web_chat/serializers.py @@ -51,6 +51,7 @@ class ConfigSerializer(serializers.Serializer): customCss = serializers.CharField(required=False) timeBetweenMessages = serializers.IntegerField(default=1) tooltipMessage = serializers.CharField(required=False) + showAttachmentsButton = serializers.BooleanField(default=True) def to_internal_value(self, data): self.app = self.parent.instance