Feature/vtex clients (#43)

* feature: vtex clients to vtex source

docker/docker-compose.yaml

@@ -12,6 +12,10 @@ services:
       - POSTGRES_USER=${POSTGRES_USER:-insights}
       - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-insights}
       - POSTGRES_DB=${POSTGRES_DB:-insights}
+  redis:
+    image: redis:6.2
+    ports:
+      - 6379:6379
   insights:
     image: ${DOCKER_IMAGE_NAME:-ilha/insights}:${TAG:-latest}
     build:

insights/conftest.py

@@ -3,6 +3,11 @@
 from insights.dashboards.models import Dashboard, DashboardTemplate
 from insights.projects.models import Project, ProjectAuth
 from insights.users.models import User
+from insights.dashboards.models import (
+    Dashboard,
+    DashboardTemplate,
+)
+from insights.widgets.models import Widget
 
 
 @fixture
@@ -75,3 +80,13 @@ def create_project_auth(create_project, create_user):
     role = 1
 
     return ProjectAuth.objects.create(project=proj, user=user, role=role)
+
+
+@fixture
+def create_widget(create_default_dashboard):
+    return Widget.objects.create(
+        dashboard=create_default_dashboard,
+        name="Example Widget",
+        source="flows",
+        config={"example": "widget config"},
+    )

insights/dashboards/usecases/flows_dashboard_creation.py

@@ -69,7 +69,7 @@ def create_graph_funnel_widgets(self, dashboard, amount):
         for position in positions[amount]:
             Widget.objects.create(
                 name="Funil",
-                type="graph_funnel",
+                type="empty_column",
                 source="",
                 config={},
                 dashboard=dashboard,

insights/internals/base.py

@@ -22,3 +22,7 @@ def headers(self):
             "Content-Type": "application/json; charset: utf-8",
             "Authorization": self.get_module_token(),
         }
+
+
+class VtexAuthentication:
+    pass

insights/settings.py

@@ -12,6 +12,7 @@
 
 import os
 from pathlib import Path
+import json
 
 import environ
 import sentry_sdk
@@ -273,3 +274,28 @@
 GROQ_OPEN_AI_URL = env.str("GROQ_OPEN_AI_URL", default="")
 GROQ_CHATGPT_TOKEN = env.str("GROQ_CHATGPT_TOKEN", default="")
 GROQ_OPEN_AI_GPT_VERSION = env.str("GROQ_OPEN_AI_GPT_VERSION", default="")
+
+INTEGRATIONS_URL = env("INTEGRATIONS_URL")
+
+REDIS_URL = env.str("CHANNEL_LAYERS_REDIS", default="redis://localhost:6379/1")
+
+# channels
+CHANNEL_LAYERS = {
+    "default": {
+        "BACKEND": "channels_redis.pubsub.RedisPubSubChannelLayer",
+        "CONFIG": {
+            "hosts": [REDIS_URL],
+        },
+    },
+}
+
+CACHES = {
+    "default": {
+        "BACKEND": "django_redis.cache.RedisCache",
+        "LOCATION": REDIS_URL,
+        "OPTIONS": {"CLIENT_CLASS": "django_redis.client.DefaultClient"},
+    }
+}
+
+PROJECTS_VTEX = json.loads(os.getenv("PROJECTS_VTEX", "[]"))
+PROJECT_TOKENS_VTEX = json.loads(os.getenv("PROJECT_TOKENS_VTEX", "{}"))

insights/sources/cache.py

@@ -0,0 +1,15 @@
+from django_redis import get_redis_connection
+from typing import Optional, Any
+
+
+class CacheClient:
+    def __init__(self) -> None:
+        pass
+
+    def get(self, key: str) -> Optional[Any]:
+        with get_redis_connection() as redis_connection:
+            return redis_connection.get(key)
+
+    def set(self, key: str, value: Any, ex: Optional[int] = None) -> bool:
+        with get_redis_connection() as redis_connection:
+            return redis_connection.set(key, value, ex=ex)

insights/sources/orders/clients.py

@@ -0,0 +1,133 @@
+import requests
+from insights.internals.base import VtexAuthentication
+from concurrent.futures import ThreadPoolExecutor, as_completed
+import json
+from insights.sources.vtexcredentials.clients import AuthRestClient
+from insights.sources.cache import CacheClient
+from insights.utils import format_to_iso_utc
+from django.conf import settings
+
+from datetime import datetime
+
+
+class VtexOrdersRestClient(VtexAuthentication):
+    def __init__(self, auth_params, cache_client: CacheClient) -> None:
+        self.headers = {
+            "X-VTEX-API-AppToken": auth_params["VTEX-API-AppToken"],
+            "X-VTEX-API-AppKey": auth_params["VTEX-API-AppKey"],
+        }
+        self.base_url = auth_params["domain"]
+        self.cache = cache_client
+
+    def get_cache_key(self, query_filters):
+        """Gere uma chave única para o cache baseada nos filtros de consulta."""
+        return f"vtex_data:{json.dumps(query_filters, sort_keys=True)}"
+
+    def get_vtex_endpoint(self, query_filters: dict, page_number: int = 1):
+        start_date = query_filters.get("ended_at__gte")
+        end_date = query_filters.get("ended_at__lte")
+        utm_source = query_filters.get("utm_source")
+
+        if start_date is not None:
+            url = f"https://{self.base_url}.myvtex.com/api/oms/pvt/orders/?f_UtmSource={utm_source}&per_page=100&page={page_number}&f_authorizedDate=authorizedDate:[{start_date} TO {end_date}]&f_status=invoiced"
+        else:
+            url = f"https://{self.base_url}.myvtex.com/api/oms/pvt/orders/?f_UtmSource={utm_source}&per_page=100&page={page_number}&f_status=invoiced"
+        return url
+
+    def parse_datetime(self, date_str):
+        try:
+            # Tente fazer o parse da string para datetime
+            return datetime.fromisoformat(date_str)  # Para strings ISO formatadas
+        except ValueError:
+            return None  # Retorne None se a conversão falhar
+
+    def list(self, query_filters: dict):
+        cache_key = self.get_cache_key(query_filters)
+
+        cached_data = self.cache.get(cache_key)
+        if cached_data:
+            return json.loads(cached_data)
+
+        if not query_filters.get("utm_source", None):
+            return {"error": "utm_source field is mandatory"}
+
+        if query_filters.get("ended_at__gte", None):
+            start_date_str = query_filters["ended_at__gte"]
+            start_date = self.parse_datetime(start_date_str)
+            if start_date:
+                query_filters["ended_at__gte"] = start_date.strftime(
+                    "%Y-%m-%dT%H:%M:%S.%fZ"
+                )
+
+        if query_filters.get("ended_at__lte", None):
+            end_date_str = query_filters["ended_at__lte"]
+            end_date = self.parse_datetime(end_date_str)
+            if end_date:
+                query_filters["ended_at__lte"] = end_date.strftime(
+                    "%Y-%m-%dT%H:%M:%S.%fZ"
+                )
+
+        if query_filters.get("utm_source", None):
+            query_filters["utm_source"] = query_filters.pop("utm_source")[0]
+
+        total_value = 0
+        total_sell = 0
+        max_value = float("-inf")
+        min_value = float("inf")
+
+        response = requests.get(
+            self.get_vtex_endpoint(query_filters), headers=self.headers
+        )
+        data = response.json()
+
+        if "list" not in data or not data["list"]:
+            return response.status_code, data
+
+        pages = data["paging"]["pages"] if "paging" in data else 1
+
+        # botar o max_workers em variavel de ambiente
+        with ThreadPoolExecutor(max_workers=10) as executor:
+            page_futures = {
+                executor.submit(
+                    lambda page=page: requests.get(
+                        self.get_vtex_endpoint({**query_filters, "page": page}),
+                        headers=self.headers,
+                    )
+                ): page
+                for page in range(1, pages + 1)
+            }
+
+            for page_future in as_completed(page_futures):
+                try:
+                    response = page_future.result()
+                    if response.status_code == 200:
+                        results = response.json()
+                        for result in results["list"]:
+                            if result["status"] != "canceled":
+                                total_value += result["totalValue"]
+                                total_sell += 1
+                                max_value = max(max_value, result["totalValue"])
+                                min_value = min(min_value, result["totalValue"])
+                    else:
+                        print(
+                            f"Request failed with status code: {response.status_code}"
+                        )
+                except Exception as exc:
+                    print(f"Generated an exception: {exc}")
+
+        total_value /= 100
+        max_value /= 100
+        min_value /= 100
+        medium_ticket = total_value / total_sell if total_sell > 0 else 0
+
+        result_data = {
+            "countSell": total_sell,
+            "accumulatedTotal": total_value,
+            "ticketMax": max_value,
+            "ticketMin": min_value,
+            "medium_ticket": medium_ticket,
+        }
+
+        self.cache.set(cache_key, json.dumps(result_data), ex=3600)
+
+        return response.status_code, result_data

insights/sources/orders/tests/test_orders_client.py

@@ -0,0 +1,103 @@
+import pytest
+from clients import VtexOrdersRestClient
+from unittest.mock import patch, Mock
+import json
+
+
+@pytest.mark.django_db
+@patch("clients.VtexOrdersRestClient.list")
+def test_verify_fields(mock_list):
+    mock_list.return_value = {
+        "countSell": 1,
+        "accumulatedTotal": 50.21,
+        "ticketMax": 50.21,
+        "ticketMin": 50.21,
+        "medium_ticket": 50.21,
+    }
+
+    auth_params = {
+        "app_token": "fake_token",
+        "app_key": "fake_key",
+        "domain": "fake_domain",
+    }
+    cache_client = Mock()
+
+    client = VtexOrdersRestClient(auth_params, cache_client)
+
+    query_filters = {
+        "start_date": "2024-09-01T00:00:00.000Z",
+        "end_date": "2024-09-04T00:00:00.000Z",
+        "base_url": "gbarbosa",
+        "utm_source": "gbarbosa-recuperacaochatbotvtex",
+    }
+
+    response = client.list(query_filters)
+    expected_keys = {
+        "countSell",
+        "accumulatedTotal",
+        "ticketMax",
+        "ticketMin",
+        "medium_ticket",
+    }
+    assert set(response.keys()) == expected_keys
+
+
+@pytest.mark.django_db
+def test_missing_utm_source():
+    # Mockando os auth_params e o cache_client
+    auth_params = {
+        "app_token": "fake_token",
+        "app_key": "fake_key",
+        "domain": "fake_domain",
+    }
+    cache_client = Mock()  # CacheClient mockado
+
+    # Simular que o cache não tem dados, retornando None
+    cache_client.get.return_value = None
+
+    client = VtexOrdersRestClient(auth_params, cache_client)
+
+    query_filters = {
+        "start_date": "2024-09-01T00:00:00.000Z",
+        "end_date": "2024-09-04T00:00:00.000Z",
+        "base_url": "gbarbosa",
+    }
+
+    response = client.list(query_filters)
+
+    assert response == {"error": "utm_source field is mandatory"}
+
+
+@pytest.mark.django_db
+def test_cache_behavior():
+    auth_params = {
+        "app_token": "fake_token",
+        "app_key": "fake_key",
+        "domain": "fake_domain",
+    }
+    cache_client = Mock()
+
+    client = VtexOrdersRestClient(auth_params, cache_client)
+
+    cached_response = {
+        "countSell": 1,
+        "accumulatedTotal": 50.21,
+        "ticketMax": 50.21,
+        "ticketMin": 50.21,
+        "medium_ticket": 50.21,
+    }
+
+    client.cache.get.return_value = json.dumps(cached_response)
+
+    query_filters = {
+        "start_date": "2024-09-01T00:00:00.000Z",
+        "end_date": "2024-09-04T00:00:00.000Z",
+        "base_url": "gbarbosa",
+        "utm_source": "gbarbosa-recuperacaochatbotvtex",
+    }
+
+    response = client.list(query_filters)
+
+    assert response == (200, cached_response)
+
+    client.cache.get.assert_called_once_with(client.get_cache_key(query_filters))

insights/sources/orders/usecases/query_execute.py

@@ -0,0 +1,20 @@
+from insights.sources.orders.clients import VtexOrdersRestClient
+from insights.sources.cache import CacheClient
+
+
+class QueryExecutor:
+    def execute(
+        filters: dict,
+        operation: str,
+        parser: callable,
+        query_kwargs: dict = {},
+        auth_params: dict = {},
+        *args,
+        **kwargs
+    ):
+        client = VtexOrdersRestClient(
+            auth_params=auth_params, cache_client=CacheClient()
+        )
+        list_data = client.list(query_filters=filters)
+
+        return list_data

insights/sources/vtexcredentials/clients.py

@@ -0,0 +1,23 @@
+import requests
+from django.conf import settings
+
+from insights.internals.base import InternalAuthentication
+
+
+class AuthRestClient(InternalAuthentication):
+    def __init__(self, project) -> None:
+        self.url = f"{settings.INTEGRATIONS_URL}/api/v1/apptypes/vtex/integration-details/{project}"
+
+    def get_vtex_auth(self):
+        # response = requests.get(url=self.url, headers=self.headers)
+        # print("url", self.url)
+        # print("RESPONSE", response)
+        # tokens = response.json()
+        # print("TOKEN", tokens)
+        # credentials = {
+        #     "app_key": tokens["app_key"],
+        #     "app_token": tokens["app_token"],
+        #     "domain": tokens["domain"],
+        # }
+        # print("credenciais")
+        return {}