Feature/add filterset class (#24)

* feat: add filterset class to room source, to clean the fields and return every datail of the field on a single class

* feat: update agents, flows, queues, sectors, rooms, tags sources with the new filterset pattern

* feat: update rooms and flowruns

* feat: fix flows query executor

* feat: fix imports

* feat: minor fixes

* feat: minor fixes

* feat: minor fixes

* feat: remove comments

insights/projects/viewsets.py

@@ -32,12 +32,11 @@ def retrieve_source_data(self, request, source_slug=None, *args, **kwargs):
         op_field = filters.pop("op_field", [None])[0]
         if op_field:
             query_kwargs["op_field"] = op_field
-
+        filters["project"] = str(self.get_object().uuid)
         serialized_source = SourceQuery.execute(
             filters=filters,
             operation=operation,
             parser=parse_dict_to_json,
-            project=self.get_object(),
             user_email=self.request.user.email,
             return_format="select_input",
             query_kwargs=query_kwargs,

insights/sources/agents/clients.py

@@ -2,8 +2,11 @@
 from django.conf import settings
 
 from insights.internals.base import InternalAuthentication
-from insights.sources.agents.query_builder import AgentSQLQueryBuilder
-from insights.sources.filters import PostgreSQLFilterStrategy
+from insights.sources.clients import GenericSQLQueryGenerator
+
+
+class AgentSQLQueryGenerator(GenericSQLQueryGenerator):
+    default_query_type = "list"
 
 
 class AgentsRESTClient(InternalAuthentication):
@@ -23,25 +26,3 @@ def list(self, query_filters: dict):
             url=self.url, headers=self.headers, params=query_filters
         )
         return response.json()
-
-
-def generate_sql_query(
-    filters: dict,
-    query_type: str = "list",
-    query_kwargs: dict = {},
-):
-    strategy = PostgreSQLFilterStrategy()
-    builder = AgentSQLQueryBuilder()
-
-    for key, value in filters.items():
-        table_alias = "pp"
-        if "__" in key:
-            field, operation = key.split("__", 1)
-        elif type(value) is list:
-            field = key.split("__", 1)[0]
-            operation = "in"
-        else:
-            field, operation = key, "eq"
-        builder.add_filter(strategy, field, operation, value, table_alias)
-    builder.build_query()
-    return getattr(builder, query_type)(**query_kwargs)

insights/sources/agents/filtersets.py

@@ -0,0 +1,17 @@
+# use stub files to represent it on other parts of the code
+# Use django_filters Filter class as a reference
+from insights.sources.filtersets import GenericSQLFilter
+
+
+class AgentFilterSet:
+    project = GenericSQLFilter(
+        source_field="project_id",
+        table_alias="pp",
+    )
+    project_id = project
+
+    def get_field(self, field_name):
+        try:
+            return getattr(self, field_name)
+        except AttributeError:
+            return None

insights/sources/agents/usecases/query_execute.py

@@ -1,8 +1,11 @@
 from insights.db.postgres.django.connection import dictfetchall, get_cursor
 from insights.sources.agents.clients import (
+    AgentSQLQueryGenerator,
     AgentsRESTClient,
-    generate_sql_query,
 )
+from insights.sources.agents.filtersets import AgentFilterSet
+from insights.sources.agents.query_builder import AgentSQLQueryBuilder
+from insights.sources.filter_strategies import PostgreSQLFilterStrategy
 
 
 class QueryExecutor:
@@ -13,12 +16,20 @@ def execute(
         return_format: str = None,
         project: object = None,
         user_email: str = None,
+        query_kwargs: dict = {},
         *args,
         **kwargs
     ):
         if return_format == "select_input" or operation != "list":
-            filters["project_id"] = str(project.uuid)
-            query, params = generate_sql_query(filters=filters, query_type=operation)
+            query_generator = AgentSQLQueryGenerator(
+                filter_strategy=PostgreSQLFilterStrategy,
+                query_builder=AgentSQLQueryBuilder,
+                filterset=AgentFilterSet,
+                filters=filters,
+                query_type=operation,
+                query_kwargs=query_kwargs,
+            )
+            query, params = query_generator.generate()
             with get_cursor(db_name="chats") as cur:
                 query_exec = cur.execute(query, params)
                 query_results = dictfetchall(query_exec)

insights/sources/clients.py

@@ -0,0 +1,87 @@
+class GenericSQLQueryGenerator:
+    default_query_type = "count"
+
+    def __init__(
+        self,
+        filter_strategy,
+        query_builder,
+        filterset,
+        filters: dict,
+        query_type: str = "",
+        query_kwargs: dict = {},
+    ) -> None:
+        self.filter_strategy = filter_strategy
+        self.query_builder = query_builder
+        self.filterset = filterset
+        self.filters = filters
+        self.query_type = query_type or self.default_query_type
+        self.query_kwargs = query_kwargs
+
+    def generate(self):
+        strategy = self.filter_strategy()
+        builder = self.query_builder()
+        filterset = self.filterset()
+
+        for key, value in self.filters.items():
+            if "__" in key:
+                field, operation = key.split("__", 1)
+            elif type(value) is list:
+                field = key.split("__", 1)[0]
+                operation = "in"
+            else:
+                field, operation = key, "eq"
+            field_object = filterset.get_field(field)
+            if field_object is None:
+                continue
+            source_field = field_object.source_field
+            join_clause = field_object.join_clause
+            if join_clause != {}:
+                builder.add_joins(join_clause)
+            builder.add_filter(
+                strategy, source_field, operation, value, field_object.table_alias
+            )
+        builder.build_query()
+
+        return getattr(builder, self.query_type)(**self.query_kwargs)
+
+
+class GenericElasticSearchQueryGenerator:
+    default_query_type = "count"
+
+    def __init__(
+        self,
+        filter_strategy,
+        query_builder,
+        filterset,
+        filters: dict,
+        query_type: str = "",
+        query_kwargs: dict = {},
+    ) -> None:
+        self.filter_strategy = filter_strategy
+        self.query_builder = query_builder
+        self.filterset = filterset
+        self.filters = filters
+        self.query_type = query_type or self.default_query_type
+        self.query_kwargs = query_kwargs
+
+    def generate(self):
+        strategy = self.filter_strategy()
+        builder = self.query_builder()
+        filterset = self.filterset()
+
+        for key, value in self.filters.items():
+            if "__" in key:
+                field, operation = key.split("__", 1)
+            elif type(value) is list:
+                field = key.split("__", 1)[0]
+                operation = "in"
+            else:
+                field, operation = key, "eq"
+            field_object = filterset.get_field(field)
+            if field_object is None:
+                continue
+            source_field = field_object.source_field
+            builder.add_filter(strategy, source_field, operation, value)
+        builder.build_query()
+
+        return getattr(builder, self.query_type)(**self.query_kwargs)

insights/sources/filtersets.py

@@ -0,0 +1,25 @@
+class GenericSQLFilter:
+    """Responsible for cleaning and validating Filter data"""
+
+    def __init__(
+        self,
+        source_field: str,
+        table_alias: str,
+        join_clause: dict = {},
+        value: any = None,
+    ) -> None:
+        self.source_field = source_field
+        self.table_alias = table_alias
+        self.join_clause = join_clause
+
+
+class GenericElasticSearchFilter:
+    """Responsible for cleaning and validating Filter data"""
+
+    def __init__(
+        self,
+        source_field: str,
+        field_type: str,
+    ) -> None:
+        self.source_field = source_field
+        self.field_type = field_type

insights/sources/flowruns/clients.py

@@ -1,7 +1,4 @@
-from insights.sources.filters import ElasticSearchFilterStrategy
-from insights.sources.flowruns.query_builder import (
-    FlowRunsElasticSearchQueryBuilder,
-)
+from insights.sources.clients import GenericElasticSearchQueryGenerator
 
 flow_runs_filters = {
     "created_on": {"type": "date", "to_field": "created_on"},
@@ -12,32 +9,5 @@
 }
 
 
-class FlowRunsElasticSearchClient:
-    def execute(
-        self,
-        filters: dict,
-        query_type: str = "count",
-        query_kwargs: dict = {},
-    ):
-        strategy = ElasticSearchFilterStrategy()
-        builder = FlowRunsElasticSearchQueryBuilder()
-
-        for key, value in filters.items():
-            if "__" in key:
-                field, operation = key.split("__", 1)
-            elif type(value) is list:
-                field = key.split("__", 1)[0]
-                operation = "in"
-            else:
-                field, operation = key, "eq"
-
-            if (
-                field in flow_runs_filters
-            ):  # only consider filters describred in the flow_runs_filters dict. TODO: maybe transform this dict into a class similar to django-filters filterset classes
-                field = flow_runs_filters[field]["to_field"]
-            else:
-                continue
-            builder.add_filter(strategy, field, operation, value)
-        builder.build_query()
-
-        return getattr(builder, query_type)(**query_kwargs)
+class FlowRunElasticSearchQueryGenerator(GenericElasticSearchQueryGenerator):
+    default_query_type = "count"

insights/sources/flowruns/filtersets.py

@@ -0,0 +1,32 @@
+# use stub files to represent it on other parts of the code
+# Use django_filters Filter class as a reference
+from insights.sources.filtersets import GenericElasticSearchFilter
+
+
+class FlowRunFilterSet:
+    created_on = GenericElasticSearchFilter(
+        source_field="created_on",
+        field_type="date",
+    )
+    exited_on = GenericElasticSearchFilter(
+        source_field="exited_on",
+        field_type="date",
+    )
+    ended_at = GenericElasticSearchFilter(
+        source_field="exited_on",
+        field_type="date",
+    )
+    project = GenericElasticSearchFilter(
+        source_field="project_uuid",
+        field_type="string",
+    )
+    flow = GenericElasticSearchFilter(
+        source_field="flow_uuid",
+        field_type="string",
+    )
+
+    def get_field(self, field_name):
+        try:
+            return getattr(self, field_name)
+        except AttributeError:
+            return None

insights/sources/flowruns/query_builder.py

@@ -1,4 +1,4 @@
-class FlowRunsElasticSearchQueryBuilder:
+class FlowRunElasticSearchQueryBuilder:
     def __init__(self):
         self.query_clauses = {}
         self.is_valid = False

insights/sources/flowruns/usecases/query_execute.py

@@ -1,5 +1,12 @@
 from insights.db.elasticsearch.connection import Connection
-from insights.sources.flowruns.clients import FlowRunsElasticSearchClient
+from insights.sources.filter_strategies import ElasticSearchFilterStrategy
+from insights.sources.flowruns.clients import (
+    FlowRunElasticSearchQueryGenerator,
+)
+from insights.sources.flowruns.filtersets import FlowRunFilterSet
+from insights.sources.flowruns.query_builder import (
+    FlowRunElasticSearchQueryBuilder,
+)
 
 
 def transform_terms_count_to_percentage(
@@ -21,16 +28,19 @@ def execute(
         filters: dict,
         operation: str,
         parser: callable,
-        project: object,
         query_kwargs: dict = {},
         *args,
         **kwargs,
     ) -> dict:
-        filters["project"] = str(project.uuid)
-        client = FlowRunsElasticSearchClient()
-        endpoint, params = client.execute(
-            filters=filters, query_type=operation, query_kwargs=query_kwargs
+        query_generator = FlowRunElasticSearchQueryGenerator(
+            filter_strategy=ElasticSearchFilterStrategy,
+            query_builder=FlowRunElasticSearchQueryBuilder,
+            filterset=FlowRunFilterSet,
+            filters=filters,
+            query_type=operation,
+            query_kwargs=query_kwargs,
         )
+        endpoint, params = query_generator.generate()
         response = Connection(endpoint).get(params=params)
 
         if operation == "recurrence":
@@ -42,7 +52,7 @@ def execute(
                 others=terms_agg.get("agg_value", {}).get("sum_other_doc_count", 0),
                 terms_agg_buckets=terms_agg.get("agg_value", {}).get("buckets", []),
             )
-            if len(transformed_terms) <= 1:
+            if len(transformed_terms) == 1:
                 return transformed_terms[0]
             return {
                 "results": transformed_terms,