From d084b36e51fc99fb3b27809dc21cddf477c27c8e Mon Sep 17 00:00:00 2001 From: Helder Souza <42891390+helllllllder@users.noreply.github.com> Date: Thu, 18 Jul 2024 15:58:19 -0300 Subject: [PATCH] Feature/add filterset class (#24) * feat: add filterset class to room source, to clean the fields and return every datail of the field on a single class * feat: update agents, flows, queues, sectors, rooms, tags sources with the new filterset pattern * feat: update rooms and flowruns * feat: fix flows query executor * feat: fix imports * feat: minor fixes * feat: minor fixes * feat: minor fixes * feat: remove comments --- insights/projects/viewsets.py | 3 +- insights/sources/agents/clients.py | 29 ++----- insights/sources/agents/filtersets.py | 17 ++++ .../sources/agents/usecases/query_execute.py | 17 +++- insights/sources/clients.py | 87 +++++++++++++++++++ .../{filters.py => filter_strategies.py} | 0 insights/sources/filtersets.py | 25 ++++++ insights/sources/flowruns/clients.py | 36 +------- insights/sources/flowruns/filtersets.py | 32 +++++++ insights/sources/flowruns/query_builder.py | 2 +- .../flowruns/usecases/query_execute.py | 24 +++-- insights/sources/flows/clients.py | 45 +--------- insights/sources/flows/filtersets.py | 18 ++++ .../sources/flows/usecases/query_execute.py | 21 +++-- insights/sources/queues/clients.py | 25 +----- insights/sources/queues/filtersets.py | 29 +++++++ insights/sources/queues/query_builder.py | 8 +- .../sources/queues/usecases/query_execute.py | 24 ++++- insights/sources/rooms/clients.py | 61 +------------ insights/sources/rooms/filtersets.py | 71 +++++++++++++++ insights/sources/rooms/query_builder.py | 16 ++-- .../sources/rooms/usecases/query_execute.py | 20 ++++- insights/sources/sectors/clients.py | 25 +----- insights/sources/sectors/filtersets.py | 21 +++++ .../sources/sectors/usecases/query_execute.py | 18 +++- insights/sources/tags/clients.py | 25 +----- insights/sources/tags/filtersets.py | 24 +++++ insights/sources/tags/query_builder.py | 8 +- .../sources/tags/usecases/query_execute.py | 24 ++++- insights/sources/tests/rooms/test_filters.py | 2 +- .../sources/tests/rooms/test_query_builder.py | 2 +- insights/widgets/usecases/get_source_data.py | 3 +- 32 files changed, 496 insertions(+), 266 deletions(-) create mode 100644 insights/sources/agents/filtersets.py create mode 100644 insights/sources/clients.py rename insights/sources/{filters.py => filter_strategies.py} (100%) create mode 100644 insights/sources/filtersets.py create mode 100644 insights/sources/flowruns/filtersets.py create mode 100644 insights/sources/flows/filtersets.py create mode 100644 insights/sources/queues/filtersets.py create mode 100644 insights/sources/rooms/filtersets.py create mode 100644 insights/sources/sectors/filtersets.py create mode 100644 insights/sources/tags/filtersets.py diff --git a/insights/projects/viewsets.py b/insights/projects/viewsets.py index d0ba058..723caa1 100644 --- a/insights/projects/viewsets.py +++ b/insights/projects/viewsets.py @@ -32,12 +32,11 @@ def retrieve_source_data(self, request, source_slug=None, *args, **kwargs): op_field = filters.pop("op_field", [None])[0] if op_field: query_kwargs["op_field"] = op_field - + filters["project"] = str(self.get_object().uuid) serialized_source = SourceQuery.execute( filters=filters, operation=operation, parser=parse_dict_to_json, - project=self.get_object(), user_email=self.request.user.email, return_format="select_input", query_kwargs=query_kwargs, diff --git a/insights/sources/agents/clients.py b/insights/sources/agents/clients.py index 7604f5c..194b939 100644 --- a/insights/sources/agents/clients.py +++ b/insights/sources/agents/clients.py @@ -2,8 +2,11 @@ from django.conf import settings from insights.internals.base import InternalAuthentication -from insights.sources.agents.query_builder import AgentSQLQueryBuilder -from insights.sources.filters import PostgreSQLFilterStrategy +from insights.sources.clients import GenericSQLQueryGenerator + + +class AgentSQLQueryGenerator(GenericSQLQueryGenerator): + default_query_type = "list" class AgentsRESTClient(InternalAuthentication): @@ -23,25 +26,3 @@ def list(self, query_filters: dict): url=self.url, headers=self.headers, params=query_filters ) return response.json() - - -def generate_sql_query( - filters: dict, - query_type: str = "list", - query_kwargs: dict = {}, -): - strategy = PostgreSQLFilterStrategy() - builder = AgentSQLQueryBuilder() - - for key, value in filters.items(): - table_alias = "pp" - if "__" in key: - field, operation = key.split("__", 1) - elif type(value) is list: - field = key.split("__", 1)[0] - operation = "in" - else: - field, operation = key, "eq" - builder.add_filter(strategy, field, operation, value, table_alias) - builder.build_query() - return getattr(builder, query_type)(**query_kwargs) diff --git a/insights/sources/agents/filtersets.py b/insights/sources/agents/filtersets.py new file mode 100644 index 0000000..cc150cd --- /dev/null +++ b/insights/sources/agents/filtersets.py @@ -0,0 +1,17 @@ +# use stub files to represent it on other parts of the code +# Use django_filters Filter class as a reference +from insights.sources.filtersets import GenericSQLFilter + + +class AgentFilterSet: + project = GenericSQLFilter( + source_field="project_id", + table_alias="pp", + ) + project_id = project + + def get_field(self, field_name): + try: + return getattr(self, field_name) + except AttributeError: + return None diff --git a/insights/sources/agents/usecases/query_execute.py b/insights/sources/agents/usecases/query_execute.py index 9e08e17..a6fbf66 100644 --- a/insights/sources/agents/usecases/query_execute.py +++ b/insights/sources/agents/usecases/query_execute.py @@ -1,8 +1,11 @@ from insights.db.postgres.django.connection import dictfetchall, get_cursor from insights.sources.agents.clients import ( + AgentSQLQueryGenerator, AgentsRESTClient, - generate_sql_query, ) +from insights.sources.agents.filtersets import AgentFilterSet +from insights.sources.agents.query_builder import AgentSQLQueryBuilder +from insights.sources.filter_strategies import PostgreSQLFilterStrategy class QueryExecutor: @@ -13,12 +16,20 @@ def execute( return_format: str = None, project: object = None, user_email: str = None, + query_kwargs: dict = {}, *args, **kwargs ): if return_format == "select_input" or operation != "list": - filters["project_id"] = str(project.uuid) - query, params = generate_sql_query(filters=filters, query_type=operation) + query_generator = AgentSQLQueryGenerator( + filter_strategy=PostgreSQLFilterStrategy, + query_builder=AgentSQLQueryBuilder, + filterset=AgentFilterSet, + filters=filters, + query_type=operation, + query_kwargs=query_kwargs, + ) + query, params = query_generator.generate() with get_cursor(db_name="chats") as cur: query_exec = cur.execute(query, params) query_results = dictfetchall(query_exec) diff --git a/insights/sources/clients.py b/insights/sources/clients.py new file mode 100644 index 0000000..68af322 --- /dev/null +++ b/insights/sources/clients.py @@ -0,0 +1,87 @@ +class GenericSQLQueryGenerator: + default_query_type = "count" + + def __init__( + self, + filter_strategy, + query_builder, + filterset, + filters: dict, + query_type: str = "", + query_kwargs: dict = {}, + ) -> None: + self.filter_strategy = filter_strategy + self.query_builder = query_builder + self.filterset = filterset + self.filters = filters + self.query_type = query_type or self.default_query_type + self.query_kwargs = query_kwargs + + def generate(self): + strategy = self.filter_strategy() + builder = self.query_builder() + filterset = self.filterset() + + for key, value in self.filters.items(): + if "__" in key: + field, operation = key.split("__", 1) + elif type(value) is list: + field = key.split("__", 1)[0] + operation = "in" + else: + field, operation = key, "eq" + field_object = filterset.get_field(field) + if field_object is None: + continue + source_field = field_object.source_field + join_clause = field_object.join_clause + if join_clause != {}: + builder.add_joins(join_clause) + builder.add_filter( + strategy, source_field, operation, value, field_object.table_alias + ) + builder.build_query() + + return getattr(builder, self.query_type)(**self.query_kwargs) + + +class GenericElasticSearchQueryGenerator: + default_query_type = "count" + + def __init__( + self, + filter_strategy, + query_builder, + filterset, + filters: dict, + query_type: str = "", + query_kwargs: dict = {}, + ) -> None: + self.filter_strategy = filter_strategy + self.query_builder = query_builder + self.filterset = filterset + self.filters = filters + self.query_type = query_type or self.default_query_type + self.query_kwargs = query_kwargs + + def generate(self): + strategy = self.filter_strategy() + builder = self.query_builder() + filterset = self.filterset() + + for key, value in self.filters.items(): + if "__" in key: + field, operation = key.split("__", 1) + elif type(value) is list: + field = key.split("__", 1)[0] + operation = "in" + else: + field, operation = key, "eq" + field_object = filterset.get_field(field) + if field_object is None: + continue + source_field = field_object.source_field + builder.add_filter(strategy, source_field, operation, value) + builder.build_query() + + return getattr(builder, self.query_type)(**self.query_kwargs) diff --git a/insights/sources/filters.py b/insights/sources/filter_strategies.py similarity index 100% rename from insights/sources/filters.py rename to insights/sources/filter_strategies.py diff --git a/insights/sources/filtersets.py b/insights/sources/filtersets.py new file mode 100644 index 0000000..bcd4342 --- /dev/null +++ b/insights/sources/filtersets.py @@ -0,0 +1,25 @@ +class GenericSQLFilter: + """Responsible for cleaning and validating Filter data""" + + def __init__( + self, + source_field: str, + table_alias: str, + join_clause: dict = {}, + value: any = None, + ) -> None: + self.source_field = source_field + self.table_alias = table_alias + self.join_clause = join_clause + + +class GenericElasticSearchFilter: + """Responsible for cleaning and validating Filter data""" + + def __init__( + self, + source_field: str, + field_type: str, + ) -> None: + self.source_field = source_field + self.field_type = field_type diff --git a/insights/sources/flowruns/clients.py b/insights/sources/flowruns/clients.py index 09b058d..ccd0aff 100644 --- a/insights/sources/flowruns/clients.py +++ b/insights/sources/flowruns/clients.py @@ -1,7 +1,4 @@ -from insights.sources.filters import ElasticSearchFilterStrategy -from insights.sources.flowruns.query_builder import ( - FlowRunsElasticSearchQueryBuilder, -) +from insights.sources.clients import GenericElasticSearchQueryGenerator flow_runs_filters = { "created_on": {"type": "date", "to_field": "created_on"}, @@ -12,32 +9,5 @@ } -class FlowRunsElasticSearchClient: - def execute( - self, - filters: dict, - query_type: str = "count", - query_kwargs: dict = {}, - ): - strategy = ElasticSearchFilterStrategy() - builder = FlowRunsElasticSearchQueryBuilder() - - for key, value in filters.items(): - if "__" in key: - field, operation = key.split("__", 1) - elif type(value) is list: - field = key.split("__", 1)[0] - operation = "in" - else: - field, operation = key, "eq" - - if ( - field in flow_runs_filters - ): # only consider filters describred in the flow_runs_filters dict. TODO: maybe transform this dict into a class similar to django-filters filterset classes - field = flow_runs_filters[field]["to_field"] - else: - continue - builder.add_filter(strategy, field, operation, value) - builder.build_query() - - return getattr(builder, query_type)(**query_kwargs) +class FlowRunElasticSearchQueryGenerator(GenericElasticSearchQueryGenerator): + default_query_type = "count" diff --git a/insights/sources/flowruns/filtersets.py b/insights/sources/flowruns/filtersets.py new file mode 100644 index 0000000..af9729b --- /dev/null +++ b/insights/sources/flowruns/filtersets.py @@ -0,0 +1,32 @@ +# use stub files to represent it on other parts of the code +# Use django_filters Filter class as a reference +from insights.sources.filtersets import GenericElasticSearchFilter + + +class FlowRunFilterSet: + created_on = GenericElasticSearchFilter( + source_field="created_on", + field_type="date", + ) + exited_on = GenericElasticSearchFilter( + source_field="exited_on", + field_type="date", + ) + ended_at = GenericElasticSearchFilter( + source_field="exited_on", + field_type="date", + ) + project = GenericElasticSearchFilter( + source_field="project_uuid", + field_type="string", + ) + flow = GenericElasticSearchFilter( + source_field="flow_uuid", + field_type="string", + ) + + def get_field(self, field_name): + try: + return getattr(self, field_name) + except AttributeError: + return None diff --git a/insights/sources/flowruns/query_builder.py b/insights/sources/flowruns/query_builder.py index 37c0ba0..80e8a5b 100644 --- a/insights/sources/flowruns/query_builder.py +++ b/insights/sources/flowruns/query_builder.py @@ -1,4 +1,4 @@ -class FlowRunsElasticSearchQueryBuilder: +class FlowRunElasticSearchQueryBuilder: def __init__(self): self.query_clauses = {} self.is_valid = False diff --git a/insights/sources/flowruns/usecases/query_execute.py b/insights/sources/flowruns/usecases/query_execute.py index dae638e..b396ba1 100644 --- a/insights/sources/flowruns/usecases/query_execute.py +++ b/insights/sources/flowruns/usecases/query_execute.py @@ -1,5 +1,12 @@ from insights.db.elasticsearch.connection import Connection -from insights.sources.flowruns.clients import FlowRunsElasticSearchClient +from insights.sources.filter_strategies import ElasticSearchFilterStrategy +from insights.sources.flowruns.clients import ( + FlowRunElasticSearchQueryGenerator, +) +from insights.sources.flowruns.filtersets import FlowRunFilterSet +from insights.sources.flowruns.query_builder import ( + FlowRunElasticSearchQueryBuilder, +) def transform_terms_count_to_percentage( @@ -21,16 +28,19 @@ def execute( filters: dict, operation: str, parser: callable, - project: object, query_kwargs: dict = {}, *args, **kwargs, ) -> dict: - filters["project"] = str(project.uuid) - client = FlowRunsElasticSearchClient() - endpoint, params = client.execute( - filters=filters, query_type=operation, query_kwargs=query_kwargs + query_generator = FlowRunElasticSearchQueryGenerator( + filter_strategy=ElasticSearchFilterStrategy, + query_builder=FlowRunElasticSearchQueryBuilder, + filterset=FlowRunFilterSet, + filters=filters, + query_type=operation, + query_kwargs=query_kwargs, ) + endpoint, params = query_generator.generate() response = Connection(endpoint).get(params=params) if operation == "recurrence": @@ -42,7 +52,7 @@ def execute( others=terms_agg.get("agg_value", {}).get("sum_other_doc_count", 0), terms_agg_buckets=terms_agg.get("agg_value", {}).get("buckets", []), ) - if len(transformed_terms) <= 1: + if len(transformed_terms) == 1: return transformed_terms[0] return { "results": transformed_terms, diff --git a/insights/sources/flows/clients.py b/insights/sources/flows/clients.py index b55381f..5e91b8a 100644 --- a/insights/sources/flows/clients.py +++ b/insights/sources/flows/clients.py @@ -1,44 +1,5 @@ -from insights.sources.filters import PostgreSQLFilterStrategy -from insights.sources.flows.query_builder import FlowSQLQueryBuilder +from insights.sources.clients import GenericSQLQueryGenerator -relation_schema = { - "project": {"field_name": "proj_uuid", "table_alias": "o"}, -} - -def get_joins_from_schema(field): - joins = dict() - if "project" == field: - joins["o"] = "INNER JOIN public.orgs_org AS o ON o.id=f.org_id" - - return joins - - -def generate_sql_query( - filters: dict, - schema: dict = relation_schema, - query_type: str = "count", - query_kwargs: dict = {}, -): - strategy = PostgreSQLFilterStrategy() - builder = FlowSQLQueryBuilder() - - for key, value in filters.items(): - table_alias = "f" - if "__" in key: - field, operation = key.split("__", 1) - elif type(value) is list: - field = key.split("__", 1)[0] - operation = "in" - else: - field, operation = key, "eq" - - if field in schema: - f_schema = schema[field] - builder.add_joins(get_joins_from_schema(field)) - field = f_schema["field_name"] - table_alias = f_schema["table_alias"] - builder.add_filter(strategy, field, operation, value, table_alias) - builder.build_query() - - return getattr(builder, query_type)(**query_kwargs) +class FlowSQLQueryGenerator(GenericSQLQueryGenerator): + default_query_type = "list" diff --git a/insights/sources/flows/filtersets.py b/insights/sources/flows/filtersets.py new file mode 100644 index 0000000..684d649 --- /dev/null +++ b/insights/sources/flows/filtersets.py @@ -0,0 +1,18 @@ +# use stub files to represent it on other parts of the code +# Use django_filters Filter class as a reference +from insights.sources.filtersets import GenericSQLFilter + + +class FlowFilterSet: + project = GenericSQLFilter( + source_field="proj_uuid", + table_alias="o", + join_clause={"o": "INNER JOIN public.orgs_org AS o ON o.id=f.org_id"}, + ) + project_id = project + + def get_field(self, field_name): + try: + return getattr(self, field_name) + except AttributeError: + return None diff --git a/insights/sources/flows/usecases/query_execute.py b/insights/sources/flows/usecases/query_execute.py index d1e8d6f..3c4041a 100644 --- a/insights/sources/flows/usecases/query_execute.py +++ b/insights/sources/flows/usecases/query_execute.py @@ -1,5 +1,8 @@ from insights.db.postgres.psycopg.connection import get_cursor -from insights.sources.flows.clients import generate_sql_query +from insights.sources.filter_strategies import PostgreSQLFilterStrategy +from insights.sources.flows.clients import FlowSQLQueryGenerator +from insights.sources.flows.filtersets import FlowFilterSet +from insights.sources.flows.query_builder import FlowSQLQueryBuilder class QueryExecutor: @@ -7,13 +10,21 @@ def execute( filters: dict, operation: str, parser: callable, - project: object, + query_kwargs: dict = {}, *args, **kwargs ): - filters["project"] = str(project.uuid) - query, params = generate_sql_query(filters=filters, query_type=operation) + query_generator = FlowSQLQueryGenerator( + filter_strategy=PostgreSQLFilterStrategy, + query_builder=FlowSQLQueryBuilder, + filterset=FlowFilterSet, + filters=filters, + query_type=operation, + query_kwargs=query_kwargs, + ) + query, params = query_generator.generate() + with get_cursor(db_name="flows") as cur: query_results = cur.execute(query, params).fetchall() paginated_results = {"next": None, "previous": None, "results": query_results} - return paginated_results # parser(paginated_results) + return paginated_results diff --git a/insights/sources/queues/clients.py b/insights/sources/queues/clients.py index 85c0d3f..5b892ea 100644 --- a/insights/sources/queues/clients.py +++ b/insights/sources/queues/clients.py @@ -1,24 +1,5 @@ -from insights.sources.filters import PostgreSQLFilterStrategy -from insights.sources.queues.query_builder import QueueSQLQueryBuilder +from insights.sources.clients import GenericSQLQueryGenerator -def generate_sql_query( - filters: dict, - query_type: str = "list", - query_kwargs: dict = {}, -): - strategy = PostgreSQLFilterStrategy() - builder = QueueSQLQueryBuilder() - - for key, value in filters.items(): - table_alias = "q" - if "__" in key: - field, operation = key.split("__", 1) - elif type(value) is list: - field = key.split("__", 1)[0] - operation = "in" - else: - field, operation = key, "eq" - builder.add_filter(strategy, field, operation, value, table_alias) - builder.build_query() - return getattr(builder, query_type)(**query_kwargs) +class QueueSQLQueryGenerator(GenericSQLQueryGenerator): + default_query_type = "list" diff --git a/insights/sources/queues/filtersets.py b/insights/sources/queues/filtersets.py new file mode 100644 index 0000000..eb5a21b --- /dev/null +++ b/insights/sources/queues/filtersets.py @@ -0,0 +1,29 @@ +# use stub files to represent it on other parts of the code +# Use django_filters Filter class as a reference +from insights.sources.filtersets import GenericSQLFilter + + +class QueueFilterSet: + project = GenericSQLFilter( + source_field="project_id", + table_alias="s", + join_clause={ + "s": "INNER JOIN public.sectors_sector AS s ON s.uuid=q.sector_id", + }, + ) + project_id = project + sector = GenericSQLFilter( + source_field="sector_id", + table_alias="q", + ) + sector_id = sector + uuid = GenericSQLFilter( + source_field="uuid", + table_alias="q", + ) + + def get_field(self, field_name): + try: + return getattr(self, field_name) + except AttributeError: + return None diff --git a/insights/sources/queues/query_builder.py b/insights/sources/queues/query_builder.py index 223a36f..c27244a 100644 --- a/insights/sources/queues/query_builder.py +++ b/insights/sources/queues/query_builder.py @@ -1,5 +1,6 @@ class QueueSQLQueryBuilder: def __init__(self): + self.joins = dict() self.where_clauses = [] self.params = [] self.is_valid = False @@ -10,13 +11,18 @@ def add_filter(self, strategy, field, operation, value, table_alias: str = "q"): self.where_clauses.append(clause) self.params.extend(params) + def add_joins(self, joins: set): + self.joins.update(joins) + def build_query(self): self.where_clause = " AND ".join(self.where_clauses) + self.join_clause = " ".join(self.joins.values()) + self.is_valid = True def list(self): if not self.is_valid: self.build_query() - query = f"SELECT q.uuid,q.name FROM public.queues_queue AS q WHERE {self.where_clause};" + query = f"SELECT q.uuid,q.name FROM public.queues_queue AS q {self.join_clause} WHERE {self.where_clause};" return query, self.params diff --git a/insights/sources/queues/usecases/query_execute.py b/insights/sources/queues/usecases/query_execute.py index abcd133..701392a 100644 --- a/insights/sources/queues/usecases/query_execute.py +++ b/insights/sources/queues/usecases/query_execute.py @@ -1,10 +1,28 @@ from insights.db.postgres.django.connection import dictfetchall, get_cursor -from insights.sources.queues.clients import generate_sql_query +from insights.sources.filter_strategies import PostgreSQLFilterStrategy +from insights.sources.queues.clients import QueueSQLQueryGenerator +from insights.sources.queues.filtersets import QueueFilterSet +from insights.sources.queues.query_builder import QueueSQLQueryBuilder class QueryExecutor: - def execute(filters: dict, operation: str, parser: callable, *args, **kwargs): - query, params = generate_sql_query(filters=filters, query_type=operation) + def execute( + filters: dict, + operation: str, + parser: callable, + query_kwargs: dict = {}, + *args, + **kwargs + ): + query_generator = QueueSQLQueryGenerator( + filter_strategy=PostgreSQLFilterStrategy, + query_builder=QueueSQLQueryBuilder, + filterset=QueueFilterSet, + filters=filters, + query_type=operation, + query_kwargs=query_kwargs, + ) + query, params = query_generator.generate() with get_cursor(db_name="chats") as cur: query_exec = cur.execute(query, params) query_results = dictfetchall(query_exec) diff --git a/insights/sources/rooms/clients.py b/insights/sources/rooms/clients.py index 847aca8..a8473a8 100644 --- a/insights/sources/rooms/clients.py +++ b/insights/sources/rooms/clients.py @@ -2,66 +2,11 @@ from django.conf import settings from insights.internals.base import InternalAuthentication -from insights.sources.filters import PostgreSQLFilterStrategy -from insights.sources.rooms.query_builder import RoomSQLQueryBuilder +from insights.sources.clients import GenericSQLQueryGenerator -relation_schema = { - "agent": {"field_name": "user_id", "table_alias": "r"}, - "project": {"field_name": "uuid", "table_alias": "p"}, - "tag": {"field_name": "sectortag_id", "table_alias": "tg"}, - "tags": {"field_name": "sectortag_id", "table_alias": "tg"}, - "sector": {"field_name": "sector_id", "table_alias": "q"}, - "sector_id": {"field_name": "sector_id", "table_alias": "q"}, - "queue": {"field_name": "queue_id", "table_alias": "r"}, - "contact": {"field_name": "uuid", "table_alias": "ctt"}, -} - -def get_joins_from_schema(field): - joins = dict() - if "sector" == field or "project" == field: - joins["q"] = "INNER JOIN public.queues_queue AS q ON q.uuid=r.queue_id" - if "project" == field: - joins["s"] = "INNER JOIN public.sectors_sector AS s ON s.uuid=q.sector_id" - joins["p"] = "INNER JOIN public.projects_project AS p ON p.uuid=s.project_id" - if "tag" == field or "tags" == field: - joins["tg"] = "INNER JOIN public.rooms_room_tags AS tg ON tg.room_id=r.uuid" - if "contact" == field: - joins["ctt"] = ( - "INNER JOIN public.contacts_contact AS ctt on ctt.uuid=r.contact_id" - ) - - return joins - - -def generate_sql_query( - filters: dict, - schema: dict = relation_schema, - query_type: str = "count", - query_kwargs: dict = {}, -): - strategy = PostgreSQLFilterStrategy() - builder = RoomSQLQueryBuilder() - - for key, value in filters.items(): - table_alias = "r" - if "__" in key: - field, operation = key.split("__", 1) - elif type(value) is list: - field = key.split("__", 1)[0] - operation = "in" - else: - field, operation = key, "eq" - - if field in schema: - f_schema = schema[field] - builder.add_joins(get_joins_from_schema(field)) - field = f_schema["field_name"] if field != "contact" else "contact" - table_alias = f_schema["table_alias"] - builder.add_filter(strategy, field, operation, value, table_alias) - builder.build_query() - - return getattr(builder, query_type)(**query_kwargs) +class RoomSQLQueryGenerator(GenericSQLQueryGenerator): + default_query_type = "count" class RoomRESTClient(InternalAuthentication): diff --git a/insights/sources/rooms/filtersets.py b/insights/sources/rooms/filtersets.py new file mode 100644 index 0000000..a8123c5 --- /dev/null +++ b/insights/sources/rooms/filtersets.py @@ -0,0 +1,71 @@ +from insights.sources.filtersets import GenericSQLFilter + + +# use stub files to represent it on other parts of the code +# Use django_filters Filter class as a reference + + +class RoomFilterSet: + agent = GenericSQLFilter( + source_field="user_id", + table_alias="r", + ) + project = GenericSQLFilter( + source_field="uuid", + table_alias="p", + join_clause={ + "q": "INNER JOIN public.queues_queue AS q ON q.uuid=r.queue_id", + "s": "INNER JOIN public.sectors_sector AS s ON s.uuid=q.sector_id", + "p": "INNER JOIN public.projects_project AS p ON p.uuid=s.project_id", + }, + ) + tag = GenericSQLFilter( + source_field="sectortag_id", + table_alias="tg", + join_clause={ + "tg": "INNER JOIN public.rooms_room_tags AS tg ON tg.room_id=r.uuid" + }, + ) + tags = tag + sector = GenericSQLFilter( + source_field="sector_id", + table_alias="q", + join_clause={ + "q": "INNER JOIN public.queues_queue AS q ON q.uuid=r.queue_id", + }, + ) + sector_id = sector + queue = GenericSQLFilter( + source_field="queue_id", + table_alias="r", + ) + contact = GenericSQLFilter( + source_field="uuid", + table_alias="ctt", + join_clause={ + "q": "INNER JOIN public.contacts_contact AS ctt on ctt.uuid=r.contact_id", + }, + ) + created_on = GenericSQLFilter( + source_field="created_on", + table_alias="r", + ) + ended_at = GenericSQLFilter( + source_field="ended_at", + table_alias="r", + ) + user = GenericSQLFilter( + source_field="user_id", + table_alias="r", + ) + user_id = user + is_active = GenericSQLFilter( + source_field="is_active", + table_alias="r", + ) + + def get_field(self, field_name): + try: + return getattr(self, field_name) + except AttributeError: + return None diff --git a/insights/sources/rooms/query_builder.py b/insights/sources/rooms/query_builder.py index 27717cb..6a01608 100644 --- a/insights/sources/rooms/query_builder.py +++ b/insights/sources/rooms/query_builder.py @@ -6,8 +6,12 @@ def __init__(self): self.is_valid = False def add_filter(self, strategy, field, operation, value, table_alias: str = "r"): - if field == "contact": - field = {"name": "ctt", "external_id": "ctt", "urn": "r"} + if table_alias == "ctt" and field == "uuid": + field = { + "name": "ctt", + "external_id": "ctt", + "urn": "r", + } # {field_name: table_alias} operation = "or" clause, params = strategy.apply(field, operation, value, table_alias) @@ -24,28 +28,28 @@ def build_query(self): self.is_valid = True def timeseries_hour_group_count( - self, time_field: str = "created_on", limit: int = 24 + self, time_field: str = "created_on", limit: int = 24, *args, **kwargs ): if not self.is_valid: self.build_query() query = f"WITH hourly_data AS (SELECT EXTRACT(HOUR FROM r.{time_field}) AS hour, COUNT(*) AS rooms_count FROM public.rooms_room as r {self.join_clause} WHERE {self.where_clause} GROUP BY hour) SELECT CONCAT(hours.label, 'h') AS label, COALESCE(hourly_data.rooms_count, 0) AS value FROM generate_series(0, 23) AS hours(label) LEFT JOIN hourly_data ON hours.label = hourly_data.hour ORDER BY value DESC FETCH FIRST {limit} ROWS ONLY;" return query, self.params - def count(self): + def count(self, *args, **kwargs): if not self.is_valid: self.build_query() query = f"SELECT COUNT(r.*) AS value FROM public.rooms_room as r {self.join_clause} WHERE {self.where_clause};" return query, self.params - def sum(self, op_field: str): + def sum(self, op_field: str, *args, **kwargs): if not self.is_valid: self.build_query() query = f"SELECT SUM(mr.{op_field}) AS value FROM public.rooms_room as r INNER JOIN public.dashboard_roommetrics AS mr ON mr.room_id=r.uuid {self.join_clause} WHERE {self.where_clause};" return query, self.params - def avg(self, op_field: str): + def avg(self, op_field: str, *args, **kwargs): if not self.is_valid: self.build_query() query = f"SELECT (ROUND(COALESCE(AVG(mr.{op_field}), 0)/60, 2)) AS value FROM public.rooms_room as r INNER JOIN public.dashboard_roommetrics AS mr ON mr.room_id=r.uuid {self.join_clause} WHERE {self.where_clause};" diff --git a/insights/sources/rooms/usecases/query_execute.py b/insights/sources/rooms/usecases/query_execute.py index 1399e06..f2f760f 100644 --- a/insights/sources/rooms/usecases/query_execute.py +++ b/insights/sources/rooms/usecases/query_execute.py @@ -3,7 +3,13 @@ dictfetchone, get_cursor, ) -from insights.sources.rooms.clients import RoomRESTClient, generate_sql_query +from insights.sources.filter_strategies import PostgreSQLFilterStrategy +from insights.sources.rooms.clients import ( + RoomRESTClient, + RoomSQLQueryGenerator, +) +from insights.sources.rooms.filtersets import RoomFilterSet +from insights.sources.rooms.query_builder import RoomSQLQueryBuilder class QueryExecutor: @@ -31,10 +37,16 @@ def execute( "results": query_results.get("results", []), } return paginated_results # parser(paginated_results) - filters["project"] = str(project.uuid) - query, params = generate_sql_query( - filters=filters, query_type=operation, query_kwargs=query_kwargs + + query_generator = RoomSQLQueryGenerator( + filter_strategy=PostgreSQLFilterStrategy, + query_builder=RoomSQLQueryBuilder, + filterset=RoomFilterSet, + filters=filters, + query_type=operation, + query_kwargs=query_kwargs, ) + query, params = query_generator.generate() with get_cursor(db_name="chats") as cur: query_exec = cur.execute(query, params) if operation in ["count", "avg"]: diff --git a/insights/sources/sectors/clients.py b/insights/sources/sectors/clients.py index 1281553..db57ba5 100644 --- a/insights/sources/sectors/clients.py +++ b/insights/sources/sectors/clients.py @@ -1,24 +1,5 @@ -from insights.sources.filters import PostgreSQLFilterStrategy -from insights.sources.sectors.query_builder import SectorSQLQueryBuilder +from insights.sources.clients import GenericSQLQueryGenerator -def generate_sql_query( - filters: dict, - query_type: str = "list", - query_kwargs: dict = {}, -): - strategy = PostgreSQLFilterStrategy() - builder = SectorSQLQueryBuilder() - - for key, value in filters.items(): - table_alias = "s" - if "__" in key: - field, operation = key.split("__", 1) - elif type(value) is list: - field = key.split("__", 1)[0] - operation = "in" - else: - field, operation = key, "eq" - builder.add_filter(strategy, field, operation, value, table_alias) - builder.build_query() - return getattr(builder, query_type)(**query_kwargs) +class SectorSQLQueryGenerator(GenericSQLQueryGenerator): + default_query_type = "list" diff --git a/insights/sources/sectors/filtersets.py b/insights/sources/sectors/filtersets.py new file mode 100644 index 0000000..9c6d009 --- /dev/null +++ b/insights/sources/sectors/filtersets.py @@ -0,0 +1,21 @@ +# use stub files to represent it on other parts of the code +# Use django_filters Filter class as a reference +from insights.sources.filtersets import GenericSQLFilter + + +class SectorFilterSet: + project = GenericSQLFilter( + source_field="project_id", + table_alias="s", + ) + project_id = project + uuid = GenericSQLFilter( + source_field="uuid", + table_alias="s", + ) + + def get_field(self, field_name): + try: + return getattr(self, field_name) + except AttributeError: + return None diff --git a/insights/sources/sectors/usecases/query_execute.py b/insights/sources/sectors/usecases/query_execute.py index 62530d6..83cb358 100644 --- a/insights/sources/sectors/usecases/query_execute.py +++ b/insights/sources/sectors/usecases/query_execute.py @@ -1,5 +1,8 @@ from insights.db.postgres.django.connection import dictfetchall, get_cursor -from insights.sources.sectors.clients import generate_sql_query +from insights.sources.filter_strategies import PostgreSQLFilterStrategy +from insights.sources.sectors.clients import SectorSQLQueryGenerator +from insights.sources.sectors.filtersets import SectorFilterSet +from insights.sources.sectors.query_builder import SectorSQLQueryBuilder class QueryExecutor: @@ -7,12 +10,19 @@ def execute( filters: dict, operation: str, parser: callable, - project: object, + query_kwargs: dict = {}, *args, **kwargs ): - filters["project_id"] = str(project.uuid) - query, params = generate_sql_query(filters=filters, query_type=operation) + query_generator = SectorSQLQueryGenerator( + filter_strategy=PostgreSQLFilterStrategy, + query_builder=SectorSQLQueryBuilder, + filterset=SectorFilterSet, + filters=filters, + query_type=operation, + query_kwargs=query_kwargs, + ) + query, params = query_generator.generate() with get_cursor(db_name="chats") as cur: query_exec = cur.execute(query, params) query_results = dictfetchall(query_exec) diff --git a/insights/sources/tags/clients.py b/insights/sources/tags/clients.py index e93ea39..7c146e6 100644 --- a/insights/sources/tags/clients.py +++ b/insights/sources/tags/clients.py @@ -1,24 +1,5 @@ -from insights.sources.filters import PostgreSQLFilterStrategy -from insights.sources.tags.query_builder import TagSQLQueryBuilder +from insights.sources.clients import GenericSQLQueryGenerator -def generate_sql_query( - filters: dict, - query_type: str = "list", - query_kwargs: dict = {}, -): - strategy = PostgreSQLFilterStrategy() - builder = TagSQLQueryBuilder() - - for key, value in filters.items(): - table_alias = "tg" - if "__" in key: - field, operation = key.split("__", 1) - elif type(value) is list: - field = key.split("__", 1)[0] - operation = "in" - else: - field, operation = key, "eq" - builder.add_filter(strategy, field, operation, value, table_alias) - builder.build_query() - return getattr(builder, query_type)(**query_kwargs) +class TagSQLQueryGenerator(GenericSQLQueryGenerator): + default_query_type = "list" diff --git a/insights/sources/tags/filtersets.py b/insights/sources/tags/filtersets.py new file mode 100644 index 0000000..ac234e4 --- /dev/null +++ b/insights/sources/tags/filtersets.py @@ -0,0 +1,24 @@ +# use stub files to represent it on other parts of the code +# Use django_filters Filter class as a reference +from insights.sources.filtersets import GenericSQLFilter + + +class TagFilterSet: + project = GenericSQLFilter( + source_field="project_id", + table_alias="s", + join_clause={ + "s": "INNER JOIN public.sectors_sector AS s ON s.uuid=tg.sector_id", + }, + ) + sector = GenericSQLFilter( + source_field="sector_id", + table_alias="tg", + ) + sector_id = sector + + def get_field(self, field_name): + try: + return getattr(self, field_name) + except AttributeError: + return None diff --git a/insights/sources/tags/query_builder.py b/insights/sources/tags/query_builder.py index 54b9507..1df7983 100644 --- a/insights/sources/tags/query_builder.py +++ b/insights/sources/tags/query_builder.py @@ -1,5 +1,6 @@ class TagSQLQueryBuilder: def __init__(self): + self.joins = dict() self.where_clauses = [] self.params = [] self.is_valid = False @@ -10,13 +11,18 @@ def add_filter(self, strategy, field, operation, value, table_alias: str = "tg") self.where_clauses.append(clause) self.params.extend(params) + def add_joins(self, joins: set): + self.joins.update(joins) + def build_query(self): self.where_clause = " AND ".join(self.where_clauses) + self.join_clause = " ".join(self.joins.values()) + self.is_valid = True def list(self): if not self.is_valid: self.build_query() - query = f"SELECT tg.uuid,tg.name FROM public.sectors_sectortag AS tg WHERE {self.where_clause};" + query = f"SELECT tg.uuid,tg.name FROM public.sectors_sectortag AS tg {self.join_clause} WHERE {self.where_clause};" return query, self.params diff --git a/insights/sources/tags/usecases/query_execute.py b/insights/sources/tags/usecases/query_execute.py index 3c3c76c..9f9368c 100644 --- a/insights/sources/tags/usecases/query_execute.py +++ b/insights/sources/tags/usecases/query_execute.py @@ -1,10 +1,28 @@ from insights.db.postgres.django.connection import dictfetchall, get_cursor -from insights.sources.tags.clients import generate_sql_query +from insights.sources.filter_strategies import PostgreSQLFilterStrategy +from insights.sources.tags.clients import TagSQLQueryGenerator +from insights.sources.tags.filtersets import TagFilterSet +from insights.sources.tags.query_builder import TagSQLQueryBuilder class QueryExecutor: - def execute(filters: dict, operation: str, parser: callable, *args, **kwargs): - query, params = generate_sql_query(filters=filters, query_type=operation) + def execute( + filters: dict, + operation: str, + parser: callable, + query_kwargs: dict = {}, + *args, + **kwargs + ): + query_generator = TagSQLQueryGenerator( + filter_strategy=PostgreSQLFilterStrategy, + query_builder=TagSQLQueryBuilder, + filterset=TagFilterSet, + filters=filters, + query_type=operation, + query_kwargs=query_kwargs, + ) + query, params = query_generator.generate() with get_cursor(db_name="chats") as cur: query_exec = cur.execute(query, params) query_results = dictfetchall(query_exec) diff --git a/insights/sources/tests/rooms/test_filters.py b/insights/sources/tests/rooms/test_filters.py index e9d16cd..a2bb4e7 100644 --- a/insights/sources/tests/rooms/test_filters.py +++ b/insights/sources/tests/rooms/test_filters.py @@ -2,7 +2,7 @@ import pytest -from insights.sources.filters import PostgreSQLFilterStrategy +from insights.sources.filter_strategies import PostgreSQLFilterStrategy @pytest.fixture diff --git a/insights/sources/tests/rooms/test_query_builder.py b/insights/sources/tests/rooms/test_query_builder.py index 20e24ae..f83a064 100644 --- a/insights/sources/tests/rooms/test_query_builder.py +++ b/insights/sources/tests/rooms/test_query_builder.py @@ -1,6 +1,6 @@ import pytest -from insights.sources.filters import PostgreSQLFilterStrategy +from insights.sources.filter_strategies import PostgreSQLFilterStrategy from insights.sources.rooms.query_builder import RoomSQLQueryBuilder diff --git a/insights/widgets/usecases/get_source_data.py b/insights/widgets/usecases/get_source_data.py index 5b1f3dd..0fe4359 100644 --- a/insights/widgets/usecases/get_source_data.py +++ b/insights/widgets/usecases/get_source_data.py @@ -1,4 +1,4 @@ -from datetime import datetime, time +from datetime import datetime import pytz @@ -50,6 +50,7 @@ def get_source_data_from_widget( if limit: query_kwargs["limit"] = limit + default_filters["project"] = str(widget.project.uuid) serialized_source = SourceQuery.execute( filters=default_filters, operation=operation,