From c5da65daafb086ed3c890e957f99ae83a8f0fe4b Mon Sep 17 00:00:00 2001 From: Luke Couzens Date: Mon, 12 Aug 2024 11:49:17 +0100 Subject: [PATCH 1/7] Include provider in OCP/GCP sql params --- koku/masu/database/gcp_report_db_accessor.py | 3 ++- .../sql/gcp/openshift/reporting_ocpgcptags_summary.sql | 1 + .../processor/ocp/ocp_cloud_parquet_summary_updater.py | 2 +- koku/masu/processor/tasks.py | 8 ++++---- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/koku/masu/database/gcp_report_db_accessor.py b/koku/masu/database/gcp_report_db_accessor.py index 3bb6bb052a..93e4a7148a 100644 --- a/koku/masu/database/gcp_report_db_accessor.py +++ b/koku/masu/database/gcp_report_db_accessor.py @@ -475,13 +475,14 @@ def get_openshift_on_cloud_matched_tags_trino( ) return [json.loads(result[0]) for result in results] - def populate_ocp_on_gcp_tag_information(self, gcp_bill_ids, start_date, end_date): + def populate_ocp_on_gcp_tag_information(self, gcp_bill_ids, start_date, end_date, ocp_uuid): """Populate the line item aggregated totals data table.""" sql_params = { "schema": self.schema, "gcp_bill_ids": gcp_bill_ids, "start_date": start_date, "end_date": end_date, + "source_uuid": ocp_uuid, } # Tag Summary sql = pkgutil.get_data("masu.database", "sql/gcp/openshift/reporting_ocpgcptags_summary.sql") diff --git a/koku/masu/database/sql/gcp/openshift/reporting_ocpgcptags_summary.sql b/koku/masu/database/sql/gcp/openshift/reporting_ocpgcptags_summary.sql index fb4a6c8533..b51ead8db4 100644 --- a/koku/masu/database/sql/gcp/openshift/reporting_ocpgcptags_summary.sql +++ b/koku/masu/database/sql/gcp/openshift/reporting_ocpgcptags_summary.sql @@ -12,6 +12,7 @@ WITH cte_tag_value AS ( jsonb_each_text(li.tags) labels WHERE li.usage_start >= {{start_date}} AND li.usage_start <= {{end_date}} + AND li.source_uuid = {{source_uuid}} AND value IS NOT NULL AND li.tags ?| (SELECT array_agg(DISTINCT key) FROM {{schema | sqlsafe}}.reporting_enabledtagkeys WHERE enabled=true AND provider_type = 'GCP') {% if bill_ids %} diff --git a/koku/masu/processor/ocp/ocp_cloud_parquet_summary_updater.py b/koku/masu/processor/ocp/ocp_cloud_parquet_summary_updater.py index a6fcd9d4b4..416910b675 100644 --- a/koku/masu/processor/ocp/ocp_cloud_parquet_summary_updater.py +++ b/koku/masu/processor/ocp/ocp_cloud_parquet_summary_updater.py @@ -555,7 +555,7 @@ def update_gcp_summary_tables(self, openshift_provider_uuid, gcp_provider_uuid, accessor.populate_ocp_on_gcp_ui_summary_tables_trino( start, end, openshift_provider_uuid, gcp_provider_uuid ) - accessor.populate_ocp_on_gcp_tag_information(gcp_bill_ids, start, end) + accessor.populate_ocp_on_gcp_tag_information(gcp_bill_ids, start, end, openshift_provider_uuid) with OCPReportDBAccessor(self._schema) as ocp_accessor: sql_params["source_type"] = "GCP" diff --git a/koku/masu/processor/tasks.py b/koku/masu/processor/tasks.py index abb427ec91..2b196a2bd0 100644 --- a/koku/masu/processor/tasks.py +++ b/koku/masu/processor/tasks.py @@ -513,7 +513,7 @@ def update_summary_tables( # noqa: C901 msg = f"Task {task_name} already running for {cache_args}. Requeuing." if rate_limited: msg = f"Schema {schema} is currently rate limited. Requeuing." - LOG.debug(log_json(tracing_id, msg=msg)) + LOG.info(log_json(tracing_id, msg=msg)) update_summary_tables.s( schema, provider_type, @@ -723,8 +723,6 @@ def update_openshift_on_cloud( # noqa: C901 """Update OpenShift on Cloud for a specific OpenShift and cloud source.""" # Get latest manifest id for running OCP provider ocp_manifest_id = get_latest_openshift_on_cloud_manifest(start_date, openshift_provider_uuid) - # Set OpenShift summary started time - set_summary_timestamp(ManifestState.START, ocp_manifest_id) task_name = "masu.processor.tasks.update_openshift_on_cloud" if is_ocp_on_cloud_summary_disabled(schema_name): msg = f"OCP on Cloud summary disabled for {schema_name}." @@ -748,7 +746,7 @@ def update_openshift_on_cloud( # noqa: C901 msg = f"Task {task_name} already running for {cache_args}. Requeuing." if rate_limited: msg = f"Schema {schema_name} is currently rate limited. Requeuing." - LOG.debug(log_json(tracing_id, msg=msg)) + LOG.info(log_json(tracing_id, msg=msg)) update_openshift_on_cloud.s( schema_name, openshift_provider_uuid, @@ -764,6 +762,8 @@ def update_openshift_on_cloud( # noqa: C901 return worker_cache.lock_single_task(task_name, cache_args, timeout=timeout) + # Set OpenShift summary started time + set_summary_timestamp(ManifestState.START, ocp_manifest_id) ctx = { "schema": schema_name, "ocp_provider_uuid": openshift_provider_uuid, From 8c1e9aadd1d42ecc39eaaef3d42b305f84ca65fe Mon Sep 17 00:00:00 2001 From: Luke Couzens Date: Mon, 12 Aug 2024 13:37:50 +0100 Subject: [PATCH 2/7] switch to cluster_id --- koku/api/report/test/util/model_bakery_loader.py | 2 +- koku/masu/database/gcp_report_db_accessor.py | 4 ++-- .../sql/gcp/openshift/reporting_ocpgcptags_summary.sql | 2 +- .../masu/processor/ocp/ocp_cloud_parquet_summary_updater.py | 2 +- koku/masu/test/database/test_gcp_report_db_accessor.py | 6 ++++-- 5 files changed, 9 insertions(+), 7 deletions(-) diff --git a/koku/api/report/test/util/model_bakery_loader.py b/koku/api/report/test/util/model_bakery_loader.py index 2bd5f1a5b5..e2b7be0bb7 100644 --- a/koku/api/report/test/util/model_bakery_loader.py +++ b/koku/api/report/test/util/model_bakery_loader.py @@ -528,7 +528,7 @@ def load_openshift_on_cloud_data(self, provider_type, cluster_id, bills, report_ with dbaccessor(self.schema) as accessor: # update tags cls_method = getattr(accessor, tags_update_method) - cls_method([bill.id for bill in bills], self.first_start_date, self.last_end_date) + cls_method([bill.id for bill in bills], self.first_start_date, self.last_end_date, cluster_id) # update ui tables sql_params = { diff --git a/koku/masu/database/gcp_report_db_accessor.py b/koku/masu/database/gcp_report_db_accessor.py index 93e4a7148a..381fc2505c 100644 --- a/koku/masu/database/gcp_report_db_accessor.py +++ b/koku/masu/database/gcp_report_db_accessor.py @@ -475,14 +475,14 @@ def get_openshift_on_cloud_matched_tags_trino( ) return [json.loads(result[0]) for result in results] - def populate_ocp_on_gcp_tag_information(self, gcp_bill_ids, start_date, end_date, ocp_uuid): + def populate_ocp_on_gcp_tag_information(self, gcp_bill_ids, start_date, end_date, cluster_id): """Populate the line item aggregated totals data table.""" sql_params = { "schema": self.schema, "gcp_bill_ids": gcp_bill_ids, "start_date": start_date, "end_date": end_date, - "source_uuid": ocp_uuid, + "cluster_id": cluster_id, } # Tag Summary sql = pkgutil.get_data("masu.database", "sql/gcp/openshift/reporting_ocpgcptags_summary.sql") diff --git a/koku/masu/database/sql/gcp/openshift/reporting_ocpgcptags_summary.sql b/koku/masu/database/sql/gcp/openshift/reporting_ocpgcptags_summary.sql index b51ead8db4..31c8a4d835 100644 --- a/koku/masu/database/sql/gcp/openshift/reporting_ocpgcptags_summary.sql +++ b/koku/masu/database/sql/gcp/openshift/reporting_ocpgcptags_summary.sql @@ -12,7 +12,7 @@ WITH cte_tag_value AS ( jsonb_each_text(li.tags) labels WHERE li.usage_start >= {{start_date}} AND li.usage_start <= {{end_date}} - AND li.source_uuid = {{source_uuid}} + AND li.cluster_id = {{cluster_id}} AND value IS NOT NULL AND li.tags ?| (SELECT array_agg(DISTINCT key) FROM {{schema | sqlsafe}}.reporting_enabledtagkeys WHERE enabled=true AND provider_type = 'GCP') {% if bill_ids %} diff --git a/koku/masu/processor/ocp/ocp_cloud_parquet_summary_updater.py b/koku/masu/processor/ocp/ocp_cloud_parquet_summary_updater.py index 416910b675..d54a08d83d 100644 --- a/koku/masu/processor/ocp/ocp_cloud_parquet_summary_updater.py +++ b/koku/masu/processor/ocp/ocp_cloud_parquet_summary_updater.py @@ -555,7 +555,7 @@ def update_gcp_summary_tables(self, openshift_provider_uuid, gcp_provider_uuid, accessor.populate_ocp_on_gcp_ui_summary_tables_trino( start, end, openshift_provider_uuid, gcp_provider_uuid ) - accessor.populate_ocp_on_gcp_tag_information(gcp_bill_ids, start, end, openshift_provider_uuid) + accessor.populate_ocp_on_gcp_tag_information(gcp_bill_ids, start, end, cluster_id) with OCPReportDBAccessor(self._schema) as ocp_accessor: sql_params["source_type"] = "GCP" diff --git a/koku/masu/test/database/test_gcp_report_db_accessor.py b/koku/masu/test/database/test_gcp_report_db_accessor.py index 7efd5de7a2..1888307cd2 100644 --- a/koku/masu/test/database/test_gcp_report_db_accessor.py +++ b/koku/masu/test/database/test_gcp_report_db_accessor.py @@ -395,7 +395,7 @@ def test_populate_ocp_on_gcp_tag_information_assert_call(self, mock_trino): end_date = self.dh.this_month_end.date() mock_gcp_bills = [Mock(), Mock()] - self.accessor.populate_ocp_on_gcp_tag_information(mock_gcp_bills, start_date, end_date) + self.accessor.populate_ocp_on_gcp_tag_information(mock_gcp_bills, start_date, end_date, self.ocp_cluster_id) mock_trino.assert_called() @patch("masu.database.gcp_report_db_accessor.GCPReportDBAccessor.schema_exists_trino") @@ -514,7 +514,9 @@ def test_populate_ocp_on_gcp_tag_information(self, mock_unleash): parent_key, parent_obj, parent_count = populated_keys[0] child_key, child_obj, child_count = populated_keys[1] TagMapping.objects.create(parent=parent_obj, child=child_obj) - self.accessor.populate_ocp_on_gcp_tag_information(bill_ids, self.dh.this_month_start, self.dh.today) + self.accessor.populate_ocp_on_gcp_tag_information( + bill_ids, self.dh.this_month_start, self.dh.today, self.ocp_cluster_id + ) expected_parent_count = parent_count + child_count actual_parent_count = OCPGCPCostLineItemProjectDailySummaryP.objects.filter( tags__has_key=parent_key, usage_start__gte=self.dh.this_month_start, usage_start__lte=self.dh.today From e0052eaef232f883ee620951463967b24743150b Mon Sep 17 00:00:00 2001 From: Luke Couzens Date: Tue, 13 Aug 2024 09:39:55 +0100 Subject: [PATCH 3/7] Update model_bakery_loader.py --- koku/api/report/test/util/model_bakery_loader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/koku/api/report/test/util/model_bakery_loader.py b/koku/api/report/test/util/model_bakery_loader.py index e2b7be0bb7..2bd5f1a5b5 100644 --- a/koku/api/report/test/util/model_bakery_loader.py +++ b/koku/api/report/test/util/model_bakery_loader.py @@ -528,7 +528,7 @@ def load_openshift_on_cloud_data(self, provider_type, cluster_id, bills, report_ with dbaccessor(self.schema) as accessor: # update tags cls_method = getattr(accessor, tags_update_method) - cls_method([bill.id for bill in bills], self.first_start_date, self.last_end_date, cluster_id) + cls_method([bill.id for bill in bills], self.first_start_date, self.last_end_date) # update ui tables sql_params = { From 88a972f5747ade326991de9c7a528f40fe1284b2 Mon Sep 17 00:00:00 2001 From: Luke Couzens Date: Tue, 13 Aug 2024 11:38:51 +0100 Subject: [PATCH 4/7] update all providers --- koku/api/report/test/util/model_bakery_loader.py | 2 +- koku/masu/database/aws_report_db_accessor.py | 10 ++++++++-- koku/masu/database/azure_report_db_accessor.py | 10 ++++++++-- .../masu/database/sql/reporting_ocpawstags_summary.sql | 1 + .../database/sql/reporting_ocpazuretags_summary.sql | 1 + .../processor/ocp/ocp_cloud_parquet_summary_updater.py | 4 ++-- koku/masu/test/database/test_aws_report_db_accessor.py | 4 +++- .../test/database/test_azure_report_db_accessor.py | 4 +++- .../test_ocp_cloud_parquet_report_summary_updater.py | 4 ++-- 9 files changed, 29 insertions(+), 11 deletions(-) diff --git a/koku/api/report/test/util/model_bakery_loader.py b/koku/api/report/test/util/model_bakery_loader.py index 2bd5f1a5b5..e2b7be0bb7 100644 --- a/koku/api/report/test/util/model_bakery_loader.py +++ b/koku/api/report/test/util/model_bakery_loader.py @@ -528,7 +528,7 @@ def load_openshift_on_cloud_data(self, provider_type, cluster_id, bills, report_ with dbaccessor(self.schema) as accessor: # update tags cls_method = getattr(accessor, tags_update_method) - cls_method([bill.id for bill in bills], self.first_start_date, self.last_end_date) + cls_method([bill.id for bill in bills], self.first_start_date, self.last_end_date, cluster_id) # update ui tables sql_params = { diff --git a/koku/masu/database/aws_report_db_accessor.py b/koku/masu/database/aws_report_db_accessor.py index cac1c5f3e2..b234d4071b 100644 --- a/koku/masu/database/aws_report_db_accessor.py +++ b/koku/masu/database/aws_report_db_accessor.py @@ -275,9 +275,15 @@ def back_populate_ocp_infrastructure_costs(self, start_date, end_date, report_pe } self._prepare_and_execute_raw_sql_query(table_name, sql, sql_params) - def populate_ocp_on_aws_tag_information(self, bill_ids, start_date, end_date): + def populate_ocp_on_aws_tag_information(self, bill_ids, start_date, end_date, cluster_id): """Populate the line item aggregated totals data table.""" - sql_params = {"schema": self.schema, "bill_ids": bill_ids, "start_date": start_date, "end_date": end_date} + sql_params = { + "schema": self.schema, + "bill_ids": bill_ids, + "start_date": start_date, + "end_date": end_date, + "cluster_id": cluster_id, + } # Tag Summary sql = pkgutil.get_data("masu.database", "sql/reporting_ocpawstags_summary.sql") sql = sql.decode("utf-8") diff --git a/koku/masu/database/azure_report_db_accessor.py b/koku/masu/database/azure_report_db_accessor.py index 6e930665ca..013e922fa8 100644 --- a/koku/masu/database/azure_report_db_accessor.py +++ b/koku/masu/database/azure_report_db_accessor.py @@ -152,9 +152,15 @@ def get_bill_query_before_date(self, date): """Get the cost entry bill objects with billing period before provided date.""" return AzureCostEntryBill.objects.filter(billing_period_start__lte=date) - def populate_ocp_on_azure_tag_information(self, bill_ids, start_date, end_date): + def populate_ocp_on_azure_tag_information(self, bill_ids, start_date, end_date, cluster_id): """Populate the line item aggregated totals data table.""" - sql_params = {"schema": self.schema, "bill_ids": bill_ids, "start_date": start_date, "end_date": end_date} + sql_params = { + "schema": self.schema, + "bill_ids": bill_ids, + "start_date": start_date, + "end_date": end_date, + "cluster_id": cluster_id, + } # Tag summary sql = pkgutil.get_data("masu.database", "sql/reporting_ocpazuretags_summary.sql") sql = sql.decode("utf-8") diff --git a/koku/masu/database/sql/reporting_ocpawstags_summary.sql b/koku/masu/database/sql/reporting_ocpawstags_summary.sql index 0201cb322c..76ef7f6132 100644 --- a/koku/masu/database/sql/reporting_ocpawstags_summary.sql +++ b/koku/masu/database/sql/reporting_ocpawstags_summary.sql @@ -11,6 +11,7 @@ WITH cte_tag_value AS ( jsonb_each_text(li.tags) labels WHERE li.usage_start >= {{start_date}} AND li.usage_start <= {{end_date}} + AND li.cluster_id = {{cluster_id}} AND li.tags ?| (SELECT array_agg(DISTINCT key) FROM {{schema | sqlsafe}}.reporting_enabledtagkeys WHERE enabled=true AND provider_type='AWS') {% if bill_ids %} AND li.cost_entry_bill_id IN {{ bill_ids | inclause }} diff --git a/koku/masu/database/sql/reporting_ocpazuretags_summary.sql b/koku/masu/database/sql/reporting_ocpazuretags_summary.sql index a614c424d6..d343efeb06 100644 --- a/koku/masu/database/sql/reporting_ocpazuretags_summary.sql +++ b/koku/masu/database/sql/reporting_ocpazuretags_summary.sql @@ -10,6 +10,7 @@ WITH cte_tag_value AS ( jsonb_each_text(li.tags) labels WHERE li.usage_start >= {{start_date}} AND li.usage_start <= {{end_date}} + AND li.cluster_id = {{cluster_id}} AND li.tags ?| (SELECT array_agg(DISTINCT key) FROM {{schema | sqlsafe}}.reporting_enabledtagkeys WHERE enabled=true AND provider_type='Azure') {% if bill_ids %} AND li.cost_entry_bill_id IN {{ bill_ids | inclause }} diff --git a/koku/masu/processor/ocp/ocp_cloud_parquet_summary_updater.py b/koku/masu/processor/ocp/ocp_cloud_parquet_summary_updater.py index d54a08d83d..1c09af6059 100644 --- a/koku/masu/processor/ocp/ocp_cloud_parquet_summary_updater.py +++ b/koku/masu/processor/ocp/ocp_cloud_parquet_summary_updater.py @@ -299,7 +299,7 @@ def update_aws_summary_tables(self, openshift_provider_uuid, aws_provider_uuid, sql_params["start_date"] = start sql_params["end_date"] = end accessor.back_populate_ocp_infrastructure_costs(start, end, current_ocp_report_period_id) - accessor.populate_ocp_on_aws_tag_information(aws_bill_ids, start, end) + accessor.populate_ocp_on_aws_tag_information(aws_bill_ids, start, end, cluster_id) accessor.populate_ocp_on_aws_ui_summary_tables_trino( start, end, openshift_provider_uuid, aws_provider_uuid ) @@ -434,7 +434,7 @@ def update_azure_summary_tables(self, openshift_provider_uuid, azure_provider_uu sql_params["start_date"] = start sql_params["end_date"] = end accessor.back_populate_ocp_infrastructure_costs(start, end, current_ocp_report_period_id) - accessor.populate_ocp_on_azure_tag_information(azure_bill_ids, start, end) + accessor.populate_ocp_on_azure_tag_information(azure_bill_ids, start, end, cluster_id) accessor.populate_ocp_on_azure_ui_summary_tables_trino( start, end, openshift_provider_uuid, azure_provider_uuid ) diff --git a/koku/masu/test/database/test_aws_report_db_accessor.py b/koku/masu/test/database/test_aws_report_db_accessor.py index 26d854ce74..803442bd99 100644 --- a/koku/masu/test/database/test_aws_report_db_accessor.py +++ b/koku/masu/test/database/test_aws_report_db_accessor.py @@ -446,7 +446,9 @@ def test_populate_ocp_on_aws_tag_information(self, mock_unleash): parent_key, parent_obj, parent_count = populated_keys[0] child_key, child_obj, child_count = populated_keys[1] TagMapping.objects.create(parent=parent_obj, child=child_obj) - self.accessor.populate_ocp_on_aws_tag_information(bill_ids, self.dh.this_month_start, self.dh.today) + self.accessor.populate_ocp_on_aws_tag_information( + bill_ids, self.dh.this_month_start, self.dh.today, self.cluster_id + ) expected_parent_count = parent_count + child_count actual_parent_count = OCPAWSCostLineItemProjectDailySummaryP.objects.filter( tags__has_key=parent_key, usage_start__gte=self.dh.this_month_start, usage_start__lte=self.dh.today diff --git a/koku/masu/test/database/test_azure_report_db_accessor.py b/koku/masu/test/database/test_azure_report_db_accessor.py index c1313134f9..8a7687a8b0 100644 --- a/koku/masu/test/database/test_azure_report_db_accessor.py +++ b/koku/masu/test/database/test_azure_report_db_accessor.py @@ -369,7 +369,9 @@ def test_populate_ocp_on_aZURE_tag_information(self, mock_unleash): parent_key, parent_obj, parent_count = populated_keys[0] child_key, child_obj, child_count = populated_keys[1] TagMapping.objects.create(parent=parent_obj, child=child_obj) - self.accessor.populate_ocp_on_azure_tag_information(bill_ids, self.dh.this_month_start, self.dh.today) + self.accessor.populate_ocp_on_azure_tag_information( + bill_ids, self.dh.this_month_start, self.dh.today, self.ocp_cluster_id + ) expected_parent_count = parent_count + child_count actual_parent_count = OCPAzureCostLineItemProjectDailySummaryP.objects.filter( tags__has_key=parent_key, usage_start__gte=self.dh.this_month_start, usage_start__lte=self.dh.today diff --git a/koku/masu/test/processor/ocp/test_ocp_cloud_parquet_report_summary_updater.py b/koku/masu/test/processor/ocp/test_ocp_cloud_parquet_report_summary_updater.py index 77cdf4ccb4..e937905f3f 100644 --- a/koku/masu/test/processor/ocp/test_ocp_cloud_parquet_report_summary_updater.py +++ b/koku/masu/test/processor/ocp/test_ocp_cloud_parquet_report_summary_updater.py @@ -307,7 +307,7 @@ def test_update_gcp_summary_tables( mock_ui_summary.assert_called_with( start_date, end_date, self.ocpgcp_provider_uuid, self.gcp_test_provider_uuid ) - mock_tag_summary.assert_called_with([1, 2], start_date, end_date) + mock_tag_summary.assert_called_with([1, 2], start_date, end_date, cluster_id) @patch( "masu.processor.ocp.ocp_cloud_parquet_summary_updater.OCPReportDBAccessor.populate_ocp_on_all_ui_summary_tables" # noqa: E501 @@ -379,7 +379,7 @@ def test_update_gcp_summary_tables_with_string_dates( start_date, end_date, self.ocpgcp_provider_uuid, self.gcp_test_provider_uuid ) - mock_tag_summary.assert_called_with([1, 2], start_date, end_date) + mock_tag_summary.assert_called_with([1, 2], start_date, end_date, cluster_id) @patch("masu.processor.ocp.ocp_cloud_parquet_summary_updater.OCPReportDBAccessor.get_cluster_for_provider") @patch( From 145281c8478248eea87f13092c67e105225d8c7f Mon Sep 17 00:00:00 2001 From: Luke Couzens Date: Sat, 17 Aug 2024 12:20:40 +0100 Subject: [PATCH 5/7] switch from cluster id to report bill id --- koku/masu/database/aws_report_db_accessor.py | 4 ++-- koku/masu/database/azure_report_db_accessor.py | 4 ++-- koku/masu/database/gcp_report_db_accessor.py | 4 ++-- .../sql/gcp/openshift/reporting_ocpgcptags_summary.sql | 2 +- koku/masu/database/sql/reporting_ocpawstags_summary.sql | 2 +- koku/masu/database/sql/reporting_ocpazuretags_summary.sql | 2 +- .../processor/ocp/ocp_cloud_parquet_summary_updater.py | 8 +++++--- koku/masu/test/database/test_aws_report_db_accessor.py | 3 ++- koku/masu/test/database/test_azure_report_db_accessor.py | 3 ++- koku/masu/test/database/test_gcp_report_db_accessor.py | 6 ++++-- 10 files changed, 22 insertions(+), 16 deletions(-) diff --git a/koku/masu/database/aws_report_db_accessor.py b/koku/masu/database/aws_report_db_accessor.py index b234d4071b..19f83bc685 100644 --- a/koku/masu/database/aws_report_db_accessor.py +++ b/koku/masu/database/aws_report_db_accessor.py @@ -275,14 +275,14 @@ def back_populate_ocp_infrastructure_costs(self, start_date, end_date, report_pe } self._prepare_and_execute_raw_sql_query(table_name, sql, sql_params) - def populate_ocp_on_aws_tag_information(self, bill_ids, start_date, end_date, cluster_id): + def populate_ocp_on_aws_tag_information(self, bill_ids, start_date, end_date, current_ocp_report_period_id): """Populate the line item aggregated totals data table.""" sql_params = { "schema": self.schema, "bill_ids": bill_ids, "start_date": start_date, "end_date": end_date, - "cluster_id": cluster_id, + "report_period_id": current_ocp_report_period_id, } # Tag Summary sql = pkgutil.get_data("masu.database", "sql/reporting_ocpawstags_summary.sql") diff --git a/koku/masu/database/azure_report_db_accessor.py b/koku/masu/database/azure_report_db_accessor.py index 013e922fa8..3eaa4f493c 100644 --- a/koku/masu/database/azure_report_db_accessor.py +++ b/koku/masu/database/azure_report_db_accessor.py @@ -152,14 +152,14 @@ def get_bill_query_before_date(self, date): """Get the cost entry bill objects with billing period before provided date.""" return AzureCostEntryBill.objects.filter(billing_period_start__lte=date) - def populate_ocp_on_azure_tag_information(self, bill_ids, start_date, end_date, cluster_id): + def populate_ocp_on_azure_tag_information(self, bill_ids, start_date, end_date, report_period_id): """Populate the line item aggregated totals data table.""" sql_params = { "schema": self.schema, "bill_ids": bill_ids, "start_date": start_date, "end_date": end_date, - "cluster_id": cluster_id, + "report_period_id": report_period_id, } # Tag summary sql = pkgutil.get_data("masu.database", "sql/reporting_ocpazuretags_summary.sql") diff --git a/koku/masu/database/gcp_report_db_accessor.py b/koku/masu/database/gcp_report_db_accessor.py index 381fc2505c..5d17f18ace 100644 --- a/koku/masu/database/gcp_report_db_accessor.py +++ b/koku/masu/database/gcp_report_db_accessor.py @@ -475,14 +475,14 @@ def get_openshift_on_cloud_matched_tags_trino( ) return [json.loads(result[0]) for result in results] - def populate_ocp_on_gcp_tag_information(self, gcp_bill_ids, start_date, end_date, cluster_id): + def populate_ocp_on_gcp_tag_information(self, gcp_bill_ids, start_date, end_date, report_period_id): """Populate the line item aggregated totals data table.""" sql_params = { "schema": self.schema, "gcp_bill_ids": gcp_bill_ids, "start_date": start_date, "end_date": end_date, - "cluster_id": cluster_id, + "report_period_id": report_period_id, } # Tag Summary sql = pkgutil.get_data("masu.database", "sql/gcp/openshift/reporting_ocpgcptags_summary.sql") diff --git a/koku/masu/database/sql/gcp/openshift/reporting_ocpgcptags_summary.sql b/koku/masu/database/sql/gcp/openshift/reporting_ocpgcptags_summary.sql index 31c8a4d835..526c13a4b9 100644 --- a/koku/masu/database/sql/gcp/openshift/reporting_ocpgcptags_summary.sql +++ b/koku/masu/database/sql/gcp/openshift/reporting_ocpgcptags_summary.sql @@ -12,7 +12,7 @@ WITH cte_tag_value AS ( jsonb_each_text(li.tags) labels WHERE li.usage_start >= {{start_date}} AND li.usage_start <= {{end_date}} - AND li.cluster_id = {{cluster_id}} + AND li.report_period_id = {{report_period_id}} AND value IS NOT NULL AND li.tags ?| (SELECT array_agg(DISTINCT key) FROM {{schema | sqlsafe}}.reporting_enabledtagkeys WHERE enabled=true AND provider_type = 'GCP') {% if bill_ids %} diff --git a/koku/masu/database/sql/reporting_ocpawstags_summary.sql b/koku/masu/database/sql/reporting_ocpawstags_summary.sql index 76ef7f6132..0d105bfebe 100644 --- a/koku/masu/database/sql/reporting_ocpawstags_summary.sql +++ b/koku/masu/database/sql/reporting_ocpawstags_summary.sql @@ -11,7 +11,7 @@ WITH cte_tag_value AS ( jsonb_each_text(li.tags) labels WHERE li.usage_start >= {{start_date}} AND li.usage_start <= {{end_date}} - AND li.cluster_id = {{cluster_id}} + AND li.report_period_id = {{report_period_id}} AND li.tags ?| (SELECT array_agg(DISTINCT key) FROM {{schema | sqlsafe}}.reporting_enabledtagkeys WHERE enabled=true AND provider_type='AWS') {% if bill_ids %} AND li.cost_entry_bill_id IN {{ bill_ids | inclause }} diff --git a/koku/masu/database/sql/reporting_ocpazuretags_summary.sql b/koku/masu/database/sql/reporting_ocpazuretags_summary.sql index d343efeb06..c78fba95e3 100644 --- a/koku/masu/database/sql/reporting_ocpazuretags_summary.sql +++ b/koku/masu/database/sql/reporting_ocpazuretags_summary.sql @@ -10,7 +10,7 @@ WITH cte_tag_value AS ( jsonb_each_text(li.tags) labels WHERE li.usage_start >= {{start_date}} AND li.usage_start <= {{end_date}} - AND li.cluster_id = {{cluster_id}} + AND li.report_period_id = {{report_period_id}} AND li.tags ?| (SELECT array_agg(DISTINCT key) FROM {{schema | sqlsafe}}.reporting_enabledtagkeys WHERE enabled=true AND provider_type='Azure') {% if bill_ids %} AND li.cost_entry_bill_id IN {{ bill_ids | inclause }} diff --git a/koku/masu/processor/ocp/ocp_cloud_parquet_summary_updater.py b/koku/masu/processor/ocp/ocp_cloud_parquet_summary_updater.py index 1c09af6059..7eb52f224c 100644 --- a/koku/masu/processor/ocp/ocp_cloud_parquet_summary_updater.py +++ b/koku/masu/processor/ocp/ocp_cloud_parquet_summary_updater.py @@ -299,7 +299,7 @@ def update_aws_summary_tables(self, openshift_provider_uuid, aws_provider_uuid, sql_params["start_date"] = start sql_params["end_date"] = end accessor.back_populate_ocp_infrastructure_costs(start, end, current_ocp_report_period_id) - accessor.populate_ocp_on_aws_tag_information(aws_bill_ids, start, end, cluster_id) + accessor.populate_ocp_on_aws_tag_information(aws_bill_ids, start, end, current_ocp_report_period_id) accessor.populate_ocp_on_aws_ui_summary_tables_trino( start, end, openshift_provider_uuid, aws_provider_uuid ) @@ -434,7 +434,9 @@ def update_azure_summary_tables(self, openshift_provider_uuid, azure_provider_uu sql_params["start_date"] = start sql_params["end_date"] = end accessor.back_populate_ocp_infrastructure_costs(start, end, current_ocp_report_period_id) - accessor.populate_ocp_on_azure_tag_information(azure_bill_ids, start, end, cluster_id) + accessor.populate_ocp_on_azure_tag_information( + azure_bill_ids, start, end, current_ocp_report_period_id + ) accessor.populate_ocp_on_azure_ui_summary_tables_trino( start, end, openshift_provider_uuid, azure_provider_uuid ) @@ -555,7 +557,7 @@ def update_gcp_summary_tables(self, openshift_provider_uuid, gcp_provider_uuid, accessor.populate_ocp_on_gcp_ui_summary_tables_trino( start, end, openshift_provider_uuid, gcp_provider_uuid ) - accessor.populate_ocp_on_gcp_tag_information(gcp_bill_ids, start, end, cluster_id) + accessor.populate_ocp_on_gcp_tag_information(gcp_bill_ids, start, end, current_ocp_report_period_id) with OCPReportDBAccessor(self._schema) as ocp_accessor: sql_params["source_type"] = "GCP" diff --git a/koku/masu/test/database/test_aws_report_db_accessor.py b/koku/masu/test/database/test_aws_report_db_accessor.py index 803442bd99..2ce03b09fb 100644 --- a/koku/masu/test/database/test_aws_report_db_accessor.py +++ b/koku/masu/test/database/test_aws_report_db_accessor.py @@ -425,6 +425,7 @@ def test_populate_ocp_on_aws_tag_information(self, mock_unleash): """ mock_unleash.return_value = True populated_keys = [] + report_period_id = 1 with schema_context(self.schema): enabled_tags = EnabledTagKeys.objects.filter(provider_type=Provider.PROVIDER_AWS, enabled=True) for enabled_tag in enabled_tags: @@ -447,7 +448,7 @@ def test_populate_ocp_on_aws_tag_information(self, mock_unleash): child_key, child_obj, child_count = populated_keys[1] TagMapping.objects.create(parent=parent_obj, child=child_obj) self.accessor.populate_ocp_on_aws_tag_information( - bill_ids, self.dh.this_month_start, self.dh.today, self.cluster_id + bill_ids, self.dh.this_month_start, self.dh.today, report_period_id ) expected_parent_count = parent_count + child_count actual_parent_count = OCPAWSCostLineItemProjectDailySummaryP.objects.filter( diff --git a/koku/masu/test/database/test_azure_report_db_accessor.py b/koku/masu/test/database/test_azure_report_db_accessor.py index 8a7687a8b0..ca29931807 100644 --- a/koku/masu/test/database/test_azure_report_db_accessor.py +++ b/koku/masu/test/database/test_azure_report_db_accessor.py @@ -348,6 +348,7 @@ def test_populate_ocp_on_aZURE_tag_information(self, mock_unleash): """ mock_unleash.return_value = True populated_keys = [] + report_period_id = 1 with schema_context(self.schema): enabled_tags = EnabledTagKeys.objects.filter(provider_type=Provider.PROVIDER_AZURE, enabled=True) for enabled_tag in enabled_tags: @@ -370,7 +371,7 @@ def test_populate_ocp_on_aZURE_tag_information(self, mock_unleash): child_key, child_obj, child_count = populated_keys[1] TagMapping.objects.create(parent=parent_obj, child=child_obj) self.accessor.populate_ocp_on_azure_tag_information( - bill_ids, self.dh.this_month_start, self.dh.today, self.ocp_cluster_id + bill_ids, self.dh.this_month_start, self.dh.today, report_period_id ) expected_parent_count = parent_count + child_count actual_parent_count = OCPAzureCostLineItemProjectDailySummaryP.objects.filter( diff --git a/koku/masu/test/database/test_gcp_report_db_accessor.py b/koku/masu/test/database/test_gcp_report_db_accessor.py index 1888307cd2..4b84184ba5 100644 --- a/koku/masu/test/database/test_gcp_report_db_accessor.py +++ b/koku/masu/test/database/test_gcp_report_db_accessor.py @@ -391,11 +391,12 @@ def test_populate_ocp_gcp_ui_summary_tables(self, mock_sql): @patch("masu.database.gcp_report_db_accessor.GCPReportDBAccessor._execute_raw_sql_query") def test_populate_ocp_on_gcp_tag_information_assert_call(self, mock_trino): """Test that we construst our SQL and execute our query.""" + report_period_id = 1 start_date = self.dh.this_month_start.date() end_date = self.dh.this_month_end.date() mock_gcp_bills = [Mock(), Mock()] - self.accessor.populate_ocp_on_gcp_tag_information(mock_gcp_bills, start_date, end_date, self.ocp_cluster_id) + self.accessor.populate_ocp_on_gcp_tag_information(mock_gcp_bills, start_date, end_date, report_period_id) mock_trino.assert_called() @patch("masu.database.gcp_report_db_accessor.GCPReportDBAccessor.schema_exists_trino") @@ -493,6 +494,7 @@ def test_populate_ocp_on_gcp_tag_information(self, mock_unleash): """ mock_unleash.return_value = True populated_keys = [] + report_period_id = 1 with schema_context(self.schema): enabled_tags = EnabledTagKeys.objects.filter(provider_type=Provider.PROVIDER_GCP, enabled=True) for enabled_tag in enabled_tags: @@ -515,7 +517,7 @@ def test_populate_ocp_on_gcp_tag_information(self, mock_unleash): child_key, child_obj, child_count = populated_keys[1] TagMapping.objects.create(parent=parent_obj, child=child_obj) self.accessor.populate_ocp_on_gcp_tag_information( - bill_ids, self.dh.this_month_start, self.dh.today, self.ocp_cluster_id + bill_ids, self.dh.this_month_start, self.dh.today, report_period_id ) expected_parent_count = parent_count + child_count actual_parent_count = OCPGCPCostLineItemProjectDailySummaryP.objects.filter( From befd9d2cfd0b10d79b5c6c2c03e6dab583d3e4ce Mon Sep 17 00:00:00 2001 From: Luke Couzens Date: Sun, 18 Aug 2024 08:46:16 +0100 Subject: [PATCH 6/7] units --- koku/api/report/test/util/model_bakery_loader.py | 3 ++- .../ocp/test_ocp_cloud_parquet_report_summary_updater.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/koku/api/report/test/util/model_bakery_loader.py b/koku/api/report/test/util/model_bakery_loader.py index 9f6a2b9cb5..1643cd7c79 100644 --- a/koku/api/report/test/util/model_bakery_loader.py +++ b/koku/api/report/test/util/model_bakery_loader.py @@ -528,7 +528,8 @@ def load_openshift_on_cloud_data(self, provider_type, cluster_id, bills, report_ with dbaccessor(self.schema) as accessor: # update tags cls_method = getattr(accessor, tags_update_method) - cls_method([bill.id for bill in bills], self.first_start_date, self.last_end_date, report_period) + for report_period in report_periods: + cls_method([bill.id for bill in bills], self.first_start_date, self.last_end_date, report_period.id) # update ui tables sql_params = { diff --git a/koku/masu/test/processor/ocp/test_ocp_cloud_parquet_report_summary_updater.py b/koku/masu/test/processor/ocp/test_ocp_cloud_parquet_report_summary_updater.py index e937905f3f..f87138b358 100644 --- a/koku/masu/test/processor/ocp/test_ocp_cloud_parquet_report_summary_updater.py +++ b/koku/masu/test/processor/ocp/test_ocp_cloud_parquet_report_summary_updater.py @@ -307,7 +307,7 @@ def test_update_gcp_summary_tables( mock_ui_summary.assert_called_with( start_date, end_date, self.ocpgcp_provider_uuid, self.gcp_test_provider_uuid ) - mock_tag_summary.assert_called_with([1, 2], start_date, end_date, cluster_id) + mock_tag_summary.assert_called_with([1, 2], start_date, end_date, current_ocp_report_period_id) @patch( "masu.processor.ocp.ocp_cloud_parquet_summary_updater.OCPReportDBAccessor.populate_ocp_on_all_ui_summary_tables" # noqa: E501 @@ -379,7 +379,7 @@ def test_update_gcp_summary_tables_with_string_dates( start_date, end_date, self.ocpgcp_provider_uuid, self.gcp_test_provider_uuid ) - mock_tag_summary.assert_called_with([1, 2], start_date, end_date, cluster_id) + mock_tag_summary.assert_called_with([1, 2], start_date, end_date, current_ocp_report_period_id) @patch("masu.processor.ocp.ocp_cloud_parquet_summary_updater.OCPReportDBAccessor.get_cluster_for_provider") @patch( From b06e01d189e4809c486fe230ff8fcc04b710efc5 Mon Sep 17 00:00:00 2001 From: Luke Couzens Date: Mon, 19 Aug 2024 16:20:51 +0100 Subject: [PATCH 7/7] Apply suggestions from code review --- koku/masu/database/aws_report_db_accessor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/koku/masu/database/aws_report_db_accessor.py b/koku/masu/database/aws_report_db_accessor.py index 19f83bc685..c00651846c 100644 --- a/koku/masu/database/aws_report_db_accessor.py +++ b/koku/masu/database/aws_report_db_accessor.py @@ -275,14 +275,14 @@ def back_populate_ocp_infrastructure_costs(self, start_date, end_date, report_pe } self._prepare_and_execute_raw_sql_query(table_name, sql, sql_params) - def populate_ocp_on_aws_tag_information(self, bill_ids, start_date, end_date, current_ocp_report_period_id): + def populate_ocp_on_aws_tag_information(self, bill_ids, start_date, end_date, report_period_id): """Populate the line item aggregated totals data table.""" sql_params = { "schema": self.schema, "bill_ids": bill_ids, "start_date": start_date, "end_date": end_date, - "report_period_id": current_ocp_report_period_id, + "report_period_id": report_period_id, } # Tag Summary sql = pkgutil.get_data("masu.database", "sql/reporting_ocpawstags_summary.sql")