diff --git a/koku/masu/database/ocp_report_db_accessor.py b/koku/masu/database/ocp_report_db_accessor.py index efba1bd8f1..dd72650b31 100644 --- a/koku/masu/database/ocp_report_db_accessor.py +++ b/koku/masu/database/ocp_report_db_accessor.py @@ -407,7 +407,7 @@ def populate_distributed_cost_sql(self, start_date, end_date, provider_uuid, dis key_to_file_mapping = { metric_constants.PLATFORM_COST: "distribute_platform_cost.sql", metric_constants.WORKER_UNALLOCATED: "distribute_worker_cost.sql", - # metric_constants.STORAGE_UNATTRIBUTED: "distribute_unattributed_storage_cost.sql", + metric_constants.STORAGE_UNATTRIBUTED: "distribute_unattributed_storage_cost.sql", # metric_constants.NETWORK_UNATTRIBUTED: "distribute_unattributed_network_cost.sql", } @@ -415,7 +415,7 @@ def populate_distributed_cost_sql(self, start_date, end_date, provider_uuid, dis table_name = self._table_map["line_item_daily_summary"] report_period = self.report_periods_for_provider_uuid(provider_uuid, start_date) if not report_period: - msg = "no report period for OCP provider, skipping platform_and_worker_distributed_cost_sql update" + msg = "no report period for OCP provider, skipping distribution update" context = {"schema": self.schema, "provider_uuid": provider_uuid, "start_date": start_date} LOG.info(log_json(msg=msg, context=context)) return diff --git a/koku/masu/database/sql/openshift/cost_model/distribute_cost/distribute_unattributed_storage_cost.sql b/koku/masu/database/sql/openshift/cost_model/distribute_cost/distribute_unattributed_storage_cost.sql index e69de29bb2..dbed3b8067 100644 --- a/koku/masu/database/sql/openshift/cost_model/distribute_cost/distribute_unattributed_storage_cost.sql +++ b/koku/masu/database/sql/openshift/cost_model/distribute_cost/distribute_unattributed_storage_cost.sql @@ -0,0 +1,175 @@ +DELETE FROM {{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary AS lids +WHERE lids.usage_start >= {{start_date}}::date + AND lids.usage_start <= {{end_date}}::date + AND lids.report_period_id = {{report_period_id}} + AND lids.cost_model_rate_type = 'unattributed_storage' +; + +{% if populate %} +WITH unattributed_storage_cost AS ( + SELECT SUM( + COALESCE(infrastructure_raw_cost, 0) + + COALESCE(infrastructure_markup_cost, 0)+ + COALESCE(cost_model_cpu_cost, 0) + + COALESCE(cost_model_memory_cost, 0) + + COALESCE(cost_model_volume_cost, 0) + ) as unattributed_cost, + lids.usage_start + FROM {{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as lids + WHERE lids.usage_start >= {{start_date}}::date + AND lids.usage_start <= {{end_date}}::date + AND lids.report_period_id = {{report_period_id}} + AND lids.namespace = 'Storage unattributed' + GROUP BY lids.usage_start +), +user_defined_project_sum as ( + SELECT sum(pod_effective_usage_cpu_core_hours) as usage_cpu_sum, + sum(pod_effective_usage_memory_gigabyte_hours) as usage_memory_sum, + cluster_id, + usage_start, + source_uuid + FROM {{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as lids + LEFT JOIN {{schema | sqlsafe}}.reporting_ocp_cost_category AS cat + ON lids.cost_category_id = cat.id + WHERE lids.usage_start >= {{start_date}}::date + AND lids.usage_start <= {{end_date}}::date + AND report_period_id = {{report_period_id}} + AND lids.namespace not in ('Worker unallocated', 'Platform unallocated', 'Storage unattributed', 'Network unattributed') + AND (cost_category_id IS NULL OR cat.name != 'Platform') + GROUP BY usage_start, cluster_id, source_uuid +), +cte_line_items as ( + SELECT + max(report_period_id) as report_period_id, + lids.cluster_id, + max(cluster_alias) as cluster_alias, + lids.data_source as data_source, + lids.usage_start, + max(usage_end) as usage_end, + lids.namespace, + lids.node, + max(resource_id) as resource_id, + max(node_capacity_cpu_cores) as node_capacity_cpu_cores, + max(node_capacity_cpu_core_hours) as node_capacity_cpu_core_hours, + max(node_capacity_memory_gigabytes) as node_capacity_memory_gigabytes, + max(node_capacity_memory_gigabyte_hours) as node_capacity_memory_gigabyte_hours, + max(cluster_capacity_cpu_core_hours) as cluster_capacity_cpu_core_hours, + max(cluster_capacity_memory_gigabyte_hours) as cluster_capacity_memory_gigabyte_hours, + CASE WHEN {{distribution}} = 'cpu' AND namespace != 'Storage unattributed' THEN + CASE WHEN max(udps.usage_cpu_sum) <= 0 THEN + 0 + ELSE + (sum(pod_effective_usage_cpu_core_hours) / max(udps.usage_cpu_sum)) * max(usc.unattributed_cost)::decimal + END + WHEN {{distribution}} = 'memory' AND namespace != 'Storage unattributed' THEN + CASE WHEN max(udps.usage_memory_sum) <= 0 THEN + 0 + ELSE + (sum(pod_effective_usage_memory_gigabyte_hours) / max(udps.usage_memory_sum)) * max(usc.unattributed_cost)::decimal + END + WHEN namespace = 'Storage unattributed' THEN + 0 - SUM( + COALESCE(infrastructure_raw_cost, 0) + + COALESCE(infrastructure_markup_cost, 0) + + COALESCE(cost_model_cpu_cost, 0) + + COALESCE(cost_model_memory_cost, 0) + + COALESCE(cost_model_volume_cost, 0) + ) + END AS distributed_cost, + max(cost_category_id) as cost_category_id + FROM {{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary AS lids + LEFT JOIN unattributed_storage_cost as usc + ON usc.usage_start = lids.usage_start + LEFT JOIN user_defined_project_sum as udps + ON udps.usage_start = lids.usage_start + AND udps.cluster_id = lids.cluster_id + LEFT JOIN {{schema | sqlsafe}}.reporting_ocp_cost_category AS cat + ON lids.cost_category_id = cat.id + WHERE lids.usage_start >= {{start_date}}::date + AND lids.usage_start <= {{end_date}}::date + AND report_period_id = {{report_period_id}} + AND lids.namespace IS NOT NULL + AND lids.namespace not in ('Worker unallocated', 'Platform unallocated', 'Network unattributed') + AND (cost_category_id IS NULL OR cat.name != 'Platform') + GROUP BY lids.usage_start, lids.node, lids.namespace, lids.cluster_id, cost_category_id, lids.data_source +) +INSERT INTO {{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary ( + uuid, + report_period_id, + cluster_id, + cluster_alias, + data_source, + usage_start, + usage_end, + namespace, + node, + resource_id, + pod_labels, + pod_usage_cpu_core_hours, + pod_request_cpu_core_hours, + pod_effective_usage_cpu_core_hours, + pod_limit_cpu_core_hours, + pod_usage_memory_gigabyte_hours, + pod_request_memory_gigabyte_hours, + pod_effective_usage_memory_gigabyte_hours, + pod_limit_memory_gigabyte_hours, + node_capacity_cpu_cores, + node_capacity_cpu_core_hours, + node_capacity_memory_gigabytes, + node_capacity_memory_gigabyte_hours, + cluster_capacity_cpu_core_hours, + cluster_capacity_memory_gigabyte_hours, + persistentvolumeclaim, + persistentvolume, + storageclass, + volume_labels, + persistentvolumeclaim_capacity_gigabyte, + persistentvolumeclaim_capacity_gigabyte_months, + volume_request_storage_gigabyte_months, + persistentvolumeclaim_usage_gigabyte_months, + source_uuid, + cost_model_rate_type, + distributed_cost, + cost_category_id +) +SELECT + uuid_generate_v4(), + ctl.report_period_id, + ctl.cluster_id, + ctl.cluster_alias, + ctl.data_source as data_source, + ctl.usage_start, + ctl.usage_end, + ctl.namespace, + ctl.node, + ctl.resource_id, + NULL as pod_labels, + NULL as pod_usage_cpu_core_hours, + NULL as pod_request_cpu_core_hours, + NULL as pod_effective_usage_cpu_core_hours, + NULL as pod_limit_cpu_core_hours, + NULL as pod_usage_memory_gigabyte_hours, + NULL as pod_request_memory_gigabyte_hours, + NULL as pod_effective_usage_memory_gigabyte_hours, + NULL as pod_limit_memory_gigabyte_hours, + ctl.node_capacity_cpu_cores, + ctl.node_capacity_cpu_core_hours, + ctl.node_capacity_memory_gigabytes, + ctl.node_capacity_memory_gigabyte_hours, + ctl.cluster_capacity_cpu_core_hours, + ctl.cluster_capacity_memory_gigabyte_hours, + NULL as persistentvolumeclaim, + NULL as persistentvolume, + NULL as storageclass, + NULL as volume_labels, + NULL as persistentvolumeclaim_capacity_gigabyte, + NULL as persistentvolumeclaim_capacity_gigabyte_months, + NULL as volume_request_storage_gigabyte_months, + NULL as persistentvolumeclaim_usage_gigabyte_months, + UUID '{{source_uuid | sqlsafe}}' as source_uuid, + 'unattributed_storage' as cost_model_rate_type, + ctl.distributed_cost, + ctl.cost_category_id +FROM cte_line_items as ctl +WHERE ctl.distributed_cost != 0; +{% endif %} diff --git a/koku/masu/test/database/test_ocp_report_db_accessor.py b/koku/masu/test/database/test_ocp_report_db_accessor.py index 4c8c06c50c..c483e3b352 100644 --- a/koku/masu/test/database/test_ocp_report_db_accessor.py +++ b/koku/masu/test/database/test_ocp_report_db_accessor.py @@ -966,6 +966,7 @@ def get_pkgutil_values(file): side_effect = [ [get_pkgutil_values("distribute_worker_cost.sql"), default_sql_params], [get_pkgutil_values("distribute_platform_cost.sql"), default_sql_params], + [get_pkgutil_values("distribute_unattributed_storage_cost.sql"), default_sql_params], ] mock_jinja = Mock() mock_jinja.side_effect = side_effect @@ -978,11 +979,12 @@ def get_pkgutil_values(file): expected_calls = [ call(masu_database, "sql/openshift/cost_model/distribute_worker_cost.sql"), call(masu_database, "sql/openshift/cost_model/distribute_platform_cost.sql"), + call(masu_database, "sql/openshift/cost_model/distribute_unattributed_storage_cost.sql"), ] for expected_call in expected_calls: self.assertIn(expected_call, mock_data_get.call_args_list) mock_sql_execute.assert_called() - self.assertEqual(len(mock_sql_execute.call_args_list), 2) + self.assertEqual(len(mock_sql_execute.call_args_list), 3) @patch("masu.database.ocp_report_db_accessor.is_feature_cost_3592_tag_mapping_enabled") def test_update_line_item_daily_summary_with_tag_mapping(self, mock_unleash):