[COST-4913] Add ability to distribute unattributed storage cost.

koku/masu/database/ocp_report_db_accessor.py

@@ -407,15 +407,15 @@ def populate_distributed_cost_sql(self, start_date, end_date, provider_uuid, dis
         key_to_file_mapping = {
             metric_constants.PLATFORM_COST: "distribute_platform_cost.sql",
             metric_constants.WORKER_UNALLOCATED: "distribute_worker_cost.sql",
-            # metric_constants.STORAGE_UNATTRIBUTED: "distribute_unattributed_storage_cost.sql",
+            metric_constants.STORAGE_UNATTRIBUTED: "distribute_unattributed_storage_cost.sql",
             # metric_constants.NETWORK_UNATTRIBUTED: "distribute_unattributed_network_cost.sql",
         }
 
         distribution = distribution_info.get("distribution_type", DEFAULT_DISTRIBUTION_TYPE)
         table_name = self._table_map["line_item_daily_summary"]
         report_period = self.report_periods_for_provider_uuid(provider_uuid, start_date)
         if not report_period:
-            msg = "no report period for OCP provider, skipping platform_and_worker_distributed_cost_sql update"
+            msg = "no report period for OCP provider, skipping distribution update"
             context = {"schema": self.schema, "provider_uuid": provider_uuid, "start_date": start_date}
             LOG.info(log_json(msg=msg, context=context))
             return

koku/masu/database/sql/openshift/cost_model/distribute_cost/distribute_unattributed_storage_cost.sql

@@ -0,0 +1,175 @@
+DELETE FROM {{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary AS lids
+WHERE lids.usage_start >= {{start_date}}::date
+    AND lids.usage_start <= {{end_date}}::date
+    AND lids.report_period_id = {{report_period_id}}
+    AND lids.cost_model_rate_type = 'unattributed_storage'
+;
+
+{% if populate %}
+WITH unattributed_storage_cost AS (
+    SELECT SUM(
+            COALESCE(infrastructure_raw_cost, 0) +
+            COALESCE(infrastructure_markup_cost, 0)+
+            COALESCE(cost_model_cpu_cost, 0) +
+            COALESCE(cost_model_memory_cost, 0) +
+            COALESCE(cost_model_volume_cost, 0)
+        ) as unattributed_cost,
+        lids.usage_start
+    FROM {{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as lids
+    WHERE lids.usage_start >= {{start_date}}::date
+        AND lids.usage_start <= {{end_date}}::date
+        AND lids.report_period_id = {{report_period_id}}
+        AND lids.namespace = 'Storage unattributed'
+    GROUP BY lids.usage_start
+),
+user_defined_project_sum as (
+    SELECT sum(pod_effective_usage_cpu_core_hours) as usage_cpu_sum,
+        sum(pod_effective_usage_memory_gigabyte_hours) as usage_memory_sum,
+        cluster_id,
+        usage_start,
+        source_uuid
+    FROM {{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as lids
+    LEFT JOIN {{schema | sqlsafe}}.reporting_ocp_cost_category AS cat
+        ON lids.cost_category_id = cat.id
+    WHERE lids.usage_start >= {{start_date}}::date
+        AND lids.usage_start <= {{end_date}}::date
+        AND report_period_id = {{report_period_id}}
+        AND lids.namespace not in ('Worker unallocated', 'Platform unallocated', 'Storage unattributed', 'Network unattributed')
+        AND (cost_category_id IS NULL OR cat.name != 'Platform')
+    GROUP BY usage_start, cluster_id, source_uuid
+),
+cte_line_items as (
+    SELECT
+        max(report_period_id) as report_period_id,
+        lids.cluster_id,
+        max(cluster_alias) as cluster_alias,
+        lids.data_source as data_source,
+        lids.usage_start,
+        max(usage_end) as usage_end,
+        lids.namespace,
+        lids.node,
+        max(resource_id) as resource_id,
+        max(node_capacity_cpu_cores) as node_capacity_cpu_cores,
+        max(node_capacity_cpu_core_hours) as node_capacity_cpu_core_hours,
+        max(node_capacity_memory_gigabytes) as node_capacity_memory_gigabytes,
+        max(node_capacity_memory_gigabyte_hours) as node_capacity_memory_gigabyte_hours,
+        max(cluster_capacity_cpu_core_hours) as cluster_capacity_cpu_core_hours,
+        max(cluster_capacity_memory_gigabyte_hours) as cluster_capacity_memory_gigabyte_hours,
+        CASE WHEN {{distribution}} = 'cpu' AND namespace != 'Storage unattributed' THEN
+            CASE WHEN max(udps.usage_cpu_sum) <= 0 THEN
+                0
+            ELSE
+                (sum(pod_effective_usage_cpu_core_hours) / max(udps.usage_cpu_sum)) * max(usc.unattributed_cost)::decimal
+            END
+        WHEN {{distribution}} = 'memory' AND namespace != 'Storage unattributed'  THEN
+            CASE WHEN max(udps.usage_memory_sum) <= 0 THEN
+                0
+            ELSE
+                (sum(pod_effective_usage_memory_gigabyte_hours) / max(udps.usage_memory_sum)) * max(usc.unattributed_cost)::decimal
+            END
+        WHEN namespace = 'Storage unattributed' THEN
+            0 - SUM(
+                    COALESCE(infrastructure_raw_cost, 0) +
+                    COALESCE(infrastructure_markup_cost, 0) +
+                    COALESCE(cost_model_cpu_cost, 0) +
+                    COALESCE(cost_model_memory_cost, 0) +
+                    COALESCE(cost_model_volume_cost, 0)
+                )
+        END AS distributed_cost,
+        max(cost_category_id) as cost_category_id
+    FROM {{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary AS lids
+    LEFT JOIN unattributed_storage_cost as usc
+        ON usc.usage_start = lids.usage_start
+    LEFT JOIN user_defined_project_sum as udps
+        ON udps.usage_start = lids.usage_start
+        AND udps.cluster_id = lids.cluster_id
+    LEFT JOIN {{schema | sqlsafe}}.reporting_ocp_cost_category AS cat
+        ON lids.cost_category_id = cat.id
+    WHERE lids.usage_start >= {{start_date}}::date
+        AND lids.usage_start <= {{end_date}}::date
+        AND report_period_id = {{report_period_id}}
+        AND lids.namespace IS NOT NULL
+        AND lids.namespace not in ('Worker unallocated', 'Platform unallocated', 'Network unattributed')
+        AND (cost_category_id IS NULL OR cat.name != 'Platform')
+    GROUP BY lids.usage_start, lids.node, lids.namespace, lids.cluster_id, cost_category_id, lids.data_source
+)
+INSERT INTO {{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary (
+    uuid,
+    report_period_id,
+    cluster_id,
+    cluster_alias,
+    data_source,
+    usage_start,
+    usage_end,
+    namespace,
+    node,
+    resource_id,
+    pod_labels,
+    pod_usage_cpu_core_hours,
+    pod_request_cpu_core_hours,
+    pod_effective_usage_cpu_core_hours,
+    pod_limit_cpu_core_hours,
+    pod_usage_memory_gigabyte_hours,
+    pod_request_memory_gigabyte_hours,
+    pod_effective_usage_memory_gigabyte_hours,
+    pod_limit_memory_gigabyte_hours,
+    node_capacity_cpu_cores,
+    node_capacity_cpu_core_hours,
+    node_capacity_memory_gigabytes,
+    node_capacity_memory_gigabyte_hours,
+    cluster_capacity_cpu_core_hours,
+    cluster_capacity_memory_gigabyte_hours,
+    persistentvolumeclaim,
+    persistentvolume,
+    storageclass,
+    volume_labels,
+    persistentvolumeclaim_capacity_gigabyte,
+    persistentvolumeclaim_capacity_gigabyte_months,
+    volume_request_storage_gigabyte_months,
+    persistentvolumeclaim_usage_gigabyte_months,
+    source_uuid,
+    cost_model_rate_type,
+    distributed_cost,
+    cost_category_id
+)
+SELECT
+    uuid_generate_v4(),
+    ctl.report_period_id,
+    ctl.cluster_id,
+    ctl.cluster_alias,
+    ctl.data_source as data_source,
+    ctl.usage_start,
+    ctl.usage_end,
+    ctl.namespace,
+    ctl.node,
+    ctl.resource_id,
+    NULL as pod_labels,
+    NULL as pod_usage_cpu_core_hours,
+    NULL as pod_request_cpu_core_hours,
+    NULL as pod_effective_usage_cpu_core_hours,
+    NULL as pod_limit_cpu_core_hours,
+    NULL as pod_usage_memory_gigabyte_hours,
+    NULL as pod_request_memory_gigabyte_hours,
+    NULL as pod_effective_usage_memory_gigabyte_hours,
+    NULL as pod_limit_memory_gigabyte_hours,
+    ctl.node_capacity_cpu_cores,
+    ctl.node_capacity_cpu_core_hours,
+    ctl.node_capacity_memory_gigabytes,
+    ctl.node_capacity_memory_gigabyte_hours,
+    ctl.cluster_capacity_cpu_core_hours,
+    ctl.cluster_capacity_memory_gigabyte_hours,
+    NULL as persistentvolumeclaim,
+    NULL as persistentvolume,
+    NULL as storageclass,
+    NULL as volume_labels,
+    NULL as persistentvolumeclaim_capacity_gigabyte,
+    NULL as persistentvolumeclaim_capacity_gigabyte_months,
+    NULL as volume_request_storage_gigabyte_months,
+    NULL as persistentvolumeclaim_usage_gigabyte_months,
+    UUID '{{source_uuid | sqlsafe}}' as source_uuid,
+    'unattributed_storage' as cost_model_rate_type,
+    ctl.distributed_cost,
+    ctl.cost_category_id
+FROM cte_line_items as ctl
+WHERE ctl.distributed_cost != 0;
+{% endif %}

koku/masu/test/database/test_ocp_report_db_accessor.py

@@ -966,6 +966,7 @@ def get_pkgutil_values(file):
         side_effect = [
             [get_pkgutil_values("distribute_worker_cost.sql"), default_sql_params],
             [get_pkgutil_values("distribute_platform_cost.sql"), default_sql_params],
+            [get_pkgutil_values("distribute_unattributed_storage_cost.sql"), default_sql_params],
         ]
         mock_jinja = Mock()
         mock_jinja.side_effect = side_effect
@@ -978,11 +979,12 @@ def get_pkgutil_values(file):
             expected_calls = [
                 call(masu_database, "sql/openshift/cost_model/distribute_worker_cost.sql"),
                 call(masu_database, "sql/openshift/cost_model/distribute_platform_cost.sql"),
+                call(masu_database, "sql/openshift/cost_model/distribute_unattributed_storage_cost.sql"),
             ]
             for expected_call in expected_calls:
                 self.assertIn(expected_call, mock_data_get.call_args_list)
             mock_sql_execute.assert_called()
-            self.assertEqual(len(mock_sql_execute.call_args_list), 2)
+            self.assertEqual(len(mock_sql_execute.call_args_list), 3)
 
     @patch("masu.database.ocp_report_db_accessor.is_feature_cost_3592_tag_mapping_enabled")
     def test_update_line_item_daily_summary_with_tag_mapping(self, mock_unleash):