[COST-4906] Update resource matching to use CSI volume handle

dev/scripts/nise_ymls/ocp_on_azure/azure_static_data.yml

@@ -55,7 +55,7 @@ generators:
       end_date: {{end_date}}
       meter_id: 55555555-4444-3333-2222-111111111123
       resource_location: "US North Central"
-      instance_id: '/subscriptions/99999999-9999-9999-9999-999999999999/resourceGroups/koku-99hqd-rg/providers/Microsoft.Compute/disks/azure-cloud-prefix-pvc-volume_2'
+      instance_id: '/subscriptions/99999999-9999-9999-9999-999999999999/resourceGroups/koku-99hqd-rg/providers/Microsoft.Compute/disks/azure-cloud-prefix-pvc-partial-matching'
       amount: 10
       rate: 0.01
       tags:
@@ -65,7 +65,7 @@ generators:
       end_date: {{end_date}}
       meter_id: 55555555-4444-3333-2222-111111111122
       resource_location: "US North Central"
-      instance_id: '/subscriptions/99999999-9999-9999-9999-999999999999/resourceGroups/koku-99hqd-rg/providers/Microsoft.Compute/disks/pvc-volume_1-azure_cloud_suffix'
+      instance_id: '/subscriptions/99999999-9999-9999-9999-999999999999/resourceGroups/koku-99hqd-rg/providers/Microsoft.Compute/disks/disk-id-1234567'
       amount: 10
       rate: 0.01
       tags:
@@ -75,7 +75,7 @@ generators:
       end_date: {{end_date}}
       meter_id: 55555555-4444-3333-2222-111111111121
       resource_location: "US North Central"
-      instance_id: '/subscriptions/99999999-9999-9999-9999-999999999999/resourceGroups/koku-99hqd-rg/providers/Microsoft.Compute/disks/pvc-1234567-33333333-3333-3333-3333'
+      instance_id: '/subscriptions/99999999-9999-9999-9999-999999999999/resourceGroups/koku-99hqd-rg/providers/Microsoft.Compute/disks/pv-123-claimless'
       amount: 10
       rate: 0.01
       tags:

dev/scripts/nise_ymls/ocp_on_azure/ocp_static_data.yml

@@ -31,9 +31,10 @@ generators:
                   labels: label_environment:Jupiter|label_app:mobile|label_version:Mars|label_qa:approved|label_dashed-key-on-azure:dashed-value
               volumes:
                 - volume:
-                  volume_name: pvc-volume_1
+                  volume_name: pv-volume-handle-with-pvc
                   storage_class: managed
                   volume_request_gig: 20
+                  csi_volume_handle: disk-id-1234567
                   labels: label_environment:Jupiter|label_app:mobile|label_version:Mars|label_storageclass:Baldur|label_dashed-key-on-azure:dashed-value
                   volume_claims:
                   - volume_claim:
@@ -61,7 +62,7 @@ generators:
                   labels: label_environment:Jupiter|label_app:mobile|label_version:Mars|label_qa:approved
               volumes:
                 - volume:
-                  volume_name: pvc-volume_2
+                  volume_name: pvc-partial-matching
                   storage_class: managed
                   volume_request_gig: 20
                   labels: label_environment:Jupiter|label_app:mobile|label_version:Mars|label_storageclass:Loki
@@ -98,9 +99,10 @@ generators:
                   labels: label_environment:qe|label_app:banking|label_version:MilkyWay|label_qa:approved
               volumes:
                 - volume:
-                  volume_name: pvc-volume_1
+                  volume_name: pv-volume-handle-with-pvc
                   storage_class: managed
                   volume_request_gig: 20
+                  csi_volume_handle: disk-id-1234567
                   labels: label_environment:Jupiter|label_app:mobile|label_version:Mars|label_storageclass:Baldur
                   volume_claims:
                   - volume_claim:
@@ -119,6 +121,12 @@ generators:
                     pod_name: pod_name3a
                     labels: label_environment:qe|label_app:banking|label_version:MilkyWay|label_storageclass:Odin
                     capacity_gig: 20
+                - volume:
+                  volume_name: pv-volume_claimless
+                  storage_class: gp3-csi
+                  csi_driver: ebs.csi.azure.com
+                  csi_volume_handle: pv-123-claimless
+                  volume_request_gig: 20
         - node:
           node_name: azure_compute3
           node_labels: label_nodeclass:compute|label_node_role_kubernetes_io:infra
@@ -146,7 +154,7 @@ generators:
                   labels: label_environment:Mars|label_app:weather|label_version:Andromeda|label_qa:approved
               volumes:
                 - volume:
-                  volume_name: pvc-volume_1
+                  volume_name: pv-volume-handle-with-pvc
                   storage_class: managed
                   volume_request_gig: 20
                   labels: label_environment:Jupiter|label_app:mobile|label_version:Mars|label_storageclass:Baldur

koku/masu/database/ocp_report_db_accessor.py

@@ -909,7 +909,16 @@ def populate_pvc_table(self, cluster, pvcs):
         """Get or create an entry in the OCP cluster table."""
         LOG.info(log_json(msg="populating reporting_ocp_pvcs table", schema=self.schema, cluster=cluster))
         for pvc in pvcs:
-            OCPPVC.objects.get_or_create(persistent_volume=pvc[0], persistent_volume_claim=pvc[1], cluster=cluster)
+            try:
+                ocppvc = OCPPVC.objects.get(persistent_volume=pvc[0], persistent_volume_claim=pvc[1], cluster=cluster)
+                # Update the existing record's csi_volume_handle
+                ocppvc.csi_volume_handle = pvc[2]
+                ocppvc.save()
+            except OCPPVC.DoesNotExist:
+                # If the record does not exist, create a new one
+                OCPPVC.objects.create(
+                    persistent_volume=pvc[0], persistent_volume_claim=pvc[1], csi_volume_handle=pvc[2], cluster=cluster
+                )
 
     def populate_project_table(self, cluster, projects):
         """Get or create an entry in the OCP cluster table."""
@@ -951,7 +960,8 @@ def get_pvcs_trino(self, source_uuid, start_date, end_date):
         """Get the nodes from an OpenShift cluster."""
         sql = f"""
             SELECT distinct persistentvolume,
-                persistentvolumeclaim
+                persistentvolumeclaim,
+                csi_volume_handle
             FROM hive.{self.schema}.openshift_storage_usage_line_items_daily as ocp
             WHERE ocp.source = '{source_uuid}'
                 AND ocp.year = '{start_date.strftime("%Y")}'
@@ -995,9 +1005,9 @@ def get_pvcs_for_cluster(self, cluster_id):
         pvcs = (
             OCPPVC.objects.filter(cluster_id=cluster_id)
             .exclude(persistent_volume__exact="")
-            .values_list("persistent_volume", "persistent_volume_claim")
+            .values_list("persistent_volume", "persistent_volume_claim", "csi_volume_handle")
         )
-        pvcs = [(pvc[0], pvc[1]) for pvc in pvcs]
+        pvcs = [(pvc[0], pvc[1], pvc[2]) for pvc in pvcs]
         return pvcs
 
     def get_projects_for_cluster(self, cluster_id):
@@ -1023,6 +1033,7 @@ def get_openshift_topology_for_multiple_providers(self, provider_uuids):
                     "resource_ids": [node[1] for node in nodes_tuple],
                     "persistent_volumes": [pvc[0] for pvc in pvc_tuple],
                     "persistent_volume_claims": [pvc[1] for pvc in pvc_tuple],
+                    "csi_volume_handle": [pvc[2] for pvc in pvc_tuple],
                     "projects": [project for project in project_tuple],
                 }
             )

koku/masu/database/trino_sql/reporting_ocpazurecostlineitem_daily_summary.sql

@@ -381,13 +381,15 @@ SELECT azure.uuid as azure_uuid,
                 (replace(lower(azure.resource_id), '_osdisk', '') = lower(ocp.node) AND ocp.data_source = 'Pod')
             OR
                 (strpos(azure.resource_id, ocp.persistentvolume) > 0 AND ocp.data_source = 'Storage')
+            OR
+                (lower(ocp.csi_volume_handle) = lower(azure.resource_id))
             )
     WHERE ocp.source = {{ocp_source_uuid}}
         AND ocp.year = {{year}}
         AND lpad(ocp.month, 2, '0') = {{month}} -- Zero pad the month when fewer than 2 characters
         AND ocp.usage_start >= {{start_date}}
         AND ocp.usage_start < date_add('day', 1, {{end_date}})
-        AND (ocp.resource_id IS NOT NULL AND ocp.resource_id != '')
+        AND (ocp.resource_id IS NOT NULL AND ocp.resource_id != '') -- This excludes claimless PVs
         -- Filter out Node Network Costs because they cannot be tied to namespace level
         AND azure.data_transfer_direction IS NULL
         AND azure.ocp_source = {{ocp_source_uuid}}

koku/masu/test/database/test_ocp_report_db_accessor.py

@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: Apache-2.0
 #
 """Test the OCPReportDBAccessor utility object."""
+import logging
 import pkgutil
 import random
 import uuid
@@ -34,6 +35,8 @@
 from reporting.provider.ocp.models import OCPPVC
 from reporting.provider.ocp.models import OCPUsageReportPeriod
 
+LOG = logging.getLogger(__name__)
+
 
 class OCPReportDBAccessorTest(MasuTestCase):
     """Test Cases for the OCPReportDBAccessor object."""
@@ -540,10 +543,11 @@ def test_populate_openshift_cluster_information_tables(
         capacity = [1, 1]
         volumes = ["vol_1", "vol_2"]
         pvcs = ["pvc_1", "pvc_2"]
+        csi_volume_handles = ["csi1", "csi2"]
         projects = ["project_1", "project_2"]
         roles = ["master", "worker"]
         mock_get_nodes.return_value = zip(nodes, resource_ids, capacity, roles)
-        mock_get_pvcs.return_value = zip(volumes, pvcs)
+        mock_get_pvcs.return_value = zip(volumes, pvcs, csi_volume_handles)
         mock_get_projects.return_value = projects
         mock_table.return_value = True
         cluster_id = uuid.uuid4()
@@ -553,7 +557,10 @@ def test_populate_openshift_cluster_information_tables(
         end_date = self.dh.this_month_end.date()
 
         with self.accessor as acc:
-
+            cluster = acc.populate_cluster_table(self.aws_provider, cluster_id, cluster_alias)
+            OCPPVC.objects.get_or_create(
+                persistent_volume_claim=pvcs[0], persistent_volume=volumes[0], cluster=cluster
+            )
             acc.populate_openshift_cluster_information_tables(
                 self.aws_provider, cluster_id, cluster_alias, start_date, end_date
             )
@@ -594,10 +601,11 @@ def test_get_openshift_topology_for_multiple_providers(
         capacity = [1, 1]
         volumes = ["vol_1", "vol_2"]
         pvcs = ["pvc_1", "pvc_2"]
+        csi_volume_handles = ["csi1", "csi2"]
         projects = ["project_1", "project_2"]
         roles = ["master", "worker"]
         mock_get_nodes.return_value = zip(nodes, resource_ids, capacity, roles)
-        mock_get_pvcs.return_value = zip(volumes, pvcs)
+        mock_get_pvcs.return_value = zip(volumes, pvcs, csi_volume_handles)
         mock_get_projects.return_value = projects
         mock_table.return_value = True
         cluster_id = str(uuid.uuid4())
@@ -627,6 +635,7 @@ def test_get_openshift_topology_for_multiple_providers(
             for pvc in pvcs:
                 self.assertIn(pvc.persistent_volume_claim, topo.get("persistent_volume_claims"))
                 self.assertIn(pvc.persistent_volume, topo.get("persistent_volumes"))
+                self.assertIn(pvc.csi_volume_handle, topo.get("csi_volume_handle"))
             for project in projects:
                 self.assertIn(project.project, topo.get("projects"))
 

koku/masu/util/azure/common.py

@@ -179,7 +179,10 @@ def match_openshift_resources_and_labels(data_frame, cluster_topologies, matched
     volumes = chain.from_iterable(
         cluster_topology.get("persistent_volumes", []) for cluster_topology in cluster_topologies
     )
-    matchable_resources = list(nodes) + list(volumes)
+    csi_volume_handles = chain.from_iterable(
+        cluster_topology.get("csi_volume_handle", []) for cluster_topology in cluster_topologies
+    )
+    matchable_resources = [*nodes, *volumes, *csi_volume_handles]
     data_frame["resource_id_matched"] = False
     resource_id_df = data_frame["resourceid"]
     if resource_id_df.eq("").all():

koku/reporting/migrations/0324_ocppvc_csi_volume_handle.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.11 on 2024-05-16 18:41
+from django.db import migrations
+from django.db import models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("reporting", "0323_remove_awscostentrylineitemsummarybyec2compute_availability_zone_and_org_unit"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="ocppvc",
+            name="csi_volume_handle",
+            field=models.TextField(null=True),
+        ),
+    ]

koku/reporting/provider/ocp/models.py

@@ -406,6 +406,7 @@ class Meta:
     persistent_volume_claim = models.TextField()
     persistent_volume = models.TextField()
     cluster = models.ForeignKey("OCPCluster", on_delete=models.CASCADE)
+    csi_volume_handle = models.TextField(null=True)
 
 
 class OpenshiftCostCategory(models.Model):