diff --git a/dev/scripts/nise_ymls/ocp_on_azure/azure_static_data.yml b/dev/scripts/nise_ymls/ocp_on_azure/azure_static_data.yml index 353e5daf46..7eebb7d8a5 100644 --- a/dev/scripts/nise_ymls/ocp_on_azure/azure_static_data.yml +++ b/dev/scripts/nise_ymls/ocp_on_azure/azure_static_data.yml @@ -55,7 +55,7 @@ generators: end_date: {{end_date}} meter_id: 55555555-4444-3333-2222-111111111123 resource_location: "US North Central" - instance_id: '/subscriptions/99999999-9999-9999-9999-999999999999/resourceGroups/koku-99hqd-rg/providers/Microsoft.Compute/disks/azure-cloud-prefix-pvc-volume_2' + instance_id: '/subscriptions/99999999-9999-9999-9999-999999999999/resourceGroups/koku-99hqd-rg/providers/Microsoft.Compute/disks/azure-cloud-prefix-pvc-partial-matching' amount: 10 rate: 0.01 tags: @@ -65,7 +65,7 @@ generators: end_date: {{end_date}} meter_id: 55555555-4444-3333-2222-111111111122 resource_location: "US North Central" - instance_id: '/subscriptions/99999999-9999-9999-9999-999999999999/resourceGroups/koku-99hqd-rg/providers/Microsoft.Compute/disks/pvc-volume_1-azure_cloud_suffix' + instance_id: '/subscriptions/99999999-9999-9999-9999-999999999999/resourceGroups/koku-99hqd-rg/providers/Microsoft.Compute/disks/disk-id-1234567' amount: 10 rate: 0.01 tags: @@ -75,7 +75,7 @@ generators: end_date: {{end_date}} meter_id: 55555555-4444-3333-2222-111111111121 resource_location: "US North Central" - instance_id: '/subscriptions/99999999-9999-9999-9999-999999999999/resourceGroups/koku-99hqd-rg/providers/Microsoft.Compute/disks/pvc-1234567-33333333-3333-3333-3333' + instance_id: '/subscriptions/99999999-9999-9999-9999-999999999999/resourceGroups/koku-99hqd-rg/providers/Microsoft.Compute/disks/pv-123-claimless' amount: 10 rate: 0.01 tags: diff --git a/dev/scripts/nise_ymls/ocp_on_azure/ocp_static_data.yml b/dev/scripts/nise_ymls/ocp_on_azure/ocp_static_data.yml index 1d0977eb09..96db350cb2 100644 --- a/dev/scripts/nise_ymls/ocp_on_azure/ocp_static_data.yml +++ b/dev/scripts/nise_ymls/ocp_on_azure/ocp_static_data.yml @@ -31,9 +31,10 @@ generators: labels: label_environment:Jupiter|label_app:mobile|label_version:Mars|label_qa:approved|label_dashed-key-on-azure:dashed-value volumes: - volume: - volume_name: pvc-volume_1 + volume_name: pv-volume-handle-with-pvc storage_class: managed volume_request_gig: 20 + csi_volume_handle: disk-id-1234567 labels: label_environment:Jupiter|label_app:mobile|label_version:Mars|label_storageclass:Baldur|label_dashed-key-on-azure:dashed-value volume_claims: - volume_claim: @@ -61,7 +62,7 @@ generators: labels: label_environment:Jupiter|label_app:mobile|label_version:Mars|label_qa:approved volumes: - volume: - volume_name: pvc-volume_2 + volume_name: pvc-partial-matching storage_class: managed volume_request_gig: 20 labels: label_environment:Jupiter|label_app:mobile|label_version:Mars|label_storageclass:Loki @@ -98,9 +99,10 @@ generators: labels: label_environment:qe|label_app:banking|label_version:MilkyWay|label_qa:approved volumes: - volume: - volume_name: pvc-volume_1 + volume_name: pv-volume-handle-with-pvc storage_class: managed volume_request_gig: 20 + csi_volume_handle: disk-id-1234567 labels: label_environment:Jupiter|label_app:mobile|label_version:Mars|label_storageclass:Baldur volume_claims: - volume_claim: @@ -119,6 +121,12 @@ generators: pod_name: pod_name3a labels: label_environment:qe|label_app:banking|label_version:MilkyWay|label_storageclass:Odin capacity_gig: 20 + - volume: + volume_name: pv-volume_claimless + storage_class: gp3-csi + csi_driver: ebs.csi.azure.com + csi_volume_handle: pv-123-claimless + volume_request_gig: 20 - node: node_name: azure_compute3 node_labels: label_nodeclass:compute|label_node_role_kubernetes_io:infra @@ -146,7 +154,7 @@ generators: labels: label_environment:Mars|label_app:weather|label_version:Andromeda|label_qa:approved volumes: - volume: - volume_name: pvc-volume_1 + volume_name: pv-volume-handle-with-pvc storage_class: managed volume_request_gig: 20 labels: label_environment:Jupiter|label_app:mobile|label_version:Mars|label_storageclass:Baldur diff --git a/koku/masu/database/ocp_report_db_accessor.py b/koku/masu/database/ocp_report_db_accessor.py index 00d002448e..d2a424ff5f 100644 --- a/koku/masu/database/ocp_report_db_accessor.py +++ b/koku/masu/database/ocp_report_db_accessor.py @@ -909,7 +909,17 @@ def populate_pvc_table(self, cluster, pvcs): """Get or create an entry in the OCP cluster table.""" LOG.info(log_json(msg="populating reporting_ocp_pvcs table", schema=self.schema, cluster=cluster)) for pvc in pvcs: - OCPPVC.objects.get_or_create(persistent_volume=pvc[0], persistent_volume_claim=pvc[1], cluster=cluster) + try: + ocppvc = OCPPVC.objects.get(persistent_volume=pvc[0], persistent_volume_claim=pvc[1], cluster=cluster) + if not ocppvc.csi_volume_handle: + # Update the existing record's csi_volume_handle + ocppvc.csi_volume_handle = pvc[2] + ocppvc.save(update_fields=["csi_volume_handle"]) + except OCPPVC.DoesNotExist: + # If the record does not exist, create a new one + OCPPVC.objects.create( + persistent_volume=pvc[0], persistent_volume_claim=pvc[1], csi_volume_handle=pvc[2], cluster=cluster + ) def populate_project_table(self, cluster, projects): """Get or create an entry in the OCP cluster table.""" @@ -951,7 +961,8 @@ def get_pvcs_trino(self, source_uuid, start_date, end_date): """Get the nodes from an OpenShift cluster.""" sql = f""" SELECT distinct persistentvolume, - persistentvolumeclaim + persistentvolumeclaim, + csi_volume_handle FROM hive.{self.schema}.openshift_storage_usage_line_items_daily as ocp WHERE ocp.source = '{source_uuid}' AND ocp.year = '{start_date.strftime("%Y")}' @@ -995,9 +1006,9 @@ def get_pvcs_for_cluster(self, cluster_id): pvcs = ( OCPPVC.objects.filter(cluster_id=cluster_id) .exclude(persistent_volume__exact="") - .values_list("persistent_volume", "persistent_volume_claim") + .values_list("persistent_volume", "persistent_volume_claim", "csi_volume_handle") ) - pvcs = [(pvc[0], pvc[1]) for pvc in pvcs] + pvcs = [(pvc[0], pvc[1], pvc[2]) for pvc in pvcs] return pvcs def get_projects_for_cluster(self, cluster_id): @@ -1023,6 +1034,7 @@ def get_openshift_topology_for_multiple_providers(self, provider_uuids): "resource_ids": [node[1] for node in nodes_tuple], "persistent_volumes": [pvc[0] for pvc in pvc_tuple], "persistent_volume_claims": [pvc[1] for pvc in pvc_tuple], + "csi_volume_handle": [pvc[2] for pvc in pvc_tuple], "projects": [project for project in project_tuple], } ) diff --git a/koku/masu/database/trino_sql/reporting_ocpazurecostlineitem_daily_summary.sql b/koku/masu/database/trino_sql/reporting_ocpazurecostlineitem_daily_summary.sql index 73f02dba1f..e13eed9b15 100644 --- a/koku/masu/database/trino_sql/reporting_ocpazurecostlineitem_daily_summary.sql +++ b/koku/masu/database/trino_sql/reporting_ocpazurecostlineitem_daily_summary.sql @@ -381,13 +381,15 @@ SELECT azure.uuid as azure_uuid, (replace(lower(azure.resource_id), '_osdisk', '') = lower(ocp.node) AND ocp.data_source = 'Pod') OR (strpos(azure.resource_id, ocp.persistentvolume) > 0 AND ocp.data_source = 'Storage') + OR + (lower(ocp.csi_volume_handle) = lower(azure.resource_id)) ) WHERE ocp.source = {{ocp_source_uuid}} AND ocp.year = {{year}} AND lpad(ocp.month, 2, '0') = {{month}} -- Zero pad the month when fewer than 2 characters AND ocp.usage_start >= {{start_date}} AND ocp.usage_start < date_add('day', 1, {{end_date}}) - AND (ocp.resource_id IS NOT NULL AND ocp.resource_id != '') + AND (ocp.resource_id IS NOT NULL AND ocp.resource_id != '') -- This excludes claimless PVs -- Filter out Node Network Costs because they cannot be tied to namespace level AND azure.data_transfer_direction IS NULL AND azure.ocp_source = {{ocp_source_uuid}} diff --git a/koku/masu/test/database/test_ocp_report_db_accessor.py b/koku/masu/test/database/test_ocp_report_db_accessor.py index 17ad969597..3a2f650709 100644 --- a/koku/masu/test/database/test_ocp_report_db_accessor.py +++ b/koku/masu/test/database/test_ocp_report_db_accessor.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 # """Test the OCPReportDBAccessor utility object.""" +import logging import pkgutil import random import uuid @@ -34,6 +35,8 @@ from reporting.provider.ocp.models import OCPPVC from reporting.provider.ocp.models import OCPUsageReportPeriod +LOG = logging.getLogger(__name__) + class OCPReportDBAccessorTest(MasuTestCase): """Test Cases for the OCPReportDBAccessor object.""" @@ -540,10 +543,11 @@ def test_populate_openshift_cluster_information_tables( capacity = [1, 1] volumes = ["vol_1", "vol_2"] pvcs = ["pvc_1", "pvc_2"] + csi_volume_handles = ["csi1", "csi2"] projects = ["project_1", "project_2"] roles = ["master", "worker"] mock_get_nodes.return_value = zip(nodes, resource_ids, capacity, roles) - mock_get_pvcs.return_value = zip(volumes, pvcs) + mock_get_pvcs.return_value = zip(volumes, pvcs, csi_volume_handles) mock_get_projects.return_value = projects mock_table.return_value = True cluster_id = uuid.uuid4() @@ -553,7 +557,10 @@ def test_populate_openshift_cluster_information_tables( end_date = self.dh.this_month_end.date() with self.accessor as acc: - + cluster = acc.populate_cluster_table(self.aws_provider, cluster_id, cluster_alias) + OCPPVC.objects.get_or_create( + persistent_volume_claim=pvcs[0], persistent_volume=volumes[0], cluster=cluster + ) acc.populate_openshift_cluster_information_tables( self.aws_provider, cluster_id, cluster_alias, start_date, end_date ) @@ -594,10 +601,11 @@ def test_get_openshift_topology_for_multiple_providers( capacity = [1, 1] volumes = ["vol_1", "vol_2"] pvcs = ["pvc_1", "pvc_2"] + csi_volume_handles = ["csi1", "csi2"] projects = ["project_1", "project_2"] roles = ["master", "worker"] mock_get_nodes.return_value = zip(nodes, resource_ids, capacity, roles) - mock_get_pvcs.return_value = zip(volumes, pvcs) + mock_get_pvcs.return_value = zip(volumes, pvcs, csi_volume_handles) mock_get_projects.return_value = projects mock_table.return_value = True cluster_id = str(uuid.uuid4()) @@ -627,6 +635,7 @@ def test_get_openshift_topology_for_multiple_providers( for pvc in pvcs: self.assertIn(pvc.persistent_volume_claim, topo.get("persistent_volume_claims")) self.assertIn(pvc.persistent_volume, topo.get("persistent_volumes")) + self.assertIn(pvc.csi_volume_handle, topo.get("csi_volume_handle")) for project in projects: self.assertIn(project.project, topo.get("projects")) diff --git a/koku/masu/util/azure/common.py b/koku/masu/util/azure/common.py index ace240d9f4..0f13b11e16 100644 --- a/koku/masu/util/azure/common.py +++ b/koku/masu/util/azure/common.py @@ -179,7 +179,11 @@ def match_openshift_resources_and_labels(data_frame, cluster_topologies, matched volumes = chain.from_iterable( cluster_topology.get("persistent_volumes", []) for cluster_topology in cluster_topologies ) - matchable_resources = list(nodes) + list(volumes) + csi_volume_handles = chain.from_iterable( + cluster_topology.get("csi_volume_handle", []) for cluster_topology in cluster_topologies + ) + matchable_resources = [*nodes, *volumes, *csi_volume_handles] + matchable_resources = [x for x in matchable_resources if x is not None] data_frame["resource_id_matched"] = False resource_id_df = data_frame["resourceid"] if resource_id_df.eq("").all(): diff --git a/koku/reporting/provider/ocp/models.py b/koku/reporting/provider/ocp/models.py index 06c53f506c..746c42b8e7 100644 --- a/koku/reporting/provider/ocp/models.py +++ b/koku/reporting/provider/ocp/models.py @@ -406,6 +406,7 @@ class Meta: persistent_volume_claim = models.TextField() persistent_volume = models.TextField() cluster = models.ForeignKey("OCPCluster", on_delete=models.CASCADE) + csi_volume_handle = models.TextField(null=True) class OpenshiftCostCategory(models.Model):