Merge pull request #2610 from project-koku/rc-2021.01.22

Rc 2021.01.22

.github/workflows/unittests.yml

@@ -54,7 +54,7 @@ jobs:
           path: |
             ~/.cache/pipenv
             ~/.local/share/virtualenvs
-          key: os-${{ runner.os }}-env-${{ matrix.python-version }}-${{ hashFiles('**/Pipfile.lock') }}-${{ github.ref }}
+          key: ${{ runner.os }}-env-${{ matrix.python-version }}-${{ hashFiles('**/Pipfile.lock') }}-${{ github.ref }}
 
       - name: Install dependencies
         if: steps.cache-dependencies.outputs.cache-hit != 'true'

docs/source/specs/openapi.json

@@ -396,7 +396,7 @@
                 }]
             }
         },
-        "/forecasts/aws/costs": {
+        "/forecasts/aws/costs/": {
             "summary": "AWS Cost Forecasts",
             "get": {
                 "tags":["Forecasts"],
@@ -441,7 +441,7 @@
                 }]
             }
         },
-        "/forecasts/azure/costs": {
+        "/forecasts/azure/costs/": {
             "summary": "Azure Cost Forecasts",
             "get": {
                 "tags":["Forecasts"],
@@ -486,7 +486,52 @@
                 }]
             }
         },
-        "/forecasts/openshift/costs": {
+        "/forecasts/gcp/costs/": {
+            "summary": "GCP Cost Forecasts",
+            "get": {
+                "tags":["Forecasts"],
+                "parameters": [{
+                    "$ref": "#/components/parameters/QueryFilter",
+                    "name":"QueryFilter"
+                }],
+                "responses": {
+                    "200": {
+                        "description": "An object describing the cost forecast.",
+                        "content": {
+                            "application/json": {
+                                "schema": {
+                                    "$ref": "#/components/schemas/Forecast"
+                                }
+                            }
+                        }
+                    },
+                    "400": {
+                        "description": "Request Error",
+                        "content": {
+                            "application/json": {
+                                "schema": {
+                                    "$ref": "#/components/schemas/Error"
+                                }
+                            }
+                        }
+                    },
+                    "500": {
+                        "description": "Unexpected Error",
+                        "content": {
+                            "application/json": {
+                                "schema": {
+                                    "$ref": "#/components/schemas/Error"
+                                }
+                            }
+                        }
+                    }
+                },
+                "security": [{
+                    "basic_auth": []
+                }]
+            }
+        },
+        "/forecasts/openshift/costs/": {
             "summary": "OpenShift Cost Forecasts",
             "get": {
                 "tags":["Forecasts"],
@@ -531,7 +576,7 @@
                 }]
             }
         },
-        "/forecasts/openshift/infrastructures/aws/costs": {
+        "/forecasts/openshift/infrastructures/aws/costs/": {
             "summary": "OpenShift on AWS Cost Forecasts",
             "get": {
                 "tags":["Forecasts"],
@@ -576,7 +621,7 @@
                 }]
             }
         },
-        "/forecasts/openshift/infrastructures/all/costs": {
+        "/forecasts/openshift/infrastructures/all/costs/": {
             "summary": "OpenShift on Cloud Cost Forecasts",
             "get": {
                 "tags":["Forecasts"],
@@ -621,7 +666,7 @@
                 }]
             }
         },
-        "/forecasts/openshift/infrastructures/azure/costs": {
+        "/forecasts/openshift/infrastructures/azure/costs/": {
             "summary": "OpenShift on Azure Cost Forecasts",
             "get": {
                 "tags":["Forecasts"],
@@ -2406,6 +2451,31 @@
                 }
             }
         },
+        "/user-access/": {
+            "get": {
+                "tags": [
+                    "UserAccess"
+                ],
+                "summary": "Returns user permission status.",
+                "operationId": "listUserAccess",
+                "parameters": [{
+                        "$ref": "#/components/parameters/QueryType"
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "description": "| - 200 response",
+                        "content": {
+                            "application/json": {
+                                "schema": {
+                                    "$ref": "#/components/schemas/UserAccessListPagination"
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        },
         "/sources/": {
             "get": {
                 "tags": [
@@ -3822,6 +3892,14 @@
                     "type": "boolean"
                 }
             },
+            "QueryType": {
+                "name": "type",
+                "in": "query",
+                "description": "String to identify user access permission type (i.e. AWS, cost_model).",
+                "schema": {
+                    "type": "string"
+                }
+            },
             "QueryValue": {
                 "in": "query",
                 "name": "value",
@@ -4476,6 +4554,26 @@
                     }
                 ]
             },
+            "UserAccessListPagination": {
+                "allOf": [{
+                    "$ref": "#/components/schemas/ListPagination"
+                  },
+                  {
+                        "type": "object",
+                        "required": [
+                            "data"
+                        ],
+                        "properties": {
+                            "data": {
+                                "type": "array",
+                                "items": {
+                                    "$ref": "#/components/schemas/UserAccessTypeOut"
+                                }
+                            }
+                        }
+                    }
+                ]
+            },
             "ResourceTypeListPagination": {
                 "allOf": [{
                     "$ref": "#/components/schemas/ListPagination"
@@ -4523,6 +4621,16 @@
                     }
                 }
             },
+            "UserAccessTypeOut": {
+                "properties": {
+                    "type": {
+                        "type": "string"
+                    },
+                    "access": {
+                        "type": "boolean"
+                    }
+                }
+            },
             "ResourceTypeListOut": {
                 "properties": {
                     "value": {

koku/api/common/permissions/__init__.py

@@ -16,17 +16,20 @@
 #
 from api.common.permissions.aws_access import AwsAccessPermission
 from api.common.permissions.azure_access import AzureAccessPermission
+from api.common.permissions.gcp_access import GcpAccessPermission
 from api.common.permissions.openshift_access import OpenShiftAccessPermission
 from api.provider.models import Provider
 
 RESOURCE_TYPES = [
     AwsAccessPermission.resource_type,
     AzureAccessPermission.resource_type,
     OpenShiftAccessPermission.resource_type,
+    GcpAccessPermission.resource_type,
 ]
 
 RESOURCE_TYPE_MAP = {
     AwsAccessPermission.resource_type: [Provider.PROVIDER_AWS, Provider.PROVIDER_AWS_LOCAL],
     AzureAccessPermission.resource_type: [Provider.PROVIDER_AZURE, Provider.PROVIDER_AZURE_LOCAL],
     OpenShiftAccessPermission.resource_type: [Provider.PROVIDER_OCP],
+    GcpAccessPermission.resource_type: [Provider.PROVIDER_GCP, Provider.PROVIDER_GCP_LOCAL],
 }

koku/api/forecast/serializers.py

@@ -50,6 +50,10 @@ class AWSCostForecastParamSerializer(ForecastParamSerializer):
     """AWS Cost Forecast Serializer."""
 
 
+class GCPCostForecastParamSerializer(ForecastParamSerializer):
+    """GCP Cost Forecast Serializer."""
+
+
 class AzureCostForecastParamSerializer(ForecastParamSerializer):
     """Azure Cost Forecast Serializer."""
 

koku/api/forecast/views.py

@@ -27,22 +27,26 @@
 from api.common.pagination import ForecastListPaginator
 from api.common.permissions import AwsAccessPermission
 from api.common.permissions import AzureAccessPermission
+from api.common.permissions import GcpAccessPermission
 from api.common.permissions import OpenShiftAccessPermission
 from api.common.permissions.openshift_all_access import OpenshiftAllAccessPermission
 from api.forecast.serializers import AWSCostForecastParamSerializer
 from api.forecast.serializers import AzureCostForecastParamSerializer
+from api.forecast.serializers import GCPCostForecastParamSerializer
 from api.forecast.serializers import OCPAllCostForecastParamSerializer
 from api.forecast.serializers import OCPAWSCostForecastParamSerializer
 from api.forecast.serializers import OCPAzureCostForecastParamSerializer
 from api.forecast.serializers import OCPCostForecastParamSerializer
 from api.query_params import QueryParameters
 from forecast import AWSForecast
 from forecast import AzureForecast
+from forecast import GCPForecast
 from forecast import OCPAllForecast
 from forecast import OCPAWSForecast
 from forecast import OCPAzureForecast
 from forecast import OCPForecast
 from reporting.models import AzureTagsSummary
+from reporting.models import GCPTagsSummary
 from reporting.models import OCPAWSTagsSummary
 from reporting.models import OCPAzureTagsSummary
 from reporting.models import OCPStorageVolumeLabelSummary
@@ -128,3 +132,12 @@ class OCPAllCostForecastView(ForecastView):
     query_handler = OCPAllForecast
     serializer = OCPAllCostForecastParamSerializer
     tag_handler = [OCPAWSTagsSummary, OCPAzureTagsSummary]
+
+
+class GCPForecastCostView(ForecastView):
+    """GCP Cost Forecast View."""
+
+    permission_classes = (GcpAccessPermission,)
+    query_handler = GCPForecast
+    serializer = GCPCostForecastParamSerializer
+    tag_handler = [GCPTagsSummary]

koku/api/migrations/0034_remove_sources_endpoint_id.py

@@ -0,0 +1,9 @@
+# Generated by Django 3.1.3 on 2021-01-15 15:32
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [("api", "0033_sources_name_text")]
+
+    operations = [migrations.RemoveField(model_name="sources", name="endpoint_id")]

koku/api/organizations/queries.py

@@ -28,7 +28,6 @@
 from api.query_filter import QueryFilter
 from api.query_filter import QueryFilterCollection
 from api.query_handler import QueryHandler
-from api.utils import DateHelper
 
 
 LOG = logging.getLogger(__name__)
@@ -71,8 +70,6 @@ class OrgQueryHandler(QueryHandler):
     SUPPORTED_FILTERS = []
     FILTER_MAP = {}
 
-    dh = DateHelper()
-
     def __init__(self, parameters):
         """Establish org query handler.
 

koku/api/provider/models.py

@@ -190,9 +190,6 @@ class Meta:
     # Kafka message offset for Platform-Sources kafka stream
     offset = models.IntegerField(null=False)
 
-    # Endpoint ID.  Identifier to connect source to authentication.
-    endpoint_id = models.IntegerField(null=True)
-
     # Koku Specific data.
     # Customer Account ID
     account_id = models.TextField(null=True)

koku/api/query_handler.py

@@ -19,6 +19,7 @@
 import logging
 
 from dateutil import relativedelta
+from django.core.exceptions import FieldDoesNotExist
 from django.db.models.functions import TruncDay
 from django.db.models.functions import TruncMonth
 
@@ -59,6 +60,7 @@ def __init__(self, parameters):
 
         """
         LOG.debug(f"Query Params: {parameters}")
+        self.dh = DateHelper()
         parameters = self.filter_to_order_by(parameters)
         self.tenant = parameters.tenant
         self.access = parameters.access
@@ -215,25 +217,24 @@ def _get_timeframe(self):
         time_scope_units = self.get_time_scope_units()
         start = None
         end = None
-        dh = DateHelper()
         if time_scope_units == "month":
             if time_scope_value == -1:
                 # get current month
-                start = dh.this_month_start
-                end = dh.today
+                start = self.dh.this_month_start
+                end = self.dh.today
             else:
                 # get previous month
-                start = dh.last_month_start
-                end = dh.last_month_end
+                start = self.dh.last_month_start
+                end = self.dh.last_month_end
         else:
             if time_scope_value == -10:
                 # get last 10 days
-                start = dh.n_days_ago(dh.this_hour, 9)
-                end = dh.this_hour
+                start = self.dh.n_days_ago(self.dh.this_hour, 9)
+                end = self.dh.this_hour
             else:
                 # get last 30 days
-                start = dh.n_days_ago(dh.this_hour, 29)
-                end = dh.this_hour
+                start = self.dh.n_days_ago(self.dh.this_hour, 29)
+                end = self.dh.this_hour
 
         self.start_datetime = start
         self.end_datetime = end
@@ -344,5 +345,20 @@ def set_access_filters(self, access, filt, filters):
                 filters.add(q_filter)
         else:
             filt["operation"] = "in"
+            try:
+                check_field_type = None
+                if hasattr(self, "query_table"):
+                    # Reports APIs
+                    check_field_type = self.query_table._meta.get_field(filt.get("field", "")).get_internal_type()
+                elif hasattr(self, "data_sources"):
+                    # Tags APIs
+                    check_field_type = (
+                        self.data_sources[0].get("db_table")._meta.get_field(filt.get("field", "")).get_internal_type()
+                    )
+                if check_field_type == "ArrayField":
+                    filt["operation"] = "contains"
+            except FieldDoesNotExist:
+                pass
+
             q_filter = QueryFilter(parameter=access, **filt)
             filters.add(q_filter)

koku/api/report/all/openshift/query_handler.py

@@ -20,7 +20,7 @@
 from api.models import Provider
 from api.report.all.openshift.provider_map import OCPAllProviderMap
 from api.report.aws.openshift.query_handler import OCPInfrastructureReportQueryHandlerBase
-from api.report.queries import is_grouped_or_filtered_by_project
+from api.report.queries import is_grouped_by_project
 
 LOG = logging.getLogger(__name__)
 
@@ -61,7 +61,7 @@ def __init__(self, parameters):
         """
         self._mapper = OCPAllProviderMap(provider=self.provider, report_type=parameters.report_type)
         # Update which field is used to calculate cost by group by param.
-        if is_grouped_or_filtered_by_project(parameters):
+        if is_grouped_by_project(parameters):
             self._report_type = parameters.report_type + "_by_project"
             self._mapper = OCPAllProviderMap(provider=self.provider, report_type=self._report_type)