Rc 2021.03.29

.pre-commit-config.yaml

@@ -1,5 +1,5 @@
 default_language_version:
-  python: python3.8
+  python: python3
 repos:
 - repo: https://github.com/asottile/reorder_python_imports
   rev: v1.6.0

deploy/presto/presto-coordinator-config/jvm.config

@@ -8,7 +8,7 @@
 
 -XX:InitialRAMPercentage=50.0
 
--XX:MaxRAMPercentage=50.0
+-XX:MaxRAMPercentage=75.0
 
 -XX:+ExplicitGCInvokesConcurrent
 

docker-compose.yml

@@ -385,6 +385,7 @@ services:
         - GOOGLE_APPLICATION_CREDENTIALS=${GOOGLE_APPLICATION_CREDENTIALS}
         - DEMO_ACCOUNTS
         - prometheus_multiproc_dir=/tmp
+        - AUTO_DATA_INGEST=${AUTO_DATA_INGEST-True}
       privileged: true
       ports:
           - 4000:8080

koku/api/report/aws/openshift/query_handler.py

@@ -19,9 +19,7 @@
 import logging
 
 from django.db.models import F
-from django.db.models import Window
 from django.db.models.functions import Coalesce
-from django.db.models.functions import RowNumber
 from tenant_schemas.utils import tenant_context
 
 from api.models import Provider
@@ -51,7 +49,7 @@ def execute_query(self):  # noqa: C901
             group_by_value = self._get_group_by()
             query_group_by = ["date"] + group_by_value
             query_order_by = ["-date"]
-            query_order_by.extend([self.order])
+            query_order_by.extend([self.order])  # add implicit ordering
 
             annotations = self._mapper.report_type_map.get("annotations")
             query_data = query_data.values(*query_group_by).annotate(**annotations)
@@ -62,15 +60,10 @@ def execute_query(self):  # noqa: C901
                 )
 
             if self._limit and query_data:
-                rank_orders = []
-                if self.order_field == "delta":
-                    rank_orders.append(getattr(F(self._delta), self.order_direction)())
-                else:
-                    rank_orders.append(getattr(F(self.order_field), self.order_direction)())
-                rank_by_total = Window(expression=RowNumber(), partition_by=F("date"), order_by=rank_orders)
-                query_data = query_data.annotate(rank=rank_by_total)
-                query_order_by.insert(1, "rank")
-                query_data = self._ranked_list(query_data)
+                query_data = self._group_by_ranks(query, query_data)
+                if not self.parameters.get("order_by"):
+                    # override implicit ordering when using ranked ordering.
+                    query_order_by[-1] = "rank"
 
             if query.exists():
                 aggregates = self._mapper.report_type_map.get("aggregates")

koku/api/report/aws/query_handler.py

@@ -24,10 +24,8 @@
 from django.db.models import F
 from django.db.models import Q
 from django.db.models import Value
-from django.db.models import Window
 from django.db.models.expressions import Func
 from django.db.models.functions import Coalesce
-from django.db.models.functions import RowNumber
 from tenant_schemas.utils import tenant_context
 
 from api.models import Provider
@@ -594,7 +592,7 @@ def execute_individual_query(self, org_unit_applied=False):  # noqa: C901
             query_data = query.annotate(**self.annotations)
             query_group_by = ["date"] + self._get_group_by()
             query_order_by = ["-date"]
-            query_order_by.extend([self.order])
+            query_order_by.extend([self.order])  # add implicit ordering
 
             annotations = copy.deepcopy(self._mapper.report_type_map.get("annotations", {}))
             if not self.parameters.parameters.get("compute_count"):
@@ -615,15 +613,10 @@ def execute_individual_query(self, org_unit_applied=False):  # noqa: C901
             query_sum = self._build_sum(query, annotations)
 
             if self._limit and query_data and not org_unit_applied:
-                rank_orders = []
-                if self.order_field == "delta":
-                    rank_orders.append(getattr(F(self._delta), self.order_direction)())
-                else:
-                    rank_orders.append(getattr(F(self.order_field), self.order_direction)())
-                rank_by_total = Window(expression=RowNumber(), partition_by=F("date"), order_by=rank_orders)
-                query_data = query_data.annotate(rank=rank_by_total)
-                query_order_by.insert(1, "rank")
-                query_data = self._ranked_list(query_data)
+                query_data = self._group_by_ranks(query, query_data)
+                if not self.parameters.get("order_by"):
+                    # override implicit ordering when using ranked ordering.
+                    query_order_by[-1] = "rank"
 
             if self._delta:
                 query_data = self.add_deltas(query_data, query_sum)

koku/api/report/azure/openshift/query_handler.py

@@ -19,8 +19,6 @@
 import logging
 
 from django.db.models import F
-from django.db.models import Window
-from django.db.models.functions import RowNumber
 from tenant_schemas.utils import tenant_context
 
 from api.models import Provider
@@ -94,17 +92,16 @@ def execute_query(self):  # noqa: C901
             group_by_value = self._get_group_by()
             query_group_by = ["date"] + group_by_value
             query_order_by = ["-date"]
-            query_order_by.extend([self.order])
+            query_order_by.extend([self.order])  # add implicit ordering
 
             annotations = self._mapper.report_type_map.get("annotations")
             query_data = query_data.values(*query_group_by).annotate(**annotations)
 
-            if self._limit:
-                rank_order = getattr(F(self.order_field), self.order_direction)()
-                rank_by_total = Window(expression=RowNumber(), partition_by=F("date"), order_by=rank_order)
-                query_data = query_data.annotate(rank=rank_by_total)
-                query_order_by.insert(1, "rank")
-                query_data = self._ranked_list(query_data)
+            if self._limit and query_data:
+                query_data = self._group_by_ranks(query, query_data)
+                if not self.parameters.get("order_by"):
+                    # override implicit ordering when using ranked ordering.
+                    query_order_by[-1] = "rank"
 
             if query.exists():
                 aggregates = self._mapper.report_type_map.get("aggregates")

koku/api/report/azure/query_handler.py

@@ -18,12 +18,9 @@
 import copy
 import logging
 
-from django.db.models import F
 from django.db.models import Value
-from django.db.models import Window
 from django.db.models.functions import Coalesce
 from django.db.models.functions import Concat
-from django.db.models.functions import RowNumber
 from tenant_schemas.utils import tenant_context
 
 from api.models import Provider
@@ -155,18 +152,17 @@ def execute_query(self):
             query_data = query.annotate(**self.annotations)
             query_group_by = ["date"] + self._get_group_by()
             query_order_by = ["-date"]
-            query_order_by.extend([self.order])
+            query_order_by.extend([self.order])  # add implicit ordering
 
             annotations = self._mapper.report_type_map.get("annotations")
             query_data = query_data.values(*query_group_by).annotate(**annotations)
             query_sum = self._build_sum(query)
 
-            if self._limit:
-                rank_order = getattr(F(self.order_field), self.order_direction)()
-                rank_by_total = Window(expression=RowNumber(), partition_by=F("date"), order_by=rank_order)
-                query_data = query_data.annotate(rank=rank_by_total)
-                query_order_by.insert(1, "rank")
-                query_data = self._ranked_list(query_data)
+            if self._limit and query_data:
+                query_data = self._group_by_ranks(query, query_data)
+                if not self.parameters.get("order_by"):
+                    # override implicit ordering when using ranked ordering.
+                    query_order_by[-1] = "rank"
 
             if self._delta:
                 query_data = self.add_deltas(query_data, query_sum)

koku/api/report/gcp/query_handler.py

@@ -20,10 +20,8 @@
 
 from django.db.models import F
 from django.db.models import Value
-from django.db.models import Window
 from django.db.models.functions import Coalesce
 from django.db.models.functions import Concat
-from django.db.models.functions import RowNumber
 from tenant_schemas.utils import tenant_context
 
 from api.models import Provider
@@ -163,7 +161,7 @@ def execute_query(self):
             query_data = query.annotate(**self.annotations)
             query_group_by = ["date"] + self._get_group_by()
             query_order_by = ["-date"]
-            query_order_by.extend([self.order])
+            query_order_by.extend([self.order])  # add implicit ordering
 
             annotations = self._mapper.report_type_map.get("annotations")
             for alias_key, alias_value in self.group_by_alias.items():
@@ -173,11 +171,10 @@ def execute_query(self):
             query_sum = self._build_sum(query)
 
             if self._limit:
-                rank_order = getattr(F(self.order_field), self.order_direction)()
-                rank_by_total = Window(expression=RowNumber(), partition_by=F("date"), order_by=rank_order)
-                query_data = query_data.annotate(rank=rank_by_total)
-                query_order_by.insert(1, "rank")
-                query_data = self._ranked_list(query_data)
+                query_data = self._group_by_ranks(query, query_data)
+                if not self.parameters.get("order_by"):
+                    # override implicit ordering when using ranked ordering.
+                    query_order_by[-1] = "rank"
 
             if self._delta:
                 query_data = self.add_deltas(query_data, query_sum)

koku/api/report/ocp/query_handler.py

@@ -24,8 +24,6 @@
 from decimal import InvalidOperation
 
 from django.db.models import F
-from django.db.models import Window
-from django.db.models.functions import RowNumber
 from tenant_schemas.utils import tenant_context
 
 from api.models import Provider
@@ -84,11 +82,6 @@ def annotations(self):
 
         return annotations
 
-    @property
-    def report_annotations(self):
-        """Return annotations with the correct capacity field."""
-        return self._mapper.report_type_map.get("annotations")
-
     def _format_query_response(self):
         """Format the query response with data.
 
@@ -124,15 +117,15 @@ def execute_query(self):
 
             query_group_by = ["date"] + group_by_value
             query_order_by = ["-date"]
-            query_order_by.extend([self.order])
+            query_order_by.extend([self.order])  # add implicit ordering
 
             query_data = query_data.values(*query_group_by).annotate(**self.report_annotations)
 
-            if self._limit and group_by_value:
-                rank_by_total = self.get_rank_window_function(group_by_value)
-                query_data = query_data.annotate(rank=rank_by_total)
-                query_order_by.insert(1, "rank")
-                query_data = self._ranked_list(query_data)
+            if self._limit and query_data:
+                query_data = self._group_by_ranks(query, query_data)
+                if not self.parameters.get("order_by"):
+                    # override implicit ordering when using ranked ordering.
+                    query_order_by[-1] = "rank"
 
             # Populate the 'total' section of the API response
             if query.exists():
@@ -177,25 +170,6 @@ def execute_query(self):
         self.query_data = data
         return self._format_query_response()
 
-    def get_rank_window_function(self, group_by_value):
-        """Generate a limit ranking window function."""
-        tag_column = self._mapper.tag_column
-        rank_orders = []
-        rank_field = group_by_value.pop()
-        default_ordering = self._mapper.report_type_map.get("default_ordering")
-
-        if self.order_field == "delta" and "__" in self._delta:
-            delta_field_one, delta_field_two = self._delta.split("__")
-            rank_orders.append(getattr(F(delta_field_one) / F(delta_field_two), self.order_direction)())
-        elif self.parameters.get("order_by", default_ordering):
-            rank_orders.append(getattr(F(self.order_field), self.order_direction)())
-        if tag_column in rank_field:
-            rank_orders.append(self.get_tag_order_by(rank_field))
-        else:
-            rank_orders.append(getattr(F(rank_field), self.order_direction)())
-
-        return Window(expression=RowNumber(), partition_by=F("date"), order_by=rank_orders)
-
     def get_cluster_capacity(self, query_data):  # noqa: C901
         """Calculate cluster capacity for all nodes over the date range."""
         annotations = self._mapper.report_type_map.get("capacity_aggregate")