Adds log rate limit fields in bytes per second

Updates the `/v3/organization_quotas/` and
`/v3/space_quotas/` endpoints to allow setting and retrieving of a new
parameter (`log_rate_limit_in_bytes_per_second`). This will eventually
permit the user to set log line production limits in bytes per second,
rather than lines per second.

Updates v3/processes and v3/tasks endpoints to support
`log_rate_limit_in_bytes_per_second`

An unlimited log rate limit is represented as -1

Tracker Story ID: [#182311424]
Tracker Story ID: [#182353823]
Tracker Story ID: [#182311433]
Tracker Story ID: [#182624538]
Tracker Story ID: [#182624530]
Github Issue: cloudfoundry/capi-release#245

Signed-off-by: Carson Long <[email protected]>
Signed-off-by: Kenneth Lakin <[email protected]>
Signed-off-by: Duane May <[email protected]>
Signed-off-by: Matthew Kocher <[email protected]>
Signed-off-by: Ben Fuller <[email protected]>
Signed-off-by: Seth Boyles <[email protected]>
Co-authored-by: Matthew Kocher <[email protected]>

app/actions/organization_quotas_create.rb

@@ -16,6 +16,7 @@ def create(message)
           instance_memory_limit: message.per_process_memory_in_mb || QuotaDefinition::UNLIMITED,
           app_instance_limit: message.total_instances || QuotaDefinition::UNLIMITED,
           app_task_limit: message.per_app_tasks || QuotaDefinition::UNLIMITED,
+          log_rate_limit: message.log_rate_limit_in_bytes_per_second || QuotaDefinition::UNLIMITED,
 
           # Services
           total_services: message.total_service_instances || QuotaDefinition::DEFAULT_TOTAL_SERVICES,

app/actions/organization_quotas_update.rb

@@ -13,6 +13,7 @@ def self.update(quota, message)
         quota.instance_memory_limit = instance_memory_limit(message) if message.apps_limits_message.requested? :per_process_memory_in_mb
         quota.app_instance_limit = app_instance_limit(message) if message.apps_limits_message.requested? :total_instances
         quota.app_task_limit = app_task_limit(message) if message.apps_limits_message.requested? :per_app_tasks
+        quota.log_rate_limit = log_rate_limit(message) if message.apps_limits_message.requested? :log_rate_limit_in_bytes_per_second
 
         quota.total_services = total_services(message) if message.services_limits_message.requested? :total_service_instances
         quota.total_service_keys = total_service_keys(message) if message.services_limits_message.requested? :total_service_keys
@@ -56,6 +57,10 @@ def self.app_task_limit(message)
       default_if_nil(message.per_app_tasks, QuotaDefinition::UNLIMITED)
     end
 
+    def self.log_rate_limit(message)
+      default_if_nil(message.log_rate_limit_in_bytes_per_second, QuotaDefinition::UNLIMITED)
+    end
+
     def self.total_services(message)
       default_if_nil(message.total_service_instances, QuotaDefinition::UNLIMITED)
     end

app/actions/process_scale.rb

@@ -22,6 +22,7 @@ def scale
         @process.instances = @message.instances if @message.requested?(:instances)
         @process.memory = @message.memory_in_mb if @message.requested?(:memory_in_mb)
         @process.disk_quota = @message.disk_in_mb if @message.requested?(:disk_in_mb)
+        @process.log_rate_limit = @message.log_rate_limit_in_bytes_per_second if @message.requested?(:log_rate_limit_in_bytes_per_second)
 
         @process.save
 

app/actions/space_diff_manifest.rb

@@ -15,7 +15,7 @@ class << self
       def generate_diff(app_manifests, space)
         json_diff = []
         recognized_top_level_keys = AppManifestMessage.allowed_keys.map(&:to_s)
-        app_manifests = convert_byte_measurements_to_mb(app_manifests)
+        app_manifests = normalize_units(app_manifests)
         app_manifests.each_with_index do |manifest_app_hash, index|
           manifest_app_hash = filter_manifest_app_hash(manifest_app_hash)
           existing_app = space.app_models.find { |app| app.name == manifest_app_hash['name'] }
@@ -90,7 +90,7 @@ def filter_manifest_app_hash(manifest_app_hash)
               'memory'
             )
           end
-          manifest_app_hash['sidecars'] = convert_byte_measurements_to_mb(manifest_app_hash['sidecars'])
+          manifest_app_hash['sidecars'] = normalize_units(manifest_app_hash['sidecars'])
           manifest_app_hash = manifest_app_hash.except('sidecars') if manifest_app_hash['sidecars'] == [{}]
         end
         if manifest_app_hash.key? 'processes'
@@ -99,6 +99,7 @@ def filter_manifest_app_hash(manifest_app_hash)
               'type',
               'command',
               'disk_quota',
+              'log_rate_limit_per_second',
               'health-check-http-endpoint',
               'health-check-invocation-timeout',
               'health-check-type',
@@ -107,7 +108,7 @@ def filter_manifest_app_hash(manifest_app_hash)
               'timeout'
             )
           end
-          manifest_app_hash['processes'] = convert_byte_measurements_to_mb(manifest_app_hash['processes'])
+          manifest_app_hash['processes'] = normalize_units(manifest_app_hash['processes'])
           manifest_app_hash = manifest_app_hash.except('processes') if manifest_app_hash['processes'] == [{}]
         end
 
@@ -151,14 +152,22 @@ def create_similarity
         end
       end
 
-      def convert_byte_measurements_to_mb(manifest_app_hash)
+      def normalize_units(manifest_app_hash)
         byte_measurement_key_words = ['memory', 'disk-quota', 'disk_quota']
         manifest_app_hash.each_with_index do |process_hash, index|
           byte_measurement_key_words.each do |key|
             value = process_hash[key]
             manifest_app_hash[index][key] = convert_to_mb(value, key) unless value.nil?
           end
         end
+
+        byte_measurement_key_words = ['log_rate_limit_per_second']
+        manifest_app_hash.each_with_index do |process_hash, index|
+          byte_measurement_key_words.each do |key|
+            value = process_hash[key]
+            manifest_app_hash[index][key] = normalize_unit(value, key) unless value.nil?
+          end
+        end
         manifest_app_hash
       end
 
@@ -170,6 +179,14 @@ def convert_to_mb(human_readable_byte_value, attribute_name)
         "#{attribute_name} is not a number"
       end
 
+      def normalize_unit(non_normalized_value, attribute_name)
+        byte_converter.human_readable_byte_value(byte_converter.convert_to_b(non_normalized_value))
+      rescue ByteConverter::InvalidUnitsError
+        "#{attribute_name} must use a supported unit: B, K, KB, M, MB, G, GB, T, or TB"
+      rescue ByteConverter::NonNumericError
+        "#{attribute_name} is not a number"
+      end
+
       def byte_converter
         ByteConverter.new
       end

app/actions/space_quota_update.rb

@@ -14,6 +14,7 @@ def self.update(quota, message)
         quota.instance_memory_limit = instance_memory_limit(message) if message.apps_limits_message.requested? :per_process_memory_in_mb
         quota.app_instance_limit = app_instance_limit(message) if message.apps_limits_message.requested? :total_instances
         quota.app_task_limit = app_task_limit(message) if message.apps_limits_message.requested? :per_app_tasks
+        quota.log_rate_limit = log_rate_limit(message) if message.apps_limits_message.requested? :log_rate_limit_in_bytes_per_second
 
         quota.total_services = total_services(message) if message.services_limits_message.requested? :total_service_instances
         quota.total_service_keys = total_service_keys(message) if message.services_limits_message.requested? :total_service_keys
@@ -55,6 +56,10 @@ def self.app_task_limit(message)
       default_if_nil(message.per_app_tasks, SpaceQuotaDefinition::UNLIMITED)
     end
 
+    def self.log_rate_limit(message)
+      default_if_nil(message.log_rate_limit_in_bytes_per_second, SpaceQuotaDefinition::UNLIMITED)
+    end
+
     def self.total_services(message)
       default_if_nil(message.total_service_instances, SpaceQuotaDefinition::UNLIMITED)
     end

app/actions/space_quotas_create.rb

@@ -3,6 +3,7 @@ class SpaceQuotasCreate
     class Error < ::StandardError
     end
 
+    # rubocop:todo Metrics/CyclomaticComplexity
     def create(message, organization:)
       space_quota = nil
 
@@ -16,6 +17,7 @@ def create(message, organization:)
           instance_memory_limit: message.per_process_memory_in_mb || SpaceQuotaDefinition::UNLIMITED,
           app_instance_limit: message.total_instances || SpaceQuotaDefinition::UNLIMITED,
           app_task_limit: message.per_app_tasks || SpaceQuotaDefinition::UNLIMITED,
+          log_rate_limit: message.log_rate_limit_in_bytes_per_second || QuotaDefinition::UNLIMITED,
 
           # Services
           total_services: message.total_service_instances || SpaceQuotaDefinition::DEFAULT_TOTAL_SERVICES,
@@ -35,6 +37,7 @@ def create(message, organization:)
     rescue Sequel::ValidationFailed => e
       validation_error!(e, message)
     end
+    # rubocop:enable Metrics/CyclomaticComplexity
 
     private
 

app/actions/task_create.rb

@@ -29,6 +29,7 @@ def create(app, message, user_audit_info, droplet: nil)
           command:               command(message, template_process),
           disk_in_mb:            disk_in_mb(message, template_process),
           memory_in_mb:          memory_in_mb(message, template_process),
+          log_rate_limit:        log_rate_limit(message, template_process),
           sequence_id:           app.max_task_sequence_id
         )
 
@@ -70,15 +71,15 @@ def command(message, template_process)
     end
 
     def memory_in_mb(message, template_process)
-      return message.memory_in_mb if message.memory_in_mb
+      message.memory_in_mb || template_process.try(:memory) || config.get(:default_app_memory)
+    end
 
-      template_process.present? ? template_process.memory : config.get(:default_app_memory)
+    def log_rate_limit(message, template_process)
+      message.log_rate_limit_in_bytes_per_second || template_process.try(:log_rate_limit) || config.get(:default_app_log_rate_limit_in_bytes_per_second)
     end
 
     def disk_in_mb(message, template_process)
-      return message.disk_in_mb if message.disk_in_mb
-
-      template_process.present? ? template_process.disk_quota : config.get(:default_app_disk_in_mb)
+      message.disk_in_mb || template_process.try(:disk_quota) || config.get(:default_app_disk_in_mb)
     end
 
     def submit_task(task)

app/controllers/runtime/apps_controller.rb

@@ -33,6 +33,7 @@ def self.dependencies
       attribute :docker_credentials, Hash, default: {}
       attribute :debug, String, default: nil
       attribute :disk_quota, Integer, default: nil
+      attribute :log_rate_limit, Integer, default: nil
       attribute :environment_json, Hash, default: {}, redact_in: [:create, :update]
       attribute :health_check_http_endpoint, String, default: nil
       attribute :health_check_type, String, default: 'port'

app/controllers/v3/processes_controller.rb

@@ -89,6 +89,7 @@ def scale
         'instance-count' => message.instances,
         'memory-in-mb' => message.memory_in_mb,
         'disk-in-mb' => message.disk_in_mb,
+        'log-rate-in-bytes-per-second' => message.log_rate_limit_in_bytes_per_second,
         'process-type' => @process.type
       }
     )

app/messages/app_manifest_message.rb

@@ -16,6 +16,7 @@ class AppManifestMessage < BaseMessage
       :buildpacks,
       :command,
       :disk_quota,
+      :log_rate_limit_per_second,
       :docker,
       :env,
       :health_check_http_endpoint,
@@ -130,12 +131,16 @@ def manifest_buildpack_message
     end
 
     def process_scale_attribute_mappings
-      process_scale_attributes_from_app_level = process_scale_attributes(memory: memory, disk_quota: disk_quota, instances: instances)
+      process_scale_attributes_from_app_level = process_scale_attributes(memory: memory,
+                                                                         disk_quota: disk_quota,
+                                                                         log_rate_limit_per_second: log_rate_limit_per_second,
+                                                                         instances: instances)
 
       process_attributes(process_scale_attributes_from_app_level) do |process|
         process_scale_attributes(
           memory: process[:memory],
           disk_quota: process[:disk_quota],
+          log_rate_limit_per_second: process[:log_rate_limit_per_second],
           instances: process[:instances],
           type: process[:type]
         )
@@ -163,13 +168,15 @@ def process_attributes(app_attributes)
       process_attributes
     end
 
-    def process_scale_attributes(memory: nil, disk_quota: nil, instances:, type: nil)
+    def process_scale_attributes(memory: nil, disk_quota: nil, log_rate_limit_per_second: nil, instances:, type: nil)
       memory_in_mb = convert_to_mb(memory)
       disk_in_mb = convert_to_mb(disk_quota)
+      log_rate_limit_in_bytes_per_second = convert_to_bytes_per_second(log_rate_limit_per_second)
       {
         instances: instances,
         memory: memory_in_mb,
         disk_quota: disk_in_mb,
+        log_rate_limit: log_rate_limit_in_bytes_per_second,
         type: type
       }.compact
     end
@@ -283,6 +290,13 @@ def convert_to_mb(human_readable_byte_value)
     rescue ByteConverter::InvalidUnitsError, ByteConverter::NonNumericError
     end
 
+    def convert_to_bytes_per_second(human_readable_byte_value)
+      return nil unless human_readable_byte_value.present?
+
+      byte_converter.convert_to_b(human_readable_byte_value.strip)
+    rescue ByteConverter::InvalidUnitsError, ByteConverter::NonNumericError
+    end
+
     def validate_byte_format(human_readable_byte_value, attribute_name)
       byte_converter.convert_to_mb(human_readable_byte_value)
 
@@ -378,8 +392,10 @@ def validate_processes!
         type = process[:type]
         memory_error = validate_byte_format(process[:memory], 'Memory')
         disk_error = validate_byte_format(process[:disk_quota], 'Disk quota')
+        log_rate_limit_error = validate_byte_format(process[:log_rate_limit_per_second], 'Log rate limit per second')
         add_process_error!(memory_error, type) if memory_error
         add_process_error!(disk_error, type) if disk_error
+        add_process_error!(log_rate_limit_error, type) if log_rate_limit_error
       end
     end
 
@@ -400,8 +416,10 @@ def validate_sidecars!
     def validate_top_level_web_process!
       memory_error = validate_byte_format(memory, 'Memory')
       disk_error = validate_byte_format(disk_quota, 'Disk quota')
+      log_rate_limit_error = validate_byte_format(log_rate_limit_per_second, 'Log rate limit per second')
       add_process_error!(memory_error, ProcessTypes::WEB) if memory_error
       add_process_error!(disk_error, ProcessTypes::WEB) if disk_error
+      add_process_error!(log_rate_limit_error, ProcessTypes::WEB) if log_rate_limit_error
     end
 
     def validate_buildpack_and_buildpacks_combination!

app/messages/base_message.rb

@@ -8,6 +8,7 @@ class BaseMessage
     include Validators
 
     MAX_DB_INT = 2**31 - 1
+    MAX_DB_BIGINT = 2**63 - 1
 
     attr_accessor :requested_keys, :extra_keys
 

app/messages/manifest_process_scale_message.rb

@@ -2,20 +2,24 @@
 
 module VCAP::CloudController
   class ManifestProcessScaleMessage < BaseMessage
-    register_allowed_keys [:instances, :memory, :disk_quota, :type]
+    register_allowed_keys [:instances, :memory, :disk_quota, :log_rate_limit, :type]
     INVALID_MB_VALUE_ERROR = 'must be greater than 0MB'.freeze
+    # NOTE: -1 is valid for log_rate_limit representing unlimited
+    INVALID_QUOTA_VALUE_ERROR = 'must be an integer greater than or equal to -1'.freeze
 
     validates_with NoAdditionalKeysValidator
 
     validates :instances, numericality: { only_integer: true, greater_than_or_equal_to: 0, less_than_or_equal_to: 2000000 }, allow_nil: true
     validates :memory, numericality: { only_integer: true, greater_than: 0, message: INVALID_MB_VALUE_ERROR }, allow_nil: true
     validates :disk_quota, numericality: { only_integer: true, greater_than: 0, message: INVALID_MB_VALUE_ERROR }, allow_nil: true
+    validates :log_rate_limit, numericality: { only_integer: true, greater_than_or_equal_to: -1, message: INVALID_QUOTA_VALUE_ERROR }, allow_nil: true
 
     def to_process_scale_message
       ProcessScaleMessage.new({
         instances: instances,
         memory_in_mb: memory,
         disk_in_mb: disk_quota,
+        log_rate_limit_in_bytes_per_second: log_rate_limit,
       }.compact)
     end
   end

app/messages/organization_quotas_update_message.rb

@@ -26,7 +26,7 @@ def self.key_requested?(key)
     validate :domains_validator, if: key_requested?(:domains)
 
     # Apps validations
-    delegate :total_memory_in_mb, :per_process_memory_in_mb, :total_instances, :per_app_tasks, to: :apps_limits_message
+    delegate :total_memory_in_mb, :per_process_memory_in_mb, :total_instances, :per_app_tasks, :log_rate_limit_in_bytes_per_second, to: :apps_limits_message
 
     def validates_hash(key, sym)
       return true if key.is_a?(Hash)

app/messages/process_scale_message.rb

@@ -2,12 +2,13 @@
 
 module VCAP::CloudController
   class ProcessScaleMessage < BaseMessage
-    register_allowed_keys [:instances, :memory_in_mb, :disk_in_mb]
+    register_allowed_keys [:instances, :memory_in_mb, :disk_in_mb, :log_rate_limit_in_bytes_per_second]
 
     validates_with NoAdditionalKeysValidator
 
     validates :instances, numericality: { only_integer: true, greater_than_or_equal_to: 0, less_than_or_equal_to: MAX_DB_INT }, allow_nil: true
     validates :memory_in_mb, numericality: { only_integer: true, greater_than: 0, less_than_or_equal_to: MAX_DB_INT }, allow_nil: true
     validates :disk_in_mb, numericality: { only_integer: true, greater_than: 0, less_than_or_equal_to: MAX_DB_INT }, allow_nil: true
+    validates :log_rate_limit_in_bytes_per_second, numericality: { only_integer: true, greater_than_or_equal_to: -1, less_than_or_equal_to: MAX_DB_BIGINT }, allow_nil: true
   end
 end

app/messages/quotas_apps_message.rb

@@ -3,7 +3,7 @@
 
 module VCAP::CloudController
   class QuotasAppsMessage < BaseMessage
-    register_allowed_keys [:total_memory_in_mb, :per_process_memory_in_mb, :total_instances, :per_app_tasks]
+    register_allowed_keys [:total_memory_in_mb, :per_process_memory_in_mb, :total_instances, :per_app_tasks, :log_rate_limit_in_bytes_per_second]
 
     validates_with NoAdditionalKeysValidator
 
@@ -22,5 +22,9 @@ class QuotasAppsMessage < BaseMessage
     validates :per_app_tasks,
       numericality: { only_integer: true, greater_than_or_equal_to: 0, less_than_or_equal_to: MAX_DB_INT },
       allow_nil: true
+
+    validates :log_rate_limit_in_bytes_per_second,
+      numericality: { only_integer: true, greater_than_or_equal_to: 0, less_than_or_equal_to: MAX_DB_BIGINT },
+      allow_nil: true
   end
 end

app/messages/space_quota_update_message.rb

@@ -22,7 +22,7 @@ def self.key_requested?(key)
     validate :services_validator, if: key_requested?(:services)
     validate :routes_validator, if: key_requested?(:routes)
 
-    delegate :total_memory_in_mb, :per_process_memory_in_mb, :total_instances, :per_app_tasks, to: :apps_limits_message
+    delegate :total_memory_in_mb, :per_process_memory_in_mb, :total_instances, :per_app_tasks, :log_rate_limit_in_bytes_per_second, to: :apps_limits_message
     delegate :paid_services_allowed, :total_service_instances, :total_service_keys, to: :services_limits_message
     delegate :total_routes, :total_reserved_ports, to: :routes_limits_message
 

app/messages/task_create_message.rb

@@ -2,7 +2,7 @@
 
 module VCAP::CloudController
   class TaskCreateMessage < MetadataBaseMessage
-    register_allowed_keys [:name, :command, :disk_in_mb, :memory_in_mb, :droplet_guid, :template]
+    register_allowed_keys [:name, :command, :disk_in_mb, :memory_in_mb, :log_rate_limit_in_bytes_per_second, :droplet_guid, :template]
 
     validates_with NoAdditionalKeysValidator
 
@@ -12,6 +12,7 @@ def self.validate_template?
 
     validates :disk_in_mb, numericality: { only_integer: true, greater_than: 0 }, allow_nil: true
     validates :memory_in_mb, numericality: { only_integer: true, greater_than: 0 }, allow_nil: true
+    validates :log_rate_limit_in_bytes_per_second, numericality: { only_integer: true, greater_than: -2 }, allow_nil: true
     validates :droplet_guid, guid: true, allow_nil: true
     validates :template_process_guid, guid: true, if: validate_template?
     validate :has_command

app/models.rb

@@ -47,6 +47,8 @@
 require 'models/runtime/quota_constraints/max_service_keys_policy'
 require 'models/runtime/constraints/max_disk_quota_policy'
 require 'models/runtime/constraints/min_disk_quota_policy'
+require 'models/runtime/constraints/min_log_rate_limit_policy'
+require 'models/runtime/constraints/max_log_rate_limit_policy'
 require 'models/runtime/constraints/max_memory_policy'
 require 'models/runtime/constraints/max_instance_memory_policy'
 require 'models/runtime/constraints/min_memory_policy'