From 6f115cdf6cf9493f8e2e239a76b50843f3364d38 Mon Sep 17 00:00:00 2001 From: Carson Long Date: Mon, 6 Jun 2022 18:13:21 +0000 Subject: [PATCH 01/20] Adds log rate limit fields in bytes per second Updates the `/v3/organization_quotas/` and `/v3/space_quotas/` endpoints to allow setting and retrieving of a new parameter (`log_rate_limit_in_bytes_per_second`). This will eventually permit the user to set log line production limits in bytes per second, rather than lines per second. Updates v3/processes and v3/tasks endpoints to support `log_rate_limit_in_bytes_per_second` An unlimited log rate limit is represented as -1 Tracker Story ID: [#182311424] Tracker Story ID: [#182353823] Tracker Story ID: [#182311433] Tracker Story ID: [#182624538] Tracker Story ID: [#182624530] Github Issue: https://github.com/cloudfoundry/capi-release/issues/245 Signed-off-by: Carson Long Signed-off-by: Kenneth Lakin Signed-off-by: Duane May Signed-off-by: Matthew Kocher Signed-off-by: Ben Fuller Signed-off-by: Seth Boyles Co-authored-by: Matthew Kocher --- app/actions/organization_quotas_create.rb | 1 + app/actions/organization_quotas_update.rb | 5 + app/actions/process_scale.rb | 1 + app/actions/space_diff_manifest.rb | 25 ++- app/actions/space_quota_update.rb | 5 + app/actions/space_quotas_create.rb | 3 + app/actions/task_create.rb | 11 +- app/controllers/runtime/apps_controller.rb | 1 + app/controllers/v3/processes_controller.rb | 1 + app/messages/app_manifest_message.rb | 22 ++- app/messages/base_message.rb | 1 + .../manifest_process_scale_message.rb | 6 +- .../organization_quotas_update_message.rb | 2 +- app/messages/process_scale_message.rb | 3 +- app/messages/quotas_apps_message.rb | 6 +- app/messages/space_quota_update_message.rb | 2 +- app/messages/task_create_message.rb | 3 +- app/models.rb | 2 + .../constraints/max_log_rate_limit_policy.rb | 78 +++++++++ .../constraints/min_log_rate_limit_policy.rb | 16 ++ app/models/runtime/organization.rb | 22 ++- app/models/runtime/process_model.rb | 22 ++- app/models/runtime/quota_definition.rb | 25 +-- app/models/runtime/space.rb | 26 +++ app/models/runtime/space_quota_definition.rb | 20 ++- app/models/runtime/task_model.rb | 4 + app/presenters/v2/process_model_presenter.rb | 1 + .../process_properties_presenter.rb | 13 ++ .../v3/organization_quota_presenter.rb | 1 + app/presenters/v3/process_presenter.rb | 17 +- app/presenters/v3/process_stats_presenter.rb | 2 + app/presenters/v3/space_quota_presenter.rb | 1 + app/presenters/v3/task_presenter.rb | 1 + config/bosh-lite.yml | 1 + config/cloud_controller.yml | 1 + ...add_log_rate_limit_to_quota_definitions.rb | 5 + ...g_rate_limit_to_space_quota_definitions.rb | 5 + ...8165055_add_log_rate_limit_to_processes.rb | 5 + ...75820_add_log_rate_limit_to_tasks_table.rb | 5 + .../app_manifest/byte_converter.rb | 30 ++++ .../process_log_rate_limit_calculator.rb | 51 ++++++ .../config_schemas/base/api_schema.rb | 1 + .../config_schemas/base/clock_schema.rb | 1 + .../config_schemas/base/worker_schema.rb | 1 + spec/request/app_manifests_spec.rb | 6 + spec/request/apps_spec.rb | 146 ++++++++++++++++ spec/request/organization_quotas_spec.rb | 23 ++- spec/request/processes_spec.rb | 115 +++++++++++-- spec/request/space_manifests_spec.rb | 57 ++++++ spec/request/space_quotas_spec.rb | 23 ++- spec/request/tasks_spec.rb | 124 ++++++++++++-- spec/request/v2/apps_spec.rb | 11 ++ spec/request/v2/routes_spec.rb | 2 + spec/request/v2/service_bindings_spec.rb | 1 + spec/request/v2/spaces_spec.rb | 1 + .../organization_quotas_create_spec.rb | 5 +- .../organization_quotas_update_spec.rb | 5 +- spec/unit/actions/process_scale_spec.rb | 30 +++- spec/unit/actions/space_diff_manifest_spec.rb | 8 +- spec/unit/actions/space_quota_update_spec.rb | 5 +- spec/unit/actions/space_quotas_create_spec.rb | 3 + spec/unit/actions/task_create_spec.rb | 51 +++++- .../runtime/apps_controller_spec.rb | 2 + .../app_manifest/byte_converter_spec.rb | 72 ++++++++ .../process_log_rate_limit_calculator_spec.rb | 132 ++++++++++++++ .../messages/app_manifest_message_spec.rb | 67 +++++++- .../manifest_process_scale_message_spec.rb | 51 +++++- ...organization_quotas_create_message_spec.rb | 68 ++++++++ .../messages/process_scale_message_spec.rb | 48 ++++++ .../unit/messages/quotas_apps_message_spec.rb | 62 +++++++ .../space_quota_update_message_spec.rb | 2 + .../unit/messages/task_create_message_spec.rb | 53 ++++++ .../max_log_rate_limit_policy_spec.rb | 162 ++++++++++++++++++ .../min_log_rate_limit_policy_spec.rb | 27 +++ spec/unit/models/runtime/organization_spec.rb | 53 ++++++ .../unit/models/runtime/process_model_spec.rb | 88 +++++++++- .../models/runtime/quota_definition_spec.rb | 15 +- .../runtime/space_quota_definition_spec.rb | 15 +- spec/unit/models/runtime/space_spec.rb | 51 ++++++ spec/unit/models/runtime/task_model_spec.rb | 85 +++++++++ .../v2/process_model_presenter_spec.rb | 1 + .../v3/app_manifest_presenter_spec.rb | 5 + .../process_properties_presenter_spec.rb | 26 +++ .../presenters/v3/process_presenter_spec.rb | 9 + .../v3/process_stats_presenter_spec.rb | 16 +- .../v3/space_quota_presenter_spec.rb | 4 + .../unit/presenters/v3/task_presenter_spec.rb | 2 + 87 files changed, 2066 insertions(+), 121 deletions(-) create mode 100644 app/models/runtime/constraints/max_log_rate_limit_policy.rb create mode 100644 app/models/runtime/constraints/min_log_rate_limit_policy.rb create mode 100644 db/migrations/20220606172913_add_log_rate_limit_to_quota_definitions.rb create mode 100644 db/migrations/20220607222309_add_log_rate_limit_to_space_quota_definitions.rb create mode 100644 db/migrations/20220608165055_add_log_rate_limit_to_processes.rb create mode 100644 db/migrations/20220621175820_add_log_rate_limit_to_tasks_table.rb create mode 100644 lib/cloud_controller/app_services/process_log_rate_limit_calculator.rb create mode 100644 spec/unit/lib/cloud_controller/app_services/process_log_rate_limit_calculator_spec.rb create mode 100644 spec/unit/models/runtime/constraints/max_log_rate_limit_policy_spec.rb create mode 100644 spec/unit/models/runtime/constraints/min_log_rate_limit_policy_spec.rb diff --git a/app/actions/organization_quotas_create.rb b/app/actions/organization_quotas_create.rb index 07e0096dc04..cf3f7e85815 100644 --- a/app/actions/organization_quotas_create.rb +++ b/app/actions/organization_quotas_create.rb @@ -16,6 +16,7 @@ def create(message) instance_memory_limit: message.per_process_memory_in_mb || QuotaDefinition::UNLIMITED, app_instance_limit: message.total_instances || QuotaDefinition::UNLIMITED, app_task_limit: message.per_app_tasks || QuotaDefinition::UNLIMITED, + log_rate_limit: message.log_rate_limit_in_bytes_per_second || QuotaDefinition::UNLIMITED, # Services total_services: message.total_service_instances || QuotaDefinition::DEFAULT_TOTAL_SERVICES, diff --git a/app/actions/organization_quotas_update.rb b/app/actions/organization_quotas_update.rb index 0cf74c7d98f..12f7cfcd35c 100644 --- a/app/actions/organization_quotas_update.rb +++ b/app/actions/organization_quotas_update.rb @@ -13,6 +13,7 @@ def self.update(quota, message) quota.instance_memory_limit = instance_memory_limit(message) if message.apps_limits_message.requested? :per_process_memory_in_mb quota.app_instance_limit = app_instance_limit(message) if message.apps_limits_message.requested? :total_instances quota.app_task_limit = app_task_limit(message) if message.apps_limits_message.requested? :per_app_tasks + quota.log_rate_limit = log_rate_limit(message) if message.apps_limits_message.requested? :log_rate_limit_in_bytes_per_second quota.total_services = total_services(message) if message.services_limits_message.requested? :total_service_instances quota.total_service_keys = total_service_keys(message) if message.services_limits_message.requested? :total_service_keys @@ -56,6 +57,10 @@ def self.app_task_limit(message) default_if_nil(message.per_app_tasks, QuotaDefinition::UNLIMITED) end + def self.log_rate_limit(message) + default_if_nil(message.log_rate_limit_in_bytes_per_second, QuotaDefinition::UNLIMITED) + end + def self.total_services(message) default_if_nil(message.total_service_instances, QuotaDefinition::UNLIMITED) end diff --git a/app/actions/process_scale.rb b/app/actions/process_scale.rb index d13362fb9d9..2f6763a516b 100644 --- a/app/actions/process_scale.rb +++ b/app/actions/process_scale.rb @@ -22,6 +22,7 @@ def scale @process.instances = @message.instances if @message.requested?(:instances) @process.memory = @message.memory_in_mb if @message.requested?(:memory_in_mb) @process.disk_quota = @message.disk_in_mb if @message.requested?(:disk_in_mb) + @process.log_rate_limit = @message.log_rate_limit_in_bytes_per_second if @message.requested?(:log_rate_limit_in_bytes_per_second) @process.save diff --git a/app/actions/space_diff_manifest.rb b/app/actions/space_diff_manifest.rb index 5a860535b46..2963896dd85 100644 --- a/app/actions/space_diff_manifest.rb +++ b/app/actions/space_diff_manifest.rb @@ -15,7 +15,7 @@ class << self def generate_diff(app_manifests, space) json_diff = [] recognized_top_level_keys = AppManifestMessage.allowed_keys.map(&:to_s) - app_manifests = convert_byte_measurements_to_mb(app_manifests) + app_manifests = normalize_units(app_manifests) app_manifests.each_with_index do |manifest_app_hash, index| manifest_app_hash = filter_manifest_app_hash(manifest_app_hash) existing_app = space.app_models.find { |app| app.name == manifest_app_hash['name'] } @@ -90,7 +90,7 @@ def filter_manifest_app_hash(manifest_app_hash) 'memory' ) end - manifest_app_hash['sidecars'] = convert_byte_measurements_to_mb(manifest_app_hash['sidecars']) + manifest_app_hash['sidecars'] = normalize_units(manifest_app_hash['sidecars']) manifest_app_hash = manifest_app_hash.except('sidecars') if manifest_app_hash['sidecars'] == [{}] end if manifest_app_hash.key? 'processes' @@ -99,6 +99,7 @@ def filter_manifest_app_hash(manifest_app_hash) 'type', 'command', 'disk_quota', + 'log_rate_limit_per_second', 'health-check-http-endpoint', 'health-check-invocation-timeout', 'health-check-type', @@ -107,7 +108,7 @@ def filter_manifest_app_hash(manifest_app_hash) 'timeout' ) end - manifest_app_hash['processes'] = convert_byte_measurements_to_mb(manifest_app_hash['processes']) + manifest_app_hash['processes'] = normalize_units(manifest_app_hash['processes']) manifest_app_hash = manifest_app_hash.except('processes') if manifest_app_hash['processes'] == [{}] end @@ -151,7 +152,7 @@ def create_similarity end end - def convert_byte_measurements_to_mb(manifest_app_hash) + def normalize_units(manifest_app_hash) byte_measurement_key_words = ['memory', 'disk-quota', 'disk_quota'] manifest_app_hash.each_with_index do |process_hash, index| byte_measurement_key_words.each do |key| @@ -159,6 +160,14 @@ def convert_byte_measurements_to_mb(manifest_app_hash) manifest_app_hash[index][key] = convert_to_mb(value, key) unless value.nil? end end + + byte_measurement_key_words = ['log_rate_limit_per_second'] + manifest_app_hash.each_with_index do |process_hash, index| + byte_measurement_key_words.each do |key| + value = process_hash[key] + manifest_app_hash[index][key] = normalize_unit(value, key) unless value.nil? + end + end manifest_app_hash end @@ -170,6 +179,14 @@ def convert_to_mb(human_readable_byte_value, attribute_name) "#{attribute_name} is not a number" end + def normalize_unit(non_normalized_value, attribute_name) + byte_converter.human_readable_byte_value(byte_converter.convert_to_b(non_normalized_value)) + rescue ByteConverter::InvalidUnitsError + "#{attribute_name} must use a supported unit: B, K, KB, M, MB, G, GB, T, or TB" + rescue ByteConverter::NonNumericError + "#{attribute_name} is not a number" + end + def byte_converter ByteConverter.new end diff --git a/app/actions/space_quota_update.rb b/app/actions/space_quota_update.rb index d20aee42a21..99c58cbee99 100644 --- a/app/actions/space_quota_update.rb +++ b/app/actions/space_quota_update.rb @@ -14,6 +14,7 @@ def self.update(quota, message) quota.instance_memory_limit = instance_memory_limit(message) if message.apps_limits_message.requested? :per_process_memory_in_mb quota.app_instance_limit = app_instance_limit(message) if message.apps_limits_message.requested? :total_instances quota.app_task_limit = app_task_limit(message) if message.apps_limits_message.requested? :per_app_tasks + quota.log_rate_limit = log_rate_limit(message) if message.apps_limits_message.requested? :log_rate_limit_in_bytes_per_second quota.total_services = total_services(message) if message.services_limits_message.requested? :total_service_instances quota.total_service_keys = total_service_keys(message) if message.services_limits_message.requested? :total_service_keys @@ -55,6 +56,10 @@ def self.app_task_limit(message) default_if_nil(message.per_app_tasks, SpaceQuotaDefinition::UNLIMITED) end + def self.log_rate_limit(message) + default_if_nil(message.log_rate_limit_in_bytes_per_second, SpaceQuotaDefinition::UNLIMITED) + end + def self.total_services(message) default_if_nil(message.total_service_instances, SpaceQuotaDefinition::UNLIMITED) end diff --git a/app/actions/space_quotas_create.rb b/app/actions/space_quotas_create.rb index 5b10fd617fd..fe33cfd1a4b 100644 --- a/app/actions/space_quotas_create.rb +++ b/app/actions/space_quotas_create.rb @@ -3,6 +3,7 @@ class SpaceQuotasCreate class Error < ::StandardError end + # rubocop:todo Metrics/CyclomaticComplexity def create(message, organization:) space_quota = nil @@ -16,6 +17,7 @@ def create(message, organization:) instance_memory_limit: message.per_process_memory_in_mb || SpaceQuotaDefinition::UNLIMITED, app_instance_limit: message.total_instances || SpaceQuotaDefinition::UNLIMITED, app_task_limit: message.per_app_tasks || SpaceQuotaDefinition::UNLIMITED, + log_rate_limit: message.log_rate_limit_in_bytes_per_second || QuotaDefinition::UNLIMITED, # Services total_services: message.total_service_instances || SpaceQuotaDefinition::DEFAULT_TOTAL_SERVICES, @@ -35,6 +37,7 @@ def create(message, organization:) rescue Sequel::ValidationFailed => e validation_error!(e, message) end + # rubocop:enable Metrics/CyclomaticComplexity private diff --git a/app/actions/task_create.rb b/app/actions/task_create.rb index 39301b6711b..dae42de0508 100644 --- a/app/actions/task_create.rb +++ b/app/actions/task_create.rb @@ -29,6 +29,7 @@ def create(app, message, user_audit_info, droplet: nil) command: command(message, template_process), disk_in_mb: disk_in_mb(message, template_process), memory_in_mb: memory_in_mb(message, template_process), + log_rate_limit: log_rate_limit(message, template_process), sequence_id: app.max_task_sequence_id ) @@ -70,15 +71,15 @@ def command(message, template_process) end def memory_in_mb(message, template_process) - return message.memory_in_mb if message.memory_in_mb + message.memory_in_mb || template_process.try(:memory) || config.get(:default_app_memory) + end - template_process.present? ? template_process.memory : config.get(:default_app_memory) + def log_rate_limit(message, template_process) + message.log_rate_limit_in_bytes_per_second || template_process.try(:log_rate_limit) || config.get(:default_app_log_rate_limit_in_bytes_per_second) end def disk_in_mb(message, template_process) - return message.disk_in_mb if message.disk_in_mb - - template_process.present? ? template_process.disk_quota : config.get(:default_app_disk_in_mb) + message.disk_in_mb || template_process.try(:disk_quota) || config.get(:default_app_disk_in_mb) end def submit_task(task) diff --git a/app/controllers/runtime/apps_controller.rb b/app/controllers/runtime/apps_controller.rb index d8d2d046e36..9e475490ad7 100644 --- a/app/controllers/runtime/apps_controller.rb +++ b/app/controllers/runtime/apps_controller.rb @@ -33,6 +33,7 @@ def self.dependencies attribute :docker_credentials, Hash, default: {} attribute :debug, String, default: nil attribute :disk_quota, Integer, default: nil + attribute :log_rate_limit, Integer, default: nil attribute :environment_json, Hash, default: {}, redact_in: [:create, :update] attribute :health_check_http_endpoint, String, default: nil attribute :health_check_type, String, default: 'port' diff --git a/app/controllers/v3/processes_controller.rb b/app/controllers/v3/processes_controller.rb index f02977d7eac..59f627d8f31 100644 --- a/app/controllers/v3/processes_controller.rb +++ b/app/controllers/v3/processes_controller.rb @@ -89,6 +89,7 @@ def scale 'instance-count' => message.instances, 'memory-in-mb' => message.memory_in_mb, 'disk-in-mb' => message.disk_in_mb, + 'log-rate-in-bytes-per-second' => message.log_rate_limit_in_bytes_per_second, 'process-type' => @process.type } ) diff --git a/app/messages/app_manifest_message.rb b/app/messages/app_manifest_message.rb index d4b02c257c2..f7589da5865 100644 --- a/app/messages/app_manifest_message.rb +++ b/app/messages/app_manifest_message.rb @@ -16,6 +16,7 @@ class AppManifestMessage < BaseMessage :buildpacks, :command, :disk_quota, + :log_rate_limit_per_second, :docker, :env, :health_check_http_endpoint, @@ -130,12 +131,16 @@ def manifest_buildpack_message end def process_scale_attribute_mappings - process_scale_attributes_from_app_level = process_scale_attributes(memory: memory, disk_quota: disk_quota, instances: instances) + process_scale_attributes_from_app_level = process_scale_attributes(memory: memory, + disk_quota: disk_quota, + log_rate_limit_per_second: log_rate_limit_per_second, + instances: instances) process_attributes(process_scale_attributes_from_app_level) do |process| process_scale_attributes( memory: process[:memory], disk_quota: process[:disk_quota], + log_rate_limit_per_second: process[:log_rate_limit_per_second], instances: process[:instances], type: process[:type] ) @@ -163,13 +168,15 @@ def process_attributes(app_attributes) process_attributes end - def process_scale_attributes(memory: nil, disk_quota: nil, instances:, type: nil) + def process_scale_attributes(memory: nil, disk_quota: nil, log_rate_limit_per_second: nil, instances:, type: nil) memory_in_mb = convert_to_mb(memory) disk_in_mb = convert_to_mb(disk_quota) + log_rate_limit_in_bytes_per_second = convert_to_bytes_per_second(log_rate_limit_per_second) { instances: instances, memory: memory_in_mb, disk_quota: disk_in_mb, + log_rate_limit: log_rate_limit_in_bytes_per_second, type: type }.compact end @@ -283,6 +290,13 @@ def convert_to_mb(human_readable_byte_value) rescue ByteConverter::InvalidUnitsError, ByteConverter::NonNumericError end + def convert_to_bytes_per_second(human_readable_byte_value) + return nil unless human_readable_byte_value.present? + + byte_converter.convert_to_b(human_readable_byte_value.strip) + rescue ByteConverter::InvalidUnitsError, ByteConverter::NonNumericError + end + def validate_byte_format(human_readable_byte_value, attribute_name) byte_converter.convert_to_mb(human_readable_byte_value) @@ -378,8 +392,10 @@ def validate_processes! type = process[:type] memory_error = validate_byte_format(process[:memory], 'Memory') disk_error = validate_byte_format(process[:disk_quota], 'Disk quota') + log_rate_limit_error = validate_byte_format(process[:log_rate_limit_per_second], 'Log rate limit per second') add_process_error!(memory_error, type) if memory_error add_process_error!(disk_error, type) if disk_error + add_process_error!(log_rate_limit_error, type) if log_rate_limit_error end end @@ -400,8 +416,10 @@ def validate_sidecars! def validate_top_level_web_process! memory_error = validate_byte_format(memory, 'Memory') disk_error = validate_byte_format(disk_quota, 'Disk quota') + log_rate_limit_error = validate_byte_format(log_rate_limit_per_second, 'Log rate limit per second') add_process_error!(memory_error, ProcessTypes::WEB) if memory_error add_process_error!(disk_error, ProcessTypes::WEB) if disk_error + add_process_error!(log_rate_limit_error, ProcessTypes::WEB) if log_rate_limit_error end def validate_buildpack_and_buildpacks_combination! diff --git a/app/messages/base_message.rb b/app/messages/base_message.rb index e798f14dce2..17922378621 100644 --- a/app/messages/base_message.rb +++ b/app/messages/base_message.rb @@ -8,6 +8,7 @@ class BaseMessage include Validators MAX_DB_INT = 2**31 - 1 + MAX_DB_BIGINT = 2**63 - 1 attr_accessor :requested_keys, :extra_keys diff --git a/app/messages/manifest_process_scale_message.rb b/app/messages/manifest_process_scale_message.rb index 6c916da69f6..67975869fb0 100644 --- a/app/messages/manifest_process_scale_message.rb +++ b/app/messages/manifest_process_scale_message.rb @@ -2,20 +2,24 @@ module VCAP::CloudController class ManifestProcessScaleMessage < BaseMessage - register_allowed_keys [:instances, :memory, :disk_quota, :type] + register_allowed_keys [:instances, :memory, :disk_quota, :log_rate_limit, :type] INVALID_MB_VALUE_ERROR = 'must be greater than 0MB'.freeze + # NOTE: -1 is valid for log_rate_limit representing unlimited + INVALID_QUOTA_VALUE_ERROR = 'must be an integer greater than or equal to -1'.freeze validates_with NoAdditionalKeysValidator validates :instances, numericality: { only_integer: true, greater_than_or_equal_to: 0, less_than_or_equal_to: 2000000 }, allow_nil: true validates :memory, numericality: { only_integer: true, greater_than: 0, message: INVALID_MB_VALUE_ERROR }, allow_nil: true validates :disk_quota, numericality: { only_integer: true, greater_than: 0, message: INVALID_MB_VALUE_ERROR }, allow_nil: true + validates :log_rate_limit, numericality: { only_integer: true, greater_than_or_equal_to: -1, message: INVALID_QUOTA_VALUE_ERROR }, allow_nil: true def to_process_scale_message ProcessScaleMessage.new({ instances: instances, memory_in_mb: memory, disk_in_mb: disk_quota, + log_rate_limit_in_bytes_per_second: log_rate_limit, }.compact) end end diff --git a/app/messages/organization_quotas_update_message.rb b/app/messages/organization_quotas_update_message.rb index b13c10ef29a..536a970266a 100644 --- a/app/messages/organization_quotas_update_message.rb +++ b/app/messages/organization_quotas_update_message.rb @@ -26,7 +26,7 @@ def self.key_requested?(key) validate :domains_validator, if: key_requested?(:domains) # Apps validations - delegate :total_memory_in_mb, :per_process_memory_in_mb, :total_instances, :per_app_tasks, to: :apps_limits_message + delegate :total_memory_in_mb, :per_process_memory_in_mb, :total_instances, :per_app_tasks, :log_rate_limit_in_bytes_per_second, to: :apps_limits_message def validates_hash(key, sym) return true if key.is_a?(Hash) diff --git a/app/messages/process_scale_message.rb b/app/messages/process_scale_message.rb index d57ca116b93..e6cd9f80804 100644 --- a/app/messages/process_scale_message.rb +++ b/app/messages/process_scale_message.rb @@ -2,12 +2,13 @@ module VCAP::CloudController class ProcessScaleMessage < BaseMessage - register_allowed_keys [:instances, :memory_in_mb, :disk_in_mb] + register_allowed_keys [:instances, :memory_in_mb, :disk_in_mb, :log_rate_limit_in_bytes_per_second] validates_with NoAdditionalKeysValidator validates :instances, numericality: { only_integer: true, greater_than_or_equal_to: 0, less_than_or_equal_to: MAX_DB_INT }, allow_nil: true validates :memory_in_mb, numericality: { only_integer: true, greater_than: 0, less_than_or_equal_to: MAX_DB_INT }, allow_nil: true validates :disk_in_mb, numericality: { only_integer: true, greater_than: 0, less_than_or_equal_to: MAX_DB_INT }, allow_nil: true + validates :log_rate_limit_in_bytes_per_second, numericality: { only_integer: true, greater_than_or_equal_to: -1, less_than_or_equal_to: MAX_DB_BIGINT }, allow_nil: true end end diff --git a/app/messages/quotas_apps_message.rb b/app/messages/quotas_apps_message.rb index fe488dc319c..e1029133dc8 100644 --- a/app/messages/quotas_apps_message.rb +++ b/app/messages/quotas_apps_message.rb @@ -3,7 +3,7 @@ module VCAP::CloudController class QuotasAppsMessage < BaseMessage - register_allowed_keys [:total_memory_in_mb, :per_process_memory_in_mb, :total_instances, :per_app_tasks] + register_allowed_keys [:total_memory_in_mb, :per_process_memory_in_mb, :total_instances, :per_app_tasks, :log_rate_limit_in_bytes_per_second] validates_with NoAdditionalKeysValidator @@ -22,5 +22,9 @@ class QuotasAppsMessage < BaseMessage validates :per_app_tasks, numericality: { only_integer: true, greater_than_or_equal_to: 0, less_than_or_equal_to: MAX_DB_INT }, allow_nil: true + + validates :log_rate_limit_in_bytes_per_second, + numericality: { only_integer: true, greater_than_or_equal_to: 0, less_than_or_equal_to: MAX_DB_BIGINT }, + allow_nil: true end end diff --git a/app/messages/space_quota_update_message.rb b/app/messages/space_quota_update_message.rb index dbb6728488d..de1f3c51fd3 100644 --- a/app/messages/space_quota_update_message.rb +++ b/app/messages/space_quota_update_message.rb @@ -22,7 +22,7 @@ def self.key_requested?(key) validate :services_validator, if: key_requested?(:services) validate :routes_validator, if: key_requested?(:routes) - delegate :total_memory_in_mb, :per_process_memory_in_mb, :total_instances, :per_app_tasks, to: :apps_limits_message + delegate :total_memory_in_mb, :per_process_memory_in_mb, :total_instances, :per_app_tasks, :log_rate_limit_in_bytes_per_second, to: :apps_limits_message delegate :paid_services_allowed, :total_service_instances, :total_service_keys, to: :services_limits_message delegate :total_routes, :total_reserved_ports, to: :routes_limits_message diff --git a/app/messages/task_create_message.rb b/app/messages/task_create_message.rb index 5f230151e5d..32f90554a80 100644 --- a/app/messages/task_create_message.rb +++ b/app/messages/task_create_message.rb @@ -2,7 +2,7 @@ module VCAP::CloudController class TaskCreateMessage < MetadataBaseMessage - register_allowed_keys [:name, :command, :disk_in_mb, :memory_in_mb, :droplet_guid, :template] + register_allowed_keys [:name, :command, :disk_in_mb, :memory_in_mb, :log_rate_limit_in_bytes_per_second, :droplet_guid, :template] validates_with NoAdditionalKeysValidator @@ -12,6 +12,7 @@ def self.validate_template? validates :disk_in_mb, numericality: { only_integer: true, greater_than: 0 }, allow_nil: true validates :memory_in_mb, numericality: { only_integer: true, greater_than: 0 }, allow_nil: true + validates :log_rate_limit_in_bytes_per_second, numericality: { only_integer: true, greater_than: -2 }, allow_nil: true validates :droplet_guid, guid: true, allow_nil: true validates :template_process_guid, guid: true, if: validate_template? validate :has_command diff --git a/app/models.rb b/app/models.rb index 6c1fe448fb6..34672088446 100644 --- a/app/models.rb +++ b/app/models.rb @@ -47,6 +47,8 @@ require 'models/runtime/quota_constraints/max_service_keys_policy' require 'models/runtime/constraints/max_disk_quota_policy' require 'models/runtime/constraints/min_disk_quota_policy' +require 'models/runtime/constraints/min_log_rate_limit_policy' +require 'models/runtime/constraints/max_log_rate_limit_policy' require 'models/runtime/constraints/max_memory_policy' require 'models/runtime/constraints/max_instance_memory_policy' require 'models/runtime/constraints/min_memory_policy' diff --git a/app/models/runtime/constraints/max_log_rate_limit_policy.rb b/app/models/runtime/constraints/max_log_rate_limit_policy.rb new file mode 100644 index 00000000000..a5a6ea14c0a --- /dev/null +++ b/app/models/runtime/constraints/max_log_rate_limit_policy.rb @@ -0,0 +1,78 @@ +require 'cloud_controller/app_services/process_log_rate_limit_calculator' + +class BaseMaxLogRateLimitPolicy + def initialize(resource, policy_target, error_name) + @resource = resource + @policy_target = policy_target + @error_name = error_name + end + + def validate + return unless policy_target + return unless additional_checks + + if requested_log_rate_limit == VCAP::CloudController::QuotaDefinition::UNLIMITED && + policy_target.log_rate_limit != VCAP::CloudController::QuotaDefinition::UNLIMITED + + policy_target_type = if policy_target.respond_to?(:organization_guid) + 'space' + else + 'organization' + end + + resource.errors.add(field, "cannot be unlimited in #{policy_target_type} '#{policy_target.name}'.") + end + + unless policy_target.has_remaining_log_rate_limit(requested_log_rate_limit) + resource.errors.add(field, error_name) + end + end + + private + + attr_reader :resource, :policy_target, :error_name + + def additional_checks + true + end + + def requested_log_rate_limit + resource.public_send field + end + + def field + :log_rate_limit + end +end + +class AppMaxLogRateLimitPolicy < BaseMaxLogRateLimitPolicy + private + + def additional_checks + resource.started? && + (resource.column_changed?(:state) || resource.column_changed?(:instances)) + end + + def requested_log_rate_limit + calculator = VCAP::CloudController::ProcessLogRateLimitCalculator.new(resource) + calculator.additional_log_rate_limit_requested + end +end + +class TaskMaxLogRateLimitPolicy < BaseMaxLogRateLimitPolicy + IGNORED_STATES = [ + VCAP::CloudController::TaskModel::CANCELING_STATE, + VCAP::CloudController::TaskModel::SUCCEEDED_STATE, + VCAP::CloudController::TaskModel::FAILED_STATE, + ].freeze + + private + + def additional_checks + IGNORED_STATES.exclude?(resource.state) + end + + def field + :log_rate_limit + end +end diff --git a/app/models/runtime/constraints/min_log_rate_limit_policy.rb b/app/models/runtime/constraints/min_log_rate_limit_policy.rb new file mode 100644 index 00000000000..62c2dbfdb66 --- /dev/null +++ b/app/models/runtime/constraints/min_log_rate_limit_policy.rb @@ -0,0 +1,16 @@ +class MinLogRateLimitPolicy + ERROR_MSG = 'log_rate_limit must be greater than or equal to -1 (where -1 is unlimited)'.freeze + + def initialize(process) + @process = process + @errors = process.errors + end + + def validate + return unless @process.log_rate_limit + + if @process.log_rate_limit < -1 + @errors.add(:log_rate_limit, ERROR_MSG) + end + end +end diff --git a/app/models/runtime/organization.rb b/app/models/runtime/organization.rb index 661f5382e56..755257323e4 100644 --- a/app/models/runtime/organization.rb +++ b/app/models/runtime/organization.rb @@ -230,13 +230,21 @@ def memory_used end def has_remaining_memory(mem) - quota_definition.memory_limit == -1 || memory_remaining >= mem + quota_definition.memory_limit == QuotaDefinition::UNLIMITED || memory_remaining >= mem + end + + def has_remaining_log_rate_limit(log_rate_limit_desired) + quota_definition.log_rate_limit == QuotaDefinition::UNLIMITED || log_rate_limit_remaining >= log_rate_limit_desired end def instance_memory_limit quota_definition ? quota_definition.instance_memory_limit : QuotaDefinition::UNLIMITED end + def log_rate_limit + quota_definition ? quota_definition.log_rate_limit : QuotaDefinition::UNLIMITED + end + def app_task_limit quota_definition ? quota_definition.app_task_limit : QuotaDefinition::UNLIMITED end @@ -314,6 +322,10 @@ def memory_remaining quota_definition.memory_limit - memory_used end + def log_rate_limit_remaining + quota_definition.log_rate_limit - (started_app_log_rate_limit + running_task_log_rate_limit) + end + def running_task_memory tasks_dataset.where(state: TaskModel::RUNNING_STATE).sum(:memory_in_mb) || 0 end @@ -322,6 +334,14 @@ def started_app_memory processes_dataset.where(state: ProcessModel::STARTED).sum(Sequel.*(:memory, :instances)) || 0 end + def running_task_log_rate_limit + tasks_dataset.where(state: TaskModel::RUNNING_STATE).sum(:log_rate_limit) || 0 + end + + def started_app_log_rate_limit + processes_dataset.where(state: ProcessModel::STARTED).sum(Sequel.*(:log_rate_limit, :instances)) || 0 + end + def running_and_pending_tasks_count tasks_dataset.where(state: [TaskModel::PENDING_STATE, TaskModel::RUNNING_STATE]).count end diff --git a/app/models/runtime/process_model.rb b/app/models/runtime/process_model.rb index f454a0ed73e..a76b5172163 100644 --- a/app/models/runtime/process_model.rb +++ b/app/models/runtime/process_model.rb @@ -27,6 +27,7 @@ def after_initialize self.memory ||= Config.config.get(:default_app_memory) self.disk_quota ||= Config.config.get(:default_app_disk_in_mb) self.file_descriptors ||= Config.config.get(:instance_file_descriptor_limit) + self.log_rate_limit ||= Config.config.get(:default_app_log_rate_limit_in_bytes_per_second) self.metadata ||= {} end @@ -144,17 +145,19 @@ def non_docker_type add_association_dependencies annotations: :destroy export_attributes :name, :production, :space_guid, :stack_guid, :buildpack, - :detected_buildpack, :detected_buildpack_guid, :environment_json, :memory, :instances, :disk_quota, - :state, :version, :command, :console, :debug, :staging_task_id, - :package_state, :health_check_type, :health_check_timeout, :health_check_http_endpoint, - :staging_failed_reason, :staging_failed_description, :diego, :docker_image, :package_updated_at, - :detected_start_command, :enable_ssh, :ports + :detected_buildpack, :detected_buildpack_guid, :environment_json, + :memory, :instances, :disk_quota, :log_rate_limit, :state, :version, :command, + :console, :debug, :staging_task_id, :package_state, :health_check_type, + :health_check_timeout, :health_check_http_endpoint, :staging_failed_reason, + :staging_failed_description, :diego, :docker_image, :package_updated_at, + :detected_start_command, :enable_ssh, :ports import_attributes :name, :production, :space_guid, :stack_guid, :buildpack, :detected_buildpack, :environment_json, :memory, :instances, :disk_quota, - :state, :command, :console, :debug, :staging_task_id, - :service_binding_guids, :route_guids, :health_check_type, :health_check_http_endpoint, - :health_check_timeout, :diego, :docker_image, :app_guid, :enable_ssh, :ports + :log_rate_limit, :state, :command, :console, :debug, :staging_task_id, + :service_binding_guids, :route_guids, :health_check_type, + :health_check_http_endpoint, :health_check_timeout, :diego, + :docker_image, :app_guid, :enable_ssh, :ports serialize_attributes :json, :metadata serialize_attributes :integer_array, :ports @@ -258,6 +261,9 @@ def validation_policies InstancesPolicy.new(self), MaxAppInstancesPolicy.new(self, organization, organization && organization.quota_definition, :app_instance_limit_exceeded), MaxAppInstancesPolicy.new(self, space, space && space.space_quota_definition, :space_app_instance_limit_exceeded), + MinLogRateLimitPolicy.new(self), + AppMaxLogRateLimitPolicy.new(self, space, 'exceeds space log rate quota'), + AppMaxLogRateLimitPolicy.new(self, organization, 'exceeds organization log rate quota'), HealthCheckPolicy.new(self, health_check_timeout, health_check_invocation_timeout), DockerPolicy.new(self), PortsPolicy.new(self) diff --git a/app/models/runtime/quota_definition.rb b/app/models/runtime/quota_definition.rb index 81374dda85d..b4d24cae2c2 100644 --- a/app/models/runtime/quota_definition.rb +++ b/app/models/runtime/quota_definition.rb @@ -12,12 +12,13 @@ class QuotaDefinition < Sequel::Model export_attributes :name, :non_basic_services_allowed, :total_services, :total_routes, :total_private_domains, :memory_limit, :trial_db_allowed, :instance_memory_limit, - :app_instance_limit, :app_task_limit, :total_service_keys, :total_reserved_route_ports + :app_instance_limit, :app_task_limit, :total_service_keys, :total_reserved_route_ports, + :log_rate_limit import_attributes :name, :non_basic_services_allowed, :total_services, :total_routes, :total_private_domains, :memory_limit, :trial_db_allowed, :instance_memory_limit, - :app_instance_limit, :app_task_limit, :total_service_keys, :total_reserved_route_ports + :app_instance_limit, :app_task_limit, :total_service_keys, :total_reserved_route_ports, + :log_rate_limit - # rubocop:disable Metrics/CyclomaticComplexity def validate validates_presence :name validates_unique :name @@ -27,14 +28,14 @@ def validate validates_presence :memory_limit validate_total_reserved_route_ports - errors.add(:memory_limit, :invalid_memory_limit) if memory_limit && memory_limit < UNLIMITED - errors.add(:instance_memory_limit, :invalid_instance_memory_limit) if instance_memory_limit && instance_memory_limit < UNLIMITED - errors.add(:total_private_domains, :invalid_total_private_domains) if total_private_domains && total_private_domains < UNLIMITED - errors.add(:app_instance_limit, :invalid_app_instance_limit) if app_instance_limit && app_instance_limit < UNLIMITED - errors.add(:app_task_limit, :invalid_app_task_limit) if app_task_limit && app_task_limit < UNLIMITED - errors.add(:total_service_keys, :invalid_total_service_keys) if total_service_keys && total_service_keys < UNLIMITED + validates_limit(:memory_limit, memory_limit) + validates_limit(:instance_memory_limit, instance_memory_limit) + validates_limit(:total_private_domains, total_private_domains) + validates_limit(:app_instance_limit, app_instance_limit) + validates_limit(:app_task_limit, app_task_limit) + validates_limit(:log_rate_limit, log_rate_limit) + validates_limit(:total_service_keys, total_service_keys) end - # rubocop:enable Metrics/CyclomaticComplexity def before_destroy if organizations.present? @@ -66,6 +67,10 @@ def self.user_visibility_filter(user) private + def validates_limit(limit_name, limit) + errors.add(limit_name, :"invalid_#{limit_name}") if limit && limit < UNLIMITED + end + def validate_total_reserved_route_ports return unless total_reserved_route_ports diff --git a/app/models/runtime/space.rb b/app/models/runtime/space.rb index a7566a84f9d..65daf1b7fb6 100644 --- a/app/models/runtime/space.rb +++ b/app/models/runtime/space.rb @@ -286,6 +286,12 @@ def has_remaining_memory(mem) space_quota_definition.memory_limit == SpaceQuotaDefinition::UNLIMITED || memory_remaining >= mem end + def has_remaining_log_rate_limit(log_rate_limit_desired) + return true unless space_quota_definition + + space_quota_definition.log_rate_limit == SpaceQuotaDefinition::UNLIMITED || log_rate_limit_remaining >= log_rate_limit_desired + end + def instance_memory_limit if space_quota_definition space_quota_definition.instance_memory_limit @@ -294,6 +300,14 @@ def instance_memory_limit end end + def log_rate_limit + if space_quota_definition + space_quota_definition.log_rate_limit + else + SpaceQuotaDefinition::UNLIMITED + end + end + def app_task_limit if space_quota_definition space_quota_definition.app_task_limit @@ -329,6 +343,10 @@ def memory_remaining space_quota_definition.memory_limit - memory_used end + def log_rate_limit_remaining + space_quota_definition.log_rate_limit - (started_app_log_rate_limit + running_task_log_rate_limit) + end + def running_task_memory tasks_dataset.where(state: TaskModel::RUNNING_STATE).sum(:memory_in_mb) || 0 end @@ -337,6 +355,14 @@ def started_app_memory processes_dataset.where(state: ProcessModel::STARTED).sum(Sequel.*(:memory, :instances)) || 0 end + def running_task_log_rate_limit + tasks_dataset.where(state: TaskModel::RUNNING_STATE).sum(:log_rate_limit) || 0 + end + + def started_app_log_rate_limit + processes_dataset.where(state: ProcessModel::STARTED).sum(Sequel.*(:log_rate_limit, :instances)) || 0 + end + def running_and_pending_tasks_count tasks_dataset.where(state: [TaskModel::PENDING_STATE, TaskModel::RUNNING_STATE]).count end diff --git a/app/models/runtime/space_quota_definition.rb b/app/models/runtime/space_quota_definition.rb index 4be195c6a4d..bb0668cc42d 100644 --- a/app/models/runtime/space_quota_definition.rb +++ b/app/models/runtime/space_quota_definition.rb @@ -18,10 +18,10 @@ class SpaceQuotaDefinition < Sequel::Model export_attributes :name, :organization_guid, :non_basic_services_allowed, :total_services, :total_routes, :memory_limit, :instance_memory_limit, :app_instance_limit, :app_task_limit, - :total_service_keys, :total_reserved_route_ports + :total_service_keys, :total_reserved_route_ports, :log_rate_limit import_attributes :name, :organization_guid, :non_basic_services_allowed, :total_services, :total_routes, :memory_limit, :instance_memory_limit, :app_instance_limit, :app_task_limit, - :total_service_keys, :total_reserved_route_ports + :total_service_keys, :total_reserved_route_ports, :log_rate_limit add_association_dependencies spaces: :nullify @@ -34,11 +34,13 @@ def validate validates_presence :organization validates_unique [:organization_id, :name] - errors.add(:memory_limit, :invalid_memory_limit) if memory_limit && memory_limit < UNLIMITED - errors.add(:instance_memory_limit, :invalid_instance_memory_limit) if instance_memory_limit && instance_memory_limit < -1 - errors.add(:app_instance_limit, :invalid_app_instance_limit) if app_instance_limit && app_instance_limit < UNLIMITED - errors.add(:app_task_limit, :invalid_app_task_limit) if app_task_limit && app_task_limit < UNLIMITED - errors.add(:total_service_keys, :invalid_total_service_keys) if total_service_keys && total_service_keys < UNLIMITED + validates_limit(:memory_limit, memory_limit) + validates_limit(:instance_memory_limit, instance_memory_limit) + validates_limit(:app_instance_limit, app_instance_limit) + validates_limit(:app_task_limit, app_task_limit) + validates_limit(:log_rate_limit, log_rate_limit) + validates_limit(:total_service_keys, total_service_keys) + validate_total_reserved_ports end @@ -58,6 +60,10 @@ def self.user_visibility_filter(user) private + def validates_limit(limit_name, limit) + errors.add(limit_name, :"invalid_#{limit_name}") if limit && limit < UNLIMITED + end + def validate_total_reserved_ports return unless total_reserved_route_ports diff --git a/app/models/runtime/task_model.rb b/app/models/runtime/task_model.rb index c3541bb9bdd..be692b0f5c2 100644 --- a/app/models/runtime/task_model.rb +++ b/app/models/runtime/task_model.rb @@ -67,17 +67,21 @@ def validate validates_presence :name validate_org_quotas validate_space_quotas + + MinLogRateLimitPolicy.new(self).validate end def validate_space_quotas TaskMaxMemoryPolicy.new(self, space, 'exceeds space memory quota').validate TaskMaxInstanceMemoryPolicy.new(self, space, 'exceeds space instance memory quota').validate + TaskMaxLogRateLimitPolicy.new(self, space, 'exceeds space log rate quota').validate new? && MaxAppTasksPolicy.new(self, space, 'quota exceeded').validate end def validate_org_quotas TaskMaxMemoryPolicy.new(self, organization, 'exceeds organization memory quota').validate TaskMaxInstanceMemoryPolicy.new(self, organization, 'exceeds organization instance memory quota').validate + TaskMaxLogRateLimitPolicy.new(self, organization, 'exceeds organization log rate quota').validate new? && MaxAppTasksPolicy.new(self, organization, 'quota exceeded').validate end diff --git a/app/presenters/v2/process_model_presenter.rb b/app/presenters/v2/process_model_presenter.rb index 956ff1b3b77..a87330d54d5 100644 --- a/app/presenters/v2/process_model_presenter.rb +++ b/app/presenters/v2/process_model_presenter.rb @@ -21,6 +21,7 @@ def entity_hash(controller, process, opts, depth, parents, orphans=nil) 'memory' => process.memory, 'instances' => process.instances, 'disk_quota' => process.disk_quota, + 'log_rate_limit' => process.log_rate_limit, 'state' => process.state, 'version' => process.version, 'command' => process.command.presence, diff --git a/app/presenters/v3/app_manifest_presenters/process_properties_presenter.rb b/app/presenters/v3/app_manifest_presenters/process_properties_presenter.rb index 0535df9a8e5..155a5386976 100644 --- a/app/presenters/v3/app_manifest_presenters/process_properties_presenter.rb +++ b/app/presenters/v3/app_manifest_presenters/process_properties_presenter.rb @@ -14,6 +14,7 @@ def process_hash(process) 'instances' => process.instances, 'memory' => add_units(process.memory), 'disk_quota' => add_units(process.disk_quota), + 'log_rate_limit_per_second' => add_units_log_rate_limit(process.log_rate_limit), 'command' => process.command, 'health-check-type' => process.health_check_type, 'health-check-http-endpoint' => process.health_check_http_endpoint, @@ -24,6 +25,18 @@ def process_hash(process) def add_units(val) "#{val}M" end + + def add_units_log_rate_limit(val) + if val == -1 + '-1B' + else + byte_converter.human_readable_byte_value(val) + end + end + + def byte_converter + ByteConverter.new + end end end end diff --git a/app/presenters/v3/organization_quota_presenter.rb b/app/presenters/v3/organization_quota_presenter.rb index 5f0d1593a65..f4c15b04434 100644 --- a/app/presenters/v3/organization_quota_presenter.rb +++ b/app/presenters/v3/organization_quota_presenter.rb @@ -26,6 +26,7 @@ def to_hash per_process_memory_in_mb: convert_unlimited_to_nil(organization_quota.instance_memory_limit), total_instances: convert_unlimited_to_nil(organization_quota.app_instance_limit), per_app_tasks: convert_unlimited_to_nil(organization_quota.app_task_limit), + log_rate_limit_in_bytes_per_second: convert_unlimited_to_nil(organization_quota.log_rate_limit), }, services: { paid_services_allowed: organization_quota.non_basic_services_allowed, diff --git a/app/presenters/v3/process_presenter.rb b/app/presenters/v3/process_presenter.rb index 26d0f7fb2ce..d68eb4f5c5c 100644 --- a/app/presenters/v3/process_presenter.rb +++ b/app/presenters/v3/process_presenter.rb @@ -19,14 +19,15 @@ def to_hash health_check_data = { timeout: process.health_check_timeout, invocation_timeout: process.health_check_invocation_timeout } health_check_data[:endpoint] = process.health_check_http_endpoint if process.health_check_type == HealthCheckTypes::HTTP { - guid: process.guid, - created_at: process.created_at, - updated_at: process.updated_at, - type: process.type, - command: redact(process.specified_or_detected_command), - instances: process.instances, - memory_in_mb: process.memory, - disk_in_mb: process.disk_quota, + guid: process.guid, + created_at: process.created_at, + updated_at: process.updated_at, + type: process.type, + command: redact(process.specified_or_detected_command), + instances: process.instances, + memory_in_mb: process.memory, + disk_in_mb: process.disk_quota, + log_rate_limit_in_bytes_per_second: process.log_rate_limit, health_check: { type: process.health_check_type, data: health_check_data diff --git a/app/presenters/v3/process_stats_presenter.rb b/app/presenters/v3/process_stats_presenter.rb index 47fbc14802d..4c17c083c62 100644 --- a/app/presenters/v3/process_stats_presenter.rb +++ b/app/presenters/v3/process_stats_presenter.rb @@ -39,6 +39,7 @@ def found_instance_stats_hash(index, stats) uptime: stats[:stats][:uptime], mem_quota: stats[:stats][:mem_quota], disk_quota: stats[:stats][:disk_quota], + log_rate_limit: stats[:stats][:log_rate_limit], fds_quota: stats[:stats][:fds_quota], isolation_segment: stats[:isolation_segment], details: stats[:details] @@ -74,6 +75,7 @@ def add_usage_info(presented_stats, stats) cpu: stats[:stats][:usage][:cpu], mem: stats[:stats][:usage][:mem], disk: stats[:stats][:usage][:disk], + log_rate: stats[:stats][:usage][:log_rate], } else {} diff --git a/app/presenters/v3/space_quota_presenter.rb b/app/presenters/v3/space_quota_presenter.rb index 1c1b3ab2a8d..4d4da24c28e 100644 --- a/app/presenters/v3/space_quota_presenter.rb +++ b/app/presenters/v3/space_quota_presenter.rb @@ -26,6 +26,7 @@ def to_hash per_process_memory_in_mb: unlimited_to_nil(space_quota.instance_memory_limit), total_instances: unlimited_to_nil(space_quota.app_instance_limit), per_app_tasks: unlimited_to_nil(space_quota.app_task_limit), + log_rate_limit_in_bytes_per_second: unlimited_to_nil(space_quota.log_rate_limit), }, services: { paid_services_allowed: space_quota.non_basic_services_allowed, diff --git a/app/presenters/v3/task_presenter.rb b/app/presenters/v3/task_presenter.rb index 616a9ec99b6..2ba114fc33a 100644 --- a/app/presenters/v3/task_presenter.rb +++ b/app/presenters/v3/task_presenter.rb @@ -18,6 +18,7 @@ def to_hash state: task.state, memory_in_mb: task.memory_in_mb, disk_in_mb: task.disk_in_mb, + log_rate_limit_in_bytes_per_second: task.log_rate_limit, result: { failure_reason: task.failure_reason }, droplet_guid: task.droplet_guid, relationships: { app: { data: { guid: task.app_guid } } }, diff --git a/config/bosh-lite.yml b/config/bosh-lite.yml index 3931c6bec2a..c2c3f001251 100644 --- a/config/bosh-lite.yml +++ b/config/bosh-lite.yml @@ -36,6 +36,7 @@ completed_tasks: default_app_memory: 256 default_app_disk_in_mb: 1024 +default_app_log_rate_limit_in_bytes_per_second: 1_048_576 maximum_app_disk_in_mb: 2048 instance_file_descriptor_limit: 16384 diff --git a/config/cloud_controller.yml b/config/cloud_controller.yml index eb3e61c0249..3609f71c879 100644 --- a/config/cloud_controller.yml +++ b/config/cloud_controller.yml @@ -53,6 +53,7 @@ completed_tasks: default_app_memory: 1024 #mb default_app_disk_in_mb: 1024 +default_app_log_rate_limit_in_bytes_per_second: 1_048_576 maximum_app_disk_in_mb: 2048 max_retained_deployments_per_app: 100 max_retained_builds_per_app: 100 diff --git a/db/migrations/20220606172913_add_log_rate_limit_to_quota_definitions.rb b/db/migrations/20220606172913_add_log_rate_limit_to_quota_definitions.rb new file mode 100644 index 00000000000..475e005dc17 --- /dev/null +++ b/db/migrations/20220606172913_add_log_rate_limit_to_quota_definitions.rb @@ -0,0 +1,5 @@ +Sequel.migration do + change do + add_column :quota_definitions, :log_rate_limit, :Bignum, null: false, default: -1 + end +end diff --git a/db/migrations/20220607222309_add_log_rate_limit_to_space_quota_definitions.rb b/db/migrations/20220607222309_add_log_rate_limit_to_space_quota_definitions.rb new file mode 100644 index 00000000000..4abca30f194 --- /dev/null +++ b/db/migrations/20220607222309_add_log_rate_limit_to_space_quota_definitions.rb @@ -0,0 +1,5 @@ +Sequel.migration do + change do + add_column :space_quota_definitions, :log_rate_limit, :Bignum, null: false, default: -1 + end +end diff --git a/db/migrations/20220608165055_add_log_rate_limit_to_processes.rb b/db/migrations/20220608165055_add_log_rate_limit_to_processes.rb new file mode 100644 index 00000000000..d2825f53908 --- /dev/null +++ b/db/migrations/20220608165055_add_log_rate_limit_to_processes.rb @@ -0,0 +1,5 @@ +Sequel.migration do + change do + add_column :processes, :log_rate_limit, :Bignum, null: false, default: -1 + end +end diff --git a/db/migrations/20220621175820_add_log_rate_limit_to_tasks_table.rb b/db/migrations/20220621175820_add_log_rate_limit_to_tasks_table.rb new file mode 100644 index 00000000000..be1062df2b1 --- /dev/null +++ b/db/migrations/20220621175820_add_log_rate_limit_to_tasks_table.rb @@ -0,0 +1,5 @@ +Sequel.migration do + change do + add_column :tasks, :log_rate_limit, :Bignum, null: false, default: -1 + end +end diff --git a/lib/cloud_controller/app_manifest/byte_converter.rb b/lib/cloud_controller/app_manifest/byte_converter.rb index e5c2828083b..1dbf589367e 100644 --- a/lib/cloud_controller/app_manifest/byte_converter.rb +++ b/lib/cloud_controller/app_manifest/byte_converter.rb @@ -2,6 +2,7 @@ module VCAP::CloudController class ByteConverter + class InvalidBytesError < StandardError; end class InvalidUnitsError < StandardError; end class NonNumericError < StandardError; end @@ -15,5 +16,34 @@ def convert_to_mb(human_readable_byte_value) rescue PalmCivet::InvalidByteQuantityError raise InvalidUnitsError end + + def convert_to_b(human_readable_byte_value) + return nil unless human_readable_byte_value.present? + if !human_readable_byte_value.to_s.match?(/\A-?\d+(?:\.\d+)?/) + raise NonNumericError + end + + PalmCivet.to_bytes(human_readable_byte_value.to_s) + rescue PalmCivet::InvalidByteQuantityError + raise InvalidUnitsError + end + + def human_readable_byte_value(bytes) + return nil unless bytes.present? + + if !bytes.is_a?(Integer) + raise InvalidBytesError + end + + units = %w(B K M G T) + while units.any? + unit_in_bytes = 1024**(units.length - 1) + if bytes.remainder(unit_in_bytes).zero? + return "#{bytes / unit_in_bytes}#{units.last}" + end + + units.pop + end + end end end diff --git a/lib/cloud_controller/app_services/process_log_rate_limit_calculator.rb b/lib/cloud_controller/app_services/process_log_rate_limit_calculator.rb new file mode 100644 index 00000000000..eae51fec2d2 --- /dev/null +++ b/lib/cloud_controller/app_services/process_log_rate_limit_calculator.rb @@ -0,0 +1,51 @@ +module VCAP::CloudController + class ProcessLogRateLimitCalculator + attr_reader :process + + def initialize(process) + @process = process + end + + def additional_log_rate_limit_requested + return 0 if process.stopped? + return QuotaDefinition::UNLIMITED if is_process_log_rate_unlimited? + + total_requested_log_rate_limit - currently_used_log_rate_limit + end + + def total_requested_log_rate_limit + return 0 if process.log_rate_limit == QuotaDefinition::UNLIMITED + + process.log_rate_limit * process.instances + end + + def currently_used_log_rate_limit + return 0 if process.new? + + db_process = process_from_db + return 0 if db_process.stopped? || db_process[:log_rate_limit] == QuotaDefinition::UNLIMITED + + db_process[:log_rate_limit] * db_process[:instances] + end + + private + + def is_process_log_rate_unlimited? + process.log_rate_limit == QuotaDefinition::UNLIMITED + end + + def process_from_db + error_message = 'Expected process record not found in database with guid %s' + process_fetched_from_db = ProcessModel.find(guid: process.guid) + if process_fetched_from_db.nil? + logger.fatal('process.find.missing', guid: process.guid, self: process.inspect) + raise CloudController::Errors::ApplicationMissing.new(sprintf(error_message, guid: process.guid)) + end + process_fetched_from_db + end + + def logger + @logger ||= Steno.logger('cc.process_log_rate_limit_calculator') + end + end +end diff --git a/lib/cloud_controller/config_schemas/base/api_schema.rb b/lib/cloud_controller/config_schemas/base/api_schema.rb index 6438bc707ae..157e86b6f09 100644 --- a/lib/cloud_controller/config_schemas/base/api_schema.rb +++ b/lib/cloud_controller/config_schemas/base/api_schema.rb @@ -38,6 +38,7 @@ class ApiSchema < VCAP::Config default_app_memory: Integer, default_app_disk_in_mb: Integer, + default_app_log_rate_limit_in_bytes_per_second: Integer, maximum_app_disk_in_mb: Integer, default_health_check_timeout: Integer, maximum_health_check_timeout: Integer, diff --git a/lib/cloud_controller/config_schemas/base/clock_schema.rb b/lib/cloud_controller/config_schemas/base/clock_schema.rb index 92f50b6a54e..10f14a576cf 100644 --- a/lib/cloud_controller/config_schemas/base/clock_schema.rb +++ b/lib/cloud_controller/config_schemas/base/clock_schema.rb @@ -155,6 +155,7 @@ class ClockSchema < VCAP::Config default_app_memory: Integer, default_app_disk_in_mb: Integer, + default_app_log_rate_limit_in_bytes_per_second: Integer, instance_file_descriptor_limit: Integer, maximum_app_disk_in_mb: Integer, max_retained_deployments_per_app: Integer, diff --git a/lib/cloud_controller/config_schemas/base/worker_schema.rb b/lib/cloud_controller/config_schemas/base/worker_schema.rb index ceb64038f3c..dadb554bde2 100644 --- a/lib/cloud_controller/config_schemas/base/worker_schema.rb +++ b/lib/cloud_controller/config_schemas/base/worker_schema.rb @@ -166,6 +166,7 @@ class WorkerSchema < VCAP::Config default_app_disk_in_mb: Integer, instance_file_descriptor_limit: Integer, maximum_app_disk_in_mb: Integer, + default_app_log_rate_limit_in_bytes_per_second: Integer, default_app_ssh_access: bool, jobs: { diff --git a/spec/request/app_manifests_spec.rb b/spec/request/app_manifests_spec.rb index 5ccb23e9ca4..1fe2ae2a693 100644 --- a/spec/request/app_manifests_spec.rb +++ b/spec/request/app_manifests_spec.rb @@ -39,6 +39,7 @@ health_check_type: 'http', health_check_http_endpoint: '/foobar', health_check_timeout: 5, + log_rate_limit: 1_048_576, ) end @@ -107,6 +108,7 @@ 'instances' => process.instances, 'memory' => "#{process.memory}M", 'disk_quota' => "#{process.disk_quota}M", + 'log_rate_limit_per_second' => '1M', 'health-check-type' => process.health_check_type, }, { @@ -114,6 +116,7 @@ 'instances' => worker_process.instances, 'memory' => "#{worker_process.memory}M", 'disk_quota' => "#{worker_process.disk_quota}M", + 'log_rate_limit_per_second' => '1M', 'command' => worker_process.command, 'health-check-type' => worker_process.health_check_type, 'health-check-http-endpoint' => worker_process.health_check_http_endpoint, @@ -170,6 +173,7 @@ before do app_model.update(droplet: droplet) + process.update(log_rate_limit: -1) end let(:expected_yml_manifest) do @@ -203,6 +207,7 @@ 'instances' => process.instances, 'memory' => "#{process.memory}M", 'disk_quota' => "#{process.disk_quota}M", + 'log_rate_limit_per_second' => '-1B', 'health-check-type' => process.health_check_type, }, { @@ -210,6 +215,7 @@ 'instances' => worker_process.instances, 'memory' => "#{worker_process.memory}M", 'disk_quota' => "#{worker_process.disk_quota}M", + 'log_rate_limit_per_second' => '1M', 'command' => worker_process.command, 'health-check-type' => worker_process.health_check_type, 'health-check-http-endpoint' => worker_process.health_check_http_endpoint, diff --git a/spec/request/apps_spec.rb b/spec/request/apps_spec.rb index 831bb07450c..41e5a08f589 100644 --- a/spec/request/apps_spec.rb +++ b/spec/request/apps_spec.rb @@ -2378,6 +2378,152 @@ it_behaves_like 'permissions for single object endpoint', ALL_PERMISSIONS end + + describe 'limiting the application log rates' do + let(:log_rate_limit) { -1 } + let(:space_log_rate_limit) { -1 } + let(:org_log_rate_limit) { -1 } + let(:org_quota_definition) { VCAP::CloudController::QuotaDefinition.make(log_rate_limit: org_log_rate_limit) } + let(:org) { VCAP::CloudController::Organization.make(quota_definition: org_quota_definition) } + let(:space_quota_definition) { VCAP::CloudController::SpaceQuotaDefinition.make(organization: org, log_rate_limit: space_log_rate_limit) } + let(:space) { VCAP::CloudController::Space.make(organization: org, space_quota_definition: space_quota_definition) } + let!(:process_model) { VCAP::CloudController::ProcessModel.make(app: app_model, log_rate_limit: log_rate_limit) } + let(:app_model) { + VCAP::CloudController::AppModel.make( + :buildpack, + name: 'app-name', + space: space, + desired_state: 'STOPPED', + ) + } + let(:droplet) { VCAP::CloudController::DropletModel.make(app: app_model, process_types: { web: 'webby' }) } + + before do + app_model.update(droplet_guid: droplet.guid) + end + + describe 'space quotas' do + context 'when both the space and the app do not specify a log rate limit' do + let(:log_rate_limit) { -1 } + let(:space_log_rate_limit) { -1 } + + it 'starts the app successfully' do + post "/v3/apps/#{app_model.guid}/actions/start", nil, admin_header + + expect(last_response.status).to eq(200) + end + end + + context "when the app fits in the space's log rate limit" do + let(:log_rate_limit) { 199 } + let(:space_log_rate_limit) { 200 } + + it 'starts the app successfully' do + post "/v3/apps/#{app_model.guid}/actions/start", nil, admin_header + + expect(last_response.status).to eq(200) + end + end + + context "when the app's log rate limit is unspecified, but the space specifies a log rate limit" do + let(:log_rate_limit) { -1 } + let(:space_log_rate_limit) { 200 } + + it 'fails to start the app' do + post "/v3/apps/#{app_model.guid}/actions/start", nil, admin_header + + expect(last_response.status).to eq(422) + expect(last_response).to have_error_message("log_rate_limit cannot be unlimited in space '#{space.name}'.") + end + end + + context "when the app's log rate limit is larger than the limit specified by the space" do + let(:log_rate_limit) { 201 } + let(:space_log_rate_limit) { 200 } + + it 'fails to start the app' do + post "/v3/apps/#{app_model.guid}/actions/start", nil, admin_header + + expect(last_response.status).to eq(422) + expect(last_response).to have_error_message('log_rate_limit exceeds space log rate quota') + end + end + + context "when the space's quota is more strict that the org's quota, the space quota controls" do + let(:log_rate_limit) { 201 } + let(:space_log_rate_limit) { 200 } + let(:org_log_rate_limit) { 201 } + + it 'fails to start the app' do + post "/v3/apps/#{app_model.guid}/actions/start", nil, admin_header + + expect(last_response.status).to eq(422) + expect(last_response).to have_error_message('log_rate_limit exceeds space log rate quota') + end + end + end + + describe 'organization quotas' do + context 'when both the org and the app do not specify a log rate limit' do + let(:log_rate_limit) { -1 } + let(:org_log_rate_limit) { -1 } + + it 'starts the app successfully' do + post "/v3/apps/#{app_model.guid}/actions/start", nil, admin_header + + expect(last_response.status).to eq(200) + end + end + + context "when the app fits in the org's log rate limit" do + let(:log_rate_limit) { 199 } + let(:org_log_rate_limit) { 200 } + + it 'starts the app successfully' do + post "/v3/apps/#{app_model.guid}/actions/start", nil, admin_header + + expect(last_response.status).to eq(200) + end + end + + context "when the app's log rate limit is unspecified, but the org specifies a log rate limit" do + let(:log_rate_limit) { -1 } + let(:org_log_rate_limit) { 200 } + + it 'fails to start the app' do + post "/v3/apps/#{app_model.guid}/actions/start", nil, admin_header + + expect(last_response.status).to eq(422) + expect(last_response).to have_error_message("log_rate_limit cannot be unlimited in organization '#{org.name}'.") + end + end + + context "when the app's log rate limit is larger than the limit specified by the org" do + let(:log_rate_limit) { 201 } + let(:org_log_rate_limit) { 200 } + + it 'fails to start the app' do + post "/v3/apps/#{app_model.guid}/actions/start", nil, admin_header + + expect(last_response.status).to eq(422) + expect(last_response).to have_error_message('log_rate_limit exceeds organization log rate quota') + end + end + + context "when the org's quota is more strict that the space's quota, the org quota controls" do + let(:log_rate_limit) { 201 } + let(:space_log_rate_limit) { 202 } + let(:org_log_rate_limit) { 200 } + + it 'fails to start the app' do + post "/v3/apps/#{app_model.guid}/actions/start", nil, admin_header + + expect(last_response.status).to eq(422) + expect(last_response).to have_error_message('log_rate_limit exceeds organization log rate quota') + end + end + end + end end context 'events' do diff --git a/spec/request/organization_quotas_spec.rb b/spec/request/organization_quotas_spec.rb index 6fcbec4699a..e3dcf90c3f2 100644 --- a/spec/request/organization_quotas_spec.rb +++ b/spec/request/organization_quotas_spec.rb @@ -35,7 +35,8 @@ module VCAP::CloudController total_memory_in_mb: nil, per_process_memory_in_mb: nil, total_instances: nil, - per_app_tasks: nil + per_app_tasks: nil, + log_rate_limit_in_bytes_per_second: nil }, services: { paid_services_allowed: true, @@ -91,7 +92,8 @@ module VCAP::CloudController total_memory_in_mb: 5120, per_process_memory_in_mb: 1024, total_instances: 10, - per_app_tasks: 5 + per_app_tasks: 5, + log_rate_limit_in_bytes_per_second: 2000 }, services: { paid_services_allowed: false, @@ -118,7 +120,8 @@ module VCAP::CloudController total_memory_in_mb: 5120, per_process_memory_in_mb: 1024, total_instances: 10, - per_app_tasks: 5 + per_app_tasks: 5, + log_rate_limit_in_bytes_per_second: 2000 }, services: { paid_services_allowed: false, @@ -319,7 +322,8 @@ module VCAP::CloudController total_memory_in_mb: 5120, per_process_memory_in_mb: 1024, total_instances: nil, - per_app_tasks: 5 + per_app_tasks: 5, + log_rate_limit_in_bytes_per_second: 2000 }, services: { paid_services_allowed: false, @@ -346,7 +350,8 @@ module VCAP::CloudController total_memory_in_mb: 5120, per_process_memory_in_mb: 1024, total_instances: nil, - per_app_tasks: 5 + per_app_tasks: 5, + log_rate_limit_in_bytes_per_second: 2000 }, services: { paid_services_allowed: false, @@ -423,6 +428,7 @@ module VCAP::CloudController expect(org_quota_to_update.reload.app_task_limit).to eq(9) expect(org_quota_to_update.reload.memory_limit).to eq(-1) expect(org_quota_to_update.reload.total_services).to eq(14) + expect(org_quota_to_update.reload.log_rate_limit).to eq(-1) expect(org_quota_to_update.reload.non_basic_services_allowed).to be_falsey end @@ -434,6 +440,7 @@ module VCAP::CloudController expect(org_quota_to_update.reload.app_task_limit).to eq(9) expect(org_quota_to_update.reload.memory_limit).to eq(-1) expect(org_quota_to_update.reload.total_services).to eq(14) + expect(org_quota_to_update.reload.log_rate_limit).to eq(-1) expect(org_quota_to_update.reload.non_basic_services_allowed).to be_falsey end end @@ -591,7 +598,8 @@ def generate_org_quota_single_response(list_of_orgs) total_memory_in_mb: 20480, per_process_memory_in_mb: nil, total_instances: nil, - per_app_tasks: nil + per_app_tasks: nil, + log_rate_limit_in_bytes_per_second: nil }, services: { paid_services_allowed: true, @@ -639,7 +647,8 @@ def generate_default_org_quota_response(global_read) total_memory_in_mb: 10240, per_process_memory_in_mb: nil, total_instances: nil, - per_app_tasks: nil + per_app_tasks: nil, + log_rate_limit_in_bytes_per_second: nil }, services: { paid_services_allowed: true, diff --git a/spec/request/processes_spec.rb b/spec/request/processes_spec.rb index 4ad1144cf99..7a766fce183 100644 --- a/spec/request/processes_spec.rb +++ b/spec/request/processes_spec.rb @@ -37,6 +37,7 @@ instances: 2, memory: 1024, disk_quota: 1024, + log_rate_limit: 1_048_576, command: 'rackup', ) } @@ -48,6 +49,7 @@ instances: 1, memory: 100, disk_quota: 200, + log_rate_limit: 400, command: 'start worker', ) } @@ -111,11 +113,12 @@ } }, }, - 'type' => 'web', - 'command' => '[PRIVATE DATA HIDDEN IN LISTS]', - 'instances' => 2, - 'memory_in_mb' => 1024, - 'disk_in_mb' => 1024, + 'type' => 'web', + 'command' => '[PRIVATE DATA HIDDEN IN LISTS]', + 'instances' => 2, + 'memory_in_mb' => 1024, + 'disk_in_mb' => 1024, + 'log_rate_limit_in_bytes_per_second' => 1_048_576, 'health_check' => { 'type' => 'port', 'data' => { @@ -140,11 +143,12 @@ 'app' => { 'data' => { 'guid' => app_model.guid } }, 'revision' => nil, }, - 'type' => 'worker', - 'command' => '[PRIVATE DATA HIDDEN IN LISTS]', - 'instances' => 1, - 'memory_in_mb' => 100, - 'disk_in_mb' => 200, + 'type' => 'worker', + 'command' => '[PRIVATE DATA HIDDEN IN LISTS]', + 'instances' => 1, + 'memory_in_mb' => 100, + 'disk_in_mb' => 200, + 'log_rate_limit_in_bytes_per_second' => 400, 'health_check' => { 'type' => 'port', 'data' => { @@ -229,6 +233,7 @@ instances: 2, memory: 1024, disk_quota: 1024, + log_rate_limit: 1_048_576, command: 'rackup', ) } @@ -270,6 +275,7 @@ instances: 2, memory: 1024, disk_quota: 1024, + log_rate_limit: 1_048_576, command: 'rackup', ) } @@ -307,6 +313,7 @@ instances: 3, memory: 2048, disk_quota: 2048, + log_rate_limit: 2_097_152, command: 'at ease' ) end @@ -383,6 +390,7 @@ instances: 2, memory: 1024, disk_quota: 1024, + log_rate_limit: 1_048_576, command: 'rackup', ) end @@ -394,10 +402,11 @@ 'app' => { 'data' => { 'guid' => app_model.guid } }, 'revision' => { 'data' => { 'guid' => revision.guid } }, }, - 'command' => 'rackup', - 'instances' => 2, - 'memory_in_mb' => 1024, - 'disk_in_mb' => 1024, + 'command' => 'rackup', + 'instances' => 2, + 'memory_in_mb' => 1024, + 'disk_in_mb' => 1024, + 'log_rate_limit_in_bytes_per_second' => 1_048_576, 'health_check' => { 'type' => 'port', 'data' => { @@ -494,12 +503,14 @@ uptime: 12345, mem_quota: process[:memory] * 1024 * 1024, disk_quota: process[:disk_quota] * 1024 * 1024, + log_rate_limit: process[:log_rate_limit], fds_quota: process.file_descriptors, usage: { time: usage_time, cpu: 80, mem: 128, disk: 1024, + log_rate: 1024, } } }, @@ -522,6 +533,7 @@ 'cpu' => 80, 'mem' => 128, 'disk' => 1024, + 'log_rate' => 1024, }, 'host' => 'toast', 'instance_ports' => [ @@ -541,7 +553,8 @@ 'uptime' => 12345, 'mem_quota' => 1073741824, 'disk_quota' => 1073741824, - 'fds_quota' => 16384 + 'fds_quota' => 16384, + 'log_rate_limit' => 1_048_576 }] } end @@ -602,6 +615,7 @@ instances: 2, memory: 1024, disk_quota: 1024, + log_rate_limit: 1_048_576, command: 'rackup', ports: [4444, 5555], health_check_type: 'port', @@ -634,6 +648,7 @@ 'instances' => 2, 'memory_in_mb' => 1024, 'disk_in_mb' => 1024, + 'log_rate_limit_in_bytes_per_second' => 1_048_576, 'health_check' => { 'type' => 'process', 'data' => { @@ -748,15 +763,17 @@ instances: 2, memory: 1024, disk_quota: 1024, + log_rate_limit: 1_048_576, command: 'rackup', ) end let(:scale_request) do { - instances: 5, + instances: 5, memory_in_mb: 10, - disk_in_mb: 20, + disk_in_mb: 20, + log_rate_limit_in_bytes_per_second: 40, } end @@ -773,6 +790,7 @@ 'instances' => 5, 'memory_in_mb' => 10, 'disk_in_mb' => 20, + 'log_rate_limit_in_bytes_per_second' => 40, 'health_check' => { 'type' => 'port', 'data' => { @@ -805,6 +823,7 @@ expect(process.instances).to eq(5) expect(process.memory).to eq(10) expect(process.disk_quota).to eq(20) + expect(process.log_rate_limit).to eq(40) events = VCAP::CloudController::Event.where(actor: developer.guid).all @@ -826,7 +845,8 @@ 'request' => { 'instances' => 5, 'memory_in_mb' => 10, - 'disk_in_mb' => 20 + 'disk_in_mb' => 20, + 'log_rate_limit_in_bytes_per_second' => 40, } }) end @@ -875,6 +895,7 @@ instances: 5, memory_in_mb: 10, disk_in_mb: 20, + log_rate_limit_in_bytes_per_second: 40, } post "/v3/processes/#{process.guid}/actions/scale", scale_request.to_json, headers_for(space_supporter) @@ -886,6 +907,7 @@ expect(process.instances).to eq(5) expect(process.memory).to eq(10) expect(process.disk_quota).to eq(20) + expect(process.log_rate_limit).to eq(40) end end @@ -924,6 +946,20 @@ expect(process.memory).to eq(1024) end + it 'returns a helpful error when the log quota is too small' do + scale_request = { + log_rate_limit_in_bytes_per_second: -2, + } + + post "/v3/processes/#{process.guid}/actions/scale", scale_request.to_json, developer_headers + + expect(last_response.status).to eq(422) + expect(parsed_response['errors'][0]['detail']).to eq 'Log rate limit in bytes per second must be greater than or equal to -1' + + process.reload + expect(process.log_rate_limit).to eq(1_048_576) + end + context 'telemetry' do let(:process) { VCAP::CloudController::ProcessModel.make( @@ -933,6 +969,7 @@ instances: 2, memory: 1024, disk_quota: 1024, + log_rate_limit: 1_048_576, command: 'rackup', ) } @@ -941,6 +978,7 @@ instances: 5, memory_in_mb: 10, disk_in_mb: 20, + log_rate_limit_in_bytes_per_second: 40, } end @@ -954,6 +992,7 @@ 'instance-count' => 5, 'memory-in-mb' => 10, 'disk-in-mb' => 20, + 'log-rate-in-bytes-per-second' => 40, 'process-type' => 'web', 'app-id' => Digest::SHA256.hexdigest(process.app.guid), 'user-id' => Digest::SHA256.hexdigest(developer.guid), @@ -1042,6 +1081,7 @@ instances: 2, memory: 1024, disk_quota: 1024, + log_rate_limit: 1_048_576, command: 'rackup', ) } @@ -1055,6 +1095,7 @@ instances: 1, memory: 100, disk_quota: 200, + log_rate_limit: 400, command: 'start worker', ) } @@ -1103,6 +1144,7 @@ 'instances' => 2, 'memory_in_mb' => 1024, 'disk_in_mb' => 1024, + 'log_rate_limit_in_bytes_per_second' => 1_048_576, 'health_check' => { 'type' => 'port', 'data' => { @@ -1136,6 +1178,7 @@ 'instances' => 1, 'memory_in_mb' => 100, 'disk_in_mb' => 200, + 'log_rate_limit_in_bytes_per_second' => 400, 'health_check' => { 'type' => 'port', 'data' => { @@ -1237,6 +1280,7 @@ instances: 2, memory: 1024, disk_quota: 1024, + log_rate_limit: 1_048_576, command: 'rackup', ) } @@ -1253,6 +1297,7 @@ 'instances' => 2, 'memory_in_mb' => 1024, 'disk_in_mb' => 1024, + 'log_rate_limit_in_bytes_per_second' => 1_048_576, 'health_check' => { 'type' => 'port', 'data' => { @@ -1322,6 +1367,7 @@ instances: 2, memory: 1024, disk_quota: 1024, + log_rate_limit: 1_048_576, command: 'rackup', ports: [4444, 5555], health_check_type: 'port', @@ -1353,6 +1399,7 @@ 'instances' => 2, 'memory_in_mb' => 1024, 'disk_in_mb' => 1024, + 'log_rate_limit_in_bytes_per_second' => 1_048_576, 'health_check' => { 'type' => 'http', 'data' => { @@ -1435,7 +1482,9 @@ instances: 2, memory: 1024, disk_quota: 1024, + log_rate_limit: 1_048_576, command: 'rackup', + state: 'STARTED', ) end @@ -1443,6 +1492,7 @@ instances: 5, memory_in_mb: 10, disk_in_mb: 20, + log_rate_limit_in_bytes_per_second: 40, } end @@ -1459,6 +1509,7 @@ 'instances' => 5, 'memory_in_mb' => 10, 'disk_in_mb' => 20, + 'log_rate_limit_in_bytes_per_second' => 40, 'health_check' => { 'type' => 'port', 'data' => { @@ -1490,6 +1541,7 @@ expect(process.instances).to eq(5) expect(process.memory).to eq(10) expect(process.disk_quota).to eq(20) + expect(process.log_rate_limit).to eq(40) events = VCAP::CloudController::Event.where(actor: developer.guid).all @@ -1511,11 +1563,34 @@ 'request' => { 'instances' => 5, 'memory_in_mb' => 10, - 'disk_in_mb' => 20 + 'disk_in_mb' => 20, + 'log_rate_limit_in_bytes_per_second' => 40, } }) end + context 'when the log rate limit would be exceeded by adding additional instances' do + let(:scale_request) do + { + instances: 5 + } + end + + before do + org.update(quota_definition: VCAP::CloudController::QuotaDefinition.make(log_rate_limit: 2_097_152)) + end + + it 'fails to scale the process' do + post "/v3/apps/#{app_model.guid}/processes/web/actions/scale", scale_request.to_json, developer_headers + + expect(last_response.status).to eq(422) + expect(parsed_response['errors'][0]['detail']).to eq 'log_rate_limit exceeds organization log rate quota' + + process.reload + expect(process.instances).to eq(2) + end + end + context 'when the user is assigned the space_supporter role' do let(:space_supporter) do user = VCAP::CloudController::User.make @@ -1534,6 +1609,7 @@ expect(process.instances).to eq(5) expect(process.memory).to eq(10) expect(process.disk_quota).to eq(20) + expect(process.log_rate_limit).to eq(40) end end @@ -1548,6 +1624,7 @@ 'instance-count' => 5, 'memory-in-mb' => 10, 'disk-in-mb' => 20, + 'log-rate-in-bytes-per-second' => 40, 'process-type' => 'web', 'app-id' => Digest::SHA256.hexdigest(app_model.guid), 'user-id' => Digest::SHA256.hexdigest(developer.guid), diff --git a/spec/request/space_manifests_spec.rb b/spec/request/space_manifests_spec.rb index 3c2c4f6da9c..529eac982b1 100644 --- a/spec/request/space_manifests_spec.rb +++ b/spec/request/space_manifests_spec.rb @@ -27,6 +27,7 @@ 'instances' => 4, 'memory' => '2048MB', 'disk_quota' => '1.5GB', + 'log_rate_limit_per_second' => '1MB', 'buildpack' => buildpack.name, 'stack' => buildpack.stack, 'command' => 'new-command', @@ -73,6 +74,7 @@ 'instances' => 3, 'memory' => '2048MB', 'disk_quota' => '1.5GB', + 'log_rate_limit_per_second' => '100KB', 'buildpack' => buildpack.name, 'stack' => buildpack.stack, 'command' => 'newer-command', @@ -172,6 +174,7 @@ expect(web_process.instances).to eq(4) expect(web_process.memory).to eq(2048) expect(web_process.disk_quota).to eq(1536) + expect(web_process.log_rate_limit).to eq(1_048_576) expect(web_process.command).to eq('new-command') expect(web_process.health_check_type).to eq('http') expect(web_process.health_check_http_endpoint).to eq('/health') @@ -244,6 +247,7 @@ 'instances' => 4, 'memory' => '2048MB', 'disk_quota' => '1.5GB', + 'log_rate_limit_per_second' => '1GB', 'buildpack' => buildpack.name, 'stack' => buildpack.stack, 'command' => 'new-command', @@ -267,6 +271,7 @@ 'instances' => 4, 'memory' => '2048MB', 'disk_quota' => '1.5GB', + 'log_rate_limit_per_second' => '-1B', 'buildpack' => buildpack.name, 'stack' => buildpack.stack, 'command' => 'new-command', @@ -304,6 +309,7 @@ expect(web_process.instances).to eq(4) expect(web_process.memory).to eq(2048) expect(web_process.disk_quota).to eq(1536) + expect(web_process.log_rate_limit).to eq(-1) expect(web_process.command).to eq('new-command') expect(web_process.health_check_type).to eq('http') expect(web_process.health_check_http_endpoint).to eq('/health') @@ -455,6 +461,56 @@ end end + context 'when -1 is given as a log rate limit' do + let(:yml_manifest) do + { + 'version' => 1, + 'applications' => [ + { 'name' => app1_model.name, + 'log_rate_limit_per_second' => -1 + }, + ] + }.to_yaml + end + + it 'interprets the log rate limit as unlimited' do + post "/v3/spaces/#{space.guid}/actions/apply_manifest", yml_manifest, yml_headers(user_header) + + expect(last_response.status).to eq(202) + job_guid = VCAP::CloudController::PollableJobModel.last.guid + expect(last_response.headers['Location']).to match(%r(/v3/jobs/#{job_guid})) + + Delayed::Worker.new.work_off + expect(VCAP::CloudController::PollableJobModel.find(guid: job_guid)).to be_complete, + VCAP::CloudController::PollableJobModel.find(guid: job_guid).cf_api_error + + app1_model.reload + expect(app1_model.processes.first.log_rate_limit).to eq(-1) + end + end + + context 'when applying the manifest to an app which is already exceeding the log rate limit' do + before do + app1_model.web_processes.first.update(state: VCAP::CloudController::ProcessModel::STARTED, instances: 4) + space.update(space_quota_definition: + VCAP::CloudController::SpaceQuotaDefinition.make(organization: space.organization, log_rate_limit: 0)) + end + + it 'successfully applies the manifest' do + post "/v3/spaces/#{space.guid}/actions/apply_manifest", yml_manifest, yml_headers(user_header) + + expect(last_response.status).to eq(202) + + job_guid = VCAP::CloudController::PollableJobModel.last.guid + expect(last_response.headers['Location']).to match(%r(/v3/jobs/#{job_guid})) + + Delayed::Worker.new.work_off + # job does not restart app, so applying the manifest succeeds + expect(VCAP::CloudController::PollableJobModel.find(guid: job_guid)).to be_complete, + VCAP::CloudController::PollableJobModel.find(guid: job_guid).cf_api_error + end + end + describe 'audit events' do let!(:process1) { nil } @@ -465,6 +521,7 @@ 'instances' => 4, 'memory' => '2048MB', 'disk_quota' => '1.5GB', + 'log_rate_limit_per_second' => '300B', 'buildpack' => buildpack.name, 'stack' => buildpack.stack, 'command' => 'new-command', diff --git a/spec/request/space_quotas_spec.rb b/spec/request/space_quotas_spec.rb index 5c67463dc72..67a3dfed230 100644 --- a/spec/request/space_quotas_spec.rb +++ b/spec/request/space_quotas_spec.rb @@ -88,7 +88,8 @@ module VCAP::CloudController total_memory_in_mb: 5120, per_process_memory_in_mb: 1024, total_instances: nil, - per_app_tasks: 5 + per_app_tasks: 5, + log_rate_limit_in_bytes_per_second: 2000 }, services: { paid_services_allowed: false, @@ -112,7 +113,8 @@ module VCAP::CloudController total_memory_in_mb: 5120, per_process_memory_in_mb: 1024, total_instances: nil, - per_app_tasks: 5 + per_app_tasks: 5, + log_rate_limit_in_bytes_per_second: 2000 }, services: { paid_services_allowed: false, @@ -208,6 +210,7 @@ module VCAP::CloudController expect(last_response).to have_status_code(200) expect(space_quota_to_update.reload.app_task_limit).to eq(9) expect(space_quota_to_update.reload.memory_limit).to eq(-1) + expect(space_quota_to_update.reload.log_rate_limit).to eq(-1) expect(space_quota_to_update.reload.total_services).to eq(14) expect(space_quota_to_update.reload.non_basic_services_allowed).to be_falsey end @@ -219,6 +222,7 @@ module VCAP::CloudController expect(last_response).to have_status_code(200) expect(space_quota_to_update.reload.app_task_limit).to eq(9) expect(space_quota_to_update.reload.memory_limit).to eq(-1) + expect(space_quota_to_update.reload.log_rate_limit).to eq(-1) expect(space_quota_to_update.reload.total_services).to eq(14) expect(space_quota_to_update.reload.non_basic_services_allowed).to be_falsey end @@ -406,7 +410,8 @@ module VCAP::CloudController total_memory_in_mb: nil, per_process_memory_in_mb: nil, total_instances: nil, - per_app_tasks: nil + per_app_tasks: nil, + log_rate_limit_in_bytes_per_second: nil }, services: { paid_services_allowed: true, @@ -495,7 +500,8 @@ module VCAP::CloudController total_memory_in_mb: nil, per_process_memory_in_mb: nil, total_instances: nil, - per_app_tasks: nil + per_app_tasks: nil, + log_rate_limit_in_bytes_per_second: nil }, services: { paid_services_allowed: true, @@ -555,7 +561,8 @@ module VCAP::CloudController total_memory_in_mb: 5120, per_process_memory_in_mb: 1024, total_instances: 10, - per_app_tasks: 5 + per_app_tasks: 5, + log_rate_limit_in_bytes_per_second: 3000 }, services: { paid_services_allowed: false, @@ -589,7 +596,8 @@ module VCAP::CloudController total_memory_in_mb: 5120, per_process_memory_in_mb: 1024, total_instances: 10, - per_app_tasks: 5 + per_app_tasks: 5, + log_rate_limit_in_bytes_per_second: 3000 }, services: { paid_services_allowed: false, @@ -968,7 +976,8 @@ def make_space_quota_json(space_quota, associated_spaces=space_quota.spaces) total_memory_in_mb: 20480, per_process_memory_in_mb: nil, total_instances: nil, - per_app_tasks: 5 + per_app_tasks: 5, + log_rate_limit_in_bytes_per_second: nil }, services: { paid_services_allowed: true, diff --git a/spec/request/tasks_spec.rb b/spec/request/tasks_spec.rb index 0b030affc90..c2946239d48 100644 --- a/spec/request/tasks_spec.rb +++ b/spec/request/tasks_spec.rb @@ -2,9 +2,14 @@ require 'request_spec_shared_examples' RSpec.describe 'Tasks' do + let(:org_quota_definition) { VCAP::CloudController::QuotaDefinition.make(log_rate_limit: org_log_rate_limit) } + let(:space_quota_definition) { VCAP::CloudController::SpaceQuotaDefinition.make(organization: org, log_rate_limit: space_log_rate_limit) } + let(:space_log_rate_limit) { -1 } + let(:org_log_rate_limit) { -1 } + let(:task_log_rate_limit_in_bytes_per_second) { -1 } let(:user) { VCAP::CloudController::User.make } - let(:org) { space.organization } - let(:space) { VCAP::CloudController::Space.make } + let(:org) { VCAP::CloudController::Organization.make(quota_definition: org_quota_definition) } + let(:space) { VCAP::CloudController::Space.make(space_quota_definition: space_quota_definition, organization: org) } let(:app_model) { VCAP::CloudController::AppModel.make(space_guid: space.guid) } let(:droplet) do VCAP::CloudController::DropletModel.make( @@ -101,6 +106,7 @@ droplet: app_model.droplet, memory_in_mb: 5, disk_in_mb: 10, + log_rate_limit: 20, ) task2 = VCAP::CloudController::TaskModel.make( name: 'task two', @@ -109,6 +115,7 @@ droplet: app_model.droplet, memory_in_mb: 100, disk_in_mb: 500, + log_rate_limit: 1024, ) VCAP::CloudController::TaskModel.make( app_guid: app_model.guid, @@ -137,7 +144,8 @@ 'state' => 'RUNNING', 'memory_in_mb' => 5, 'disk_in_mb' => 10, - 'result' => { + 'log_rate_limit_in_bytes_per_second' => 20, + 'result' => { 'failure_reason' => nil }, 'droplet_guid' => task1.droplet.guid, @@ -168,7 +176,8 @@ 'state' => 'RUNNING', 'memory_in_mb' => 100, 'disk_in_mb' => 500, - 'result' => { + 'log_rate_limit_in_bytes_per_second' => 1024, + 'result' => { 'failure_reason' => nil }, 'droplet_guid' => task2.droplet.guid, @@ -216,6 +225,7 @@ droplet: app_model.droplet, memory_in_mb: 5, disk_in_mb: 10, + log_rate_limit: 20, ) end @@ -228,7 +238,8 @@ 'state' => 'RUNNING', 'memory_in_mb' => 256, 'disk_in_mb' => nil, - 'result' => { + 'log_rate_limit_in_bytes_per_second' => -1, + 'result' => { 'failure_reason' => nil }, 'droplet_guid' => task1.droplet.guid, @@ -259,7 +270,8 @@ 'state' => 'RUNNING', 'memory_in_mb' => 5, 'disk_in_mb' => 10, - 'result' => { + 'log_rate_limit_in_bytes_per_second' => 20, + 'result' => { 'failure_reason' => nil }, 'droplet_guid' => task2.droplet.guid, @@ -407,6 +419,7 @@ droplet: app_model.droplet, memory_in_mb: 5, disk_in_mb: 50, + log_rate_limit: 64, ) end let(:api_call) { lambda { |user_headers| get "/v3/tasks/#{task.guid}", nil, user_headers } } @@ -418,7 +431,8 @@ 'state' => 'RUNNING', 'memory_in_mb' => 5, 'disk_in_mb' => 50, - 'result' => { + 'log_rate_limit_in_bytes_per_second' => 64, + 'result' => { 'failure_reason' => nil }, 'droplet_guid' => task.droplet.guid, @@ -498,8 +512,10 @@ command: 'echo task', app_guid: app_model.guid, droplet_guid: app_model.droplet.guid, - disk_in_mb: 50, - memory_in_mb: 5) + disk_in_mb: 50, + memory_in_mb: 5, + log_rate_limit: 10, + ) } let(:request_body) do { @@ -528,7 +544,8 @@ 'state' => 'RUNNING', 'memory_in_mb' => 5, 'disk_in_mb' => 50, - 'result' => { + 'log_rate_limit_in_bytes_per_second' => 10, + 'result' => { 'failure_reason' => nil }, 'droplet_guid' => task.droplet.guid, @@ -605,6 +622,7 @@ state: 'CANCELING', memory_in_mb: 256, disk_in_mb: nil, + log_rate_limit_in_bytes_per_second: -1, result: { failure_reason: nil }, @@ -683,6 +701,7 @@ droplet: app_model.droplet, memory_in_mb: 5, disk_in_mb: 50, + log_rate_limit: 64, ) task2 = VCAP::CloudController::TaskModel.make( name: 'task two', @@ -691,6 +710,7 @@ droplet: app_model.droplet, memory_in_mb: 100, disk_in_mb: 500, + log_rate_limit: 256, ) VCAP::CloudController::TaskModel.make( app_guid: app_model.guid, @@ -718,7 +738,8 @@ 'state' => 'RUNNING', 'memory_in_mb' => 5, 'disk_in_mb' => 50, - 'result' => { + 'log_rate_limit_in_bytes_per_second' => 64, + 'result' => { 'failure_reason' => nil }, 'droplet_guid' => task1.droplet.guid, @@ -750,7 +771,8 @@ 'state' => 'RUNNING', 'memory_in_mb' => 100, 'disk_in_mb' => 500, - 'result' => { + 'log_rate_limit_in_bytes_per_second' => 256, + 'result' => { 'failure_reason' => nil }, 'droplet_guid' => task2.droplet.guid, @@ -815,7 +837,8 @@ 'state' => 'RUNNING', 'memory_in_mb' => 256, 'disk_in_mb' => nil, - 'result' => { + 'log_rate_limit_in_bytes_per_second' => -1, + 'result' => { 'failure_reason' => nil }, 'droplet_guid' => task1.droplet.guid, @@ -846,7 +869,8 @@ 'state' => 'RUNNING', 'memory_in_mb' => 5, 'disk_in_mb' => 10, - 'result' => { + 'log_rate_limit_in_bytes_per_second' => -1, + 'result' => { 'failure_reason' => nil }, 'droplet_guid' => task2.droplet.guid, @@ -997,6 +1021,7 @@ command: 'be rake && true', memory_in_mb: 1234, disk_in_mb: 1000, + log_rate_limit_in_bytes_per_second: task_log_rate_limit_in_bytes_per_second, metadata: { labels: { bananas: 'gros_michel', @@ -1029,7 +1054,8 @@ 'state' => 'RUNNING', 'memory_in_mb' => 1234, 'disk_in_mb' => 1000, - 'result' => { + 'log_rate_limit_in_bytes_per_second' => task_log_rate_limit_in_bytes_per_second, + 'result' => { 'failure_reason' => nil }, 'droplet_guid' => droplet.guid, @@ -1082,6 +1108,71 @@ }) end + describe 'log_rate_limit' do + context 'when the request does not specify a log rate limit' do + before do + TestConfig.config[:default_app_log_rate_limit_in_bytes_per_second] = 9876 + end + + it 'the default is applied' do + post "/v3/apps/#{app_model.guid}/tasks", body.except(:log_rate_limit_in_bytes_per_second).to_json, developer_headers + expect(last_response.status).to eq(202) + expect(VCAP::CloudController::TaskModel.last.log_rate_limit).to eq(9876) + end + end + + context 'when there are org or space log rate limits' do + let(:space_log_rate_limit) { 200 } + let(:org_log_rate_limit) { 201 } + + context 'when the task specifies a rate limit that fits in the quota' do + let(:task_log_rate_limit_in_bytes_per_second) { 199 } + + it 'succeeds' do + post "/v3/apps/#{app_model.guid}/tasks", body.to_json, developer_headers + expect(last_response.status).to eq(202) + end + end + + context 'when the task specifies unlimited rate limit' do + let(:task_log_rate_limit_in_bytes_per_second) { -1 } + + it 'returns an error' do + post "/v3/apps/#{app_model.guid}/tasks", body.to_json, developer_headers + expect(last_response.status).to eq(422) + expect(last_response).to have_error_message("log_rate_limit cannot be unlimited in organization '#{org.name}'.") + expect(last_response).to have_error_message("log_rate_limit cannot be unlimited in space '#{space.name}'.") + end + end + + context 'when the task specifies a rate limit that does not fit in the quota' do + let(:task_log_rate_limit_in_bytes_per_second) { 202 } + + context 'fails to fit in space quota' do + let(:space_log_rate_limit) { 200 } + let(:org_log_rate_limit) { -1 } + + it 'returns an error' do + post "/v3/apps/#{app_model.guid}/tasks", body.to_json, developer_headers + expect(last_response.status).to eq(422) + expect(last_response).to have_error_message('log_rate_limit exceeds space log rate quota') + end + end + + context 'fails to fit in org quota' do + let(:space_log_rate_limit) { -1 } + let(:org_log_rate_limit) { 200 } + + it 'returns an error' do + post "/v3/apps/#{app_model.guid}/tasks", body.to_json, developer_headers + expect(last_response.status).to eq(422) + expect(last_response).to have_error_message('log_rate_limit exceeds organization log rate quota') + end + end + end + end + end + context 'when requesting a specific droplet' do let(:non_assigned_droplet) do VCAP::CloudController::DropletModel.make( @@ -1137,7 +1228,8 @@ 'state' => 'RUNNING', 'memory_in_mb' => 1234, 'disk_in_mb' => 1000, - 'result' => { + 'log_rate_limit_in_bytes_per_second' => process.log_rate_limit, + 'result' => { 'failure_reason' => nil }, 'droplet_guid' => droplet.guid, diff --git a/spec/request/v2/apps_spec.rb b/spec/request/v2/apps_spec.rb index 4233bd67008..85a0fc26271 100644 --- a/spec/request/v2/apps_spec.rb +++ b/spec/request/v2/apps_spec.rb @@ -61,6 +61,7 @@ 'memory' => 1024, 'instances' => 1, 'disk_quota' => 1024, + 'log_rate_limit' => 1_048_576, 'state' => 'STOPPED', 'version' => process.version, 'command' => 'hello_world', @@ -228,6 +229,7 @@ 'memory' => 1024, 'instances' => 1, 'disk_quota' => 1024, + 'log_rate_limit' => 1_048_576, 'state' => 'STOPPED', 'version' => process.version, 'command' => 'hello_world', @@ -447,6 +449,7 @@ 'memory' => 1024, 'instances' => 1, 'disk_quota' => 1024, + 'log_rate_limit' => 1_048_576, 'state' => 'STOPPED', 'version' => process.version, 'command' => 'app-command', @@ -522,6 +525,7 @@ 'memory' => 1024, 'instances' => 1, 'disk_quota' => 1024, + 'log_rate_limit' => 1_048_576, 'state' => 'STOPPED', 'version' => process.version, 'command' => nil, @@ -667,6 +671,7 @@ 'memory' => 1024, 'instances' => 1, 'disk_quota' => 1024, + 'log_rate_limit' => 1_048_576, 'state' => 'STOPPED', 'version' => process.version, 'command' => nil, @@ -746,6 +751,7 @@ }, 'memory' => 1024, 'instances' => 1, + 'log_rate_limit' => 1_048_576, 'disk_quota' => 1024, 'state' => 'STARTED', 'version' => process.version, @@ -1012,6 +1018,7 @@ 'memory' => 1024, 'instances' => 1, 'disk_quota' => 1024, + 'log_rate_limit' => 1_048_576, 'state' => 'STARTED', 'version' => process.version, 'command' => nil, @@ -1089,6 +1096,7 @@ 'memory' => 1024, 'instances' => 1, 'disk_quota' => 1024, + 'log_rate_limit' => 1_048_576, 'state' => 'STARTED', 'version' => process.version, 'command' => nil, @@ -1217,6 +1225,7 @@ 'environment_json' => nil, 'memory' => 1024, 'instances' => 1, + 'log_rate_limit' => 1_048_576, 'disk_quota' => 1024, 'state' => 'STOPPED', 'version' => process.version, @@ -1497,6 +1506,7 @@ def make_actual_lrp(instance_guid:, index:, state:, error:, since:) 'memory' => 1024, 'instances' => 1, 'disk_quota' => 1024, + 'log_rate_limit' => 1_048_576, 'state' => 'STARTED', 'version' => process.version, 'command' => nil, @@ -1852,6 +1862,7 @@ def make_actual_lrp(instance_guid:, index:, state:, error:, since:) 'memory' => 1024, 'instances' => 1, 'disk_quota' => 1024, + 'log_rate_limit' => 1_048_576, 'state' => 'STOPPED', 'version' => process.version, 'command' => nil, diff --git a/spec/request/v2/routes_spec.rb b/spec/request/v2/routes_spec.rb index c34d02d72e3..ae786b43285 100644 --- a/spec/request/v2/routes_spec.rb +++ b/spec/request/v2/routes_spec.rb @@ -147,6 +147,7 @@ 'memory' => 1024, 'instances' => 1, 'disk_quota' => 1024, + 'log_rate_limit' => 1048576, 'state' => 'STOPPED', 'version' => process.version, 'command' => nil, @@ -349,6 +350,7 @@ 'memory' => 1024, 'instances' => 1, 'disk_quota' => 1024, + 'log_rate_limit' => 1048576, 'state' => 'STOPPED', 'version' => process.version, 'command' => nil, diff --git a/spec/request/v2/service_bindings_spec.rb b/spec/request/v2/service_bindings_spec.rb index d8705fff879..7485e5d6745 100644 --- a/spec/request/v2/service_bindings_spec.rb +++ b/spec/request/v2/service_bindings_spec.rb @@ -166,6 +166,7 @@ 'memory' => 1024, 'instances' => 1, 'disk_quota' => 1024, + 'log_rate_limit' => 1048576, 'state' => 'STOPPED', 'version' => process1.version, 'command' => nil, diff --git a/spec/request/v2/spaces_spec.rb b/spec/request/v2/spaces_spec.rb index e69b00436ec..713af2d6554 100644 --- a/spec/request/v2/spaces_spec.rb +++ b/spec/request/v2/spaces_spec.rb @@ -293,6 +293,7 @@ 'health_check_timeout' => nil, 'health_check_type' => 'port', 'instances' => 1, + 'log_rate_limit' => 1_048_576, 'memory' => 1024, 'name' => process.name, 'package_state' => 'STAGED', diff --git a/spec/unit/actions/organization_quotas_create_spec.rb b/spec/unit/actions/organization_quotas_create_spec.rb index 04349c77d44..3efe2d75acc 100644 --- a/spec/unit/actions/organization_quotas_create_spec.rb +++ b/spec/unit/actions/organization_quotas_create_spec.rb @@ -17,7 +17,8 @@ module VCAP::CloudController total_memory_in_mb: 1, per_process_memory_in_mb: 2, total_instances: 3, - per_app_tasks: 4 + per_app_tasks: 4, + log_rate_limit_in_bytes_per_second: 2000 }, services: { paid_services_allowed: false, @@ -57,6 +58,7 @@ module VCAP::CloudController expect(organization_quota.instance_memory_limit).to eq(2) expect(organization_quota.app_instance_limit).to eq(3) expect(organization_quota.app_task_limit).to eq(4) + expect(organization_quota.log_rate_limit).to eq(2000) expect(organization_quota.total_services).to eq(5) expect(organization_quota.total_service_keys).to eq(6) @@ -79,6 +81,7 @@ module VCAP::CloudController expect(organization_quota.instance_memory_limit).to eq(-1) expect(organization_quota.app_instance_limit).to eq(-1) expect(organization_quota.app_task_limit).to eq(-1) + expect(organization_quota.log_rate_limit).to eq(-1) expect(organization_quota.total_services).to eq(-1) expect(organization_quota.total_service_keys).to eq(-1) diff --git a/spec/unit/actions/organization_quotas_update_spec.rb b/spec/unit/actions/organization_quotas_update_spec.rb index f4a9bdefc6e..e75f9151bae 100644 --- a/spec/unit/actions/organization_quotas_update_spec.rb +++ b/spec/unit/actions/organization_quotas_update_spec.rb @@ -15,7 +15,8 @@ module VCAP::CloudController total_memory_in_mb: 5120, per_process_memory_in_mb: 1024, total_instances: 8, - per_app_tasks: nil + per_app_tasks: nil, + log_rate_limit_in_bytes_per_second: 2000 }, services: { paid_services_allowed: false, @@ -49,6 +50,7 @@ module VCAP::CloudController expect(updated_organization_quota.instance_memory_limit).to eq(1024) expect(updated_organization_quota.app_instance_limit).to eq(8) expect(updated_organization_quota.app_task_limit).to eq(-1) + expect(updated_organization_quota.log_rate_limit).to eq(2000) expect(updated_organization_quota.total_services).to eq(10) expect(updated_organization_quota.total_service_keys).to eq(20) @@ -64,6 +66,7 @@ module VCAP::CloudController updated_organization_quota = OrganizationQuotasUpdate.update(org_quota, minimum_message) expect(updated_organization_quota.name).to eq('org_quota_name') + expect(updated_organization_quota.log_rate_limit).to eq(-1) end context 'when a model validation fails' do diff --git a/spec/unit/actions/process_scale_spec.rb b/spec/unit/actions/process_scale_spec.rb index 45688d65dd0..9d56a1a6ba5 100644 --- a/spec/unit/actions/process_scale_spec.rb +++ b/spec/unit/actions/process_scale_spec.rb @@ -4,7 +4,14 @@ module VCAP::CloudController RSpec.describe ProcessScale do subject(:process_scale) { ProcessScale.new(user_audit_info, process, message) } - let(:valid_message_params) { { instances: 2, memory_in_mb: 100, disk_in_mb: 200 } } + let(:valid_message_params) do + { + instances: 2, + memory_in_mb: 100, + disk_in_mb: 200, + log_rate_limit_in_bytes_per_second: 409_600 + } + end let(:message) { ProcessScaleMessage.new(valid_message_params) } let(:app) { AppModel.make } let!(:process) { ProcessModelFactory.make(disk_quota: 50, app: app) } @@ -15,12 +22,14 @@ module VCAP::CloudController expect(process.instances).not_to eq(2) expect(process.memory).not_to eq(100) expect(process.disk_quota).not_to eq(200) + expect(process.log_rate_limit).not_to eq(409_600) process_scale.scale expect(process.reload.instances).to eq(2) expect(process.reload.memory).to eq(100) expect(process.reload.disk_quota).to eq(200) + expect(process.log_rate_limit).to eq(409_600) end it 'does not set instances if the user did not request it' do @@ -50,6 +59,15 @@ module VCAP::CloudController expect(process.disk_quota).to eq(original_value) end + it 'does not set log quota if the user did not request it' do + valid_message_params.delete(:log_rate_limit_in_bytes_per_second) + original_value = process.log_rate_limit + + process_scale.scale + + expect(process.log_rate_limit).to eq(original_value) + end + describe 'audit events' do it 'creates a process audit event' do expect(Repositories::ProcessEventRepository).to receive(:record_scale).with( @@ -58,7 +76,8 @@ module VCAP::CloudController { 'instances' => 2, 'memory_in_mb' => 100, - 'disk_in_mb' => 200 + 'disk_in_mb' => 200, + 'log_rate_limit_in_bytes_per_second' => 409_600, }, manifest_triggered: false ) @@ -76,7 +95,8 @@ module VCAP::CloudController { 'instances' => 2, 'memory_in_mb' => 100, - 'disk_in_mb' => 200 + 'disk_in_mb' => 200, + 'log_rate_limit_in_bytes_per_second' => 409_600, }, manifest_triggered: true ) @@ -109,12 +129,14 @@ module VCAP::CloudController expect(process.instances).to eq(1) expect(process.memory).to eq(1024) expect(process.disk_quota).to eq(50) + expect(process.log_rate_limit).to eq(1_048_576) process_scale.scale expect(process.reload.instances).to eq(2) expect(process.reload.memory).to eq(100) expect(process.reload.disk_quota).to eq(200) + expect(process.reload.log_rate_limit).to eq(409_600) end it 'fails if the process is web' do @@ -123,12 +145,14 @@ module VCAP::CloudController expect(process.instances).to eq(1) expect(process.memory).to eq(1024) expect(process.disk_quota).to eq(50) + expect(process.log_rate_limit).to eq(1_048_576) expect { process_scale.scale }.to raise_error(ProcessScale::InvalidProcess, 'Cannot scale this process while a deployment is in flight.') expect(process.reload.instances).to eq(1) expect(process.reload.memory).to eq(1024) expect(process.reload.disk_quota).to eq(50) + expect(process.reload.log_rate_limit).to eq(1_048_576) end end end diff --git a/spec/unit/actions/space_diff_manifest_spec.rb b/spec/unit/actions/space_diff_manifest_spec.rb index 341e36004c9..b410175fa01 100644 --- a/spec/unit/actions/space_diff_manifest_spec.rb +++ b/spec/unit/actions/space_diff_manifest_spec.rb @@ -21,6 +21,7 @@ module VCAP::CloudController 'instances' => process1.instances, 'memory' => '1024M', 'disk_quota' => '1024M', + 'log_rate_limit_per_second' => '1M', 'health-check-type' => process1.health_check_type }, { @@ -28,6 +29,7 @@ module VCAP::CloudController 'instances' => process2.instances, 'memory' => '1024M', 'disk_quota' => '1024M', + 'log_rate_limit_per_second' => '1M', 'health-check-type' => process2.health_check_type } ] @@ -264,6 +266,7 @@ module VCAP::CloudController before do default_manifest['applications'][0]['processes'][0]['memory'] = '1G' default_manifest['applications'][0]['processes'][0]['disk_quota'] = '1G' + default_manifest['applications'][0]['processes'][0]['log_rate_limit_per_second'] = '1024K' end it 'returns an empty diff' do expect(subject).to eq([]) @@ -274,11 +277,13 @@ module VCAP::CloudController before do default_manifest['applications'][0]['processes'][0]['memory'] = '2G' default_manifest['applications'][0]['processes'][0]['disk_quota'] = '4G' + default_manifest['applications'][0]['processes'][0]['log_rate_limit_per_second'] = '2G' end - it 'returns the diff formatted as megabytes' do + it 'returns the diff formatted' do expect(subject).to eq([ { 'op' => 'replace', 'path' => '/applications/0/processes/0/memory', 'value' => '2048M', 'was' => '1024M' }, { 'op' => 'replace', 'path' => '/applications/0/processes/0/disk_quota', 'value' => '4096M', 'was' => '1024M' }, + { 'op' => 'replace', 'path' => '/applications/0/processes/0/log_rate_limit_per_second', 'value' => '2G', 'was' => '1M' }, ]) end end @@ -320,6 +325,7 @@ module VCAP::CloudController before do default_manifest['applications'][0]['memory'] = '1G' default_manifest['applications'][0]['disk_quota'] = '1G' + default_manifest['applications'][0]['log_rate_limit_per_second'] = '1024K' end it 'returns an empty diff if the field is equivalent' do diff --git a/spec/unit/actions/space_quota_update_spec.rb b/spec/unit/actions/space_quota_update_spec.rb index 86b7feae606..e98a366a899 100644 --- a/spec/unit/actions/space_quota_update_spec.rb +++ b/spec/unit/actions/space_quota_update_spec.rb @@ -23,7 +23,8 @@ module VCAP::CloudController total_memory_in_mb: 5120, per_process_memory_in_mb: 1024, total_instances: 8, - per_app_tasks: nil + per_app_tasks: nil, + log_rate_limit_in_bytes_per_second: 2000, }, services: { paid_services_allowed: false, @@ -50,6 +51,7 @@ module VCAP::CloudController expect(updated_space_quota.instance_memory_limit).to eq(1024) expect(updated_space_quota.app_instance_limit).to eq(8) expect(updated_space_quota.app_task_limit).to eq(-1) + expect(updated_space_quota.log_rate_limit).to eq(2000) expect(updated_space_quota.total_services).to eq(10) expect(updated_space_quota.total_service_keys).to eq(20) @@ -63,6 +65,7 @@ module VCAP::CloudController updated_space_quota = SpaceQuotaUpdate.update(space_quota, minimum_message) expect(updated_space_quota.name).to eq('space_quota_name') + expect(updated_space_quota.log_rate_limit).to eq(-1) end context 'when a model validation fails' do diff --git a/spec/unit/actions/space_quotas_create_spec.rb b/spec/unit/actions/space_quotas_create_spec.rb index aac7774bd0a..a40d9a45032 100644 --- a/spec/unit/actions/space_quotas_create_spec.rb +++ b/spec/unit/actions/space_quotas_create_spec.rb @@ -30,6 +30,7 @@ module VCAP::CloudController per_process_memory_in_mb: 6, total_instances: 7, per_app_tasks: 8, + log_rate_limit_in_bytes_per_second: 2000, }, services: { paid_services_allowed: false, @@ -68,6 +69,7 @@ module VCAP::CloudController expect(space_quota.instance_memory_limit).to eq(-1) expect(space_quota.app_instance_limit).to eq(-1) expect(space_quota.app_task_limit).to eq(-1) + expect(space_quota.log_rate_limit).to eq(-1) expect(space_quota.total_services).to eq(-1) expect(space_quota.total_service_keys).to eq(-1) @@ -90,6 +92,7 @@ module VCAP::CloudController expect(space_quota.instance_memory_limit).to eq(6) expect(space_quota.app_instance_limit).to eq(7) expect(space_quota.app_task_limit).to eq(8) + expect(space_quota.log_rate_limit).to eq(2000) expect(space_quota.total_services).to eq(9) expect(space_quota.total_service_keys).to eq(10) diff --git a/spec/unit/actions/task_create_spec.rb b/spec/unit/actions/task_create_spec.rb index 09119ba6245..d029f136acd 100644 --- a/spec/unit/actions/task_create_spec.rb +++ b/spec/unit/actions/task_create_spec.rb @@ -8,7 +8,8 @@ module VCAP::CloudController Config.new({ maximum_app_disk_in_mb: 4096, default_app_memory: 1024, - default_app_disk_in_mb: 1024 + default_app_disk_in_mb: 1024, + default_app_log_rate_limit_in_bytes_per_second: 999 }) end @@ -193,6 +194,28 @@ module VCAP::CloudController end end + describe 'log_rate_limit' do + context 'when log_rate_limit is specified' do + let(:message) { TaskCreateMessage.new name: name, command: command, log_rate_limit_in_bytes_per_second: 100 } + + it 'returns what is in the message' do + task = task_create_action.create(app, message, user_audit_info) + + expect(task.log_rate_limit).to eq(100) + end + end + + context 'when log_rate_limit is not specified' do + let(:message) { TaskCreateMessage.new name: name, command: command } + + it 'returns the default' do + task = task_create_action.create(app, message, user_audit_info) + + expect(task.log_rate_limit).to eq(999) + end + end + end + describe 'disk_in_mb' do before { config.set(:default_app_disk_in_mb, 200) } @@ -327,6 +350,32 @@ module VCAP::CloudController end end + describe 'log_rate_limit' do + before do + config.set(:default_app_log_rate_limit_in_bytes_per_second, 999) + end + + context 'when there is a template and the message does not specify log_rate_limit_in_bytes_per_second' do + let(:process) { VCAP::CloudController::ProcessModel.make(app: app, type: 'web', log_rate_limit: 23) } + let(:message) { TaskCreateMessage.new(name: name, command: 'ok', disk_in_mb: 2048, template: { process: { guid: process.guid } }) } + + it 'uses the memory from the template process' do + task = task_create_action.create(app, message, user_audit_info) + expect(task.log_rate_limit).to eq(23) + end + end + + context 'when there is a template and the message specifies log_rate_limit_in_bytes_per_second' do + let(:process) { VCAP::CloudController::ProcessModel.make(app: app, type: 'web', log_rate_limit: 23) } + let(:message) { TaskCreateMessage.new(name: name, command: 'ok', log_rate_limit_in_bytes_per_second: 2048, template: { process: { guid: process.guid } }) } + + it 'uses the memory from the message' do + task = task_create_action.create(app, message, user_audit_info) + expect(task.log_rate_limit).to eq(2048) + end + end + end + describe 'disk_in_mb' do before do config.set(:default_app_disk_in_mb, 4096) diff --git a/spec/unit/controllers/runtime/apps_controller_spec.rb b/spec/unit/controllers/runtime/apps_controller_spec.rb index 30ef40adf9f..7e86b70c17d 100644 --- a/spec/unit/controllers/runtime/apps_controller_spec.rb +++ b/spec/unit/controllers/runtime/apps_controller_spec.rb @@ -61,6 +61,7 @@ module VCAP::CloudController console: { type: 'bool', default: false }, debug: { type: 'string' }, disk_quota: { type: 'integer' }, + log_rate_limit: { type: 'integer' }, environment_json: { type: 'hash', default: {} }, health_check_http_endpoint: { type: 'string' }, health_check_timeout: { type: 'integer' }, @@ -88,6 +89,7 @@ module VCAP::CloudController console: { type: 'bool' }, debug: { type: 'string' }, disk_quota: { type: 'integer' }, + log_rate_limit: { type: 'integer' }, environment_json: { type: 'hash' }, health_check_http_endpoint: { type: 'string' }, health_check_timeout: { type: 'integer' }, diff --git a/spec/unit/lib/cloud_controller/app_manifest/byte_converter_spec.rb b/spec/unit/lib/cloud_controller/app_manifest/byte_converter_spec.rb index 8c9d8274f1a..bbc0fff0073 100644 --- a/spec/unit/lib/cloud_controller/app_manifest/byte_converter_spec.rb +++ b/spec/unit/lib/cloud_controller/app_manifest/byte_converter_spec.rb @@ -126,5 +126,77 @@ module VCAP::CloudController end end end + + describe '#convert_to_b' do + context 'when given 1M' do + let(:byte_value) { '1M' } + + it 'returns the value in bytes' do + expect(subject.convert_to_b(byte_value)).to eq(1_048_576) + end + end + end + + describe '#human_readable_byte_value' do + context 'when given nil' do + let(:byte_value) { nil } + + it 'returns nil' do + expect(subject.human_readable_byte_value(byte_value)).to be_nil + end + end + + context 'when given 1M in bytes' do + let(:byte_value) { 1_048_576 } + + it 'returns the human readable value' do + expect(subject.human_readable_byte_value(byte_value)).to eq('1M') + end + end + + context 'when given 1G in bytes' do + let(:byte_value) { 1_073_741_824 } + + it 'returns the human readable value' do + expect(subject.human_readable_byte_value(byte_value)).to eq('1G') + end + end + + context 'when given 1.1M in bytes' do + let(:byte_value) { 1_153_434 } + + it 'returns the human readable value in bytes to avoid losing precision' do + expect(subject.human_readable_byte_value(byte_value)).to eq('1153434B') + end + end + + context 'when given 1M + 1K' do + let(:byte_value) { 1049600 } + + it 'returns the human readable value in kilobytes to avoid losing precision' do + expect(subject.human_readable_byte_value(byte_value)).to eq('1025K') + end + end + + context 'when given a string' do + let(:byte_value) { 'not-a-number' } + + it 'raises an error' do + expect { + subject.human_readable_byte_value(byte_value) + }.to raise_error(ByteConverter::InvalidBytesError) + end + end + + context 'when given a float' do + let(:byte_value) { 1.1 } + + it 'raises an error' do + expect { + subject.human_readable_byte_value(byte_value) + }.to raise_error(ByteConverter::InvalidBytesError) + end + end + end end end diff --git a/spec/unit/lib/cloud_controller/app_services/process_log_rate_limit_calculator_spec.rb b/spec/unit/lib/cloud_controller/app_services/process_log_rate_limit_calculator_spec.rb new file mode 100644 index 00000000000..bfbd21af3f4 --- /dev/null +++ b/spec/unit/lib/cloud_controller/app_services/process_log_rate_limit_calculator_spec.rb @@ -0,0 +1,132 @@ +require 'spec_helper' + +module VCAP::CloudController + RSpec.describe ProcessLogRateLimitCalculator do + subject { ProcessLogRateLimitCalculator.new(process) } + let(:process_guid) { 'i-do-not-match-the-app-guid' } + let(:app_model) { AppModel.make } + let(:process) { ProcessModel.make(guid: process_guid, app: app_model) } + let(:stopped_state) { 'STOPPED' } + let(:started_state) { 'STARTED' } + + describe '#additional_log_rate_limit_requested' do + context 'when the app state is STOPPED' do + before do + process.state = stopped_state + process.save(validate: false) + end + + it 'returns 0' do + expect(subject.additional_log_rate_limit_requested).to eq(0) + end + end + + context 'when the app state is STARTED' do + let(:process) { ProcessModel.make(state: started_state, guid: process_guid, app: app_model) } + + context 'and the app is already in the db' do + it 'raises ApplicationMissing if the app no longer exists in the db' do + process.delete + expect { subject.additional_log_rate_limit_requested }.to raise_error(CloudController::Errors::ApplicationMissing) + end + + context 'and it is changing from STOPPED' do + before do + db_app = ProcessModel.find(guid: process.guid) + db_app.state = stopped_state + db_app.save(validate: false) + end + + it 'returns the total requested log_rate_limit' do + process.state = started_state + expect(subject.additional_log_rate_limit_requested).to eq(subject.total_requested_log_rate_limit) + end + end + + context 'and the app is already STARTED' do + before do + db_app = ProcessModel.find(guid: process.guid) + db_app.state = started_state + db_app.save(validate: false) + end + + it 'returns only newly requested log_rate_limit' do + expected = process.log_rate_limit + process.instances += 1 + + expect(subject.additional_log_rate_limit_requested).to eq(expected) + end + end + end + + context 'and the app is new' do + let(:process) { ProcessModel.new } + before do + process.instances = 1 + process.log_rate_limit = 100 + end + + it 'returns the total requested log_rate_limit' do + process.state = started_state + expect(subject.additional_log_rate_limit_requested).to eq(subject.total_requested_log_rate_limit) + end + end + end + + context 'and the app is requesting unlimited log quota' do + let(:process) { ProcessModel.new } + before do + process.instances = 1 + process.log_rate_limit = -1 + end + + it 'returns the fact that it is requesting unlimited quota' do + process.state = started_state + expect(subject.additional_log_rate_limit_requested).to eq(-1) + end + end + end + + describe '#total_requested_log_rate_limit' do + it 'returns requested log_rate_limit * requested instances' do + expected = process.log_rate_limit * process.instances + expect(subject.total_requested_log_rate_limit).to eq(expected) + end + end + + describe '#currently_used_log_rate_limit' do + context 'when the app is new' do + let(:process) { ProcessModel.new } + it 'returns 0' do + expect(subject.currently_used_log_rate_limit).to eq(0) + end + end + + it 'raises ApplicationMissing if the app no longer exists in the db' do + process.delete + expect { subject.currently_used_log_rate_limit }.to raise_error(CloudController::Errors::ApplicationMissing) + end + + context 'when the app in the db is STOPPED' do + it 'returns 0' do + expect(subject.currently_used_log_rate_limit).to eq(0) + end + end + + context 'when the app in the db is STARTED' do + before do + process.state = started_state + process.save(validate: false) + end + + it 'returns the log_rate_limit * instances of the db row' do + expected = process.instances * process.log_rate_limit + process.instances += 5 + process.log_rate_limit += 100 + + expect(subject.currently_used_log_rate_limit).to eq(expected) + end + end + end + end +end diff --git a/spec/unit/messages/app_manifest_message_spec.rb b/spec/unit/messages/app_manifest_message_spec.rb index caf0995f231..887f7cea778 100644 --- a/spec/unit/messages/app_manifest_message_spec.rb +++ b/spec/unit/messages/app_manifest_message_spec.rb @@ -104,6 +104,67 @@ module VCAP::CloudController end end + describe 'log_rate_limit_per_second' do + context 'when log_rate_limit_per_second unit is not part of expected set of values' do + let(:params_from_yaml) { { name: 'eugene', log_rate_limit_per_second: '200INVALID' } } + + it 'is not valid' do + message = AppManifestMessage.create_from_yml(params_from_yaml) + + expect(message).not_to be_valid + expect(message.errors).to have(1).items + expect(message.errors.full_messages).to include( + 'Process "web": Log rate limit per second must use a supported unit: B, K, KB, M, MB, G, GB, T, or TB' + ) + end + end + + context 'when log_rate_limit_per_second is less than -1 bytes' do + let(:params_from_yaml) { { name: 'eugene', log_rate_limit_per_second: '-1M' } } + + it 'is not valid' do + message = AppManifestMessage.create_from_yml(params_from_yaml) + + expect(message).not_to be_valid + expect(message.errors).to have(1).items + expect(message.errors.full_messages).to include('Process "web": Log rate limit must be an integer greater than or equal to -1B') + end + end + + context 'when log_rate_limit_per_second is not numeric' do + let(:params_from_yaml) { { name: 'eugene', log_rate_limit_per_second: 'gerg herscheisers' } } + + it 'is not valid' do + message = AppManifestMessage.create_from_yml(params_from_yaml) + + expect(message).not_to be_valid + expect(message.errors).to have(1).items + expect(message.errors.full_messages).to include( + 'Process "web": Log rate limit per second is not a number' + ) + end + end + + context 'when log_rate_limit_per_second is unlimited amount' do + let(:params_from_yaml) { { name: 'eugene', log_rate_limit_per_second: '-1B' } } + + it 'is valid' do + message = AppManifestMessage.create_from_yml(params_from_yaml) + expect(message).to be_valid + end + end + + context 'when log_rate_limit_per_second is in megabytes' do + let(:params_from_yaml) { { name: 'eugene', log_rate_limit_per_second: '1MB' } } + + it 'is valid' do + message = AppManifestMessage.create_from_yml(params_from_yaml) + expect(message).to be_valid + end + end + + end + describe 'buildpack' do context 'when providing a valid buildpack name' do let(:buildpack) { Buildpack.make } @@ -572,6 +633,7 @@ module VCAP::CloudController 'instances' => -30, 'memory' => 'potato', 'disk_quota' => '100', + 'log_rate_limit_per_second' => 'kumara', 'health_check_type' => 'sweet_potato', 'health_check_http_endpoint' => '/healthcheck_potato', 'health_check_invocation_timeout' => 'yucca', @@ -585,6 +647,7 @@ module VCAP::CloudController 'instances' => 'cassava', 'memory' => 'potato', 'disk_quota' => '100', + 'log_rate_limit_per_second' => '100', 'health_check_type' => 'sweet_potato', 'health_check_http_endpoint' => '/healthcheck_potato', 'health_check_invocation_timeout' => 'yucca', @@ -602,10 +665,11 @@ module VCAP::CloudController it 'includes the type of the process in the error message' do message = AppManifestMessage.create_from_yml(params_from_yaml) expect(message).to_not be_valid - expect(message.errors).to have(16).items + expect(message.errors).to have(18).items expect(message.errors.full_messages).to match_array([ 'Process "type1": Command must be between 1 and 4096 characters', 'Process "type1": Disk quota must use a supported unit: B, K, KB, M, MB, G, GB, T, or TB', + 'Process "type1": Log rate limit per second is not a number', 'Process "type1": Instances must be greater than or equal to 0', 'Process "type1": Memory is not a number', 'Process "type1": Timeout is not a number', @@ -614,6 +678,7 @@ module VCAP::CloudController 'Process "type1": Health check invocation timeout is not a number', 'Process "type2": Command must be between 1 and 4096 characters', 'Process "type2": Disk quota must use a supported unit: B, K, KB, M, MB, G, GB, T, or TB', + 'Process "type2": Log rate limit per second must use a supported unit: B, K, KB, M, MB, G, GB, T, or TB', 'Process "type2": Instances is not a number', 'Process "type2": Memory is not a number', 'Process "type2": Timeout is not a number', diff --git a/spec/unit/messages/manifest_process_scale_message_spec.rb b/spec/unit/messages/manifest_process_scale_message_spec.rb index 059924ba94c..5f31604935a 100644 --- a/spec/unit/messages/manifest_process_scale_message_spec.rb +++ b/spec/unit/messages/manifest_process_scale_message_spec.rb @@ -141,6 +141,44 @@ module VCAP::CloudController end end + describe '#log_rate_limit' do + context 'when log_rate_limit is not an number' do + let(:params) { { log_rate_limit: 'silly string thing' } } + + it 'is not valid' do + message = ManifestProcessScaleMessage.new(params) + + expect(message).not_to be_valid + expect(message.errors.count).to eq(1) + expect(message.errors.full_messages).to include('Log rate limit must be an integer greater than or equal to -1B') + end + end + + context 'when log_rate_limit is < -1' do + let(:params) { { log_rate_limit: -2 } } + + it 'is not valid' do + message = ManifestProcessScaleMessage.new(params) + + expect(message).not_to be_valid + expect(message.errors.count).to eq(1) + expect(message.errors.full_messages).to include('Log rate limit must be an integer greater than or equal to -1B') + end + end + + context 'when log_rate_limit is not an integer' do + let(:params) { { log_rate_limit: 3.5 } } + + it 'is not valid' do + message = ManifestProcessScaleMessage.new(params) + + expect(message).not_to be_valid + expect(message.errors.count).to eq(1) + expect(message.errors.full_messages).to include('Log rate limit must be an integer greater than or equal to -1B') + end + end + end + context 'when we have more than one error' do let(:params) { { disk_quota: 3.5, memory: 'smiling greg' } } @@ -161,7 +199,7 @@ module VCAP::CloudController let(:manifest_message) { ManifestProcessScaleMessage.new(params) } context 'when all params are given' do - let(:params) { { instances: 3, memory: 1024, disk_quota: 2048 } } + let(:params) { { instances: 3, memory: 1024, disk_quota: 2048, log_rate_limit: 1024 } } it 'returns a process_scale_message with the appropriate values' do scale_message = manifest_message.to_process_scale_message @@ -169,6 +207,7 @@ module VCAP::CloudController expect(scale_message.instances).to eq(3) expect(scale_message.memory_in_mb).to eq(1024) expect(scale_message.disk_in_mb).to eq(2048) + expect(scale_message.log_rate_limit_in_bytes_per_second).to eq(1024) end end @@ -184,6 +223,16 @@ module VCAP::CloudController end end + context 'when no log_rate_limit is given' do + let(:params) { { instances: 3, memory: 1024 } } + + it 'does not set anything for log_rate_limit_in_bytes_per_second' do + scale_message = manifest_message.to_process_scale_message + + expect(scale_message.log_rate_limit_in_bytes_per_second).to be_falsey + end + end + context 'when no instances is given' do let(:params) { { memory: 1024, disk_quota: 2048 } } diff --git a/spec/unit/messages/organization_quotas_create_message_spec.rb b/spec/unit/messages/organization_quotas_create_message_spec.rb index 38122e4c0af..5d183fe1c8a 100644 --- a/spec/unit/messages/organization_quotas_create_message_spec.rb +++ b/spec/unit/messages/organization_quotas_create_message_spec.rb @@ -240,6 +240,74 @@ module VCAP::CloudController end end + describe 'log_rate_limit_in_bytes_per_second' do + context 'when the type is a string' do + let(:params) { + { + name: 'my-name', + apps: { log_rate_limit_in_bytes_per_second: 'bob' }, + relationships: relationships, + } + } + + it 'is not valid' do + expect(subject).to be_invalid + expect(subject.errors[:apps]).to contain_exactly('Log rate limit in bytes per second is not a number') + end + end + context 'when the type is decimal' do + let(:params) { + { + name: 'my-name', + apps: { log_rate_limit_in_bytes_per_second: 1.1 }, + relationships: relationships, + } + } + + it 'is not valid' do + expect(subject).to be_invalid + expect(subject.errors[:apps]).to contain_exactly('Log rate limit in bytes per second must be an integer') + end + end + context 'when the type is a negative integer' do + let(:params) { + { + name: 'my-name', + apps: { log_rate_limit_in_bytes_per_second: -1 }, + relationships: relationships, + } + } + + it 'is not valid because "unlimited" is set with null, not -1, in V3' do + expect(subject).to be_invalid + expect(subject.errors[:apps]).to contain_exactly('Log rate limit in bytes per second must be greater than or equal to 0') + end + end + + context 'when the type is zero' do + let(:params) { + { + name: 'my-name', + apps: { log_rate_limit_in_bytes_per_second: 0 }, + relationships: relationships, + } + } + + it { is_expected.to be_valid } + end + context 'when the type is nil (unlimited)' do + let(:params) { + { + name: 'my-name', + apps: { log_rate_limit_in_bytes_per_second: nil }, + relationships: relationships, + } + } + + it { is_expected.to be_valid } + end + end + describe 'total_instances' do context 'when the type is a string' do let(:params) { diff --git a/spec/unit/messages/process_scale_message_spec.rb b/spec/unit/messages/process_scale_message_spec.rb index 8de05bbf7d4..0a45eeda210 100644 --- a/spec/unit/messages/process_scale_message_spec.rb +++ b/spec/unit/messages/process_scale_message_spec.rb @@ -158,5 +158,53 @@ module VCAP::CloudController expect(message.errors[:disk_in_mb]).to include('must be an integer') end end + + context 'when log_rate_limit_in_bytes_per_second is not an number' do + let(:params) { { log_rate_limit_in_bytes_per_second: 'silly string thing' } } + + it 'is not valid' do + message = ProcessScaleMessage.new(params) + + expect(message).not_to be_valid + expect(message.errors.count).to eq(1) + expect(message.errors[:log_rate_limit_in_bytes_per_second]).to include('is not a number') + end + end + + context 'when log_rate_limit_in_bytes_per_second is < -1' do + let(:params) { { log_rate_limit_in_bytes_per_second: -2 } } + + it 'is not valid' do + message = ProcessScaleMessage.new(params) + + expect(message).not_to be_valid + expect(message.errors.count).to eq(1) + expect(message.errors[:log_rate_limit_in_bytes_per_second]).to include('must be greater than or equal to -1') + end + end + + context 'when log_rate_limit_in_bytes_per_second is > the max value allowed in the database' do + let(:params) { { log_rate_limit_in_bytes_per_second: BaseMessage::MAX_DB_BIGINT + 1 } } + + it 'is not valid' do + message = ProcessScaleMessage.new(params) + + expect(message).not_to be_valid + expect(message.errors.count).to eq(1) + expect(message.errors[:log_rate_limit_in_bytes_per_second]).to include('must be less than or equal to 9223372036854775807') + end + end + + context 'when log_rate_limit_in_bytes_per_second is not an integer' do + let(:params) { { log_rate_limit_in_bytes_per_second: 3.5 } } + + it 'is not valid' do + message = ProcessScaleMessage.new(params) + + expect(message).not_to be_valid + expect(message.errors.count).to eq(1) + expect(message.errors[:log_rate_limit_in_bytes_per_second]).to include('must be an integer') + end + end end end diff --git a/spec/unit/messages/quotas_apps_message_spec.rb b/spec/unit/messages/quotas_apps_message_spec.rb index 77e215314c5..6b3caff77a3 100644 --- a/spec/unit/messages/quotas_apps_message_spec.rb +++ b/spec/unit/messages/quotas_apps_message_spec.rb @@ -256,6 +256,68 @@ module VCAP::CloudController end end end + + describe 'log_rate_limit_in_bytes_per_second' do + context 'when the type is a string' do + let(:params) do + { log_rate_limit_in_bytes_per_second: 'bob' } + end + + it 'is not valid' do + expect(subject).to be_invalid + expect(subject.errors).to contain_exactly('Log rate limit in bytes per second is not a number') + end + end + + context 'when the type is decimal' do + let(:params) do + { log_rate_limit_in_bytes_per_second: 1.1 } + end + + it 'is not valid' do + expect(subject).to be_invalid + expect(subject.errors).to contain_exactly('Log rate limit in bytes per second must be an integer') + end + end + + context 'when the type is a negative integer' do + let(:params) do + { log_rate_limit_in_bytes_per_second: -1 } + end + + it 'is not valid because "unlimited" is set with null, not -1, in V3' do + expect(subject).to be_invalid + expect(subject.errors).to contain_exactly('Log rate limit in bytes per second must be greater than or equal to 0') + end + end + + context 'when the type is zero' do + let(:params) do + { log_rate_limit_in_bytes_per_second: 0 } + end + + it { is_expected.to be_valid } + end + + context 'when the type is nil (unlimited)' do + let(:params) do + { log_rate_limit_in_bytes_per_second: nil } + end + + it { is_expected.to be_valid } + end + + context 'when the value is greater than the maximum allowed value in the DB' do + let(:params) do + { log_rate_limit_in_bytes_per_second: 2**64 } + end + + it 'is not valid' do + expect(subject).to be_invalid + expect(subject.errors).to contain_exactly('Log rate limit in bytes per second must be less than or equal to 9223372036854775807') + end + end + end end end end diff --git a/spec/unit/messages/space_quota_update_message_spec.rb b/spec/unit/messages/space_quota_update_message_spec.rb index bd0cba290cf..d8b0adaccd6 100644 --- a/spec/unit/messages/space_quota_update_message_spec.rb +++ b/spec/unit/messages/space_quota_update_message_spec.rb @@ -20,6 +20,7 @@ module VCAP::CloudController per_process_memory_in_mb: 1024, total_instances: 2, per_app_tasks: 4, + log_rate_limit_in_bytes_per_second: 2000, } end @@ -49,6 +50,7 @@ module VCAP::CloudController expect(subject.per_process_memory_in_mb).to eq(1024) expect(subject.total_instances).to eq(2) expect(subject.per_app_tasks).to eq(4) + expect(subject.log_rate_limit_in_bytes_per_second).to eq(2000) expect(subject.paid_services_allowed).to be_truthy expect(subject.total_service_instances).to eq(17) expect(subject.total_service_keys).to eq(19) diff --git a/spec/unit/messages/task_create_message_spec.rb b/spec/unit/messages/task_create_message_spec.rb index 45635e04e9b..fac1a3d449e 100644 --- a/spec/unit/messages/task_create_message_spec.rb +++ b/spec/unit/messages/task_create_message_spec.rb @@ -141,6 +141,59 @@ module VCAP::CloudController end end + describe 'log_rate_limit_in_bytes_per_second' do + it 'can be nil' do + body.delete 'log_rate_limit_in_bytes_per_second' + + message = TaskCreateMessage.new(body) + + expect(message).to be_valid + end + + it 'must be numerical' do + body['log_rate_limit_in_bytes_per_second'] = 'trout' + + message = TaskCreateMessage.new(body) + + expect(message).to_not be_valid + expect(message.errors.full_messages).to include('Log rate limit in bytes per second is not a number') + end + + it 'may not have a floating point' do + body['log_rate_limit_in_bytes_per_second'] = 4.5 + + message = TaskCreateMessage.new(body) + + expect(message).to_not be_valid + expect(message.errors.full_messages).to include('Log rate limit in bytes per second must be an integer') + end + + it 'may be -1' do + body['log_rate_limit_in_bytes_per_second'] = -1 + + message = TaskCreateMessage.new(body) + + expect(message).to be_valid + end + + it 'may be zero' do + body['log_rate_limit_in_bytes_per_second'] = 0 + + message = TaskCreateMessage.new(body) + + expect(message).to be_valid + end + + it 'may not be smaller than -1' do + body['log_rate_limit_in_bytes_per_second'] = -2 + + message = TaskCreateMessage.new(body) + + expect(message).to_not be_valid + expect(message.errors.full_messages).to include('Log rate limit in bytes per second must be greater than -2') + end + end + describe 'template' do it 'can be nil' do body.delete 'template' diff --git a/spec/unit/models/runtime/constraints/max_log_rate_limit_policy_spec.rb b/spec/unit/models/runtime/constraints/max_log_rate_limit_policy_spec.rb new file mode 100644 index 00000000000..a9c3c075e9e --- /dev/null +++ b/spec/unit/models/runtime/constraints/max_log_rate_limit_policy_spec.rb @@ -0,0 +1,162 @@ +require 'spec_helper' + +RSpec.describe 'max log_rate_limit policies' do + let(:org_or_space) { double(:org_or_space, has_remaining_log_rate_limit: false) } + let(:error_name) { :random_log_rate_limit_error } + + describe AppMaxLogRateLimitPolicy do + subject(:validator) { AppMaxLogRateLimitPolicy.new(process, org_or_space, error_name) } + + context 'when the app specifies a log quota' do + let(:process) { VCAP::CloudController::ProcessModelFactory.make(log_rate_limit: 100, state: 'STOPPED') } + + context 'when the app is being started' do + before do + process.state = 'STARTED' + process.log_rate_limit = 150 + end + + it 'registers an error when quota is exceeded' do + expect(org_or_space).to receive(:has_remaining_log_rate_limit).with(150).and_return(false) + expect(validator).to validate_with_error(process, :log_rate_limit, error_name) + end + + it 'does not register an error when quota is not exceeded' do + expect(org_or_space).to receive(:has_remaining_log_rate_limit).with(150).and_return(true) + expect(validator).to validate_without_error(process) + end + + it 'adds the given error to the model' do + expect(org_or_space).to receive(:has_remaining_log_rate_limit).with(150).and_return(false) + validator.validate + expect(process.errors.on(:log_rate_limit)).to include(error_name) + end + end + + context 'when the app is already started' do + let(:process) { VCAP::CloudController::ProcessModelFactory.make(log_rate_limit: 100, state: 'STARTED') } + + it 'does not register an error when quota is exceeded' do + expect(validator).to validate_without_error(process) + end + + context 'when the instance count has changed and would exceed the quota' do + before do + process.instances = 5 + end + it 'registers an error' do + expect(org_or_space).to receive(:has_remaining_log_rate_limit).with(400).and_return(false) + expect(validator).to validate_with_error(process, :log_rate_limit, error_name) + end + end + end + + context 'when the app is being stopped' do + before do + process.state = 'STOPPED' + end + + it 'does not register an error' do + expect(validator).to validate_without_error(process) + end + + context 'when the instance count has changed and would exceed the quota' do + before do + process.instances = 5 + end + it 'does not register an error' do + expect(validator).to validate_without_error(process) + end + end + end + end + + context 'when the app does not specify a log quota' do + let(:process) { VCAP::CloudController::ProcessModelFactory.make(log_rate_limit: -1, state: 'STOPPED') } + before do + process.state = 'STARTED' + end + + context 'when the org specifies a log quota' do + before do + allow(org_or_space).to receive(:name).and_return('some-org') + allow(org_or_space).to receive(:log_rate_limit).and_return(5000) + end + + it 'is unhappy and adds an error' do + validator.validate + expect(process.errors.on(:log_rate_limit)).to include("cannot be unlimited in organization 'some-org'.") + end + end + + context 'when the space specifies a log quota' do + before do + allow(org_or_space).to receive(:name).and_return('some-space') + allow(org_or_space).to receive(:log_rate_limit).and_return(5000) + allow(org_or_space).to receive(:organization_guid).and_return('some-org-guid') + end + + it 'is unhappy and adds an error' do + validator.validate + expect(process.errors.on(:log_rate_limit)).to include("cannot be unlimited in space 'some-space'.") + end + end + + context 'when both the space and the org specify an infinite log quota' do + before do + expect(org_or_space).to receive(:log_rate_limit).and_return(VCAP::CloudController::QuotaDefinition::UNLIMITED) + expect(org_or_space).to receive(:has_remaining_log_rate_limit).and_return(true) + end + + it 'is happy' do + expect(validator).to validate_without_error(process) + end + end + end + end + + describe TaskMaxLogRateLimitPolicy do + subject(:validator) { TaskMaxLogRateLimitPolicy.new(task, org_or_space, error_name) } + + let(:task) { VCAP::CloudController::TaskModel.make log_rate_limit: 150, state: VCAP::CloudController::TaskModel::RUNNING_STATE } + + context 'when not cancelling a task' do + it 'registers error when quota is exceeded' do + allow(org_or_space).to receive(:has_remaining_log_rate_limit).with(150).and_return(false) + expect(validator).to validate_with_error(task, :log_rate_limit, error_name) + end + + it 'does not register error when quota is not exceeded' do + allow(org_or_space).to receive(:has_remaining_log_rate_limit).with(150).and_return(true) + expect(validator).to validate_without_error(task) + end + + it 'adds the given error to the model' do + allow(org_or_space).to receive(:has_remaining_log_rate_limit).with(150).and_return(false) + validator.validate + expect(task.errors.on(:log_rate_limit)).to include(error_name) + end + end + + context 'when canceling a task' do + it 'does not register an error' do + task.state = VCAP::CloudController::TaskModel::CANCELING_STATE + expect(validator).to validate_without_error(task) + end + end + + context 'when the task is SUCCEEDED' do + it 'does not register error' do + task.state = VCAP::CloudController::TaskModel::SUCCEEDED_STATE + expect(validator).to validate_without_error(task) + end + end + + context 'when the task is FAILED' do + it 'does not register error' do + task.state = VCAP::CloudController::TaskModel::FAILED_STATE + expect(validator).to validate_without_error(task) + end + end + end +end diff --git a/spec/unit/models/runtime/constraints/min_log_rate_limit_policy_spec.rb b/spec/unit/models/runtime/constraints/min_log_rate_limit_policy_spec.rb new file mode 100644 index 00000000000..c106da7283d --- /dev/null +++ b/spec/unit/models/runtime/constraints/min_log_rate_limit_policy_spec.rb @@ -0,0 +1,27 @@ +require 'spec_helper' + +RSpec.describe MinLogRateLimitPolicy do + let(:process) { VCAP::CloudController::ProcessModelFactory.make } + + subject(:validator) { MinLogRateLimitPolicy.new(process) } + + it 'when requested size is negative 2' do + allow(process).to receive(:log_rate_limit).and_return(-2) + expect(validator).to validate_with_error(process, :log_rate_limit, MinLogRateLimitPolicy::ERROR_MSG) + end + + it 'when requested size is negative 1' do + allow(process).to receive(:log_rate_limit).and_return(-1) + expect(validator).to validate_without_error(process) + end + + it 'when requested size is zero' do + allow(process).to receive(:log_rate_limit).and_return(0) + expect(validator).to validate_without_error(process) + end + + it 'when requested size is positive' do + allow(process).to receive(:log_rate_limit).and_return(1) + expect(validator).to validate_without_error(process) + end +end diff --git a/spec/unit/models/runtime/organization_spec.rb b/spec/unit/models/runtime/organization_spec.rb index 8ff2b5b450a..84ef6da4a84 100644 --- a/spec/unit/models/runtime/organization_spec.rb +++ b/spec/unit/models/runtime/organization_spec.rb @@ -508,6 +508,59 @@ module VCAP::CloudController end end + describe '#has_remaining_log_rate_limit' do + let(:log_rate_limit) { 10 } + let(:quota) { QuotaDefinition.make(log_rate_limit: log_rate_limit) } + let(:org) { Organization.make(quota_definition: quota) } + let(:org2) { Organization.make(quota_definition: quota) } + let(:space) { Space.make(organization: org) } + let(:space2) { Space.make(organization: org) } + let(:space_org2) { Space.make(organization: org2) } + let!(:app_model) { AppModel.make(space: space2) } + + context 'when the quota is unlimited' do + let(:log_rate_limit) { QuotaDefinition::UNLIMITED } + + it 'handles large log quotas' do + expect(org.has_remaining_log_rate_limit(10_000_000)).to be_truthy + end + end + + context 'when nothing is running' do + it 'uses the log_rate_limit' do + expect(org.has_remaining_log_rate_limit(10)).to be_truthy + expect(org.has_remaining_log_rate_limit(11)).to be_falsey + end + end + + context 'when something else is running' do + it 'takes all things in the org into account' do + ProcessModelFactory.make(space: space, log_rate_limit: 5, state: 'STARTED') + expect(org.has_remaining_log_rate_limit(5)).to be_truthy + expect(org.has_remaining_log_rate_limit(6)).to be_falsey + + ProcessModelFactory.make(space: space, log_rate_limit: 1, state: 'STARTED') + expect(org.has_remaining_log_rate_limit(4)).to be_truthy + expect(org.has_remaining_log_rate_limit(5)).to be_falsey + + TaskModel.make(app: app_model, log_rate_limit: 1, state: TaskModel::RUNNING_STATE) + expect(org.has_remaining_log_rate_limit(3)).to be_truthy + expect(org.has_remaining_log_rate_limit(4)).to be_falsey + end + + context 'when something else is running in another org' do + it 'only accounts for things running in the owning org' do + ProcessModelFactory.make(space: space_org2, log_rate_limit: 1, instances: 2, state: 'STARTED') + expect(org.has_remaining_log_rate_limit(10)).to be_truthy + expect(org.has_remaining_log_rate_limit(11)).to be_falsey + + expect(org2.has_remaining_log_rate_limit(8)).to be_truthy + expect(org2.has_remaining_log_rate_limit(9)).to be_falsey + end + end + end + end + describe '#instance_memory_limit' do let(:quota) { QuotaDefinition.make(instance_memory_limit: 50) } let(:org) { Organization.make quota_definition: quota } diff --git a/spec/unit/models/runtime/process_model_spec.rb b/spec/unit/models/runtime/process_model_spec.rb index 7049f84707c..e571187cd86 100644 --- a/spec/unit/models/runtime/process_model_spec.rb +++ b/spec/unit/models/runtime/process_model_spec.rb @@ -74,6 +74,11 @@ def expect_no_validator(validator_class) expect(process.ports).to eq([8081, 8082]) end end + + it 'has a default log_rate_limit' do + TestConfig.override(default_app_log_rate_limit_in_bytes_per_second: 873565) + expect(process.log_rate_limit).to eq(873565) + end end describe 'Associations' do @@ -193,6 +198,7 @@ def expect_no_validator(validator_class) expect_validator(InstancesPolicy) expect_validator(MaxDiskQuotaPolicy) expect_validator(MinDiskQuotaPolicy) + expect_validator(MinLogRateLimitPolicy) expect_validator(MinMemoryPolicy) expect_validator(AppMaxInstanceMemoryPolicy) expect_validator(InstancesPolicy) @@ -289,6 +295,15 @@ def expect_no_validator(validator_class) end end + describe 'log_rate_limit' do + subject(:process) { ProcessModelFactory.make } + + it 'does not allow a log_rate_limit below the minimum' do + process.log_rate_limit = -2 + expect(process).to_not be_valid + end + end + describe 'health_check_http_endpoint' do subject(:process) { ProcessModelFactory.make } @@ -402,11 +417,12 @@ def expect_no_validator(validator_class) describe 'quota' do subject(:process) { ProcessModelFactory.make } + let(:log_rate_limit) { 1024 } let(:quota) do - QuotaDefinition.make(memory_limit: 128) + QuotaDefinition.make(memory_limit: 128, log_rate_limit: log_rate_limit) end let(:space_quota) do - SpaceQuotaDefinition.make(memory_limit: 128, organization: org) + SpaceQuotaDefinition.make(memory_limit: 128, organization: org, log_rate_limit: log_rate_limit) end context 'app update' do @@ -420,15 +436,52 @@ def act_as_cf_admin let(:org) { Organization.make(quota_definition: quota) } let(:space) { Space.make(name: 'hi', organization: org, space_quota_definition: space_quota) } let(:parent_app) { AppModel.make(space: space) } - subject!(:process) { ProcessModelFactory.make(app: parent_app, memory: 64, instances: 2, state: 'STARTED') } + subject!(:process) { ProcessModelFactory.make(app: parent_app, memory: 64, log_rate_limit: 512, instances: 2, state: 'STOPPED') } it 'should raise error when quota is exceeded' do process.memory = 65 - expect { process.save }.to raise_error(/quota_exceeded/) + process.state = 'STARTED' + expect { process.save }.to raise_error(/memory quota_exceeded/) + end + + it 'should raise error when log quota is exceeded' do + number = (log_rate_limit / 2) + 1 + process.log_rate_limit = number + process.state = 'STARTED' + expect { process.save }.to raise_error(/exceeds space log rate quota/) + end + + context 'when only exceeding the org quota' do + before do + org.quota_definition = QuotaDefinition.make(log_rate_limit: 5) + org.save + end + + it 'raises an error' do + process.log_rate_limit = 10 + process.state = 'STARTED' + expect { process.save }.to raise_error(/exceeds organization log rate quota/) + end + end + + it 'should not raise error when log quota is not exceeded' do + number = (log_rate_limit / 2) + process.log_rate_limit = number + process.state = 'STARTED' + expect { process.save }.not_to raise_error + end + + it 'should raise an error when starting an app with unlimited log rate and a limited quota' do + process.log_rate_limit = -1 + process.state = 'STARTED' + expect { process.save }.to raise_error(Sequel::ValidationFailed) + expect(process.errors.on(:log_rate_limit)).to include("cannot be unlimited in organization '#{org.name}'.") + expect(process.errors.on(:log_rate_limit)).to include("cannot be unlimited in space '#{space.name}'.") end it 'should not raise error when quota is not exceeded' do process.memory = 63 + process.state = 'STARTED' expect { process.save }.to_not raise_error end @@ -436,12 +489,15 @@ def act_as_cf_admin quota.memory_limit = 32 quota.save process.memory = 100 + process.state = 'STARTED' process.save(validate: false) expect(process.reload).to_not be_valid expect { process.delete }.not_to raise_error end it 'allows scaling down instances of an app from above quota to below quota' do + process.update(state: 'STARTED') + org.quota_definition = QuotaDefinition.make(memory_limit: 72) act_as_cf_admin { org.save } @@ -460,6 +516,7 @@ def act_as_cf_admin quota.save process.instances = 5 + process.state = 'STARTED' expect { process.save }.to raise_error(/instance_limit_exceeded/) end @@ -471,10 +528,13 @@ def act_as_cf_admin quota.save process.instances = 5 + process.state = 'STARTED' expect { process.save }.to raise_error(/instance_limit_exceeded/) end it 'raises when scaling down number of instances but remaining above quota' do + process.update(state: 'STARTED') + org.quota_definition = QuotaDefinition.make(memory_limit: 32) act_as_cf_admin { org.save } @@ -487,6 +547,7 @@ def act_as_cf_admin end it 'allows stopping an app that is above quota' do + process.update(state: 'STARTED') org.quota_definition = QuotaDefinition.make(memory_limit: 72) act_as_cf_admin { org.save } @@ -503,6 +564,7 @@ def act_as_cf_admin act_as_cf_admin { org.save } process.memory = 40 + process.state = 'STARTED' expect { process.save }.to raise_error(Sequel::ValidationFailed, /quota_exceeded/) process.memory = 32 @@ -565,6 +627,7 @@ def act_as_cf_admin :health_check_timeout, :health_check_type, :instances, + :log_rate_limit, :memory, :name, :package_state, @@ -598,6 +661,7 @@ def act_as_cf_admin :health_check_timeout, :health_check_type, :instances, + :log_rate_limit, :memory, :name, :production, @@ -1333,6 +1397,22 @@ def act_as_cf_admin end end + describe 'default log_rate_limit' do + before do + TestConfig.override(default_app_log_rate_limit_in_bytes_per_second: 1024) + end + + it 'should use the provided quota' do + process = ProcessModel.make(log_rate_limit: 256) + expect(process.log_rate_limit).to eq(256) + end + + it 'should use the default quota' do + process = ProcessModel.make + expect(process.log_rate_limit).to eq(1024) + end + end + describe 'instance_file_descriptor_limit' do before do TestConfig.override(instance_file_descriptor_limit: 200) diff --git a/spec/unit/models/runtime/quota_definition_spec.rb b/spec/unit/models/runtime/quota_definition_spec.rb index a886605b92d..7c690a4d27d 100644 --- a/spec/unit/models/runtime/quota_definition_spec.rb +++ b/spec/unit/models/runtime/quota_definition_spec.rb @@ -110,6 +110,15 @@ module VCAP::CloudController expect(quota_definition).to be_valid end + it 'log_rate_limit cannot be less than -1 (unlimited)' do + quota_definition.log_rate_limit = -2 + expect(quota_definition).not_to be_valid + expect(quota_definition.errors.on(:log_rate_limit)).to include(:invalid_log_rate_limit) + + quota_definition.log_rate_limit = -1 + expect(quota_definition).to be_valid + end + it 'app_task_limit cannot be less than -1 ("unlimited")' do quota_definition.app_task_limit = -2 expect(quota_definition).not_to be_valid @@ -133,12 +142,14 @@ module VCAP::CloudController it { is_expected.to export_attributes :name, :non_basic_services_allowed, :total_services, :total_routes, :total_private_domains, :memory_limit, :trial_db_allowed, :instance_memory_limit, - :app_instance_limit, :app_task_limit, :total_service_keys, :total_reserved_route_ports + :app_instance_limit, :app_task_limit, :total_service_keys, :total_reserved_route_ports, + :log_rate_limit } it { is_expected.to import_attributes :name, :non_basic_services_allowed, :total_services, :total_routes, :total_private_domains, :memory_limit, :trial_db_allowed, :instance_memory_limit, - :app_instance_limit, :app_task_limit, :total_service_keys, :total_reserved_route_ports + :app_instance_limit, :app_task_limit, :total_service_keys, :total_reserved_route_ports, + :log_rate_limit } end diff --git a/spec/unit/models/runtime/space_quota_definition_spec.rb b/spec/unit/models/runtime/space_quota_definition_spec.rb index c5438e62fe2..73d986b5632 100644 --- a/spec/unit/models/runtime/space_quota_definition_spec.rb +++ b/spec/unit/models/runtime/space_quota_definition_spec.rb @@ -68,6 +68,17 @@ module VCAP::CloudController end end + describe 'log_rate_limit' do + it 'cannot be less than -1' do + space_quota_definition.log_rate_limit = -2 + expect(space_quota_definition).not_to be_valid + expect(space_quota_definition.errors.on(:log_rate_limit)).to include(:invalid_log_rate_limit) + + space_quota_definition.log_rate_limit = -1 + expect(space_quota_definition).to be_valid + end + end + describe 'total_reserved_route_ports' do let(:err_msg) do 'Total reserved ports must be -1, 0, or a positive integer, must ' \ @@ -163,13 +174,13 @@ module VCAP::CloudController it do is_expected.to export_attributes :name, :organization_guid, :non_basic_services_allowed, :total_services, :total_routes, :memory_limit, :instance_memory_limit, :app_instance_limit, :app_task_limit, - :total_service_keys, :total_reserved_route_ports + :total_service_keys, :total_reserved_route_ports, :log_rate_limit end it do is_expected.to import_attributes :name, :organization_guid, :non_basic_services_allowed, :total_services, :total_routes, :memory_limit, :instance_memory_limit, :app_instance_limit, :app_task_limit, - :total_service_keys, :total_reserved_route_ports + :total_service_keys, :total_reserved_route_ports, :log_rate_limit end end diff --git a/spec/unit/models/runtime/space_spec.rb b/spec/unit/models/runtime/space_spec.rb index 2a0812dd497..ebb4f77f504 100644 --- a/spec/unit/models/runtime/space_spec.rb +++ b/spec/unit/models/runtime/space_spec.rb @@ -665,6 +665,57 @@ module VCAP::CloudController end end + describe '#has_remaining_log_rate_limit' do + let(:log_rate_limit) { 10 } + let(:quota) { SpaceQuotaDefinition.make(log_rate_limit: log_rate_limit, organization: org) } + let(:org) { Organization.make } + let(:space) { Space.make(organization: org, space_quota_definition: quota) } + let(:space2) { Space.make(organization: org, space_quota_definition: quota) } + let!(:app_model) { AppModel.make(space: space) } + + context 'when the quota is unlimited' do + let(:log_rate_limit) { QuotaDefinition::UNLIMITED } + + it 'handles large log quotas' do + expect(space.has_remaining_log_rate_limit(10_000_000)).to be_truthy + end + end + + context 'when nothing is running' do + it 'uses the log_rate_limit' do + expect(space.has_remaining_log_rate_limit(10)).to be_truthy + expect(space.has_remaining_log_rate_limit(11)).to be_falsey + end + end + + context 'when something else is running' do + it 'takes all things in the space into account' do + ProcessModelFactory.make(space: space, log_rate_limit: 5, state: 'STARTED') + expect(space.has_remaining_log_rate_limit(5)).to be_truthy + expect(space.has_remaining_log_rate_limit(6)).to be_falsey + + ProcessModelFactory.make(space: space, log_rate_limit: 1, state: 'STARTED') + expect(space.has_remaining_log_rate_limit(4)).to be_truthy + expect(space.has_remaining_log_rate_limit(5)).to be_falsey + + TaskModel.make(app: app_model, log_rate_limit: 1, state: TaskModel::RUNNING_STATE) + expect(space.has_remaining_log_rate_limit(3)).to be_truthy + expect(space.has_remaining_log_rate_limit(4)).to be_falsey + end + + context 'when processes are running in another space' do + it 'only accounts for processes running in the owning space' do + ProcessModelFactory.make(space: space2, log_rate_limit: 1, instances: 2, state: 'STARTED') + + expect(space.has_remaining_log_rate_limit(10)).to be_truthy + expect(space.has_remaining_log_rate_limit(11)).to be_falsey + expect(space2.has_remaining_log_rate_limit(8)).to be_truthy + expect(space2.has_remaining_log_rate_limit(9)).to be_falsey + end + end + end + end + describe '#instance_memory_limit' do let(:org) { Organization.make } let(:space_quota) { SpaceQuotaDefinition.make(instance_memory_limit: 50, organization: org) } diff --git a/spec/unit/models/runtime/task_model_spec.rb b/spec/unit/models/runtime/task_model_spec.rb index c9482fda8bd..deb8af74bfa 100644 --- a/spec/unit/models/runtime/task_model_spec.rb +++ b/spec/unit/models/runtime/task_model_spec.rb @@ -289,6 +289,15 @@ module VCAP::CloudController end describe 'quotas' do + it 'errors when log_rate_limit is below -1' do + expect { + TaskModel.make( + log_rate_limit: -2, + app: app, + ) + }.to raise_error + end + describe 'space quotas' do let(:space) { Space.make organization: org, space_quota_definition: quota } @@ -305,6 +314,59 @@ module VCAP::CloudController end end + describe 'when the log rate limit quota is unlimited' do + let(:quota) { SpaceQuotaDefinition.make(log_rate_limit: -1, organization: org) } + + it 'allows tasks to run with unlimited rate limits' do + expect { + TaskModel.make( + log_rate_limit: -1, + app: app, + ) + }.not_to raise_error + end + + it 'allows tasks to run with rate limits' do + expect { + TaskModel.make( + log_rate_limit: 1_000_000_000_000, + app: app, + ) + }.not_to raise_error + end + end + + describe 'when the quota has a log_rate_limit' do + let(:quota) { SpaceQuotaDefinition.make(log_rate_limit: 200, organization: org) } + + it 'allows tasks that fit in the available log rate' do + expect { + TaskModel.make( + log_rate_limit: 100, + app: app, + ) + }.not_to raise_error + end + + it 'raises an error if the task does not fit in the remaining space' do + expect { + TaskModel.make( + log_rate_limit: 201, + app: app, + ) + }.to raise_error Sequel::ValidationFailed, 'log_rate_limit exceeds space log rate quota' + end + + it 'raises an error if the task has an unlimited rate limit' do + expect { + TaskModel.make( + log_rate_limit: -1, + app: app, + ) + }.to raise_error Sequel::ValidationFailed, "log_rate_limit cannot be unlimited in space '#{space.name}'." + end + end + describe 'when the quota has a memory_limit' do let(:quota) { SpaceQuotaDefinition.make(memory_limit: 20, organization: org) } @@ -407,12 +469,35 @@ module VCAP::CloudController expect { TaskModel.make( memory_in_mb: 21, + log_rate_limit: 21_000, app: app, ) }.not_to raise_error end end + describe 'when the quota has a log_rate_limit' do + let(:quota) { QuotaDefinition.make(log_rate_limit: 200) } + + it 'does allow a task that fits in the limit to start' do + expect { + TaskModel.make( + log_rate_limit: 199, + app: app, + ) + }.to_not raise_error + end + + it 'does not allow a task that exceeds the limit to start' do + expect { + TaskModel.make( + log_rate_limit: 10_000, + app: app, + ) + }.to raise_error /log_rate_limit exceeds organization log rate/ + end + end + describe 'when the quota has a memory_limit' do let(:quota) { QuotaDefinition.make(memory_limit: 20) } diff --git a/spec/unit/presenters/v2/process_model_presenter_spec.rb b/spec/unit/presenters/v2/process_model_presenter_spec.rb index d1c03c13256..9ff7ebcbb8b 100644 --- a/spec/unit/presenters/v2/process_model_presenter_spec.rb +++ b/spec/unit/presenters/v2/process_model_presenter_spec.rb @@ -69,6 +69,7 @@ module CloudController::Presenters::V2 'memory' => 1024, 'instances' => 1, 'disk_quota' => 1024, + 'log_rate_limit' => 1_048_576, 'state' => 'STOPPED', 'version' => process.version, 'command' => 'start', diff --git a/spec/unit/presenters/v3/app_manifest_presenter_spec.rb b/spec/unit/presenters/v3/app_manifest_presenter_spec.rb index e67b1d71e00..8d76975cc81 100644 --- a/spec/unit/presenters/v3/app_manifest_presenter_spec.rb +++ b/spec/unit/presenters/v3/app_manifest_presenter_spec.rb @@ -78,6 +78,7 @@ module VCAP::CloudController::Presenters::V3 health_check_type: 'http', health_check_http_endpoint: '/foobar', health_check_timeout: 5, + log_rate_limit: 1_048_576, command: 'Do it now!', type: 'aaaaa', ) @@ -119,6 +120,7 @@ module VCAP::CloudController::Presenters::V3 'type' => process1.type, 'instances' => process1.instances, 'memory' => "#{process1.memory}M", + 'log_rate_limit_per_second' => '1M', 'disk_quota' => "#{process1.disk_quota}M", 'command' => process1.command, 'health-check-type' => process1.health_check_type, @@ -128,6 +130,7 @@ module VCAP::CloudController::Presenters::V3 { 'type' => process2.type, 'instances' => process2.instances, + 'log_rate_limit_per_second' => '1M', 'memory' => "#{process2.memory}M", 'disk_quota' => "#{process2.disk_quota}M", 'health-check-type' => process2.health_check_type, @@ -166,6 +169,7 @@ module VCAP::CloudController::Presenters::V3 { 'type' => process1.type, 'instances' => process1.instances, + 'log_rate_limit_per_second' => '1M', 'memory' => "#{process1.memory}M", 'disk_quota' => "#{process1.disk_quota}M", 'health-check-type' => process1.health_check_type, @@ -173,6 +177,7 @@ module VCAP::CloudController::Presenters::V3 { 'type' => process2.type, 'instances' => process2.instances, + 'log_rate_limit_per_second' => '1M', 'memory' => "#{process2.memory}M", 'disk_quota' => "#{process2.disk_quota}M", 'health-check-type' => process2.health_check_type, diff --git a/spec/unit/presenters/v3/app_manifest_presenters/process_properties_presenter_spec.rb b/spec/unit/presenters/v3/app_manifest_presenters/process_properties_presenter_spec.rb index e4684dd1fe6..c75f3c17cd6 100644 --- a/spec/unit/presenters/v3/app_manifest_presenters/process_properties_presenter_spec.rb +++ b/spec/unit/presenters/v3/app_manifest_presenters/process_properties_presenter_spec.rb @@ -32,5 +32,31 @@ module VCAP::CloudController::Presenters::V3::AppManifestPresenters end end end + + describe '#process_hash' do + let(:process) { VCAP::CloudController::ProcessModel.make } + + it 'renders a compact hash of the process' do + hash = subject.process_hash(process) + expect(hash).to eq({ + 'type' => 'web', + 'instances' => 1, + 'memory' => '1024M', + 'disk_quota' => '1024M', + 'log_rate_limit_per_second' => '1M', + 'health-check-type' => 'port', + }) + end + end + + describe '#add_units_log_rate_limit' do + it 'is consistant with other quotas with output' do + expect(subject.add_units_log_rate_limit(-1)).to eq('-1B') + expect(subject.add_units_log_rate_limit(256)).to eq('256B') + expect(subject.add_units_log_rate_limit(2_048)).to eq('2K') + expect(subject.add_units_log_rate_limit(4_194_304)).to eq('4M') + expect(subject.add_units_log_rate_limit(6_442_450_944)).to eq('6G') + end + end end end diff --git a/spec/unit/presenters/v3/process_presenter_spec.rb b/spec/unit/presenters/v3/process_presenter_spec.rb index 09894f8c890..affccd4c66b 100644 --- a/spec/unit/presenters/v3/process_presenter_spec.rb +++ b/spec/unit/presenters/v3/process_presenter_spec.rb @@ -147,6 +147,15 @@ module VCAP::CloudController::Presenters::V3 expect(result[:command]).to eq('[PRIVATE DATA HIDDEN]') end end + + context 'log quota is -1' do + before do + process.log_rate_limit = -1 + end + it 'displays it as unlimited' do + expect(result[:log_rate_limit_in_bytes_per_second]).to eq(-1) + end + end end end end diff --git a/spec/unit/presenters/v3/process_stats_presenter_spec.rb b/spec/unit/presenters/v3/process_stats_presenter_spec.rb index 53df8e43f12..1599806f5ce 100644 --- a/spec/unit/presenters/v3/process_stats_presenter_spec.rb +++ b/spec/unit/presenters/v3/process_stats_presenter_spec.rb @@ -66,12 +66,14 @@ module VCAP::CloudController::Presenters::V3 uptime: 12345, mem_quota: process[:memory] * 1024 * 1024, disk_quota: process[:disk_quota] * 1024 * 1024, + log_rate_limit: process[:log_rate_limit], fds_quota: process.file_descriptors, usage: { time: '2015-12-08 16:54:48 -0800', cpu: 80, mem: 128, disk: 1024, + log_rate: 2048, } } }, @@ -86,12 +88,14 @@ module VCAP::CloudController::Presenters::V3 uptime: 42, mem_quota: process[:memory] * 1024 * 1024, disk_quota: process[:disk_quota] * 1024 * 1024, + log_rate_limit: process[:log_rate_limit], fds_quota: process.file_descriptors, usage: { time: '2015-03-13 16:54:48 -0800', cpu: 70, mem: 128, disk: 1024, + log_rate: 7168, } } }, @@ -116,11 +120,13 @@ module VCAP::CloudController::Presenters::V3 expect(result[0][:uptime]).to eq(12345) expect(result[0][:mem_quota]).to eq(process[:memory] * 1024 * 1024) expect(result[0][:disk_quota]).to eq(process[:disk_quota] * 1024 * 1024) + expect(result[0][:log_rate_limit]).to eq(process[:log_rate_limit]) expect(result[0][:fds_quota]).to eq(process.file_descriptors) expect(result[0][:usage]).to eq({ time: '2015-12-08 16:54:48 -0800', cpu: 80, mem: 128, - disk: 1024 }) + disk: 1024, + log_rate: 2048 }) expect(result[1][:type]).to eq(process.type) expect(result[1][:index]).to eq(1) @@ -133,7 +139,8 @@ module VCAP::CloudController::Presenters::V3 expect(result[1][:usage]).to eq({ time: '2015-03-13 16:54:48 -0800', cpu: 70, mem: 128, - disk: 1024 }) + disk: 1024, + log_rate: 7168 }) expect(result[2]).to eq( type: process.type, @@ -190,11 +197,13 @@ module VCAP::CloudController::Presenters::V3 expect(result[0][:uptime]).to eq(12345) expect(result[0][:mem_quota]).to eq(process[:memory] * 1024 * 1024) expect(result[0][:disk_quota]).to eq(process[:disk_quota] * 1024 * 1024) + expect(result[0][:log_rate_limit]).to eq(process[:log_rate_limit]) expect(result[0][:fds_quota]).to eq(process.file_descriptors) expect(result[0][:usage]).to eq({ time: '2015-12-08 16:54:48 -0800', cpu: 80, mem: 128, - disk: 1024 }) + disk: 1024, + log_rate: 2048 }) end end @@ -230,6 +239,7 @@ module VCAP::CloudController::Presenters::V3 expect(result[0][:uptime]).to eq(12345) expect(result[0][:mem_quota]).to be_nil expect(result[0][:disk_quota]).to be_nil + expect(result[0][:log_rate_limit]).to be_nil expect(result[0][:fds_quota]).to eq(process.file_descriptors) expect(result[0][:usage]).to eq({}) end diff --git a/spec/unit/presenters/v3/space_quota_presenter_spec.rb b/spec/unit/presenters/v3/space_quota_presenter_spec.rb index 20b3cfddac1..2229ff81240 100644 --- a/spec/unit/presenters/v3/space_quota_presenter_spec.rb +++ b/spec/unit/presenters/v3/space_quota_presenter_spec.rb @@ -17,6 +17,7 @@ module VCAP::CloudController::Presenters::V3 instance_memory_limit: 3, app_instance_limit: 4, app_task_limit: 5, + log_rate_limit: 2000, non_basic_services_allowed: false, total_services: 6, total_service_keys: 7, @@ -42,6 +43,7 @@ module VCAP::CloudController::Presenters::V3 expect(result[:apps][:per_process_memory_in_mb]).to eq(space_quota.instance_memory_limit) expect(result[:apps][:total_instances]).to eq(space_quota.app_instance_limit) expect(result[:apps][:per_app_tasks]).to eq(space_quota.app_task_limit) + expect(result[:apps][:log_rate_limit_in_bytes_per_second]).to eq(space_quota.log_rate_limit) expect(result[:services][:paid_services_allowed]).to eq(space_quota.non_basic_services_allowed) expect(result[:services][:total_service_instances]).to eq(space_quota.total_services) expect(result[:services][:total_service_keys]).to eq(space_quota.total_service_keys) @@ -67,6 +69,7 @@ module VCAP::CloudController::Presenters::V3 instance_memory_limit: -1, app_instance_limit: -1, app_task_limit: -1, + log_rate_limit: -1, total_services: -1, total_service_keys: -1, total_routes: -1, @@ -79,6 +82,7 @@ module VCAP::CloudController::Presenters::V3 expect(result[:apps][:per_process_memory_in_mb]).to be_nil expect(result[:apps][:total_instances]).to be_nil expect(result[:apps][:per_app_tasks]).to be_nil + expect(result[:apps][:log_rate_limit_in_bytes_per_second]).to be_nil expect(result[:services][:total_service_instances]).to be_nil expect(result[:services][:total_service_keys]).to be_nil expect(result[:routes][:total_routes]).to be_nil diff --git a/spec/unit/presenters/v3/task_presenter_spec.rb b/spec/unit/presenters/v3/task_presenter_spec.rb index 4befd87aa89..a1db39405a7 100644 --- a/spec/unit/presenters/v3/task_presenter_spec.rb +++ b/spec/unit/presenters/v3/task_presenter_spec.rb @@ -9,6 +9,7 @@ module VCAP::CloudController::Presenters::V3 failure_reason: 'sup dawg', memory_in_mb: 2048, disk_in_mb: 4048, + log_rate_limit: 1024, created_at: Time.at(1), sequence_id: 5 ) @@ -64,6 +65,7 @@ module VCAP::CloudController::Presenters::V3 expect(result[:result][:failure_reason]).to eq 'sup dawg' expect(result[:memory_in_mb]).to eq(task.memory_in_mb) expect(result[:disk_in_mb]).to eq(task.disk_in_mb) + expect(result[:log_rate_limit_in_bytes_per_second]).to eq(task.log_rate_limit) expect(result[:sequence_id]).to eq(5) expect(result[:created_at]).to eq(task.created_at.iso8601) expect(result[:updated_at]).to eq(task.updated_at.iso8601) From 20d07d8861f1c9d0903f8a2e17586e1270db03fd Mon Sep 17 00:00:00 2001 From: Rebecca Roberts Date: Tue, 12 Jul 2022 19:41:15 +0000 Subject: [PATCH 02/20] Handle -1 without a byte suffix - We accept -1 without a byte suffix to mean unlimited - We also return -1 in the rendered manifest [#182624538](https://www.pivotaltracker.com/story/show/182624538) Github Issue: https://github.com/cloudfoundry/capi-release/issues/245 Co-authored-by: Rebecca Roberts Co-authored-by: Andrew Crump --- app/actions/space_diff_manifest.rb | 2 +- app/messages/app_manifest_message.rb | 9 +++- .../process_properties_presenter.rb | 8 ++-- spec/request/app_manifests_spec.rb | 2 +- spec/unit/actions/space_diff_manifest_spec.rb | 17 +++++++ .../messages/app_manifest_message_spec.rb | 44 ++++++++++++++++--- .../manifest_process_scale_message_spec.rb | 6 +-- .../process_properties_presenter_spec.rb | 4 +- 8 files changed, 71 insertions(+), 21 deletions(-) diff --git a/app/actions/space_diff_manifest.rb b/app/actions/space_diff_manifest.rb index 2963896dd85..63992b66a67 100644 --- a/app/actions/space_diff_manifest.rb +++ b/app/actions/space_diff_manifest.rb @@ -165,7 +165,7 @@ def normalize_units(manifest_app_hash) manifest_app_hash.each_with_index do |process_hash, index| byte_measurement_key_words.each do |key| value = process_hash[key] - manifest_app_hash[index][key] = normalize_unit(value, key) unless value.nil? + manifest_app_hash[index][key] = normalize_unit(value, key) unless value.nil? || value == -1 end end manifest_app_hash diff --git a/app/messages/app_manifest_message.rb b/app/messages/app_manifest_message.rb index f7589da5865..a9c4b846af0 100644 --- a/app/messages/app_manifest_message.rb +++ b/app/messages/app_manifest_message.rb @@ -292,6 +292,7 @@ def convert_to_mb(human_readable_byte_value) def convert_to_bytes_per_second(human_readable_byte_value) return nil unless human_readable_byte_value.present? + return -1 if human_readable_byte_value == -1 # unlimited byte_converter.convert_to_b(human_readable_byte_value.strip) rescue ByteConverter::InvalidUnitsError, ByteConverter::NonNumericError @@ -392,7 +393,9 @@ def validate_processes! type = process[:type] memory_error = validate_byte_format(process[:memory], 'Memory') disk_error = validate_byte_format(process[:disk_quota], 'Disk quota') - log_rate_limit_error = validate_byte_format(process[:log_rate_limit_per_second], 'Log rate limit per second') + if process[:log_rate_limit_per_second] != -1 + log_rate_limit_error = validate_byte_format(process[:log_rate_limit_per_second], 'Log rate limit per second') + end add_process_error!(memory_error, type) if memory_error add_process_error!(disk_error, type) if disk_error add_process_error!(log_rate_limit_error, type) if log_rate_limit_error @@ -416,7 +419,9 @@ def validate_sidecars! def validate_top_level_web_process! memory_error = validate_byte_format(memory, 'Memory') disk_error = validate_byte_format(disk_quota, 'Disk quota') - log_rate_limit_error = validate_byte_format(log_rate_limit_per_second, 'Log rate limit per second') + if log_rate_limit_per_second != -1 + log_rate_limit_error = validate_byte_format(log_rate_limit_per_second, 'Log rate limit per second') + end add_process_error!(memory_error, ProcessTypes::WEB) if memory_error add_process_error!(disk_error, ProcessTypes::WEB) if disk_error add_process_error!(log_rate_limit_error, ProcessTypes::WEB) if log_rate_limit_error diff --git a/app/presenters/v3/app_manifest_presenters/process_properties_presenter.rb b/app/presenters/v3/app_manifest_presenters/process_properties_presenter.rb index 155a5386976..82180777924 100644 --- a/app/presenters/v3/app_manifest_presenters/process_properties_presenter.rb +++ b/app/presenters/v3/app_manifest_presenters/process_properties_presenter.rb @@ -27,11 +27,9 @@ def add_units(val) end def add_units_log_rate_limit(val) - if val == -1 - '-1B' - else - byte_converter.human_readable_byte_value(val) - end + return -1 if val == -1 + + byte_converter.human_readable_byte_value(val) end def byte_converter diff --git a/spec/request/app_manifests_spec.rb b/spec/request/app_manifests_spec.rb index 1fe2ae2a693..62a27bddaf7 100644 --- a/spec/request/app_manifests_spec.rb +++ b/spec/request/app_manifests_spec.rb @@ -207,7 +207,7 @@ 'instances' => process.instances, 'memory' => "#{process.memory}M", 'disk_quota' => "#{process.disk_quota}M", - 'log_rate_limit_per_second' => '-1B', + 'log_rate_limit_per_second' => -1, 'health-check-type' => process.health_check_type, }, { diff --git a/spec/unit/actions/space_diff_manifest_spec.rb b/spec/unit/actions/space_diff_manifest_spec.rb index b410175fa01..a02b26a5cf3 100644 --- a/spec/unit/actions/space_diff_manifest_spec.rb +++ b/spec/unit/actions/space_diff_manifest_spec.rb @@ -334,6 +334,23 @@ module VCAP::CloudController end end + context 'when the log_rate_limit_per_second is unlimited' do + before do + default_manifest['applications'][0]['log_rate_limit_per_second'] = -1 + end + + it 'displays -1 in the diff' do + expect(subject).to include( + { + 'op' => 'replace', + 'path' => '/applications/0/log_rate_limit_per_second', + 'value' => -1, + 'was' => '1M' + } + ) + end + end + context 'when the user passes in a v2 manifest' do let(:default_manifest) { { diff --git a/spec/unit/messages/app_manifest_message_spec.rb b/spec/unit/messages/app_manifest_message_spec.rb index 887f7cea778..d3c4f19b769 100644 --- a/spec/unit/messages/app_manifest_message_spec.rb +++ b/spec/unit/messages/app_manifest_message_spec.rb @@ -127,7 +127,7 @@ module VCAP::CloudController expect(message).not_to be_valid expect(message.errors).to have(1).items - expect(message.errors.full_messages).to include('Process "web": Log rate limit must be an integer greater than or equal to -1B') + expect(message.errors.full_messages).to include('Process "web": Log rate limit must be an integer greater than or equal to -1') end end @@ -145,12 +145,22 @@ module VCAP::CloudController end end - context 'when log_rate_limit_per_second is unlimited amount' do - let(:params_from_yaml) { { name: 'eugene', log_rate_limit_per_second: '-1B' } } + context 'when log_rate_limit_per_second is an unlimited amount' do + context 'specified as -1' do + let(:params_from_yaml) { { name: 'eugene', log_rate_limit_per_second: -1 } } - it 'is valid' do - message = AppManifestMessage.create_from_yml(params_from_yaml) - expect(message).to be_valid + it 'is valid' do + message = AppManifestMessage.create_from_yml(params_from_yaml) + expect(message).to be_valid + end + end + context 'with a bytes suffix' do + let(:params_from_yaml) { { name: 'eugene', log_rate_limit_per_second: '-1B' } } + + it 'is valid' do + message = AppManifestMessage.create_from_yml(params_from_yaml) + expect(message).to be_valid + end end end @@ -162,7 +172,6 @@ module VCAP::CloudController expect(message).to be_valid end end - end describe 'buildpack' do @@ -704,6 +713,27 @@ module VCAP::CloudController expect(message.errors.full_messages).to include('Process "bob" may only be present once') end end + + context 'log_rate_limit_per_second' do + context 'when log_rate_limit_per_second is an unlimited amount' do + context 'specified as -1' do + let(:params_from_yaml) { { name: 'eugene', processes: [{ 'type' => 'foo', 'log_rate_limit_per_second' => -1 }] } } + + it 'is valid' do + message = AppManifestMessage.create_from_yml(params_from_yaml) + expect(message).to be_valid + end + end + context 'with a bytes suffix' do + let(:params_from_yaml) { { name: 'eugene', processes: [{ 'type' => 'foo', 'log_rate_limit_per_second' => '-1B' }] } } + + it 'is valid' do + message = AppManifestMessage.create_from_yml(params_from_yaml) + expect(message).to be_valid + end + end + end + end end describe 'sidecars' do diff --git a/spec/unit/messages/manifest_process_scale_message_spec.rb b/spec/unit/messages/manifest_process_scale_message_spec.rb index 5f31604935a..bf7571fb9f6 100644 --- a/spec/unit/messages/manifest_process_scale_message_spec.rb +++ b/spec/unit/messages/manifest_process_scale_message_spec.rb @@ -150,7 +150,7 @@ module VCAP::CloudController expect(message).not_to be_valid expect(message.errors.count).to eq(1) - expect(message.errors.full_messages).to include('Log rate limit must be an integer greater than or equal to -1B') + expect(message.errors.full_messages).to include('Log rate limit must be an integer greater than or equal to -1') end end @@ -162,7 +162,7 @@ module VCAP::CloudController expect(message).not_to be_valid expect(message.errors.count).to eq(1) - expect(message.errors.full_messages).to include('Log rate limit must be an integer greater than or equal to -1B') + expect(message.errors.full_messages).to include('Log rate limit must be an integer greater than or equal to -1') end end @@ -174,7 +174,7 @@ module VCAP::CloudController expect(message).not_to be_valid expect(message.errors.count).to eq(1) - expect(message.errors.full_messages).to include('Log rate limit must be an integer greater than or equal to -1B') + expect(message.errors.full_messages).to include('Log rate limit must be an integer greater than or equal to -1') end end end diff --git a/spec/unit/presenters/v3/app_manifest_presenters/process_properties_presenter_spec.rb b/spec/unit/presenters/v3/app_manifest_presenters/process_properties_presenter_spec.rb index c75f3c17cd6..50e19d29680 100644 --- a/spec/unit/presenters/v3/app_manifest_presenters/process_properties_presenter_spec.rb +++ b/spec/unit/presenters/v3/app_manifest_presenters/process_properties_presenter_spec.rb @@ -50,8 +50,8 @@ module VCAP::CloudController::Presenters::V3::AppManifestPresenters end describe '#add_units_log_rate_limit' do - it 'is consistant with other quotas with output' do - expect(subject.add_units_log_rate_limit(-1)).to eq('-1B') + it 'is consistent with other quotas with output' do + expect(subject.add_units_log_rate_limit(-1)).to eq(-1) expect(subject.add_units_log_rate_limit(256)).to eq('256B') expect(subject.add_units_log_rate_limit(2_048)).to eq('2K') expect(subject.add_units_log_rate_limit(4_194_304)).to eq('4M') From 5030793f8f93eddb3786b228f5460c0d80599e1a Mon Sep 17 00:00:00 2001 From: Rebecca Roberts Date: Tue, 12 Jul 2022 21:10:58 +0000 Subject: [PATCH 03/20] Replace scaling_operation? with started? - We found scaling_operation? to be a misleading name - Just refer to started? to make it clearer when validation will be performed [#182624538](https://www.pivotaltracker.com/story/show/182624538) Co-authored-by: Andrew Crump --- app/models/runtime/constraints/max_app_instances_policy.rb | 2 +- app/models/runtime/constraints/max_instance_memory_policy.rb | 2 +- app/models/runtime/constraints/max_memory_policy.rb | 2 +- app/models/runtime/process_model.rb | 4 ---- 4 files changed, 3 insertions(+), 7 deletions(-) diff --git a/app/models/runtime/constraints/max_app_instances_policy.rb b/app/models/runtime/constraints/max_app_instances_policy.rb index 2a591662181..8ae62497b90 100644 --- a/app/models/runtime/constraints/max_app_instances_policy.rb +++ b/app/models/runtime/constraints/max_app_instances_policy.rb @@ -11,7 +11,7 @@ def initialize(process, space_or_org, quota_definition, error_name) def validate return unless @quota_definition - return unless @process.scaling_operation? + return unless @process.started? return if @quota_definition.app_instance_limit == -1 || @process.stopped? other_apps = @space_or_org.processes.reject { |process| process.guid == @process.guid } diff --git a/app/models/runtime/constraints/max_instance_memory_policy.rb b/app/models/runtime/constraints/max_instance_memory_policy.rb index b5a8956e798..5ac48a24a2e 100644 --- a/app/models/runtime/constraints/max_instance_memory_policy.rb +++ b/app/models/runtime/constraints/max_instance_memory_policy.rb @@ -39,7 +39,7 @@ class AppMaxInstanceMemoryPolicy < BaseMaxInstanceMemoryPolicy private def additional_checks - resource.scaling_operation? + resource.started? end end diff --git a/app/models/runtime/constraints/max_memory_policy.rb b/app/models/runtime/constraints/max_memory_policy.rb index 6209953f4ee..00163e0350f 100644 --- a/app/models/runtime/constraints/max_memory_policy.rb +++ b/app/models/runtime/constraints/max_memory_policy.rb @@ -37,7 +37,7 @@ class AppMaxMemoryPolicy < BaseMaxMemoryPolicy private def additional_checks - resource.scaling_operation? + resource.started? end def requested_memory diff --git a/app/models/runtime/process_model.rb b/app/models/runtime/process_model.rb index a76b5172163..097e346a25e 100644 --- a/app/models/runtime/process_model.rb +++ b/app/models/runtime/process_model.rb @@ -379,10 +379,6 @@ def being_stopped? column_changed?(:state) && stopped? end - def scaling_operation? - started? - end - def desired_instances started? ? instances : 0 end From bb31b14cf64182b26d40c388665fe0efe64b3694 Mon Sep 17 00:00:00 2001 From: Andrew Crump Date: Fri, 15 Jul 2022 19:52:47 +0000 Subject: [PATCH 04/20] Pass log rate limit to Diego [#182311441](https://www.pivotaltracker.com/story/show/182311441) Co-authored-by: Rebecca Roberts --- lib/cloud_controller/diego/app_recipe_builder.rb | 1 + lib/diego/bbs/models/desired_lrp_pb.rb | 2 ++ .../unit/lib/cloud_controller/diego/app_recipe_builder_spec.rb | 3 +++ 3 files changed, 6 insertions(+) diff --git a/lib/cloud_controller/diego/app_recipe_builder.rb b/lib/cloud_controller/diego/app_recipe_builder.rb index 1ccec7e8d6d..26d4da69e2c 100644 --- a/lib/cloud_controller/diego/app_recipe_builder.rb +++ b/lib/cloud_controller/diego/app_recipe_builder.rb @@ -68,6 +68,7 @@ def app_lrp_arguments start_timeout_ms: health_check_timeout_in_seconds * 1000, disk_mb: process.disk_quota, memory_mb: process.memory, # sums up + log_rate_limit_bytes_per_second: process.log_rate_limit, privileged: desired_lrp_builder.privileged?, ports: ports, log_source: LRP_LOG_SOURCE, diff --git a/lib/diego/bbs/models/desired_lrp_pb.rb b/lib/diego/bbs/models/desired_lrp_pb.rb index c2b3a317326..f4397f6a629 100644 --- a/lib/diego/bbs/models/desired_lrp_pb.rb +++ b/lib/diego/bbs/models/desired_lrp_pb.rb @@ -53,6 +53,7 @@ repeated :image_layers, :message, 24, "diego.bbs.models.ImageLayer" map :metric_tags, :string, :message, 25, "diego.bbs.models.MetricTagValue" repeated :sidecars, :message, 26, "diego.bbs.models.Sidecar" + optional :log_rate_limit_bytes_per_second, :int64, 27 end add_message "diego.bbs.models.ProtoRoutes" do map :routes, :string, :bytes, 1 @@ -114,6 +115,7 @@ repeated :image_layers, :message, 34, "diego.bbs.models.ImageLayer" map :metric_tags, :string, :message, 35, "diego.bbs.models.MetricTagValue" repeated :sidecars, :message, 36, "diego.bbs.models.Sidecar" + optional :log_rate_limit_bytes_per_second, :int64, 37 end end diff --git a/spec/unit/lib/cloud_controller/diego/app_recipe_builder_spec.rb b/spec/unit/lib/cloud_controller/diego/app_recipe_builder_spec.rb index 9f0e3a347c7..2e1af40320b 100644 --- a/spec/unit/lib/cloud_controller/diego/app_recipe_builder_spec.rb +++ b/spec/unit/lib/cloud_controller/diego/app_recipe_builder_spec.rb @@ -47,6 +47,7 @@ module Diego instances: 21, memory: 128, disk_quota: 256, + log_rate_limit: 1024, command: command, file_descriptors: 32, health_check_type: 'port', @@ -285,6 +286,7 @@ module Diego expect(lrp.log_source).to eq(LRP_LOG_SOURCE) expect(lrp.max_pids).to eq(100) expect(lrp.memory_mb).to eq(128) + expect(lrp.log_rate_limit_bytes_per_second).to eq(1024) expect(lrp.metrics_guid).to eq(process.app.guid) expect(lrp.metric_tags.keys.size).to eq(11) @@ -872,6 +874,7 @@ module Diego expect(lrp.log_guid).to eq(process.app.guid) expect(lrp.max_pids).to eq(100) expect(lrp.memory_mb).to eq(128) + expect(lrp.log_rate_limit_bytes_per_second).to eq(1024) expect(lrp.metrics_guid).to eq(process.app.guid) expect(lrp.metric_tags.keys.size).to eq(11) From fc2220988756be2bd86c921959403d5255d82314 Mon Sep 17 00:00:00 2001 From: Andrew Crump Date: Fri, 15 Jul 2022 23:58:45 +0000 Subject: [PATCH 05/20] Pass task log rate limit to Diego [#182311441](https://www.pivotaltracker.com/story/show/182311441) Co-authored-by: Matthew Kocher --- lib/cloud_controller/diego/task_recipe_builder.rb | 1 + lib/diego/bbs/models/task_pb.rb | 1 + .../lib/cloud_controller/diego/task_recipe_builder_spec.rb | 3 +++ 3 files changed, 5 insertions(+) diff --git a/lib/cloud_controller/diego/task_recipe_builder.rb b/lib/cloud_controller/diego/task_recipe_builder.rb index ba9ae460ca9..c6cd9d5bee5 100644 --- a/lib/cloud_controller/diego/task_recipe_builder.rb +++ b/lib/cloud_controller/diego/task_recipe_builder.rb @@ -26,6 +26,7 @@ def build_app_task(config, task) disk_mb: task.disk_in_mb, egress_rules: @egress_rules.running_protobuf_rules(task.app), log_guid: task.app.guid, + log_rate_limit_bytes_per_second: task.log_rate_limit, log_source: TASK_LOG_SOURCE, max_pids: config.get(:diego, :pid_limit), memory_mb: task.memory_in_mb, diff --git a/lib/diego/bbs/models/task_pb.rb b/lib/diego/bbs/models/task_pb.rb index 68eb2386c91..ac30f329781 100644 --- a/lib/diego/bbs/models/task_pb.rb +++ b/lib/diego/bbs/models/task_pb.rb @@ -38,6 +38,7 @@ optional :image_username, :string, 23 optional :image_password, :string, 24 repeated :image_layers, :message, 25, "diego.bbs.models.ImageLayer" + optional :log_rate_limit_bytes_per_second, :int64, 26 end add_message "diego.bbs.models.Task" do optional :task_definition, :message, 1, "diego.bbs.models.TaskDefinition" diff --git a/spec/unit/lib/cloud_controller/diego/task_recipe_builder_spec.rb b/spec/unit/lib/cloud_controller/diego/task_recipe_builder_spec.rb index f7b8f1acb8b..7fdda5cbd51 100644 --- a/spec/unit/lib/cloud_controller/diego/task_recipe_builder_spec.rb +++ b/spec/unit/lib/cloud_controller/diego/task_recipe_builder_spec.rb @@ -344,6 +344,7 @@ module Diego app: app, disk_in_mb: 1024, memory_in_mb: 2048, + log_rate_limit: 3072, sequence_id: 9 ) end @@ -468,6 +469,7 @@ module Diego expect(result.log_guid).to eq(app.guid) expect(result.memory_mb).to eq(2048) expect(result.disk_mb).to eq(1024) + expect(result.log_rate_limit_bytes_per_second).to eq(3072) expect(result.environment_variables).to eq(lifecycle_environment_variables) expect(result.legacy_download_user).to eq('vcap') expect(result.root_fs).to eq('preloaded:potato-stack') @@ -610,6 +612,7 @@ module Diego expect(result.disk_mb).to eq(1024) expect(result.memory_mb).to eq(2048) + expect(result.log_rate_limit_bytes_per_second).to eq(3072) expect(result.log_guid).to eq(app.guid) expect(result.privileged).to be(false) expect(result.egress_rules).to eq([ From 6d3e2e8fe107281315948a624a2ba760fac171f1 Mon Sep 17 00:00:00 2001 From: Andrew Crump Date: Tue, 19 Jul 2022 02:03:59 +0000 Subject: [PATCH 06/20] Apply log rate limit to staging tasks - The log rate limit from the application web process is applied to the staging task - The staging log rate limit can be customized when creating a build, for consistency with memory and disk limits [#182311441](https://www.pivotaltracker.com/story/show/182311441) Co-authored-by: Rebecca Roberts --- app/actions/build_create.rb | 13 +++- app/messages/build_create_message.rb | 3 +- app/presenters/v3/build_presenter.rb | 1 + ...22_add_staging_log_rate_limit_to_builds.rb | 7 ++ ...ating_staging_log_rate_limit_calculator.rb | 28 ++++++++ lib/cloud_controller/diego/staging_details.rb | 2 +- .../diego/task_recipe_builder.rb | 1 + spec/request/apps_spec.rb | 4 ++ spec/request/builds_spec.rb | 9 ++- spec/unit/actions/build_create_spec.rb | 24 ++++++- ..._staging_log_rate_limit_calculator_spec.rb | 68 +++++++++++++++++++ .../diego/task_recipe_builder_spec.rb | 2 + .../messages/build_create_message_spec.rb | 29 ++++++++ .../presenters/v3/build_presenter_spec.rb | 2 + 14 files changed, 188 insertions(+), 5 deletions(-) create mode 100644 db/migrations/20220718211322_add_staging_log_rate_limit_to_builds.rb create mode 100644 lib/cloud_controller/backends/quota_validating_staging_log_rate_limit_calculator.rb create mode 100644 spec/unit/lib/cloud_controller/backends/quota_validating_staging_log_rate_limit_calculator_spec.rb diff --git a/app/actions/build_create.rb b/app/actions/build_create.rb index 379f4643ca9..b8c02e53bb5 100644 --- a/app/actions/build_create.rb +++ b/app/actions/build_create.rb @@ -1,3 +1,4 @@ +require 'cloud_controller/backends/quota_validating_staging_log_rate_limit_calculator' require 'cloud_controller/backends/quota_validating_staging_memory_calculator' require 'cloud_controller/backends/staging_disk_calculator' require 'cloud_controller/backends/staging_environment_builder' @@ -25,12 +26,14 @@ class StagingInProgress < BuildError def initialize(user_audit_info: UserAuditInfo.from_context(SecurityContext), memory_limit_calculator: QuotaValidatingStagingMemoryCalculator.new, disk_limit_calculator: StagingDiskCalculator.new, + log_rate_limit_calculator: QuotaValidatingStagingLogRateLimitCalculator.new, environment_presenter: StagingEnvironmentBuilder.new) @user_audit_info = user_audit_info @memory_limit_calculator = memory_limit_calculator @disk_limit_calculator = disk_limit_calculator - @environment_builder = environment_presenter + @log_rate_limit_calculator = log_rate_limit_calculator + @environment_builder = environment_presenter end def create_and_stage(package:, lifecycle:, metadata: nil, start_after_staging: false) @@ -49,6 +52,7 @@ def create_and_stage(package:, lifecycle:, metadata: nil, start_after_staging: f app: package.app, staging_memory_in_mb: staging_details.staging_memory_in_mb, staging_disk_in_mb: staging_details.staging_disk_in_mb, + staging_log_rate_limit: staging_details.staging_log_rate_limit_bytes_per_second, created_by_user_guid: @user_audit_info.user_guid, created_by_user_name: @user_audit_info.user_name, created_by_user_email: @user_audit_info.user_email @@ -117,6 +121,7 @@ def get_staging_details(package, lifecycle) memory_limit = get_memory_limit(lifecycle.staging_message.staging_memory_in_mb, app, space, org) disk_limit = get_disk_limit(lifecycle.staging_message.staging_disk_in_mb, app) + log_rate_limit = get_log_rate_limit(lifecycle.staging_message.staging_log_rate_limit_bytes_per_second, app, space, org) environment_variables = @environment_builder.build(app, space, lifecycle, @@ -128,6 +133,7 @@ def get_staging_details(package, lifecycle) staging_details.package = package staging_details.staging_memory_in_mb = memory_limit staging_details.staging_disk_in_mb = disk_limit + staging_details.staging_log_rate_limit_bytes_per_second = log_rate_limit staging_details.environment_variables = environment_variables staging_details.lifecycle = lifecycle staging_details.isolation_segment = IsolationSegmentSelector.for_space(space) @@ -151,6 +157,11 @@ def get_memory_limit(requested_limit, app, space, org) raise OrgQuotaExceeded.new e.message end + def get_log_rate_limit(requested_limit, app, space, org) + limit = requested_limit || app.newest_web_process&.log_rate_limit + @log_rate_limit_calculator.get_limit(limit, space, org) + end + def logger @logger ||= Steno.logger('cc.action.build_create') end diff --git a/app/messages/build_create_message.rb b/app/messages/build_create_message.rb index cad48bc7d58..c07f0b6bc72 100644 --- a/app/messages/build_create_message.rb +++ b/app/messages/build_create_message.rb @@ -4,7 +4,7 @@ module VCAP::CloudController class BuildCreateMessage < MetadataBaseMessage - register_allowed_keys [:staging_memory_in_mb, :staging_disk_in_mb, :environment_variables, :lifecycle, :package] + register_allowed_keys [:staging_memory_in_mb, :staging_disk_in_mb, :staging_log_rate_limit_bytes_per_second, :environment_variables, :lifecycle, :package] def self.lifecycle_requested? @lifecycle_requested ||= proc { |a| a.requested?(:lifecycle) } @@ -12,6 +12,7 @@ def self.lifecycle_requested? validates :staging_disk_in_mb, numericality: { only_integer: true, greater_than: 0 }, allow_nil: true validates :staging_memory_in_mb, numericality: { only_integer: true, greater_than: 0 }, allow_nil: true + validates :staging_log_rate_limit_bytes_per_second, numericality: { only_integer: true, greater_than_or_equal_to: -1 }, allow_nil: true validates_with NoAdditionalKeysValidator validates_with LifecycleValidator, if: lifecycle_requested? diff --git a/app/presenters/v3/build_presenter.rb b/app/presenters/v3/build_presenter.rb index 855349f0fe7..4676bf39772 100644 --- a/app/presenters/v3/build_presenter.rb +++ b/app/presenters/v3/build_presenter.rb @@ -22,6 +22,7 @@ def to_hash state: build.state, staging_memory_in_mb: build.staging_memory_in_mb, staging_disk_in_mb: build.staging_disk_in_mb, + staging_log_rate_limit_bytes_per_second: build.staging_log_rate_limit, error: error, lifecycle: { type: build.lifecycle_type, diff --git a/db/migrations/20220718211322_add_staging_log_rate_limit_to_builds.rb b/db/migrations/20220718211322_add_staging_log_rate_limit_to_builds.rb new file mode 100644 index 00000000000..13652db0978 --- /dev/null +++ b/db/migrations/20220718211322_add_staging_log_rate_limit_to_builds.rb @@ -0,0 +1,7 @@ +Sequel.migration do + change do + alter_table :builds do + add_column :staging_log_rate_limit, :Bignum, null: false, default: -1 + end + end +end diff --git a/lib/cloud_controller/backends/quota_validating_staging_log_rate_limit_calculator.rb b/lib/cloud_controller/backends/quota_validating_staging_log_rate_limit_calculator.rb new file mode 100644 index 00000000000..a6ec550bfd2 --- /dev/null +++ b/lib/cloud_controller/backends/quota_validating_staging_log_rate_limit_calculator.rb @@ -0,0 +1,28 @@ +module VCAP::CloudController + class QuotaValidatingStagingLogRateLimitCalculator + class SpaceQuotaExceeded < StandardError; end + class OrgQuotaExceeded < StandardError; end + + def get_limit(requested_limit, space, org) + if requested_limit.nil? + requested_limit = -1 + end + + requested_limit = requested_limit.to_i + + space_quota_exceeded!(requested_limit) unless space.has_remaining_log_rate_limit(requested_limit) + org_quota_exceeded!(requested_limit) unless org.has_remaining_log_rate_limit(requested_limit) + requested_limit + end + + private + + def org_quota_exceeded!(staging_log_rate_limit) + raise OrgQuotaExceeded.new("staging requires #{staging_log_rate_limit} bytes per second") + end + + def space_quota_exceeded!(staging_log_rate_limit) + raise SpaceQuotaExceeded.new("staging requires #{staging_log_rate_limit} bytes per second") + end + end +end diff --git a/lib/cloud_controller/diego/staging_details.rb b/lib/cloud_controller/diego/staging_details.rb index 152fa322980..51018788ac4 100644 --- a/lib/cloud_controller/diego/staging_details.rb +++ b/lib/cloud_controller/diego/staging_details.rb @@ -1,7 +1,7 @@ module VCAP::CloudController module Diego class StagingDetails - attr_accessor :staging_guid, :staging_memory_in_mb, :staging_disk_in_mb, :package, + attr_accessor :staging_guid, :staging_memory_in_mb, :staging_disk_in_mb, :staging_log_rate_limit_bytes_per_second, :package, :environment_variables, :lifecycle, :start_after_staging, :isolation_segment end end diff --git a/lib/cloud_controller/diego/task_recipe_builder.rb b/lib/cloud_controller/diego/task_recipe_builder.rb index c6cd9d5bee5..3a0307bb171 100644 --- a/lib/cloud_controller/diego/task_recipe_builder.rb +++ b/lib/cloud_controller/diego/task_recipe_builder.rb @@ -65,6 +65,7 @@ def build_staging_task(config, staging_details) log_guid: staging_details.package.app_guid, log_source: STAGING_LOG_SOURCE, memory_mb: staging_details.staging_memory_in_mb, + log_rate_limit_bytes_per_second: staging_details.staging_log_rate_limit_bytes_per_second, network: generate_network(staging_details.package, Protocol::ContainerNetworkInfo::STAGING), privileged: config.get(:diego, :use_privileged_containers_for_staging), result_file: STAGING_RESULT_FILE, diff --git a/spec/request/apps_spec.rb b/spec/request/apps_spec.rb index 41e5a08f589..b22a1d3a896 100644 --- a/spec/request/apps_spec.rb +++ b/spec/request/apps_spec.rb @@ -1778,6 +1778,7 @@ app: app_model, staging_memory_in_mb: 123, staging_disk_in_mb: 456, + staging_log_rate_limit: 789, created_by_user_name: 'bob the builder', created_by_user_guid: user.guid, created_by_user_email: 'bob@loblaw.com' @@ -1789,6 +1790,7 @@ app: app_model, staging_memory_in_mb: 123, staging_disk_in_mb: 456, + staging_log_rate_limit: 789, created_at: build.created_at - 1.day, created_by_user_name: 'bob the builder', created_by_user_guid: user.guid, @@ -1874,6 +1876,7 @@ 'error' => nil, 'staging_memory_in_mb' => 123, 'staging_disk_in_mb' => 456, + 'staging_log_rate_limit_bytes_per_second' => 789, 'lifecycle' => { 'type' => 'buildpack', 'data' => { @@ -1902,6 +1905,7 @@ 'error' => nil, 'staging_memory_in_mb' => 123, 'staging_disk_in_mb' => 456, + 'staging_log_rate_limit_bytes_per_second' => 789, 'lifecycle' => { 'type' => 'buildpack', 'data' => { diff --git a/spec/request/builds_spec.rb b/spec/request/builds_spec.rb index 219eebbaf60..34d67ee852b 100644 --- a/spec/request/builds_spec.rb +++ b/spec/request/builds_spec.rb @@ -78,6 +78,7 @@ 'state' => 'STAGING', 'staging_memory_in_mb' => 42, 'staging_disk_in_mb' => 42, + 'staging_log_rate_limit_bytes_per_second' => -1, 'metadata' => { 'labels' => { 'release' => 'stable', 'seriouseats.com/potato' => 'mashed' }, 'annotations' => { 'potato' => 'idaho' } }, 'error' => nil, 'lifecycle' => { @@ -347,6 +348,7 @@ created_by_user_email: 'bob@loblaw.com', staging_memory_in_mb: 123, staging_disk_in_mb: 456, + staging_log_rate_limit: 234 ) end let!(:second_build) do @@ -358,7 +360,8 @@ created_by_user_guid: developer.guid, created_by_user_email: 'bob@loblaw.com', staging_memory_in_mb: 789, - staging_disk_in_mb: 12 + staging_disk_in_mb: 12, + staging_log_rate_limit: 345 ) end let(:package) { VCAP::CloudController::PackageModel.make(app_guid: app_model.guid) } @@ -468,6 +471,7 @@ 'state' => 'STAGED', 'staging_memory_in_mb' => 123, 'staging_disk_in_mb' => 456, + 'staging_log_rate_limit_bytes_per_second' => 234, 'error' => nil, 'lifecycle' => { 'type' => 'buildpack', @@ -496,6 +500,7 @@ 'state' => 'STAGED', 'staging_memory_in_mb' => 789, 'staging_disk_in_mb' => 12, + 'staging_log_rate_limit_bytes_per_second' => 345, 'error' => nil, 'lifecycle' => { 'type' => 'buildpack', @@ -557,6 +562,7 @@ app: app_model, staging_memory_in_mb: 123, staging_disk_in_mb: 456, + staging_log_rate_limit: 789, created_by_user_name: 'bob the builder', created_by_user_guid: developer.guid, created_by_user_email: 'bob@loblaw.com' @@ -593,6 +599,7 @@ 'state' => 'STAGED', 'staging_memory_in_mb' => 123, 'staging_disk_in_mb' => 456, + 'staging_log_rate_limit_bytes_per_second' => 789, 'error' => nil, 'lifecycle' => { 'type' => 'buildpack', diff --git a/spec/unit/actions/build_create_spec.rb b/spec/unit/actions/build_create_spec.rb index 5173244a3a2..7b60d748683 100644 --- a/spec/unit/actions/build_create_spec.rb +++ b/spec/unit/actions/build_create_spec.rb @@ -10,12 +10,14 @@ module VCAP::CloudController user_audit_info: user_audit_info, memory_limit_calculator: memory_limit_calculator, disk_limit_calculator: disk_limit_calculator, + log_rate_limit_calculator: log_rate_limit_calculator, environment_presenter: environment_builder ) end let(:memory_limit_calculator) { double(:memory_limit_calculator) } let(:disk_limit_calculator) { double(:disk_limit_calculator) } + let(:log_rate_limit_calculator) { double(:log_rate_limit_calculator) } let(:environment_builder) { double(:environment_builder) } let(:user_audit_info) { UserAuditInfo.new(user_email: 'charles@las.gym', user_guid: '1234', user_name: 'charles') } @@ -24,6 +26,7 @@ module VCAP::CloudController { staging_memory_in_mb: staging_memory_in_mb, staging_disk_in_mb: staging_disk_in_mb, + staging_log_rate_limit_bytes_per_second: staging_log_rate_limit_bytes_per_second, lifecycle: request_lifecycle, }.deep_stringify_keys end @@ -50,7 +53,7 @@ module VCAP::CloudController let(:space) { Space.make } let(:org) { space.organization } let(:app) { AppModel.make(space: space) } - let!(:process) { ProcessModel.make(app: app, memory: 8192, disk_quota: 512) } + let!(:process) { ProcessModel.make(app: app, memory: 8192, disk_quota: 512, log_rate_limit: 7168) } let(:buildpack_git_url) { 'http://example.com/repo.git' } let(:stack) { Stack.default } @@ -65,9 +68,11 @@ module VCAP::CloudController let(:stager) { instance_double(Diego::Stager) } let(:calculated_mem_limit) { 32 } let(:calculated_staging_disk_in_mb) { 64 } + let(:calculated_staging_log_rate_limit) { 96 } let(:staging_memory_in_mb) { 1024 } let(:staging_disk_in_mb) { 2048 } + let(:staging_log_rate_limit_bytes_per_second) { 3072 } let(:environment_variables) { 'random string' } before do @@ -76,6 +81,7 @@ module VCAP::CloudController allow(stager).to receive(:stage) allow(memory_limit_calculator).to receive(:get_limit).with(staging_memory_in_mb, space, org).and_return(calculated_mem_limit) allow(disk_limit_calculator).to receive(:get_limit).with(staging_disk_in_mb).and_return(calculated_staging_disk_in_mb) + allow(log_rate_limit_calculator).to receive(:get_limit).with(staging_log_rate_limit_bytes_per_second, space, org).and_return(calculated_staging_log_rate_limit) allow(environment_builder).to receive(:build).and_return(environment_variables) end @@ -93,6 +99,7 @@ module VCAP::CloudController expect(build.package_guid).to eq(package.guid) expect(build.staging_memory_in_mb).to eq(calculated_mem_limit) expect(build.staging_disk_in_mb).to eq(calculated_staging_disk_in_mb) + expect(build.staging_log_rate_limit).to eq(calculated_staging_log_rate_limit) expect(build.lifecycle_data.to_hash).to eq(lifecycle_data) expect(build.created_by_user_guid).to eq('1234') expect(build.created_by_user_name).to eq('charles') @@ -174,6 +181,7 @@ module VCAP::CloudController expect(staging_details.staging_guid).to eq(build.guid) expect(staging_details.staging_memory_in_mb).to eq(calculated_mem_limit) expect(staging_details.staging_disk_in_mb).to eq(calculated_staging_disk_in_mb) + expect(staging_details.staging_log_rate_limit_bytes_per_second).to eq(calculated_staging_log_rate_limit) expect(staging_details.environment_variables).to eq(environment_variables) expect(staging_details.lifecycle).to eq(lifecycle) expect(staging_details.isolation_segment).to be_nil @@ -210,6 +218,20 @@ module VCAP::CloudController end end + context 'when staging log rate limit is not specified in the message' do + before do + allow(log_rate_limit_calculator).to receive(:get_limit).with(process.log_rate_limit, space, org).and_return(process.log_rate_limit) + end + let(:staging_log_rate_limit_bytes_per_second) { nil } + + it 'uses the app web process log rate limit for staging log rate limit' do + expect(log_rate_limit_calculator).to receive(:get_limit).with(process.log_rate_limit, space, org) + + build = action.create_and_stage(package: package, lifecycle: lifecycle) + expect(build.staging_log_rate_limit).to eq(process.log_rate_limit) + end + end + describe 'isolation segments' do let(:assigner) { VCAP::CloudController::IsolationSegmentAssign.new } let(:isolation_segment_model) { VCAP::CloudController::IsolationSegmentModel.make } diff --git a/spec/unit/lib/cloud_controller/backends/quota_validating_staging_log_rate_limit_calculator_spec.rb b/spec/unit/lib/cloud_controller/backends/quota_validating_staging_log_rate_limit_calculator_spec.rb new file mode 100644 index 00000000000..c24e9af1c49 --- /dev/null +++ b/spec/unit/lib/cloud_controller/backends/quota_validating_staging_log_rate_limit_calculator_spec.rb @@ -0,0 +1,68 @@ +require 'spec_helper' +require 'cloud_controller/backends/quota_validating_staging_log_rate_limit_calculator' + +module VCAP::CloudController + RSpec.describe QuotaValidatingStagingLogRateLimitCalculator do + let(:calculator) { QuotaValidatingStagingLogRateLimitCalculator.new } + + describe '#get_limit' do + let(:space_quota_limit) { 200 } + let(:org_quota_limit) { 200 } + let(:requested_limit) { 100 } + let(:space) { Space.make } + let(:org) { space.organization } + let(:space_quota_definition) { SpaceQuotaDefinition.make(organization: org, log_rate_limit: space_quota_limit) } + let(:quota_definition) { QuotaDefinition.make(log_rate_limit: org_quota_limit) } + + before do + space.space_quota_definition = space_quota_definition + org.quota_definition = quota_definition + space.save + org.save + end + + it 'uses the requested_limit' do + limit = calculator.get_limit(requested_limit, space, org) + expect(limit).to eq(requested_limit) + end + + context 'when the requested_limit is passed as an integer string' do + let(:requested_limit) { '100' } + + it 'uses the requested_limit' do + limit = calculator.get_limit(requested_limit, space, org) + expect(limit).to eq(100) + end + end + + context 'when the requested_limit exceeds the space quota' do + let(:space_quota_limit) { requested_limit - 1 } + + it 'raises a SpaceQuotaExceeded error' do + expect { + calculator.get_limit(requested_limit, space, org) + }.to raise_error(QuotaValidatingStagingLogRateLimitCalculator::SpaceQuotaExceeded, /staging requires 100 bytes per second/) + end + end + + context 'when the requested_limit exceeds the org quota' do + let(:org_quota_limit) { requested_limit - 1 } + + it 'raises a OrgQuotaExceeded error' do + expect { + calculator.get_limit(requested_limit, space, org) + }.to raise_error(QuotaValidatingStagingLogRateLimitCalculator::OrgQuotaExceeded, /staging requires 100 bytes per second/) + end + end + + context 'when the requested_limit is nil' do + let(:requested_limit) { nil } + + it 'uses no limit' do + limit = calculator.get_limit(requested_limit, space, org) + expect(limit).to eq(-1) + end + end + end + end +end diff --git a/spec/unit/lib/cloud_controller/diego/task_recipe_builder_spec.rb b/spec/unit/lib/cloud_controller/diego/task_recipe_builder_spec.rb index 7fdda5cbd51..952a53825ed 100644 --- a/spec/unit/lib/cloud_controller/diego/task_recipe_builder_spec.rb +++ b/spec/unit/lib/cloud_controller/diego/task_recipe_builder_spec.rb @@ -14,6 +14,7 @@ module Diego details.environment_variables = [::Diego::Bbs::Models::EnvironmentVariable.new(name: 'nightshade_fruit', value: 'potato')] details.staging_memory_in_mb = 42 details.staging_disk_in_mb = 51 + details.staging_log_rate_limit_bytes_per_second = 67 details.start_after_staging = true details.lifecycle = lifecycle details.isolation_segment = isolation_segment @@ -152,6 +153,7 @@ module Diego expect(result.memory_mb).to eq(42) expect(result.disk_mb).to eq(51) + expect(result.log_rate_limit_bytes_per_second).to eq(67) expect(result.image_layers).to eq(lifecycle_image_layers) expect(result.cpu_weight).to eq(50) diff --git a/spec/unit/messages/build_create_message_spec.rb b/spec/unit/messages/build_create_message_spec.rb index e3e46df6210..a9d6b1cc46b 100644 --- a/spec/unit/messages/build_create_message_spec.rb +++ b/spec/unit/messages/build_create_message_spec.rb @@ -211,6 +211,35 @@ module VCAP::CloudController end end + describe '#staging_log_rate_limit_bytes_per_second' do + subject(:build_create_message) { BuildCreateMessage.new(params) } + let(:params) { { staging_log_rate_limit_bytes_per_second: -1 } } + + it 'returns the staging_log_rate_limit_bytes_per_second' do + expect(build_create_message.staging_log_rate_limit_bytes_per_second).to eq(-1) + end + + context 'when not provided' do + let(:params) { nil } + + it 'returns nil' do + expect(build_create_message.staging_log_rate_limit_bytes_per_second).to eq(nil) + end + end + + context 'when the value is out of bounds' do + let(:params) do + { + package: { guid: 'some-guid' }, + staging_log_rate_limit_bytes_per_second: -2 + } + end + it 'is invalid' do + expect(build_create_message.valid?).to be false + end + end + end + describe '#environment variables' do subject(:build_create_message) { BuildCreateMessage.new(params) } diff --git a/spec/unit/presenters/v3/build_presenter_spec.rb b/spec/unit/presenters/v3/build_presenter_spec.rb index 11f21d900cd..3c66ef5e2d7 100644 --- a/spec/unit/presenters/v3/build_presenter_spec.rb +++ b/spec/unit/presenters/v3/build_presenter_spec.rb @@ -15,6 +15,7 @@ module VCAP::CloudController::Presenters::V3 app: app, staging_memory_in_mb: 1024, staging_disk_in_mb: 1024, + staging_log_rate_limit: 2048, created_by_user_guid: 'happy user guid', created_by_user_name: 'happier user name', created_by_user_email: 'this user emailed in' @@ -46,6 +47,7 @@ module VCAP::CloudController::Presenters::V3 expect(result[:staging_memory_in_mb]).to eq(1024) expect(result[:staging_disk_in_mb]).to eq(1024) + expect(result[:staging_log_rate_limit_bytes_per_second]).to eq(2048) expect(result[:created_at]).to be_a(Time) expect(result[:updated_at]).to be_a(Time) From 230e6a57e6344da3fd994b772ae850e48e2243f4 Mon Sep 17 00:00:00 2001 From: Andrew Crump Date: Tue, 19 Jul 2022 18:42:07 +0000 Subject: [PATCH 07/20] Wrap log rate limit quota errors [#182311441](https://www.pivotaltracker.com/story/show/182311441) https://github.com/cloudfoundry/capi-release/issues/245 Co-authored-by: Rebecca Roberts --- app/actions/build_create.rb | 16 ++++++-- app/actions/v2/app_stage.rb | 8 +++- app/controllers/v3/builds_controller.rb | 8 +++- errors/v2.yml | 10 +++++ spec/unit/actions/build_create_spec.rb | 22 ++++++++++ spec/unit/actions/v2/app_stage_spec.rb | 34 ++++++++++++++-- .../controllers/v3/builds_controller_spec.rb | 40 +++++++++++++++++-- 7 files changed, 122 insertions(+), 16 deletions(-) diff --git a/app/actions/build_create.rb b/app/actions/build_create.rb index b8c02e53bb5..8bce29f0571 100644 --- a/app/actions/build_create.rb +++ b/app/actions/build_create.rb @@ -12,12 +12,16 @@ class BuildError < StandardError end class InvalidPackage < BuildError end - class SpaceQuotaExceeded < BuildError + class MemorySpaceQuotaExceeded < BuildError end - class OrgQuotaExceeded < BuildError + class MemoryOrgQuotaExceeded < BuildError end class DiskLimitExceeded < BuildError end + class LogRateLimitSpaceQuotaExceeded < BuildError + end + class LogRateLimitOrgQuotaExceeded < BuildError + end class StagingInProgress < BuildError end @@ -152,14 +156,18 @@ def get_memory_limit(requested_limit, app, space, org) limit = requested_limit || app.newest_web_process&.memory @memory_limit_calculator.get_limit(limit, space, org) rescue QuotaValidatingStagingMemoryCalculator::SpaceQuotaExceeded => e - raise SpaceQuotaExceeded.new e.message + raise MemorySpaceQuotaExceeded.new e.message rescue QuotaValidatingStagingMemoryCalculator::OrgQuotaExceeded => e - raise OrgQuotaExceeded.new e.message + raise MemoryOrgQuotaExceeded.new e.message end def get_log_rate_limit(requested_limit, app, space, org) limit = requested_limit || app.newest_web_process&.log_rate_limit @log_rate_limit_calculator.get_limit(limit, space, org) + rescue QuotaValidatingStagingLogRateLimitCalculator::SpaceQuotaExceeded => e + raise LogRateLimitSpaceQuotaExceeded.new e.message + rescue QuotaValidatingStagingLogRateLimitCalculator::OrgQuotaExceeded => e + raise LogRateLimitOrgQuotaExceeded.new e.message end def logger diff --git a/app/actions/v2/app_stage.rb b/app/actions/v2/app_stage.rb index b46e220ddff..c9f2426cfcb 100644 --- a/app/actions/v2/app_stage.rb +++ b/app/actions/v2/app_stage.rb @@ -40,12 +40,16 @@ def stage(process) process.last_stager_response = build_creator.staging_response rescue Diego::Runner::CannotCommunicateWithDiegoError => e logger.error("failed communicating with diego backend: #{e.message}") - rescue BuildCreate::SpaceQuotaExceeded => e + rescue BuildCreate::MemorySpaceQuotaExceeded => e raise CloudController::Errors::ApiError.new_from_details('SpaceQuotaMemoryLimitExceeded', e.message) - rescue BuildCreate::OrgQuotaExceeded => e + rescue BuildCreate::MemoryOrgQuotaExceeded => e raise CloudController::Errors::ApiError.new_from_details('AppMemoryQuotaExceeded', e.message) rescue BuildCreate::DiskLimitExceeded raise CloudController::Errors::ApiError.new_from_details('AppInvalid', 'too much disk requested') + rescue BuildCreate::LogRateLimitSpaceQuotaExceeded => e + raise CloudController::Errors::ApiError.new_from_details('SpaceQuotaLogRateLimitExceeded', e.message) + rescue BuildCreate::LogRateLimitOrgQuotaExceeded => e + raise CloudController::Errors::ApiError.new_from_details('OrgQuotaLogRateLimitExceeded', e.message) rescue BuildCreate::BuildError => e raise CloudController::Errors::ApiError.new_from_details('AppInvalid', e.message) end diff --git a/app/controllers/v3/builds_controller.rb b/app/controllers/v3/builds_controller.rb index 3195104dd69..74716b9e399 100644 --- a/app/controllers/v3/builds_controller.rb +++ b/app/controllers/v3/builds_controller.rb @@ -60,12 +60,16 @@ def create render status: :created, json: Presenters::V3::BuildPresenter.new(build) rescue BuildCreate::InvalidPackage => e bad_request!(e.message) - rescue BuildCreate::SpaceQuotaExceeded => e + rescue BuildCreate::MemorySpaceQuotaExceeded => e unprocessable!("space's memory limit exceeded: #{e.message}") - rescue BuildCreate::OrgQuotaExceeded => e + rescue BuildCreate::MemoryOrgQuotaExceeded => e unprocessable!("organization's memory limit exceeded: #{e.message}") rescue BuildCreate::DiskLimitExceeded unprocessable!('disk limit exceeded') + rescue BuildCreate::LogRateLimitSpaceQuotaExceeded => e + unprocessable!("space's log rate limit exceeded: #{e.message}") + rescue BuildCreate::LogRateLimitOrgQuotaExceeded => e + unprocessable!("organization's log rate limit exceeded: #{e.message}") rescue BuildCreate::StagingInProgress raise CloudController::Errors::ApiError.new_from_details('StagingInProgress') rescue BuildCreate::BuildError => e diff --git a/errors/v2.yml b/errors/v2.yml index 48d3f2058f9..3e18ca1c9bc 100644 --- a/errors/v2.yml +++ b/errors/v2.yml @@ -473,6 +473,11 @@ http_code: 400 message: "The requested memory allocation is not large enough to run all of your sidecar processes." +100010: + name: OrgQuotaLogRateLimitExceeded + http_code: 400 + message: "You have exceeded your organization's log rate limit: %s" + 110001: name: ServicePlanInvalid http_code: 400 @@ -1118,6 +1123,11 @@ http_code: 400 message: "You have exceeded the total reserved route ports for your space's quota." +310011: + name: SpaceQuotaLogRateLimitExceeded + http_code: 400 + message: "You have exceeded your space's log rate limit: %s" + 320001: name: DiegoDisabled http_code: 400 diff --git a/spec/unit/actions/build_create_spec.rb b/spec/unit/actions/build_create_spec.rb index 7b60d748683..cb4752ef97b 100644 --- a/spec/unit/actions/build_create_spec.rb +++ b/spec/unit/actions/build_create_spec.rb @@ -365,6 +365,28 @@ module VCAP::CloudController end end + context 'when the org quota is exceeded' do + before do + allow(log_rate_limit_calculator).to receive(:get_limit).and_raise(QuotaValidatingStagingLogRateLimitCalculator::OrgQuotaExceeded, 'some-message') + end + it 'raises a LogRateLimitOrgQuotaExceeded error' do + expect { + action.create_and_stage(package: package, lifecycle: lifecycle, metadata: metadata) + }.to raise_error(::VCAP::CloudController::BuildCreate::LogRateLimitOrgQuotaExceeded, 'some-message') + end + end + + context 'when the space quota is exceeded' do + before do + allow(log_rate_limit_calculator).to receive(:get_limit).and_raise(QuotaValidatingStagingLogRateLimitCalculator::SpaceQuotaExceeded, 'some-message') + end + it 'raises a LogRateLimitSpaceQuotaExceeded error' do + expect { + action.create_and_stage(package: package, lifecycle: lifecycle, metadata: metadata) + }.to raise_error(::VCAP::CloudController::BuildCreate::LogRateLimitSpaceQuotaExceeded, 'some-message') + end + end + describe 'using custom buildpacks' do let!(:app) { AppModel.make(space: space) } diff --git a/spec/unit/actions/v2/app_stage_spec.rb b/spec/unit/actions/v2/app_stage_spec.rb index cf8ecfc6243..9ee71ece3e2 100644 --- a/spec/unit/actions/v2/app_stage_spec.rb +++ b/spec/unit/actions/v2/app_stage_spec.rb @@ -100,10 +100,10 @@ module V2 end end - context 'when SpaceQuotaExceeded error is raised' do + context 'when MemorySpaceQuotaExceeded error is raised' do before do allow(build_create).to receive(:create_and_stage_without_event).and_raise( - BuildCreate::SpaceQuotaExceeded.new('helpful message') + BuildCreate::MemorySpaceQuotaExceeded.new('helpful message') ) end @@ -115,10 +115,10 @@ module V2 end end - context 'when OrgQuotaExceeded error is raised' do + context 'when MemoryOrgQuotaExceeded error is raised' do before do allow(build_create).to receive(:create_and_stage_without_event).and_raise( - BuildCreate::OrgQuotaExceeded.new('helpful message') + BuildCreate::MemoryOrgQuotaExceeded.new('helpful message') ) end @@ -141,6 +141,32 @@ module V2 end end end + + context 'when LogRateLimitSpaceQuotaExceeded error is raised' do + before do + allow(build_create).to receive(:create_and_stage_without_event).and_raise( + BuildCreate::LogRateLimitSpaceQuotaExceeded.new('helpful message') + ) + end + + it 'translates it to an ApiError' do + expect { action.stage(process) }.to(raise_error(CloudController::Errors::ApiError, + /helpful message/)) { |err| expect(err.details.name).to eq('SpaceQuotaLogRateLimitExceeded') } + end + end + + context 'when LogRateLimitOrgQuotaExceeded error is raised' do + before do + allow(build_create).to receive(:create_and_stage_without_event).and_raise( + BuildCreate::LogRateLimitOrgQuotaExceeded.new('helpful message') + ) + end + + it 'translates it to an ApiError' do + expect { action.stage(process) }.to(raise_error(CloudController::Errors::ApiError, + /helpful message/)) { |err| expect(err.details.name).to eq('OrgQuotaLogRateLimitExceeded') } + end + end end context 'telemetry' do diff --git a/spec/unit/controllers/v3/builds_controller_spec.rb b/spec/unit/controllers/v3/builds_controller_spec.rb index 263dc929717..e1900af52fb 100644 --- a/spec/unit/controllers/v3/builds_controller_spec.rb +++ b/spec/unit/controllers/v3/builds_controller_spec.rb @@ -521,10 +521,10 @@ end end - context 'when the space quota is exceeded' do + context 'when the space memory quota is exceeded' do before do allow(build_create).to receive(:create_and_stage).and_raise( - VCAP::CloudController::BuildCreate::SpaceQuotaExceeded.new('helpful message') + VCAP::CloudController::BuildCreate::MemorySpaceQuotaExceeded.new('helpful message') ) end @@ -537,10 +537,10 @@ end end - context 'when the org quota is exceeded' do + context 'when the org memory quota is exceeded' do before do allow(build_create).to receive(:create_and_stage).and_raise( - VCAP::CloudController::BuildCreate::OrgQuotaExceeded.new('helpful message') + VCAP::CloudController::BuildCreate::MemoryOrgQuotaExceeded.new('helpful message') ) end @@ -565,6 +565,38 @@ expect(response.body).to include('disk limit exceeded') end end + + context 'when the space log rate limit quota is exceeded' do + before do + allow(build_create).to receive(:create_and_stage).and_raise( + VCAP::CloudController::BuildCreate::LogRateLimitSpaceQuotaExceeded.new('helpful message') + ) + end + + it 'returns 422 Unprocessable' do + post :create, params: req_body, as: :json + + expect(response.status).to eq(422) + expect(response.body).to include("space's log rate limit exceeded") + expect(response.body).to include('helpful message') + end + end + + context 'when the org log rate limit quota is exceeded' do + before do + allow(build_create).to receive(:create_and_stage).and_raise( + VCAP::CloudController::BuildCreate::LogRateLimitOrgQuotaExceeded.new('helpful message') + ) + end + + it 'returns 422 Unprocessable' do + post :create, params: req_body, as: :json + + expect(response.status).to eq(422) + expect(response.body).to include("organization's log rate limit exceeded") + expect(response.body).to include('helpful message') + end + end end describe 'metadata' do From a19a6ffb5e6051d891c321cde852472d832ac733 Mon Sep 17 00:00:00 2001 From: Rebecca Roberts Date: Thu, 21 Jul 2022 21:22:50 +0000 Subject: [PATCH 08/20] Change log_rate_limit from an int to MessageType - Found that if there is a new version of Diego and an old version of cloud controller, the field would not be provided and defaulted to 0. - Wrapped the integer in a MessageType so it would default to null. Co-authored-by: Duane May --- .../diego/app_recipe_builder.rb | 2 +- .../diego/task_recipe_builder.rb | 4 ++-- lib/diego/bbs/models/desired_lrp_pb.rb | 5 +++-- lib/diego/bbs/models/log_rate_limit_pb.rb | 18 ++++++++++++++++++ lib/diego/bbs/models/task_pb.rb | 3 ++- .../diego/app_recipe_builder_spec.rb | 4 ++-- .../diego/task_recipe_builder_spec.rb | 6 +++--- 7 files changed, 31 insertions(+), 11 deletions(-) create mode 100644 lib/diego/bbs/models/log_rate_limit_pb.rb diff --git a/lib/cloud_controller/diego/app_recipe_builder.rb b/lib/cloud_controller/diego/app_recipe_builder.rb index 26d4da69e2c..cb5add91d94 100644 --- a/lib/cloud_controller/diego/app_recipe_builder.rb +++ b/lib/cloud_controller/diego/app_recipe_builder.rb @@ -68,7 +68,7 @@ def app_lrp_arguments start_timeout_ms: health_check_timeout_in_seconds * 1000, disk_mb: process.disk_quota, memory_mb: process.memory, # sums up - log_rate_limit_bytes_per_second: process.log_rate_limit, + log_rate_limit: ::Diego::Bbs::Models::LogRateLimit.new(bytes_per_second: process.log_rate_limit), privileged: desired_lrp_builder.privileged?, ports: ports, log_source: LRP_LOG_SOURCE, diff --git a/lib/cloud_controller/diego/task_recipe_builder.rb b/lib/cloud_controller/diego/task_recipe_builder.rb index 3a0307bb171..a73c71400d1 100644 --- a/lib/cloud_controller/diego/task_recipe_builder.rb +++ b/lib/cloud_controller/diego/task_recipe_builder.rb @@ -26,7 +26,7 @@ def build_app_task(config, task) disk_mb: task.disk_in_mb, egress_rules: @egress_rules.running_protobuf_rules(task.app), log_guid: task.app.guid, - log_rate_limit_bytes_per_second: task.log_rate_limit, + log_rate_limit: ::Diego::Bbs::Models::LogRateLimit.new(bytes_per_second: task.log_rate_limit), log_source: TASK_LOG_SOURCE, max_pids: config.get(:diego, :pid_limit), memory_mb: task.memory_in_mb, @@ -65,7 +65,7 @@ def build_staging_task(config, staging_details) log_guid: staging_details.package.app_guid, log_source: STAGING_LOG_SOURCE, memory_mb: staging_details.staging_memory_in_mb, - log_rate_limit_bytes_per_second: staging_details.staging_log_rate_limit_bytes_per_second, + log_rate_limit: ::Diego::Bbs::Models::LogRateLimit.new(bytes_per_second: staging_details.staging_log_rate_limit_bytes_per_second), network: generate_network(staging_details.package, Protocol::ContainerNetworkInfo::STAGING), privileged: config.get(:diego, :use_privileged_containers_for_staging), result_file: STAGING_RESULT_FILE, diff --git a/lib/diego/bbs/models/desired_lrp_pb.rb b/lib/diego/bbs/models/desired_lrp_pb.rb index f4397f6a629..e73bdae1094 100644 --- a/lib/diego/bbs/models/desired_lrp_pb.rb +++ b/lib/diego/bbs/models/desired_lrp_pb.rb @@ -15,6 +15,7 @@ require 'image_layer_pb' require 'metric_tags_pb' require 'sidecar_pb' +require 'log_rate_limit_pb' Google::Protobuf::DescriptorPool.generated_pool.build do add_message "diego.bbs.models.DesiredLRPSchedulingInfo" do optional :desired_lrp_key, :message, 1, "diego.bbs.models.DesiredLRPKey" @@ -53,7 +54,7 @@ repeated :image_layers, :message, 24, "diego.bbs.models.ImageLayer" map :metric_tags, :string, :message, 25, "diego.bbs.models.MetricTagValue" repeated :sidecars, :message, 26, "diego.bbs.models.Sidecar" - optional :log_rate_limit_bytes_per_second, :int64, 27 + optional :log_rate_limit, :message, 27, "diego.bbs.models.LogRateLimit" end add_message "diego.bbs.models.ProtoRoutes" do map :routes, :string, :bytes, 1 @@ -115,7 +116,7 @@ repeated :image_layers, :message, 34, "diego.bbs.models.ImageLayer" map :metric_tags, :string, :message, 35, "diego.bbs.models.MetricTagValue" repeated :sidecars, :message, 36, "diego.bbs.models.Sidecar" - optional :log_rate_limit_bytes_per_second, :int64, 37 + optional :log_rate_limit, :message, 37, "diego.bbs.models.LogRateLimit" end end diff --git a/lib/diego/bbs/models/log_rate_limit_pb.rb b/lib/diego/bbs/models/log_rate_limit_pb.rb new file mode 100644 index 00000000000..a549296f05c --- /dev/null +++ b/lib/diego/bbs/models/log_rate_limit_pb.rb @@ -0,0 +1,18 @@ +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: log_rate_limit.proto + +require 'google/protobuf' + +Google::Protobuf::DescriptorPool.generated_pool.build do + add_message "diego.bbs.models.LogRateLimit" do + optional :bytes_per_second, :int64, 1 + end +end + +module Diego + module Bbs + module Models + LogRateLimit = Google::Protobuf::DescriptorPool.generated_pool.lookup("diego.bbs.models.LogRateLimit").msgclass + end + end +end diff --git a/lib/diego/bbs/models/task_pb.rb b/lib/diego/bbs/models/task_pb.rb index ac30f329781..e4afd6f8041 100644 --- a/lib/diego/bbs/models/task_pb.rb +++ b/lib/diego/bbs/models/task_pb.rb @@ -11,6 +11,7 @@ require 'network_pb' require 'certificate_properties_pb' require 'image_layer_pb' +require 'log_rate_limit_pb' Google::Protobuf::DescriptorPool.generated_pool.build do add_message "diego.bbs.models.TaskDefinition" do optional :root_fs, :string, 1 @@ -38,7 +39,7 @@ optional :image_username, :string, 23 optional :image_password, :string, 24 repeated :image_layers, :message, 25, "diego.bbs.models.ImageLayer" - optional :log_rate_limit_bytes_per_second, :int64, 26 + optional :log_rate_limit, :message, 26, "diego.bbs.models.LogRateLimit" end add_message "diego.bbs.models.Task" do optional :task_definition, :message, 1, "diego.bbs.models.TaskDefinition" diff --git a/spec/unit/lib/cloud_controller/diego/app_recipe_builder_spec.rb b/spec/unit/lib/cloud_controller/diego/app_recipe_builder_spec.rb index 2e1af40320b..872264c4100 100644 --- a/spec/unit/lib/cloud_controller/diego/app_recipe_builder_spec.rb +++ b/spec/unit/lib/cloud_controller/diego/app_recipe_builder_spec.rb @@ -286,7 +286,7 @@ module Diego expect(lrp.log_source).to eq(LRP_LOG_SOURCE) expect(lrp.max_pids).to eq(100) expect(lrp.memory_mb).to eq(128) - expect(lrp.log_rate_limit_bytes_per_second).to eq(1024) + expect(lrp.log_rate_limit.bytes_per_second).to eq(1024) expect(lrp.metrics_guid).to eq(process.app.guid) expect(lrp.metric_tags.keys.size).to eq(11) @@ -874,7 +874,7 @@ module Diego expect(lrp.log_guid).to eq(process.app.guid) expect(lrp.max_pids).to eq(100) expect(lrp.memory_mb).to eq(128) - expect(lrp.log_rate_limit_bytes_per_second).to eq(1024) + expect(lrp.log_rate_limit.bytes_per_second).to eq(1024) expect(lrp.metrics_guid).to eq(process.app.guid) expect(lrp.metric_tags.keys.size).to eq(11) diff --git a/spec/unit/lib/cloud_controller/diego/task_recipe_builder_spec.rb b/spec/unit/lib/cloud_controller/diego/task_recipe_builder_spec.rb index 952a53825ed..8d5315f5ef1 100644 --- a/spec/unit/lib/cloud_controller/diego/task_recipe_builder_spec.rb +++ b/spec/unit/lib/cloud_controller/diego/task_recipe_builder_spec.rb @@ -153,7 +153,7 @@ module Diego expect(result.memory_mb).to eq(42) expect(result.disk_mb).to eq(51) - expect(result.log_rate_limit_bytes_per_second).to eq(67) + expect(result.log_rate_limit.bytes_per_second).to eq(67) expect(result.image_layers).to eq(lifecycle_image_layers) expect(result.cpu_weight).to eq(50) @@ -471,7 +471,7 @@ module Diego expect(result.log_guid).to eq(app.guid) expect(result.memory_mb).to eq(2048) expect(result.disk_mb).to eq(1024) - expect(result.log_rate_limit_bytes_per_second).to eq(3072) + expect(result.log_rate_limit.bytes_per_second).to eq(3072) expect(result.environment_variables).to eq(lifecycle_environment_variables) expect(result.legacy_download_user).to eq('vcap') expect(result.root_fs).to eq('preloaded:potato-stack') @@ -614,7 +614,7 @@ module Diego expect(result.disk_mb).to eq(1024) expect(result.memory_mb).to eq(2048) - expect(result.log_rate_limit_bytes_per_second).to eq(3072) + expect(result.log_rate_limit.bytes_per_second).to eq(3072) expect(result.log_guid).to eq(app.guid) expect(result.privileged).to be(false) expect(result.egress_rules).to eq([ From 5f0da2eaadc0f45a9fc0071e6dede32c71d65216 Mon Sep 17 00:00:00 2001 From: Ben Fuller Date: Wed, 10 Aug 2022 20:35:25 +0000 Subject: [PATCH 09/20] Add Log Rate container metrics and remove Traffic Controller support * Add Log Rate container metrics. These metrics allow users to see how much each application is logging. * Remove support for getting metrics from Traffic Controller. Log Cache has been the default for retreiving metrics for a long time. Removing "temporary" config flag and support for the old way of connecting to Traffic Controller for metrics We did the two above items together because we did not want to add support for getting the new container metrics from Traffic Controller. The code was manually creating Traffic Controller protobufs and shoving Log Cache data into them, which we had to move away from to make this work without delving into Traffic Controller protobufs. Signed-off-by: Duane May Co-authored-by: Duane May Signed-off-by: Matthew Kocher Co-authored-by: Matthew Kocher Signed-off-by: Ben Fuller Co-authored-by: Ben Fuller --- config/cloud_controller.yml | 1 - .../backends/instances_reporters.rb | 7 +- .../config_schemas/base/api_schema.rb | 1 - lib/cloud_controller/dependency_locator.rb | 17 +- .../reporters/instances_stats_reporter.rb | 28 +- lib/logcache/container_metric_batch.rb | 7 + ...corator.rb => container_metric_batcher.rb} | 43 ++-- lib/traffic_controller/client.rb | 71 ------ lib/traffic_controller/errors.rb | 16 -- lib/traffic_controller/models/envelope.pb.rb | 59 ----- lib/traffic_controller/models/error.pb.rb | 24 -- lib/traffic_controller/models/http.pb.rb | 95 ------- lib/traffic_controller/models/log.pb.rb | 32 --- lib/traffic_controller/models/metric.pb.rb | 47 ---- lib/traffic_controller/models/uuid.pb.rb | 23 -- lib/traffic_controller/traffic_controller.rb | 8 - lib/utils/multipart_parser_wrapper.rb | 45 ---- scripts/cf-release/restore-bosh-lite-cc.sh | 1 - scripts/cf-release/setup-local-cc.sh | 4 - scripts/generate-traffic-controller-models.sh | 36 --- scripts/short-circuit-cc | 1 - ...ec.rb => container_metric_batcher_spec.rb} | 240 ++++++++---------- spec/traffic_controller/client_spec.rb | 101 -------- .../backends/instances_reporters_spec.rb | 31 +-- .../dependency_locator_spec.rb | 12 +- .../instances_stats_reporter_spec.rb | 105 ++++---- .../utils/multipart_parser_wrapper_spec.rb | 65 ----- 27 files changed, 218 insertions(+), 902 deletions(-) create mode 100644 lib/logcache/container_metric_batch.rb rename lib/logcache/{traffic_controller_decorator.rb => container_metric_batcher.rb} (69%) delete mode 100644 lib/traffic_controller/client.rb delete mode 100644 lib/traffic_controller/errors.rb delete mode 100644 lib/traffic_controller/models/envelope.pb.rb delete mode 100644 lib/traffic_controller/models/error.pb.rb delete mode 100644 lib/traffic_controller/models/http.pb.rb delete mode 100644 lib/traffic_controller/models/log.pb.rb delete mode 100644 lib/traffic_controller/models/metric.pb.rb delete mode 100644 lib/traffic_controller/models/uuid.pb.rb delete mode 100644 lib/traffic_controller/traffic_controller.rb delete mode 100644 lib/utils/multipart_parser_wrapper.rb delete mode 100755 scripts/generate-traffic-controller-models.sh rename spec/logcache/{traffic_controller_decorator_spec.rb => container_metric_batcher_spec.rb} (75%) delete mode 100644 spec/traffic_controller/client_spec.rb delete mode 100644 spec/unit/lib/utils/multipart_parser_wrapper_spec.rb diff --git a/config/cloud_controller.yml b/config/cloud_controller.yml index 3609f71c879..008f76b658a 100644 --- a/config/cloud_controller.yml +++ b/config/cloud_controller.yml @@ -13,7 +13,6 @@ readiness_port: external_protocol: http external_domain: api2.vcap.me temporary_disable_deployments: true -temporary_use_logcache: false deployment_updater: update_frequency_in_seconds: 30 lock_key: 'cf-deployment-updater' diff --git a/lib/cloud_controller/backends/instances_reporters.rb b/lib/cloud_controller/backends/instances_reporters.rb index f39328d2c02..5199fee7cdf 100644 --- a/lib/cloud_controller/backends/instances_reporters.rb +++ b/lib/cloud_controller/backends/instances_reporters.rb @@ -38,12 +38,7 @@ def diego_reporter end def diego_stats_reporter - client = if Config.config.get(:temporary_use_logcache) - dependency_locator.traffic_controller_compatible_logcache_client - else - dependency_locator.traffic_controller_client - end - Diego::InstancesStatsReporter.new(dependency_locator.bbs_instances_client, client) + Diego::InstancesStatsReporter.new(dependency_locator.bbs_instances_client, dependency_locator.log_cache_metrics_client) end def dependency_locator diff --git a/lib/cloud_controller/config_schemas/base/api_schema.rb b/lib/cloud_controller/config_schemas/base/api_schema.rb index 157e86b6f09..d1f6903acef 100644 --- a/lib/cloud_controller/config_schemas/base/api_schema.rb +++ b/lib/cloud_controller/config_schemas/base/api_schema.rb @@ -11,7 +11,6 @@ class ApiSchema < VCAP::Config external_port: Integer, external_domain: String, temporary_disable_deployments: bool, - temporary_use_logcache: bool, optional(:temporary_disable_v2_staging) => bool, tls_port: Integer, external_protocol: String, diff --git a/lib/cloud_controller/dependency_locator.rb b/lib/cloud_controller/dependency_locator.rb index db2e77cd4e6..3b031d2e57c 100644 --- a/lib/cloud_controller/dependency_locator.rb +++ b/lib/cloud_controller/dependency_locator.rb @@ -12,9 +12,8 @@ require 'cloud_controller/blob_sender/nginx_blob_sender' require 'cloud_controller/blob_sender/default_blob_sender' require 'cloud_controller/blob_sender/missing_blob_handler' -require 'traffic_controller/client' require 'logcache/client' -require 'logcache/traffic_controller_decorator' +require 'logcache/container_metric_batcher' require 'cloud_controller/diego/task_recipe_builder' require 'cloud_controller/diego/app_recipe_builder' require 'cloud_controller/diego/bbs_apps_client' @@ -102,17 +101,13 @@ def bbs_instances_client @dependencies[:bbs_instances_client] || register(:bbs_instances_client, build_instances_client) end - def traffic_controller_client - @dependencies[:traffic_controller_client] || register(:traffic_controller_client, build_traffic_controller_client) - end - def logcache_client @dependencies[:logcache_client] || register(:logcache_client, build_logcache_client) end - def traffic_controller_compatible_logcache_client - @dependencies[:traffic_controller_compatible_logcache_client] || - register(:traffic_controller_compatible_logcache_client, Logcache::TrafficControllerDecorator.new(logcache_client)) + def log_cache_metrics_client + @dependencies[:log_cache_metrics_client] || + register(:log_cache_metrics_client, Logcache::ContainerMetricBatcher.new(logcache_client)) end def upload_handler @@ -516,10 +511,6 @@ def build_bbs_client ) end - def build_traffic_controller_client - TrafficController::Client.new(url: config.get(:loggregator, :internal_url)) - end - def build_logcache_client Logcache::Client.new( host: config.get(:logcache, :host), diff --git a/lib/cloud_controller/diego/reporters/instances_stats_reporter.rb b/lib/cloud_controller/diego/reporters/instances_stats_reporter.rb index a37352776b6..2fe140f1aef 100644 --- a/lib/cloud_controller/diego/reporters/instances_stats_reporter.rb +++ b/lib/cloud_controller/diego/reporters/instances_stats_reporter.rb @@ -1,4 +1,3 @@ -require 'traffic_controller/client' require 'logcache/client' require 'cloud_controller/diego/reporters/reporter_mixins' @@ -70,14 +69,16 @@ def stats_for_app(process) def metrics_data_for_instance(stats, quota_stats, log_cache_errors, formatted_current_time, index) if log_cache_errors.blank? { - mem_quota: quota_stats[index]&.memoryBytesQuota, - disk_quota: quota_stats[index]&.diskBytesQuota, + mem_quota: quota_stats[index]&.memory_bytes_quota, + disk_quota: quota_stats[index]&.disk_bytes_quota, + log_rate_limit: quota_stats[index]&.log_rate_limit, usage: stats[index] || missing_process_stats(formatted_current_time) } else { mem_quota: nil, disk_quota: nil, + log_rate_limit: nil, usage: {} } end @@ -89,6 +90,7 @@ def missing_process_stats(formatted_current_time) cpu: 0, mem: 0, disk: 0, + log_rate: 0, } end @@ -96,8 +98,8 @@ def formatted_process_stats(log_cache_data, formatted_current_time) log_cache_data. map { |e| [ - e.containerMetric.instanceIndex, - converted_container_metrics(e.containerMetric, formatted_current_time), + e.instance_index, + converted_container_metrics(e, formatted_current_time), ] }.to_h end @@ -106,8 +108,8 @@ def formatted_quota_stats(log_cache_data) log_cache_data. map { |e| [ - e.containerMetric.instanceIndex, - e.containerMetric, + e.instance_index, + e ] }.to_h end @@ -136,18 +138,20 @@ def logger end def converted_container_metrics(container_metrics, formatted_current_time) - cpu = container_metrics.cpuPercentage - mem = container_metrics.memoryBytes - disk = container_metrics.diskBytes + cpu = container_metrics.cpu_percentage + mem = container_metrics.memory_bytes + disk = container_metrics.disk_bytes + log_rate = container_metrics.log_rate - if cpu.nil? || mem.nil? || disk.nil? + if cpu.nil? || mem.nil? || disk.nil? || log_rate.nil? missing_process_stats(formatted_current_time) else { time: formatted_current_time, cpu: cpu / 100, mem: mem, - disk: disk + disk: disk, + log_rate: log_rate } end end diff --git a/lib/logcache/container_metric_batch.rb b/lib/logcache/container_metric_batch.rb new file mode 100644 index 00000000000..a3f4cf8bc6b --- /dev/null +++ b/lib/logcache/container_metric_batch.rb @@ -0,0 +1,7 @@ +module Logcache + class ContainerMetricBatch + attr_accessor :cpu_percentage, :memory_bytes, :disk_bytes, :log_rate, + :disk_bytes_quota, :memory_bytes_quota, :log_rate_limit, + :instance_index + end +end diff --git a/lib/logcache/traffic_controller_decorator.rb b/lib/logcache/container_metric_batcher.rb similarity index 69% rename from lib/logcache/traffic_controller_decorator.rb rename to lib/logcache/container_metric_batcher.rb index d6378992fb8..456572c6df0 100644 --- a/lib/logcache/traffic_controller_decorator.rb +++ b/lib/logcache/container_metric_batcher.rb @@ -1,8 +1,9 @@ require 'logcache/client' require 'utils/time_utils' +require 'logcache/container_metric_batch' module Logcache - class TrafficControllerDecorator + class ContainerMetricBatcher MAX_REQUEST_COUNT = 100 def initialize(logcache_client) @@ -40,7 +41,7 @@ def container_metrics(auth_token: nil, source_guid:, logcache_filter:) uniq { |e| e.gauge.metrics.keys << e.instance_id }. sort_by(&:instance_id). chunk(&:instance_id). - map { |envelopes_by_instance| convert_to_traffic_controller_envelope(source_guid, envelopes_by_instance) } + map { |envelopes_by_instance| batch_metrics(source_guid, envelopes_by_instance) } end private @@ -66,47 +67,45 @@ def has_container_metrics_fields?(envelope) envelope.gauge.metrics.has_key?('memory') || envelope.gauge.metrics.has_key?('memory_quota') || envelope.gauge.metrics.has_key?('disk') || - envelope.gauge.metrics.has_key?('disk_quota') + envelope.gauge.metrics.has_key?('disk_quota') || + envelope.gauge.metrics.has_key?('log_rate') || + envelope.gauge.metrics.has_key?('log_rate_limit') # rubocop:enable Style/PreferredHashMethods end - def convert_to_traffic_controller_envelope(source_guid, envelopes_by_instance) - tc_envelope = TrafficController::Models::Envelope.new( - containerMetric: TrafficController::Models::ContainerMetric.new({ - applicationId: source_guid, - instanceIndex: envelopes_by_instance.first, - }), - ) + def batch_metrics(source_guid, envelopes_by_instance) + metric_batch = ContainerMetricBatch.new + metric_batch.instance_index = envelopes_by_instance.first.to_i - tags = {} envelopes_by_instance.second.each { |e| - tc_envelope.containerMetric.instanceIndex = e.instance_id # rubocop seems to think that there is a 'key?' method # on envelope.gauge.metrics - but it does not # rubocop:disable Style/PreferredHashMethods if e.gauge.metrics.has_key?('cpu') - tc_envelope.containerMetric.cpuPercentage = e.gauge.metrics['cpu'].value + metric_batch.cpu_percentage = e.gauge.metrics['cpu'].value end if e.gauge.metrics.has_key?('memory') - tc_envelope.containerMetric.memoryBytes = e.gauge.metrics['memory'].value + metric_batch.memory_bytes = e.gauge.metrics['memory'].value.to_i end if e.gauge.metrics.has_key?('disk') - tc_envelope.containerMetric.diskBytes = e.gauge.metrics['disk'].value + metric_batch.disk_bytes = e.gauge.metrics['disk'].value.to_i + end + if e.gauge.metrics.has_key?('log_rate') + metric_batch.log_rate = e.gauge.metrics['log_rate'].value.to_i end if e.gauge.metrics.has_key?('disk_quota') - tc_envelope.containerMetric.diskBytesQuota = e.gauge.metrics['disk_quota'].value + metric_batch.disk_bytes_quota = e.gauge.metrics['disk_quota'].value.to_i end if e.gauge.metrics.has_key?('memory_quota') - tc_envelope.containerMetric.memoryBytesQuota = e.gauge.metrics['memory_quota'].value + metric_batch.memory_bytes_quota = e.gauge.metrics['memory_quota'].value.to_i + end + if e.gauge.metrics.has_key?('log_rate_limit') + metric_batch.log_rate_limit = e.gauge.metrics['log_rate_limit'].value.to_i end # rubocop:enable Style/PreferredHashMethods - - tags.merge!(e.tags.to_h) } - tc_envelope.tags = tags.map { |k, v| TrafficController::Models::Envelope::TagsEntry.new(key: k, value: v) } - - tc_envelope + metric_batch end def logger diff --git a/lib/traffic_controller/client.rb b/lib/traffic_controller/client.rb deleted file mode 100644 index 63c96953a12..00000000000 --- a/lib/traffic_controller/client.rb +++ /dev/null @@ -1,71 +0,0 @@ -require 'traffic_controller/traffic_controller' -require 'traffic_controller/errors' -require 'utils/multipart_parser_wrapper' - -module TrafficController - class Client - BOUNDARY_REGEXP = /boundary=(.+)/.freeze - - def initialize(url:) - @url = url - end - - def container_metrics(auth_token:, source_guid:, logcache_filter: nil) - response = with_request_error_handling do - client.get("/apps/#{source_guid}/containermetrics", nil, { 'Authorization' => auth_token }) - end - - validate_status!(response: response, statuses: [200]) - - envelopes = [] - boundary = extract_boundary!(response.contenttype) - parser = VCAP::MultipartParserWrapper.new(body: response.body, boundary: boundary) - until (next_part = parser.next_part).nil? - envelopes << protobuf_decode!(next_part, Models::Envelope) - end - envelopes - end - - def with_request_error_handling(&blk) - tries ||= 3 - yield - rescue => e - retry unless (tries -= 1).zero? - raise RequestError.new(e.message) - end - - private - - attr_reader :url - - def extract_boundary!(content_type) - match_data = BOUNDARY_REGEXP.match(content_type) - raise ResponseError.new('failed to find multipart boundary in Content-Type header') if match_data.nil? - - match_data.captures.first - end - - def validate_status!(response:, statuses:) - raise ResponseError.new("failed with status: #{response.status}, body: #{response.body}") unless statuses.include?(response.status) - end - - def protobuf_decode!(message, protobuf_decoder) - protobuf_decoder.decode(message) - rescue => e - raise DecodeError.new(e.message) - end - - def client - @client ||= build_client - end - - def build_client - client = HTTPClient.new(base_url: url) - client.connect_timeout = 10 - client.send_timeout = 10 - client.receive_timeout = 10 - client.ssl_config.verify_mode = OpenSSL::SSL::VERIFY_NONE - client - end - end -end diff --git a/lib/traffic_controller/errors.rb b/lib/traffic_controller/errors.rb deleted file mode 100644 index 0deb07e55f7..00000000000 --- a/lib/traffic_controller/errors.rb +++ /dev/null @@ -1,16 +0,0 @@ -module TrafficController - class Error < StandardError - end - - class RequestError < Error - end - - class ResponseError < Error - end - - class DecodeError < Error - end - - class ParseError < Error - end -end diff --git a/lib/traffic_controller/models/envelope.pb.rb b/lib/traffic_controller/models/envelope.pb.rb deleted file mode 100644 index 76e4a97a053..00000000000 --- a/lib/traffic_controller/models/envelope.pb.rb +++ /dev/null @@ -1,59 +0,0 @@ -# encoding: utf-8 - -## -# This file is auto-generated. DO NOT EDIT! -# -require 'protobuf/message' - -## -# Imports -# -require 'http.pb' -require 'log.pb' -require 'metric.pb' -require 'error.pb' - -module TrafficController - module Models - ## - # Message Classes - # - class Envelope < ::Protobuf::Message - class EventType < ::Protobuf::Enum - define :HttpStartStop, 4 - define :LogMessage, 5 - define :ValueMetric, 6 - define :CounterEvent, 7 - define :Error, 8 - define :ContainerMetric, 9 - end - - class TagsEntry < ::Protobuf::Message; end - end - - ## - # Message Fields - # - class Envelope - class TagsEntry - optional :string, :key, 1 - optional :string, :value, 2 - end - - required :string, :origin, 1 - required ::TrafficController::Models::Envelope::EventType, :eventType, 2 - optional :int64, :timestamp, 6 - optional :string, :deployment, 13 - optional :string, :job, 14 - optional :string, :index, 15 - optional :string, :ip, 16 - repeated ::TrafficController::Models::Envelope::TagsEntry, :tags, 17 - optional ::TrafficController::Models::HttpStartStop, :httpStartStop, 7 - optional ::TrafficController::Models::LogMessage, :logMessage, 8 - optional ::TrafficController::Models::ValueMetric, :valueMetric, 9 - optional ::TrafficController::Models::CounterEvent, :counterEvent, 10 - optional ::TrafficController::Models::Error, :error, 11 - optional ::TrafficController::Models::ContainerMetric, :containerMetric, 12 - end - end -end diff --git a/lib/traffic_controller/models/error.pb.rb b/lib/traffic_controller/models/error.pb.rb deleted file mode 100644 index 5caba72c6d0..00000000000 --- a/lib/traffic_controller/models/error.pb.rb +++ /dev/null @@ -1,24 +0,0 @@ -# encoding: utf-8 - -## -# This file is auto-generated. DO NOT EDIT! -# -require 'protobuf/message' - -module TrafficController - module Models - ## - # Message Classes - # - class Error < ::Protobuf::Message; end - - ## - # Message Fields - # - class Error - required :string, :source, 1 - required :int32, :code, 2 - required :string, :message, 3 - end - end -end diff --git a/lib/traffic_controller/models/http.pb.rb b/lib/traffic_controller/models/http.pb.rb deleted file mode 100644 index 750525f9cf9..00000000000 --- a/lib/traffic_controller/models/http.pb.rb +++ /dev/null @@ -1,95 +0,0 @@ -# encoding: utf-8 - -## -# This file is auto-generated. DO NOT EDIT! -# -require 'protobuf/message' - -## -# Imports -# -require 'uuid.pb' - -module TrafficController - module Models - ## - # Enum Classes - # - class PeerType < ::Protobuf::Enum - define :Client, 1 - define :Server, 2 - end - - class Method < ::Protobuf::Enum - define :GET, 1 - define :POST, 2 - define :PUT, 3 - define :DELETE, 4 - define :HEAD, 5 - define :ACL, 6 - define :BASELINE_CONTROL, 7 - define :BIND, 8 - define :CHECKIN, 9 - define :CHECKOUT, 10 - define :CONNECT, 11 - define :COPY, 12 - define :DEBUG, 13 - define :LABEL, 14 - define :LINK, 15 - define :LOCK, 16 - define :MERGE, 17 - define :MKACTIVITY, 18 - define :MKCALENDAR, 19 - define :MKCOL, 20 - define :MKREDIRECTREF, 21 - define :MKWORKSPACE, 22 - define :MOVE, 23 - define :OPTIONS, 24 - define :ORDERPATCH, 25 - define :PATCH, 26 - define :PRI, 27 - define :PROPFIND, 28 - define :PROPPATCH, 29 - define :REBIND, 30 - define :REPORT, 31 - define :SEARCH, 32 - define :SHOWMETHOD, 33 - define :SPACEJUMP, 34 - define :TEXTSEARCH, 35 - define :TRACE, 36 - define :TRACK, 37 - define :UNBIND, 38 - define :UNCHECKOUT, 39 - define :UNLINK, 40 - define :UNLOCK, 41 - define :UPDATE, 42 - define :UPDATEREDIRECTREF, 43 - define :VERSION_CONTROL, 44 - end - - ## - # Message Classes - # - class HttpStartStop < ::Protobuf::Message; end - - ## - # Message Fields - # - class HttpStartStop - required :int64, :startTimestamp, 1 - required :int64, :stopTimestamp, 2 - required ::TrafficController::Models::UUID, :requestId, 3 - required ::TrafficController::Models::PeerType, :peerType, 4 - required ::TrafficController::Models::Method, :method, 5 - required :string, :uri, 6 - required :string, :remoteAddress, 7 - required :string, :userAgent, 8 - required :int32, :statusCode, 9 - required :int64, :contentLength, 10 - optional ::TrafficController::Models::UUID, :applicationId, 12 - optional :int32, :instanceIndex, 13 - optional :string, :instanceId, 14 - repeated :string, :forwarded, 15 - end - end -end diff --git a/lib/traffic_controller/models/log.pb.rb b/lib/traffic_controller/models/log.pb.rb deleted file mode 100644 index a8d24fb4e44..00000000000 --- a/lib/traffic_controller/models/log.pb.rb +++ /dev/null @@ -1,32 +0,0 @@ -# encoding: utf-8 - -## -# This file is auto-generated. DO NOT EDIT! -# -require 'protobuf/message' - -module TrafficController - module Models - ## - # Message Classes - # - class LogMessage < ::Protobuf::Message - class MessageType < ::Protobuf::Enum - define :OUT, 1 - define :ERR, 2 - end - end - - ## - # Message Fields - # - class LogMessage - required :bytes, :message, 1 - required ::TrafficController::Models::LogMessage::MessageType, :message_type, 2 - required :int64, :timestamp, 3 - optional :string, :app_id, 4 - optional :string, :source_type, 5 - optional :string, :source_instance, 6 - end - end -end diff --git a/lib/traffic_controller/models/metric.pb.rb b/lib/traffic_controller/models/metric.pb.rb deleted file mode 100644 index e38d531fcc6..00000000000 --- a/lib/traffic_controller/models/metric.pb.rb +++ /dev/null @@ -1,47 +0,0 @@ -# encoding: utf-8 - -## -# This file is auto-generated. DO NOT EDIT! -# -require 'protobuf/message' - -## -# Imports -# -require 'uuid.pb' - -module TrafficController - module Models - ## - # Message Classes - # - class ValueMetric < ::Protobuf::Message; end - class CounterEvent < ::Protobuf::Message; end - class ContainerMetric < ::Protobuf::Message; end - - ## - # Message Fields - # - class ValueMetric - required :string, :name, 1 - required :double, :value, 2 - required :string, :unit, 3 - end - - class CounterEvent - required :string, :name, 1 - required :uint64, :delta, 2 - optional :uint64, :total, 3 - end - - class ContainerMetric - required :string, :applicationId, 1 - required :int32, :instanceIndex, 2 - required :double, :cpuPercentage, 3 - required :uint64, :memoryBytes, 4 - required :uint64, :diskBytes, 5 - optional :uint64, :memoryBytesQuota, 6 - optional :uint64, :diskBytesQuota, 7 - end - end -end diff --git a/lib/traffic_controller/models/uuid.pb.rb b/lib/traffic_controller/models/uuid.pb.rb deleted file mode 100644 index 16b1b5a2fff..00000000000 --- a/lib/traffic_controller/models/uuid.pb.rb +++ /dev/null @@ -1,23 +0,0 @@ -# encoding: utf-8 - -## -# This file is auto-generated. DO NOT EDIT! -# -require 'protobuf/message' - -module TrafficController - module Models - ## - # Message Classes - # - class UUID < ::Protobuf::Message; end - - ## - # Message Fields - # - class UUID - required :uint64, :low, 1 - required :uint64, :high, 2 - end - end -end diff --git a/lib/traffic_controller/traffic_controller.rb b/lib/traffic_controller/traffic_controller.rb deleted file mode 100644 index 60f9159c88f..00000000000 --- a/lib/traffic_controller/traffic_controller.rb +++ /dev/null @@ -1,8 +0,0 @@ -$LOAD_PATH.unshift(File.expand_path('models', __dir__)) - -require 'traffic_controller/models/envelope.pb' -require 'traffic_controller/models/error.pb' -require 'traffic_controller/models/http.pb' -require 'traffic_controller/models/log.pb' -require 'traffic_controller/models/metric.pb' -require 'traffic_controller/models/uuid.pb' diff --git a/lib/utils/multipart_parser_wrapper.rb b/lib/utils/multipart_parser_wrapper.rb deleted file mode 100644 index 7c03d9787c3..00000000000 --- a/lib/utils/multipart_parser_wrapper.rb +++ /dev/null @@ -1,45 +0,0 @@ -require 'multipart_parser/reader' - -module VCAP - class MultipartParserWrapper - NEW_LINE = "\r\n".freeze - - def initialize(body:, boundary:) - @body = body - @boundary = boundary - end - - def next_part - @chunks ||= parse(@body, @boundary) - @chunks.next - rescue StopIteration, ParseError - nil - end - - private - - def parse(body, boundary) - parts = [] - - reader = ::MultipartParser::Reader.new(boundary) - - reader.on_part do |part| - p = [] - - part.on_data do |partial_data| - p << partial_data - end - - parts << p - end - - reader.write body - - unless reader.ended? - raise ParseError.new('truncated multipart message') - end - - parts.map(&:join).to_enum - end - end -end diff --git a/scripts/cf-release/restore-bosh-lite-cc.sh b/scripts/cf-release/restore-bosh-lite-cc.sh index 2a1e6aed7ad..99d834d99da 100755 --- a/scripts/cf-release/restore-bosh-lite-cc.sh +++ b/scripts/cf-release/restore-bosh-lite-cc.sh @@ -10,4 +10,3 @@ sed "/bbs.service.cf.internal/d" /etc/hosts | sudo tee /etc/hosts > /dev/null sed "/bits-service.service.cf.internal/d" /etc/hosts | sudo tee /etc/hosts > /dev/null sed "/bits-service.bosh-lite.com/d" /etc/hosts | sudo tee /etc/hosts > /dev/null sed "/uaa.service.cf.internal/d" /etc/hosts | sudo tee /etc/hosts > /dev/null -sed "/loggregator-trafficcontroller.service.cf.internal/d" /etc/hosts | sudo tee /etc/hosts > /dev/null diff --git a/scripts/cf-release/setup-local-cc.sh b/scripts/cf-release/setup-local-cc.sh index 55c61aed108..1331ce12da2 100755 --- a/scripts/cf-release/setup-local-cc.sh +++ b/scripts/cf-release/setup-local-cc.sh @@ -24,7 +24,3 @@ fi if ! cat /etc/hosts | grep "uaa.service.cf.internal" > /dev/null; then echo "10.244.0.134 uaa.service.cf.internal" | sudo tee -a /etc/hosts > /dev/null fi - -if ! cat /etc/hosts | grep "loggregator-trafficcontroller.service.cf.internal" > /dev/null; then - echo "10.244.0.150 loggregator-trafficcontroller.service.cf.internal" | sudo tee -a /etc/hosts > /dev/null -fi diff --git a/scripts/generate-traffic-controller-models.sh b/scripts/generate-traffic-controller-models.sh deleted file mode 100755 index e7a9bf2b92e..00000000000 --- a/scripts/generate-traffic-controller-models.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env bash - -ruby_generated_files_path="src/cloud_controller_ng/lib/traffic_controller/models" -traffic_controller_proto_path="${GOPATH}/src/dropsonde-protocol/events" -generated_ruby_destination="${GOPATH}/${ruby_generated_files_path}" - -if [[ ! -d $traffic_controller_proto_path ]]; then - echo "bbs models were not available at ${traffic_controller_proto_path}" - exit 1 -fi - -if [[ ! -d "${generated_ruby_destination}" ]]; then - echo "directory not available for generated ruby classes at ${generated_ruby_destination}" - exit 1 -fi - -if [[ ! $(protoc --version) ]]; then - echo "must install protoc" - exit 1 -fi - -# this gem contains the plugin used to generate ruby models that are proto2 compatible -if [[ ! $(gem list | grep protobuf) ]]; then - echo "must 'gem install protobuf'" - exit 1 -fi - -pushd "${traffic_controller_proto_path}" - # the ruby modules are created based on the package name - sed -i'' -e 's/package events/package TrafficController.models/' ./*.proto - - # this is a hack to allow protoc to use a plugin that supports proto2 generation since protoc only supports proto3 by default - # see: https://github.com/ruby-protobuf/protobuf/issues/341 - protoc --proto_path="${GOPATH}/src":. --plugin="protoc-gen-bob=$(which protoc-gen-ruby)" --bob_out="${generated_ruby_destination}" ./*.proto - git checkout . -popd diff --git a/scripts/short-circuit-cc b/scripts/short-circuit-cc index 2048080c1b6..47decb8dbdb 100755 --- a/scripts/short-circuit-cc +++ b/scripts/short-circuit-cc @@ -45,7 +45,6 @@ blobstore.service.cf.internal \ cc-uploader.service.cf.internal \ cell.service.cf.internal \ cloud-controller-ng.service.cf.internal \ -loggregator-trafficcontroller.service.cf.internal \ sql-db.service.cf.internal \ uaa.service.cf.internal" ip_to_dns="$(bosh ssh "${BOSH_API_INSTANCE}" -c "for addr in ${internal_hostnames}; do ip=\"\$(dig +short \${addr})\" && echo \"\${ip:-\"UNKNOWN\"} \${addr}\"; done" -r --column=Stdout | cat)" diff --git a/spec/logcache/traffic_controller_decorator_spec.rb b/spec/logcache/container_metric_batcher_spec.rb similarity index 75% rename from spec/logcache/traffic_controller_decorator_spec.rb rename to spec/logcache/container_metric_batcher_spec.rb index 1dda6d03234..b95a9676cff 100644 --- a/spec/logcache/traffic_controller_decorator_spec.rb +++ b/spec/logcache/container_metric_batcher_spec.rb @@ -1,8 +1,8 @@ require 'spec_helper' -require 'logcache/traffic_controller_decorator' +require 'logcache/container_metric_batcher' require 'utils/time_utils' -RSpec.describe Logcache::TrafficControllerDecorator do +RSpec.describe Logcache::ContainerMetricBatcher do subject { described_class.new(wrapped_logcache_client).container_metrics(source_guid: process_guid, logcache_filter: filter) } let(:wrapped_logcache_client) { instance_double(Logcache::Client, container_metrics: logcache_response) } @@ -22,12 +22,9 @@ def generate_batch(size, offset: 0, last_timestamp: TimeUtils.to_nanoseconds(Tim 'cpu' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: cpu_percentage), 'memory' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 100 * i + 2), 'disk' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 100 * i + 3), + 'log_rate' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 100 * i + 4), }), instance_id: (offset + i).to_s, - tags: { - 'source_id' => process.app.guid, - 'process_id' => process.guid, - }, ), Loggregator::V2::Envelope.new( timestamp: last_timestamp, @@ -38,17 +35,13 @@ def generate_batch(size, offset: 0, last_timestamp: TimeUtils.to_nanoseconds(Tim 'container_age' => Loggregator::V2::GaugeValue.new(unit: 'nanoseconds', value: 100 * i + 3), }), instance_id: (offset + i).to_s, - tags: { - 'source_id' => process.app.guid, - 'process_id' => process.guid, - }, ) ] end Loggregator::V2::EnvelopeBatch.new(batch: batch) end - describe 'converting from Logcache to TrafficController' do + describe 'batches envelopes' do let(:filter) { ->(_) { true } } before do @@ -73,6 +66,7 @@ def generate_batch(size, offset: 0, last_timestamp: TimeUtils.to_nanoseconds(Tim 'cpu' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 10), 'memory' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 11), 'disk' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 12), + 'log_rate' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 13), }), instance_id: '1' ), @@ -82,6 +76,7 @@ def generate_batch(size, offset: 0, last_timestamp: TimeUtils.to_nanoseconds(Tim 'cpu' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 13), 'memory' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 10), 'disk' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 10), + 'log_rate' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 10), }), instance_id: '2' ), @@ -91,6 +86,7 @@ def generate_batch(size, offset: 0, last_timestamp: TimeUtils.to_nanoseconds(Tim 'cpu' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 10), 'memory' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 9), 'disk' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 8), + 'log_rate' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 7), }), instance_id: '1' ), @@ -103,11 +99,11 @@ def generate_batch(size, offset: 0, last_timestamp: TimeUtils.to_nanoseconds(Tim subject expect(subject).to have(1).items - expect(subject.first.containerMetric.applicationId).to eq(process_guid) - expect(subject.first.containerMetric.instanceIndex).to eq(1) - expect(subject.first.containerMetric.cpuPercentage).to eq(10) - expect(subject.first.containerMetric.memoryBytes).to eq(11) - expect(subject.first.containerMetric.diskBytes).to eq(12) + expect(subject.first.instance_index).to eq(1) + expect(subject.first.cpu_percentage).to eq(10) + expect(subject.first.memory_bytes).to eq(11) + expect(subject.first.disk_bytes).to eq(12) + expect(subject.first.log_rate).to eq(13) end end @@ -138,6 +134,7 @@ def generate_batch(size, offset: 0, last_timestamp: TimeUtils.to_nanoseconds(Tim 'cpu' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 10), 'memory' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 11), 'disk' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 12), + 'log_rate' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 13), }), instance_id: '1' ), @@ -156,6 +153,7 @@ def generate_batch(size, offset: 0, last_timestamp: TimeUtils.to_nanoseconds(Tim 'cpu' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 20), 'memory' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 21), 'disk' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 22), + 'log_rate' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 23), }), instance_id: '2' ), @@ -174,6 +172,7 @@ def generate_batch(size, offset: 0, last_timestamp: TimeUtils.to_nanoseconds(Tim 'cpu' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 30), 'memory' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 31), 'disk' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 32), + 'log_rate' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 33), }), instance_id: '3' ) @@ -182,25 +181,25 @@ def generate_batch(size, offset: 0, last_timestamp: TimeUtils.to_nanoseconds(Tim } let(:num_instances) { 3 } - it 'returns an array of envelopes, formatted as Traffic Controller would' do - expect(subject.first.containerMetric.applicationId).to eq(process_guid) - expect(subject.first.containerMetric.instanceIndex).to eq(1) - expect(subject.first.containerMetric.cpuPercentage).to eq(10) - expect(subject.first.containerMetric.memoryBytes).to eq(11) - expect(subject.first.containerMetric.diskBytes).to eq(12) - - expect(subject.second.containerMetric.applicationId).to eq(process_guid) - expect(subject.second.containerMetric.instanceIndex).to eq(2) - expect(subject.second.containerMetric.cpuPercentage).to eq(20) - expect(subject.second.containerMetric.memoryBytes).to eq(21) - expect(subject.second.containerMetric.diskBytes).to eq(22) - - cm = subject[2].containerMetric - expect(cm.applicationId).to eq(process_guid) - expect(cm.instanceIndex).to eq(3) - expect(cm.cpuPercentage).to eq(30) - expect(cm.memoryBytes).to eq(31) - expect(cm.diskBytes).to eq(32) + it 'returns an array of batched metrics' do + expect(subject.first.instance_index).to eq(1) + expect(subject.first.cpu_percentage).to eq(10) + expect(subject.first.memory_bytes).to eq(11) + expect(subject.first.disk_bytes).to eq(12) + expect(subject.first.log_rate).to eq(13) + + expect(subject.second.instance_index).to eq(2) + expect(subject.second.cpu_percentage).to eq(20) + expect(subject.second.memory_bytes).to eq(21) + expect(subject.second.disk_bytes).to eq(22) + expect(subject.second.log_rate).to eq(23) + + cm = subject[2] + expect(cm.instance_index).to eq(3) + expect(cm.cpu_percentage).to eq(30) + expect(cm.memory_bytes).to eq(31) + expect(cm.disk_bytes).to eq(32) + expect(cm.log_rate).to eq(33) end end @@ -210,48 +209,28 @@ def generate_batch(size, offset: 0, last_timestamp: TimeUtils.to_nanoseconds(Tim batch: [Loggregator::V2::Envelope.new( source_id: process_guid, gauge: Loggregator::V2::Gauge.new(metrics: { - 'cpu' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 10), - 'memory' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 11), - 'disk' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 12), - 'disk_quota' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 24), - 'memory_quota' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 25), + 'cpu' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 0.10), + 'memory' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 11.0), + 'disk' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 12.0), + 'log_rate' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 13.0), + 'disk_quota' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 24.0), + 'memory_quota' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 25.0), + 'log_rate_limit' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 26.0), }), instance_id: '1' )] ) } - it 'returns an array of one envelope, formatted as Traffic Controller would' do - expect(subject.first.containerMetric.applicationId).to eq(process_guid) - expect(subject.first.containerMetric.instanceIndex).to eq(1) - expect(subject.first.containerMetric.cpuPercentage).to eq(10) - expect(subject.first.containerMetric.memoryBytes).to eq(11) - expect(subject.first.containerMetric.diskBytes).to eq(12) - expect(subject.first.containerMetric.diskBytesQuota).to eq(24) - expect(subject.first.containerMetric.memoryBytesQuota).to eq(25) - end - end - - context 'when the envelope does not have tags' do - let(:envelopes) { - Loggregator::V2::EnvelopeBatch.new( - batch: [ - Loggregator::V2::Envelope.new( - source_id: process_guid, - gauge: Loggregator::V2::Gauge.new(metrics: { - 'cpu' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 10), - 'memory' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 11), - 'disk' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 12), - }), - instance_id: '1', - tags: {}, - ), - ] - ) - } - - it 'returns an envelope with an empty array of tags' do - expect(subject.first.tags).to be_empty + it 'returns an array of one batched envelope' do + expect(subject.first.instance_index).to eql(1) + expect(subject.first.cpu_percentage).to eql(0.10) + expect(subject.first.memory_bytes).to eql(11) + expect(subject.first.disk_bytes).to eql(12) + expect(subject.first.log_rate).to eql(13) + expect(subject.first.disk_bytes_quota).to eql(24) + expect(subject.first.memory_bytes_quota).to eql(25) + expect(subject.first.log_rate_limit).to eql(26) end end @@ -265,14 +244,12 @@ def generate_batch(size, offset: 0, last_timestamp: TimeUtils.to_nanoseconds(Tim 'cpu' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 10), 'memory' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 11), 'disk' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 12), + 'log_rate' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 13), 'disk_quota' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 24), - 'memory_quota' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 25), + 'memory_quota' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 25), + 'log_rate_limit' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 26), }), instance_id: '1', - tags: { - 'source_id' => process.app.guid, - 'process_id' => process.guid, - }, ), Loggregator::V2::Envelope.new( source_id: process_guid, @@ -280,14 +257,12 @@ def generate_batch(size, offset: 0, last_timestamp: TimeUtils.to_nanoseconds(Tim 'cpu' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 20), 'memory' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 21), 'disk' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 22), + 'log_rate' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 23), 'disk_quota' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 34), 'memory_quota' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 35), + 'log_rate_limit' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 36), }), instance_id: '2', - tags: { - 'source_id' => process.app.guid, - 'process_id' => process.guid, - }, ), Loggregator::V2::Envelope.new( source_id: process_guid, @@ -295,50 +270,46 @@ def generate_batch(size, offset: 0, last_timestamp: TimeUtils.to_nanoseconds(Tim 'cpu' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 30), 'memory' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 31), 'disk' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 32), + 'log_rate' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 33), 'disk_quota' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 44), 'memory_quota' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 45), + 'log_rate_limit' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 46), }), instance_id: '3', - tags: { - 'source_id' => process.app.guid, - 'process_id' => process.guid, - }, ) ] ) } let(:num_instances) { 3 } - it 'returns an array of envelopes, formatted as Traffic Controller would' do - expect(subject.first.containerMetric.applicationId).to eq(process_guid) - expect(subject.first.containerMetric.instanceIndex).to eq(1) - expect(subject.first.containerMetric.cpuPercentage).to eq(10) - expect(subject.first.containerMetric.memoryBytes).to eq(11) - expect(subject.first.containerMetric.diskBytes).to eq(12) - expect(subject.first.containerMetric.diskBytesQuota).to eq(24) - expect(subject.first.containerMetric.memoryBytesQuota).to eq(25) - - expect(subject.first.tags[0].key).to eq('source_id') - expect(subject.first.tags[0].value).to eq(process.app.guid) - expect(subject.first.tags[1].key).to eq('process_id') - expect(subject.first.tags[1].value).to eq(process.guid) - - expect(subject.second.containerMetric.applicationId).to eq(process_guid) - expect(subject.second.containerMetric.instanceIndex).to eq(2) - expect(subject.second.containerMetric.cpuPercentage).to eq(20) - expect(subject.second.containerMetric.memoryBytes).to eq(21) - expect(subject.second.containerMetric.diskBytes).to eq(22) - expect(subject.second.containerMetric.diskBytesQuota).to eq(34) - expect(subject.second.containerMetric.memoryBytesQuota).to eq(35) - - cm = subject[2].containerMetric - expect(cm.applicationId).to eq(process_guid) - expect(cm.instanceIndex).to eq(3) - expect(cm.cpuPercentage).to eq(30) - expect(cm.memoryBytes).to eq(31) - expect(cm.diskBytes).to eq(32) - expect(cm.diskBytesQuota).to eq(44) - expect(cm.memoryBytesQuota).to eq(45) + it 'returns an array of batched metrics' do + expect(subject.first.instance_index).to eq(1) + expect(subject.first.cpu_percentage).to eq(10) + expect(subject.first.memory_bytes).to eq(11) + expect(subject.first.disk_bytes).to eq(12) + expect(subject.first.log_rate).to eq(13) + expect(subject.first.disk_bytes_quota).to eq(24) + expect(subject.first.memory_bytes_quota).to eq(25) + expect(subject.first.log_rate_limit).to eq(26) + + expect(subject.second.instance_index).to eq(2) + expect(subject.second.cpu_percentage).to eq(20) + expect(subject.second.memory_bytes).to eq(21) + expect(subject.second.disk_bytes).to eq(22) + expect(subject.second.log_rate).to eq(23) + expect(subject.second.disk_bytes_quota).to eq(34) + expect(subject.second.memory_bytes_quota).to eq(35) + expect(subject.second.log_rate_limit).to eq(36) + + cm = subject[2] + expect(cm.instance_index).to eq(3) + expect(cm.cpu_percentage).to eq(30) + expect(cm.memory_bytes).to eq(31) + expect(cm.disk_bytes).to eq(32) + expect(cm.log_rate).to eq(33) + expect(cm.disk_bytes_quota).to eq(44) + expect(cm.memory_bytes_quota).to eq(45) + expect(cm.log_rate_limit).to eq(46) end end @@ -352,6 +323,7 @@ def generate_batch(size, offset: 0, last_timestamp: TimeUtils.to_nanoseconds(Tim 'cpu' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 10), 'memory' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 11), 'disk' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 12), + 'log_rate' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 13), }), instance_id: '1' ), @@ -361,6 +333,7 @@ def generate_batch(size, offset: 0, last_timestamp: TimeUtils.to_nanoseconds(Tim 'cpu' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 20), 'memory' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 21), 'disk' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 22), + 'log_rate' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 23), }), instance_id: '2' ), @@ -370,6 +343,7 @@ def generate_batch(size, offset: 0, last_timestamp: TimeUtils.to_nanoseconds(Tim 'cpu' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 30), 'memory' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 31), 'disk' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 32), + 'log_rate' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 33), }), instance_id: '1' ) @@ -380,9 +354,9 @@ def generate_batch(size, offset: 0, last_timestamp: TimeUtils.to_nanoseconds(Tim it 'returns only the newest metric' do expect(subject.count).to eq(2) - expect(subject.first.containerMetric.instanceIndex).to eq(1) - expect(subject.first.containerMetric.cpuPercentage).to eq(10) - expect(subject.second.containerMetric.instanceIndex).to eq(2) + expect(subject.first.instance_index).to eq(1) + expect(subject.first.cpu_percentage).to eq(10) + expect(subject.second.instance_index).to eq(2) end end @@ -409,8 +383,8 @@ def generate_batch(size, offset: 0, last_timestamp: TimeUtils.to_nanoseconds(Tim Timecop.freeze(call_time) do expect(subject).to have(1000).items - expect(subject.first.containerMetric.instanceIndex).to eq(1) - expect(subject.first.containerMetric.cpuPercentage).to eq(34.0) + expect(subject.first.instance_index).to eq(1) + expect(subject.first.cpu_percentage).to eq(34.0) end end end @@ -440,12 +414,20 @@ def generate_batch(size, offset: 0, last_timestamp: TimeUtils.to_nanoseconds(Tim }), instance_id: '1' ), + Loggregator::V2::Envelope.new( + source_id: process_guid, + gauge: Loggregator::V2::Gauge.new(metrics: { + 'log_rate' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 13), + }), + instance_id: '1' + ), Loggregator::V2::Envelope.new( source_id: process_guid, gauge: Loggregator::V2::Gauge.new(metrics: { 'cpu' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 20), 'memory' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 21), 'disk' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 22), + 'log_rate' => Loggregator::V2::GaugeValue.new(unit: 'bytes', value: 23), }), instance_id: '2' ) @@ -457,15 +439,17 @@ def generate_batch(size, offset: 0, last_timestamp: TimeUtils.to_nanoseconds(Tim it 'returns a single envelope per instance' do expect(subject.count).to eq(2) - expect(subject.first.containerMetric.instanceIndex).to eq(1) - expect(subject.first.containerMetric.cpuPercentage).to eq(10) - expect(subject.first.containerMetric.memoryBytes).to eq(11) - expect(subject.first.containerMetric.diskBytes).to eq(12) - - expect(subject.second.containerMetric.instanceIndex).to eq(2) - expect(subject.second.containerMetric.cpuPercentage).to eq(20) - expect(subject.second.containerMetric.memoryBytes).to eq(21) - expect(subject.second.containerMetric.diskBytes).to eq(22) + expect(subject.first.instance_index).to eq(1) + expect(subject.first.cpu_percentage).to eq(10) + expect(subject.first.memory_bytes).to eq(11) + expect(subject.first.disk_bytes).to eq(12) + expect(subject.first.log_rate).to eq(13) + + expect(subject.second.instance_index).to eq(2) + expect(subject.second.cpu_percentage).to eq(20) + expect(subject.second.memory_bytes).to eq(21) + expect(subject.second.disk_bytes).to eq(22) + expect(subject.second.log_rate).to eq(23) end end diff --git a/spec/traffic_controller/client_spec.rb b/spec/traffic_controller/client_spec.rb deleted file mode 100644 index 7e9aaca1a54..00000000000 --- a/spec/traffic_controller/client_spec.rb +++ /dev/null @@ -1,101 +0,0 @@ -require 'spec_helper' -require 'traffic_controller/client' -require 'openssl' - -module TrafficController - RSpec.describe Client do - let(:doppler_url) { 'https://doppler.example.com:4443' } - - subject(:client) { Client.new(url: doppler_url) } - let(:expected_request_options) { { 'headers' => { 'Authorization' => 'bearer oauth-token' } } } - - def build_response_body(boundary, encoded_envelopes) - body = [] - encoded_envelopes.each do |env| - body << "--#{boundary}" - body << '' - body << env - end - body << "--#{boundary}--" - - body.join("\r\n") - end - - describe '#container_metrics' do - let(:auth_token) { 'bearer oauth-token' } - let(:response_boundary) { SecureRandom.uuid } - let(:response_body) do - build_response_body(response_boundary, [ - Models::Envelope.new(origin: 'a', eventType: Models::Envelope::EventType::ContainerMetric).encode.to_s, - Models::Envelope.new(origin: 'b', eventType: Models::Envelope::EventType::ContainerMetric).encode.to_s, - ]) - end - let(:response_status) { 200 } - let(:response_headers) { { 'Content-Type' => "multipart/x-protobuf; boundary=#{response_boundary}" } } - - before do - stub_request(:get, 'https://doppler.example.com:4443/apps/example-app-guid/containermetrics'). - with(expected_request_options). - to_return(status: response_status, body: response_body, headers: response_headers) - end - - it 'returns an array of Envelopes' do - expect(client.container_metrics(auth_token: auth_token, source_guid: 'example-app-guid')).to match_array([ - Models::Envelope.new(origin: 'a', eventType: Models::Envelope::EventType::ContainerMetric), - Models::Envelope.new(origin: 'b', eventType: Models::Envelope::EventType::ContainerMetric), - ]) - expect(a_request(:get, 'https://doppler.example.com:4443/apps/example-app-guid/containermetrics')).to have_been_made - end - - context 'when it does not return successfully' do - let(:response_status) { 404 } - let(:response_body) { 'not found' } - - it 'raises' do - expect { client.container_metrics(auth_token: auth_token, source_guid: 'example-app-guid') }.to raise_error(ResponseError, /status: 404, body: not found/) - end - end - - context 'when it fails to make the request' do - before do - stub_request(:get, 'https://doppler.example.com:4443/apps/example-app-guid/containermetrics').to_raise(StandardError.new('error message')) - end - - it 'raises' do - expect { client.container_metrics(auth_token: auth_token, source_guid: 'example-app-guid') }.to raise_error(RequestError, /error message/) - end - end - - context 'when the response is not a valid multipart body' do - let(:response_body) { '' } - - it 'returns an empty array' do - expect(client.container_metrics(auth_token: auth_token, source_guid: 'example-app-guid')).to be_empty - end - end - - context 'when the response does not contain the boundary in the "Content-Type" header' do - let(:response_headers) { { 'Content-Type' => 'potato' } } - - it 'raises' do - expect { - client.container_metrics(auth_token: auth_token, source_guid: 'example-app-guid') - }.to raise_error(ResponseError, 'failed to find multipart boundary in Content-Type header') - end - end - - context 'when a part cannot be decoded by ProtoBuf' do - let(:response_body) do - build_response_body(response_boundary, [ - Models::Envelope.new(origin: 'a', eventType: Models::Envelope::EventType::ContainerMetric).encode.to_s, - 'potato', - ]) - end - - it 'raises' do - expect { client.container_metrics(auth_token: auth_token, source_guid: 'example-app-guid') }.to raise_error(DecodeError) - end - end - end - end -end diff --git a/spec/unit/lib/cloud_controller/backends/instances_reporters_spec.rb b/spec/unit/lib/cloud_controller/backends/instances_reporters_spec.rb index 231e201cf33..40d33a94ebc 100644 --- a/spec/unit/lib/cloud_controller/backends/instances_reporters_spec.rb +++ b/spec/unit/lib/cloud_controller/backends/instances_reporters_spec.rb @@ -5,26 +5,20 @@ module VCAP::CloudController subject(:instances_reporters) { InstancesReporters.new } let(:bbs_instances_client) { instance_double(Diego::BbsInstancesClient) } - let(:traffic_controller_client) { instance_double(::TrafficController::Client) } let(:logcache_client) { instance_double(::Logcache::Client) } - let(:tc_compatible_logcache_client) { instance_double(Logcache::TrafficControllerDecorator) } + let(:log_cache_metrics_client) { instance_double(Logcache::ContainerMetricBatcher) } let(:diego_process) { ProcessModelFactory.make(diego: true) } let(:diego_instances_reporter) { instance_double(Diego::InstancesReporter) } let(:diego_instances_stats_reporter) { instance_double(Diego::InstancesStatsReporter) } - let(:temporary_use_logcache) { false } before do - TestConfig.override(temporary_use_logcache: temporary_use_logcache) - CloudController::DependencyLocator.instance.register(:bbs_instances_client, bbs_instances_client) - CloudController::DependencyLocator.instance.register(:traffic_controller_client, traffic_controller_client) CloudController::DependencyLocator.instance.register(:logcache_client, logcache_client) - CloudController::DependencyLocator.instance.register(:traffic_controller_compatible_logcache_client, tc_compatible_logcache_client) + CloudController::DependencyLocator.instance.register(:log_cache_metrics_client, log_cache_metrics_client) allow(Diego::InstancesReporter).to receive(:new).with(bbs_instances_client).and_return(diego_instances_reporter) - allow(Diego::InstancesStatsReporter).to receive(:new).with(bbs_instances_client, traffic_controller_client).and_return(diego_instances_stats_reporter) - allow(Diego::InstancesStatsReporter).to receive(:new).with(bbs_instances_client, tc_compatible_logcache_client).and_return(diego_instances_stats_reporter) + allow(Diego::InstancesStatsReporter).to receive(:new).with(bbs_instances_client, log_cache_metrics_client).and_return(diego_instances_stats_reporter) end describe '#number_of_starting_and_running_instances_for_process' do @@ -106,22 +100,9 @@ module VCAP::CloudController allow(diego_instances_stats_reporter).to receive(:stats_for_app).with(app) end - context 'when temporary_use_logcache is true' do - let(:temporary_use_logcache) { true } - - it 'uses the logcache' do - instances_reporters.stats_for_app(app) - expect(Diego::InstancesStatsReporter).to have_received(:new).with(bbs_instances_client, tc_compatible_logcache_client) - end - end - - context 'when temporary_use_logcache is false' do - let(:temporary_use_logcache) { false } - - it 'uses the trafficcontroller' do - instances_reporters.stats_for_app(app) - expect(Diego::InstancesStatsReporter).to have_received(:new).with(bbs_instances_client, traffic_controller_client) - end + it 'uses the logcache' do + instances_reporters.stats_for_app(app) + expect(Diego::InstancesStatsReporter).to have_received(:new).with(bbs_instances_client, log_cache_metrics_client) end end end diff --git a/spec/unit/lib/cloud_controller/dependency_locator_spec.rb b/spec/unit/lib/cloud_controller/dependency_locator_spec.rb index 1728e1d515e..297f0ecc500 100644 --- a/spec/unit/lib/cloud_controller/dependency_locator_spec.rb +++ b/spec/unit/lib/cloud_controller/dependency_locator_spec.rb @@ -477,13 +477,7 @@ end end - context '#traffic_controller_client' do - it 'returns the tc client' do - expect(locator.traffic_controller_client).to be_an_instance_of(TrafficController::Client) - end - end - - describe '#traffic_controller_compatible_logcache_client' do + describe '#log_cache_metrics_client' do let(:logcache_client) { instance_double(Logcache::Client) } before do allow(Logcache::Client).to receive(:new).and_return(logcache_client) @@ -493,7 +487,7 @@ TestConfig.override( logcache_tls: nil ) - expect(locator.traffic_controller_compatible_logcache_client).to be_an_instance_of(Logcache::TrafficControllerDecorator) + expect(locator.log_cache_metrics_client).to be_an_instance_of(Logcache::ContainerMetricBatcher) expect(Logcache::Client).to have_received(:new).with( host: 'http://doppler.service.cf.internal', port: 8080, @@ -517,7 +511,7 @@ subject_name: 'some-tls-cert-san' } ) - expect(locator.traffic_controller_compatible_logcache_client).to be_an_instance_of(Logcache::TrafficControllerDecorator) + expect(locator.log_cache_metrics_client).to be_an_instance_of(Logcache::ContainerMetricBatcher) expect(Logcache::Client).to have_received(:new).with( host: 'some-logcache-host', port: 1234, diff --git a/spec/unit/lib/cloud_controller/diego/reporters/instances_stats_reporter_spec.rb b/spec/unit/lib/cloud_controller/diego/reporters/instances_stats_reporter_spec.rb index 5fe14ae4baa..2cda59e841e 100644 --- a/spec/unit/lib/cloud_controller/diego/reporters/instances_stats_reporter_spec.rb +++ b/spec/unit/lib/cloud_controller/diego/reporters/instances_stats_reporter_spec.rb @@ -4,12 +4,12 @@ module VCAP::CloudController module Diego RSpec.describe InstancesStatsReporter do - subject(:instances_reporter) { InstancesStatsReporter.new(bbs_instances_client, traffic_controller_client) } + subject(:instances_reporter) { InstancesStatsReporter.new(bbs_instances_client, log_cache_client) } let(:app) { AppModel.make } let(:process) { ProcessModel.make(instances: desired_instances, app: app) } let(:desired_instances) { 1 } let(:bbs_instances_client) { instance_double(BbsInstancesClient) } - let(:traffic_controller_client) { instance_double(TrafficController::Client) } + let(:log_cache_client) { instance_double(Logcache::ContainerMetricBatcher) } let(:two_days_ago_since_epoch_ns) { 2.days.ago.to_f * 1e9 } let(:two_days_in_seconds) { 60 * 60 * 24 * 2 } @@ -61,22 +61,17 @@ def make_actual_lrp(instance_guid:, index:, state:, error:, since:) actual_lrp.actual_lrp_net_info = lrp_1_net_info end end - let(:traffic_controller_response) do - [ - ::TrafficController::Models::Envelope.new( - origin: 'does-anyone-even-know?', - eventType: ::TrafficController::Models::Envelope::EventType::ContainerMetric, - containerMetric: ::TrafficController::Models::ContainerMetric.new( - instanceIndex: 0, - cpuPercentage: 3.92, - memoryBytes: 564, - diskBytes: 5000, - memoryBytesQuota: 1234, - diskBytesQuota: 10234, - ), - tags: [::TrafficController::Models::Envelope::TagsEntry.new(key: 'process_id', value: process.guid)], - ), - ] + let(:log_cache_response) do + container_metric_batch = ::Logcache::ContainerMetricBatch.new + container_metric_batch.instance_index = 0 + container_metric_batch.cpu_percentage = 3.92 + container_metric_batch.memory_bytes = 564 + container_metric_batch.disk_bytes = 5000 + container_metric_batch.memory_bytes_quota = 1234 + container_metric_batch.disk_bytes_quota = 10234 + container_metric_batch.log_rate = 5 + container_metric_batch.log_rate_limit = 10 + [container_metric_batch] end let(:expected_stats_response) do @@ -93,12 +88,14 @@ def make_actual_lrp(instance_guid:, index:, state:, error:, since:) uptime: two_days_in_seconds, mem_quota: 1234, disk_quota: 10234, + log_rate_limit: 10, fds_quota: process.file_descriptors, usage: { time: formatted_current_time, cpu: 0.0392, mem: 564, disk: 5000, + log_rate: 5, } }, details: 'some-details', @@ -109,9 +106,9 @@ def make_actual_lrp(instance_guid:, index:, state:, error:, since:) before do allow(bbs_instances_client).to receive(:lrp_instances).and_return(bbs_actual_lrps_response) allow(bbs_instances_client).to receive(:desired_lrp_instance).and_return(bbs_desired_lrp_response) - allow(traffic_controller_client).to receive(:container_metrics). + allow(log_cache_client).to receive(:container_metrics). with(auth_token: 'my-token', source_guid: process.app.guid, logcache_filter: anything). - and_return(traffic_controller_response) + and_return(log_cache_response) allow(VCAP::CloudController::SecurityContext).to receive(:auth_token).and_return('my-token') end @@ -122,9 +119,9 @@ def make_actual_lrp(instance_guid:, index:, state:, error:, since:) it 'passes a process_id filter' do filter = nil - allow(traffic_controller_client).to receive(:container_metrics) { |args| + allow(log_cache_client).to receive(:container_metrics) { |args| filter = args[:logcache_filter] - }.and_return(traffic_controller_response) + }.and_return(log_cache_response) expected_envelope = Loggregator::V2::Envelope.new( source_id: process.app.guid, @@ -162,28 +159,24 @@ def make_actual_lrp(instance_guid:, index:, state:, error:, since:) 'source_id' => ::Diego::Bbs::Models::MetricTagValue.new(static: process.guid), } } - let(:traffic_controller_response) do - [ - ::TrafficController::Models::Envelope.new( - origin: 'does-anyone-even-know?', - eventType: ::TrafficController::Models::Envelope::EventType::ContainerMetric, - containerMetric: ::TrafficController::Models::ContainerMetric.new( - instanceIndex: 0, - cpuPercentage: 3.92, - memoryBytes: 564, - diskBytes: 5000, - memoryBytesQuota: 1234, - diskBytesQuota: 10234, - ), - ), - ] + let(:log_cache_response) do + container_metric_batch = ::Logcache::ContainerMetricBatch.new + container_metric_batch.instance_index = 0 + container_metric_batch.cpu_percentage = 3.92 + container_metric_batch.memory_bytes = 564 + container_metric_batch.disk_bytes = 5000 + container_metric_batch.log_rate = 5 + container_metric_batch.memory_bytes_quota = 1234 + container_metric_batch.disk_bytes_quota = 10234 + container_metric_batch.log_rate_limit = 10 + [container_metric_batch] end it 'gets metrics for the process and does not filter on the source_id' do - expect(traffic_controller_client). + expect(log_cache_client). to receive(:container_metrics). with(auth_token: 'my-token', source_guid: process.guid, logcache_filter: anything). - and_return(traffic_controller_response) + and_return(log_cache_response) expect(instances_reporter.stats_for_app(process)).to eq([expected_stats_response, []]) end @@ -215,19 +208,13 @@ def make_actual_lrp(instance_guid:, index:, state:, error:, since:) end end - context 'when traffic controller somehow returns a partial response without cpuPercentage' do + context 'when log cache somehow returns a partial response without cpu_percentage' do # We aren't exactly sure how this happens, but it can happen on an overloaded deployment, see #156707836 - let(:traffic_controller_response) do - [ - ::TrafficController::Models::Envelope.new( - origin: 'does-anyone-even-know?', - eventType: ::TrafficController::Models::Envelope::EventType::ContainerMetric, - containerMetric: ::TrafficController::Models::ContainerMetric.new( - instanceIndex: 0, - memoryBytes: 564, - ), - ), - ] + let(:log_cache_response) do + container_metric_batch = ::Logcache::ContainerMetricBatch.new + container_metric_batch.instance_index = 0 + container_metric_batch.memory_bytes = 564 + [container_metric_batch] end it 'sets all the stats to zero' do @@ -237,6 +224,7 @@ def make_actual_lrp(instance_guid:, index:, state:, error:, since:) cpu: 0, mem: 0, disk: 0, + log_rate: 0, }) end end @@ -278,6 +266,7 @@ def make_actual_lrp(instance_guid:, index:, state:, error:, since:) uptime: two_days_in_seconds, mem_quota: nil, disk_quota: nil, + log_rate_limit: nil, fds_quota: process.file_descriptors, usage: {} }, @@ -287,7 +276,7 @@ def make_actual_lrp(instance_guid:, index:, state:, error:, since:) end before do - allow(traffic_controller_client).to receive(:container_metrics).and_raise(error) + allow(log_cache_client).to receive(:container_metrics).and_raise(error) allow(instances_reporter).to receive(:logger).and_return(mock_logger) end @@ -316,7 +305,7 @@ def make_actual_lrp(instance_guid:, index:, state:, error:, since:) context 'when number of actual lrps > desired number of instances' do let(:desired_instances) { 0 } - let(:traffic_controller_response) { [] } + let(:log_cache_response) { [] } it 'ignores superfluous instances' do expect(instances_reporter.stats_for_app(process)).to eq([{}, []]) @@ -324,7 +313,7 @@ def make_actual_lrp(instance_guid:, index:, state:, error:, since:) end context 'when number of container metrics < desired number of instances' do - let(:traffic_controller_response) { [] } + let(:log_cache_response) { [] } it 'provides defaults for unreported instances' do result, _ = instances_reporter.stats_for_app(process) @@ -334,6 +323,7 @@ def make_actual_lrp(instance_guid:, index:, state:, error:, since:) cpu: 0, mem: 0, disk: 0, + log_rate: 0, }) end end @@ -360,11 +350,11 @@ def make_actual_lrp(instance_guid:, index:, state:, error:, since:) end end - context 'when an error is raised communicating with traffic controller' do + context 'when an error is raised communicating with log cache' do let(:error) { StandardError.new('tomato') } let(:mock_logger) { double(:logger, error: nil, debug: nil) } before do - allow(traffic_controller_client).to receive(:container_metrics). + allow(log_cache_client).to receive(:container_metrics). with(auth_token: 'my-token', source_guid: process.app.guid, logcache_filter: anything). and_raise(error) allow(instances_reporter).to receive(:logger).and_return(mock_logger) @@ -402,6 +392,7 @@ def make_actual_lrp(instance_guid:, index:, state:, error:, since:) uptime: two_days_in_seconds, mem_quota: nil, disk_quota: nil, + log_rate_limit: nil, fds_quota: process.file_descriptors, usage: {} }, @@ -411,7 +402,7 @@ def make_actual_lrp(instance_guid:, index:, state:, error:, since:) end before do - allow(traffic_controller_client).to receive(:container_metrics). + allow(log_cache_client).to receive(:container_metrics). with(auth_token: 'my-token', source_guid: process.app.guid, logcache_filter: anything). and_raise(error) allow(instances_reporter).to receive(:logger).and_return(mock_logger) diff --git a/spec/unit/lib/utils/multipart_parser_wrapper_spec.rb b/spec/unit/lib/utils/multipart_parser_wrapper_spec.rb deleted file mode 100644 index fa711e9c9be..00000000000 --- a/spec/unit/lib/utils/multipart_parser_wrapper_spec.rb +++ /dev/null @@ -1,65 +0,0 @@ -require 'spec_helper' - -RSpec.describe VCAP::MultipartParserWrapper do - subject(:parser) { VCAP::MultipartParserWrapper.new(body: body, boundary: boundary) } - describe '#next_part' do - let(:body) do - [ - "--#{boundary}", - "\r\n", - "\r\n", - first_part, - "\r\n", - "--#{boundary}", - "\r\n", - "\r\n", - second_part, - "\r\n", - "--#{boundary}--", - ].join - end - let(:boundary) { 'boundary-guid' } - let(:first_part) { "part one\r\n data" } - let(:second_part) { 'part two data' } - - it 'can return the first part' do - expect(parser.next_part).to eq("part one\r\n data") - end - - it 'can read more than one part' do - expect(parser.next_part).to eq("part one\r\n data") - expect(parser.next_part).to eq('part two data') - end - - context 'when there are no parts left' do - it 'returns nil' do - expect(parser.next_part).to eq("part one\r\n data") - expect(parser.next_part).to eq('part two data') - expect(parser.next_part).to be_nil - end - end - - context 'when there body contains no parts' do - let(:body) { "\r\n--#{boundary}--\r\n" } - it 'returns nil' do - expect(parser.next_part).to be_nil - end - end - - context 'when the body is empty' do - let(:body) { '' } - - it 'returns nil' do - expect(parser.next_part).to be_nil - end - end - - context 'when the body is not a valid multipart response' do - let(:body) { 'potato' } - - it 'returns nil' do - expect(parser.next_part).to be_nil - end - end - end -end From bb70156fe78cf9cafa2a1b4eabb4228c13340d56 Mon Sep 17 00:00:00 2001 From: Matthew Kocher Date: Thu, 18 Aug 2022 18:38:05 +0000 Subject: [PATCH 10/20] Rename log_rate_limit_per_second to log-rate-limit-per-second All our archeology seems to indicate that dashes should be preferred in manifests as they are yaml and yaml prefers kebab-case to snake_case. We also uncovered some oddities around setting process level properties on the top level app object. We added tests to document that disk and memory don't show up in the diff when changed on the app instead of the process. We also discovered that proprties like health-check that shouldbe kebab-case were not working at the app level and fixed them to be consistent. Signed-off-by: Rebecca Roberts Co-authored-by: Rebecca Roberts --- app/actions/space_diff_manifest.rb | 8 +- .../process_properties_presenter.rb | 2 +- spec/request/app_manifests_spec.rb | 8 +- spec/request/space_manifests_spec.rb | 12 +-- spec/unit/actions/space_diff_manifest_spec.rb | 78 +++++++++++++++---- .../messages/app_manifest_message_spec.rb | 24 +++--- .../v3/app_manifest_presenter_spec.rb | 8 +- .../process_properties_presenter_spec.rb | 2 +- 8 files changed, 94 insertions(+), 48 deletions(-) diff --git a/app/actions/space_diff_manifest.rb b/app/actions/space_diff_manifest.rb index 63992b66a67..1ab45b5bf2f 100644 --- a/app/actions/space_diff_manifest.rb +++ b/app/actions/space_diff_manifest.rb @@ -14,7 +14,9 @@ class << self # rubocop:todo Metrics/CyclomaticComplexity def generate_diff(app_manifests, space) json_diff = [] - recognized_top_level_keys = AppManifestMessage.allowed_keys.map(&:to_s) + + recognized_top_level_keys = AppManifestMessage.allowed_keys.map(&:to_s).map(&:dasherize) + app_manifests = normalize_units(app_manifests) app_manifests.each_with_index do |manifest_app_hash, index| manifest_app_hash = filter_manifest_app_hash(manifest_app_hash) @@ -99,7 +101,7 @@ def filter_manifest_app_hash(manifest_app_hash) 'type', 'command', 'disk_quota', - 'log_rate_limit_per_second', + 'log-rate-limit-per-second', 'health-check-http-endpoint', 'health-check-invocation-timeout', 'health-check-type', @@ -161,7 +163,7 @@ def normalize_units(manifest_app_hash) end end - byte_measurement_key_words = ['log_rate_limit_per_second'] + byte_measurement_key_words = ['log-rate-limit-per-second'] manifest_app_hash.each_with_index do |process_hash, index| byte_measurement_key_words.each do |key| value = process_hash[key] diff --git a/app/presenters/v3/app_manifest_presenters/process_properties_presenter.rb b/app/presenters/v3/app_manifest_presenters/process_properties_presenter.rb index 82180777924..904e5a8a9db 100644 --- a/app/presenters/v3/app_manifest_presenters/process_properties_presenter.rb +++ b/app/presenters/v3/app_manifest_presenters/process_properties_presenter.rb @@ -14,7 +14,7 @@ def process_hash(process) 'instances' => process.instances, 'memory' => add_units(process.memory), 'disk_quota' => add_units(process.disk_quota), - 'log_rate_limit_per_second' => add_units_log_rate_limit(process.log_rate_limit), + 'log-rate-limit-per-second' => add_units_log_rate_limit(process.log_rate_limit), 'command' => process.command, 'health-check-type' => process.health_check_type, 'health-check-http-endpoint' => process.health_check_http_endpoint, diff --git a/spec/request/app_manifests_spec.rb b/spec/request/app_manifests_spec.rb index 62a27bddaf7..14ad274fcb0 100644 --- a/spec/request/app_manifests_spec.rb +++ b/spec/request/app_manifests_spec.rb @@ -108,7 +108,7 @@ 'instances' => process.instances, 'memory' => "#{process.memory}M", 'disk_quota' => "#{process.disk_quota}M", - 'log_rate_limit_per_second' => '1M', + 'log-rate-limit-per-second' => '1M', 'health-check-type' => process.health_check_type, }, { @@ -116,7 +116,7 @@ 'instances' => worker_process.instances, 'memory' => "#{worker_process.memory}M", 'disk_quota' => "#{worker_process.disk_quota}M", - 'log_rate_limit_per_second' => '1M', + 'log-rate-limit-per-second' => '1M', 'command' => worker_process.command, 'health-check-type' => worker_process.health_check_type, 'health-check-http-endpoint' => worker_process.health_check_http_endpoint, @@ -207,7 +207,7 @@ 'instances' => process.instances, 'memory' => "#{process.memory}M", 'disk_quota' => "#{process.disk_quota}M", - 'log_rate_limit_per_second' => -1, + 'log-rate-limit-per-second' => -1, 'health-check-type' => process.health_check_type, }, { @@ -215,7 +215,7 @@ 'instances' => worker_process.instances, 'memory' => "#{worker_process.memory}M", 'disk_quota' => "#{worker_process.disk_quota}M", - 'log_rate_limit_per_second' => '1M', + 'log-rate-limit-per-second' => '1M', 'command' => worker_process.command, 'health-check-type' => worker_process.health_check_type, 'health-check-http-endpoint' => worker_process.health_check_http_endpoint, diff --git a/spec/request/space_manifests_spec.rb b/spec/request/space_manifests_spec.rb index 529eac982b1..7fe1263b092 100644 --- a/spec/request/space_manifests_spec.rb +++ b/spec/request/space_manifests_spec.rb @@ -27,7 +27,7 @@ 'instances' => 4, 'memory' => '2048MB', 'disk_quota' => '1.5GB', - 'log_rate_limit_per_second' => '1MB', + 'log-rate-limit-per-second' => '1MB', 'buildpack' => buildpack.name, 'stack' => buildpack.stack, 'command' => 'new-command', @@ -74,7 +74,7 @@ 'instances' => 3, 'memory' => '2048MB', 'disk_quota' => '1.5GB', - 'log_rate_limit_per_second' => '100KB', + 'log-rate-limit-per-second' => '100KB', 'buildpack' => buildpack.name, 'stack' => buildpack.stack, 'command' => 'newer-command', @@ -247,7 +247,7 @@ 'instances' => 4, 'memory' => '2048MB', 'disk_quota' => '1.5GB', - 'log_rate_limit_per_second' => '1GB', + 'log-rate-limit-per-second' => '1GB', 'buildpack' => buildpack.name, 'stack' => buildpack.stack, 'command' => 'new-command', @@ -271,7 +271,7 @@ 'instances' => 4, 'memory' => '2048MB', 'disk_quota' => '1.5GB', - 'log_rate_limit_per_second' => '-1B', + 'log-rate-limit-per-second' => '-1B', 'buildpack' => buildpack.name, 'stack' => buildpack.stack, 'command' => 'new-command', @@ -467,7 +467,7 @@ 'version' => 1, 'applications' => [ { 'name' => app1_model.name, - 'log_rate_limit_per_second' => -1 + 'log-rate-limit-per-second' => -1 }, ] }.to_yaml @@ -521,7 +521,7 @@ 'instances' => 4, 'memory' => '2048MB', 'disk_quota' => '1.5GB', - 'log_rate_limit_per_second' => '300B', + 'log-rate-limit-per-second' => '300B', 'buildpack' => buildpack.name, 'stack' => buildpack.stack, 'command' => 'new-command', diff --git a/spec/unit/actions/space_diff_manifest_spec.rb b/spec/unit/actions/space_diff_manifest_spec.rb index a02b26a5cf3..1c8b4a849a8 100644 --- a/spec/unit/actions/space_diff_manifest_spec.rb +++ b/spec/unit/actions/space_diff_manifest_spec.rb @@ -21,7 +21,7 @@ module VCAP::CloudController 'instances' => process1.instances, 'memory' => '1024M', 'disk_quota' => '1024M', - 'log_rate_limit_per_second' => '1M', + 'log-rate-limit-per-second' => '1M', 'health-check-type' => process1.health_check_type }, { @@ -29,7 +29,7 @@ module VCAP::CloudController 'instances' => process2.instances, 'memory' => '1024M', 'disk_quota' => '1024M', - 'log_rate_limit_per_second' => '1M', + 'log-rate-limit-per-second' => '1M', 'health-check-type' => process2.health_check_type } ] @@ -71,13 +71,13 @@ module VCAP::CloudController context 'when there are changes in the manifest' do before do - default_manifest['applications'][0]['random_route'] = true + default_manifest['applications'][0]['random-route'] = true default_manifest['applications'][0]['stack'] = 'big brother' end it 'returns the correct diff' do expect(subject).to match_array([ - { 'op' => 'add', 'path' => '/applications/0/random_route', 'value' => true }, + { 'op' => 'add', 'path' => '/applications/0/random-route', 'value' => true }, { 'op' => 'replace', 'path' => '/applications/0/stack', 'was' => process1.stack.name, 'value' => 'big brother' }, ]) end @@ -266,7 +266,7 @@ module VCAP::CloudController before do default_manifest['applications'][0]['processes'][0]['memory'] = '1G' default_manifest['applications'][0]['processes'][0]['disk_quota'] = '1G' - default_manifest['applications'][0]['processes'][0]['log_rate_limit_per_second'] = '1024K' + default_manifest['applications'][0]['processes'][0]['log-rate-limit-per-second'] = '1024K' end it 'returns an empty diff' do expect(subject).to eq([]) @@ -277,13 +277,15 @@ module VCAP::CloudController before do default_manifest['applications'][0]['processes'][0]['memory'] = '2G' default_manifest['applications'][0]['processes'][0]['disk_quota'] = '4G' - default_manifest['applications'][0]['processes'][0]['log_rate_limit_per_second'] = '2G' + default_manifest['applications'][0]['processes'][0]['health-check-type'] = 'process' + default_manifest['applications'][0]['processes'][0]['log-rate-limit-per-second'] = '2G' end it 'returns the diff formatted' do expect(subject).to eq([ { 'op' => 'replace', 'path' => '/applications/0/processes/0/memory', 'value' => '2048M', 'was' => '1024M' }, { 'op' => 'replace', 'path' => '/applications/0/processes/0/disk_quota', 'value' => '4096M', 'was' => '1024M' }, - { 'op' => 'replace', 'path' => '/applications/0/processes/0/log_rate_limit_per_second', 'value' => '2G', 'was' => '1M' }, + { 'op' => 'replace', 'path' => '/applications/0/processes/0/log-rate-limit-per-second', 'value' => '2G', 'was' => '1M' }, + { 'op' => 'replace', 'path' => '/applications/0/processes/0/health-check-type', 'value' => 'process', 'was' => 'port' }, ]) end end @@ -322,29 +324,71 @@ module VCAP::CloudController end context 'when updating app-level configurations' do - before do - default_manifest['applications'][0]['memory'] = '1G' - default_manifest['applications'][0]['disk_quota'] = '1G' - default_manifest['applications'][0]['log_rate_limit_per_second'] = '1024K' + context 'when nothing has changed' do + before do + default_manifest['applications'][0]['log-rate-limit-per-second'] = '1024K' + end + + it 'returns an empty diff if the field is equivalent' do + expect(subject).to eq([]) + end end - it 'returns an empty diff if the field is equivalent' do - expect(subject).to eq([]) + context 'when trying to change memory and disk quota' do + before do + default_manifest['applications'][0]['memory'] = '99G' + default_manifest['applications'][0]['disk_quota'] = '99G' + end + + it 'returns an empty diff because for some reason we ignore these fields at the app level' do + expect(subject).to eq([]) + end + end + + context 'when things have changed' do + before do + default_manifest['applications'][0]['health-check-type'] = 'process' + default_manifest['applications'][0]['instances'] = 9 + default_manifest['applications'][0]['log-rate-limit-per-second'] = '1G' + end + + it 'displays in the diff' do + expect(subject).to eq([ + { + 'op' => 'replace', + 'path' => '/applications/0/health-check-type', + 'was' => 'port', + 'value' => 'process' + }, + { + 'op' => 'replace', + 'path' => '/applications/0/instances', + 'was' => 1, + 'value' => 9 + }, + { + 'op' => 'replace', + 'path' => '/applications/0/log-rate-limit-per-second', + 'was' => '1M', + 'value' => '1G' + } + ]) + end end end end - context 'when the log_rate_limit_per_second is unlimited' do + context 'when the log-rate-limit-per-second is unlimited' do before do - default_manifest['applications'][0]['log_rate_limit_per_second'] = -1 + default_manifest['applications'][0]['log-rate-limit-per-second'] = '-1B' end it 'displays -1 in the diff' do expect(subject).to include( { 'op' => 'replace', - 'path' => '/applications/0/log_rate_limit_per_second', - 'value' => -1, + 'path' => '/applications/0/log-rate-limit-per-second', + 'value' => '-1B', 'was' => '1M' } ) diff --git a/spec/unit/messages/app_manifest_message_spec.rb b/spec/unit/messages/app_manifest_message_spec.rb index d3c4f19b769..37cb3030eb3 100644 --- a/spec/unit/messages/app_manifest_message_spec.rb +++ b/spec/unit/messages/app_manifest_message_spec.rb @@ -104,8 +104,8 @@ module VCAP::CloudController end end - describe 'log_rate_limit_per_second' do - context 'when log_rate_limit_per_second unit is not part of expected set of values' do + describe 'log-rate-limit-per-second' do + context 'when log-rate-limit-per-second unit is not part of expected set of values' do let(:params_from_yaml) { { name: 'eugene', log_rate_limit_per_second: '200INVALID' } } it 'is not valid' do @@ -119,7 +119,7 @@ module VCAP::CloudController end end - context 'when log_rate_limit_per_second is less than -1 bytes' do + context 'when log-rate-limit-per-second is less than -1 bytes' do let(:params_from_yaml) { { name: 'eugene', log_rate_limit_per_second: '-1M' } } it 'is not valid' do @@ -131,7 +131,7 @@ module VCAP::CloudController end end - context 'when log_rate_limit_per_second is not numeric' do + context 'when log-rate-limit-per-second is not numeric' do let(:params_from_yaml) { { name: 'eugene', log_rate_limit_per_second: 'gerg herscheisers' } } it 'is not valid' do @@ -145,7 +145,7 @@ module VCAP::CloudController end end - context 'when log_rate_limit_per_second is an unlimited amount' do + context 'when log-rate-limit-per-second is an unlimited amount' do context 'specified as -1' do let(:params_from_yaml) { { name: 'eugene', log_rate_limit_per_second: -1 } } @@ -164,7 +164,7 @@ module VCAP::CloudController end end - context 'when log_rate_limit_per_second is in megabytes' do + context 'when log-rate-limit-per-second is in megabytes' do let(:params_from_yaml) { { name: 'eugene', log_rate_limit_per_second: '1MB' } } it 'is valid' do @@ -642,7 +642,7 @@ module VCAP::CloudController 'instances' => -30, 'memory' => 'potato', 'disk_quota' => '100', - 'log_rate_limit_per_second' => 'kumara', + 'log-rate-limit-per-second' => 'kumara', 'health_check_type' => 'sweet_potato', 'health_check_http_endpoint' => '/healthcheck_potato', 'health_check_invocation_timeout' => 'yucca', @@ -656,7 +656,7 @@ module VCAP::CloudController 'instances' => 'cassava', 'memory' => 'potato', 'disk_quota' => '100', - 'log_rate_limit_per_second' => '100', + 'log-rate-limit-per-second' => '100', 'health_check_type' => 'sweet_potato', 'health_check_http_endpoint' => '/healthcheck_potato', 'health_check_invocation_timeout' => 'yucca', @@ -714,10 +714,10 @@ module VCAP::CloudController end end - context 'log_rate_limit_per_second' do - context 'when log_rate_limit_per_second is an unlimited amount' do + context 'log-rate-limit-per-second' do + context 'when log-rate-limit-per-second is an unlimited amount' do context 'specified as -1' do - let(:params_from_yaml) { { name: 'eugene', processes: [{ 'type' => 'foo', 'log_rate_limit_per_second' => -1 }] } } + let(:params_from_yaml) { { name: 'eugene', processes: [{ 'type' => 'foo', 'log-rate-limit-per-second' => -1 }] } } it 'is valid' do message = AppManifestMessage.create_from_yml(params_from_yaml) @@ -725,7 +725,7 @@ module VCAP::CloudController end end context 'with a bytes suffix' do - let(:params_from_yaml) { { name: 'eugene', processes: [{ 'type' => 'foo', 'log_rate_limit_per_second' => '-1B' }] } } + let(:params_from_yaml) { { name: 'eugene', processes: [{ 'type' => 'foo', 'log-rate-limit-per-second' => '-1B' }] } } it 'is valid' do message = AppManifestMessage.create_from_yml(params_from_yaml) diff --git a/spec/unit/presenters/v3/app_manifest_presenter_spec.rb b/spec/unit/presenters/v3/app_manifest_presenter_spec.rb index 8d76975cc81..31df7131d89 100644 --- a/spec/unit/presenters/v3/app_manifest_presenter_spec.rb +++ b/spec/unit/presenters/v3/app_manifest_presenter_spec.rb @@ -120,7 +120,7 @@ module VCAP::CloudController::Presenters::V3 'type' => process1.type, 'instances' => process1.instances, 'memory' => "#{process1.memory}M", - 'log_rate_limit_per_second' => '1M', + 'log-rate-limit-per-second' => '1M', 'disk_quota' => "#{process1.disk_quota}M", 'command' => process1.command, 'health-check-type' => process1.health_check_type, @@ -130,7 +130,7 @@ module VCAP::CloudController::Presenters::V3 { 'type' => process2.type, 'instances' => process2.instances, - 'log_rate_limit_per_second' => '1M', + 'log-rate-limit-per-second' => '1M', 'memory' => "#{process2.memory}M", 'disk_quota' => "#{process2.disk_quota}M", 'health-check-type' => process2.health_check_type, @@ -169,7 +169,7 @@ module VCAP::CloudController::Presenters::V3 { 'type' => process1.type, 'instances' => process1.instances, - 'log_rate_limit_per_second' => '1M', + 'log-rate-limit-per-second' => '1M', 'memory' => "#{process1.memory}M", 'disk_quota' => "#{process1.disk_quota}M", 'health-check-type' => process1.health_check_type, @@ -177,7 +177,7 @@ module VCAP::CloudController::Presenters::V3 { 'type' => process2.type, 'instances' => process2.instances, - 'log_rate_limit_per_second' => '1M', + 'log-rate-limit-per-second' => '1M', 'memory' => "#{process2.memory}M", 'disk_quota' => "#{process2.disk_quota}M", 'health-check-type' => process2.health_check_type, diff --git a/spec/unit/presenters/v3/app_manifest_presenters/process_properties_presenter_spec.rb b/spec/unit/presenters/v3/app_manifest_presenters/process_properties_presenter_spec.rb index 50e19d29680..5945871e56a 100644 --- a/spec/unit/presenters/v3/app_manifest_presenters/process_properties_presenter_spec.rb +++ b/spec/unit/presenters/v3/app_manifest_presenters/process_properties_presenter_spec.rb @@ -43,7 +43,7 @@ module VCAP::CloudController::Presenters::V3::AppManifestPresenters 'instances' => 1, 'memory' => '1024M', 'disk_quota' => '1024M', - 'log_rate_limit_per_second' => '1M', + 'log-rate-limit-per-second' => '1M', 'health-check-type' => 'port', }) end From a9e65c0713e4122586eeccf70f98def783e69573 Mon Sep 17 00:00:00 2001 From: Matthew Kocher Date: Thu, 18 Aug 2022 21:29:32 +0000 Subject: [PATCH 11/20] Manifests handle -1 and 0 log-rate-limits without units Signed-off-by: Rebecca Roberts Co-authored-by: Rebecca Roberts wip: group with app manifest message updates for unlimited --- app/actions/space_diff_manifest.rb | 6 ++++- app/messages/app_manifest_message.rb | 17 ++++++------- spec/request/space_manifests_spec.rb | 2 +- spec/unit/actions/space_diff_manifest_spec.rb | 21 ++++++++++++---- .../messages/app_manifest_message_spec.rb | 25 +++++++++++++++++-- 5 files changed, 53 insertions(+), 18 deletions(-) diff --git a/app/actions/space_diff_manifest.rb b/app/actions/space_diff_manifest.rb index 1ab45b5bf2f..6210833da98 100644 --- a/app/actions/space_diff_manifest.rb +++ b/app/actions/space_diff_manifest.rb @@ -182,7 +182,11 @@ def convert_to_mb(human_readable_byte_value, attribute_name) end def normalize_unit(non_normalized_value, attribute_name) - byte_converter.human_readable_byte_value(byte_converter.convert_to_b(non_normalized_value)) + if %w[-1 0].include?(non_normalized_value) + non_normalized_value + else + byte_converter.human_readable_byte_value(byte_converter.convert_to_b(non_normalized_value)) + end rescue ByteConverter::InvalidUnitsError "#{attribute_name} must use a supported unit: B, K, KB, M, MB, G, GB, T, or TB" rescue ByteConverter::NonNumericError diff --git a/app/messages/app_manifest_message.rb b/app/messages/app_manifest_message.rb index a9c4b846af0..8e21faf9c94 100644 --- a/app/messages/app_manifest_message.rb +++ b/app/messages/app_manifest_message.rb @@ -292,14 +292,17 @@ def convert_to_mb(human_readable_byte_value) def convert_to_bytes_per_second(human_readable_byte_value) return nil unless human_readable_byte_value.present? - return -1 if human_readable_byte_value == -1 # unlimited + return -1 if human_readable_byte_value == '-1' + return 0 if human_readable_byte_value == '0' byte_converter.convert_to_b(human_readable_byte_value.strip) rescue ByteConverter::InvalidUnitsError, ByteConverter::NonNumericError end - def validate_byte_format(human_readable_byte_value, attribute_name) - byte_converter.convert_to_mb(human_readable_byte_value) + def validate_byte_format(human_readable_byte_value, attribute_name, allow_unlimited: false) + unless (allow_unlimited && human_readable_byte_value == '-1') || human_readable_byte_value == '0' + byte_converter.convert_to_mb(human_readable_byte_value) + end nil rescue ByteConverter::InvalidUnitsError @@ -393,9 +396,7 @@ def validate_processes! type = process[:type] memory_error = validate_byte_format(process[:memory], 'Memory') disk_error = validate_byte_format(process[:disk_quota], 'Disk quota') - if process[:log_rate_limit_per_second] != -1 - log_rate_limit_error = validate_byte_format(process[:log_rate_limit_per_second], 'Log rate limit per second') - end + log_rate_limit_error = validate_byte_format(process[:log_rate_limit_per_second], 'Log rate limit per second', allow_unlimited: true) add_process_error!(memory_error, type) if memory_error add_process_error!(disk_error, type) if disk_error add_process_error!(log_rate_limit_error, type) if log_rate_limit_error @@ -419,9 +420,7 @@ def validate_sidecars! def validate_top_level_web_process! memory_error = validate_byte_format(memory, 'Memory') disk_error = validate_byte_format(disk_quota, 'Disk quota') - if log_rate_limit_per_second != -1 - log_rate_limit_error = validate_byte_format(log_rate_limit_per_second, 'Log rate limit per second') - end + log_rate_limit_error = validate_byte_format(log_rate_limit_per_second, 'Log rate limit per second', allow_unlimited: true) add_process_error!(memory_error, ProcessTypes::WEB) if memory_error add_process_error!(disk_error, ProcessTypes::WEB) if disk_error add_process_error!(log_rate_limit_error, ProcessTypes::WEB) if log_rate_limit_error diff --git a/spec/request/space_manifests_spec.rb b/spec/request/space_manifests_spec.rb index 7fe1263b092..1c4a72b7858 100644 --- a/spec/request/space_manifests_spec.rb +++ b/spec/request/space_manifests_spec.rb @@ -467,7 +467,7 @@ 'version' => 1, 'applications' => [ { 'name' => app1_model.name, - 'log-rate-limit-per-second' => -1 + 'log-rate-limit-per-second' => '-1' }, ] }.to_yaml diff --git a/spec/unit/actions/space_diff_manifest_spec.rb b/spec/unit/actions/space_diff_manifest_spec.rb index 1c8b4a849a8..b0fa6b2a78a 100644 --- a/spec/unit/actions/space_diff_manifest_spec.rb +++ b/spec/unit/actions/space_diff_manifest_spec.rb @@ -378,17 +378,28 @@ module VCAP::CloudController end end - context 'when the log-rate-limit-per-second is unlimited' do - before do - default_manifest['applications'][0]['log-rate-limit-per-second'] = '-1B' + describe 'log-rate-limit-per-second' do + it 'can handle -1 for unlimted' do + default_manifest['applications'][0]['log-rate-limit-per-second'] = '-1' + + expect(subject).to include( + { + 'op' => 'replace', + 'path' => '/applications/0/log-rate-limit-per-second', + 'value' => '-1', + 'was' => '1M' + } + ) end - it 'displays -1 in the diff' do + it 'can handle 0 without units' do + default_manifest['applications'][0]['log-rate-limit-per-second'] = '0' + expect(subject).to include( { 'op' => 'replace', 'path' => '/applications/0/log-rate-limit-per-second', - 'value' => '-1B', + 'value' => '0', 'was' => '1M' } ) diff --git a/spec/unit/messages/app_manifest_message_spec.rb b/spec/unit/messages/app_manifest_message_spec.rb index 37cb3030eb3..84ed40cc421 100644 --- a/spec/unit/messages/app_manifest_message_spec.rb +++ b/spec/unit/messages/app_manifest_message_spec.rb @@ -147,7 +147,7 @@ module VCAP::CloudController context 'when log-rate-limit-per-second is an unlimited amount' do context 'specified as -1' do - let(:params_from_yaml) { { name: 'eugene', log_rate_limit_per_second: -1 } } + let(:params_from_yaml) { { name: 'eugene', 'log-rate-limit-per-second' => '-1' } } it 'is valid' do message = AppManifestMessage.create_from_yml(params_from_yaml) @@ -172,6 +172,26 @@ module VCAP::CloudController expect(message).to be_valid end end + + context 'when log-rate-limit-per-second is 0' do + let(:params_from_yaml) { { name: 'eugene', log_rate_limit_per_second: '0' } } + + it 'is valid' do + message = AppManifestMessage.create_from_yml(params_from_yaml) + expect(message).to be_valid + expect(message.manifest_process_scale_messages.first.log_rate_limit).to eq(0) + end + end + + context 'when log-rate-limit-per-second is 0TB' do + let(:params_from_yaml) { { name: 'eugene', log_rate_limit_per_second: '0TB' } } + + it 'is valid' do + message = AppManifestMessage.create_from_yml(params_from_yaml) + expect(message).to be_valid + expect(message.manifest_process_scale_messages.first.log_rate_limit).to eq(0) + end + end end describe 'buildpack' do @@ -717,11 +737,12 @@ module VCAP::CloudController context 'log-rate-limit-per-second' do context 'when log-rate-limit-per-second is an unlimited amount' do context 'specified as -1' do - let(:params_from_yaml) { { name: 'eugene', processes: [{ 'type' => 'foo', 'log-rate-limit-per-second' => -1 }] } } + let(:params_from_yaml) { { name: 'eugene', processes: [{ 'type' => 'foo', 'log-rate-limit-per-second' => '-1' }] } } it 'is valid' do message = AppManifestMessage.create_from_yml(params_from_yaml) expect(message).to be_valid + expect(message.manifest_process_scale_messages.first.log_rate_limit).to eq(-1) end end context 'with a bytes suffix' do From 5c42fada685f753d0f387ea43f9a81a742f3bb0a Mon Sep 17 00:00:00 2001 From: Andrew Crump Date: Fri, 19 Aug 2022 00:48:42 +0000 Subject: [PATCH 12/20] Validate log rate limit is not too large - Adds validation to messages for builds, manifest processes and tasks - Avoids 'An unknown error occurred' caused by the database insert failing due to an out of range error. [#182969510](https://www.pivotaltracker.com/story/show/182969510) Co-authored-by: Matthew Kocher --- app/messages/build_create_message.rb | 2 +- app/messages/manifest_process_scale_message.rb | 4 +++- app/messages/task_create_message.rb | 2 +- spec/unit/messages/build_create_message_spec.rb | 14 +++++++++++++- .../manifest_process_scale_message_spec.rb | 12 ++++++++++++ spec/unit/messages/task_create_message_spec.rb | 9 +++++++++ 6 files changed, 39 insertions(+), 4 deletions(-) diff --git a/app/messages/build_create_message.rb b/app/messages/build_create_message.rb index c07f0b6bc72..cd51643edc7 100644 --- a/app/messages/build_create_message.rb +++ b/app/messages/build_create_message.rb @@ -12,7 +12,7 @@ def self.lifecycle_requested? validates :staging_disk_in_mb, numericality: { only_integer: true, greater_than: 0 }, allow_nil: true validates :staging_memory_in_mb, numericality: { only_integer: true, greater_than: 0 }, allow_nil: true - validates :staging_log_rate_limit_bytes_per_second, numericality: { only_integer: true, greater_than_or_equal_to: -1 }, allow_nil: true + validates :staging_log_rate_limit_bytes_per_second, numericality: { only_integer: true, greater_than_or_equal_to: -1, less_than_or_equal_to: MAX_DB_BIGINT }, allow_nil: true validates_with NoAdditionalKeysValidator validates_with LifecycleValidator, if: lifecycle_requested? diff --git a/app/messages/manifest_process_scale_message.rb b/app/messages/manifest_process_scale_message.rb index 67975869fb0..1905f089f57 100644 --- a/app/messages/manifest_process_scale_message.rb +++ b/app/messages/manifest_process_scale_message.rb @@ -12,7 +12,9 @@ class ManifestProcessScaleMessage < BaseMessage validates :instances, numericality: { only_integer: true, greater_than_or_equal_to: 0, less_than_or_equal_to: 2000000 }, allow_nil: true validates :memory, numericality: { only_integer: true, greater_than: 0, message: INVALID_MB_VALUE_ERROR }, allow_nil: true validates :disk_quota, numericality: { only_integer: true, greater_than: 0, message: INVALID_MB_VALUE_ERROR }, allow_nil: true - validates :log_rate_limit, numericality: { only_integer: true, greater_than_or_equal_to: -1, message: INVALID_QUOTA_VALUE_ERROR }, allow_nil: true + validates :log_rate_limit, + numericality: { only_integer: true, greater_than_or_equal_to: -1, less_than_or_equal_to: MAX_DB_BIGINT, message: INVALID_QUOTA_VALUE_ERROR }, + allow_nil: true def to_process_scale_message ProcessScaleMessage.new({ diff --git a/app/messages/task_create_message.rb b/app/messages/task_create_message.rb index 32f90554a80..8392f2030d0 100644 --- a/app/messages/task_create_message.rb +++ b/app/messages/task_create_message.rb @@ -12,7 +12,7 @@ def self.validate_template? validates :disk_in_mb, numericality: { only_integer: true, greater_than: 0 }, allow_nil: true validates :memory_in_mb, numericality: { only_integer: true, greater_than: 0 }, allow_nil: true - validates :log_rate_limit_in_bytes_per_second, numericality: { only_integer: true, greater_than: -2 }, allow_nil: true + validates :log_rate_limit_in_bytes_per_second, numericality: { only_integer: true, greater_than: -2, less_than_or_equal_to: MAX_DB_BIGINT }, allow_nil: true validates :droplet_guid, guid: true, allow_nil: true validates :template_process_guid, guid: true, if: validate_template? validate :has_command diff --git a/spec/unit/messages/build_create_message_spec.rb b/spec/unit/messages/build_create_message_spec.rb index a9d6b1cc46b..dae6f928f95 100644 --- a/spec/unit/messages/build_create_message_spec.rb +++ b/spec/unit/messages/build_create_message_spec.rb @@ -227,7 +227,7 @@ module VCAP::CloudController end end - context 'when the value is out of bounds' do + context 'when the value is less than -1' do let(:params) do { package: { guid: 'some-guid' }, @@ -238,6 +238,18 @@ module VCAP::CloudController expect(build_create_message.valid?).to be false end end + + context 'when the value is too large' do + let(:params) do + { + package: { guid: 'some-guid' }, + staging_log_rate_limit_bytes_per_second: 2**63 + } + end + it 'is invalid' do + expect(build_create_message.valid?).to be false + end + end end describe '#environment variables' do diff --git a/spec/unit/messages/manifest_process_scale_message_spec.rb b/spec/unit/messages/manifest_process_scale_message_spec.rb index bf7571fb9f6..7831ed9e1dd 100644 --- a/spec/unit/messages/manifest_process_scale_message_spec.rb +++ b/spec/unit/messages/manifest_process_scale_message_spec.rb @@ -177,6 +177,18 @@ module VCAP::CloudController expect(message.errors.full_messages).to include('Log rate limit must be an integer greater than or equal to -1') end end + + context 'when log_rate_limit is too large' do + let(:params) { { log_rate_limit: 2**63 } } + + it 'is not valid' do + message = ManifestProcessScaleMessage.new(params) + + expect(message).not_to be_valid + expect(message.errors.count).to eq(1) + expect(message.errors.full_messages).to include('Log rate limit must be an integer greater than or equal to -1') + end + end end context 'when we have more than one error' do diff --git a/spec/unit/messages/task_create_message_spec.rb b/spec/unit/messages/task_create_message_spec.rb index fac1a3d449e..3d7730d348d 100644 --- a/spec/unit/messages/task_create_message_spec.rb +++ b/spec/unit/messages/task_create_message_spec.rb @@ -192,6 +192,15 @@ module VCAP::CloudController expect(message).to_not be_valid expect(message.errors.full_messages).to include('Log rate limit in bytes per second must be greater than -2') end + + it 'may not be too large' do + body['log_rate_limit_in_bytes_per_second'] = 2**63 + + message = TaskCreateMessage.new(body) + + expect(message).to_not be_valid + expect(message.errors.full_messages).to include('Log rate limit in bytes per second must be less than or equal to 9223372036854775807') + end end describe 'template' do From e30a91f1d8fbcf949e04a517243eae3e9b2e80a2 Mon Sep 17 00:00:00 2001 From: Andrew Crump Date: Fri, 19 Aug 2022 02:15:06 +0000 Subject: [PATCH 13/20] Address rspec warning for potential false positive Fixes: Using the `raise_error` matcher without providing a specific error or message risks false positives [#182969510](https://www.pivotaltracker.com/story/show/182969510) Co-authored-by: Rebecca Roberts --- spec/unit/models/runtime/task_model_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/unit/models/runtime/task_model_spec.rb b/spec/unit/models/runtime/task_model_spec.rb index deb8af74bfa..88934bd206e 100644 --- a/spec/unit/models/runtime/task_model_spec.rb +++ b/spec/unit/models/runtime/task_model_spec.rb @@ -295,7 +295,7 @@ module VCAP::CloudController log_rate_limit: -2, app: app, ) - }.to raise_error + }.to raise_error(Sequel::ValidationFailed, /log_rate_limit must be greater than or equal to -1/) end describe 'space quotas' do From c072bfdb12cdd8c168c27b2301ed1b8d48a7e550 Mon Sep 17 00:00:00 2001 From: Andrew Crump Date: Sat, 20 Aug 2022 02:30:44 +0000 Subject: [PATCH 14/20] Preserve log rate limit after rolling deployment - The log rate limit was discarded when cloning the web process [#183038057](https://www.pivotaltracker.com/story/show/183038057) --- app/actions/deployment_create.rb | 1 + spec/unit/actions/deployment_create_spec.rb | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/app/actions/deployment_create.rb b/app/actions/deployment_create.rb index 9f69f9be8d7..af04731279d 100644 --- a/app/actions/deployment_create.rb +++ b/app/actions/deployment_create.rb @@ -109,6 +109,7 @@ def clone_existing_web_process(app, revision) memory: web_process.memory, file_descriptors: web_process.file_descriptors, disk_quota: web_process.disk_quota, + log_rate_limit: web_process.log_rate_limit, metadata: web_process.metadata, # execution_metadata, not labels & annotations metadata! detected_buildpack: web_process.detected_buildpack, health_check_timeout: web_process.health_check_timeout, diff --git a/spec/unit/actions/deployment_create_spec.rb b/spec/unit/actions/deployment_create_spec.rb index e683742beed..fe9e414f1fd 100644 --- a/spec/unit/actions/deployment_create_spec.rb +++ b/spec/unit/actions/deployment_create_spec.rb @@ -5,7 +5,7 @@ module VCAP::CloudController RSpec.describe DeploymentCreate do let(:app) { AppModel.make(desired_state: ProcessModel::STARTED) } - let!(:web_process) { ProcessModel.make(app: app, instances: 3) } + let!(:web_process) { ProcessModel.make(app: app, instances: 3, log_rate_limit: 101) } let(:original_droplet) { DropletModel.make(app: app, process_types: { 'web' => 'asdf' }) } let(:next_droplet) { DropletModel.make(app: app, process_types: { 'web' => '1234' }) } let!(:route1) { Route.make(space: app.space) } @@ -169,6 +169,7 @@ module VCAP::CloudController expect(deploying_web_process.memory).to eq(web_process.memory) expect(deploying_web_process.file_descriptors).to eq(web_process.file_descriptors) expect(deploying_web_process.disk_quota).to eq(web_process.disk_quota) + expect(deploying_web_process.log_rate_limit).to eq(web_process.log_rate_limit) expect(deploying_web_process.metadata).to eq(web_process.metadata) expect(deploying_web_process.detected_buildpack).to eq(web_process.detected_buildpack) expect(deploying_web_process.health_check_timeout).to eq(web_process.health_check_timeout) @@ -206,6 +207,7 @@ module VCAP::CloudController health_check_type: 'http', health_check_http_endpoint: '/old_dawg', health_check_invocation_timeout: 9, + log_rate_limit: 11, enable_ssh: true, ports: [], ) @@ -227,6 +229,7 @@ module VCAP::CloudController health_check_type: 'port', health_check_http_endpoint: '/new_cat', health_check_invocation_timeout: 10, + log_rate_limit: 12, enable_ssh: false, ports: nil, ) @@ -244,6 +247,7 @@ module VCAP::CloudController expect(deploying_web_process.memory).to eq(newer_web_process.memory) expect(deploying_web_process.file_descriptors).to eq(newer_web_process.file_descriptors) expect(deploying_web_process.disk_quota).to eq(newer_web_process.disk_quota) + expect(deploying_web_process.log_rate_limit).to eq(newer_web_process.log_rate_limit) expect(deploying_web_process.metadata).to eq(newer_web_process.metadata) expect(deploying_web_process.detected_buildpack).to eq(newer_web_process.detected_buildpack) expect(deploying_web_process.health_check_timeout).to eq(newer_web_process.health_check_timeout) From afef7c89e98feb45509e5ee64d802c96694f0bfe Mon Sep 17 00:00:00 2001 From: Andrew Crump Date: Mon, 22 Aug 2022 23:09:24 +0000 Subject: [PATCH 15/20] Combine sequel migrations [#183053488](https://www.pivotaltracker.com/story/show/183053488) --- ...0606172913_add_log_rate_limit_to_quota_definitions.rb | 5 ----- ...2309_add_log_rate_limit_to_space_quota_definitions.rb | 5 ----- .../20220608165055_add_log_rate_limit_to_processes.rb | 5 ----- .../20220621175820_add_log_rate_limit_to_tasks_table.rb | 5 ----- ...0220718211322_add_staging_log_rate_limit_to_builds.rb | 7 ------- db/migrations/20220822224900_add_log_rate_limit.rb | 9 +++++++++ 6 files changed, 9 insertions(+), 27 deletions(-) delete mode 100644 db/migrations/20220606172913_add_log_rate_limit_to_quota_definitions.rb delete mode 100644 db/migrations/20220607222309_add_log_rate_limit_to_space_quota_definitions.rb delete mode 100644 db/migrations/20220608165055_add_log_rate_limit_to_processes.rb delete mode 100644 db/migrations/20220621175820_add_log_rate_limit_to_tasks_table.rb delete mode 100644 db/migrations/20220718211322_add_staging_log_rate_limit_to_builds.rb create mode 100644 db/migrations/20220822224900_add_log_rate_limit.rb diff --git a/db/migrations/20220606172913_add_log_rate_limit_to_quota_definitions.rb b/db/migrations/20220606172913_add_log_rate_limit_to_quota_definitions.rb deleted file mode 100644 index 475e005dc17..00000000000 --- a/db/migrations/20220606172913_add_log_rate_limit_to_quota_definitions.rb +++ /dev/null @@ -1,5 +0,0 @@ -Sequel.migration do - change do - add_column :quota_definitions, :log_rate_limit, :Bignum, null: false, default: -1 - end -end diff --git a/db/migrations/20220607222309_add_log_rate_limit_to_space_quota_definitions.rb b/db/migrations/20220607222309_add_log_rate_limit_to_space_quota_definitions.rb deleted file mode 100644 index 4abca30f194..00000000000 --- a/db/migrations/20220607222309_add_log_rate_limit_to_space_quota_definitions.rb +++ /dev/null @@ -1,5 +0,0 @@ -Sequel.migration do - change do - add_column :space_quota_definitions, :log_rate_limit, :Bignum, null: false, default: -1 - end -end diff --git a/db/migrations/20220608165055_add_log_rate_limit_to_processes.rb b/db/migrations/20220608165055_add_log_rate_limit_to_processes.rb deleted file mode 100644 index d2825f53908..00000000000 --- a/db/migrations/20220608165055_add_log_rate_limit_to_processes.rb +++ /dev/null @@ -1,5 +0,0 @@ -Sequel.migration do - change do - add_column :processes, :log_rate_limit, :Bignum, null: false, default: -1 - end -end diff --git a/db/migrations/20220621175820_add_log_rate_limit_to_tasks_table.rb b/db/migrations/20220621175820_add_log_rate_limit_to_tasks_table.rb deleted file mode 100644 index be1062df2b1..00000000000 --- a/db/migrations/20220621175820_add_log_rate_limit_to_tasks_table.rb +++ /dev/null @@ -1,5 +0,0 @@ -Sequel.migration do - change do - add_column :tasks, :log_rate_limit, :Bignum, null: false, default: -1 - end -end diff --git a/db/migrations/20220718211322_add_staging_log_rate_limit_to_builds.rb b/db/migrations/20220718211322_add_staging_log_rate_limit_to_builds.rb deleted file mode 100644 index 13652db0978..00000000000 --- a/db/migrations/20220718211322_add_staging_log_rate_limit_to_builds.rb +++ /dev/null @@ -1,7 +0,0 @@ -Sequel.migration do - change do - alter_table :builds do - add_column :staging_log_rate_limit, :Bignum, null: false, default: -1 - end - end -end diff --git a/db/migrations/20220822224900_add_log_rate_limit.rb b/db/migrations/20220822224900_add_log_rate_limit.rb new file mode 100644 index 00000000000..4045a7388fb --- /dev/null +++ b/db/migrations/20220822224900_add_log_rate_limit.rb @@ -0,0 +1,9 @@ +Sequel.migration do + change do + add_column :quota_definitions, :log_rate_limit, :Bignum, null: false, default: -1 + add_column :space_quota_definitions, :log_rate_limit, :Bignum, null: false, default: -1 + add_column :processes, :log_rate_limit, :Bignum, null: false, default: -1 + add_column :tasks, :log_rate_limit, :Bignum, null: false, default: -1 + add_column :builds, :staging_log_rate_limit, :Bignum, null: false, default: -1 + end +end From b0384816b2a5f0eedb8c9485e62814784a464574 Mon Sep 17 00:00:00 2001 From: Rebecca Roberts Date: Wed, 31 Aug 2022 21:22:05 +0000 Subject: [PATCH 16/20] update docs for log rate limiting Signed-off-by: Ben Fuller Co-authored-by: Ben Fuller --- docs/v3/source/includes/api_resources/_builds.erb | 3 +++ .../v3/source/includes/api_resources/_organization_quotas.erb | 3 +++ docs/v3/source/includes/api_resources/_processes.erb | 3 +++ docs/v3/source/includes/api_resources/_space_quotas.erb | 3 +++ docs/v3/source/includes/api_resources/_tasks.erb | 4 ++++ docs/v3/source/includes/resources/builds/_create.md.erb | 1 + docs/v3/source/includes/resources/builds/_object.md.erb | 1 + docs/v3/source/includes/resources/feature_flags/_flags.md.erb | 2 +- docs/v3/source/includes/resources/manifests/_get.md | 1 + docs/v3/source/includes/resources/manifests/_object.md.erb | 4 ++++ .../includes/resources/organization_quotas/_create.md.erb | 1 + .../source/includes/resources/organization_quotas/_header.md | 2 +- .../includes/resources/organization_quotas/_object.md.erb | 1 + .../includes/resources/organization_quotas/_update.md.erb | 2 ++ docs/v3/source/includes/resources/processes/_object.md.erb | 1 + docs/v3/source/includes/resources/processes/_scale.md.erb | 4 +++- .../source/includes/resources/processes/_stats_object.md.erb | 2 ++ docs/v3/source/includes/resources/space_quotas/_create.md.erb | 1 + docs/v3/source/includes/resources/space_quotas/_object.md.erb | 1 + docs/v3/source/includes/resources/space_quotas/_update.md.erb | 2 ++ docs/v3/source/includes/resources/tasks/_create.md.erb | 1 + docs/v3/source/includes/resources/tasks/_object.md.erb | 1 + 22 files changed, 41 insertions(+), 3 deletions(-) diff --git a/docs/v3/source/includes/api_resources/_builds.erb b/docs/v3/source/includes/api_resources/_builds.erb index 478847084f6..5c77c4af345 100644 --- a/docs/v3/source/includes/api_resources/_builds.erb +++ b/docs/v3/source/includes/api_resources/_builds.erb @@ -25,6 +25,7 @@ "state": "STAGING", "staging_memory_in_mb": 1024, "staging_disk_in_mb": 1024, + "staging_log_rate_limit_bytes_per_second": 1024, "error": null, "lifecycle": { "type": "buildpack", @@ -74,6 +75,7 @@ "state": "STAGING", "staging_memory_in_mb": 1024, "staging_disk_in_mb": 1024, + "staging_log_rate_limit_bytes_per_second": 1024, "error": null, "lifecycle": { "type": "buildpack", @@ -121,6 +123,7 @@ "state": "STAGED", "staging_memory_in_mb": 1024, "staging_disk_in_mb": 1024, + "staging_log_rate_limit_bytes_per_second": 1024, "error": null, "lifecycle": { "type": "buildpack", diff --git a/docs/v3/source/includes/api_resources/_organization_quotas.erb b/docs/v3/source/includes/api_resources/_organization_quotas.erb index 80c9a0ac3fe..e1bf9bc058b 100644 --- a/docs/v3/source/includes/api_resources/_organization_quotas.erb +++ b/docs/v3/source/includes/api_resources/_organization_quotas.erb @@ -7,6 +7,7 @@ "apps": { "total_memory_in_mb": 5120, "per_process_memory_in_mb": 1024, + "log_rate_limit_in_bytes_per_second": 1024, "total_instances": 10, "per_app_tasks": 5 }, @@ -71,6 +72,7 @@ "apps": { "total_memory_in_mb": 5120, "per_process_memory_in_mb": 1024, + "log_rate_limit_in_bytes_per_second": 1024, "total_instances": 10, "per_app_tasks": 5 }, @@ -105,6 +107,7 @@ "apps": { "total_memory_in_mb": 2048, "per_process_memory_in_mb": 1024, + "log_rate_limit_in_bytes_per_second": 1024, "total_instances": 5, "per_app_tasks": 2 }, diff --git a/docs/v3/source/includes/api_resources/_processes.erb b/docs/v3/source/includes/api_resources/_processes.erb index 2a3eddd8ff5..b2eb729391a 100644 --- a/docs/v3/source/includes/api_resources/_processes.erb +++ b/docs/v3/source/includes/api_resources/_processes.erb @@ -6,6 +6,7 @@ "instances": 5, "memory_in_mb": 256, "disk_in_mb": 1024, + "log_rate_limit_in_bytes_per_second": 1024, "health_check": { "type": "port", "data": { @@ -75,6 +76,7 @@ "instances": 5, "memory_in_mb": 256, "disk_in_mb": 1024, + "log_rate_limit_in_bytes_per_second": 1024, "health_check": { "type": "port", "data": { @@ -125,6 +127,7 @@ "instances": 1, "memory_in_mb": 256, "disk_in_mb": 1024, + "log_rate_limit_in_bytes_per_second": 1024, "health_check": { "type": "process", "data": { diff --git a/docs/v3/source/includes/api_resources/_space_quotas.erb b/docs/v3/source/includes/api_resources/_space_quotas.erb index a0172bac79e..4b15d082d5d 100644 --- a/docs/v3/source/includes/api_resources/_space_quotas.erb +++ b/docs/v3/source/includes/api_resources/_space_quotas.erb @@ -7,6 +7,7 @@ "apps": { "total_memory_in_mb": 5120, "per_process_memory_in_mb": 1024, + "log_rate_limit_in_bytes_per_second": 1024, "total_instances": 10, "per_app_tasks": null }, @@ -67,6 +68,7 @@ "apps": { "total_memory_in_mb": 5120, "per_process_memory_in_mb": 1024, + "log_rate_limit_in_bytes_per_second": 1024, "total_instances": 10, "per_app_tasks": null }, @@ -110,6 +112,7 @@ "apps": { "total_memory_in_mb": 2048, "per_process_memory_in_mb": 1024, + "log_rate_limit_in_bytes_per_second": 1024, "total_instances": 5, "per_app_tasks": 2 }, diff --git a/docs/v3/source/includes/api_resources/_tasks.erb b/docs/v3/source/includes/api_resources/_tasks.erb index 189e4e0fd3c..33dda9c7723 100644 --- a/docs/v3/source/includes/api_resources/_tasks.erb +++ b/docs/v3/source/includes/api_resources/_tasks.erb @@ -7,6 +7,7 @@ "state": "RUNNING", "memory_in_mb": 512, "disk_in_mb": 1024, + "log_rate_limit_in_bytes_per_second": 1024, "result": { "failure_reason": null }, @@ -51,6 +52,7 @@ "state": "CANCELING", "memory_in_mb": 512, "disk_in_mb": 1024, + "log_rate_limit_in_bytes_per_second": 1024, "result": { "failure_reason": null }, @@ -110,6 +112,7 @@ "state": "SUCCEEDED", "memory_in_mb": 512, "disk_in_mb": 1024, + "log_rate_limit_in_bytes_per_second": 1024, "result": { "failure_reason": null }, @@ -150,6 +153,7 @@ "state": "FAILED", "memory_in_mb": 512, "disk_in_mb": 1024, + "log_rate_limit_in_bytes_per_second": 1024, "result": { "failure_reason": "Exited with status 1" }, diff --git a/docs/v3/source/includes/resources/builds/_create.md.erb b/docs/v3/source/includes/resources/builds/_create.md.erb index 49e8673cc40..1ff9e360122 100644 --- a/docs/v3/source/includes/resources/builds/_create.md.erb +++ b/docs/v3/source/includes/resources/builds/_create.md.erb @@ -43,6 +43,7 @@ Name | Type | Description | Default **lifecycle** | [_lifecycle object_](#the-lifecycle-object) | Lifecycle information for a build | lifecycle on the app **staging_memory_in_mb** | _integer_ | Memory in MB allocated for staging of the build **staging_disk_in_mb** | _integer_ | Disk space in MB allocated for staging of the build +**staging_log_rate_limit_bytes_per_second** | _integer_ | Log rate limit in bytes per second allocated for staging of the build **metadata.labels** | [_label object_](#labels) | Labels applied to the build **metadata.annotations** | [_annotation object_](#annotations) | Annotations applied to the build diff --git a/docs/v3/source/includes/resources/builds/_object.md.erb b/docs/v3/source/includes/resources/builds/_object.md.erb index 381cfea2808..aee39c46375 100644 --- a/docs/v3/source/includes/resources/builds/_object.md.erb +++ b/docs/v3/source/includes/resources/builds/_object.md.erb @@ -15,6 +15,7 @@ Name | Type | Description **state** | _string_ | State of the build; valid states are `STAGING`, `STAGED`, or `FAILED` **staging_memory_in_mb** | _integer_ | Memory in MB allocated for staging of the build **staging_disk_in_mb** | _integer_ | Disk space in MB allocated for staging of the build +**staging_log_rate_limit_bytes_per_second** | _integer_ | Log rate limit in bytes per second allocated for staging of the build **error** | _string_ | A string describing errors during the build process **lifecycle** | [_lifecycle object_](#the-lifecycle-object) | Provides the lifecycle object to use during staging; this will override the build's application's default lifecycle for this build **package.guid** | _string_ | The package that is the input to the staging process diff --git a/docs/v3/source/includes/resources/feature_flags/_flags.md.erb b/docs/v3/source/includes/resources/feature_flags/_flags.md.erb index 4f0de9eb32a..2222d010adc 100644 --- a/docs/v3/source/includes/resources/feature_flags/_flags.md.erb +++ b/docs/v3/source/includes/resources/feature_flags/_flags.md.erb @@ -1,7 +1,7 @@ ### List of feature flags <% feature_flags = [ ["app_bits_upload", "true", "When enabled, space developers can upload app bits. When disabled, only admin users can upload app bits."], -["app_scaling", "true", "When enabled, space developers can perform scaling operations (i.e. change memory, disk or instances). When disabled, only admins can perform scaling operations."], +["app_scaling", "true", "When enabled, space developers can perform scaling operations (i.e. change memory, disk, log rate, or instances). When disabled, only admins can perform scaling operations."], ["diego_docker", "false", "When enabled, Docker applications are supported by Diego. When disabled, Docker applications will stop running. It will still be possible to stop and delete them and update their configurations."], ["env_var_visibility", "true", "When enabled, all users can see their environment variables. When disabled, no users can see environment variables."], ["hide_marketplace_from_unauthenticated_users", "false", "When enabled, service offerings available in the marketplace will be hidden from unauthenticated users. When disabled, unauthenticated users will be able to see the service offerings available in the marketplace."], diff --git a/docs/v3/source/includes/resources/manifests/_get.md b/docs/v3/source/includes/resources/manifests/_get.md index d6eea6f0a81..fb3cdfec81d 100644 --- a/docs/v3/source/includes/resources/manifests/_get.md +++ b/docs/v3/source/includes/resources/manifests/_get.md @@ -31,6 +31,7 @@ applications: - type: web instances: 2 memory: 512M + log-rate-limit-per-second: 1KB disk_quota: 1024M health-check-type: port ``` diff --git a/docs/v3/source/includes/resources/manifests/_object.md.erb b/docs/v3/source/includes/resources/manifests/_object.md.erb index 7245878ead0..2cccf51a136 100644 --- a/docs/v3/source/includes/resources/manifests/_object.md.erb +++ b/docs/v3/source/includes/resources/manifests/_object.md.erb @@ -46,6 +46,7 @@ applications: health-check-invocation-timeout: 10 instances: 3 memory: 500M + log-rate-limit-per-second: 1KB timeout: 10 - type: worker command: start-worker.sh @@ -53,6 +54,7 @@ applications: health-check-type: process instances: 2 memory: 256M + log-rate-limit-per-second: 1KB timeout: 15 - name: app2 env: @@ -61,6 +63,7 @@ applications: - type: web instances: 1 memory: 256M + log-rate-limit-per-second: 1KB sidecars: - name: authenticator process_types: [ 'web', 'worker' ] @@ -117,6 +120,7 @@ Name | Type | Description **health-check-type** | _string_ | Type of health check to perform; `none` is deprecated and an alias to `process` **instances** | _integer_ | The number of instances to run **memory** | _string_ | The memory limit for all instances of the web process;
this attribute requires a unit of measurement: `B`, `K`, `KB`, `M`, `MB`, `G`, `GB`, `T`, or `TB` in upper case or lower case +**log-rate-limit-per-second** | _string_ | The log rate limit for all the instances of the process;
this attribute requires a unit of measurement: `B`, `K`, `KB`, `M`, `MB`, `G`, `GB`, `T`, or `TB` in upper case or lower case, or -1 or 0 **timeout** | _integer_ | Time in seconds at which the health-check will report failure #### Route-level configuration diff --git a/docs/v3/source/includes/resources/organization_quotas/_create.md.erb b/docs/v3/source/includes/resources/organization_quotas/_create.md.erb index c578c4a05c2..eb07babdcf6 100644 --- a/docs/v3/source/includes/resources/organization_quotas/_create.md.erb +++ b/docs/v3/source/includes/resources/organization_quotas/_create.md.erb @@ -46,6 +46,7 @@ Name | Type | Description | Default | | **apps.per_process_memory_in_mb** | _integer_ or `null` | Maximum memory for a single process or task | null (infinite) | | **apps.total_memory_in_mb** | _integer_ or `null` | Total memory allowed for all the started processes and running tasks in an organization | null (infinite) | | **apps.total_instances** | _integer_ or `null` | Total instances of all the started processes allowed in an organization | null (infinite) | +| **apps.log_rate_limit_in_bytes_per_second** | _integer_ or `null` | Total log rate limit allowed for all the started processes and running tasks in an organization | null (infinite) | | **apps.per_app_tasks** | _integer_ or `null` | Maximum number of running tasks in an organization | null (infinite) | | **services** | _object_ | Quotas that affect services | | **services.paid_services_allowed** | _boolean_ | Specifies whether instances of paid service plans can be created | true | diff --git a/docs/v3/source/includes/resources/organization_quotas/_header.md b/docs/v3/source/includes/resources/organization_quotas/_header.md index 4388905386c..7f644d0931f 100644 --- a/docs/v3/source/includes/resources/organization_quotas/_header.md +++ b/docs/v3/source/includes/resources/organization_quotas/_header.md @@ -1,6 +1,6 @@ ## Organization Quotas -Organization quotas are named sets of memory, service, and instance usage quotas. For example, one organization quota might allow up to 10 services, 10 routes, and 2 GB of RAM, while another might offer 100 services, 100 routes, and 10 GB of RAM. +Organization quotas are named sets of memory, log rate, service, and instance usage quotas. For example, one organization quota might allow up to 10 services, 10 routes, and 2 GB of RAM, while another might offer 100 services, 100 routes, and 10 GB of RAM. An organization has exactly one organization quota. If not specifically assigned a quota, it will have the default quota. diff --git a/docs/v3/source/includes/resources/organization_quotas/_object.md.erb b/docs/v3/source/includes/resources/organization_quotas/_object.md.erb index be670c1f2d7..3274bc010f3 100644 --- a/docs/v3/source/includes/resources/organization_quotas/_object.md.erb +++ b/docs/v3/source/includes/resources/organization_quotas/_object.md.erb @@ -17,6 +17,7 @@ Name | Type | Description| | **apps.per_process_memory_in_mb** | _integer_ or `null` | Maximum memory for a single process or task | | **apps.total_memory_in_mb** | _integer_ or `null` | Total memory allowed for all the started processes and running tasks in an organization | | **apps.total_instances** | _integer_ or `null` | Total instances of all the started processes allowed in an organization | +| **apps.log_rate_limit_in_bytes_per_second** | _integer_ or `null` | Total log rate limit allowed for all the started processes and running tasks in an organization | null (infinite) | | **apps.per_app_tasks** | _integer_ or `null` | Maximum number of running tasks in an organization | | **services** | _object_ | Quotas that affect services | | **services.paid_services_allowed** | _boolean_ | Specifies whether instances of paid service plans can be created | diff --git a/docs/v3/source/includes/resources/organization_quotas/_update.md.erb b/docs/v3/source/includes/resources/organization_quotas/_update.md.erb index 8338b4f509b..93ff5e14e6c 100644 --- a/docs/v3/source/includes/resources/organization_quotas/_update.md.erb +++ b/docs/v3/source/includes/resources/organization_quotas/_update.md.erb @@ -14,6 +14,7 @@ curl "https://api.example.org/v3/organization_quotas/[guid]" \ "apps": { "total_memory_in_mb": 5120, "per_process_memory_in_mb": 1024, + "log_rate_limit_in_bytes_per_second": 1024, "total_instances": 10, "per_app_tasks": 5 }, @@ -56,6 +57,7 @@ Name | Type | Description | **apps.per_process_memory_in_mb** | _integer_ or `null` | Maximum memory for a single process or task | **apps.total_memory_in_mb** | _integer_ or `null` | Total memory of all the started processes and running tasks in an organization | **apps.total_instances** | _integer_ or `null` | Total instances of all the started processes in an organization +| **apps.log_rate_limit_in_bytes_per_second** | _integer_ or `null` | Total log rate limit allowed for all the started processes and running tasks in an organization | null (infinite) | | **apps.per_app_tasks** | _integer_ or `null` | Maximum number of running tasks in an organization | **services** | _object_ | Quotas that affect services | **services.paid_services_allowed** | _boolean_ | If instances of paid service plans can be created diff --git a/docs/v3/source/includes/resources/processes/_object.md.erb b/docs/v3/source/includes/resources/processes/_object.md.erb index 3d07fbc1f34..bcb24be3926 100644 --- a/docs/v3/source/includes/resources/processes/_object.md.erb +++ b/docs/v3/source/includes/resources/processes/_object.md.erb @@ -16,6 +16,7 @@ Name | Type | Description **command** | _string_ or _null_ | The command used to start the process; use _null_ to revert to the buildpack-detected or procfile-provided start command **instances** | _integer_ | The number of instances to run **memory_in_mb** | _integer_ | The memory in MB allocated per instance +**log_rate_limit_in_bytes_per_second** | _integer_ | The log rate in bytes per second allocated per instance **disk_in_mb** | _integer_ | The disk in MB allocated per instance **health_check** | [_health check object_](#the-health-check-object) | The health check to perform on the process **relationships.app** | [_to-one relationship_](#to-one-relationships) | The app the process belongs to diff --git a/docs/v3/source/includes/resources/processes/_scale.md.erb b/docs/v3/source/includes/resources/processes/_scale.md.erb index 59ae2b7fb83..1bbd7f45a4c 100644 --- a/docs/v3/source/includes/resources/processes/_scale.md.erb +++ b/docs/v3/source/includes/resources/processes/_scale.md.erb @@ -12,7 +12,8 @@ curl "https://api.example.org/v3/processes/[guid]/actions/scale" \ -d '{ "instances": 5, "memory_in_mb": 256, - "disk_in_mb": 1024 + "disk_in_mb": 1024, + "log_rate_limit_in_bytes_per_second": 1024 }' ``` @@ -38,6 +39,7 @@ Name | Type | Description **instances** | _integer_ | The number of instances to run **memory_in_mb** | _integer_ | The memory in mb allocated per instance **disk_in_mb** | _integer_ | The disk in mb allocated per instance +**log_rate_limit_in_bytes_per_second** | _integer_ | The log rate in bytes per second allocated per instance #### Permitted roles | diff --git a/docs/v3/source/includes/resources/processes/_stats_object.md.erb b/docs/v3/source/includes/resources/processes/_stats_object.md.erb index c72a8729954..db3ba33c14e 100644 --- a/docs/v3/source/includes/resources/processes/_stats_object.md.erb +++ b/docs/v3/source/includes/resources/processes/_stats_object.md.erb @@ -17,11 +17,13 @@ Name | Type | Description **usage.cpu** | _number_ | The current cpu usage of the instance **usage.mem** | _integer_ | The current memory usage of the instance **usage.disk** | _integer_ | The current disk usage of the instance +**usage.log_rate** | _integer_ | The current logging usage of the instance **host** | _string_ | The host the instance is running on **instance_ports** | _object_ | JSON array of port mappings between the network-exposed port used to communicate with the app (`external`) and port opened to the running process that it can listen on (`internal`) **uptime** | _integer_ | The uptime in seconds for the instance **mem_quota** | _integer_ | The current maximum memory allocated for the instance; the value is `null` when memory quota data is unavailable **disk_quota** | _integer_ | The current maximum disk allocated for the instance; the value is `null` when disk quota data is unavailable +**log_rate_limit** | _integer_ | The current maximum log rate allocated for the instance; the value `-1` is unlimited, the value is `null` when the log_rate_limit is unavailable **fds_quota** | _integer_ | The maximum file descriptors the instance is allowed to use **isolation_segment** | _string_ | The current isolation segment that the instance is running on; the value is `null` when the instance is not placed on a particular isolation segment **details** | _string_ | Information about errors placing the instance; the value is `null` if there are no placement errors diff --git a/docs/v3/source/includes/resources/space_quotas/_create.md.erb b/docs/v3/source/includes/resources/space_quotas/_create.md.erb index 66fdab82dae..4952def69cc 100644 --- a/docs/v3/source/includes/resources/space_quotas/_create.md.erb +++ b/docs/v3/source/includes/resources/space_quotas/_create.md.erb @@ -51,6 +51,7 @@ Name | Type | Description | Default | | **apps.per_process_memory_in_mb** | _integer_ or `null` | Maximum memory for a single process or task | null (infinite) | | **apps.total_memory_in_mb** | _integer_ or `null` | Total memory allowed for all the started processes and running tasks in a space | null (infinite) | | **apps.total_instances** | _integer_ or `null` | Total instances of all the started processes allowed in a space | null (infinite) | +| **apps.log_rate_limit_in_bytes_per_second** | _integer_ or `null` | Total log rate limit allowed for all the started processes and running tasks in an organization | null (infinite) | | **apps.per_app_tasks** | _integer_ or `null` | Maximum number of running tasks in a space | null (infinite) | | **services** | _object_ | Quotas that affect services | | **services.paid_services_allowed** | _boolean_ | Specifies whether instances of paid service plans can be created | true | diff --git a/docs/v3/source/includes/resources/space_quotas/_object.md.erb b/docs/v3/source/includes/resources/space_quotas/_object.md.erb index 28a8735b602..6f9b4a451e8 100644 --- a/docs/v3/source/includes/resources/space_quotas/_object.md.erb +++ b/docs/v3/source/includes/resources/space_quotas/_object.md.erb @@ -17,6 +17,7 @@ Name | Type | Description| | **apps.per_process_memory_in_mb** | _integer_ or `null` | Maximum memory for a single process or task | | **apps.total_memory_in_mb** | _integer_ or `null` | Total memory allowed for all the started processes and running tasks in a space | | **apps.total_instances** | _integer_ or `null` | Total instances of all the started processes allowed in a space | +| **apps.log_rate_limit_in_bytes_per_second** | _integer_ or `null` | Total log rate limit allowed for all the started processes and running tasks in an organization | null (infinite) | | **apps.per_app_tasks** | _integer_ or `null` | Maximum number of running tasks in a space | | **services** | _object_ | Quotas that affect services | | **services.paid_services_allowed** | _boolean_ | Specifies whether instances of paid service plans can be created | diff --git a/docs/v3/source/includes/resources/space_quotas/_update.md.erb b/docs/v3/source/includes/resources/space_quotas/_update.md.erb index 876b2eac29b..fab9b05dda2 100644 --- a/docs/v3/source/includes/resources/space_quotas/_update.md.erb +++ b/docs/v3/source/includes/resources/space_quotas/_update.md.erb @@ -14,6 +14,7 @@ curl "https://api.example.org/v3/space_quotas/[guid]" \ "apps": { "total_memory_in_mb": 5120, "per_process_memory_in_mb": 1024, + "log_rate_limit_in_bytes_per_second": 1024, "total_instances": 10, "per_app_tasks": 5 }, @@ -53,6 +54,7 @@ Name | Type | Description | **apps.per_process_memory_in_mb** | _integer_ or `null` | Maximum memory for a single process or task | **apps.total_memory_in_mb** | _integer_ or `null` | Total memory of all the started processes and running tasks in a space | **apps.total_instances** | _integer_ or `null` | Total instances of all the started processes in a space +| **apps.log_rate_limit_in_bytes_per_second** | _integer_ or `null` | Total log rate limit allowed for all the started processes and running tasks in an organization | null (infinite) | | **apps.per_app_tasks** | _integer_ or `null` | Maximum number of running tasks in a space | **services** | _object_ | Quotas that affect services | **services.paid_services_allowed** | _boolean_ | If instances of paid service plans can be created diff --git a/docs/v3/source/includes/resources/tasks/_create.md.erb b/docs/v3/source/includes/resources/tasks/_create.md.erb index cb0f9b35355..4366a275152 100644 --- a/docs/v3/source/includes/resources/tasks/_create.md.erb +++ b/docs/v3/source/includes/resources/tasks/_create.md.erb @@ -51,6 +51,7 @@ Name | Type | Description | Default **name** | _string_ | Name of the task | auto-generated **disk_in_mb**[1] | _integer_ | Amount of disk to allocate for the task in MB | operator-configured `default_app_disk_in_mb` **memory_in_mb**[1] | _integer_ | Amount of memory to allocate for the task in MB | operator-configured `default_app_memory` +**log_rate_limit_per_second**[1] | _integer_ | Amount of log rate to allocate for the task in bytes | operator-configured `default_app_log_rate_limit_in_bytes_per_second` **droplet_guid** | _uuid_ | The guid of the droplet that will be used to run the command | the app's current droplet **template.process.guid** | _uuid_ | The guid of the process that will be used as a template | `null` **metadata.labels** | [_label object_](#labels) | Labels applied to the package diff --git a/docs/v3/source/includes/resources/tasks/_object.md.erb b/docs/v3/source/includes/resources/tasks/_object.md.erb index cc6ba76a623..6d5a9340c13 100644 --- a/docs/v3/source/includes/resources/tasks/_object.md.erb +++ b/docs/v3/source/includes/resources/tasks/_object.md.erb @@ -18,6 +18,7 @@ Name | Type | Description **state** | _string_ | State of the task Possible states are `PENDING`, `RUNNING`, `SUCCEEDED`, `CANCELING`, and `FAILED` **memory_in_mb** | _integer_ | Amount of memory to allocate for the task in MB **disk_in_mb** | _integer_ | Amount of disk to allocate for the task in MB +**log_rate_limit_per_second** | _integer_ | Amount of log rate to allocate for the task in bytes **result** | _object_ | Results from the task **result.failure_reason** | _string_ | Null if the task succeeds, contains the error message if it fails **droplet_guid** | _uuid_ | The guid of the droplet that will be used to run the command From b041419bcf03b4bdbcd08f2d11da6917d8a89d54 Mon Sep 17 00:00:00 2001 From: Andrew Crump Date: Thu, 1 Sep 2022 20:53:03 +0000 Subject: [PATCH 17/20] Handle -1 and 0 when applying/diffing manifests The cf CLI would provide these values as strings, however directly curling the endpoint with the values as integers would result in a 500. Co-authored-by: Carson Long --- app/actions/space_diff_manifest.rb | 6 ++-- app/messages/app_manifest_message.rb | 4 +-- spec/request/space_manifests_spec.rb | 2 +- spec/unit/actions/space_diff_manifest_spec.rb | 30 +++++++++++++++++-- .../messages/app_manifest_message_spec.rb | 20 +++++++++++++ 5 files changed, 54 insertions(+), 8 deletions(-) diff --git a/app/actions/space_diff_manifest.rb b/app/actions/space_diff_manifest.rb index 6210833da98..ec791f32711 100644 --- a/app/actions/space_diff_manifest.rb +++ b/app/actions/space_diff_manifest.rb @@ -167,7 +167,7 @@ def normalize_units(manifest_app_hash) manifest_app_hash.each_with_index do |process_hash, index| byte_measurement_key_words.each do |key| value = process_hash[key] - manifest_app_hash[index][key] = normalize_unit(value, key) unless value.nil? || value == -1 + manifest_app_hash[index][key] = normalize_unit(value, key) unless value.nil? end end manifest_app_hash @@ -182,8 +182,8 @@ def convert_to_mb(human_readable_byte_value, attribute_name) end def normalize_unit(non_normalized_value, attribute_name) - if %w[-1 0].include?(non_normalized_value) - non_normalized_value + if %w(-1 0).include?(non_normalized_value.to_s) + non_normalized_value.to_s else byte_converter.human_readable_byte_value(byte_converter.convert_to_b(non_normalized_value)) end diff --git a/app/messages/app_manifest_message.rb b/app/messages/app_manifest_message.rb index 8e21faf9c94..e33da75f90a 100644 --- a/app/messages/app_manifest_message.rb +++ b/app/messages/app_manifest_message.rb @@ -292,8 +292,8 @@ def convert_to_mb(human_readable_byte_value) def convert_to_bytes_per_second(human_readable_byte_value) return nil unless human_readable_byte_value.present? - return -1 if human_readable_byte_value == '-1' - return 0 if human_readable_byte_value == '0' + return -1 if human_readable_byte_value.to_s == '-1' + return 0 if human_readable_byte_value.to_s == '0' byte_converter.convert_to_b(human_readable_byte_value.strip) rescue ByteConverter::InvalidUnitsError, ByteConverter::NonNumericError diff --git a/spec/request/space_manifests_spec.rb b/spec/request/space_manifests_spec.rb index 1c4a72b7858..7fe1263b092 100644 --- a/spec/request/space_manifests_spec.rb +++ b/spec/request/space_manifests_spec.rb @@ -467,7 +467,7 @@ 'version' => 1, 'applications' => [ { 'name' => app1_model.name, - 'log-rate-limit-per-second' => '-1' + 'log-rate-limit-per-second' => -1 }, ] }.to_yaml diff --git a/spec/unit/actions/space_diff_manifest_spec.rb b/spec/unit/actions/space_diff_manifest_spec.rb index b0fa6b2a78a..85340f15173 100644 --- a/spec/unit/actions/space_diff_manifest_spec.rb +++ b/spec/unit/actions/space_diff_manifest_spec.rb @@ -379,7 +379,7 @@ module VCAP::CloudController end describe 'log-rate-limit-per-second' do - it 'can handle -1 for unlimted' do + it 'can handle -1 as a string for unlimited' do default_manifest['applications'][0]['log-rate-limit-per-second'] = '-1' expect(subject).to include( @@ -392,7 +392,20 @@ module VCAP::CloudController ) end - it 'can handle 0 without units' do + it 'can handle -1 as a number for unlimited' do + default_manifest['applications'][0]['log-rate-limit-per-second'] = -1 + + expect(subject).to include( + { + 'op' => 'replace', + 'path' => '/applications/0/log-rate-limit-per-second', + 'value' => '-1', + 'was' => '1M' + } + ) + end + + it 'can handle 0 as a string without units' do default_manifest['applications'][0]['log-rate-limit-per-second'] = '0' expect(subject).to include( @@ -404,6 +417,19 @@ module VCAP::CloudController } ) end + + it 'can handle 0 as a number without units' do + default_manifest['applications'][0]['log-rate-limit-per-second'] = 0 + + expect(subject).to include( + { + 'op' => 'replace', + 'path' => '/applications/0/log-rate-limit-per-second', + 'value' => '0', + 'was' => '1M' + } + ) + end end context 'when the user passes in a v2 manifest' do diff --git a/spec/unit/messages/app_manifest_message_spec.rb b/spec/unit/messages/app_manifest_message_spec.rb index 84ed40cc421..cb23b463b8b 100644 --- a/spec/unit/messages/app_manifest_message_spec.rb +++ b/spec/unit/messages/app_manifest_message_spec.rb @@ -154,6 +154,16 @@ module VCAP::CloudController expect(message).to be_valid end end + + context 'specified as the number -1' do + let(:params_from_yaml) { { name: 'eugene', 'log-rate-limit-per-second' => -1 } } + + it 'is valid' do + message = AppManifestMessage.create_from_yml(params_from_yaml) + expect(message).to be_valid + end + end + context 'with a bytes suffix' do let(:params_from_yaml) { { name: 'eugene', log_rate_limit_per_second: '-1B' } } @@ -183,6 +193,16 @@ module VCAP::CloudController end end + context 'when log-rate-limit-per-second is the number 0' do + let(:params_from_yaml) { { name: 'eugene', log_rate_limit_per_second: 0 } } + + it 'is valid' do + message = AppManifestMessage.create_from_yml(params_from_yaml) + expect(message).to be_valid + expect(message.manifest_process_scale_messages.first.log_rate_limit).to eq(0) + end + end + context 'when log-rate-limit-per-second is 0TB' do let(:params_from_yaml) { { name: 'eugene', log_rate_limit_per_second: '0TB' } } From adfb5cabce9cbd2168ac5c9d8718fc77f5c5072a Mon Sep 17 00:00:00 2001 From: Andrew Crump Date: Thu, 1 Sep 2022 22:10:14 +0000 Subject: [PATCH 18/20] Raise error when disk quota is zero Co-authored-by: Carson Long --- app/messages/app_manifest_message.rb | 2 +- spec/unit/messages/app_manifest_message_spec.rb | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/app/messages/app_manifest_message.rb b/app/messages/app_manifest_message.rb index e33da75f90a..ab60a752fe7 100644 --- a/app/messages/app_manifest_message.rb +++ b/app/messages/app_manifest_message.rb @@ -300,7 +300,7 @@ def convert_to_bytes_per_second(human_readable_byte_value) end def validate_byte_format(human_readable_byte_value, attribute_name, allow_unlimited: false) - unless (allow_unlimited && human_readable_byte_value == '-1') || human_readable_byte_value == '0' + unless allow_unlimited && (human_readable_byte_value.to_s == '-1' || human_readable_byte_value.to_s == '0') byte_converter.convert_to_mb(human_readable_byte_value) end diff --git a/spec/unit/messages/app_manifest_message_spec.rb b/spec/unit/messages/app_manifest_message_spec.rb index cb23b463b8b..2e3fdfbb7b9 100644 --- a/spec/unit/messages/app_manifest_message_spec.rb +++ b/spec/unit/messages/app_manifest_message_spec.rb @@ -79,6 +79,20 @@ module VCAP::CloudController end end + context 'when disk_quota is zero' do + let(:params_from_yaml) { { name: 'eugene', disk_quota: 0 } } + + it 'is not valid' do + message = AppManifestMessage.create_from_yml(params_from_yaml) + + expect(message).not_to be_valid + expect(message.errors).to have(1).items + expect(message.errors.full_messages).to include( + 'Process "web": Disk quota must use a supported unit: B, K, KB, M, MB, G, GB, T, or TB' + ) + end + end + context 'when disk_quota is not a positive amount' do let(:params_from_yaml) { { name: 'eugene', disk_quota: '-1MB' } } From b98d3ab36fb812e9896a96dcfeaff99bc638784c Mon Sep 17 00:00:00 2001 From: Andrew Crump Date: Fri, 2 Sep 2022 19:23:23 +0000 Subject: [PATCH 19/20] Don't error when limit is passed as an integer --- app/messages/app_manifest_message.rb | 1 + spec/unit/messages/app_manifest_message_spec.rb | 14 ++++++++++++++ 2 files changed, 15 insertions(+) diff --git a/app/messages/app_manifest_message.rb b/app/messages/app_manifest_message.rb index ab60a752fe7..9fce84c6971 100644 --- a/app/messages/app_manifest_message.rb +++ b/app/messages/app_manifest_message.rb @@ -291,6 +291,7 @@ def convert_to_mb(human_readable_byte_value) end def convert_to_bytes_per_second(human_readable_byte_value) + human_readable_byte_value = human_readable_byte_value.to_s return nil unless human_readable_byte_value.present? return -1 if human_readable_byte_value.to_s == '-1' return 0 if human_readable_byte_value.to_s == '0' diff --git a/spec/unit/messages/app_manifest_message_spec.rb b/spec/unit/messages/app_manifest_message_spec.rb index 2e3fdfbb7b9..7d13737b181 100644 --- a/spec/unit/messages/app_manifest_message_spec.rb +++ b/spec/unit/messages/app_manifest_message_spec.rb @@ -226,6 +226,20 @@ module VCAP::CloudController expect(message.manifest_process_scale_messages.first.log_rate_limit).to eq(0) end end + + context 'when log-rate-limit-per-second is a positive integer without a suffix' do + let(:params_from_yaml) { { name: 'eugene', log_rate_limit_per_second: 9999 } } + + it 'is not valid' do + message = AppManifestMessage.create_from_yml(params_from_yaml) + + expect(message).not_to be_valid + expect(message.errors).to have(1).items + expect(message.errors.full_messages).to include( + 'Process "web": Log rate limit per second must use a supported unit: B, K, KB, M, MB, G, GB, T, or TB' + ) + end + end end describe 'buildpack' do From b11e37ef1dfa175011f334bdac522582a0f0a830 Mon Sep 17 00:00:00 2001 From: Duane May Date: Wed, 7 Sep 2022 21:45:23 +0000 Subject: [PATCH 20/20] Ignore a null log rate limit on process scale Signed-off-by: Andrew Crump --- app/actions/process_scale.rb | 5 +++-- spec/unit/actions/process_scale_spec.rb | 9 +++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/app/actions/process_scale.rb b/app/actions/process_scale.rb index 2f6763a516b..42d349b0746 100644 --- a/app/actions/process_scale.rb +++ b/app/actions/process_scale.rb @@ -22,8 +22,9 @@ def scale @process.instances = @message.instances if @message.requested?(:instances) @process.memory = @message.memory_in_mb if @message.requested?(:memory_in_mb) @process.disk_quota = @message.disk_in_mb if @message.requested?(:disk_in_mb) - @process.log_rate_limit = @message.log_rate_limit_in_bytes_per_second if @message.requested?(:log_rate_limit_in_bytes_per_second) - + if @message.requested?(:log_rate_limit_in_bytes_per_second) && !@message.log_rate_limit_in_bytes_per_second.nil? + @process.log_rate_limit = @message.log_rate_limit_in_bytes_per_second + end @process.save record_audit_event diff --git a/spec/unit/actions/process_scale_spec.rb b/spec/unit/actions/process_scale_spec.rb index 9d56a1a6ba5..7d603313059 100644 --- a/spec/unit/actions/process_scale_spec.rb +++ b/spec/unit/actions/process_scale_spec.rb @@ -68,6 +68,15 @@ module VCAP::CloudController expect(process.log_rate_limit).to eq(original_value) end + it 'does not set log quota if null is provided' do + valid_message_params[:log_rate_limit_in_bytes_per_second] = nil + original_value = process.log_rate_limit + + process_scale.scale + + expect(process.log_rate_limit).to eq(original_value) + end + describe 'audit events' do it 'creates a process audit event' do expect(Repositories::ProcessEventRepository).to receive(:record_scale).with(