From 63027247e2e4bbf1a7d4f95b3cb3136ca2205115 Mon Sep 17 00:00:00 2001 From: Katharina Przybill <30441792+kathap@users.noreply.github.com> Date: Mon, 17 Jul 2023 13:24:53 +0200 Subject: [PATCH 1/4] Adapt failed_jobs to run frequently --- jobs/cloud_controller_clock/spec | 7 +++++++ .../templates/cloud_controller_ng.yml.erb | 2 ++ 2 files changed, 9 insertions(+) diff --git a/jobs/cloud_controller_clock/spec b/jobs/cloud_controller_clock/spec index 7161024336..a5e7fe7960 100644 --- a/jobs/cloud_controller_clock/spec +++ b/jobs/cloud_controller_clock/spec @@ -114,6 +114,10 @@ properties: description: "The age at which completed tasks are pruned from the Cloud Controller database" default: 31 + cc.failed_jobs.max_number_of_failed_delayed_jobs: + description: "Maximum number of failed jobs that should stay in cloud controller database before being cleaned up" + default: 100000 + cc.pending_droplets.frequency_in_seconds: description: "How often the pending droplets cleanup job runs" default: 300 @@ -123,6 +127,9 @@ properties: cc.diego_sync.frequency_in_seconds: description: "How often to synchronize CC's database with Diego's" default: 30 + cc.failed_jobs.frequency_in_seconds: + description: "How often the failed_jobs cleanup job runs" + default: 144000 # 4 hours cc.external_protocol: default: "https" diff --git a/jobs/cloud_controller_clock/templates/cloud_controller_ng.yml.erb b/jobs/cloud_controller_clock/templates/cloud_controller_ng.yml.erb index aedc4010c5..1c1d7b2eef 100644 --- a/jobs/cloud_controller_clock/templates/cloud_controller_ng.yml.erb +++ b/jobs/cloud_controller_clock/templates/cloud_controller_ng.yml.erb @@ -85,6 +85,8 @@ audit_events: failed_jobs: cutoff_age_in_days: <%= p("cc.failed_jobs.cutoff_age_in_days") %> + max_number_of_failed_delayed_jobs: <%= p("cc.failed_jobs.max_number_of_failed_delayed_jobs") %> + frequency_in_seconds: <%= p("cc.failed_jobs.frequency_in_seconds") %> completed_tasks: cutoff_age_in_days: <%= p("cc.completed_tasks.cutoff_age_in_days") %> From 8f9baa2708de396eb5a88c945554c37f243df5f3 Mon Sep 17 00:00:00 2001 From: Katharina Przybill <30441792+kathap@users.noreply.github.com> Date: Fri, 28 Jul 2023 17:35:34 +0200 Subject: [PATCH 2/4] adapt to optional cc.failed_jobs parameter and add unit test --- jobs/cloud_controller_clock/spec | 1 - .../templates/cloud_controller_ng.yml.erb | 4 +++- .../cloud_controller_ng_spec.rb | 17 +++++++++++++++++ 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/jobs/cloud_controller_clock/spec b/jobs/cloud_controller_clock/spec index a5e7fe7960..f2d24366ff 100644 --- a/jobs/cloud_controller_clock/spec +++ b/jobs/cloud_controller_clock/spec @@ -116,7 +116,6 @@ properties: cc.failed_jobs.max_number_of_failed_delayed_jobs: description: "Maximum number of failed jobs that should stay in cloud controller database before being cleaned up" - default: 100000 cc.pending_droplets.frequency_in_seconds: description: "How often the pending droplets cleanup job runs" diff --git a/jobs/cloud_controller_clock/templates/cloud_controller_ng.yml.erb b/jobs/cloud_controller_clock/templates/cloud_controller_ng.yml.erb index 1c1d7b2eef..9227e77aff 100644 --- a/jobs/cloud_controller_clock/templates/cloud_controller_ng.yml.erb +++ b/jobs/cloud_controller_clock/templates/cloud_controller_ng.yml.erb @@ -85,7 +85,9 @@ audit_events: failed_jobs: cutoff_age_in_days: <%= p("cc.failed_jobs.cutoff_age_in_days") %> - max_number_of_failed_delayed_jobs: <%= p("cc.failed_jobs.max_number_of_failed_delayed_jobs") %> + <% if_p("cc.failed_jobs.max_number_of_failed_delayed_jobs") do |max_number_of_failed_delayed_jobs| %> + max_number_of_failed_delayed_jobs: <%= max_number_of_failed_delayed_jobs %> + <% end %> frequency_in_seconds: <%= p("cc.failed_jobs.frequency_in_seconds") %> completed_tasks: diff --git a/spec/cloud_controller_ng/cloud_controller_ng_spec.rb b/spec/cloud_controller_ng/cloud_controller_ng_spec.rb index 1afd57d0c2..e5aaefa972 100644 --- a/spec/cloud_controller_ng/cloud_controller_ng_spec.rb +++ b/spec/cloud_controller_ng/cloud_controller_ng_spec.rb @@ -624,6 +624,23 @@ module Test end end end + + describe 'max_number_of_failed_delayed_jobs' do + context "when 'cc.failed_jobs.max_number_of_failed_delayed_jobs' is set" do + it 'renders max_number_of_failed_delayed_jobs into the ccng config' do + merged_manifest_properties['cc'].store('renderer', { 'max_number_of_failed_delayed_jobs' => 1000 }) + template_hash = YAML.safe_load(template.render(merged_manifest_properties, consumes: links)) + expect(template_hash['failed_jobs']['max_number_of_failed_delayed_jobs']).to eq(1000) + end + end + + context "when 'cc.failed_jobs.max_number_of_failed_delayed_jobs' is not set (default)" do + it 'does not render max_number_of_failed_delayed_jobs into the ccng config' do + template_hash = YAML.safe_load(template.render(merged_manifest_properties, consumes: links)) + expect(template_hash['failed_jobs']).not_to have_key(:max_number_of_failed_delayed_jobs) + end + end + end end end end From 92c838217f11d2ab6bbf123e29ceca51fe0acdd9 Mon Sep 17 00:00:00 2001 From: Katharina Przybill <30441792+kathap@users.noreply.github.com> Date: Mon, 31 Jul 2023 11:44:38 +0200 Subject: [PATCH 3/4] add test for cloud_controller_clock failed_jobs optional parameter --- .../cloud_controller_clock_spec.rb | 137 ++++++++++++++++++ 1 file changed, 137 insertions(+) create mode 100644 spec/cloud_controller_clock/cloud_controller_clock_spec.rb diff --git a/spec/cloud_controller_clock/cloud_controller_clock_spec.rb b/spec/cloud_controller_clock/cloud_controller_clock_spec.rb new file mode 100644 index 0000000000..7b79c41941 --- /dev/null +++ b/spec/cloud_controller_clock/cloud_controller_clock_spec.rb @@ -0,0 +1,137 @@ +# frozen_string_literal: true + +require 'rspec' +require 'bosh/template/test' +require 'yaml' +require 'json' + +module Bosh + module Template + module Test + describe 'cloud_controller_ng job template rendering' do + let(:release_path) { File.join(File.dirname(__FILE__), '../..') } + let(:release) { ReleaseDir.new(release_path) } + let(:job) { release.job('cloud_controller_clock') } + + let(:manifest_properties) do + { + 'system_domain' => 'brook-sentry.capi.land', + + 'cc' => { + + 'db_logging_level' => 100, + 'staging_upload_user' => 'staging_user', + 'staging_upload_password' => 'hunter2', + 'database_encryption' => { + 'experimental_pbkdf2_hmac_iterations' => 123, + 'skip_validation' => false, + 'current_key_label' => 'encryption_key_0', + :keys => { 'encryption_key_0' => '((cc_db_encryption_key))' } + } + }, + 'ccdb' => { + 'db_scheme' => 'mysql', + 'max_connections' => 'foo2', + 'databases' => [{ 'tag' => 'cc' }], + 'roles' => [{ + 'tag' => 'admin', + 'name' => 'alex', + 'password' => 'pass' + }], + 'address' => 'foo5', + 'port' => 'foo7', + 'pool_timeout' => 'foo11', + 'ssl_verify_hostname' => 'foo12', + 'read_timeout' => 'foo13', + 'connection_validation_timeout' => 'foo14', + 'ca_cert' => 'foo15' + } + } + end + + let(:properties) do + { + 'router' => { 'route_services_secret' => '((router_route_services_secret))' }, + 'cc' => { + 'system_hostnames' => '', + 'logging_max_retries' => 'bar1', + 'default_app_ssh_access' => 'something', + 'logging_level' => 'other thing', + 'log_db_queries' => 'balsdkj', + 'logging' => { 'format' => { 'timestamp' => 'rfc3339' } }, + 'db_logging_level' => 'bar2', + 'db_encryption_key' => 'bar3', + 'volume_services_enabled' => true, + 'uaa' => { + 'client_timeout' => 10 + }, + 'database_encryption' => { + 'experimental_pbkdf2_hmac_iterations' => 123, + 'skip_validation' => false, + 'current_key_label' => 'encryption_key_0', + :keys => { 'encryption_key_0' => '((cc_db_encryption_key))' } + }, + 'max_labels_per_resource' => true, + 'max_annotations_per_resource' => 'yus', + 'disable_private_domain_cross_space_context_path_route_sharing' => false, + 'custom_metric_tag_prefix_list' => ['heck.yes.example.com'] + } + } + end + let(:cloud_controller_internal_link) do + Link.new(name: 'cloud_controller_internal', properties:, instances: [LinkInstance.new(address: 'default_app_ssh_access')]) + end + + let(:cloud_controller_db_link) do + properties = { + 'ccdb' => { + 'db_scheme' => 'mysql', + 'max_connections' => 'foo2', + 'databases' => [{ 'tag' => 'cc' }], + 'roles' => [{ + 'tag' => 'admin', + 'name' => 'alex', + 'password' => 'pass' + }], + 'address' => 'foo5', + 'port' => 'foo7', + 'pool_timeout' => 'foo11', + 'ssl_verify_hostname' => 'foo12', + 'read_timeout' => 'foo13', + 'connection_validation_timeout' => 'foo14', + 'ca_cert' => 'foo15' + } + } + Link.new(name: 'database', properties:, instances: [LinkInstance.new(address: 'cloud_controller_db')]) + end + + let(:links) { [cloud_controller_internal_link, cloud_controller_db_link] } + + let(:template) { job.template('config/cloud_controller_ng.yml') } + + it 'creates the cloud_controller_ng.yml config file' do + expect do + YAML.safe_load(template.render(manifest_properties, consumes: links)) + end.not_to raise_error + end + + describe 'max_number_of_failed_delayed_jobs' do + context "when 'cc.failed_jobs.max_number_of_failed_delayed_jobs' is set" do + it 'renders max_number_of_failed_delayed_jobs into the ccng config' do + manifest_properties['cc'].store('failed_jobs', { 'max_number_of_failed_delayed_jobs' => 1000 }) + template_hash = YAML.safe_load(template.render(manifest_properties, consumes: links)) + expect(template_hash['failed_jobs']['max_number_of_failed_delayed_jobs']).to eq(1000) + end + end + + context "when 'cc.failed_jobs.max_number_of_failed_delayed_jobs' is not set (default)" do + it 'does not render max_number_of_failed_delayed_jobs into the ccng config' do + template_hash = YAML.safe_load(template.render(manifest_properties, consumes: links)) + expect(template_hash['failed_jobs']).not_to have_key(:max_number_of_failed_delayed_jobs) + end + end + end + end + end + end +end From 552b2205ca69de46cadd8486347ae9f0b4aa8c6d Mon Sep 17 00:00:00 2001 From: Katharina Przybill <30441792+kathap@users.noreply.github.com> Date: Mon, 31 Jul 2023 11:47:56 +0200 Subject: [PATCH 4/4] clean up --- .../cloud_controller_ng_spec.rb | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/spec/cloud_controller_ng/cloud_controller_ng_spec.rb b/spec/cloud_controller_ng/cloud_controller_ng_spec.rb index f57efc9653..187eb9db97 100644 --- a/spec/cloud_controller_ng/cloud_controller_ng_spec.rb +++ b/spec/cloud_controller_ng/cloud_controller_ng_spec.rb @@ -625,23 +625,6 @@ module Test end end - describe 'max_number_of_failed_delayed_jobs' do - context "when 'cc.failed_jobs.max_number_of_failed_delayed_jobs' is set" do - it 'renders max_number_of_failed_delayed_jobs into the ccng config' do - merged_manifest_properties['cc'].store('failed_jobs', { 'max_number_of_failed_delayed_jobs' => 1000 }) - template_hash = YAML.safe_load(template.render(merged_manifest_properties, consumes: links)) - expect(template_hash['failed_jobs']['max_number_of_failed_delayed_jobs']).to eq(1000) - end - end - - context "when 'cc.failed_jobs.max_number_of_failed_delayed_jobs' is not set (default)" do - it 'does not render max_number_of_failed_delayed_jobs into the ccng config' do - template_hash = YAML.safe_load(template.render(merged_manifest_properties, consumes: links)) - expect(template_hash['failed_jobs']).not_to have_key(:max_number_of_failed_delayed_jobs) - end - end - end - describe 'max_total_results' do context "when 'cc.renderer.max_total_results' is set" do it 'renders max_total_results into the ccng config' do