Adjust unit tests

lib/cloud_controller/metrics/periodic_updater.rb

@@ -46,7 +46,7 @@ def catch_error
     def update_task_stats
       running_tasks = VCAP::CloudController::TaskModel.where(state: VCAP::CloudController::TaskModel::RUNNING_STATE)
       running_task_count = running_tasks.count
-      running_task_memory = running_task_memory.nil? ? 0 : running_tasks.sum(:memory_in_mb)
+      running_task_memory = running_tasks.sum(:memory_in_mb) || 0
       @statsd_updater.update_task_stats(running_task_count, running_task_memory)
       @prometheus_updater.update_task_stats(running_task_count, running_task_memory * 1000 * 1000)
     end

lib/cloud_controller/metrics/prometheus_updater.rb

@@ -9,12 +9,17 @@ def initialize(registry=Prometheus::Client.registry)
       @registry.gauge(:cc_job_queues_length_total, docstring: 'Job queues length of worker processes', labels: [:queue]) unless @registry.exist?(:cc_job_queues_length_total)
       @registry.gauge(:cc_failed_jobs_total, docstring: 'Number of failed jobs of worker processes', labels: [:queue]) unless @registry.exist?(:cc_failed_jobs_total)
       @registry.counter(:cc_staging_requested, docstring: 'Number of staging requests') unless @registry.exist?(:cc_staging_requested)
+
       unless @registry.exist?(:cc_staging_succeeded_duration_seconds)
         @registry.histogram(:cc_staging_succeeded_duration_seconds,
                             docstring: 'Durations of successful staging events',
                             buckets: duration_buckets)
       end
-      @registry.histogram(:cc_staging_failed_duration_seconds, docstring: 'Durations of failed staging events', buckets: duration_buckets) unless @registry.exist?(:cc_staging_failed_duration_seconds)
+      unless @registry.exist?(:cc_staging_failed_duration_seconds)
+        @registry.histogram(:cc_staging_failed_duration_seconds,
+                            docstring: 'Durations of failed staging events',
+                            buckets: duration_buckets)
+      end
 
       @registry.gauge(:cc_requests_outstanding_gauge, docstring: 'Requests Outstanding Gauge') unless @registry.exist?(:cc_requests_outstanding_gauge)
       @registry.counter(:cc_requests_completed, docstring: 'Requests Completed') unless @registry.exist?(:cc_requests_completed)

spec/request/internal/metrics_spec.rb

@@ -75,11 +75,11 @@
       Delayed::Job.dataset.delete
     end
 
-    it 'includes job queue length metric in output' do
+    it 'includes job queue length metric labelled for each queue' do
       get '/internal/v4/metrics', nil
 
-      expect(last_response.body).to match(/cc_job_queue_length_cc_api_0 1\.0/)
-      expect(last_response.body).to match(/cc_job_queue_length_total 2\.0/)
+      expect(last_response.body).to match(/cc_job_queues_length_total{queue="cc_api_0"} 1\.0/)
+      expect(last_response.body).to match(/cc_job_queues_length_total{queue="cc_generic"} 1\.0/)
     end
   end
 
@@ -97,11 +97,22 @@
   end
 
   context 'cc_failed_job_count' do
+    before do
+      Delayed::Job.enqueue(VCAP::CloudController::Jobs::Runtime::EventsCleanup.new(1), { queue: 'cc_api_0', run_at: Time.now + 1.day })
+      Delayed::Job.enqueue(VCAP::CloudController::Jobs::Runtime::EventsCleanup.new(1), { queue: 'cc_generic', run_at: Time.now + 1.day })
+
+      Delayed::Job.dataset.update(failed_at: Time.now.utc)
+    end
+
+    after do
+      Delayed::Job.dataset.delete
+    end
+
     it 'reports failed job count' do
       get '/internal/v4/metrics', nil
 
-      expect(last_response.body).to match(/cc_failed_job_count_cc_api_0 [0-9][0-9]*\.\d+/)
-      expect(last_response.body).to match(/cc_failed_job_count_total [0-9][0-9]*\.\d+/)
+      expect(last_response.body).to match(/cc_failed_jobs_total{queue="cc_api_0"} 1\.0/)
+      expect(last_response.body).to match(/cc_failed_jobs_total{queue="cc_generic"} 1\.0/)
     end
   end
 

spec/unit/lib/cloud_controller/metrics/periodic_updater_spec.rb

@@ -56,7 +56,7 @@ module VCAP::CloudController::Metrics
         periodic_updater.update_task_stats
 
         expect(statsd_updater).to have_received(:update_task_stats).with(anything, 513)
-        expect(prometheus_updater).to have_received(:update_task_stats).with(anything, 513)
+        expect(prometheus_updater).to have_received(:update_task_stats).with(anything, 513_000_000)
       end
 
       context 'when there are no running tasks' do
@@ -256,7 +256,7 @@ module VCAP::CloudController::Metrics
           expected_total = 1
 
           expect(statsd_updater).to have_received(:update_job_queue_length).with(expected_pending_job_count_by_queue, expected_total)
-          expect(prometheus_updater).to have_received(:update_job_queue_length).with(expected_pending_job_count_by_queue, expected_total)
+          expect(prometheus_updater).to have_received(:update_job_queue_length).with(expected_pending_job_count_by_queue)
         end
       end
 
@@ -270,7 +270,7 @@ module VCAP::CloudController::Metrics
           expected_total = 0
 
           expect(statsd_updater).to have_received(:update_job_queue_length).with(expected_pending_job_count_by_queue, expected_total)
-          expect(prometheus_updater).to have_received(:update_job_queue_length).with(expected_pending_job_count_by_queue, expected_total)
+          expect(prometheus_updater).to have_received(:update_job_queue_length).with(expected_pending_job_count_by_queue)
         end
       end
 
@@ -289,7 +289,7 @@ module VCAP::CloudController::Metrics
         expected_total = 3
 
         expect(statsd_updater).to have_received(:update_job_queue_length).with(expected_pending_job_count_by_queue, expected_total)
-        expect(prometheus_updater).to have_received(:update_job_queue_length).with(expected_pending_job_count_by_queue, expected_total)
+        expect(prometheus_updater).to have_received(:update_job_queue_length).with(expected_pending_job_count_by_queue)
       end
 
       it 'finds jobs which have not been attempted yet' do
@@ -306,7 +306,7 @@ module VCAP::CloudController::Metrics
         expected_total = 2
 
         expect(statsd_updater).to have_received(:update_job_queue_length).with(expected_pending_job_count_by_queue, expected_total)
-        expect(prometheus_updater).to have_received(:update_job_queue_length).with(expected_pending_job_count_by_queue, expected_total)
+        expect(prometheus_updater).to have_received(:update_job_queue_length).with(expected_pending_job_count_by_queue)
       end
 
       it 'ignores jobs that have already been attempted' do
@@ -321,7 +321,7 @@ module VCAP::CloudController::Metrics
         expected_total = 0
 
         expect(statsd_updater).to have_received(:update_job_queue_length).with(expected_pending_job_count_by_queue, expected_total)
-        expect(prometheus_updater).to have_received(:update_job_queue_length).with(expected_pending_job_count_by_queue, expected_total)
+        expect(prometheus_updater).to have_received(:update_job_queue_length).with(expected_pending_job_count_by_queue)
       end
 
       it '"resets" pending job count to 0 after they have been emitted' do
@@ -337,7 +337,7 @@ module VCAP::CloudController::Metrics
         expected_total = 2
 
         expect(statsd_updater).to have_received(:update_job_queue_length).with(expected_pending_job_count_by_queue, expected_total)
-        expect(prometheus_updater).to have_received(:update_job_queue_length).with(expected_pending_job_count_by_queue, expected_total)
+        expect(prometheus_updater).to have_received(:update_job_queue_length).with(expected_pending_job_count_by_queue)
 
         Delayed::Job.dataset.delete
         periodic_updater.update_job_queue_length
@@ -349,7 +349,7 @@ module VCAP::CloudController::Metrics
         expected_total = 0
 
         expect(statsd_updater).to have_received(:update_job_queue_length).with(expected_pending_job_count_by_queue, expected_total)
-        expect(prometheus_updater).to have_received(:update_job_queue_length).with(expected_pending_job_count_by_queue, expected_total)
+        expect(prometheus_updater).to have_received(:update_job_queue_length).with(expected_pending_job_count_by_queue)
       end
     end
 
@@ -374,7 +374,7 @@ module VCAP::CloudController::Metrics
           expected_total = 1
 
           expect(statsd_updater).to have_received(:update_failed_job_count).with(expected_failed_jobs_by_queue, expected_total)
-          expect(prometheus_updater).to have_received(:update_failed_job_count).with(expected_failed_jobs_by_queue, expected_total)
+          expect(prometheus_updater).to have_received(:update_failed_job_count).with(expected_failed_jobs_by_queue)
         end
       end
 
@@ -392,7 +392,7 @@ module VCAP::CloudController::Metrics
           expected_total = 0
 
           expect(statsd_updater).to have_received(:update_failed_job_count).with(expected_failed_jobs_by_queue, expected_total)
-          expect(prometheus_updater).to have_received(:update_failed_job_count).with(expected_failed_jobs_by_queue, expected_total)
+          expect(prometheus_updater).to have_received(:update_failed_job_count).with(expected_failed_jobs_by_queue)
         end
       end
 
@@ -414,7 +414,7 @@ module VCAP::CloudController::Metrics
         expected_total = 3
 
         expect(statsd_updater).to have_received(:update_failed_job_count).with(expected_failed_jobs_by_queue, expected_total)
-        expect(prometheus_updater).to have_received(:update_failed_job_count).with(expected_failed_jobs_by_queue, expected_total)
+        expect(prometheus_updater).to have_received(:update_failed_job_count).with(expected_failed_jobs_by_queue)
       end
 
       it '"resets" failed job count to 0 after they have been emitted' do
@@ -431,7 +431,7 @@ module VCAP::CloudController::Metrics
         expected_total = 2
 
         expect(statsd_updater).to have_received(:update_failed_job_count).with(expected_failed_jobs_by_queue, expected_total)
-        expect(prometheus_updater).to have_received(:update_failed_job_count).with(expected_failed_jobs_by_queue, expected_total)
+        expect(prometheus_updater).to have_received(:update_failed_job_count).with(expected_failed_jobs_by_queue)
 
         Delayed::Job.dataset.delete
         periodic_updater.update_failed_job_count
@@ -443,7 +443,7 @@ module VCAP::CloudController::Metrics
         expected_total = 0
 
         expect(statsd_updater).to have_received(:update_failed_job_count).with(expected_failed_jobs_by_queue, expected_total)
-        expect(prometheus_updater).to have_received(:update_failed_job_count).with(expected_failed_jobs_by_queue, expected_total)
+        expect(prometheus_updater).to have_received(:update_failed_job_count).with(expected_failed_jobs_by_queue)
       end
     end
 

spec/unit/lib/cloud_controller/metrics/prometheus_updater_spec.rb

@@ -60,47 +60,35 @@ module VCAP::CloudController::Metrics
       it 'records the length of the delayed job queues and total' do
         expected_local_length   = 5
         expected_generic_length = 6
-        total                   = expected_local_length + expected_generic_length
 
         pending_job_count_by_queue = {
           cc_local: expected_local_length,
           cc_generic: expected_generic_length
         }
 
-        updater.update_job_queue_length(pending_job_count_by_queue, total)
+        updater.update_job_queue_length(pending_job_count_by_queue)
 
-        metric = prom_client.metrics.find { |m| m.name == :cc_job_queue_length_cc_local }
-        expect(metric.get).to eq 5
-
-        metric = prom_client.metrics.find { |m| m.name == :cc_job_queue_length_cc_generic }
-        expect(metric.get).to eq 6
-
-        metric = prom_client.metrics.find { |m| m.name == :cc_job_queue_length_total }
-        expect(metric.get).to eq 11
+        metric = prom_client.get :cc_job_queues_length_total
+        expect(metric.get(labels: { queue: 'cc_local' })).to eq 5
+        expect(metric.get(labels: { queue: 'cc_generic' })).to eq 6
       end
     end
 
     describe '#update_failed_job_count' do
       it 'records the number of failed jobs in the delayed job queue and the total to statsd' do
         expected_local_length   = 5
         expected_generic_length = 6
-        total                   = expected_local_length + expected_generic_length
 
         failed_jobs_by_queue = {
           cc_local: expected_local_length,
           cc_generic: expected_generic_length
         }
 
-        updater.update_failed_job_count(failed_jobs_by_queue, total)
-
-        metric = prom_client.metrics.find { |m| m.name == :cc_failed_job_count_cc_local }
-        expect(metric.get).to eq 5
+        updater.update_failed_job_count(failed_jobs_by_queue)
 
-        metric = prom_client.metrics.find { |m| m.name == :cc_failed_job_count_cc_generic }
-        expect(metric.get).to eq 6
-
-        metric = prom_client.metrics.find { |m| m.name == :cc_failed_job_count_total }
-        expect(metric.get).to eq 11
+        metric = prom_client.get :cc_failed_jobs_total
+        expect(metric.get(labels: { queue: 'cc_local' })).to eq 5
+        expect(metric.get(labels: { queue: 'cc_generic' })).to eq 6
       end
     end
 
@@ -209,29 +197,27 @@ module VCAP::CloudController::Metrics
 
     describe '#report_staging_success_metrics' do
       it 'records staging success metrics' do
+        # 20 seconds
         duration_ns = 20 * 1e9
 
         updater.report_staging_success_metrics(duration_ns)
-        metric = prom_client.metrics.find { |m| m.name == :cc_staging_succeeded }
-        expect(metric.get).to eq 1
 
-        metric = prom_client.metrics.find { |m| m.name == :cc_staging_succeeded_duration }
+        metric = prom_client.get :cc_staging_succeeded_duration_seconds
         # expected buckets for duration, in millis : 10000, 15000, 20000, 25000, 30000
-        expect(metric.get).to eq({ '10000.0' => 0, '15000.0' => 0, '20000.0' => 1, '25000.0' => 1, '30000.0' => 1, 'sum' => 20_000, '+Inf' => 1 })
+        expect(metric.get).to eq({ '+Inf' => 1.0, '10' => 0.0, '30' => 1.0, '300' => 1.0, '5' => 0.0, '60' => 1.0, '600' => 1.0, '890' => 1.0, 'sum' => 20.0 })
       end
     end
 
     describe '#report_staging_failure_metrics' do
       it 'emits staging failure metrics' do
-        duration_ns = 20 * 1e9
+        # 20 seconds
+        duration_ns = 900 * 1e9
 
         updater.report_staging_failure_metrics(duration_ns)
-        metric = prom_client.metrics.find { |m| m.name == :cc_staging_failed }
-        expect(metric.get).to eq 1
 
-        metric = prom_client.metrics.find { |m| m.name == :cc_staging_failed_duration }
+        metric = prom_client.get :cc_staging_failed_duration_seconds
         # expected buckets for duration, in millis : 10000, 15000, 20000, 25000, 30000
-        expect(metric.get).to eq({ '10000.0' => 0, '15000.0' => 0, '20000.0' => 1, '25000.0' => 1, '30000.0' => 1, 'sum' => 20_000, '+Inf' => 1 })
+        expect(metric.get).to eq({ '+Inf' => 1.0, '10' => 0.0, '30' => 0.0, '300' => 0.0, '5' => 0.0, '60' => 0.0, '600' => 0.0, '890' => 0.0, 'sum' => 900.0 })
       end
     end
   end

spec/unit/lib/cloud_controller/metrics/request_metrics_spec.rb

@@ -11,6 +11,7 @@ module VCAP::CloudController::Metrics
       allow(prometheus_client).to receive(:update_gauge_metric)
       allow(prometheus_client).to receive(:decrement_gauge_metric)
       allow(prometheus_client).to receive(:increment_gauge_metric)
+      allow(prometheus_client).to receive(:increment_counter_metric)
     end
 
     describe '#start_request' do
@@ -57,7 +58,7 @@ module VCAP::CloudController::Metrics
         request_metrics.complete_request(status)
 
         expect(prometheus_client).to have_received(:decrement_gauge_metric).with(:cc_requests_outstanding_gauge, kind_of(String))
-        expect(prometheus_client).to have_received(:increment_gauge_metric).with(:cc_requests_completed, kind_of(String))
+        expect(prometheus_client).to have_received(:increment_counter_metric).with(:cc_requests_completed, kind_of(String))
       end
 
       it 'normalizes http status codes in statsd' do