Add Redis job

- Redis will be used for rate limiting and (at a later point in time)
  also for metrics.
- The monit process will only be started if either the Puma webserver is
  used or 'cc.experimental.use_redis' is set to 'true' (defaults to
  'false').
- If Redis is used, the CC config 'redis.socket' is set; this switches
  the rate limit implementation from using in-memory counters to Redis.

jobs/cloud_controller_ng/spec

@@ -224,6 +224,8 @@ provides:
   - ssl.skip_cert_verify
   - system_domain
   - uaa.clients.cc_routing.secret
+  - cc.experimental.use_puma_webserver
+  - cc.experimental.use_redis
 
 consumes:
 - name: database
@@ -1210,6 +1212,10 @@ properties:
     description: "Use Puma in place of Thin as the webserver. This may increase performance as Puma forks Cloud Controller processes to avoid relying on threads"
     default: false
 
+  cc.experimental.use_redis:
+    description: "Use co-deployed Redis for rate limiting and metrics. If the Puma webserver is enabled, Redis will automatically be used."
+    default: false
+
   cc.puma.workers:
     description: "Number of workers for Puma webserver."
     default: 3

jobs/cloud_controller_ng/templates/bpm.yml.erb

@@ -12,6 +12,16 @@ def mount_nfs_volume!(config)
   end
 end
 
+def mount_redis_volume!(config)
+  if p("cc.experimental.use_puma_webserver") || p("cc.experimental.use_redis")
+    config['additional_volumes'] = [] unless config.key?('additional_volumes')
+    config['additional_volumes'] << {
+      "path" => "/var/vcap/data/redis",
+      "mount_only" => true,
+    }
+  end
+end
+
 def enable_fog_debugging!(config)
     if p("cc.log_fog_requests")
         case p("cc.packages.fog_connection.provider", "").downcase
@@ -67,6 +77,7 @@ if properties.env
     end
 end
 mount_nfs_volume!(cloud_controller_ng_config)
+mount_redis_volume!(cloud_controller_ng_config)
 
 nginx_config = {
   "name" => "nginx",

jobs/cloud_controller_ng/templates/cloud_controller_ng.yml.erb

@@ -20,7 +20,7 @@
 
   def validate_app_domains(app_domains)
     app_domains.each do |app_domain|
-      if app_domain.is_a?(Hash) && app_domain['internal'] == true && app_domain.has_key?('router_group_name')
+      if app_domain.is_a?(Hash) && app_domain['internal'] == true && app_domain.key?('router_group_name')
         raise 'Error for app_domains: Router groups cannot be specified for internal domains.'
       end
     end
@@ -456,6 +456,11 @@ rate_limiter_v2_api:
   per_process_admin_limit: <%= (p("cc.rate_limiter_v2_api.admin_limit").to_f/instances).ceil %>
   reset_interval_in_minutes: <%= p("cc.rate_limiter_v2_api.reset_interval_in_minutes") %>
 
+<% if p("cc.experimental.use_puma_webserver") || p("cc.experimental.use_redis") %>
+redis:
+  socket: "/var/vcap/data/redis/redis.sock"
+<% end %>
+
 <%
 cc_uploader_url = nil
 if_link("cc_uploader") do |cc_uploader|

jobs/redis/monit

@@ -0,0 +1,14 @@
+<%
+  cloud_controller_internal = link("cloud_controller_internal")
+  if cloud_controller_internal.p("cc.experimental.use_puma_webserver") || cloud_controller_internal.p("cc.experimental.use_redis")
+%>
+
+check process redis
+  with pidfile /var/vcap/sys/run/bpm/redis/redis.pid
+  start program "/var/vcap/jobs/bpm/bin/bpm start redis"
+  stop program "/var/vcap/jobs/bpm/bin/bpm stop redis"
+  group vcap
+
+<%
+  end
+%>

jobs/redis/spec

@@ -0,0 +1,18 @@
+---
+name: redis
+
+description: "Co-deployed Redis used for rate limiting and metrics"
+
+templates:
+  bpm.yml.erb: config/bpm.yml
+  pre-start.sh.erb: bin/pre-start.sh
+  redis-cli.sh.erb: bin/redis-cli.sh
+
+packages:
+  - redis
+
+consumes:
+  - name: cloud_controller_internal
+    type: cloud_controller_internal
+
+properties: {}

jobs/redis/templates/bpm.yml.erb

@@ -0,0 +1,16 @@
+---
+processes:
+- name: redis
+  executable: /var/vcap/packages/redis/redis-server
+  args:
+    - --port
+    - 0
+    - --unixsocket
+    - /var/vcap/data/redis/redis.sock
+    - --save
+    - ""
+  ephemeral_disk: true
+  limits:
+    open_files: 10032
+  hooks:
+    pre_start: /var/vcap/jobs/redis/bin/pre-start.sh

jobs/redis/templates/pre-start.sh.erb

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+set -e
+
+sysctl -w vm.overcommit_memory=1

jobs/redis/templates/redis-cli.sh.erb

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+set -e
+
+/var/vcap/packages/redis/redis-cli -s /var/vcap/data/redis/redis.sock

packages/redis/README.md

@@ -0,0 +1,9 @@
+redis
+============
+Redis in-memory data store.
+
+This file can be downloaded from the following locations:
+
+| Filename | Download URL |
+| -------- | ------------ |
+| 7.0.11.tar.gz | https://github.com/redis/redis/archive/7.0.11.tar.gz |

packages/redis/packaging

@@ -0,0 +1,16 @@
+#!/bin/bash -eu
+
+function main() {
+  local redis_version
+  redis_version="7.0.11"
+
+  tar xzf "redis/${redis_version}.tar.gz"
+
+  pushd "redis-${redis_version}" > /dev/null
+    make
+
+    cp src/redis-server src/redis-cli "${BOSH_INSTALL_TARGET}"
+  popd > /dev/null
+}
+
+main

packages/redis/spec

@@ -0,0 +1,4 @@
+---
+name: redis
+files:
+- redis/7.0.11.tar.gz

spec/cloud_controller_ng/bpm_spec.rb

@@ -17,6 +17,21 @@ def expect_default_debug_env_vars(env_vars)
           expect(env_vars['FOG_DEBUG']).to be(true)
         end
 
+        def redis_volume_mounted?(process)
+          return false unless process.key?('additional_volumes')
+
+          results = process['additional_volumes'].select { |v| v['path'] == '/var/vcap/data/redis' }
+          return false unless results.length == 1
+
+          redis_volume = results[0]
+          return false unless redis_volume.key?('mount_only')
+
+          mount_only = redis_volume['mount_only']
+          return false unless mount_only.is_a?(TrueClass) && mount_only == true
+
+          true
+        end
+
         let(:release_path) { File.join(File.dirname(__FILE__), '../..') }
         let(:release) { ReleaseDir.new(release_path) }
         let(:job) { release.job('cloud_controller_ng') }
@@ -96,6 +111,36 @@ def expect_default_debug_env_vars(env_vars)
               expect(env_vars).not_to have_key('ALIYUN_OSS_SDK_LOG_LEVEL')
             end
           end
+
+          context 'when the puma webserver is used' do
+            it 'mounts the redis volume into the ccng job container' do
+              template_hash = YAML.safe_load(template.render({ 'cc' => { 'experimental' => { 'use_puma_webserver' => true } } }, consumes: {}))
+
+              results = template_hash['processes'].select { |p| p['name'].include?('cloud_controller_ng') }
+              expect(results.length).to eq(1)
+              expect(redis_volume_mounted?(results[0])).to be_truthy
+            end
+          end
+
+          context "when 'cc.experimental.use_redis' is set to 'true'" do
+            it 'mounts the redis volume into the ccng job container' do
+              template_hash = YAML.safe_load(template.render({ 'cc' => { 'experimental' => { 'use_redis' => true } } }, consumes: {}))
+
+              results = template_hash['processes'].select { |p| p['name'].include?('cloud_controller_ng') }
+              expect(results.length).to eq(1)
+              expect(redis_volume_mounted?(results[0])).to be_truthy
+            end
+          end
+
+          context "when neither the puma webserver is used nor 'cc.experimental.use_redis' is set to 'true'" do
+            it 'does not mount the redis volume into the ccng job container' do
+              template_hash = YAML.safe_load(template.render({}, consumes: {}))
+
+              results = template_hash['processes'].select { |p| p['name'].include?('cloud_controller_ng') }
+              expect(results.length).to eq(1)
+              expect(redis_volume_mounted?(results[0])).to be_falsey
+            end
+          end
         end
       end
     end

spec/cloud_controller_ng/cloud_controller_ng_spec.rb

@@ -599,6 +599,31 @@ module Test
               end
             end
           end
+
+          describe 'redis config' do
+            context 'when the puma webserver is used' do
+              it 'renders the redis socket into the ccng config' do
+                merged_manifest_properties['cc']['experimental'] = { 'use_puma_webserver' => true }
+                template_hash = YAML.safe_load(template.render(merged_manifest_properties, consumes: links))
+                expect(template_hash['redis']['socket']).to eq('/var/vcap/data/redis/redis.sock')
+              end
+            end
+
+            context "when 'cc.experimental.use_redis' is set to 'true'" do
+              it 'renders the redis socket into the ccng config' do
+                merged_manifest_properties['cc']['experimental'] = { 'use_redis' => true }
+                template_hash = YAML.safe_load(template.render(merged_manifest_properties, consumes: links))
+                expect(template_hash['redis']['socket']).to eq('/var/vcap/data/redis/redis.sock')
+              end
+            end
+
+            context "when neither the puma webserver is used nor 'cc.experimental.use_redis' is set to 'true'" do
+              it 'does not render the redis socket into the ccng config' do
+                template_hash = YAML.safe_load(template.render(merged_manifest_properties, consumes: links))
+                expect(template_hash).not_to have_key('redis')
+              end
+            end
+          end
         end
       end
     end

spec/redis/monit_spec.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require 'rspec'
+require 'bosh/template/test'
+
+module Bosh
+  module Template
+    module Test
+      describe 'redis job template rendering' do
+        let(:release_path) { File.join(File.dirname(__FILE__), '../..') }
+        let(:release) { ReleaseDir.new(release_path) }
+        let(:job) { release.job('redis') }
+        let(:spec) { job.instance_variable_get(:@spec) }
+        let(:job_path) { job.instance_variable_get(:@job_path) }
+        let(:cc_internal_properties) { { 'cc' => { 'experimental' => { 'use_puma_webserver' => false, 'use_redis' => false } } } }
+        let(:link) { Link.new(name: 'cloud_controller_internal', properties: cc_internal_properties) }
+
+        describe 'monit' do
+          let(:template) { Template.new(spec, File.join(job_path, 'monit')) }
+
+          context 'when the puma webserver is used' do
+            it 'renders the monit directives' do
+              cc_internal_properties['cc']['experimental']['use_puma_webserver'] = true
+              result = template.render({}, consumes: [link])
+              expect(result).to include('check process redis')
+            end
+          end
+
+          context "when 'cc.experimental.use_redis' is set to 'true'" do
+            it 'renders the monit directives' do
+              cc_internal_properties['cc']['experimental']['use_redis'] = true
+              result = template.render({}, consumes: [link])
+              expect(result).to include('check process redis')
+            end
+          end
+
+          context "when neither the puma webserver is used nor 'cc.experimental.use_redis' is set to 'true'" do
+            it 'does not render the monit directives' do
+              result = template.render({}, consumes: [link])
+              expect(result.strip).to eq('')
+            end
+          end
+        end
+      end
+    end
+  end
+end