Merge remote-tracking branch 'origin/master' into puma-pr

Signed-off-by: Beyhan Veli <[email protected]>

ci/pipeline.yml

@@ -5,7 +5,7 @@ groups:
       - unit-2.4
       - unit-2.4-mysql
       - unit-2.4-postgres
-      - unit-2.4-rds
+      - unit-2.4-db-migrations-rds
       - blobstore-client-integration
       - integration-postgres-gocli-sha2
       - integration-mysql-gocli-sha1
@@ -19,20 +19,19 @@ groups:
       - bats-centos
       - bats-ubuntu
       - brats-ubuntu
-      - brats-ubuntu-non-tls-agent
       - upgrade-tests
       - blobstore-performance
 
   - name: mysql
     jobs:
       - unit-2.4-mysql
-      - unit-2.4-rds
+      - unit-2.4-db-migrations-rds
       - integration-mysql-gocli-sha1
 
   - name: postgres
     jobs:
       - unit-2.4-postgres
-      - unit-2.4-rds
+      - unit-2.4-db-migrations-rds
 
   - name: finalize
     jobs:
@@ -241,7 +240,7 @@ jobs:
           on_failure:
             <<: *slack-alert
 
-  - name: unit-2.4-rds
+  - name: unit-2.4-db-migrations-rds
     public: true
     serial: true
     build_logs_to_retain: 250
@@ -680,28 +679,6 @@ jobs:
         input_mapping:
           stemcell: candidate-warden-ubuntu-stemcell
 
-  - name: brats-ubuntu-non-tls-agent
-    serial: true
-    plan:
-    - do:
-      - aggregate:
-        - get: bosh-src
-          passed:
-          - candidate-release
-        - get: bosh-dns-release
-        - get: warden-ubuntu-trusty-3363-37
-        - get: bosh-release
-          resource: bosh-candidate-release-tarballs
-          trigger: true
-          passed:
-            - candidate-release
-      - task: test-brats
-        file: bosh-src/ci/tasks/test-brats.yml
-        tags: ["worker-brats"]
-        privileged: true
-        input_mapping:
-          stemcell: warden-ubuntu-trusty-3363-37
-
   - name: finalize-bosh-release
     serial: true
     plan:
@@ -809,7 +786,7 @@ resources:
   - name: bosh-dns-release
     type: bosh-io-release
     source:
-      repository: cloudfoundry/dns-release
+      repository: cloudfoundry/bosh-dns-release
 
   #
   # BATS
@@ -846,12 +823,6 @@ resources:
       bucket: bosh-core-stemcells-candidate
       regexp: warden/bosh-stemcell-(.+)-warden-boshlite-ubuntu-trusty-go_agent.tgz
 
-  - name: warden-ubuntu-trusty-3363-37
-    type: bosh-io-stemcell
-    source:
-      name: bosh-warden-boshlite-ubuntu-trusty-go_agent
-      version_family: "3363.37"
-
   - name: warden-ubuntu-trusty
     type: bosh-io-stemcell
     source:

ci/tasks/test-integration-gocli.sh

@@ -40,8 +40,8 @@ case "$DB" in
     exit 1
 esac
 
-mv ./bosh-cli/*bosh-cli-*-linux-amd64 /usr/local/bin/gobosh
-chmod +x /usr/local/bin/gobosh
+mv ./bosh-cli/*bosh-cli-*-linux-amd64 /usr/local/bin/bosh
+chmod +x /usr/local/bin/bosh
 
 source /etc/profile.d/chruby.sh
 chruby $RUBY_VERSION

ci/tasks/test-upgrade.sh

@@ -40,8 +40,8 @@ case "$DB" in
     exit 1
 esac
 
-mv ./bosh-cli/*bosh-cli-*-linux-amd64 /usr/local/bin/gobosh
-chmod +x /usr/local/bin/gobosh
+mv ./bosh-cli/*bosh-cli-*-linux-amd64 /usr/local/bin/bosh
+chmod +x /usr/local/bin/bosh
 
 source /etc/profile.d/chruby.sh
 chruby $RUBY_VERSION

docs/workstation_setup.md

@@ -64,6 +64,27 @@
     cd ~/workspace/bosh/src
     bundle install
     ```
+9. Special instructions for nginx on  Mac
+
+    Before running `rake spec:integration:install_dependencies`, modify the nginx packaging script to fix compilation on OSX.
+    
+    ```
+    diff --git a/packages/nginx/packaging b/packages/nginx/packaging
+    index 007e408a5..cc8956efe 100755
+    --- a/packages/nginx/packaging
+    +++ b/packages/nginx/packaging
+    @@ -27,7 +27,9 @@ pushd nginx-1.12.1
+             --add-module=../headers-more-nginx-module-0.30 \
+             --with-http_ssl_module \
+             --with-http_dav_module \
+    -    --add-module=../nginx-upload-module-2.2
+    +    --add-module=../nginx-upload-module-2.2 \
+    +    --with-ld-opt="-L/usr/local/opt/openssl/lib" \
+    +    --with-cc-opt="-I/usr/local/opt/openssl/include"
+    
+         make
+         make install
+    ```
 
 ## Issues
 
@@ -77,7 +98,7 @@ If you have trouble bundling, you may have to install pg gem manually by specify
 
 ### Custom bosh-cli
 
-To use a custom go-cli in integration tests change `gobosh` in  `src/spec/gocli/support/bosh_go_cli_runner.rb`.
+To use a custom go-cli in integration tests change `bosh` in  `src/spec/gocli/support/bosh_go_cli_runner.rb`.
 
 ### Special instructions for nginx on  Mac
 

jobs/nats/spec

@@ -32,11 +32,11 @@ properties:
     description: Timeout (in seconds) for clients to send auth credentials
     default: 30
   nats.tls.ca:
-    description: CA cert to trust when communicating with NATS server
+    description: CA cert used by NATS server to verify clients certificates (For Mutual TLS Connections)
   nats.tls.server.certificate:
-    description: Certificate for NATs mutual TLS
+    description: Certificate used by the NATS server to serve TLS connections
   nats.tls.server.private_key:
-    description: Private Key for NATs mutual TLS
+    description: Private Key used by the NATS server to serve TLS connections
   nats.tls.timeout:
     description: Timeout (in seconds) for TLS handshake
     default: 30

src/bosh-dev/assets/sandbox/nats_server/certs/generate.sh

@@ -12,21 +12,21 @@ mkdir -p ./director
 mkdir -p ./health_monitor
 mkdir -p ./test_client
 
-gobosh int --vars-store=${CREDS_FILE} -v hostname=$HOSTNAME ${TEMPLATE_FILE}
+bosh int --vars-store=${CREDS_FILE} -v hostname=$HOSTNAME ${TEMPLATE_FILE}
 
-gobosh int --path=/default_ca/ca ${CREDS_FILE} | sed '/^$/d' > rootCA.pem
-gobosh int --path=/default_ca/private_key ${CREDS_FILE} | sed '/^$/d' > rootCA.key
+bosh int --path=/default_ca/ca ${CREDS_FILE} | sed '/^$/d' > rootCA.pem
+bosh int --path=/default_ca/private_key ${CREDS_FILE} | sed '/^$/d' > rootCA.key
 
-gobosh int --path=/nats/certificate ${CREDS_FILE} | sed '/^$/d' > nats/certificate.pem
-gobosh int --path=/nats/private_key ${CREDS_FILE} | sed '/^$/d' > nats/private_key
+bosh int --path=/nats/certificate ${CREDS_FILE} | sed '/^$/d' > nats/certificate.pem
+bosh int --path=/nats/private_key ${CREDS_FILE} | sed '/^$/d' > nats/private_key
 
-gobosh int --path=/director_client/certificate ${CREDS_FILE} | sed '/^$/d' > director/certificate.pem
-gobosh int --path=/director_client/private_key ${CREDS_FILE} | sed '/^$/d' > director/private_key
+bosh int --path=/director_client/certificate ${CREDS_FILE} | sed '/^$/d' > director/certificate.pem
+bosh int --path=/director_client/private_key ${CREDS_FILE} | sed '/^$/d' > director/private_key
 
-gobosh int --path=/hm_client/certificate ${CREDS_FILE} | sed '/^$/d' > health_monitor/certificate.pem
-gobosh int --path=/hm_client/private_key ${CREDS_FILE} | sed '/^$/d' > health_monitor/private_key
+bosh int --path=/hm_client/certificate ${CREDS_FILE} | sed '/^$/d' > health_monitor/certificate.pem
+bosh int --path=/hm_client/private_key ${CREDS_FILE} | sed '/^$/d' > health_monitor/private_key
 
 # This cert can subscribe and publish to everything
-gobosh int --path=/test_client/certificate ${CREDS_FILE} | sed '/^$/d' > test_client/certificate.pem
-gobosh int --path=/test_client/private_key ${CREDS_FILE} | sed '/^$/d' > test_client/private_key
+bosh int --path=/test_client/certificate ${CREDS_FILE} | sed '/^$/d' > test_client/certificate.pem
+bosh int --path=/test_client/private_key ${CREDS_FILE} | sed '/^$/d' > test_client/private_key
 

src/bosh-dev/lib/bosh/dev/sandbox/nginx.rb

@@ -44,7 +44,7 @@ def shasum(directory)
     end
 
     def sync_release_blobs
-      Dir.chdir(RELEASE_ROOT) { @runner.run('gobosh sync-blobs') }
+      Dir.chdir(RELEASE_ROOT) { @runner.run('bosh sync-blobs') }
     end
 
     def compile

src/bosh-dev/lib/bosh/dev/tasks/spec.rake

@@ -11,6 +11,7 @@ require 'bosh/dev/sandbox/services/config_server_service'
 require 'bosh/dev/legacy_agent_manager'
 require 'bosh/dev/verify_multidigest_manager'
 require 'bosh/dev/gnatsd_manager'
+require 'bosh/dev/test_runner'
 require 'parallel_tests/tasks'
 
 namespace :spec do
@@ -130,29 +131,6 @@ namespace :spec do
 
   task :upgrade => %w(spec:integration:upgrade)
 
-  def unit_exec(build)
-    command = "rspec --tty --backtrace -c -f p #{unit_files(build)}"
-
-    puts "----- BEGIN #{build}"
-    system({'BOSH_BUILD_NAME' => build}, "cd #{build} && #{command}")
-    puts "----- END   #{build}\n\n"
-  end
-
-  def unit_files(build)
-    cpi_builds.include?(build) ? 'spec/unit/' : 'spec/'
-  end
-
-  def unit_builds
-    @unit_builds ||= begin
-      builds = Dir['*'].select { |f| File.directory?(f) && File.exists?("#{f}/spec") }.sort
-      builds -= %w(bat)
-    end
-  end
-
-  def cpi_builds
-    @cpi_builds ||= unit_builds.select { |f| File.directory?(f) && f.end_with?("_cpi") }
-  end
-
   desc 'Run all release unit tests (ERB templates)'
   task :release_unit do
     puts "Release unit tests (ERB templates)"
@@ -166,22 +144,19 @@ namespace :spec do
   end
 
   namespace :unit do
+    runner = Bosh::Dev::TestRunner.new
+
     desc 'Run all unit tests for ruby components'
     task :ruby do
       trap('INT') { exit }
-      log_dir = Dir.mktmpdir
-      puts "Logging spec results in #{log_dir}"
-
-      unit_builds.each do |build|
-        unit_exec(build)
-      end
+      runner.ruby
     end
 
-    (unit_builds - cpi_builds).each do |build|
+    runner.unit_builds.each do |build|
       desc "Run unit tests for the #{build} component"
       task build.sub(/^bosh[_-]/, '').intern do
         trap('INT') { exit }
-        unit_exec(build)
+        runner.unit_exec(build)
       end
     end
 

src/bosh-dev/lib/bosh/dev/test_runner.rb

@@ -0,0 +1,54 @@
+require 'common/thread_pool'
+
+module Bosh::Dev
+  class TestRunner
+    def ruby
+      log_dir = Dir.mktmpdir
+      puts "Logging spec results in #{log_dir}"
+
+      max_threads = ENV.fetch('BOSH_MAX_THREADS', 10).to_i
+      null_logger = Logging::Logger.new('Ignored')
+      Bosh::ThreadPool.new(max_threads: max_threads, logger: null_logger).wrap do |pool|
+        unit_builds.each do |build|
+          pool.process do
+            unit_exec(build, "#{log_dir}/#{build}.log")
+          end
+        end
+
+        pool.wait
+      end
+    end
+
+    def unit_exec(build, log_file = nil)
+      command = unit_cmd(log_file)
+
+      # inject command name so coverage results for each component don't clobber others
+      if Kernel.system({'BOSH_BUILD_NAME' => build}, "cd #{build} && #{command}")
+        puts "----- BEGIN #{build}"
+        puts "            #{command}"
+        print File.read(log_file) if log_file && File.exists?(log_file)
+        puts "----- END   #{build}\n\n"
+      else
+        error_message = "#{build} failed to build unit tests"
+        error_message += ": #{File.read(log_file)}" if log_file && File.exists?(log_file)
+        raise error_message
+      end
+    end
+
+    def unit_cmd(log_file = nil)
+      "".tap do |cmd|
+        cmd << 'rspec --tty --backtrace -c -f p spec'
+        cmd << " > #{log_file} 2>&1" if log_file
+      end
+    end
+
+    def unit_builds
+      @unit_builds ||= begin
+                         builds = Dir['*'].select do |f|
+                           File.directory?(f) && File.exists?("#{f}/spec")
+                         end.sort
+                         builds -= %w(bat)
+                       end
+    end
+  end
+end