Merge remote-tracking branch 'origin/master' into puma-pr

.final_builds/jobs/director/index.yml

@@ -187,6 +187,10 @@ builds:
     version: 93aa5c2fd4b7a0cae4c3db8751497b1b27097a9d
     blobstore_id: 06c8321a-d725-46ed-a1eb-9b6c3ccabc3c
     sha1: 815cde9cd22882fa678d44714d322cb5f3ab5727
+  959e33f30fd99a08c44c9f8a57abe937d90f9ab8:
+    version: 959e33f30fd99a08c44c9f8a57abe937d90f9ab8
+    blobstore_id: e9bf3188-5641-41c9-7c91-dc05d1a54590
+    sha1: 54c6f09087cc9536d1b176e0c2bd5e47fab9eafd
   963e768cf08a4f319367a50d91e0308e80279e61:
     version: "14"
     blobstore_id: 669696b4-22bf-47eb-b501-0c424f474035

.final_builds/packages/director/index.yml

@@ -899,6 +899,10 @@ builds:
     version: "23"
     blobstore_id: b96b0710-7ebf-4a16-baad-bda36feb3817
     sha1: ac4edbd5450dba2d5c813533140d20cc7a9b2ac8
+  b75f5a45a82d0403dae2413883b4c1b1ec358bd8:
+    version: b75f5a45a82d0403dae2413883b4c1b1ec358bd8
+    blobstore_id: ed0c44fa-534f-4b55-7ef1-fc059323eaec
+    sha1: d767bf8a211113bf841624130042e6df189fa553
   b9389705f21e33cca4fcd42fa7e2a4b9cbcf09b0:
     version: b9389705f21e33cca4fcd42fa7e2a4b9cbcf09b0
     blobstore_id: 89b59665-2fc6-4e15-921b-e17e18e80de3

.final_builds/packages/ruby-2.4/index.yml

@@ -15,6 +15,10 @@ builds:
     version: 962fd56b54466da82d604d7361fab1661a86563e
     blobstore_id: ebcf4021-5b66-49a3-5d6e-76815e500c61
     sha1: fb3e2c1adb6370c86563046fdc9d7dcf48fce798
+  cff0ddb99e62cd9dfa146248dcbfd6df2a62a54e:
+    version: cff0ddb99e62cd9dfa146248dcbfd6df2a62a54e
+    blobstore_id: 078f68c0-d137-4220-46d9-2f3c8abff3c9
+    sha1: d439cbfc58ea7504e8ecd735b258af2b5d09c17d
   de0e84b8dc18147ef8378cd172a048c497824a73:
     version: de0e84b8dc18147ef8378cd172a048c497824a73
     blobstore_id: 281d95a2-a840-4c34-69d3-d13e0cf36018

.gitignore

@@ -25,8 +25,6 @@ aws_rds_receipt.yml
 aws_route53_receipt.yml
 aws_vpc_receipt.yml
 
-/bin/
-
 create-rds-output*.yml
 create-vpc-output*.yml
 
@@ -35,6 +33,7 @@ create-vpc-output*.yml
 /src/go/src/code.google.com/p/go.tools
 /src/go/src/golang.org
 /src/go/src/github.com/golang
+/src/go/src/github.com/cloudfoundry/gnatsd/
 
 pkg/
 

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "src/go/src/github.com/cloudfoundry/bosh-agent"]
+	path = src/go/src/github.com/cloudfoundry/bosh-agent
+	url = https://github.com/cloudfoundry/bosh-agent.git

ci/bats/tasks/deploy-director.sh

@@ -23,6 +23,7 @@ bosh-cli interpolate bosh-deployment/bosh.yml \
   -o bosh-deployment/jumpbox-user.yml \
   -o bosh-src/ci/bats/ops/remove-health-monitor.yml \
   -o bosh-deployment/local-bosh-release.yml \
+  -o bosh-deployment/experimental/nats-tls.yml \
   -v dns_recursor_ip=8.8.8.8 \
   -v director_name=bats-director \
   -v local_bosh_release=$(realpath bosh-release/*.tgz) \

ci/docker/main-bosh-docker/start-bosh.sh

@@ -163,6 +163,7 @@ function main() {
       command bosh int bosh.yml \
         -o docker/cpi.yml \
         -o jumpbox-user.yml \
+        -o experimental/nats-tls.yml \
         -v director_name=docker \
         -v internal_cidr=10.245.0.0/16 \
         -v internal_gw=10.245.0.1 \

ci/docker/main-bosh-docker/start-inner-bosh.sh

@@ -6,6 +6,9 @@ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 bosh_path="${bosh_release_path:-${script_dir}/../../../}"
 bosh_release_path=""
 
+src_dir="${script_dir}/../../../"
+stemcell="${src_dir}/../candidate-warden-ubuntu-stemcell/bosh-stemcell-*-go_agent.tgz"
+
 pushd "${bosh_path}" > /dev/null
   if [[ ! -e $(find . -maxdepth 1 -name "*.tgz") ]]; then
     bosh create-release --tarball release.tgz
@@ -29,6 +32,7 @@ mkdir -p ${inner_bosh_dir}
 
 bosh int bosh.yml \
   -o "$script_dir/inner-bosh-ops.yml" \
+  -o ./experimental/nats-tls.yml \
   -o jumpbox-user.yml \
   -v director_name=docker-inner \
   -v internal_cidr=10.245.0.0/16 \
@@ -41,11 +45,7 @@ bosh int bosh.yml \
   -v local_bosh_release="${bosh_release_path}" \
   ${@} > "${inner_bosh_dir}/bosh-director.yml"
 
-bosh upload-stemcell \
-  --sha1=70c2584a8ad8e2b417c32809c34377728a6d6f86 \
-  --name=bosh-warden-boshlite-ubuntu-trusty-go_agent \
-  --version=3363.20 \
-  https://bosh.io/d/stemcells/bosh-warden-boshlite-ubuntu-trusty-go_agent?v=3363.20
+bosh upload-stemcell ${stemcell}
 
 # point to our outer director and launch the inner director
 source "${local_bosh_dir}/env"

ci/pipeline.yml

@@ -278,23 +278,24 @@ jobs:
     build_logs_to_retain: 250
     plan:
       - { get: bosh-src, trigger: true }
-      - task: test-s3
-        file: bosh-src/ci/tasks/test-s3-blobstore-client-integration.yml
-        params:
-          access_key_id: ((blobstore_client_aws_access_key_id))
-          secret_access_key: ((blobstore_client_aws_secret_access_key))
-          s3_region: ((blobstore_client_aws_s3_region))
-          s3_host: ((blobstore_client_aws_s3_host))
-          run_aws_tests: "Not null"
-      - task: test-local
-        file: bosh-src/ci/tasks/test-local-blobstore-client-integration.yml
-      - task: test-dav
-        file: bosh-src/ci/tasks/test-dav-blobstore-client-integration.yml
-      - task: test-gcs
-        file: bosh-src/ci/tasks/test-gcs-blobstore-client-integration.yml
-        params:
-          google_project: ((blobstore_client_google_project))
-          google_json_key_data: ((blobstore_client_google_json_key_data))
+      - aggregate:
+        - task: test-s3
+          file: bosh-src/ci/tasks/test-s3-blobstore-client-integration.yml
+          params:
+            access_key_id: ((blobstore_client_aws_access_key_id))
+            secret_access_key: ((blobstore_client_aws_secret_access_key))
+            s3_region: ((blobstore_client_aws_s3_region))
+            s3_host: ((blobstore_client_aws_s3_host))
+            run_aws_tests: "Not null"
+        - task: test-local
+          file: bosh-src/ci/tasks/test-local-blobstore-client-integration.yml
+        - task: test-dav
+          file: bosh-src/ci/tasks/test-dav-blobstore-client-integration.yml
+        - task: test-gcs
+          file: bosh-src/ci/tasks/test-gcs-blobstore-client-integration.yml
+          params:
+            google_project: ((blobstore_client_google_project))
+            google_json_key_data: ((blobstore_client_google_json_key_data))
 
   - name: blobstore-performance
     public: true
@@ -768,7 +769,7 @@ resources:
   - name: load-tests-interval
     type: time
     source:
-      interval: 1h
+      interval: 5h
 
   - name: slack-alert
     type: slack-notification

ci/tasks/test-gcs-blobstore-client-integration.sh

@@ -10,55 +10,48 @@ chruby 2.1.2
 check_param google_project
 check_param google_json_key_data
 
-pushd bosh-src
-  bosh sync blobs
-  chmod +x ./blobs/bosh-gcscli/bosh-gcscli-*-amd64
-popd
-
 function clean_up_bucket {
   local bucket=$1
-  gsutil rm gs://${bucket_name}/public # This may fail, which is fine.
-  gsutil rb "gs://${bucket}"
-}
-
-function clean_up {
-  local bucket=$1
-  clean_up_bucket ${bucket}
+  gsutil rm -rf gs://${bucket_name}/
 }
 
-gcloud config set project $google_project
-
-echo $google_json_key_data > key.json
-gcloud auth activate-service-account --key-file=key.json
+function gcs_login {
+  gcloud config set project $google_project
 
-export GCS_SERVICE_ACCOUNT_KEY=$google_json_key_data
+  echo $google_json_key_data > key.json
+  gcloud auth activate-service-account --key-file=key.json
+}
 
-pushd bosh-src/src
-  bundle install
-  pushd bosh-director
+function gcs_logout {
+  gcloud auth revoke
+}
 
-    # Create bucket in US region
+function setup_bucket {
     bucket_name="bosh-blobstore-bucket-$RANDOM"
     echo -n "foobar" > public
-    gsutil mb -c MULTI_REGIONAL -l us gs://${bucket_name}
-    gsutil iam ch allUsers:objectViewer gs://${bucket_name}
-    gsutil iam ch allUsers:legacyObjectReader gs://${bucket_name}
-		gsutil iam ch allUsers:legacyBucketReader gs://${bucket_name}
-		echo "waiting for IAM to propagate" && \
-		until curl -s \
-			https://storage.googleapis.com/${bucket_name}/non-existent \
-			| grep -q "NoSuchKey"; do sleep 1; done; \
-    trap 'clean_up_bucket ${bucket_name}; exit 1' ERR
-    gsutil cp public gs://${bucket_name}/public
-    retry_command "gsutil acl ch -r -u AllUsers:R gs://${bucket_name}/public"
-    trap 'clean_up ${bucket_name}; exit 1' ERR
+    gcs_login
+    gsutil mb -c MULTI_REGIONAL -l us "gs://${bucket_name}"
+
+    trap 'clean_up_bucket ${bucket_name}' EXIT
+
+    gsutil acl set public-read "gs://${bucket_name}"
+    gsutil cp -a public-read public "gs://${bucket_name}/"
 
     export GCS_BUCKET_NAME=${bucket_name}
+}
+
+setup_bucket
 
-    bundle exec rspec spec/functional/gcs_spec.rb --tag general_gcs
+pushd bosh-src
+  bosh sync blobs
+  chmod +x ./blobs/bosh-gcscli/bosh-gcscli-*-amd64
+popd
 
-    trap - ERR
-    clean_up ${bucket_name}
+pushd bosh-src/src
+  bundle install
+popd
 
-  popd
+pushd bosh-src/src/bosh-director
+  export GCS_SERVICE_ACCOUNT_KEY="${google_json_key_data}"
+  bundle exec rspec spec/functional/gcs_spec.rb --tag general_gcs
 popd

config/blobs.yml

@@ -1,11 +1,15 @@
-bosh-gcscli/bosh-gcscli-0.0.2-linux-amd64:
+bosh-gcscli/bosh-gcscli-0.0.3-linux-amd64:
   size: 12777526
-  object_id: 32a1f288-c0ba-4d44-6a18-fc699afa4778
-  sha: a151d3829f53b68bd15edfd3e6a527d187591992
-davcli/davcli-0.0.6-linux-amd64:
-  size: 8705864
-  object_id: fbe6b42b-cd0e-47a6-a0b4-501be4b52426
-  sha: 6b42b9833ad8f4945ce2d7f995f4dbb0e3503b08
+  object_id: e561b18d-0d52-4579-7bd5-9823bdbe7aba
+  sha: a6ca3153856b40c1af6d79da8f11efdde0899b7d
+davcli/davcli-0.0.24-linux-amd64:
+  size: 6165337
+  object_id: aa46e12b-6aa9-4de7-5694-955799bea8ad
+  sha: de7307b184f03caff746ae2ad78bf94ba04c9c71
+gnatsd/gnatsd-0.9.6-bosh.12-linux-amd64:
+  size: 9505349
+  object_id: 54d32ad0-9d4a-4b5d-7431-7f504bd9fb9a
+  sha: a801c9fa84f29f9885d54739a3c284c7f58d56bb
 mysql/mariadb-connector-c-2.2.1-src.tar.gz:
   size: 519628
   object_id: b7f2ab15-c9c3-4441-bb7b-f41dbf43f668
@@ -18,30 +22,6 @@ mysql/server-5.7.10-3-Linux.x86_64.ssl100.tar.gz:
   size: 4639216
   object_id: 4d2023eb-9e8e-415d-9247-fc5481270eb1
   sha: f00c86bfa13b64e54f2a08551c3fe61d8966fd6e
-nats/vendor/cache/daemons-1.1.9.gem:
-  size: 32768
-  object_id: 688133be-f542-4a6a-b139-cd1290cedcd7
-  sha: e73ba58fe9e948f7edc5017f91f53b8f0a955a37
-nats/vendor/cache/eventmachine-1.0.4.gem:
-  size: 227840
-  object_id: b6a8625f-3e04-41d1-ae69-57841921e254
-  sha: 10fffa2d326e5fedc6aeaf65d2bd01d3c5e6f3ab
-nats/vendor/cache/json_pure-1.8.1.gem:
-  size: 148992
-  object_id: ce934969-81b7-4d2b-a50c-48a9ee54d50b
-  sha: 7b8f08852f734ef287468fe1676c1c4e7a9f678a
-nats/vendor/cache/nats-0.5.0.beta.12.gem:
-  size: 30208
-  object_id: 714da93b-b55f-4adf-a5e5-b11ea991cd1f
-  sha: f39b79844741d0019cd6e277ded2b2025b69bca3
-nats/vendor/cache/rack-1.5.4.gem:
-  size: 216064
-  object_id: 61e5a9ed-2a1a-41d4-5fa8-6ec6a966ff61
-  sha: d71ea9c90d7ef2a0787722f233da8fcbfb5e55d5
-nats/vendor/cache/thin-1.5.1.gem:
-  size: 55296
-  object_id: 5b19908f-9a4c-4b03-a377-29d7b9d00a9b
-  sha: 4280f5a3db7c4d6470c15f6acec9a0dded057ab8
 nginx/headers-more-nginx-module-0.30.tar.gz:
   size: 27793
   object_id: ee5b4828-67f9-42f7-a0fb-23b4d4c9805f
@@ -70,10 +50,10 @@ powerdns/pdns-static_3.3.1-1_amd64.deb:
   size: 9353694
   object_id: 6e861aa6-2e83-4027-a1e8-0ebec995c274
   sha: 22bb46aa0b3e7671e0ee9aa30ed95c38841106ab
-s3cli/s3cli-0.0.55-linux-amd64:
-  size: 10330042
-  object_id: 7131652f-fdc7-4a84-4437-c22c908e0148
-  sha: 2b345d8476970bc9668af6d602e1c3fa65cff89a
+s3cli/s3cli-0.0.64-linux-amd64:
+  size: 10341472
+  object_id: 90012a1c-100d-4d73-56e8-039b88d8ea72
+  sha: 73278ee6dca087f675026a8e4ecbac66b8982b99
 verify-multidigest/verify-multidigest-0.0.29-linux-amd64:
   size: 3277550
   object_id: e5925f0e-d420-4759-4c4e-9b964183efc1

docs/README.md

@@ -9,5 +9,6 @@ The contents of docs include:
 * [Hotfix Workflow](hotfix_workflow.md)
 * [Manual Release of BOSH](manual_bosh_release_workflow.md)
 * [Running Tests](running_tests.md)
+* [NATS Server](nats.md)
 * [Director Database Schema Diagram](director_schema/README.md)
 * [Code style](code_style.md)

docs/docker-compose.yml

@@ -19,3 +19,5 @@ services:
         environment:
             - MYSQL_ROOT_PASSWORD=password
             - MYSQL_DB=bosh
+        volumes:
+            - /tmp/mysql:/var/lib/mysql

docs/nats.md

@@ -0,0 +1,11 @@
+# NATS 
+
+BOSH uses a custom version of GNATSD (Go edition of NATS) to support TLS & NON-TLS on the same port.
+
+The fork lives here : [https://github.com/cloudfoundry/gnatsd](https://github.com/cloudfoundry/gnatsd)
+
+### How to update GNATSD version: 
+
+- Get the latest binary from the [gnatsd
+  pipeline](https://main.bosh-ci.cf-app.com/teams/main/pipelines/gnatsd)
+and update the blob

docs/running_tests.md

@@ -49,7 +49,7 @@ You can also use a [Concourse CI](https://concourse.ci/) instance with the rake
 bosh/src$ CONCOURSE_TARGET=bosh CONCOURSE_TAG= bundle exec rake fly:integration_gocli
 ```
 
-You can run individual tests by invoking `rspec` directly after setting up the sandbox with `rake spec:integration:install_dependencies` as described in the [workstation setup docs](workstation_setup.md).
+You can run individual tests by invoking `rspec` directly after setting up the sandbox with `rake spec:integration:install_dependencies` and `rake  spec:integration:download_bosh_agent`. More information about the integration test set up can  be found in the [workstation setup docs](workstation_setup.md).
 
 ```
 bosh/src$ bundle exec rspec spec/gocli/integration/cli_env_spec.rb
@@ -91,7 +91,7 @@ Sometimes type of infrastructure does not make a difference for changes made. Fo
 
 ### Build stemcell
 
-The stemcell building process is described in [bosh-stemcell's README](../bosh-stemcell/README.md). One thing to note is that rake tasks were initially created to run tests on BOSH CI. For development purposes there should be some modifications:
+The stemcell building process is described in [bosh-stemcell's README](https://github.com/cloudfoundry/bosh-linux-stemcell-builder). One thing to note is that rake tasks were initially created to run tests on BOSH CI. For development purposes there should be some modifications:
 
 * DO NOT set `CANDIDATE_BUILD_NUMBER` when building stemcell. This will allow you to build stemcell of version `0000` which is understood by rake tasks as a local stemcell.
 * Generated stemcells of version `0000` should be put into the `bosh/tmp` directory before running BATs.

docs/workstation_setup.md

@@ -25,6 +25,9 @@
     `brew install postgresql`
 
 5. Setup and Start postgresql (required for running integration tests with postgresql (default))
+
+    The server does not use any password, the authentication method should be set to `trust` for all authentication methods in `pg_hba.conf`.
+
     * start postgres
 
         ```
@@ -35,16 +38,12 @@
     * create postgres user: $USER/\<no-password\>
 
         `createuser -U $USER --superuser postgres`
+
     * create postgres database
 
         `createdb -U $USER`
 
-6. Get Golang 1.3.3: As homebrew has a golang version >1.3.3 as current version, we need to install the `homebrew versions` command to check the correct git revision of golang 1.3.3
-    * `brew tap homebrew/boneyard`
-    * `brew versions go` and get the revision for version 1.3.3
-    * `cd /usr/local/Library/Formula/`
-    * `git checkout <revision> go.rb`
-    * `brew install go`
+6. Get Golang dependencies
 
     Install vet and golint
     * `go get code.google.com/p/go.tools/cmd/vet`
@@ -104,12 +103,12 @@ To use a custom go-cli in integration tests change `gobosh` in  `src/spec/gocli/
 
 ### Cleaning the sandbox cache manually
 
-Preparing the sandbox for integration tests caches dependencies like nginx. 
+Preparing the sandbox for integration tests caches dependencies like nginx.
 To force a recompilation either delete the complete `src/tmp` folder or just the 'work' folder:
 
 ```
 bosh/src$ rm -fr tmp/integration-nginx-work/
-```  
+```
 
 ### Running integration test databases in docker