.circleci/config.yml

@@ -61,8 +61,6 @@ workflows:
           test_packages: "./client/fingerprint"
           goarch: "386"
           <<: *IGNORE_FOR_UI_BRANCHES
-      - test-rkt:
-          <<: *IGNORE_FOR_UI_BRANCHES
       - test-e2e:
           <<: *IGNORE_FOR_UI_BRANCHES
       - test-ui
@@ -80,7 +78,7 @@ executors:
   go:
     working_directory: /go/src/github.com/hashicorp/nomad
     docker:
-      - image: circleci/golang:1.12.10
+      - image: golang:1.12.12
   go-machine:
     working_directory: ~/go/src/github.com/hashicorp/nomad
     machine:
@@ -111,8 +109,8 @@ jobs:
       GOPATH: /go
     steps:
       - checkout
+      - run: apt-get update; apt-get install -y shellcheck sudo unzip
       - install-protoc
-      - run: sudo apt-get update && sudo apt-get install shellcheck
       - run: make deps lint-deps
       - run: make check
       - run: make checkscripts
@@ -137,6 +135,7 @@ jobs:
       GOTESTARCH: "<< parameters.goarch >>"
     steps:
       - checkout
+      - run: apt-get update; apt-get install -y shellcheck sudo unzip
       - run: make deps
       - install-protoc
       - install-consul
@@ -154,8 +153,20 @@ jobs:
       GOPATH: /go
     steps:
       - checkout
-      - run: make deps
-      - run: make e2e-test
+      - run: apt-get update; apt-get install -y sudo unzip
+      # e2e tests require privileged mount/umount permissions when running as root
+      # TODO: switch to using machine executor and run as root to test e2e path
+      - run:
+          name: prepare non-root user
+          command: |
+            groupadd --gid 3434 circleci
+            useradd --uid 3434 --gid circleci --shell /bin/bash --create-home circleci
+            echo 'circleci ALL=NOPASSWD: ALL' >> /etc/sudoers.d/50-circleci
+            echo 'Defaults    env_keep += "DEBIAN_FRONTEND"' >> /etc/sudoers.d/env_keep
+            chown -R circleci:circleci /go
+
+      - run: sudo -E -H -u circleci PATH=${PATH} make deps
+      - run: sudo -E -H -u circleci PATH=${PATH} make e2e-test
 
   test-website:
     executor: go-machine-recent
@@ -165,32 +176,6 @@ jobs:
       - checkout
       - run: make test-website
 
-  test-rkt:
-    executor: go-machine-recent
-    environment:
-      <<: *COMMON_ENVS
-      GOTEST_PKGS: "./drivers/rkt"
-      GOPATH: /home/circleci/go
-      RKT_VERSION: 1.29.0
-    steps:
-      - checkout
-      - install-golang
-      - install-protoc
-      - run:
-          name: install rkt
-          command: |
-            gpg --recv-key 18AD5014C99EF7E3BA5F6CE950BDD3E0FC8A365E
-            wget https://github.com/rkt/rkt/releases/download/v$RKT_VERSION/rkt_$RKT_VERSION-1_amd64.deb
-            wget https://github.com/rkt/rkt/releases/download/v$RKT_VERSION/rkt_$RKT_VERSION-1_amd64.deb.asc
-            gpg --verify rkt_$RKT_VERSION-1_amd64.deb.asc
-            sudo dpkg -i rkt_$RKT_VERSION-1_amd64.deb
-      - run: PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make bootstrap
-      - run-tests
-      - store_test_results:
-          path: /tmp/test-reports
-      - store_artifacts:
-          path: /tmp/test-reports
-
   test-machine:
     executor: "<< parameters.executor >>"
     parameters:
@@ -299,7 +284,7 @@ commands:
     parameters:
       version:
         type: string
-        default: "1.12.10"
+        default: "1.12.12"
     steps:
       - run:
           name: install golang << parameters.version >>
@@ -313,7 +298,7 @@ commands:
     parameters:
       version:
         type: string
-        default: 1.0.0
+        default: 1.2.3
     steps:
       - run:
           name: Install Vault << parameters.version >>
@@ -326,7 +311,7 @@ commands:
     parameters:
       version:
         type: string
-        default: 1.6.0-rc1
+        default: 1.6.1
     steps:
       - run:
           name: Install Consul << parameters.version >>

.github/ISSUE_TEMPLATE.md

@@ -1,3 +1,5 @@
+For reporting security vulnerabilities [please refer to the website.](https://www.nomadproject.io/security.html)
+
 If you have a question, prepend your issue with `[question]` or preferably use the [nomad mailing list](https://www.nomadproject.io/community.html).
 
 If filing a bug please include the following:

.github/SECURITY.md

@@ -0,0 +1,3 @@
+# Security Policy
+
+Please see https://www.nomadproject.io/security.html

.gitignore

@@ -86,6 +86,8 @@ rkt-*
 /ui/libpeerconnection.log
 /ui/npm-debug.log*
 /ui/testem.log
+/ui/.env*
+/ui/.pnp*
 .ignore
 
 # ember-try

CHANGELOG.md

@@ -1,4 +1,46 @@
-## 0.10.0 (Unreleased)
+## 0.10.2 (Unreleased)
+
+FEATURES:
+ * core: Add `nomad monitor` command to stream logs at a specified level for debugging [[GH-6499](https://github.com/hashicorp/nomad/issues/6499)]
+
+IMPROVEMENTS:
+ * api: Add `StartedAt` field to `Node.DrainStrategy` [[GH-6698](https://github.com/hashicorp/nomad/issues/6698)]
+ * core: Add support for running under Windows Service Manager [[GH-6220](https://github.com/hashicorp/nomad/issues/6220)]
+ * cli: Show full ID in node and alloc individual status views [[GH-6425](https://github.com/hashicorp/nomad/issues/6425)]
+ * client: Enable setting tags on Consul Connect sidecar service [[GH-6448](https://github.com/hashicorp/nomad/issues/6448)]
+ * client: Add support for downloading artifacts from Google Cloud Storage [[GH-6692](https://github.com/hashicorp/nomad/pull/6692)]
+
+BUG FIXES:
+
+ * core: Ignore `server` config values if `server` is disabled [[GH-6047](https://github.com/hashicorp/nomad/issues/6047)]
+ * core: Added `semver` constraint for strict Semver 2.0 version comparisons [[GH-6699](https://github.com/hashicorp/nomad/issues/6699)]
+ * api: Return a 404 if endpoint not found instead of redirecting to /ui/ [[GH-6658](https://github.com/hashicorp/nomad/issues/6658)]
+ * api: Decompress web socket response body if gzipped on error responses [[GH-6650](https://github.com/hashicorp/nomad/issues/6650)]
+ * api: Fixed a bug where some FS/Allocation API endpoints didn't return error messages [[GH-6427](https://github.com/hashicorp/nomad/issues/6427)]
+ * api: Return 40X status code for failing ACL requests, rather than 500 [[GH-6421](https://github.com/hashicorp/nomad/issues/6421)]
+ * cli: Make scoring column orders consistent `nomad alloc status` [[GH-6609](https://github.com/hashicorp/nomad/issues/6609)]
+ * cli: Fixed a bug where a cli user may fail to query FS/Allocation API endpoints if they lack `node:read` capability [[GH-6423](https://github.com/hashicorp/nomad/issues/6423)]
+ * client: Fixed a bug where a client may not restart dead internal processes upon client's restart on Windows [[GH-6426](https://github.com/hashicorp/nomad/issues/6426)]
+ * driver/docker: Added mechanism for detecting running unexpectedly running docker containers [[GH-6325](https://github.com/hashicorp/nomad/issues/6325)]
+ * nomad: Multiple connect enabled services in the same taskgroup failed to
+   register [[GH-6646](https://github.com/hashicorp/nomad/issues/6646)]
+ * scheduler: Changes to devices in resource stanza should cause rescheduling [[GH-6644](https://github.com/hashicorp/nomad/issues/6644)]
+ * vault: Allow overriding implicit Vault version constraint [[GH-6687](https://github.com/hashicorp/nomad/issues/6687)]
+ * vault: Supported Vault auth role's new field, `token_period` [[GH-6574](https://github.com/hashicorp/nomad/issues/6574)]
+ * scheduler: Fixed a bug that allowed inplace updates after a constraint, affinity, or spread was changed [[GH-6703](https://github.com/hashicorp/nomad/issues/6703)]
+
+## 0.10.1 (November 4, 2019)
+
+BUG FIXES:
+
+ * core: Fixed server panic when upgrading from 0.8 -> 0.10 and performing an
+   inplace update of an allocation. [[GH-6541](https://github.com/hashicorp/nomad/issues/6541)]
+ * api: Fixed panic when submitting Connect-enabled job without using a bridge
+   network [[GH-6575](https://github.com/hashicorp/nomad/issues/6575)]
+ * client: Fixed client panic when upgrading from 0.8 -> 0.10 and performing an
+   inplace update of an allocation. [[GH-6605](https://github.com/hashicorp/nomad/issues/6605)]
+
+## 0.10.0 (October 22, 2019)
 
 FEATURES:
  * **Consul Connect**: Nomad may now register Consul Connect services and
@@ -27,16 +69,41 @@ IMPROVEMENTS:
 
 BUG FIXES:
 
- * core: Fixed a bug where scheduler may schedule an allocation on a node without required drivers [[GH-6227](https://github.com/hashicorp/nomad/issues/6227)]
  * cli: Fixed `nomad run ...` on Windows so it works with unprivileged accounts [[GH-6009](https://github.com/hashicorp/nomad/issues/6009)]
  * client: Fixed a bug in client fingerprinting on 32-bit nodes [[GH-6239](https://github.com/hashicorp/nomad/issues/6239)]
  * client: Fixed a bug where completed allocations may re-run after client restart [[GH-6216](https://github.com/hashicorp/nomad/issues/6216)]
  * client: Fixed failure to start if another client is already running with the same data directory [[GH-6348](https://github.com/hashicorp/nomad/pull/6348)]
  * devices: Fixed a bug causing CPU usage spike when a device is detected [[GH-6201](https://github.com/hashicorp/nomad/issues/6201)]
+ * drivers: Allowd user-defined environment variable keys to contain dashes [[GH-6080](https://github.com/hashicorp/nomad/issues/6080)]
+ * driver/docker: Set gc image_delay default to 3 minutes [[GH-6078](https://github.com/hashicorp/nomad/pull/6078)]
+ * driver/docker: Improved docker driver handling of container creation or starting failures [[GH-6326](https://github.com/hashicorp/nomad/issues/6326)], [[GH-6346](https://github.com/hashicorp/nomad/issues/6346)]
+ * ui: Fixed a bug where the allocation log viewer would render HTML or hide content that matched XML syntax [[GH-6048](https://github.com/hashicorp/nomad/issues/6048)]
+ * ui: Fixed a bug where allocation log viewer doesn't show all content in Firefox [[GH-6466](https://github.com/hashicorp/nomad/issues/6466)]
+ * ui: Fixed navigation via clicking recent allocation row [[GH-6087](https://github.com/hashicorp/nomad/pull/6087)]
+ * ui: Fixed a bug where the allocation log viewer would render HTML or hide content that matched XML syntax [[GH-6048](https://github.com/hashicorp/nomad/issues/6048)]
+ * ui: Fixed a bug where allocation log viewer doesn't show all content in Firefox [[GH-6466](https://github.com/hashicorp/nomad/issues/6466)]
+
+## 0.9.6 (October 7, 2019)
+
+SECURITY:
+
+ * core: Redacted replication token in agent/self API endpoint.  The replication token is a management token that can be used for further privilege escalation. CVE-2019-12741 [[GH-6430](https://github.com/hashicorp/nomad/issues/6430)]
+ * core: Fixed a bug where a user may start raw_exec task on clients despite driver being disabled. CVE-2019-15928 [[GH-6227](https://github.com/hashicorp/nomad/issues/6227)] [[GH-6431](https://github.com/hashicorp/nomad/issues/6431)]
+ * enterprise/acl: Fix ACL access checks in Nomad Enterprise where users may query allocation information and perform lifecycle actions in namespaces they are not authorized to. CVE-2019-16742 [[GH-6432](https://github.com/hashicorp/nomad/issues/6432)]
+
+IMPROVEMENTS:
+
+ * client: Reduced memory footprint of nomad logging and executor processes [[GH-6341](https://github.com/hashicorp/nomad/issues/6341)]
+
+BUG FIXES:
+
+ * core: Fixed a bug where scheduler may schedule an allocation on a node without required drivers [[GH-6227](https://github.com/hashicorp/nomad/issues/6227)]
+ * client: Fixed a bug where completed allocations may re-run after client restart [[GH-6216](https://github.com/hashicorp/nomad/issues/6216)] [[GH-6207](https://github.com/hashicorp/nomad/issues/6207)]
+ * client: Fixed a panic that may occur when an `nomad alloc exec` is initiated while process is terminating [[GH-6065](https://github.com/hashicorp/nomad/issues/6065)]
+ * devices: Fixed a bug causing CPU usage spike when a device is detected [[GH-6201](https://github.com/hashicorp/nomad/issues/6201)]
  * drivers: Fixed port mapping for docker and qemu drivers [[GH-6251](https://github.com/hashicorp/nomad/pull/6251)]
  * drivers/docker: Fixed a case where a `nomad alloc exec` would never time out [[GH-6144](https://github.com/hashicorp/nomad/pull/6144)]
- * drivers/docker: Set gc image_delay default to 3 minutes [[GH-6078](https://github.com/hashicorp/nomad/pull/6078)]
- * ui: Fixed navigation via clicking recent allocation row [[GH-6087](https://github.com/hashicorp/nomad/pull/6087)]
+ * ui: Fixed a bug where allocation log viewer doesn't show all content. [[GH-6048](https://github.com/hashicorp/nomad/issues/6048)]
 
 ## 0.9.5 (21 August 2019)
 

GNUmakefile

@@ -6,7 +6,7 @@ GIT_COMMIT := $(shell git rev-parse HEAD)
 GIT_DIRTY := $(if $(shell git status --porcelain),+CHANGES)
 
 GO_LDFLAGS := "-X github.com/hashicorp/nomad/version.GitCommit=$(GIT_COMMIT)$(GIT_DIRTY)"
-GO_TAGS ?=
+GO_TAGS ?= codegen_generated
 
 GO_TEST_CMD = $(if $(shell which gotestsum),gotestsum --,go test)
 
@@ -147,6 +147,7 @@ deps:  ## Install build and development dependencies
 	go get -u github.com/a8m/tree/cmd/tree
 	go get -u github.com/magiconair/vendorfmt/cmd/vendorfmt
 	go get -u gotest.tools/gotestsum
+	go get -u github.com/fatih/hclfmt
 	@bash -C "$(PROJECT_ROOT)/scripts/install-codecgen.sh"
 	@bash -C "$(PROJECT_ROOT)/scripts/install-protoc-gen-go.sh"
 
@@ -195,7 +196,7 @@ check: ## Lint the source code
 	@if (git status | grep -q .pb.go); then echo the following proto files are out of sync; git status |grep .pb.go; exit 1; fi
 
 	@echo "==> Check API package is isolated from rest"
-	@! go list -f '{{ join .Deps "\n" }}' ./api | grep github.com/hashicorp/nomad/ | grep -v -e /vendor/ -e /nomad/api/
+	@! go list --test -f '{{ join .Deps "\n" }}' ./api | grep github.com/hashicorp/nomad/ | grep -v -e /vendor/ -e /nomad/api/ -e nomad/api.test
 
 .PHONY: checkscripts
 checkscripts: ## Lint shell scripts
@@ -224,21 +225,30 @@ generate-examples: command/job_init.bindata_assetfs.go
 command/job_init.bindata_assetfs.go: command/assets/*
 	go-bindata-assetfs -pkg command -o command/job_init.bindata_assetfs.go ./command/assets/...
 
+.PHONY: vendorfmt
 vendorfmt:
 	@echo "--> Formatting vendor/vendor.json"
 	test -x $(GOPATH)/bin/vendorfmt || go get -u github.com/magiconair/vendorfmt/cmd/vendorfmt
 		vendorfmt
+
+.PHONY: changelogfmt
 changelogfmt:
 	@echo "--> Making [GH-xxxx] references clickable..."
 	@sed -E 's|([^\[])\[GH-([0-9]+)\]|\1[[GH-\2](https://github.com/hashicorp/nomad/issues/\2)]|g' CHANGELOG.md > changelog.tmp && mv changelog.tmp CHANGELOG.md
 
+## We skip the terraform directory as there are templated hcl configurations
+## that do not successfully compile without rendering
+.PHONY: hclfmt
+hclfmt:
+	@echo "--> Formatting HCL"
+	@find . -path ./terraform -prune -o -name 'upstart.nomad' -prune -o \( -name '*.nomad' -o -name '*.hcl' \) -exec hclfmt -w {} +
 
 .PHONY: dev
 dev: GOOS=$(shell go env GOOS)
 dev: GOARCH=$(shell go env GOARCH)
 dev: GOPATH=$(shell go env GOPATH)
 dev: DEV_TARGET=pkg/$(GOOS)_$(GOARCH)/nomad
-dev: vendorfmt changelogfmt ## Build for the current development platform
+dev: vendorfmt changelogfmt hclfmt ## Build for the current development platform
 	@echo "==> Removing old development build..."
 	@rm -f $(PROJECT_ROOT)/$(DEV_TARGET)
 	@rm -f $(PROJECT_ROOT)/bin/nomad
@@ -252,11 +262,11 @@ dev: vendorfmt changelogfmt ## Build for the current development platform
 	@cp $(PROJECT_ROOT)/$(DEV_TARGET) $(GOPATH)/bin
 
 .PHONY: prerelease
-prerelease: GO_TAGS=ui release
+prerelease: GO_TAGS=ui codegen_generated release
 prerelease: generate-all ember-dist static-assets ## Generate all the static assets for a Nomad release
 
 .PHONY: release
-release: GO_TAGS=ui release
+release: GO_TAGS=ui codegen_generated release
 release: clean $(foreach t,$(ALL_TARGETS),pkg/$(t).zip) ## Build all release packages which can be built on this platform.
 	@echo "==> Results:"
 	@tree --dirsfirst $(PROJECT_ROOT)/pkg
@@ -283,6 +293,7 @@ test-nomad: dev ## Run Nomad test suites
 		$(if $(ENABLE_RACE),-race) $(if $(VERBOSE),-v) \
 		-cover \
 		-timeout=15m \
+		-tags "$(GO_TAGS)" \
 		$(GOTEST_PKGS) $(if $(VERBOSE), >test.log ; echo $$? > exit-code)
 	@if [ $(VERBOSE) ] ; then \
 		bash -C "$(PROJECT_ROOT)/scripts/test_check.sh" ; \
@@ -295,6 +306,7 @@ e2e-test: dev ## Run the Nomad e2e test suite
 		$(if $(ENABLE_RACE),-race) $(if $(VERBOSE),-v) \
 		-cover \
 		-timeout=900s \
+		-tags "$(GO_TAGS)" \
 		github.com/hashicorp/nomad/e2e/vault/ \
 		-integration