Release Flux v1.12.3

.circleci/config.yml

@@ -1,20 +1,41 @@
 version: 2
 jobs:
   build:
-    working_directory: /go/src/github.com/weaveworks/flux
-    docker:
-      - image: circleci/golang:1.10
-      - image: memcached
+    working_directory: ~/go/src/github.com/weaveworks/flux
+    machine: true
+    environment:
+      GO_VERSION: 1.12.5
+      GOPATH: /home/circleci/go
+      PATH: /bin:/usr/bin:/usr/local/go/bin:/home/circleci/go/bin
     steps:
       - checkout
-      - setup_remote_docker
-
-      - run: curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
+      - run:
+          name: Install Golang
+          command: |
+            curl -OL https://storage.googleapis.com/golang/go${GO_VERSION}.linux-amd64.tar.gz
+            tar -xf go${GO_VERSION}.linux-amd64.tar.gz
+            sudo rm -rf /usr/local/go
+            sudo mv go /usr/local
+            mkdir -p "$HOME/go/bin"
+            go version
+      - run:
+          name: Update packages and Start Memcached
+          command: |
+            # These repos fail and we don't need them:
+            sudo rm /etc/apt/sources.list.d/circleci_trusty.list /etc/apt/sources.list.d/google-chrome.list
+            sudo apt-get update
+            sudo apt-get install -y git rng-tools docker-ce memcached
+            git version
+            docker version
+      - run: 
+          name: Install Dep
+          command: curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
       - run: dep ensure -vendor-only
       - run: make check-generated
       - run: make test TEST_FLAGS="-race -tags integration -timeout 60s"
       - run: make all
-
+      - run: make e2e
+
       - deploy:
           name: Maybe push prerelease images
           command: |
@@ -27,7 +48,6 @@ jobs:
               docker tag "docker.io/weaveworks/helm-operator:$(docker/image-tag)" "docker.io/weaveworks/helm-operator-prerelease:$(docker/image-tag)"
               docker push "docker.io/weaveworks/helm-operator-prerelease:$(docker/image-tag)"
             fi
-
       - deploy:
           name: Maybe push release image and upload binaries
           command: |
@@ -43,17 +63,6 @@ jobs:
               RELEASE_TAG=$(echo "$CIRCLE_TAG" | cut -c 6-)
               docker push "docker.io/weaveworks/helm-operator:${RELEASE_TAG}"
             fi
-  e2e-testing:
-    machine: true
-    working_directory: ~/go/src/github.com/weaveworks/flux
-    steps:
-      - checkout
-      - run: test/e2e/e2e-golang.sh
-      - run: test/e2e/e2e-flux-build.sh
-      - run: test/e2e/e2e-kind.sh
-      - run: test/e2e/e2e-helm.sh
-      - run: test/e2e/e2e-git.sh
-      - run: test/e2e/e2e-flux-chart.sh
 
 workflows:
   version: 2
@@ -63,6 +72,4 @@ workflows:
           filters:
             tags:
               only: /(helm-)?[0-9]+(\.[0-9]+)*(-[a-z]+)?/
-      - e2e-testing:
-          requires:
-            - build
+

.gitignore

@@ -40,3 +40,4 @@ docker/fluxy-dumbconf.priv
 test/profiles
 test/bin/kubectl
 test/bin/helm
+test/bin/kind

CHANGELOG.md

@@ -1,6 +1,46 @@
 This is the changelog for the Flux daemon; the changelog for the Helm
 operator is in [./CHANGELOG-helmop.md](./CHANGELOG-helmop.md).
 
+## 1.12.3 (2019-05-22)
+
+This is a patch release.
+
+### Fixes
+
+- Show tag image for workload in list-images
+  [weaveworks/flux#2024][]
+- Log warning when not applying resource by namespace
+  [weaveworks/flux#2034][]
+- Always list the status of a workload in `fluxctl`
+  [weaveworks/flux#2035][]
+- Ensure Flux installs gnutls >=3.6.7, to resolve security scan issues
+  [weaveworks/flux#2001][]
+- Rename controller to workload in `fluxctl release`
+  [weaveworks/flux#2048][]
+- Give full output of git command on errors
+  [weaveworks/flux#2054][]
+
+### Maintenance and documentation
+
+- Warn about Flux only supporting YAML and not JSON
+  [weaveworks/flux#2010][]
+- Fix and refactor end-to-end tests
+  [weaveworks/flux#2050][] [weaveworks/flux#2058][]
+
+### Thanks
+
+Thanks to @2opremio, @hiddeco, @squaremo and @xtellurian for contributions.
+
+[weaveworks/flux#2010]: https://github.com/weaveworks/flux/pull/2010
+[weaveworks/flux#2024]: https://github.com/weaveworks/flux/pull/2024
+[weaveworks/flux#2034]: https://github.com/weaveworks/flux/pull/2034
+[weaveworks/flux#2035]: https://github.com/weaveworks/flux/pull/2035
+[weaveworks/flux#2044]: https://github.com/weaveworks/flux/pull/2044
+[weaveworks/flux#2048]: https://github.com/weaveworks/flux/pull/2048
+[weaveworks/flux#2050]: https://github.com/weaveworks/flux/pull/2050
+[weaveworks/flux#2054]: https://github.com/weaveworks/flux/pull/2054
+[weaveworks/flux#2058]: https://github.com/weaveworks/flux/pull/2058
+
 ## 1.12.2 (2019-05-08)
 
 This is a patch release.

Makefile

@@ -47,14 +47,17 @@ release-bins:
 clean:
 	go clean
 	rm -rf ./build
-	rm -f test/bin/kubectl test/bin/helm
+	rm -f test/bin/kubectl test/bin/helm test/bin/kind
 
 realclean: clean
 	rm -rf ./cache
 
 test: test/bin/helm test/bin/kubectl
 	PATH="${PWD}/bin:${PWD}/test/bin:${PATH}" go test ${TEST_FLAGS} $(shell go list ./... | grep -v "^github.com/weaveworks/flux/vendor" | sort -u)
 
+e2e: test/bin/helm test/bin/kubectl build/.flux.done build/.helm-operator.done
+	PATH="${PWD}/test/bin:${PATH}" CURRENT_OS_ARCH=$(CURRENT_OS_ARCH) test/e2e/run.sh
+
 build/.%.done: docker/Dockerfile.%
 	mkdir -p ./build/docker/$*
 	cp $^ ./build/docker/$*/

cluster/kubernetes/sync.go

@@ -62,12 +62,14 @@ func (c *Cluster) Sync(syncSet cluster.SyncSet) error {
 
 	cs := makeChangeSet()
 	var errs cluster.SyncError
+	var excluded []string
 	for _, res := range syncSet.Resources {
 		resID := res.ResourceID()
+		id := resID.String()
 		if !c.IsAllowedResource(resID) {
+			excluded = append(excluded, id)
 			continue
 		}
-		id := resID.String()
 		// make a record of the checksum, whether we stage it to
 		// be applied or not, so that we don't delete it later.
 		csum := sha1.Sum(res.Bytes())
@@ -93,6 +95,10 @@ func (c *Cluster) Sync(syncSet cluster.SyncSet) error {
 		}
 	}
 
+	if len(excluded) > 0 {
+		logger.Log("warning", "not applying resources; excluded by namespace constraints", "resources", strings.Join(excluded, ","))
+	}
+
 	c.mu.Lock()
 	defer c.mu.Unlock()
 	c.muSyncErrors.RLock()

cmd/fluxctl/list_images_cmd.go

@@ -136,6 +136,14 @@ func (opts *imageListOpts) RunE(cmd *cobra.Command, args []string) error {
 					fmt.Fprintf(out, "\t\t%s %s\t%s\n", running, tag, createdAt)
 				}
 			}
+			if !foundRunning {
+				running := "'->"
+				if currentTag == "" {
+					currentTag = "(untagged)"
+				}
+				fmt.Fprintf(out, "\t\t%s %s\t%s\n", running, currentTag, "?")
+
+			}
 			workloadName = ""
 		}
 	}

cmd/fluxctl/list_workloads_cmd.go

@@ -63,7 +63,7 @@ func (opts *workloadListOpts) RunE(cmd *cobra.Command, args []string) error {
 				fmt.Fprintf(w, "\t%s\t%s\t\t\n", c.Name, c.Current.ID)
 			}
 		} else {
-			fmt.Fprintf(w, "%s\t\t\t\t\n", workload.ID)
+			fmt.Fprintf(w, "%s\t\t\t%s\t%s\n", workload.ID, workload.Status, policies(workload))
 		}
 	}
 	w.Flush()

cmd/fluxctl/release_cmd.go

@@ -42,7 +42,7 @@ func newWorkloadRelease(parent *rootOpts) *workloadReleaseOpts {
 func (opts *workloadReleaseOpts) Command() *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "release",
-		Short: "Release a new version of a controller.",
+		Short: "Release a new version of a workload.",
 		Example: makeExample(
 			"fluxctl release -n default --workload=deployment/foo --update-image=library/hello:v2",
 			"fluxctl release --all --update-image=library/hello:v2",
@@ -56,10 +56,10 @@ func (opts *workloadReleaseOpts) Command() *cobra.Command {
 	cmd.Flags().StringVarP(&opts.namespace, "namespace", "n", "default", "Workload namespace")
 	// Note: we cannot define a shorthand for --workload since it clashes with the shorthand of --watch
 	cmd.Flags().StringSliceVarP(&opts.workloads, "workload", "", []string{}, "List of workloads to release <namespace>:<kind>/<name>")
-	cmd.Flags().BoolVar(&opts.allWorkloads, "all", false, "Release all controllers")
+	cmd.Flags().BoolVar(&opts.allWorkloads, "all", false, "Release all workloads")
 	cmd.Flags().StringVarP(&opts.image, "update-image", "i", "", "Update a specific image")
 	cmd.Flags().BoolVar(&opts.allImages, "update-all-images", false, "Update all images to latest versions")
-	cmd.Flags().StringSliceVar(&opts.exclude, "exclude", []string{}, "List of controllers to exclude")
+	cmd.Flags().StringSliceVar(&opts.exclude, "exclude", []string{}, "List of workloads to exclude")
 	cmd.Flags().BoolVar(&opts.dryRun, "dry-run", false, "Do not release anything; just report back what would have been done")
 	cmd.Flags().BoolVar(&opts.interactive, "interactive", false, "Select interactively which containers to update")
 	cmd.Flags().BoolVarP(&opts.force, "force", "f", false, "Disregard locks and container image filters (has no effect when used with --all or --update-all-images)")

deploy/flux-deployment.yaml

@@ -54,7 +54,7 @@ spec:
         # There are no ":latest" images for flux. Find the most recent
         # release or image version at https://hub.docker.com/r/weaveworks/flux/tags
         # and replace the tag here.
-        image: docker.io/weaveworks/flux:1.12.2
+        image: docker.io/weaveworks/flux:1.12.3
         imagePullPolicy: IfNotPresent
         resources:
           requests:

docker/Dockerfile.flux

@@ -2,7 +2,7 @@ FROM alpine:3.9
 
 WORKDIR /home/flux
 
-RUN apk add --no-cache openssh ca-certificates tini 'git>=2.3.0' gnupg
+RUN apk add --no-cache openssh ca-certificates tini 'git>=2.12.0' 'gnutls>=3.6.7' gnupg
 
 # Add git hosts to known hosts file so we can use
 # StrickHostKeyChecking with git+ssh

docker/Dockerfile.helm-operator

@@ -2,7 +2,7 @@ FROM alpine:3.9
 
 WORKDIR /home/flux
 
-RUN apk add --no-cache openssh ca-certificates tini 'git>=2.3.0'
+RUN apk add --no-cache openssh ca-certificates tini 'git>=2.12.0'
 
 # Add git hosts to known hosts file so we can use
 # StrickHostKeyChecking with git+ssh

git/operations.go

@@ -3,15 +3,14 @@ package git
 import (
 	"bufio"
 	"bytes"
+	"context"
 	"encoding/json"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"os/exec"
 	"strings"
-
-	"context"
+	"sync"
 
 	"github.com/pkg/errors"
 )
@@ -290,7 +289,7 @@ func changed(ctx context.Context, workingDir, ref string, subPaths []string) ([]
 }
 
 // traceGitCommand returns a log line that can be useful when debugging and developing git activity
-func traceGitCommand(args []string, config gitCmdConfig, stdout string, stderr string) string {
+func traceGitCommand(args []string, config gitCmdConfig, stdOutAndStdErr string) string {
 	for _, exemptedCommand := range exemptedTraceCommands {
 		if exemptedCommand == args[0] {
 			return ""
@@ -305,19 +304,34 @@ func traceGitCommand(args []string, config gitCmdConfig, stdout string, stderr s
 	}
 
 	command := `git ` + strings.Join(args, " ")
-	out := prepare(stdout)
-	err := prepare(stderr)
+	out := prepare(stdOutAndStdErr)
 
 	return fmt.Sprintf(
-		"TRACE: command=%q out=%q err=%q dir=%q env=%q",
+		"TRACE: command=%q out=%q dir=%q env=%q",
 		command,
 		out,
-		err,
 		config.dir,
 		strings.Join(config.env, ","),
 	)
 }
 
+type threadSafeBuffer struct {
+	bytes.Buffer
+	sync.Mutex
+}
+
+func (b *threadSafeBuffer) Write(p []byte) (n int, err error) {
+	b.Lock()
+	defer b.Unlock()
+	return b.Buffer.Write(p)
+}
+
+func (b *threadSafeBuffer) ReadFrom(r io.Reader) (n int64, err error) {
+	b.Lock()
+	defer b.Unlock()
+	return b.Buffer.ReadFrom(r)
+}
+
 // execGitCmd runs a `git` command with the supplied arguments.
 func execGitCmd(ctx context.Context, args []string, config gitCmdConfig) error {
 	c := exec.CommandContext(ctx, "git", args...)
@@ -326,30 +340,26 @@ func execGitCmd(ctx context.Context, args []string, config gitCmdConfig) error {
 		c.Dir = config.dir
 	}
 	c.Env = append(env(), config.env...)
-	c.Stdout = ioutil.Discard
+	stdOutAndStdErr := &threadSafeBuffer{}
+	c.Stdout = stdOutAndStdErr
+	c.Stderr = stdOutAndStdErr
 	if config.out != nil {
-		c.Stdout = config.out
-	}
-	errOut := &bytes.Buffer{}
-	c.Stderr = errOut
-
-	traceStdout := &bytes.Buffer{}
-	traceStderr := &bytes.Buffer{}
-	if trace {
-		c.Stdout = io.MultiWriter(c.Stdout, traceStdout)
-		c.Stderr = io.MultiWriter(c.Stderr, traceStderr)
+		c.Stdout = io.MultiWriter(c.Stdout, config.out)
 	}
 
 	err := c.Run()
 	if err != nil {
-		msg := findErrorMessage(errOut)
-		if msg != "" {
-			err = errors.New(msg)
+		if len(stdOutAndStdErr.Bytes()) > 0 {
+			err = errors.New(stdOutAndStdErr.String())
+			msg := findErrorMessage(stdOutAndStdErr)
+			if msg != "" {
+				err = fmt.Errorf("%s, full output:\n %s", msg, err.Error())
+			}
 		}
 	}
 
 	if trace {
-		if traceCommand := traceGitCommand(args, config, traceStdout.String(), traceStderr.String()); traceCommand != "" {
+		if traceCommand := traceGitCommand(args, config, stdOutAndStdErr.String()); traceCommand != "" {
 			println(traceCommand)
 		}
 	}