e2e: Add image release test

Makefile

@@ -62,7 +62,7 @@ realclean: clean
 test: test/bin/helm test/bin/kubectl test/bin/sops test/bin/kustomize $(GENERATED_TEMPLATES_FILE)
 	PATH="${PWD}/bin:${PWD}/test/bin:${PATH}" go test ${TEST_FLAGS} $(shell go list ./... | sort -u)
 
-e2e: lint-e2e test/bin/helm test/bin/kubectl test/bin/sops test/e2e/bats $(GOBIN)/fluxctl build/.flux.done
+e2e: lint-e2e test/bin/helm test/bin/kubectl test/bin/sops test/bin/crane test/e2e/bats $(GOBIN)/fluxctl build/.flux.done
 	PATH="${PWD}/test/bin:${PATH}" CURRENT_OS_ARCH=$(CURRENT_OS_ARCH) test/e2e/run.bash
 
 E2E_BATS_FILES := test/e2e/*.bats
@@ -152,6 +152,10 @@ cache/bats-core-$(BATS_COMMIT).tar.gz:
 	# Use 2opremio's fork until https://github.com/bats-core/bats-core/pull/255 is merged
 	curl --fail -L -o $@ https://github.com/2opremio/bats-core/archive/$(BATS_COMMIT).tar.gz
 
+test/bin/crane:
+	mkdir -p $(@D)
+	go build -o $@ github.com/google/go-containerregistry/cmd/crane
+
 $(GOBIN)/fluxctl: $(FLUXCTL_DEPS) $(GENERATED_TEMPLATES_FILE)
 	go install ./cmd/fluxctl
 

go.mod

@@ -5,6 +5,9 @@ go 1.13
 // remove when https://github.com/docker/distribution/pull/2905 is released.
 replace github.com/docker/distribution => github.com/2opremio/distribution v0.0.0-20190419185413-6c9727e5e5de
 
+// remove when https://github.com/google/go-containerregistry/pull/661 is merged.
+replace github.com/google/go-containerregistry => github.com/2opremio/go-containerregistry v0.0.0-20200120142434-5d8cba6ae83d
+
 // fix go-autorest ambiguous import caused by sops
 // sops needs to update their deps ref: https://github.com/kubernetes/client-go/issues/628
 replace github.com/Azure/go-autorest => github.com/Azure/go-autorest v12.2.0+incompatible
@@ -21,7 +24,7 @@ require (
 	github.com/Masterminds/semver/v3 v3.0.3
 	github.com/Masterminds/sprig v2.22.0+incompatible // indirect
 	github.com/VividCortex/gohistogram v1.0.0 // indirect
-	github.com/aws/aws-sdk-go v1.27.0
+	github.com/aws/aws-sdk-go v1.27.1
 	github.com/bradfitz/gomemcache v0.0.0-20190329173943-551aad21a668
 	github.com/cheggaaa/pb/v3 v3.0.2
 	github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect
@@ -33,8 +36,9 @@ require (
 	github.com/gogo/googleapis v1.3.1 // indirect
 	github.com/gogo/status v1.1.0 // indirect
 	github.com/golang/gddo v0.0.0-20190312205958-5a2505f3dbf0
+	github.com/google/go-containerregistry v0.0.0-20200115214256-379933c9c22b // indirect
 	github.com/google/go-github/v28 v28.1.1
-	github.com/gorilla/mux v1.7.1
+	github.com/gorilla/mux v1.7.3
 	github.com/gorilla/websocket v1.4.0
 	github.com/imdario/mergo v0.3.8
 	github.com/instrumenta/kubeval v0.0.0-20190804145309-805845b47dfc
@@ -62,9 +66,9 @@ require (
 	golang.org/x/sys v0.0.0-20191028164358-195ce5e7f934
 	golang.org/x/time v0.0.0-20191024005414-555d28b269f0
 	gopkg.in/yaml.v2 v2.2.4
-	k8s.io/api v0.0.0-20191016110408-35e52d86657a // kubernetes-1.16.2
+	k8s.io/api v0.17.0 // kubernetes-1.16.2
 	k8s.io/apiextensions-apiserver v0.0.0-20191016113550-5357c4baaf65 // kubernetes-1.16.2
-	k8s.io/apimachinery v0.0.0-20191004115801-a2eda9f80ab8 // kubernetes-1.16.2
+	k8s.io/apimachinery v0.17.0 // kubernetes-1.16.2
 	k8s.io/client-go v11.0.0+incompatible
 	k8s.io/helm v2.16.1+incompatible
 	k8s.io/klog v1.0.0

test/e2e/13_sync_gc.bats

@@ -29,10 +29,8 @@ function setup() {
   defer rm -rf "'$clone_dir'"
   git clone -b master ssh://git@localhost/git-server/repos/cluster.git "$clone_dir"
   cd "$clone_dir"
-  local sync_tag_hash
-  sync_tag_hash=$(git rev-list -n 1 flux)
   head_hash=$(git rev-list -n 1 HEAD)
-  [ "$sync_tag_hash" = "$head_hash" ]
+  poll_until_equals "sync tag" "$head_hash" 'git pull -f --tags > /dev/null 2>&1; git rev-list -n 1 flux'
 
   # Remove a manifest and commit that
   git rm workloads/podinfo-dep.yaml

test/e2e/14_release_image.bats

@@ -0,0 +1,86 @@
+#!/usr/bin/env bats
+
+function setup() {
+  load lib/env
+  load lib/install
+  load lib/poll
+  load lib/defer
+  load lib/registry
+
+  kubectl create namespace "$FLUX_NAMESPACE"
+
+  # Install the git server, allowing external access
+  install_git_srv git_srv_result
+  # shellcheck disable=SC2154
+  export GIT_SSH_COMMAND="${git_srv_result[0]}"
+  # Teardown the created port-forward to gitsrv.
+  defer kill "${git_srv_result[1]}"
+
+  # Install a local registry, with some empty images to be used later in the test
+  install_registry registry_result
+  # shellcheck disable=SC2154
+  REGISTRY_PORT="${registry_result[0]}"
+  # Teardown the created port-forward to the registry.
+  defer kill "${registry_result[1]}"
+  echo "REGISTRY_PORT=$REGISTRY_PORT" >&3
+  # create empty images for the test
+  push_empty_image "localhost:$REGISTRY_PORT" 'bitnami/ghost:3.0.2-debian-9-r3' '2020-01-20T13:53:05.47178071Z'
+  push_empty_image "localhost:$REGISTRY_PORT" 'bitnami/ghost:3.1.1-debian-9-r0' '2020-02-20T13:53:05.47178071Z'
+  push_empty_image "localhost:$REGISTRY_PORT" 'stefanprodan/podinfo:3.1.0' '2020-03-20T13:53:05.47178071Z'
+  push_empty_image "localhost:$REGISTRY_PORT" 'stefanprodan/podinfo:3.0.5' '2020-04-20T13:53:05.47178071Z'
+  REGISTRY_SERVICE_IP=$(kubectl -n "$FLUX_NAMESPACE" get service registry -o 'jsonpath={.spec.clusterIP}')
+
+  # Finally, install Flux
+  local -A template_values
+  # shellcheck disable=SC2034
+  template_values['REGISTRY_SERVICE_IP']="$REGISTRY_SERVICE_IP"
+  install_flux_with_fluxctl '14_release_image' 'template_values'
+}
+
+@test "Image releases" {
+  # Wait until flux deploys the workloads, which indicates it has at least started a sync
+  poll_until_true 'workload podinfo' 'kubectl -n demo describe deployment/podinfo'
+
+  # make sure we have _finished_ a sync run
+  fluxctl --k8s-fwd-ns "${FLUX_NAMESPACE}" sync
+
+  # Clone the repo and check the sync tag
+  local clone_dir
+  clone_dir="$(mktemp -d)"
+  defer rm -rf "'$clone_dir'"
+  git clone -b master ssh://git@localhost/git-server/repos/cluster.git "$clone_dir"
+  cd "$clone_dir"
+  head_hash=$(git rev-list -n 1 HEAD)
+  poll_until_equals "sync tag" "$head_hash" 'git pull -f --tags > /dev/null 2>&1; git rev-list -n 1 flux'
+
+  # Wait for the registry scanner to fo its magic on stefanprodan/podinfo and bitnami/ghost
+  poll_until_true "stefanprodan/podinfo to be scanned" "kubectl logs -n $FLUX_NAMESPACE deploy/flux | grep -q \"component=warmer updated=stefanprodan/podinfo\"" 5 50
+  poll_until_true "bitnami/ghost to be scanned" "kubectl logs -n $FLUX_NAMESPACE deploy/flux | grep -q \"component=warmer updated=bitnami/ghost\"" 5 50
+
+  # Manually release podinfo to version 3.0.5
+  fluxctl --k8s-fwd-ns "${FLUX_NAMESPACE}" deautomate --workload=demo:deployment/podinfo
+  fluxctl --k8s-fwd-ns "${FLUX_NAMESPACE}" release --force --workload=demo:deployment/podinfo --update-image=stefanprodan/podinfo:3.0.5
+  poll_until_true "deployment/podinfo version 3.0.5 to be released" 'git pull > /dev/null 2>&1; grep -q stefanprodan/podinfo:3.0.5 workloads/podinfo-dep.yaml'
+
+  # Manually release ghost to version 3.0.2-debian-9-r3
+  fluxctl --k8s-fwd-ns "${FLUX_NAMESPACE}" release --force --workload=demo:helmrelease/ghost --update-image=bitnami/ghost:3.0.2-debian-9-r3
+  poll_until_true "helmrelease/ghost version 3.0.2-debian-9-r3 to be released" 'git pull > /dev/null 2>&1; grep -q 3.0.2-debian-9-r3 releases/ghost.yaml'
+
+  # Automate the resources and make sure that podinfo and ghost are updated according to their annotations
+  # (semver:~3.1 for podinfo and glob:3.1.1-debian-9-* for ghost)
+  fluxctl --k8s-fwd-ns "${FLUX_NAMESPACE}" automate --workload=demo:deployment/podinfo
+  fluxctl --k8s-fwd-ns "${FLUX_NAMESPACE}" automate --workload=demo:helmrelease/ghost
+  poll_until_true "deployment/podinfo semver:~3.1 to be released" 'git pull > /dev/null 2>&1; grep -q stefanprodan/podinfo:3.1. workloads/podinfo-dep.yaml'
+  poll_until_true "helmrelease/ghost glob:3.1.1-debian-9-* to be released" 'git pull > /dev/null 2>&1; grep -q 3.1.1-debian-9-r0 releases/ghost.yaml'
+}
+
+function teardown() {
+  run_deferred
+  # Although the namespace delete below takes care of removing most Flux
+  # elements, the global resources will not be removed without this.
+  uninstall_flux_with_fluxctl
+  # Removing the namespace also takes care of removing Flux and gitsrv.
+  kubectl delete namespace "$FLUX_NAMESPACE"
+  # Only remove the demo workloads after Flux, so that they cannot be recreated.
+  kubectl delete namespace "$DEMO_NAMESPACE"
+}

test/e2e/fixtures/crane_empty_img_tmpl/config.json

@@ -0,0 +1,68 @@
+{
+  "architecture": "amd64",
+  "config": {
+    "Hostname": "",
+    "Domainname": "",
+    "User": "",
+    "AttachStdin": false,
+    "AttachStdout": false,
+    "AttachStderr": false,
+    "Tty": false,
+    "OpenStdin": false,
+    "StdinOnce": false,
+    "Env": [
+      "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
+      "foo="
+    ],
+    "Cmd": null,
+    "Image": "",
+    "Volumes": null,
+    "WorkingDir": "",
+    "Entrypoint": null,
+    "OnBuild": null,
+    "Labels": null
+  },
+  "container": "622070b2ab6961b448c41a0ca3f694e7301ceaef4676aafc0f5302500a17300d",
+  "container_config": {
+    "Hostname": "622070b2ab69",
+    "Domainname": "",
+    "User": "",
+    "AttachStdin": false,
+    "AttachStdout": false,
+    "AttachStderr": false,
+    "Tty": false,
+    "OpenStdin": false,
+    "StdinOnce": false,
+    "Env": [
+      "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+    ],
+    "Cmd": [
+      "/bin/sh",
+      "-c",
+      "#(nop) ",
+      "ENV foo="
+    ],
+    "Image": "",
+    "Volumes": null,
+    "WorkingDir": "",
+    "Entrypoint": null,
+    "OnBuild": null,
+    "Labels": {
+      "org.opencontainers.image.created": "${CREATION_TIME}",
+      "org.label-schema.build-date": "${CREATION_TIME}"
+    }
+  },
+  "created": "${CREATION_TIME}",
+  "docker_version": "19.03.5",
+  "history": [
+    {
+      "created": "${CREATION_TIME}",
+      "created_by": "/bin/sh -c #(nop)  ENV foo=",
+      "empty_layer": true
+    }
+  ],
+  "os": "linux",
+  "rootfs": {
+    "type": "layers"
+  }
+}

test/e2e/fixtures/crane_empty_img_tmpl/manifest.json

@@ -0,0 +1,5 @@
+[
+  {
+    "Config": "config.json"
+  }
+]

test/e2e/fixtures/kustom/13_sync_gc/kustomization.yaml

@@ -1,9 +1,6 @@
 bases:
 - "../base/flux"
 patchesJson6902:
-## this patch is for test-specific patches; supply a filename to
-## install_flux_with_fluxctl and it will use that rather than the
-## (empty) default.
 - target:
     group: apps
     version: v1

test/e2e/fixtures/kustom/14_release_image/kustomization.yaml

@@ -0,0 +1,9 @@
+bases:
+- "../base/flux"
+patchesJson6902:
+- target:
+    group: apps
+    version: v1
+    kind: Deployment
+    name: flux
+  path: release_image_patch.yaml

test/e2e/fixtures/kustom/14_release_image/release_image_patch.yaml

@@ -0,0 +1,16 @@
+- op: add
+  path: /spec/template/spec/containers/0/args/-
+  value: --registry-scanning=true
+- op: add
+  path: /spec/template/spec/containers/0/args/-
+  value: --registry-exclude-image=*bitnami/mongodb,*bitnami/redis,*k8s.gcr.io*,*docker/kube-*,*fluxcd/flux,*alpine,*memcached,*stefanprodan/gitsrv,*registry
+# replace docker's registry by our local one
+- op: add
+  path: /spec/template/spec/containers/0/args/-
+  value: --registry-insecure-host=index.docker.io
+- op: add
+  path: /spec/template/spec/hostAliases
+  value:
+    - ip: "${REGISTRY_SERVICE_IP}"
+      hostnames:
+      - "index.docker.io"

test/e2e/fixtures/kustom/base/flux/e2e_patch.yaml

@@ -10,7 +10,7 @@
   value: --sync-interval=10s
 - op: add
   path: /spec/template/spec/containers/0/args/-
-  value: --registry-exclude-image=*
+  value: --registry-scanning=false
 - op: add
   path: /spec/template/spec/containers/0/volumeMounts/-
   value:

test/e2e/fixtures/kustom/base/gitsrv/gitsrv.yaml

@@ -22,7 +22,7 @@ spec:
         - name: REPO
           value: "cluster.git"
         - name: TAR_URL
-          value: "https://github.com/fluxcd/flux-get-started/archive/master.tar.gz"
+          value: "https://github.com/fluxcd/flux-get-started/archive/a4bdf4bb92c35cf9e944788368510168ee0bedf4.tar.gz"
         ports:
         - containerPort: 22
           name: ssh

test/e2e/fixtures/kustom/base/registry/kustomization.yaml

@@ -0,0 +1,2 @@
+resources:
+- registry.yaml

test/e2e/fixtures/kustom/base/registry/registry.yaml

@@ -0,0 +1,32 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: registry
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      name: registry
+  template:
+    metadata:
+      labels:
+        name: registry
+    spec:
+      containers:
+      - image: registry:2
+        name: registry
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: registry
+spec:
+  ports:
+    - name: http
+      port: 80
+      protocol: TCP
+      targetPort: 5000
+  selector:
+    name: registry
+  type: ClusterIP