feat(controller): Basic support for multi-cluster/multi-namespace workflows. Closes #3523

.github/workflows/ci-build.yaml

@@ -43,7 +43,7 @@ jobs:
       fail-fast: false
       matrix:
         # kubelet is not included because it'd take ages to get it working methinks
-        test: [ "smoke", "test-e2e", "test-cli", "test-e2e-cron" ]
+        test: [ "smoke", "test-e2e", "test-cli", "test-e2e-mc", "test-e2e-cron" ]
         containerRuntimeExecutor: [ "docker", "k8sapi", "pns" ]
         # ok, so we're only running `smoke` for all CREs,
         exclude:
@@ -55,6 +55,11 @@ jobs:
             containerRuntimeExecutor: k8sapi
           - test: test-cli
             containerRuntimeExecutor: pns
+          # counter-intuitively, this suite will work on pns, won't work on docker
+          - test: test-e2e-mc
+            containerRuntimeExecutor: docker
+          - test: test-e2e-mc
+            containerRuntimeExecutor: k8sapi
           - test: test-e2e-cron
             containerRuntimeExecutor: k8sapi
           - test: test-e2e-cron
@@ -76,46 +81,51 @@ jobs:
           echo /home/runner/go/bin >> $GITHUB_PATH
           echo /usr/local/bin >> $GITHUB_PATH
       - name: Install and start K3S v1.18.8+k3s1
-        run: curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=v1.18.8+k3s1 INSTALL_K3S_CHANNEL=stable INSTALL_K3S_EXEC=--docker K3S_KUBECONFIG_MODE=644 sh - &
-      - name: Pre-pull images
-        env:
-          GOPATH: /home/runner/go
-        run: make pull-build-images test-images &
-      - name: Create Kubeconfig
         run: |
-          mkdir -p ~/.kube
+          curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=v1.18.8+k3s1 INSTALL_K3S_CHANNEL=stable INSTALL_K3S_EXEC=--docker K3S_KUBECONFIG_MODE=644 sh - &
           until stat /etc/rancher/k3s/k3s.yaml ; do sleep 10s ; done
+          mkdir -p ~/.kube
           cat /etc/rancher/k3s/k3s.yaml | sed "s/127.0.0.1/$(hostname)/g" > ~/.kube/config
-          echo "- name: fake_token_user" >> ~/.kube/config
-          echo "  user:" >> ~/.kube/config
-          echo "    token: xxxxxx" >> ~/.kube/config
-      - name: Start Argo
+          kubectl cluster-info
+      - name: Install K3D
+        run: curl -s https://raw.githubusercontent.com/rancher/k3d/main/install.sh | bash
+      - name: Fetch tags
+        run: git fetch --tags
+      - name: Download Go modules
+        run: go mod download
+      - name: Pre-pull images
         env:
           GOPATH: /home/runner/go
-          PROFILE: mysql
+        run: make pull-build-images test-images &
+      - name: Add hosts
         run: |
           echo '127.0.0.1 dex'      | sudo tee -a /etc/hosts
           echo '127.0.0.1 minio'    | sudo tee -a /etc/hosts
           echo '127.0.0.1 postgres' | sudo tee -a /etc/hosts
           echo '127.0.0.1 mysql'    | sudo tee -a /etc/hosts
+      - name: Start Argo
+        env:
+          KUBECONFIG: /home/runner/.kube/config
+          GOPATH: /home/runner/go
+          PROFILE: mysql
+        run: |
           mkdir -p /tmp/log/argo-e2e
-          git fetch --tags
-          KUBECONFIG=~/.kube/config make start PROFILE=$PROFILE E2E_EXECUTOR=${{matrix.containerRuntimeExecutor}} DEV_IMAGE=true STATIC_FILES=false 2>&1 > /tmp/log/argo-e2e/argo.log &
-      - name: Install gotestsum
-        run: go install gotest.tools/gotestsum
+          touch /tmp/log/argo-e2e/argo.log
+          make pre-start PROFILE=$PROFILE E2E_EXECUTOR=${{matrix.containerRuntimeExecutor}} DEV_IMAGE=true STATIC_FILES=false
+          make start     PROFILE=$PROFILE E2E_EXECUTOR=${{matrix.containerRuntimeExecutor}} DEV_IMAGE=true STATIC_FILES=false 2>&1 > /tmp/log/argo-e2e/argo.log &
       - name: Wait for Argo Server to be ready
         env:
           GOPATH: /home/runner/go
         run: make wait
       - name: Run tests
         env:
           GOPATH: /home/runner/go
-        run: make ${{ matrix.test }} GOTEST='gotestsum --format testname --'
+        run: make ${{ matrix.test }}
       - name: Upload logs
         if: ${{ always() }}
         uses: actions/upload-artifact@v1
         with:
-          name: ${{ matrix.test }}-${{ github.run_id }}-argo.log
+          name: ${{ matrix.test }}-${{matrix.containerRuntimeExecutor}}-${{ github.run_id }}-argo.log
           path: /tmp/log/argo-e2e/argo.log
 
   codegen:

.gitignore

@@ -1,5 +1,6 @@
 .vscode/
 .idea/
+.run/
 .DS_Store
 vendor/
 dist/
@@ -18,8 +19,8 @@ git-ask-pass.sh
 /workflow-controller
 /.scannerwork/
 /test-results/
-/argo-server.crt
-/argo-server.key
+/*.crt
+/*.key
 /package-lock.json
 /pkg/apiclient/_.secondary.swagger.json
 /pkg/apiclient/clusterworkflowtemplate/cluster-workflow-template.swagger.json

.golangci.yml

@@ -9,6 +9,7 @@ run:
   build-tags:
     - e2e
     - cli
+    - e2emc
 linters:
   enable:
     - goimports

Dockerfile.dev

@@ -73,7 +73,7 @@ COPY --from=builder /usr/local/bin/docker /usr/local/bin/
 # argoexec
 ####################################################################################################
 FROM argoexec-base as argoexec
-COPY argoexec /usr/local/bin/
+COPY argoexec-linux-amd64 /usr/local/bin/argoexec
 ENTRYPOINT [ "argoexec" ]
 
 ####################################################################################################
@@ -83,7 +83,7 @@ FROM scratch as workflow-controller
 USER 8737
 # Add timezone data
 COPY --from=builder /usr/share/zoneinfo /usr/share/zoneinfo
-COPY workflow-controller /bin/
+COPY workflow-controller-linux-amd64 /bin/workflow-controller
 ENTRYPOINT [ "workflow-controller" ]
 
 ####################################################################################################
@@ -95,5 +95,14 @@ COPY --from=argoexec-base /etc/ssh/ssh_known_hosts /etc/ssh/ssh_known_hosts
 COPY --from=argoexec-base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
 COPY argo-server.crt argo-server.crt
 COPY argo-server.key argo-server.key
-COPY argo /bin/
+COPY argo-linux-amd64 /bin/argo
 ENTRYPOINT [ "argo" ]
+
+FROM scratch as agent
+USER 8737
+COPY --from=argoexec-base /etc/ssh/ssh_known_hosts /etc/ssh/ssh_known_hosts
+COPY --from=argoexec-base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
+COPY agent.crt agent.crt
+COPY agent.key agent.key
+COPY agent-linux-amd64 /bin/agent
+ENTRYPOINT [ "agent" ]

Makefile

@@ -44,11 +44,11 @@ endif
 CLI_IMAGE_FILE         := dist/cli-image.marker
 EXECUTOR_IMAGE_FILE    := dist/executor-image.marker
 CONTROLLER_IMAGE_FILE  := dist/controller-image.marker
+AGENT_IMAGE_FILE       := dist/agent-image.marker
 
 # perform static compilation
 STATIC_BUILD          ?= true
 STATIC_FILES          ?= true
-GOTEST                ?= go test
 PROFILE               ?= minimal
 # whether or not to start the Argo Service in TLS mode
 SECURE                := false
@@ -63,8 +63,9 @@ endif
 # * `local` run the workflow–controller and argo-server as single replicas on the local machine (default)
 # * `kubernetes` run the workflow-controller and argo-server on the Kubernetes cluster
 RUN_MODE              := local
-K3D                   := $(shell if [[ "`which kubectl`" != '' ]] && [[ "`kubectl config current-context`" == "k3d-"* ]]; then echo true; else echo false; fi)
-LOG_LEVEL             := debug
+CI                    := false
+K3D                   := $(shell [ "`which kubectl`" != '' ] && [ "`kubectl config current-context`" = "k3d-*" ] && echo true || echo false)
+LOG_LEVEL             := info
 UPPERIO_DB_DEBUG      := 0
 NAMESPACED            := true
 
@@ -91,6 +92,7 @@ override LDFLAGS += -X github.com/argoproj/argo.gitTag=${GIT_TAG}
 endif
 
 ARGOEXEC_PKGS    := $(shell echo cmd/argoexec            && go list -f '{{ join .Deps "\n" }}' ./cmd/argoexec/            | grep 'argoproj/argo' | cut -c 26-)
+AGENT_PKGS       := $(shell echo cmd/agent               && go list -f '{{ join .Deps "\n" }}' ./cmd/agent/               | grep 'argoproj/argo' | cut -c 26-)
 CLI_PKGS         := $(shell echo cmd/argo                && go list -f '{{ join .Deps "\n" }}' ./cmd/argo/                | grep 'argoproj/argo' | cut -c 26-)
 CONTROLLER_PKGS  := $(shell echo cmd/workflow-controller && go list -f '{{ join .Deps "\n" }}' ./cmd/workflow-controller/ | grep 'argoproj/argo' | cut -c 26-)
 MANIFESTS        := $(shell find manifests -mindepth 2 -type f)
@@ -134,15 +136,15 @@ endef
 # docker_build,image_name,binary_name,marker_file_name
 define docker_build
 	# If we're making a dev build, we build this locally (this will be faster due to existing Go build caches).
-	if [ $(DEV_IMAGE) = true ]; then $(MAKE) dist/$(2)-$(OUTPUT_IMAGE_OS)-$(OUTPUT_IMAGE_ARCH) && mv dist/$(2)-$(OUTPUT_IMAGE_OS)-$(OUTPUT_IMAGE_ARCH) $(2); fi
+	if [ $(DEV_IMAGE) = true ]; then $(MAKE) dist/$(2)-$(OUTPUT_IMAGE_OS)-$(OUTPUT_IMAGE_ARCH) && mv dist/$(2)-$(OUTPUT_IMAGE_OS)-$(OUTPUT_IMAGE_ARCH) .; fi
 	docker build --progress plain -t $(IMAGE_NAMESPACE)/$(1):$(VERSION) --target $(1) -f $(DOCKERFILE) --build-arg IMAGE_OS=$(OUTPUT_IMAGE_OS) --build-arg IMAGE_ARCH=$(OUTPUT_IMAGE_ARCH) .
-	if [ $(DEV_IMAGE) = true ]; then mv $(2) dist/$(2)-$(OUTPUT_IMAGE_OS)-$(OUTPUT_IMAGE_ARCH); fi
-	if [ $(K3D) = true ]; then k3d image import $(IMAGE_NAMESPACE)/$(1):$(VERSION); fi
+	if [ $(DEV_IMAGE) = true ]; then mv $(2)-$(OUTPUT_IMAGE_OS)-$(OUTPUT_IMAGE_ARCH) dist/; fi
+	if [ $(K3D) = true ] ; then k3d image import $(IMAGE_NAMESPACE)/$(1):$(VERSION); fi
 	touch $(3)
 endef
 define docker_pull
 	docker pull $(1)
-	if [ $(K3D) = true ]; then k3d image import $(1); fi
+	if [ $(K3D) = true ] ; then k3d image import $(1); fi
 endef
 
 ifndef $(GOPATH)
@@ -159,7 +161,7 @@ images: cli-image executor-image controller-image
 # cli
 
 .PHONY: cli
-cli: dist/argo argo-server.crt argo-server.key
+cli: dist/argo argo-server.key
 
 ui/dist/app/index.html: $(shell find ui/src -type f && find ui -maxdepth 1 -type f)
 	# Build UI
@@ -196,10 +198,8 @@ dist/argo-%.gz: dist/argo-%
 dist/argo-%: server/static/files.go $(CLI_PKGS)
 	CGO_ENABLED=0 $(GOARGS) go build -v -i -ldflags '${LDFLAGS}' -o $@ ./cmd/argo
 
-argo-server.crt: argo-server.key
-
-argo-server.key:
-	openssl req -x509 -newkey rsa:4096 -keyout argo-server.key -out argo-server.crt -days 365 -nodes -subj /CN=localhost/O=ArgoProj
+%.key:
+	openssl req -x509 -newkey rsa:4096 -keyout $*.key -out $*.crt -days 365 -nodes -subj /CN=localhost/O=ArgoProj
 
 .PHONY: cli-image
 cli-image: $(CLI_IMAGE_FILE)
@@ -231,6 +231,22 @@ controller-image: $(CONTROLLER_IMAGE_FILE)
 $(CONTROLLER_IMAGE_FILE): $(CONTROLLER_PKGS)
 	$(call docker_build,workflow-controller,workflow-controller,$(CONTROLLER_IMAGE_FILE))
 
+# agent
+
+.PHONY: agent-image
+agent-image: $(AGENT_IMAGE_FILE)
+
+$(AGENT_IMAGE_FILE): $(AGENT_PKGS) agent.key
+	$(call docker_build,agent,agent,$(AGENT_IMAGE_FILE))
+
+dist/agent-linux-amd64: GOARGS = GOOS=linux GOARCH=amd64
+dist/agent-linux-arm64: GOARGS = GOOS=linux GOARCH=arm64
+dist/agent-linux-ppc64le: GOARGS = GOOS=linux GOARCH=ppc64le
+dist/agent-linux-s390x: GOARGS = GOOS=linux GOARCH=s390x
+
+dist/agent-%: $(AGENT_PKGS)
+	CGO_ENABLED=0 $(GOARGS) go build -v -i -ldflags '${LDFLAGS}' -o $@ ./cmd/agent
+
 # argoexec
 
 dist/argoexec-linux-amd64: GOARGS = GOOS=linux GOARCH=amd64
@@ -382,18 +398,31 @@ endif
 # for local we have a faster target that prints to stdout, does not use json, and can cache because it has no coverage
 .PHONY: test
 test: server/static/files.go
-	env KUBECONFIG=/dev/null $(GOTEST) ./...
+	env KUBECONFIG=/dev/null go test ./...
 
-dist/$(PROFILE).yaml: $(MANIFESTS) $(E2E_MANIFESTS) /usr/local/bin/kustomize
-	mkdir -p dist
-	kustomize build --load_restrictor=none test/e2e/manifests/$(PROFILE) | sed 's/:latest/:$(VERSION)/' | sed 's/pns/$(E2E_EXECUTOR)/'  > dist/$(PROFILE).yaml
+dist/main-context:
+	kubectl config current-context > dist/main-context
 
 .PHONY: install
-install: dist/$(PROFILE).yaml
-	cat test/e2e/manifests/argo-ns.yaml | sed 's/argo/$(KUBE_NAMESPACE)/' > dist/argo-ns.yaml
-	kubectl apply -f dist/argo-ns.yaml
-	kubectl -n $(KUBE_NAMESPACE) apply -l app.kubernetes.io/part-of=argo --prune --force -f dist/$(PROFILE).yaml
+install: /usr/local/bin/kustomize dist/argo dist/main-context agent-image
+	# create other cluster (if not exists)
+	k3d cluster get other || k3d cluster create other -p "2468:80@loadbalancer" --wait --update-default-kubeconfig
+	# configure other cluster
+	kubectl config use-context k3d-other
+	kubectl create ns $(KUBE_NAMESPACE) --dry-run=client -o yaml | kubectl apply -f -
+	kustomize build --load_restrictor=none test/e2e/manifests/other-cluster | kubectl -n $(KUBE_NAMESPACE) apply --force -f-
+	k3d image import argoproj/agent:latest -c other
+	kubectl -n $(KUBE_NAMESPACE) wait --for=condition=Ready pod --all -l app --timeout 2m
+	# configure main cluster
+	kubectl config use-context `cat dist/main-context`
+	kubectl create ns $(KUBE_NAMESPACE) --dry-run=client -o yaml | kubectl apply -f -
+	kustomize build --load_restrictor=none test/e2e/manifests/$(PROFILE) | sed 's/:latest/:$(VERSION)/' | sed 's/pns/$(E2E_EXECUTOR)/' | kubectl -n $(KUBE_NAMESPACE) apply -l app.kubernetes.io/part-of=argo --prune --force -f -
+	kubectl -n $(KUBE_NAMESPACE) create secret generic clusters --dry-run=client -o yaml | kubectl apply -f -
+	./dist/argo -n $(KUBE_NAMESPACE) cluster add other k3d-other -v
+	KUBECONFIG=$(HOME)/.kube/config:test/e2e/testdata/agent.kubeconfig ./dist/argo -n $(KUBE_NAMESPACE) cluster add agent agent -v
+
 
+KUBECONFIG=~/.kube/config:test/e2e/testdata/agent.kubeconfig ./dist/argo -n argo cluster add agent agent -v
 .PHONY: pull-build-images
 pull-build-images:
 	./hack/pull-build-images.sh
@@ -422,11 +451,17 @@ stop:
 $(GOPATH)/bin/goreman:
 	go get github.com/mattn/goreman
 
+.PHONY: pre-start
+ifeq ($(RUN_MODE),kubernetes)
+pre-start: stop install agent-image executor-image controller-image cli-image
+else
+pre-start: stop install agent-image executor-image controller cli $(GOPATH)/bin/goreman
+endif
+
 .PHONY: start
-start: stop install controller cli executor-image $(GOPATH)/bin/goreman
+start: pre-start
 	kubectl config set-context --current --namespace=$(KUBE_NAMESPACE)
 ifeq ($(RUN_MODE),kubernetes)
-	$(MAKE) controller-image cli-image
 	kubectl -n $(KUBE_NAMESPACE) scale deploy/workflow-controller --replicas 1
 	kubectl -n $(KUBE_NAMESPACE) scale deploy/argo-server --replicas 1
 endif
@@ -460,26 +495,32 @@ mysql-cli:
 
 .PHONY: test-e2e
 test-e2e:
-	$(GOTEST) -timeout 10m -count 1 --tags e2e -p 1 --short ./test/e2e
+	# set-up bad user for e2e tests
+	kubectl config set-credentials fake_token_user --token=xxxxxx
+	go test -timeout 10m -count 1 --tags e2e -p 1 --short ./test/e2e
 
 .PHONY: test-cli
 test-cli:
-	$(GOTEST) -timeout 15m -count 1 --tags cli -p 1 --short ./test/e2e
+	go test -timeout 15m -count 1 --tags cli -p 1 --short ./test/e2e
+
+.PHONY: test-e2e-mc
+test-e2e-mc:
+	go test -timeout 15m -count 1 --tags e2emc -p 1 --short ./test/e2e
 
 .PHONY: test-e2e-cron
 test-e2e-cron:
-	$(GOTEST) -count 1 --tags e2e -parallel 10 -run CronSuite ./test/e2e
+	go test -count 1 --tags e2e -parallel 10 -run CronSuite ./test/e2e
 
 .PHONY: smoke
 smoke:
-	$(GOTEST) -count 1 --tags e2e -p 1 -run SmokeSuite ./test/e2e
+	go test -count 1 --tags e2e -p 1 -run SmokeSuite ./test/e2e
 
 # clean
 
 .PHONY: clean
 clean:
 	go clean
-	rm -Rf test-results node_modules vendor dist/* ui/dist
+	rm -Rf test-results node_modules vendor dist/* ui/dist go-diagrams
 
 # swagger