Expand on the recognised fluxd flags

README.md

@@ -38,7 +38,8 @@ Start by setting up a minikube instance to run the tests on:
 
 Run the tests:
 ```
-make integration-tests
+make
+make integration-tests WEAVE_CLOUD_TOKEN=<YOUR_TEST_INSTANCE_ON_FRONTEND.DEV.WEAVE.WORKS>
 ```
 
 This script will first ensure the dependencies are built and then run:

agent/flux.go

@@ -1,36 +1,152 @@
 package main
 
 import (
+	"net/url"
+	"strconv"
 	"strings"
+	"time"
 
-	flags "github.com/jessevdk/go-flags"
+	"github.com/spf13/pflag"
 	"github.com/weaveworks/launcher/pkg/kubectl"
 )
 
 // FluxConfig stores existing flux arguments which will be used when updating WC agents
 type FluxConfig struct {
-	GitLabel  string `long:"git-label"`
-	GitURL    string `long:"git-url"`
-	GitPath   string `long:"git-path"`
-	GitBranch string `long:"git-branch"`
+	// git parameters
+	GitLabel string
+	GitURL   string
+	// This arg is multi-valued, or can be passed as comma-separated
+	// values. This accounts for either form.
+	GitPath         []string
+	GitBranch       string
+	GitTimeout      time.Duration
+	GitPollInterval time.Duration
+	GitSetAuthor    bool
+	GitCISkip       bool
+
+	// sync behaviour
+	GarbageCollection bool
+
+	// For specifying ECR region from outside AWS (fluxd detects it when inside AWS)
+	RegistryECRRegion []string
+	// For requiring a particular registry to be accessible, else crash
+	RegistryRequire []string
+	// Can be used to switch image registry scanning off for some or
+	// all images (with glob patterns)
+	RegistryExcludeImage []string
+
+	// This is now hard-wired to empty in launch-generator, to tell flux
+	// _not_ to use service discovery. But: maybe someone needs to use
+	// service discovery.
+	MemcachedService string
+
+	// Just in case we more explicitly support restricting Weave
+	// Cloud, or just Flux to particular namespaces
+	AllowNamespace []string
+
+	fs *pflag.FlagSet
 }
 
-func getFluxConfig(k kubectl.Client, namespace string) (*FluxConfig, error) {
-	out, err := k.Execute("get", "pod", "-n", namespace, "-l", "name=weave-flux-agent", "-o", "jsonpath='{.items[?(@.metadata.labels.name==\"weave-flux-agent\")].spec.containers[0].args[*]}'")
-	if err != nil {
-		return nil, err
+// AsQueryParams returns the configuration as a fragment of query
+// string, so it can be interpolated into a text template.
+func (c *FluxConfig) AsQueryParams() string {
+	// Nothing clever here.
+	vals := url.Values{}
+
+	if c == nil {
+		return ""
 	}
 
-	cfg := &FluxConfig{}
-	parser := flags.NewParser(cfg, flags.IgnoreUnknown)
-	_, err = parser.ParseArgs(strings.Split(out, " "))
-	if err != nil {
-		return nil, err
+	// String-valued arguments
+	for arg, val := range map[string]string{
+		"git-label":         c.GitLabel,
+		"git-url":           c.GitURL,
+		"git-branch":        c.GitBranch,
+		"memcached-service": c.MemcachedService,
+	} {
+		if c.fs.Changed(arg) {
+			vals.Add(arg, val)
+		}
+	}
+
+	// []string-valued arguments
+	for arg, slice := range map[string][]string{
+		"git-path":               c.GitPath,
+		"registry-ecr-region":    c.RegistryECRRegion,
+		"registry-require":       c.RegistryRequire,
+		"registry-exclude-image": c.RegistryExcludeImage,
+		"k8s-allow-namespace":    c.AllowNamespace,
+	} {
+		for _, val := range slice {
+			vals.Add(arg, val)
+		}
+	}
+
+	// duration-valued arguments
+	for arg, dur := range map[string]time.Duration{
+		"git-timeout":       c.GitTimeout,
+		"git-poll-interval": c.GitPollInterval,
+	} {
+		if c.fs.Changed(arg) {
+			vals.Add(arg, dur.String())
+		}
 	}
 
-	if cfg.GitBranch == "" && cfg.GitLabel == "" && cfg.GitPath == "" && cfg.GitURL == "" {
-		return nil, nil
+	for arg, flag := range map[string]bool{
+		"sync-garbage-collection": c.GarbageCollection,
+		"git-set-author":          c.GitSetAuthor,
+		"git-ci-skip":             c.GitCISkip,
+	} {
+		if c.fs.Changed(arg) {
+			vals.Add(arg, strconv.FormatBool(flag))
+		}
+	}
+
+	return vals.Encode()
+}
+
+// ParseFluxArgs parses a string of args into a nice FluxConfig
+func ParseFluxArgs(argString string) (*FluxConfig, error) {
+	fs := pflag.NewFlagSet("default", pflag.ContinueOnError)
+	fs.ParseErrorsWhitelist.UnknownFlags = true
+	cfg := &FluxConfig{fs: fs}
+
+	// strings
+	fs.StringVar(&cfg.GitLabel, "git-label", "", "")
+	fs.StringVar(&cfg.GitURL, "git-url", "", "")
+	fs.StringVar(&cfg.GitBranch, "git-branch", "", "")
+	fs.StringVar(&cfg.MemcachedService, "memcached-service", "", "")
+
+	// durations
+	fs.DurationVar(&cfg.GitTimeout, "git-timeout", time.Second, "")
+	fs.DurationVar(&cfg.GitPollInterval, "git-poll-interval", time.Minute, "")
+
+	// bools
+	fs.BoolVar(&cfg.GitSetAuthor, "git-set-author", false, "")
+	fs.BoolVar(&cfg.GitCISkip, "git-ci-skip", false, "")
+	fs.BoolVar(&cfg.GarbageCollection, "sync-garbage-collection", false, "")
+
+	// string slices
+	fs.StringSliceVar(&cfg.GitPath, "git-path", nil, "")
+	fs.StringSliceVar(&cfg.RegistryECRRegion, "registry-ecr-region", nil, "")
+	fs.StringSliceVar(&cfg.RegistryRequire, "registry-require", nil, "")
+	fs.StringSliceVar(&cfg.RegistryExcludeImage, "registry-exclude-image", nil, "")
+	fs.StringSliceVar(&cfg.AllowNamespace, "k8s-allow-namespace", nil, "")
+
+	// Parse it all
+	fs.Parse(strings.Split(argString, " "))
+
+	if fs.NFlag() > 0 {
+		return cfg, nil
+	}
+	return nil, nil
+}
+
+func getFluxConfig(k kubectl.Client, namespace string) (*FluxConfig, error) {
+	out, err := k.Execute("get", "pod", "-n", namespace, "-l", "name=weave-flux-agent", "-o", "jsonpath='{.items[?(@.metadata.labels.name==\"weave-flux-agent\")].spec.containers[0].args[*]}'")
+	if err != nil {
+		return nil, err
 	}
 
-	return cfg, nil
+	return ParseFluxArgs(out)
 }

agent/main.go

@@ -39,10 +39,7 @@ const (
 	defaultWCHostname        = "cloud.weave.works"
 	defaultWCPollURL         = "https://{{.WCHostname}}/k8s.yaml" +
 		"?k8s-version={{.KubernetesVersion}}&t={{.Token}}&omit-support-info=true" +
-		"{{if .FluxConfig}}" +
-		"&git-label={{.FluxConfig.GitLabel}}&git-url={{.FluxConfig.GitURL}}" +
-		"&git-path={{.FluxConfig.GitPath}}&git-branch={{.FluxConfig.GitBranch}}" +
-		"{{end}}" +
+		"{{if .FluxConfig.AsQueryParams}}&{{.FluxConfig.AsQueryParams}}{{end}}" +
 		"{{if .CRIEndpoint}}" +
 		"&cri-endpoint={{.CRIEndpoint}}" +
 		"{{end}}" +

agent/main_test.go

@@ -2,8 +2,10 @@ package main
 
 import (
 	"errors"
+	"fmt"
 	"net/url"
 	"reflect"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -37,7 +39,7 @@ func TestGetMinorMajorVersion(t *testing.T) {
 			return
 		}
 		if v != c.version {
-			t.Errorf("Version was wrongl expected: %s got %s", c.version, v)
+			t.Errorf("Version was wrong; expected: %s got %s", c.version, v)
 		}
 	}
 }
@@ -54,3 +56,85 @@ func TestAgentManifestURL(t *testing.T) {
 	}
 	assert.Contains(t, manifestURL, v.Encode())
 }
+
+func TestAgentFluxURL(t *testing.T) {
+	// Not an exhaustive test; just representative
+	gitPath := []string{"config/helloworld", "config/hej-world"}
+
+	argsStr := fmt.Sprintf(`--git-path=%s --memcached-service= --git-ci-skip=false --git-timeout=40s`, strings.Join(gitPath, ","))
+
+	fluxCfg, err := ParseFluxArgs(argsStr)
+	assert.NoError(t, err)
+
+	cfg := &agentConfig{
+		AgentPollURLTemplate: defaultWCPollURL,
+		FluxConfig:           fluxCfg,
+	}
+
+	manifestURL := agentManifestURL(cfg)
+
+	v := url.Values{
+		"git-path":          gitPath,
+		"memcached-service": []string{""},
+		"git-ci-skip":       []string{"false"},
+		"git-timeout":       []string{"40s"},
+	}
+
+	assert.Contains(t, manifestURL, v.Encode())
+}
+
+func TestParseFluxArgs(t *testing.T) {
+	// nothing
+	argString := ""
+	fluxCfg, err := ParseFluxArgs(argString)
+	assert.Equal(t, nil, err)
+	assert.Equal(t, "", fluxCfg.AsQueryParams())
+
+	// Test handling boolean flags w/out `=true|false`
+	argString = "--git-ci-skip"
+	fluxCfg, err = ParseFluxArgs(argString)
+	assert.Equal(t, nil, err)
+	assert.Equal(t, true, fluxCfg.GitCISkip)
+
+	// Test handling boolean flags w `=true|false`
+	argString = "--git-ci-skip=true"
+	fluxCfg, err = ParseFluxArgs(argString)
+	assert.Equal(t, nil, err)
+	assert.Equal(t, true, fluxCfg.GitCISkip)
+
+	argString = "--git-ci-skip=false"
+	fluxCfg, err = ParseFluxArgs(argString)
+	assert.Equal(t, nil, err)
+	assert.Equal(t, false, fluxCfg.GitCISkip)
+
+	// Test we only serialize props that we provided
+	argString = "--git-label=foo --git-path=derp"
+	fluxCfg, err = ParseFluxArgs(argString)
+	assert.Equal(t, nil, err)
+	assert.Equal(t, "foo", fluxCfg.GitLabel)
+	assert.Equal(t, "git-label=foo&git-path=derp", fluxCfg.AsQueryParams())
+
+	// string[]
+	argString = "--git-path=zing --git-path=derp"
+	fluxCfg, err = ParseFluxArgs(argString)
+	assert.Equal(t, nil, err)
+	assert.Equal(t, "git-path=zing&git-path=derp", fluxCfg.AsQueryParams())
+
+	// unknown
+	argString = "--token=derp"
+	fluxCfg, err = ParseFluxArgs(argString)
+	assert.Equal(t, nil, err)
+	assert.Equal(t, "", fluxCfg.AsQueryParams())
+
+	// some unknown
+	argString = "--git-path=zing --token=derp"
+	fluxCfg, err = ParseFluxArgs(argString)
+	assert.Equal(t, nil, err)
+	assert.Equal(t, "git-path=zing", fluxCfg.AsQueryParams())
+
+	// Preserves empty values
+	argString = "--memcached-service="
+	fluxCfg, err = ParseFluxArgs(argString)
+	assert.Equal(t, nil, err)
+	assert.Equal(t, "memcached-service=", fluxCfg.AsQueryParams())
+}

integration-tests/tests/flux-config.sh

@@ -55,7 +55,8 @@ sleep 40
 
 echo "• Check flux configuration still exists"
 args=$(kubectl get pod -n weave -l name=weave-flux-agent -o jsonpath='{.items[?(@.metadata.labels.name=="weave-flux-agent")].spec.containers[?(@.name=="flux-agent")].args[*]}')
-if [[ $args != *"[email protected]:weaveworks/example --git-path=k8s/example --git-branch=example --git-label=example"* ]]; then
-    echo "Missing existing flux args"
+expected="[email protected]:weaveworks/example --git-path=k8s/example --git-branch=example --git-label=example"
+if [[ $args != *"$expected"* ]]; then
+    echo "Missing existing flux args: \"$expected\" not found in \"$args\""
     exit 1
 fi

integration-tests/tests/kube-system-migration.sh

@@ -35,6 +35,6 @@ while [ $(kubectl get pods -n kube-system -l 'app in (weave-flux, weave-cortex,
 echo "• Check old flux arguments have been applied to the new agent"
 args=$(kubectl get pod -n weave -l name=weave-flux-agent -o jsonpath='{.items[?(@.metadata.labels.name=="weave-flux-agent")].spec.containers[?(@.name=="flux-agent")].args[*]}')
 if [[ $args != *"[email protected]:weaveworks/example --git-path=k8s/example --git-branch=master --git-label=example"* ]]; then
-    echo "Missing existing flux args"
+    echo "Missing existing flux args: got $args"
     exit 1
 fi

service/static/install.sh

@@ -50,8 +50,9 @@ else
 fi
 
 # Download the bootstrap binary
-echo "Downloading the Weave Cloud installer...  "
-curl -Ls "{{.Scheme}}://{{.LauncherHostname}}/bootstrap?dist=$dist" >> "$TMPFILE"
+installerurl="{{.Scheme}}://{{.LauncherHostname}}/bootstrap?dist=$dist"
+echo "Downloading the Weave Cloud installer from $installerurl"
+curl -Ls "$installerurl" >> "$TMPFILE"
 
 # Make the bootstrap binary executable
 chmod +x "$TMPFILE"