From d2e0b88548aeffbda35636e3ce9eb87ac112af62 Mon Sep 17 00:00:00 2001 From: Colin Chartier Date: Tue, 20 Aug 2019 21:33:42 -0400 Subject: [PATCH] Add k3s support. --- provisioners/common.go | 12 ++++- provisioners/external/external.go | 4 ++ provisioners/k3s/k3s.go | 80 +++++++++++++++++++++++++++++++ provisioners/localdev/localdev.go | 5 +- provisioners/util/registry.go | 9 ++-- 5 files changed, 104 insertions(+), 6 deletions(-) create mode 100644 provisioners/k3s/k3s.go diff --git a/provisioners/common.go b/provisioners/common.go index 1910a7b..0441eb8 100644 --- a/provisioners/common.go +++ b/provisioners/common.go @@ -1,8 +1,9 @@ - package provisioners +package provisioners import ( "fmt" "github.com/distributed-containers-inc/sanic/provisioners/external" + "github.com/distributed-containers-inc/sanic/provisioners/k3s" "github.com/distributed-containers-inc/sanic/provisioners/localdev" "github.com/distributed-containers-inc/sanic/provisioners/provisioner" ) @@ -13,6 +14,9 @@ var provisionerBuilders = map[string]provisionerBuilder{ "external": func(args map[string]string) provisioner.Provisioner { return external.Create(args) }, + "k3s": func(args map[string]string) provisioner.Provisioner { + return &k3s.ProvisionerK3s{} + }, "localdev": func(args map[string]string) provisioner.Provisioner { return &localdev.ProvisionerLocalDev{} }, @@ -24,6 +28,12 @@ var provisionerValidators = map[string]provisionerConfigValidator{ "external": func(args map[string]string) error { return external.ValidateConfig(args) }, + "k3s": func(args map[string]string) error { + for k := range args { + return fmt.Errorf("k3s takes no arguments, got %s", k) + } + return nil + }, "localdev": func(args map[string]string) error { for k := range args { return fmt.Errorf("localdev takes no arguments, got %s", k) diff --git a/provisioners/external/external.go b/provisioners/external/external.go index b7cdf4f..0cae6e1 100644 --- a/provisioners/external/external.go +++ b/provisioners/external/external.go @@ -23,6 +23,10 @@ func (provisioner *ProvisionerExternal) EnsureCluster() error { //KubectlCommand for external just returns a provisioner that has KUBECONFIG pointing to the configured directory func (provisioner *ProvisionerExternal) KubectlCommand(args ...string) (*exec.Cmd, error) { + if _, err := exec.LookPath("kubectl"); err != nil { + return nil, errors.Wrap(err, "could not find kubectl executable in path - is it installed?") + } + cmd := exec.Command("kubectl", args...) kubeconfig := provisioner.kubeConfigLocation diff --git a/provisioners/k3s/k3s.go b/provisioners/k3s/k3s.go new file mode 100644 index 0000000..50e4e2c --- /dev/null +++ b/provisioners/k3s/k3s.go @@ -0,0 +1,80 @@ +package k3s + +import ( + "context" + "fmt" + provisionerutil "github.com/distributed-containers-inc/sanic/provisioners/util" + "github.com/pkg/errors" + "os" + "os/exec" + "strings" +) + +//ProvisionerK3s starts k3s server +type ProvisionerK3s struct { +} + +//EnsureCluster just ensures the registry is running +func (provisioner *ProvisionerK3s) EnsureCluster() error { + return provisionerutil.StartRegistry(provisioner, context.Background(), map[string]string{"node-role.kubernetes.io/master": "true"}) +} + +//KubectlCommand for k3s is just a wrapper around "k3s kubectl" +func (provisioner *ProvisionerK3s) KubectlCommand(args ...string) (*exec.Cmd, error) { + if _, err := exec.LookPath("k3s"); err != nil { + return nil, errors.Wrap(err, "could not find k3s executable in path - is it installed?") + } + cmd := exec.Command("k3s", append([]string{"kubectl"}, args...)...) + cmd.Env = os.Environ() + + return cmd, nil +} + +func (provisioner *ProvisionerK3s) Registry() (registryAddr string, registryInsecure bool, err error) { + cmd, err := provisioner.KubectlCommand( + "get", "service", + "--namespace", "kube-system", + "sanic-registry", + "--output", "jsonpath={.spec.clusterIP}", + ) + if err != nil { + return + } + out, err := cmd.Output() + if err != nil { + return + } + ip := strings.TrimSpace(string(out)) + if ip == "" { + err = fmt.Errorf("could not connect to registry - try 'sanic deploy' and waiting 90 seconds") + return + } + registryAddr = fmt.Sprintf("%s:5000", ip) + registryInsecure = true + return +} + +func (provisioner *ProvisionerK3s) EdgeNodes() ([]string, error) { + cmd, err := provisioner.KubectlCommand( + "get", "services", + "-n", "kube-system", + "-o", "jsonpath={.spec.clusterIP}", + "traefik", + ) + if err != nil { + return nil, errors.Wrap(err, "could not get the traefik service") + } + out, err := cmd.Output() + if err != nil { + return nil, errors.Wrap(err, "could not get the traefik service") + } + ip := strings.TrimSpace(string(out)) + if ip == "" { + return nil, fmt.Errorf("could not get the IP of the traefik service") + } + return []string{ip}, nil +} + +func (provisioner *ProvisionerK3s) InClusterDir(hostDir string) string { + return hostDir //k3s runs the server on the computer itself +} diff --git a/provisioners/localdev/localdev.go b/provisioners/localdev/localdev.go index 795e18d..fcd7283 100644 --- a/provisioners/localdev/localdev.go +++ b/provisioners/localdev/localdev.go @@ -31,6 +31,9 @@ type ProvisionerLocalDev struct{} //KubectlCommand : In ProvisionerLocalDev, returns kubectl pointing to kind's own generated configuration func (provisioner *ProvisionerLocalDev) KubectlCommand(args ...string) (*exec.Cmd, error) { + if _, err := exec.LookPath("kubectl"); err != nil { + return nil, errors.Wrap(err, "could not find kubectl executable in path - is it installed?") + } cmd := exec.Command("kubectl", args...) configPath := kindContext.KubeConfigPath() if _, err := os.Stat(configPath); err != nil { @@ -55,7 +58,7 @@ func (provisioner *ProvisionerLocalDev) EnsureCluster() error { return err } err = util.RunContextuallyInParallel(context.Background(), - func(ctx context.Context) error { return provisionerutil.StartRegistry(provisioner, ctx) }, + func(ctx context.Context) error { return provisionerutil.StartRegistry(provisioner, ctx, map[string]string{"node-role.kubernetes.io/master": ""}) }, provisioner.patchRegistryContainers, provisioner.startIngressController, ) diff --git a/provisioners/util/registry.go b/provisioners/util/registry.go index 5a02134..d8461ae 100644 --- a/provisioners/util/registry.go +++ b/provisioners/util/registry.go @@ -36,9 +36,9 @@ spec: name: sanic-registry spec: terminationGracePeriodSeconds: 10 - hostNetwork: true nodeSelector: - node-role.kubernetes.io/master: "" + {{range $key, $value := .NodeSelectors}}{{$key}}: "{{$value}}" +{{end}} tolerations: - key: node-role.kubernetes.io/master operator: Exists @@ -67,7 +67,7 @@ spec: ` //StartRegistry : makes a pod definition using registry:2 -func StartRegistry(provisioner provisioner.Provisioner, ctx context.Context) error { +func StartRegistry(provisioner provisioner.Provisioner, ctx context.Context, nodeSelectors map[string]string) error { cmd, err := provisioner.KubectlCommand("apply", "-f", "-") if err != nil { @@ -76,6 +76,7 @@ func StartRegistry(provisioner provisioner.Provisioner, ctx context.Context) err type yamlConfig struct { RegistryNodePort int + NodeSelectors map[string]string } t, err := template.New("").Parse(registryYaml) if err != nil { @@ -83,7 +84,7 @@ func StartRegistry(provisioner provisioner.Provisioner, ctx context.Context) err } stdinBuffer := &bytes.Buffer{} - err = t.Execute(stdinBuffer, &yamlConfig{RegistryNodePort: RegistryNodePort}) + err = t.Execute(stdinBuffer, &yamlConfig{RegistryNodePort: RegistryNodePort, NodeSelectors: nodeSelectors}) if err != nil { panic(err) }