k8s-tester: initial commit, run test against existing cluster (#210)

* k8s-tester: initial commit

Signed-off-by: Gyuho Lee <[email protected]>

* CHANGELOG: update

Signed-off-by: Gyuho Lee <[email protected]>

CHANGELOG/CHANGELOG-1.5.md

@@ -18,6 +18,11 @@
 - Add [`AWS_K8S_TESTER_EKS_ADD_ON_CLUSTER_LOADER_REMOTE_CL2_SCHEDULER_THROUGHPUT_THRESHOLD`](https://github.com/aws/aws-k8s-tester/pull/208).
 - Ignore error [for an unknown region](https://github.com/aws/aws-k8s-tester/pull/204).
 
+### `k8s-tester`
+
+- [Initial commit](https://github.com/aws/aws-k8s-tester/pull/210).
+  - To run [tests against existing clusters](https://github.com/aws/aws-k8s-tester/issues/123).
+
 ### Go
 
 - Compile with [*Go 1.16.3*](https://golang.org/doc/devel/release.html#go1.16).

k8s-tester/README.md

@@ -0,0 +1,28 @@
+
+`k8s-tester` implements defines Kubernetes "tester client" interface without "cluster provisioner" dependency. This replaces all test cases under `eks/*` (< `aws-k8s-tester` v1.6). The tester assumes an existing Kubernetes cluster (e.g., EKS, vanilla Kubernetes) and worker nodes to run testing components.
+
+The test case:
+ - may be opinionated but must comply with `"github.com/aws/aws-k8s-tester/k8s-tester/tester.Tester"` interface
+ - may require certain AWS API calls and assume correct IAM or instance role for required AWS actions
+ - must be generic enough to run against any Kubernetes cluster on AWS
+ - must implement clean-up in a non-destrutive way
+ - must implement a package that can be easily imported as a library (e.g., integrates with EKS tester)
+ - must control their own dependencies (e.g., vending Kubernetes client-go) in case a user does not want to carry out other dependencies
+ - must be easy to use -- a single command for install and clean-up
+ - must implement a CLI with the sub-commands of "apply" and "delete"
+
+To add a new tester,
+
+**Step 1.** Create a new directory under `"k8s-tester"`.
+
+**Step 2.** Implement `"github.com/aws/aws-k8s-tester/k8s-tester/tester.Tester"` interface.
+
+**Step 3.** Run
+
+```bash
+go mod init
+go mod tidy -v
+
+# don't run
+# go mod vendor -v
+```

k8s-tester/fmt.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -e
+
+goimports -w ./nlb-hello-world
+goimports -w ./tester
+
+gofmt -s -w ./nlb-hello-world
+gofmt -s -w ./tester

k8s-tester/nlb-hello-world/cmd/k8s-tester-nlb-hello-world/main.go

@@ -0,0 +1,165 @@
+// k8s-tester-nlb-hello-world installs Kubernetes NLB hello world tester.
+package main
+
+import (
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/aws/aws-k8s-tester/client"
+	nlb_hello_world "github.com/aws/aws-k8s-tester/k8s-tester/nlb-hello-world"
+	"github.com/manifoldco/promptui"
+	"github.com/spf13/cobra"
+	"go.uber.org/zap"
+)
+
+var rootCmd = &cobra.Command{
+	Use:        "k8s-tester-nlb-hello-world",
+	Short:      "Kubernetes NLB hello world tester",
+	SuggestFor: []string{"nlb-hello-world"},
+}
+
+func init() {
+	cobra.EnablePrefixMatching = true
+}
+
+var (
+	enablePrompt   bool
+	namespace      string
+	kubectlPath    string
+	kubeConfigPath string
+)
+
+// TODO: make logging configurable
+// TODO: make deployment replicas, node selector configurable
+
+func init() {
+	rootCmd.PersistentFlags().BoolVar(&enablePrompt, "enable-prompt", true, "'true' to enable prompt mode")
+	rootCmd.PersistentFlags().StringVar(&namespace, "namespace", "test-namespace", "'true' to auto-generate path for create config/cluster, overwrites existing --path value")
+	rootCmd.PersistentFlags().StringVar(&kubectlPath, "kubectl-path", "", "kubectl path")
+	rootCmd.PersistentFlags().StringVar(&kubeConfigPath, "kubeconfig-path", "", "KUBECONFIG path")
+
+	rootCmd.AddCommand(
+		newApply(),
+		newDelete(),
+	)
+}
+
+func main() {
+	if err := rootCmd.Execute(); err != nil {
+		fmt.Fprintf(os.Stderr, "k8s-tester-nlb-hello-world failed %v\n", err)
+		os.Exit(1)
+	}
+	os.Exit(0)
+}
+
+func newApply() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "apply",
+		Short: "Apply tests",
+		Run:   createApplyFunc,
+	}
+	return cmd
+}
+
+func createApplyFunc(cmd *cobra.Command, args []string) {
+	if enablePrompt {
+		prompt := promptui.Select{
+			Label: "Ready to apply resources, should we continue?",
+			Items: []string{
+				"No, cancel it!",
+				"Yes, let's apply!",
+			},
+		}
+		idx, answer, err := prompt.Run()
+		if err != nil {
+			panic(err)
+		}
+		if idx != 1 {
+			fmt.Printf("returning 'apply' [index %d, answer %q]\n", idx, answer)
+			return
+		}
+	}
+
+	time.Sleep(5 * time.Second)
+
+	lg, err := zap.NewDevelopment()
+	if err != nil {
+		panic(err)
+	}
+	cfg := nlb_hello_world.Config{
+		Logger:    lg,
+		LogWriter: os.Stderr,
+		Namespace: namespace,
+		ClientConfig: &client.Config{
+			Logger:         lg,
+			KubectlPath:    kubectlPath,
+			KubeConfigPath: kubeConfigPath,
+		},
+		DeploymentReplicas: 1,
+	}
+
+	ts := nlb_hello_world.New(cfg)
+	if err := ts.Apply(); err != nil {
+		fmt.Fprintf(os.Stderr, "failed to apply (%v)\n", err)
+		os.Exit(1)
+	}
+
+	fmt.Printf("\n*********************************\n")
+	fmt.Printf("'k8s-tester-nlb-hello-world apply' success\n")
+}
+
+func newDelete() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "delete",
+		Short: "Delete resources",
+		Run:   createDeleteFunc,
+	}
+	return cmd
+}
+
+func createDeleteFunc(cmd *cobra.Command, args []string) {
+	if enablePrompt {
+		prompt := promptui.Select{
+			Label: "Ready to delete resources, should we continue?",
+			Items: []string{
+				"No, cancel it!",
+				"Yes, let's delete!",
+			},
+		}
+		idx, answer, err := prompt.Run()
+		if err != nil {
+			panic(err)
+		}
+		if idx != 1 {
+			fmt.Printf("returning 'delete' [index %d, answer %q]\n", idx, answer)
+			return
+		}
+	}
+
+	time.Sleep(5 * time.Second)
+
+	lg, err := zap.NewDevelopment()
+	if err != nil {
+		panic(err)
+	}
+	cfg := nlb_hello_world.Config{
+		Logger:    lg,
+		LogWriter: os.Stderr,
+		Namespace: namespace,
+		ClientConfig: &client.Config{
+			Logger:         lg,
+			KubectlPath:    kubectlPath,
+			KubeConfigPath: kubeConfigPath,
+		},
+	}
+
+	ts := nlb_hello_world.New(cfg)
+	if err := ts.Delete(); err != nil {
+		fmt.Fprintf(os.Stderr, "failed to delete (%v)\n", err)
+		os.Exit(1)
+	}
+
+	fmt.Printf("\n*********************************\n")
+	fmt.Printf("'k8s-tester-nlb-hello-world delete' success\n")
+}

k8s-tester/nlb-hello-world/go.mod

@@ -0,0 +1,23 @@
+module github.com/aws/aws-k8s-tester/k8s-tester/nlb-hello-world
+
+go 1.16
+
+require (
+	github.com/aws/aws-k8s-tester/client v0.0.0-00010101000000-000000000000
+	github.com/aws/aws-k8s-tester/k8s-tester/tester v0.0.0-00010101000000-000000000000
+	github.com/aws/aws-sdk-go v1.38.21
+	github.com/dustin/go-humanize v1.0.0
+	github.com/manifoldco/promptui v0.8.0
+	github.com/mitchellh/ioprogress v0.0.0-20180201004757-6a23b12fa88e
+	github.com/spf13/cobra v1.1.3
+	go.uber.org/zap v1.16.0
+	k8s.io/api v0.21.0
+	k8s.io/apimachinery v0.21.0
+	k8s.io/client-go v0.21.0
+	k8s.io/utils v0.0.0-20210305010621-2afb4311ab10
+)
+
+replace (
+	github.com/aws/aws-k8s-tester/client => ../../client
+	github.com/aws/aws-k8s-tester/k8s-tester/tester => ../tester
+)