From 2e31b8180e4da1e1e6d611fd60ccd84f669f9468 Mon Sep 17 00:00:00 2001 From: Yash Singh Date: Thu, 29 Sep 2022 20:16:52 +0530 Subject: [PATCH 1/4] Adding installing steps for testing webhooks Adding installing steps for testing webhooks Adding installing steps for testing webhooks Adding installing steps for testing webhooks Adding installing steps for testing webhooks Adding installing steps for testing webhooks Adding installing steps for testing webhooks Adding installing steps for testing webhooks Adding installing steps for testing webhooks Adding installing steps for testing webhooks Adding installing steps for testing webhooks Adding installing steps for testing webhooks --- docs/book/src/reference/envtest.md | 163 +++++++++++++++++++++++++++-- 1 file changed, 153 insertions(+), 10 deletions(-) diff --git a/docs/book/src/reference/envtest.md b/docs/book/src/reference/envtest.md index 6eb65c7ece3..86ade0cac27 100644 --- a/docs/book/src/reference/envtest.md +++ b/docs/book/src/reference/envtest.md @@ -12,7 +12,7 @@ The make targets require `bash` to run. ## Installation in Air Gaped/disconnected environments If you would like to download the tarball containing the binaries, to use in a disconnected environment you can use -`setup-envtest` to download the required binaries locally. There are a lot of ways to configure `setup-envtest` to avoid talking to +[`setup-envtest`][setup-envtest] to download the required binaries locally. There are a lot of ways to configure `setup-envtest` to avoid talking to the internet you can read about them [here](https://github.com/kubernetes-sigs/controller-runtime/tree/master/tools/setup-envtest#what-if-i-dont-want-to-talk-to-the-internet). The examples below will show how to install the Kubernetes API binaries using mostly defaults set by `setup-envtest`. @@ -76,6 +76,15 @@ err = testEnv.Stop() Logs from the test runs are prefixed with `test-env`. + + ### Configuring your test control plane Controller-runtime’s [envtest][envtest] framework requires `kubectl`, `kube-apiserver`, and `etcd` binaries be present locally to simulate the API portions of a real cluster. @@ -97,12 +106,11 @@ You can use environment variables and/or flags to specify the `kubectl`,`api-ser ### Environment Variables | Variable name | Type | When to use | -| --- | :--- | :--- | -| `USE_EXISTING_CLUSTER` | boolean | Instead of setting up a local control plane, point to the control plane of an existing cluster. | -| `KUBEBUILDER_ASSETS` | path to directory | Point integration tests to a directory containing all binaries (api-server, etcd and kubectl). | -| `TEST_ASSET_KUBE_APISERVER`, `TEST_ASSET_ETCD`, `TEST_ASSET_KUBECTL` | paths to, respectively, api-server, etcd and kubectl binaries | Similar to `KUBEBUILDER_ASSETS`, but more granular. Point integration tests to use binaries other than the default ones. These environment variables can also be used to ensure specific tests run with expected versions of these binaries. | -| `KUBEBUILDER_CONTROLPLANE_START_TIMEOUT` and `KUBEBUILDER_CONTROLPLANE_STOP_TIMEOUT` | durations in format supported by [`time.ParseDuration`](https://golang.org/pkg/time/#ParseDuration) | Specify timeouts different from the default for the test control plane to (respectively) start and stop; any test run that exceeds them will fail. | -| `KUBEBUILDER_ATTACH_CONTROL_PLANE_OUTPUT` | boolean | Set to `true` to attach the control plane's stdout and stderr to os.Stdout and os.Stderr. This can be useful when debugging test failures, as output will include output from the control plane. | +| --- | :--- | :--- | +| `KUBEBUILDER_ASSETS` | path to directory | Point integration tests to a directory containing all binaries (api-server, etcd and kubectl). | +| `TEST_ASSET_KUBE_APISERVER`, `TEST_ASSET_ETCD`, `TEST_ASSET_KUBECTL` | paths to, respectively, api-server, etcd and kubectl binaries | Similar to `KUBEBUILDER_ASSETS`, but more granular. Point integration tests to use binaries other than the default ones. These environment variables can also be used to ensure specific tests run with expected versions of these binaries. | +| `KUBEBUILDER_CONTROLPLANE_START_TIMEOUT` and `KUBEBUILDER_CONTROLPLANE_STOP_TIMEOUT` | durations in format supported by [`time.ParseDuration`](https://golang.org/pkg/time/#ParseDuration) | Specify timeouts different from the default for the test control plane to (respectively) start and stop; any test run that exceeds them will fail. | +| `KUBEBUILDER_ATTACH_CONTROL_PLANE_OUTPUT` | boolean | Set to `true` to attach the control plane's stdout and stderr to os.Stdout and os.Stderr. This can be useful when debugging test failures, as output will include output from the control plane. | See that the `test` makefile target will ensure that all is properly setup when you are using it. However, if you would like to run the tests without use the Makefile targets, for example via an IDE, then you can set the environment variables directly in the code of your `suite_test.go`: @@ -116,7 +124,7 @@ var _ = BeforeSuite(func(done Done) { logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) testenv = &envtest.Environment{} - + _, err := testenv.Start() Expect(err).NotTo(HaveOccurred()) @@ -133,6 +141,14 @@ var _ = AfterSuite(func() { }) ``` + + ### Flags Here's an example of modifying the flags with which to start the API server in your integration tests, compared to the default values in `envtest.DefaultKubeAPIServerFlags`: @@ -167,5 +183,132 @@ expectedOwnerReference := v1.OwnerReference{ Expect(deployment.ObjectMeta.OwnerReferences).To(ContainElement(expectedOwnerReference)) ``` -[envtest]:https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/envtest -[setup-envtest]:https://pkg.go.dev/sigs.k8s.io/controller-runtime/tools/setup-envtest +## Cert-Manager and Prometheus options + +Projects scaffolded with Kubebuilder can enable the [`metrics`][metrics] and the [`cert-manager`][cert-manager] options. Note that when we are using the ENV TEST we are looking to test the controllers and their reconciliation. It is considered an integrated test because the ENV TEST API will do the test against a cluster and because of this the binaries are downloaded and used to configure its pre-requirements, however, its purpose is mainly to `unit` test the controllers. + +Therefore, to test a reconciliation in common cases you do not need to care about these options. However, if you would like to do tests with the Prometheus and the Cert-manager installed you can add the required steps to install them before running the tests. +Following an example. + +```go + // Add the operations to install the Prometheus operator and the cert-manager + // before the tests. + BeforeEach(func() { + By("installing prometheus operator") + Expect(utils.InstallPrometheusOperator()).To(Succeed()) + + By("installing the cert-manager") + Expect(utils.InstallCertManager()).To(Succeed()) + } + + // You can also remove them after the tests:: + AfterEach(func() { + By("uninstalling the Prometheus manager bundle") + utils.UninstallPrometheusOperManager() + + By("uninstalling the cert-manager bundle") + utils.UninstallCertManager() + }) +``` + +Check the following example of how you can implement the above operations: + +```go +const ( + prometheusOperatorVersion = "0.51" + prometheusOperatorURL = "https://raw.githubusercontent.com/prometheus-operator/" + "prometheus-operator/release-%s/bundle.yaml" + certmanagerVersion = "v1.5.3" + certmanagerURLTmpl = "https://github.com/jetstack/cert-manager/releases/download/%s/cert-manager.yaml" +) + +func warnError(err error) { + fmt.Fprintf(GinkgoWriter, "warning: %v\n", err) +} + +// InstallPrometheusOperator installs the prometheus Operator to be used to export the enabled metrics. +func InstallPrometheusOperator() error { + url := fmt.Sprintf(prometheusOperatorURL, prometheusOperatorVersion) + cmd := exec.Command("kubectl", "apply", "-f", url) + _, err := Run(cmd) + return err +} + +// UninstallPrometheusOperator uninstalls the prometheus +func UninstallPrometheusOperator() { + url := fmt.Sprintf(prometheusOperatorURL, prometheusOperatorVersion) + cmd := exec.Command("kubectl", "delete", "-f", url) + if _, err := Run(cmd); err != nil { + warnError(err) + } +} + +// UninstallCertManager uninstalls the cert manager +func UninstallCertManager() { + url := fmt.Sprintf(certmanagerURLTmpl, certmanagerVersion) + cmd := exec.Command("kubectl", "delete", "-f", url) + if _, err := Run(cmd); err != nil { + warnError(err) + } +} + +// InstallCertManager installs the cert manager bundle. +func InstallCertManager() error { + url := fmt.Sprintf(certmanagerURLTmpl, certmanagerVersion) + cmd := exec.Command("kubectl", "apply", "-f", url) + if _, err := Run(cmd); err != nil { + return err + } + // Wait for cert-manager-webhook to be ready, which can take time if cert-manager + //was re-installed after uninstalling on a cluster. + cmd = exec.Command("kubectl", "wait", "deployment.apps/cert-manager-webhook", + "--for", "condition=Available", + "--namespace", "cert-manager", + "--timeout", "5m", + ) + + _, err := Run(cmd) + return err +} + +// LoadImageToKindCluster loads a local docker image to the kind cluster +func LoadImageToKindClusterWithName(name string) error { + cluster := "kind" + if v, ok := os.LookupEnv("KIND_CLUSTER"); ok { + cluster = v + } + + kindOptions := []string{"load", "docker-image", name, "--name", cluster} + cmd := exec.Command("kind", kindOptions...) + _, err := Run(cmd) + return err +} + +// GetNonEmptyLines converts given command output string into individual objects +// according to line breakers, and ignores the empty elements in it. +func GetNonEmptyLines(output string) []string { + var res []string + elements := strings.Split(output, "\n") + for _, element := range elements { + if element != "" { + res = append(res, element) + } + } + + return res +} +``` +However, see that tests for the metrics and cert-manager might fit better well as e2e tests and not under the tests done using ENV TEST for the controllers. You might want to give a look at the [sample example][sdk-e2e-sample-example] implemented into [Operator-SDK][sdk] repository to know how can you can write your e2e tests to ensure the basic workflows of your project. +Also, see that you can run the tests against a cluster where you have some configurations in place they can use the option to test using an existing cluster: + +```go +testEnv = &envtest.Environment{ + UseExistingCluster: true, +} +``` + +[metrics]: https://book.kubebuilder.io/reference/metrics.html +[envtest]: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/envtest +[setup-envtest]: https://pkg.go.dev/sigs.k8s.io/controller-runtime/tools/setup-envtest +[cert-manager]: https://book.kubebuilder.io/cronjob-tutorial/cert-manager.html +[sdk-e2e-sample-example]: https://github.com/operator-framework/operator-sdk/tree/master/testdata/go/v3/memcached-operator/test/e2e +[sdk]: https://github.com/operator-framework/operator-sdk \ No newline at end of file From b257ea047eb58c63936120741cd775637c8185d4 Mon Sep 17 00:00:00 2001 From: Yash Singh Date: Fri, 7 Oct 2022 21:15:22 +0530 Subject: [PATCH 2/4] Adding installing steps for testing webhooks --- docs/book/src/reference/envtest.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/book/src/reference/envtest.md b/docs/book/src/reference/envtest.md index 86ade0cac27..103a0642a0b 100644 --- a/docs/book/src/reference/envtest.md +++ b/docs/book/src/reference/envtest.md @@ -107,6 +107,7 @@ You can use environment variables and/or flags to specify the `kubectl`,`api-ser | Variable name | Type | When to use | | --- | :--- | :--- | +| `USE_EXISTING_CLUSTER` | boolean | Instead of setting up a local control plane, point to the control plane of an existing cluster. | | `KUBEBUILDER_ASSETS` | path to directory | Point integration tests to a directory containing all binaries (api-server, etcd and kubectl). | | `TEST_ASSET_KUBE_APISERVER`, `TEST_ASSET_ETCD`, `TEST_ASSET_KUBECTL` | paths to, respectively, api-server, etcd and kubectl binaries | Similar to `KUBEBUILDER_ASSETS`, but more granular. Point integration tests to use binaries other than the default ones. These environment variables can also be used to ensure specific tests run with expected versions of these binaries. | | `KUBEBUILDER_CONTROLPLANE_START_TIMEOUT` and `KUBEBUILDER_CONTROLPLANE_STOP_TIMEOUT` | durations in format supported by [`time.ParseDuration`](https://golang.org/pkg/time/#ParseDuration) | Specify timeouts different from the default for the test control plane to (respectively) start and stop; any test run that exceeds them will fail. | From 04f302ae027e287a72bbd169d9996ed98d724bac Mon Sep 17 00:00:00 2001 From: Yash Singh Date: Fri, 7 Oct 2022 21:16:56 +0530 Subject: [PATCH 3/4] Adding installing steps for testing webhooks --- docs/book/src/reference/envtest.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/book/src/reference/envtest.md b/docs/book/src/reference/envtest.md index 103a0642a0b..3bb7a201021 100644 --- a/docs/book/src/reference/envtest.md +++ b/docs/book/src/reference/envtest.md @@ -298,7 +298,7 @@ func GetNonEmptyLines(output string) []string { return res } ``` -However, see that tests for the metrics and cert-manager might fit better well as e2e tests and not under the tests done using ENV TEST for the controllers. You might want to give a look at the [sample example][sdk-e2e-sample-example] implemented into [Operator-SDK][sdk] repository to know how can you can write your e2e tests to ensure the basic workflows of your project. +However, see that tests for the metrics and cert-manager might fit better well as e2e tests and not under the tests done using ENV TEST for the controllers. You might want to give a look at the [sample example][sdk-e2e-sample-example] implemented into [Operator-SDK][sdk] repository to know how you can write your e2e tests to ensure the basic workflows of your project. Also, see that you can run the tests against a cluster where you have some configurations in place they can use the option to test using an existing cluster: ```go From 0c9247cc59c98511af710397413dfa5f096b0c40 Mon Sep 17 00:00:00 2001 From: Yash Singh Date: Tue, 11 Oct 2022 08:35:21 +0530 Subject: [PATCH 4/4] Adding installing steps for testing webhooks --- docs/book/src/reference/envtest.md | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/docs/book/src/reference/envtest.md b/docs/book/src/reference/envtest.md index 3bb7a201021..45a18ddbc8e 100644 --- a/docs/book/src/reference/envtest.md +++ b/docs/book/src/reference/envtest.md @@ -283,20 +283,6 @@ func LoadImageToKindClusterWithName(name string) error { _, err := Run(cmd) return err } - -// GetNonEmptyLines converts given command output string into individual objects -// according to line breakers, and ignores the empty elements in it. -func GetNonEmptyLines(output string) []string { - var res []string - elements := strings.Split(output, "\n") - for _, element := range elements { - if element != "" { - res = append(res, element) - } - } - - return res -} ``` However, see that tests for the metrics and cert-manager might fit better well as e2e tests and not under the tests done using ENV TEST for the controllers. You might want to give a look at the [sample example][sdk-e2e-sample-example] implemented into [Operator-SDK][sdk] repository to know how you can write your e2e tests to ensure the basic workflows of your project. Also, see that you can run the tests against a cluster where you have some configurations in place they can use the option to test using an existing cluster: