diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index a02ec19e8..479a9f982 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -6,33 +6,38 @@ on: merge_group: push: branches: - - main + - main jobs: e2e-kind: runs-on: ubuntu-latest steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 - - uses: actions/checkout@v4 - with: - fetch-depth: 0 + - uses: actions/setup-go@v4 + with: + go-version-file: go.mod - - uses: actions/setup-go@v4 - with: - go-version-file: go.mod + - name: Run e2e tests + run: | + # By default make stops building on first non-zero exit code which + # in case of E2E tests will mean that code coverage will only be + # collected on successful runs. We want to collect coverage even + # after failing tests. + # With -k flag make will continue the build, but will return non-zero + # exit code in case of any errors. + ARTIFACT_PATH=/tmp/artifacts make -k test-e2e - - name: Run e2e tests - run: | - # By default make stops building on first non-zero exit code which - # in case of E2E tests will mean that code coverage will only be - # collected on successful runs. We want to collect coverage even - # after failing tests. - # With -k flag make will continue the build, but will return non-zero - # exit code in case of any errors. - make -k test-e2e + - uses: cytopia/upload-artifact-retry-action@v0.1.7 + if: failure() + with: + name: e2e-artifacts + path: /tmp/artifacts/ - - uses: codecov/codecov-action@v3 - with: - files: e2e-cover.out - flags: e2e - functionalities: fixes + - uses: codecov/codecov-action@v3 + with: + files: e2e-cover.out + flags: e2e + functionalities: fixes diff --git a/Makefile b/Makefile index a023bd2ce..cad1265a6 100644 --- a/Makefile +++ b/Makefile @@ -19,6 +19,10 @@ export XDG_DATA_HOME ?= /tmp/.local/share # bingo manages consistent tooling versions for things like kind, kustomize, etc. include .bingo/Variables.mk +# ARTIFACT_PATH is the absolute path to the directory where the operator-controller e2e tests will store the artifacts +# for example: ARTIFACT_PATH=/tmp/artifacts make test +export ARTIFACT_PATH ?= + OPERATOR_CONTROLLER_NAMESPACE ?= operator-controller-system KIND_CLUSTER_NAME ?= operator-controller diff --git a/go.mod b/go.mod index 678b1d4da..9691ca2fb 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,8 @@ require ( github.com/stretchr/testify v1.8.4 go.uber.org/zap v1.25.0 golang.org/x/exp v0.0.0-20230315142452-642cacee5cc0 + gopkg.in/yaml.v2 v2.4.0 + k8s.io/api v0.26.1 k8s.io/apiextensions-apiserver v0.26.1 k8s.io/apimachinery v0.26.1 k8s.io/client-go v0.26.1 @@ -134,9 +136,7 @@ require ( google.golang.org/protobuf v1.28.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/api v0.26.1 // indirect k8s.io/apiserver v0.26.1 // indirect k8s.io/klog/v2 v2.80.1 // indirect k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 // indirect diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 2d3d34bdc..3d970321d 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -8,6 +8,9 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/utils/env" "k8s.io/apimachinery/pkg/api/errors" apimeta "k8s.io/apimachinery/pkg/api/meta" @@ -61,6 +64,13 @@ var _ = BeforeSuite(func() { Expect(catalogd.AddToScheme(scheme)).To(Succeed()) var err error + + err = appsv1.AddToScheme(scheme) + Expect(err).ToNot(HaveOccurred()) + + err = corev1.AddToScheme(scheme) + Expect(err).ToNot(HaveOccurred()) + c, err = client.New(cfg, client.Options{Scheme: scheme}) Expect(err).To(Not(HaveOccurred())) @@ -82,7 +92,10 @@ var _ = BeforeSuite(func() { var _ = AfterSuite(func() { ctx := context.Background() - + if basePath := env.GetString("ARTIFACT_PATH", ""); basePath != "" { + // get all the artifacts from the test run and save them to the artifact path + getArtifactsOutput(ctx, basePath) + } Expect(c.Delete(ctx, operatorCatalog)).To(Succeed()) Eventually(func(g Gomega) { err := c.Get(ctx, types.NamespacedName{Name: operatorCatalog.Name}, &catalogd.Catalog{}) diff --git a/test/e2e/install_test.go b/test/e2e/install_test.go index b38f650bb..1905cfb7a 100644 --- a/test/e2e/install_test.go +++ b/test/e2e/install_test.go @@ -3,21 +3,36 @@ package e2e import ( "context" "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" catalogd "github.com/operator-framework/catalogd/api/core/v1alpha1" "github.com/operator-framework/operator-registry/alpha/declcfg" rukpakv1alpha1 "github.com/operator-framework/rukpak/api/v1alpha1" + "gopkg.in/yaml.v2" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/rand" + kubeclient "k8s.io/client-go/kubernetes" + "k8s.io/utils/env" + "sigs.k8s.io/controller-runtime/pkg/client" operatorv1alpha1 "github.com/operator-framework/operator-controller/api/v1alpha1" ) +const ( + artifactName = "operator-controller-e2e" +) + var _ = Describe("Operator Install", func() { var ( ctx context.Context @@ -212,6 +227,10 @@ var _ = Describe("Operator Install", func() { }) AfterEach(func() { + if basePath := env.GetString("ARTIFACT_PATH", ""); basePath != "" { + // get all the artifacts from the test run and save them to the artifact path + getArtifactsOutput(ctx, basePath) + } Expect(c.Delete(ctx, operator)).To(Succeed()) Eventually(func(g Gomega) { err := c.Get(ctx, types.NamespacedName{Name: operator.Name}, &operatorv1alpha1.Operator{}) @@ -221,3 +240,162 @@ var _ = Describe("Operator Install", func() { }) }) + +// getArtifactsOutput gets all the artifacts from the test run and saves them to the artifact path. +// Currently it saves: +// - operators +// - pods logs +// - deployments +// - bundle +// - bundledeployments +// - catalogsources +func getArtifactsOutput(ctx context.Context, basePath string) { + kubeClient, err := kubeclient.NewForConfig(cfg) + Expect(err).To(Not(HaveOccurred())) + + // sanitize the artifact name for use as a directory name + testName := strings.ReplaceAll(strings.ToLower(CurrentSpecReport().LeafNodeText), " ", "-") + // Get the test description and sanitize it for use as a directory name + artifactPath := filepath.Join(basePath, artifactName, fmt.Sprint(time.Now().UnixNano()), testName) + + // Create the full artifact path + err = os.MkdirAll(artifactPath, 0755) + Expect(err).To(Not(HaveOccurred())) + + // Get all namespaces + namespaces := corev1.NamespaceList{} + if err := c.List(ctx, &namespaces); err != nil { + GinkgoWriter.Printf("Failed to list namespaces %w", err) + } + + // get all operators save them to the artifact path. + operators := operatorv1alpha1.OperatorList{} + if err := c.List(ctx, &operators, client.InNamespace("")); err != nil { + GinkgoWriter.Printf("Failed to list operators %w", err) + } + for _, operator := range operators.Items { + // Save operator to artifact path + operatorYaml, err := yaml.Marshal(operator) + if err != nil { + GinkgoWriter.Printf("Failed to marshal operator %w", err) + continue + } + if err := os.WriteFile(filepath.Join(artifactPath, operator.Name+"-operator.yaml"), operatorYaml, 0600); err != nil { + GinkgoWriter.Printf("Failed to write operator to file %w", err) + } + } + + // get all catalogsources save them to the artifact path. + catalogsources := catalogd.CatalogList{} + if err := c.List(ctx, &catalogsources, client.InNamespace("")); err != nil { + GinkgoWriter.Printf("Failed to list catalogsources %w", err) + } + for _, catalogsource := range catalogsources.Items { + // Save catalogsource to artifact path + catalogsourceYaml, err := yaml.Marshal(catalogsource) + if err != nil { + GinkgoWriter.Printf("Failed to marshal catalogsource %w", err) + continue + } + if err := os.WriteFile(filepath.Join(artifactPath, catalogsource.Name+"-catalogsource.yaml"), catalogsourceYaml, 0600); err != nil { + GinkgoWriter.Printf("Failed to write catalogsource to file %w", err) + } + } + + // Get all Bundles in the namespace and save them to the artifact path. + bundles := rukpakv1alpha1.BundleList{} + if err := c.List(ctx, &bundles, client.InNamespace("")); err != nil { + GinkgoWriter.Printf("Failed to list bundles %w", err) + } + for _, bundle := range bundles.Items { + // Save bundle to artifact path + bundleYaml, err := yaml.Marshal(bundle) + if err != nil { + GinkgoWriter.Printf("Failed to marshal bundle %w", err) + continue + } + if err := os.WriteFile(filepath.Join(artifactPath, bundle.Name+"-bundle.yaml"), bundleYaml, 0600); err != nil { + GinkgoWriter.Printf("Failed to write bundle to file %w", err) + } + } + + // Get all BundleDeployments in the namespace and save them to the artifact path. + bundleDeployments := rukpakv1alpha1.BundleDeploymentList{} + if err := c.List(ctx, &bundleDeployments, client.InNamespace("")); err != nil { + GinkgoWriter.Printf("Failed to list bundleDeployments %w", err) + } + for _, bundleDeployment := range bundleDeployments.Items { + // Save bundleDeployment to artifact path + bundleDeploymentYaml, err := yaml.Marshal(bundleDeployment) + if err != nil { + GinkgoWriter.Printf("Failed to marshal bundleDeployment %w", err) + continue + } + if err := os.WriteFile(filepath.Join(artifactPath, bundleDeployment.Name+"-bundleDeployment.yaml"), bundleDeploymentYaml, 0600); err != nil { + GinkgoWriter.Printf("Failed to write bundleDeployment to file %w", err) + } + } + + for _, namespace := range namespaces.Items { + // let's ignore kube-* namespaces. + if strings.Contains(namespace.Name, "kube-") { + continue + } + + namespacedArtifactPath := filepath.Join(artifactPath, namespace.Name) + if err := os.Mkdir(namespacedArtifactPath, 0755); err != nil { + GinkgoWriter.Printf("Failed to create namespaced artifact path %w", err) + continue + } + + // get all deployments in the namespace and save them to the artifact path. + deployments := appsv1.DeploymentList{} + if err := c.List(ctx, &deployments, client.InNamespace(namespace.Name)); err != nil { + GinkgoWriter.Printf("Failed to list deployments %w in namespace: %q", err, namespace.Name) + continue + } + + for _, deployment := range deployments.Items { + // Save deployment to artifact path + deploymentYaml, err := yaml.Marshal(deployment) + if err != nil { + GinkgoWriter.Printf("Failed to marshal deployment %w", err) + continue + } + if err := os.WriteFile(filepath.Join(namespacedArtifactPath, deployment.Name+"-deployment.yaml"), deploymentYaml, 0600); err != nil { + GinkgoWriter.Printf("Failed to write deployment to file %w", err) + } + } + + // Get logs from all pods in all namespaces + pods := corev1.PodList{} + if err := c.List(ctx, &pods, client.InNamespace(namespace.Name)); err != nil { + GinkgoWriter.Printf("Failed to list pods %w in namespace: %q", err, namespace.Name) + } + for _, pod := range pods.Items { + if pod.Status.Phase != corev1.PodRunning && pod.Status.Phase != corev1.PodSucceeded && pod.Status.Phase != corev1.PodFailed { + continue + } + for _, container := range pod.Spec.Containers { + logs, err := kubeClient.CoreV1().Pods(namespace.Name).GetLogs(pod.Name, &corev1.PodLogOptions{Container: container.Name}).Stream(ctx) + if err != nil { + GinkgoWriter.Printf("Failed to get logs for pod %q in namespace %q: %w", pod.Name, namespace.Name, err) + continue + } + defer logs.Close() + + outFile, err := os.Create(filepath.Join(namespacedArtifactPath, pod.Name+"-"+container.Name+"-logs.txt")) + if err != nil { + GinkgoWriter.Printf("Failed to create file for pod %q in namespace %q: %w", pod.Name, namespace.Name, err) + continue + } + defer outFile.Close() + + if _, err := io.Copy(outFile, logs); err != nil { + GinkgoWriter.Printf("Failed to copy logs for pod %q in namespace %q: %w", pod.Name, namespace.Name, err) + continue + } + } + } + } +}