Testing: Save pod logs and resources after e2e test run (#234)

* Expose logs/resources after test run

Signed-off-by: Joaquim Moreno Prusi <[email protected]>

* CI: Generate e2e artifacts and upload

Signed-off-by: Joaquim Moreno Prusi <[email protected]>

---------

Signed-off-by: Joaquim Moreno Prusi <[email protected]>

.github/workflows/e2e.yaml

@@ -6,33 +6,38 @@ on:
   merge_group:
   push:
     branches:
-    - main
+      - main
 
 jobs:
   e2e-kind:
     runs-on: ubuntu-latest
     steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
 
-    - uses: actions/checkout@v4
-      with:
-        fetch-depth: 0
+      - uses: actions/setup-go@v4
+        with:
+          go-version-file: go.mod
 
-    - uses: actions/setup-go@v4
-      with:
-        go-version-file: go.mod
+      - name: Run e2e tests
+        run: |
+          # By default make stops building on first non-zero exit code which
+          # in case of E2E tests will mean that code coverage will only be
+          # collected on successful runs. We want to collect coverage even
+          # after failing tests.
+          # With -k flag make will continue the build, but will return non-zero
+          # exit code in case of any errors.
+          ARTIFACT_PATH=/tmp/artifacts make -k test-e2e
 
-    - name: Run e2e tests
-      run: |
-        # By default make stops building on first non-zero exit code which
-        # in case of E2E tests will mean that code coverage will only be
-        # collected on successful runs. We want to collect coverage even
-        # after failing tests.
-        # With -k flag make will continue the build, but will return non-zero
-        # exit code in case of any errors.
-        make -k test-e2e
+      - uses: cytopia/[email protected]
+        if: failure()
+        with:
+          name: e2e-artifacts
+          path: /tmp/artifacts/
 
-    - uses: codecov/codecov-action@v3
-      with:
-        files: e2e-cover.out
-        flags: e2e
-        functionalities: fixes
+      - uses: codecov/codecov-action@v3
+        with:
+          files: e2e-cover.out
+          flags: e2e
+          functionalities: fixes

Makefile

@@ -19,6 +19,10 @@ export XDG_DATA_HOME ?= /tmp/.local/share
 # bingo manages consistent tooling versions for things like kind, kustomize, etc.
 include .bingo/Variables.mk
 
+# ARTIFACT_PATH is the absolute path to the directory where the operator-controller e2e tests will store the artifacts
+# for example: ARTIFACT_PATH=/tmp/artifacts make test 
+export ARTIFACT_PATH ?= 
+
 OPERATOR_CONTROLLER_NAMESPACE ?= operator-controller-system
 KIND_CLUSTER_NAME ?= operator-controller
 

go.mod

@@ -17,6 +17,8 @@ require (
 	github.com/stretchr/testify v1.8.4
 	go.uber.org/zap v1.25.0
 	golang.org/x/exp v0.0.0-20230315142452-642cacee5cc0
+	gopkg.in/yaml.v2 v2.4.0
+	k8s.io/api v0.26.1
 	k8s.io/apiextensions-apiserver v0.26.1
 	k8s.io/apimachinery v0.26.1
 	k8s.io/client-go v0.26.1
@@ -134,9 +136,7 @@ require (
 	google.golang.org/protobuf v1.28.1 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/api v0.26.1 // indirect
 	k8s.io/apiserver v0.26.1 // indirect
 	k8s.io/klog/v2 v2.80.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 // indirect

test/e2e/e2e_suite_test.go

@@ -8,6 +8,9 @@ import (
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/utils/env"
 
 	"k8s.io/apimachinery/pkg/api/errors"
 	apimeta "k8s.io/apimachinery/pkg/api/meta"
@@ -61,6 +64,13 @@ var _ = BeforeSuite(func() {
 	Expect(catalogd.AddToScheme(scheme)).To(Succeed())
 
 	var err error
+
+	err = appsv1.AddToScheme(scheme)
+	Expect(err).ToNot(HaveOccurred())
+
+	err = corev1.AddToScheme(scheme)
+	Expect(err).ToNot(HaveOccurred())
+
 	c, err = client.New(cfg, client.Options{Scheme: scheme})
 	Expect(err).To(Not(HaveOccurred()))
 
@@ -82,7 +92,10 @@ var _ = BeforeSuite(func() {
 
 var _ = AfterSuite(func() {
 	ctx := context.Background()
-
+	if basePath := env.GetString("ARTIFACT_PATH", ""); basePath != "" {
+		// get all the artifacts from the test run and save them to the artifact path
+		getArtifactsOutput(ctx, basePath)
+	}
 	Expect(c.Delete(ctx, operatorCatalog)).To(Succeed())
 	Eventually(func(g Gomega) {
 		err := c.Get(ctx, types.NamespacedName{Name: operatorCatalog.Name}, &catalogd.Catalog{})

test/e2e/install_test.go

@@ -3,21 +3,36 @@ package e2e
 import (
 	"context"
 	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	catalogd "github.com/operator-framework/catalogd/api/core/v1alpha1"
 	"github.com/operator-framework/operator-registry/alpha/declcfg"
 	rukpakv1alpha1 "github.com/operator-framework/rukpak/api/v1alpha1"
+	"gopkg.in/yaml.v2"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	apimeta "k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/rand"
+	kubeclient "k8s.io/client-go/kubernetes"
+	"k8s.io/utils/env"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	operatorv1alpha1 "github.com/operator-framework/operator-controller/api/v1alpha1"
 )
 
+const (
+	artifactName = "operator-controller-e2e"
+)
+
 var _ = Describe("Operator Install", func() {
 	var (
 		ctx          context.Context
@@ -212,6 +227,10 @@ var _ = Describe("Operator Install", func() {
 		})
 
 		AfterEach(func() {
+			if basePath := env.GetString("ARTIFACT_PATH", ""); basePath != "" {
+				// get all the artifacts from the test run and save them to the artifact path
+				getArtifactsOutput(ctx, basePath)
+			}
 			Expect(c.Delete(ctx, operator)).To(Succeed())
 			Eventually(func(g Gomega) {
 				err := c.Get(ctx, types.NamespacedName{Name: operator.Name}, &operatorv1alpha1.Operator{})
@@ -221,3 +240,162 @@ var _ = Describe("Operator Install", func() {
 
 	})
 })
+
+// getArtifactsOutput gets all the artifacts from the test run and saves them to the artifact path.
+// Currently it saves:
+// - operators
+// - pods logs
+// - deployments
+// - bundle
+// - bundledeployments
+// - catalogsources
+func getArtifactsOutput(ctx context.Context, basePath string) {
+	kubeClient, err := kubeclient.NewForConfig(cfg)
+	Expect(err).To(Not(HaveOccurred()))
+
+	// sanitize the artifact name for use as a directory name
+	testName := strings.ReplaceAll(strings.ToLower(CurrentSpecReport().LeafNodeText), " ", "-")
+	// Get the test description and sanitize it for use as a directory name
+	artifactPath := filepath.Join(basePath, artifactName, fmt.Sprint(time.Now().UnixNano()), testName)
+
+	// Create the full artifact path
+	err = os.MkdirAll(artifactPath, 0755)
+	Expect(err).To(Not(HaveOccurred()))
+
+	// Get all namespaces
+	namespaces := corev1.NamespaceList{}
+	if err := c.List(ctx, &namespaces); err != nil {
+		GinkgoWriter.Printf("Failed to list namespaces %w", err)
+	}
+
+	// get all operators save them to the artifact path.
+	operators := operatorv1alpha1.OperatorList{}
+	if err := c.List(ctx, &operators, client.InNamespace("")); err != nil {
+		GinkgoWriter.Printf("Failed to list operators %w", err)
+	}
+	for _, operator := range operators.Items {
+		// Save operator to artifact path
+		operatorYaml, err := yaml.Marshal(operator)
+		if err != nil {
+			GinkgoWriter.Printf("Failed to marshal operator %w", err)
+			continue
+		}
+		if err := os.WriteFile(filepath.Join(artifactPath, operator.Name+"-operator.yaml"), operatorYaml, 0600); err != nil {
+			GinkgoWriter.Printf("Failed to write operator to file %w", err)
+		}
+	}
+
+	// get all catalogsources save them to the artifact path.
+	catalogsources := catalogd.CatalogList{}
+	if err := c.List(ctx, &catalogsources, client.InNamespace("")); err != nil {
+		GinkgoWriter.Printf("Failed to list catalogsources %w", err)
+	}
+	for _, catalogsource := range catalogsources.Items {
+		// Save catalogsource to artifact path
+		catalogsourceYaml, err := yaml.Marshal(catalogsource)
+		if err != nil {
+			GinkgoWriter.Printf("Failed to marshal catalogsource %w", err)
+			continue
+		}
+		if err := os.WriteFile(filepath.Join(artifactPath, catalogsource.Name+"-catalogsource.yaml"), catalogsourceYaml, 0600); err != nil {
+			GinkgoWriter.Printf("Failed to write catalogsource to file %w", err)
+		}
+	}
+
+	// Get all Bundles in the namespace and save them to the artifact path.
+	bundles := rukpakv1alpha1.BundleList{}
+	if err := c.List(ctx, &bundles, client.InNamespace("")); err != nil {
+		GinkgoWriter.Printf("Failed to list bundles %w", err)
+	}
+	for _, bundle := range bundles.Items {
+		// Save bundle to artifact path
+		bundleYaml, err := yaml.Marshal(bundle)
+		if err != nil {
+			GinkgoWriter.Printf("Failed to marshal bundle %w", err)
+			continue
+		}
+		if err := os.WriteFile(filepath.Join(artifactPath, bundle.Name+"-bundle.yaml"), bundleYaml, 0600); err != nil {
+			GinkgoWriter.Printf("Failed to write bundle to file %w", err)
+		}
+	}
+
+	// Get all BundleDeployments in the namespace and save them to the artifact path.
+	bundleDeployments := rukpakv1alpha1.BundleDeploymentList{}
+	if err := c.List(ctx, &bundleDeployments, client.InNamespace("")); err != nil {
+		GinkgoWriter.Printf("Failed to list bundleDeployments %w", err)
+	}
+	for _, bundleDeployment := range bundleDeployments.Items {
+		// Save bundleDeployment to artifact path
+		bundleDeploymentYaml, err := yaml.Marshal(bundleDeployment)
+		if err != nil {
+			GinkgoWriter.Printf("Failed to marshal bundleDeployment %w", err)
+			continue
+		}
+		if err := os.WriteFile(filepath.Join(artifactPath, bundleDeployment.Name+"-bundleDeployment.yaml"), bundleDeploymentYaml, 0600); err != nil {
+			GinkgoWriter.Printf("Failed to write bundleDeployment to file %w", err)
+		}
+	}
+
+	for _, namespace := range namespaces.Items {
+		// let's ignore kube-* namespaces.
+		if strings.Contains(namespace.Name, "kube-") {
+			continue
+		}
+
+		namespacedArtifactPath := filepath.Join(artifactPath, namespace.Name)
+		if err := os.Mkdir(namespacedArtifactPath, 0755); err != nil {
+			GinkgoWriter.Printf("Failed to create namespaced artifact path %w", err)
+			continue
+		}
+
+		// get all deployments in the namespace and save them to the artifact path.
+		deployments := appsv1.DeploymentList{}
+		if err := c.List(ctx, &deployments, client.InNamespace(namespace.Name)); err != nil {
+			GinkgoWriter.Printf("Failed to list deployments %w in namespace: %q", err, namespace.Name)
+			continue
+		}
+
+		for _, deployment := range deployments.Items {
+			// Save deployment to artifact path
+			deploymentYaml, err := yaml.Marshal(deployment)
+			if err != nil {
+				GinkgoWriter.Printf("Failed to marshal deployment %w", err)
+				continue
+			}
+			if err := os.WriteFile(filepath.Join(namespacedArtifactPath, deployment.Name+"-deployment.yaml"), deploymentYaml, 0600); err != nil {
+				GinkgoWriter.Printf("Failed to write deployment to file %w", err)
+			}
+		}
+
+		// Get logs from all pods in all namespaces
+		pods := corev1.PodList{}
+		if err := c.List(ctx, &pods, client.InNamespace(namespace.Name)); err != nil {
+			GinkgoWriter.Printf("Failed to list pods %w in namespace: %q", err, namespace.Name)
+		}
+		for _, pod := range pods.Items {
+			if pod.Status.Phase != corev1.PodRunning && pod.Status.Phase != corev1.PodSucceeded && pod.Status.Phase != corev1.PodFailed {
+				continue
+			}
+			for _, container := range pod.Spec.Containers {
+				logs, err := kubeClient.CoreV1().Pods(namespace.Name).GetLogs(pod.Name, &corev1.PodLogOptions{Container: container.Name}).Stream(ctx)
+				if err != nil {
+					GinkgoWriter.Printf("Failed to get logs for pod %q in namespace %q: %w", pod.Name, namespace.Name, err)
+					continue
+				}
+				defer logs.Close()
+
+				outFile, err := os.Create(filepath.Join(namespacedArtifactPath, pod.Name+"-"+container.Name+"-logs.txt"))
+				if err != nil {
+					GinkgoWriter.Printf("Failed to create file for pod %q in namespace %q: %w", pod.Name, namespace.Name, err)
+					continue
+				}
+				defer outFile.Close()
+
+				if _, err := io.Copy(outFile, logs); err != nil {
+					GinkgoWriter.Printf("Failed to copy logs for pod %q in namespace %q: %w", pod.Name, namespace.Name, err)
+					continue
+				}
+			}
+		}
+	}
+}