Collect K8S events and pod configuration on K8S based tests

integration-tests/README.md

@@ -82,3 +82,56 @@ COLLECTOR_BCC_COMMAND='syscount --latency' make benchmark
 # Record perf events, writing /tmp
 COLLECTOR_PERF_COMMAND='record -o /tmp/perf.data' make benchmark
 ```
+
+## Useful jq queries for K8S based log files
+The K8S based tests that run on KinD are able to dump events for namespaces
+and configuration used by pods involved in tests for later inspection. However,
+these dumps are done in JSON files and can be quite bulky, so using `jq` for
+narrowing down data and formatting is highly recommended.
+
+For reference and other use cases not covered here, please refer to the jq
+manual: https://jqlang.github.io/jq/manual/
+
+### Get the container configuration from a pod
+```sh
+$ jq '.spec.containers[]' TestK8sNamespace-collector-tests-collector-config.json
+{
+  "name": "collector",
+  "image": "quay.io/stackrox-io/collector:3.18.x-146-g3538c194be-dirty",
+  "ports": [
+    {
+      "containerPort": 8080,
+      "protocol": "TCP"
+    }
+  ],
+  "env": [
+    {
+      "name": "GRPC_SERVER",
+      "value": "tester-svc:9999"
+    },
+...
+```
+
+### Get all events from the collector pod
+```sh
+$ jq 'select(.involvedObject.name == "collector")' core-bpf/TestK8sNamespace-collector-tests-events.jsonl
+{
+  "metadata": {
+    "name": "collector.17c772d099db6bb6",
+    "namespace": "collector-tests",
+    "uid": "f467c96e-b8c1-4ef2-8505-773fe711af92",
+    "resourceVersion": "475",
+    "creationTimestamp": "2024-04-18T18:20:23Z",
+    "managedFields": [
+      {
+        "manager": "kube-scheduler",
+        "operation": "Update",
+        "apiVersion": "v1",
+        "time": "2024-04-18T18:20:23Z",
+        "fieldsType": "FieldsV1",
+        "fieldsV1": {
+          "f:count": {},
+          "f:firstTimestamp": {},
+          "f:involvedObject": {},
+...
+```

integration-tests/pkg/collector/collector.go

@@ -1,11 +1,6 @@
 package collector
 
 import (
-	"os"
-	"path/filepath"
-	"strings"
-
-	"github.com/stackrox/collector/integration-tests/pkg/config"
 	"github.com/stackrox/collector/integration-tests/pkg/executor"
 )
 
@@ -32,14 +27,3 @@ func New(e executor.Executor, name string) Manager {
 	}
 	return newDockerManager(e, name)
 }
-
-func prepareLog(m Manager) (*os.File, error) {
-	logDirectory := filepath.Join(".", "container-logs", config.VMInfo().Config, config.CollectionMethod())
-	err := os.MkdirAll(logDirectory, os.ModePerm)
-	if err != nil {
-		return nil, err
-	}
-
-	logPath := filepath.Join(logDirectory, strings.ReplaceAll(m.TestName(), "/", "_")+"-collector.log")
-	return os.Create(logPath)
-}

integration-tests/pkg/collector/collector_docker.go

@@ -190,7 +190,7 @@ func (c *DockerCollectorManager) captureLogs(containerName string) (string, erro
 		return "", err
 	}
 
-	logFile, err := prepareLog(c)
+	logFile, err := common.PrepareLog(c.testName, "collector.log")
 	if err != nil {
 		return "", err
 	}

integration-tests/pkg/collector/collector_k8s.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io"
 
+	"github.com/stackrox/collector/integration-tests/pkg/common"
 	"github.com/stackrox/collector/integration-tests/pkg/config"
 	"github.com/stackrox/collector/integration-tests/pkg/executor"
 	"golang.org/x/exp/maps"
@@ -197,14 +198,29 @@ func replaceOrAppendEnvVar(list []coreV1.EnvVar, newVar coreV1.EnvVar) []coreV1.
 }
 
 func (k *K8sCollectorManager) captureLogs() error {
+	err := k.capturePodLogs()
+	if err != nil {
+		return err
+	}
+
+	err = k.capturePodConfiguration()
+	if err != nil {
+		return err
+	}
+
+	err = k.captureNamespaceEvents()
+	return err
+}
+
+func (k *K8sCollectorManager) capturePodLogs() error {
 	req := k.executor.ClientSet().CoreV1().Pods(TEST_NAMESPACE).GetLogs("collector", &coreV1.PodLogOptions{})
 	podLogs, err := req.Stream(context.Background())
 	if err != nil {
 		return err
 	}
 	defer podLogs.Close()
 
-	logFile, err := prepareLog(k)
+	logFile, err := common.PrepareLog(k.testName, "collector.log")
 	if err != nil {
 		return err
 	}
@@ -213,3 +229,11 @@ func (k *K8sCollectorManager) captureLogs() error {
 	_, err = io.Copy(logFile, podLogs)
 	return err
 }
+
+func (k *K8sCollectorManager) capturePodConfiguration() error {
+	return k.executor.CapturePodConfiguration(k.testName, TEST_NAMESPACE, "collector")
+}
+
+func (k *K8sCollectorManager) captureNamespaceEvents() error {
+	return k.executor.CaptureNamespaceEvents(k.testName, TEST_NAMESPACE)
+}

integration-tests/pkg/common/utils.go

@@ -2,9 +2,12 @@ package common
 
 import (
 	"bytes"
+	"os"
+	"path/filepath"
 	"strconv"
 	"strings"
 
+	"github.com/stackrox/collector/integration-tests/pkg/config"
 	"golang.org/x/sys/unix"
 	"k8s.io/utils/strings/slices"
 )
@@ -49,3 +52,15 @@ func ArchSupported(supported ...string) (bool, string) {
 	arch := string(bytes.Trim(u.Machine[:], "\x00"))
 	return slices.Contains(supported, arch), arch
 }
+
+// Creates a new file to dump logs into
+func PrepareLog(testName string, logName string) (*os.File, error) {
+	logDirectory := filepath.Join(".", "container-logs", config.VMInfo().Config, config.CollectionMethod())
+	err := os.MkdirAll(logDirectory, os.ModePerm)
+	if err != nil {
+		return nil, err
+	}
+
+	logPath := filepath.Join(logDirectory, strings.ReplaceAll(testName, "/", "_")+"-"+logName)
+	return os.Create(logPath)
+}

integration-tests/pkg/executor/executor_k8s.go

@@ -2,6 +2,7 @@ package executor
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"strings"
 
@@ -172,3 +173,55 @@ func (e *K8sExecutor) CreatePod(ns string, pod *coreV1.Pod) (*coreV1.Pod, error)
 func (e *K8sExecutor) ClientSet() *kubernetes.Clientset {
 	return e.clientset
 }
+
+func (e *K8sExecutor) CapturePodConfiguration(testName, ns, podName string) error {
+	pod, err := e.clientset.CoreV1().Pods(ns).Get(context.Background(), podName, metaV1.GetOptions{})
+	if err != nil {
+		return err
+	}
+
+	logFile, err := common.PrepareLog(testName, ns+"-"+podName+"-config.json")
+	if err != nil {
+		return err
+	}
+	defer logFile.Close()
+
+	podJson, err := json.Marshal(pod)
+	if err != nil {
+		return err
+	}
+
+	_, err = logFile.Write(podJson)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (e *K8sExecutor) CaptureNamespaceEvents(testName, ns string) error {
+	events, err := e.clientset.CoreV1().Events(ns).List(context.Background(), metaV1.ListOptions{})
+	if err != nil {
+		return err
+	}
+
+	logFile, err := common.PrepareLog(testName, ns+"-events.jsonl")
+	if err != nil {
+		return err
+	}
+	defer logFile.Close()
+
+	for _, event := range events.Items {
+		eventJson, err := json.Marshal(event)
+		if err != nil {
+			return err
+		}
+
+		_, err = logFile.WriteString(string(eventJson) + "\n")
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

integration-tests/suites/k8s_namespace.go

@@ -51,13 +51,21 @@ func (k *K8sNamespaceTestSuite) SetupSuite() {
 			k.sensor.Stop()
 		}
 
+		// Dump logs for the target namespace
+		e, ok := k.Executor().(*executor.K8sExecutor)
+		k.Require().True(ok)
+		err = e.CaptureNamespaceEvents(k.collector.TestName(), NAMESPACE)
+		k.Require().NoError(err)
+
 		nginxPodFilter := executor.ContainerFilter{
 			Name:      "nginx",
 			Namespace: NAMESPACE,
 		}
 		exists, _ = k.Executor().ContainerExists(nginxPodFilter)
 
 		if exists {
+			err = e.CapturePodConfiguration(k.collector.TestName(), NAMESPACE, "nginx")
+			k.Require().NoError(err)
 			k.Executor().RemoveContainer(nginxPodFilter)
 		}