From 45537f3e50c68ef7664a5a44054d04f8388717be Mon Sep 17 00:00:00 2001 From: Ryan Old Date: Fri, 22 Nov 2024 13:15:43 -0800 Subject: [PATCH 01/22] Enable stats in common recommendations --- test/kubernetes/e2e/tests/manifests/common-recommendations.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/kubernetes/e2e/tests/manifests/common-recommendations.yaml b/test/kubernetes/e2e/tests/manifests/common-recommendations.yaml index 2529898d53a..d3d707f480b 100644 --- a/test/kubernetes/e2e/tests/manifests/common-recommendations.yaml +++ b/test/kubernetes/e2e/tests/manifests/common-recommendations.yaml @@ -74,6 +74,8 @@ gloo: limits: cpu: 1000m memory: 10Gi + stats: + enabled: true # enable stats server for gloo so we can collect the metrics in CI # Configuration for the statically deployed gateway-proxy that ships by default with Gloo Gateway gatewayProxies: From ff88db6454d48c7697e0fd7d1bb71bb30ac555f9 Mon Sep 17 00:00:00 2001 From: Ryan Old Date: Fri, 22 Nov 2024 17:29:56 -0800 Subject: [PATCH 02/22] Fetch and save Gloo metrics and some snapshots on failure --- .../virtualhost_options/vhost_opt_suite.go | 2 + test/kubernetes/e2e/test.go | 54 +++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/test/kubernetes/e2e/features/virtualhost_options/vhost_opt_suite.go b/test/kubernetes/e2e/features/virtualhost_options/vhost_opt_suite.go index bf0f3bb0fc4..d7fd61e838f 100644 --- a/test/kubernetes/e2e/features/virtualhost_options/vhost_opt_suite.go +++ b/test/kubernetes/e2e/features/virtualhost_options/vhost_opt_suite.go @@ -225,6 +225,8 @@ func (s *testingSuite) TestConfigureVirtualHostOptionsWithSectionNameManualSetup []string{"conflict with more specific or older VirtualHostOptions"}, defaults.KubeGatewayReporter, ) + + s.Assert().Equal(true, false, "intentionally failing to trigger drump, remove when done debugging") } // The goal here is to test the behavior when multiple VHOs are targeting a gateway without sectionName. The expected diff --git a/test/kubernetes/e2e/test.go b/test/kubernetes/e2e/test.go index 7690d5cb474..bf800c10363 100644 --- a/test/kubernetes/e2e/test.go +++ b/test/kubernetes/e2e/test.go @@ -1,6 +1,7 @@ package e2e import ( + "bytes" "context" "errors" "fmt" @@ -314,6 +315,59 @@ func (i *TestInstallation) PreFailHandler(ctx context.Context) { kubectlGetResourcesCmd := i.Actions.Kubectl().Command(ctx, "get", strings.Join(resourcesToGet, ","), "-A", "-owide") _ = kubectlGetResourcesCmd.WithStdout(clusterStateFile).WithStderr(clusterStateFile).Run() clusterStateFile.WriteString("\n") + + podStdOut := bytes.NewBuffer(nil) + podStdErr := bytes.NewBuffer(nil) + + // Fetch the name of the Gloo Gateway controller pod + getGlooPodNameCmd := i.Actions.Kubectl().Command(ctx, "get", "pod", "-n", i.Metadata.InstallNamespace, + "--selector", "gloo=gloo", "--output", "jsonpath='{.items[0].metadata.name}'") + _ = getGlooPodNameCmd.WithStdout(podStdOut).WithStderr(podStdErr).Run() + + // Clean up and check the output + glooPodName := strings.Trim(podStdOut.String(), "'") + if glooPodName == "" { + fmt.Printf("Failed to get the name of the Gloo Gateway controller pod: %s\n", podStdErr.String()) + return + } + + // Get the metrics from the Gloo Gateway controller pod and write them to a file + metricsFilePath := filepath.Join(failureDir, "metrics.log") + metricsFile, err := os.OpenFile(metricsFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, os.ModePerm) + i.Assertions.Require.NoError(err) + + // Using an ephemeral debug pod fetch the metrics from the Gloo Gateway controller + metricsCmd := i.Actions.Kubectl().Command(ctx, "debug", "-n", i.Metadata.InstallNamespace, + "-it", "--image=curlimages/curl:7.83.1", glooPodName, "--", + "curl", "http://localhost:9091/metrics") + _ = metricsCmd.WithStdout(metricsFile).WithStderr(metricsFile).Run() + metricsFile.Close() + + // Get krt snapshot from the Gloo Gateway controller pod and write it to a file + krtSnapshotFilePath := filepath.Join(failureDir, "krt_snapshot.log") + krtSnapshotFile, err := os.OpenFile(krtSnapshotFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, os.ModePerm) + i.Assertions.Require.NoError(err) + + // Using an ephemeral debug pod fetch the krt snapshot from the Gloo Gateway controller + krtSnapshotCmd := i.Actions.Kubectl().Command(ctx, "debug", "-n", i.Metadata.InstallNamespace, + "-it", "--image=curlimages/curl:7.83.1", glooPodName, "--", + "curl", "http://localhost:9095/snapshots/krt") + _ = krtSnapshotCmd.WithStdout(krtSnapshotFile).WithStderr(krtSnapshotFile).Run() + krtSnapshotFile.Close() + + // Get xds snapshot from the Gloo Gateway controller pod and write it to a file + xdsSnapshotFilePath := filepath.Join(failureDir, "xds_snapshot.log") + xdsSnapshotFile, err := os.OpenFile(xdsSnapshotFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, os.ModePerm) + i.Assertions.Require.NoError(err) + + // Using an ephemeral debug pod fetch the xds snapshot from the Gloo Gateway controller + xdsSnapshotCmd := i.Actions.Kubectl().Command(ctx, "debug", "-n", i.Metadata.InstallNamespace, + "-it", "--image=curlimages/curl:7.83.1", glooPodName, "--", + "curl", "http://localhost:9095/snapshots/xds") + _ = xdsSnapshotCmd.WithStdout(xdsSnapshotFile).WithStderr(xdsSnapshotFile).Run() + xdsSnapshotFile.Close() + + fmt.Printf("Test failed. Logs and cluster state are available in %s\n", failureDir) } // GeneratedFiles is a collection of files that are generated during the execution of a set of tests From e2b43ec2db5129ee1e39afcdf269a793770837d8 Mon Sep 17 00:00:00 2001 From: Ryan Old Date: Fri, 22 Nov 2024 17:41:57 -0800 Subject: [PATCH 03/22] Added changelog --- .../collect-more-artifacts-on-ci-failure.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 changelog/v1.18.0-rc2/collect-more-artifacts-on-ci-failure.yaml diff --git a/changelog/v1.18.0-rc2/collect-more-artifacts-on-ci-failure.yaml b/changelog/v1.18.0-rc2/collect-more-artifacts-on-ci-failure.yaml new file mode 100644 index 00000000000..6e1cb3b66c0 --- /dev/null +++ b/changelog/v1.18.0-rc2/collect-more-artifacts-on-ci-failure.yaml @@ -0,0 +1,9 @@ +changelog: + - type: NON_USER_FACING + description: >- + Gloo Gateway controller metrics and xds/krt snaphots are now collected and included + the test failure artifacts. + After encountering some test failures that proved difficult to debug without knowing more + about the state of the cluster, we have added additional artifacts to be collected when + a test fails. + This will help us to more easily diagnose the cause of test failures. From 31d5f5b67f5422ec63bd17e13ae7822887efc854 Mon Sep 17 00:00:00 2001 From: changelog-bot Date: Mon, 25 Nov 2024 18:00:06 +0000 Subject: [PATCH 04/22] Adding changelog file to new location --- .../collect-more-artifacts-on-ci-failure.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 changelog/v1.18.0-rc3/collect-more-artifacts-on-ci-failure.yaml diff --git a/changelog/v1.18.0-rc3/collect-more-artifacts-on-ci-failure.yaml b/changelog/v1.18.0-rc3/collect-more-artifacts-on-ci-failure.yaml new file mode 100644 index 00000000000..6e1cb3b66c0 --- /dev/null +++ b/changelog/v1.18.0-rc3/collect-more-artifacts-on-ci-failure.yaml @@ -0,0 +1,9 @@ +changelog: + - type: NON_USER_FACING + description: >- + Gloo Gateway controller metrics and xds/krt snaphots are now collected and included + the test failure artifacts. + After encountering some test failures that proved difficult to debug without knowing more + about the state of the cluster, we have added additional artifacts to be collected when + a test fails. + This will help us to more easily diagnose the cause of test failures. From ba62682db51d7f6097c19ee5d49e6f1a222cd923 Mon Sep 17 00:00:00 2001 From: changelog-bot Date: Mon, 25 Nov 2024 18:00:06 +0000 Subject: [PATCH 05/22] Deleting changelog file from old location --- .../collect-more-artifacts-on-ci-failure.yaml | 9 --------- 1 file changed, 9 deletions(-) delete mode 100644 changelog/v1.18.0-rc2/collect-more-artifacts-on-ci-failure.yaml diff --git a/changelog/v1.18.0-rc2/collect-more-artifacts-on-ci-failure.yaml b/changelog/v1.18.0-rc2/collect-more-artifacts-on-ci-failure.yaml deleted file mode 100644 index 6e1cb3b66c0..00000000000 --- a/changelog/v1.18.0-rc2/collect-more-artifacts-on-ci-failure.yaml +++ /dev/null @@ -1,9 +0,0 @@ -changelog: - - type: NON_USER_FACING - description: >- - Gloo Gateway controller metrics and xds/krt snaphots are now collected and included - the test failure artifacts. - After encountering some test failures that proved difficult to debug without knowing more - about the state of the cluster, we have added additional artifacts to be collected when - a test fails. - This will help us to more easily diagnose the cause of test failures. From 2cbdc1e7fc1a1251689596272e1f3a5fe2aa8977 Mon Sep 17 00:00:00 2001 From: Ryan Old Date: Mon, 25 Nov 2024 10:47:28 -0800 Subject: [PATCH 06/22] Move to using glood admin cli package --- pkg/utils/glooadminutils/admincli/client.go | 12 ++++++ test/kubernetes/e2e/test.go | 47 +++++++++++++++------ 2 files changed, 47 insertions(+), 12 deletions(-) diff --git a/pkg/utils/glooadminutils/admincli/client.go b/pkg/utils/glooadminutils/admincli/client.go index f49d994082d..7eee99091fa 100644 --- a/pkg/utils/glooadminutils/admincli/client.go +++ b/pkg/utils/glooadminutils/admincli/client.go @@ -14,6 +14,8 @@ import ( const ( InputSnapshotPath = "/snapshots/input" + xdsSnapshotPath = "/snapshots/xds" + krtSnapshotPath = "/snapshots/krt" ) // Client is a utility for executing requests against the Gloo Admin API @@ -84,6 +86,16 @@ func (c *Client) InputSnapshotCmd(ctx context.Context) cmdutils.Cmd { return c.Command(ctx, curl.WithPath(InputSnapshotPath)) } +// XdsSnapshotCmd returns the cmdutils.Cmd that can be run, and will execute a request against the XDS Snapshot path +func (c *Client) XdsSnapshotCmd(ctx context.Context) cmdutils.Cmd { + return c.Command(ctx, curl.WithPath(xdsSnapshotPath)) +} + +// KrtSnapshotCmd returns the cmdutils.Cmd that can be run, and will execute a request against the KRT Snapshot path +func (c *Client) KrtSnapshotCmd(ctx context.Context) cmdutils.Cmd { + return c.Command(ctx, curl.WithPath(krtSnapshotPath)) +} + // GetInputSnapshot returns the data that is available at the input snapshot endpoint func (c *Client) GetInputSnapshot(ctx context.Context) ([]interface{}, error) { var outLocation threadsafe.Buffer diff --git a/test/kubernetes/e2e/test.go b/test/kubernetes/e2e/test.go index bf800c10363..1b12367e51c 100644 --- a/test/kubernetes/e2e/test.go +++ b/test/kubernetes/e2e/test.go @@ -5,6 +5,7 @@ import ( "context" "errors" "fmt" + "io" "io/fs" "os" "path/filepath" @@ -13,6 +14,10 @@ import ( "testing" "time" + "github.com/solo-io/gloo/pkg/utils/glooadminutils/admincli" + "github.com/solo-io/gloo/pkg/utils/kubeutils/portforward" + "github.com/solo-io/gloo/pkg/utils/requestutils/curl" + "github.com/solo-io/gloo/projects/gloo/pkg/servers/admin" "github.com/solo-io/gloo/test/kubernetes/testutils/actions" "github.com/solo-io/gloo/test/kubernetes/testutils/assertions" "github.com/solo-io/gloo/test/kubernetes/testutils/cluster" @@ -343,29 +348,47 @@ func (i *TestInstallation) PreFailHandler(ctx context.Context) { _ = metricsCmd.WithStdout(metricsFile).WithStderr(metricsFile).Run() metricsFile.Close() + // Open a port-forward to the Gloo Gateway controller pod's admin port + portForwarder, err := i.Actions.Kubectl().StartPortForward(ctx, + portforward.WithDeployment("gloo", i.Metadata.InstallNamespace), + portforward.WithPorts(int(admin.AdminPort), int(admin.AdminPort)), + ) + if err != nil { + fmt.Printf("Failed to open port-forward: %v\n", err) + return + } + + defer func() { + portForwarder.Close() + portForwarder.WaitForStop() + }() + + adminClient := admincli.NewClient(). + WithReceiver(io.Discard). // adminAssertion can overwrite this + WithCurlOptions( + curl.WithRetries(3, 0, 10), + curl.WithPort(int(admin.AdminPort)), + ) + // Get krt snapshot from the Gloo Gateway controller pod and write it to a file krtSnapshotFilePath := filepath.Join(failureDir, "krt_snapshot.log") krtSnapshotFile, err := os.OpenFile(krtSnapshotFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, os.ModePerm) i.Assertions.Require.NoError(err) - // Using an ephemeral debug pod fetch the krt snapshot from the Gloo Gateway controller - krtSnapshotCmd := i.Actions.Kubectl().Command(ctx, "debug", "-n", i.Metadata.InstallNamespace, - "-it", "--image=curlimages/curl:7.83.1", glooPodName, "--", - "curl", "http://localhost:9095/snapshots/krt") - _ = krtSnapshotCmd.WithStdout(krtSnapshotFile).WithStderr(krtSnapshotFile).Run() - krtSnapshotFile.Close() + adminClient.KrtSnapshotCmd(ctx). + WithStdout(krtSnapshotFile). + WithStderr(krtSnapshotFile). + Run() // Get xds snapshot from the Gloo Gateway controller pod and write it to a file xdsSnapshotFilePath := filepath.Join(failureDir, "xds_snapshot.log") xdsSnapshotFile, err := os.OpenFile(xdsSnapshotFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, os.ModePerm) i.Assertions.Require.NoError(err) - // Using an ephemeral debug pod fetch the xds snapshot from the Gloo Gateway controller - xdsSnapshotCmd := i.Actions.Kubectl().Command(ctx, "debug", "-n", i.Metadata.InstallNamespace, - "-it", "--image=curlimages/curl:7.83.1", glooPodName, "--", - "curl", "http://localhost:9095/snapshots/xds") - _ = xdsSnapshotCmd.WithStdout(xdsSnapshotFile).WithStderr(xdsSnapshotFile).Run() - xdsSnapshotFile.Close() + adminClient.XdsSnapshotCmd(ctx). + WithStdout(xdsSnapshotFile). + WithStderr(xdsSnapshotFile). + Run() fmt.Printf("Test failed. Logs and cluster state are available in %s\n", failureDir) } From 788c36f661bd6def1509e6265e6bb1180b9a851b Mon Sep 17 00:00:00 2001 From: Ryan Old Date: Mon, 25 Nov 2024 11:00:24 -0800 Subject: [PATCH 07/22] Adjusting error handling --- test/kubernetes/e2e/test.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/test/kubernetes/e2e/test.go b/test/kubernetes/e2e/test.go index 1b12367e51c..5a4467b7819 100644 --- a/test/kubernetes/e2e/test.go +++ b/test/kubernetes/e2e/test.go @@ -353,10 +353,7 @@ func (i *TestInstallation) PreFailHandler(ctx context.Context) { portforward.WithDeployment("gloo", i.Metadata.InstallNamespace), portforward.WithPorts(int(admin.AdminPort), int(admin.AdminPort)), ) - if err != nil { - fmt.Printf("Failed to open port-forward: %v\n", err) - return - } + i.Assertions.Require.NoError(err) defer func() { portForwarder.Close() @@ -375,20 +372,22 @@ func (i *TestInstallation) PreFailHandler(ctx context.Context) { krtSnapshotFile, err := os.OpenFile(krtSnapshotFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, os.ModePerm) i.Assertions.Require.NoError(err) - adminClient.KrtSnapshotCmd(ctx). + cmdErr := adminClient.KrtSnapshotCmd(ctx). WithStdout(krtSnapshotFile). WithStderr(krtSnapshotFile). Run() + i.Assertions.Require.NoError(cmdErr) // Get xds snapshot from the Gloo Gateway controller pod and write it to a file xdsSnapshotFilePath := filepath.Join(failureDir, "xds_snapshot.log") xdsSnapshotFile, err := os.OpenFile(xdsSnapshotFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, os.ModePerm) i.Assertions.Require.NoError(err) - adminClient.XdsSnapshotCmd(ctx). + cmdErr = adminClient.XdsSnapshotCmd(ctx). WithStdout(xdsSnapshotFile). WithStderr(xdsSnapshotFile). Run() + i.Assertions.Require.NoError(cmdErr) fmt.Printf("Test failed. Logs and cluster state are available in %s\n", failureDir) } From 1e353f25012acb6e5108365bd74ca3b9081ef1fa Mon Sep 17 00:00:00 2001 From: Ryan Old Date: Mon, 25 Nov 2024 11:01:24 -0800 Subject: [PATCH 08/22] Adjusting error handling --- test/kubernetes/e2e/test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/kubernetes/e2e/test.go b/test/kubernetes/e2e/test.go index 5a4467b7819..a90cc18d897 100644 --- a/test/kubernetes/e2e/test.go +++ b/test/kubernetes/e2e/test.go @@ -332,8 +332,7 @@ func (i *TestInstallation) PreFailHandler(ctx context.Context) { // Clean up and check the output glooPodName := strings.Trim(podStdOut.String(), "'") if glooPodName == "" { - fmt.Printf("Failed to get the name of the Gloo Gateway controller pod: %s\n", podStdErr.String()) - return + i.Assertions.Require.NoError(fmt.Errorf("failed to get the Gloo Gateway controller pod name: %s", podStdErr.String())) } // Get the metrics from the Gloo Gateway controller pod and write them to a file From f40c5a00f3254a0b8680fc6ee96250ce63f29eab Mon Sep 17 00:00:00 2001 From: Ryan Old Date: Mon, 25 Nov 2024 11:02:44 -0800 Subject: [PATCH 09/22] Adjusting error handling --- test/kubernetes/e2e/test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/kubernetes/e2e/test.go b/test/kubernetes/e2e/test.go index a90cc18d897..97870fb9525 100644 --- a/test/kubernetes/e2e/test.go +++ b/test/kubernetes/e2e/test.go @@ -327,7 +327,8 @@ func (i *TestInstallation) PreFailHandler(ctx context.Context) { // Fetch the name of the Gloo Gateway controller pod getGlooPodNameCmd := i.Actions.Kubectl().Command(ctx, "get", "pod", "-n", i.Metadata.InstallNamespace, "--selector", "gloo=gloo", "--output", "jsonpath='{.items[0].metadata.name}'") - _ = getGlooPodNameCmd.WithStdout(podStdOut).WithStderr(podStdErr).Run() + cmdErr := getGlooPodNameCmd.WithStdout(podStdOut).WithStderr(podStdErr).Run() + i.Assertions.Require.NoError(cmdErr) // Clean up and check the output glooPodName := strings.Trim(podStdOut.String(), "'") From a32eac9ea4be35c02af5fd3b9649fa7615f0bf85 Mon Sep 17 00:00:00 2001 From: Ryan Old Date: Mon, 25 Nov 2024 11:35:50 -0800 Subject: [PATCH 10/22] Fix error check failing --- test/kubernetes/e2e/test.go | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/test/kubernetes/e2e/test.go b/test/kubernetes/e2e/test.go index 97870fb9525..e821a9950a6 100644 --- a/test/kubernetes/e2e/test.go +++ b/test/kubernetes/e2e/test.go @@ -328,12 +328,15 @@ func (i *TestInstallation) PreFailHandler(ctx context.Context) { getGlooPodNameCmd := i.Actions.Kubectl().Command(ctx, "get", "pod", "-n", i.Metadata.InstallNamespace, "--selector", "gloo=gloo", "--output", "jsonpath='{.items[0].metadata.name}'") cmdErr := getGlooPodNameCmd.WithStdout(podStdOut).WithStderr(podStdErr).Run() - i.Assertions.Require.NoError(cmdErr) + if cmdErr != nil { + i.Assertions.Require.NoError(cmdErr) + } // Clean up and check the output glooPodName := strings.Trim(podStdOut.String(), "'") if glooPodName == "" { - i.Assertions.Require.NoError(fmt.Errorf("failed to get the Gloo Gateway controller pod name: %s", podStdErr.String())) + i.Assertions.Require.NoError(fmt.Errorf("failed to get the Gloo Gateway controller pod name: %s", + podStdErr.String())) } // Get the metrics from the Gloo Gateway controller pod and write them to a file @@ -345,8 +348,10 @@ func (i *TestInstallation) PreFailHandler(ctx context.Context) { metricsCmd := i.Actions.Kubectl().Command(ctx, "debug", "-n", i.Metadata.InstallNamespace, "-it", "--image=curlimages/curl:7.83.1", glooPodName, "--", "curl", "http://localhost:9091/metrics") - _ = metricsCmd.WithStdout(metricsFile).WithStderr(metricsFile).Run() - metricsFile.Close() + cmdErr = metricsCmd.WithStdout(metricsFile).WithStderr(metricsFile).Run() + if cmdErr != nil { + i.Assertions.Require.NoError(cmdErr) + } // Open a port-forward to the Gloo Gateway controller pod's admin port portForwarder, err := i.Actions.Kubectl().StartPortForward(ctx, @@ -372,11 +377,13 @@ func (i *TestInstallation) PreFailHandler(ctx context.Context) { krtSnapshotFile, err := os.OpenFile(krtSnapshotFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, os.ModePerm) i.Assertions.Require.NoError(err) - cmdErr := adminClient.KrtSnapshotCmd(ctx). + cmdErr = adminClient.KrtSnapshotCmd(ctx). WithStdout(krtSnapshotFile). WithStderr(krtSnapshotFile). Run() - i.Assertions.Require.NoError(cmdErr) + if cmdErr != nil { + i.Assertions.Require.NoError(cmdErr) + } // Get xds snapshot from the Gloo Gateway controller pod and write it to a file xdsSnapshotFilePath := filepath.Join(failureDir, "xds_snapshot.log") @@ -387,7 +394,9 @@ func (i *TestInstallation) PreFailHandler(ctx context.Context) { WithStdout(xdsSnapshotFile). WithStderr(xdsSnapshotFile). Run() - i.Assertions.Require.NoError(cmdErr) + if cmdErr != nil { + i.Assertions.Require.NoError(cmdErr) + } fmt.Printf("Test failed. Logs and cluster state are available in %s\n", failureDir) } From 1b100f56984814d6b01c3c6766e966761ace934d Mon Sep 17 00:00:00 2001 From: Ryan Old Date: Mon, 25 Nov 2024 11:37:33 -0800 Subject: [PATCH 11/22] Removed intentional failure --- .../e2e/features/virtualhost_options/vhost_opt_suite.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/kubernetes/e2e/features/virtualhost_options/vhost_opt_suite.go b/test/kubernetes/e2e/features/virtualhost_options/vhost_opt_suite.go index d7fd61e838f..bf0f3bb0fc4 100644 --- a/test/kubernetes/e2e/features/virtualhost_options/vhost_opt_suite.go +++ b/test/kubernetes/e2e/features/virtualhost_options/vhost_opt_suite.go @@ -225,8 +225,6 @@ func (s *testingSuite) TestConfigureVirtualHostOptionsWithSectionNameManualSetup []string{"conflict with more specific or older VirtualHostOptions"}, defaults.KubeGatewayReporter, ) - - s.Assert().Equal(true, false, "intentionally failing to trigger drump, remove when done debugging") } // The goal here is to test the behavior when multiple VHOs are targeting a gateway without sectionName. The expected From 11d433616386433c5cda90407824a280acb8c815 Mon Sep 17 00:00:00 2001 From: Ryan Old Date: Tue, 26 Nov 2024 16:08:50 -0800 Subject: [PATCH 12/22] Partial work unifying the dump logic --- test/helpers/kube_dump.go | 215 +++++++++++++++--- test/kube2e/gateway/gateway_suite_test.go | 6 +- test/kube2e/gloo/gloo_suite_test.go | 6 +- test/kube2e/upgrade/upgrade_suite_test.go | 3 +- .../virtualhost_options/vhost_opt_suite.go | 2 + test/kubernetes/e2e/test.go | 192 ++++------------ 6 files changed, 238 insertions(+), 186 deletions(-) diff --git a/test/helpers/kube_dump.go b/test/helpers/kube_dump.go index b04b9d88fac..059e88486ba 100644 --- a/test/helpers/kube_dump.go +++ b/test/helpers/kube_dump.go @@ -15,6 +15,8 @@ import ( "github.com/solo-io/go-utils/threadsafe" "github.com/solo-io/gloo/pkg/utils/kubeutils/kubectl" + "github.com/solo-io/gloo/pkg/utils/kubeutils/portforward" + "github.com/solo-io/gloo/pkg/utils/requestutils/curl" "github.com/onsi/ginkgo/v2" "github.com/solo-io/gloo/pkg/cliutil/install" @@ -22,18 +24,14 @@ import ( gateway_defaults "github.com/solo-io/gloo/projects/gateway/pkg/defaults" "github.com/solo-io/gloo/projects/gloo/pkg/defaults" - "github.com/solo-io/skv2/codegen/util" + "github.com/solo-io/gloo/projects/gloo/pkg/servers/admin" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -var ( - kubeOutDir = filepath.Join(util.GetModuleRoot(), "_output", "kube2e-artifacts") - envoyOutDir = filepath.Join(kubeOutDir, "envoy-dump") -) - -// StandardGlooDumpOnFail creates adump of the kubernetes state and certain envoy data from the admin interface when a test fails +// StandardGlooDumpOnFail creates adump of the kubernetes state and certain envoy data from +// the admin interface when a test fails. // Look at `KubeDumpOnFail` && `EnvoyDumpOnFail` for more details -func StandardGlooDumpOnFail(out io.Writer, proxies ...metav1.ObjectMeta) func() { +func StandardGlooDumpOnFail(outLog io.Writer, outDir string, proxies ...metav1.ObjectMeta) func() { return func() { var namespaces []string for _, proxy := range proxies { @@ -42,29 +40,31 @@ func StandardGlooDumpOnFail(out io.Writer, proxies ...metav1.ObjectMeta) func() } } - KubeDumpOnFail(out, namespaces...)() - EnvoyDumpOnFail(out, proxies...)() + KubeDumpOnFail(outLog, outDir, namespaces...)() + ControllerDumpOnFail(outLog, outDir, namespaces...)() + EnvoyDumpOnFail(outLog, outDir, proxies...)() + + fmt.Printf("Test failed. Logs and cluster state are available in %s\n", outDir) } } // KubeDumpOnFail creates a small dump of the kubernetes state when a test fails. // This is useful for debugging test failures. -// The dump is written to _output/kube2e-artifacts. // The dump includes: // - docker state // - process state // - kubernetes state // - logs from all pods in the given namespaces // - yaml representations of all solo.io CRs in the given namespaces -func KubeDumpOnFail(out io.Writer, namespaces ...string) func() { +func KubeDumpOnFail(outLog io.Writer, outDir string, namespaces ...string) func() { return func() { - setupOutDir(kubeOutDir) + setupOutDir(outDir) - recordDockerState(fileAtPath(filepath.Join(kubeOutDir, "docker-state.log"))) - recordProcessState(fileAtPath(filepath.Join(kubeOutDir, "process-state.log"))) - recordKubeState(fileAtPath(filepath.Join(kubeOutDir, "kube-state.log"))) + recordDockerState(fileAtPath(filepath.Join(outDir, "docker-state.log"))) + recordProcessState(fileAtPath(filepath.Join(outDir, "process-state.log"))) + recordKubeState(fileAtPath(filepath.Join(outDir, "kube-state.log"))) - recordKubeDump(namespaces...) + recordKubeDump(outDir, namespaces...) } } @@ -110,11 +110,49 @@ func recordKubeState(f *os.File) { defer f.Close() kubeCli := &install.CmdKubectl{} - kubeState, err := kubeCli.KubectlOut(nil, "get", "all", "-A") + kubeState, err := kubeCli.KubectlOut(nil, "get", "all", "-A", "-o", "wide") if err != nil { f.WriteString("*** Unable to get kube state ***\n") return } + + resourcesToGet := []string{ + // Kubernetes resources + "secrets", + // Kube GW API resources + "gateways.gateway.networking.k8s.io", + "gatewayclasses.gateway.networking.k8s.io", + "httproutes.gateway.networking.k8s.io", + "referencegrants.gateway.networking.k8s.io", + // GG Kube GW resources + "gatewayparameters.gateway.gloo.solo.io", + "listeneroptions.gateway.solo.io", // only implemented for kube gw as of now + "httplisteneroptions.gateway.solo.io", // only implemented for kube gw as of now + // GG Gloo resources + "graphqlapis.graphql.gloo.solo.io", + "proxies.gloo.solo.io", + "settings.gloo.solo.io", + "upstreamgroups.gloo.solo.io", + "upstreams.gloo.solo.io", + // GG Edge GW resources + "gateways.gateway.solo.io", + "httpgateways.gateway.solo.io", + "tcpgateways.gateway.solo.io", + "virtualservices.gateway.solo.io", + // Shared GW resources + "routeoptions.gateway.solo.io", + "virtualhostoptions.gateway.solo.io", + // Dataplane extensions resources + "authconfigs.enterprise.gloo.solo.io", + "ratelimitconfigs.ratelimit.solo.io", + } + + kubeResources, err := kubeCli.KubectlOut(nil, "get", strings.Join(resourcesToGet, ","), "-A", "-owide") + if err != nil { + f.WriteString("*** Unable to get kube resources ***. Reason: " + err.Error() + " \n") + return + } + // Describe everything to identify the reason for issues such as Pods, LoadBalancers stuck in pending state // (insufficient resources, unable to acquire an IP), etc. // Ie: More context around the output of the previous command `kubectl get all -A` @@ -123,28 +161,32 @@ func recordKubeState(f *os.File) { f.WriteString("*** Unable to get kube describe ***. Reason: " + err.Error() + " \n") return } + kubeEndpointsState, err := kubeCli.KubectlOut(nil, "get", "endpoints", "-A") if err != nil { f.WriteString("*** Unable to get endpoint state ***. Reason: " + err.Error() + " \n") return } + f.WriteString("*** Kube state ***\n") f.WriteString(string(kubeState) + "\n") + f.WriteString(string(kubeResources) + "\n") f.WriteString(string(kubeDescribe) + "\n") f.WriteString(string(kubeEndpointsState) + "\n") + f.WriteString("*** End Kube state ***\n") } -func recordKubeDump(namespaces ...string) { +func recordKubeDump(outDir string, namespaces ...string) { // for each namespace, create a namespace directory that contains... for _, ns := range namespaces { // ...a pod logs subdirectoy - if err := recordPods(filepath.Join(kubeOutDir, ns, "_pods"), ns); err != nil { + if err := recordPods(filepath.Join(outDir, ns, "_pods"), ns); err != nil { fmt.Printf("error recording pod logs: %f, \n", err) } // ...and a subdirectory for each solo.io CRD with non-zero resources - if err := recordCRs(filepath.Join(kubeOutDir, ns), ns); err != nil { + if err := recordCRs(filepath.Join(outDir, ns), ns); err != nil { fmt.Printf("error recording pod logs: %f, \n", err) } } @@ -283,18 +325,123 @@ func kubeList(namespace string, target string) ([]string, string, error) { return toReturn, "", nil } +// ControllerDumpOnFail creates a small dump of the controller state when a test fails. +// This is useful for debugging test failures. +func ControllerDumpOnFail(outLog io.Writer, outDir string, namespaces ...string) func() { + return func() { + for _, ns := range namespaces { + namespaceOutDir := filepath.Join(outDir, ns) + setupOutDir(namespaceOutDir) + + // Get the Gloo Gateway controller logs + controllerLogsFilePath := filepath.Join(namespaceOutDir, "controller.log") + controllerLogsFile, err := os.OpenFile(controllerLogsFilePath, + os.O_APPEND|os.O_CREATE|os.O_WRONLY, os.ModePerm) + if err != nil { + fmt.Printf("error opening controller log file: %f\n", err) + } + + controllerLogsCmd := kubectl.NewCli().WithReceiver(controllerLogsFile).Command(context.Background(), + "-n", ns, "logs", "deployment/gloo", "-c", "gloo", "--tail=1000") + err = controllerLogsCmd.Run().Cause() + if err != nil { + fmt.Printf("error running controller logs command: %f\n", err) + } + + // podStdOut := bytes.NewBuffer(nil) + // podStdErr := bytes.NewBuffer(nil) + + // Fetch the name of the Gloo Gateway controller pod + getGlooPodNameCmd := i.Actions.Kubectl().Command(ctx, "get", "pod", "-n", i.Metadata.InstallNamespace, + "--selector", "gloo=gloo", "--output", "jsonpath='{.items[0].metadata.name}'") + cmdErr := getGlooPodNameCmd.WithStdout(podStdOut).WithStderr(podStdErr).Run() + if cmdErr != nil { + i.Assertions.Require.NoError(cmdErr) + } + + // Clean up and check the output + glooPodName := strings.Trim(podStdOut.String(), "'") + if glooPodName == "" { + i.Assertions.Require.NoError(fmt.Errorf("failed to get the Gloo Gateway controller pod name: %s", + podStdErr.String())) + } + + // Get the metrics from the Gloo Gateway controller pod and write them to a file + metricsFilePath := filepath.Join(failureDir, "metrics.log") + metricsFile, err := os.OpenFile(metricsFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, os.ModePerm) + i.Assertions.Require.NoError(err) + + // Using an ephemeral debug pod fetch the metrics from the Gloo Gateway controller + metricsCmd := i.Actions.Kubectl().Command(ctx, "debug", "-n", i.Metadata.InstallNamespace, + "-it", "--image=curlimages/curl:7.83.1", glooPodName, "--", + "curl", "http://localhost:9091/metrics") + cmdErr = metricsCmd.WithStdout(metricsFile).WithStderr(metricsFile).Run() + if cmdErr != nil { + i.Assertions.Require.NoError(cmdErr) + } + + // Open a port-forward to the Gloo Gateway controller pod's admin port + portForwarder, err := i.Actions.Kubectl().StartPortForward(ctx, + portforward.WithDeployment("gloo", i.Metadata.InstallNamespace), + portforward.WithPorts(int(admin.AdminPort), int(admin.AdminPort)), + ) + i.Assertions.Require.NoError(err) + + defer func() { + portForwarder.Close() + portForwarder.WaitForStop() + }() + + adminClient := admincli.NewClient(). + WithReceiver(io.Discard). + WithCurlOptions( + curl.WithRetries(3, 0, 10), + curl.WithPort(int(admin.AdminPort)), + ) + + // Get krt snapshot from the Gloo Gateway controller pod and write it to a file + krtSnapshotFilePath := filepath.Join(failureDir, "krt_snapshot.log") + krtSnapshotFile, err := os.OpenFile(krtSnapshotFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, os.ModePerm) + i.Assertions.Require.NoError(err) + + cmdErr = adminClient.KrtSnapshotCmd(ctx). + WithStdout(krtSnapshotFile). + WithStderr(krtSnapshotFile). + Run() + if cmdErr != nil { + i.Assertions.Require.NoError(cmdErr) + } + + // Get xds snapshot from the Gloo Gateway controller pod and write it to a file + xdsSnapshotFilePath := filepath.Join(failureDir, "xds_snapshot.log") + xdsSnapshotFile, err := os.OpenFile(xdsSnapshotFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, os.ModePerm) + i.Assertions.Require.NoError(err) + + cmdErr = adminClient.XdsSnapshotCmd(ctx). + WithStdout(xdsSnapshotFile). + WithStderr(xdsSnapshotFile). + Run() + if cmdErr != nil { + i.Assertions.Require.NoError(cmdErr) + } + + } + } +} + // EnvoyDumpOnFail creates a small dump of the envoy admin interface when a test fails. // This is useful for debugging test failures. -// The dump is written to _output/envoy-dump. // The dump includes: // - config dump // - stats // - clusters // - listeners -func EnvoyDumpOnFail(_ io.Writer, proxies ...metav1.ObjectMeta) func() { +func EnvoyDumpOnFail(_ io.Writer, outDir string, proxies ...metav1.ObjectMeta) func() { return func() { - setupOutDir(envoyOutDir) for _, proxy := range proxies { + envoyOutDir := filepath.Join(outDir, proxy.Namespace, proxy.Name) + setupOutDir(envoyOutDir) + proxyName := proxy.GetName() if proxyName == "" { proxyName = gateway_defaults.GatewayProxyName @@ -304,13 +451,23 @@ func EnvoyDumpOnFail(_ io.Writer, proxies ...metav1.ObjectMeta) func() { proxyNamespace = defaults.GlooSystem } - adminCli, shutdown, _ := admincli.NewPortForwardedClient(context.Background(), fmt.Sprintf("deployment/%s", proxyName), proxyNamespace) + adminCli, shutdown, err := admincli.NewPortForwardedClient(context.Background(), + fmt.Sprintf("deployment/%s", proxyName), proxyNamespace) + if err != nil { + fmt.Printf("error creating admin cli: %f\n", err) + return + } + defer shutdown() - adminCli.ConfigDumpCmd(context.Background(), nil).WithStdout(fileAtPath(filepath.Join(envoyOutDir, "config.log"))).Run().Cause() - adminCli.StatsCmd(context.Background()).WithStdout(fileAtPath(filepath.Join(envoyOutDir, "stats.log"))).Run().Cause() - adminCli.ClustersCmd(context.Background()).WithStdout(fileAtPath(filepath.Join(envoyOutDir, "clusters.log"))).Run().Cause() - adminCli.ListenersCmd(context.Background()).WithStdout(fileAtPath(filepath.Join(envoyOutDir, "listeners.log"))).Run().Cause() + adminCli.ConfigDumpCmd(context.Background(), nil). + WithStdout(fileAtPath(filepath.Join(envoyOutDir, "config.log"))).Run().Cause() + adminCli.StatsCmd(context.Background()). + WithStdout(fileAtPath(filepath.Join(envoyOutDir, "stats.log"))).Run().Cause() + adminCli.ClustersCmd(context.Background()). + WithStdout(fileAtPath(filepath.Join(envoyOutDir, "clusters.log"))).Run().Cause() + adminCli.ListenersCmd(context.Background()). + WithStdout(fileAtPath(filepath.Join(envoyOutDir, "listeners.log"))).Run().Cause() } } } diff --git a/test/kube2e/gateway/gateway_suite_test.go b/test/kube2e/gateway/gateway_suite_test.go index f98a9db07b3..de60c581679 100644 --- a/test/kube2e/gateway/gateway_suite_test.go +++ b/test/kube2e/gateway/gateway_suite_test.go @@ -8,6 +8,7 @@ import ( "time" "github.com/solo-io/gloo/test/kubernetes/testutils/cluster" + "github.com/solo-io/skv2/codegen/util" kubetestclients "github.com/solo-io/gloo/test/kubernetes/testutils/clients" @@ -63,7 +64,10 @@ func StartTestHelper() { testHelper, err = kube2e.GetTestHelper(ctx, namespace) Expect(err).NotTo(HaveOccurred()) - skhelpers.RegisterPreFailHandler(helpers.StandardGlooDumpOnFail(GinkgoWriter, metav1.ObjectMeta{Namespace: testHelper.InstallNamespace})) + + outDir := filepath.Join(util.GetModuleRoot(), "_output", "kube2e-artifacts") + skhelpers.RegisterPreFailHandler(helpers.StandardGlooDumpOnFail(GinkgoWriter, outDir, + metav1.ObjectMeta{Namespace: testHelper.InstallNamespace})) kubeCli = kubectl.NewCli().WithReceiver(GinkgoWriter) diff --git a/test/kube2e/gloo/gloo_suite_test.go b/test/kube2e/gloo/gloo_suite_test.go index e8f13f8bc7f..206094c271f 100644 --- a/test/kube2e/gloo/gloo_suite_test.go +++ b/test/kube2e/gloo/gloo_suite_test.go @@ -8,6 +8,7 @@ import ( "time" "github.com/solo-io/gloo/test/kubernetes/testutils/cluster" + "github.com/solo-io/skv2/codegen/util" "github.com/solo-io/gloo/pkg/utils/kubeutils/kubectl" @@ -65,7 +66,10 @@ var _ = BeforeSuite(func() { testHelper, err = kube2e.GetTestHelper(ctx, namespace) Expect(err).NotTo(HaveOccurred()) testHelper.SetKubeCli(kubectl.NewCli().WithReceiver(GinkgoWriter)) - skhelpers.RegisterPreFailHandler(helpers.StandardGlooDumpOnFail(GinkgoWriter, metav1.ObjectMeta{Namespace: testHelper.InstallNamespace})) + + outDir := filepath.Join(util.GetModuleRoot(), "_output", "kube2e-artifacts") + skhelpers.RegisterPreFailHandler(helpers.StandardGlooDumpOnFail(GinkgoWriter, outDir, + metav1.ObjectMeta{Namespace: testHelper.InstallNamespace})) // Allow skipping of install step for running multiple times if !glootestutils.ShouldSkipInstall() { diff --git a/test/kube2e/upgrade/upgrade_suite_test.go b/test/kube2e/upgrade/upgrade_suite_test.go index afed671e336..2650a226d30 100644 --- a/test/kube2e/upgrade/upgrade_suite_test.go +++ b/test/kube2e/upgrade/upgrade_suite_test.go @@ -52,7 +52,8 @@ var _ = BeforeSuite(func() { testHelper, err := kube2e.GetTestHelper(suiteCtx, namespace) Expect(err).NotTo(HaveOccurred()) - skhelpers.RegisterPreFailHandler(helpers.StandardGlooDumpOnFail(GinkgoWriter, + outDir := filepath.Join(util.GetModuleRoot(), "_output", "kube2e-artifacts") + skhelpers.RegisterPreFailHandler(helpers.StandardGlooDumpOnFail(GinkgoWriter, outDir, metav1.ObjectMeta{Namespace: "upgrade"}, metav1.ObjectMeta{Namespace: testHelper.InstallNamespace}, metav1.ObjectMeta{Namespace: "other-ns"})) diff --git a/test/kubernetes/e2e/features/virtualhost_options/vhost_opt_suite.go b/test/kubernetes/e2e/features/virtualhost_options/vhost_opt_suite.go index bf0f3bb0fc4..7958a14ba3b 100644 --- a/test/kubernetes/e2e/features/virtualhost_options/vhost_opt_suite.go +++ b/test/kubernetes/e2e/features/virtualhost_options/vhost_opt_suite.go @@ -225,6 +225,8 @@ func (s *testingSuite) TestConfigureVirtualHostOptionsWithSectionNameManualSetup []string{"conflict with more specific or older VirtualHostOptions"}, defaults.KubeGatewayReporter, ) + + s.Assert().Equal(true, false, "intentionally failing to trigger dump, remove when done debugging") } // The goal here is to test the behavior when multiple VHOs are targeting a gateway without sectionName. The expected diff --git a/test/kubernetes/e2e/test.go b/test/kubernetes/e2e/test.go index 0a875483748..943652c0b94 100644 --- a/test/kubernetes/e2e/test.go +++ b/test/kubernetes/e2e/test.go @@ -5,19 +5,15 @@ import ( "context" "errors" "fmt" - "io" "io/fs" "os" "path/filepath" "runtime" - "strings" "testing" "time" - "github.com/solo-io/gloo/pkg/utils/glooadminutils/admincli" - "github.com/solo-io/gloo/pkg/utils/kubeutils/portforward" - "github.com/solo-io/gloo/pkg/utils/requestutils/curl" - "github.com/solo-io/gloo/projects/gloo/pkg/servers/admin" + "github.com/solo-io/gloo/pkg/utils/kubeutils/kubectl" + "github.com/solo-io/gloo/test/helpers" "github.com/solo-io/gloo/test/kubernetes/testutils/actions" "github.com/solo-io/gloo/test/kubernetes/testutils/assertions" "github.com/solo-io/gloo/test/kubernetes/testutils/cluster" @@ -25,6 +21,7 @@ import ( "github.com/solo-io/gloo/test/kubernetes/testutils/helper" testruntime "github.com/solo-io/gloo/test/kubernetes/testutils/runtime" "github.com/solo-io/gloo/test/testutils" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // MustTestHelper returns the SoloTestHelper used for e2e tests @@ -233,7 +230,6 @@ func (i *TestInstallation) UninstallGlooGateway(ctx context.Context, uninstallFn // PreFailHandler is the function that is invoked if a test in the given TestInstallation fails func (i *TestInstallation) PreFailHandler(ctx context.Context) { - // This is a work in progress // The idea here is we want to accumulate ALL information about this TestInstallation into a single directory // That way we can upload it in CI, or inspect it locally @@ -246,159 +242,20 @@ func (i *TestInstallation) PreFailHandler(ctx context.Context) { i.Assertions.Require.NoError(err) } - glooLogFilePath := filepath.Join(failureDir, "gloo.log") - glooLogFile, err := os.OpenFile(glooLogFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, os.ModePerm) - i.Assertions.Require.NoError(err) - defer glooLogFile.Close() - - glooLogsCmd := i.Actions.Kubectl().Command(ctx, "logs", "-n", i.Metadata.InstallNamespace, "deployments/gloo") - _ = glooLogsCmd.WithStdout(glooLogFile).WithStderr(glooLogFile).Run() - - edgeGatewayLogFilePath := filepath.Join(failureDir, "edge_gateway.log") - edgeGatewayLogFile, err := os.OpenFile(edgeGatewayLogFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, os.ModePerm) - i.Assertions.Require.NoError(err) - defer edgeGatewayLogFile.Close() - - kubeGatewayLogFilePath := filepath.Join(failureDir, "kube_gateway.log") - kubeGatewayLogFile, err := os.OpenFile(kubeGatewayLogFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, os.ModePerm) - i.Assertions.Require.NoError(err) - defer kubeGatewayLogFile.Close() - namespaces, err := i.Actions.Kubectl().Namespaces(ctx) i.Assertions.Require.NoError(err) - for _, n := range namespaces { - edgeGatewayLogFile.WriteString(fmt.Sprintf("Logs for edge gateway proxies in namespace %s\n", n)) - edgeGatewayLogsCmd := i.Actions.Kubectl().Command(ctx, "logs", "--all-containers", "--namespace", n, "--prefix", "-l", "gloo=gateway-proxy") - _ = edgeGatewayLogsCmd.WithStdout(edgeGatewayLogFile).WithStderr(edgeGatewayLogFile).Run() - edgeGatewayLogFile.WriteString("----------------------------------------------------------------------------------------------------------\n") - - kubeGatewayLogFile.WriteString(fmt.Sprintf("Logs for kube gateway proxies in namespace %s\n", n)) - kubeGatewayLogsCmd := i.Actions.Kubectl().Command(ctx, "logs", "--all-containers", "--namespace", n, "--prefix", "-l", "gloo=kube-gateway") - _ = kubeGatewayLogsCmd.WithStdout(kubeGatewayLogFile).WithStderr(kubeGatewayLogFile).Run() - kubeGatewayLogFile.WriteString("----------------------------------------------------------------------------------------------------------\n") - } - - clusterStateFilePath := filepath.Join(failureDir, "cluster_state.log") - clusterStateFile, err := os.OpenFile(clusterStateFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, os.ModePerm) - i.Assertions.Require.NoError(err) - defer clusterStateFile.Close() - - kubectlGetAllCmd := i.Actions.Kubectl().Command(ctx, "get", "all", "-A", "-owide") - _ = kubectlGetAllCmd.WithStdout(clusterStateFile).WithStderr(clusterStateFile).Run() - clusterStateFile.WriteString("\n") - - resourcesToGet := []string{ - // Kubernetes resources - "secrets", - // Kube GW API resources - "gateways.gateway.networking.k8s.io", - "gatewayclasses.gateway.networking.k8s.io", - "httproutes.gateway.networking.k8s.io", - "referencegrants.gateway.networking.k8s.io", - // GG Kube GW resources - "gatewayparameters.gateway.gloo.solo.io", - "listeneroptions.gateway.solo.io", // only implemented for kube gw as of now - "httplisteneroptions.gateway.solo.io", // only implemented for kube gw as of now - // GG Gloo resources - "graphqlapis.graphql.gloo.solo.io", - "proxies.gloo.solo.io", - "settings.gloo.solo.io", - "upstreamgroups.gloo.solo.io", - "upstreams.gloo.solo.io", - // GG Edge GW resources - "gateways.gateway.solo.io", - "httpgateways.gateway.solo.io", - "tcpgateways.gateway.solo.io", - "virtualservices.gateway.solo.io", - // Shared GW resources - "routeoptions.gateway.solo.io", - "virtualhostoptions.gateway.solo.io", - // Dataplane extensions resources - "authconfigs.enterprise.gloo.solo.io", - "ratelimitconfigs.ratelimit.solo.io", - } - kubectlGetResourcesCmd := i.Actions.Kubectl().Command(ctx, "get", strings.Join(resourcesToGet, ","), "-A", "-owide") - _ = kubectlGetResourcesCmd.WithStdout(clusterStateFile).WithStderr(clusterStateFile).Run() - clusterStateFile.WriteString("\n") - - podStdOut := bytes.NewBuffer(nil) - podStdErr := bytes.NewBuffer(nil) - - // Fetch the name of the Gloo Gateway controller pod - getGlooPodNameCmd := i.Actions.Kubectl().Command(ctx, "get", "pod", "-n", i.Metadata.InstallNamespace, - "--selector", "gloo=gloo", "--output", "jsonpath='{.items[0].metadata.name}'") - cmdErr := getGlooPodNameCmd.WithStdout(podStdOut).WithStderr(podStdErr).Run() - if cmdErr != nil { - i.Assertions.Require.NoError(cmdErr) - } - - // Clean up and check the output - glooPodName := strings.Trim(podStdOut.String(), "'") - if glooPodName == "" { - i.Assertions.Require.NoError(fmt.Errorf("failed to get the Gloo Gateway controller pod name: %s", - podStdErr.String())) - } - - // Get the metrics from the Gloo Gateway controller pod and write them to a file - metricsFilePath := filepath.Join(failureDir, "metrics.log") - metricsFile, err := os.OpenFile(metricsFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, os.ModePerm) - i.Assertions.Require.NoError(err) - - // Using an ephemeral debug pod fetch the metrics from the Gloo Gateway controller - metricsCmd := i.Actions.Kubectl().Command(ctx, "debug", "-n", i.Metadata.InstallNamespace, - "-it", "--image=curlimages/curl:7.83.1", glooPodName, "--", - "curl", "http://localhost:9091/metrics") - cmdErr = metricsCmd.WithStdout(metricsFile).WithStderr(metricsFile).Run() - if cmdErr != nil { - i.Assertions.Require.NoError(cmdErr) - } - - // Open a port-forward to the Gloo Gateway controller pod's admin port - portForwarder, err := i.Actions.Kubectl().StartPortForward(ctx, - portforward.WithDeployment("gloo", i.Metadata.InstallNamespace), - portforward.WithPorts(int(admin.AdminPort), int(admin.AdminPort)), - ) - i.Assertions.Require.NoError(err) - defer func() { - portForwarder.Close() - portForwarder.WaitForStop() - }() - - adminClient := admincli.NewClient(). - WithReceiver(io.Discard). - WithCurlOptions( - curl.WithRetries(3, 0, 10), - curl.WithPort(int(admin.AdminPort)), - ) - - // Get krt snapshot from the Gloo Gateway controller pod and write it to a file - krtSnapshotFilePath := filepath.Join(failureDir, "krt_snapshot.log") - krtSnapshotFile, err := os.OpenFile(krtSnapshotFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, os.ModePerm) - i.Assertions.Require.NoError(err) - - cmdErr = adminClient.KrtSnapshotCmd(ctx). - WithStdout(krtSnapshotFile). - WithStderr(krtSnapshotFile). - Run() - if cmdErr != nil { - i.Assertions.Require.NoError(cmdErr) - } - - // Get xds snapshot from the Gloo Gateway controller pod and write it to a file - xdsSnapshotFilePath := filepath.Join(failureDir, "xds_snapshot.log") - xdsSnapshotFile, err := os.OpenFile(xdsSnapshotFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, os.ModePerm) - i.Assertions.Require.NoError(err) + proxies := []metav1.ObjectMeta{} + for _, ns := range namespaces { + proxies, err = fetchAndAddProxies(i.Actions.Kubectl(), ns, "gloo=kube-gateway", proxies) + i.Assertions.Require.NoError(err) - cmdErr = adminClient.XdsSnapshotCmd(ctx). - WithStdout(xdsSnapshotFile). - WithStderr(xdsSnapshotFile). - Run() - if cmdErr != nil { - i.Assertions.Require.NoError(cmdErr) + proxies, err = fetchAndAddProxies(i.Actions.Kubectl(), ns, "gloo=gateway-proxy", proxies) + i.Assertions.Require.NoError(err) } - fmt.Printf("Test failed. Logs and cluster state are available in %s\n", failureDir) + // Dump the logs and state of the cluster + helpers.StandardGlooDumpOnFail(os.Stdout, failureDir, proxies...)() } // GeneratedFiles is a collection of files that are generated during the execution of a set of tests @@ -434,3 +291,30 @@ func MustGeneratedFiles(tmpDirId, clusterId string) GeneratedFiles { FailureDir: failureDir, } } + +func fetchAndAddProxies(kubectl *kubectl.Cli, namespace string, label string, + proxies []metav1.ObjectMeta) ([]metav1.ObjectMeta, error) { + + stdout := bytes.NewBuffer(nil) + stderr := bytes.NewBuffer(nil) + + // get all deployments in the namespace with the label + lookupKubeGatewaysCmd := kubectl.Command(context.Background(), "get", + "deployment", "-n", namespace, "--selector", "gloo=kube-gateway", + "--no-headers", "-o", `custom-columns=":metadata.name"`) + err := lookupKubeGatewaysCmd.WithStdout(stdout).WithStderr(stderr). + Run().Cause() + if err != nil { + return nil, fmt.Errorf("failed to get proxies: %s (%s)", err, stderr.String()) + } + + // iterate lines in the output and append to the list of proxies + for _, line := range bytes.Split(stdout.Bytes(), []byte("\n")) { + proxyName := string(bytes.TrimSpace(line)) + if proxyName != "" { + proxies = append(proxies, metav1.ObjectMeta{Namespace: namespace, Name: proxyName}) + } + } + + return proxies, nil +} From d526096b61f0322b44766d420b9692fd32e67ac3 Mon Sep 17 00:00:00 2001 From: Ryan Old Date: Wed, 27 Nov 2024 09:35:06 -0800 Subject: [PATCH 13/22] More work, should be at least building and running now. Still fixing things --- test/helpers/kube_dump.go | 262 +++++++++++++--------- test/kube2e/gateway/gateway_suite_test.go | 4 +- test/kube2e/gloo/gloo_suite_test.go | 5 +- test/kube2e/upgrade/upgrade_suite_test.go | 7 +- test/kubernetes/e2e/test.go | 42 +--- 5 files changed, 158 insertions(+), 162 deletions(-) diff --git a/test/helpers/kube_dump.go b/test/helpers/kube_dump.go index 059e88486ba..e3705d7d2b1 100644 --- a/test/helpers/kube_dump.go +++ b/test/helpers/kube_dump.go @@ -11,6 +11,7 @@ import ( "strings" "github.com/hashicorp/go-multierror" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/solo-io/go-utils/threadsafe" @@ -22,27 +23,18 @@ import ( "github.com/solo-io/gloo/pkg/cliutil/install" "github.com/solo-io/gloo/pkg/utils/envoyutils/admincli" - gateway_defaults "github.com/solo-io/gloo/projects/gateway/pkg/defaults" - "github.com/solo-io/gloo/projects/gloo/pkg/defaults" + glooAdminCli "github.com/solo-io/gloo/pkg/utils/glooadminutils/admincli" "github.com/solo-io/gloo/projects/gloo/pkg/servers/admin" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // StandardGlooDumpOnFail creates adump of the kubernetes state and certain envoy data from // the admin interface when a test fails. // Look at `KubeDumpOnFail` && `EnvoyDumpOnFail` for more details -func StandardGlooDumpOnFail(outLog io.Writer, outDir string, proxies ...metav1.ObjectMeta) func() { +func StandardGlooDumpOnFail(outLog io.Writer, outDir string, namespaces []string) func() { return func() { - var namespaces []string - for _, proxy := range proxies { - if proxy.GetNamespace() != "" { - namespaces = append(namespaces, proxy.Namespace) - } - } - - KubeDumpOnFail(outLog, outDir, namespaces...)() - ControllerDumpOnFail(outLog, outDir, namespaces...)() - EnvoyDumpOnFail(outLog, outDir, proxies...)() + KubeDumpOnFail(outLog, outDir, namespaces)() + ControllerDumpOnFail(outLog, outDir, namespaces)() + EnvoyDumpOnFail(outLog, outDir, namespaces)() fmt.Printf("Test failed. Logs and cluster state are available in %s\n", outDir) } @@ -56,7 +48,7 @@ func StandardGlooDumpOnFail(outLog io.Writer, outDir string, proxies ...metav1.O // - kubernetes state // - logs from all pods in the given namespaces // - yaml representations of all solo.io CRs in the given namespaces -func KubeDumpOnFail(outLog io.Writer, outDir string, namespaces ...string) func() { +func KubeDumpOnFail(outLog io.Writer, outDir string, namespaces []string) func() { return func() { setupOutDir(outDir) @@ -327,7 +319,7 @@ func kubeList(namespace string, target string) ([]string, string, error) { // ControllerDumpOnFail creates a small dump of the controller state when a test fails. // This is useful for debugging test failures. -func ControllerDumpOnFail(outLog io.Writer, outDir string, namespaces ...string) func() { +func ControllerDumpOnFail(outLog io.Writer, outDir string, namespaces []string) func() { return func() { for _, ns := range namespaces { namespaceOutDir := filepath.Join(outDir, ns) @@ -341,90 +333,106 @@ func ControllerDumpOnFail(outLog io.Writer, outDir string, namespaces ...string) fmt.Printf("error opening controller log file: %f\n", err) } - controllerLogsCmd := kubectl.NewCli().WithReceiver(controllerLogsFile).Command(context.Background(), + kubeCli := kubectl.NewCli() + + controllerLogsCmd := kubeCli.WithReceiver(controllerLogsFile).Command(context.Background(), "-n", ns, "logs", "deployment/gloo", "-c", "gloo", "--tail=1000") err = controllerLogsCmd.Run().Cause() if err != nil { fmt.Printf("error running controller logs command: %f\n", err) } - // podStdOut := bytes.NewBuffer(nil) - // podStdErr := bytes.NewBuffer(nil) + podStdOut := bytes.NewBuffer(nil) + podStdErr := bytes.NewBuffer(nil) // Fetch the name of the Gloo Gateway controller pod - getGlooPodNameCmd := i.Actions.Kubectl().Command(ctx, "get", "pod", "-n", i.Metadata.InstallNamespace, - "--selector", "gloo=gloo", "--output", "jsonpath='{.items[0].metadata.name}'") - cmdErr := getGlooPodNameCmd.WithStdout(podStdOut).WithStderr(podStdErr).Run() - if cmdErr != nil { - i.Assertions.Require.NoError(cmdErr) + getGlooPodNamesCmd := kubeCli.Command(context.Background(), "get", "pod", "-n", + ns, "--selector", "gloo=gloo", "--output", "jsonpath='{.items[*].metadata.name}'") + err = getGlooPodNamesCmd.WithStdout(podStdOut).WithStderr(podStdErr).Run().Cause() + if err != nil { + fmt.Printf("error running get gloo pod name command: %f\n", err) } // Clean up and check the output - glooPodName := strings.Trim(podStdOut.String(), "'") - if glooPodName == "" { - i.Assertions.Require.NoError(fmt.Errorf("failed to get the Gloo Gateway controller pod name: %s", - podStdErr.String())) - } - - // Get the metrics from the Gloo Gateway controller pod and write them to a file - metricsFilePath := filepath.Join(failureDir, "metrics.log") - metricsFile, err := os.OpenFile(metricsFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, os.ModePerm) - i.Assertions.Require.NoError(err) - - // Using an ephemeral debug pod fetch the metrics from the Gloo Gateway controller - metricsCmd := i.Actions.Kubectl().Command(ctx, "debug", "-n", i.Metadata.InstallNamespace, - "-it", "--image=curlimages/curl:7.83.1", glooPodName, "--", - "curl", "http://localhost:9091/metrics") - cmdErr = metricsCmd.WithStdout(metricsFile).WithStderr(metricsFile).Run() - if cmdErr != nil { - i.Assertions.Require.NoError(cmdErr) + glooPodNamesString := strings.TrimSpace(podStdOut.String()) + if glooPodNamesString == "" { + fmt.Printf("error getting gloo pod names: %s\n", podStdErr.String()) + return } - // Open a port-forward to the Gloo Gateway controller pod's admin port - portForwarder, err := i.Actions.Kubectl().StartPortForward(ctx, - portforward.WithDeployment("gloo", i.Metadata.InstallNamespace), - portforward.WithPorts(int(admin.AdminPort), int(admin.AdminPort)), - ) - i.Assertions.Require.NoError(err) - - defer func() { - portForwarder.Close() - portForwarder.WaitForStop() - }() - - adminClient := admincli.NewClient(). - WithReceiver(io.Discard). - WithCurlOptions( - curl.WithRetries(3, 0, 10), - curl.WithPort(int(admin.AdminPort)), + // Split the string on whitespace to get the pod names + glooPodNames := strings.Fields(glooPodNamesString) + + for _, podName := range glooPodNames { + // Get the metrics from the Gloo Gateway controller pod and write them to a file + metricsFilePath := filepath.Join(namespaceOutDir, fmt.Sprintf("%s.metrics.log", podName)) + metricsFile, err := os.OpenFile(metricsFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, os.ModePerm) + if err != nil { + fmt.Printf("error opening metrics file: %f\n", err) + } + + // Using an ephemeral debug pod fetch the metrics from the Gloo Gateway controller + metricsCmd := kubeCli.Command(context.Background(), "debug", "-n", ns, + "-it", "--image=curlimages/curl:7.83.1", glooPodNamesString, "--", + "curl", "http://localhost:9091/metrics") + err = metricsCmd.WithStdout(metricsFile).WithStderr(metricsFile).Run().Cause() + if err != nil { + fmt.Printf("error running metrics command: %f\n", err) + } + + // Open a port-forward to the Gloo Gateway controller pod's admin port + portForwarder, err := kubeCli.StartPortForward(context.Background(), + portforward.WithDeployment("gloo", ns), + portforward.WithPorts(int(admin.AdminPort), int(admin.AdminPort)), ) - - // Get krt snapshot from the Gloo Gateway controller pod and write it to a file - krtSnapshotFilePath := filepath.Join(failureDir, "krt_snapshot.log") - krtSnapshotFile, err := os.OpenFile(krtSnapshotFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, os.ModePerm) - i.Assertions.Require.NoError(err) - - cmdErr = adminClient.KrtSnapshotCmd(ctx). - WithStdout(krtSnapshotFile). - WithStderr(krtSnapshotFile). - Run() - if cmdErr != nil { - i.Assertions.Require.NoError(cmdErr) - } - - // Get xds snapshot from the Gloo Gateway controller pod and write it to a file - xdsSnapshotFilePath := filepath.Join(failureDir, "xds_snapshot.log") - xdsSnapshotFile, err := os.OpenFile(xdsSnapshotFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, os.ModePerm) - i.Assertions.Require.NoError(err) - - cmdErr = adminClient.XdsSnapshotCmd(ctx). - WithStdout(xdsSnapshotFile). - WithStderr(xdsSnapshotFile). - Run() - if cmdErr != nil { - i.Assertions.Require.NoError(cmdErr) + if err != nil { + fmt.Printf("error starting port forward: %f\n", err) + } + + defer func() { + portForwarder.Close() + portForwarder.WaitForStop() + }() + + adminClient := glooAdminCli.NewClient(). + WithReceiver(io.Discard). + WithCurlOptions( + curl.WithRetries(3, 0, 10), + curl.WithPort(int(admin.AdminPort)), + ) + + // Get krt snapshot from the Gloo Gateway controller pod and write it to a file + krtSnapshotFilePath := filepath.Join(namespaceOutDir, fmt.Sprintf("%s.krt_snapshot.log", podName)) + krtSnapshotFile, err := os.OpenFile(krtSnapshotFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, + os.ModePerm) + if err != nil { + fmt.Printf("error opening krt snapshot file: %f\n", err) + } + + err = adminClient.KrtSnapshotCmd(context.Background()). + WithStdout(krtSnapshotFile). + WithStderr(krtSnapshotFile). + Run().Cause() + if err != nil { + fmt.Printf("error running krt snapshot command: %f\n", err) + } + + // Get xds snapshot from the Gloo Gateway controller pod and write it to a file + xdsSnapshotFilePath := filepath.Join(namespaceOutDir, fmt.Sprintf("%s.xds_snapshot.log", podName)) + xdsSnapshotFile, err := os.OpenFile(xdsSnapshotFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, + os.ModePerm) + if err != nil { + fmt.Printf("error opening xds snapshot file: %f\n", err) + } + + err = adminClient.XdsSnapshotCmd(context.Background()). + WithStdout(xdsSnapshotFile). + WithStderr(xdsSnapshotFile). + Run().Cause() + if err != nil { + fmt.Printf("error running xds snapshot command: %f\n", err) + } } - } } } @@ -436,38 +444,43 @@ func ControllerDumpOnFail(outLog io.Writer, outDir string, namespaces ...string) // - stats // - clusters // - listeners -func EnvoyDumpOnFail(_ io.Writer, outDir string, proxies ...metav1.ObjectMeta) func() { +func EnvoyDumpOnFail(_ io.Writer, outDir string, namespaces []string) func() { return func() { - for _, proxy := range proxies { - envoyOutDir := filepath.Join(outDir, proxy.Namespace, proxy.Name) - setupOutDir(envoyOutDir) + kubectl := kubectl.NewCli() - proxyName := proxy.GetName() - if proxyName == "" { - proxyName = gateway_defaults.GatewayProxyName - } - proxyNamespace := proxy.GetNamespace() - if proxyNamespace == "" { - proxyNamespace = defaults.GlooSystem + for _, ns := range namespaces { + proxies, err := fetchAndAddProxies(kubectl, ns, "gloo=kube-gateway", []metav1.ObjectMeta{}) + if err != nil { + fmt.Printf("error fetching kube-gateway proxies: %f\n", err) } - adminCli, shutdown, err := admincli.NewPortForwardedClient(context.Background(), - fmt.Sprintf("deployment/%s", proxyName), proxyNamespace) + proxies, err = fetchAndAddProxies(kubectl, ns, "gloo=gateway-proxy", proxies) if err != nil { - fmt.Printf("error creating admin cli: %f\n", err) - return + fmt.Printf("error fetching gateway-proxy proxies: %f\n", err) } - defer shutdown() - - adminCli.ConfigDumpCmd(context.Background(), nil). - WithStdout(fileAtPath(filepath.Join(envoyOutDir, "config.log"))).Run().Cause() - adminCli.StatsCmd(context.Background()). - WithStdout(fileAtPath(filepath.Join(envoyOutDir, "stats.log"))).Run().Cause() - adminCli.ClustersCmd(context.Background()). - WithStdout(fileAtPath(filepath.Join(envoyOutDir, "clusters.log"))).Run().Cause() - adminCli.ListenersCmd(context.Background()). - WithStdout(fileAtPath(filepath.Join(envoyOutDir, "listeners.log"))).Run().Cause() + for _, proxy := range proxies { + envoyOutDir := filepath.Join(outDir, ns) + setupOutDir(envoyOutDir) + + adminCli, shutdown, err := admincli.NewPortForwardedClient(context.Background(), + fmt.Sprintf("deployment/%s", proxy.GetName()), proxy.GetNamespace()) + if err != nil { + fmt.Printf("error creating admin cli: %f\n", err) + return + } + + defer shutdown() + + adminCli.ConfigDumpCmd(context.Background(), nil). + WithStdout(fileAtPath(filepath.Join(envoyOutDir, "config.log"))).Run().Cause() + adminCli.StatsCmd(context.Background()). + WithStdout(fileAtPath(filepath.Join(envoyOutDir, "stats.log"))).Run().Cause() + adminCli.ClustersCmd(context.Background()). + WithStdout(fileAtPath(filepath.Join(envoyOutDir, "clusters.log"))).Run().Cause() + adminCli.ListenersCmd(context.Background()). + WithStdout(fileAtPath(filepath.Join(envoyOutDir, "listeners.log"))).Run().Cause() + } } } } @@ -482,8 +495,6 @@ func setupOutDir(outdir string) { if err != nil { fmt.Printf("error creating log directory: %f\n", err) } - - fmt.Println("kube dump artifacts will be stored at: " + outdir) } // fileAtPath creates a file at the given path, and returns the file object @@ -494,3 +505,30 @@ func fileAtPath(path string) *os.File { } return f } + +func fetchAndAddProxies(kubectl *kubectl.Cli, namespace string, label string, + proxies []metav1.ObjectMeta) ([]metav1.ObjectMeta, error) { + + stdout := bytes.NewBuffer(nil) + stderr := bytes.NewBuffer(nil) + + // get all deployments in the namespace with the label + lookupKubeGatewaysCmd := kubectl.Command(context.Background(), "get", + "deployment", "-n", namespace, "--selector", "gloo=kube-gateway", + "--no-headers", "-o", `custom-columns=":metadata.name"`) + err := lookupKubeGatewaysCmd.WithStdout(stdout).WithStderr(stderr). + Run().Cause() + if err != nil { + return proxies, fmt.Errorf("failed to get proxies: %s (%s)", err, stderr.String()) + } + + // iterate lines in the output and append to the list of proxies + for _, line := range bytes.Split(stdout.Bytes(), []byte("\n")) { + proxyName := string(bytes.TrimSpace(line)) + if proxyName != "" { + proxies = append(proxies, metav1.ObjectMeta{Namespace: namespace, Name: proxyName}) + } + } + + return proxies, nil +} diff --git a/test/kube2e/gateway/gateway_suite_test.go b/test/kube2e/gateway/gateway_suite_test.go index de60c581679..660e8f7812e 100644 --- a/test/kube2e/gateway/gateway_suite_test.go +++ b/test/kube2e/gateway/gateway_suite_test.go @@ -66,8 +66,8 @@ func StartTestHelper() { Expect(err).NotTo(HaveOccurred()) outDir := filepath.Join(util.GetModuleRoot(), "_output", "kube2e-artifacts") - skhelpers.RegisterPreFailHandler(helpers.StandardGlooDumpOnFail(GinkgoWriter, outDir, - metav1.ObjectMeta{Namespace: testHelper.InstallNamespace})) + namespaces := []string{testHelper.InstallNamespace} + skhelpers.RegisterPreFailHandler(helpers.StandardGlooDumpOnFail(GinkgoWriter, outDir, namespaces)) kubeCli = kubectl.NewCli().WithReceiver(GinkgoWriter) diff --git a/test/kube2e/gloo/gloo_suite_test.go b/test/kube2e/gloo/gloo_suite_test.go index 206094c271f..069f39cf51d 100644 --- a/test/kube2e/gloo/gloo_suite_test.go +++ b/test/kube2e/gloo/gloo_suite_test.go @@ -13,7 +13,6 @@ import ( "github.com/solo-io/gloo/pkg/utils/kubeutils/kubectl" "github.com/avast/retry-go" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/log/zap" "github.com/solo-io/gloo/test/services/envoy" @@ -68,8 +67,8 @@ var _ = BeforeSuite(func() { testHelper.SetKubeCli(kubectl.NewCli().WithReceiver(GinkgoWriter)) outDir := filepath.Join(util.GetModuleRoot(), "_output", "kube2e-artifacts") - skhelpers.RegisterPreFailHandler(helpers.StandardGlooDumpOnFail(GinkgoWriter, outDir, - metav1.ObjectMeta{Namespace: testHelper.InstallNamespace})) + namespaces := []string{testHelper.InstallNamespace} + skhelpers.RegisterPreFailHandler(helpers.StandardGlooDumpOnFail(GinkgoWriter, outDir, namespaces)) // Allow skipping of install step for running multiple times if !glootestutils.ShouldSkipInstall() { diff --git a/test/kube2e/upgrade/upgrade_suite_test.go b/test/kube2e/upgrade/upgrade_suite_test.go index 2650a226d30..5ea1cd2e84f 100644 --- a/test/kube2e/upgrade/upgrade_suite_test.go +++ b/test/kube2e/upgrade/upgrade_suite_test.go @@ -13,7 +13,6 @@ import ( "github.com/solo-io/gloo/test/kubernetes/testutils/helper" "github.com/solo-io/go-utils/versionutils" "github.com/solo-io/skv2/codegen/util" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" . "github.com/onsi/ginkgo/v2" "github.com/solo-io/gloo/test/helpers" @@ -53,10 +52,8 @@ var _ = BeforeSuite(func() { Expect(err).NotTo(HaveOccurred()) outDir := filepath.Join(util.GetModuleRoot(), "_output", "kube2e-artifacts") - skhelpers.RegisterPreFailHandler(helpers.StandardGlooDumpOnFail(GinkgoWriter, outDir, - metav1.ObjectMeta{Namespace: "upgrade"}, - metav1.ObjectMeta{Namespace: testHelper.InstallNamespace}, - metav1.ObjectMeta{Namespace: "other-ns"})) + namespaces := []string{"upgrade", testHelper.InstallNamespace, "other-ns"} + skhelpers.RegisterPreFailHandler(helpers.StandardGlooDumpOnFail(GinkgoWriter, outDir, namespaces)) crdDir = filepath.Join(util.GetModuleRoot(), "install", "helm", "gloo", "crds") targetReleasedVersion = kube2e.GetTestReleasedVersion(suiteCtx, "gloo") diff --git a/test/kubernetes/e2e/test.go b/test/kubernetes/e2e/test.go index 943652c0b94..efb71e60652 100644 --- a/test/kubernetes/e2e/test.go +++ b/test/kubernetes/e2e/test.go @@ -1,7 +1,6 @@ package e2e import ( - "bytes" "context" "errors" "fmt" @@ -12,7 +11,6 @@ import ( "testing" "time" - "github.com/solo-io/gloo/pkg/utils/kubeutils/kubectl" "github.com/solo-io/gloo/test/helpers" "github.com/solo-io/gloo/test/kubernetes/testutils/actions" "github.com/solo-io/gloo/test/kubernetes/testutils/assertions" @@ -21,7 +19,6 @@ import ( "github.com/solo-io/gloo/test/kubernetes/testutils/helper" testruntime "github.com/solo-io/gloo/test/kubernetes/testutils/runtime" "github.com/solo-io/gloo/test/testutils" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // MustTestHelper returns the SoloTestHelper used for e2e tests @@ -242,20 +239,12 @@ func (i *TestInstallation) PreFailHandler(ctx context.Context) { i.Assertions.Require.NoError(err) } + // The kubernetes/e2e tests may use multiple namespaces, so we need to dump all of them namespaces, err := i.Actions.Kubectl().Namespaces(ctx) i.Assertions.Require.NoError(err) - proxies := []metav1.ObjectMeta{} - for _, ns := range namespaces { - proxies, err = fetchAndAddProxies(i.Actions.Kubectl(), ns, "gloo=kube-gateway", proxies) - i.Assertions.Require.NoError(err) - - proxies, err = fetchAndAddProxies(i.Actions.Kubectl(), ns, "gloo=gateway-proxy", proxies) - i.Assertions.Require.NoError(err) - } - // Dump the logs and state of the cluster - helpers.StandardGlooDumpOnFail(os.Stdout, failureDir, proxies...)() + helpers.StandardGlooDumpOnFail(os.Stdout, failureDir, namespaces)() } // GeneratedFiles is a collection of files that are generated during the execution of a set of tests @@ -291,30 +280,3 @@ func MustGeneratedFiles(tmpDirId, clusterId string) GeneratedFiles { FailureDir: failureDir, } } - -func fetchAndAddProxies(kubectl *kubectl.Cli, namespace string, label string, - proxies []metav1.ObjectMeta) ([]metav1.ObjectMeta, error) { - - stdout := bytes.NewBuffer(nil) - stderr := bytes.NewBuffer(nil) - - // get all deployments in the namespace with the label - lookupKubeGatewaysCmd := kubectl.Command(context.Background(), "get", - "deployment", "-n", namespace, "--selector", "gloo=kube-gateway", - "--no-headers", "-o", `custom-columns=":metadata.name"`) - err := lookupKubeGatewaysCmd.WithStdout(stdout).WithStderr(stderr). - Run().Cause() - if err != nil { - return nil, fmt.Errorf("failed to get proxies: %s (%s)", err, stderr.String()) - } - - // iterate lines in the output and append to the list of proxies - for _, line := range bytes.Split(stdout.Bytes(), []byte("\n")) { - proxyName := string(bytes.TrimSpace(line)) - if proxyName != "" { - proxies = append(proxies, metav1.ObjectMeta{Namespace: namespace, Name: proxyName}) - } - } - - return proxies, nil -} From 432a4b31311b53e5b7ae15e1cfa9937a3629c210 Mon Sep 17 00:00:00 2001 From: Ryan Old Date: Wed, 27 Nov 2024 10:57:13 -0800 Subject: [PATCH 14/22] Working dump --- test/helpers/kube_dump.go | 253 ++++++++++++++++++++------------------ 1 file changed, 134 insertions(+), 119 deletions(-) diff --git a/test/helpers/kube_dump.go b/test/helpers/kube_dump.go index e3705d7d2b1..222c3736cc9 100644 --- a/test/helpers/kube_dump.go +++ b/test/helpers/kube_dump.go @@ -9,9 +9,9 @@ import ( "os/exec" "path/filepath" "strings" + "time" "github.com/hashicorp/go-multierror" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/solo-io/go-utils/threadsafe" @@ -32,9 +32,16 @@ import ( // Look at `KubeDumpOnFail` && `EnvoyDumpOnFail` for more details func StandardGlooDumpOnFail(outLog io.Writer, outDir string, namespaces []string) func() { return func() { - KubeDumpOnFail(outLog, outDir, namespaces)() - ControllerDumpOnFail(outLog, outDir, namespaces)() - EnvoyDumpOnFail(outLog, outDir, namespaces)() + fmt.Printf("Test failed. Dumping state from %s...\n", strings.Join(namespaces, ", ")) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + kubeCli := kubectl.NewCli() + + KubeDumpOnFail(ctx, kubeCli, outLog, outDir, namespaces)() + ControllerDumpOnFail(ctx, kubeCli, outLog, outDir, namespaces)() + EnvoyDumpOnFail(ctx, kubeCli, outLog, outDir, namespaces)() fmt.Printf("Test failed. Logs and cluster state are available in %s\n", outDir) } @@ -48,7 +55,8 @@ func StandardGlooDumpOnFail(outLog io.Writer, outDir string, namespaces []string // - kubernetes state // - logs from all pods in the given namespaces // - yaml representations of all solo.io CRs in the given namespaces -func KubeDumpOnFail(outLog io.Writer, outDir string, namespaces []string) func() { +func KubeDumpOnFail(ctx context.Context, kubeCli *kubectl.Cli, outLog io.Writer, outDir string, + namespaces []string) func() { return func() { setupOutDir(outDir) @@ -57,6 +65,8 @@ func KubeDumpOnFail(outLog io.Writer, outDir string, namespaces []string) func() recordKubeState(fileAtPath(filepath.Join(outDir, "kube-state.log"))) recordKubeDump(outDir, namespaces...) + + fmt.Printf("Finished dumping kubernetes state\n") } } @@ -319,74 +329,37 @@ func kubeList(namespace string, target string) ([]string, string, error) { // ControllerDumpOnFail creates a small dump of the controller state when a test fails. // This is useful for debugging test failures. -func ControllerDumpOnFail(outLog io.Writer, outDir string, namespaces []string) func() { +func ControllerDumpOnFail(ctx context.Context, kubeCli *kubectl.Cli, outLog io.Writer, + outDir string, namespaces []string) func() { return func() { for _, ns := range namespaces { - namespaceOutDir := filepath.Join(outDir, ns) - setupOutDir(namespaceOutDir) - - // Get the Gloo Gateway controller logs - controllerLogsFilePath := filepath.Join(namespaceOutDir, "controller.log") - controllerLogsFile, err := os.OpenFile(controllerLogsFilePath, - os.O_APPEND|os.O_CREATE|os.O_WRONLY, os.ModePerm) - if err != nil { - fmt.Printf("error opening controller log file: %f\n", err) - } - - kubeCli := kubectl.NewCli() - - controllerLogsCmd := kubeCli.WithReceiver(controllerLogsFile).Command(context.Background(), - "-n", ns, "logs", "deployment/gloo", "-c", "gloo", "--tail=1000") - err = controllerLogsCmd.Run().Cause() + glooPodNames, err := fetchPodNamesByNsAndLabel(ctx, kubeCli, ns, "gloo=gloo") if err != nil { - fmt.Printf("error running controller logs command: %f\n", err) + fmt.Printf("error fetching controller pod names: %f\n", err) + continue } - podStdOut := bytes.NewBuffer(nil) - podStdErr := bytes.NewBuffer(nil) - - // Fetch the name of the Gloo Gateway controller pod - getGlooPodNamesCmd := kubeCli.Command(context.Background(), "get", "pod", "-n", - ns, "--selector", "gloo=gloo", "--output", "jsonpath='{.items[*].metadata.name}'") - err = getGlooPodNamesCmd.WithStdout(podStdOut).WithStderr(podStdErr).Run().Cause() - if err != nil { - fmt.Printf("error running get gloo pod name command: %f\n", err) + if len(glooPodNames) == 0 { + fmt.Printf("no controller pods found in namespace %s\n", ns) + continue } - // Clean up and check the output - glooPodNamesString := strings.TrimSpace(podStdOut.String()) - if glooPodNamesString == "" { - fmt.Printf("error getting gloo pod names: %s\n", podStdErr.String()) - return - } + fmt.Printf("found controller pods: %s\n", strings.Join(glooPodNames, ", ")) - // Split the string on whitespace to get the pod names - glooPodNames := strings.Fields(glooPodNamesString) + namespaceOutDir := filepath.Join(outDir, ns) + setupOutDir(namespaceOutDir) for _, podName := range glooPodNames { - // Get the metrics from the Gloo Gateway controller pod and write them to a file - metricsFilePath := filepath.Join(namespaceOutDir, fmt.Sprintf("%s.metrics.log", podName)) - metricsFile, err := os.OpenFile(metricsFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, os.ModePerm) - if err != nil { - fmt.Printf("error opening metrics file: %f\n", err) - } - - // Using an ephemeral debug pod fetch the metrics from the Gloo Gateway controller - metricsCmd := kubeCli.Command(context.Background(), "debug", "-n", ns, - "-it", "--image=curlimages/curl:7.83.1", glooPodNamesString, "--", - "curl", "http://localhost:9091/metrics") - err = metricsCmd.WithStdout(metricsFile).WithStderr(metricsFile).Run().Cause() - if err != nil { - fmt.Printf("error running metrics command: %f\n", err) - } + writeControllerLog(ctx, namespaceOutDir, ns, podName, kubeCli) + writeMetricsLog(ctx, namespaceOutDir, ns, podName, kubeCli) - // Open a port-forward to the Gloo Gateway controller pod's admin port - portForwarder, err := kubeCli.StartPortForward(context.Background(), - portforward.WithDeployment("gloo", ns), + // Open a port-forward to the controller pod's admin port + portForwarder, err := kubeCli.StartPortForward(ctx, + portforward.WithPod(podName, ns), portforward.WithPorts(int(admin.AdminPort), int(admin.AdminPort)), ) if err != nil { - fmt.Printf("error starting port forward: %f\n", err) + fmt.Printf("error starting port forward to controller admin port: %f\n", err) } defer func() { @@ -401,37 +374,19 @@ func ControllerDumpOnFail(outLog io.Writer, outDir string, namespaces []string) curl.WithPort(int(admin.AdminPort)), ) - // Get krt snapshot from the Gloo Gateway controller pod and write it to a file - krtSnapshotFilePath := filepath.Join(namespaceOutDir, fmt.Sprintf("%s.krt_snapshot.log", podName)) - krtSnapshotFile, err := os.OpenFile(krtSnapshotFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, - os.ModePerm) - if err != nil { - fmt.Printf("error opening krt snapshot file: %f\n", err) - } - - err = adminClient.KrtSnapshotCmd(context.Background()). - WithStdout(krtSnapshotFile). - WithStderr(krtSnapshotFile). - Run().Cause() + krtSnapshotFile := fileAtPath(filepath.Join(namespaceOutDir, fmt.Sprintf("%s.krt_snapshot.log", podName))) + err = adminClient.KrtSnapshotCmd(ctx).WithStdout(krtSnapshotFile).Run().Cause() if err != nil { fmt.Printf("error running krt snapshot command: %f\n", err) } - // Get xds snapshot from the Gloo Gateway controller pod and write it to a file - xdsSnapshotFilePath := filepath.Join(namespaceOutDir, fmt.Sprintf("%s.xds_snapshot.log", podName)) - xdsSnapshotFile, err := os.OpenFile(xdsSnapshotFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, - os.ModePerm) - if err != nil { - fmt.Printf("error opening xds snapshot file: %f\n", err) - } - - err = adminClient.XdsSnapshotCmd(context.Background()). - WithStdout(xdsSnapshotFile). - WithStderr(xdsSnapshotFile). - Run().Cause() + xdsSnapshotFile := fileAtPath(filepath.Join(namespaceOutDir, fmt.Sprintf("%s.xds_snapshot.log", podName))) + err = adminClient.XdsSnapshotCmd(ctx).WithStdout(xdsSnapshotFile).Run().Cause() if err != nil { fmt.Printf("error running xds snapshot command: %f\n", err) } + + fmt.Printf("finished dumping controller state\n") } } } @@ -444,27 +399,38 @@ func ControllerDumpOnFail(outLog io.Writer, outDir string, namespaces []string) // - stats // - clusters // - listeners -func EnvoyDumpOnFail(_ io.Writer, outDir string, namespaces []string) func() { +func EnvoyDumpOnFail(ctx context.Context, kubectl *kubectl.Cli, _ io.Writer, outDir string, namespaces []string) func() { return func() { - kubectl := kubectl.NewCli() - for _, ns := range namespaces { - proxies, err := fetchAndAddProxies(kubectl, ns, "gloo=kube-gateway", []metav1.ObjectMeta{}) + proxies := []string{} + + kubeGatewayProxies, err := fetchPodNamesByNsAndLabel(ctx, kubectl, ns, "gloo=kube-gateway") if err != nil { fmt.Printf("error fetching kube-gateway proxies: %f\n", err) + } else { + proxies = append(proxies, kubeGatewayProxies...) } - proxies, err = fetchAndAddProxies(kubectl, ns, "gloo=gateway-proxy", proxies) + gatewayProxies, err := fetchPodNamesByNsAndLabel(ctx, kubectl, ns, "gloo=gateway-proxy") if err != nil { fmt.Printf("error fetching gateway-proxy proxies: %f\n", err) + } else { + proxies = append(proxies, gatewayProxies...) } - for _, proxy := range proxies { - envoyOutDir := filepath.Join(outDir, ns) - setupOutDir(envoyOutDir) + if len(proxies) == 0 { + fmt.Printf("no proxies found in namespace %s\n", ns) + continue + } + + fmt.Printf("found proxies: %s\n", strings.Join(proxies, ", ")) - adminCli, shutdown, err := admincli.NewPortForwardedClient(context.Background(), - fmt.Sprintf("deployment/%s", proxy.GetName()), proxy.GetNamespace()) + envoyOutDir := filepath.Join(outDir, ns) + setupOutDir(envoyOutDir) + + for _, proxy := range proxies { + adminCli, shutdown, err := admincli.NewPortForwardedClient(ctx, + fmt.Sprintf("deployment/%s", proxy), ns) if err != nil { fmt.Printf("error creating admin cli: %f\n", err) return @@ -472,14 +438,31 @@ func EnvoyDumpOnFail(_ io.Writer, outDir string, namespaces []string) func() { defer shutdown() - adminCli.ConfigDumpCmd(context.Background(), nil). - WithStdout(fileAtPath(filepath.Join(envoyOutDir, "config.log"))).Run().Cause() - adminCli.StatsCmd(context.Background()). - WithStdout(fileAtPath(filepath.Join(envoyOutDir, "stats.log"))).Run().Cause() - adminCli.ClustersCmd(context.Background()). - WithStdout(fileAtPath(filepath.Join(envoyOutDir, "clusters.log"))).Run().Cause() - adminCli.ListenersCmd(context.Background()). - WithStdout(fileAtPath(filepath.Join(envoyOutDir, "listeners.log"))).Run().Cause() + configDumpFile := fileAtPath(filepath.Join(envoyOutDir, fmt.Sprintf("%s.config.log", proxy))) + err = adminCli.ConfigDumpCmd(ctx, nil).WithStdout(configDumpFile).Run().Cause() + if err != nil { + fmt.Printf("error running config dump command: %f\n", err) + } + + statsFile := fileAtPath(filepath.Join(envoyOutDir, fmt.Sprintf("%s.stats.log", proxy))) + err = adminCli.StatsCmd(ctx).WithStdout(statsFile).Run().Cause() + if err != nil { + fmt.Printf("error running stats command: %f\n", err) + } + + clustersFile := fileAtPath(filepath.Join(envoyOutDir, fmt.Sprintf("%s.clusters.log", proxy))) + err = adminCli.ClustersCmd(ctx).WithStdout(clustersFile).Run().Cause() + if err != nil { + fmt.Printf("error running clusters command: %f\n", err) + } + + listenersFile := fileAtPath(filepath.Join(envoyOutDir, fmt.Sprintf("%s.listeners.log", proxy))) + err = adminCli.ListenersCmd(ctx).WithStdout(listenersFile).Run().Cause() + if err != nil { + fmt.Printf("error running listeners command: %f\n", err) + } + + fmt.Printf("finished dumping envoy state\n") } } } @@ -506,29 +489,61 @@ func fileAtPath(path string) *os.File { return f } -func fetchAndAddProxies(kubectl *kubectl.Cli, namespace string, label string, - proxies []metav1.ObjectMeta) ([]metav1.ObjectMeta, error) { +func fetchPodNamesByNsAndLabel(ctx context.Context, kubectl *kubectl.Cli, ns, label string) ([]string, error) { + podStdOut := bytes.NewBuffer(nil) + podStdErr := bytes.NewBuffer(nil) + + // Fetch the name of the Gloo Gateway controller pod + getGlooPodNamesCmd := kubectl.Command(ctx, "get", "pod", "-n", ns, + "--selector", label, "--output", "jsonpath='{.items[*].metadata.name}'") + err := getGlooPodNamesCmd.WithStdout(podStdOut).WithStderr(podStdErr).Run().Cause() + if err != nil { + fmt.Printf("error running get gloo pod name command: %v\n", err) + } + + // Clean up and check the output + glooPodNamesString := strings.Trim(podStdOut.String(), "'") + if glooPodNamesString == "" { + fmt.Printf("no %s pods found in namespace %s\n", label, ns) + return []string{}, nil + } - stdout := bytes.NewBuffer(nil) - stderr := bytes.NewBuffer(nil) + // Split the string on whitespace to get the pod names + glooPodNames := strings.Fields(glooPodNamesString) + return glooPodNames, nil +} - // get all deployments in the namespace with the label - lookupKubeGatewaysCmd := kubectl.Command(context.Background(), "get", - "deployment", "-n", namespace, "--selector", "gloo=kube-gateway", - "--no-headers", "-o", `custom-columns=":metadata.name"`) - err := lookupKubeGatewaysCmd.WithStdout(stdout).WithStderr(stderr). - Run().Cause() +func writeControllerLog(ctx context.Context, outDir string, ns string, podName string, kubectl *kubectl.Cli) { + // Get the Gloo Gateway controller logs + controllerLogsFilePath := filepath.Join(outDir, fmt.Sprintf("%s.controller.log", podName)) + controllerLogsFile, err := os.OpenFile(controllerLogsFilePath, + os.O_APPEND|os.O_CREATE|os.O_WRONLY, os.ModePerm) if err != nil { - return proxies, fmt.Errorf("failed to get proxies: %s (%s)", err, stderr.String()) + fmt.Printf("error opening controller log file: %f\n", err) } - // iterate lines in the output and append to the list of proxies - for _, line := range bytes.Split(stdout.Bytes(), []byte("\n")) { - proxyName := string(bytes.TrimSpace(line)) - if proxyName != "" { - proxies = append(proxies, metav1.ObjectMeta{Namespace: namespace, Name: proxyName}) - } + controllerLogsCmd := kubectl.WithReceiver(controllerLogsFile).Command(ctx, + "-n", ns, "logs", podName, "-c", "gloo", "--tail=1000") + err = controllerLogsCmd.Run().Cause() + if err != nil { + fmt.Printf("error running controller logs for %s in %s command: %v\n", podName, ns, err) } +} - return proxies, nil +func writeMetricsLog(ctx context.Context, outDir string, ns string, podName string, kubectl *kubectl.Cli) { + // Get the metrics from the Gloo Gateway controller pod and write them to a file + metricsFilePath := filepath.Join(outDir, fmt.Sprintf("%s.metrics.log", podName)) + metricsFile, err := os.OpenFile(metricsFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, os.ModePerm) + if err != nil { + fmt.Printf("error opening metrics file: %f\n", err) + } + + // Using an ephemeral debug pod fetch the metrics from the Gloo Gateway controller + metricsCmd := kubectl.Command(ctx, "debug", "-n", ns, + "-it", "--image=curlimages/curl:7.83.1", podName, "--", + "curl", "http://localhost:9091/metrics") + err = metricsCmd.WithStdout(metricsFile).WithStderr(metricsFile).Run().Cause() + if err != nil { + fmt.Printf("error running metrics command: %f\n", err) + } } From 690e5beff916dc1f105e570fc0c3c7a618a2f57c Mon Sep 17 00:00:00 2001 From: Ryan Old Date: Wed, 27 Nov 2024 11:03:58 -0800 Subject: [PATCH 15/22] Intentionally fail a kube2e test --- test/kube2e/gloo/happypath_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/kube2e/gloo/happypath_test.go b/test/kube2e/gloo/happypath_test.go index 6aefb1ddb18..6bfb0ade41b 100644 --- a/test/kube2e/gloo/happypath_test.go +++ b/test/kube2e/gloo/happypath_test.go @@ -257,6 +257,8 @@ var _ = Describe("Happy path", func() { Expect(err).NotTo(HaveOccurred()) TestUpstreamReachable() + + Expect(true).NotTo(BeTrue(), "intentionally failing a test will remove before merging") }) }) }) From 068f973654b599617531c8a32f13f412b3847911 Mon Sep 17 00:00:00 2001 From: Ryan Old Date: Wed, 27 Nov 2024 11:04:17 -0800 Subject: [PATCH 16/22] Minor log adjustment --- test/helpers/kube_dump.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/helpers/kube_dump.go b/test/helpers/kube_dump.go index 222c3736cc9..e50db6dc3f9 100644 --- a/test/helpers/kube_dump.go +++ b/test/helpers/kube_dump.go @@ -340,7 +340,7 @@ func ControllerDumpOnFail(ctx context.Context, kubeCli *kubectl.Cli, outLog io.W } if len(glooPodNames) == 0 { - fmt.Printf("no controller pods found in namespace %s\n", ns) + fmt.Printf("no gloo=gloo pods found in namespace %s\n", ns) continue } From afe6f109417cc75c524e3a303eaa7479575f8d2d Mon Sep 17 00:00:00 2001 From: Ryan Old Date: Wed, 27 Nov 2024 11:36:14 -0800 Subject: [PATCH 17/22] Clean up --- .../collect-more-artifacts-on-ci-failure.yaml | 5 + pkg/utils/kubeutils/kubectl/cli.go | 25 +++++ test/helpers/kube_dump.go | 91 ++++++------------- 3 files changed, 56 insertions(+), 65 deletions(-) diff --git a/changelog/v1.18.0-rc3/collect-more-artifacts-on-ci-failure.yaml b/changelog/v1.18.0-rc3/collect-more-artifacts-on-ci-failure.yaml index 6e1cb3b66c0..10a86e4c627 100644 --- a/changelog/v1.18.0-rc3/collect-more-artifacts-on-ci-failure.yaml +++ b/changelog/v1.18.0-rc3/collect-more-artifacts-on-ci-failure.yaml @@ -7,3 +7,8 @@ changelog: about the state of the cluster, we have added additional artifacts to be collected when a test fails. This will help us to more easily diagnose the cause of test failures. + - type: NON_USER_FACING + description: >- + Unified the kube2e and kubernetes/e2e test failure artifact collection. + Previously, the test failure artifacts for kube2e and kubernetes/e2e tests were different + and produced by their own logic. diff --git a/pkg/utils/kubeutils/kubectl/cli.go b/pkg/utils/kubeutils/kubectl/cli.go index b78a115afe9..3332675f080 100644 --- a/pkg/utils/kubeutils/kubectl/cli.go +++ b/pkg/utils/kubeutils/kubectl/cli.go @@ -342,3 +342,28 @@ func (c *Cli) GetContainerLogs(ctx context.Context, namespace string, name strin stdout, stderr, err := c.Execute(ctx, "-n", namespace, "logs", name) return stdout + stderr, err } + +// GetPodsInNsWithLabel returns the pods in the specified namespace with the specified label +func (c *Cli) GetPodsInNsWithLabel(ctx context.Context, namespace string, label string) ([]string, error) { + podStdOut := bytes.NewBuffer(nil) + podStdErr := bytes.NewBuffer(nil) + + // Fetch the name of the Gloo Gateway controller pod + getGlooPodNamesCmd := c.Command(ctx, "get", "pod", "-n", namespace, + "--selector", label, "--output", "jsonpath='{.items[*].metadata.name}'") + err := getGlooPodNamesCmd.WithStdout(podStdOut).WithStderr(podStdErr).Run().Cause() + if err != nil { + fmt.Printf("error running get gloo pod name command: %v\n", err) + } + + // Clean up and check the output + glooPodNamesString := strings.Trim(podStdOut.String(), "'") + if glooPodNamesString == "" { + fmt.Printf("no %s pods found in namespace %s\n", label, namespace) + return []string{}, nil + } + + // Split the string on whitespace to get the pod names + glooPodNames := strings.Fields(glooPodNamesString) + return glooPodNames, nil +} diff --git a/test/helpers/kube_dump.go b/test/helpers/kube_dump.go index e50db6dc3f9..8b11c0c6edf 100644 --- a/test/helpers/kube_dump.go +++ b/test/helpers/kube_dump.go @@ -12,19 +12,16 @@ import ( "time" "github.com/hashicorp/go-multierror" - - "github.com/solo-io/go-utils/threadsafe" - - "github.com/solo-io/gloo/pkg/utils/kubeutils/kubectl" - "github.com/solo-io/gloo/pkg/utils/kubeutils/portforward" - "github.com/solo-io/gloo/pkg/utils/requestutils/curl" - "github.com/onsi/ginkgo/v2" + "github.com/solo-io/gloo/pkg/cliutil/install" "github.com/solo-io/gloo/pkg/utils/envoyutils/admincli" - glooAdminCli "github.com/solo-io/gloo/pkg/utils/glooadminutils/admincli" + "github.com/solo-io/gloo/pkg/utils/kubeutils/kubectl" + "github.com/solo-io/gloo/pkg/utils/kubeutils/portforward" + "github.com/solo-io/gloo/pkg/utils/requestutils/curl" "github.com/solo-io/gloo/projects/gloo/pkg/servers/admin" + "github.com/solo-io/go-utils/threadsafe" ) // StandardGlooDumpOnFail creates adump of the kubernetes state and certain envoy data from @@ -37,11 +34,11 @@ func StandardGlooDumpOnFail(outLog io.Writer, outDir string, namespaces []string ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) defer cancel() - kubeCli := kubectl.NewCli() + kubectlCli := kubectl.NewCli() - KubeDumpOnFail(ctx, kubeCli, outLog, outDir, namespaces)() - ControllerDumpOnFail(ctx, kubeCli, outLog, outDir, namespaces)() - EnvoyDumpOnFail(ctx, kubeCli, outLog, outDir, namespaces)() + KubeDumpOnFail(ctx, kubectlCli, outLog, outDir, namespaces)() + ControllerDumpOnFail(ctx, kubectlCli, outLog, outDir, namespaces)() + EnvoyDumpOnFail(ctx, kubectlCli, outLog, outDir, namespaces)() fmt.Printf("Test failed. Logs and cluster state are available in %s\n", outDir) } @@ -55,7 +52,7 @@ func StandardGlooDumpOnFail(outLog io.Writer, outDir string, namespaces []string // - kubernetes state // - logs from all pods in the given namespaces // - yaml representations of all solo.io CRs in the given namespaces -func KubeDumpOnFail(ctx context.Context, kubeCli *kubectl.Cli, outLog io.Writer, outDir string, +func KubeDumpOnFail(ctx context.Context, kubectlCli *kubectl.Cli, outLog io.Writer, outDir string, namespaces []string) func() { return func() { setupOutDir(outDir) @@ -329,11 +326,11 @@ func kubeList(namespace string, target string) ([]string, string, error) { // ControllerDumpOnFail creates a small dump of the controller state when a test fails. // This is useful for debugging test failures. -func ControllerDumpOnFail(ctx context.Context, kubeCli *kubectl.Cli, outLog io.Writer, +func ControllerDumpOnFail(ctx context.Context, kubectlCli *kubectl.Cli, outLog io.Writer, outDir string, namespaces []string) func() { return func() { for _, ns := range namespaces { - glooPodNames, err := fetchPodNamesByNsAndLabel(ctx, kubeCli, ns, "gloo=gloo") + glooPodNames, err := kubectlCli.GetPodsInNsWithLabel(ctx, ns, "gloo=gloo") if err != nil { fmt.Printf("error fetching controller pod names: %f\n", err) continue @@ -350,11 +347,11 @@ func ControllerDumpOnFail(ctx context.Context, kubeCli *kubectl.Cli, outLog io.W setupOutDir(namespaceOutDir) for _, podName := range glooPodNames { - writeControllerLog(ctx, namespaceOutDir, ns, podName, kubeCli) - writeMetricsLog(ctx, namespaceOutDir, ns, podName, kubeCli) + writeControllerLog(ctx, namespaceOutDir, ns, podName, kubectlCli) + writeMetricsLog(ctx, namespaceOutDir, ns, podName, kubectlCli) // Open a port-forward to the controller pod's admin port - portForwarder, err := kubeCli.StartPortForward(ctx, + portForwarder, err := kubectlCli.StartPortForward(ctx, portforward.WithPod(podName, ns), portforward.WithPorts(int(admin.AdminPort), int(admin.AdminPort)), ) @@ -399,19 +396,19 @@ func ControllerDumpOnFail(ctx context.Context, kubeCli *kubectl.Cli, outLog io.W // - stats // - clusters // - listeners -func EnvoyDumpOnFail(ctx context.Context, kubectl *kubectl.Cli, _ io.Writer, outDir string, namespaces []string) func() { +func EnvoyDumpOnFail(ctx context.Context, kubectlCli *kubectl.Cli, _ io.Writer, outDir string, namespaces []string) func() { return func() { for _, ns := range namespaces { proxies := []string{} - kubeGatewayProxies, err := fetchPodNamesByNsAndLabel(ctx, kubectl, ns, "gloo=kube-gateway") + kubeGatewayProxies, err := kubectlCli.GetPodsInNsWithLabel(ctx, ns, "gloo=kube-gateway") if err != nil { fmt.Printf("error fetching kube-gateway proxies: %f\n", err) } else { proxies = append(proxies, kubeGatewayProxies...) } - gatewayProxies, err := fetchPodNamesByNsAndLabel(ctx, kubectl, ns, "gloo=gateway-proxy") + gatewayProxies, err := kubectlCli.GetPodsInNsWithLabel(ctx, ns, "gloo=gateway-proxy") if err != nil { fmt.Printf("error fetching gateway-proxy proxies: %f\n", err) } else { @@ -489,60 +486,24 @@ func fileAtPath(path string) *os.File { return f } -func fetchPodNamesByNsAndLabel(ctx context.Context, kubectl *kubectl.Cli, ns, label string) ([]string, error) { - podStdOut := bytes.NewBuffer(nil) - podStdErr := bytes.NewBuffer(nil) - - // Fetch the name of the Gloo Gateway controller pod - getGlooPodNamesCmd := kubectl.Command(ctx, "get", "pod", "-n", ns, - "--selector", label, "--output", "jsonpath='{.items[*].metadata.name}'") - err := getGlooPodNamesCmd.WithStdout(podStdOut).WithStderr(podStdErr).Run().Cause() - if err != nil { - fmt.Printf("error running get gloo pod name command: %v\n", err) - } - - // Clean up and check the output - glooPodNamesString := strings.Trim(podStdOut.String(), "'") - if glooPodNamesString == "" { - fmt.Printf("no %s pods found in namespace %s\n", label, ns) - return []string{}, nil - } - - // Split the string on whitespace to get the pod names - glooPodNames := strings.Fields(glooPodNamesString) - return glooPodNames, nil -} - -func writeControllerLog(ctx context.Context, outDir string, ns string, podName string, kubectl *kubectl.Cli) { +func writeControllerLog(ctx context.Context, outDir string, ns string, podName string, kubectlCli *kubectl.Cli) { // Get the Gloo Gateway controller logs - controllerLogsFilePath := filepath.Join(outDir, fmt.Sprintf("%s.controller.log", podName)) - controllerLogsFile, err := os.OpenFile(controllerLogsFilePath, - os.O_APPEND|os.O_CREATE|os.O_WRONLY, os.ModePerm) - if err != nil { - fmt.Printf("error opening controller log file: %f\n", err) - } - - controllerLogsCmd := kubectl.WithReceiver(controllerLogsFile).Command(ctx, + controllerLogsFile := fileAtPath(filepath.Join(outDir, fmt.Sprintf("%s.controller.log", podName))) + controllerLogsCmd := kubectlCli.WithReceiver(controllerLogsFile).Command(ctx, "-n", ns, "logs", podName, "-c", "gloo", "--tail=1000") - err = controllerLogsCmd.Run().Cause() + err := controllerLogsCmd.Run().Cause() if err != nil { fmt.Printf("error running controller logs for %s in %s command: %v\n", podName, ns, err) } } -func writeMetricsLog(ctx context.Context, outDir string, ns string, podName string, kubectl *kubectl.Cli) { - // Get the metrics from the Gloo Gateway controller pod and write them to a file - metricsFilePath := filepath.Join(outDir, fmt.Sprintf("%s.metrics.log", podName)) - metricsFile, err := os.OpenFile(metricsFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, os.ModePerm) - if err != nil { - fmt.Printf("error opening metrics file: %f\n", err) - } - +func writeMetricsLog(ctx context.Context, outDir string, ns string, podName string, kubectlCli *kubectl.Cli) { // Using an ephemeral debug pod fetch the metrics from the Gloo Gateway controller - metricsCmd := kubectl.Command(ctx, "debug", "-n", ns, + metricsFile := fileAtPath(filepath.Join(outDir, fmt.Sprintf("%s.metrics.log", podName))) + metricsCmd := kubectlCli.Command(ctx, "debug", "-n", ns, "-it", "--image=curlimages/curl:7.83.1", podName, "--", "curl", "http://localhost:9091/metrics") - err = metricsCmd.WithStdout(metricsFile).WithStderr(metricsFile).Run().Cause() + err := metricsCmd.WithStdout(metricsFile).WithStderr(metricsFile).Run().Cause() if err != nil { fmt.Printf("error running metrics command: %f\n", err) } From 6709c45fc6cf3e0100b601e36f0ff35df3439c78 Mon Sep 17 00:00:00 2001 From: Ryan Old Date: Wed, 27 Nov 2024 11:57:23 -0800 Subject: [PATCH 18/22] Fixing import cycle --- .../pkg/servers/iosnapshot/history_test.go | 37 ++++++++++--------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/projects/gloo/pkg/servers/iosnapshot/history_test.go b/projects/gloo/pkg/servers/iosnapshot/history_test.go index e0209632284..2a4aef8cbf8 100644 --- a/projects/gloo/pkg/servers/iosnapshot/history_test.go +++ b/projects/gloo/pkg/servers/iosnapshot/history_test.go @@ -1,4 +1,4 @@ -package iosnapshot +package iosnapshot_test import ( "context" @@ -8,6 +8,7 @@ import ( gomegatypes "github.com/onsi/gomega/types" "github.com/solo-io/gloo/pkg/schemes" gloov1 "github.com/solo-io/gloo/projects/gloo/pkg/api/v1/kube/apis/gloo.solo.io/v1" + "github.com/solo-io/gloo/projects/gloo/pkg/servers/iosnapshot" apiv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" wellknownkube "github.com/solo-io/gloo/projects/gloo/pkg/api/v1/kube/wellknown" @@ -52,16 +53,16 @@ var _ = Describe("History", func() { ctx context.Context clientBuilder *fake.ClientBuilder - history History + history iosnapshot.History - historyFactorParams HistoryFactoryParameters + historyFactorParams iosnapshot.HistoryFactoryParameters ) BeforeEach(func() { ctx = context.Background() clientBuilder = fake.NewClientBuilder().WithScheme(schemes.DefaultScheme()) - historyFactorParams = HistoryFactoryParameters{ + historyFactorParams = iosnapshot.HistoryFactoryParameters{ Settings: &v1.Settings{ Metadata: &core.Metadata{ Name: "my-settings", @@ -98,11 +99,11 @@ var _ = Describe("History", func() { }, } - history = NewHistory( + history = iosnapshot.NewHistory( historyFactorParams.Cache, historyFactorParams.Settings, clientBuilder.WithObjects(clientObjects...).Build(), - append(CompleteInputSnapshotGVKs, deploymentGvk), // include the Deployment GVK + append(iosnapshot.CompleteInputSnapshotGVKs, deploymentGvk), // include the Deployment GVK ) }) @@ -136,7 +137,7 @@ var _ = Describe("History", func() { }, } - history = NewHistory(&xds.MockXdsCache{}, + history = iosnapshot.NewHistory(&xds.MockXdsCache{}, &v1.Settings{ Metadata: &core.Metadata{ Name: "my-settings", @@ -144,7 +145,7 @@ var _ = Describe("History", func() { }, }, clientBuilder.WithObjects(clientObjects...).Build(), - CompleteInputSnapshotGVKs, // do not include the Deployment GVK + iosnapshot.CompleteInputSnapshotGVKs, // do not include the Deployment GVK ) }) @@ -377,11 +378,11 @@ var _ = Describe("History", func() { }, } - history = NewHistory( + history = iosnapshot.NewHistory( historyFactorParams.Cache, historyFactorParams.Settings, clientBuilder.WithObjects(clientObjects...).Build(), - CompleteInputSnapshotGVKs) + iosnapshot.CompleteInputSnapshotGVKs) }) Context("Kubernetes Core Resources", func() { @@ -663,11 +664,11 @@ var _ = Describe("History", func() { Context("GetEdgeApiSnapshot", func() { BeforeEach(func() { - history = NewHistory( + history = iosnapshot.NewHistory( historyFactorParams.Cache, historyFactorParams.Settings, clientBuilder.Build(), // no objects, because this API doesn't rely on the kube client - CompleteInputSnapshotGVKs, + iosnapshot.CompleteInputSnapshotGVKs, ) }) @@ -758,11 +759,11 @@ var _ = Describe("History", func() { Context("GetProxySnapshot", func() { BeforeEach(func() { - history = NewHistory( + history = iosnapshot.NewHistory( historyFactorParams.Cache, historyFactorParams.Settings, clientBuilder.Build(), // no objects, because this API doesn't rely on the kube client - CompleteInputSnapshotGVKs, + iosnapshot.CompleteInputSnapshotGVKs, ) }) @@ -794,7 +795,7 @@ var _ = Describe("History", func() { }) -func getInputSnapshotObjects(ctx context.Context, history History) []client.Object { +func getInputSnapshotObjects(ctx context.Context, history iosnapshot.History) []client.Object { snapshotResponse := history.GetInputSnapshot(ctx) Expect(snapshotResponse.Error).NotTo(HaveOccurred()) @@ -804,7 +805,7 @@ func getInputSnapshotObjects(ctx context.Context, history History) []client.Obje return responseObjects } -func getProxySnapshotResources(ctx context.Context, history History) []crdv1.Resource { +func getProxySnapshotResources(ctx context.Context, history iosnapshot.History) []crdv1.Resource { snapshotResponse := history.GetProxySnapshot(ctx) Expect(snapshotResponse.Error).NotTo(HaveOccurred()) @@ -814,7 +815,7 @@ func getProxySnapshotResources(ctx context.Context, history History) []crdv1.Res return responseObjects } -func getEdgeApiSnapshot(ctx context.Context, history History) *v1snap.ApiSnapshot { +func getEdgeApiSnapshot(ctx context.Context, history iosnapshot.History) *v1snap.ApiSnapshot { snapshotResponse := history.GetEdgeApiSnapshot(ctx) Expect(snapshotResponse.Error).NotTo(HaveOccurred()) @@ -827,7 +828,7 @@ func getEdgeApiSnapshot(ctx context.Context, history History) *v1snap.ApiSnapsho // setSnapshotOnHistory sets the ApiSnapshot on the history, and blocks until it has been processed // This is a utility method to help developers write tests, without having to worry about the asynchronous // nature of the `Set` API on the History -func setSnapshotOnHistory(ctx context.Context, history History, snap *v1snap.ApiSnapshot) { +func setSnapshotOnHistory(ctx context.Context, history iosnapshot.History, snap *v1snap.ApiSnapshot) { gwSignal := &gatewayv1.Gateway{ // We append a custom Gateway to the Snapshot, and then use that object // to verify the Snapshot has been processed From d184371c394f46f7a8526f56f3ccc58182ab0a41 Mon Sep 17 00:00:00 2001 From: Ryan Old Date: Wed, 27 Nov 2024 12:48:41 -0800 Subject: [PATCH 19/22] Adjusting the port forwarding for fetching envoy state --- test/helpers/kube_dump.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/helpers/kube_dump.go b/test/helpers/kube_dump.go index 8b11c0c6edf..5b7b1c0f436 100644 --- a/test/helpers/kube_dump.go +++ b/test/helpers/kube_dump.go @@ -427,7 +427,7 @@ func EnvoyDumpOnFail(ctx context.Context, kubectlCli *kubectl.Cli, _ io.Writer, for _, proxy := range proxies { adminCli, shutdown, err := admincli.NewPortForwardedClient(ctx, - fmt.Sprintf("deployment/%s", proxy), ns) + fmt.Sprintf("pod/%s", proxy), ns) if err != nil { fmt.Printf("error creating admin cli: %f\n", err) return From 3f6369736290207bcda8ea59124881c9ed04b7ee Mon Sep 17 00:00:00 2001 From: Ryan Old Date: Wed, 27 Nov 2024 12:50:35 -0800 Subject: [PATCH 20/22] Making the envoy dumping a little more fault tollerant --- test/helpers/kube_dump.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/helpers/kube_dump.go b/test/helpers/kube_dump.go index 5b7b1c0f436..85ede47a83b 100644 --- a/test/helpers/kube_dump.go +++ b/test/helpers/kube_dump.go @@ -326,6 +326,11 @@ func kubeList(namespace string, target string) ([]string, string, error) { // ControllerDumpOnFail creates a small dump of the controller state when a test fails. // This is useful for debugging test failures. +// The dump includes: +// - controller logs +// - controller metrics +// - controller xds snapshot +// - controller krt snapshot func ControllerDumpOnFail(ctx context.Context, kubectlCli *kubectl.Cli, outLog io.Writer, outDir string, namespaces []string) func() { return func() { @@ -430,7 +435,7 @@ func EnvoyDumpOnFail(ctx context.Context, kubectlCli *kubectl.Cli, _ io.Writer, fmt.Sprintf("pod/%s", proxy), ns) if err != nil { fmt.Printf("error creating admin cli: %f\n", err) - return + continue } defer shutdown() From 8c494c4aa85518ed0677b421a414ba92d1882f5a Mon Sep 17 00:00:00 2001 From: Ryan Old Date: Wed, 27 Nov 2024 13:57:50 -0800 Subject: [PATCH 21/22] Don't wipe while making subdirs --- test/helpers/kube_dump.go | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/test/helpers/kube_dump.go b/test/helpers/kube_dump.go index 85ede47a83b..95e74d141b1 100644 --- a/test/helpers/kube_dump.go +++ b/test/helpers/kube_dump.go @@ -36,6 +36,9 @@ func StandardGlooDumpOnFail(outLog io.Writer, outDir string, namespaces []string kubectlCli := kubectl.NewCli() + // only wipe at the start of the dump + wipeOutDir(outDir) + KubeDumpOnFail(ctx, kubectlCli, outLog, outDir, namespaces)() ControllerDumpOnFail(ctx, kubectlCli, outLog, outDir, namespaces)() EnvoyDumpOnFail(ctx, kubectlCli, outLog, outDir, namespaces)() @@ -470,13 +473,16 @@ func EnvoyDumpOnFail(ctx context.Context, kubectlCli *kubectl.Cli, _ io.Writer, } } -// setupOutDir forcibly deletes/creates the output directory -func setupOutDir(outdir string) { - err := os.RemoveAll(outdir) +func wipeOutDir(outDir string) { + err := os.RemoveAll(outDir) if err != nil { - fmt.Printf("error removing log directory: %f\n", err) + fmt.Printf("error wiping out directory: %f\n", err) } - err = os.MkdirAll(outdir, os.ModePerm) +} + +// setupOutDir forcibly deletes/creates the output directory +func setupOutDir(outdir string) { + err := os.MkdirAll(outdir, os.ModePerm) if err != nil { fmt.Printf("error creating log directory: %f\n", err) } From 04c342b5324fb8578aa5320e42626e1094c43ec5 Mon Sep 17 00:00:00 2001 From: Ryan Old Date: Wed, 27 Nov 2024 14:49:00 -0800 Subject: [PATCH 22/22] Removed intentional failures --- test/kube2e/gloo/happypath_test.go | 2 -- .../e2e/features/virtualhost_options/vhost_opt_suite.go | 2 -- 2 files changed, 4 deletions(-) diff --git a/test/kube2e/gloo/happypath_test.go b/test/kube2e/gloo/happypath_test.go index 6bfb0ade41b..6aefb1ddb18 100644 --- a/test/kube2e/gloo/happypath_test.go +++ b/test/kube2e/gloo/happypath_test.go @@ -257,8 +257,6 @@ var _ = Describe("Happy path", func() { Expect(err).NotTo(HaveOccurred()) TestUpstreamReachable() - - Expect(true).NotTo(BeTrue(), "intentionally failing a test will remove before merging") }) }) }) diff --git a/test/kubernetes/e2e/features/virtualhost_options/vhost_opt_suite.go b/test/kubernetes/e2e/features/virtualhost_options/vhost_opt_suite.go index 506b706859f..51d1c1fb336 100644 --- a/test/kubernetes/e2e/features/virtualhost_options/vhost_opt_suite.go +++ b/test/kubernetes/e2e/features/virtualhost_options/vhost_opt_suite.go @@ -283,8 +283,6 @@ func (s *testingSuite) TestConfigureVirtualHostOptionsWithSectionNameManualSetup ), Body: gstruct.Ignore(), }) - - s.Assert().Equal(true, false, "intentionally failing to trigger dump, remove when done debugging") } // TestMultipleVirtualHostOptionsSetup tests a complex scenario where multiple VirtualHostOptions conflict