From 78ef5303cc61ce2ed30b9354be4f7f265a9e1909 Mon Sep 17 00:00:00 2001 From: Tom Pantelis Date: Thu, 21 Nov 2024 14:51:44 -0500 Subject: [PATCH] Output init containers in subctl gather Refactored "gatherPodLogs" to take the desired container names. If none specified, gather logs for all init and normal containers confgiured in the pod. Also, if a pod fails, it's useful to see the pod details so gather the YAML for pod resources for daemonsets and deployments. Signed-off-by: Tom Pantelis --- internal/gather/connectivity.go | 4 +- internal/gather/logs.go | 51 +++++++++++++++++--------- internal/gather/resource.go | 2 + internal/gather/servicediscovery.go | 2 +- scripts/test/lib_subctl_gather_test.sh | 19 +++++++--- scripts/test/system.sh | 2 +- 6 files changed, 54 insertions(+), 26 deletions(-) diff --git a/internal/gather/connectivity.go b/internal/gather/connectivity.go index 7a987876e..fa03ec6ea 100644 --- a/internal/gather/connectivity.go +++ b/internal/gather/connectivity.go @@ -40,10 +40,10 @@ func gatherGatewayPodLogs(info *Info) { } func gatherMetricsProxyPodLogs(info *Info) { - gatherPodLogsByContainer(metricsProxyPodLabel, "gateway-metrics-proxy", info) + gatherPodLogs(metricsProxyPodLabel, info, "gateway-metrics-proxy") if info.Submariner.Spec.GlobalCIDR != "" { - gatherPodLogsByContainer(metricsProxyPodLabel, "globalnet-metrics-proxy", info) + gatherPodLogs(metricsProxyPodLabel, info, "globalnet-metrics-proxy") } } diff --git a/internal/gather/logs.go b/internal/gather/logs.go index ac776b4dd..a8ce28208 100644 --- a/internal/gather/logs.go +++ b/internal/gather/logs.go @@ -31,11 +31,7 @@ import ( "k8s.io/client-go/kubernetes" ) -func gatherPodLogs(podLabelSelector string, info *Info) { - gatherPodLogsByContainer(podLabelSelector, "", info) -} - -func gatherPodLogsByContainer(podLabelSelector, container string, info *Info) { +func gatherPodLogs(podLabelSelector string, info *Info, fromContainers ...string) { err := func() error { pods, err := findPods(info.ClientProducer.ForKubernetes(), podLabelSelector) if err != nil { @@ -44,11 +40,23 @@ func gatherPodLogsByContainer(podLabelSelector, container string, info *Info) { info.Status.Success("Found %d pods matching label selector %q", len(pods.Items), podLabelSelector) - podLogOptions := corev1.PodLogOptions{ - Container: container, - } for i := range pods.Items { - info.Summary.PodLogs = append(info.Summary.PodLogs, outputPodLogs(&pods.Items[i], podLogOptions, info)) + containers := fromContainers + if len(containers) == 0 { + containers = append(getContainerNames(pods.Items[i].Spec.InitContainers), + getContainerNames(pods.Items[i].Spec.Containers)...) + } + + for _, container := range containers { + logName := pods.Items[i].Name + if len(containers) > 1 { + logName = logName + "-" + container + } + + info.Summary.PodLogs = append(info.Summary.PodLogs, outputPodLogs(&pods.Items[i], corev1.PodLogOptions{ + Container: container, + }, logName, info)) + } } return nil @@ -59,19 +67,28 @@ func gatherPodLogsByContainer(podLabelSelector, container string, info *Info) { } } +func getContainerNames(containers []corev1.Container) []string { + names := make([]string, len(containers)) + for i := range containers { + names[i] = containers[i].Name + } + + return names +} + //nolint:gocritic // hugeParam: podLogOptions - purposely passed by value. -func outputPodLogs(pod *corev1.Pod, podLogOptions corev1.PodLogOptions, info *Info) (podLogInfo LogInfo) { +func outputPodLogs(pod *corev1.Pod, podLogOptions corev1.PodLogOptions, logName string, info *Info) (podLogInfo LogInfo) { podLogInfo.Namespace = pod.Namespace podLogInfo.PodState = pod.Status.Phase podLogInfo.PodName = pod.Name podLogInfo.NodeName = pod.Spec.NodeName - err := outputPreviousPodLog(pod, podLogOptions, info, &podLogInfo) + err := outputPreviousPodLog(pod, podLogOptions, logName, info, &podLogInfo) if err != nil { info.Status.Failure("Error outputting previous log for pod %q: %v", pod.Name, err) } - err = outputCurrentPodLog(pod, podLogOptions, info, &podLogInfo) + err = outputCurrentPodLog(pod, podLogOptions, logName, info, &podLogInfo) if err != nil { info.Status.Failure("Error outputting current log for pod %q: %v", pod.Name, err) } @@ -129,7 +146,7 @@ func findPods(clientSet kubernetes.Interface, byLabelSelector string) (*corev1.P } //nolint:gocritic // hugeParam: podLogOptions - purposely passed by value. -func outputPreviousPodLog(pod *corev1.Pod, podLogOptions corev1.PodLogOptions, info *Info, podLogInfo *LogInfo) error { +func outputPreviousPodLog(pod *corev1.Pod, podLogOptions corev1.PodLogOptions, logName string, info *Info, podLogInfo *LogInfo) error { podLogOptions.Previous = true logRequest := info.ClientProducer.ForKubernetes().CoreV1().Pods(pod.Namespace).GetLogs(pod.Name, &podLogOptions) logStream, _ := logRequest.Stream(context.TODO()) @@ -138,9 +155,9 @@ func outputPreviousPodLog(pod *corev1.Pod, podLogOptions corev1.PodLogOptions, i // if no previous pods found, logstream == nil, ignore it if logStream != nil { - info.Status.Warning("Found logs for previous instances of pod %s", pod.Name) + info.Status.Warning("Found logs for previous instances of pod %q", pod.Name) - fileName, err := writePodLogToFile(logStream, info, pod.Name, ".log.prev") + fileName, err := writePodLogToFile(logStream, info, logName, ".log.prev") if err != nil { return err } @@ -158,7 +175,7 @@ func outputPreviousPodLog(pod *corev1.Pod, podLogOptions corev1.PodLogOptions, i } //nolint:gocritic // hugeParam: podLogOptions - purposely passed by value. -func outputCurrentPodLog(pod *corev1.Pod, podLogOptions corev1.PodLogOptions, info *Info, podLogInfo *LogInfo) error { +func outputCurrentPodLog(pod *corev1.Pod, podLogOptions corev1.PodLogOptions, logName string, info *Info, podLogInfo *LogInfo) error { // Running with Previous = false on the same pod podLogOptions.Previous = false logRequest := info.ClientProducer.ForKubernetes().CoreV1().Pods(pod.Namespace).GetLogs(pod.Name, &podLogOptions) @@ -170,7 +187,7 @@ func outputCurrentPodLog(pod *corev1.Pod, podLogOptions corev1.PodLogOptions, in defer logStream.Close() - fileName, err := writePodLogToFile(logStream, info, pod.Name, ".log") + fileName, err := writePodLogToFile(logStream, info, logName, ".log") podLogInfo.LogFileName = append(podLogInfo.LogFileName, fileName) return err diff --git a/internal/gather/resource.go b/internal/gather/resource.go index f7adead0a..dd9a5c8cd 100644 --- a/internal/gather/resource.go +++ b/internal/gather/resource.go @@ -97,11 +97,13 @@ func ResourcesToYAMLFile(info *Info, ofType schema.GroupVersionResource, namespa //nolint:gocritic // hugeParam: listOptions - match K8s API. func gatherDaemonSet(info *Info, namespace string, listOptions metav1.ListOptions) { ResourcesToYAMLFile(info, appsv1.SchemeGroupVersion.WithResource("daemonsets"), namespace, listOptions) + ResourcesToYAMLFile(info, corev1.SchemeGroupVersion.WithResource("pods"), namespace, listOptions) } //nolint:gocritic // hugeParam: listOptions - match K8s API. func gatherDeployment(info *Info, namespace string, listOptions metav1.ListOptions) { ResourcesToYAMLFile(info, appsv1.SchemeGroupVersion.WithResource("deployments"), namespace, listOptions) + ResourcesToYAMLFile(info, corev1.SchemeGroupVersion.WithResource("pods"), namespace, listOptions) } //nolint:gocritic // hugeParam: listOptions - match K8s API. diff --git a/internal/gather/servicediscovery.go b/internal/gather/servicediscovery.go index 959800817..1e5b43d8a 100644 --- a/internal/gather/servicediscovery.go +++ b/internal/gather/servicediscovery.go @@ -42,7 +42,7 @@ func gatherServiceDiscoveryPodLogs(info *Info) { func gatherCoreDNSPodLogs(info *Info) { if isCoreDNSTypeOcp(info) { - gatherPodLogsByContainer(ocpCoreDNSPodLabel, "dns", info) + gatherPodLogs(ocpCoreDNSPodLabel, info, "dns") } else { gatherPodLogs(k8sCoreDNSPodLabel, info) } diff --git a/scripts/test/lib_subctl_gather_test.sh b/scripts/test/lib_subctl_gather_test.sh index d847efc09..87b3bf114 100644 --- a/scripts/test/lib_subctl_gather_test.sh +++ b/scripts/test/lib_subctl_gather_test.sh @@ -19,8 +19,8 @@ function validate_gathered_files () { validate_resource_files all 'globalegressips.submariner.io' 'GlobalEgressIP' validate_resource_files all 'globalingressips.submariner.io' 'GlobalIngressIP' - validate_pod_log_files "$subm_ns" '-l app=submariner-gateway' - validate_pod_log_files "$subm_ns" '-l app=submariner-routeagent' + validate_pod_log_files "$subm_ns" '-l app=submariner-gateway' 'submariner-gateway submariner-gateway-init' + validate_pod_log_files "$subm_ns" '-l app=submariner-routeagent' 'submariner-routeagent submariner-routeagent-init' validate_pod_log_files "$subm_ns" '-l app=submariner-globalnet' validate_pod_log_files "$subm_ns" '-l app=submariner-networkplugin-syncer' @@ -53,6 +53,7 @@ function validate_gathered_files () { function validate_pod_log_files() { local ns=$1 local selector=$2 + local containers=$3 local nsarg="--namespace=${ns}" if [[ "$ns" == "all" ]]; then @@ -61,10 +62,18 @@ function validate_pod_log_files() { pod_names=$(kubectl get pods "$nsarg" "$selector" -o=jsonpath='{.items..metadata.name}') read -ra pod_names_array <<< "$pod_names" - for pod_name in "${pod_names_array[@]}"; do - file=$gather_out_dir/${cluster}/$pod_name.log - cat "$file" + read -ra containers_array <<< "$containers" + for pod_name in "${pod_names_array[@]}"; do + if [[ "${#containers_array[@]}" == 0 ]]; then + file=$gather_out_dir/${cluster}/$pod_name.log + cat "$file" + else + for container in "${containers_array[@]}"; do + file=$gather_out_dir/${cluster}/$pod_name-$container.log + cat "$file" + done + fi done } diff --git a/scripts/test/system.sh b/scripts/test/system.sh index 6f9e4b05c..520ccd139 100755 --- a/scripts/test/system.sh +++ b/scripts/test/system.sh @@ -44,7 +44,7 @@ function test_subctl_gather() { _subctl gather --dir "$gather_out_dir" echo "::group::Validating 'subctl gather'" - ls $gather_out_dir + ls -R $gather_out_dir for cluster in "${clusters[@]}"; do with_context "${cluster}" validate_gathered_files