From 25d2d826de4d2c0be79a37b672eb39f98c7e7647 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jose=20Luis=20Ojosnegros=20Manch=C3=B3n?= Date: Mon, 20 Jun 2022 09:00:53 +0200 Subject: [PATCH] Add new `pod-info` command. This command will allow us to gather some pod information filtered by node. The pod information gathered is filtered with a go-template. Right now that template is hard-coded but it would be easy to use an incoming param to read it from a file name if needed in the future. Pod information is filtered because we want to avoid to show sensible information in the must-gather tool. When, if ever, change the output go-template consider: 1.- this output is going to be parsed at must-gather-pao so any change should be carefully synced 2.- be sure no sensible information is in the final output (like environmental variables or command parameters that can contain passwords) --- cmd/knit/main.go | 1 + go.mod | 8 +- go.sum | 2 + pkg/knit/cmd/k8s/podinfo.go | 206 +++++++++++++++++++++++++++++++ pkg/knit/cmd/k8s/podinfo_test.go | 152 +++++++++++++++++++++++ 5 files changed, 366 insertions(+), 3 deletions(-) create mode 100644 pkg/knit/cmd/k8s/podinfo.go create mode 100644 pkg/knit/cmd/k8s/podinfo_test.go diff --git a/cmd/knit/main.go b/cmd/knit/main.go index 3edb63fe..f16b2de1 100644 --- a/cmd/knit/main.go +++ b/cmd/knit/main.go @@ -30,6 +30,7 @@ import ( func main() { root := cmd.NewRootCommand( k8s.NewPodResourcesCommand, + k8s.NewPodInfoCommand, ghw.NewLscpuCommand, ghw.NewLspciCommand, ghw.NewLstopoCommand, diff --git a/go.mod b/go.mod index 20ab0e5a..3d3ffccd 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,9 @@ require ( github.com/safchain/ethtool v0.2.0 github.com/spf13/cobra v1.2.1 github.com/spf13/pflag v1.0.5 + k8s.io/api v0.23.0 + k8s.io/apimachinery v0.23.0 + k8s.io/client-go v0.23.0 k8s.io/klog/v2 v2.30.0 k8s.io/kubelet v0.23.0 k8s.io/kubernetes v0.23.0 @@ -23,6 +26,7 @@ require ( github.com/blang/semver v3.5.1+incompatible // indirect github.com/cespare/xxhash/v2 v2.1.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/evanphx/json-patch v4.12.0+incompatible // indirect github.com/fsnotify/fsnotify v1.4.9 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/go-logr/logr v1.2.0 // indirect @@ -32,6 +36,7 @@ require ( github.com/google/gofuzz v1.1.0 // indirect github.com/google/uuid v1.1.2 // indirect github.com/googleapis/gnostic v0.5.5 // indirect + github.com/imdario/mergo v0.3.5 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jaypipes/pcidb v0.6.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -62,10 +67,7 @@ require ( gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect - k8s.io/api v0.23.0 // indirect - k8s.io/apimachinery v0.23.0 // indirect k8s.io/apiserver v0.23.0 // indirect - k8s.io/client-go v0.23.0 // indirect k8s.io/component-base v0.23.0 // indirect k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b // indirect diff --git a/go.sum b/go.sum index 34e1c404..b85111c1 100644 --- a/go.sum +++ b/go.sum @@ -176,6 +176,7 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/euank/go-kmsg-parser v2.0.0+incompatible/go.mod h1:MhmAMZ8V4CYH4ybgdRwPr2TU5ThnS43puaKEMpja1uw= github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4= github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= @@ -347,6 +348,7 @@ github.com/heketi/tests v0.0.0-20151005000721-f3775cbcefd6/go.mod h1:xGMAM8JLi7U github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= diff --git a/pkg/knit/cmd/k8s/podinfo.go b/pkg/knit/cmd/k8s/podinfo.go new file mode 100644 index 00000000..32bb6205 --- /dev/null +++ b/pkg/knit/cmd/k8s/podinfo.go @@ -0,0 +1,206 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright 2022 Red Hat, Inc. + */ +package k8s + +import ( + "context" + "fmt" + "io" + "os" + "os/user" + "path/filepath" + "text/template" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + + "github.com/openshift-kni/debug-tools/pkg/knit/cmd" + "github.com/spf13/cobra" +) + +type podInfoOptions struct { + nodeName string +} + +//Only need some info about the pod. +// Right now is: +// - pod name +// - pod namespace +// - node name +// - status.qosClass +// - containers +// - requests cpu +// - limits cpu +// Note this output format could change but it would be parsed on insight rules +// so the change should be sync with it. +// Caution: We filter the data from pods to avoid exposing sensible information +// (like environment variables or input parameters which can contain passwords) +// so take care of that when changing this template. +const defaultTemplate string = ` +[ + {{- range $idx, $item := .Items}} + {{- if (ne $idx 0)}},{{end}} + { + "namespace":"{{.ObjectMeta.Namespace}}", + "name":"{{.ObjectMeta.Name}}", + "nodeName":"{{.Spec.NodeName}}", + "qosClass": "{{.Status.QOSClass}}", + {{- if .Spec.Containers }} + "containers": [ + {{- range $cdx, $cont := .Spec.Containers -}} + {{- if (ne $cdx 0) }},{{ end }} + { + "name":"{{.Name}}" + {{- if or .Resources.Requests .Resources.Limits -}} + , + "resources": { + {{- if .Resources.Limits}} {{if .Resources.Limits.Cpu}} + "limits": { + "cpu": "{{.Resources.Limits.Cpu}}" + } + {{- end }}{{end}} + {{- if .Resources.Requests}}{{if .Resources.Requests.Cpu -}} + , + "requests": { + "cpu": "{{.Resources.Requests.Cpu}}" + } + {{- end }}{{end}} + } + {{- end }} + } + {{- end }} + ] + {{- end }} + } + {{- end }} +]` + +func NewPodInfoCommand(knitOpts *cmd.KnitOptions) *cobra.Command { + + opts := &podInfoOptions{} + podInfo := &cobra.Command{ + Use: "podinfo", + Short: "get pod information complementing podresources data", + RunE: func(cmd *cobra.Command, args []string) error { + + clientset, err := getClientSetFromClusterConfig() + if err != nil { + return fmt.Errorf("unable to get clientset: %w", err) + } + + podInfoTemplate, err := createOutputTemplate("pod_info", defaultTemplate) + if err != nil { + return fmt.Errorf("unable to get output template: %w", err) + } + + nodeFieldSelector := buildNodeFieldSelector(opts.nodeName) + + return showPodInfo(nodeFieldSelector, clientset, podInfoTemplate, os.Stdout) + }, + } + + podInfo.Flags().StringVar(&opts.nodeName, "node-name", "", "node name to get pod info from.") + + return podInfo +} + +// GetConfig creates a *rest.Config for talking to a Kubernetes apiserver. +// +// Config precedence +// +// - KUBECONFIG environment variable pointing at a file +// - $HOME/.kube/config if exists +// - In-cluster config if running in cluster +func getKubeConfig() (*rest.Config, error) { + kubeconfigFromFilePath := func(kubeConfigFilePath string) (*rest.Config, error) { + if _, err := os.Stat(kubeConfigFilePath); err != nil { + return nil, fmt.Errorf("cannot stat kubeconfig '%s'", kubeConfigFilePath) + } + return clientcmd.BuildConfigFromFlags("", kubeConfigFilePath) + } + + // If an env variable is specified with the config location, use that + kubeConfig := os.Getenv("KUBECONFIG") + if len(kubeConfig) > 0 { + return kubeconfigFromFilePath(kubeConfig) + } + + // try the default location in the user's home directory + if usr, err := user.Current(); err == nil { + kubeConfig := filepath.Join(usr.HomeDir, ".kube", "config") + return kubeconfigFromFilePath(kubeConfig) + } + + // try the in-cluster config + if c, err := rest.InClusterConfig(); err == nil { + return c, nil + } + + return nil, fmt.Errorf("could not locate a kubeconfig") +} + +func getClientSetFromClusterConfig() (kubernetes.Interface, error) { + + config, err := getKubeConfig() + if err != nil { + return nil, err + } + // creates the clientset + return kubernetes.NewForConfig(config) +} + +func createOutputTemplate(name string, tmplStr string) (*template.Template, error) { + podInfoTemplate, err := template.New(name).Parse(tmplStr) + if err != nil { + return nil, err + } + return podInfoTemplate, nil +} + +func buildNodeFieldSelector(nodeName string) string { + fieldSelector := "" + if len(nodeName) != 0 { + fieldSelector = fmt.Sprintf("spec.nodeName=%s,", nodeName) + } + fieldSelector += "status.phase=Running" + + return fieldSelector +} + +func showPodInfo(nodeFieldSelector string, clientset kubernetes.Interface, podInfoTemplate *template.Template, output io.Writer) error { + + if nil == podInfoTemplate { + return fmt.Errorf("wrong incoming params: need an output template") + } + + listOptions := metav1.ListOptions{ + FieldSelector: nodeFieldSelector, + } + // get pods in all the namespaces by omitting namespace + // Or specify namespace to get pods in particular namespace + pods, err := clientset.CoreV1().Pods("").List(context.TODO(), listOptions) + if err != nil { + return fmt.Errorf("error while getting pods list: %w", err) + } + + if err := podInfoTemplate.Execute(output, pods); err != nil { + return fmt.Errorf("error while trying to format output: %w", err) + } + + return nil +} diff --git a/pkg/knit/cmd/k8s/podinfo_test.go b/pkg/knit/cmd/k8s/podinfo_test.go new file mode 100644 index 00000000..34a7a1d7 --- /dev/null +++ b/pkg/knit/cmd/k8s/podinfo_test.go @@ -0,0 +1,152 @@ +package k8s + +import ( + "bytes" + "encoding/json" + "fmt" + "reflect" + "testing" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" +) + +var fakeClientset = fake.NewSimpleClientset( + &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-one", + Namespace: "namespaceOne", + Annotations: map[string]string{}, + }, + Spec: v1.PodSpec{ + NodeName: "nodeNameOne", + Containers: []v1.Container{ + { + Name: "pod-one-c-one", + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("200m"), + }, + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + }, + }, + }, + }, + }, + Status: v1.PodStatus{ + QOSClass: v1.PodQOSBurstable, + }, + }, + &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-two", + Namespace: "myOtherNamespace", + Annotations: map[string]string{}, + }, + Spec: v1.PodSpec{ + NodeName: "nodeNameOne", + Containers: []v1.Container{ + { + Name: "pod-two-c-one", + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("2"), + }, + }, + }, + { + Name: "pod-two-c-two", + }, + }, + }, + Status: v1.PodStatus{ + QOSClass: v1.PodQOSGuaranteed, + }, + }, +) + +func TestDummy(t *testing.T) { + + expectedOutput := ` + [ + { + "namespace":"myOtherNamespace", + "name":"pod-two", + "nodeName":"nodeNameOne", + "qosClass": "Guaranteed", + "containers": [ + { + "name":"pod-two-c-one", + "resources": { + "limits": { + "cpu": "2" + } + } + }, + { + "name":"pod-two-c-two" + } + ] + }, + { + "namespace":"namespaceOne", + "name":"pod-one", + "nodeName":"nodeNameOne", + "qosClass": "Burstable", + "containers": [ + { + "name":"pod-one-c-one", + "resources": { + "limits": { + "cpu": "200m" + }, + "requests": { + "cpu": "100m" + } + } + } + ] + } +]` + + template, err := createOutputTemplate("test-template", defaultTemplate) + if err != nil { + t.Errorf("Unable to build template %v", err) + } + + buffer := new(bytes.Buffer) + ret := showPodInfo("", fakeClientset, template, buffer) + if ret != nil { + t.Errorf("showPodInfo failed with: %v", ret) + } + + ok, err := AreEqualJSON(buffer.String(), expectedOutput) + if err != nil { + t.Errorf("Error while trying to check json output: %v", err) + } + + if !ok { + t.Errorf("showPodInfo unexpected output:\n\tactual:%v\n\texpected:%v\n", buffer, expectedOutput) + } + +} + +func AreEqualJSON(s1, s2 string) (bool, error) { + var o1 interface{} + var o2 interface{} + + var err error + err = json.Unmarshal([]byte(s1), &o1) + if err != nil { + return false, fmt.Errorf("Error mashalling string 1 :: %s", err.Error()) + } + err = json.Unmarshal([]byte(s2), &o2) + if err != nil { + return false, fmt.Errorf("Error mashalling string 2 :: %s", err.Error()) + } + + return reflect.DeepEqual(o1, o2), nil +}