From 8936aa18a14b1e51ed0bf35971a5e4c2dbd1e98a Mon Sep 17 00:00:00 2001 From: Sophie Zhao Date: Mon, 19 Apr 2021 15:43:27 -0400 Subject: [PATCH 1/2] introduce arcmode --- cmd/aks-periscope/aks-periscope.go | 32 ++++++++++++++++------- deployment/aks-periscope.yaml | 20 +++++++++----- pkg/collector/collector.go | 8 +++++- pkg/collector/customresource_collector.go | 27 +++++++++++++++++++ pkg/collector/exec_collector.go | 27 +++++++++++++++++++ pkg/collector/helm_collector.go | 27 +++++++++++++++++++ 6 files changed, 125 insertions(+), 16 deletions(-) create mode 100644 pkg/collector/customresource_collector.go create mode 100644 pkg/collector/exec_collector.go create mode 100644 pkg/collector/helm_collector.go diff --git a/cmd/aks-periscope/aks-periscope.go b/cmd/aks-periscope/aks-periscope.go index aa36bb39..2327a296 100644 --- a/cmd/aks-periscope/aks-periscope.go +++ b/cmd/aks-periscope/aks-periscope.go @@ -2,6 +2,7 @@ package main import ( "log" + "os" "strings" "sync" @@ -22,25 +23,35 @@ func main() { log.Printf("Failed to create CRD: %+v", err) } + clusterType := os.Getenv("CLUSTER_TYPE") + log.Printf("Cluster Type: %s", clusterType) + log.Printf(clusterType) + + storageAccount := os.Getenv("AZURE_BLOB_ACCOUNT_NAME") + log.Printf("Storage Account: %s", storageAccount) collectors := []interfaces.Collector{} containerLogsCollector := collector.NewContainerLogsCollector(exporter) collectors = append(collectors, containerLogsCollector) - systemLogsCollector := collector.NewSystemLogsCollector(exporter) - collectors = append(collectors, systemLogsCollector) networkOutboundCollector := collector.NewNetworkOutboundCollector(5, exporter) collectors = append(collectors, networkOutboundCollector) - ipTablesCollector := collector.NewIPTablesCollector(exporter) - collectors = append(collectors, ipTablesCollector) - nodeLogsCollector := collector.NewNodeLogsCollector(exporter) - collectors = append(collectors, nodeLogsCollector) dnsCollector := collector.NewDNSCollector(exporter) collectors = append(collectors, dnsCollector) kubeObjectsCollector := collector.NewKubeObjectsCollector(exporter) collectors = append(collectors, kubeObjectsCollector) + + systemLogsCollector := collector.NewSystemLogsCollector(exporter) + ipTablesCollector := collector.NewIPTablesCollector(exporter) + nodeLogsCollector := collector.NewNodeLogsCollector(exporter) kubeletCmdCollector := collector.NewKubeletCmdCollector(exporter) - collectors = append(collectors, kubeletCmdCollector) systemPerfCollector := collector.NewSystemPerfCollector(exporter) - collectors = append(collectors, systemPerfCollector) + + if clusterType != "connectedcluster" { + collectors = append(collectors, systemLogsCollector) + collectors = append(collectors, ipTablesCollector) + collectors = append(collectors, nodeLogsCollector) + collectors = append(collectors, kubeletCmdCollector) + collectors = append(collectors, systemPerfCollector) + } for _, c := range collectors { waitgroup.Add(1) @@ -63,9 +74,12 @@ func main() { waitgroup.Wait() diagnosers := []interfaces.Diagnoser{} - diagnosers = append(diagnosers, diagnoser.NewNetworkConfigDiagnoser(dnsCollector, kubeletCmdCollector, exporter)) diagnosers = append(diagnosers, diagnoser.NewNetworkOutboundDiagnoser(networkOutboundCollector, exporter)) + if clusterType != "connectedcluster" { + diagnosers = append(diagnosers, diagnoser.NewNetworkConfigDiagnoser(dnsCollector, kubeletCmdCollector, exporter)) + } + for _, d := range diagnosers { waitgroup.Add(1) go func(d interfaces.Diagnoser) { diff --git a/deployment/aks-periscope.yaml b/deployment/aks-periscope.yaml index 03ef1167..b97c0cc8 100644 --- a/deployment/aks-periscope.yaml +++ b/deployment/aks-periscope.yaml @@ -14,8 +14,8 @@ kind: ClusterRole metadata: name: aks-periscope-role rules: -- apiGroups: ["","metrics.k8s.io"] - resources: ["pods", "nodes"] +- apiGroups: ["","metrics.k8s.io", "apiextensions.k8s.io"] + resources: ["pods", "nodes", "customresourcedefinitions"] verbs: ["get", "watch", "list"] - apiGroups: ["aks-periscope.azure.github.com"] resources: ["diagnostics"] @@ -69,7 +69,7 @@ spec: beta.kubernetes.io/os: linux containers: - name: aks-periscope - image: aksrepos.azurecr.io/staging/aks-periscope:v0.3 + image: aksrepos.azurecr.io/staging/aks-periscope:v0.2 securityContext: privileged: true imagePullPolicy: Always @@ -114,7 +114,7 @@ metadata: name: containerlogs-config namespace: aks-periscope data: - DIAGNOSTIC_CONTAINERLOGS_LIST: kube-system + DIAGNOSTIC_CONTAINERLOGS_LIST: azure-arc --- apiVersion: v1 kind: ConfigMap @@ -122,7 +122,7 @@ metadata: name: kubeobjects-config namespace: aks-periscope data: - DIAGNOSTIC_KUBEOBJECTS_LIST: kube-system/pod kube-system/service kube-system/deployment + DIAGNOSTIC_KUBEOBJECTS_LIST: azure-arc/pod azure-arc/service azure-arc/deployment --- apiVersion: v1 kind: ConfigMap @@ -132,6 +132,14 @@ metadata: data: DIAGNOSTIC_NODELOGS_LIST: /var/log/azure/cluster-provision.log /var/log/cloud-init.log --- +apiVersion: v1 +kind: ConfigMap +metadata: + name: clustertype-config + namespace: aks-periscope +data: + CLUSTER_TYPE: connectedcluster +--- apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: @@ -159,4 +167,4 @@ spec: singular: diagnostic kind: Diagnostic shortNames: - - apd \ No newline at end of file + - apd diff --git a/pkg/collector/collector.go b/pkg/collector/collector.go index 1c342617..471e423f 100644 --- a/pkg/collector/collector.go +++ b/pkg/collector/collector.go @@ -10,6 +10,12 @@ const ( DNS Type = iota // ContainerLogs defines ContainerLogs Collector Type ContainerLogs + //CustomResource defines CustomResource Collector Type + CustomResource + //Exec defines Exec Collector Type + Exec + //Helm defines Helm Collector Type + Helm // IPTables defines IPTables Collector Type IPTables // KubeletCmd defines KubeletCmd Collector Type @@ -28,7 +34,7 @@ const ( // Name returns type name func (t Type) name() string { - return [...]string{"dns", "containerlogs", "iptables", "kubeletcmd", "kubeobjects", "networkoutbound", "nodelogs", "systemlogs", "systemperf"}[t] + return [...]string{"dns", "containerlogs", "customresource", "exec", "helm", "iptables", "kubeletcmd", "kubeobjects", "networkoutbound", "nodelogs", "systemlogs", "systemperf"}[t] } // BaseCollector defines Base Collector diff --git a/pkg/collector/customresource_collector.go b/pkg/collector/customresource_collector.go new file mode 100644 index 00000000..588d097f --- /dev/null +++ b/pkg/collector/customresource_collector.go @@ -0,0 +1,27 @@ +package collector + +import ( + "github.com/Azure/aks-periscope/pkg/interfaces" +) + +// CustomResourceCollector defines a CustomResources Collector struct +type CustomResourceCollector struct { + BaseCollector +} + +var _ interfaces.Collector = &CustomResourceCollector{} + +// CustomResourceCollector is a constructor +func NewCustomResourceCollector(exporter interfaces.Exporter) *CustomResourceCollector { + return &CustomResourceCollector{ + BaseCollector: BaseCollector{ + collectorType: CustomResource, + exporter: exporter, + }, + } +} + +// Collect implements the interface method +func (collector *CustomResourceCollector) Collect() error { + return nil +} diff --git a/pkg/collector/exec_collector.go b/pkg/collector/exec_collector.go new file mode 100644 index 00000000..fa511fc3 --- /dev/null +++ b/pkg/collector/exec_collector.go @@ -0,0 +1,27 @@ +package collector + +import ( + "github.com/Azure/aks-periscope/pkg/interfaces" +) + +// ExecCollector defines a Exec Collector struct +type ExecCollector struct { + BaseCollector +} + +var _ interfaces.Collector = &ExecCollector{} + +// ExecCollector is a constructor +func NewExecCollector(exporter interfaces.Exporter) *ExecCollector { + return &ExecCollector{ + BaseCollector: BaseCollector{ + collectorType: Exec, + exporter: exporter, + }, + } +} + +// Collect implements the interface method +func (collector *ExecCollector) Collect() error { + return nil +} diff --git a/pkg/collector/helm_collector.go b/pkg/collector/helm_collector.go new file mode 100644 index 00000000..6ab6a9be --- /dev/null +++ b/pkg/collector/helm_collector.go @@ -0,0 +1,27 @@ +package collector + +import ( + "github.com/Azure/aks-periscope/pkg/interfaces" +) + +// HelmCollector defines a Helm Collector struct +type HelmCollector struct { + BaseCollector +} + +var _ interfaces.Collector = &IPTablesCollector{} + +// NewHelmCollector is a constructor +func NewHelmCollector(exporter interfaces.Exporter) *HelmCollector { + return &HelmCollector{ + BaseCollector: BaseCollector{ + collectorType: Helm, + exporter: exporter, + }, + } +} + +// Collect implements the interface method +func (collector *HelmCollector) Collect() error { + return nil +} From eda7ba906eb868ed232fc97543e4f7d583015eaa Mon Sep 17 00:00:00 2001 From: sophsoph321 Date: Thu, 22 Apr 2021 12:01:44 -0400 Subject: [PATCH 2/2] add helm collector --- builder/Dockerfile | 5 +++- cmd/aks-periscope/aks-periscope.go | 30 +++++++++---------- deployment/aks-periscope.yaml | 16 +++++----- pkg/collector/collector.go | 6 +--- pkg/collector/customresource_collector.go | 27 ----------------- pkg/collector/exec_collector.go | 27 ----------------- pkg/collector/helm_collector.go | 36 ++++++++++++++++++++++- 7 files changed, 64 insertions(+), 83 deletions(-) delete mode 100644 pkg/collector/customresource_collector.go delete mode 100644 pkg/collector/exec_collector.go diff --git a/builder/Dockerfile b/builder/Dockerfile index 0ab571e9..a0d4e580 100644 --- a/builder/Dockerfile +++ b/builder/Dockerfile @@ -5,9 +5,12 @@ WORKDIR /app RUN CGO_ENABLED=0 GOOS=linux go build -mod=vendor github.com/Azure/aks-periscope/cmd/aks-periscope FROM alpine -RUN apk --no-cache add ca-certificates +RUN apk --no-cache add ca-certificates curl openssl bash ADD https://storage.googleapis.com/kubernetes-release/release/v1.16.0/bin/linux/amd64/kubectl /usr/local/bin/kubectl RUN chmod +x /usr/local/bin/kubectl +RUN curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 \ + && chmod +x get_helm.sh \ + && ./get_helm.sh RUN mkdir /app WORKDIR /app COPY --from=builder /app/aks-periscope . diff --git a/cmd/aks-periscope/aks-periscope.go b/cmd/aks-periscope/aks-periscope.go index 2327a296..26fe6c17 100644 --- a/cmd/aks-periscope/aks-periscope.go +++ b/cmd/aks-periscope/aks-periscope.go @@ -24,28 +24,31 @@ func main() { } clusterType := os.Getenv("CLUSTER_TYPE") - log.Printf("Cluster Type: %s", clusterType) - log.Printf(clusterType) - storageAccount := os.Getenv("AZURE_BLOB_ACCOUNT_NAME") - log.Printf("Storage Account: %s", storageAccount) collectors := []interfaces.Collector{} containerLogsCollector := collector.NewContainerLogsCollector(exporter) - collectors = append(collectors, containerLogsCollector) networkOutboundCollector := collector.NewNetworkOutboundCollector(5, exporter) - collectors = append(collectors, networkOutboundCollector) dnsCollector := collector.NewDNSCollector(exporter) - collectors = append(collectors, dnsCollector) kubeObjectsCollector := collector.NewKubeObjectsCollector(exporter) - collectors = append(collectors, kubeObjectsCollector) - systemLogsCollector := collector.NewSystemLogsCollector(exporter) ipTablesCollector := collector.NewIPTablesCollector(exporter) nodeLogsCollector := collector.NewNodeLogsCollector(exporter) kubeletCmdCollector := collector.NewKubeletCmdCollector(exporter) systemPerfCollector := collector.NewSystemPerfCollector(exporter) - - if clusterType != "connectedcluster" { + helmCollector := collector.NewHelmCollector(exporter) + + if strings.EqualFold(clusterType, "connectedCluster") { + collectors = append(collectors, containerLogsCollector) + collectors = append(collectors, dnsCollector) + collectors = append(collectors, helmCollector) + collectors = append(collectors, kubeObjectsCollector) + collectors = append(collectors, networkOutboundCollector) + + } else { + collectors = append(collectors, containerLogsCollector) + collectors = append(collectors, dnsCollector) + collectors = append(collectors, kubeObjectsCollector) + collectors = append(collectors, networkOutboundCollector) collectors = append(collectors, systemLogsCollector) collectors = append(collectors, ipTablesCollector) collectors = append(collectors, nodeLogsCollector) @@ -74,12 +77,9 @@ func main() { waitgroup.Wait() diagnosers := []interfaces.Diagnoser{} + diagnosers = append(diagnosers, diagnoser.NewNetworkConfigDiagnoser(dnsCollector, kubeletCmdCollector, exporter)) diagnosers = append(diagnosers, diagnoser.NewNetworkOutboundDiagnoser(networkOutboundCollector, exporter)) - if clusterType != "connectedcluster" { - diagnosers = append(diagnosers, diagnoser.NewNetworkConfigDiagnoser(dnsCollector, kubeletCmdCollector, exporter)) - } - for _, d := range diagnosers { waitgroup.Add(1) go func(d interfaces.Diagnoser) { diff --git a/deployment/aks-periscope.yaml b/deployment/aks-periscope.yaml index b97c0cc8..df0daaaa 100644 --- a/deployment/aks-periscope.yaml +++ b/deployment/aks-periscope.yaml @@ -14,8 +14,8 @@ kind: ClusterRole metadata: name: aks-periscope-role rules: -- apiGroups: ["","metrics.k8s.io", "apiextensions.k8s.io"] - resources: ["pods", "nodes", "customresourcedefinitions"] +- apiGroups: ["","metrics.k8s.io"] + resources: ["pods", "nodes", "secrets"] verbs: ["get", "watch", "list"] - apiGroups: ["aks-periscope.azure.github.com"] resources: ["diagnostics"] @@ -69,7 +69,7 @@ spec: beta.kubernetes.io/os: linux containers: - name: aks-periscope - image: aksrepos.azurecr.io/staging/aks-periscope:v0.2 + image: aksrepos.azurecr.io/staging/aks-periscope:v0.3 securityContext: privileged: true imagePullPolicy: Always @@ -80,6 +80,8 @@ spec: name: kubeobjects-config - configMapRef: name: nodelogs-config + - configMapRef: + name: clustertype-config - secretRef: name: azureblob-secret volumeMounts: @@ -114,7 +116,7 @@ metadata: name: containerlogs-config namespace: aks-periscope data: - DIAGNOSTIC_CONTAINERLOGS_LIST: azure-arc + DIAGNOSTIC_CONTAINERLOGS_LIST: kube-system --- apiVersion: v1 kind: ConfigMap @@ -122,7 +124,7 @@ metadata: name: kubeobjects-config namespace: aks-periscope data: - DIAGNOSTIC_KUBEOBJECTS_LIST: azure-arc/pod azure-arc/service azure-arc/deployment + DIAGNOSTIC_KUBEOBJECTS_LIST: kube-system/pod kube-system/service kube-system/deployment --- apiVersion: v1 kind: ConfigMap @@ -138,7 +140,7 @@ metadata: name: clustertype-config namespace: aks-periscope data: - CLUSTER_TYPE: connectedcluster + CLUSTER_TYPE: "managedCluster" # --- apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition @@ -167,4 +169,4 @@ spec: singular: diagnostic kind: Diagnostic shortNames: - - apd + - apd \ No newline at end of file diff --git a/pkg/collector/collector.go b/pkg/collector/collector.go index 471e423f..971a01ed 100644 --- a/pkg/collector/collector.go +++ b/pkg/collector/collector.go @@ -10,10 +10,6 @@ const ( DNS Type = iota // ContainerLogs defines ContainerLogs Collector Type ContainerLogs - //CustomResource defines CustomResource Collector Type - CustomResource - //Exec defines Exec Collector Type - Exec //Helm defines Helm Collector Type Helm // IPTables defines IPTables Collector Type @@ -34,7 +30,7 @@ const ( // Name returns type name func (t Type) name() string { - return [...]string{"dns", "containerlogs", "customresource", "exec", "helm", "iptables", "kubeletcmd", "kubeobjects", "networkoutbound", "nodelogs", "systemlogs", "systemperf"}[t] + return [...]string{"dns", "containerlogs", "helm", "iptables", "kubeletcmd", "kubeobjects", "networkoutbound", "nodelogs", "systemlogs", "systemperf"}[t] } // BaseCollector defines Base Collector diff --git a/pkg/collector/customresource_collector.go b/pkg/collector/customresource_collector.go deleted file mode 100644 index 588d097f..00000000 --- a/pkg/collector/customresource_collector.go +++ /dev/null @@ -1,27 +0,0 @@ -package collector - -import ( - "github.com/Azure/aks-periscope/pkg/interfaces" -) - -// CustomResourceCollector defines a CustomResources Collector struct -type CustomResourceCollector struct { - BaseCollector -} - -var _ interfaces.Collector = &CustomResourceCollector{} - -// CustomResourceCollector is a constructor -func NewCustomResourceCollector(exporter interfaces.Exporter) *CustomResourceCollector { - return &CustomResourceCollector{ - BaseCollector: BaseCollector{ - collectorType: CustomResource, - exporter: exporter, - }, - } -} - -// Collect implements the interface method -func (collector *CustomResourceCollector) Collect() error { - return nil -} diff --git a/pkg/collector/exec_collector.go b/pkg/collector/exec_collector.go deleted file mode 100644 index fa511fc3..00000000 --- a/pkg/collector/exec_collector.go +++ /dev/null @@ -1,27 +0,0 @@ -package collector - -import ( - "github.com/Azure/aks-periscope/pkg/interfaces" -) - -// ExecCollector defines a Exec Collector struct -type ExecCollector struct { - BaseCollector -} - -var _ interfaces.Collector = &ExecCollector{} - -// ExecCollector is a constructor -func NewExecCollector(exporter interfaces.Exporter) *ExecCollector { - return &ExecCollector{ - BaseCollector: BaseCollector{ - collectorType: Exec, - exporter: exporter, - }, - } -} - -// Collect implements the interface method -func (collector *ExecCollector) Collect() error { - return nil -} diff --git a/pkg/collector/helm_collector.go b/pkg/collector/helm_collector.go index 6ab6a9be..313da0c2 100644 --- a/pkg/collector/helm_collector.go +++ b/pkg/collector/helm_collector.go @@ -1,7 +1,10 @@ package collector import ( + "path/filepath" + "github.com/Azure/aks-periscope/pkg/interfaces" + "github.com/Azure/aks-periscope/pkg/utils" ) // HelmCollector defines a Helm Collector struct @@ -9,7 +12,7 @@ type HelmCollector struct { BaseCollector } -var _ interfaces.Collector = &IPTablesCollector{} +var _ interfaces.Collector = &HelmCollector{} // NewHelmCollector is a constructor func NewHelmCollector(exporter interfaces.Exporter) *HelmCollector { @@ -23,5 +26,36 @@ func NewHelmCollector(exporter interfaces.Exporter) *HelmCollector { // Collect implements the interface method func (collector *HelmCollector) Collect() error { + rootPath, err := utils.CreateCollectorDir(collector.GetName()) + if err != nil { + return err + } + + helmListFile := filepath.Join(rootPath, "helm_list") + helm_list_output, helm_list_err := utils.RunCommandOnContainer("helm", "list", "--all-namespaces") + if helm_list_err != nil { + return helm_list_err + } + + helm_list_err = utils.WriteToFile(helmListFile, helm_list_output) + if helm_list_err != nil { + return helm_list_err + } + + collector.AddToCollectorFiles(helmListFile) + + helmHistoryFile := filepath.Join(rootPath, "helm_history") + helm_history_output, helm_history_err := utils.RunCommandOnContainer("helm", "history", "-n", "default", "azure-arc") + if helm_history_err != nil { + return helm_history_err + } + + helm_history_err = utils.WriteToFile(helmHistoryFile, helm_history_output) + if helm_history_err != nil { + return helm_history_err + } + + collector.AddToCollectorFiles(helmHistoryFile) + return nil }