diff --git a/kubectl-fdb/cmd/profile_analyzer.go b/kubectl-fdb/cmd/profile_analyzer.go new file mode 100644 index 000000000..a2064a2d0 --- /dev/null +++ b/kubectl-fdb/cmd/profile_analyzer.go @@ -0,0 +1,171 @@ +/* + * profile_analyzer.go + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2022 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cmd + +import ( + "bytes" + "context" + "fmt" + "log" + "strings" + "text/template" + + fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2" + "github.com/spf13/cobra" + batchv1 "k8s.io/api/batch/v1" + v1 "k8s.io/api/core/v1" + yamlutil "k8s.io/apimachinery/pkg/util/yaml" + "k8s.io/cli-runtime/pkg/genericclioptions" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type profileConfig struct { + Namespace string + ClusterName string + JobName string + CommandArgs string +} + +func newProfileAnalyzerCmd(streams genericclioptions.IOStreams) *cobra.Command { + o := newFDBOptions(streams) + + cmd := &cobra.Command{ + Use: "analyze-profile", + Short: "Analyze FDB shards to find the busiest team", + Long: "Analyze FDB shards to find the busiest team", + RunE: func(cmd *cobra.Command, args []string) error { + kubeClient, err := getKubeClient(o) + if err != nil { + return err + } + + clusterName, err := cmd.Flags().GetString("fdb-cluster") + if err != nil { + return err + } + startTime, err := cmd.Flags().GetString("start-time") + if err != nil { + return err + } + endTime, err := cmd.Flags().GetString("end-time") + if err != nil { + return err + } + topRequests, err := cmd.Flags().GetInt("top-requests") + if err != nil { + return err + } + templateName, err := cmd.Flags().GetString("template-name") + if err != nil { + return err + } + + namespace, err := getNamespace(*o.configFlags.Namespace) + if err != nil { + return err + } + + return runProfileAnalyzer(kubeClient, namespace, clusterName, startTime, endTime, topRequests, templateName) + }, + Example: ` +# * experimental* +# Run the profiler for cluster-1 in the default namespace for the provided state and end time. +kubectl fdb analyze-profile -c cluster-1 --start-time "01:01 20/07/2022 BST" --end-time "01:30 20/07/2022 BST" --top-requests 100 --template-name job.yaml +`, + } + cmd.SetOut(o.Out) + cmd.SetErr(o.ErrOut) + cmd.SetIn(o.In) + + cmd.Flags().StringP("fdb-cluster", "c", "", "cluster name for running the analyze profile against.") + cmd.Flags().String("start-time", "", "start time for the analyzing transaction '01:30 30/07/2022 BST'.") + cmd.Flags().String("end-time", "", "end time for analyzing the transaction '02:30 30/07/2022 BST'.") + cmd.Flags().String("template-name", "", "name of the Job template.") + cmd.Flags().Int("top-requests", 100, "") + err := cmd.MarkFlagRequired("fdb-cluster") + if err != nil { + log.Fatal(err) + } + err = cmd.MarkFlagRequired("start-time") + if err != nil { + log.Fatal(err) + } + err = cmd.MarkFlagRequired("end-time") + if err != nil { + log.Fatal(err) + } + err = cmd.MarkFlagRequired("template-name") + if err != nil { + log.Fatal(err) + } + + o.configFlags.AddFlags(cmd.Flags()) + return cmd +} + +func runProfileAnalyzer(kubeClient client.Client, namespace string, clusterName string, startTime string, endTime string, topRequests int, templateName string) error { + config := profileConfig{ + Namespace: namespace, + ClusterName: clusterName, + JobName: clusterName + "-hot-shard-tool", + CommandArgs: fmt.Sprintf(" -C /var/dynamic-conf/fdb.cluster -s \"%s\" -e \"%s\" --filter-get-range --top-requests %d", startTime, endTime, topRequests), + } + t, err := template.ParseFiles(templateName) + if err != nil { + return err + } + buf := bytes.Buffer{} + err = t.Execute(&buf, config) + if err != nil { + return err + } + decoder := yamlutil.NewYAMLOrJSONDecoder(&buf, 100000) + + job := &batchv1.Job{} + err = decoder.Decode(&job) + if err != nil { + return err + } + cluster, err := loadCluster(kubeClient, namespace, clusterName) + if err != nil { + return err + } + + version, err := fdbv1beta2.ParseFdbVersion(cluster.GetRunningVersion()) + if err != nil { + return err + } + for _, container := range job.Spec.Template.Spec.InitContainers { + if !strings.Contains(container.Image, version.Compact()) { + continue + } + imageVersion := strings.Split(container.Image, ":") + fdbVersion := strings.Split(imageVersion[1], "-1") + envValue := v1.EnvVar{ + Name: "FDB_NETWORK_OPTION_EXTERNAL_CLIENT_DIRECTORY", + Value: fmt.Sprintf("/usr/bin/fdb/%s/lib/", fdbVersion[0]), + } + job.Spec.Template.Spec.Containers[0].Env = append(job.Spec.Template.Spec.Containers[0].Env, envValue) + break + } + log.Printf("creating job %s", config.JobName) + return kubeClient.Create(context.TODO(), job) +} diff --git a/kubectl-fdb/cmd/profile_analyzer_test.go b/kubectl-fdb/cmd/profile_analyzer_test.go new file mode 100644 index 000000000..e15d5c42c --- /dev/null +++ b/kubectl-fdb/cmd/profile_analyzer_test.go @@ -0,0 +1,118 @@ +/* + * profile_analyzer_test.go + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2022 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package cmd + +import ( + fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2" + batchv1 "k8s.io/api/batch/v1" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" + + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + ctx "context" +) + +var _ = Describe("profile analyser", func() { + When("running profile analyzer with 6.3", func() { + clusterName := "test" + namespace := "test" + scheme := runtime.NewScheme() + var kubeClient client.Client + + BeforeEach(func() { + _ = clientgoscheme.AddToScheme(scheme) + _ = fdbv1beta2.AddToScheme(scheme) + cluster := &fdbv1beta2.FoundationDBCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Namespace: namespace, + }, + Spec: fdbv1beta2.FoundationDBClusterSpec{ + Version: "6.3.24", + }, + } + kubeClient = fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(cluster).Build() + }) + It("should match the command args", func() { + expectedEnv := v1.EnvVar{ + Name: "FDB_NETWORK_OPTION_EXTERNAL_CLIENT_DIRECTORY", + Value: "/usr/bin/fdb/6.3.24/lib/", + ValueFrom: nil, + } + err := runProfileAnalyzer(kubeClient, namespace, clusterName, "21:30 08/24/2022 BST", "22:30 08/24/2022 BST", 100, "../../sample-apps/fdb-profile-analyzer/sample_template.yaml") + Expect(err).NotTo(HaveOccurred()) + job := &batchv1.Job{} + err = kubeClient.Get(ctx.Background(), client.ObjectKey{ + Namespace: namespace, + Name: "test-hot-shard-tool", + }, job) + Expect(err).NotTo(HaveOccurred()) + Expect(job.Spec.Template.Spec.Containers[0].Args).To(Equal([]string{"-c", + "python3 ./transaction_profiling_analyzer.py -C /var/dynamic-conf/fdb.cluster -s \"21:30 08/24/2022 BST\" -e \"22:30 08/24/2022 BST\" --filter-get-range --top-requests 100"})) + Expect(job.Spec.Template.Spec.Containers[0].Env).To(ContainElements(expectedEnv)) + }) + }) + When("running profile analyzer with 7.1", func() { + clusterName := "test" + namespace := "test" + scheme := runtime.NewScheme() + var kubeClient client.Client + + BeforeEach(func() { + _ = clientgoscheme.AddToScheme(scheme) + _ = fdbv1beta2.AddToScheme(scheme) + cluster := &fdbv1beta2.FoundationDBCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Namespace: namespace, + }, + Spec: fdbv1beta2.FoundationDBClusterSpec{ + Version: "7.1.19", + }, + } + kubeClient = fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(cluster).Build() + }) + It("should match the command args", func() { + expectedEnv := v1.EnvVar{ + Name: "FDB_NETWORK_OPTION_EXTERNAL_CLIENT_DIRECTORY", + Value: "/usr/bin/fdb/7.1.19/lib/", + ValueFrom: nil, + } + err := runProfileAnalyzer(kubeClient, namespace, clusterName, "21:30 08/24/2022 BST", "22:30 08/24/2022 BST", 100, "../../sample-apps/fdb-profile-analyzer/sample_template.yaml") + Expect(err).NotTo(HaveOccurred()) + job := &batchv1.Job{} + err = kubeClient.Get(ctx.Background(), client.ObjectKey{ + Namespace: namespace, + Name: "test-hot-shard-tool", + }, job) + Expect(err).NotTo(HaveOccurred()) + Expect(job.Spec.Template.Spec.Containers[0].Args).To(Equal([]string{"-c", + "python3 ./transaction_profiling_analyzer.py -C /var/dynamic-conf/fdb.cluster -s \"21:30 08/24/2022 BST\" -e \"22:30 08/24/2022 BST\" --filter-get-range --top-requests 100"})) + Expect(job.Spec.Template.Spec.Containers[0].Env).To(ContainElements(expectedEnv)) + }) + }) +}) diff --git a/kubectl-fdb/cmd/root.go b/kubectl-fdb/cmd/root.go index adbd9543e..64f4e3365 100644 --- a/kubectl-fdb/cmd/root.go +++ b/kubectl-fdb/cmd/root.go @@ -82,6 +82,7 @@ func NewRootCmd(streams genericclioptions.IOStreams) *cobra.Command { newFixCoordinatorIPsCmd(streams), newGetCmd(streams), newBuggifyCmd(streams), + newProfileAnalyzerCmd(streams), ) return cmd diff --git a/sample-apps/fdb-profile-analyzer/Dockerfile b/sample-apps/fdb-profile-analyzer/Dockerfile new file mode 100644 index 000000000..1ab894e4b --- /dev/null +++ b/sample-apps/fdb-profile-analyzer/Dockerfile @@ -0,0 +1,25 @@ +FROM python:3.9-slim +USER root + +ARG FDB_VERSION=7.1.21 +RUN apt-get update \ + && apt-get install -y wget \ + bind9-utils \ + lsof \ + less \ + net-tools \ + jq \ + openssl \ + pkg-config + +RUN groupadd -g 4059 foundationdb +RUN useradd -m -d / -s /bin/bash -u 4059 -o foundationdb -g 4059 +RUN mkdir /app/ +RUN chown foundationdb:foundationdb -R /app/ +RUN pip install foundationdb==${FDB_VERSION} +RUN pip install dateparser==1.1.1 +RUN wget https://github.com/apple/foundationdb/releases/download/${FDB_VERSION}/foundationdb-clients_${FDB_VERSION}-1_amd64.deb && \ + dpkg -i foundationdb-clients_${FDB_VERSION}-1_amd64.deb +WORKDIR /app/ +USER foundationdb +RUN wget https://raw.githubusercontent.com/apple/foundationdb/${FDB_VERSION}/contrib/transaction_profiling_analyzer/transaction_profiling_analyzer.py diff --git a/sample-apps/fdb-profile-analyzer/sample_template.yaml b/sample-apps/fdb-profile-analyzer/sample_template.yaml new file mode 100644 index 000000000..dba1b270e --- /dev/null +++ b/sample-apps/fdb-profile-analyzer/sample_template.yaml @@ -0,0 +1,96 @@ +apiVersion: batch/v1 +kind: Job +metadata: + labels: + app: fdb-profile-analyzer + name: {{ .JobName }} + namespace: {{ .Namespace }} +spec: + template: + metadata: + labels: + app: fdb-profile-analyzer + spec: + initContainers: + - args: + - --copy-library + - "6.2" + - --output-dir + - /var/output-files/6.2.30 + - --init-mode + image: foundationdb/foundationdb-kubernetes-sidecar:6.2.30-1 + name: foundationdb-kubernetes-init-6-2 + volumeMounts: + - mountPath: /var/output-files + name: fdb-binaries + - args: + - --copy-library + - "6.3" + - --output-dir + - /var/output-files/6.3.24 + - --init-mode + image: foundationdb/foundationdb-kubernetes-sidecar:6.3.24-1 + name: foundationdb-kubernetes-init-6-3 + volumeMounts: + - mountPath: /var/output-files + name: fdb-binaries + - args: + - --copy-library + - "7.1" + - --output-dir + - /var/output-files/7.1.19 + - --init-mode + image: foundationdb/foundationdb-kubernetes-sidecar:7.1.19-1 + name: foundationdb-kubernetes-init-7-1 + volumeMounts: + - mountPath: /var/output-files + name: fdb-binaries + containers: + - command: + - /bin/bash + args: + - -c + - python3 ./transaction_profiling_analyzer.py {{ .CommandArgs }} + env: + - name: FDB_CLUSTER_FILE + value: /var/dynamic-conf/fdb.cluster + - name: FDB_NETWORK_OPTION_TRACE_ENABLE + value: /var/log/fdb-profile-analyzer + - name: FDB_NETWORK_OPTION_TRACE_FORMAT + value: json + image: fdb-profile-analyzer:latest + imagePullPolicy: Always + name: profile-analyzer + resources: + requests: + cpu: 1000m + memory: 1Gi + limits: + cpu: 1000m + memory: 1Gi + volumeMounts: + - mountPath: /var/dynamic-conf + name: config-map + readOnly: true + - mountPath: /var/log/fdb-profile-analyzer + name: fdb-profile-analyzer-logs + - mountPath: /usr/bin/fdb + name: fdb-binaries + restartPolicy: Never + securityContext: + fsGroup: 4059 + runAsGroup: 4059 + runAsUser: 4059 + volumes: + - configMap: + defaultMode: 420 + items: + - key: cluster-file + path: fdb.cluster + name: {{ .ClusterName }}-config + name: config-map + - emptyDir: {} + name: fdb-profile-analyzer-logs + - emptyDir: {} + name: fdb-binaries + ttlSecondsAfterFinished: 7200