From 20db32ba84a9ac3e2d73c6ad25e52586ce281044 Mon Sep 17 00:00:00 2001 From: ddalvi Date: Tue, 5 Dec 2023 09:44:21 -0500 Subject: [PATCH] Enable logging for pipelines components --- backend/Dockerfile | 3 ++- backend/Dockerfile.persistenceagent | 3 ++- backend/Dockerfile.scheduledworkflow | 3 ++- backend/src/agent/persistence/main.go | 13 +++++++++++++ backend/src/apiserver/main.go | 13 +++++++++++++ .../src/crd/controller/scheduledworkflow/main.go | 12 ++++++++++++ .../installs/generic/pipeline-install-config.yaml | 1 + .../pipeline/ml-pipeline-apiserver-deployment.yaml | 2 ++ .../ml-pipeline-persistenceagent-deployment.yaml | 2 ++ .../ml-pipeline-scheduledworkflow-deployment.yaml | 2 ++ 10 files changed, 51 insertions(+), 3 deletions(-) diff --git a/backend/Dockerfile b/backend/Dockerfile index c65b897da47..08fee1822fc 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -59,6 +59,7 @@ ARG COMMIT_SHA=unknown ENV COMMIT_SHA=${COMMIT_SHA} ARG TAG_NAME=unknown ENV TAG_NAME=${TAG_NAME} +ENV LOG_LEVEL info WORKDIR /bin @@ -82,4 +83,4 @@ RUN sed -E "s#/(blob|tree)/master/#/\1/${COMMIT_SHA}/#g" -i /config/sample_confi EXPOSE 8888 # Start the apiserver -CMD /bin/apiserver --config=/config --sampleconfig=/config/sample_config.json -logtostderr=true +CMD /bin/apiserver --config=/config --sampleconfig=/config/sample_config.json -logtostderr=true --logLevel=${LOG_LEVEL} diff --git a/backend/Dockerfile.persistenceagent b/backend/Dockerfile.persistenceagent index aeaff4e5122..157bdfa6345 100644 --- a/backend/Dockerfile.persistenceagent +++ b/backend/Dockerfile.persistenceagent @@ -49,5 +49,6 @@ ENV TTL_SECONDS_AFTER_WORKFLOW_FINISH 86400 # NUM_WORKERS indicates now many worker goroutines ENV NUM_WORKERS 2 +ENV LOG_LEVEL info -CMD persistence_agent --logtostderr=true --namespace=${NAMESPACE} --ttlSecondsAfterWorkflowFinish=${TTL_SECONDS_AFTER_WORKFLOW_FINISH} --numWorker ${NUM_WORKERS} +CMD persistence_agent --logtostderr=true --namespace=${NAMESPACE} --ttlSecondsAfterWorkflowFinish=${TTL_SECONDS_AFTER_WORKFLOW_FINISH} --numWorker ${NUM_WORKERS} --logLevel=${LOG_LEVEL} diff --git a/backend/Dockerfile.scheduledworkflow b/backend/Dockerfile.scheduledworkflow index 2525b904554..f2a45ae601f 100644 --- a/backend/Dockerfile.scheduledworkflow +++ b/backend/Dockerfile.scheduledworkflow @@ -45,5 +45,6 @@ COPY --from=builder /tmp/licenses.csv /third_party/licenses.csv COPY --from=builder /tmp/NOTICES /third_party/NOTICES ENV NAMESPACE "" +ENV LOG_LEVEL info -CMD /bin/controller --logtostderr=true --namespace=${NAMESPACE} +CMD /bin/controller --logtostderr=true --namespace=${NAMESPACE} --logLevel=${LOG_LEVEL} diff --git a/backend/src/agent/persistence/main.go b/backend/src/agent/persistence/main.go index 4da32a7095e..3473326f907 100644 --- a/backend/src/agent/persistence/main.go +++ b/backend/src/agent/persistence/main.go @@ -30,6 +30,7 @@ import ( var ( masterURL string + logLevel string kubeconfig string initializeTimeout time.Duration timeout time.Duration @@ -47,6 +48,7 @@ var ( ) const ( + logLevelFlagName = "logLevel" kubeconfigFlagName = "kubeconfig" masterFlagName = "master" initializationTimeoutFlagName = "initializeTimeout" @@ -86,6 +88,16 @@ func main() { log.Fatalf("Error building schedule clientset: %s", err.Error()) } + if logLevel == "" { + logLevel = "info" + } + + level, err := log.ParseLevel(logLevel) + if err != nil { + log.Fatal("Invalid log level:", err) + } + log.SetLevel(level) + clientParam := util.ClientParameters{QPS: float64(cfg.QPS), Burst: cfg.Burst} execInformer := util.NewExecutionInformerOrFatal(util.ArgoWorkflow, namespace, time.Second*30, clientParam) @@ -131,6 +143,7 @@ func main() { func init() { flag.StringVar(&kubeconfig, kubeconfigFlagName, "", "Path to a kubeconfig. Only required if out-of-cluster.") flag.StringVar(&masterURL, masterFlagName, "", "The address of the Kubernetes API server. Overrides any value in kubeconfig. Only required if out-of-cluster.") + flag.StringVar(&logLevel, logLevelFlagName, "", "Defines the log level for the application.") flag.DurationVar(&initializeTimeout, initializationTimeoutFlagName, 2*time.Minute, "Duration to wait for initialization of the ML pipeline API server.") flag.DurationVar(&timeout, timeoutFlagName, 1*time.Minute, "Duration to wait for calls to complete.") flag.StringVar(&mlPipelineAPIServerName, mlPipelineAPIServerNameFlagName, "ml-pipeline", "Name of the ML pipeline API server.") diff --git a/backend/src/apiserver/main.go b/backend/src/apiserver/main.go index 23ae0200069..926f0f35307 100644 --- a/backend/src/apiserver/main.go +++ b/backend/src/apiserver/main.go @@ -41,12 +41,14 @@ import ( "github.com/kubeflow/pipelines/backend/src/apiserver/resource" "github.com/kubeflow/pipelines/backend/src/apiserver/server" "github.com/prometheus/client_golang/prometheus/promhttp" + log "github.com/sirupsen/logrus" "github.com/spf13/viper" "google.golang.org/grpc" "google.golang.org/grpc/reflection" ) var ( + logLevelFlag = flag.String("logLevel", "", "Defines the log level for the application.") rpcPortFlag = flag.String("rpcPortFlag", ":8887", "RPC Port") httpPortFlag = flag.String("httpPortFlag", ":8888", "Http Proxy Port") configPath = flag.String("config", "", "Path to JSON file containing config") @@ -77,6 +79,17 @@ func main() { } } + logLevel := *logLevelFlag + if logLevel == "" { + logLevel = "info" + } + + level, err := log.ParseLevel(logLevel) + if err != nil { + log.Fatal("Invalid log level:", err) + } + log.SetLevel(level) + go startRpcServer(resourceManager) startHttpProxy(resourceManager) diff --git a/backend/src/crd/controller/scheduledworkflow/main.go b/backend/src/crd/controller/scheduledworkflow/main.go index a99486dfda5..96527391b9c 100644 --- a/backend/src/crd/controller/scheduledworkflow/main.go +++ b/backend/src/crd/controller/scheduledworkflow/main.go @@ -32,6 +32,7 @@ import ( ) var ( + logLevel string masterURL string kubeconfig string namespace string @@ -53,6 +54,16 @@ func main() { cfg.QPS = float32(clientQPS) cfg.Burst = clientBurst + if logLevel == "" { + logLevel = "info" + } + + level, err := log.ParseLevel(logLevel) + if err != nil { + log.Fatal("Invalid log level:", err) + } + log.SetLevel(level) + kubeClient, err := kubernetes.NewForConfig(cfg) if err != nil { log.Fatalf("Error building kubernetes clientset: %s", err.Error()) @@ -102,6 +113,7 @@ func initEnv() { func init() { initEnv() + flag.StringVar(&logLevel, "logLevel", "", "Defines the log level for the application.") flag.StringVar(&kubeconfig, "kubeconfig", "", "Path to a kubeconfig. Only required if out-of-cluster.") flag.StringVar(&masterURL, "master", "", "The address of the Kubernetes API server. Overrides any value in kubeconfig. Only required if out-of-cluster.") flag.StringVar(&namespace, "namespace", "", "The namespace name used for Kubernetes informers to obtain the listers.") diff --git a/manifests/kustomize/base/installs/generic/pipeline-install-config.yaml b/manifests/kustomize/base/installs/generic/pipeline-install-config.yaml index 99f569ad226..2b7931b6c6b 100644 --- a/manifests/kustomize/base/installs/generic/pipeline-install-config.yaml +++ b/manifests/kustomize/base/installs/generic/pipeline-install-config.yaml @@ -93,3 +93,4 @@ data: ## If this value doesn't include a unit abbreviation, the units will be assumed ## to be nanoseconds. ConMaxLifeTime: "120s" + LOG_LEVEL: "info" diff --git a/manifests/kustomize/base/pipeline/ml-pipeline-apiserver-deployment.yaml b/manifests/kustomize/base/pipeline/ml-pipeline-apiserver-deployment.yaml index d296a72b835..cd80133596f 100644 --- a/manifests/kustomize/base/pipeline/ml-pipeline-apiserver-deployment.yaml +++ b/manifests/kustomize/base/pipeline/ml-pipeline-apiserver-deployment.yaml @@ -17,6 +17,8 @@ spec: spec: containers: - env: + - name: LOG_LEVEL + value: "info" - name: AUTO_UPDATE_PIPELINE_DEFAULT_VERSION valueFrom: configMapKeyRef: diff --git a/manifests/kustomize/base/pipeline/ml-pipeline-persistenceagent-deployment.yaml b/manifests/kustomize/base/pipeline/ml-pipeline-persistenceagent-deployment.yaml index 0d8b504278f..d23cee601af 100644 --- a/manifests/kustomize/base/pipeline/ml-pipeline-persistenceagent-deployment.yaml +++ b/manifests/kustomize/base/pipeline/ml-pipeline-persistenceagent-deployment.yaml @@ -25,6 +25,8 @@ spec: value: "86400" - name: NUM_WORKERS value: "2" + - name: LOG_LEVEL + value: "info" image: gcr.io/ml-pipeline/persistenceagent:dummy imagePullPolicy: IfNotPresent name: ml-pipeline-persistenceagent diff --git a/manifests/kustomize/base/pipeline/ml-pipeline-scheduledworkflow-deployment.yaml b/manifests/kustomize/base/pipeline/ml-pipeline-scheduledworkflow-deployment.yaml index ac20e5736f2..aa19c70f706 100644 --- a/manifests/kustomize/base/pipeline/ml-pipeline-scheduledworkflow-deployment.yaml +++ b/manifests/kustomize/base/pipeline/ml-pipeline-scheduledworkflow-deployment.yaml @@ -20,6 +20,8 @@ spec: imagePullPolicy: IfNotPresent name: ml-pipeline-scheduledworkflow env: + - name: LOG_LEVEL + value: "info" - name: NAMESPACE valueFrom: fieldRef: