From 6771c34208c6acf2ba10599d229ae5d5da273dc2 Mon Sep 17 00:00:00 2001 From: VaniHaripriya Date: Fri, 13 Dec 2024 11:07:06 -0600 Subject: [PATCH] Add ilab kfp pipeline to the DSPO repo Signed-off-by: VaniHaripriya --- api/v1/dspipeline_types.go | 10 ++++--- api/v1alpha1/dspipeline_types.go | 10 ++++--- ...b.io_datasciencepipelinesapplications.yaml | 12 +++++++++ .../apiserver/default/deployment.yaml.tmpl | 8 +++--- .../sample-pipeline/sample-config.yaml.tmpl | 26 +++++++++++++------ controllers/apiserver.go | 2 +- controllers/dspipeline_params.go | 10 ++++--- tests/resources/test-pipeline-run.yaml | 2 +- ...t-pipeline-with-custom-pip-server-run.yaml | 4 +-- 9 files changed, 59 insertions(+), 25 deletions(-) diff --git a/api/v1/dspipeline_types.go b/api/v1/dspipeline_types.go index 8f6253318..716ce4ed2 100644 --- a/api/v1/dspipeline_types.go +++ b/api/v1/dspipeline_types.go @@ -69,9 +69,13 @@ type APIServer struct { // Include sample pipelines with the deployment of this DSP API Server. Default: true // +kubebuilder:default:=false // +kubebuilder:validation:Optional - EnableSamplePipeline bool `json:"enableSamplePipeline"` - ArgoLauncherImage string `json:"argoLauncherImage,omitempty"` - ArgoDriverImage string `json:"argoDriverImage,omitempty"` + EnableSamplePipeline bool `json:"enableSamplePipeline"` + // Include instructlab multi-phase training and LLM generation pipeline with the deployment of this DSP API Server. Default: false + // +kubebuilder:default:=false + // +kubebuilder:validation:Optional + EnableInstructlabPipeline bool `json:"enableInstructlabPipeline"` + ArgoLauncherImage string `json:"argoLauncherImage,omitempty"` + ArgoDriverImage string `json:"argoDriverImage,omitempty"` // Specify custom Pod resource requirements for this component. Resources *ResourceRequirements `json:"resources,omitempty"` diff --git a/api/v1alpha1/dspipeline_types.go b/api/v1alpha1/dspipeline_types.go index 6a01914ac..dd810fddd 100644 --- a/api/v1alpha1/dspipeline_types.go +++ b/api/v1alpha1/dspipeline_types.go @@ -69,9 +69,13 @@ type APIServer struct { // Include sample pipelines with the deployment of this DSP API Server. Default: true // +kubebuilder:default:=false // +kubebuilder:validation:Optional - EnableSamplePipeline bool `json:"enableSamplePipeline"` - ArgoLauncherImage string `json:"argoLauncherImage,omitempty"` - ArgoDriverImage string `json:"argoDriverImage,omitempty"` + EnableSamplePipeline bool `json:"enableSamplePipeline"` + // Include instructlab multi-phase training and LLM generation pipeline with the deployment of this DSP API Server. Default: false + // +kubebuilder:default:=false + // +kubebuilder:validation:Optional + EnableInstructlabPipeline bool `json:"enableInstructlabPipeline"` + ArgoLauncherImage string `json:"argoLauncherImage,omitempty"` + ArgoDriverImage string `json:"argoDriverImage,omitempty"` // Specify custom Pod resource requirements for this component. Resources *ResourceRequirements `json:"resources,omitempty"` diff --git a/config/crd/bases/datasciencepipelinesapplications.opendatahub.io_datasciencepipelinesapplications.yaml b/config/crd/bases/datasciencepipelinesapplications.opendatahub.io_datasciencepipelinesapplications.yaml index 380aaeb5c..df0a6ac4d 100644 --- a/config/crd/bases/datasciencepipelinesapplications.opendatahub.io_datasciencepipelinesapplications.yaml +++ b/config/crd/bases/datasciencepipelinesapplications.opendatahub.io_datasciencepipelinesapplications.yaml @@ -105,6 +105,12 @@ spec: Server. Setting Deploy to false disables operator reconciliation. Default: true' type: boolean + enableInstructlabPipeline: + default: false + description: 'Include instructlab multi-phase training and LLM + generation pipeline with the deployment of this DSP API Server. + Default: false' + type: boolean enableOauth: default: true description: 'Create an Openshift Route for this DSP API Server. @@ -990,6 +996,12 @@ spec: Server. Setting Deploy to false disables operator reconciliation. Default: true' type: boolean + enableInstructlabPipeline: + default: false + description: 'Include instructlab multi-phase training and LLM + generation pipeline with the deployment of this DSP API Server. + Default: false' + type: boolean enableOauth: default: true description: 'Create an Openshift Route for this DSP API Server. diff --git a/config/internal/apiserver/default/deployment.yaml.tmpl b/config/internal/apiserver/default/deployment.yaml.tmpl index ee555d763..051e56144 100644 --- a/config/internal/apiserver/default/deployment.yaml.tmpl +++ b/config/internal/apiserver/default/deployment.yaml.tmpl @@ -147,7 +147,7 @@ spec: args: - --config=/config - -logtostderr=true - {{ if .APIServer.EnableSamplePipeline }} + {{ if or .APIServer.EnableSamplePipeline .APIServer.EnableInstructlabPipeline }} - --sampleconfig=/config/sample_config.json {{ end }} {{ if .PodToPodTLS }} @@ -206,8 +206,8 @@ spec: - mountPath: /etc/tls/private name: proxy-tls {{ end }} - {{ if or .APIServer.EnableSamplePipeline .CustomCABundle }} - {{ if .APIServer.EnableSamplePipeline }} + {{ if or .APIServer.EnableSamplePipeline .CustomCABundle .APIServer.EnableInstructlabPipeline}} + {{ if or .APIServer.EnableSamplePipeline .APIServer.EnableInstructlabPipeline}} - name: sample-config mountPath: /config/sample_config.json subPath: sample_config.json @@ -287,7 +287,7 @@ spec: configMap: name: {{ .CustomCABundle.ConfigMapName }} {{ end }} - {{ if .APIServer.EnableSamplePipeline }} + {{ if or .APIServer.EnableSamplePipeline .APIServer.EnableInstructlabPipeline}} - name: sample-config configMap: name: sample-config-{{.Name}} diff --git a/config/internal/apiserver/sample-pipeline/sample-config.yaml.tmpl b/config/internal/apiserver/sample-pipeline/sample-config.yaml.tmpl index 76e64d65d..9e682f3d5 100644 --- a/config/internal/apiserver/sample-pipeline/sample-config.yaml.tmpl +++ b/config/internal/apiserver/sample-pipeline/sample-config.yaml.tmpl @@ -7,11 +7,21 @@ metadata: app: {{.APIServerDefaultResourceName}} component: data-science-pipelines data: - sample_config.json: |- - [ - { - "name": "[Demo] iris-training", - "description": "[source code](https://github.com/opendatahub-io/data-science-pipelines/tree/master/samples/iris-sklearn) A simple pipeline to demonstrate a basic ML Training workflow", - "file": "/samples/iris-pipeline-compiled.yaml" - } - ] + sample_config.json: |- + [ + {{- if .EnableSamplePipeline }} + { + "name": "[Demo] iris-training", + "description": "[source code](https://github.com/opendatahub-io/data-science-pipelines/tree/master/samples/iris-sklearn) A simple pipeline to demonstrate a basic ML Training workflow", + "file": "/samples/iris-pipeline-compiled.yaml" + } + {{- if .EnableInstructlabPipeline }},{{ end }} + {{- end }} + {{- if .EnableInstructlabPipeline }} + { + "name": "[Instructlab] Multi-Phase Training Pipeline", + "description": "[source code](https://github.com/opendatahub-io/ilab-on-ocp) Instructlab Multi-Phase Training Pipeline", + "file": "/pipelines/instructlab.yaml" + } + {{- end }} + ] diff --git a/controllers/apiserver.go b/controllers/apiserver.go index 929ede232..bc789588a 100644 --- a/controllers/apiserver.go +++ b/controllers/apiserver.go @@ -68,7 +68,7 @@ func (r *DSPAReconciler) ReconcileAPIServer(ctx context.Context, dsp *dspav1.Dat } for cmName, template := range samplePipelineTemplates { - if dsp.Spec.APIServer.EnableSamplePipeline { + if dsp.Spec.APIServer.EnableSamplePipeline || dsp.Spec.APIServer.EnableInstructlabPipeline { err := r.Apply(dsp, params, template) if err != nil { return err diff --git a/controllers/dspipeline_params.go b/controllers/dspipeline_params.go index 4efdbe4af..9b3c6d457 100644 --- a/controllers/dspipeline_params.go +++ b/controllers/dspipeline_params.go @@ -92,9 +92,10 @@ type DSPAParams struct { CustomCABundle *dspa.CABundle DSPONamespace string // Use to enable tls communication between component pods. - PodToPodTLS bool - - APIServerServiceDNSName string + PodToPodTLS bool + EnableSamplePipeline bool + EnableInstructlabPipeline bool + APIServerServiceDNSName string } type DBConnection struct { @@ -601,6 +602,9 @@ func (p *DSPAParams) ExtractParams(ctx context.Context, dsp *dspa.DataSciencePip setStringDefault(argoDriverImageFromConfig, &p.APIServer.ArgoDriverImage) setResourcesDefault(config.APIServerResourceRequirements, &p.APIServer.Resources) + p.EnableSamplePipeline = dsp.Spec.APIServer.EnableSamplePipeline + p.EnableInstructlabPipeline = dsp.Spec.APIServer.EnableInstructlabPipeline + if p.APIServer.CustomServerConfig == nil { p.APIServer.CustomServerConfig = &dspa.ScriptConfigMap{ Name: config.CustomServerConfigMapNamePrefix + dsp.Name, diff --git a/tests/resources/test-pipeline-run.yaml b/tests/resources/test-pipeline-run.yaml index f92d80046..e57e88869 100644 --- a/tests/resources/test-pipeline-run.yaml +++ b/tests/resources/test-pipeline-run.yaml @@ -29,7 +29,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.10.1'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.11.0'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh diff --git a/tests/resources/test-pipeline-with-custom-pip-server-run.yaml b/tests/resources/test-pipeline-with-custom-pip-server-run.yaml index da3e6352c..704c77a56 100644 --- a/tests/resources/test-pipeline-with-custom-pip-server-run.yaml +++ b/tests/resources/test-pipeline-with-custom-pip-server-run.yaml @@ -24,7 +24,7 @@ deploymentSpec: - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ \ python3 -m pip install --quiet --no-warn-script-location --index-url https://nginx-service.test-pypiserver.svc.cluster.local/simple/\ - \ 'kfp==2.10.1' '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"\ + \ 'kfp==2.11.0' '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"\ 3.9\"' && python3 -m pip install --quiet --no-warn-script-location --index-url\ \ https://nginx-service.test-pypiserver.svc.cluster.local/simple/ 'numpy'\ \ && \"$0\" \"$@\"\n" @@ -66,4 +66,4 @@ root: Output: parameterType: STRING schemaVersion: 2.1.0 -sdkVersion: kfp-2.10.1 +sdkVersion: kfp-2.11.0