Add ilab kfp pipeline to the DSPO repo

Signed-off-by: VaniHaripriya <[email protected]>
opendatahub-io · Jan 8, 2025 · 5d0682e · 5d0682e
1 parent d80a181
commit 5d0682e
Show file tree

Hide file tree

Showing 12 changed files with 96 additions and 29 deletions.
diff --git a/api/v1/dspipeline_types.go b/api/v1/dspipeline_types.go
@@ -69,9 +69,13 @@ type APIServer struct {
 	// Include sample pipelines with the deployment of this DSP API Server. Default: true
 	// +kubebuilder:default:=false
 	// +kubebuilder:validation:Optional
-	EnableSamplePipeline bool   `json:"enableSamplePipeline"`
-	ArgoLauncherImage    string `json:"argoLauncherImage,omitempty"`
-	ArgoDriverImage      string `json:"argoDriverImage,omitempty"`
+	EnableSamplePipeline bool `json:"enableSamplePipeline"`
+	// Enable the Instructlab Multi-Phase Training pipeline with the deployment of this DSP API server. Default: false
+	// +kubebuilder:default:=false
+	// +kubebuilder:validation:Optional
+	EnableInstructLabPipeline bool   `json:"enableInstructLabPipeline"`
+	ArgoLauncherImage         string `json:"argoLauncherImage,omitempty"`
+	ArgoDriverImage           string `json:"argoDriverImage,omitempty"`
 	// Specify custom Pod resource requirements for this component.
 	Resources *ResourceRequirements `json:"resources,omitempty"`
 

diff --git a/api/v1alpha1/dspipeline_types.go b/api/v1alpha1/dspipeline_types.go
@@ -69,9 +69,13 @@ type APIServer struct {
 	// Include sample pipelines with the deployment of this DSP API Server. Default: true
 	// +kubebuilder:default:=false
 	// +kubebuilder:validation:Optional
-	EnableSamplePipeline bool   `json:"enableSamplePipeline"`
-	ArgoLauncherImage    string `json:"argoLauncherImage,omitempty"`
-	ArgoDriverImage      string `json:"argoDriverImage,omitempty"`
+	EnableSamplePipeline bool `json:"enableSamplePipeline"`
+	// Enable the Instructlab Multi-Phase Training pipeline with the deployment of this DSP API server. Default: false
+	// +kubebuilder:default:=false
+	// +kubebuilder:validation:Optional
+	EnableInstructLabPipeline bool   `json:"enableInstructLabPipeline"`
+	ArgoLauncherImage         string `json:"argoLauncherImage,omitempty"`
+	ArgoDriverImage           string `json:"argoDriverImage,omitempty"`
 	// Specify custom Pod resource requirements for this component.
 	Resources *ResourceRequirements `json:"resources,omitempty"`
 

diff --git a/...ses/datasciencepipelinesapplications.opendatahub.io_datasciencepipelinesapplications.yaml b/...ses/datasciencepipelinesapplications.opendatahub.io_datasciencepipelinesapplications.yaml
@@ -105,6 +105,11 @@ spec:
                       Server. Setting Deploy to false disables operator reconciliation.
                       Default: true'
                     type: boolean
+                  enableInstructLabPipeline:
+                    default: false
+                    description: 'Enable the Instructlab Multi-Phase Training pipeline
+                      with the deployment of this DSP API server. Default: false'
+                    type: boolean
                   enableOauth:
                     default: true
                     description: 'Create an Openshift Route for this DSP API Server.
@@ -990,6 +995,11 @@ spec:
                       Server. Setting Deploy to false disables operator reconciliation.
                       Default: true'
                     type: boolean
+                  enableInstructLabPipeline:
+                    default: false
+                    description: 'Enable the Instructlab Multi-Phase Training pipeline
+                      with the deployment of this DSP API server. Default: false'
+                    type: boolean
                   enableOauth:
                     default: true
                     description: 'Create an Openshift Route for this DSP API Server.

diff --git a/config/internal/apiserver/default/deployment.yaml.tmpl b/config/internal/apiserver/default/deployment.yaml.tmpl
@@ -147,7 +147,7 @@ spec:
           args:
             - --config=/config
             - -logtostderr=true
-            {{ if .APIServer.EnableSamplePipeline }}
+            {{ if or .APIServer.EnableSamplePipeline .APIServer.EnableInstructLabPipeline }}
             - --sampleconfig=/config/sample_config.json
             {{ end }}
             {{ if .PodToPodTLS }}
@@ -206,19 +206,17 @@ spec:
             - mountPath: /etc/tls/private
               name: proxy-tls
           {{ end }}
-          {{ if or .APIServer.EnableSamplePipeline .CustomCABundle }}
-            {{ if .APIServer.EnableSamplePipeline }}
+          {{ if or .APIServer.EnableSamplePipeline .APIServer.EnableInstructLabPipeline}}
             - name: sample-config
               mountPath: /config/sample_config.json
               subPath: sample_config.json
             - name: sample-pipeline
               mountPath: /samples/
-            {{ end }}
-            {{ if .CustomCABundle }}
+          {{ end }}
+          {{ if .CustomCABundle }}
             - mountPath: {{ .CustomCABundleRootMountPath  }}
               name: ca-bundle
-            {{ end }}
-          {{ end }}
+          {{ end }}          
         {{ if .APIServer.EnableRoute }}
         - name: oauth-proxy
           args:
@@ -287,7 +285,7 @@ spec:
           configMap:
             name: {{ .CustomCABundle.ConfigMapName }}
         {{ end }}
-        {{ if .APIServer.EnableSamplePipeline }}
+        {{ if or .APIServer.EnableSamplePipeline .APIServer.EnableInstructLabPipeline}}
         - name: sample-config
           configMap:
             name: sample-config-{{.Name}}

diff --git a/config/internal/apiserver/sample-pipeline/sample-config.yaml.tmpl b/config/internal/apiserver/sample-pipeline/sample-config.yaml.tmpl
@@ -7,11 +7,21 @@ metadata:
         app: {{.APIServerDefaultResourceName}}
         component: data-science-pipelines
 data:
-    sample_config.json: |-
-        [
-          {
-            "name": "[Demo] iris-training",
-            "description": "[source code](https://github.com/opendatahub-io/data-science-pipelines/tree/master/samples/iris-sklearn) A simple pipeline to demonstrate a basic ML Training workflow",
-            "file": "/samples/iris-pipeline-compiled.yaml"
-          }
-        ]
+  sample_config.json: |-
+    [
+      {{- if .EnableSamplePipeline }}
+      {
+        "name": "[Demo] iris-training",
+        "description": "[source code](https://github.com/opendatahub-io/data-science-pipelines/tree/master/samples/iris-sklearn) A simple pipeline to demonstrate a basic ML Training workflow",
+        "file": "/samples/iris-pipeline-compiled.yaml"
+      }
+      {{- if and .EnableSamplePipeline .EnableInstructLabPipeline }},{{ end }}
+      {{- end }}
+      {{- if .EnableInstructLabPipeline }}
+      {
+        "name": "[InstructLab] Multi-Phase Training Pipeline",
+        "description": "[source code](https://github.com/opendatahub-io/ilab-on-ocp) Instructlab Multi-Phase Training Pipeline",
+        "file": "/pipelines/instructlab.yaml"
+      }
+      {{- end }}
+    ]
diff --git a/controllers/apiserver.go b/controllers/apiserver.go
@@ -68,7 +68,7 @@ func (r *DSPAReconciler) ReconcileAPIServer(ctx context.Context, dsp *dspav1.Dat
 	}
 
 	for cmName, template := range samplePipelineTemplates {
-		if dsp.Spec.APIServer.EnableSamplePipeline {
+		if dsp.Spec.APIServer.EnableSamplePipeline || dsp.Spec.APIServer.EnableInstructLabPipeline {
 			err := r.Apply(dsp, params, template)
 			if err != nil {
 				return err

diff --git a/controllers/dspipeline_params.go b/controllers/dspipeline_params.go
@@ -92,9 +92,10 @@ type DSPAParams struct {
 	CustomCABundle *dspa.CABundle
 	DSPONamespace  string
 	// Use to enable tls communication between component pods.
-	PodToPodTLS bool
-
-	APIServerServiceDNSName string
+	PodToPodTLS               bool
+	EnableSamplePipeline      bool
+	EnableInstructLabPipeline bool
+	APIServerServiceDNSName   string
 }
 
 type DBConnection struct {
@@ -601,6 +602,9 @@ func (p *DSPAParams) ExtractParams(ctx context.Context, dsp *dspa.DataSciencePip
 		setStringDefault(argoDriverImageFromConfig, &p.APIServer.ArgoDriverImage)
 		setResourcesDefault(config.APIServerResourceRequirements, &p.APIServer.Resources)
 
+		p.EnableSamplePipeline = dsp.Spec.APIServer.EnableSamplePipeline
+		p.EnableInstructLabPipeline = dsp.Spec.APIServer.EnableInstructLabPipeline
+
 		if p.APIServer.CustomServerConfig == nil {
 			p.APIServer.CustomServerConfig = &dspa.ScriptConfigMap{
 				Name: config.CustomServerConfigMapNamePrefix + dsp.Name,

diff --git a/tests/pipeline_test.go b/tests/pipeline_test.go
@@ -75,4 +75,29 @@ func (suite *IntegrationTestSuite) TestAPIServerDeployment() {
 		require.NoError(t, err)
 		assert.Equal(t, 200, response.StatusCode)
 	})
+
+	suite.T().Run("Should check for InstructLab pipeline existence based on EnableInstructLabPipeline flag", func(t *testing.T) {
+		expectedDisplayName := "[InstructLab] Multi-Phase Training Pipeline"
+
+		// Retrieve pipelines
+		pipelines, err := TestUtil.RetrievePipelines(t, suite.Clientmgr.httpClient, APIServerURL)
+		require.NoError(t, err, "Failed to retrieve pipelines")
+
+		found := false
+		for _, pipeline := range pipelines.Pipelines {
+			if pipeline.DisplayName == expectedDisplayName {
+				found = true
+				break
+			}
+		}
+		if suite.DSPA.Spec.APIServer.EnableInstructLabPipeline {
+			expectedCount := 4
+			assert.Equal(t, expectedCount, len(pipelines.Pipelines), "Pipeline count should match when EnableInstructLabPipeline flag is enabled")
+			assert.True(t, found, "InstructLab pipeline should exist when the flag is enabled")
+		} else {
+			expectedCount := 3
+			assert.Equal(t, expectedCount, len(pipelines.Pipelines), "Pipeline count should match when EnableInstructLabPipeline flag is disabled")
+			assert.False(t, found, "InstructLab pipeline should not exist when the flag is disabled")
+		}
+	})
 }
diff --git a/tests/resources/dspa-lite.yaml b/tests/resources/dspa-lite.yaml
@@ -9,6 +9,7 @@ spec:
     deploy: true
     enableOauth: false
     enableSamplePipeline: true
+    enableInstructLabPipeline: true
     cABundle:
       configMapName: nginx-tls-config
       configMapKey: rootCA.crt

diff --git a/tests/resources/test-pipeline-run.yaml b/tests/resources/test-pipeline-run.yaml
@@ -29,7 +29,7 @@ deploymentSpec:
         - -c
         - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
           \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
-          \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.10.1'\
+          \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.11.0'\
           \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\
           $0\" \"$@\"\n"
         - sh

diff --git a/tests/resources/test-pipeline-with-custom-pip-server-run.yaml b/tests/resources/test-pipeline-with-custom-pip-server-run.yaml
@@ -24,7 +24,7 @@ deploymentSpec:
         - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
           \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
           \ python3 -m pip install --quiet --no-warn-script-location --index-url https://nginx-service.test-pypiserver.svc.cluster.local/simple/\
-          \ 'kfp==2.10.1' '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"\
+          \ 'kfp==2.11.0' '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"\
           3.9\"'  &&  python3 -m pip install --quiet --no-warn-script-location --index-url\
           \ https://nginx-service.test-pypiserver.svc.cluster.local/simple/ 'numpy'\
           \ && \"$0\" \"$@\"\n"
@@ -66,4 +66,4 @@ root:
       Output:
         parameterType: STRING
 schemaVersion: 2.1.0
-sdkVersion: kfp-2.10.1
+sdkVersion: kfp-2.11.0
diff --git a/tests/util/rest.go b/tests/util/rest.go
@@ -94,6 +94,17 @@ func RetrievePipelineId(t *testing.T, httpClient http.Client, APIServerURL strin
 	}
 }
 
+func RetrievePipelines(t *testing.T, httpClient http.Client, APIServerURL string) (Pipeline, error) {
+	response, err := httpClient.Get(fmt.Sprintf("%s/apis/v2beta1/pipelines", APIServerURL))
+	require.NoError(t, err)
+	responseData, err := io.ReadAll(response.Body)
+	require.NoError(t, err)
+	var pipelineData Pipeline
+	err = json.Unmarshal(responseData, &pipelineData)
+	require.NoError(t, err)
+	return pipelineData, nil
+}
+
 func FormatRequestBody(t *testing.T, pipelineID string, PipelineDisplayName string) []byte {
 	requestBody := PipelineRequest{
 		DisplayName: PipelineDisplayName,