Refactor CI scripts to support more components (#473)

Signed-off-by: Dolpher Du <[email protected]>

helm-charts/CI.md

@@ -0,0 +1,25 @@
+# CI guidelines for helm charts
+
+## Table of Contents
+
+- [Infra Setup](#infra-setup)
+- [Add new test case](#add-new-test-case)
+
+## Infra Setup
+
+Helm charts are organized as:
+1). E2E examples are directly located under helm-charts/ directory
+2). Charts for components are located under helm-charts/common directory.
+
+We have Xeon servers with Gaudi accelerator card, which covers CPU and Gaudi accelerated workloads. For coverage of other configurations/features, it's case by case.
+
+CI tests values files which names are listed in `ci-*values.yaml` files.
+
+## Add new test case
+
+Add a new values file following the naming rules: `ci-*values.yaml`.
+If the values file for CI is identical to another one, a symbolic link can be used.
+
+See following files for the test scripts:
+https://github.com/opea-project/GenAIInfra/blob/main/.github/workflows/pr-chart-e2e.yaml
+https://github.com/opea-project/GenAIInfra/blob/main/.github/workflows/_helm-e2e.yaml

helm-charts/chatqna/ci-gaudi-values.yaml

@@ -0,0 +1 @@
+gaudi-values.yaml

helm-charts/chatqna/ci-guardrails-gaudi-values.yaml

@@ -0,0 +1 @@
+guardrails-gaudi-values.yaml

helm-charts/chatqna/ci-guardrails-values.yaml

@@ -0,0 +1 @@
+guardrails-values.yaml

helm-charts/chatqna/ci-values.yaml

@@ -0,0 +1 @@
+values.yaml

helm-charts/codegen/ci-gaudi-values.yaml

@@ -0,0 +1 @@
+gaudi-values.yaml

helm-charts/codegen/ci-values.yaml

@@ -0,0 +1 @@
+values.yaml

helm-charts/codetrans/ci-gaudi-values.yaml

@@ -0,0 +1 @@
+gaudi-values.yaml

helm-charts/codetrans/ci-values.yaml

@@ -0,0 +1 @@
+values.yaml

helm-charts/common/asr/Chart.yaml

@@ -12,4 +12,4 @@ dependencies:
   - name: whisper
     version: 1.0.0
     repository: file://../whisper
-    condition: autodependency.enabled
+    condition: whisper.enabled

helm-charts/common/asr/README.md

@@ -24,7 +24,7 @@ helm install asr . --set ASR_ENDPOINT=${ASR_ENDPOINT}
 ```console
 cd GenAIInfra/helm-charts/common/asr
 helm dependency update
-helm install asr . --set autodependency.enabled=true
+helm install asr . --set whisper.enabled=true
 ```
 
 ## Verify

helm-charts/common/asr/ci-values.yaml

@@ -0,0 +1,9 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+# Default values for asr.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+whisper:
+  enabled: true

helm-charts/common/asr/values.yaml

@@ -5,7 +5,7 @@
 # This is a YAML-formatted file.
 # Declare variables to be passed into your templates.
 
-autodependency:
+whisper:
   enabled: false
 
 replicaCount: 1

helm-charts/common/chathistory-usvc/Chart.yaml

@@ -12,4 +12,4 @@ dependencies:
   - name: mongodb
     version: 1.0.0
     repository: file://../mongodb
-    condition: autodependency.enabled
+    condition: mongodb.enabled

helm-charts/common/chathistory-usvc/README.md

@@ -25,7 +25,7 @@ helm install chathistory-usvc . --set MONGO_HOST=${MONGO_HOST} --set MONGO_PORT=
 ```console
 cd GenAIInfra/helm-charts/common/chathistory-usvc
 helm dependency update
-helm install chathistory-usvc . --set autodependency.enabled=true
+helm install chathistory-usvc . --set mongodb.enabled=true
 ```
 
 ## Verify

helm-charts/common/chathistory-usvc/ci-values.yaml

@@ -0,0 +1,9 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+# Default values for chathistory-usvc.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+mongodb:
+  enabled: true

helm-charts/common/chathistory-usvc/values.yaml

@@ -5,7 +5,7 @@
 # This is a YAML-formatted file.
 # Declare variables to be passed into your templates.
 
-autodependency:
+mongodb:
   enabled: false
 
 replicaCount: 1

helm-charts/common/data-prep/Chart.yaml

@@ -12,8 +12,8 @@ dependencies:
   - name: tei
     version: 1.0.0
     repository: file://../tei
-    condition: autodependency.enabled
+    condition: tei.enabled
   - name: redis-vector-db
     version: 1.0.0
     repository: file://../redis-vector-db
-    condition: autodependency.enabled
+    condition: redis-vector-db.enabled

helm-charts/common/data-prep/README.md

@@ -25,7 +25,8 @@ helm install data-prep . --set REDIS_URL=${REDIS_URL} --set TEI_EMBEDDING_ENDPOI
 ```console
 cd GenAIInfra/helm-charts/common/data-prep
 helm dependency update
-helm install data-prep . --set autodependency.enabled=true
+helm install data-prep . --set redis-vector-db.enabled=true --set tei.enabled=true
+
 ```
 
 ## Verify

helm-charts/common/data-prep/ci-values.yaml

@@ -0,0 +1,11 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+# Default values for data-prep.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+tei:
+  enabled: true
+redis-vector-db:
+  enabled: true

helm-charts/common/data-prep/values.yaml

@@ -5,7 +5,9 @@
 # This is a YAML-formatted file.
 # Declare variables to be passed into your templates.
 
-autodependency:
+tei:
+  enabled: false
+redis-vector-db:
   enabled: false
 
 replicaCount: 1

helm-charts/common/embedding-usvc/Chart.yaml

@@ -12,4 +12,4 @@ dependencies:
   - name: tei
     version: 1.0.0
     repository: file://../tei
-    condition: autodependency.enabled
+    condition: tei.enabled

helm-charts/common/embedding-usvc/README.md

@@ -24,7 +24,7 @@ helm install embedding-usvc . --set TEI_EMBEDDING_ENDPOINT=${TEI_EMBEDDING_ENDPO
 ```console
 cd GenAIInfra/helm-charts/common/embedding-usvc
 helm dependency update
-helm install embedding-usvc . --set autodependency.enabled=true
+helm install embedding-usvc . --set tei.enabled=true
 ```
 
 ## Verify

helm-charts/common/embedding-usvc/ci-values.yaml

@@ -0,0 +1,9 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+# Default values for embedding-usvc.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+tei:
+  enabled: true

helm-charts/common/embedding-usvc/values.yaml

@@ -5,7 +5,7 @@
 # This is a YAML-formatted file.
 # Declare variables to be passed into your templates.
 
-autodependency:
+tei:
   enabled: false
 
 replicaCount: 1

helm-charts/common/guardrails-usvc/Chart.yaml

@@ -11,4 +11,4 @@ dependencies:
   - name: tgi
     version: 1.0.0
     repository: file://../tgi
-    condition: autodependency.enabled
+    condition: tgi.enabled

helm-charts/common/guardrails-usvc/README.md

@@ -27,7 +27,7 @@ helm install guardrails-usvc . --set global.HUGGINGFACEHUB_API_TOKEN=${HFTOKEN}
 cd GenAIInfra/helm-charts/common/guardrails-usvc
 export HFTOKEN="insert-your-huggingface-token-here"
 helm dependency update
-helm install guardrails-usvc . --set global.HUGGINGFACEHUB_API_TOKEN=${HFTOKEN} --set autodependency.enabled=true --wait
+helm install guardrails-usvc . --set global.HUGGINGFACEHUB_API_TOKEN=${HFTOKEN} --set tgi.enabled=true --wait
 ```
 
 ## Verify

helm-charts/common/guardrails-usvc/ci-values.yaml

@@ -0,0 +1,9 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+# Default values for guardrails-usvc.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+tgi:
+  enabled: true

helm-charts/common/guardrails-usvc/values.yaml

@@ -5,7 +5,7 @@
 # This is a YAML-formatted file.
 # Declare variables to be passed into your templates.
 
-autodependency:
+tgi:
   enabled: false
 
 replicaCount: 1

helm-charts/common/llm-uservice/Chart.yaml

@@ -12,4 +12,4 @@ dependencies:
   - name: tgi
     version: 1.0.0
     repository: file://../tgi
-    condition: autodependency.enabled
+    condition: tgi.enabled

helm-charts/common/llm-uservice/README.md

@@ -26,7 +26,7 @@ helm install llm-uservice . --set global.HUGGINGFACEHUB_API_TOKEN=${HFTOKEN} --s
 cd GenAIInfra/helm-charts/common/llm-uservice
 export HFTOKEN="insert-your-huggingface-token-here"
 helm dependency update
-helm install llm-uservice . --set global.HUGGINGFACEHUB_API_TOKEN=${HFTOKEN} --set autodependency.enabled=true --wait
+helm install llm-uservice . --set global.HUGGINGFACEHUB_API_TOKEN=${HFTOKEN} --set tgi.enabled=true --wait
 ```
 
 ## Verify

helm-charts/common/llm-uservice/variant_docsum-values.yaml → helm-charts/common/llm-uservice/ci-docsum-values.yaml

@@ -4,3 +4,5 @@
 image:
   repository: opea/llm-docsum-tgi
   tag: "latest"
+tgi:
+  enabled: true

helm-charts/common/llm-uservice/variant_faqgen-values.yaml → helm-charts/common/llm-uservice/ci-faqgen-values.yaml

@@ -6,4 +6,5 @@ image:
   tag: "latest"
 
 tgi:
+  enabled: true
   LLM_MODEL_ID: meta-llama/Meta-Llama-3-8B-Instruct

helm-charts/common/llm-uservice/ci-values.yaml

@@ -0,0 +1,9 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+# Default values for llm-uservice.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+tgi:
+  enabled: true

helm-charts/common/llm-uservice/values.yaml

@@ -5,7 +5,7 @@
 # This is a YAML-formatted file.
 # Declare variables to be passed into your templates.
 
-autodependency:
+tgi:
   enabled: false
 
 replicaCount: 1

helm-charts/common/mongodb/ci-values.yaml

@@ -0,0 +1 @@
+values.yaml

helm-charts/common/prompt-usvc/Chart.yaml

@@ -12,4 +12,4 @@ dependencies:
   - name: mongodb
     version: 1.0.0
     repository: file://../mongodb
-    condition: autodependency.enabled
+    condition: mongodb.enabled

helm-charts/common/prompt-usvc/README.md

@@ -25,7 +25,7 @@ helm install prompt-usvc . --set MONGO_HOST=${MONGO_HOST} --set MONGO_PORT=${MON
 ```console
 cd GenAIInfra/helm-charts/common/prompt-usvc
 helm dependency update
-helm install prompt-usvc . --set autodependency.enabled=true
+helm install prompt-usvc . --set mongodb.enabled=true
 ```
 
 ## Verify

helm-charts/common/prompt-usvc/ci-values.yaml

@@ -0,0 +1,9 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+# Default values for prompt-usvc.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+mongodb:
+  enabled: true

helm-charts/common/prompt-usvc/values.yaml

@@ -5,7 +5,7 @@
 # This is a YAML-formatted file.
 # Declare variables to be passed into your templates.
 
-autodependency:
+mongodb:
   enabled: false
 
 replicaCount: 1

helm-charts/common/redis-vector-db/ci-values.yaml

@@ -0,0 +1 @@
+values.yaml

helm-charts/common/reranking-usvc/Chart.yaml

@@ -12,4 +12,4 @@ dependencies:
   - name: teirerank
     version: 1.0.0
     repository: file://../teirerank
-    condition: autodependency.enabled
+    condition: teirerank.enabled

helm-charts/common/reranking-usvc/README.md

@@ -24,7 +24,7 @@ helm install reranking-usvc . --set TEI_RERANKING_ENDPOINT=${TEI_RERANKING_ENDPO
 ```console
 cd GenAIInfra/helm-charts/common/reranking-usvc
 helm dependency update
-helm install reranking-usvc . --set autodependency.enabled=true
+helm install reranking-usvc . --set teirerank.enabled=true
 ```
 
 ## Verify

helm-charts/common/reranking-usvc/ci-values.yaml

@@ -0,0 +1,9 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+# Default values for reranking-usvc.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+teirerank:
+  enabled: true

helm-charts/common/reranking-usvc/values.yaml

@@ -5,7 +5,7 @@
 # This is a YAML-formatted file.
 # Declare variables to be passed into your templates.
 
-autodependency:
+teirerank:
   enabled: false
 
 replicaCount: 1

helm-charts/common/retriever-usvc/Chart.yaml

@@ -12,8 +12,8 @@ dependencies:
   - name: tei
     version: 1.0.0
     repository: file://../tei
-    condition: autodependency.enabled
+    condition: tei.enabled
   - name: redis-vector-db
     version: 1.0.0
     repository: file://../redis-vector-db
-    condition: autodependency.enabled
+    condition: redis-vector-db.enabled

helm-charts/common/retriever-usvc/README.md

@@ -25,7 +25,7 @@ helm install retriever-usvc . --set REDIS_URL=${REDIS_URL} --set TEI_EMBEDDING_E
 ```console
 cd GenAIInfra/helm-charts/common/retriever-usvc
 helm dependency update
-helm install retriever-usvc . --set autodependency.enabled=true
+helm install retriever-usvc . --set tei.enabled=true --set redis-vector-db.enabled=true
 ```
 
 ## Verify

helm-charts/common/retriever-usvc/ci-values.yaml

@@ -0,0 +1,11 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+# Default values for retriever-usvc.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+tei:
+  enabled: false
+redis-vector-db:
+  enabled: false

helm-charts/common/retriever-usvc/values.yaml

@@ -5,7 +5,9 @@
 # This is a YAML-formatted file.
 # Declare variables to be passed into your templates.
 
-autodependency:
+tei:
+  enabled: false
+redis-vector-db:
   enabled: false
 
 replicaCount: 1

helm-charts/common/speecht5/ci-gaudi-values.yaml

@@ -0,0 +1 @@
+gaudi-values.yaml

helm-charts/common/speecht5/ci-values.yaml

@@ -0,0 +1 @@
+values.yaml