Skip to content

Commit

Permalink
Sync with main and resolve conflicts
Browse files Browse the repository at this point in the history
Signed-off-by: Abolfazl Shahbazi <[email protected]>
  • Loading branch information
ashahba committed Sep 4, 2024
2 parents caa499a + 54aa943 commit 4ed3628
Show file tree
Hide file tree
Showing 212 changed files with 8,831 additions and 788 deletions.
12 changes: 12 additions & 0 deletions .github/CODEOWNERS
Validating CODEOWNERS rules …
Original file line number Diff line number Diff line change
@@ -1,9 +1,21 @@
/comps/agent/ [email protected]
/comps/finetuning/ [email protected]
/comps/guardrails/ [email protected]
/comps/asr/ [email protected]
/comps/intent_detection/ [email protected]
/comps/knowledgegraphs/ [email protected]
/comps/cores/ [email protected]
/comps/dataprep/ [email protected]
/comps/embeddings/ [email protected]
/comps/guardrails/ [email protected]
/comps/llms/ [email protected]
/comps/lvms/ [email protected]
/comps/nginx/ [email protected]
/comps/prompt_registry/ [email protected]
/comps/chathistory/ [email protected]
/comps/reranks/ [email protected]
/comps/retrievers/ [email protected]
/comps/tts/ [email protected]
/comps/ragas/ [email protected]
/comps/vectorstores/ [email protected]
/comps/web_retrievers/ [email protected]
93 changes: 33 additions & 60 deletions .github/workflows/_comps-workflow.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,17 +20,26 @@ on:
default: true
required: false
type: boolean
# scan:
# default: true
# required: false
# type: boolean
test:
default: true
description: "Test comps with docker compose"
required: false
type: boolean
mode:
default: "CD"
description: "Whether the test range is CI or CD"
required: false
type: string

jobs:
####################################################################################################
# Image Build
####################################################################################################
build-images:
runs-on: "docker-build-${{ inputs.node }}"
continue-on-error: true
outputs:
file_exists: ${{ steps.get-yaml-path.outputs.file_exists }}
steps:
- name: Clean Up Working Directory
run: sudo rm -rf ${{github.workspace}}/*
Expand All @@ -39,82 +48,46 @@ jobs:
uses: actions/checkout@v4

- name: Clone required Repo
id: get-yaml-path
run: |
cd ${{ github.workspace }}/.github/workflows/docker/compose
# service=$(echo ${{ inputs.service }} | cut -d'_' -f1)
docker_compose_yml=${{ github.workspace }}/.github/workflows/docker/compose/${{ inputs.service }}-compose.yaml
echo ${docker_compose_yml}
if [[ "${{ inputs.mode }}" == "CD" ]]; then
docker_compose_yml=${{ github.workspace }}/.github/workflows/docker/compose/${{ inputs.service }}-compose-cd.yaml
else
docker_compose_yml=${{ github.workspace }}/.github/workflows/docker/compose/${{ inputs.service }}-compose.yaml
fi
echo "docker_compose_path=${docker_compose_yml}" >> $GITHUB_OUTPUT
if [ -f "$docker_compose_yml" ]; then
echo "file_exists=true" >> $GITHUB_OUTPUT
else
echo "There is no ${{ inputs.mode }} part of ${{ inputs.service }} that needs to be executed."
echo "file_exists=false" >> $GITHUB_OUTPUT
fi
cd ${{ github.workspace }}
if [[ $(grep -c "llava-tgi:" ${docker_compose_yml}) != 0 ]]; then
git clone https://github.com/yuanwu2017/tgi-gaudi.git && cd tgi-gaudi && git checkout v2.0.4
git clone https://github.com/yuanwu2017/tgi-gaudi.git && cd tgi-gaudi && git checkout v2.0.4 && cd ../
fi
if [[ $(grep -c "vllm-openvino:" ${docker_compose_yml}) != 0 ]]; then
git clone https://github.com/vllm-project/vllm.git vllm-openvino
fi
# echo "service=$service" >> $GITHUB_ENV
- name: Build Image
if: ${{ fromJSON(inputs.build) }}
if: ${{ fromJSON(inputs.build) && steps.get-yaml-path.outputs.file_exists == 'true' }}
uses: opea-project/validation/actions/image-build@main
with:
work_dir: ${{ github.workspace }}/
docker_compose_path: ${{ github.workspace }}/.github/workflows/docker/compose/${{ inputs.service }}-compose.yaml
work_dir: ${{ github.workspace }}
docker_compose_path: ${{ steps.get-yaml-path.outputs.docker_compose_path }}
registry: ${OPEA_IMAGE_REPO}opea
tag: ${{ inputs.tag }}
# ####################################################################################################
# # Trivy Scan
# ####################################################################################################
# get-image-list:
# needs: [build-images]
# if: ${{ fromJSON(inputs.scan) && inputs.node == 'gaudi' }}
# runs-on: ubuntu-latest
# outputs:
# matrix: ${{ steps.scan-matrix.outputs.matrix }}
# steps:
# - name: Checkout out Repo
# uses: actions/checkout@v4

# - name: Set Matrix
# id: scan-matrix
# run: |
# pip install yq
# compose_path=${{ github.workspace }}/${{ inputs.example }}/docker/docker_build_compose.yaml
# echo "matrix=$(cat ${compose_path} | yq -r '.[]' | jq 'keys' | jq -c '.')" >> $GITHUB_OUTPUT

# scan-images:
# needs: [get-image-list, build-images]
# if: ${{ fromJSON(inputs.scan) && inputs.node == 'gaudi'}}
# runs-on: "docker-build-${{ inputs.node }}"
# strategy:
# matrix:
# image: ${{ fromJSON(needs.get-image-list.outputs.matrix) }}
# fail-fast: false
# steps:
# - name: Pull Image
# run: |
# docker pull ${OPEA_IMAGE_REPO}opea/${{ matrix.image }}:${{ inputs.tag }}
# echo "OPEA_IMAGE_REPO=${OPEA_IMAGE_REPO}" >> $GITHUB_ENV

# - name: Scan Container
# uses: opea-project/validation/actions/trivy-scan@main
# with:
# image-ref: ${{ env.OPEA_IMAGE_REPO }}opea/${{ matrix.image }}:${{ inputs.tag }}
# output: ${{ matrix.image }}-scan.txt

# - name: Cleanup
# if: always()
# run: docker rmi -f ${OPEA_IMAGE_REPO}opea/${{ matrix.image }}:${{ inputs.tag }}

# - uses: actions/[email protected]
# with:
# name: ${{ matrix.image }}-scan
# path: ${{ matrix.image }}-scan.txt
# overwrite: true

####################################################################################################
# Docker Compose Test
####################################################################################################
test-service-compose:
needs: [build-images]
if: ${{ fromJSON(inputs.test) && needs.build-images.outputs.file_exists == 'true' }}
uses: ./.github/workflows/_run-docker-compose.yml
with:
tag: ${{ inputs.tag }}
Expand Down
65 changes: 65 additions & 0 deletions .github/workflows/_get-image-list.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
# Copyright (C) 2024 Intel Corporation
# SPDX-License-Identifier: Apache-2.0

name: Get Image List
permissions: read-all
on:
workflow_call:
inputs:
services:
default: ""
required: false
type: string
images:
default: ""
required: false
type: string
mode:
default: "CD"
required: false
type: string
outputs:
matrix:
description: "Image List"
value: ${{ jobs.get-image-list.outputs.matrix }}

jobs:
get-image-list:
runs-on: ubuntu-latest
outputs:
matrix: ${{ steps.get-matrix.outputs.matrix }}
steps:
- name: Checkout out Repo
uses: actions/checkout@v4

- name: Set Matrix
id: get-matrix
run: |
image_list=[]
if [[ ! -z "${{ inputs.services }}" ]]; then
pip install yq
services=($(echo ${{ inputs.services }} | tr ',' ' '))
for service in ${services[@]}
do
if [[ "${{ inputs.mode }}" == "CD" ]]; then
docker_compose_yml=${{ github.workspace }}/.github/workflows/docker/compose/${service}-compose-cd.yaml
else
docker_compose_yml=${{ github.workspace }}/.github/workflows/docker/compose/${service}-compose.yaml
fi
if [ -f "$docker_compose_yml" ]; then
images=$(cat $docker_compose_yml | yq -r '.[]' | jq 'keys' | jq -c '.')
image_list=$(echo ${image_list} | jq -s '.[0] + .[1] | unique' - <(echo ${images}))
fi
done
fi
if [[ ! -z "${{ inputs.images }}" ]]; then
images=($(echo ${{ inputs.images }} | tr ',' ' '))
input_image_list=$(printf '%s\n' "${images[@]}" | sort -u | jq -R '.' | jq -sc '.')
image_list=$(echo ${image_list} | jq -s '.[0] + .[1] | unique' - <(echo ${input_image_list}))
fi
echo "print image list..."
echo "$image_list" | jq . | jq -r '.[]'
echo "end of image list..."
echo "matrix=$(echo ${image_list} | jq -c '.')" >> $GITHUB_OUTPUT
3 changes: 1 addition & 2 deletions .github/workflows/_get-test-matrix.yml
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@ jobs:
else
echo "CHECKOUT_REF=${{ github.ref }}" >> $GITHUB_ENV
fi
echo "checkout ref ${{ env.CHECKOUT_REF }}"
- name: Checkout out Repo
uses: actions/checkout@v4
Expand All @@ -39,7 +38,7 @@ jobs:
set -xe
if [ "${{ github.event_name }}" == "pull_request" ] || [ "${{ github.event_name }}" == "pull_request_target" ]; then
LATEST_COMMIT_SHA=$(curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
"https://api.github.com/repos/opea-project/GenAIComps/commits?sha=main" | jq -r '.[0].sha')
"https://api.github.com/repos/opea-project/GenAIComps/commits?sha=${{ github.event.pull_request.base.ref }}" | jq -r '.[0].sha')
echo "Latest commit SHA is $LATEST_COMMIT_SHA"
base_commit=$LATEST_COMMIT_SHA
else
Expand Down
9 changes: 9 additions & 0 deletions .github/workflows/docker/compose/agent-compose-cd.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
# Copyright (C) 2024 Intel Corporation
# SPDX-License-Identifier: Apache-2.0

# this file should be run in the root of the repo
services:
comps-agent-langchain:
build:
dockerfile: comps/agent/langchain/docker/Dockerfile
image: ${REGISTRY:-opea}/comps-agent-langchain:${TAG:-latest}
6 changes: 3 additions & 3 deletions .github/workflows/docker/compose/asr-compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,12 @@ services:
asr:
build:
dockerfile: comps/asr/Dockerfile
image: ${REGISTRY}opea/asr:${TAG:-latest}
image: ${REGISTRY:-opea}/asr:${TAG:-latest}
whisper:
build:
dockerfile: comps/asr/whisper/Dockerfile
image: ${REGISTRY}opea/whisper:${TAG:-latest}
image: ${REGISTRY:-opea}/whisper:${TAG:-latest}
whisper-gaudi:
build:
dockerfile: comps/asr/whisper/Dockerfile_hpu
image: ${REGISTRY}opea/whisper-gaudi:${TAG:-latest}
image: ${REGISTRY:-opea}/whisper-gaudi:${TAG:-latest}
9 changes: 9 additions & 0 deletions .github/workflows/docker/compose/chathistory-compose-cd.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
# Copyright (C) 2024 Intel Corporation
# SPDX-License-Identifier: Apache-2.0

# this file should be run in the root of the repo
services:
chathistory-mongo-server:
build:
dockerfile: comps/chathistory/mongo/docker/Dockerfile
image: ${REGISTRY:-opea}/chathistory-mongo-server:${TAG:-latest}
25 changes: 25 additions & 0 deletions .github/workflows/docker/compose/dataprep-compose-cd.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
# Copyright (C) 2024 Intel Corporation
# SPDX-License-Identifier: Apache-2.0

# others: dataprep-redis-llama-index,dataprep-on-ray-redis
services:
dataprep-redis-llama-index:
build:
dockerfile: comps/dataprep/redis/llama_index/docker/Dockerfile
image: ${REGISTRY:-opea}/dataprep-redis-llama-index:${TAG:-latest}
dataprep-on-ray-redis:
build:
dockerfile: comps/dataprep/redis/langchain_ray/docker/Dockerfile
image: ${REGISTRY:-opea}/dataprep-on-ray-redis:${TAG:-latest}
dataprep-milvus:
build:
dockerfile: comps/dataprep/milvus/docker/Dockerfile
image: ${REGISTRY:-opea}/dataprep-milvus:${TAG:-latest}
dataprep-pgvector:
build:
dockerfile: comps/dataprep/pgvector/langchain/docker/Dockerfile
image: ${REGISTRY:-opea}/dataprep-pgvector:${TAG:-latest}
dataprep-pinecone:
build:
dockerfile: comps/dataprep/pinecone/docker/Dockerfile
image: ${REGISTRY:-opea}/dataprep-pinecone:${TAG:-latest}
12 changes: 2 additions & 10 deletions .github/workflows/docker/compose/dataprep-compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,16 +8,8 @@ services:
dataprep-redis:
build:
dockerfile: comps/dataprep/redis/langchain/docker/Dockerfile
image: ${REGISTRY}opea/dataprep-redis:${TAG:-latest}
image: ${REGISTRY:-opea}/dataprep-redis:${TAG:-latest}
dataprep-qdrant:
build:
dockerfile: comps/dataprep/qdrant/docker/Dockerfile
image: ${REGISTRY}opea/dataprep-qdrant:${TAG:-latest}
dataprep-redis-llama-index:
build:
dockerfile: comps/dataprep/redis/llama_index/docker/Dockerfile
image: ${REGISTRY}opea/dataprep-redis-llama-index:${TAG:-latest}
dataprep-on-ray-redis:
build:
dockerfile: comps/dataprep/redis/langchain_ray/docker/Dockerfile
image: ${REGISTRY}opea/dataprep-on-ray-redis:${TAG:-latest}
image: ${REGISTRY:-opea}/dataprep-qdrant:${TAG:-latest}
16 changes: 16 additions & 0 deletions .github/workflows/docker/compose/embeddings-compose-cd.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
# Copyright (C) 2024 Intel Corporation
# SPDX-License-Identifier: Apache-2.0

services:
embedding-langchain-mosec-endpoint:
build:
dockerfile: comps/embeddings/langchain-mosec/mosec-docker/Dockerfile
image: ${REGISTRY:-opea}/embedding-langchain-mosec-endpoint:${TAG:-latest}
embedding-langchain-mosec:
build:
dockerfile: comps/embeddings/langchain-mosec/docker/Dockerfile
image: ${REGISTRY:-opea}/embedding-langchain-mosec:${TAG:-latest}
embedding-tei-llama-index:
build:
dockerfile: comps/embeddings/llama_index/docker/Dockerfile
image: ${REGISTRY:-opea}/embedding-tei-llama-index:${TAG:-latest}
2 changes: 1 addition & 1 deletion .github/workflows/docker/compose/embeddings-compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -7,4 +7,4 @@ services:
embedding-tei:
build:
dockerfile: comps/embeddings/langchain/docker/Dockerfile
image: ${REGISTRY}opea/embedding-tei:${TAG:-latest}
image: ${REGISTRY:-opea}/embedding-tei:${TAG:-latest}
8 changes: 8 additions & 0 deletions .github/workflows/docker/compose/guardrails-compose-cd.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
# Copyright (C) 2024 Intel Corporation
# SPDX-License-Identifier: Apache-2.0

services:
guardrails-pii-detection:
build:
dockerfile: comps/guardrails/pii_detection/docker/Dockerfile
image: ${REGISTRY:-opea}/guardrails-pii-detection:${TAG:-latest}
6 changes: 1 addition & 5 deletions .github/workflows/docker/compose/guardrails-compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,4 @@ services:
guardrails-tgi:
build:
dockerfile: comps/guardrails/llama_guard/docker/Dockerfile
image: ${REGISTRY}opea/guardrails-tgi:${TAG:-latest}
guardrails-pii-detection:
build:
dockerfile: comps/guardrails/pii_detection/docker/Dockerfile
image: ${REGISTRY}opea/guardrails-pii-detection:${TAG:-latest}
image: ${REGISTRY:-opea}/guardrails-tgi:${TAG:-latest}
13 changes: 13 additions & 0 deletions .github/workflows/docker/compose/llms-compose-cd.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
# Copyright (C) 2024 Intel Corporation
# SPDX-License-Identifier: Apache-2.0

services:
llm-native:
build:
dockerfile: comps/llms/text-generation/native/docker/Dockerfile
image: ${REGISTRY:-opea}/llm-native:${TAG:-latest}
vllm-openvino:
build:
context: vllm-openvino
dockerfile: Dockerfile.openvino
image: ${REGISTRY:-opea}/vllm-openvino:${TAG:-latest}
Loading

0 comments on commit 4ed3628

Please sign in to comment.