diff --git a/.github/workflows/image-build-on-push.yml b/.github/workflows/image-build-on-push.yml new file mode 100644 index 000000000..44010fdb4 --- /dev/null +++ b/.github/workflows/image-build-on-push.yml @@ -0,0 +1,29 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# Test +name: Build latest images on push event + +on: + push: + branches: ["main"] + paths: + - comps/** + - "!**.md" + - "!**.txt" + - .github/workflows/image-build-on-push.yml + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-on-push + cancel-in-progress: true + +jobs: + job1: + uses: ./.github/workflows/reuse-get-test-matrix.yml + + image-build: + needs: job1 + strategy: + matrix: ${{ fromJSON(needs.job1.outputs.run_matrix) }} + uses: ./.github/workflows/reuse-image-build.yml + with: + micro_service: "${{ matrix.service }}" diff --git a/.github/workflows/microservice-test.yml b/.github/workflows/microservice-test.yml index 836327c9e..65412cffc 100644 --- a/.github/workflows/microservice-test.yml +++ b/.github/workflows/microservice-test.yml @@ -20,47 +20,7 @@ concurrency: jobs: job1: - name: Get-test-matrix - runs-on: ubuntu-latest - outputs: - run_matrix: ${{ steps.get-test-matrix.outputs.run_matrix }} - steps: - - name: Checkout out Repo - uses: actions/checkout@v4 - with: - ref: "refs/pull/${{ github.event.number }}/merge" - fetch-depth: 0 - - name: Get test matrix - id: get-test-matrix - run: | - set -xe - merged_commit=$(git log -1 --format='%H') - changed_files="$(git diff --name-only ${{ github.event.pull_request.base.sha }} ${merged_commit} \ - | grep 'comps/' | grep -vE '*.md|*.txt|comps/cores')" || true - services=$(printf '%s\n' "${changed_files[@]}" | cut -d'/' -f2 | grep -vE '*.py' | sort -u) - path_level_1=("asr" "tts") - path_level_3=("llms_summarization" "llms_text-generation" "dataprep_redis") - run_matrix="{\"include\":[" - for service in ${services}; do - hardware="gaudi" # default hardware, set based on the changed files - if [[ "${path_level_1[@]}" =~ "${service}" ]]; then - run_matrix="${run_matrix}{\"service\":\"${service}\",\"hardware\":\"${hardware}\"}," - else - vendors=$(printf '%s\n' "${changed_files[@]}" | grep ${service} | cut -d'/' -f3 | grep -vE '*.py|Dockerfile' | sort -u) - for vendor in ${vendors}; do - if [[ "${path_level_3[@]}" =~ "${service}_${vendor}" ]]; then - sub_vendors=$(printf '%s\n' "${changed_files[@]}" | grep ${service} | grep ${vendor} | cut -d'/' -f4 | grep -vE '*.py' | sort -u) - for sub_vendor in ${sub_vendors}; do - run_matrix="${run_matrix}{\"service\":\"${service}_${vendor}_${sub_vendor}\",\"hardware\":\"${hardware}\"}," - done - else - run_matrix="${run_matrix}{\"service\":\"${service}_${vendor}\",\"hardware\":\"${hardware}\"}," - fi - done - fi - done - run_matrix=$run_matrix"]}" - echo "run_matrix=${run_matrix}" >> $GITHUB_OUTPUT + uses: ./.github/workflows/reuse-get-test-matrix.yml Microservice-test: needs: job1 @@ -80,16 +40,15 @@ jobs: - name: Run microservice test env: HF_TOKEN: ${{ secrets.HF_TOKEN }} - service: ${{ matrix.service }} + service_path: ${{ matrix.service }} hardware: ${{ matrix.hardware }} run: | cd tests - if [ -f test_${service}.sh ]; then timeout 30m bash test_${service}.sh; else echo "Test script not found, skip test!"; fi + service=$(echo $service_path | tr '/' '_') + echo "service=${service}" >> $GITHUB_ENV + if [ -f test_${service}.sh ]; then timeout 10m bash test_${service}.sh; else echo "Test script not found, skip test!"; fi - name: Clean up container - env: - service: ${{ matrix.service }} - hardware: ${{ matrix.hardware }} if: cancelled() || failure() run: | cid=$(docker ps -aq --filter "name=test-comps-*") @@ -100,5 +59,5 @@ jobs: if: ${{ !cancelled() }} uses: actions/upload-artifact@v4 with: - name: ${{ matrix.service }}-${{ matrix.hardware }} + name: ${{ env.service }} path: ${{ github.workspace }}/tests/*.log diff --git a/.github/workflows/reuse-get-test-matrix.yml b/.github/workflows/reuse-get-test-matrix.yml new file mode 100644 index 000000000..3a7704b48 --- /dev/null +++ b/.github/workflows/reuse-get-test-matrix.yml @@ -0,0 +1,72 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +# Support push and pull_request events +name: Get Test Matrix +permissions: read-all +on: + workflow_call: + outputs: + run_matrix: + description: "The matrix string" + value: ${{ jobs.job1.outputs.run_matrix }} + +jobs: + job1: + name: Get-test-matrix + runs-on: ubuntu-latest + outputs: + run_matrix: ${{ steps.get-test-matrix.outputs.run_matrix }} + steps: + - name: Get checkout ref + run: | + if [ "${{ github.event_name }}" == "pull_request" ] || [ "${{ github.event_name }}" == "pull_request_target" ]; then + echo "CHECKOUT_REF=refs/pull/${{ github.event.number }}/merge" >> $GITHUB_ENV + else + echo "CHECKOUT_REF=${{ github.ref }}" >> $GITHUB_ENV + fi + echo "checkout ref ${{ env.CHECKOUT_REF }}" + + - name: Checkout out Repo + uses: actions/checkout@v4 + with: + ref: ${{ env.CHECKOUT_REF }} + fetch-depth: 0 + + - name: Get test matrix + id: get-test-matrix + run: | + set -xe + if [ "${{ github.event_name }}" == "pull_request" ] || [ "${{ github.event_name }}" == "pull_request_target" ]; then + base_commit=${{ github.event.pull_request.base.sha }} + else + base_commit=$(git rev-parse HEAD~1) # push event + fi + merged_commit=$(git log -1 --format='%H') + + changed_files="$(git diff --name-only ${base_commit} ${merged_commit} | \ + grep 'comps/' | grep -vE '*.md|*.txt|comps/cores')" || true + services=$(printf '%s\n' "${changed_files[@]}" | cut -d'/' -f2 | grep -vE '*.py' | sort -u) + path_level_1=("asr" "tts") + path_level_3=("llms/summarization" "llms/text-generation" "dataprep/redis" "retrievers/langchain/redis") + run_matrix="{\"include\":[" + for service in ${services}; do + hardware="gaudi" # default hardware, set based on the changed files + if [[ "${path_level_1[@]}" =~ "${service}" ]]; then + run_matrix="${run_matrix}{\"service\":\"${service}\",\"hardware\":\"${hardware}\"}," + else + vendors=$(printf '%s\n' "${changed_files[@]}" | grep ${service} | cut -d'/' -f3 | grep -vE '*.py|Dockerfile|*.md' | sort -u) + for vendor in ${vendors}; do + if [[ "${path_level_3[@]}" =~ "${service}/${vendor}" ]]; then + sub_vendors=$(printf '%s\n' "${changed_files[@]}" | grep ${service} | grep ${vendor} | cut -d'/' -f4 | grep -vE '*.py' | sort -u) + for sub_vendor in ${sub_vendors}; do + run_matrix="${run_matrix}{\"service\":\"${service}/${vendor}/${sub_vendor}\",\"hardware\":\"${hardware}\"}," + done + else + run_matrix="${run_matrix}{\"service\":\"${service}/${vendor}\",\"hardware\":\"${hardware}\"}," + fi + done + fi + done + run_matrix=$run_matrix"]}" + echo "run_matrix=${run_matrix}" >> $GITHUB_OUTPUT diff --git a/.github/workflows/reuse-image-build.yml b/.github/workflows/reuse-image-build.yml new file mode 100644 index 000000000..316fc7124 --- /dev/null +++ b/.github/workflows/reuse-image-build.yml @@ -0,0 +1,31 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +name: Image Build +permissions: read-all +on: + workflow_call: + inputs: + micro_service: + required: true + type: string + +jobs: + micro-image-build: + continue-on-error: true + strategy: + matrix: + node: [docker-build-xeon, docker-build-gaudi] + runs-on: ${{ matrix.node }} + steps: + - name: Checkout out Repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Building MicroService Docker Image + id: build-microservice-image + env: + micro_service: ${{ inputs.micro_service }} + run: | + bash .github/workflows/scripts/docker_images_build_push.sh ${micro_service} diff --git a/.github/workflows/scripts/docker_images_build_push.sh b/.github/workflows/scripts/docker_images_build_push.sh new file mode 100644 index 000000000..ff8a6b8f4 --- /dev/null +++ b/.github/workflows/scripts/docker_images_build_push.sh @@ -0,0 +1,59 @@ +#!/bin/bash +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +set -xe + +WORKSPACE=$PWD +IMAGE_REPO=${IMAGE_REPO:-$OPEA_IMAGE_REPO} +IMAGE_TAG=${IMAGE_TAG:-latest} + +function docker_build() { + # docker_build + IMAGE_NAME=$1 + micro_service=$2 + dockerfile_path=${WORKSPACE}/comps/${micro_service} + if [ -f "$dockerfile_path/Dockerfile" ]; then + DOCKERFILE_PATH="$dockerfile_path/Dockerfile" + elif [ -f "$dockerfile_path/docker/Dockerfile" ]; then + DOCKERFILE_PATH="$dockerfile_path/docker/Dockerfile" + else + echo "Dockerfile not found" + exit 1 + fi + echo "Building ${IMAGE_REPO}${IMAGE_NAME}:$IMAGE_TAG using Dockerfile $DOCKERFILE_PATH" + + docker build --no-cache -t ${IMAGE_REPO}${IMAGE_NAME}:$IMAGE_TAG -f $DOCKERFILE_PATH . + docker push ${IMAGE_REPO}${IMAGE_NAME}:$IMAGE_TAG + docker rmi ${IMAGE_REPO}${IMAGE_NAME}:$IMAGE_TAG +} + +micro_service=$1 +case ${micro_service} in + "asr"|"tts") + IMAGE_NAME="opea/${micro_service}" + ;; + "embeddings/langchain") + IMAGE_NAME="opea/embedding-tei" + ;; + "retrievers/langchain") + IMAGE_NAME="opea/retriever-redis" + ;; + "reranks/langchain") + IMAGE_NAME="opea/reranking-tei" + ;; + "llms/text-generation/tgi") + IMAGE_NAME="opea/llm-tgi" + ;; + "dataprep/redis/langchain") + IMAGE_NAME="opea/dataprep-redis" + ;; + "llms/summarization/tgi") + IMAGE_NAME="opea/llm-docsum-tgi" + ;; + *) + echo "Not supported yet" + exit 0 + ;; +esac +docker_build "${IMAGE_NAME}" "${micro_service}" diff --git a/tests/test_retrievers_langchain.sh b/tests/test_retrievers_langchain_redis.sh similarity index 100% rename from tests/test_retrievers_langchain.sh rename to tests/test_retrievers_langchain_redis.sh