From 1f623c94730450769ea17d095067f7e605600865 Mon Sep 17 00:00:00 2001 From: Monu Singh Date: Wed, 18 Oct 2023 13:44:56 +0530 Subject: [PATCH] Container image (#1249) (#1273) --- .github/workflows/build-and-test.yml | 45 +++++++++++++++--------- .github/workflows/build.yml | 2 +- .github/workflows/security-knn-tests.yml | 33 +++++++++++++---- 3 files changed, 56 insertions(+), 24 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index e8a6ace5..125d13db 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -10,28 +10,39 @@ on: # We build for all combinations but run tests only on one combination (linux & latest java) jobs: - build: - continue-on-error: true + Get-CI-Image-Tag: + uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main + with: + product: opensearch + + build-test-linux: strategy: matrix: - java: - - 11 - - 17 - # Job name - name: Run integration tests on linux with Java ${{ matrix.java }} + java: [11, 17] + + name: Build CCR Plugin on Linux using Container Image runs-on: ubuntu-latest + needs: Get-CI-Image-Tag + container: + # using the same image which is used by opensearch-build team to build the OpenSearch Distribution + # this image tag is subject to change as more dependencies and updates will arrive over time + image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} + # need to switch to root so that github actions can install runner binary on container without permission issues. + options: --user root + steps: - # This step uses the setup-java Github action: https://github.com/actions/setup-java - - name: Set Up JDK ${{ matrix.java }} + - name: Checkout CCR + uses: actions/checkout@v2 + - name: Setup Java ${{ matrix.java }} uses: actions/setup-java@v1 with: java-version: ${{ matrix.java }} - # This step uses the checkout Github action: https://github.com/actions/checkout - - name: Checkout Branch - uses: actions/checkout@v2 - - name: Build and run Replication tests + + - name: Run build + # switching the user, as OpenSearch cluster can only be started as root/Administrator on linux-deb/linux-rpm/windows-zip. run: | - ./gradlew clean release -D"build.snapshot=true" + chown -R 1000:1000 `pwd` + su `id -un 1000` -c 'whoami && java -version && ./gradlew --refresh-dependencies clean release -D"build.snapshot=true"' - name: Upload failed logs uses: actions/upload-artifact@v2 if: failure() @@ -45,7 +56,7 @@ jobs: run: | mkdir -p cross-cluster-replication-artifacts cp ./build/distributions/*.zip cross-cluster-replication-artifacts - - name: Uploads coverage + - name: Upload Coverage Report + uses: codecov/codecov-action@v1 with: - fetch-depth: 2 - uses: codecov/codecov-action@v1.2.1 + token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1faf6511..ca44eebf 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -34,4 +34,4 @@ jobs: uses: actions/checkout@v2 - name: Build and run Replication tests run: | - ./gradlew clean release -D"build.snapshot=true" -x test -x IntegTest \ No newline at end of file + ./gradlew --refresh-dependencies clean release -D"build.snapshot=true" -x test -x IntegTest diff --git a/.github/workflows/security-knn-tests.yml b/.github/workflows/security-knn-tests.yml index 867df6c8..c3259a50 100644 --- a/.github/workflows/security-knn-tests.yml +++ b/.github/workflows/security-knn-tests.yml @@ -9,6 +9,11 @@ on: - '*' jobs: + Get-CI-Image-Tag: + uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main + with: + product: opensearch + req: # Job name name: plugin check @@ -45,12 +50,19 @@ jobs: cat $GITHUB_OUTPUT fi - build: - needs: req + build-linux: + needs: [req, Get-CI-Image-Tag] if: ${{ 'True' == needs.req.outputs.isSecurityPluginAvailable }} # Job name name: Build and Run Security tests runs-on: ubuntu-latest + container: + # using the same image which is used by opensearch-build team to build the OpenSearch Distribution + # this image tag is subject to change as more dependencies and updates will arrive over time + image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} + # need to switch to root so that github actions can install runner binary on container without permission issues. + options: --user root + steps: # This step uses the setup-java Github action: https://github.com/actions/setup-java - name: Set Up JDK 17 @@ -62,8 +74,9 @@ jobs: uses: actions/checkout@v2 - name: Build and run Replication tests run: | + chown -R 1000:1000 `pwd` ls -al src/test/resources/security/plugin - ./gradlew clean release -Dbuild.snapshot=true -Psecurity=true + su `id -un 1000` -c "whoami && java -version && ./gradlew --refresh-dependencies clean release -Dbuild.snapshot=true -Psecurity=true" - name: Upload failed logs uses: actions/upload-artifact@v2 if: failure() @@ -82,12 +95,19 @@ jobs: fetch-depth: 2 uses: codecov/codecov-action@v1.2.1 - knn-build: - needs: req + knn-build-linux: + needs: [req, Get-CI-Image-Tag] if: ${{ 'True' == needs.req.outputs.isKnnPluginAvailable }} # Job name name: Build and Run Knn tests runs-on: ubuntu-latest + container: + # using the same image which is used by opensearch-build team to build the OpenSearch Distribution + # this image tag is subject to change as more dependencies and updates will arrive over time + image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} + # need to switch to root so that github actions can install runner binary on container without permission issues. + options: --user root + steps: # This step uses the setup-java Github action: https://github.com/actions/setup-java - name: Set Up JDK 17 @@ -99,7 +119,8 @@ jobs: uses: actions/checkout@v2 - name: Build and run Replication tests run: | - ./gradlew clean release -Dbuild.snapshot=true -PnumNodes=1 -Dtests.class=org.opensearch.replication.BasicReplicationIT -Dtests.method="test knn index replication" -Pknn=true + chown -R 1000:1000 `pwd` + su `id -un 1000` -c 'whoami && java -version && ./gradlew --refresh-dependencies clean release -Dbuild.snapshot=true -PnumNodes=1 -Dtests.class=org.opensearch.replication.BasicReplicationIT -Dtests.method="test knn index replication" -Pknn=true' - name: Upload failed logs uses: actions/upload-artifact@v2 if: failure()