From d55411c9a2e018e7cf7df9217a398953b3fb863e Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Wed, 12 Jul 2023 14:10:47 -0400 Subject: [PATCH 1/4] Add test keys to CI related workflows Signed-off-by: Peter Zhu --- .github/workflows/CI-workflow.yml | 32 +++++++++++++++++++++++++++++++ .github/workflows/dco.yml | 18 ----------------- 2 files changed, 32 insertions(+), 18 deletions(-) delete mode 100644 .github/workflows/dco.yml diff --git a/.github/workflows/CI-workflow.yml b/.github/workflows/CI-workflow.yml index d9fc74c080..0503dec1bd 100644 --- a/.github/workflows/CI-workflow.yml +++ b/.github/workflows/CI-workflow.yml @@ -7,6 +7,11 @@ on: push: branches: - "*" + +permissions: + id-token: write + contents: read + jobs: Build-ml: strategy: @@ -14,6 +19,8 @@ jobs: java: [11, 17] name: Build and Test MLCommons Plugin + if: github.repository == 'opensearch-project/ml-commons' + environment: ml-commons-cicd-env runs-on: ubuntu-latest steps: @@ -22,6 +29,11 @@ jobs: with: java-version: ${{ matrix.java }} + - uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: ${{ secrets.ML_ROLE }} + aws-region: us-west-2 + # ml-commons - name: Checkout MLCommons uses: actions/checkout@v2 @@ -34,7 +46,12 @@ jobs: ./gradlew publishToMavenLocal - name: Multi Nodes Integration Testing run: | + export OPENAI_KEY=$(aws secretsmanager get-secret-value --secret-id github_openai_key --query SecretString --output text) + export COHERE_KEY=$(aws secretsmanager get-secret-value --secret-id github_cohere_key --query SecretString --output text) + echo "::add-mask::$OPENAI_KEY" + echo "::add-mask::$COHERE_KEY" ./gradlew integTest -PnumNodes=3 + - name: Pull and Run Docker run: | plugin=`basename $(ls plugin/build/distributions/*.zip)` @@ -71,6 +88,10 @@ jobs: if: env.imagePresent == 'true' run: | security=`curl -XGET https://localhost:9200/_cat/plugins?v -u admin:admin --insecure |grep opensearch-security|wc -l` + export OPENAI_KEY=$(aws secretsmanager get-secret-value --secret-id github_openai_key --query SecretString --output text) + export COHERE_KEY=$(aws secretsmanager get-secret-value --secret-id github_cohere_key --query SecretString --output text) + echo "::add-mask::$OPENAI_KEY" + echo "::add-mask::$COHERE_KEY" if [ $security -gt 0 ] then echo "Security plugin is available" @@ -91,6 +112,8 @@ jobs: matrix: java: [11, 17] name: Build and Test MLCommons Plugin on Windows + if: github.repository == 'opensearch-project/ml-commons' + environment: ml-commons-cicd-env runs-on: windows-latest steps: @@ -99,6 +122,11 @@ jobs: with: java-version: ${{ matrix.java }} + - uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: ${{ secrets.ML_ROLE }} + aws-region: us-west-2 + # ml-commons - name: Checkout MLCommons uses: actions/checkout@v2 @@ -111,4 +139,8 @@ jobs: ./gradlew publishToMavenLocal # - name: Multi Nodes Integration Testing # run: | +# export OPENAI_KEY=$(aws secretsmanager get-secret-value --secret-id github_openai_key --query SecretString --output text) +# export COHERE_KEY=$(aws secretsmanager get-secret-value --secret-id github_cohere_key --query SecretString --output text) +# echo "::add-mask::$OPENAI_KEY" +# echo "::add-mask::$COHERE_KEY" # ./gradlew integTest -PnumNodes=3 diff --git a/.github/workflows/dco.yml b/.github/workflows/dco.yml deleted file mode 100644 index cf30ea89dc..0000000000 --- a/.github/workflows/dco.yml +++ /dev/null @@ -1,18 +0,0 @@ -name: Developer Certificate of Origin Check - -on: [pull_request] - -jobs: - check: - runs-on: ubuntu-latest - - steps: - - name: Get PR Commits - id: 'get-pr-commits' - uses: tim-actions/get-pr-commits@v1.1.0 - with: - token: ${{ secrets.GITHUB_TOKEN }} - - name: DCO Check - uses: tim-actions/dco@v1.1.0 - with: - commits: ${{ steps.get-pr-commits.outputs.commits }} From a50f6cefffce8316d1b102db6b0ee60bec24a368 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Wed, 12 Jul 2023 14:20:59 -0400 Subject: [PATCH 2/4] test more Signed-off-by: Peter Zhu --- .github/workflows/CI-workflow.yml | 62 +++++++++++++++---------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/.github/workflows/CI-workflow.yml b/.github/workflows/CI-workflow.yml index 0503dec1bd..0728e4b711 100644 --- a/.github/workflows/CI-workflow.yml +++ b/.github/workflows/CI-workflow.yml @@ -16,7 +16,7 @@ jobs: Build-ml: strategy: matrix: - java: [11, 17] + java: [11] name: Build and Test MLCommons Plugin if: github.repository == 'opensearch-project/ml-commons' @@ -107,36 +107,36 @@ jobs: flags: ml-commons token: ${{ secrets.CODECOV_TOKEN }} - Build-ml-windows: - strategy: - matrix: - java: [11, 17] - name: Build and Test MLCommons Plugin on Windows - if: github.repository == 'opensearch-project/ml-commons' - environment: ml-commons-cicd-env - runs-on: windows-latest - - steps: - - name: Setup Java ${{ matrix.java }} - uses: actions/setup-java@v1 - with: - java-version: ${{ matrix.java }} - - - uses: aws-actions/configure-aws-credentials@v2 - with: - role-to-assume: ${{ secrets.ML_ROLE }} - aws-region: us-west-2 - - # ml-commons - - name: Checkout MLCommons - uses: actions/checkout@v2 - - - name: Build and Run Tests - run: | - ./gradlew.bat build - - name: Publish to Maven Local - run: | - ./gradlew publishToMavenLocal +# Build-ml-windows: +# strategy: +# matrix: +# java: [11, 17] +# name: Build and Test MLCommons Plugin on Windows +# if: github.repository == 'opensearch-project/ml-commons' +# environment: ml-commons-cicd-env +# runs-on: windows-latest +# +# steps: +# - name: Setup Java ${{ matrix.java }} +# uses: actions/setup-java@v1 +# with: +# java-version: ${{ matrix.java }} +# +# - uses: aws-actions/configure-aws-credentials@v2 +# with: +# role-to-assume: ${{ secrets.ML_ROLE }} +# aws-region: us-west-2 +# +# # ml-commons +# - name: Checkout MLCommons +# uses: actions/checkout@v2 +# +# - name: Build and Run Tests +# run: | +# ./gradlew.bat build +# - name: Publish to Maven Local +# run: | +# ./gradlew publishToMavenLocal # - name: Multi Nodes Integration Testing # run: | # export OPENAI_KEY=$(aws secretsmanager get-secret-value --secret-id github_openai_key --query SecretString --output text) From a887dbe8b5a7a38596711787cf4f0ce6384dfa7c Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Wed, 12 Jul 2023 14:23:21 -0400 Subject: [PATCH 3/4] more test Signed-off-by: Peter Zhu --- .github/workflows/CI-workflow.yml | 62 +++++++++++++++---------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/.github/workflows/CI-workflow.yml b/.github/workflows/CI-workflow.yml index 0728e4b711..0503dec1bd 100644 --- a/.github/workflows/CI-workflow.yml +++ b/.github/workflows/CI-workflow.yml @@ -16,7 +16,7 @@ jobs: Build-ml: strategy: matrix: - java: [11] + java: [11, 17] name: Build and Test MLCommons Plugin if: github.repository == 'opensearch-project/ml-commons' @@ -107,36 +107,36 @@ jobs: flags: ml-commons token: ${{ secrets.CODECOV_TOKEN }} -# Build-ml-windows: -# strategy: -# matrix: -# java: [11, 17] -# name: Build and Test MLCommons Plugin on Windows -# if: github.repository == 'opensearch-project/ml-commons' -# environment: ml-commons-cicd-env -# runs-on: windows-latest -# -# steps: -# - name: Setup Java ${{ matrix.java }} -# uses: actions/setup-java@v1 -# with: -# java-version: ${{ matrix.java }} -# -# - uses: aws-actions/configure-aws-credentials@v2 -# with: -# role-to-assume: ${{ secrets.ML_ROLE }} -# aws-region: us-west-2 -# -# # ml-commons -# - name: Checkout MLCommons -# uses: actions/checkout@v2 -# -# - name: Build and Run Tests -# run: | -# ./gradlew.bat build -# - name: Publish to Maven Local -# run: | -# ./gradlew publishToMavenLocal + Build-ml-windows: + strategy: + matrix: + java: [11, 17] + name: Build and Test MLCommons Plugin on Windows + if: github.repository == 'opensearch-project/ml-commons' + environment: ml-commons-cicd-env + runs-on: windows-latest + + steps: + - name: Setup Java ${{ matrix.java }} + uses: actions/setup-java@v1 + with: + java-version: ${{ matrix.java }} + + - uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: ${{ secrets.ML_ROLE }} + aws-region: us-west-2 + + # ml-commons + - name: Checkout MLCommons + uses: actions/checkout@v2 + + - name: Build and Run Tests + run: | + ./gradlew.bat build + - name: Publish to Maven Local + run: | + ./gradlew publishToMavenLocal # - name: Multi Nodes Integration Testing # run: | # export OPENAI_KEY=$(aws secretsmanager get-secret-value --secret-id github_openai_key --query SecretString --output text) From a6f239839365836edc4a04da38bbb562b314bd06 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Wed, 12 Jul 2023 14:27:04 -0400 Subject: [PATCH 4/4] test Signed-off-by: Peter Zhu --- .github/workflows/CI-workflow.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/CI-workflow.yml b/.github/workflows/CI-workflow.yml index 0503dec1bd..b7870a1660 100644 --- a/.github/workflows/CI-workflow.yml +++ b/.github/workflows/CI-workflow.yml @@ -122,10 +122,10 @@ jobs: with: java-version: ${{ matrix.java }} - - uses: aws-actions/configure-aws-credentials@v2 - with: - role-to-assume: ${{ secrets.ML_ROLE }} - aws-region: us-west-2 +# - uses: aws-actions/configure-aws-credentials@v2 +# with: +# role-to-assume: ${{ secrets.ML_ROLE }} +# aws-region: us-west-2 # ml-commons - name: Checkout MLCommons