From 51f95e3be8411df66d50b964a7c76f845224f2be Mon Sep 17 00:00:00 2001 From: Vacha Shah Date: Wed, 12 Apr 2023 08:38:27 -0700 Subject: [PATCH] Improve time taken by Github actions (#439) * Improve time taken by Github actions Signed-off-by: Vacha Shah * Add Changelog Signed-off-by: Vacha Shah * Fixing distribution and adding cache Signed-off-by: Vacha Shah --------- Signed-off-by: Vacha Shah --- .github/workflows/build.yml | 10 +++-- .github/workflows/checkstyle.yml | 8 +++- .github/workflows/dependabot_pr.yml | 2 +- .github/workflows/links.yml | 2 +- .github/workflows/publish-snapshots.yml | 33 ++++++++++++++ .github/workflows/release-drafter.yml | 6 ++- .../workflows/test-integration-unreleased.yml | 43 +++++++++++++------ .github/workflows/test-integration.yml | 10 +++-- .github/workflows/test-unit.yml | 10 +++-- 9 files changed, 94 insertions(+), 30 deletions(-) create mode 100644 .github/workflows/publish-snapshots.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 668fc8b7b5..b51c25c85a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -10,13 +10,15 @@ jobs: java: [ 11 ] os: [ubuntu-latest, windows-latest, macOS-latest] steps: + - name: Checkout Java Client + uses: actions/checkout@v3 + - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v1 + uses: actions/setup-java@v3 with: java-version: ${{ matrix.java }} - - - name: Checkout Branch - uses: actions/checkout@v2 + distribution: 'temurin' + cache: 'gradle' - name: Build with Gradle run: ./gradlew clean build -x test diff --git a/.github/workflows/checkstyle.yml b/.github/workflows/checkstyle.yml index ecc9129902..eb0ea0b887 100644 --- a/.github/workflows/checkstyle.yml +++ b/.github/workflows/checkstyle.yml @@ -9,12 +9,16 @@ jobs: matrix: java: [ 11 ] steps: + - name: Checkout Java Client + uses: actions/checkout@v3 + - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v1 + uses: actions/setup-java@v3 with: java-version: ${{ matrix.java }} + distribution: 'temurin' + cache: 'gradle' - - uses: actions/checkout@v2 - name: Check style and license headers run: | ./gradlew checkstyleMain checkstyleTest diff --git a/.github/workflows/dependabot_pr.yml b/.github/workflows/dependabot_pr.yml index f13889c13d..ffcdd521c1 100644 --- a/.github/workflows/dependabot_pr.yml +++ b/.github/workflows/dependabot_pr.yml @@ -18,7 +18,7 @@ jobs: installation_id: 22958780 - name: Check out code - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: token: ${{ steps.github_app_token.outputs.token }} diff --git a/.github/workflows/links.yml b/.github/workflows/links.yml index 2b1a29fb3d..f5d2b8dd6e 100644 --- a/.github/workflows/links.yml +++ b/.github/workflows/links.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: lychee Link Checker id: lychee uses: lycheeverse/lychee-action@v1.5.0 diff --git a/.github/workflows/publish-snapshots.yml b/.github/workflows/publish-snapshots.yml new file mode 100644 index 0000000000..b0f23c2a1d --- /dev/null +++ b/.github/workflows/publish-snapshots.yml @@ -0,0 +1,33 @@ +name: Publish snapshots to maven + +on: + push: + branches: + - main + +jobs: + build-and-publish-snapshots: + runs-on: ubuntu-latest + permissions: + id-token: write + contents: write + steps: + - uses: actions/checkout@v3 + - name: Set up JDK 11 + uses: actions/setup-java@v3 + with: + java-version: '11' + distribution: 'temurin' + cache: 'gradle' + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ secrets.PUBLISH_SNAPSHOTS_ROLE }} + aws-region: us-east-1 + - name: publish snapshots to maven + run: | + export SONATYPE_USERNAME=$(aws secretsmanager get-secret-value --secret-id maven-snapshots-username --query SecretString --output text) + export SONATYPE_PASSWORD=$(aws secretsmanager get-secret-value --secret-id maven-snapshots-password --query SecretString --output text) + echo "::add-mask::$SONATYPE_USERNAME" + echo "::add-mask::$SONATYPE_PASSWORD" + ./gradlew --no-daemon publishPublishMavenPublicationToSnapshotsRepository \ No newline at end of file diff --git a/.github/workflows/release-drafter.yml b/.github/workflows/release-drafter.yml index 72bc8a69ca..4374158a6c 100644 --- a/.github/workflows/release-drafter.yml +++ b/.github/workflows/release-drafter.yml @@ -25,9 +25,11 @@ jobs: issue-body: "Please approve or deny the release of opensearch-java. **VERSION**: ${{ steps.get_data.outputs.version }} **TAG**: ${{ github.ref_name }} **COMMIT**: ${{ github.sha }}" exclude-workflow-initiator-as-approver: true - name: Set up JDK 11 - uses: actions/setup-java@v1 + uses: actions/setup-java@v3 with: - java-version: 11 + java-version: '11' + distribution: 'temurin' + cache: 'gradle' - name: Build with Gradle run: | export VERSION=`cat gradle.properties | grep "systemProp.version" | tr -d " " | cut -d '=' -f2` diff --git a/.github/workflows/test-integration-unreleased.yml b/.github/workflows/test-integration-unreleased.yml index 7fd53b1eb1..d1509fc039 100644 --- a/.github/workflows/test-integration-unreleased.yml +++ b/.github/workflows/test-integration-unreleased.yml @@ -18,36 +18,55 @@ jobs: - { opensearch_ref: '1.x', java: 11 } - { opensearch_ref: '2.x', java: 11 } - { opensearch_ref: '2.x', java: 17 } - - { opensearch_ref: '2.0', java: 11 } - { opensearch_ref: 'main', java: 11 } - { opensearch_ref: 'main', java: 17 } steps: - name: Set up JDK ${{ matrix.entry.java }} - uses: actions/setup-java@v1 + uses: actions/setup-java@v3 with: java-version: ${{ matrix.entry.java }} + distribution: 'temurin' - name: Checkout OpenSearch - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: repository: opensearch-project/OpenSearch ref: ${{ matrix.entry.opensearch_ref }} path: opensearch + - name: Get OpenSearch branch top + id: get-key + working-directory: opensearch + run: echo key=`git log -1 --format='%H'` >> $GITHUB_OUTPUT + + - name: Restore cached build + id: cache-restore + uses: actions/cache/restore@v3 + with: + path: opensearch/distribution/archives/linux-tar/build/distributions + key: ${{ steps.get-key.outputs.key }} + - name: Assemble OpenSearch - run: | - cd opensearch - ./gradlew assemble + if: steps.cache-restore.outputs.cache-hit != 'true' + working-directory: opensearch + run: ./gradlew :distribution:archives:linux-tar:assemble + + - name: Save cached build + if: steps.cache-restore.outputs.cache-hit != 'true' + uses: actions/cache/save@v3 + with: + path: opensearch/distribution/archives/linux-tar/build/distributions + key: ${{ steps.get-key.outputs.key }} - # This step runs the docker image generated during gradle assemble in OpenSearch. It is tagged as opensearch:test. - # Reference: https://github.com/opensearch-project/OpenSearch/blob/2.0/distribution/docker/build.gradle#L190 - - name: Run Docker Image + - name: Run OpenSearch + working-directory: opensearch/distribution/archives/linux-tar/build/distributions run: | - docker run -p 9200:9200 -p 9600:9600 -d -e "discovery.type=single-node" -e "bootstrap.memory_lock=true" opensearch:test - sleep 90 + tar xf opensearch-min-* + ./opensearch-*/bin/opensearch & + for attempt in {1..20}; do sleep 5; if curl -s localhost:9200; then echo '=====> ready'; break; fi; echo '=====> waiting...'; done - name: Checkout Java Client - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: path: opensearch-java diff --git a/.github/workflows/test-integration.yml b/.github/workflows/test-integration.yml index 0a554ccf31..5f76dfd2a2 100644 --- a/.github/workflows/test-integration.yml +++ b/.github/workflows/test-integration.yml @@ -20,13 +20,15 @@ jobs: - { opensearch_version: 2.4.1, java: 11 } - { opensearch_version: 2.5.0, java: 11 } steps: + - name: Checkout Java Client + uses: actions/checkout@v3 + - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v1 + uses: actions/setup-java@v3 with: java-version: ${{ matrix.entry.java }} - - - name: Checkout Branch - uses: actions/checkout@v2 + distribution: 'temurin' + cache: 'gradle' - name: Run Docker run: | diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index fbfe15671c..70f591553c 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -10,13 +10,15 @@ jobs: java: [ 11 ] os: [ubuntu-latest, windows-latest, macOS-latest] steps: + - name: Checkout Java Client + uses: actions/checkout@v3 + - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v1 + uses: actions/setup-java@v3 with: java-version: ${{ matrix.java }} - - - name: Checkout Branch - uses: actions/checkout@v2 + distribution: 'temurin' + cache: 'gradle' - name: Run Unit Test run: ./gradlew clean unitTest