Create opensearch-cross-cluster-replication.release-notes-2.12.0.0.md

.github/workflows/auto-release.yml

@@ -0,0 +1,29 @@
+name: Releases
+
+on:
+  push:
+    tags:
+      - '*'
+
+jobs:
+
+  build:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - name: GitHub App token
+        id: github_app_token
+        uses: tibdex/[email protected]
+        with:
+          app_id: ${{ secrets.APP_ID }}
+          private_key: ${{ secrets.APP_PRIVATE_KEY }}
+          installation_id: 22958780
+      - name: Get tag
+        id: tag
+        uses: dawidd6/action-get-tag@v1
+      - uses: actions/checkout@v2
+      - uses: ncipollo/release-action@v1
+        with:
+          github_token: ${{ steps.github_app_token.outputs.token }}
+          bodyFile: release-notes/opensearch-cross-cluster-replication.release-notes-${{steps.tag.outputs.tag}}.md

.github/workflows/build-and-test.yml

@@ -0,0 +1,62 @@
+name: CCR Test Workflow
+# This workflow is triggered on pull requests to main branch
+on:
+  pull_request:
+    branches:
+      - '*'
+  push:
+    branches:
+      - '*'
+
+# We build for all combinations but run tests only on one combination (linux & latest java)
+jobs:
+  Get-CI-Image-Tag:
+    uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main
+    with:
+      product: opensearch
+
+  build-test-linux:
+    strategy:
+      matrix:
+        java: [11, 17]
+
+    name: Build CCR Plugin on Linux using Container Image
+    runs-on: ubuntu-latest
+    needs: Get-CI-Image-Tag
+    container:
+      # using the same image which is used by opensearch-build team to build the OpenSearch Distribution
+      # this image tag is subject to change as more dependencies and updates will arrive over time
+      image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
+      # need to switch to root so that github actions can install runner binary on container without permission issues.
+      options: --user root
+
+    steps:
+      - name: Checkout CCR
+        uses: actions/checkout@v2
+      - name: Setup Java ${{ matrix.java }}
+        uses: actions/setup-java@v1
+        with:
+          java-version: ${{ matrix.java }}
+
+      - name: Run build
+        # switching the user, as OpenSearch cluster can only be started as root/Administrator on linux-deb/linux-rpm/windows-zip.
+        run: |
+          chown -R 1000:1000 `pwd`
+          su `id -un 1000` -c 'whoami && java -version && ./gradlew --refresh-dependencies clean release -D"build.snapshot=true"'
+      - name: Upload failed logs
+        uses: actions/upload-artifact@v2
+        if: failure()
+        with:
+          name: logs
+          path: |
+            build/testclusters/integTest-*/logs/*
+            build/testclusters/leaderCluster-*/logs/*
+            build/testclusters/followCluster-*/logs/*
+      - name: Create Artifact Path
+        run: |
+          mkdir -p cross-cluster-replication-artifacts
+          cp ./build/distributions/*.zip cross-cluster-replication-artifacts
+      - name: Upload Coverage Report
+        uses: codecov/codecov-action@v1
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/build.yml

@@ -1,4 +1,4 @@
-name: Test and Build Workflow
+name: Build Replication plugin
 # This workflow is triggered on pull requests to main branch
 on:
   pull_request:
@@ -8,16 +8,21 @@ on:
     branches:
       - '*'
 
+# We build for other platforms except linux which is already covered in build-and-test.
+# Also, We're not running tests here as those are already covered with linux build.
 jobs:
   build:
+    continue-on-error: true
     strategy:
       matrix:
         java:
-          - 11
           - 17
+        os:
+          - windows-latest
+          - macos-latest
     # Job name
-    name: Build Replication plugin
-    runs-on: ubuntu-latest
+    name: Java ${{ matrix.java }} On ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
     steps:
       # This step uses the setup-java Github action: https://github.com/actions/setup-java
       - name: Set Up JDK ${{ matrix.java }}
@@ -29,21 +34,4 @@ jobs:
         uses: actions/checkout@v2
       - name: Build and run Replication tests
         run: |
-          ./gradlew clean release -Dbuild.snapshot=true
-      - name: Upload failed logs
-        uses: actions/upload-artifact@v2
-        if: failure()
-        with:
-          name: logs
-          path: |
-            build/testclusters/integTest-*/logs/*
-            build/testclusters/leaderCluster-*/logs/*
-            build/testclusters/followCluster-*/logs/*
-      - name: Create Artifact Path
-        run: |
-          mkdir -p cross-cluster-replication-artifacts
-          cp ./build/distributions/*.zip cross-cluster-replication-artifacts
-      - name: Uploads coverage
-        with:
-          fetch-depth: 2
-        uses: codecov/[email protected]
+          ./gradlew --refresh-dependencies clean release -D"build.snapshot=true" -x test -x IntegTest

.github/workflows/maven-publish.yml

@@ -0,0 +1,34 @@
+name: Publish snapshots to maven
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [
+        main
+          1.*
+          2.*
+    ]
+
+jobs:
+  build-and-publish-snapshots:
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: write
+    steps:
+      - uses: actions/setup-java@v3
+        with:
+          distribution: temurin # Temurin is a distribution of adoptium
+          java-version: 17
+      - uses: actions/checkout@v3
+      - uses: aws-actions/configure-aws-credentials@v1
+        with:
+          role-to-assume: ${{ secrets.PUBLISH_SNAPSHOTS_ROLE }}
+          aws-region: us-east-1
+      - name: publish snapshots to maven
+        run: |
+          export SONATYPE_USERNAME=$(aws secretsmanager get-secret-value --secret-id maven-snapshots-username --query SecretString --output text)
+          export SONATYPE_PASSWORD=$(aws secretsmanager get-secret-value --secret-id maven-snapshots-password --query SecretString --output text)
+          echo "::add-mask::$SONATYPE_USERNAME"
+          echo "::add-mask::$SONATYPE_PASSWORD"
+          ./gradlew publishPluginZipPublicationToSnapshotsRepository

.github/workflows/security-knn-tests.yml

@@ -0,0 +1,140 @@
+name: Security and knn tests
+# This workflow is triggered on pull requests to main branch
+on:
+  pull_request:
+    branches:
+      - '*'
+  push:
+    branches:
+      - '*'
+
+jobs:
+  Get-CI-Image-Tag:
+    uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main
+    with:
+      product: opensearch
+
+  req:
+    # Job name
+    name: plugin check
+    runs-on: ubuntu-latest
+    outputs:
+      isSecurityPluginAvailable: ${{ steps.plugin-availability-check.outputs.isSecurityPluginAvailable }}
+      isKnnPluginAvailable: ${{ steps.plugin-availability-check.outputs.isKnnPluginAvailable }}
+    steps:
+      # This step uses the checkout Github action: https://github.com/actions/checkout
+      - name: Checkout Branch
+        uses: actions/checkout@v2
+      - id: plugin-availability-check
+        name: "plugin check"
+        run: |
+          opensearch_version=$(grep "System.getProperty(\"opensearch.version\", \"" build.gradle | grep '\([0-9]\|[.]\)\{5\}' -o)
+          opensearch_version=$opensearch_version".0-SNAPSHOT"
+          # we publish build artifacts to the below url 
+          sec_plugin_url="https://aws.oss.sonatype.org/content/repositories/snapshots/org/opensearch/plugin/opensearch-security/"$opensearch_version"/"
+          sec_st=$(curl -s -o /dev/null -w "%{http_code}" $sec_plugin_url)
+          if [ "$sec_st" = "200" ]; then
+            echo "isSecurityPluginAvailable=True" >> $GITHUB_OUTPUT
+            cat $GITHUB_OUTPUT
+          else
+            echo "isSecurityPluginAvailable=False" >> $GITHUB_OUTPUT
+            cat $GITHUB_OUTPUT
+          fi
+          knn_plugin_url="https://aws.oss.sonatype.org/content/repositories/snapshots/org/opensearch/plugin/opensearch-knn/"$opensearch_version"/"
+          knn_st=$(curl -s -o /dev/null -w "%{http_code}" $knn_plugin_url)
+          if [ "$knn_st" = "200" ]; then
+            echo "isKnnPluginAvailable=True" >> $GITHUB_OUTPUT
+            cat $GITHUB_OUTPUT
+          else
+            echo "isKnnPluginAvailable=False" >> $GITHUB_OUTPUT
+            cat $GITHUB_OUTPUT
+          fi  
+
+  build-linux:
+    needs: [req, Get-CI-Image-Tag]
+    if: ${{ 'True' == needs.req.outputs.isSecurityPluginAvailable }}
+    # Job name
+    name: Build and Run Security tests
+    runs-on: ubuntu-latest
+    container:
+      # using the same image which is used by opensearch-build team to build the OpenSearch Distribution
+      # this image tag is subject to change as more dependencies and updates will arrive over time
+      image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
+      # need to switch to root so that github actions can install runner binary on container without permission issues.
+      options: --user root
+
+    steps:
+      # This step uses the setup-java Github action: https://github.com/actions/setup-java
+      - name: Set Up JDK 17
+        uses: actions/setup-java@v1
+        with:
+          java-version: 17
+      # This step uses the checkout Github action: https://github.com/actions/checkout
+      - name: Checkout Branch
+        uses: actions/checkout@v2
+      - name: Build and run Replication tests
+        run: |
+          chown -R 1000:1000 `pwd`
+          ls -al src/test/resources/security/plugin
+          su `id -un 1000` -c "whoami && java -version && ./gradlew --refresh-dependencies clean release -Dbuild.snapshot=true -Psecurity=true"
+      - name: Upload failed logs
+        uses: actions/upload-artifact@v2
+        if: failure()
+        with:
+          name: logs
+          path: |
+            build/testclusters/integTest-*/logs/*
+            build/testclusters/leaderCluster-*/logs/*
+            build/testclusters/followCluster-*/logs/*
+      - name: Create Artifact Path
+        run: |
+          mkdir -p cross-cluster-replication-artifacts
+          cp ./build/distributions/*.zip cross-cluster-replication-artifacts
+      - name: Uploads coverage
+        with:
+          fetch-depth: 2
+        uses: codecov/[email protected]
+
+  knn-build-linux:
+    needs: [req, Get-CI-Image-Tag]
+    if: ${{ 'True' == needs.req.outputs.isKnnPluginAvailable }}
+    # Job name
+    name: Build and Run Knn tests
+    runs-on: ubuntu-latest
+    container:
+      # using the same image which is used by opensearch-build team to build the OpenSearch Distribution
+      # this image tag is subject to change as more dependencies and updates will arrive over time
+      image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
+      # need to switch to root so that github actions can install runner binary on container without permission issues.
+      options: --user root
+
+    steps:
+      # This step uses the setup-java Github action: https://github.com/actions/setup-java
+      - name: Set Up JDK 17
+        uses: actions/setup-java@v1
+        with:
+          java-version: 17
+      # This step uses the checkout Github action: https://github.com/actions/checkout
+      - name: Checkout Branch
+        uses: actions/checkout@v2
+      - name: Build and run Replication tests
+        run: |
+          chown -R 1000:1000 `pwd`
+          su `id -un 1000` -c 'whoami && java -version && ./gradlew --refresh-dependencies clean release -Dbuild.snapshot=true -PnumNodes=1  -Dtests.class=org.opensearch.replication.BasicReplicationIT   -Dtests.method="test knn index replication"  -Pknn=true'
+      - name: Upload failed logs
+        uses: actions/upload-artifact@v2
+        if: failure()
+        with:
+          name: logs
+          path: |
+            build/testclusters/integTest-*/logs/*
+            build/testclusters/leaderCluster-*/logs/*
+            build/testclusters/followCluster-*/logs/*
+      - name: Create Artifact Path
+        run: |
+          mkdir -p cross-cluster-replication-artifacts
+          cp ./build/distributions/*.zip cross-cluster-replication-artifacts
+      - name: Uploads coverage
+        with:
+          fetch-depth: 2
+        uses: codecov/[email protected]

.github/workflows/security-tests.yml

@@ -15,25 +15,13 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       # This step uses the setup-java Github action: https://github.com/actions/setup-java
-      - name: Set Up JDK 11
+      - name: Set Up JDK 17
         uses: actions/setup-java@v1
         with:
-          java-version: 11
+          java-version: 17
       # This step uses the checkout Github action: https://github.com/actions/checkout
       - name: Checkout Branch
         uses: actions/checkout@v2
-      # Security plugin dependency
-      - name: Checkout security
-        uses: actions/checkout@v2
-        with:
-          repository: 'opensearch-project/security'
-          path: security
-          ref: 'main'
-      - name: Build security
-        working-directory: ./security
-        run: |
-          ./gradlew clean build -Dbuild.snapshot=false -x test
-          cp build/distributions/opensearch-security-*.zip ../src/test/resources/security/plugin/opensearch-security.zip
       - name: Build and run Replication tests
         run: |
           ls -al src/test/resources/security/plugin