Merge pull request #299 from opendistro-for-elasticsearch/arm64-support

Add arm64 support and correct the naming convention to the new standards
opendistro-for-elasticsearch · Jan 11, 2021 · 79f7748 · 79f7748
2 parents 8537d20 + 3d77863
commit 79f7748
Show file tree

Hide file tree

Showing 2 changed files with 129 additions and 6 deletions.
diff --git a/.github/workflows/CD.yml b/.github/workflows/CD.yml
@@ -5,13 +5,37 @@ on:
       - v*
 
 jobs:
+  Provision-Runners:
+    name: Provision-Runners
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@v1
+        with:
+          repository: opendistro-for-elasticsearch/opendistro-build
+          ref: master
+          path: opendistro-build
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_INFRATEST_EC2_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_INFRATEST_EC2_SECRET_ACCESS_KEY }}
+          aws-region: us-west-2
+      - name: AWS Cli Processing
+        run: |
+          #!/bin/bash
+          # Please add comma at the end of the RUNNERS strings
+          # Don't add space after commas
+          RUNNERS="odfe-knnlib-build-arm64"
+          export GIT_URL_REPO=${GITHUB_REPOSITORY}
+          ../opendistro-build/release-tools/scripts/setup_runners.sh run $RUNNERS ${{ secrets.ODFE_RELEASE_BOT_PUBLIC_PRIVATE_READ_WRITE_TOKEN }} ami-0ed17173ab64255b1
+
   plugin-build-and-ship-artifacts:
     strategy:
       matrix:
         java: [14]
 
     name: Build and release plugin artifacts
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-18.04
     steps:
       - name: Checkout k-NN
         uses: actions/checkout@v2
@@ -40,9 +64,9 @@ jobs:
           aws s3 cp $deb_artifact s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/debs/opendistro-knn/
           aws cloudfront create-invalidation --distribution-id E1VG5HMIWI4SA2 --paths "/*"
 
-  library-build-and-ship-artifacts:
-    name: Build and release JNI library artifacts
-    runs-on: ubuntu-latest
+  library-build-and-ship-artifacts-x64:
+    name: Build and release JNI library artifacts x64
+    runs-on: ubuntu-18.04
     container:
       image: centos:7
     strategy:
@@ -103,3 +127,102 @@ jobs:
           aws s3 cp $rpm_artifact s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/rpms/opendistro-knnlib/
           aws s3 cp $deb_artifact s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/debs/opendistro-knnlib/
           aws cloudfront create-invalidation --distribution-id E1VG5HMIWI4SA2 --paths "/*"
+
+  library-build-and-ship-artifacts-arm64:
+    needs: [Provision-Runners]
+    runs-on: [self-hosted, Linux, ARM64, odfe-knnlib-build-arm64]
+    name: Build and release JNI library artifacts arm64
+    strategy:
+      matrix:
+        java: [14]
+        compiler: [g++]
+    steps:
+      - name: Checkout k-NN
+        uses: actions/checkout@v1
+        with:
+          submodules: true
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-1
+
+      # Arm64 is not supported by setup-java actions as of now
+      # Placeholder in case it adds support later
+      # Using pre-installed jdk on arm64 self-hosted runners now
+      #- name: Setup Java ${{ matrix.java }}
+      #  uses: actions/setup-java@v1
+      #  with:
+      #    java-version: ${{ matrix.java }}
+
+      - name: Install dependencies
+        run: |
+          sudo yum update -y
+          sudo yum install -y cmake rpm-build gcc-c++ make epel-release unzip
+          sudo yum repolist
+          sudo yum install -y dpkg
+          curl "https://awscli.amazonaws.com/awscli-exe-linux-aarch64.zip" -o "awscliv2.zip"
+          unzip awscliv2.zip
+          sudo ./aws/install
+          aws --version
+
+      - name: Build and ship library artifacts
+        env:
+          CXX: ${{ matrix.compiler }}
+        run: |
+
+          export JAVA_HOME="/etc/alternatives/jre"
+          echo JAVA_HOME $JAVA_HOME
+
+          cd jni
+          cmake .
+          make package
+
+          cd packages
+          folder_name=`ls ./*.rpm | sed 's|\(.*\)\..*|\1|'`
+          zip_name=$folder_name".zip"
+          mkdir $folder_name
+          cp ../release/*.so $folder_name
+          zip -r $zip_name $folder_name/*
+          cd ..
+
+          zip_artifact=`ls packages/*.zip`
+          rpm_artifact=`ls packages/*.rpm`
+          deb_artifact=`ls packages/*.deb`
+
+          ls -l packages/
+
+
+          aws s3 cp $zip_artifact s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/opendistro-libs/opendistro-knnlib/
+          aws s3 cp $rpm_artifact s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/rpms/opendistro-knnlib/
+          aws s3 cp $deb_artifact s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/debs/opendistro-knnlib/
+          aws cloudfront create-invalidation --distribution-id E1VG5HMIWI4SA2 --paths "/*"
+
+  CleanUp-Runners:
+    needs: [library-build-and-ship-artifacts-arm64]
+    if: always()
+    name: CleanUp-Runners
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@v1
+        with:
+          repository: opendistro-for-elasticsearch/opendistro-build
+          ref: master
+          path: opendistro-build
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_INFRATEST_EC2_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_INFRATEST_EC2_SECRET_ACCESS_KEY }}
+          aws-region: us-west-2
+      - name: AWS Cli Processing
+        run: |
+          #!/bin/bash
+          # Please add comma at the end of the RUNNERS strings
+          # Don't add space after commas
+          RUNNERS="odfe-knnlib-build-arm64"
+          export GIT_URL_REPO=${GITHUB_REPOSITORY}
+          ../opendistro-build/release-tools/scripts/setup_runners.sh terminate $RUNNERS ${{ secrets.ODFE_RELEASE_BOT_PUBLIC_PRIVATE_READ_WRITE_TOKEN }}
+
diff --git a/jni/CMakeLists.txt b/jni/CMakeLists.txt
@@ -81,7 +81,7 @@ set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "KNN JNI library built off of NMSLIB for O
 set(CPACK_PACKAGE_VENDOR "Amazon")
 set(CPACK_PACKAGE_CONTACT "Maintainer: ${KNN_MAINTAINER}")
 set(CPACK_PACKAGING_INSTALL_PREFIX ${CMAKE_INSTALL_PREFIX})
-set(CPACK_PACKAGE_FILE_NAME "${CPACK_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION}-${CPACK_PACKAGE_RELEASE}_${JVM_OS_TYPE}.${CMAKE_SYSTEM_PROCESSOR}")
+set(CPACK_PACKAGE_FILE_NAME "${CPACK_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION}-${JVM_OS_TYPE}-${CMAKE_SYSTEM_PROCESSOR}")
 
 # RPM Specific variables
 set(CPACK_RPM_PACKAGE_RELEASE ${CPACK_PACKAGE_RELEASE})
@@ -96,4 +96,4 @@ set(CPACK_DEBIAN_PACKAGE_SOURCE ${CPACK_PACKAGE_NAME})
 set(CPACK_DEBIAN_PACKAGE_VERSION ${CPACK_PACKAGE_VERSION})
 set(CPACK_DEBIAN_PACKAGE_SECTION "libs")
 
-include(CPack)
+include(CPack)