Skip to content

Build and Test k-NN #603

Build and Test k-NN

Build and Test k-NN #603

Workflow file for this run

name: Build and Test k-NN
on:
schedule:
- cron: '0 0 * * *' # every night
push:
branches:
- "*"
- "feature/**"
paths:
- 'build.gradle'
- 'settings.gradle'
- 'src/**'
- 'build-tools/**'
- 'buildSrc/**'
- 'gradle/**'
- 'jni/**'
- 'micro-benchmarks/**'
- '.github/workflows/CI.yml'
pull_request:
branches:
- "*"
- "feature/**"
paths:
- 'build.gradle'
- 'settings.gradle'
- 'src/**'
- 'build-tools/**'
- 'buildSrc/**'
- 'gradle/**'
- 'jni/**'
- 'micro-benchmarks/**'
- '.github/workflows/CI.yml'
env:
ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
jobs:
Get-CI-Image-Tag:
uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main
with:
product: opensearch
Build-k-NN-Linux:
strategy:
matrix:
java: [21]
name: Build and Test k-NN Plugin on Linux
runs-on: ubuntu-latest
needs: Get-CI-Image-Tag
container:
# using the same image which is used by opensearch-build team to build the OpenSearch Distribution
# this image tag is subject to change as more dependencies and updates will arrive over time
image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
# need to switch to root so that github actions can install runner binary on container without permission issues.
options: --user root
steps:
- name: Checkout k-NN
uses: actions/checkout@v1
# Setup git user so that patches for native libraries can be applied and committed
- name: Setup git user
run: |
su `id -un 1000` -c 'git config --global user.name "github-actions[bot]"'
su `id -un 1000` -c 'git config --global user.email "github-actions[bot]@users.noreply.github.com"'
- name: Setup Java ${{ matrix.java }}
uses: actions/setup-java@v1
with:
java-version: ${{ matrix.java }}
- name: Run build
# switching the user, as OpenSearch cluster can only be started as root/Administrator on linux-deb/linux-rpm/windows-zip.
run: |
chown -R 1000:1000 `pwd`
if lscpu | grep -i avx512f | grep -i avx512cd | grep -i avx512vl | grep -i avx512dq | grep -i avx512bw
then
echo "avx512 available on system"
su `id -un 1000` -c "whoami && java -version && ./gradlew build -Dnproc.count=`nproc`"
elif lscpu | grep -i avx2
then
echo "avx2 available on system"
su `id -un 1000` -c "whoami && java -version && ./gradlew build -Dnproc.count=`nproc` -Davx512.enabled=false"
else
echo "avx512 and avx2 not available on system"
su `id -un 1000` -c "whoami && java -version && ./gradlew build -Davx2.enabled=false -Davx512.enabled=false -Dnproc.count=`nproc`"
fi
- name: Upload Coverage Report
uses: codecov/codecov-action@v1
with:
token: ${{ secrets.CODECOV_TOKEN }}
Build-k-NN-MacOS:
strategy:
matrix:
java: [ 21 ]
name: Build and Test k-NN Plugin on MacOS
needs: Get-CI-Image-Tag
runs-on: macos-13
steps:
- name: Checkout k-NN
uses: actions/checkout@v1
# Setup git user so that patches for native libraries can be applied and committed
- name: Setup git user
run: |
git config --global user.name "github-actions[bot]"
git config --global user.email "github-actions[bot]@users.noreply.github.com"
- name: Setup Java ${{ matrix.java }}
uses: actions/setup-java@v1
with:
java-version: ${{ matrix.java }}
- name: Install dependencies on macos
run: |
brew install libomp
brew reinstall gcc
export FC=/usr/local/Cellar/gcc/12.2.0/bin/gfortran
# TODO: Detect processor count and set the value of nproc.count
- name: Run build
run: |
if sysctl -n machdep.cpu.features machdep.cpu.leaf7_features | grep -i AVX2
then
echo "avx2 available on system"
./gradlew build -Dnproc.count=3
else
echo "avx2 not available on system"
./gradlew build -Davx2.enabled=false -Davx512.enabled=false -Dnproc.count=3
fi
Build-k-NN-Windows:
strategy:
matrix:
java: [ 21 ]
name: Build and Test k-NN Plugin on Windows
needs: Get-CI-Image-Tag
runs-on: windows-latest
steps:
- name: Checkout k-NN
uses: actions/checkout@v1
# Setup git user so that patches for native libraries can be applied and committed
- name: Setup git user
run: |
git config --global user.name "github-actions[bot]"
git config --global user.email "github-actions[bot]@users.noreply.github.com"
- name: Setup Java ${{ matrix.java }}
uses: actions/setup-java@v1
with:
java-version: ${{ matrix.java }}
- name: Install MinGW Using Scoop
run: |
iex "& {$(irm get.scoop.sh)} -RunAsAdmin"
scoop bucket add main
scoop install mingw
- name: Add MinGW to PATH
run: |
echo "C:/Users/runneradmin/scoop/apps/mingw/current/bin" >> $env:GITHUB_PATH
Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1"
refreshenv
- name: Install Zlib Using Scoop
run: |
echo "C:/Users/runneradmin/scoop/shims" >> $env:GITHUB_PATH
Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1"
refreshenv
scoop bucket add extras
scoop install zlib
regedit /s "C:\\Users\\runneradmin\\scoop\\apps\\zlib\\current\\register.reg"
- name: Download OpenBLAS
run: |
curl -L -O https://github.com/xianyi/OpenBLAS/releases/download/v0.3.21/OpenBLAS-0.3.21-x64.zip
mkdir OpenBLAS
Expand-Archive -Path .\OpenBLAS-0.3.21-x64.zip -DestinationPath .\OpenBLAS\
mkdir ./src/main/resources/windowsDependencies
cp ./OpenBLAS/bin/libopenblas.dll ./src/main/resources/windowsDependencies/
rm .\OpenBLAS-0.3.21-x64.zip
rm -r .\OpenBLAS\
# TODO: Detect processor count and set the value of nproc.count
- name: Run build
run: |
./gradlew.bat build -D'avx2.enabled=false' -D'avx512.enabled=false' -D'nproc.count=4'