Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add Windows Support to BWC Tests #726

Merged
merged 1 commit into from
Jan 18, 2023
Merged

Add Windows Support to BWC Tests #726

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
86 changes: 80 additions & 6 deletions .github/workflows/backwards_compatibility_tests_workflow.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,21 @@ jobs:
strategy:
matrix:
java: [ 11, 17 ]
os: [windows-latest, ubuntu-latest]
bwc_version : [ "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0-SNAPSHOT", "2.6.0-SNAPSHOT" ]
opensearch_version : [ "3.0.0-SNAPSHOT" ]
exclude:
- os: windows-latest
bwc_version: "2.0.1"
- os: windows-latest
bwc_version: "2.1.0"
- os: windows-latest
bwc_version: "2.2.1"
- os: windows-latest
bwc_version: "2.3.0"
naveentatikonda marked this conversation as resolved.
Show resolved Hide resolved

name: k-NN Restart-Upgrade BWC Tests
runs-on: ubuntu-latest
runs-on: ${{ matrix.os }}
env:
BWC_VERSION_RESTART_UPGRADE: ${{ matrix.bwc_version }}

Expand All @@ -31,12 +41,43 @@ jobs:
with:
java-version: ${{ matrix.java }}

- name: Install dependencies
- if: startsWith(matrix.os,'ubuntu')
name: Install dependencies on ubuntu
run: |
sudo apt-get install libopenblas-dev gfortran -y

- if: startsWith(matrix.os,'windows')
name: Install MinGW Using Scoop on Windows
run: |
iex "& {$(irm get.scoop.sh)} -RunAsAdmin"
scoop bucket add main
scoop install mingw

- if: startsWith(matrix.os,'windows')
name: Add MinGW to PATH on Windows
run: |
echo "C:/Users/runneradmin/scoop/apps/mingw/current/bin" >> $env:GITHUB_PATH
refreshenv

- if: startsWith(matrix.os,'windows')
name: Download OpenBLAS on Windows
run: |
curl -L -O https://github.com/xianyi/OpenBLAS/releases/download/v0.3.21/OpenBLAS-0.3.21-x64.zip
mkdir OpenBLAS
Expand-Archive -Path .\OpenBLAS-0.3.21-x64.zip -DestinationPath .\OpenBLAS\
mkdir ./src/main/resources/windowsDependencies
cp ./OpenBLAS/bin/libopenblas.dll ./src/main/resources/windowsDependencies/
rm .\OpenBLAS-0.3.21-x64.zip
rm -r .\OpenBLAS\

- name: Run k-NN Restart-Upgrade BWC Tests from BWCVersion-${{ matrix.bwc_version }} to OpenSearch Version-${{ matrix.opensearch_version }}
- if: startsWith(matrix.os,'windows')
name: Run k-NN Restart-Upgrade BWC Tests from BWCVersion-${{ matrix.bwc_version }} to OpenSearch Version-${{ matrix.opensearch_version }} on Windows
run: |
echo "Running restart-upgrade backwards compatibility tests ..."
./gradlew :qa:restart-upgrade:testRestartUpgrade -D'tests.bwc.version=${{ matrix.bwc_version }}'

- if: startsWith(matrix.os,'ubuntu')
name: Run k-NN Restart-Upgrade BWC Tests from BWCVersion-${{ matrix.bwc_version }} to OpenSearch Version-${{ matrix.opensearch_version }} on Ubuntu
run: |
echo "Running restart-upgrade backwards compatibility tests ..."
./gradlew :qa:restart-upgrade:testRestartUpgrade -Dtests.bwc.version=$BWC_VERSION_RESTART_UPGRADE
Expand All @@ -46,11 +87,12 @@ jobs:
strategy:
matrix:
java: [ 11, 17 ]
os: [windows-latest, ubuntu-latest]
bwc_version: [ "2.6.0-SNAPSHOT" ]
opensearch_version: [ "3.0.0-SNAPSHOT" ]

name: k-NN Rolling-Upgrade BWC Tests
runs-on: ubuntu-latest
runs-on: ${{ matrix.os }}
env:
BWC_VERSION_ROLLING_UPGRADE: ${{ matrix.bwc_version }}

Expand All @@ -63,11 +105,43 @@ jobs:
with:
java-version: ${{ matrix.java }}

- name: Install dependencies
- if: startsWith(matrix.os,'ubuntu')
name: Install dependencies on ubuntu
run: |
sudo apt-get install libopenblas-dev gfortran -y

- name: Run k-NN Rolling-Upgrade BWC Tests from BWCVersion-${{ matrix.bwc_version }} to OpenSearch Version-${{ matrix.opensearch_version }}
- if: startsWith(matrix.os,'windows')
name: Install MinGW Using Scoop on Windows
run: |
iex "& {$(irm get.scoop.sh)} -RunAsAdmin"
scoop bucket add main
scoop install mingw

- if: startsWith(matrix.os,'windows')
name: Add MinGW to PATH on Windows
run: |
echo "C:/Users/runneradmin/scoop/apps/mingw/current/bin" >> $env:GITHUB_PATH
refreshenv

- if: startsWith(matrix.os,'windows')
name: Download OpenBLAS on Windows
run: |
curl -L -O https://github.com/xianyi/OpenBLAS/releases/download/v0.3.21/OpenBLAS-0.3.21-x64.zip
mkdir OpenBLAS
Expand-Archive -Path .\OpenBLAS-0.3.21-x64.zip -DestinationPath .\OpenBLAS\
mkdir ./src/main/resources/windowsDependencies
naveentatikonda marked this conversation as resolved.
Show resolved Hide resolved
cp ./OpenBLAS/bin/libopenblas.dll ./src/main/resources/windowsDependencies/
rm .\OpenBLAS-0.3.21-x64.zip
rm -r .\OpenBLAS\

- if: startsWith(matrix.os,'windows')
name: Run k-NN Rolling-Upgrade BWC Tests from BWCVersion-${{ matrix.bwc_version }} to OpenSearch Version-${{ matrix.opensearch_version }} on Windows
run: |
echo "Running rolling-upgrade backwards compatibility tests ..."
./gradlew :qa:rolling-upgrade:testRollingUpgrade -D'tests.bwc.version=${{ matrix.bwc_version }}'

- if: startsWith(matrix.os,'ubuntu')
name: Run k-NN Rolling-Upgrade BWC Tests from BWCVersion-${{ matrix.bwc_version }} to OpenSearch Version-${{ matrix.opensearch_version }} on Ubuntu
run: |
echo "Running rolling-upgrade backwards compatibility tests ..."
./gradlew :qa:rolling-upgrade:testRollingUpgrade -Dtests.bwc.version=$BWC_VERSION_ROLLING_UPGRADE
32 changes: 27 additions & 5 deletions qa/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
* SPDX-License-Identifier: Apache-2.0
*/

import org.apache.tools.ant.taskdefs.condition.Os
import java.nio.file.Files

apply plugin: 'opensearch.testclusters'
apply plugin: 'opensearch.build'
Expand Down Expand Up @@ -33,18 +35,34 @@ String default_bwc_version = System.getProperty("bwc.version")
String knn_bwc_version = System.getProperty("tests.bwc.version", default_bwc_version)
boolean isSnapshot = knn_bwc_version.contains("-SNAPSHOT")
String knn_bwc_version_no_qualifier = isSnapshot ? knn_bwc_version - "-SNAPSHOT" : knn_bwc_version

String os_platform = "linux"
String artifact_type = "tar"
String file_ext = "tar.gz"

if (Os.isFamily(Os.FAMILY_WINDOWS)) {
os_platform = "windows"
artifact_type = "zip"
file_ext = "zip"
}

// Task to pull k-NN plugin from archive
task pullBwcPlugin {
doFirst {
delete java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-*")
File[] tempFiles = tmp_dir.listFiles()
for (File child : tempFiles) {
if (child.exists() && child.toString().contains("opensearch-")) {
Files.delete(child.toPath());
}
}
}

doLast {
ext{
if (isSnapshot) {
srcUrl = "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/${knn_bwc_version_no_qualifier}/latest/linux/x64/tar/dist/opensearch/opensearch-${knn_bwc_version_no_qualifier}-linux-x64.tar.gz"
srcUrl = "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/${knn_bwc_version_no_qualifier}/latest/${os_platform}/x64/${artifact_type}/dist/opensearch/opensearch-${knn_bwc_version_no_qualifier}-${os_platform}-x64.${file_ext}"
} else {
srcUrl = "https://artifacts.opensearch.org/releases/bundle/opensearch/${knn_bwc_version}/opensearch-${knn_bwc_version}-linux-x64.tar.gz"
srcUrl = "https://artifacts.opensearch.org/releases/bundle/opensearch/${knn_bwc_version}/opensearch-${knn_bwc_version}-${os_platform}-x64.${file_ext}"
}
}
ant.get(
Expand All @@ -53,14 +71,18 @@ task pullBwcPlugin {
httpusecaches: false
)
copy {
from tarTree(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${knn_bwc_version_no_qualifier}-linux-x64.tar.gz"))
if (Os.isFamily(Os.FAMILY_WINDOWS)) {
from zipTree(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${knn_bwc_version_no_qualifier}-${os_platform}-x64.${file_ext}"))
} else {
from tarTree(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${knn_bwc_version_no_qualifier}-${os_platform}-x64.${file_ext}"))
}
into tmp_dir.absolutePath
}
copy {
from(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${knn_bwc_version_no_qualifier}", "plugins", "opensearch-knn"))
into java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-knn")
}
delete java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${knn_bwc_version_no_qualifier}"), java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${knn_bwc_version_no_qualifier}-linux-x64.tar.gz")
delete java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${knn_bwc_version_no_qualifier}"), java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${knn_bwc_version_no_qualifier}-${os_platform}-x64.${file_ext}")
}
}

Expand Down
10 changes: 9 additions & 1 deletion qa/restart-upgrade/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
*/

import org.opensearch.gradle.testclusters.StandaloneRestIntegTestTask
import org.apache.tools.ant.taskdefs.condition.Os

apply from : "$rootDir/qa/build.gradle"

Expand All @@ -23,7 +24,14 @@ testClusters {
setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
setting 'http.content_type.required', 'true'
environment "LD_LIBRARY_PATH", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib;${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib"
systemProperty "java.library.path", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib:${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib"
if (Os.isFamily(Os.FAMILY_WINDOWS)) {
// While running on Windows OS, setting the PATH environment variable to include the paths to dlls of JNI libraries and windows dependencies
environment('PATH', System.getenv('PATH') + ";$rootDir/jni/release" + ";$rootDir/src/main/resources/windowsDependencies")
naveentatikonda marked this conversation as resolved.
Show resolved Hide resolved
systemProperty "java.library.path", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib;${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib"
} else {
systemProperty "java.library.path", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib:${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib"
}

}
}

Expand Down
9 changes: 8 additions & 1 deletion qa/rolling-upgrade/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
*/

import org.opensearch.gradle.testclusters.StandaloneRestIntegTestTask
import org.apache.tools.ant.taskdefs.condition.Os

apply from : "$rootDir/qa/build.gradle"

Expand All @@ -23,7 +24,13 @@ testClusters {
setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
setting 'http.content_type.required', 'true'
environment "LD_LIBRARY_PATH", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib;${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib"
systemProperty "java.library.path", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib:${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib"
if (Os.isFamily(Os.FAMILY_WINDOWS)) {
// While running on Windows OS, setting the PATH environment variable to include the paths to dlls of JNI libraries and windows dependencies
environment('PATH', System.getenv('PATH') + ";$rootDir/jni/release" + ";$rootDir/src/main/resources/windowsDependencies")
systemProperty "java.library.path", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib;${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib"
} else {
systemProperty "java.library.path", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib:${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib"
}
}
}

Expand Down