Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Gradle 7 and Opensearch 2.0 upgrade #464

Merged
merged 1 commit into from
Mar 24, 2022
Merged

Gradle 7 and Opensearch 2.0 upgrade #464

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
99 changes: 50 additions & 49 deletions .github/workflows/CI.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,63 +29,64 @@ jobs:

- name: Assemble anomaly-detection
run: |
./gradlew assemble -Dopensearch.version=1.3.0-SNAPSHOT
echo "Creating ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/1.3.0.0-SNAPSHOT ..."
mkdir -p ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/1.3.0.0-SNAPSHOT
echo "Copying ./build/distributions/*.zip to ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/1.3.0.0-SNAPSHOT ..."
./gradlew assemble -Dopensearch.version=2.0.0-SNAPSHOT
echo "Creating ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/2.0.0.0-SNAPSHOT ..."
mkdir -p ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/2.0.0.0-SNAPSHOT
echo "Copying ./build/distributions/*.zip to ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/2.0.0.0-SNAPSHOT ..."
ls ./build/distributions/
cp ./build/distributions/*.zip ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/1.3.0.0-SNAPSHOT
echo "Copied ./build/distributions/*.zip to ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/1.3.0.0-SNAPSHOT ..."
ls ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/1.3.0.0-SNAPSHOT
cp ./build/distributions/*.zip ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/2.0.0.0-SNAPSHOT
echo "Copied ./build/distributions/*.zip to ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/2.0.0.0-SNAPSHOT ..."
ls ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/2.0.0.0-SNAPSHOT
- name: Build and Run Tests
run: |
./gradlew build -Dopensearch.version=1.3.0-SNAPSHOT
./gradlew build -Dopensearch.version=2.0.0-SNAPSHOT
- name: Publish to Maven Local
run: |
./gradlew publishToMavenLocal -Dopensearch.version=1.3.0-SNAPSHOT
./gradlew publishToMavenLocal -Dopensearch.version=2.0.0-SNAPSHOT
- name: Multi Nodes Integration Testing
run: |
./gradlew integTest -PnumNodes=3
- name: Pull and Run Docker
run: |
plugin=`ls build/distributions/*.zip`
version=1.3.0
plugin_version=1.3.0.0-SNAPSHOT
echo Using OpenSearch $version with AD $plugin_version
cd ..
if docker pull opensearchstaging/opensearch:$version
then
echo "FROM opensearchstaging/opensearch:$version" >> Dockerfile
echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-anomaly-detection ]; then /usr/share/opensearch/bin/opensearch-plugin remove opensearch-anomaly-detection; fi" >> Dockerfile
echo "ADD anomaly-detection/build/distributions/opensearch-anomaly-detection-$plugin_version.zip /tmp/" >> Dockerfile
echo "RUN /usr/share/opensearch/bin/opensearch-plugin install --batch file:/tmp/opensearch-anomaly-detection-$plugin_version.zip" >> Dockerfile
docker build -t opensearch-ad:test .
echo "imagePresent=true" >> $GITHUB_ENV
else
echo "imagePresent=false" >> $GITHUB_ENV
fi
- name: Run Docker Image
if: env.imagePresent == 'true'
run: |
cd ..
docker run -p 9200:9200 -d -p 9600:9600 -e "discovery.type=single-node" opensearch-ad:test
sleep 90
- name: Run AD Test
if: env.imagePresent == 'true'
run: |
security=`curl -XGET https://localhost:9200/_cat/plugins?v -u admin:admin --insecure |grep opensearch-security|wc -l`
if [ $security -gt 0 ]
then
echo "Security plugin is available"
./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="docker-cluster" -Dhttps=true -Duser=admin -Dpassword=admin
else
echo "Security plugin is NOT available"
./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="docker-cluster"
fi
- name: Run AD Backwards Compatibility Tests
run: |
echo "Running backwards compatibility tests ..."
./gradlew bwcTestSuite -Dtests.security.manager=false
# ToDo: Check infra team for 2.0 docker availability and re-enable
# - name: Pull and Run Docker
# run: |
# plugin=`ls build/distributions/*.zip`
# version=2.0.0
# plugin_version=2.0.0.0-SNAPSHOT
# echo Using OpenSearch $version with AD $plugin_version
# cd ..
# if docker pull opensearchstaging/opensearch:$version
# then
# echo "FROM opensearchstaging/opensearch:$version" >> Dockerfile
# echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-anomaly-detection ]; then /usr/share/opensearch/bin/opensearch-plugin remove opensearch-anomaly-detection; fi" >> Dockerfile
# echo "ADD anomaly-detection/build/distributions/opensearch-anomaly-detection-$plugin_version.zip /tmp/" >> Dockerfile
# echo "RUN /usr/share/opensearch/bin/opensearch-plugin install --batch file:/tmp/opensearch-anomaly-detection-$plugin_version.zip" >> Dockerfile
# docker build -t opensearch-ad:test .
# echo "imagePresent=true" >> $GITHUB_ENV
# else
# echo "imagePresent=false" >> $GITHUB_ENV
# fi
# - name: Run Docker Image
# if: env.imagePresent == 'true'
# run: |
# cd ..
# docker run -p 9200:9200 -d -p 9600:9600 -e "discovery.type=single-node" opensearch-ad:test
# sleep 90
# - name: Run AD Test
# if: env.imagePresent == 'true'
# run: |
# security=`curl -XGET https://localhost:9200/_cat/plugins?v -u admin:admin --insecure |grep opensearch-security|wc -l`
# if [ $security -gt 0 ]
# then
# echo "Security plugin is available"
# ./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="docker-cluster" -Dhttps=true -Duser=admin -Dpassword=admin
# else
# echo "Security plugin is NOT available"
# ./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="docker-cluster"
# fi
# - name: Run AD Backwards Compatibility Tests
amitgalitz marked this conversation as resolved.
Show resolved Hide resolved
# run: |
# echo "Running backwards compatibility tests ..."
# ./gradlew bwcTestSuite -Dtests.security.manager=false
- name: Upload Coverage Report
uses: codecov/codecov-action@v1
with:
Expand Down
55 changes: 30 additions & 25 deletions build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ import org.opensearch.gradle.testclusters.StandaloneRestIntegTestTask
buildscript {
ext {
opensearch_group = "org.opensearch"
opensearch_version = System.getProperty("opensearch.version", "1.3.0-SNAPSHOT")
opensearch_version = System.getProperty("opensearch.version", "2.0.0-SNAPSHOT")
// 1.2.0 -> 1.2.0.0, and 1.2.0-SNAPSHOT -> 1.2.0.0-SNAPSHOT
opensearch_build = opensearch_version.replaceAll(/(\.\d)([^\d]*)$/, '$1.0$2')
common_utils_version = System.getProperty("common_utils.version", opensearch_build)
Expand Down Expand Up @@ -116,7 +116,7 @@ configurations.all {
}

configurations {
testCompile {
testImplementation {
exclude group: 'org.hamcrest', module: 'hamcrest-core'
}
}
Expand Down Expand Up @@ -503,6 +503,7 @@ List<String> jacocoExclusions = [
'org.opensearch.ad.transport.GetAnomalyDetectorTransportAction*',
'org.opensearch.ad.transport.SearchAnomalyResultTransportAction*',
'org.opensearch.ad.transport.SearchAnomalyDetectorInfoTransportAction*',
'org.opensearch.ad.transport.AnomalyDetectorJobRequest',

// TODO: unified flow caused coverage drop
'org.opensearch.ad.transport.DeleteAnomalyResultsTransportAction',
Expand Down Expand Up @@ -549,40 +550,44 @@ check.dependsOn jacocoTestCoverageVerification
jacocoTestCoverageVerification.dependsOn jacocoTestReport

dependencies {
compile "org.opensearch:opensearch:${opensearch_version}"
implementation "org.opensearch:opensearch:${opensearch_version}"
compileOnly "org.opensearch.plugin:opensearch-scripting-painless-spi:${opensearch_version}"
compileOnly "org.opensearch:opensearch-job-scheduler-spi:${job_scheduler_version}"
compile "org.opensearch:common-utils:${common_utils_version}"
compile "org.opensearch.client:opensearch-rest-client:${opensearch_version}"
compile group: 'com.google.guava', name: 'guava', version:'31.0.1-jre'
compile group: 'org.apache.commons', name: 'commons-math3', version: '3.6.1'
compile group: 'com.google.code.gson', name: 'gson', version: '2.8.9'
compile group: 'com.yahoo.datasketches', name: 'sketches-core', version: '0.13.4'
compile group: 'com.yahoo.datasketches', name: 'memory', version: '0.12.2'
compile group: 'commons-lang', name: 'commons-lang', version: '2.6'
compile group: 'org.apache.commons', name: 'commons-pool2', version: '2.10.0'
compileOnly "org.opensearch:opensearch-job-scheduler-spi:2.0.0.0-alpha1-SNAPSHOT"
implementation "org.opensearch:common-utils:${common_utils_version}"
implementation "org.opensearch.client:opensearch-rest-client:${opensearch_version}"
implementation group: 'com.google.guava', name: 'guava', version:'31.0.1-jre'
implementation group: 'com.google.guava', name: 'failureaccess', version:'1.0.1'
implementation group: 'org.javassist', name: 'javassist', version:'3.28.0-GA'
implementation group: 'org.apache.commons', name: 'commons-math3', version: '3.6.1'
implementation group: 'com.google.code.gson', name: 'gson', version: '2.8.9'
implementation group: 'com.yahoo.datasketches', name: 'sketches-core', version: '0.13.4'
implementation group: 'com.yahoo.datasketches', name: 'memory', version: '0.12.2'
implementation group: 'commons-lang', name: 'commons-lang', version: '2.6'
implementation group: 'org.apache.commons', name: 'commons-pool2', version: '2.10.0'

// force Jackson version to avoid version conflict issue
implementation 'software.amazon.randomcutforest:randomcutforest-serialization:2.0.1'
implementation "com.fasterxml.jackson.core:jackson-core:2.12.6"
implementation "com.fasterxml.jackson.core:jackson-databind:2.12.6"
implementation "com.fasterxml.jackson.core:jackson-annotations:2.12.6"
compile files('lib/randomcutforest-parkservices-2.0.1.jar')
compile files('lib/randomcutforest-core-2.0.1.jar')
implementation files('lib/randomcutforest-parkservices-2.0.1.jar')
implementation files('lib/randomcutforest-core-2.0.1.jar')

// used for serializing/deserializing rcf models.
compile group: 'io.protostuff', name: 'protostuff-core', version: '1.8.0'
compile group: 'io.protostuff', name: 'protostuff-runtime', version: '1.8.0'
compile group: 'org.apache.commons', name: 'commons-lang3', version: '3.12.0'

compile "org.jacoco:org.jacoco.agent:0.8.5"
compile ("org.jacoco:org.jacoco.ant:0.8.5") {
implementation group: 'io.protostuff', name: 'protostuff-core', version: '1.8.0'
implementation group: 'io.protostuff', name: 'protostuff-runtime', version: '1.8.0'
implementation group: 'io.protostuff', name: 'protostuff-api', version: '1.8.0'
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why add these new dependencies: protostuff-api and protostuff-collectionschema?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These required to be explicitly added as the compilation errors for Linkedbuffer and Schema were coming from these.
Compile failures : https://paste.amazon.com/show/snghes/1647551203

implementation group: 'io.protostuff', name: 'protostuff-collectionschema', version: '1.8.0'
implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.12.0'

implementation "org.jacoco:org.jacoco.agent:0.8.5"
implementation ("org.jacoco:org.jacoco.ant:0.8.5") {
exclude group: 'org.ow2.asm', module: 'asm-commons'
exclude group: 'org.ow2.asm', module: 'asm'
exclude group: 'org.ow2.asm', module: 'asm-tree'
}

testCompile group: 'pl.pragmatists', name: 'JUnitParams', version: '1.1.1'
testImplementation group: 'pl.pragmatists', name: 'JUnitParams', version: '1.1.1'
testImplementation group: 'org.mockito', name: 'mockito-core', version: '2.25.0'
testImplementation group: 'org.powermock', name: 'powermock-api-mockito2', version: '2.0.2'
testImplementation group: 'org.powermock', name: 'powermock-module-junit4', version: '2.0.2'
Expand All @@ -591,11 +596,11 @@ dependencies {
testImplementation group: 'org.powermock', name: 'powermock-api-support', version: '2.0.2'
testImplementation group: 'org.powermock', name: 'powermock-reflect', version: '2.0.7'
testImplementation group: 'org.objenesis', name: 'objenesis', version: '3.0.1'
testCompile group: 'net.bytebuddy', name: 'byte-buddy', version: '1.9.15'
testCompile group: 'net.bytebuddy', name: 'byte-buddy-agent', version: '1.9.15'
testImplementation group: 'net.bytebuddy', name: 'byte-buddy', version: '1.9.15'
testImplementation group: 'net.bytebuddy', name: 'byte-buddy-agent', version: '1.9.15'
testCompileOnly 'org.apiguardian:apiguardian-api:1.1.0'
testImplementation 'org.junit.jupiter:junit-jupiter-api:5.7.2'
testImplementation 'org.junit.jupiter:junit-jupiter-params:5.7.2'
testImplementation 'org.junit.jupiter:junit-jupiter-params:5.7.2'
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

minor: extra space

testImplementation 'org.junit.jupiter:junit-jupiter-engine:5.7.2'
testRuntimeOnly 'org.junit.vintage:junit-vintage-engine:5.7.2'
testCompileOnly 'junit:junit:4.13.2'
Expand Down
Binary file modified gradle/wrapper/gradle-wrapper.jar
View file
Open in desktop
Binary file not shown.
13 changes: 1 addition & 12 deletions gradle/wrapper/gradle-wrapper.properties
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,16 +1,5 @@
/*
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*
* Modifications Copyright OpenSearch Contributors. See
* GitHub history for details.
*/

distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-7.3.3-bin.zip
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-6.6.1-all.zip
53 changes: 33 additions & 20 deletions gradlew
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,21 @@
#!/usr/bin/env sh

#
# Copyright 2015 the original author or authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

##############################################################################
##
## Gradle start up script for UN*X
Expand Down Expand Up @@ -28,7 +44,7 @@ APP_NAME="Gradle"
APP_BASE_NAME=`basename "$0"`

# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
DEFAULT_JVM_OPTS=""
DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
amitgalitz marked this conversation as resolved.
Show resolved Hide resolved

# Use the maximum available, or set MAX_FD != -1 to use that value.
MAX_FD="maximum"
Expand Down Expand Up @@ -66,6 +82,7 @@ esac

CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar


# Determine the Java command to use to start the JVM.
if [ -n "$JAVA_HOME" ] ; then
if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
Expand Down Expand Up @@ -109,10 +126,11 @@ if $darwin; then
GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
fi

# For Cygwin, switch paths to Windows format before running java
if $cygwin ; then
# For Cygwin or MSYS, switch paths to Windows format before running java
if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then
APP_HOME=`cygpath --path --mixed "$APP_HOME"`
CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`

JAVACMD=`cygpath --unix "$JAVACMD"`

# We build the pattern for arguments to be converted via cygpath
Expand All @@ -138,19 +156,19 @@ if $cygwin ; then
else
eval `echo args$i`="\"$arg\""
fi
i=$((i+1))
i=`expr $i + 1`
done
case $i in
(0) set -- ;;
(1) set -- "$args0" ;;
(2) set -- "$args0" "$args1" ;;
(3) set -- "$args0" "$args1" "$args2" ;;
(4) set -- "$args0" "$args1" "$args2" "$args3" ;;
(5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
(6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
(7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
(8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
(9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
0) set -- ;;
1) set -- "$args0" ;;
2) set -- "$args0" "$args1" ;;
3) set -- "$args0" "$args1" "$args2" ;;
4) set -- "$args0" "$args1" "$args2" "$args3" ;;
5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
esac
fi

Expand All @@ -159,14 +177,9 @@ save () {
for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
echo " "
}
APP_ARGS=$(save "$@")
APP_ARGS=`save "$@"`

# Collect all arguments for the java command, following the shell quoting and substitution rules
eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"

# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong
if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then
cd "$(dirname "$0")"
fi

exec "$JAVACMD" "$@"
Loading