Merge branch 'master' into enh/ability-to-pass-ef-search-in-query

* master:
  added github action to build library artifacts (opendistro-for-elasticsearch#132)
  FIX: buildDir->rootDir
  Build separate artifacts for library using CPack (opendistro-for-elasticsearch#123)
  Fix test structure (opendistro-for-elasticsearch#125)

.github/workflows/CD.yml

@@ -9,13 +9,14 @@ jobs:
     strategy:
       matrix:
         java: [14]
+        compiler: [g++-5]
 
     name: Build and Release k-NN Plugin
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-16.04
 
     steps:
       - name: Checkout k-NN
-        uses: actions/checkout@v1
+        uses: actions/checkout@v2
 
       - name: Configure AWS
         uses: aws-actions/configure-aws-credentials@v1
@@ -29,7 +30,24 @@ jobs:
         with:
           java-version: ${{ matrix.java }}
 
-      - name: Run build
+      - name: Build and ship library artifacts
+        env:
+          CXX: ${{ matrix.compiler }}
+        run: |
+            cd jni
+            cmake .
+            make package
+
+            artifact=`ls release/*.so`
+            rpm_artifact=`ls packages/*.rpm`
+            deb_artifact=`ls packages/*.deb`
+
+            aws s3 cp $artifact s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/elasticsearch-plugins/opendistro-knn/
+            aws s3 cp $rpm_artifact s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/rpms/opendistro-knn/
+            aws s3 cp $deb_artifact s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/debs/opendistro-knn/
+            aws cloudfront create-invalidation --distribution-id E1VG5HMIWI4SA2 --paths "/*"
+
+      - name: Build and ship plugin artifacts
         run: |
           ./gradlew buildPackages --console=plain -Dbuild.snapshot=false
           artifact=`ls build/distributions/*.zip`

.gitignore

@@ -13,4 +13,8 @@ oss/*
 jni/CMakeCache.txt
 jni/CMakeFiles
 jni/Makefile
-jni/cmake_install.cmake
+jni/cmake*
+jni/CPack*
+jni/_CPack*
+jni/release
+jni/packages

README.md

@@ -30,7 +30,21 @@ cmake .
 make
 ```
 
-The library will be placed in the `buildSrc` directory.
+The library will be placed in the `jni/release` directory.
+
+## Build JNI Library RPM/DEB
+
+To build an RPM or DEB of the JNI library, follow these steps:
+
+```
+cd jni
+cmake .
+make package
+```
+
+The artifacts will be placed in the `jni/packages` directory.
+
+Additionally, we build the RPM and DEB in [this GitHub action](https://github.com/opendistro-for-elasticsearch/k-NN/blob/master/.github/workflows/CD.yml). We use Ubuntu 16.04 with g++ 5.4.0.
 
 ## Running Multi-node Cluster Locally
 

build.gradle

@@ -148,16 +148,23 @@ task buildJniLib(type:Exec) {
     commandLine 'make'
 }
 
-test {
+task makeJniLib(type: Copy) {
     dependsOn buildJniLib
+    from "$rootDir/jni/release"
+    include "*"
+    into "$rootDir/buildSrc"
+}
+
+test {
+    dependsOn makeJniLib
     systemProperty 'tests.security.manager', 'false'
     systemProperty "java.library.path", "$rootDir/buildSrc"
 }
 
 def _numNodes = findProperty('numNodes') as Integer ?: 1
 integTest {
     if (integTestDependOnJniLib) {
-        dependsOn buildJniLib
+        dependsOn makeJniLib
     }
     runner {
         systemProperty 'tests.security.manager', 'false'
@@ -205,7 +212,7 @@ testClusters.integTest {
 }
 
 run {
-    dependsOn buildJniLib
+    dependsOn makeJniLib
     doFirst {
         // There seems to be an issue when running multi node run or integ tasks with unicast_hosts
         // not being written, the waitForAllConditions ensures it's written
@@ -248,6 +255,7 @@ afterEvaluate {
         dirMode 0755
 
         requires('elasticsearch-oss', versions.elasticsearch, EQUAL)
+        requires("opendistro-knnlib",  "1.8.0.0",  EQUAL)
         packager = 'Amazon'
         vendor = 'Amazon'
         os = 'LINUX'

jni/CMakeLists.txt

@@ -22,6 +22,7 @@ set(CMAKE_CXX_STANDARD_REQUIRED True)
 
 # Target Library to be built
 set(KNN_INDEX KNNIndexV1_7_3_6)
+set(KNN_PACKAGE_NAME opendistro-knnlib)
 
 # Check if similarity search exists
 find_path(NMS_REPO_DIR NAMES similarity_search PATHS ${CMAKE_CURRENT_SOURCE_DIR}/external/nmslib REQUIRED)
@@ -33,10 +34,11 @@ if (NOT EXISTS ${NMS_REPO_DIR})
 endif ()
 
 # Add the subdirectory so it is possible to use its targets
-add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/external/nmslib/similarity_search)
+add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/external/nmslib/similarity_search EXCLUDE_FROM_ALL)
 
 # Set OS specific variables
 if (${CMAKE_SYSTEM_NAME} STREQUAL Darwin)
+    set(CMAKE_MACOSX_RPATH 1)
     set(JVM_OS_TYPE darwin)
     set(LIB_EXT .jnilib)
 elseif(${CMAKE_SYSTEM_NAME} STREQUAL Linux)
@@ -47,14 +49,45 @@ else()
 endif()
 
 # Compile the library
-add_library(${KNN_INDEX} SHARED ${CMAKE_CURRENT_SOURCE_DIR}/src/v1736/com_amazon_opendistroforelasticsearch_knn_index_v1736_KNNIndex.cpp)
+add_library(${KNN_INDEX} SHARED ${CMAKE_CURRENT_SOURCE_DIR}/src/com_amazon_opendistroforelasticsearch_knn_index_v1736_KNNIndex.cpp)
 target_link_libraries(${KNN_INDEX} NonMetricSpaceLib)
-target_include_directories(${KNN_INDEX} PRIVATE $ENV{JAVA_HOME}/include $ENV{JAVA_HOME}/include/${JVM_OS_TYPE} ${CMAKE_CURRENT_SOURCE_DIR}/external/nmslib/similarity_search/include)
+target_include_directories(${KNN_INDEX} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include $ENV{JAVA_HOME}/include $ENV{JAVA_HOME}/include/${JVM_OS_TYPE} ${CMAKE_CURRENT_SOURCE_DIR}/external/nmslib/similarity_search/include)
 
 set_target_properties(${KNN_INDEX} PROPERTIES SUFFIX ${LIB_EXT})
 set_target_properties(${KNN_INDEX} PROPERTIES POSITION_INDEPENDENT_CODE ON)
-set_target_properties(${KNN_INDEX} PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/../buildSrc)
+set_target_properties(${KNN_INDEX} PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/release)
 
-# Log directory where library will be placed
-get_target_property(JNILibDir ${KNN_INDEX} LIBRARY_OUTPUT_DIRECTORY)
-message(STATUS "JNI Library Output Directory: ${JNILibDir}")
+# Installation rules for shared library
+install(TARGETS ${KNN_INDEX}
+        LIBRARY DESTINATION lib
+        COMPONENT library)
+
+# CPack section to build artifacts
+set(KNN_MAINTAINER "OpenDistro for Elasticsearch Team <[email protected]>")
+set(ODFE_DOWNLOAD_URL "https://opendistro.github.io/elasticsearch/downloads")
+set(CPACK_PACKAGE_NAME ${KNN_PACKAGE_NAME})
+set(CPACK_PACKAGE_VERSION 1.8.0.0)
+set(CMAKE_INSTALL_PREFIX /usr)
+set(CPACK_GENERATOR "RPM;DEB")
+SET(CPACK_OUTPUT_FILE_PREFIX packages)
+set(CPACK_PACKAGE_RELEASE 0.1)
+set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "KNN JNI library built off of NMSLIB for OpenDistro for Elasticsearch. Reference documentation can be found at https://opendistro.github.io/for-elasticsearch-docs/.")
+set(CPACK_PACKAGE_VENDOR "Amazon")
+set(CPACK_PACKAGE_CONTACT "Maintainer: ${KNN_MAINTAINER}")
+set(CPACK_PACKAGING_INSTALL_PREFIX ${CMAKE_INSTALL_PREFIX})
+set(CPACK_PACKAGE_FILE_NAME "${CPACK_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION}-${CPACK_PACKAGE_RELEASE}_${JVM_OS_TYPE}.${CMAKE_SYSTEM_PROCESSOR}")
+
+# RPM Specific variables
+set(CPACK_RPM_PACKAGE_RELEASE ${CPACK_PACKAGE_RELEASE})
+set(CPACK_RPM_PACKAGE_URL ${ODFE_DOWNLOAD_URL})
+set(CPACK_RPM_PACKAGE_DESCRIPTION "Open Distro for Elasticsearch KNN JNI Library")
+set(CPACK_RPM_PACKAGE_LICENSE "ASL-2.0")
+
+# DEB Specific variables
+set(CPACK_DEBIAN_PACKAGE_HOMEPAGE ${ODFE_DOWNLOAD_URL})
+set(CPACK_DEBIAN_PACKAGE_MAINTAINER ${KNN_MAINTAINER})
+set(CPACK_DEBIAN_PACKAGE_SOURCE ${CPACK_PACKAGE_NAME})
+set(CPACK_DEBIAN_PACKAGE_VERSION ${CPACK_PACKAGE_VERSION})
+set(CPACK_DEBIAN_PACKAGE_SECTION "libs")
+
+include(CPack)

src/main/java/com/amazon/opendistroforelasticsearch/knn/plugin/stats/KNNCounter.java

@@ -64,4 +64,11 @@ public Long getCount() {
     public void increment() {
         count.getAndIncrement();
     }
+
+    /**
+     * Set the value of a counter
+     */
+    public void set(long value) {
+        count.set(value);
+    }
 }

src/test/java/com/amazon/opendistroforelasticsearch/knn/index/KNNRestTestCase.java → src/test/java/com/amazon/opendistroforelasticsearch/knn/KNNRestTestCase.java

@@ -13,8 +13,9 @@
  *   permissions and limitations under the License.
  */
 
-package com.amazon.opendistroforelasticsearch.knn.index;
+package com.amazon.opendistroforelasticsearch.knn;
 
+import com.amazon.opendistroforelasticsearch.knn.index.KNNQueryBuilder;
 import com.amazon.opendistroforelasticsearch.knn.plugin.KNNPlugin;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;

src/test/java/com/amazon/opendistroforelasticsearch/knn/index/KNNResult.java → src/test/java/com/amazon/opendistroforelasticsearch/knn/KNNResult.java

@@ -13,7 +13,7 @@
  *   permissions and limitations under the License.
  */
 
-package com.amazon.opendistroforelasticsearch.knn.index;
+package com.amazon.opendistroforelasticsearch.knn;
 
 public class KNNResult {
     private String docId;

src/test/java/com/amazon/opendistroforelasticsearch/knn/KNNTestCase.java

@@ -0,0 +1,33 @@
+/*
+ *   Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ *   Licensed under the Apache License, Version 2.0 (the "License").
+ *   You may not use this file except in compliance with the License.
+ *   A copy of the License is located at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *   or in the "license" file accompanying this file. This file is distributed
+ *   on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ *   express or implied. See the License for the specific language governing
+ *   permissions and limitations under the License.
+ */
+
+package com.amazon.opendistroforelasticsearch.knn;
+
+import com.amazon.opendistroforelasticsearch.knn.plugin.stats.KNNCounter;
+import org.elasticsearch.test.ESTestCase;
+import org.junit.Before;
+
+/**
+ * Base class for integration tests for KNN plugin. Contains several methods for testing KNN ES functionality.
+ */
+public class KNNTestCase extends ESTestCase {
+    @Before
+    public void resetCounters() {
+        // Reset all of the counters
+        for (KNNCounter knnCounter : KNNCounter.values()) {
+            knnCounter.set(0L);
+        }
+    }
+}

src/test/java/com/amazon/opendistroforelasticsearch/knn/index/KNNJNITests.java

@@ -15,14 +15,14 @@
 
 package com.amazon.opendistroforelasticsearch.knn.index;
 
+import com.amazon.opendistroforelasticsearch.knn.KNNTestCase;
 import com.amazon.opendistroforelasticsearch.knn.index.v1736.KNNIndex;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.FSDirectory;
 import org.apache.lucene.store.FilterDirectory;
-import org.elasticsearch.test.ESTestCase;
 
 import java.nio.file.Paths;
 import java.security.AccessController;
@@ -31,7 +31,7 @@
 import java.util.Map;
 import java.util.stream.Collectors;
 
-public class KNNJNITests extends ESTestCase {
+public class KNNJNITests extends KNNTestCase {
     private static final Logger logger = LogManager.getLogger(KNNJNITests.class);
 
     public void testCreateHnswIndex() throws Exception {

src/test/java/com/amazon/opendistroforelasticsearch/knn/plugin/stats/KNNCounterTests.java

@@ -15,9 +15,9 @@
 
 package com.amazon.opendistroforelasticsearch.knn.plugin.stats;
 
-import org.elasticsearch.test.ESTestCase;
+import com.amazon.opendistroforelasticsearch.knn.KNNTestCase;
 
-public class KNNCounterTests extends ESTestCase {
+public class KNNCounterTests extends KNNTestCase {
     public void testGetName() {
         assertEquals(StatNames.GRAPH_QUERY_ERRORS.getName(), KNNCounter.GRAPH_QUERY_ERRORS.getName());
     }

src/test/java/com/amazon/opendistroforelasticsearch/knn/plugin/stats/suppliers/KNNCounterSupplierTests.java

@@ -15,10 +15,10 @@
 
 package com.amazon.opendistroforelasticsearch.knn.plugin.stats.suppliers;
 
+import com.amazon.opendistroforelasticsearch.knn.KNNTestCase;
 import com.amazon.opendistroforelasticsearch.knn.plugin.stats.KNNCounter;
-import org.elasticsearch.test.ESTestCase;
 
-public class KNNCounterSupplierTests extends ESTestCase {
+public class KNNCounterSupplierTests extends KNNTestCase {
     public void testNormal() {
         KNNCounterSupplier knnCounterSupplier = new KNNCounterSupplier(KNNCounter.GRAPH_QUERY_REQUESTS);
         assertEquals((Long) 0L, knnCounterSupplier.get());