[FEATURE] Add z-score for the normalization processor #376

CHANGELOG.md

@@ -14,6 +14,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ## [Unreleased 2.x](https://github.com/opensearch-project/neural-search/compare/2.11...2.x)
 ### Features
+adding z-score normalization for hybrid query [#470](https://github.com/opensearch-project/neural-search/pull/470)
 ### Enhancements
 ### Bug Fixes
 ### Infrastructure

build.gradle

@@ -19,7 +19,7 @@ apply plugin: "com.diffplug.spotless"
 apply plugin: 'io.freefair.lombok'
 
 def pluginName = 'opensearch-neural-search'
-def pluginDescription = 'A plugin that adds dense neural retrieval into the OpenSearch ecosytem'
+def pluginDescription = 'A plugin that adds dense neural retrieval into the OpenSearch ecosystem'
 def projectPath = 'org.opensearch'
 def pathToPlugin = 'neuralsearch.plugin'
 def pluginClassName = 'NeuralSearch'
@@ -219,6 +219,12 @@ integTest {
     if (System.getProperty("test.debug") != null) {
         jvmArgs '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005'
     }
+
+    systemProperty 'log4j2.configurationFile', "${projectDir}/src/test/resources/log4j2-test.xml"
+
+    // Set this to true this if you want to see the logs in the terminal test output.
+    // note: if left false the log output will still show in your IDE
+    testLogging.showStandardStreams = true
 }
 
 testClusters.integTest {

scripts/Dockerfile

@@ -0,0 +1,9 @@
+# Build with the following command
+# docker build --tag="opensearch-zscore-test:2.11.0" .
+FROM opensearchproject/opensearch:2.11.0
+# Remove previous neural search plugin and install new one
+RUN /usr/share/opensearch/bin/opensearch-plugin remove opensearch-neural-search
+# Make sure the build is preset, to create it `./gradlew clean assemble -Dopensearch.version="2.11.0"`
+# Then copy it locally `cp ../build/distributions/opensearch-neural-search-2.11.0.0-SNAPSHOT.zip .`
+COPY opensearch-neural-search-2.11.0.0-SNAPSHOT.zip /usr/share/opensearch/
+RUN /usr/share/opensearch/bin/opensearch-plugin install -b file:/usr/share/opensearch/opensearch-neural-search-2.11.0.0-SNAPSHOT.zip

scripts/cleanup-previous-docker.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+#### Procedure to cleanup all previous experiments with docker compose ####
+
+# Stop the container(s) using the following command:
+docker-compose down
+
+# Delete all containers using the following command:
+docker rm -f $(docker ps -a -q)
+
+# Delete all volumes using the following command:
+docker volume rm $(docker volume ls -q)
+
+# Restart the containers using the following command:
+docker-compose up
+

scripts/docker-compose.yml

@@ -0,0 +1,48 @@
+version: '3'
+services:
+  opensearch-node1: # This is also the hostname of the container within the Docker network (i.e. https://opensearch-node1/)
+    #image: opensearchproject/opensearch:2.11.0
+    image: opensearch-zscore-test:2.11.0
+    container_name: opensearch-node1
+    environment:
+      - cluster.name=opensearch-cluster # Name the cluster
+      - node.name=opensearch-node1 # Name the node that will run in this container
+      - discovery.seed_hosts=opensearch-node1 # Nodes to look for when discovering the cluster
+      - cluster.initial_cluster_manager_nodes=opensearch-node1 # Nodes eligibile to serve as cluster manager
+      - bootstrap.memory_lock=true # Disable JVM heap memory swapping
+      - plugins.security.disabled=true # Disable security plugin so it's easy to test
+      - "DISABLE_INSTALL_DEMO_CONFIG=true" # Disable security plugin so it's easy to test
+      - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" # Set min and max JVM heap sizes to at least 50% of system RAM
+    ulimits:
+      memlock:
+        soft: -1 # Set memlock to unlimited (no soft or hard limit)
+        hard: -1
+      nofile:
+        soft: 65536 # Maximum number of open files for the opensearch user - set to at least 65536
+        hard: 65536
+    volumes:
+      - opensearch-data1:/usr/share/opensearch/data # Creates volume called opensearch-data1 and mounts it to the container
+    ports:
+      - 9200:9200 # REST API
+      - 9600:9600 # Performance Analyzer
+    networks:
+      - opensearch-net # All of the containers will join the same Docker bridge network
+  opensearch-dashboards:
+    image: opensearchproject/opensearch-dashboards:2.11.0 # Make sure the version of opensearch-dashboards matches the version of opensearch installed on other nodes
+    container_name: opensearch-dashboards
+    ports:
+      - 5601:5601 # Map host port 5601 to container port 5601
+    expose:
+      - "5601" # Expose port 5601 for web access to OpenSearch Dashboards
+    environment:
+      - 'OPENSEARCH_HOSTS=["http://opensearch-node1:9200"]' # Define the OpenSearch nodes that OpenSearch Dashboards will query (Plain HTTP,  security disabled)
+      - "DISABLE_SECURITY_DASHBOARDS_PLUGIN=true" # disables security dashboards plugin in OpenSearch Dashboards
+    networks:
+      - opensearch-net
+
+volumes:
+  opensearch-data1:
+
+networks:
+  opensearch-net:
+