Merge branch 'main' into ml-similarity

jonathan-buttner · Mar 25, 2024 · b7141ce · b7141ce
2 parents 6429598 + c15f727
commit b7141ce
Show file tree

Hide file tree

Showing 26 changed files with 86 additions and 1,142 deletions.
diff --git a/build-tools-internal/gradle/wrapper/gradle-wrapper.properties b/build-tools-internal/gradle/wrapper/gradle-wrapper.properties
@@ -1,7 +1,7 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionSha256Sum=85719317abd2112f021d4f41f09ec370534ba288432065f4b477b6a3b652910d
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.6-all.zip
+distributionSha256Sum=194717442575a6f96e1c1befa2c30e9a4fc90f701d7aee33eb879b79e7ff05c0
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.7-all.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME

diff --git a/build-tools-internal/src/main/resources/minimumGradleVersion b/build-tools-internal/src/main/resources/minimumGradleVersion
@@ -1 +1 @@
-8.6
+8.7
diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar
diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties
@@ -1,7 +1,7 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionSha256Sum=85719317abd2112f021d4f41f09ec370534ba288432065f4b477b6a3b652910d
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.6-all.zip
+distributionSha256Sum=194717442575a6f96e1c1befa2c30e9a4fc90f701d7aee33eb879b79e7ff05c0
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.7-all.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME

diff --git a/...s/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml b/...s/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml
@@ -9,6 +9,7 @@ setup:
         index: tsdb
         body:
           settings:
+            number_of_shards: 1
             mode: time_series
             routing_path: [key]
             time_series:

diff --git a/plugins/examples/gradle/wrapper/gradle-wrapper.properties b/plugins/examples/gradle/wrapper/gradle-wrapper.properties
@@ -1,7 +1,7 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionSha256Sum=85719317abd2112f021d4f41f09ec370534ba288432065f4b477b6a3b652910d
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.6-all.zip
+distributionSha256Sum=194717442575a6f96e1c1befa2c30e9a4fc90f701d7aee33eb879b79e7ff05c0
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.7-all.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME

diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/count/30_min_score.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/count/30_min_score.yml
@@ -3,6 +3,9 @@
   - do:
       indices.create:
           index:  test_count_min_score
+          body:
+            settings:
+              number_of_shards: 1
 
   - do:
       index:

diff --git a/...yamlRestTest/resources/rest-api-spec/test/search.vectors/160_knn_query_missing_params.yml b/...yamlRestTest/resources/rest-api-spec/test/search.vectors/160_knn_query_missing_params.yml
@@ -6,6 +6,8 @@ setup:
       indices.create:
         index: knn_query_test_index
         body:
+          settings:
+            number_of_shards: 1
           mappings:
             properties:
                   vector:

diff --git a/...rc/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java b/...rc/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java
@@ -292,12 +292,14 @@ private RolloverResult rolloverDataStream(
         DataStreamAutoShardingEvent dataStreamAutoShardingEvent = autoShardingResult == null
             ? dataStream.getAutoShardingEvent()
             : switch (autoShardingResult.type()) {
-                case NO_CHANGE_REQUIRED -> {
-                    logger.info(
-                        "Rolling over data stream [{}] using existing auto-sharding recommendation [{}]",
-                        dataStreamName,
-                        dataStream.getAutoShardingEvent()
-                    );
+                case NO_CHANGE_REQUIRED, COOLDOWN_PREVENTED_INCREASE, COOLDOWN_PREVENTED_DECREASE -> {
+                    if (dataStream.getAutoShardingEvent() != null) {
+                        logger.info(
+                            "Rolling over data stream [{}] using existing auto-sharding recommendation [{}]",
+                            dataStreamName,
+                            dataStream.getAutoShardingEvent()
+                        );
+                    }
                     yield dataStream.getAutoShardingEvent();
                 }
                 case INCREASE_SHARDS, DECREASE_SHARDS -> {
@@ -308,18 +310,6 @@ yield new DataStreamAutoShardingEvent(
                         now.toEpochMilli()
                     );
                 }
-                case COOLDOWN_PREVENTED_INCREASE, COOLDOWN_PREVENTED_DECREASE -> {
-                    // we're in the cooldown period for this particular recommendation so perhaps use a previous autosharding
-                    // recommendation (or the value configured in the backing index template otherwise)
-                    if (dataStream.getAutoShardingEvent() != null) {
-                        logger.info(
-                            "Rolling over data stream [{}] using existing auto-sharding recommendation [{}]",
-                            dataStreamName,
-                            dataStream.getAutoShardingEvent()
-                        );
-                    }
-                    yield dataStream.getAutoShardingEvent();
-                }
                 // data sharding might not be available due to the feature not being available/enabled or due to cluster level excludes
                 // being configured. the index template will dictate the number of shards as usual
                 case NOT_APPLICABLE -> {

diff --git a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java
@@ -329,7 +329,7 @@ public IndexRequest id(String id) {
      */
     @Override
     public IndexRequest routing(String routing) {
-        if (routing != null && routing.length() == 0) {
+        if (routing != null && routing.isEmpty()) {
             this.routing = null;
         } else {
             this.routing = routing;

diff --git a/...asticsearch/index/codec/tsdb/ForUtil.java → ...rg/elasticsearch/index/codec/ForUtil.java b/...asticsearch/index/codec/tsdb/ForUtil.java → ...rg/elasticsearch/index/codec/ForUtil.java
@@ -6,7 +6,7 @@
  * Side Public License, v 1.
  */
 
-package org.elasticsearch.index.codec.tsdb;
+package org.elasticsearch.index.codec;
 
 import org.apache.lucene.store.DataInput;
 import org.apache.lucene.store.DataOutput;
@@ -20,20 +20,9 @@
 // else we pack 2 ints per long
 public final class ForUtil {
 
-    static final int DEFAULT_BLOCK_SIZE = 128;
-    private final int blockSize;
-    private final int blockSizeLog2;
-    private final long[] tmp;
-
-    public ForUtil() {
-        this(DEFAULT_BLOCK_SIZE);
-    }
-
-    private ForUtil(int blockSize) {
-        this.blockSize = blockSize;
-        this.blockSizeLog2 = (int) (Math.log(blockSize) / Math.log(2));
-        this.tmp = new long[blockSize / 2];
-    }
+    public static final int BLOCK_SIZE = 128;
+    private static final int BLOCK_SIZE_LOG2 = 7;
+    private final long[] tmp = new long[BLOCK_SIZE / 2];
 
     private static long expandMask32(long mask32) {
         return mask32 | (mask32 << 32);
@@ -129,20 +118,20 @@ private static void collapse32(long[] arr) {
     }
 
     /** Encode 128 integers from {@code longs} into {@code out}. */
-    void encode(long[] longs, int bitsPerValue, DataOutput out) throws IOException {
+    public void encode(long[] longs, int bitsPerValue, DataOutput out) throws IOException {
         final int nextPrimitive;
         final int numLongs;
         if (bitsPerValue <= 8) {
             nextPrimitive = 8;
-            numLongs = blockSize / 8;
+            numLongs = BLOCK_SIZE / 8;
             collapse8(longs);
         } else if (bitsPerValue <= 16) {
             nextPrimitive = 16;
-            numLongs = blockSize / 4;
+            numLongs = BLOCK_SIZE / 4;
             collapse16(longs);
         } else {
             nextPrimitive = 32;
-            numLongs = blockSize / 2;
+            numLongs = BLOCK_SIZE / 2;
             collapse32(longs);
         }
 
@@ -202,11 +191,11 @@ void encode(long[] longs, int bitsPerValue, DataOutput out) throws IOException {
     }
 
     /** Number of bytes required to encode 128 integers of {@code bitsPerValue} bits per value. */
-    int numBytes(int bitsPerValue) {
-        return bitsPerValue << (blockSizeLog2 - 3);
+    public int numBytes(int bitsPerValue) {
+        return bitsPerValue << (BLOCK_SIZE_LOG2 - 3);
     }
 
-    private static void decodeSlow(int blockSize, int bitsPerValue, DataInput in, long[] tmp, long[] longs) throws IOException {
+    private static void decodeSlow(int bitsPerValue, DataInput in, long[] tmp, long[] longs) throws IOException {
         final int numLongs = bitsPerValue << 1;
         in.readLongs(tmp, 0, numLongs);
         final long mask = MASKS32[bitsPerValue];
@@ -220,7 +209,7 @@ private static void decodeSlow(int blockSize, int bitsPerValue, DataInput in, lo
         final long mask32RemainingBitsPerLong = MASKS32[remainingBitsPerLong];
         int tmpIdx = 0;
         int remainingBits = remainingBitsPerLong;
-        for (; longsIdx < blockSize / 2; ++longsIdx) {
+        for (; longsIdx < BLOCK_SIZE / 2; ++longsIdx) {
             int b = bitsPerValue - remainingBits;
             long l = (tmp[tmpIdx++] & MASKS32[remainingBits]) << b;
             while (b >= remainingBitsPerLong) {
@@ -310,7 +299,7 @@ private static void shiftLongs(long[] a, int count, long[] b, int bi, int shift,
     private static final long MASK32_24 = MASKS32[24];
 
     /** Decode 128 integers into {@code longs}. */
-    void decode(int bitsPerValue, DataInput in, long[] longs) throws IOException {
+    public void decode(int bitsPerValue, DataInput in, long[] longs) throws IOException {
         switch (bitsPerValue) {
             case 1:
                 decode1(in, tmp, longs);
@@ -409,7 +398,7 @@ void decode(int bitsPerValue, DataInput in, long[] longs) throws IOException {
                 expand32(longs);
                 break;
             default:
-                decodeSlow(blockSize, bitsPerValue, in, tmp, longs);
+                decodeSlow(bitsPerValue, in, tmp, longs);
                 expand32(longs);
                 break;
         }
@@ -421,7 +410,7 @@ void decode(int bitsPerValue, DataInput in, long[] longs) throws IOException {
      * [0..63], and values [64..127] are encoded in the low-order bits of {@code longs} [0..63]. This
      * representation may allow subsequent operations to be performed on two values at a time.
      */
-    void decodeTo32(int bitsPerValue, DataInput in, long[] longs) throws IOException {
+    public void decodeTo32(int bitsPerValue, DataInput in, long[] longs) throws IOException {
         switch (bitsPerValue) {
             case 1:
                 decode1(in, tmp, longs);
@@ -512,7 +501,7 @@ void decodeTo32(int bitsPerValue, DataInput in, long[] longs) throws IOException
                 decode24(in, tmp, longs);
                 break;
             default:
-                decodeSlow(blockSize, bitsPerValue, in, tmp, longs);
+                decodeSlow(bitsPerValue, in, tmp, longs);
                 break;
         }
     }

diff --git a/server/src/main/java/org/elasticsearch/index/codec/postings/ES812PostingsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/postings/ES812PostingsFormat.java
@@ -36,6 +36,7 @@
 import org.apache.lucene.store.DataOutput;
 import org.apache.lucene.util.packed.PackedInts;
 import org.elasticsearch.core.IOUtils;
+import org.elasticsearch.index.codec.ForUtil;
 
 import java.io.IOException;
 

diff --git a/server/src/main/java/org/elasticsearch/index/codec/postings/ES812PostingsReader.java b/server/src/main/java/org/elasticsearch/index/codec/postings/ES812PostingsReader.java
@@ -36,19 +36,20 @@
 import org.apache.lucene.util.BitUtil;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.core.IOUtils;
+import org.elasticsearch.index.codec.ForUtil;
 import org.elasticsearch.index.codec.postings.ES812PostingsFormat.IntBlockTermState;
 
 import java.io.IOException;
 import java.util.Arrays;
 
+import static org.elasticsearch.index.codec.ForUtil.BLOCK_SIZE;
 import static org.elasticsearch.index.codec.postings.ES812PostingsFormat.DOC_CODEC;
 import static org.elasticsearch.index.codec.postings.ES812PostingsFormat.MAX_SKIP_LEVELS;
 import static org.elasticsearch.index.codec.postings.ES812PostingsFormat.PAY_CODEC;
 import static org.elasticsearch.index.codec.postings.ES812PostingsFormat.POS_CODEC;
 import static org.elasticsearch.index.codec.postings.ES812PostingsFormat.TERMS_CODEC;
 import static org.elasticsearch.index.codec.postings.ES812PostingsFormat.VERSION_CURRENT;
 import static org.elasticsearch.index.codec.postings.ES812PostingsFormat.VERSION_START;
-import static org.elasticsearch.index.codec.postings.ForUtil.BLOCK_SIZE;
 
 /**
  * Concrete class that reads docId(maybe frq,pos,offset,payloads) list with postings format.

diff --git a/server/src/main/java/org/elasticsearch/index/codec/postings/ES812PostingsWriter.java b/server/src/main/java/org/elasticsearch/index/codec/postings/ES812PostingsWriter.java
@@ -35,6 +35,7 @@
 import org.apache.lucene.util.BitUtil;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.core.IOUtils;
+import org.elasticsearch.index.codec.ForUtil;
 import org.elasticsearch.index.codec.postings.ES812PostingsFormat.IntBlockTermState;
 
 import java.io.IOException;

diff --git a/server/src/main/java/org/elasticsearch/index/codec/postings/ES812SkipReader.java b/server/src/main/java/org/elasticsearch/index/codec/postings/ES812SkipReader.java
@@ -21,6 +21,7 @@
 
 import org.apache.lucene.codecs.MultiLevelSkipListReader;
 import org.apache.lucene.store.IndexInput;
+import org.elasticsearch.index.codec.ForUtil;
 
 import java.io.IOException;
 import java.util.Arrays;