Static and dynamic configuration for VectorCollection [HZAI-18, HZAI-…

…19, HZAI-20] (#1151)

Implemented static and dynamic (member/client) configurations for vector
collections in accordance with the checklist
https://github.com/hazelcast/hazelcast/wiki/Checklist-for-Hazelcast-config-changes.

Programmatic:
```
var vectorIndexConfig = new VectorIndexConfig().setMetric(Metric.EUCLIDEAN).setDimension(1).setName("index-1");
var vectorCollectionConfig = new VectorCollectionConfig("vector-collection-1").addVectorIndexConfig(vectorIndexConfig);
Config config = new Config().addVectorCollectionConfig(vectorCollectionConfig);
```

XML:
```
    <vector-collection name="vector-1">
        <indexes>
            <index name="index-1">
                <dimension>2</dimension>
                <metric>DOT</metric>
            </index>
        </indexes>
    </vector-collection>
```

YAML:
```
  vector-collection:
    vector-1:
      indexes:
        - name: index-1
          dimension: 2
          metric: DOT
```

Restrictions and questions:

1) Permissions haven't been implemented.
2) The wildcard feature for vector collections operates akin to IMap.
This implies that creating a vector collection configuration such as
"*_OneIndexCollection" allows vector collections like
"myvector_OneIndexCollection" to inherit the configuration. Is this
functionality necessary?
3) The allowed pattern for vector collection name is
```[a-zA-Z0-9\\-*_]+```. The allowed pattern for vector collection index
name is ```[a-zA-Z0-9\\-_]+```.
4) com.hazelcast.internal.config.*ReadOnly are not implemented as they
are not used at the moment.

Checklist:
- [x] Codecs were generated using hazelcast-client-protocol but fixed
after that. Understand how to generate the correct codec without
requiring further modifications. -
b79fb809c6db9f066d212cdfe42d923a85068efd
- [x] check
https://github.com/hazelcast/hazelcast/wiki/Checklist-for-Hazelcast-config-changes
again
- [x] Labels (`Team:`, `Type:`, `Source:`, `Module:`) and Milestone set
- [x] Add `Add to Release Notes` label if changes should be mentioned in
release notes or `Not Release Notes content` if changes are not relevant
for release notes
- [x] Request reviewers if possible
- [x] New public APIs have `@Nonnull/@Nullable` annotations
- [x] New public APIs have `@since` tags in Javadoc
- [x] Send backports/forwardports if fix needs to be applied to
past/future releases

---------

Co-authored-by: Krzysztof Jamróz <[email protected]>
Co-authored-by: Vassilis Bekiaris <[email protected]>
GitOrigin-RevId: 8c01060ff2e7a4e161e0f2323fdb1b4356523412

hazelcast/src/main/java/com/hazelcast/client/impl/clientside/ClientDynamicClusterConfig.java

@@ -34,6 +34,7 @@
 import com.hazelcast.client.impl.protocol.codec.DynamicConfigAddScheduledExecutorConfigCodec;
 import com.hazelcast.client.impl.protocol.codec.DynamicConfigAddSetConfigCodec;
 import com.hazelcast.client.impl.protocol.codec.DynamicConfigAddTopicConfigCodec;
+import com.hazelcast.client.impl.protocol.codec.DynamicConfigAddVectorCollectionConfigCodec;
 import com.hazelcast.client.impl.protocol.codec.DynamicConfigAddWanReplicationConfigCodec;
 import com.hazelcast.client.impl.protocol.codec.holder.WanBatchPublisherConfigHolder;
 import com.hazelcast.client.impl.protocol.codec.holder.WanConsumerConfigHolder;
@@ -96,6 +97,7 @@
 import com.hazelcast.config.cp.CPSubsystemConfig;
 import com.hazelcast.config.rest.RestConfig;
 import com.hazelcast.config.tpc.TpcConfig;
+import com.hazelcast.config.vector.VectorCollectionConfig;
 import com.hazelcast.core.ManagedContext;
 import com.hazelcast.internal.config.DataPersistenceAndHotRestartMerger;
 import com.hazelcast.internal.config.ServicesConfig;
@@ -1196,6 +1198,31 @@ public Config setRestConfig(@Nonnull RestConfig restConfig) {
         throw new UnsupportedOperationException(UNSUPPORTED_ERROR_MESSAGE);
     }
 
+    @Override
+    public VectorCollectionConfig getVectorCollectionConfigOrNull(String name) {
+        throw new UnsupportedOperationException(UNSUPPORTED_ERROR_MESSAGE);
+    }
+
+    @Override
+    public Map<String, VectorCollectionConfig> getVectorCollectionConfigs() {
+        throw new UnsupportedOperationException(UNSUPPORTED_ERROR_MESSAGE);
+    }
+
+    @Override
+    @Nonnull
+    public Config addVectorCollectionConfig(@Nonnull VectorCollectionConfig vectorCollectionConfig) {
+        ClientMessage request = DynamicConfigAddVectorCollectionConfigCodec.encodeRequest(
+                vectorCollectionConfig.getName(),
+                vectorCollectionConfig.getVectorIndexConfigs());
+        invoke(request);
+        return this;
+    }
+
+    @Override
+    public Config setVectorCollectionConfigs(List<VectorCollectionConfig> vectorConfigs) {
+        throw new UnsupportedOperationException(UNSUPPORTED_ERROR_MESSAGE);
+    }
+
     @Override
     public String toString() {
         return "DynamicClusterConfig{instance=" + instance + "}";

hazelcast/src/main/java/com/hazelcast/client/impl/protocol/codec/DynamicConfigAddVectorCollectionConfigCodec.java

@@ -0,0 +1,97 @@
+/*
+ * Copyright (c) 2008-2024, Hazelcast, Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.hazelcast.client.impl.protocol.codec;
+
+import com.hazelcast.client.impl.protocol.ClientMessage;
+import com.hazelcast.client.impl.protocol.Generated;
+import com.hazelcast.client.impl.protocol.codec.builtin.*;
+import com.hazelcast.client.impl.protocol.codec.custom.*;
+
+import javax.annotation.Nullable;
+
+import static com.hazelcast.client.impl.protocol.ClientMessage.*;
+import static com.hazelcast.client.impl.protocol.codec.builtin.FixedSizeTypesCodec.*;
+
+/*
+ * This file is auto-generated by the Hazelcast Client Protocol Code Generator.
+ * To change this file, edit the templates or the protocol
+ * definitions on the https://github.com/hazelcast/hazelcast-client-protocol
+ * and regenerate it.
+ */
+
+/**
+ * Adds a new vector collection configuration to a running cluster.
+ */
+@SuppressWarnings("unused")
+@Generated("52a7d98dfd816d848ac7f4d5bb84d6d3")
+public final class DynamicConfigAddVectorCollectionConfigCodec {
+    //hex: 0x1B1400
+    public static final int REQUEST_MESSAGE_TYPE = 1774592;
+    //hex: 0x1B1401
+    public static final int RESPONSE_MESSAGE_TYPE = 1774593;
+    private static final int REQUEST_INITIAL_FRAME_SIZE = PARTITION_ID_FIELD_OFFSET + INT_SIZE_IN_BYTES;
+    private static final int RESPONSE_INITIAL_FRAME_SIZE = RESPONSE_BACKUP_ACKS_FIELD_OFFSET + BYTE_SIZE_IN_BYTES;
+
+    private DynamicConfigAddVectorCollectionConfigCodec() {
+    }
+
+    @edu.umd.cs.findbugs.annotations.SuppressFBWarnings({"URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD"})
+    public static class RequestParameters {
+
+        /**
+         * vector collection's name
+         */
+        public java.lang.String name;
+
+        /**
+         * item vector index configurations
+         */
+        public java.util.List<com.hazelcast.config.vector.VectorIndexConfig> indexConfigs;
+    }
+
+    public static ClientMessage encodeRequest(java.lang.String name, java.util.List<com.hazelcast.config.vector.VectorIndexConfig> indexConfigs) {
+        ClientMessage clientMessage = ClientMessage.createForEncode();
+        clientMessage.setRetryable(false);
+        clientMessage.setOperationName("DynamicConfig.AddVectorCollectionConfig");
+        ClientMessage.Frame initialFrame = new ClientMessage.Frame(new byte[REQUEST_INITIAL_FRAME_SIZE], UNFRAGMENTED_MESSAGE);
+        encodeInt(initialFrame.content, TYPE_FIELD_OFFSET, REQUEST_MESSAGE_TYPE);
+        encodeInt(initialFrame.content, PARTITION_ID_FIELD_OFFSET, -1);
+        clientMessage.add(initialFrame);
+        StringCodec.encode(clientMessage, name);
+        ListMultiFrameCodec.encode(clientMessage, indexConfigs, VectorIndexConfigCodec::encode);
+        return clientMessage;
+    }
+
+    public static DynamicConfigAddVectorCollectionConfigCodec.RequestParameters decodeRequest(ClientMessage clientMessage) {
+        ClientMessage.ForwardFrameIterator iterator = clientMessage.frameIterator();
+        RequestParameters request = new RequestParameters();
+        //empty initial frame
+        iterator.next();
+        request.name = StringCodec.decode(iterator);
+        request.indexConfigs = ListMultiFrameCodec.decode(iterator, VectorIndexConfigCodec::decode);
+        return request;
+    }
+
+    public static ClientMessage encodeResponse() {
+        ClientMessage clientMessage = ClientMessage.createForEncode();
+        ClientMessage.Frame initialFrame = new ClientMessage.Frame(new byte[RESPONSE_INITIAL_FRAME_SIZE], UNFRAGMENTED_MESSAGE);
+        encodeInt(initialFrame.content, TYPE_FIELD_OFFSET, RESPONSE_MESSAGE_TYPE);
+        clientMessage.add(initialFrame);
+
+        return clientMessage;
+    }
+}

hazelcast/src/main/java/com/hazelcast/client/impl/protocol/codec/builtin/CustomTypeFactory.java

@@ -33,6 +33,8 @@
 import com.hazelcast.config.MerkleTreeConfig;
 import com.hazelcast.config.NearCachePreloaderConfig;
 import com.hazelcast.config.TieredStoreConfig;
+import com.hazelcast.config.vector.Metric;
+import com.hazelcast.config.vector.VectorIndexConfig;
 import com.hazelcast.core.HazelcastException;
 import com.hazelcast.core.HazelcastJsonValue;
 import com.hazelcast.instance.EndpointQualifier;
@@ -329,4 +331,12 @@ public static JobAndSqlSummary createJobAndSqlSummary(
                 completionTime, failureText, sqlSummary, isSuspensionCauseExists ? suspensionCause : null,
                 isUserCancelledExists ? userCancelled : false);
     }
+
+    public static VectorIndexConfig createVectorIndexConfig(String name, int metric, int dimension) {
+        var vectorConfig = new VectorIndexConfig().setMetric(Metric.getById(metric)).setDimension(dimension);
+        if (name != null) {
+            vectorConfig.setName(name);
+        }
+        return vectorConfig;
+    }
 }

hazelcast/src/main/java/com/hazelcast/client/impl/protocol/codec/builtin/FixedSizeTypesCodec.java

@@ -20,6 +20,7 @@
 import com.hazelcast.config.BitmapIndexOptions.UniqueKeyTransformation;
 import com.hazelcast.config.CacheSimpleConfig.ExpiryPolicyFactoryConfig.TimedExpiryPolicyFactoryConfig.ExpiryPolicyType;
 import com.hazelcast.config.IndexType;
+import com.hazelcast.config.vector.Metric;
 import com.hazelcast.instance.ProtocolType;
 import com.hazelcast.internal.management.dto.ClientBwListEntryDTO;
 import com.hazelcast.internal.nio.Bits;
@@ -127,6 +128,10 @@ public static void encodeInt(byte[] buffer, int pos, JobStatus jobStatus) {
         encodeInt(buffer, pos, jobStatus.getId());
     }
 
+    public static void encodeInt(byte[] buffer, int pos, Metric metric) {
+        encodeInt(buffer, pos, metric.getId());
+    }
+
     public static void encodeShort(byte[] buffer, int pos, short value) {
         Bits.writeShortL(buffer, pos, value);
     }

hazelcast/src/main/java/com/hazelcast/client/impl/protocol/codec/custom/VectorIndexConfigCodec.java

@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2008-2024, Hazelcast, Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.hazelcast.client.impl.protocol.codec.custom;
+
+import com.hazelcast.client.impl.protocol.ClientMessage;
+import com.hazelcast.client.impl.protocol.Generated;
+import com.hazelcast.client.impl.protocol.codec.builtin.*;
+
+import static com.hazelcast.client.impl.protocol.codec.builtin.CodecUtil.fastForwardToEndFrame;
+import static com.hazelcast.client.impl.protocol.ClientMessage.*;
+import static com.hazelcast.client.impl.protocol.codec.builtin.FixedSizeTypesCodec.*;
+
+@SuppressWarnings("unused")
+@Generated("68fdb687656a1ea263358c750a00aa99")
+public final class VectorIndexConfigCodec {
+    private static final int METRIC_FIELD_OFFSET = 0;
+    private static final int DIMENSION_FIELD_OFFSET = METRIC_FIELD_OFFSET + INT_SIZE_IN_BYTES;
+    private static final int INITIAL_FRAME_SIZE = DIMENSION_FIELD_OFFSET + INT_SIZE_IN_BYTES;
+
+    private VectorIndexConfigCodec() {
+    }
+
+    public static void encode(ClientMessage clientMessage, com.hazelcast.config.vector.VectorIndexConfig vectorIndexConfig) {
+        clientMessage.add(BEGIN_FRAME.copy());
+
+        ClientMessage.Frame initialFrame = new ClientMessage.Frame(new byte[INITIAL_FRAME_SIZE]);
+        encodeInt(initialFrame.content, METRIC_FIELD_OFFSET, vectorIndexConfig.getMetric());
+        encodeInt(initialFrame.content, DIMENSION_FIELD_OFFSET, vectorIndexConfig.getDimension());
+        clientMessage.add(initialFrame);
+
+        CodecUtil.encodeNullable(clientMessage, vectorIndexConfig.getName(), StringCodec::encode);
+
+        clientMessage.add(END_FRAME.copy());
+    }
+
+    public static com.hazelcast.config.vector.VectorIndexConfig decode(ClientMessage.ForwardFrameIterator iterator) {
+        // begin frame
+        iterator.next();
+
+        ClientMessage.Frame initialFrame = iterator.next();
+        int metric = decodeInt(initialFrame.content, METRIC_FIELD_OFFSET);
+        int dimension = decodeInt(initialFrame.content, DIMENSION_FIELD_OFFSET);
+
+        java.lang.String name = CodecUtil.decodeNullable(iterator, StringCodec::decode);
+
+        fastForwardToEndFrame(iterator);
+
+        return CustomTypeFactory.createVectorIndexConfig(name, metric, dimension);
+    }
+}

hazelcast/src/main/java/com/hazelcast/config/Config.java

@@ -23,6 +23,7 @@
 import com.hazelcast.config.tpc.TpcConfig;
 import com.hazelcast.config.cp.CPSubsystemConfig;
 import com.hazelcast.config.matcher.MatchingPointConfigPatternMatcher;
+import com.hazelcast.config.vector.VectorCollectionConfig;
 import com.hazelcast.core.HazelcastInstance;
 import com.hazelcast.core.ManagedContext;
 import com.hazelcast.flakeidgen.FlakeIdGenerator;
@@ -82,6 +83,8 @@
 import java.util.Properties;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
+import java.util.function.Function;
+import java.util.stream.Collectors;
 
 import static com.hazelcast.config.LocalDeviceConfig.DEFAULT_DEVICE_NAME;
 import static com.hazelcast.internal.config.ConfigUtils.lookupByPattern;
@@ -210,6 +213,8 @@ public class Config {
 
     private SqlConfig sqlConfig = new SqlConfig();
 
+    private final Map<String, VectorCollectionConfig> vectorCollectionConfigs = new ConcurrentHashMap<>();
+
     private AuditlogConfig auditlogConfig = new AuditlogConfig();
 
     private MetricsConfig metricsConfig = new MetricsConfig();
@@ -3047,6 +3052,57 @@ public Config setSqlConfig(@Nonnull SqlConfig sqlConfig) {
         return this;
     }
 
+    /**
+     * Get vector collection config.
+     * If no configuration is found for the given name, returns null.
+     *
+     * @return vector collection config or {@code null}
+     * @since 5.5
+     */
+    public VectorCollectionConfig getVectorCollectionConfigOrNull(String name) {
+        return vectorCollectionConfigs.getOrDefault(name, null);
+    }
+
+    /**
+     * Retrieve configurations for all vector collections.
+     *
+     * @return a map where the key represents the name of the vector collection config
+     * and the value corresponds to the respective vector collection config.
+     * @since 5.5
+     */
+    public Map<String, VectorCollectionConfig> getVectorCollectionConfigs() {
+        return vectorCollectionConfigs;
+    }
+
+    /**
+     * Add vector collection config.
+     *
+     * @param vectorCollectionConfig the vector configuration to add
+     * @return this config instance
+     * @since 5.5
+     */
+    @Nonnull
+    public Config addVectorCollectionConfig(@Nonnull VectorCollectionConfig vectorCollectionConfig) {
+        Preconditions.checkNotNull(vectorCollectionConfig, "vectorCollectionConfig");
+        this.vectorCollectionConfigs.put(vectorCollectionConfig.getName(), vectorCollectionConfig);
+        return this;
+    }
+
+    /**
+     * Set the list of vector collection configurations.
+     *
+     * @param vectorConfigs the list of vector configuration to set
+     * @return this config instance
+     * @since 5.5
+     */
+    public Config setVectorCollectionConfigs(List<VectorCollectionConfig> vectorConfigs) {
+        this.vectorCollectionConfigs.clear();
+        this.vectorCollectionConfigs.putAll(
+                vectorConfigs.stream().collect(Collectors.toMap(VectorCollectionConfig::getName, Function.identity()))
+        );
+        return this;
+    }
+
     /**
      * Returns the configuration for tracking use of this Hazelcast instance.
      */

hazelcast/src/main/java/com/hazelcast/config/ConfigXmlGenerator.java

@@ -83,6 +83,7 @@
 import static com.hazelcast.internal.dynamicconfig.DynamicConfigXmlGenerator.scheduledExecutorXmlGenerator;
 import static com.hazelcast.internal.dynamicconfig.DynamicConfigXmlGenerator.setXmlGenerator;
 import static com.hazelcast.internal.dynamicconfig.DynamicConfigXmlGenerator.topicXmlGenerator;
+import static com.hazelcast.internal.dynamicconfig.DynamicConfigXmlGenerator.vectorCollectionXmlGenerator;
 import static com.hazelcast.internal.dynamicconfig.DynamicConfigXmlGenerator.wanReplicationXmlGenerator;
 import static com.hazelcast.internal.util.Preconditions.isNotNull;
 import static com.hazelcast.internal.util.StringUtil.isNullOrEmpty;
@@ -207,6 +208,7 @@ public String generate(Config config) {
         tpcConfiguration(gen, config);
         namespacesConfiguration(gen, config);
         restServerConfiguration(gen, config);
+        vectorCollectionXmlGenerator(gen, config);
         xml.append("</hazelcast>");
 
         String xmlString = xml.toString();