Initial support for vector search.

.github/workflows/dotnet.yml

@@ -13,7 +13,7 @@ jobs:
 
     services:
       redis:
-        image: redislabs/redismod:preview
+        image: redis/redis-stack-server:latest
         ports:
           - 6379:6379
 
@@ -22,7 +22,7 @@ jobs:
     - name: Setup .NET
       uses: actions/setup-dotnet@v1
       with:
-        dotnet-version: 6.0.x
+        dotnet-version: 8.0.x
     - name: Restore dependencies
       run: dotnet restore
     - name: Build

.github/workflows/publish.yml

@@ -18,7 +18,7 @@ jobs:
     - name: Setup .NET
       uses: actions/setup-dotnet@v1
       with:
-        dotnet-version: 6.0.x
+        dotnet-version: 8.0.x
     - name: Clean
       run: dotnet clean ./RediSearchClient.sln --configuration Release && dotnet nuget locals all --clear
     - name: Restore dependencies

README.md

@@ -28,6 +28,7 @@ So here we are.
     - [Custom Dictionaries](#custom-dictionaries)
 * [Tag Values](#tag-values)
 * [Synonyms](#synonyms)
+* [Vector Support **\*NEW\***](https://github.com/tombatron/RediSearchClient/wiki/VectorSearchSupport)
 
 ## Installation
 
@@ -108,6 +109,46 @@ var indexDefintion = RediSearchIndex
       .Build();
 ```
 
+If you're using RediSearch as a vector database, you can now define a `VECTOR` field on hash-based and JSON-based indexes. 
+
+The following is an example of defining a `VECTOR` field on a hash-based index:
+
+```csharp
+var indexDefinition = RediSearchIndex
+    .OnHash()
+    .WithSchema(
+        x=> x.Text("Name"),
+        x=> x.Vector("Embedding", 
+            VectorIndexAlgorithm.FLAT(
+                type: VectorType.FLOAT32, 
+                dimensions: 32, 
+                distanceMetric: DistanceMetric.L2, 
+                initialCap: 30, 
+                blockSize: 20))
+    )
+    .Build();
+```
+
+Next is an example of defining a `VECTOR` field on a JSON-based index:
+
+```csharp
+var indexDefinition = RediSearchIndex
+    .OnJson()
+    .WithSchema(
+        x=> x.Text("$.Id", "Id"),
+        x=> x.Vector("$.Embedding", alias: "Embedded",
+            VectorIndexAlgorithm.FLAT(
+                type: VectorType.FLOAT64,
+                dimensions: 33,
+                distanceMetric: DistanceMetric.L2,
+                initialCap: 23,
+                blockSize: 22))
+    )
+    .Build();
+```
+
+Note, that while the above examples specify a "FLAT" vector index, the `VectorIndexAlgorithm` factory class will also allow you to define an HNSW index as well. 
+
 #### Dates and Times
 
 The only field types that RediSearch supports are "Text", "Tag", "Numeric", and "Geo". No dates and times.

RediSearchClient.IntegrationTests/BaseIntegrationTest.cs

@@ -1,87 +1,156 @@
-using System;
+using NReJSON;
+using RediSearchClient.Indexes;
 using StackExchange.Redis;
+using System;
+using System.Text.Json;
+using System.Threading;
 using static RediSearchClient.IntegrationTests.SampleData;
 
-namespace RediSearchClient.IntegrationTests
+namespace RediSearchClient.IntegrationTests;
+
+public abstract class BaseIntegrationTest : IDisposable
 {
-    public abstract class BaseIntegrationTest : IDisposable
+    private static bool HasIndexCleanupRun = false;
+
+    protected const string MovieDataPrefix = "movie::";
+
+    private ConnectionMultiplexer _muxr;
+    protected IDatabase _db;
+    protected string _indexName;
+    protected string _recordPrefix;
+    protected string _dictionaryName;
+    protected string _hashVectorIndexName;
+    protected string _jsonVectorIndexName;
+
+    protected virtual void Setup()
     {
-        private static bool HasIndexCleanupRun = false;
+        NReJSONSerializer.SerializerProxy = new SystemTextJsonSerializer();
 
-        protected const string MovieDataPrefix = "movie::";
+        _muxr = ConnectionMultiplexer.Connect("localhost");
 
-        private ConnectionMultiplexer _muxr;
-        protected IDatabase _db;
-        protected string _indexName;
-        protected string _recordPrefix;
-        protected string _dictionaryName;
+        _db = _muxr.GetDatabase(0);
 
-        protected virtual void Setup()
-        {
-            _muxr = ConnectionMultiplexer.Connect("localhost");
+        _indexName = Guid.NewGuid().ToString("n");
+        _recordPrefix = Guid.NewGuid().ToString("n");
+        _dictionaryName = Guid.NewGuid().ToString("n");
+        _hashVectorIndexName = Guid.NewGuid().ToString("n");
+        _jsonVectorIndexName = Guid.NewGuid().ToString("n");
 
-            _db = _muxr.GetDatabase(0);
+        SetupDemoMovieData();
+        SetupTestVectorData();
+    }
 
-            _indexName = Guid.NewGuid().ToString("n");
-            _recordPrefix = Guid.NewGuid().ToString("n");
-            _dictionaryName = Guid.NewGuid().ToString("n");
+    public virtual void TearDown()
+    {
+        //CleanupIndexes();
 
-            SetupDemoMovieData();
+        _muxr.Dispose();
+    }
 
-            CleanupIndexes();
-        }
+    public BaseIntegrationTest()
+    {
+        Setup();
+    }
 
-        public virtual void TearDown()
-        {
-            _muxr.Dispose();
-        }
+    public void Dispose()
+    {
+        TearDown();
+    }
 
-        public BaseIntegrationTest()
+    private void SetupDemoMovieData()
+    {
+        if (_db.KeyExists($"{MovieDataPrefix}1"))
         {
-            Setup();
+            // Movies are already in the database, bail. 
+            return;
         }
 
-        public void Dispose()
+        for (var i = 0; i < Movies.Length; i++)
         {
-            TearDown();
+            _db.HashSet($"{MovieDataPrefix}{i + 1}", Movies[i]);
         }
+    }
 
-        private void SetupDemoMovieData()
+    private void SetupTestVectorData()
+    {
+        foreach (var vec in SampleData.SampleVectorData)
         {
-            if (_db.KeyExists($"{MovieDataPrefix}1"))
+            _db.HashSet($"test_hash_vector:{vec.Name}", new[]
             {
-                // Movies are already in the database, bail. 
-                return;
-            }
+                new HashEntry("name", vec.Name),
+                new HashEntry("feature_embeddings", vec.FileBytes)
+            });
 
-            for (var i = 0; i < Movies.Length; i++)
+            _db.JsonSet($"test_json_vector:{vec.Name}", new
             {
-                _db.HashSet($"{MovieDataPrefix}{i + 1}", Movies[i]);
-            }
+                name = vec.Name,
+                feature_embeddings = vec.FileFloats
+            });
         }
 
-        private static object locker = new object();
+        // Create the test indexes. 
+        var hashIndex = RediSearchIndex
+            .OnHash()
+            .ForKeysWithPrefix("test_hash_vector:")
+            .WithSchema(
+                s => s.Text("name"),
+                s => s.Vector("feature_embeddings",
+                    VectorIndexAlgorithm.HNSW(
+                        type: VectorType.FLOAT32,
+                        dimensions: 512, // Used ResNet34 to generate feature embeddings...
+                        distanceMetric: DistanceMetric.COSINE
+                     ))
+                ).Build();
+
+        var jsonIndex = RediSearchIndex
+            .OnJson()
+            .ForKeysWithPrefix("test_hash_vector:")
+            .WithSchema(
+                s => s.Text("$.name", "name"),
+                s => s.Vector("$.feature_embeddings", "feature_embeddings",
+                    VectorIndexAlgorithm.HNSW(
+                        type: VectorType.FLOAT32,
+                        dimensions: 512, // Used ResNet34 to generate feature embeddings...
+                        distanceMetric: DistanceMetric.COSINE
+                     ))
+                ).Build();
+
+        _db.CreateIndex(_hashVectorIndexName, hashIndex);
+        _db.CreateIndex(_jsonVectorIndexName, jsonIndex);
+
+        Thread.Sleep(500);
+    }
 
-        private void CleanupIndexes()
+    private static object locker = new object();
+
+    private void CleanupIndexes()
+    {
+        if (!HasIndexCleanupRun)
         {
-            if (!HasIndexCleanupRun)
+            lock (locker)
             {
-                lock (locker)
+                if (!HasIndexCleanupRun)
                 {
-                    if (!HasIndexCleanupRun)
+                    foreach (var index in _db.ListIndexes())
                     {
-                        foreach (var index in _db.ListIndexes())
+                        if(Guid.TryParse(index, out var _))
                         {
-                            if(Guid.TryParse(index, out var _))
-                            {
-                                _db.DropIndex(index);
-                            }
+                            _db.DropIndex(index);
                         }
-
-                        HasIndexCleanupRun = true;
                     }
+
+                    HasIndexCleanupRun = true;
                 }
             }
         }
     }
+}
+
+public sealed class SystemTextJsonSerializer : ISerializerProxy
+{
+    public TResult Deserialize<TResult>(RedisResult serializedValue) =>
+        JsonSerializer.Deserialize<TResult>(serializedValue.ToString());
+
+    public string Serialize<TObjectType>(TObjectType obj) =>
+        JsonSerializer.Serialize(obj);
 }