feat: support enable/disable mmap for index (#29005)

support enable/disable mmap for index, the user could alter the index's mode by `AlterIndex` method related: #21866 --------- Signed-off-by: yah01 <[email protected]> Signed-off-by: yah01 <[email protected]>
milvus-io · Dec 21, 2023 · a0e1a1e · a0e1a1e
1 parent 7a2374e
commit a0e1a1e
Show file tree

Hide file tree

Showing 30 changed files with 1,053 additions and 67 deletions.
diff --git a/go.mod b/go.mod
@@ -23,8 +23,7 @@ require (
 	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
 	github.com/klauspost/compress v1.16.7
 	github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d
-	github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20231208092431-02cbad30332f
-	github.com/milvus-io/milvus/pkg v0.0.1
+	github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20231211073628-ce99324c276c
 	github.com/minio/minio-go/v7 v7.0.61
 	github.com/prometheus/client_golang v1.14.0
 	github.com/prometheus/client_model v0.3.0
@@ -62,6 +61,7 @@ require github.com/apache/arrow/go/v12 v12.0.1
 require github.com/milvus-io/milvus-storage/go v0.0.0-20231109072809-1cd7b0866092
 
 require (
+	github.com/milvus-io/milvus/pkg v0.0.0-00010101000000-000000000000
 	github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81
 	github.com/quasilyte/go-ruleguard/dsl v0.3.22
 	golang.org/x/net v0.17.0

diff --git a/go.sum b/go.sum
@@ -581,10 +581,8 @@ github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/le
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b h1:TfeY0NxYxZzUfIfYe5qYDBzt4ZYRqzUjTR6CvUzjat8=
 github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b/go.mod h1:iwW+9cWfIzzDseEBCCeDSN5SD16Tidvy8cwQ7ZY8Qj4=
-github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20231114080011-9a495865219e h1:IH1WAXwEF8vbwahPdupi4zzRNWViT4B7fZzIjtRLpG4=
-github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20231114080011-9a495865219e/go.mod h1:1OIl0v5PQeNxIJhCvY+K55CBUOYDZevw9g9380u1Wek=
-github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20231208092431-02cbad30332f h1:0cAMN9OsgBxlEUY8i1e1ocrBZ/cpu/Kdguz4JWz9fUc=
-github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20231208092431-02cbad30332f/go.mod h1:1OIl0v5PQeNxIJhCvY+K55CBUOYDZevw9g9380u1Wek=
+github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20231211073628-ce99324c276c h1:Wbc2IZt/13+B5jc8JPU/dOxGYy+1jeOsChVgcza+qgw=
+github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20231211073628-ce99324c276c/go.mod h1:1OIl0v5PQeNxIJhCvY+K55CBUOYDZevw9g9380u1Wek=
 github.com/milvus-io/milvus-storage/go v0.0.0-20231109072809-1cd7b0866092 h1:UYJ7JB+QlMOoFHNdd8mUa3/lV63t9dnBX7ILXmEEWPY=
 github.com/milvus-io/milvus-storage/go v0.0.0-20231109072809-1cd7b0866092/go.mod h1:GPETMcTZq1gLY1WA6Na5kiNAKnq8SEMMiVKUZrM3sho=
 github.com/milvus-io/pulsar-client-go v0.6.10 h1:eqpJjU+/QX0iIhEo3nhOqMNXL+TyInAs1IAHZCrCM/A=

diff --git a/internal/datacoord/index_meta.go b/internal/datacoord/index_meta.go
@@ -18,6 +18,7 @@
 package datacoord
 
 import (
+	"context"
 	"fmt"
 	"strconv"
 
@@ -211,6 +212,22 @@ func (m *meta) CreateIndex(index *model.Index) error {
 	return nil
 }
 
+func (m *meta) AlterIndex(ctx context.Context, indexes ...*model.Index) error {
+	m.Lock()
+	defer m.Unlock()
+
+	err := m.catalog.AlterIndexes(ctx, indexes)
+	if err != nil {
+		return err
+	}
+
+	for _, index := range indexes {
+		m.updateCollectionIndex(index)
+	}
+
+	return nil
+}
+
 // AddSegmentIndex adds the index meta corresponding the indexBuildID to meta table.
 func (m *meta) AddSegmentIndex(segIndex *model.SegmentIndex) error {
 	m.Lock()

diff --git a/internal/datacoord/index_service.go b/internal/datacoord/index_service.go
@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/samber/lo"
 	"go.uber.org/zap"
 
 	"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
@@ -236,6 +237,51 @@ func (s *Server) CreateIndex(ctx context.Context, req *indexpb.CreateIndexReques
 	return merr.Success(), nil
 }
 
+func (s *Server) AlterIndex(ctx context.Context, req *indexpb.AlterIndexRequest) (*commonpb.Status, error) {
+	log := log.Ctx(ctx).With(
+		zap.Int64("collectionID", req.GetCollectionID()),
+		zap.String("indexName", req.GetIndexName()),
+	)
+	log.Info("received AlterIndex request", zap.Any("params", req.GetParams()))
+
+	if err := merr.CheckHealthy(s.GetStateCode()); err != nil {
+		log.Warn(msgDataCoordIsUnhealthy(paramtable.GetNodeID()), zap.Error(err))
+		return merr.Status(err), nil
+	}
+
+	indexes := s.meta.GetIndexesForCollection(req.GetCollectionID(), req.GetIndexName())
+	params := make(map[string]string)
+	for _, index := range indexes {
+		for _, param := range index.UserIndexParams {
+			params[param.GetKey()] = param.GetValue()
+		}
+
+		// update the index params
+		for _, param := range req.GetParams() {
+			params[param.GetKey()] = param.GetValue()
+		}
+
+		log.Info("prepare to alter index",
+			zap.String("indexName", index.IndexName),
+			zap.Any("params", params),
+		)
+		index.UserIndexParams = lo.MapToSlice(params, func(k string, v string) *commonpb.KeyValuePair {
+			return &commonpb.KeyValuePair{
+				Key:   k,
+				Value: v,
+			}
+		})
+	}
+
+	err := s.meta.AlterIndex(ctx, indexes...)
+	if err != nil {
+		log.Warn("failed to alter index", zap.Error(err))
+		return merr.Status(err), nil
+	}
+
+	return merr.Success(), nil
+}
+
 // GetIndexState gets the index state of the index name in the request from Proxy.
 // Deprecated
 func (s *Server) GetIndexState(ctx context.Context, req *indexpb.GetIndexStateRequest) (*indexpb.GetIndexStateResponse, error) {