Skip to content

Commit

Permalink
Merge pull request #2 from WayneCao/main
Browse files Browse the repository at this point in the history 
support filtering index && support array type
  • Loading branch information
@WayneCao
WayneCao authored Jan 7, 2025
2 parents 6d3b4cf + c7e9e6c commit aff1e23
Show file tree
Hide file tree
Showing 3 changed files with 263 additions and 23 deletions.
33 changes: 28 additions & 5 deletions example/example.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -116,9 +116,15 @@ func (m *MochowTest) createDatabaseAndTable() error {
},
{
FieldName: "segment",
FieldType: "TEXT",
FieldType: api.FieldTypeString,
NotNull: true,
},
{
FieldName: "arrField",
FieldType: api.FieldTypeArray,
ElementType: api.ElementTypeString,
NotNull: true,
},
}

// Indexes
Expand All @@ -130,6 +136,18 @@ func (m *MochowTest) createDatabaseAndTable() error {
Field: "bookName",
IndexType: api.SecondaryIndex,
},
{
IndexName: "compound_filtering_idx",
IndexType: api.FilteringIndex,
FilterIndexFields: []api.FilteringIndexField{
{
Field: "bookName",
},
{
Field: "page",
},
},
},
{
IndexName: "vector_idx",
Field: "vector",
Expand All @@ -143,10 +161,10 @@ func (m *MochowTest) createDatabaseAndTable() error {
AutoBuildPolicy: autoBuildPolicy.Params(),
},
{
IndexName: "book_segment_inverted_idx",
IndexType: api.InvertedIndex,
Fields: []string{"segment"},
FieldAttributes: []api.InvertedIndexFieldAttribute{api.Analyzed},
IndexName: "book_segment_inverted_idx",
IndexType: api.InvertedIndex,
InvertedIndexFields: []string{"segment"},
InvertedIndexFieldAttributes: []api.InvertedIndexFieldAttribute{api.Analyzed},
Params: api.InvertedIndexParams{
"analyzer": api.ChineseAnalyzer,
"parseMode": api.FineMode,
Expand Down Expand Up @@ -194,6 +212,7 @@ func (m *MochowTest) upsertData() error {
"author": "吴承恩",
"page": 21,
"vector": []float32{0.2123, 0.21, 0.213},
"arrField": []string{},
"segment": "富贵功名，前缘分定，为人切莫欺心。",
},
},
Expand All @@ -204,6 +223,7 @@ func (m *MochowTest) upsertData() error {
"author": "吴承恩",
"page": 22,
"vector": []float32{0.2123, 0.22, 0.213},
"arrField": []string{},
"segment": "正大光明，忠良善果弥深。些些狂妄天加谴，眼前不遇待时临。",
},
},
Expand All @@ -214,6 +234,7 @@ func (m *MochowTest) upsertData() error {
"author": "罗贯中",
"page": 23,
"vector": []float32{0.2123, 0.23, 0.213},
"arrField": []string{"细作", "吕布"},
"segment": "细作探知这个消息，飞报吕布。",
},
},
Expand All @@ -224,6 +245,7 @@ func (m *MochowTest) upsertData() error {
"author": "罗贯中",
"page": 24,
"vector": []float32{0.2123, 0.24, 0.213},
"arrField": []string{"吕布", "陈宫", "刘玄德"},
"segment": "布大惊，与陈宫商议。宫曰：“闻刘玄德新领徐州，可往投之。” 布从其言，竟投徐州来。有人报知玄德。",
},
},
Expand All @@ -234,6 +256,7 @@ func (m *MochowTest) upsertData() error {
"author": "罗贯中",
"page": 25,
"vector": []float32{0.2123, 0.24, 0.213},
"arrField": []string{"玄德", "吕布", "糜竺"},
"segment": "玄德曰：“布乃当今英勇之士，可出迎之。”糜竺曰：“吕布乃虎狼之徒，不可收留；收则伤人矣。",
},
},
Expand Down
214 changes: 196 additions & 18 deletions mochow/api/entity.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,13 +30,15 @@ type PartitionParams struct {
}

type FieldSchema struct {
FieldName string `json:"fieldName"`
FieldType FieldType `json:"fieldType"`
PrimaryKey bool `json:"primaryKey"`
PartitionKey bool `json:"partitionKey"`
AutoIncrement bool `json:"autoIncrement"`
NotNull bool `json:"notNull"`
Dimension uint32 `json:"dimension"`
FieldName string `json:"fieldName"`
FieldType FieldType `json:"fieldType"`
PrimaryKey bool `json:"primaryKey"`
PartitionKey bool `json:"partitionKey"`
AutoIncrement bool `json:"autoIncrement"`
NotNull bool `json:"notNull"`
Dimension uint32 `json:"dimension"`
ElementType ElementType `json:"elementType"`
MaxCapacity uint32 `json:"maxCapacity"`
}

func (f *FieldSchema) MarshalJSON() ([]byte, error) {
Expand All @@ -54,6 +56,14 @@ func (f *FieldSchema) MarshalJSON() ([]byte, error) {
fields["partitionKey"] = f.PartitionKey
fields["autoIncrement"] = f.AutoIncrement
fields["notNull"] = f.NotNull

if len(f.ElementType) > 0 {
fields["elementType"] = f.ElementType
}
if f.MaxCapacity != 0 {
fields["maxCapacity"] = f.MaxCapacity
}

field, err := sonic.Marshal(fields)
if err != nil {
return nil, err
Expand All @@ -64,20 +74,186 @@ func (f *FieldSchema) MarshalJSON() ([]byte, error) {
type IndexParams interface{}
type VectorIndexParams map[string]interface{}
type InvertedIndexParams map[string]interface{}

type AutoBuildParams map[string]interface{}

type FilteringIndexField struct {
Field string `json:"field"`
IndexStructureType IndexStructureType `json:"indexStructureType"`
}

func (f *FilteringIndexField) FromMapInterface(i interface{}) error {
var ok bool
switch v := i.(type) {
case map[string]interface{}:
if field, exist := v["field"]; exist {
if f.Field, ok = field.(string); !ok {
return fmt.Errorf("field should be string")
}
}
if indexStructureType, exist := v["indexStructureType"]; exist {
var structureTypeStr string
if structureTypeStr, ok = indexStructureType.(string); !ok {
return fmt.Errorf("invalid indexStructureType enum value")
}
f.IndexStructureType = IndexStructureType(structureTypeStr)
}
}
return nil
}

type IndexSchema struct {
IndexName string `json:"indexName,omitempty"`
IndexType IndexType `json:"indexType,omitempty"`
MetricType MetricType `json:"metricType,omitempty"`
Params IndexParams `json:"params,omitempty"`
Field string `json:"field,omitempty"`
Fields []string `json:"fields,omitempty"`
FieldAttributes []InvertedIndexFieldAttribute `json:"fieldsIndexAttributes,omitempty"`
State IndexState `json:"state,omitempty"`
AutoBuild bool `json:"autoBuild,omitempty"`
AutoBuildPolicy AutoBuildParams `json:"autoBuildPolicy,omitempty"`
IndexName string
IndexType IndexType
MetricType MetricType
Params IndexParams
Field string
InvertedIndexFields []string // for inverted index
InvertedIndexFieldAttributes []InvertedIndexFieldAttribute // for inverted index
FilterIndexFields []FilteringIndexField // for filtering index
State IndexState
AutoBuild bool
AutoBuildPolicy AutoBuildParams
}

func (index IndexSchema) MarshalJSON() ([]byte, error) {
params := make(map[string]interface{})
// index name
if len(index.IndexName) > 0 {
params["indexName"] = index.IndexName
}
// index type
params["indexType"] = index.IndexType
// metric type
params["metricType"] = index.MetricType
// params
if index.Params != nil {
params["params"] = index.Params
}
// auto build
params["autoBuild"] = index.AutoBuild
params["autoBuildPolicy"] = index.AutoBuildPolicy
params["state"] = index.State

// vector index and secondary index field
if len(index.Field) > 0 {
params["field"] = index.Field
}

// handle conflicted index fields
switch index.IndexType {
case InvertedIndex:
// inverted index fields
if len(index.InvertedIndexFields) > 0 {
params["fields"] = index.InvertedIndexFields
}
// inverted index field attributes
if len(index.InvertedIndexFieldAttributes) > 0 {
params["fieldsIndexAttributes"] = index.InvertedIndexFieldAttributes
}
case FilteringIndex:
// filtering index fields
if len(index.FilterIndexFields) > 0 {
params["fields"] = index.FilterIndexFields
}
}
return sonic.Marshal(&params)
}

func (index *IndexSchema) UnmarshalJSON(data []byte) error {
ds := decoder.NewStreamDecoder(bytes.NewReader(data))
ds.UseNumber()
params := make(map[string]interface{})
err := ds.Decode(&params)
if err != nil {
return err
}

// index name
var ok bool
if indexName, exist := params["indexName"]; exist {
if index.IndexName, ok = indexName.(string); !ok {
return fmt.Errorf("indexName should be string")
}
}
// index type
if indexType, exist := params["indexType"]; exist {
var indexTypeStr string
if indexTypeStr, ok = indexType.(string); !ok {
return fmt.Errorf("indexType should be string")
}
index.IndexType = IndexType(indexTypeStr)
}
// metric type
if metricType, exist := params["metricType"]; exist {
var metricTypeStr string
if metricTypeStr, ok = metricType.(string); !ok {
return fmt.Errorf("metricType should be string")
}
index.MetricType = MetricType(metricTypeStr)
}
// index params
if indexParams, exist := params["params"]; exist {
index.Params = indexParams
}
// field
if indexField, exist := params["field"]; exist {
if index.Field, ok = indexField.(string); !ok {
return fmt.Errorf("field should be string")
}
}
// auto build
if autoBuild, exist := params["autoBuild"]; exist {
if index.AutoBuild, ok = autoBuild.(bool); !ok {
return fmt.Errorf("autoBuild should be bool")
}
}
if autoBuildPolicy, exist := params["autoBuildPolicy"]; exist {
if index.AutoBuildPolicy, ok = autoBuildPolicy.(map[string]interface{}); !ok {
return fmt.Errorf("invalid autoBuildPolicy")
}
}
// index state
if state, exist := params["state"]; exist {
index.State = IndexState(state.(string))
}
switch index.IndexType {
case InvertedIndex:
// inverted index fields
if fields, ok := params["fields"]; ok {
switch v := fields.(type) {
case []interface{}:
index.InvertedIndexFields = make([]string, len(v))
for i := 0; i < len(v); i++ {
index.InvertedIndexFields[i] = v[i].(string)
}
}
}
if attributes, ok := params["fieldsIndexAttributes"]; ok {
switch v := attributes.(type) {
case []interface{}:
index.InvertedIndexFieldAttributes = make([]InvertedIndexFieldAttribute, len(v))
for i := 0; i < len(v); i++ {
index.InvertedIndexFieldAttributes[i] = InvertedIndexFieldAttribute(v[i].(string))
}
}
}
case FilteringIndex:
// fields for filtering index
if fields, ok := params["fields"]; ok {
switch t := fields.(type) {
case []interface{}:
index.FilterIndexFields = make([]FilteringIndexField, len(t))
for i := 0; i < len(t); i++ {
var filteringIndexField FilteringIndexField
if err := filteringIndexField.FromMapInterface(t[i]); err != nil {
return err
}
index.FilterIndexFields[i] = filteringIndexField
}
}
}
}
return nil
}

type TableSchema struct {
Expand Down Expand Up @@ -851,3 +1027,5 @@ func NewAutoBuildIncrementPolicy() *AutoBuildIncrementPolicy {
baseAutoBuildPolicy: newBaseAutoBuildPolicy(AutoBuildPolicyIncrement),
}
}

//
39 changes: 39 additions & 0 deletions mochow/api/enum.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,9 @@ const (
// scalar index type
SecondaryIndex IndexType = "SECONDARY"

// filtering index type
FilteringIndex IndexType = "FILTERING"

// inverted index type
InvertedIndex IndexType = "INVERTED"
)
Expand Down Expand Up @@ -89,6 +92,34 @@ const (

// vector field type
FieldTypeFloatVector FieldType = "FLOAT_VECTOR"

// array field type
FieldTypeArray FieldType = "ARRAY"
)

type ElementType string

const (
ElementTypeBool ElementType = "BOOL"
ElementTypeInt8 ElementType = "INT8"
ElementTypeUint8 ElementType = "UINT8"
ElementTypeInt16 ElementType = "INT16"
ElementTypeUint16 ElementType = "UINT16"
ElementTypeInt32 ElementType = "INT32"
ElementTypeUint32 ElementType = "UINT32"
ElementTypeInt64 ElementType = "INT64"
ElementTypeUint64 ElementType = "UINT64"
ElementTypeFloat ElementType = "FLOAT"
ElementTypeDouble ElementType = "DOUBLE"
ElementTypeDate ElementType = "DATE"
ElementTypeDatetime ElementType = "DATETIME"
ElementTypeTimestamp ElementType = "TIMESTAMP"
ElementTypeString ElementType = "STRING"
ElementTypeBinary ElementType = "BINARY"
ElementTypeUUID ElementType = "UUID"
ElementTypeText ElementType = "TEXT"
ElementTypeTextGBK ElementType = "TEXT_GBK"
ElementTypeTextGB18030 ElementType = "TEXT_GB18030"
)

type AutoBuildPolicyType string
Expand Down Expand Up @@ -123,10 +154,18 @@ const (
type IndexState string

const (
IndexStateInvalid IndexState = "INVALID"
IndexStateBuilding IndexState = "BUILDING"
IndexStateNormal IndexState = "NORMAL"
)

type IndexStructureType string

const (
IndexStructureTypeDefault IndexStructureType = "DEFAULT"
IndexStructureTypeBitmap IndexStructureType = "BITMAP"
)

type ServerErrCode int32

const (
Expand Down

0 comments on commit aff1e23

Please sign in to comment.