Skip to content

Commit

Permalink
Merge pull request #16 from varfrog/feat/supportSlices
Browse files Browse the repository at this point in the history
  • Loading branch information
@varfrog
varfrog authored Mar 11, 2024
2 parents bcb2255 + ed7f7f9 commit bb36be7
Show file tree
Hide file tree
Showing 3 changed files with 141 additions and 63 deletions.
89 changes: 44 additions & 45 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ func main() {
}
type person struct {
Name string
Aliases []string
Email string `opensearch:"type:keyword"`
DOB opensearchutil.TimeBasicDateTimeNoMillis
Age uint8
Expand Down Expand Up @@ -71,6 +72,9 @@ Output:
"age": {
"type": "integer"
},
"aliases": {
"type": "text"
},
"dob": {
"format": "basic_date_time_no_millis",
"type": "date"
Expand Down Expand Up @@ -129,46 +133,58 @@ The resulting JSON contents is then used in a request to the [Create index API r
package main

import (
"fmt"
"github.com/varfrog/opensearchutil"
"os"
"fmt"
"github.com/varfrog/opensearchutil"
"os"
)

func main() {
type person struct {
Name string
Email string `opensearch:"type:keyword"`
}
type address struct {
streetName string
}

type person struct {
Name string
Email string `opensearch:"type:keyword"`
Addresses []address
}

mappingPropertiesBuilder := opensearchutil.NewMappingPropertiesBuilder()
generator := opensearchutil.NewIndexGenerator()
mappingPropertiesBuilder := opensearchutil.NewMappingPropertiesBuilder()
generator := opensearchutil.NewIndexGenerator()

mappingProperties, err := mappingPropertiesBuilder.BuildMappingProperties(person{})
if err != nil {
fmt.Printf("BuildMappingProperties: %v", err)
os.Exit(1)
}
mappingProperties, err := mappingPropertiesBuilder.BuildMappingProperties(person{})
if err != nil {
fmt.Printf("BuildMappingProperties: %v", err)
os.Exit(1)
}

indexJson, err := generator.GenerateMappingsJson(mappingProperties)
if err != nil {
fmt.Printf("GenerateMappingsJson: %v", err)
os.Exit(1)
}
fmt.Printf("%s\n", string(indexJson))
indexJson, err := generator.GenerateMappingsJson(mappingProperties)
if err != nil {
fmt.Printf("GenerateMappingsJson: %v", err)
os.Exit(1)
}
fmt.Printf("%s\n", string(indexJson))
}
```

Output:
```json
{
"properties": {
"email": {
"type": "keyword"
},
"name": {
"type": "text"
{
"properties": {
"addresses": {
"properties": {
"street_name": {
"type": "text"
}
}
}
},
"email": {
"type": "keyword"
},
"name": {
"type": "text"
}
}
}
```

Expand Down Expand Up @@ -253,20 +269,3 @@ Document body:
"date_c": "20230223T224633.808+02:00"
}
```

## Best Practice

I recommend using **separate structs for generating index mappings and indexing documents**. This allows to address the issue of storing an array of some value. For example, if a User has multiple Address objects, in code we need to have a slice of these, i.e. `[]Address`. But in the mapping we need only to define a mapping of the `object` itself, not of its array, so in the mapping it's just `Address`.

For example,
```go
// UserIndexMapping is used to generate index mappings.
type UserIndexMapping struct {
HomeAddresses Address // No slice
}

// User is used to index documents and to unmarshal into this struct when retrieving them.
type User struct {
HomeAddresses []Address // Slice
}
```
48 changes: 35 additions & 13 deletions mapping_properties_builder.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ func (b *MappingPropertiesBuilder) doBuildMappingProperties(
tField := t.Field(i)
fieldName := tField.Name
resolvedField := b.resolveField(tField, v.Field(i))
resolvedField = b.unslice(resolvedField)

if err := validateField(resolvedField); err != nil {
return nil, errors.Wrapf(err, "validateField")
Expand Down Expand Up @@ -100,13 +101,6 @@ func (b *MappingPropertiesBuilder) doBuildMappingProperties(
})
continue
} else if !b.optionContainer.omitUnsupportedTypes {
if resolvedField.kind == reflect.Slice {
return nil, fmt.Errorf(
"slices are not supported (field '%s'), please use just the object and not its slice, you will still "+
"be able to index an array of objects in that field",
resolvedField.field.Name)
}

return nil, fmt.Errorf(
"field not supported: %s, please use opensearchutil.OmitUnsupportedTypes to skip"+
" fields of unsupported types",
Expand All @@ -116,7 +110,7 @@ func (b *MappingPropertiesBuilder) doBuildMappingProperties(
return mappingProperties, nil
}

func (b *MappingPropertiesBuilder) addProperties(resolvedField fieldWrapper, mappingProperty *MappingProperty) error {
func (b *MappingPropertiesBuilder) addProperties(resolvedField *fieldWrapper, mappingProperty *MappingProperty) error {
indexPrefixes := getTagOptionValue(resolvedField.field, tagKey, tagOptionIndexPrefixes)
if indexPrefixes != "" {
opts := parseCustomPropertyValue(indexPrefixes)
Expand All @@ -134,7 +128,7 @@ func (b *MappingPropertiesBuilder) addProperties(resolvedField fieldWrapper, map
return nil
}

func validateField(field fieldWrapper) error {
func validateField(field *fieldWrapper) error {
if field.kind == reflect.Struct {
switch field.value.Interface().(type) {
case time.Time:
Expand All @@ -144,7 +138,7 @@ func validateField(field fieldWrapper) error {
return nil
}

func (b *MappingPropertiesBuilder) resolveFieldType(field fieldWrapper) (string, error) {
func (b *MappingPropertiesBuilder) resolveFieldType(field *fieldWrapper) (string, error) {
fieldTypeOverride := getTagOptionValue(field.field, tagKey, tagOptionType)
if fieldTypeOverride != "" {
return fieldTypeOverride, nil
Expand All @@ -162,7 +156,7 @@ func (b *MappingPropertiesBuilder) resolveFieldType(field fieldWrapper) (string,
return "", nil
}

func (b *MappingPropertiesBuilder) resolveFieldFormat(field fieldWrapper) (*string, error) {
func (b *MappingPropertiesBuilder) resolveFieldFormat(field *fieldWrapper) (*string, error) {
fieldFormatOverride := getTagOptionValue(field.field, tagKey, tagOptionFormat)
if fieldFormatOverride != "" {
return &fieldFormatOverride, nil
Expand All @@ -179,7 +173,7 @@ func (b *MappingPropertiesBuilder) resolveFieldFormat(field fieldWrapper) (*stri

// resolveField returns a wrapper object for the given field. If the field is a pointer, it returns a wrapper
// for the dereferenced field, since we treat both pointer and value fields the same.
func (b *MappingPropertiesBuilder) resolveField(structField reflect.StructField, value reflect.Value) fieldWrapper {
func (b *MappingPropertiesBuilder) resolveField(structField reflect.StructField, value reflect.Value) *fieldWrapper {
var kind reflect.Kind
var val reflect.Value
if structField.Type.Kind() == reflect.Ptr {
Expand All @@ -190,14 +184,42 @@ func (b *MappingPropertiesBuilder) resolveField(structField reflect.StructField,
val = value
}

return fieldWrapper{
return &fieldWrapper{
field: structField,
kind: kind,
value: val,
isPrimitive: b.isPrimitive(kind),
}
}

// unslice "un-slices" the field by resolving the underlying element type. I.e. if it's a slice of struct Foo
// the returned object contains reflection objects for just Foo (not its slice).
func (b *MappingPropertiesBuilder) unslice(wrapper *fieldWrapper) *fieldWrapper {
if wrapper.kind != reflect.Slice {
return wrapper
}

elemType := wrapper.field.Type.Elem()
var (
newKind reflect.Kind
newVal reflect.Value
)
if elemType.Kind() == reflect.Ptr {
newKind = elemType.Elem().Kind()
newVal = reflect.New(elemType.Elem()).Elem()
} else {
newKind = elemType.Kind()
newVal = reflect.New(elemType).Elem()
}

return &fieldWrapper{
field: wrapper.field,
kind: newKind,
value: newVal,
isPrimitive: b.isPrimitive(newKind),
}
}

func (b *MappingPropertiesBuilder) isPrimitive(kind reflect.Kind) bool {
switch kind {
case reflect.Bool,
Expand Down
67 changes: 62 additions & 5 deletions mapping_properties_builder_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -262,18 +262,75 @@ func TestMappingPropertiesBuilder_BuildMappingProperties_DoesNotErrorWithUnsuppo
g.Expect(err).To(gomega.BeNil())
}

func TestMappingPropertiesBuilder_BuildMappingProperties_ErrorsSpecificErrorForSlices(t *testing.T) {
func TestMappingPropertiesBuilder_BuildMappingProperties_SupportsObjectSlices(t *testing.T) {
g := gomega.NewGomegaWithT(t)

type location struct {
city string
}
type person struct {
addresses []location // no need for slices, can just use the object in the mapping and still store arrays
addresses []location
}

builder := NewMappingPropertiesBuilder()
_, err := builder.BuildMappingProperties(person{})
g.Expect(err).ToNot(gomega.BeNil())
g.Expect(err.Error()).To(gomega.ContainSubstring("slices are not supported"))
mps, err := builder.BuildMappingProperties(person{})
g.Expect(err).ToNot(gomega.HaveOccurred())
g.Expect(mps).To(gomega.HaveLen(1))
g.Expect(mps[0]).To(gomega.Equal(MappingProperty{
FieldName: "addresses",
Children: []MappingProperty{
{
FieldName: "city",
FieldType: "text",
},
},
}))
}

func TestMappingPropertiesBuilder_BuildMappingProperties_SupportsPrimitiveSlices(t *testing.T) {
g := gomega.NewGomegaWithT(t)

type person struct {
names []string
}

builder := NewMappingPropertiesBuilder()
mps, err := builder.BuildMappingProperties(person{})
g.Expect(err).ToNot(gomega.HaveOccurred())
g.Expect(mps).To(gomega.HaveLen(1))
g.Expect(mps[0]).To(gomega.Equal(MappingProperty{
FieldName: "names",
FieldType: "text",
}))
}

func TestMappingPropertiesBuilder_BuildMappingProperties_SliceRecursiveMaxDepth(t *testing.T) {
g := gomega.NewGomegaWithT(t)

type person struct {
siblings []person
}

const depth = 3

builder := NewMappingPropertiesBuilder(WithMaxDepth(depth))

expectedMappingProperties := []MappingProperty{ // Level 1
{
FieldName: "siblings",
Children: []MappingProperty{ // Level 2
{
FieldName: "siblings",
Children: nil, // Level 3, nothing to map, recursion stopped
},
},
},
}

mps, err := builder.BuildMappingProperties(person{})
g.Expect(err).ToNot(gomega.HaveOccurred())
g.Expect(mps).To(gomega.Equal(expectedMappingProperties))
for _, mp := range mps {
g.Expect(mp.GetDepth() <= depth).To(gomega.BeTrue())
}
}

0 comments on commit bb36be7

Please sign in to comment.