diff --git a/internal/search/doc/doc.go b/internal/search/doc/doc.go index 5a87c184db..38aa6505bf 100644 --- a/internal/search/doc/doc.go +++ b/internal/search/doc/doc.go @@ -6,117 +6,53 @@ import ( "time" "sigs.k8s.io/yaml" - - "google.golang.org/appengine/search" -) - -const ( - identifierStr = "identifier" - documentStr = "document" - repoURLStr = "repo_url" - filePathStr = "file_path" - creationTimeStr = "creation_time" ) -// Represents an unbreakable character stream. -type Atom = search.Atom - -// Implements search.FieldLoadSaver in order to index this representation of a kustomization.yaml -// file. +// This document is meant to be used at the elasticsearch document type. +// Fields are serialized as-is to elasticsearch, where indices are built +// to facilitate text search queries. Identifiers, Values, FilePath, +// RepositoryURL and DocumentData are meant to be searched for text queries +// directly, while the other fields can either be used as a filter, or as +// additional metadata displayed in the UI. +// +// The fields of the document and their purpose are listed below: +// - DocumentData contains the contents of the kustomization file. +// - Kinds Represents the kubernetes Kinds that are in this file. +// - Identifiers are a list of (partial and full) identifier paths that can be +// found by users. Each part of a path is delimited by ":" e.g. spec:replicas. +// - Values are a list of identifier paths and their values that can be found by +// search queries. The path is delimited by ":" and the value follows the "=" +// symbol e.g. spec:replicas=4. +// - FilePath is the path of the file. +// - RepositoryURL is the URL of the source repository. +// - CreationTime is the time at which the file was created. +// +// Representing each Identifier and Value as a flat string representation +// facilitates the use of complex text search features from elasticsearch such +// as fuzzy searching, regex, wildcards, etc. type KustomizationDocument struct { - identifiers []Atom - FilePath Atom - RepositoryURL Atom - DocumentData string - CreationTime time.Time + DocumentData string `json:"document,omitempty"` + Kinds []string `json:"kinds,omitempty"` + Identifiers []string `json:"identifiers,omitempty"` + Values []string `json:"values,omitempty"` + FilePath string `json:"filePath,omitempty"` + RepositoryURL string `json:"repositoryUrl,omitempty"` + CreationTime time.Time `json:"creationTime,omitempty"` } -// Partially implements search.FieldLoadSaver. -func (k *KustomizationDocument) Load(fields []search.Field, metadata *search.DocumentMetadata) error { - k.identifiers = make([]search.Atom, 0) - wrongTypeError := func(name string, expected interface{}, actual interface{}) error { - return fmt.Errorf("%s expects type %T, found %#v", name, expected, actual) - } - - for _, f := range fields { - switch f.Name { - case identifierStr: - identifier, ok := f.Value.(search.Atom) - if !ok { - return wrongTypeError(f.Name, identifier, f.Value) - } - k.identifiers = append(k.identifiers, identifier) - - case documentStr: - document, ok := f.Value.(string) - if !ok { - return wrongTypeError(f.Name, document, f.Value) - } - k.DocumentData = document - - case filePathStr: - fp, ok := f.Value.(search.Atom) - if !ok { - return wrongTypeError(f.Name, fp, f.Value) - } - k.FilePath = fp - - case repoURLStr: - url, ok := f.Value.(search.Atom) - if !ok { - return wrongTypeError(f.Name, url, f.Value) - } - k.RepositoryURL = url - - case creationTimeStr: - time, ok := f.Value.(time.Time) - if !ok { - return wrongTypeError(f.Name, time, f.Value) - } - k.CreationTime = time - default: - return fmt.Errorf("KustomizationDocument field %s not recognized", f.Name) - } - } - - return nil -} - -// Partially implements search.FieldLoadSaver. -func (k *KustomizationDocument) Save() ([]search.Field, *search.DocumentMetadata, error) { - err := k.ParseYAML() - if err != nil { - return nil, nil, err - } - - extraFields := []search.Field{ - {Name: documentStr, Value: k.DocumentData}, - {Name: filePathStr, Value: k.FilePath}, - {Name: repoURLStr, Value: k.RepositoryURL}, - {Name: creationTimeStr, Value: k.CreationTime}, - } - - fields := make([]search.Field, 0, len(k.identifiers)+len(extraFields)) - for _, identifier := range k.identifiers { - fields = append(fields, search.Field{Name: identifierStr, Value: identifier}) - } - fields = append(fields, extraFields...) - - return fields, nil, nil -} - -func (k *KustomizationDocument) ParseYAML() error { - k.identifiers = make([]Atom, 0) +func (doc *KustomizationDocument) ParseYAML() error { + doc.Identifiers = make([]string, 0) + doc.Values = make([]string, 0) var kustomization map[string]interface{} - err := yaml.Unmarshal([]byte(k.DocumentData), &kustomization) + err := yaml.Unmarshal([]byte(doc.DocumentData), &kustomization) if err != nil { return fmt.Errorf("unable to parse kustomization file: %s", err) } type Map struct { data map[string]interface{} - prefix Atom + prefix string } toVisit := []Map{ @@ -126,43 +62,53 @@ func (k *KustomizationDocument) ParseYAML() error { }, } - atomJoin := func(vals ...interface{}) Atom { - strs := make([]string, 0, len(vals)) - for _, val := range vals { - strs = append(strs, fmt.Sprint(val)) - } - return Atom(strings.Trim(strings.Join(strs, " "), " ")) - } - - set := make(map[Atom]struct{}) - + identifierSet := make(map[string]struct{}) + valueSet := make(map[string]struct{}) for i := 0; i < len(toVisit); i++ { visiting := toVisit[i] for k, v := range visiting.data { - set[atomJoin(visiting.prefix, k)] = struct{}{} - switch value := v.(type) { - case map[string]interface{}: - toVisit = append(toVisit, Map{ - data: value, - prefix: atomJoin(visiting.prefix, fmt.Sprint(k)), - }) - case []interface{}: - for _, val := range value { - submap, ok := val.(map[string]interface{}) - if !ok { - continue - } + identifier := fmt.Sprintf("%s:%s", visiting.prefix, + strings.Replace(k, ":", "%3A", -1)) + // noop after the first iteration. + identifier = strings.TrimLeft(identifier, ":") + + // Recursive function traverses structure to find + // identifiers and values. These later get formatted + // into doc.Identifiers and doc.Values respectively. + var traverseStructure func(interface{}) + traverseStructure = func(arg interface{}) { + switch value := arg.(type) { + case map[string]interface{}: toVisit = append(toVisit, Map{ - data: submap, - prefix: atomJoin(visiting.prefix, fmt.Sprint(k)), + data: value, + prefix: identifier, }) + case []interface{}: + for _, val := range value { + traverseStructure(val) + } + case interface{}: + esc := strings.Replace(fmt.Sprintf("%v", + value), ":", "%3A", -1) + + valuePath := fmt.Sprintf("%s=%v", + identifier, esc) + valueSet[valuePath] = struct{}{} } } + traverseStructure(v) + + identifierSet[identifier] = struct{}{} + } } - for key := range set { - k.identifiers = append(k.identifiers, key) + for val := range valueSet { + doc.Values = append(doc.Values, val) + } + + for key := range identifierSet { + doc.Identifiers = append(doc.Identifiers, key) } return nil diff --git a/internal/search/doc/doc_test.go b/internal/search/doc/doc_test.go index a28ef68146..919d2bd94e 100644 --- a/internal/search/doc/doc_test.go +++ b/internal/search/doc/doc_test.go @@ -1,82 +1,30 @@ package doc import ( - "fmt" "reflect" "sort" "strings" "testing" - "time" - - "google.golang.org/appengine/search" ) -func TestLoadFailures(t *testing.T) { - type sentinelType struct{} - sentinel := sentinelType{} - - testCases := [][]search.Field{ - {{Name: identifierStr, Value: sentinel}}, - {{Name: documentStr, Value: sentinel}}, - {{Name: repoURLStr, Value: sentinel}}, - {{Name: filePathStr, Value: sentinel}}, - {{Name: creationTimeStr, Value: sentinel}}, - } - - for _, test := range testCases { - var k KustomizationDocument - err := k.Load(test, nil) - if err == nil { - t.Errorf("Type missmatch %#v should not be loadable", test) - } - } -} - -func TestFieldLoadSaver(t *testing.T) { - - commonTestCases := []KustomizationDocument{ - { - identifiers: []Atom{"namePrefix", "metadata.name", "kind"}, - FilePath: "some/path/kustomization.yaml", - RepositoryURL: "https://example.com/kustomize", - CreationTime: time.Now(), - DocumentData: ` -namePrefix: dev- -metadata: - name: app -kind: Deployment -`, - }, - } - - for _, test := range commonTestCases { - fields, metadata, err := test.Save() - if err != nil { - t.Errorf("Error calling Save(): %s\n", err) - } - doc := KustomizationDocument{} - err = doc.Load(fields, metadata) - if err != nil { - t.Errorf("Doc failed to load: %s\n", err) - } - if !reflect.DeepEqual(test, doc) { - t.Errorf("Expected loaded document (%+v) to be equal to (%+v)\n", doc, test) - } - } -} - func TestParseYAML(t *testing.T) { testCases := []struct { - identifiers []Atom + identifiers []string + values []string yaml string }{ { - identifiers: []Atom{ + identifiers: []string{ "namePrefix", "metadata", - "metadata name", + "metadata:name", "kind", }, + values: []string{ + "namePrefix=dev-", + "metadata:name=app", + "kind=Deployment", + }, yaml: ` namePrefix: dev- metadata: @@ -85,18 +33,29 @@ kind: Deployment `, }, { - identifiers: []Atom{ + identifiers: []string{ "namePrefix", "metadata", - "metadata name", - "metadata spec", - "metadata spec replicas", + "metadata:name", + "metadata:spec", + "metadata:spec:replicas", "kind", "replicas", - "replicas name", - "replicas count", + "replicas:name", + "replicas:count", "resource", }, + values: []string{ + "namePrefix=dev-", + "metadata:name=n1", + "metadata:spec:replicas=3", + "kind=Deployment", + "replicas:name=n1", + "replicas:name=n2", + "replicas:count=3", + "resource=file1.yaml", + "resource=file2.yaml", + }, yaml: ` namePrefix: dev- # map of map @@ -121,14 +80,6 @@ resource: }, } - atomStrs := func(atoms []Atom) []string { - strs := make([]string, 0, len(atoms)) - for _, val := range atoms { - strs = append(strs, fmt.Sprintf("%v", val)) - } - return strs - } - for _, test := range testCases { doc := KustomizationDocument{ DocumentData: test.yaml, @@ -140,14 +91,20 @@ resource: t.Errorf("Document error error: %s", err) } - docIDs := atomStrs(doc.identifiers) - expectedIDs := atomStrs(test.identifiers) - sort.Strings(docIDs) - sort.Strings(expectedIDs) + cmpStrings := func(got, expected []string, label string) { + sort.Strings(got) + sort.Strings(expected) + + if !reflect.DeepEqual(got, expected) { + t.Errorf("Expected %s (%v) to be equal to (%v)\n", + label, + strings.Join(got, ","), + strings.Join(expected, ",")) + } - if !reflect.DeepEqual(docIDs, expectedIDs) { - t.Errorf("Expected loaded document (%v) to be equal to (%v)\n", - strings.Join(docIDs, ","), strings.Join(expectedIDs, ",")) } + + cmpStrings(doc.Identifiers, test.identifiers, "identifiers") + cmpStrings(doc.Values, test.values, "values") } } diff --git a/internal/search/go.mod b/internal/search/go.mod index 7bde012906..e408ff68cf 100644 --- a/internal/search/go.mod +++ b/internal/search/go.mod @@ -3,7 +3,6 @@ module sigs.k8s.io/kustomize/internal/search go 1.12 require ( - google.golang.org/appengine v1.6.1 gopkg.in/yaml.v2 v2.2.2 // indirect sigs.k8s.io/yaml v1.1.0 ) diff --git a/internal/search/go.sum b/internal/search/go.sum index a93f1159ea..60aa01b568 100644 --- a/internal/search/go.sum +++ b/internal/search/go.sum @@ -1,21 +1,3 @@ -github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65 h1:+rhAzEzT3f4JtomfC371qB+0Ola2caSKcY69NUBZrRQ= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -google.golang.org/appengine v1.6.1 h1:QzqyMA1tlu6CgqCDUtU9V+ZKhLFT2dkJuANu5QaxI3I= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=