diff --git a/services/search/pkg/content/cs3.go b/services/search/pkg/content/cs3.go index e9c17fd6b1f..6df113c1bae 100644 --- a/services/search/pkg/content/cs3.go +++ b/services/search/pkg/content/cs3.go @@ -34,13 +34,13 @@ func newCS3Retriever(client gateway.GatewayAPIClient, logger log.Logger, insecur // Retrieve downloads the file from a cs3 service // The caller MUST make sure to close the returned ReadCloser -func (s cs3) Retrieve(ctx context.Context, rid *provider.ResourceId) (io.ReadCloser, error) { +func (s cs3) Retrieve(ctx context.Context, rID *provider.ResourceId) (io.ReadCloser, error) { at, ok := contextGet(ctx, revactx.TokenHeader) if !ok { return nil, fmt.Errorf("context without %s", revactx.TokenHeader) } - res, err := s.gwClient.InitiateFileDownload(ctx, &provider.InitiateFileDownloadRequest{Ref: &provider.Reference{ResourceId: rid, Path: "."}}) + res, err := s.gwClient.InitiateFileDownload(ctx, &provider.InitiateFileDownloadRequest{Ref: &provider.Reference{ResourceId: rID, Path: "."}}) if err != nil { return nil, err } diff --git a/services/search/pkg/content/mocks/Extractor.go b/services/search/pkg/content/mocks/Extractor.go index 1bb1053cad9..d1ab8d08188 100644 --- a/services/search/pkg/content/mocks/Extractor.go +++ b/services/search/pkg/content/mocks/Extractor.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.14.0. DO NOT EDIT. +// Code generated by mockery v2.15.0. DO NOT EDIT. package mocks diff --git a/services/search/pkg/content/mocks/Retriever.go b/services/search/pkg/content/mocks/Retriever.go index 0c05f0f3e6f..9b50186c64b 100644 --- a/services/search/pkg/content/mocks/Retriever.go +++ b/services/search/pkg/content/mocks/Retriever.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.14.0. DO NOT EDIT. +// Code generated by mockery v2.15.0. DO NOT EDIT. package mocks @@ -16,13 +16,13 @@ type Retriever struct { mock.Mock } -// Retrieve provides a mock function with given fields: ctx, rid -func (_m *Retriever) Retrieve(ctx context.Context, rid *providerv1beta1.ResourceId) (io.ReadCloser, error) { - ret := _m.Called(ctx, rid) +// Retrieve provides a mock function with given fields: ctx, rID +func (_m *Retriever) Retrieve(ctx context.Context, rID *providerv1beta1.ResourceId) (io.ReadCloser, error) { + ret := _m.Called(ctx, rID) var r0 io.ReadCloser if rf, ok := ret.Get(0).(func(context.Context, *providerv1beta1.ResourceId) io.ReadCloser); ok { - r0 = rf(ctx, rid) + r0 = rf(ctx, rID) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(io.ReadCloser) @@ -31,7 +31,7 @@ func (_m *Retriever) Retrieve(ctx context.Context, rid *providerv1beta1.Resource var r1 error if rf, ok := ret.Get(1).(func(context.Context, *providerv1beta1.ResourceId) error); ok { - r1 = rf(ctx, rid) + r1 = rf(ctx, rID) } else { r1 = ret.Error(1) } diff --git a/services/search/pkg/content/retriever.go b/services/search/pkg/content/retriever.go index bbd625732e3..1548a5bb9ca 100644 --- a/services/search/pkg/content/retriever.go +++ b/services/search/pkg/content/retriever.go @@ -13,7 +13,7 @@ import ( // //go:generate mockery --name=Retriever type Retriever interface { - Retrieve(ctx context.Context, rid *provider.ResourceId) (io.ReadCloser, error) + Retrieve(ctx context.Context, rID *provider.ResourceId) (io.ReadCloser, error) } func contextGet(ctx context.Context, k string) (string, bool) { diff --git a/services/search/pkg/engine/bleve.go b/services/search/pkg/engine/bleve.go index 8a1240b6ee5..12fec54d0a8 100644 --- a/services/search/pkg/engine/bleve.go +++ b/services/search/pkg/engine/bleve.go @@ -3,20 +3,20 @@ package engine import ( "context" "errors" - "fmt" "math" "path" "path/filepath" "strings" "time" + "github.com/blevesearch/bleve/v2/analysis/token/porter" + "github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode" + "github.com/blevesearch/bleve/v2" "github.com/blevesearch/bleve/v2/analysis/analyzer/custom" "github.com/blevesearch/bleve/v2/analysis/analyzer/keyword" "github.com/blevesearch/bleve/v2/analysis/token/lowercase" - "github.com/blevesearch/bleve/v2/analysis/token/porter" "github.com/blevesearch/bleve/v2/analysis/tokenizer/single" - "github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode" "github.com/blevesearch/bleve/v2/mapping" "github.com/blevesearch/bleve/v2/search/query" storageProvider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" @@ -25,9 +25,6 @@ import ( searchMessage "github.com/owncloud/ocis/v2/protogen/gen/ocis/messages/search/v0" searchService "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/search/v0" "github.com/owncloud/ocis/v2/services/search/pkg/content" - sq "github.com/owncloud/ocis/v2/services/search/pkg/query" - "golang.org/x/text/cases" - "golang.org/x/text/language" "google.golang.org/protobuf/types/known/timestamppb" ) @@ -119,26 +116,31 @@ func (b *Bleve) Search(_ context.Context, sir *searchService.SearchIndexRequest) Bool: false, FieldVal: "Deleted", }, - &query.TermQuery{ - FieldVal: "RootID", - Term: storagespace.FormatResourceID( - storageProvider.ResourceId{ - StorageId: sir.Ref.GetResourceId().GetStorageId(), - SpaceId: sir.Ref.GetResourceId().GetSpaceId(), - OpaqueId: sir.Ref.GetResourceId().GetOpaqueId(), - }, - ), - }, - // investigate what's wrong and why this is slow, see filter in for loop workaround - //&query.PrefixQuery{ - // Prefix: escapeQuery(utils.MakeRelativePath(path.Join(sir.Ref.Path, "/"))), - // FieldVal: "Path", - //}, &query.QueryStringQuery{ - Query: b.buildQuery(sir.Query), + Query: formatQuery(sir.Query), }, ) + if sir.Ref != nil { + q.Conjuncts = append( + q.Conjuncts, + &query.TermQuery{ + FieldVal: "RootID", + Term: storagespace.FormatResourceID( + storageProvider.ResourceId{ + StorageId: sir.Ref.GetResourceId().GetStorageId(), + SpaceId: sir.Ref.GetResourceId().GetSpaceId(), + OpaqueId: sir.Ref.GetResourceId().GetOpaqueId(), + }, + ), + }, + &query.PrefixQuery{ + Prefix: utils.MakeRelativePath(path.Join(sir.Ref.Path, "/")), + FieldVal: "Path", + }, + ) + } + bleveReq := bleve.NewSearchRequest(q) switch { @@ -158,14 +160,6 @@ func (b *Bleve) Search(_ context.Context, sir *searchService.SearchIndexRequest) matches := []*searchMessage.Match{} for _, hit := range res.Hits { - // Limit search to this directory in the space - if !strings.HasPrefix( - getValue[string](hit.Fields, "Path"), - utils.MakeRelativePath(path.Join(sir.Ref.Path, "/")), - ) { - continue - } - rootID, err := storagespace.ParseID(getValue[string](hit.Fields, "RootID")) if err != nil { return nil, err @@ -356,56 +350,17 @@ func (b *Bleve) setDeleted(id string, deleted bool) error { return nil } -func (b *Bleve) buildQuery(si string) string { - var queries [][]string - var so []string - lexer := sq.NewLexer(strings.NewReader(si)) - allowedFields := []string{"content", "title", "tags"} - - for { - tok, lit := lexer.Scan() - if tok == sq.TField { - for _, field := range allowedFields { - if strings.EqualFold(field, lit) { - queries = append(queries, []string{cases.Title(language.Und, cases.NoLower).String(lit)}) - } - } - } - - if tok == sq.TValue { - if len(queries) == 0 { - queries = append(queries, []string{"*"}) - } - - queries[len(queries)-1] = append(queries[len(queries)-1], lit) - } - - if tok == sq.TEof { - break - } +func formatQuery(q string) string { + cq := q + fields := []string{"RootID", "Path", "ID", "Name", "Size", "Mtime", "MimeType", "Type"} + for _, field := range fields { + cq = strings.ReplaceAll(cq, strings.ToLower(field)+":", field+":") } - for _, q := range queries { - if len(q) <= 1 { - continue - } - - fields := []string{q[0]} - - if fields[0] == "*" { - fields = []string{"Content", "Name", "Tags"} - } - - for _, field := range fields { - ss := strings.ToLower(strings.Join(q[1:], `\ `)) - - if !strings.Contains(ss, "*") && field != "Content" { - ss = "*" + ss + "*" - } - - so = append(so, fmt.Sprintf("%s:%s", field, ss)) - } + if strings.Contains(cq, ":") { + return cq // Sophisticated field based search } - return strings.Join(so, " ") + // this is a basic filename search + return "Name:*" + strings.ReplaceAll(strings.ToLower(cq), " ", `\ `) + "*" } diff --git a/services/search/pkg/engine/bleve_test.go b/services/search/pkg/engine/bleve_test.go index 1a4abd079d4..29048ba0b69 100644 --- a/services/search/pkg/engine/bleve_test.go +++ b/services/search/pkg/engine/bleve_test.go @@ -4,6 +4,8 @@ import ( "context" "fmt" + "github.com/cs3org/reva/v2/pkg/storagespace" + "github.com/blevesearch/bleve/v2" sprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" . "github.com/onsi/ginkgo/v2" @@ -20,42 +22,35 @@ var _ = Describe("Bleve", func() { idx bleve.Index ctx context.Context - createEntity = func(id sprovider.ResourceId, parentOpaque string, doc content.Document) engine.Resource { - name := doc.Name - - if name == "" { - name = "default.pdf" - } - - return engine.Resource{ - ID: fmt.Sprintf("%s$%s!%s", id.StorageId, id.SpaceId, id.OpaqueId), - RootID: fmt.Sprintf("%s$%s!%s", id.StorageId, id.SpaceId, id.OpaqueId), - ParentID: fmt.Sprintf("%s$%s!%s", id.StorageId, id.SpaceId, parentOpaque), - Path: fmt.Sprintf("./%s", name), - Document: doc, + doSearch = func(id string, query string) (*searchsvc.SearchIndexResponse, error) { + rID, err := storagespace.ParseID(id) + if err != nil { + return nil, err } - } - doSearch = func(id sprovider.ResourceId, query string) (*searchsvc.SearchIndexResponse, error) { return eng.Search(ctx, &searchsvc.SearchIndexRequest{ Query: query, Ref: &searchmsg.Reference{ ResourceId: &searchmsg.ResourceID{ - StorageId: id.StorageId, - SpaceId: id.SpaceId, - OpaqueId: id.OpaqueId, + StorageId: rID.StorageId, + SpaceId: rID.SpaceId, + OpaqueId: rID.OpaqueId, }, }, }) } - assertDocCount = func(id sprovider.ResourceId, query string, expectedCount int) []*searchmsg.Match { + assertDocCount = func(id string, query string, expectedCount int) []*searchmsg.Match { res, err := doSearch(id, query) ExpectWithOffset(1, err).ToNot(HaveOccurred()) ExpectWithOffset(1, len(res.Matches)).To(Equal(expectedCount), "query returned unexpected number of results: "+query) return res.Matches } + + rootResource engine.Resource + parentResource engine.Resource + childResource engine.Resource ) BeforeEach(func() { @@ -67,6 +62,31 @@ var _ = Describe("Bleve", func() { eng = engine.NewBleveEngine(idx) Expect(err).ToNot(HaveOccurred()) + + rootResource = engine.Resource{ + ID: "1$2!2", + RootID: "1$2!2", + Path: ".", + Document: content.Document{}, + } + + parentResource = engine.Resource{ + ID: "1$2!3", + ParentID: rootResource.ID, + RootID: rootResource.ID, + Path: "./parent d!r", + Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_CONTAINER), + Document: content.Document{Name: "parent d!r"}, + } + + childResource = engine.Resource{ + ID: "1$2!4", + ParentID: parentResource.ID, + RootID: rootResource.ID, + Path: "./parent d!r/child.pdf", + Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE), + Document: content.Document{Name: "child.pdf"}, + } }) Describe("New", func() { @@ -79,215 +99,278 @@ var _ = Describe("Bleve", func() { Describe("Search", func() { Context("by other fields than filename", func() { It("finds files by tags", func() { - rid := sprovider.ResourceId{ - StorageId: "1", - SpaceId: "2", - OpaqueId: "3", - } - r := createEntity(rid, "p", content.Document{Tags: []string{"foo", "bar"}}) - err := eng.Upsert(r.ID, r) + parentResource.Document.Tags = []string{"foo", "bar"} + err := eng.Upsert(parentResource.ID, parentResource) + Expect(err).ToNot(HaveOccurred()) + + assertDocCount(rootResource.ID, "Tags:foo", 1) + assertDocCount(rootResource.ID, "Tags:bar", 1) + assertDocCount(rootResource.ID, "Tags:foo Tags:bar", 1) + assertDocCount(rootResource.ID, "Tags:foo Tags:bar Tags:baz", 1) + assertDocCount(rootResource.ID, "Tags:foo Tags:bar Tags:baz", 1) + assertDocCount(rootResource.ID, "Tags:baz", 0) + }) + + It("finds files by size", func() { + parentResource.Document.Size = 12345 + err := eng.Upsert(parentResource.ID, parentResource) Expect(err).ToNot(HaveOccurred()) - assertDocCount(rid, `Tags:foo`, 1) - assertDocCount(rid, `Tags:bar`, 1) - assertDocCount(rid, `Tags:foo Tags:bar`, 1) - assertDocCount(rid, `Tags:foo Tags:bar Tags:baz`, 1) - assertDocCount(rid, `Tags:foo Tags:bar Tags:baz`, 1) - assertDocCount(rid, `Tags:baz`, 0) + assertDocCount(rootResource.ID, "Size:12345", 1) + assertDocCount(rootResource.ID, "Size:>1000", 1) + assertDocCount(rootResource.ID, "Size:<100000", 1) + assertDocCount(rootResource.ID, "Size:12344", 0) + assertDocCount(rootResource.ID, "Size:<1000", 0) + assertDocCount(rootResource.ID, "Size:>100000", 0) }) }) Context("by filename", func() { It("finds files with spaces in the filename", func() { - rid := sprovider.ResourceId{ - StorageId: "1", - SpaceId: "2", - OpaqueId: "3", - } - r := createEntity(rid, "p", content.Document{Name: "Foo oo.pdf"}) - err := eng.Upsert(r.ID, r) + parentResource.Document.Name = "Foo oo.pdf" + err := eng.Upsert(parentResource.ID, parentResource) Expect(err).ToNot(HaveOccurred()) - assertDocCount(rid, `Name:foo o*`, 1) + + assertDocCount(rootResource.ID, `Name:foo\ o*`, 1) }) It("finds files by digits in the filename", func() { - rid := sprovider.ResourceId{ - StorageId: "1", - SpaceId: "2", - OpaqueId: "3", - } - r := createEntity(rid, "p", content.Document{Name: "12345.pdf"}) - err := eng.Upsert(r.ID, r) + parentResource.Document.Name = "12345.pdf" + err := eng.Upsert(parentResource.ID, parentResource) + Expect(err).ToNot(HaveOccurred()) + + assertDocCount(rootResource.ID, "Name:1234*", 1) + }) + + It("filters hidden files", func() { + childResource.Hidden = true + err := eng.Upsert(childResource.ID, childResource) Expect(err).ToNot(HaveOccurred()) - assertDocCount(rid, `Name:1234*`, 1) + assertDocCount(rootResource.ID, "Hidden:T", 1) + assertDocCount(rootResource.ID, "Hidden:F", 0) }) Context("with a file in the root of the space", func() { It("scopes the search to the specified space", func() { - rid := sprovider.ResourceId{ - StorageId: "1", - SpaceId: "2", - OpaqueId: "3", - } - r := createEntity(rid, "p", content.Document{Name: "foo.pdf"}) - err := eng.Upsert(r.ID, r) + parentResource.Document.Name = "foo.pdf" + err := eng.Upsert(parentResource.ID, parentResource) Expect(err).ToNot(HaveOccurred()) - assertDocCount(rid, `Name:foo.pdf`, 1) - assertDocCount(sprovider.ResourceId{ - StorageId: "9", - SpaceId: "8", - OpaqueId: "7", - }, `Name:foo.pdf`, 0) + assertDocCount(rootResource.ID, "Name:foo.pdf", 1) + assertDocCount("9$8!7", "Name:foo.pdf", 0) }) }) It("limits the search to the specified fields", func() { - rid := sprovider.ResourceId{ - StorageId: "1", - SpaceId: "2", - OpaqueId: "3", - } - r := createEntity(rid, "p", content.Document{Name: "bar.pdf", Size: 789}) - err := eng.Upsert(r.ID, r) + parentResource.Document.Name = "bar.pdf" + err := eng.Upsert(parentResource.ID, parentResource) Expect(err).ToNot(HaveOccurred()) - assertDocCount(rid, `Name:bar.pdf`, 1) - assertDocCount(rid, `Unknown:field`, 0) + assertDocCount(rootResource.ID, "Name:bar.pdf", 1) + assertDocCount(rootResource.ID, "Unknown:field", 0) }) It("returns the total number of hits", func() { - rid := sprovider.ResourceId{ - StorageId: "1", - SpaceId: "2", - OpaqueId: "3", - } - r := createEntity(rid, "p", content.Document{Name: "bar.pdf"}) - err := eng.Upsert(r.ID, r) + parentResource.Document.Name = "bar.pdf" + err := eng.Upsert(parentResource.ID, parentResource) Expect(err).ToNot(HaveOccurred()) - res, err := doSearch(rid, "Name:bar*") + res, err := doSearch(rootResource.ID, "Name:bar*") Expect(err).ToNot(HaveOccurred()) Expect(res.TotalMatches).To(Equal(int32(1))) }) It("returns all desired fields", func() { - rid := sprovider.ResourceId{ - StorageId: "1", - SpaceId: "2", - OpaqueId: "3", - } - r := createEntity(rid, "p", content.Document{Name: "bar.pdf"}) - r.Type = 3 - r.MimeType = "application/pdf" + parentResource.Document.Name = "bar.pdf" + parentResource.Type = 3 + parentResource.MimeType = "application/pdf" - err := eng.Upsert(r.ID, r) + err := eng.Upsert(parentResource.ID, parentResource) Expect(err).ToNot(HaveOccurred()) - matches := assertDocCount(rid, fmt.Sprintf("Name:%s", r.Name), 1) + matches := assertDocCount(rootResource.ID, fmt.Sprintf("Name:%s", parentResource.Name), 1) match := matches[0] - Expect(match.Entity.Ref.ResourceId.OpaqueId).To(Equal(rid.OpaqueId)) - Expect(match.Entity.Ref.Path).To(Equal(r.Path)) - Expect(match.Entity.Name).To(Equal(r.Name)) - Expect(match.Entity.Size).To(Equal(r.Size)) - Expect(match.Entity.Type).To(Equal(r.Type)) - Expect(match.Entity.MimeType).To(Equal(r.MimeType)) + Expect(match.Entity.Ref.Path).To(Equal(parentResource.Path)) + Expect(match.Entity.Name).To(Equal(parentResource.Name)) + Expect(match.Entity.Size).To(Equal(parentResource.Size)) + Expect(match.Entity.Type).To(Equal(parentResource.Type)) + Expect(match.Entity.MimeType).To(Equal(parentResource.MimeType)) Expect(match.Entity.Deleted).To(BeFalse()) Expect(match.Score > 0).To(BeTrue()) }) It("finds files by name, prefix or substring match", func() { - queries := []string{"foo.pdf", "foo*", "*oo.p*"} - for i, query := range queries { - rid := sprovider.ResourceId{ - StorageId: string(rune(i + 1)), - SpaceId: string(rune(i + 2)), - OpaqueId: string(rune(i + 3)), - } + parentResource.Document.Name = "foo.pdf" - r := createEntity(rid, "p", content.Document{Name: "foo.pdf"}) - r.Size = uint64(i + 250) - err := eng.Upsert(r.ID, r) + err := eng.Upsert(parentResource.ID, parentResource) + Expect(err).ToNot(HaveOccurred()) + + queries := []string{"foo.pdf", "foo*", "*oo.p*"} + for _, query := range queries { + err := eng.Upsert(parentResource.ID, parentResource) Expect(err).ToNot(HaveOccurred()) - matches := assertDocCount(rid, query, 1) - Expect(matches[0].Entity.Ref.ResourceId.OpaqueId).To(Equal(rid.OpaqueId)) - Expect(matches[0].Entity.Ref.Path).To(Equal(r.Path)) - Expect(matches[0].Entity.Id.OpaqueId).To(Equal(rid.OpaqueId)) - Expect(matches[0].Entity.Name).To(Equal(r.Name)) - Expect(matches[0].Entity.Size).To(Equal(r.Size)) + assertDocCount(rootResource.ID, query, 1) } }) - It("ignores case", func() { - rid := sprovider.ResourceId{ - StorageId: "1", - SpaceId: "2", - OpaqueId: "3", - } - r := createEntity(rid, "p", content.Document{Name: "foo.pdf"}) - r.Type = 3 - r.MimeType = "application/pdf" + It("uses a lower-case index", func() { + parentResource.Document.Name = "foo.pdf" - err := eng.Upsert(r.ID, r) + err := eng.Upsert(parentResource.ID, parentResource) Expect(err).ToNot(HaveOccurred()) - assertDocCount(rid, "Name:foo*", 1) - assertDocCount(rid, "Name:Foo*", 1) + assertDocCount(rootResource.ID, "Name:foo*", 1) + assertDocCount(rootResource.ID, "Name:Foo*", 0) }) Context("and an additional file in a subdirectory", func() { - var ( - ridT sprovider.ResourceId - entT engine.Resource - ridD sprovider.ResourceId - entD engine.Resource - ) - BeforeEach(func() { - ridT = sprovider.ResourceId{ - StorageId: "1", - SpaceId: "2", - OpaqueId: "3", - } - entT = createEntity(ridT, "p", content.Document{Name: "top.pdf"}) - Expect(eng.Upsert(entT.ID, entT)).ToNot(HaveOccurred()) + err := eng.Upsert(parentResource.ID, parentResource) + Expect(err).ToNot(HaveOccurred()) - ridD = sprovider.ResourceId{ - StorageId: "1", - SpaceId: "2", - OpaqueId: "4", - } - entD = createEntity(ridD, "p", content.Document{Name: "deep.pdf"}) - entD.Path = "./nested/deep.pdf" - Expect(eng.Upsert(entD.ID, entD)).ToNot(HaveOccurred()) + err = eng.Upsert(childResource.ID, childResource) + Expect(err).ToNot(HaveOccurred()) }) It("finds files living deeper in the tree by filename, prefix or substring match", func() { - queries := []string{"deep.pdf", "dee*", "*ep.*"} + queries := []string{"child.pdf", "child*", "*ld.*"} for _, query := range queries { - assertDocCount(ridD, query, 1) + assertDocCount(rootResource.ID, query, 1) } }) - - It("does not find the higher levels when limiting the searched directory", func() { - res, err := eng.Search(ctx, &searchsvc.SearchIndexRequest{ - Ref: &searchmsg.Reference{ - ResourceId: &searchmsg.ResourceID{ - StorageId: ridT.StorageId, - SpaceId: ridT.SpaceId, - OpaqueId: ridT.OpaqueId, - }, - Path: "./nested/", - }, - Query: "Name:top.pdf", - }) - Expect(err).ToNot(HaveOccurred()) - Expect(res).ToNot(BeNil()) - Expect(len(res.Matches)).To(Equal(0)) - }) }) }) }) + + Describe("Upsert", func() { + It("adds a resourceInfo to the index", func() { + err := eng.Upsert(childResource.ID, childResource) + Expect(err).ToNot(HaveOccurred()) + + count, err := idx.DocCount() + Expect(err).ToNot(HaveOccurred()) + Expect(count).To(Equal(uint64(1))) + + query := bleve.NewMatchQuery("child.pdf") + res, err := idx.Search(bleve.NewSearchRequest(query)) + Expect(err).ToNot(HaveOccurred()) + Expect(res.Hits.Len()).To(Equal(1)) + }) + + It("updates an existing resource in the index", func() { + + err := eng.Upsert(childResource.ID, childResource) + Expect(err).ToNot(HaveOccurred()) + + countA, err := idx.DocCount() + Expect(err).ToNot(HaveOccurred()) + Expect(countA).To(Equal(uint64(1))) + + err = eng.Upsert(childResource.ID, childResource) + Expect(err).ToNot(HaveOccurred()) + + countB, err := idx.DocCount() + Expect(err).ToNot(HaveOccurred()) + Expect(countB).To(Equal(uint64(1))) + }) + }) + + Describe("Delete", func() { + It("marks a resource as deleted", func() { + err := eng.Upsert(childResource.ID, childResource) + Expect(err).ToNot(HaveOccurred()) + + assertDocCount(rootResource.ID, "Name:*child*", 1) + + err = eng.Delete(childResource.ID) + Expect(err).ToNot(HaveOccurred()) + + assertDocCount(rootResource.ID, "Name:*child*", 0) + }) + + It("marks a child resources as deleted", func() { + err := eng.Upsert(parentResource.ID, parentResource) + Expect(err).ToNot(HaveOccurred()) + + err = eng.Upsert(childResource.ID, childResource) + Expect(err).ToNot(HaveOccurred()) + + assertDocCount(rootResource.ID, parentResource.Document.Name, 1) + assertDocCount(rootResource.ID, childResource.Document.Name, 1) + + err = eng.Delete(parentResource.ID) + Expect(err).ToNot(HaveOccurred()) + + assertDocCount(rootResource.ID, parentResource.Document.Name, 0) + assertDocCount(rootResource.ID, childResource.Document.Name, 0) + }) + }) + + Describe("Restore", func() { + It("also marks child resources as restored", func() { + err := eng.Upsert(parentResource.ID, parentResource) + Expect(err).ToNot(HaveOccurred()) + + err = eng.Upsert(childResource.ID, childResource) + Expect(err).ToNot(HaveOccurred()) + + err = eng.Delete(parentResource.ID) + Expect(err).ToNot(HaveOccurred()) + + assertDocCount(rootResource.ID, parentResource.Name, 0) + assertDocCount(rootResource.ID, childResource.Name, 0) + + err = eng.Restore(parentResource.ID) + Expect(err).ToNot(HaveOccurred()) + + assertDocCount(rootResource.ID, parentResource.Name, 1) + assertDocCount(rootResource.ID, childResource.Name, 1) + }) + }) + + Describe("Move", func() { + It("renames the parent and its child resources", func() { + err := eng.Upsert(parentResource.ID, parentResource) + Expect(err).ToNot(HaveOccurred()) + + err = eng.Upsert(childResource.ID, childResource) + Expect(err).ToNot(HaveOccurred()) + + parentResource.Path = "newname" + err = eng.Move(parentResource.ID, parentResource.ParentID, "./my/newname") + Expect(err).ToNot(HaveOccurred()) + + assertDocCount(rootResource.ID, parentResource.Name, 0) + + matches := assertDocCount(rootResource.ID, "Name:child.pdf", 1) + Expect(matches[0].Entity.ParentId.OpaqueId).To(Equal("3")) + Expect(matches[0].Entity.Ref.Path).To(Equal("./my/newname/child.pdf")) + }) + + It("moves the parent and its child resources", func() { + err := eng.Upsert(parentResource.ID, parentResource) + Expect(err).ToNot(HaveOccurred()) + + err = eng.Upsert(childResource.ID, childResource) + Expect(err).ToNot(HaveOccurred()) + + parentResource.Path = " " + parentResource.ParentID = "1$2!somewhereopaqueid" + + err = eng.Move(parentResource.ID, parentResource.ParentID, "./somewhere/else/newname") + Expect(err).ToNot(HaveOccurred()) + assertDocCount(rootResource.ID, `parent d!r`, 0) + + matches := assertDocCount(rootResource.ID, "Name:child.pdf", 1) + Expect(matches[0].Entity.ParentId.OpaqueId).To(Equal("3")) + Expect(matches[0].Entity.Ref.Path).To(Equal("./somewhere/else/newname/child.pdf")) + + matches = assertDocCount(rootResource.ID, `newname`, 1) + Expect(matches[0].Entity.ParentId.OpaqueId).To(Equal("somewhereopaqueid")) + Expect(matches[0].Entity.Ref.Path).To(Equal("./somewhere/else/newname")) + + }) + }) }) diff --git a/services/search/pkg/engine/engine.go b/services/search/pkg/engine/engine.go index 5c20c976dec..c24ca3d7ae1 100644 --- a/services/search/pkg/engine/engine.go +++ b/services/search/pkg/engine/engine.go @@ -33,6 +33,7 @@ type Resource struct { ParentID string Type uint64 Deleted bool + Hidden bool } func resourceIDtoSearchID(id storageProvider.ResourceId) *searchMessage.ResourceID { diff --git a/services/search/pkg/engine/mocks/engine.go b/services/search/pkg/engine/mocks/engine.go index 09bb02da01a..61037adc514 100644 --- a/services/search/pkg/engine/mocks/engine.go +++ b/services/search/pkg/engine/mocks/engine.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.10.4. DO NOT EDIT. +// Code generated by mockery v2.15.0. DO NOT EDIT. package mocks @@ -129,3 +129,18 @@ func (_m *Engine) Upsert(id string, r engine.Resource) error { return r0 } + +type mockConstructorTestingTNewEngine interface { + mock.TestingT + Cleanup(func()) +} + +// NewEngine creates a new instance of Engine. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewEngine(t mockConstructorTestingTNewEngine) *Engine { + mock := &Engine{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/services/search/pkg/query/lexer.go b/services/search/pkg/query/lexer.go deleted file mode 100644 index f540f921ad3..00000000000 --- a/services/search/pkg/query/lexer.go +++ /dev/null @@ -1,103 +0,0 @@ -package query - -import ( - "bufio" - "bytes" - "io" - "unicode" - "unicode/utf8" -) - -// Lexer is responsible for lexing the query. -type Lexer struct { - r *bufio.Reader - plain bool -} - -// NewLexer creates a new Lexer -func NewLexer(r io.Reader) Lexer { - return Lexer{r: bufio.NewReader(r)} -} - -// Scan reads a query section and returns a Token and literal -func (l *Lexer) Scan() (Token, string) { - - for { - r := l.read() - - if r == eof { - return TEof, "" - } - - if unicode.IsSpace(r) { - continue - } - - if r == '"' { - l.plain = !l.plain - return TQuotationMark, "" - } - - if r == '-' { - return TNegation, "" - } - - if r == '+' { - return TAddition, "" - } - - if r != ':' { - l.unread() - return l.scanUnknown(TValue) - } - - if r == ':' && (unicode.IsLetter(l.peek(1)) || unicode.IsNumber(l.peek(1))) { - return l.scanUnknown(TShortcut) - } - - return TUnknown, string(r) - } -} - -func (l *Lexer) scanUnknown(t Token) (Token, string) { - var buf bytes.Buffer - - for { - r := l.read() - - if r == eof || unicode.IsSpace(r) { - break - } - - if r == '"' { - l.unread() - break - } - - if r == ':' && !l.plain { - return TField, buf.String() - } - - buf.WriteRune(r) - } - - return t, buf.String() -} - -func (l *Lexer) peek(n int) rune { - b, _ := l.r.Peek(n) - r, _ := utf8.DecodeRune(b) - return r -} - -func (l *Lexer) read() rune { - r, _, err := l.r.ReadRune() - if err != nil { - return eof - } - return r -} - -func (l *Lexer) unread() { - _ = l.r.UnreadRune() -} diff --git a/services/search/pkg/query/lexer_test.go b/services/search/pkg/query/lexer_test.go deleted file mode 100644 index 8d2d0bed025..00000000000 --- a/services/search/pkg/query/lexer_test.go +++ /dev/null @@ -1,347 +0,0 @@ -package query - -import ( - "strings" - "testing" - - . "github.com/onsi/gomega" -) - -type kv[K any, V any] struct { - k K - v V -} - -// ios AST https://github.com/owncloud/ios-app/pull/933 -func TestLexer(t *testing.T) { - g := NewGomegaWithT(t) - - cases := []struct { - input string - exp []kv[Token, string] - }{ - { - input: "engineering", - exp: []kv[Token, string]{ - {k: TValue, v: "engineering"}, - {k: TEof}, - }, - }, - { - input: "engineering demos", - exp: []kv[Token, string]{ - {k: TValue, v: "engineering"}, - {k: TValue, v: "demos"}, - {k: TEof}, - }, - }, - { - input: "\"engineering demos\"", - exp: []kv[Token, string]{ - {k: TQuotationMark}, - {k: TValue, v: "engineering"}, - {k: TValue, v: "demos"}, - {k: TQuotationMark}, - {k: TEof}, - }, - }, - { - input: "\"engineering \"demos", - exp: []kv[Token, string]{ - {k: TQuotationMark}, - {k: TValue, v: "engineering"}, - {k: TQuotationMark}, - {k: TValue, v: "demos"}, - {k: TEof}, - }, - }, - { - input: "type:pdf", - exp: []kv[Token, string]{ - {k: TField, v: "type"}, - {k: TValue, v: "pdf"}, - {k: TEof}, - }, - }, - { - input: "before:2021", - exp: []kv[Token, string]{ - {k: TField, v: "before"}, - {k: TValue, v: "2021"}, - {k: TEof}, - }, - }, - { - input: "before:2021-02", - exp: []kv[Token, string]{ - {k: TField, v: "before"}, - {k: TValue, v: "2021-02"}, - {k: TEof}, - }, - }, - { - input: "before:2021-02-03", - exp: []kv[Token, string]{ - {k: TField, v: "before"}, - {k: TValue, v: "2021-02-03"}, - {k: TEof}, - }, - }, - { - input: "after:2020", - exp: []kv[Token, string]{ - {k: TField, v: "after"}, - {k: TValue, v: "2020"}, - {k: TEof}, - }, - }, - { - input: "after:2020-02", - exp: []kv[Token, string]{ - {k: TField, v: "after"}, - {k: TValue, v: "2020-02"}, - {k: TEof}, - }, - }, - { - input: "after:2020-02-03", - exp: []kv[Token, string]{ - {k: TField, v: "after"}, - {k: TValue, v: "2020-02-03"}, - {k: TEof}, - }, - }, - { - input: "on:2020-02-03", - exp: []kv[Token, string]{ - {k: TField, v: "on"}, - {k: TValue, v: "2020-02-03"}, - {k: TEof}, - }, - }, - { - input: "on:2020-02-03,2020-02-05", - exp: []kv[Token, string]{ - {k: TField, v: "on"}, - {k: TValue, v: "2020-02-03,2020-02-05"}, - {k: TEof}, - }, - }, - { - input: "smaller:200mb", - exp: []kv[Token, string]{ - {k: TField, v: "smaller"}, - {k: TValue, v: "200mb"}, - {k: TEof}, - }, - }, - { - input: "greater:1gb", - exp: []kv[Token, string]{ - {k: TField, v: "greater"}, - {k: TValue, v: "1gb"}, - {k: TEof}, - }, - }, - { - input: "owner:cscherm", - exp: []kv[Token, string]{ - {k: TField, v: "owner"}, - {k: TValue, v: "cscherm"}, - {k: TEof}, - }, - }, - { - input: ":file", - exp: []kv[Token, string]{ - {k: TShortcut, v: "file"}, - {k: TEof}, - }, - }, - { - input: ":folder", - exp: []kv[Token, string]{ - {k: TShortcut, v: "folder"}, - {k: TEof}, - }, - }, - { - input: ":image", - exp: []kv[Token, string]{ - {k: TShortcut, v: "image"}, - {k: TEof}, - }, - }, - { - input: ":video", - exp: []kv[Token, string]{ - {k: TShortcut, v: "video"}, - {k: TEof}, - }, - }, - { - input: ":year", - exp: []kv[Token, string]{ - {k: TShortcut, v: "year"}, - {k: TEof}, - }, - }, - { - input: ":month", - exp: []kv[Token, string]{ - {k: TShortcut, v: "month"}, - {k: TEof}, - }, - }, - { - input: ":week", - exp: []kv[Token, string]{ - {k: TShortcut, v: "week"}, - {k: TEof}, - }, - }, - { - input: ":today", - exp: []kv[Token, string]{ - {k: TShortcut, v: "today"}, - {k: TEof}, - }, - }, - { - input: ":d", - exp: []kv[Token, string]{ - {k: TShortcut, v: "d"}, - {k: TEof}, - }, - }, - { - input: ":5d", - exp: []kv[Token, string]{ - {k: TShortcut, v: "5d"}, - {k: TEof}, - }, - }, - { - input: ":2w", - exp: []kv[Token, string]{ - {k: TShortcut, v: "2w"}, - {k: TEof}, - }, - }, - { - input: ":2w", - exp: []kv[Token, string]{ - {k: TShortcut, v: "2w"}, - {k: TEof}, - }, - }, - { - input: ":1m", - exp: []kv[Token, string]{ - {k: TShortcut, v: "1m"}, - {k: TEof}, - }, - }, - { - input: ":m", - exp: []kv[Token, string]{ - {k: TShortcut, v: "m"}, - {k: TEof}, - }, - }, - { - input: ":1y", - exp: []kv[Token, string]{ - {k: TShortcut, v: "1y"}, - {k: TEof}, - }, - }, - { - input: ":y", - exp: []kv[Token, string]{ - {k: TShortcut, v: "y"}, - {k: TEof}, - }, - }, - { - input: "-:image", - exp: []kv[Token, string]{ - {k: TNegation}, - {k: TShortcut, v: "image"}, - {k: TEof}, - }, - }, - { - input: "\"engineering demos\" \"engineering \"demos type:pdf,mov on:2020-02-03,2020-02-05 :image :pdf -:image", - exp: []kv[Token, string]{ - {k: TQuotationMark}, - {k: TValue, v: "engineering"}, - {k: TValue, v: "demos"}, - {k: TQuotationMark}, - {k: TQuotationMark}, - {k: TValue, v: "engineering"}, - {k: TQuotationMark}, - {k: TValue, v: "demos"}, - {k: TField, v: "type"}, - {k: TValue, v: "pdf,mov"}, - {k: TField, v: "on"}, - {k: TValue, v: "2020-02-03,2020-02-05"}, - {k: TShortcut, v: "image"}, - {k: TShortcut, v: "pdf"}, - {k: TNegation}, - {k: TShortcut, v: "image"}, - {k: TEof}, - }, - }, - { - input: ": a b: c :::d:::\"e \"f\"\"\"a\"", - exp: []kv[Token, string]{ - {k: TUnknown, v: ":"}, - {k: TValue, v: "a"}, - {k: TField, v: "b"}, - {k: TValue, v: "c"}, - {k: TUnknown, v: ":"}, - {k: TUnknown, v: ":"}, - {k: TField, v: "d"}, - {k: TUnknown, v: ":"}, - {k: TUnknown, v: ":"}, - {k: TQuotationMark}, - {k: TValue, v: "e"}, - {k: TQuotationMark}, - {k: TValue, v: "f"}, - {k: TQuotationMark}, - {k: TQuotationMark}, - {k: TQuotationMark}, - {k: TValue, v: "a"}, - {k: TQuotationMark}, - {k: TEof}, - }, - }, - { - input: "+ID:1284d238-aa92-42ce-bdc4-0b0000009157$4c510ada-c86b-4815-8820-42cdf82c3d51!4c510ada-c86b-4815-8820-42cdf82c3d51 +Mtime:>=\"2022-10-19T10:35:21.064277496+02:00\"", - exp: []kv[Token, string]{ - {k: TAddition}, - {k: TField, v: "ID"}, - {k: TValue, v: "1284d238-aa92-42ce-bdc4-0b0000009157$4c510ada-c86b-4815-8820-42cdf82c3d51!4c510ada-c86b-4815-8820-42cdf82c3d51"}, - {k: TAddition}, - {k: TField, v: "Mtime"}, - {k: TValue, v: ">="}, - {k: TQuotationMark}, - {k: TValue, v: "2022-10-19T10:35:21.064277496+02:00"}, - {k: TQuotationMark}, - {k: TEof}, - }, - }, - } - - for _, c := range cases { - t.Run(c.input, func(t *testing.T) { - l := NewLexer(strings.NewReader(c.input)) - for _, exp := range c.exp { - tok, lit := l.Scan() - g.Expect(tok).To(Equal(exp.k)) - g.Expect(lit).To(Equal(exp.v)) - } - }) - } -} diff --git a/services/search/pkg/query/query.go b/services/search/pkg/query/query.go deleted file mode 100644 index 21288094af3..00000000000 --- a/services/search/pkg/query/query.go +++ /dev/null @@ -1,40 +0,0 @@ -package query - -// Token maps to token type -type Token int - -const ( - // TEof end of file token type - TEof Token = iota - // TUnknown unknown token type - TUnknown - // TAddition addition token type - TAddition - // TNegation negation token type - TNegation - // TQuotationMark quotation-mark token type, e.g. not of type - "-" - TQuotationMark - // TField field token type, e.g. - '"' - TField - // TValue value token type, e.g. - "content:" - TValue - // TShortcut shortcut token type, e.g. all images - ":image" - TShortcut -) - -var eof = rune(0) - -var tokens = map[Token]string{ - TEof: "EOF", - TUnknown: "UNKNOWN", - TNegation: "NEGATION", - TQuotationMark: "QUOTATION_MARK", - TField: "FIELD", - TValue: "VALUE", - TShortcut: "SHORTCUT", -} - -// String returns the Token human-readable. -func (t Token) String() string { - return tokens[t] -} diff --git a/services/search/pkg/search/events.go b/services/search/pkg/search/events.go index 1717657aa71..b344532cfcc 100644 --- a/services/search/pkg/search/events.go +++ b/services/search/pkg/search/events.go @@ -1,31 +1,16 @@ package search import ( - "context" - - gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" - user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/v2/pkg/events" "github.com/cs3org/reva/v2/pkg/storagespace" - "github.com/cs3org/reva/v2/pkg/utils" "github.com/owncloud/ocis/v2/ocis-pkg/log" "github.com/owncloud/ocis/v2/services/search/pkg/config" - "github.com/owncloud/ocis/v2/services/search/pkg/content" - "github.com/owncloud/ocis/v2/services/search/pkg/engine" ) -type eventHandler struct { - logger log.Logger - engine engine.Engine - gateway gateway.GatewayAPIClient - extractor content.Extractor - secret string -} - // HandleEvents listens to the needed events, // it handles the whole resource indexing livecycle. -func HandleEvents(eng engine.Engine, extractor content.Extractor, gw gateway.GatewayAPIClient, bus events.Consumer, logger log.Logger, cfg *config.Config) error { +func HandleEvents(s Searcher, bus events.Consumer, logger log.Logger, cfg *config.Config) error { evts := []events.Unmarshaller{ events.ItemTrashed{}, events.ItemRestored{}, @@ -52,125 +37,59 @@ func HandleEvents(eng engine.Engine, extractor content.Extractor, gw gateway.Gat cfg.Events.NumConsumers = 1 } + spaceID := func(ref *provider.Reference) *provider.StorageSpaceId { + return &provider.StorageSpaceId{ + OpaqueId: storagespace.FormatResourceID( + provider.ResourceId{ + StorageId: ref.GetResourceId().GetStorageId(), + SpaceId: ref.GetResourceId().GetSpaceId(), + }, + ), + } + } + for i := 0; i < cfg.Events.NumConsumers; i++ { - go func(eh *eventHandler, ch <-chan interface{}) { + go func(s Searcher, ch <-chan interface{}) { for e := range ch { - eh.logger.Debug().Interface("event", e).Msg("updating index") + logger.Debug().Interface("event", e).Msg("updating index") + + var err error switch ev := e.(type) { case events.ItemTrashed: - eh.trashItem(ev.ID) + s.TrashItem(ev.ID) + err = s.IndexSpace(spaceID(ev.Ref), ev.Executant) case events.ItemMoved: - eh.moveItem(ev.Ref, ev.Executant) + s.MoveItem(ev.Ref, ev.Executant) + err = s.IndexSpace(spaceID(ev.Ref), ev.Executant) case events.ItemRestored: - eh.restoreItem(ev.Ref, ev.Executant) + s.RestoreItem(ev.Ref, ev.Executant) + err = s.IndexSpace(spaceID(ev.Ref), ev.Executant) case events.ContainerCreated: - eh.upsertItem(ev.Ref, ev.Executant) + err = s.IndexSpace(spaceID(ev.Ref), ev.Executant) case events.FileTouched: - eh.upsertItem(ev.Ref, ev.Executant) + err = s.IndexSpace(spaceID(ev.Ref), ev.Executant) case events.FileVersionRestored: - eh.upsertItem(ev.Ref, ev.Executant) - case events.FileUploaded: - eh.upsertItem(ev.Ref, ev.Executant) - case events.UploadReady: - eh.upsertItem(ev.FileRef, ev.ExecutingUser.Id) + err = s.IndexSpace(spaceID(ev.Ref), ev.Executant) case events.TagsAdded: - eh.upsertItem(ev.Ref, ev.Executant) + err = s.IndexSpace(spaceID(ev.Ref), ev.Executant) case events.TagsRemoved: - eh.upsertItem(ev.Ref, ev.Executant) + err = s.IndexSpace(spaceID(ev.Ref), ev.Executant) + case events.FileUploaded: + err = s.IndexSpace(spaceID(ev.Ref), ev.Executant) + case events.UploadReady: + err = s.IndexSpace(spaceID(ev.FileRef), ev.ExecutingUser.Id) + } + + if err != nil { + logger.Error().Err(err).Interface("event", e) } } }( - &eventHandler{ - logger: logger, - engine: eng, - secret: cfg.MachineAuthAPIKey, - gateway: gw, - extractor: extractor, - }, + s, ch, ) } return nil } - -func (eh *eventHandler) trashItem(rid *provider.ResourceId) { - err := eh.engine.Delete(storagespace.FormatResourceID(*rid)) - if err != nil { - eh.logger.Error().Err(err).Interface("Id", rid).Msg("failed to remove item from index") - } -} - -func (eh *eventHandler) upsertItem(ref *provider.Reference, uid *user.UserId) { - ctx, stat, path := eh.resInfo(uid, ref) - if ctx == nil || stat == nil || path == "" { - return - } - - doc, err := eh.extractor.Extract(ctx, stat.Info) - if err != nil { - eh.logger.Error().Err(err).Msg("failed to extract resource content") - return - } - - r := engine.Resource{ - ID: storagespace.FormatResourceID(*stat.Info.Id), - RootID: storagespace.FormatResourceID(provider.ResourceId{ - StorageId: stat.Info.Id.StorageId, - OpaqueId: stat.Info.Id.SpaceId, - SpaceId: stat.Info.Id.SpaceId, - }), - ParentID: storagespace.FormatResourceID(*stat.GetInfo().GetParentId()), - Path: utils.MakeRelativePath(path), - Type: uint64(stat.Info.Type), - Document: doc, - } - - if err = eh.engine.Upsert(r.ID, r); err != nil { - eh.logger.Error().Err(err).Msg("error adding updating the resource in the index") - } else { - logDocCount(eh.engine, eh.logger) - } -} - -func (eh *eventHandler) restoreItem(ref *provider.Reference, uid *user.UserId) { - ctx, stat, path := eh.resInfo(uid, ref) - if ctx == nil || stat == nil || path == "" { - return - } - - if err := eh.engine.Restore(storagespace.FormatResourceID(*stat.Info.Id)); err != nil { - eh.logger.Error().Err(err).Msg("failed to restore the changed resource in the index") - } -} - -func (eh *eventHandler) moveItem(ref *provider.Reference, uid *user.UserId) { - ctx, stat, path := eh.resInfo(uid, ref) - if ctx == nil || stat == nil || path == "" { - return - } - - if err := eh.engine.Move(storagespace.FormatResourceID(*stat.GetInfo().GetId()), storagespace.FormatResourceID(*stat.GetInfo().GetParentId()), path); err != nil { - eh.logger.Error().Err(err).Msg("failed to move the changed resource in the index") - } -} - -func (eh *eventHandler) resInfo(uid *user.UserId, ref *provider.Reference) (context.Context, *provider.StatResponse, string) { - ownerCtx, err := getAuthContext(&user.User{Id: uid}, eh.gateway, eh.secret, eh.logger) - if err != nil { - return nil, nil, "" - } - - statRes, err := statResource(ownerCtx, ref, eh.gateway, eh.logger) - if err != nil { - return nil, nil, "" - } - - r, err := ResolveReference(ownerCtx, ref, statRes.GetInfo(), eh.gateway) - if err != nil { - return nil, nil, "" - } - - return ownerCtx, statRes, r.GetPath() -} diff --git a/services/search/pkg/search/events_test.go b/services/search/pkg/search/events_test.go index 3f98c3e17ce..d06debb56cf 100644 --- a/services/search/pkg/search/events_test.go +++ b/services/search/pkg/search/events_test.go @@ -1,186 +1,54 @@ package search_test import ( - "context" - - gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" userv1beta1 "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/v2/pkg/events" - "github.com/cs3org/reva/v2/pkg/rgrpc/status" - cs3mocks "github.com/cs3org/reva/v2/tests/cs3mocks/mocks" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/owncloud/ocis/v2/ocis-pkg/log" "github.com/owncloud/ocis/v2/services/search/pkg/config" - "github.com/owncloud/ocis/v2/services/search/pkg/content" - contentMocks "github.com/owncloud/ocis/v2/services/search/pkg/content/mocks" - engineMocks "github.com/owncloud/ocis/v2/services/search/pkg/engine/mocks" "github.com/owncloud/ocis/v2/services/search/pkg/search" + searchMocks "github.com/owncloud/ocis/v2/services/search/pkg/search/mocks" "github.com/stretchr/testify/mock" mEvents "go-micro.dev/v4/events" ) -var _ = Describe("Events", func() { - var ( - gw *cs3mocks.GatewayAPIClient - engine *engineMocks.Engine - extractor *contentMocks.Extractor - bus events.Stream - ctx context.Context +var _ = DescribeTable("events", + func(mcks []string, e interface{}, asyncUploads bool) { + var ( + s = &searchMocks.Searcher{} + calls int + ) - logger = log.NewLogger() - user = &userv1beta1.User{ - Id: &userv1beta1.UserId{ - OpaqueId: "user", - }, - } + bus, _ := mEvents.NewStream() - ref = &provider.Reference{ - ResourceId: &provider.ResourceId{ - StorageId: "storageid", - OpaqueId: "rootopaqueid", - }, - Path: "./foo.pdf", - } - ri = &provider.ResourceInfo{ - Id: &provider.ResourceId{ - StorageId: "storageid", - OpaqueId: "opaqueid", - }, - ParentId: &provider.ResourceId{ - StorageId: "storageid", - OpaqueId: "parentopaqueid", + search.HandleEvents(s, bus, log.NewLogger(), &config.Config{ + Events: config.Events{ + AsyncUploads: asyncUploads, }, - Path: "foo.pdf", - Size: 12345, - } - ) - - BeforeEach(func() { - ctx = context.Background() - gw = &cs3mocks.GatewayAPIClient{} - engine = &engineMocks.Engine{} - bus, _ = mEvents.NewStream() - extractor = &contentMocks.Extractor{} - - _ = search.HandleEvents(engine, extractor, gw, bus, logger, &config.Config{}) - - gw.On("Authenticate", mock.Anything, mock.Anything).Return(&gateway.AuthenticateResponse{ - Status: status.NewOK(ctx), - Token: "authtoken", - }, nil) - gw.On("Stat", mock.Anything, mock.Anything).Return(&provider.StatResponse{ - Status: status.NewOK(ctx), - Info: ri, - }, nil) - gw.On("GetPath", mock.Anything, mock.Anything).Return(&provider.GetPathResponse{ - Status: status.NewOK(ctx), - Path: "", - }, nil) - engine.On("DocCount").Return(uint64(1), nil) - }) - Describe("events", func() { - It("triggers an index update when a file has been uploaded", func() { - called := false - extractor.Mock.On("Extract", mock.Anything, mock.Anything).Return(content.Document{}, nil) - engine.On("Upsert", mock.Anything, mock.Anything).Return(nil).Run(func(args mock.Arguments) { - called = true - }) - - _ = events.Publish(bus, events.FileUploaded{ - Ref: ref, - Executant: user.Id, - }) - - Eventually(func() bool { - return called - }, "2s").Should(BeTrue()) - }) - - It("triggers an index update when a file has been touched", func() { - called := false - extractor.Mock.On("Extract", mock.Anything, mock.Anything, mock.Anything).Return(content.Document{}, nil) - engine.On("Upsert", mock.Anything, mock.Anything).Return(nil).Run(func(args mock.Arguments) { - called = true - }) - _ = events.Publish(bus, events.FileTouched{ - Ref: ref, - Executant: user.Id, - }) - - Eventually(func() bool { - return called - }, "2s").Should(BeTrue()) - }) - - It("removes an entry from the index when the file has been deleted", func() { - called := false - gw.On("Stat", mock.Anything, mock.Anything).Return(&provider.StatResponse{ - Status: status.NewNotFound(context.Background(), ""), - }, nil) - engine.On("Delete", mock.Anything).Return(nil).Run(func(args mock.Arguments) { - called = true - }) - - _ = events.Publish(bus, events.ItemTrashed{ - Ref: ref, - ID: ri.Id, - Executant: user.Id, - }) - - Eventually(func() bool { - return called - }, "2s").Should(BeTrue()) }) - It("indexes items when they are being restored", func() { - called := false - engine.On("Restore", mock.Anything).Return(nil).Run(func(args mock.Arguments) { - called = true - }) - - _ = events.Publish(bus, events.ItemRestored{ - Ref: ref, - Executant: user.Id, - }) - - Eventually(func() bool { - return called - }, "2s").Should(BeTrue()) - }) - - It("indexes items when a version has been restored", func() { - called := false - extractor.Mock.On("Extract", mock.Anything, mock.Anything, mock.Anything).Return(content.Document{}, nil) - engine.On("Upsert", mock.Anything, mock.Anything).Return(nil).Run(func(args mock.Arguments) { - called = true - }) - - _ = events.Publish(bus, events.FileVersionRestored{ - Ref: ref, - Executant: user.Id, - }) - - Eventually(func() bool { - return called - }, "2s").Should(BeTrue()) - }) - - It("indexes items when they are being moved", func() { - called := false - engine.On("Move", mock.Anything, mock.Anything, mock.Anything).Return(nil).Run(func(args mock.Arguments) { - called = true - }) - - _ = events.Publish(bus, events.ItemMoved{ - Ref: ref, - Executant: user.Id, + for _, mck := range mcks { + s.On(mck, mock.Anything, mock.Anything).Return(nil).Run(func(args mock.Arguments) { + calls += 1 }) + } - Eventually(func() bool { - return called - }, "2s").Should(BeTrue()) - }) - }) -}) + err := events.Publish(bus, e) + + Expect(err).To(BeNil()) + Eventually(func() int { + return calls + }, "2s").Should(Equal(len(mcks))) + }, + Entry("ItemTrashed", []string{"TrashItem", "IndexSpace"}, events.ItemTrashed{}, false), + Entry("ItemMoved", []string{"MoveItem", "IndexSpace"}, events.ItemMoved{}, false), + Entry("ItemRestored", []string{"RestoreItem", "IndexSpace"}, events.ItemRestored{}, false), + Entry("ContainerCreated", []string{"IndexSpace"}, events.ContainerCreated{}, false), + Entry("FileTouched", []string{"IndexSpace"}, events.FileTouched{}, false), + Entry("FileVersionRestored", []string{"IndexSpace"}, events.FileVersionRestored{}, false), + Entry("TagsAdded", []string{"IndexSpace"}, events.TagsAdded{}, false), + Entry("TagsRemoved", []string{"IndexSpace"}, events.TagsRemoved{}, false), + Entry("FileUploaded", []string{"IndexSpace"}, events.FileUploaded{}, false), + Entry("UploadReady", []string{"IndexSpace"}, events.UploadReady{ExecutingUser: &userv1beta1.User{}}, true), +) diff --git a/services/search/pkg/search/mocks/Searcher.go b/services/search/pkg/search/mocks/Searcher.go new file mode 100644 index 00000000000..2f94ad0ad1b --- /dev/null +++ b/services/search/pkg/search/mocks/Searcher.go @@ -0,0 +1,91 @@ +// Code generated by mockery v2.15.0. DO NOT EDIT. + +package mocks + +import ( + context "context" + + providerv1beta1 "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + mock "github.com/stretchr/testify/mock" + + userv1beta1 "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + + v0 "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/search/v0" +) + +// Searcher is an autogenerated mock type for the Searcher type +type Searcher struct { + mock.Mock +} + +// IndexSpace provides a mock function with given fields: rID, uID +func (_m *Searcher) IndexSpace(rID *providerv1beta1.StorageSpaceId, uID *userv1beta1.UserId) error { + ret := _m.Called(rID, uID) + + var r0 error + if rf, ok := ret.Get(0).(func(*providerv1beta1.StorageSpaceId, *userv1beta1.UserId) error); ok { + r0 = rf(rID, uID) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// MoveItem provides a mock function with given fields: ref, uID +func (_m *Searcher) MoveItem(ref *providerv1beta1.Reference, uID *userv1beta1.UserId) { + _m.Called(ref, uID) +} + +// RestoreItem provides a mock function with given fields: ref, uID +func (_m *Searcher) RestoreItem(ref *providerv1beta1.Reference, uID *userv1beta1.UserId) { + _m.Called(ref, uID) +} + +// Search provides a mock function with given fields: ctx, req +func (_m *Searcher) Search(ctx context.Context, req *v0.SearchRequest) (*v0.SearchResponse, error) { + ret := _m.Called(ctx, req) + + var r0 *v0.SearchResponse + if rf, ok := ret.Get(0).(func(context.Context, *v0.SearchRequest) *v0.SearchResponse); ok { + r0 = rf(ctx, req) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*v0.SearchResponse) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, *v0.SearchRequest) error); ok { + r1 = rf(ctx, req) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// TrashItem provides a mock function with given fields: rID +func (_m *Searcher) TrashItem(rID *providerv1beta1.ResourceId) { + _m.Called(rID) +} + +// UpsertItem provides a mock function with given fields: ref, uID +func (_m *Searcher) UpsertItem(ref *providerv1beta1.Reference, uID *userv1beta1.UserId) { + _m.Called(ref, uID) +} + +type mockConstructorTestingTNewSearcher interface { + mock.TestingT + Cleanup(func()) +} + +// NewSearcher creates a new instance of Searcher. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewSearcher(t mockConstructorTestingTNewSearcher) *Searcher { + mock := &Searcher{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/services/search/pkg/search/provider.go b/services/search/pkg/search/provider.go deleted file mode 100644 index 157364036b5..00000000000 --- a/services/search/pkg/search/provider.go +++ /dev/null @@ -1,342 +0,0 @@ -package search - -import ( - "context" - "fmt" - "path/filepath" - "sort" - "strings" - - gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" - rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" - rpcv1beta1 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - ctxpkg "github.com/cs3org/reva/v2/pkg/ctx" - "github.com/cs3org/reva/v2/pkg/errtypes" - "github.com/cs3org/reva/v2/pkg/events" - sdk "github.com/cs3org/reva/v2/pkg/sdk/common" - "github.com/cs3org/reva/v2/pkg/storage/utils/walker" - "github.com/cs3org/reva/v2/pkg/storagespace" - "github.com/cs3org/reva/v2/pkg/utils" - "github.com/owncloud/ocis/v2/ocis-pkg/log" - "github.com/owncloud/ocis/v2/services/search/pkg/content" - "github.com/owncloud/ocis/v2/services/search/pkg/engine" - "google.golang.org/grpc/metadata" - - searchmsg "github.com/owncloud/ocis/v2/protogen/gen/ocis/messages/search/v0" - searchsvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/search/v0" -) - -// Permissions is copied from reva internal conversion pkg -type Permissions uint - -// consts are copied from reva internal conversion pkg -const ( - // PermissionInvalid represents an invalid permission - PermissionInvalid Permissions = 0 - // PermissionRead grants read permissions on a resource - PermissionRead Permissions = 1 << (iota - 1) - // PermissionWrite grants write permissions on a resource - PermissionWrite - // PermissionCreate grants create permissions on a resource - PermissionCreate - // PermissionDelete grants delete permissions on a resource - PermissionDelete - // PermissionShare grants share permissions on a resource - PermissionShare -) - -// ListenEvents are the events the search service is listening to -var ListenEvents = []events.Unmarshaller{ - events.ItemTrashed{}, - events.ItemRestored{}, - events.ItemMoved{}, - events.ContainerCreated{}, - events.FileUploaded{}, - events.FileTouched{}, - events.FileVersionRestored{}, -} - -// Provider is responsible for indexing spaces and pass on a search -// to it's underlying engine. -type Provider struct { - logger log.Logger - gateway gateway.GatewayAPIClient - engine engine.Engine - extractor content.Extractor - secret string -} - -// NewProvider creates a new Provider instance. -func NewProvider(gw gateway.GatewayAPIClient, eng engine.Engine, extractor content.Extractor, logger log.Logger, secret string) *Provider { - return &Provider{ - gateway: gw, - engine: eng, - secret: secret, - logger: logger, - extractor: extractor, - } -} - -// Search processes a search request and passes it down to the engine. -func (p *Provider) Search(ctx context.Context, req *searchsvc.SearchRequest) (*searchsvc.SearchResponse, error) { - if req.Query == "" { - return nil, errtypes.BadRequest("empty query provided") - } - p.logger.Debug().Str("query", req.Query).Msg("performing a search") - - listSpacesRes, err := p.gateway.ListStorageSpaces(ctx, &provider.ListStorageSpacesRequest{ - Filters: []*provider.ListStorageSpacesRequest_Filter{ - { - Type: provider.ListStorageSpacesRequest_Filter_TYPE_SPACE_TYPE, - Term: &provider.ListStorageSpacesRequest_Filter_SpaceType{SpaceType: "+grant"}, - }, - }, - }) - if err != nil { - p.logger.Error().Err(err).Msg("failed to list the user's storage spaces") - return nil, err - } - - mountpointMap := map[string]string{} - for _, space := range listSpacesRes.StorageSpaces { - if space.SpaceType != "mountpoint" { - continue - } - opaqueMap := sdk.DecodeOpaqueMap(space.Opaque) - grantSpaceID := storagespace.FormatResourceID(provider.ResourceId{ - StorageId: opaqueMap["grantStorageID"], - SpaceId: opaqueMap["grantSpaceID"], - OpaqueId: opaqueMap["grantOpaqueID"], - }) - mountpointMap[grantSpaceID] = space.Id.OpaqueId - } - - matches := matchArray{} - total := int32(0) - for _, space := range listSpacesRes.StorageSpaces { - searchRootID := &searchmsg.ResourceID{ - StorageId: space.Root.StorageId, - SpaceId: space.Root.SpaceId, - OpaqueId: space.Root.OpaqueId, - } - - if req.Ref != nil && - (req.Ref.ResourceId.StorageId != searchRootID.StorageId || - req.Ref.ResourceId.SpaceId != searchRootID.SpaceId || - req.Ref.ResourceId.OpaqueId != searchRootID.OpaqueId) { - continue - } - - var ( - mountpointRootID *searchmsg.ResourceID - rootName string - permissions *provider.ResourcePermissions - ) - mountpointPrefix := "" - switch space.SpaceType { - case "mountpoint": - continue // mountpoint spaces are only "links" to the shared spaces. we have to search the shared "grant" space instead - case "grant": - // In case of grant spaces we search the root of the outer space and translate the paths to the according mountpoint - searchRootID.OpaqueId = space.Root.SpaceId - mountpointID, ok := mountpointMap[space.Id.OpaqueId] - if !ok { - p.logger.Warn().Interface("space", space).Msg("could not find mountpoint space for grant space") - continue - } - gpRes, err := p.gateway.GetPath(ctx, &provider.GetPathRequest{ - ResourceId: space.Root, - }) - if err != nil { - p.logger.Error().Err(err).Str("space", space.Id.OpaqueId).Msg("failed to get path for grant space root") - continue - } - if gpRes.Status.Code != rpcv1beta1.Code_CODE_OK { - p.logger.Error().Interface("status", gpRes.Status).Str("space", space.Id.OpaqueId).Msg("failed to get path for grant space root") - continue - } - mountpointPrefix = utils.MakeRelativePath(gpRes.Path) - sid, spid, oid, err := storagespace.SplitID(mountpointID) - if err != nil { - p.logger.Error().Err(err).Str("space", space.Id.OpaqueId).Str("mountpointId", mountpointID).Msg("invalid mountpoint space id") - continue - } - mountpointRootID = &searchmsg.ResourceID{ - StorageId: sid, - SpaceId: spid, - OpaqueId: oid, - } - rootName = space.GetRootInfo().GetPath() - permissions = space.GetRootInfo().GetPermissionSet() - p.logger.Debug().Interface("grantSpace", space).Interface("mountpointRootId", mountpointRootID).Msg("searching a grant") - case "personal": - permissions = space.GetRootInfo().GetPermissionSet() - } - - res, err := p.engine.Search(ctx, &searchsvc.SearchIndexRequest{ - Query: req.Query, - Ref: &searchmsg.Reference{ - ResourceId: searchRootID, - Path: mountpointPrefix, - }, - PageSize: req.PageSize, - }) - if err != nil { - p.logger.Error().Err(err).Str("space", space.Id.OpaqueId).Msg("failed to search the index") - return nil, err - } - p.logger.Debug().Str("space", space.Id.OpaqueId).Int("hits", len(res.Matches)).Msg("space search done") - - total += res.TotalMatches - for _, match := range res.Matches { - if mountpointPrefix != "" { - match.Entity.Ref.Path = utils.MakeRelativePath(strings.TrimPrefix(match.Entity.Ref.Path, mountpointPrefix)) - } - if mountpointRootID != nil { - match.Entity.Ref.ResourceId = mountpointRootID - } - match.Entity.ShareRootName = rootName - - isShared := match.GetEntity().GetRef().GetResourceId().GetSpaceId() == utils.ShareStorageSpaceID - isMountpoint := isShared && match.GetEntity().GetRef().GetPath() == "." - isDir := match.GetEntity().GetMimeType() == "httpd/unix-directory" - match.Entity.Permissions = convertToWebDAVPermissions(isShared, isMountpoint, isDir, permissions) - matches = append(matches, match) - } - } - - // compile one sorted list of matches from all spaces and apply the limit if needed - sort.Sort(matches) - limit := req.PageSize - if limit == 0 { - limit = 200 - } - if int32(len(matches)) > limit && limit != -1 { - matches = matches[0:limit] - } - - return &searchsvc.SearchResponse{ - Matches: matches, - TotalMatches: total, - }, nil -} - -// IndexSpace (re)indexes all resources of a given space. -func (p *Provider) IndexSpace(ctx context.Context, req *searchsvc.IndexSpaceRequest) (*searchsvc.IndexSpaceResponse, error) { - // Get auth context - authRes, err := p.gateway.Authenticate(ctx, &gateway.AuthenticateRequest{ - Type: "machine", - ClientId: "userid:" + req.UserId, - ClientSecret: p.secret, - }) - if err != nil || authRes.GetStatus().GetCode() != rpc.Code_CODE_OK { - return nil, err - } - - if authRes.GetStatus().GetCode() != rpc.Code_CODE_OK { - return nil, fmt.Errorf("could not get authenticated context for user") - } - ownerCtx := ctxpkg.ContextSetUser(context.Background(), authRes.User) - ownerCtx = metadata.AppendToOutgoingContext(ownerCtx, ctxpkg.TokenHeader, authRes.Token) - - // Walk the space and index all files - w := walker.NewWalker(p.gateway) - rootID, err := storagespace.ParseID(req.SpaceId) - if err != nil { - p.logger.Error().Err(err).Msg(err.Error()) - return nil, err - } - err = w.Walk(ownerCtx, &rootID, func(wd string, info *provider.ResourceInfo, err error) error { - if err != nil { - p.logger.Error().Err(err).Msg("error walking the tree") - } - ref, err := ResolveReference(ownerCtx, &provider.Reference{ - Path: utils.MakeRelativePath(filepath.Join(wd, info.Path)), - ResourceId: &rootID, - }, info, p.gateway) - if err != nil { - p.logger.Error().Err(err).Msg("error resolving reference") - return nil - } - - doc, err := p.extractor.Extract(ownerCtx, info) - if err != nil { - p.logger.Error().Err(err).Msg("error extracting content") - } - - var pid string - if info.ParentId != nil { - pid = storagespace.FormatResourceID(*info.ParentId) - } - r := engine.Resource{ - ID: storagespace.FormatResourceID(*info.Id), - RootID: storagespace.FormatResourceID(provider.ResourceId{ - StorageId: info.Id.StorageId, - OpaqueId: info.Id.SpaceId, - SpaceId: info.Id.SpaceId, - }), - ParentID: pid, - Path: ref.Path, - Type: uint64(info.Type), - Document: doc, - } - - err = p.engine.Upsert(r.ID, r) - if err != nil { - p.logger.Error().Err(err).Msg("error adding resource to the index") - } else { - p.logger.Debug().Interface("ref", ref).Msg("added resource to index") - } - return nil - }) - if err != nil { - return nil, err - } - - return &searchsvc.IndexSpaceResponse{}, nil -} - -// NOTE: this converts CS3 to WebDAV permissions -// since conversions pkg is reva internal we have no other choice than to duplicate the logic -func convertToWebDAVPermissions(isShared, isMountpoint, isDir bool, p *provider.ResourcePermissions) string { - if p == nil { - return "" - } - var b strings.Builder - if isShared { - fmt.Fprintf(&b, "S") - } - if p.ListContainer && - p.ListFileVersions && - p.ListRecycle && - p.Stat && - p.GetPath && - p.GetQuota && - p.InitiateFileDownload { - fmt.Fprintf(&b, "R") - } - if isMountpoint { - fmt.Fprintf(&b, "M") - } - if p.Delete && - p.PurgeRecycle { - fmt.Fprintf(&b, "D") - } - if p.InitiateFileUpload && - p.RestoreFileVersion && - p.RestoreRecycleItem { - fmt.Fprintf(&b, "NV") - if !isDir { - fmt.Fprintf(&b, "W") - } - } - if isDir && - p.ListContainer && - p.Stat && - p.CreateContainer && - p.InitiateFileUpload { - fmt.Fprintf(&b, "CK") - } - return b.String() -} diff --git a/services/search/pkg/search/search.go b/services/search/pkg/search/search.go index 662a14ae7ce..954ea55f546 100644 --- a/services/search/pkg/search/search.go +++ b/services/search/pkg/search/search.go @@ -3,6 +3,8 @@ package search import ( "context" "errors" + "fmt" + "strings" gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" @@ -93,3 +95,47 @@ func statResource(ctx context.Context, ref *provider.Reference, gw gateway.Gatew return res, nil } + +// NOTE: this converts CS3 to WebDAV permissions +// since conversions pkg is reva internal we have no other choice than to duplicate the logic +func convertToWebDAVPermissions(isShared, isMountpoint, isDir bool, p *provider.ResourcePermissions) string { + if p == nil { + return "" + } + var b strings.Builder + if isShared { + fmt.Fprintf(&b, "S") + } + if p.ListContainer && + p.ListFileVersions && + p.ListRecycle && + p.Stat && + p.GetPath && + p.GetQuota && + p.InitiateFileDownload { + fmt.Fprintf(&b, "R") + } + if isMountpoint { + fmt.Fprintf(&b, "M") + } + if p.Delete && + p.PurgeRecycle { + fmt.Fprintf(&b, "D") + } + if p.InitiateFileUpload && + p.RestoreFileVersion && + p.RestoreRecycleItem { + fmt.Fprintf(&b, "NV") + if !isDir { + fmt.Fprintf(&b, "W") + } + } + if isDir && + p.ListContainer && + p.Stat && + p.CreateContainer && + p.InitiateFileUpload { + fmt.Fprintf(&b, "CK") + } + return b.String() +} diff --git a/services/search/pkg/search/service.go b/services/search/pkg/search/service.go new file mode 100644 index 00000000000..7717a457c29 --- /dev/null +++ b/services/search/pkg/search/service.go @@ -0,0 +1,352 @@ +package search + +import ( + "context" + "fmt" + "path/filepath" + "sort" + "strings" + "time" + + gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" + user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + rpcv1beta1 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/v2/pkg/errtypes" + sdk "github.com/cs3org/reva/v2/pkg/sdk/common" + "github.com/cs3org/reva/v2/pkg/storage/utils/walker" + "github.com/cs3org/reva/v2/pkg/storagespace" + "github.com/cs3org/reva/v2/pkg/utils" + "github.com/owncloud/ocis/v2/ocis-pkg/log" + searchmsg "github.com/owncloud/ocis/v2/protogen/gen/ocis/messages/search/v0" + searchsvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/search/v0" + "github.com/owncloud/ocis/v2/services/search/pkg/content" + "github.com/owncloud/ocis/v2/services/search/pkg/engine" +) + +//go:generate mockery --name=Searcher + +// Searcher is the interface to the SearchService +type Searcher interface { + Search(ctx context.Context, req *searchsvc.SearchRequest) (*searchsvc.SearchResponse, error) + IndexSpace(rID *provider.StorageSpaceId, uID *user.UserId) error + TrashItem(rID *provider.ResourceId) + UpsertItem(ref *provider.Reference, uID *user.UserId) + RestoreItem(ref *provider.Reference, uID *user.UserId) + MoveItem(ref *provider.Reference, uID *user.UserId) +} + +// Service is responsible for indexing spaces and pass on a search +// to it's underlying engine. +type Service struct { + logger log.Logger + gateway gateway.GatewayAPIClient + engine engine.Engine + extractor content.Extractor + secret string +} + +// NewService creates a new Provider instance. +func NewService(gw gateway.GatewayAPIClient, eng engine.Engine, extractor content.Extractor, logger log.Logger, secret string) *Service { + return &Service{ + gateway: gw, + engine: eng, + secret: secret, + logger: logger, + extractor: extractor, + } +} + +// Search processes a search request and passes it down to the engine. +func (s *Service) Search(ctx context.Context, req *searchsvc.SearchRequest) (*searchsvc.SearchResponse, error) { + if req.Query == "" { + return nil, errtypes.BadRequest("empty query provided") + } + s.logger.Debug().Str("query", req.Query).Msg("performing a search") + + listSpacesRes, err := s.gateway.ListStorageSpaces(ctx, &provider.ListStorageSpacesRequest{ + Filters: []*provider.ListStorageSpacesRequest_Filter{ + { + Type: provider.ListStorageSpacesRequest_Filter_TYPE_SPACE_TYPE, + Term: &provider.ListStorageSpacesRequest_Filter_SpaceType{SpaceType: "+grant"}, + }, + }, + }) + if err != nil { + s.logger.Error().Err(err).Msg("failed to list the user's storage spaces") + return nil, err + } + + mountpointMap := map[string]string{} + for _, space := range listSpacesRes.StorageSpaces { + if space.SpaceType != "mountpoint" { + continue + } + opaqueMap := sdk.DecodeOpaqueMap(space.Opaque) + grantSpaceID := storagespace.FormatResourceID(provider.ResourceId{ + StorageId: opaqueMap["grantStorageID"], + SpaceId: opaqueMap["grantSpaceID"], + OpaqueId: opaqueMap["grantOpaqueID"], + }) + mountpointMap[grantSpaceID] = space.Id.OpaqueId + } + + matches := matchArray{} + total := int32(0) + for _, space := range listSpacesRes.StorageSpaces { + searchRootID := &searchmsg.ResourceID{ + StorageId: space.Root.StorageId, + SpaceId: space.Root.SpaceId, + OpaqueId: space.Root.OpaqueId, + } + + if req.Ref != nil && + (req.Ref.ResourceId.StorageId != searchRootID.StorageId || + req.Ref.ResourceId.SpaceId != searchRootID.SpaceId || + req.Ref.ResourceId.OpaqueId != searchRootID.OpaqueId) { + continue + } + + var ( + mountpointRootID *searchmsg.ResourceID + rootName string + permissions *provider.ResourcePermissions + ) + mountpointPrefix := "" + switch space.SpaceType { + case "mountpoint": + continue // mountpoint spaces are only "links" to the shared spaces. we have to search the shared "grant" space instead + case "grant": + // In case of grant spaces we search the root of the outer space and translate the paths to the according mountpoint + searchRootID.OpaqueId = space.Root.SpaceId + mountpointID, ok := mountpointMap[space.Id.OpaqueId] + if !ok { + s.logger.Warn().Interface("space", space).Msg("could not find mountpoint space for grant space") + continue + } + gpRes, err := s.gateway.GetPath(ctx, &provider.GetPathRequest{ + ResourceId: space.Root, + }) + if err != nil { + s.logger.Error().Err(err).Str("space", space.Id.OpaqueId).Msg("failed to get path for grant space root") + continue + } + if gpRes.Status.Code != rpcv1beta1.Code_CODE_OK { + s.logger.Error().Interface("status", gpRes.Status).Str("space", space.Id.OpaqueId).Msg("failed to get path for grant space root") + continue + } + mountpointPrefix = utils.MakeRelativePath(gpRes.Path) + sid, spid, oid, err := storagespace.SplitID(mountpointID) + if err != nil { + s.logger.Error().Err(err).Str("space", space.Id.OpaqueId).Str("mountpointId", mountpointID).Msg("invalid mountpoint space id") + continue + } + mountpointRootID = &searchmsg.ResourceID{ + StorageId: sid, + SpaceId: spid, + OpaqueId: oid, + } + rootName = space.GetRootInfo().GetPath() + permissions = space.GetRootInfo().GetPermissionSet() + s.logger.Debug().Interface("grantSpace", space).Interface("mountpointRootId", mountpointRootID).Msg("searching a grant") + case "personal": + permissions = space.GetRootInfo().GetPermissionSet() + } + + res, err := s.engine.Search(ctx, &searchsvc.SearchIndexRequest{ + Query: req.Query, + Ref: &searchmsg.Reference{ + ResourceId: searchRootID, + Path: mountpointPrefix, + }, + PageSize: req.PageSize, + }) + if err != nil { + s.logger.Error().Err(err).Str("space", space.Id.OpaqueId).Msg("failed to search the index") + return nil, err + } + s.logger.Debug().Str("space", space.Id.OpaqueId).Int("hits", len(res.Matches)).Msg("space search done") + + total += res.TotalMatches + for _, match := range res.Matches { + if mountpointPrefix != "" { + match.Entity.Ref.Path = utils.MakeRelativePath(strings.TrimPrefix(match.Entity.Ref.Path, mountpointPrefix)) + } + if mountpointRootID != nil { + match.Entity.Ref.ResourceId = mountpointRootID + } + match.Entity.ShareRootName = rootName + + isShared := match.GetEntity().GetRef().GetResourceId().GetSpaceId() == utils.ShareStorageSpaceID + isMountpoint := isShared && match.GetEntity().GetRef().GetPath() == "." + isDir := match.GetEntity().GetMimeType() == "httpd/unix-directory" + match.Entity.Permissions = convertToWebDAVPermissions(isShared, isMountpoint, isDir, permissions) + matches = append(matches, match) + } + } + + // compile one sorted list of matches from all spaces and apply the limit if needed + sort.Sort(matches) + limit := req.PageSize + if limit == 0 { + limit = 200 + } + if int32(len(matches)) > limit && limit != -1 { + matches = matches[0:limit] + } + + return &searchsvc.SearchResponse{ + Matches: matches, + TotalMatches: total, + }, nil +} + +// IndexSpace (re)indexes all resources of a given space. +func (s *Service) IndexSpace(spaceID *provider.StorageSpaceId, uID *user.UserId) error { + ownerCtx, err := getAuthContext(&user.User{Id: uID}, s.gateway, s.secret, s.logger) + if err != nil { + return err + } + + rootID, err := storagespace.ParseID(spaceID.OpaqueId) + if err != nil { + s.logger.Error().Err(err).Msg("invalid space id") + return err + } + if rootID.StorageId == "" || rootID.SpaceId == "" { + s.logger.Error().Err(err).Msg("invalid space id") + return fmt.Errorf("invalid space id") + } + rootID.OpaqueId = rootID.SpaceId + + w := walker.NewWalker(s.gateway) + err = w.Walk(ownerCtx, &rootID, func(wd string, info *provider.ResourceInfo, err error) error { + if err != nil { + s.logger.Error().Err(err).Msg("error walking the tree") + return err + } + + if info == nil { + return nil + } + + ref := &provider.Reference{ + Path: utils.MakeRelativePath(filepath.Join(wd, info.Path)), + ResourceId: &rootID, + } + s.logger.Debug().Str("path", ref.Path).Msg("Walking tree") + + searchRes, err := s.engine.Search(ownerCtx, &searchsvc.SearchIndexRequest{ + Query: "+ID:" + storagespace.FormatResourceID(*info.Id) + ` +Mtime:>="` + utils.TSToTime(info.Mtime).Format(time.RFC3339Nano) + `"`, + }) + + if err == nil && len(searchRes.Matches) >= 1 { + if info.Type == provider.ResourceType_RESOURCE_TYPE_CONTAINER { + s.logger.Debug().Str("path", ref.Path).Msg("subtree hasn't changed. Skipping.") + return filepath.SkipDir + } + s.logger.Debug().Str("path", ref.Path).Msg("element hasn't changed. Skipping.") + return nil + } + + s.UpsertItem(ref, uID) + + return nil + }) + + if err != nil { + return err + } + + logDocCount(s.engine, s.logger) + + return nil +} + +// TrashItem marks the item as deleted. +func (s *Service) TrashItem(rID *provider.ResourceId) { + err := s.engine.Delete(storagespace.FormatResourceID(*rID)) + if err != nil { + s.logger.Error().Err(err).Interface("Id", rID).Msg("failed to remove item from index") + } +} + +// UpsertItem indexes or stores Resource data fields. +func (s *Service) UpsertItem(ref *provider.Reference, uID *user.UserId) { + ctx, stat, path := s.resInfo(uID, ref) + if ctx == nil || stat == nil || path == "" { + return + } + + doc, err := s.extractor.Extract(ctx, stat.Info) + if err != nil { + s.logger.Error().Err(err).Msg("failed to extract resource content") + return + } + + r := engine.Resource{ + ID: storagespace.FormatResourceID(*stat.Info.Id), + RootID: storagespace.FormatResourceID(provider.ResourceId{ + StorageId: stat.Info.Id.StorageId, + OpaqueId: stat.Info.Id.SpaceId, + SpaceId: stat.Info.Id.SpaceId, + }), + Path: utils.MakeRelativePath(path), + Type: uint64(stat.Info.Type), + Document: doc, + } + r.Hidden = strings.HasPrefix(r.Path, ".") + + if parentID := stat.GetInfo().GetParentId(); parentID != nil { + r.ParentID = storagespace.FormatResourceID(*parentID) + } + + if err = s.engine.Upsert(r.ID, r); err != nil { + s.logger.Error().Err(err).Msg("error adding updating the resource in the index") + } else { + logDocCount(s.engine, s.logger) + } +} + +// RestoreItem makes the item available again. +func (s *Service) RestoreItem(ref *provider.Reference, uID *user.UserId) { + ctx, stat, path := s.resInfo(uID, ref) + if ctx == nil || stat == nil || path == "" { + return + } + + if err := s.engine.Restore(storagespace.FormatResourceID(*stat.Info.Id)); err != nil { + s.logger.Error().Err(err).Msg("failed to restore the changed resource in the index") + } +} + +// MoveItem updates the resource location and all of its necessary fields. +func (s *Service) MoveItem(ref *provider.Reference, uID *user.UserId) { + ctx, stat, path := s.resInfo(uID, ref) + if ctx == nil || stat == nil || path == "" { + return + } + + if err := s.engine.Move(storagespace.FormatResourceID(*stat.GetInfo().GetId()), storagespace.FormatResourceID(*stat.GetInfo().GetParentId()), path); err != nil { + s.logger.Error().Err(err).Msg("failed to move the changed resource in the index") + } +} + +func (s *Service) resInfo(uID *user.UserId, ref *provider.Reference) (context.Context, *provider.StatResponse, string) { + ownerCtx, err := getAuthContext(&user.User{Id: uID}, s.gateway, s.secret, s.logger) + if err != nil { + return nil, nil, "" + } + + statRes, err := statResource(ownerCtx, ref, s.gateway, s.logger) + if err != nil { + return nil, nil, "" + } + + r, err := ResolveReference(ownerCtx, ref, statRes.GetInfo(), s.gateway) + if err != nil { + return nil, nil, "" + } + + return ownerCtx, statRes, r.GetPath() +} diff --git a/services/search/pkg/search/provider_test.go b/services/search/pkg/search/service_test.go similarity index 92% rename from services/search/pkg/search/provider_test.go rename to services/search/pkg/search/service_test.go index 16bd13f4059..aa671bdf2b5 100644 --- a/services/search/pkg/search/provider_test.go +++ b/services/search/pkg/search/service_test.go @@ -23,7 +23,7 @@ import ( var _ = Describe("Searchprovider", func() { var ( - p *search.Provider + s search.Searcher extractor *contentMocks.Extractor gw *cs3mocks.GatewayAPIClient indexClient *engineMocks.Engine @@ -73,7 +73,7 @@ var _ = Describe("Searchprovider", func() { indexClient = &engineMocks.Engine{} extractor = &contentMocks.Extractor{} - p = search.NewProvider(gw, indexClient, extractor, logger, "") + s = search.NewService(gw, indexClient, extractor, logger, "") gw.On("Authenticate", mock.Anything, mock.Anything).Return(&gateway.AuthenticateResponse{ Status: status.NewOK(ctx), @@ -94,8 +94,8 @@ var _ = Describe("Searchprovider", func() { Describe("New", func() { It("returns a new instance", func() { - p := search.NewProvider(gw, indexClient, extractor, logger, "") - Expect(p).ToNot(BeNil()) + s := search.NewService(gw, indexClient, extractor, logger, "") + Expect(s).ToNot(BeNil()) }) }) @@ -105,21 +105,18 @@ var _ = Describe("Searchprovider", func() { Status: status.NewOK(context.Background()), User: user, }, nil) - extractor.Mock.On("Extract", mock.Anything, mock.Anything, mock.Anything).Return(content.Document{}, nil) + extractor.On("Extract", mock.Anything, mock.Anything, mock.Anything).Return(content.Document{}, nil) indexClient.On("Upsert", mock.Anything, mock.Anything).Return(nil) + indexClient.On("Search", mock.Anything, mock.Anything).Return(&searchsvc.SearchIndexResponse{}, nil) - res, err := p.IndexSpace(ctx, &searchsvc.IndexSpaceRequest{ - SpaceId: "storageid$spaceid!spaceid", - UserId: "user", - }) - Expect(err).ToNot(HaveOccurred()) - Expect(res).ToNot(BeNil()) + err := s.IndexSpace(&sprovider.StorageSpaceId{OpaqueId: "storageid$spaceid!spaceid"}, user.Id) + Expect(err).ShouldNot(HaveOccurred()) }) }) Describe("Search", func() { It("fails when an empty query is given", func() { - res, err := p.Search(ctx, &searchsvc.SearchRequest{ + res, err := s.Search(ctx, &searchsvc.SearchRequest{ Query: "", }) Expect(err).To(HaveOccurred()) @@ -158,7 +155,7 @@ var _ = Describe("Searchprovider", func() { }) It("does not mess with field-based searches", func() { - _, err := p.Search(ctx, &searchsvc.SearchRequest{ + _, err := s.Search(ctx, &searchsvc.SearchRequest{ Query: "Size:<10", }) Expect(err).ToNot(HaveOccurred()) @@ -168,7 +165,7 @@ var _ = Describe("Searchprovider", func() { }) It("searches the personal user space", func() { - res, err := p.Search(ctx, &searchsvc.SearchRequest{ + res, err := s.Search(ctx, &searchsvc.SearchRequest{ Query: "foo", }) Expect(err).ToNot(HaveOccurred()) @@ -245,7 +242,7 @@ var _ = Describe("Searchprovider", func() { }, }, nil) - res, err := p.Search(ctx, &searchsvc.SearchRequest{ + res, err := s.Search(ctx, &searchsvc.SearchRequest{ Query: "Foo", }) Expect(err).ToNot(HaveOccurred()) @@ -337,7 +334,7 @@ var _ = Describe("Searchprovider", func() { }) It("considers the search Ref parameter", func() { - res, err := p.Search(ctx, &searchsvc.SearchRequest{ + res, err := s.Search(ctx, &searchsvc.SearchRequest{ Query: "foo", Ref: &searchmsg.Reference{ ResourceId: &searchmsg.ResourceID{ @@ -354,7 +351,7 @@ var _ = Describe("Searchprovider", func() { }) It("finds matches in both the personal space AND the grant", func() { - res, err := p.Search(ctx, &searchsvc.SearchRequest{ + res, err := s.Search(ctx, &searchsvc.SearchRequest{ Query: "foo", }) Expect(err).ToNot(HaveOccurred()) @@ -365,7 +362,7 @@ var _ = Describe("Searchprovider", func() { }) It("sorts and limits the combined results from all spaces", func() { - res, err := p.Search(ctx, &searchsvc.SearchRequest{ + res, err := s.Search(ctx, &searchsvc.SearchRequest{ Query: "foo", PageSize: 2, }) diff --git a/services/search/pkg/service/grpc/v0/service.go b/services/search/pkg/service/grpc/v0/service.go index 549887309b1..6a9a2187c66 100644 --- a/services/search/pkg/service/grpc/v0/service.go +++ b/services/search/pkg/service/grpc/v0/service.go @@ -9,25 +9,25 @@ import ( "os" "time" + user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + ctxpkg "github.com/cs3org/reva/v2/pkg/ctx" revactx "github.com/cs3org/reva/v2/pkg/ctx" "github.com/cs3org/reva/v2/pkg/errtypes" "github.com/cs3org/reva/v2/pkg/events/server" "github.com/cs3org/reva/v2/pkg/rgrpc/todo/pool" "github.com/go-micro/plugins/v4/events/natsjs" "github.com/jellydator/ttlcache/v2" + ociscrypto "github.com/owncloud/ocis/v2/ocis-pkg/crypto" + "github.com/owncloud/ocis/v2/ocis-pkg/log" + v0 "github.com/owncloud/ocis/v2/protogen/gen/ocis/messages/search/v0" + searchsvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/search/v0" "github.com/owncloud/ocis/v2/services/search/pkg/content" "github.com/owncloud/ocis/v2/services/search/pkg/engine" "github.com/owncloud/ocis/v2/services/search/pkg/search" merrors "go-micro.dev/v4/errors" "go-micro.dev/v4/metadata" grpcmetadata "google.golang.org/grpc/metadata" - - user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" - ctxpkg "github.com/cs3org/reva/v2/pkg/ctx" - ociscrypto "github.com/owncloud/ocis/v2/ocis-pkg/crypto" - "github.com/owncloud/ocis/v2/ocis-pkg/log" - v0 "github.com/owncloud/ocis/v2/protogen/gen/ocis/messages/search/v0" - searchsvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/search/v0" ) // NewHandler returns a service implementation for Service. @@ -103,8 +103,10 @@ func NewHandler(opts ...Option) (searchsvc.SearchProviderHandler, func(), error) return nil, teardown, err } + ss := search.NewService(gw, eng, extractor, logger, cfg.MachineAuthAPIKey) + // setup event handling - if err := search.HandleEvents(eng, extractor, gw, bus, logger, cfg); err != nil { + if err := search.HandleEvents(ss, bus, logger, cfg); err != nil { return nil, teardown, err } @@ -116,7 +118,7 @@ func NewHandler(opts ...Option) (searchsvc.SearchProviderHandler, func(), error) return &Service{ id: cfg.GRPC.Namespace + "." + cfg.Service.Name, log: logger, - provider: search.NewProvider(gw, eng, extractor, logger, cfg.MachineAuthAPIKey), + searcher: ss, cache: cache, }, teardown, nil } @@ -125,7 +127,7 @@ func NewHandler(opts ...Option) (searchsvc.SearchProviderHandler, func(), error) type Service struct { id string log log.Logger - provider *search.Provider + searcher search.Searcher cache *ttlcache.Cache } @@ -144,7 +146,7 @@ func (s Service) Search(ctx context.Context, in *searchsvc.SearchRequest, out *s res, ok := s.FromCache(key) if !ok { var err error - res, err = s.provider.Search(ctx, &searchsvc.SearchRequest{ + res, err = s.searcher.Search(ctx, &searchsvc.SearchRequest{ Query: in.Query, PageSize: in.PageSize, Ref: in.Ref, @@ -169,8 +171,7 @@ func (s Service) Search(ctx context.Context, in *searchsvc.SearchRequest, out *s // IndexSpace (re)indexes all resources of a given space. func (s Service) IndexSpace(ctx context.Context, in *searchsvc.IndexSpaceRequest, _ *searchsvc.IndexSpaceResponse) error { - _, err := s.provider.IndexSpace(ctx, in) - return err + return s.searcher.IndexSpace(&provider.StorageSpaceId{OpaqueId: in.SpaceId}, &user.UserId{OpaqueId: in.UserId}) } // FromCache pulls a search result from cache