From 9426a5a45d4c2fd07f84261f6d602680e79cdc48 Mon Sep 17 00:00:00 2001 From: Chris Cotter Date: Wed, 21 Jun 2023 16:18:14 -0400 Subject: [PATCH] feat(storage): add support for MatchGlob (#8097) Adds support for MatchGlob in listing objects. Only JSON is supported because the parameter is not yet available via the gRPC API. Updates #7727 Supercedes #7728 --- storage/grpc_client.go | 5 +++ storage/http_client.go | 1 + storage/integration_test.go | 62 +++++++++++++++++++++++++++++++++++++ storage/storage.go | 12 +++++-- 4 files changed, 78 insertions(+), 2 deletions(-) diff --git a/storage/grpc_client.go b/storage/grpc_client.go index 26a87fad09d1..60f192299fe3 100644 --- a/storage/grpc_client.go +++ b/storage/grpc_client.go @@ -415,6 +415,11 @@ func (c *grpcStorageClient) ListObjects(ctx context.Context, bucket string, q *Q } gitr := c.raw.ListObjects(it.ctx, req, s.gax...) fetch := func(pageSize int, pageToken string) (token string, err error) { + // MatchGlob not yet supported for gRPC. + // TODO: add support when b/287306063 resolved. + if q != nil && q.MatchGlob != "" { + return "", status.Errorf(codes.Unimplemented, "MatchGlob is not supported for gRPC") + } var objects []*storagepb.Object err = run(it.ctx, func() error { objects, token, err = gitr.InternalFetch(pageSize, pageToken) diff --git a/storage/http_client.go b/storage/http_client.go index 263b077becac..4ca0e02bfd81 100644 --- a/storage/http_client.go +++ b/storage/http_client.go @@ -347,6 +347,7 @@ func (c *httpStorageClient) ListObjects(ctx context.Context, bucket string, q *Q req.EndOffset(it.query.EndOffset) req.Versions(it.query.Versions) req.IncludeTrailingDelimiter(it.query.IncludeTrailingDelimiter) + req.MatchGlob(it.query.MatchGlob) if selection := it.query.toFieldSelection(); selection != "" { req.Fields("nextPageToken", googleapi.Field(selection)) } diff --git a/storage/integration_test.go b/storage/integration_test.go index 7961806aa1b4..d13a0c30ed7c 100644 --- a/storage/integration_test.go +++ b/storage/integration_test.go @@ -1293,6 +1293,68 @@ func TestIntegration_ObjectIteration(t *testing.T) { }) } +func TestIntegration_ObjectIterationMatchGlob(t *testing.T) { + // This is a separate test from the Object Iteration test above because + // MatchGlob is not yet implemented for gRPC. + ctx := skipGRPC("https://github.com/googleapis/google-cloud-go/issues/7727") + multiTransportTest(skipJSONReads(ctx, "no reads in test"), t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) { + // Reset testTime, 'cause object last modification time should be within 5 min + // from test (test iteration if -count passed) start time. + testTime = time.Now().UTC() + newBucketName := prefix + uidSpace.New() + h := testHelper{t} + bkt := client.Bucket(newBucketName).Retryer(WithPolicy(RetryAlways)) + + h.mustCreate(bkt, testutil.ProjID(), nil) + defer func() { + if err := killBucket(ctx, client, newBucketName); err != nil { + log.Printf("deleting %q: %v", newBucketName, err) + } + }() + const defaultType = "text/plain" + + // Populate object names and make a map for their contents. + objects := []string{ + "obj1", + "obj2", + "obj/with/slashes", + "obj/", + "other/obj1", + } + contents := make(map[string][]byte) + + // Test Writer. + for _, obj := range objects { + c := randomContents() + if err := writeObject(ctx, bkt.Object(obj), defaultType, c); err != nil { + t.Errorf("Write for %v failed with %v", obj, err) + } + contents[obj] = c + } + query := &Query{MatchGlob: "**obj1"} + + var gotNames []string + it := bkt.Objects(context.Background(), query) + for { + attrs, err := it.Next() + if err == iterator.Done { + break + } + if err != nil { + t.Fatalf("iterator.Next: %v", err) + } + if attrs.Name != "" { + gotNames = append(gotNames, attrs.Name) + } + } + + sortedNames := []string{"obj1", "other/obj1"} + if !cmp.Equal(sortedNames, gotNames) { + t.Errorf("names = %v, want %v", gotNames, sortedNames) + } + }) +} + func TestIntegration_ObjectUpdate(t *testing.T) { ctx := skipJSONReads(context.Background(), "no reads in test") multiTransportTest(ctx, t, func(t *testing.T, ctx context.Context, bucket string, _ string, client *Client) { diff --git a/storage/storage.go b/storage/storage.go index c2babd58594f..d76546676086 100644 --- a/storage/storage.go +++ b/storage/storage.go @@ -1486,6 +1486,8 @@ type Query struct { // aside from the prefix, contain delimiter will have their name, // truncated after the delimiter, returned in prefixes. // Duplicate prefixes are omitted. + // Must be set to / when used with the MatchGlob parameter to filter results + // in a directory-like mode. // Optional. Delimiter string @@ -1499,9 +1501,9 @@ type Query struct { Versions bool // attrSelection is used to select only specific fields to be returned by - // the query. It is set by the user calling calling SetAttrSelection. These + // the query. It is set by the user calling SetAttrSelection. These // are used by toFieldMask and toFieldSelection for gRPC and HTTP/JSON - // clients repsectively. + // clients respectively. attrSelection []string // StartOffset is used to filter results to objects whose names are @@ -1527,6 +1529,12 @@ type Query struct { // true, they will also be included as objects and their metadata will be // populated in the returned ObjectAttrs. IncludeTrailingDelimiter bool + + // MatchGlob is a glob pattern used to filter results (for example, foo*bar). See + // https://cloud.google.com/storage/docs/json_api/v1/objects/list#list-object-glob + // for syntax details. When Delimiter is set in conjunction with MatchGlob, + // it must be set to /. + MatchGlob string } // attrToFieldMap maps the field names of ObjectAttrs to the underlying field