From cb278c838b7848bb6ade1808a03745206d896c15 Mon Sep 17 00:00:00 2001 From: Mark Phelps <209477+markphelps@users.noreply.github.com> Date: Fri, 17 Jan 2025 10:50:35 -0500 Subject: [PATCH] fix: bucket prefix (#3808) * chore: prep 1.54 for release Signed-off-by: Mark Phelps <209477+markphelps@users.noreply.github.com> * chore: update sdk/go Signed-off-by: Mark Phelps <209477+markphelps@users.noreply.github.com> * chore: mod tidy Signed-off-by: Mark Phelps <209477+markphelps@users.noreply.github.com> * chore: update changelog Signed-off-by: Mark Phelps <209477+markphelps@users.noreply.github.com> * fix: issue with bucket prefix and gcp Signed-off-by: Mark Phelps <209477+markphelps@users.noreply.github.com> * chore: fix storage test Signed-off-by: Mark Phelps <209477+markphelps@users.noreply.github.com> * chore: only add / to prefix if prefix exists Signed-off-by: Mark Phelps <209477+markphelps@users.noreply.github.com> * chore: rm / adding to prefix Signed-off-by: Mark Phelps <209477+markphelps@users.noreply.github.com> --------- Signed-off-by: Mark Phelps <209477+markphelps@users.noreply.github.com> --- go.work.sum | 14 ++++++++++++++ internal/storage/fs/object/store.go | 23 +++++------------------ internal/storage/fs/object/store_test.go | 18 ++++++++---------- internal/storage/fs/store/store.go | 13 +++++++++---- 4 files changed, 36 insertions(+), 32 deletions(-) diff --git a/go.work.sum b/go.work.sum index 8b513af529..5e7f68a056 100644 --- a/go.work.sum +++ b/go.work.sum @@ -494,6 +494,7 @@ cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3f cloud.google.com/go/storage v1.35.1/go.mod h1:M6M/3V/D3KpzMTJyPOR/HU6n2Si5QdaXYEsng2xgOs8= cloud.google.com/go/storage v1.39.1/go.mod h1:xK6xZmxZmo+fyP7+DEF6FhNc24/JAe95OLyOHCXFH1o= cloud.google.com/go/storage v1.43.0/go.mod h1:ajvxEa7WmZS1PxvKRq4bq0tFT3vMd502JwstCcYv0Q0= +cloud.google.com/go/storage v1.49.0/go.mod h1:k1eHhhpLvrPjVGfo0mOUPEJ4Y2+a/Hv5PiwehZI9qGU= cloud.google.com/go/storagetransfer v1.10.5/go.mod h1:086WXPZlWXLfql+/nlmcc8ZzFWvITqfSGUQyMdf5eBk= cloud.google.com/go/storagetransfer v1.10.10/go.mod h1:8+nX+WgQ2ZJJnK8e+RbK/zCXk8T7HdwyQAJeY7cEcm0= cloud.google.com/go/storagetransfer v1.11.0/go.mod h1:arcvgzVC4HPcSikqV8D4h4PwrvGQHfKtbL4OwKPirjs= @@ -1062,6 +1063,11 @@ github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go. github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/go-control-plane v0.12.0/go.mod h1:ZBTaoJ23lqITozF0M6G4/IragXCQKCnYbmlmtHvwRG0= github.com/envoyproxy/go-control-plane v0.12.1-0.20240621013728-1eb8caab5155/go.mod h1:5Wkq+JduFtdAXihLmeTJf+tRYIT4KBc2vPXDhwVo1pA= +github.com/envoyproxy/go-control-plane v0.13.4 h1:zEqyPVyku6IvWCFwux4x9RxkLOMUL+1vC9xUFv5l2/M= +github.com/envoyproxy/go-control-plane v0.13.4/go.mod h1:kDfuBlDVsSj2MjrLEtRWtHlsWIFcGyB2RMO44Dc5GZA= +github.com/envoyproxy/go-control-plane/envoy v1.32.3 h1:hVEaommgvzTjTd4xCaFd+kEQ2iYBtGxP6luyLrx6uOk= +github.com/envoyproxy/go-control-plane/envoy v1.32.3/go.mod h1:F6hWupPfh75TBXGKA++MCT/CZHFq5r9/uwt/kQYkZfE= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= github.com/envoyproxy/protoc-gen-validate v0.0.14/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v1.0.4/go.mod h1:qys6tmnRsYrQqIhm2bvKZH4Blx/1gTIZ2UKVY1M+Yew= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM= @@ -1253,6 +1259,7 @@ github.com/googleapis/gax-go/v2 v2.12.2/go.mod h1:61M8vcyyXR2kqKFxKrfA22jaA8JGF7 github.com/googleapis/gax-go/v2 v2.12.3/go.mod h1:AKloxT6GtNbaLm8QTNSidHUVsHYcBHwWRvkNFJUQcS4= github.com/googleapis/gax-go/v2 v2.12.5/go.mod h1:BUDKcWo+RaKq5SC9vVYL0wLADa3VcfswbOMMRmB9H3E= github.com/googleapis/gax-go/v2 v2.13.0/go.mod h1:Z/fvTZXF8/uw7Xu5GuslPw+bplx6SS338j1Is2S+B7A= +github.com/googleapis/gax-go/v2 v2.14.0/go.mod h1:lhBCnjdLrWRaPvLWhmc8IS24m9mr07qSYnHncrgo+zk= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= @@ -1288,6 +1295,7 @@ github.com/grpc-ecosystem/grpc-gateway v1.12.1/go.mod h1:8XEsbTttt/W+VvjtQhLACqC github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0/go.mod h1:qztMSjm835F2bXf+5HKAPIS5qsmQDqZna/PgVt4rWtI= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c/go.mod h1:NMPJylDgVpX0MLRlPy15sqSwOFv/U1GZ2m21JhFfek0= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= github.com/hanwen/go-fuse/v2 v2.4.0/go.mod h1:xKwi1cF7nXAOBCXujD5ie0ZKsxc8GGSA1rlMJc+8IJs= @@ -1637,6 +1645,7 @@ github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5/go.mod h1:iIss55rK github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pkg/sftp v1.13.6/go.mod h1:tz1ryNURKu77RL+GuCzmoJYxQczL3wLNNpPWagdg4Qk= +github.com/pkg/sftp v1.13.7/go.mod h1:KMKI0t3T6hfA+lTR/ssZdunHo+uwq7ghoN09/FSu3DY= github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= @@ -2036,6 +2045,7 @@ golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt7 golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= golang.org/x/crypto v0.29.0/go.mod h1:+F4F4N5hv6v38hfeYwTdx20oUvLLc+QfrE9Ax9HtgRg= golang.org/x/crypto v0.30.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= @@ -2420,6 +2430,8 @@ google.golang.org/api v0.197.0/go.mod h1:AuOuo20GoQ331nq7DquGHlU6d+2wN2fZ8O0ta60 google.golang.org/api v0.203.0/go.mod h1:BuOVyCSYEPwJb3npWvDnNmFI92f3GeRnHNkETneT3SI= google.golang.org/api v0.205.0/go.mod h1:NrK1EMqO8Xk6l6QwRAmrXXg2v6dzukhlOyvkYtnvUuc= google.golang.org/api v0.210.0/go.mod h1:B9XDZGnx2NtyjzVkOVTGrFSAVZgPcbedzKg/gTLwqBs= +google.golang.org/api v0.211.0/go.mod h1:XOloB4MXFH4UTlQSGuNUxw0UT74qdENK8d6JNsXKLi0= +google.golang.org/api v0.214.0/go.mod h1:bYPpLG8AyeMWwDU6NXoB00xC0DFkikVvd5MfwoxjLqE= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.2/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= @@ -2523,12 +2535,14 @@ google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53/go. google.golang.org/genproto/googleapis/api v0.0.0-20241104194629-dd2ea8efbc28/go.mod h1:dguCy7UOdZhTvLzDyt15+rOrawrpM4q7DD9dQ1P11P4= google.golang.org/genproto/googleapis/api v0.0.0-20241118233622-e639e219e697/go.mod h1:+D9ySVjN8nY8YCVjc5O7PZDIdZporIDY3KaGfJunh88= google.golang.org/genproto/googleapis/api v0.0.0-20241206012308-a4fef0638583/go.mod h1:jehYqy3+AhJU9ve55aNOaSml7wUXjF9x6z2LcCfpAhY= +google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576/go.mod h1:1R3kvZ1dtP3+4p4d3G8uJ8rFk/fWlScl38vanWACI08= google.golang.org/genproto/googleapis/bytestream v0.0.0-20240429193739-8cf5692501f6/go.mod h1:ULqtoQMxDLNRfW+pJbKA68wtIy1OiYjdIsJs3PMpzh8= google.golang.org/genproto/googleapis/bytestream v0.0.0-20240730163845-b1a4ccb954bf/go.mod h1:5/MT647Cn/GGhwTpXC7QqcaR5Cnee4v4MKCU1/nwnIQ= google.golang.org/genproto/googleapis/bytestream v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:q0eWNnCW04EJlyrmLT+ZHsjuoUiZ36/eAEdCCezZoco= google.golang.org/genproto/googleapis/bytestream v0.0.0-20241015192408-796eee8c2d53/go.mod h1:T8O3fECQbif8cez15vxAcjbwXxvL2xbnvbQ7ZfiMAMs= google.golang.org/genproto/googleapis/bytestream v0.0.0-20241118233622-e639e219e697/go.mod h1:qUsLYwbwz5ostUWtuFuXPlHmSJodC5NI/88ZlHj4M1o= google.golang.org/genproto/googleapis/bytestream v0.0.0-20241209162323-e6fa225c2576/go.mod h1:qUsLYwbwz5ostUWtuFuXPlHmSJodC5NI/88ZlHj4M1o= +google.golang.org/genproto/googleapis/bytestream v0.0.0-20241223144023-3abc09e42ca8/go.mod h1:bLYPejkLzwgJuAHlIk1gdPOlx9CUYXLZi2rZxL/ursM= google.golang.org/genproto/googleapis/rpc v0.0.0-20230731190214-cbb8c96f2d6d/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM= google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5/go.mod h1:zBEcrKX2ZOcEkHWxBPAIvYUWOKKMIhYcmNiUIu2ji3I= google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97/go.mod h1:v7nGkzlmW8P3n/bKmWBn2WpBjpOEx8Q6gMueudAmKfY= diff --git a/internal/storage/fs/object/store.go b/internal/storage/fs/object/store.go index 4175c78b84..695831f9e4 100644 --- a/internal/storage/fs/object/store.go +++ b/internal/storage/fs/object/store.go @@ -6,7 +6,6 @@ import ( "errors" "io" "io/fs" - "strings" "sync" "time" @@ -26,7 +25,6 @@ type SnapshotStore struct { logger *zap.Logger scheme string bucket *gcblob.Bucket - prefix string pollOpts []containers.Option[storagefs.Poller] mu sync.RWMutex @@ -58,13 +56,6 @@ func NewSnapshotStore(ctx context.Context, logger *zap.Logger, scheme string, bu return s, nil } -// WithPrefix configures the prefix for object store -func WithPrefix(prefix string) containers.Option[SnapshotStore] { - return func(s *SnapshotStore) { - s.prefix = prefix - } -} - // WithPollOptions configures the poller options used when periodically updating snapshot state func WithPollOptions(opts ...containers.Option[storagefs.Poller]) containers.Option[SnapshotStore] { return func(s *SnapshotStore) { @@ -105,9 +96,7 @@ func (s *SnapshotStore) build(ctx context.Context) (*storagefs.Snapshot, error) return nil, err } - iterator := s.bucket.List(&gcblob.ListOptions{ - Prefix: s.prefix, - }) + iterator := s.bucket.List(&gcblob.ListOptions{}) var files []fs.File for { @@ -119,18 +108,17 @@ func (s *SnapshotStore) build(ctx context.Context) (*storagefs.Snapshot, error) return nil, err } - key := strings.TrimPrefix(item.Key, s.prefix) - if !idx.Match(key) { + if !idx.Match(item.Key) { continue } - rd, err := s.bucket.NewReader(ctx, s.prefix+key, &gcblob.ReaderOptions{}) + rd, err := s.bucket.NewReader(ctx, item.Key, &gcblob.ReaderOptions{}) if err != nil { return nil, err } files = append(files, NewFile( - key, + item.Key, item.Size, rd, item.ModTime, @@ -142,7 +130,7 @@ func (s *SnapshotStore) build(ctx context.Context) (*storagefs.Snapshot, error) } func (s *SnapshotStore) getIndex(ctx context.Context) (*storagefs.FliptIndex, error) { - rd, err := s.bucket.NewReader(ctx, s.prefix+storagefs.IndexFileName, &gcblob.ReaderOptions{}) + rd, err := s.bucket.NewReader(ctx, storagefs.IndexFileName, &gcblob.ReaderOptions{}) if err != nil { if gcerrors.Code(err) != gcerrors.NotFound { @@ -161,5 +149,4 @@ func (s *SnapshotStore) getIndex(ctx context.Context) (*storagefs.FliptIndex, er } return idx, nil - } diff --git a/internal/storage/fs/object/store_test.go b/internal/storage/fs/object/store_test.go index 6c890bc344..d2dbe9e285 100644 --- a/internal/storage/fs/object/store_test.go +++ b/internal/storage/fs/object/store_test.go @@ -219,17 +219,15 @@ flags: require.NoError(t, store.View(ctx, func(s storage.ReadOnlyStore) error { _, err = s.GetNamespace(context.TODO(), storage.NewNamespace("production")) - if err != nil { - return err - } + require.NoError(t, err) _, err = s.GetFlag(context.TODO(), storage.NewResource("production", "foo")) - if err != nil { - return err - } + require.NoError(t, err) _, err = s.GetNamespace(context.TODO(), storage.NewNamespace("prefix")) - return err + require.NoError(t, err) + + return nil })) }) @@ -242,6 +240,8 @@ flags: bucket, err := OpenBucket(ctx, u) require.NoError(t, err) + bucket = gcblob.PrefixedBucket(bucket, "prefix/") + t.Cleanup(func() { _ = bucket.Close() }) writeTestDataToBucket(t, ctx, bucket) @@ -253,7 +253,6 @@ flags: zaptest.NewLogger(t), u.Scheme, bucket, - WithPrefix("prefix"), ) require.NoError(t, err) @@ -265,8 +264,7 @@ flags: // however, both should be empty require.NoError(t, store.View(ctx, func(s storage.ReadOnlyStore) error { _, err := s.GetNamespace(ctx, storage.NewNamespace("production")) - require.Error(t, err, "should not exist in prefixed bucket") - + require.NoError(t, err) _, err = s.GetFlag(ctx, storage.NewResource("production", "foo")) require.Error(t, err, "flag should not be defined yet") diff --git a/internal/storage/fs/store/store.go b/internal/storage/fs/store/store.go index 4886451d7f..adce939ed7 100644 --- a/internal/storage/fs/store/store.go +++ b/internal/storage/fs/store/store.go @@ -21,6 +21,7 @@ import ( "go.flipt.io/flipt/internal/storage/fs/object" storageoci "go.flipt.io/flipt/internal/storage/fs/oci" "go.uber.org/zap" + "gocloud.dev/blob" "gocloud.dev/blob/azureblob" "gocloud.dev/blob/gcsblob" "gocloud.dev/blob/s3blob" @@ -179,6 +180,7 @@ func newObjectStore(ctx context.Context, cfg *config.Config, logger *zap.Logger) scheme string bucketName string values = url.Values{} + prefix string ) // keep this as a case statement in anticipation of // more object types in the future @@ -196,14 +198,13 @@ func newObjectStore(ctx context.Context, cfg *config.Config, logger *zap.Logger) } if ocfg.S3.Prefix != "" { - values.Set("prefix", ocfg.S3.Prefix) + prefix = ocfg.S3.Prefix } opts = append(opts, object.WithPollOptions( storagefs.WithInterval(ocfg.S3.PollInterval), ), - object.WithPrefix(ocfg.S3.Prefix), ) case config.AZBlobObjectSubStorageType: scheme = azureblob.Scheme @@ -226,15 +227,15 @@ func newObjectStore(ctx context.Context, cfg *config.Config, logger *zap.Logger) case config.GSBlobObjectSubStorageType: scheme = gcsblob.Scheme bucketName = ocfg.GS.Bucket + if ocfg.GS.Prefix != "" { - values.Set("prefix", ocfg.GS.Prefix) + prefix = ocfg.GS.Prefix } opts = append(opts, object.WithPollOptions( storagefs.WithInterval(ocfg.GS.PollInterval), ), - object.WithPrefix(ocfg.GS.Prefix), ) default: return nil, fmt.Errorf("unexpected object storage subtype: %q", ocfg.Type) @@ -251,6 +252,10 @@ func newObjectStore(ctx context.Context, cfg *config.Config, logger *zap.Logger) return nil, err } + if prefix != "" { + bucket = blob.PrefixedBucket(bucket, prefix) + } + snap, err := object.NewSnapshotStore(ctx, logger, scheme, bucket, opts...) if err != nil { return nil, err