Skip to content

Commit

Permalink
Merge branch 'main' of https://github.com/flipt-io/flipt
Browse files Browse the repository at this point in the history 
* 'main' of https://github.com/flipt-io/flipt:
  fix: bucket prefix (#3808)
  • Loading branch information
@markphelps
markphelps committed Jan 17, 2025
2 parents 6ea664c + cb278c8 commit b69cfb6
Show file tree
Hide file tree
Showing 4 changed files with 36 additions and 32 deletions.
14 changes: 14 additions & 0 deletions go.work.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -494,6 +494,7 @@ cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3f
cloud.google.com/go/storage v1.35.1/go.mod h1:M6M/3V/D3KpzMTJyPOR/HU6n2Si5QdaXYEsng2xgOs8=
cloud.google.com/go/storage v1.39.1/go.mod h1:xK6xZmxZmo+fyP7+DEF6FhNc24/JAe95OLyOHCXFH1o=
cloud.google.com/go/storage v1.43.0/go.mod h1:ajvxEa7WmZS1PxvKRq4bq0tFT3vMd502JwstCcYv0Q0=
cloud.google.com/go/storage v1.49.0/go.mod h1:k1eHhhpLvrPjVGfo0mOUPEJ4Y2+a/Hv5PiwehZI9qGU=
cloud.google.com/go/storagetransfer v1.10.5/go.mod h1:086WXPZlWXLfql+/nlmcc8ZzFWvITqfSGUQyMdf5eBk=
cloud.google.com/go/storagetransfer v1.10.10/go.mod h1:8+nX+WgQ2ZJJnK8e+RbK/zCXk8T7HdwyQAJeY7cEcm0=
cloud.google.com/go/storagetransfer v1.11.0/go.mod h1:arcvgzVC4HPcSikqV8D4h4PwrvGQHfKtbL4OwKPirjs=
Expand Down Expand Up @@ -1062,6 +1063,11 @@ github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.
github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE=
github.com/envoyproxy/go-control-plane v0.12.0/go.mod h1:ZBTaoJ23lqITozF0M6G4/IragXCQKCnYbmlmtHvwRG0=
github.com/envoyproxy/go-control-plane v0.12.1-0.20240621013728-1eb8caab5155/go.mod h1:5Wkq+JduFtdAXihLmeTJf+tRYIT4KBc2vPXDhwVo1pA=
github.com/envoyproxy/go-control-plane v0.13.4 h1:zEqyPVyku6IvWCFwux4x9RxkLOMUL+1vC9xUFv5l2/M=
github.com/envoyproxy/go-control-plane v0.13.4/go.mod h1:kDfuBlDVsSj2MjrLEtRWtHlsWIFcGyB2RMO44Dc5GZA=
github.com/envoyproxy/go-control-plane/envoy v1.32.3 h1:hVEaommgvzTjTd4xCaFd+kEQ2iYBtGxP6luyLrx6uOk=
github.com/envoyproxy/go-control-plane/envoy v1.32.3/go.mod h1:F6hWupPfh75TBXGKA++MCT/CZHFq5r9/uwt/kQYkZfE=
github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4=
github.com/envoyproxy/protoc-gen-validate v0.0.14/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/envoyproxy/protoc-gen-validate v1.0.4/go.mod h1:qys6tmnRsYrQqIhm2bvKZH4Blx/1gTIZ2UKVY1M+Yew=
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM=
Expand Down Expand Up @@ -1253,6 +1259,7 @@ github.com/googleapis/gax-go/v2 v2.12.2/go.mod h1:61M8vcyyXR2kqKFxKrfA22jaA8JGF7
github.com/googleapis/gax-go/v2 v2.12.3/go.mod h1:AKloxT6GtNbaLm8QTNSidHUVsHYcBHwWRvkNFJUQcS4=
github.com/googleapis/gax-go/v2 v2.12.5/go.mod h1:BUDKcWo+RaKq5SC9vVYL0wLADa3VcfswbOMMRmB9H3E=
github.com/googleapis/gax-go/v2 v2.13.0/go.mod h1:Z/fvTZXF8/uw7Xu5GuslPw+bplx6SS338j1Is2S+B7A=
github.com/googleapis/gax-go/v2 v2.14.0/go.mod h1:lhBCnjdLrWRaPvLWhmc8IS24m9mr07qSYnHncrgo+zk=
github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg=
github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU=
github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA=
Expand Down Expand Up @@ -1288,6 +1295,7 @@ github.com/grpc-ecosystem/grpc-gateway v1.12.1/go.mod h1:8XEsbTttt/W+VvjtQhLACqC
github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0/go.mod h1:qztMSjm835F2bXf+5HKAPIS5qsmQDqZna/PgVt4rWtI=
github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c/go.mod h1:NMPJylDgVpX0MLRlPy15sqSwOFv/U1GZ2m21JhFfek0=
github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4=
github.com/hanwen/go-fuse/v2 v2.4.0/go.mod h1:xKwi1cF7nXAOBCXujD5ie0ZKsxc8GGSA1rlMJc+8IJs=
Expand Down Expand Up @@ -1637,6 +1645,7 @@ github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5/go.mod h1:iIss55rK
github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg=
github.com/pkg/sftp v1.13.6/go.mod h1:tz1ryNURKu77RL+GuCzmoJYxQczL3wLNNpPWagdg4Qk=
github.com/pkg/sftp v1.13.7/go.mod h1:KMKI0t3T6hfA+lTR/ssZdunHo+uwq7ghoN09/FSu3DY=
github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s=
Expand Down Expand Up @@ -2036,6 +2045,7 @@ golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt7
golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U=
golang.org/x/crypto v0.29.0/go.mod h1:+F4F4N5hv6v38hfeYwTdx20oUvLLc+QfrE9Ax9HtgRg=
golang.org/x/crypto v0.30.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
Expand Down Expand Up @@ -2420,6 +2430,8 @@ google.golang.org/api v0.197.0/go.mod h1:AuOuo20GoQ331nq7DquGHlU6d+2wN2fZ8O0ta60
google.golang.org/api v0.203.0/go.mod h1:BuOVyCSYEPwJb3npWvDnNmFI92f3GeRnHNkETneT3SI=
google.golang.org/api v0.205.0/go.mod h1:NrK1EMqO8Xk6l6QwRAmrXXg2v6dzukhlOyvkYtnvUuc=
google.golang.org/api v0.210.0/go.mod h1:B9XDZGnx2NtyjzVkOVTGrFSAVZgPcbedzKg/gTLwqBs=
google.golang.org/api v0.211.0/go.mod h1:XOloB4MXFH4UTlQSGuNUxw0UT74qdENK8d6JNsXKLi0=
google.golang.org/api v0.214.0/go.mod h1:bYPpLG8AyeMWwDU6NXoB00xC0DFkikVvd5MfwoxjLqE=
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
google.golang.org/appengine v1.6.2/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
Expand Down Expand Up @@ -2523,12 +2535,14 @@ google.golang.org/genproto/googleapis/api v0.0.0-20241015192408-796eee8c2d53/go.
google.golang.org/genproto/googleapis/api v0.0.0-20241104194629-dd2ea8efbc28/go.mod h1:dguCy7UOdZhTvLzDyt15+rOrawrpM4q7DD9dQ1P11P4=
google.golang.org/genproto/googleapis/api v0.0.0-20241118233622-e639e219e697/go.mod h1:+D9ySVjN8nY8YCVjc5O7PZDIdZporIDY3KaGfJunh88=
google.golang.org/genproto/googleapis/api v0.0.0-20241206012308-a4fef0638583/go.mod h1:jehYqy3+AhJU9ve55aNOaSml7wUXjF9x6z2LcCfpAhY=
google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576/go.mod h1:1R3kvZ1dtP3+4p4d3G8uJ8rFk/fWlScl38vanWACI08=
google.golang.org/genproto/googleapis/bytestream v0.0.0-20240429193739-8cf5692501f6/go.mod h1:ULqtoQMxDLNRfW+pJbKA68wtIy1OiYjdIsJs3PMpzh8=
google.golang.org/genproto/googleapis/bytestream v0.0.0-20240730163845-b1a4ccb954bf/go.mod h1:5/MT647Cn/GGhwTpXC7QqcaR5Cnee4v4MKCU1/nwnIQ=
google.golang.org/genproto/googleapis/bytestream v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:q0eWNnCW04EJlyrmLT+ZHsjuoUiZ36/eAEdCCezZoco=
google.golang.org/genproto/googleapis/bytestream v0.0.0-20241015192408-796eee8c2d53/go.mod h1:T8O3fECQbif8cez15vxAcjbwXxvL2xbnvbQ7ZfiMAMs=
google.golang.org/genproto/googleapis/bytestream v0.0.0-20241118233622-e639e219e697/go.mod h1:qUsLYwbwz5ostUWtuFuXPlHmSJodC5NI/88ZlHj4M1o=
google.golang.org/genproto/googleapis/bytestream v0.0.0-20241209162323-e6fa225c2576/go.mod h1:qUsLYwbwz5ostUWtuFuXPlHmSJodC5NI/88ZlHj4M1o=
google.golang.org/genproto/googleapis/bytestream v0.0.0-20241223144023-3abc09e42ca8/go.mod h1:bLYPejkLzwgJuAHlIk1gdPOlx9CUYXLZi2rZxL/ursM=
google.golang.org/genproto/googleapis/rpc v0.0.0-20230731190214-cbb8c96f2d6d/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM=
google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5/go.mod h1:zBEcrKX2ZOcEkHWxBPAIvYUWOKKMIhYcmNiUIu2ji3I=
google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97/go.mod h1:v7nGkzlmW8P3n/bKmWBn2WpBjpOEx8Q6gMueudAmKfY=
Expand Down
23 changes: 5 additions & 18 deletions internal/storage/fs/object/store.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ import (
"errors"
"io"
"io/fs"
"strings"
"sync"
"time"

Expand All @@ -26,7 +25,6 @@ type SnapshotStore struct {
logger *zap.Logger
scheme string
bucket *gcblob.Bucket
prefix string
pollOpts []containers.Option[storagefs.Poller]

mu sync.RWMutex
Expand Down Expand Up @@ -58,13 +56,6 @@ func NewSnapshotStore(ctx context.Context, logger *zap.Logger, scheme string, bu
return s, nil
}

// WithPrefix configures the prefix for object store
func WithPrefix(prefix string) containers.Option[SnapshotStore] {
return func(s *SnapshotStore) {
s.prefix = prefix
}
}

// WithPollOptions configures the poller options used when periodically updating snapshot state
func WithPollOptions(opts ...containers.Option[storagefs.Poller]) containers.Option[SnapshotStore] {
return func(s *SnapshotStore) {
Expand Down Expand Up @@ -105,9 +96,7 @@ func (s *SnapshotStore) build(ctx context.Context) (*storagefs.Snapshot, error)
return nil, err
}

iterator := s.bucket.List(&gcblob.ListOptions{
Prefix: s.prefix,
})
iterator := s.bucket.List(&gcblob.ListOptions{})

var files []fs.File
for {
Expand All @@ -119,18 +108,17 @@ func (s *SnapshotStore) build(ctx context.Context) (*storagefs.Snapshot, error)
return nil, err
}

key := strings.TrimPrefix(item.Key, s.prefix)
if !idx.Match(key) {
if !idx.Match(item.Key) {
continue
}

rd, err := s.bucket.NewReader(ctx, s.prefix+key, &gcblob.ReaderOptions{})
rd, err := s.bucket.NewReader(ctx, item.Key, &gcblob.ReaderOptions{})
if err != nil {
return nil, err
}

files = append(files, NewFile(
key,
item.Key,
item.Size,
rd,
item.ModTime,
Expand All @@ -142,7 +130,7 @@ func (s *SnapshotStore) build(ctx context.Context) (*storagefs.Snapshot, error)
}

func (s *SnapshotStore) getIndex(ctx context.Context) (*storagefs.FliptIndex, error) {
rd, err := s.bucket.NewReader(ctx, s.prefix+storagefs.IndexFileName, &gcblob.ReaderOptions{})
rd, err := s.bucket.NewReader(ctx, storagefs.IndexFileName, &gcblob.ReaderOptions{})

if err != nil {
if gcerrors.Code(err) != gcerrors.NotFound {
Expand All @@ -161,5 +149,4 @@ func (s *SnapshotStore) getIndex(ctx context.Context) (*storagefs.FliptIndex, er
}

return idx, nil

}
18 changes: 8 additions & 10 deletions internal/storage/fs/object/store_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -219,17 +219,15 @@ flags:

require.NoError(t, store.View(ctx, func(s storage.ReadOnlyStore) error {
_, err = s.GetNamespace(context.TODO(), storage.NewNamespace("production"))
if err != nil {
return err
}
require.NoError(t, err)

_, err = s.GetFlag(context.TODO(), storage.NewResource("production", "foo"))
if err != nil {
return err
}
require.NoError(t, err)

_, err = s.GetNamespace(context.TODO(), storage.NewNamespace("prefix"))
return err
require.NoError(t, err)

return nil
}))
})

Expand All @@ -242,6 +240,8 @@ flags:
bucket, err := OpenBucket(ctx, u)
require.NoError(t, err)

bucket = gcblob.PrefixedBucket(bucket, "prefix/")

t.Cleanup(func() { _ = bucket.Close() })

writeTestDataToBucket(t, ctx, bucket)
Expand All @@ -253,7 +253,6 @@ flags:
zaptest.NewLogger(t),
u.Scheme,
bucket,
WithPrefix("prefix"),
)
require.NoError(t, err)

Expand All @@ -265,8 +264,7 @@ flags:
// however, both should be empty
require.NoError(t, store.View(ctx, func(s storage.ReadOnlyStore) error {
_, err := s.GetNamespace(ctx, storage.NewNamespace("production"))
require.Error(t, err, "should not exist in prefixed bucket")

require.NoError(t, err)
_, err = s.GetFlag(ctx, storage.NewResource("production", "foo"))
require.Error(t, err, "flag should not be defined yet")

Expand Down
13 changes: 9 additions & 4 deletions internal/storage/fs/store/store.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"go.flipt.io/flipt/internal/storage/fs/object"
storageoci "go.flipt.io/flipt/internal/storage/fs/oci"
"go.uber.org/zap"
"gocloud.dev/blob"
"gocloud.dev/blob/azureblob"
"gocloud.dev/blob/gcsblob"
"gocloud.dev/blob/s3blob"
Expand Down Expand Up @@ -179,6 +180,7 @@ func newObjectStore(ctx context.Context, cfg *config.Config, logger *zap.Logger)
scheme string
bucketName string
values = url.Values{}
prefix string
)
// keep this as a case statement in anticipation of
// more object types in the future
Expand All @@ -196,14 +198,13 @@ func newObjectStore(ctx context.Context, cfg *config.Config, logger *zap.Logger)
}

if ocfg.S3.Prefix != "" {
values.Set("prefix", ocfg.S3.Prefix)
prefix = ocfg.S3.Prefix
}

opts = append(opts,
object.WithPollOptions(
storagefs.WithInterval(ocfg.S3.PollInterval),
),
object.WithPrefix(ocfg.S3.Prefix),
)
case config.AZBlobObjectSubStorageType:
scheme = azureblob.Scheme
Expand All @@ -226,15 +227,15 @@ func newObjectStore(ctx context.Context, cfg *config.Config, logger *zap.Logger)
case config.GSBlobObjectSubStorageType:
scheme = gcsblob.Scheme
bucketName = ocfg.GS.Bucket

if ocfg.GS.Prefix != "" {
values.Set("prefix", ocfg.GS.Prefix)
prefix = ocfg.GS.Prefix
}

opts = append(opts,
object.WithPollOptions(
storagefs.WithInterval(ocfg.GS.PollInterval),
),
object.WithPrefix(ocfg.GS.Prefix),
)
default:
return nil, fmt.Errorf("unexpected object storage subtype: %q", ocfg.Type)
Expand All @@ -251,6 +252,10 @@ func newObjectStore(ctx context.Context, cfg *config.Config, logger *zap.Logger)
return nil, err
}

if prefix != "" {
bucket = blob.PrefixedBucket(bucket, prefix)
}

snap, err := object.NewSnapshotStore(ctx, logger, scheme, bucket, opts...)
if err != nil {
return nil, err
Expand Down

0 comments on commit b69cfb6

Please sign in to comment.