From 0fc64613017ad6417d142596168ace12a5ece7f4 Mon Sep 17 00:00:00 2001 From: Shubham Vernekar Date: Mon, 20 Nov 2023 22:45:29 +0530 Subject: [PATCH] Added AWS cloud storage functionality * Added AWS cloud functionality * adding init flow * error fix * read aws default creds directly via sdk --- client/client.go | 30 ++++- client/{gcs_service.go => cloud_service.go} | 12 +- client/service_test.go | 9 +- cmd/dolores/init.go | 77 ++++++++--- config/config.go | 41 ++++-- go.mod | 29 +++- go.sum | 56 ++++++-- store/aws/aws.go | 141 ++++++++++++++++++++ store/cloud/object.go | 10 ++ store/google/gcs.go | 15 +-- 10 files changed, 348 insertions(+), 72 deletions(-) rename client/{gcs_service.go => cloud_service.go} (94%) create mode 100644 store/aws/aws.go create mode 100644 store/cloud/object.go diff --git a/client/client.go b/client/client.go index 10b2a57..90aef1b 100644 --- a/client/client.go +++ b/client/client.go @@ -8,6 +8,7 @@ import ( "github.com/rs/zerolog" "github.com/rs/zerolog/log" "github.com/scalescape/dolores/config" + "github.com/scalescape/dolores/store/aws" "github.com/scalescape/dolores/store/google" ) @@ -106,12 +107,37 @@ func (c *Client) GetSecretList(_ SecretListConfig) ([]SecretObject, error) { return objs, nil } +func getStore(ctx context.Context, cfg config.Client) (clouldStore, error) { + var store clouldStore + var err error + switch cfg.Provider { + case config.AWS: + { + store, err = aws.NewStore(ctx) + if err != nil { + return nil, err + } + } + case config.GCS: + { + gcfg := google.Config{ServiceAccountFile: cfg.Cloud.ApplicationCredentials} + store, err = google.NewStore(ctx, gcfg) + if err != nil { + return nil, err + } + } + default: + err = fmt.Errorf("failed to get store: %w", config.ErrCloudProviderNotFound) + } + + return store, err +} + func New(ctx context.Context, cfg config.Client) (*Client, error) { if err := cfg.Valid(); err != nil { return nil, err } - gcfg := google.Config{ServiceAccountFile: cfg.Google.ApplicationCredentials} - st, err := google.NewStore(ctx, gcfg) + st, err := getStore(ctx, cfg) if err != nil { return nil, err } diff --git a/client/gcs_service.go b/client/cloud_service.go similarity index 94% rename from client/gcs_service.go rename to client/cloud_service.go index 1301390..bb79c00 100644 --- a/client/gcs_service.go +++ b/client/cloud_service.go @@ -10,7 +10,7 @@ import ( "github.com/rs/zerolog/log" "github.com/scalescape/dolores/config" - "github.com/scalescape/dolores/store/google" + "github.com/scalescape/dolores/store/cloud" ) var ErrInvalidPublicKeys = errors.New("invalid public keys") @@ -18,13 +18,13 @@ var ErrInvalidPublicKeys = errors.New("invalid public keys") const metadataFile = "dolores.md" type Service struct { - store gcsStore + store clouldStore } -type gcsStore interface { +type clouldStore interface { WriteToObject(ctx context.Context, bucketName, fileName string, data []byte) error ReadObject(ctx context.Context, bucketName, fileName string) ([]byte, error) - ListObject(ctx context.Context, bucketName, path string) ([]google.Object, error) + ListObject(ctx context.Context, bucketName, path string) ([]cloud.Object, error) ExistsObject(ctx context.Context, bucketName, fileName string) (bool, error) } @@ -138,7 +138,7 @@ func (s Service) saveObject(ctx context.Context, bucket, fname string, md any) e return s.store.WriteToObject(ctx, bucket, fname, data) } -func (s Service) ListObject(ctx context.Context, bucket, path string) ([]google.Object, error) { +func (s Service) ListObject(ctx context.Context, bucket, path string) ([]cloud.Object, error) { resp, err := s.store.ListObject(ctx, bucket, path) if err != nil { return nil, err @@ -146,6 +146,6 @@ func (s Service) ListObject(ctx context.Context, bucket, path string) ([]google. return resp, nil } -func NewService(st gcsStore) Service { +func NewService(st clouldStore) Service { return Service{store: st} } diff --git a/client/service_test.go b/client/service_test.go index 26bcef0..d138ba4 100644 --- a/client/service_test.go +++ b/client/service_test.go @@ -7,7 +7,7 @@ import ( "github.com/scalescape/dolores/client" "github.com/scalescape/dolores/config" - "github.com/scalescape/dolores/store/google" + "github.com/scalescape/dolores/store/cloud" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" @@ -39,9 +39,9 @@ func (m *mockGCS) ReadObject(ctx context.Context, bucketName, fileName string) ( return args.Get(0).([]byte), args.Error(1) } -func (m *mockGCS) ListObject(ctx context.Context, bucketName, path string) ([]google.Object, error) { +func (m *mockGCS) ListObject(ctx context.Context, bucketName, path string) ([]cloud.Object, error) { args := m.Called(ctx, bucketName, path) - return args.Get(0).([]google.Object), args.Error(1) + return args.Get(0).([]cloud.Object), args.Error(1) } func (m *mockGCS) ExistsObject(ctx context.Context, bucketName, fileName string) (bool, error) { @@ -69,7 +69,8 @@ func (s *serviceSuite) TestShouldNotOverwriteMetadata() { cfg := client.Configuration{ PublicKey: "public_key", Metadata: config.Metadata{Location: "secrets"}, - UserID: "test_user"} + UserID: "test_user", + } s.gcs.On("ExistsObject", mock.AnythingOfType("context.backgroundCtx"), s.bucket, name).Return(true, nil).Once() s.gcs.On("WriteToObject", mock.AnythingOfType("context.backgroundCtx"), s.bucket, "secrets/keys/test_user.key", []byte(cfg.PublicKey)).Return(nil).Once() diff --git a/cmd/dolores/init.go b/cmd/dolores/init.go index f3b89b4..4c43b23 100644 --- a/cmd/dolores/init.go +++ b/cmd/dolores/init.go @@ -41,14 +41,16 @@ func NewInitCommand(newCli GetClient) *cli.Command { } type Input struct { + CloudProvider string `survey:"cloud_provider"` UserID string `survey:"user_id"` Bucket string Location string - ApplicationCredentials string `survey:"google_creds"` + ApplicationCredentials string `survey:"creds"` } func (inp Input) ToMetadata(env string) config.Metadata { return config.Metadata{ + CloudProvider: inp.CloudProvider, Bucket: inp.Bucket, Location: inp.Location, CreatedAt: time.Now(), @@ -57,13 +59,61 @@ func (inp Input) ToMetadata(env string) config.Metadata { } } +func (c *InitCommand) getCred(res *Input) error { + qs := []*survey.Question{} + + switch res.CloudProvider { + case config.GCS: + { + credFile := os.Getenv("GOOGLE_APPLICATION_CREDENTIALS") + if credFile != "" { + qs = append(qs, &survey.Question{ + Name: "creds", + Validate: survey.Required, + Prompt: &survey.Select{ + Message: "Use GOOGLE_APPLICATION_CREDENTIALS env as credentials file", + Options: []string{credFile}, + }, + }) + } else { + qs = append(qs, &survey.Question{ + Name: "creds", + Prompt: &survey.Input{ + Message: "Enter google service account file path", + }, + Validate: survey.Required, + }) + } + } + case config.AWS: + { + res.ApplicationCredentials = "aws_default" + return nil + } + } + + credRes := new(Input) + if err := survey.Ask(qs, credRes); err != nil { + return fmt.Errorf("failed to get appropriate input: %w", err) + } + res.ApplicationCredentials = credRes.ApplicationCredentials + return nil +} + func (c *InitCommand) getData(env string) (*Input, error) { - credFile := os.Getenv("GOOGLE_APPLICATION_CREDENTIALS") qs := []*survey.Question{ + { + Name: "cloud_provider", + Validate: survey.Required, + Prompt: &survey.Select{ + Message: "Select Cloud provider", + Options: []string{config.AWS, config.GCS}, + }, + }, { Name: "bucket", Prompt: &survey.Input{ - Message: "Enter the GCS bucket name:", + Message: "Enter the bucket name:", }, Validate: survey.Required, }, @@ -84,28 +134,13 @@ func (c *InitCommand) getData(env string) (*Input, error) { }, }, } - if credFile != "" { - qs = append(qs, &survey.Question{ - Name: "google_creds", - Validate: survey.Required, - Prompt: &survey.Select{ - Message: "Use GOOGLE_APPLICATION_CREDENTIALS env as credentials file", - Options: []string{credFile}, - }, - }) - } else { - qs = append(qs, &survey.Question{ - Name: "google_creds", - Prompt: &survey.Input{ - Message: "Enter google service account file path", - }, - Validate: survey.Required, - }) - } res := new(Input) if err := survey.Ask(qs, res); err != nil { return nil, fmt.Errorf("failed to get appropriate input: %w", err) } + if err := c.getCred(res); err != nil { + return nil, fmt.Errorf("failed to get appropriate input: %w", err) + } return res, nil } diff --git a/config/config.go b/config/config.go index ea81703..1d636c6 100644 --- a/config/config.go +++ b/config/config.go @@ -12,9 +12,15 @@ import ( ) var ( - ErrInvalidGoogleCreds = errors.New("invalid google application credentials") - ErrInvalidStorageBucket = errors.New("invalid storage bucket") - ErrInvalidKeyFile = errors.New("invalid key file") + ErrInvalidGoogleCreds = errors.New("invalid google application credentials") + ErrInvalidStorageBucket = errors.New("invalid storage bucket") + ErrInvalidKeyFile = errors.New("invalid key file") + ErrCloudProviderNotFound = errors.New("cloud provider not found") +) + +var ( + AWS = "AWS" + GCS = "GCS" ) type CtxKey string @@ -27,13 +33,14 @@ var ( File = filepath.Join(Dir, "dolores.json") ) -type Google struct { +type Cloud struct { ApplicationCredentials string `split_words:"true"` StorageBucket string `split_words:"true"` StoragePrefix string } type Metadata struct { + CloudProvider string `json:"cloud_provider"` Bucket string `json:"bucket"` Location string `json:"location"` Environment string `json:"environment"` @@ -42,18 +49,22 @@ type Metadata struct { } type Client struct { - Google + Cloud + Provider string } func (c Client) BucketName() string { - return c.Google.StorageBucket + return c.Cloud.StorageBucket } func (c Client) Valid() error { - if c.Google.ApplicationCredentials == "" { + if c.Provider == "" { + return ErrCloudProviderNotFound + } + if c.Cloud.ApplicationCredentials == "" { return ErrInvalidGoogleCreds } - if c.Google.StorageBucket == "" { + if c.Cloud.StorageBucket == "" { return ErrInvalidStorageBucket } return nil @@ -65,23 +76,27 @@ func LoadClient(ctx context.Context, env string) (Client, error) { if err != nil { return Client{}, fmt.Errorf("dolores not initialized yet: %w", err) } - if err := envconfig.Process("GOOGLE", &cfg.Google); err != nil { + if err := envconfig.Process("GOOGLE", &cfg.Cloud); err != nil { return Client{}, fmt.Errorf("processing config: %w", err) } md := d.Environments[env].Metadata - if cfg.Google.ApplicationCredentials == "" { + if cloudProvider := md.CloudProvider; cloudProvider != "" { + cfg.Provider = cloudProvider + } + + if cfg.Cloud.ApplicationCredentials == "" { if creds := md.ApplicationCredentials; creds != "" { - cfg.Google.ApplicationCredentials = creds + cfg.Cloud.ApplicationCredentials = creds } } if bucket := md.Bucket; bucket != "" { - cfg.Google.StorageBucket = bucket + cfg.Cloud.StorageBucket = bucket } if location := md.Location; location != "" { - cfg.Google.StoragePrefix = location + cfg.Cloud.StoragePrefix = location } if err := cfg.Valid(); err != nil { diff --git a/go.mod b/go.mod index 2290e1d..256f898 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,8 @@ require ( cloud.google.com/go/storage v1.30.1 filippo.io/age v1.1.1 github.com/AlecAivazis/survey/v2 v2.3.7 + github.com/aws/aws-sdk-go-v2/config v1.23.0 + github.com/aws/aws-sdk-go-v2/service/s3 v1.42.1 github.com/kelseyhightower/envconfig v1.4.0 github.com/rs/zerolog v1.29.1 github.com/scalescape/go-metrics v0.0.0-20230825040750-1888415fe69a @@ -14,11 +16,28 @@ require ( google.golang.org/api v0.129.0 ) +require github.com/aws/aws-sdk-go-v2/credentials v1.15.2 // indirect + require ( cloud.google.com/go v0.110.0 // indirect cloud.google.com/go/compute v1.19.3 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect cloud.google.com/go/iam v0.13.0 // indirect + github.com/aws/aws-sdk-go-v2 v1.22.2 + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.0 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.3 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.2 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.2 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.6.0 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.0 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.2 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.17.1 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.19.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.25.1 // indirect + github.com/aws/smithy-go v1.16.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cactus/go-statsd-client/v5 v5.0.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect @@ -45,12 +64,12 @@ require ( github.com/stretchr/objx v0.5.0 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect go.opencensus.io v0.24.0 // indirect - golang.org/x/crypto v0.10.0 // indirect - golang.org/x/net v0.11.0 // indirect + golang.org/x/crypto v0.14.0 // indirect + golang.org/x/net v0.17.0 // indirect golang.org/x/oauth2 v0.9.0 // indirect - golang.org/x/sys v0.9.0 // indirect - golang.org/x/term v0.9.0 // indirect - golang.org/x/text v0.10.0 // indirect + golang.org/x/sys v0.13.0 // indirect + golang.org/x/term v0.13.0 // indirect + golang.org/x/text v0.13.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc // indirect diff --git a/go.sum b/go.sum index cf068cf..cf2b6b6 100644 --- a/go.sum +++ b/go.sum @@ -55,6 +55,42 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/aws/aws-sdk-go-v2 v1.22.2 h1:lV0U8fnhAnPz8YcdmZVV60+tr6CakHzqA6P8T46ExJI= +github.com/aws/aws-sdk-go-v2 v1.22.2/go.mod h1:Kd0OJtkW3Q0M0lUWGszapWjEvrXDzRW+D21JNsroB+c= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.0 h1:hHgLiIrTRtddC0AKcJr5s7i/hLgcpTt+q/FKxf1Zayk= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.0/go.mod h1:w4I/v3NOWgD+qvs1NPEwhd++1h3XPHFaVxasfY6HlYQ= +github.com/aws/aws-sdk-go-v2/config v1.23.0 h1:kqzEfGGDIrRJpfJckgwuZfFTbU9NB1jZnRcaO9MpOqE= +github.com/aws/aws-sdk-go-v2/config v1.23.0/go.mod h1:p7wbxKXXjS1GGQOss7VXOazVMFF9bjUGq85/4wR/fSw= +github.com/aws/aws-sdk-go-v2/credentials v1.15.2 h1:rKH7khRMxPdD0u3dHecd0Q7NOVw3EUe7AqdkUOkiOGI= +github.com/aws/aws-sdk-go-v2/credentials v1.15.2/go.mod h1:tXM8wmaeAhfC7nZoCxb0FzM/aRaB1m1WQ7x0qlBLq80= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.3 h1:G5KawTAkyHH6WyKQCdHiW4h3PmAXNJpOgwKg3H7sDRE= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.3/go.mod h1:hugKmSFnZB+HgNI1sYGT14BUPZkO6alC/e0AWu+0IAQ= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.2 h1:AaQsr5vvGR7rmeSWBtTCcw16tT9r51mWijuCQhzLnq8= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.2/go.mod h1:o1IiRn7CWocIFTXJjGKJDOwxv1ibL53NpcvcqGWyRBA= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.2 h1:UZx8SXZ0YtzRiALzYAWcjb9Y9hZUR7MBKaBQ5ouOjPs= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.2/go.mod h1:ipuRpcSaklmxR6C39G187TpBAO132gUfleTGccUPs8c= +github.com/aws/aws-sdk-go-v2/internal/ini v1.6.0 h1:hwZB07/beLiCopuRKF0t+dEHmP39iN4YtDh3X5d3hrg= +github.com/aws/aws-sdk-go-v2/internal/ini v1.6.0/go.mod h1:rdAuXeHWhI/zkpYcO5n8WCpaIgY9MUxFyBsuqq3kjyA= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.2 h1:pyVrNAf7Hwz0u39dLKN5t+n0+K/3rMYKuiOoIum3AsU= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.2/go.mod h1:mydrfOb9uiOYCxuCPR8YHQNQyGQwUQ7gPMZGBKbH8NY= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.0 h1:CJxo7ZBbaIzmXfV3hjcx36n9V87gJsIUPJflwqEHl3Q= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.0/go.mod h1:yjVfjuY4nD1EW9i387Kau+I6V5cBA5YnC/mWNopjZrI= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.2 h1:f2LhPofnjcdOQKRtumKjMvIHkfSQ8aH/rwKUDEQ/SB4= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.2/go.mod h1:q+xX0H4OfuWDuBy7y/LDi4v8IBOWuF+vtp8Z6ex+lw4= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.2 h1:h7j73yuAVVjic8pqswh+L/7r2IHP43QwRyOu6zcCDDE= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.2/go.mod h1:H07AHdK5LSy8F7EJUQhoxyiCNkePoHj2D8P2yGTWafo= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.2 h1:gbIaOzpXixUpoPK+js/bCBK1QBDXM22SigsnzGZio0U= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.2/go.mod h1:p+S7RNbdGN8qgHDSg2SCQJ9FeMAmvcETQiVpeGhYnNM= +github.com/aws/aws-sdk-go-v2/service/s3 v1.42.1 h1:o6MCcX1rJW8Y3g+hvg2xpjF6JR6DftuYhfl3Nc1WV9Q= +github.com/aws/aws-sdk-go-v2/service/s3 v1.42.1/go.mod h1:UDtxEWbREX6y4KREapT+jjtjoH0TiVSS6f5nfaY1UaM= +github.com/aws/aws-sdk-go-v2/service/sso v1.17.1 h1:km+ZNjtLtpXYf42RdaDZnNHm9s7SYAuDGTafy6nd89A= +github.com/aws/aws-sdk-go-v2/service/sso v1.17.1/go.mod h1:aHBr3pvBSD5MbzOvQtYutyPLLRPbl/y9x86XyJJnUXQ= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.19.1 h1:iRFNqZH4a67IqPvK8xxtyQYnyrlsvwmpHOe9r55ggBA= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.19.1/go.mod h1:pTy5WM+6sNv2tB24JNKFtn6EvciQ5k40ZJ0pq/Iaxj0= +github.com/aws/aws-sdk-go-v2/service/sts v1.25.1 h1:txgVXIXWPXyqdiVn92BV6a/rgtpX31HYdsOYj0sVQQQ= +github.com/aws/aws-sdk-go-v2/service/sts v1.25.1/go.mod h1:VAiJiNaoP1L89STFlEMgmHX1bKixY+FaP+TpRFrmyZ4= +github.com/aws/smithy-go v1.16.0 h1:gJZEH/Fqh+RsvlJ1Zt4tVAtV6bKkp3cC+R6FCZMNzik= +github.com/aws/smithy-go v1.16.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -303,8 +339,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= -golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -371,8 +407,8 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU= -golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -437,12 +473,12 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= -golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.9.0 h1:GRRCnKYhdQrD8kfRAdQ6Zcw1P0OcELxGLKJvtjVMZ28= -golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= +golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -452,8 +488,8 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= -golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/store/aws/aws.go b/store/aws/aws.go new file mode 100644 index 0000000..f55845c --- /dev/null +++ b/store/aws/aws.go @@ -0,0 +1,141 @@ +package aws + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/rs/zerolog/log" + "github.com/scalescape/dolores/store/cloud" +) + +var ErrInvalidServiceAccount = errors.New("invalid service account") + +type StorageClient struct { + client *s3.Client + region string +} + +func (s StorageClient) bucketExists(ctx context.Context, bucketName string) (bool, error) { + _, err := s.client.HeadBucket(ctx, &s3.HeadBucketInput{ + Bucket: aws.String(bucketName), + }) + if err != nil { + var notFoundType *types.NotFound + if errors.As(err, ¬FoundType) { + return false, nil + } + } + return true, err +} + +func (s StorageClient) CreateBucket(ctx context.Context, bucketName string) error { + lconst := types.BucketLocationConstraint(s.region) + cbCfg := &types.CreateBucketConfiguration{LocationConstraint: lconst} + bucket := &s3.CreateBucketInput{ + Bucket: aws.String(bucketName), + CreateBucketConfiguration: cbCfg, + } + _, err := s.client.CreateBucket(ctx, bucket) + var existsErr *types.BucketAlreadyOwnedByYou = new(types.BucketAlreadyOwnedByYou) + if errors.As(err, &existsErr) { + log.Debug().Msgf("bucket %s already exists", bucketName) + return nil + } + if err != nil { + return fmt.Errorf("error creating bucket: %s at region %s: %w", bucketName, s.region, err) + } + return nil +} + +func (s StorageClient) ListObject(ctx context.Context, bucket, path string) ([]cloud.Object, error) { + resp, err := s.client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ + Bucket: aws.String(bucket), + Prefix: aws.String(path), + }) + if err != nil { + return nil, fmt.Errorf("failed to get object list: %w", err) + } + + items := resp.Contents + objs := make([]cloud.Object, len(items)) + for i, item := range items { + o := cloud.Object{Name: *item.Key, Updated: *item.LastModified, Bucket: bucket} + objs[i] = o + } + log.Trace().Msgf("list of objects from path: %s length: %+v", path, len(objs)) + return objs, nil +} + +func (s StorageClient) WriteToObject(ctx context.Context, bucketName, fileName string, data []byte) error { + log.Debug().Msgf("writing to %s/%s", bucketName, fileName) + bucketExist, err := s.bucketExists(ctx, bucketName) + if err != nil { + return fmt.Errorf("failed to fetch bucket: %w", err) + } + if !bucketExist { + if err := s.CreateBucket(ctx, bucketName); err != nil { + return err + } + } + + fileReader := bytes.NewReader(data) + _, err = s.client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(bucketName), + Key: aws.String(fileName), + Body: fileReader, + }) + + if err != nil { + return fmt.Errorf("failed to upload secret: %w", err) + } + return nil +} + +func (s StorageClient) ReadObject(ctx context.Context, bucketName, fileName string) ([]byte, error) { + resp, err := s.client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(bucketName), + Key: aws.String(fileName), + }) + if err != nil { + return nil, fmt.Errorf("failed to read object : %w", err) + } + defer resp.Body.Close() + data, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read response body : %w", err) + } + return data, nil +} + +func (s StorageClient) ExistsObject(ctx context.Context, bucketName, fileName string) (bool, error) { + _, err := s.client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(bucketName), + Key: aws.String(fileName), + }) + if err != nil { + var notFoundType *types.NoSuchKey + if errors.As(err, ¬FoundType) { + return false, nil + } + return false, err + } + + return true, nil +} + +func NewStore(ctx context.Context) (StorageClient, error) { + cfg, err := config.LoadDefaultConfig(ctx) + if err != nil { + return StorageClient{}, err + } + + cli := s3.NewFromConfig(cfg) + return StorageClient{client: cli, region: cfg.Region}, nil +} diff --git a/store/cloud/object.go b/store/cloud/object.go new file mode 100644 index 0000000..c254400 --- /dev/null +++ b/store/cloud/object.go @@ -0,0 +1,10 @@ +package cloud + +import "time" + +type Object struct { + Name string `json:"name"` + Bucket string `json:"bucket"` + Created time.Time `json:"created"` + Updated time.Time `json:"updated"` +} diff --git a/store/google/gcs.go b/store/google/gcs.go index a0dd500..03468b2 100644 --- a/store/google/gcs.go +++ b/store/google/gcs.go @@ -7,10 +7,10 @@ import ( "fmt" "io" "os" - "time" "cloud.google.com/go/storage" "github.com/rs/zerolog/log" + "github.com/scalescape/dolores/store/cloud" "google.golang.org/api/iterator" "google.golang.org/api/option" ) @@ -26,13 +26,6 @@ type Config struct { ServiceAccountFile string } -type Object struct { - Name string `json:"name"` - Bucket string `json:"bucket"` - Created time.Time `json:"created"` - Updated time.Time `json:"updated"` -} - type ServiceAccount struct { Type string `json:"type"` ProjectID string `json:"project_id"` @@ -113,12 +106,12 @@ func (s StorageClient) ListBuckets(ctx context.Context) ([]string, error) { return buckets, nil } -func (s StorageClient) ListObject(ctx context.Context, bucketName, path string) ([]Object, error) { +func (s StorageClient) ListObject(ctx context.Context, bucketName, path string) ([]cloud.Object, error) { bucket := s.Client.Bucket(bucketName) if _, err := bucket.Attrs(ctx); err != nil { return nil, fmt.Errorf("failed to get bucket: %w", err) } - objs := make([]Object, 0) + objs := make([]cloud.Object, 0) iter := bucket.Objects(ctx, &storage.Query{Prefix: path}) for { attrs, err := iter.Next() @@ -128,7 +121,7 @@ func (s StorageClient) ListObject(ctx context.Context, bucketName, path string) if err != nil { return nil, fmt.Errorf("failed to iterate object list: %w", err) } - o := Object{Name: attrs.Name, Created: attrs.Created, Updated: attrs.Updated, Bucket: attrs.Bucket} + o := cloud.Object{Name: attrs.Name, Created: attrs.Created, Updated: attrs.Updated, Bucket: attrs.Bucket} objs = append(objs, o) } log.Trace().Msgf("list of objects from path: %s %+v", path, objs)