From c2478711bf2cb9e2814020ece8faf73428b7671a Mon Sep 17 00:00:00 2001 From: shubhamvernekar Date: Sat, 4 Nov 2023 22:14:51 +0530 Subject: [PATCH 1/5] init commit --- client/service.go | 4 ++-- client/service_test.go | 2 +- cmd/dolores/config.go | 15 +++++++++++++++ secrets/manager.go | 17 +++++++++++++++++ store/google/gcs.go | 2 +- 5 files changed, 36 insertions(+), 4 deletions(-) diff --git a/client/service.go b/client/service.go index 072983a..778240f 100644 --- a/client/service.go +++ b/client/service.go @@ -23,7 +23,7 @@ type Service struct { type gcsStore interface { WriteToObject(ctx context.Context, bucketName, fileName string, data []byte) error ReadObject(ctx context.Context, bucketName, fileName string) ([]byte, error) - ListOjbect(ctx context.Context, bucketName, path string) ([]string, error) + ListObject(ctx context.Context, bucketName, path string) ([]string, error) ExistsObject(ctx context.Context, bucketName, fileName string) (bool, error) } @@ -77,7 +77,7 @@ func (s Service) GetOrgPublicKeys(ctx context.Context, env, bucketName, path str if pubKey != "" { return []string{pubKey}, nil } - resp, err := s.store.ListOjbect(ctx, bucketName, path) + resp, err := s.store.ListObject(ctx, bucketName, path) if err != nil { return nil, fmt.Errorf("error listing objects: %w", err) } diff --git a/client/service_test.go b/client/service_test.go index dff657a..2751b92 100644 --- a/client/service_test.go +++ b/client/service_test.go @@ -38,7 +38,7 @@ func (m *mockGCS) ReadObject(ctx context.Context, bucketName, fileName string) ( return args.Get(0).([]byte), args.Error(1) } -func (m *mockGCS) ListOjbect(ctx context.Context, bucketName, path string) ([]string, error) { +func (m *mockGCS) ListObject(ctx context.Context, bucketName, path string) ([]string, error) { args := m.Called(ctx, bucketName, path) return args.Get(0).([]string), args.Error(1) } diff --git a/cmd/dolores/config.go b/cmd/dolores/config.go index d4d3ae6..af47e03 100644 --- a/cmd/dolores/config.go +++ b/cmd/dolores/config.go @@ -2,6 +2,7 @@ package main import ( "context" + "fmt" "github.com/rs/zerolog" "github.com/rs/zerolog/log" @@ -31,6 +32,7 @@ func NewConfig(client func(context.Context) *client.Client) *ConfigCommand { cfg.Subcommands = append(cfg.Subcommands, EncryptCommand(cfg.encryptAction)) cfg.Subcommands = append(cfg.Subcommands, DecryptCommand(cfg.decryptAction)) cfg.Subcommands = append(cfg.Subcommands, EditCommand(cfg.editAction)) + cfg.Subcommands = append(cfg.Subcommands, ListSecretCommand(cfg.listSecretAction)) return cfg } @@ -75,6 +77,11 @@ func (c *ConfigCommand) decryptAction(ctx *cli.Context) error { return sec.Decrypt(req) } +func (c *ConfigCommand) listSecretAction(ctx *cli.Context) error { + fmt.Println("List secrets ") + return nil +} + func EncryptCommand(action cli.ActionFunc) *cli.Command { return &cli.Command{ Name: "encrypt", @@ -135,3 +142,11 @@ func EditCommand(action cli.ActionFunc) *cli.Command { Action: action, } } + +func ListSecretCommand(action cli.ActionFunc) *cli.Command { + return &cli.Command{ + Name: "list_secret", + Usage: "shows the list of secrets uploaded in cloud", + Action: action, + } +} diff --git a/secrets/manager.go b/secrets/manager.go index b0a6446..69ccf76 100644 --- a/secrets/manager.go +++ b/secrets/manager.go @@ -124,6 +124,23 @@ func (sm SecretManager) Decrypt(cfg DecryptConfig) error { return nil } +type ListSecretConfig struct { + Environment string + KeyFile string + Out io.Writer +} + +func (c ListSecretConfig) Valid() error { + if c.KeyFile == "" { + return ErrInvalidKeyFile + } + + if strings.ToLower(c.Environment) != "production" && strings.ToLower(c.Environment) != "staging" { + return ErrInvalidEnvironment + } + return nil +} + func NewSecretsManager(log zerolog.Logger, rcli *client.Client) SecretManager { return SecretManager{Client: rcli, log: log} } diff --git a/store/google/gcs.go b/store/google/gcs.go index 9d5e417..e5a0855 100644 --- a/store/google/gcs.go +++ b/store/google/gcs.go @@ -105,7 +105,7 @@ func (s StorageClient) ListBuckets(ctx context.Context) ([]string, error) { return buckets, nil } -func (s StorageClient) ListOjbect(ctx context.Context, bucketName, path string) ([]string, error) { +func (s StorageClient) ListObject(ctx context.Context, bucketName, path string) ([]string, error) { bucket := s.Client.Bucket(bucketName) if _, err := bucket.Attrs(ctx); err != nil { return nil, fmt.Errorf("failed to get bucket: %w", err) From a706a54f121fdb9e4bb0b76a6c7ebd3064f3ea5b Mon Sep 17 00:00:00 2001 From: shubhamvernekar Date: Sun, 5 Nov 2023 00:35:33 +0530 Subject: [PATCH 2/5] added list secrets command --- client/client.go | 9 +++++++++ client/gcs_service.go | 5 +++-- client/monart.go | 5 +++++ cmd/dolores/config.go | 12 +++++++++--- cmd/dolores/main.go | 2 ++ secrets/manager.go | 33 +++++++++++++++++++++++++++++---- store/google/gcs.go | 15 ++++++++++++--- 7 files changed, 69 insertions(+), 12 deletions(-) diff --git a/client/client.go b/client/client.go index 0d74096..b4796fc 100644 --- a/client/client.go +++ b/client/client.go @@ -77,6 +77,15 @@ func (c *Client) GetOrgPublicKeys(env string) (OrgPublicKeys, error) { return OrgPublicKeys{Recipients: recps}, nil } +func (c *Client) GetSecretList() ([]google.SecretObject, error) { + resp, err := c.Service.store.ListObject(c.ctx, c.bucket, c.prefix) + if err != nil { + return nil, err + } + + return resp, nil +} + func New(ctx context.Context, cfg config.Client) (*Client, error) { if err := cfg.Valid(); err != nil { return nil, err diff --git a/client/gcs_service.go b/client/gcs_service.go index 778240f..5dad991 100644 --- a/client/gcs_service.go +++ b/client/gcs_service.go @@ -10,6 +10,7 @@ import ( "github.com/rs/zerolog/log" "github.com/scalescape/dolores/config" + "github.com/scalescape/dolores/store/google" ) var ErrInvalidPublicKeys = errors.New("invalid public keys") @@ -23,7 +24,7 @@ type Service struct { type gcsStore interface { WriteToObject(ctx context.Context, bucketName, fileName string, data []byte) error ReadObject(ctx context.Context, bucketName, fileName string) ([]byte, error) - ListObject(ctx context.Context, bucketName, path string) ([]string, error) + ListObject(ctx context.Context, bucketName, path string) ([]google.SecretObject, error) ExistsObject(ctx context.Context, bucketName, fileName string) (bool, error) } @@ -83,7 +84,7 @@ func (s Service) GetOrgPublicKeys(ctx context.Context, env, bucketName, path str } keys := make([]string, len(resp)) for i, obj := range resp { - key, err := s.store.ReadObject(ctx, bucketName, obj) + key, err := s.store.ReadObject(ctx, bucketName, obj.Name) if err != nil { return nil, fmt.Errorf("failed to read object %s: %w", obj, err) } diff --git a/client/monart.go b/client/monart.go index 1c73e3b..6332cbc 100644 --- a/client/monart.go +++ b/client/monart.go @@ -11,6 +11,7 @@ import ( "github.com/rs/zerolog/log" "github.com/scalescape/dolores/config" + "github.com/scalescape/dolores/store/google" ) type credentials struct { @@ -109,6 +110,10 @@ func (s MonartClient) call(req *http.Request, dest any) (*http.Response, error) return resp, nil } +func (c MonartClient) GetSecretList() ([]google.SecretObject, error) { + return nil, nil +} + func NewMonart(ctx context.Context, cfg *config.Monart) MonartClient { cred := credentials{APIToken: cfg.APIToken, ID: cfg.ID} return MonartClient{cli: http.DefaultClient, cred: cred, ctx: ctx} diff --git a/cmd/dolores/config.go b/cmd/dolores/config.go index 812fa8c..c6493e4 100644 --- a/cmd/dolores/config.go +++ b/cmd/dolores/config.go @@ -2,7 +2,7 @@ package main import ( "context" - "fmt" + "os" "github.com/rs/zerolog" "github.com/rs/zerolog/log" @@ -77,7 +77,13 @@ func (c *ConfigCommand) decryptAction(ctx *cli.Context) error { } func (c *ConfigCommand) listSecretAction(ctx *cli.Context) error { - fmt.Println("List secrets ") + env := ctx.String("environment") + log := c.log.With().Str("cmd", "config.list").Str("environment", env).Logger() + secMan := secrets.NewSecretsManager(log, c.rcli(ctx.Context)) + req := secrets.ListSecretConfig{Environment: env, Out: os.Stdout} + if err := secMan.ListSecret(req); err != nil { + return err + } return nil } @@ -144,7 +150,7 @@ func EditCommand(action cli.ActionFunc) *cli.Command { func ListSecretCommand(action cli.ActionFunc) *cli.Command { return &cli.Command{ - Name: "list_secret", + Name: "list", Usage: "shows the list of secrets uploaded in cloud", Action: action, } diff --git a/cmd/dolores/main.go b/cmd/dolores/main.go index 6355cc6..013ee0a 100644 --- a/cmd/dolores/main.go +++ b/cmd/dolores/main.go @@ -9,6 +9,7 @@ import ( "github.com/rs/zerolog/log" "github.com/scalescape/dolores/client" "github.com/scalescape/dolores/config" + "github.com/scalescape/dolores/store/google" "github.com/urfave/cli/v2" ) @@ -17,6 +18,7 @@ type secretsClient interface { FetchSecrets(req client.FetchSecretRequest) ([]byte, error) GetOrgPublicKeys(env string) (client.OrgPublicKeys, error) Init(ctx context.Context, bucket string, cfg client.Configuration) error + GetSecretList() ([]google.SecretObject, error) } type CtxKey string diff --git a/secrets/manager.go b/secrets/manager.go index faa72de..9cb0944 100644 --- a/secrets/manager.go +++ b/secrets/manager.go @@ -11,12 +11,14 @@ import ( "github.com/rs/zerolog" "github.com/scalescape/dolores" "github.com/scalescape/dolores/client" + "github.com/scalescape/dolores/store/google" ) type secClient interface { FetchSecrets(req client.FetchSecretRequest) ([]byte, error) UploadSecrets(req client.EncryptedConfig) error GetOrgPublicKeys(env string) (client.OrgPublicKeys, error) + GetSecretList() ([]google.SecretObject, error) } type EncryptConfig struct { @@ -132,21 +134,44 @@ func (sm SecretManager) Decrypt(cfg DecryptConfig) error { type ListSecretConfig struct { Environment string - KeyFile string Out io.Writer } -func (c ListSecretConfig) Valid() error { - if c.KeyFile == "" { - return ErrInvalidKeyFile +func (c ListSecretConfig) Output() io.Writer { + if c.Out == nil { + return os.Stdout } + return c.Out +} +func (c ListSecretConfig) Valid() error { if strings.ToLower(c.Environment) != "production" && strings.ToLower(c.Environment) != "staging" { return ErrInvalidEnvironment } return nil } +func (sm SecretManager) ListSecret(cfg ListSecretConfig) error { + if err := cfg.Valid(); err != nil { + return fmt.Errorf("invalid config: %w", err) + } + resp, err := sm.client.GetSecretList() + if err != nil { + return fmt.Errorf("failed to get secrets: %w", err) + } + if _, err := cfg.Output().Write([]byte(fmt.Sprintf("%-20s %-40s %-40s %-40s\n", "Name", "Bucket", "Create At", "Updated At"))); err != nil { + return err + } + for _, obj := range resp { + if !strings.HasSuffix(obj.Name, ".key") && !strings.HasSuffix(obj.Name, "/") { + if _, err := cfg.Output().Write([]byte(fmt.Sprintf("%-20s %-40s %-40s %-40s\n", obj.Name, obj.Bucket, obj.CreatedAt, obj.UpdatedAt))); err != nil { + return err + } + } + } + return nil +} + func NewSecretsManager(log zerolog.Logger, rcli secClient) SecretManager { return SecretManager{client: rcli, log: log} } diff --git a/store/google/gcs.go b/store/google/gcs.go index e5a0855..8c168bc 100644 --- a/store/google/gcs.go +++ b/store/google/gcs.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "os" + "time" "cloud.google.com/go/storage" "github.com/rs/zerolog/log" @@ -25,6 +26,13 @@ type Config struct { ServiceAccountFile string } +type SecretObject struct { + Name string `json:"name"` + Bucket string `json:"bucket"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + type ServiceAccount struct { Type string `json:"type"` ProjectID string `json:"project_id"` @@ -105,12 +113,12 @@ func (s StorageClient) ListBuckets(ctx context.Context) ([]string, error) { return buckets, nil } -func (s StorageClient) ListObject(ctx context.Context, bucketName, path string) ([]string, error) { +func (s StorageClient) ListObject(ctx context.Context, bucketName, path string) ([]SecretObject, error) { bucket := s.Client.Bucket(bucketName) if _, err := bucket.Attrs(ctx); err != nil { return nil, fmt.Errorf("failed to get bucket: %w", err) } - objs := make([]string, 0) + objs := make([]SecretObject, 0) iter := bucket.Objects(ctx, &storage.Query{Prefix: path}) for { attrs, err := iter.Next() @@ -120,7 +128,8 @@ func (s StorageClient) ListObject(ctx context.Context, bucketName, path string) if err != nil { return nil, fmt.Errorf("failed to iterate object list: %w", err) } - objs = append(objs, attrs.Name) + o := SecretObject{Name: attrs.Name, CreatedAt: attrs.Created, UpdatedAt: attrs.Updated, Bucket: attrs.Bucket} + objs = append(objs, o) } log.Trace().Msgf("list of objects from path: %s %+v", path, objs) return objs, nil From bbc7f011322b2257da19825e82c7352c340aba9b Mon Sep 17 00:00:00 2001 From: shubhamvernekar Date: Mon, 6 Nov 2023 11:34:57 +0530 Subject: [PATCH 3/5] comment resolve --- client/client.go | 14 ++++++++------ client/gcs_service.go | 10 +++++++++- client/monart.go | 17 +++++++++++++---- client/service_test.go | 5 +++-- cmd/dolores/main.go | 2 +- secrets/manager.go | 21 +++++++++++++++------ store/google/gcs.go | 4 ++-- 7 files changed, 51 insertions(+), 22 deletions(-) diff --git a/client/client.go b/client/client.go index b4796fc..1d7b661 100644 --- a/client/client.go +++ b/client/client.go @@ -77,13 +77,15 @@ func (c *Client) GetOrgPublicKeys(env string) (OrgPublicKeys, error) { return OrgPublicKeys{Recipients: recps}, nil } -func (c *Client) GetSecretList() ([]google.SecretObject, error) { - resp, err := c.Service.store.ListObject(c.ctx, c.bucket, c.prefix) - if err != nil { - return nil, err - } +type GetSecretListConfig struct { + Environment string `json:"environment"` +} +type GetSecretListResponse struct { + Secrets []google.SecretObject `json:"secrets"` +} - return resp, nil +func (c *Client) GetSecretList(cfg GetSecretListConfig) ([]google.SecretObject, error) { + return c.Service.GetObjList(c.ctx, c.bucket, c.prefix) } func New(ctx context.Context, cfg config.Client) (*Client, error) { diff --git a/client/gcs_service.go b/client/gcs_service.go index 5dad991..c532b5d 100644 --- a/client/gcs_service.go +++ b/client/gcs_service.go @@ -78,7 +78,7 @@ func (s Service) GetOrgPublicKeys(ctx context.Context, env, bucketName, path str if pubKey != "" { return []string{pubKey}, nil } - resp, err := s.store.ListObject(ctx, bucketName, path) + resp, err := s.GetObjList(ctx, bucketName, path) if err != nil { return nil, fmt.Errorf("error listing objects: %w", err) } @@ -138,6 +138,14 @@ func (s Service) saveObject(ctx context.Context, bucket, fname string, md any) e return s.store.WriteToObject(ctx, bucket, fname, data) } +func (s Service) GetObjList(ctx context.Context, bucket, path string) ([]google.SecretObject, error) { + resp, err := s.store.ListObject(ctx, bucket, path) + if err != nil { + return nil, err + } + return resp, nil +} + func NewService(st gcsStore) Service { return Service{store: st} } diff --git a/client/monart.go b/client/monart.go index 6332cbc..ff60f49 100644 --- a/client/monart.go +++ b/client/monart.go @@ -87,6 +87,19 @@ func (s MonartClient) FetchSecrets(fetchReq FetchSecretRequest) ([]byte, error) return sec, nil } +func (s MonartClient) GetSecretList(cfg GetSecretListConfig) ([]google.SecretObject, error) { + path := fmt.Sprintf("/environment/%s/secrets", cfg.Environment) + req, err := http.NewRequest(http.MethodGet, s.serverURL(path), nil) + if err != nil { + return nil, fmt.Errorf("unable to build Get SecretList request: %w", err) + } + result := new(GetSecretListResponse) + if _, err := s.call(req, &result); err != nil { + return nil, err + } + return result.Secrets, nil +} + func (s MonartClient) serverURL(path string) string { return "https://relyonmetrics.com/api/" + path } @@ -110,10 +123,6 @@ func (s MonartClient) call(req *http.Request, dest any) (*http.Response, error) return resp, nil } -func (c MonartClient) GetSecretList() ([]google.SecretObject, error) { - return nil, nil -} - func NewMonart(ctx context.Context, cfg *config.Monart) MonartClient { cred := credentials{APIToken: cfg.APIToken, ID: cfg.ID} return MonartClient{cli: http.DefaultClient, cred: cred, ctx: ctx} diff --git a/client/service_test.go b/client/service_test.go index 2751b92..986830f 100644 --- a/client/service_test.go +++ b/client/service_test.go @@ -7,6 +7,7 @@ import ( "github.com/scalescape/dolores/client" "github.com/scalescape/dolores/config" + "github.com/scalescape/dolores/store/google" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" @@ -38,9 +39,9 @@ func (m *mockGCS) ReadObject(ctx context.Context, bucketName, fileName string) ( return args.Get(0).([]byte), args.Error(1) } -func (m *mockGCS) ListObject(ctx context.Context, bucketName, path string) ([]string, error) { +func (m *mockGCS) ListObject(ctx context.Context, bucketName, path string) ([]google.SecretObject, error) { args := m.Called(ctx, bucketName, path) - return args.Get(0).([]string), args.Error(1) + return args.Get(0).([]google.SecretObject), args.Error(1) } func (m *mockGCS) ExistsObject(ctx context.Context, bucketName, fileName string) (bool, error) { diff --git a/cmd/dolores/main.go b/cmd/dolores/main.go index 013ee0a..5205891 100644 --- a/cmd/dolores/main.go +++ b/cmd/dolores/main.go @@ -18,7 +18,7 @@ type secretsClient interface { FetchSecrets(req client.FetchSecretRequest) ([]byte, error) GetOrgPublicKeys(env string) (client.OrgPublicKeys, error) Init(ctx context.Context, bucket string, cfg client.Configuration) error - GetSecretList() ([]google.SecretObject, error) + GetSecretList(req client.GetSecretListConfig) ([]google.SecretObject, error) } type CtxKey string diff --git a/secrets/manager.go b/secrets/manager.go index 9cb0944..fb7d7e6 100644 --- a/secrets/manager.go +++ b/secrets/manager.go @@ -18,7 +18,7 @@ type secClient interface { FetchSecrets(req client.FetchSecretRequest) ([]byte, error) UploadSecrets(req client.EncryptedConfig) error GetOrgPublicKeys(env string) (client.OrgPublicKeys, error) - GetSecretList() ([]google.SecretObject, error) + GetSecretList(cfg client.GetSecretListConfig) ([]google.SecretObject, error) } type EncryptConfig struct { @@ -137,7 +137,7 @@ type ListSecretConfig struct { Out io.Writer } -func (c ListSecretConfig) Output() io.Writer { +func (c ListSecretConfig) output() io.Writer { if c.Out == nil { return os.Stdout } @@ -145,7 +145,8 @@ func (c ListSecretConfig) Output() io.Writer { } func (c ListSecretConfig) Valid() error { - if strings.ToLower(c.Environment) != "production" && strings.ToLower(c.Environment) != "staging" { + env := strings.ToLower(c.Environment) + if env != "production" && env != "staging" { return ErrInvalidEnvironment } return nil @@ -155,16 +156,24 @@ func (sm SecretManager) ListSecret(cfg ListSecretConfig) error { if err := cfg.Valid(); err != nil { return fmt.Errorf("invalid config: %w", err) } - resp, err := sm.client.GetSecretList() + req := client.GetSecretListConfig{Environment: cfg.Environment} + resp, err := sm.client.GetSecretList(req) if err != nil { return fmt.Errorf("failed to get secrets: %w", err) } - if _, err := cfg.Output().Write([]byte(fmt.Sprintf("%-20s %-40s %-40s %-40s\n", "Name", "Bucket", "Create At", "Updated At"))); err != nil { + if _, err := cfg.output().Write([]byte(fmt.Sprintf("%-10s %-50s %-30s %-30s\n", "Name", "Location", "Created At UTC", "Updated At UTC"))); err != nil { return err } for _, obj := range resp { if !strings.HasSuffix(obj.Name, ".key") && !strings.HasSuffix(obj.Name, "/") { - if _, err := cfg.Output().Write([]byte(fmt.Sprintf("%-20s %-40s %-40s %-40s\n", obj.Name, obj.Bucket, obj.CreatedAt, obj.UpdatedAt))); err != nil { + arr := strings.SplitN(obj.Name, "/", 2) + name := obj.Name + if len(arr) == 2 { + name = arr[1] + } + createdAt := obj.CreatedAt.Format("2000-01-02 15:04:05.999") + updatedAt := obj.UpdatedAt.Format("2000-01-02 15:04:05.999") + if _, err := cfg.output().Write([]byte(fmt.Sprintf("%-10s %-50s %-30s %-30s\n", name, obj.Location, createdAt, updatedAt))); err != nil { return err } } diff --git a/store/google/gcs.go b/store/google/gcs.go index 8c168bc..4635a21 100644 --- a/store/google/gcs.go +++ b/store/google/gcs.go @@ -28,7 +28,7 @@ type Config struct { type SecretObject struct { Name string `json:"name"` - Bucket string `json:"bucket"` + Location string `json:"location"` CreatedAt time.Time `json:"created_at"` UpdatedAt time.Time `json:"updated_at"` } @@ -128,7 +128,7 @@ func (s StorageClient) ListObject(ctx context.Context, bucketName, path string) if err != nil { return nil, fmt.Errorf("failed to iterate object list: %w", err) } - o := SecretObject{Name: attrs.Name, CreatedAt: attrs.Created, UpdatedAt: attrs.Updated, Bucket: attrs.Bucket} + o := SecretObject{Name: attrs.Name, CreatedAt: attrs.Created, UpdatedAt: attrs.Updated, Location: fmt.Sprintf("%s/%s", attrs.Bucket, attrs.Name)} objs = append(objs, o) } log.Trace().Msgf("list of objects from path: %s %+v", path, objs) From 16b46d5bf5eddc205ded0ccf0528c21899ba9d1c Mon Sep 17 00:00:00 2001 From: shubhamvernekar Date: Wed, 8 Nov 2023 14:04:53 +0530 Subject: [PATCH 4/5] comments fix --- client/client.go | 28 +++++++++++++++++++++++----- client/gcs_service.go | 4 ++-- client/monart.go | 7 +++---- cmd/dolores/main.go | 3 +-- secrets/manager.go | 5 ++--- store/google/gcs.go | 16 ++++++++-------- 6 files changed, 39 insertions(+), 24 deletions(-) diff --git a/client/client.go b/client/client.go index 1d7b661..978233d 100644 --- a/client/client.go +++ b/client/client.go @@ -2,6 +2,8 @@ package client import ( "context" + "fmt" + "time" "github.com/rs/zerolog" "github.com/rs/zerolog/log" @@ -23,6 +25,13 @@ type EncryptedConfig struct { Data string `json:"data"` } +type SecretObject struct { + Name string `json:"name"` + Location string `json:"location"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + func (c *Client) Init(ctx context.Context, bucket string, cfg Configuration) error { return c.Service.Init(ctx, bucket, cfg) } @@ -77,15 +86,24 @@ func (c *Client) GetOrgPublicKeys(env string) (OrgPublicKeys, error) { return OrgPublicKeys{Recipients: recps}, nil } -type GetSecretListConfig struct { +type SecretListConfig struct { Environment string `json:"environment"` } -type GetSecretListResponse struct { - Secrets []google.SecretObject `json:"secrets"` +type SecretListResponse struct { + Secrets []SecretObject `json:"secrets"` } -func (c *Client) GetSecretList(cfg GetSecretListConfig) ([]google.SecretObject, error) { - return c.Service.GetObjList(c.ctx, c.bucket, c.prefix) +func (c *Client) GetSecretList(_ SecretListConfig) ([]SecretObject, error) { + resp, err := c.Service.GetObjList(c.ctx, c.bucket, c.prefix) + if err != nil { + return nil, err + } + objs := make([]SecretObject, 0) + for _, obj := range resp { + o := SecretObject{Name: obj.Name, CreatedAt: obj.Created, UpdatedAt: obj.Updated, Location: fmt.Sprintf("%s/%s", obj.Bucket, obj.Name)} + objs = append(objs, o) + } + return objs, nil } func New(ctx context.Context, cfg config.Client) (*Client, error) { diff --git a/client/gcs_service.go b/client/gcs_service.go index c532b5d..00238c3 100644 --- a/client/gcs_service.go +++ b/client/gcs_service.go @@ -24,7 +24,7 @@ type Service struct { type gcsStore interface { WriteToObject(ctx context.Context, bucketName, fileName string, data []byte) error ReadObject(ctx context.Context, bucketName, fileName string) ([]byte, error) - ListObject(ctx context.Context, bucketName, path string) ([]google.SecretObject, error) + ListObject(ctx context.Context, bucketName, path string) ([]google.Object, error) ExistsObject(ctx context.Context, bucketName, fileName string) (bool, error) } @@ -138,7 +138,7 @@ func (s Service) saveObject(ctx context.Context, bucket, fname string, md any) e return s.store.WriteToObject(ctx, bucket, fname, data) } -func (s Service) GetObjList(ctx context.Context, bucket, path string) ([]google.SecretObject, error) { +func (s Service) GetObjList(ctx context.Context, bucket, path string) ([]google.Object, error) { resp, err := s.store.ListObject(ctx, bucket, path) if err != nil { return nil, err diff --git a/client/monart.go b/client/monart.go index ff60f49..fa12d3e 100644 --- a/client/monart.go +++ b/client/monart.go @@ -11,7 +11,6 @@ import ( "github.com/rs/zerolog/log" "github.com/scalescape/dolores/config" - "github.com/scalescape/dolores/store/google" ) type credentials struct { @@ -87,13 +86,13 @@ func (s MonartClient) FetchSecrets(fetchReq FetchSecretRequest) ([]byte, error) return sec, nil } -func (s MonartClient) GetSecretList(cfg GetSecretListConfig) ([]google.SecretObject, error) { - path := fmt.Sprintf("/environment/%s/secrets", cfg.Environment) +func (s MonartClient) GetSecretList(cfg SecretListConfig) ([]SecretObject, error) { + path := fmt.Sprintf("environment/%s/secrets", cfg.Environment) req, err := http.NewRequest(http.MethodGet, s.serverURL(path), nil) if err != nil { return nil, fmt.Errorf("unable to build Get SecretList request: %w", err) } - result := new(GetSecretListResponse) + result := new(SecretListResponse) if _, err := s.call(req, &result); err != nil { return nil, err } diff --git a/cmd/dolores/main.go b/cmd/dolores/main.go index 5205891..2456ae5 100644 --- a/cmd/dolores/main.go +++ b/cmd/dolores/main.go @@ -9,7 +9,6 @@ import ( "github.com/rs/zerolog/log" "github.com/scalescape/dolores/client" "github.com/scalescape/dolores/config" - "github.com/scalescape/dolores/store/google" "github.com/urfave/cli/v2" ) @@ -18,7 +17,7 @@ type secretsClient interface { FetchSecrets(req client.FetchSecretRequest) ([]byte, error) GetOrgPublicKeys(env string) (client.OrgPublicKeys, error) Init(ctx context.Context, bucket string, cfg client.Configuration) error - GetSecretList(req client.GetSecretListConfig) ([]google.SecretObject, error) + GetSecretList(req client.SecretListConfig) ([]client.SecretObject, error) } type CtxKey string diff --git a/secrets/manager.go b/secrets/manager.go index fb7d7e6..7b8c476 100644 --- a/secrets/manager.go +++ b/secrets/manager.go @@ -11,14 +11,13 @@ import ( "github.com/rs/zerolog" "github.com/scalescape/dolores" "github.com/scalescape/dolores/client" - "github.com/scalescape/dolores/store/google" ) type secClient interface { FetchSecrets(req client.FetchSecretRequest) ([]byte, error) UploadSecrets(req client.EncryptedConfig) error GetOrgPublicKeys(env string) (client.OrgPublicKeys, error) - GetSecretList(cfg client.GetSecretListConfig) ([]google.SecretObject, error) + GetSecretList(cfg client.SecretListConfig) ([]client.SecretObject, error) } type EncryptConfig struct { @@ -156,7 +155,7 @@ func (sm SecretManager) ListSecret(cfg ListSecretConfig) error { if err := cfg.Valid(); err != nil { return fmt.Errorf("invalid config: %w", err) } - req := client.GetSecretListConfig{Environment: cfg.Environment} + req := client.SecretListConfig{Environment: cfg.Environment} resp, err := sm.client.GetSecretList(req) if err != nil { return fmt.Errorf("failed to get secrets: %w", err) diff --git a/store/google/gcs.go b/store/google/gcs.go index 4635a21..a0dd500 100644 --- a/store/google/gcs.go +++ b/store/google/gcs.go @@ -26,11 +26,11 @@ type Config struct { ServiceAccountFile string } -type SecretObject struct { - Name string `json:"name"` - Location string `json:"location"` - CreatedAt time.Time `json:"created_at"` - UpdatedAt time.Time `json:"updated_at"` +type Object struct { + Name string `json:"name"` + Bucket string `json:"bucket"` + Created time.Time `json:"created"` + Updated time.Time `json:"updated"` } type ServiceAccount struct { @@ -113,12 +113,12 @@ func (s StorageClient) ListBuckets(ctx context.Context) ([]string, error) { return buckets, nil } -func (s StorageClient) ListObject(ctx context.Context, bucketName, path string) ([]SecretObject, error) { +func (s StorageClient) ListObject(ctx context.Context, bucketName, path string) ([]Object, error) { bucket := s.Client.Bucket(bucketName) if _, err := bucket.Attrs(ctx); err != nil { return nil, fmt.Errorf("failed to get bucket: %w", err) } - objs := make([]SecretObject, 0) + objs := make([]Object, 0) iter := bucket.Objects(ctx, &storage.Query{Prefix: path}) for { attrs, err := iter.Next() @@ -128,7 +128,7 @@ func (s StorageClient) ListObject(ctx context.Context, bucketName, path string) if err != nil { return nil, fmt.Errorf("failed to iterate object list: %w", err) } - o := SecretObject{Name: attrs.Name, CreatedAt: attrs.Created, UpdatedAt: attrs.Updated, Location: fmt.Sprintf("%s/%s", attrs.Bucket, attrs.Name)} + o := Object{Name: attrs.Name, Created: attrs.Created, Updated: attrs.Updated, Bucket: attrs.Bucket} objs = append(objs, o) } log.Trace().Msgf("list of objects from path: %s %+v", path, objs) From 636edc8c1640b735d0a9ac511ac82518af061569 Mon Sep 17 00:00:00 2001 From: shubhamvernekar Date: Thu, 9 Nov 2023 08:30:26 +0530 Subject: [PATCH 5/5] formating fix --- client/client.go | 2 +- client/gcs_service.go | 4 ++-- client/service_test.go | 8 ++++---- secrets/manager.go | 13 +++++++++---- 4 files changed, 16 insertions(+), 11 deletions(-) diff --git a/client/client.go b/client/client.go index 978233d..10b2a57 100644 --- a/client/client.go +++ b/client/client.go @@ -94,7 +94,7 @@ type SecretListResponse struct { } func (c *Client) GetSecretList(_ SecretListConfig) ([]SecretObject, error) { - resp, err := c.Service.GetObjList(c.ctx, c.bucket, c.prefix) + resp, err := c.Service.ListObject(c.ctx, c.bucket, c.prefix) if err != nil { return nil, err } diff --git a/client/gcs_service.go b/client/gcs_service.go index 00238c3..1301390 100644 --- a/client/gcs_service.go +++ b/client/gcs_service.go @@ -78,7 +78,7 @@ func (s Service) GetOrgPublicKeys(ctx context.Context, env, bucketName, path str if pubKey != "" { return []string{pubKey}, nil } - resp, err := s.GetObjList(ctx, bucketName, path) + resp, err := s.ListObject(ctx, bucketName, path) if err != nil { return nil, fmt.Errorf("error listing objects: %w", err) } @@ -138,7 +138,7 @@ func (s Service) saveObject(ctx context.Context, bucket, fname string, md any) e return s.store.WriteToObject(ctx, bucket, fname, data) } -func (s Service) GetObjList(ctx context.Context, bucket, path string) ([]google.Object, error) { +func (s Service) ListObject(ctx context.Context, bucket, path string) ([]google.Object, error) { resp, err := s.store.ListObject(ctx, bucket, path) if err != nil { return nil, err diff --git a/client/service_test.go b/client/service_test.go index 986830f..26bcef0 100644 --- a/client/service_test.go +++ b/client/service_test.go @@ -39,9 +39,9 @@ func (m *mockGCS) ReadObject(ctx context.Context, bucketName, fileName string) ( return args.Get(0).([]byte), args.Error(1) } -func (m *mockGCS) ListObject(ctx context.Context, bucketName, path string) ([]google.SecretObject, error) { +func (m *mockGCS) ListObject(ctx context.Context, bucketName, path string) ([]google.Object, error) { args := m.Called(ctx, bucketName, path) - return args.Get(0).([]google.SecretObject), args.Error(1) + return args.Get(0).([]google.Object), args.Error(1) } func (m *mockGCS) ExistsObject(ctx context.Context, bucketName, fileName string) (bool, error) { @@ -70,8 +70,8 @@ func (s *serviceSuite) TestShouldNotOverwriteMetadata() { PublicKey: "public_key", Metadata: config.Metadata{Location: "secrets"}, UserID: "test_user"} - s.gcs.On("ExistsObject", mock.AnythingOfType("*context.emptyCtx"), s.bucket, name).Return(true, nil).Once() - s.gcs.On("WriteToObject", mock.AnythingOfType("*context.emptyCtx"), s.bucket, "secrets/keys/test_user.key", []byte(cfg.PublicKey)).Return(nil).Once() + s.gcs.On("ExistsObject", mock.AnythingOfType("context.backgroundCtx"), s.bucket, name).Return(true, nil).Once() + s.gcs.On("WriteToObject", mock.AnythingOfType("context.backgroundCtx"), s.bucket, "secrets/keys/test_user.key", []byte(cfg.PublicKey)).Return(nil).Once() err := s.Service.Init(s.ctx, s.bucket, cfg) diff --git a/secrets/manager.go b/secrets/manager.go index 7b8c476..ba0106d 100644 --- a/secrets/manager.go +++ b/secrets/manager.go @@ -7,6 +7,7 @@ import ( "io" "os" "strings" + "time" "github.com/rs/zerolog" "github.com/scalescape/dolores" @@ -160,7 +161,10 @@ func (sm SecretManager) ListSecret(cfg ListSecretConfig) error { if err != nil { return fmt.Errorf("failed to get secrets: %w", err) } - if _, err := cfg.output().Write([]byte(fmt.Sprintf("%-10s %-50s %-30s %-30s\n", "Name", "Location", "Created At UTC", "Updated At UTC"))); err != nil { + + lineFormat := "%-10s %-65s %-30s %-30s\n" + header := []byte(fmt.Sprintf(lineFormat, "Name", "Location", "Created At (UTC)", "Updated At (UTC)")) + if _, err := cfg.output().Write(header); err != nil { return err } for _, obj := range resp { @@ -170,9 +174,10 @@ func (sm SecretManager) ListSecret(cfg ListSecretConfig) error { if len(arr) == 2 { name = arr[1] } - createdAt := obj.CreatedAt.Format("2000-01-02 15:04:05.999") - updatedAt := obj.UpdatedAt.Format("2000-01-02 15:04:05.999") - if _, err := cfg.output().Write([]byte(fmt.Sprintf("%-10s %-50s %-30s %-30s\n", name, obj.Location, createdAt, updatedAt))); err != nil { + createdAt := obj.CreatedAt.Format(time.DateTime) + updatedAt := obj.UpdatedAt.Format(time.DateTime) + line := []byte(fmt.Sprintf(lineFormat, name, obj.Location, createdAt, updatedAt)) + if _, err := cfg.output().Write(line); err != nil { return err } }