From c5ef305b910fbfcaa6cc39dc8cb0cde17524f8ec Mon Sep 17 00:00:00 2001 From: kendavis2 Date: Mon, 7 Oct 2019 07:48:18 -0400 Subject: [PATCH] store and vault refactoring --- README.md | 3 + cli/cmd/pull.go | 2 +- components/catalog/model.go | 15 +- components/contract/vault.go | 14 +- components/remote/init.go | 12 +- components/setting/setting.go | 12 +- components/store/aws.go | 4 + components/store/aws_param_store.go | 174 ++++--- components/store/aws_s3_store.go | 230 ++++----- components/store/aws_secrets_manager.go | 168 +++---- components/store/harbor_store.go | 449 ------------------ components/store/source_control_store.go | 12 - components/vault/aws.go | 13 + components/vault/aws_secrets_manager_vault.go | 104 +++- components/vault/env_vault.go | 9 +- components/vault/file_vault.go | 2 +- components/vault/keychain_vault_darwin.go | 4 +- components/vault/vault.go | 6 +- 18 files changed, 391 insertions(+), 842 deletions(-) delete mode 100644 components/store/harbor_store.go create mode 100644 components/vault/aws.go diff --git a/README.md b/README.md index 196965e..43ab550 100644 --- a/README.md +++ b/README.md @@ -202,6 +202,9 @@ $ cstore pull -t "dev|qa" # config must have either tag $ cstore pull -t dev -e # raw file contents ``` ```bash +$ cstore pull service/dev/.env -g json-object # JSON object format +``` +```bash $ eval $( cstore pull service/dev/.env -g terminal-export ) # export environment variables ``` diff --git a/cli/cmd/pull.go b/cli/cmd/pull.go index 07d2bcf..d871e99 100644 --- a/cli/cmd/pull.go +++ b/cli/cmd/pull.go @@ -298,7 +298,7 @@ func Pull(catalogPath string, opt cfg.UserOptions, io models.IO) (int, int, erro func compatibleFormat(format, fileType string) bool { switch format { - case "task-def-secrets", "task-def-env", "terminal-export": + case "task-def-secrets", "task-def-env", "terminal-export", "json-object": return fileType == "env" case "": return true diff --git a/components/catalog/model.go b/components/catalog/model.go index 7921b8a..22b1123 100644 --- a/components/catalog/model.go +++ b/components/catalog/model.go @@ -6,9 +6,6 @@ import ( "path/filepath" "strings" - "github.com/turnerlabs/cstore/components/cfg" - - "github.com/turnerlabs/cstore/components/models" "github.com/turnerlabs/cstore/components/path" ) @@ -191,12 +188,6 @@ func (f File) Missing(version string) bool { // Name ... func (f File) Name() string { return "*.yml" } -// Description ... -func (f File) Description() string { return "" } - -// Pre ... -func (f File) Pre(clog Catalog, fileEntry *File, uo cfg.UserOptions, io models.IO) error { return nil } - // Set ... func (f *File) Set(contextID, group, prop, value string) error { if f.Data == nil { @@ -215,11 +206,7 @@ func (f *File) Delete(contextID, group, prop string) error { // BuildKey ... func (f File) BuildKey(contextID, group, prop string) string { - if len(prop) > 0 { - return strings.ToUpper(fmt.Sprintf("%s_%s", group, prop)) - } - - return strings.ToUpper(group) + return strings.ToUpper(prop) } // Get ... diff --git a/components/contract/vault.go b/components/contract/vault.go index 44640bb..5ed193f 100644 --- a/components/contract/vault.go +++ b/components/contract/vault.go @@ -33,6 +33,10 @@ type IVault interface { // "fileEntry" represents the file this vault will operatate on. // If any data needs // + // "access" is used to get secrets required to access the vault. + // It is common to set the vault on the struct to allow other + // methods access to them. + // // "uo" specifies if the user requested settings. // // "io" contains readers and writers that should be used when @@ -40,7 +44,7 @@ type IVault interface { // line. // // "error" should return nil if the operation was successful. - Pre(clog catalog.Catalog, fileEntry *catalog.File, uo cfg.UserOptions, io models.IO) error + Pre(clog catalog.Catalog, fileEntry *catalog.File, access IVault, uo cfg.UserOptions, io models.IO) error // Get should return the requested secret or an error. // @@ -62,10 +66,10 @@ type IVault interface { // "contextID" is a guid which represents the context of the // catalog. It can be used to guarantee uniqueness for secret values. // - // "group" is a collection of props. The same group could be passed - // with different props and values. This method should not always - // overwrite the group, but should append/update the group to ensure - // other props in the same group are not overwritten. + // "group" is a collection of props. The same group could be set + // with different props and values. This method should not overwrite + // the group, but should append/update the group to ensure other + // props in the same group are not overwritten. // // "prop" is the name for the value being set. // diff --git a/components/remote/init.go b/components/remote/init.go index a9c10e8..041d209 100644 --- a/components/remote/init.go +++ b/components/remote/init.go @@ -21,19 +21,19 @@ type Components struct { func InitComponents(fileEntry *catalog.File, clog catalog.Catalog, uo cfg.UserOptions, io models.IO) (Components, error) { remote := Components{} - v, err := vault.GetBy(fileEntry.Vaults.Secrets, cfg.DefaultSecretsVault, clog, fileEntry, uo, io) + v, err := vault.GetBy(fileEntry.Vaults.Access, cfg.DefaultAccessVault, clog, fileEntry, nil, uo, io) if err != nil { return remote, err } - remote.Secrets = v - fileEntry.Vaults.Secrets = v.Name() + remote.Access = v + fileEntry.Vaults.Access = v.Name() - v, err = vault.GetBy(fileEntry.Vaults.Access, cfg.DefaultAccessVault, clog, fileEntry, uo, io) + v, err = vault.GetBy(fileEntry.Vaults.Secrets, cfg.DefaultSecretsVault, clog, fileEntry, remote.Access, uo, io) if err != nil { return remote, err } - remote.Access = v - fileEntry.Vaults.Access = v.Name() + remote.Secrets = v + fileEntry.Vaults.Secrets = v.Name() st, err := store.Select(fileEntry, clog, remote.Access, uo, io) if err != nil { diff --git a/components/setting/setting.go b/components/setting/setting.go index 9f49414..9875e7b 100644 --- a/components/setting/setting.go +++ b/components/setting/setting.go @@ -9,6 +9,16 @@ import ( "github.com/turnerlabs/cstore/components/prompt" ) +type IKeyValueStore interface { + Name() string + + Get(contextID, group, prop string) (string, error) + Set(contextID, group, prop, value string) error + Delete(contextID, group, prop string) error + + BuildKey(contextID, group, prop string) string +} + // Setting ... type Setting struct { Group string @@ -22,7 +32,7 @@ type Setting struct { HideInput bool AutoSave bool - Vault contract.IVault + Vault IKeyValueStore } // Key ... diff --git a/components/store/aws.go b/components/store/aws.go index decd876..6baba35 100644 --- a/components/store/aws.go +++ b/components/store/aws.go @@ -6,6 +6,10 @@ const ( awsSecretAccessKey = "AWS_SECRET_ACCESS_KEY" awsAccessKeyID = "AWS_ACCESS_KEY_ID" + awsBucketSetting = "AWS_S3_BUCKET" + + awsStoreKMSKeyID = "AWS_STORE_KMS_KEY_ID" + awsDefaultRegion = "us-east-1" awsDefaultProfile = "default" ) diff --git a/components/store/aws_param_store.go b/components/store/aws_param_store.go index da0a001..5dff0a1 100644 --- a/components/store/aws_param_store.go +++ b/components/store/aws_param_store.go @@ -4,12 +4,12 @@ import ( "bytes" "errors" "fmt" - "os" "regexp" "strings" "time" "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/ssm" "github.com/subosito/gotenv" @@ -34,14 +34,9 @@ const ( type AWSParameterStore struct { Session *session.Session - context string - settings map[string]setting.Setting - - encryptionType string - credentialType string + clog catalog.Catalog uo cfg.UserOptions - io models.IO } @@ -79,93 +74,81 @@ func (s AWSParameterStore) Description() string { // Pre ... func (s *AWSParameterStore) Pre(clog catalog.Catalog, file *catalog.File, access contract.IVault, uo cfg.UserOptions, io models.IO) error { - s.settings = map[string]setting.Setting{} - s.context = clog.Context + s.clog = clog s.uo = uo s.io = io - s.credentialType = autoDetect - s.encryptionType = getEncryptionType(*file) - - (setting.Setting{ - Group: "AWS", - Prop: "REGION", + //------------------------------------------ + //- Get AWS Region + //------------------------------------------ + region, err := setting.Setting{ + Description: fmt.Sprintf("Silence this %s store prompt by setting environment variable.", s.Name()), + Group: clog.Context, + Prop: "AWS_REGION", Prompt: uo.Prompt, Silent: uo.Silent, AutoSave: true, DefaultValue: awsDefaultRegion, Vault: vault.EnvVault{}, - }).Get(clog.Context, io) + }.Get(clog.Context, io) //------------------------------------------ - //- Auth Credentials + //- Get AWS Credentials from Environment //------------------------------------------ - if uo.Prompt { - s.credentialType = strings.ToLower(prompt.GetValFromUser("Authentication", prompt.Options{ - Description: "OPTIONS\n (P)rofile \n (U)ser", - DefaultValue: "P"}, io)) - } - - switch s.credentialType { - case cTypeProfile: - os.Unsetenv(awsSecretAccessKey) - os.Unsetenv(awsAccessKeyID) - - (setting.Setting{ - Group: "AWS", - Prop: "PROFILE", - DefaultValue: os.Getenv(awsProfile), - Prompt: uo.Prompt, - Silent: uo.Silent, - AutoSave: true, - Vault: vault.EnvVault{}, - }).Get(clog.Context, io) - - case cTypeUser: - os.Unsetenv(awsProfile) - - (setting.Setting{ - Group: "AWS", - Prop: "ACCESS_KEY_ID", - Prompt: uo.Prompt, - Silent: uo.Silent, - AutoSave: true, - Vault: access, - }).Get(clog.Context, io) - - (setting.Setting{ - Group: "AWS", - Prop: "SECRET_ACCESS_KEY", - Prompt: uo.Prompt, - Silent: uo.Silent, - AutoSave: true, - Vault: access, - }).Get(clog.Context, io) + if _, ok := access.(vault.EnvVault); ok { + s.Session, err = session.NewSession(&aws.Config{ + Region: aws.String(region), + }) + + return err } //------------------------------------------ - //- Encryption + //- Get AWS Credentials from Vault //------------------------------------------ - s.settings[serverEncryptionToken] = setting.Setting{ - Description: "KMS Key ID is used by Parameter Store to encrypt and decrypt secrets. Any role or user accessing a secret must also have access to the KMS key. When pushing updates, the default setting will preserve existing KMS keys. The aws/ssm key is the default Systems Manager KMS key.", - Group: "AWS", - Prop: "STORE_KMS_KEY_ID", - DefaultValue: clog.GetDataByStore(s.Name(), "AWS_STORE_KMS_KEY_ID", defaultPSKMSKey), - Prompt: uo.Prompt, - Silent: uo.Silent, - AutoSave: false, - Vault: file, + id, err := setting.Setting{ + Description: fmt.Sprintf("Store credential for %s in %s.", s.Name(), access.Name()), + Group: clog.Context, + Prop: "AWS_ACCESS_KEY_ID", + Prompt: uo.Prompt, + Silent: uo.Silent, + AutoSave: true, + Vault: access, + }.Get(clog.Context, io) + if err != nil { + return err } - //------------------------------------------ - //- Open Connection - //------------------------------------------ - sess, err := session.NewSession() + secret, err := setting.Setting{ + Description: fmt.Sprintf("Store credential for %s in %s.", s.Name(), access.Name()), + Group: clog.Context, + Prop: "AWS_SECRET_ACCESS_KEY", + Prompt: uo.Prompt, + Silent: uo.Silent, + AutoSave: true, + Vault: access, + }.Get(clog.Context, io) + if err != nil { + return err + } + + token, err := setting.Setting{ + Description: fmt.Sprintf("Store credential for %s in %s.", s.Name(), access.Name()), + Group: clog.Context, + Prop: "AWS_SESSION_TOKEN", + Prompt: uo.Prompt, + Silent: uo.Silent, + AutoSave: true, + Vault: access, + }.Get(clog.Context, io) if err != nil { return err } - s.Session = sess + s.Session, err = session.NewSession(&aws.Config{ + Region: aws.String(region), + Credentials: credentials.NewStaticCredentials(id, secret, token), + }) return err } @@ -183,24 +166,27 @@ func (s AWSParameterStore) Push(file *catalog.File, fileData []byte, version str input := ssm.PutParameterInput{ Overwrite: aws.Bool(true), - Type: aws.String(ssm.ParameterTypeString), + Type: aws.String(ssm.ParameterTypeSecureString), } //------------------------------------------ - //- Get encryption keys + //- Get encryption key //------------------------------------------ - key, serverEncryption := s.settings[serverEncryptionToken] - if serverEncryption { - value, err := key.Get(s.context, s.io) - if err != nil { - return err - } - - if value != defaultPSKMSKey { - input.KeyId = &value - } + value, err := setting.Setting{ + Description: "KMS Key ID is used by Parameter Store to encrypt and decrypt secrets. Any role or user accessing a secret must also have access to the KMS key. When pushing updates, the default setting will preserve existing KMS keys. The aws/ssm key is the default Systems Manager KMS key.", + Prop: awsStoreKMSKeyID, + DefaultValue: s.clog.GetDataByStore(s.Name(), awsStoreKMSKeyID, defaultPSKMSKey), + Prompt: s.uo.Prompt, + Silent: s.uo.Silent, + AutoSave: false, + Vault: file, + }.Get(s.clog.Context, s.io) + if err != nil { + return err + } - input.Type = aws.String(ssm.ParameterTypeSecureString) + if value != defaultPSKMSKey { + input.KeyId = &value } //------------------------------------------ @@ -213,13 +199,13 @@ func (s AWSParameterStore) Push(file *catalog.File, fileData []byte, version str svc := ssm.New(s.Session) - storedParams, err := getStoredParamsWithMetaData(s.context, file.Path, version, svc) + storedParams, err := getStoredParamsWithMetaData(s.clog.Context, file.Path, version, svc) if err != nil { return err } for name, value := range newParams { - remoteKey := buildRemoteKey(s.context, file.Path, name, version) + remoteKey := buildRemoteKey(s.clog.Context, file.Path, name, version) newParam := param{ name: remoteKey, @@ -254,7 +240,7 @@ func (s AWSParameterStore) Push(file *catalog.File, fileData []byte, version str //------------------------------------------ for _, remoteParam := range storedParams { - param := strings.Replace(remoteParam.name, buildRemotePath(s.context, file.Path, version)+"/", "", 1) + param := strings.Replace(remoteParam.name, buildRemotePath(s.clog.Context, file.Path, version)+"/", "", 1) if !isParamIn(param, newParams) { if _, err := svc.DeleteParameter(&ssm.DeleteParameterInput{ @@ -274,7 +260,7 @@ func (s AWSParameterStore) Pull(file *catalog.File, version string) ([]byte, con svc := ssm.New(s.Session) - storedParams, err := getStoredParams(s.context, file.Path, version, svc) + storedParams, err := getStoredParams(s.clog.Context, file.Path, version, svc) if err != nil { return []byte{}, contract.Attributes{}, err } @@ -290,7 +276,7 @@ func (s AWSParameterStore) Pull(file *catalog.File, version string) ([]byte, con v := value if s.uo.StoreCommand == cmdRefFormat { - buffer.WriteString(fmt.Sprintf("%s=%s\n", name, buildRemoteKey(s.context, file.Path, name, version))) + buffer.WriteString(fmt.Sprintf("%s=%s\n", name, buildRemoteKey(s.clog.Context, file.Path, name, version))) } else { buffer.WriteString(fmt.Sprintf("%s=%s\n", name, v)) } @@ -304,7 +290,7 @@ func (s AWSParameterStore) Purge(file *catalog.File, version string) error { svc := ssm.New(s.Session) - storedParams, err := getStoredParams(s.context, file.Path, version, svc) + storedParams, err := getStoredParams(s.clog.Context, file.Path, version, svc) if err != nil { return err } @@ -337,7 +323,7 @@ func (s AWSParameterStore) Changed(file *catalog.File, fileData []byte, version svc := ssm.New(s.Session) - storedParams, err := getStoredParams(s.context, file.Path, version, svc) + storedParams, err := getStoredParams(s.clog.Context, file.Path, version, svc) if err != nil { return time.Time{}, err } @@ -346,7 +332,7 @@ func (s AWSParameterStore) Changed(file *catalog.File, fileData []byte, version for _, p := range storedParams { for name, value := range config { - remoteKey := buildRemoteKey(s.context, file.Path, name, version) + remoteKey := buildRemoteKey(s.clog.Context, file.Path, name, version) if remoteKey == p.name && value != p.value { changedParams = append(changedParams, p) diff --git a/components/store/aws_s3_store.go b/components/store/aws_s3_store.go index 66e72d5..97f3b67 100644 --- a/components/store/aws_s3_store.go +++ b/components/store/aws_s3_store.go @@ -4,11 +4,11 @@ import ( "bytes" "fmt" "io/ioutil" - "os" - "strings" "time" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3manager" @@ -16,41 +16,20 @@ import ( "github.com/turnerlabs/cstore/components/cfg" "github.com/turnerlabs/cstore/components/contract" "github.com/turnerlabs/cstore/components/models" - "github.com/turnerlabs/cstore/components/prompt" "github.com/turnerlabs/cstore/components/setting" "github.com/turnerlabs/cstore/components/vault" ) -const ( - awsBucketName = "AWS_S3_BUCKET" - clientEncryptionToken = "CLIENT_ENCRYPTION" - serverEncryptionToken = "SERVER_ENCRYPTION" - - fileDataEncryptionKey = "ENCRYPTION" - - sep = "::" - - eTypeClient = "c" - eTypeServer = "s" - - cTypeProfile = "p" - cTypeUser = "u" - - autoDetect = "d" - none = "n" -) - // S3Store ... type S3Store struct { Session *session.Session - context string - settings map[string]setting.Setting - - encryptionType string - credentialType string + clog catalog.Catalog + uo cfg.UserOptions io models.IO + + bucket setting.Setting } // Name ... @@ -82,109 +61,95 @@ func (s S3Store) Description() string { // Pre ... func (s *S3Store) Pre(clog catalog.Catalog, file *catalog.File, access contract.IVault, uo cfg.UserOptions, io models.IO) error { - s.settings = map[string]setting.Setting{} - s.context = clog.Context - s.io = io - - s.credentialType = autoDetect - s.encryptionType = getEncryptionType(*file) - (setting.Setting{ - Group: "AWS", - Prop: "REGION", - Prompt: uo.Prompt, - Silent: uo.Silent, - AutoSave: true, - DefaultValue: awsDefaultRegion, - Vault: vault.EnvVault{}, - }).Get(clog.Context, io) - - //--------------------------------------------- - //- Store authentication and encryption options - //--------------------------------------------- - if uo.Prompt { - s.credentialType = strings.ToLower(prompt.GetValFromUser("Authentication", prompt.Options{ - Description: "OPTIONS\n (P)rofile \n (U)ser", - DefaultValue: "P"}, io)) - } - - //------------------------------------------ - //- Required auth creds - //------------------------------------------ - switch s.credentialType { - case cTypeProfile: - os.Unsetenv(awsSecretAccessKey) - os.Unsetenv(awsAccessKeyID) - - (setting.Setting{ - Group: "AWS", - Prop: "PROFILE", - DefaultValue: os.Getenv(awsProfile), - Prompt: uo.Prompt, - Silent: uo.Silent, - AutoSave: true, - Vault: vault.EnvVault{}, - }).Get(clog.Context, io) - - case cTypeUser: - os.Unsetenv(awsProfile) - - (setting.Setting{ - Group: "AWS", - Prop: "ACCESS_KEY_ID", - Prompt: uo.Prompt, - Silent: uo.Silent, - AutoSave: true, - Vault: access, - }).Get(clog.Context, io) - - (setting.Setting{ - Group: "AWS", - Prop: "SECRET_ACCESS_KEY", - Prompt: uo.Prompt, - Silent: uo.Silent, - AutoSave: true, - Vault: access, - }).Get(clog.Context, io) - } + s.clog = clog + s.io = io + s.uo = uo //------------------------------------------ //- Store Configuration //------------------------------------------ - s.settings[awsBucketName] = setting.Setting{ + s.bucket = setting.Setting{ Description: "S3 Bucket that will store the file.", - Group: "AWS", - Prop: "S3_BUCKET", + Prop: awsBucketSetting, Prompt: uo.Prompt, Silent: uo.Silent, AutoSave: true, - DefaultValue: clog.GetDataByStore(s.Name(), awsBucketName, fmt.Sprintf("cstore-%s", clog.Context)), + DefaultValue: clog.GetDataByStore(s.Name(), awsBucketSetting, fmt.Sprintf("%s-configs", clog.Context)), Vault: file, } //------------------------------------------ - //- Encryption + //- Get AWS Region //------------------------------------------ - s.settings[serverEncryptionToken] = setting.Setting{ - Description: "KMS Key ID is used by S3 to encrypt and decrypt secrets. Any role or user accessing a secret must also have access to the KMS key. Leave blank to use the default bucket encryption settings.", - Group: "AWS", - Prop: "STORE_KMS_KEY_ID", + region, err := setting.Setting{ + Description: fmt.Sprintf("Silence this %s store prompt by setting environment variable.", s.Name()), + Group: clog.Context, + Prop: "AWS_REGION", Prompt: uo.Prompt, Silent: uo.Silent, - DefaultValue: clog.GetDataByStore(s.Name(), "AWS_STORE_KMS_KEY_ID", ""), - AutoSave: false, - Vault: file, + AutoSave: true, + DefaultValue: awsDefaultRegion, + Vault: vault.EnvVault{}, + }.Get(clog.Context, io) + + //------------------------------------------ + //- Get AWS Credentials from Environment + //------------------------------------------ + if _, ok := access.(vault.EnvVault); ok { + s.Session, err = session.NewSession(&aws.Config{ + Region: aws.String(region), + }) + + return err } //------------------------------------------ - //- Open connection to store. + //- Get AWS Credentials from Vault //------------------------------------------ - sess, err := session.NewSession() + id, err := setting.Setting{ + Description: fmt.Sprintf("Store credential for %s in %s.", s.Name(), access.Name()), + Group: clog.Context, + Prop: "AWS_ACCESS_KEY_ID", + Prompt: uo.Prompt, + Silent: uo.Silent, + AutoSave: true, + Vault: access, + }.Get(clog.Context, io) + if err != nil { + return err + } + + secret, err := setting.Setting{ + Description: fmt.Sprintf("Store credential for %s in %s.", s.Name(), access.Name()), + Group: clog.Context, + Prop: "AWS_SECRET_ACCESS_KEY", + Prompt: uo.Prompt, + Silent: uo.Silent, + AutoSave: true, + Vault: access, + }.Get(clog.Context, io) + if err != nil { + return err + } + + token, err := setting.Setting{ + Description: fmt.Sprintf("Store credential for %s in %s.", s.Name(), access.Name()), + Group: clog.Context, + Prop: "AWS_SESSION_TOKEN", + Prompt: uo.Prompt, + Silent: uo.Silent, + AutoSave: true, + Vault: access, + }.Get(clog.Context, io) if err != nil { return err } - s.Session = sess + s.Session, err = session.NewSession(&aws.Config{ + Region: aws.String(region), + Credentials: credentials.NewStaticCredentials(id, secret, token), + }) return err } @@ -194,9 +159,7 @@ func (s S3Store) Purge(file *catalog.File, version string) error { contextKey := s.key(file.Path, version) - setting, _ := s.settings[awsBucketName] - - bucket, err := setting.Get(s.context, s.io) + bucket, err := s.bucket.Get(s.clog.Context, s.io) if err != nil { return err } @@ -220,15 +183,13 @@ func (s S3Store) Push(file *catalog.File, fileData []byte, version string) error contextKey := s.key(file.Path, version) - setting, _ := s.settings[awsBucketName] - - bucket, err := setting.Get(s.context, s.io) + bucket, err := s.bucket.Get(s.clog.Context, s.io) if err != nil { return err } file.AddData(map[string]string{ - awsBucketName: bucket, + awsBucketSetting: bucket, }) input := &s3manager.UploadInput{ @@ -240,19 +201,21 @@ func (s S3Store) Push(file *catalog.File, fileData []byte, version string) error //------------------------------------------ //- Set server side KMS Key encryption //------------------------------------------ - if key, found := s.settings[serverEncryptionToken]; found { - - value, err := key.Get(s.context, s.io) - if err != nil { - return err - } + value, err := setting.Setting{ + Description: "KMS Key ID is used by S3 to encrypt and decrypt secrets. Any role or user accessing a secret must also have access to the KMS key. Leave blank to use the default bucket encryption settings.", + Prop: awsStoreKMSKeyID, + Prompt: s.uo.Prompt, + Silent: s.uo.Silent, + DefaultValue: s.clog.GetDataByStore(s.Name(), awsStoreKMSKeyID, ""), + AutoSave: false, + Vault: file, + }.Get(s.clog.Context, s.io) - if len(value) > 0 { - etype := "aws:kms" + if len(value) > 0 { + etype := "aws:kms" - input.SSEKMSKeyId = &value - input.ServerSideEncryption = &etype - } + input.SSEKMSKeyId = &value + input.ServerSideEncryption = &etype } uploader := s3manager.NewUploader(s.Session) @@ -267,10 +230,10 @@ func (s S3Store) Pull(file *catalog.File, version string) ([]byte, contract.Attr contextKey := s.key(file.Path, version) - setting, _ := s.settings[awsBucketName] + setting := s.bucket setting.Prompt = false - bucket, err := setting.Get(s.context, s.io) + bucket, err := setting.Get(s.clog.Context, s.io) if err != nil { return []byte{}, contract.Attributes{}, err } @@ -301,10 +264,10 @@ func (s S3Store) Changed(file *catalog.File, fileData []byte, version string) (t contextKey := s.key(file.Path, version) - setting, _ := s.settings[awsBucketName] + setting := s.bucket setting.Prompt = false - bucket, err := setting.Get(s.context, s.io) + bucket, err := setting.Get(s.clog.Context, s.io) if err != nil { return time.Time{}, err } @@ -342,15 +305,8 @@ func init() { func (s S3Store) key(path, version string) string { if len(version) > 0 { - return fmt.Sprintf("%s/%s/%s", s.context, version, path) + return fmt.Sprintf("%s/%s/%s", s.clog.Context, version, path) } - return fmt.Sprintf("%s/%s", s.context, path) -} - -func getEncryptionType(file catalog.File) string { - if _, found := file.Data[fileDataEncryptionKey]; found { - return eTypeClient - } - return none + return fmt.Sprintf("%s/%s", s.clog.Context, path) } diff --git a/components/store/aws_secrets_manager.go b/components/store/aws_secrets_manager.go index c24eb78..f9663be 100644 --- a/components/store/aws_secrets_manager.go +++ b/components/store/aws_secrets_manager.go @@ -5,11 +5,10 @@ import ( "encoding/json" "errors" "fmt" - "os" - "strings" "time" "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/service/secretsmanager" @@ -21,7 +20,6 @@ import ( "github.com/turnerlabs/cstore/components/contract" "github.com/turnerlabs/cstore/components/display" "github.com/turnerlabs/cstore/components/models" - "github.com/turnerlabs/cstore/components/prompt" "github.com/turnerlabs/cstore/components/setting" "github.com/turnerlabs/cstore/components/vault" ) @@ -32,14 +30,9 @@ const defaultSMKMSKey = "aws/secretsmanager" type AWSSecretsManagerStore struct { Session *session.Session - context string - settings map[string]setting.Setting - - encryptionType string - credentialType string + clog catalog.Catalog uo cfg.UserOptions - io models.IO } @@ -77,93 +70,81 @@ func (s AWSSecretsManagerStore) Description() string { // Pre ... func (s *AWSSecretsManagerStore) Pre(clog catalog.Catalog, file *catalog.File, access contract.IVault, uo cfg.UserOptions, io models.IO) error { - s.settings = map[string]setting.Setting{} - s.context = clog.Context + s.clog = clog s.uo = uo s.io = io - s.credentialType = autoDetect - s.encryptionType = getEncryptionType(*file) - - (setting.Setting{ - Group: "AWS", - Prop: "REGION", + //------------------------------------------ + //- Get AWS Region + //------------------------------------------ + region, err := setting.Setting{ + Description: fmt.Sprintf("Silence this %s store prompt by setting environment variable.", s.Name()), + Group: clog.Context, + Prop: "AWS_REGION", Prompt: uo.Prompt, Silent: uo.Silent, AutoSave: true, DefaultValue: awsDefaultRegion, Vault: vault.EnvVault{}, - }).Get(clog.Context, io) + }.Get(clog.Context, io) //------------------------------------------ - //- Auth Credentials + //- Get AWS Credentials from Environment //------------------------------------------ - if uo.Prompt { - s.credentialType = strings.ToLower(prompt.GetValFromUser("Authentication", prompt.Options{ - Description: "OPTIONS\n (P)rofile \n (U)ser", - DefaultValue: "P"}, io)) - } + if _, ok := access.(vault.EnvVault); ok { + s.Session, err = session.NewSession(&aws.Config{ + Region: aws.String(region), + }) - switch s.credentialType { - case cTypeProfile: - os.Unsetenv(awsSecretAccessKey) - os.Unsetenv(awsAccessKeyID) - - (setting.Setting{ - Group: "AWS", - Prop: "PROFILE", - DefaultValue: os.Getenv(awsProfile), - Prompt: uo.Prompt, - Silent: uo.Silent, - AutoSave: true, - Vault: vault.EnvVault{}, - }).Get(clog.Context, io) - - case cTypeUser: - os.Unsetenv(awsProfile) - - (setting.Setting{ - Group: "AWS", - Prop: "ACCESS_KEY_ID", - Prompt: uo.Prompt, - Silent: uo.Silent, - AutoSave: true, - Vault: access, - }).Get(clog.Context, io) - - (setting.Setting{ - Group: "AWS", - Prop: "SECRET_ACCESS_KEY", - Prompt: uo.Prompt, - Silent: uo.Silent, - AutoSave: true, - Vault: access, - }).Get(clog.Context, io) + return err } //------------------------------------------ - //- Encryption + //- Get AWS Credentials from Vault //------------------------------------------ - s.settings[serverEncryptionToken] = setting.Setting{ - Description: "KMS Key ID is used by Parameter Store to encrypt and decrypt secrets. Any role or user accessing a secret must also have access to the KMS key. When pushing updates, the default setting will preserve existing KMS keys. The aws/ssm key is the default Systems Manager KMS key.", - Group: "AWS", - Prop: "STORE_KMS_KEY_ID", - DefaultValue: clog.GetDataByStore(s.Name(), "AWS_STORE_KMS_KEY_ID", defaultSMKMSKey), - Prompt: uo.Prompt, - Silent: uo.Silent, - AutoSave: false, - Vault: file, + id, err := setting.Setting{ + Description: fmt.Sprintf("Store credential for %s in %s.", s.Name(), access.Name()), + Group: clog.Context, + Prop: "AWS_ACCESS_KEY_ID", + Prompt: uo.Prompt, + Silent: uo.Silent, + AutoSave: true, + Vault: access, + }.Get(clog.Context, io) + if err != nil { + return err } - //------------------------------------------ - //- Open Connection - //------------------------------------------ - sess, err := session.NewSession() + secret, err := setting.Setting{ + Description: fmt.Sprintf("Store credential for %s in %s.", s.Name(), access.Name()), + Group: clog.Context, + Prop: "AWS_SECRET_ACCESS_KEY", + Prompt: uo.Prompt, + Silent: uo.Silent, + AutoSave: true, + Vault: access, + }.Get(clog.Context, io) + if err != nil { + return err + } + + token, err := setting.Setting{ + Description: fmt.Sprintf("Store credential for %s in %s.", s.Name(), access.Name()), + Group: clog.Context, + Prop: "AWS_SESSION_TOKEN", + Prompt: uo.Prompt, + Silent: uo.Silent, + AutoSave: true, + Vault: access, + }.Get(clog.Context, io) if err != nil { return err } - s.Session = sess + s.Session, err = session.NewSession(&aws.Config{ + Region: aws.String(region), + Credentials: credentials.NewStaticCredentials(id, secret, token), + }) return err } @@ -180,20 +161,23 @@ func (s AWSSecretsManagerStore) Push(file *catalog.File, fileData []byte, versio } //------------------------------------------ - //- Get encryption keys + //- Get encryption key //------------------------------------------ - KMSKeyID := "" - - key, serverEncryption := s.settings[serverEncryptionToken] - if serverEncryption { - value, err := key.Get(s.context, s.io) - if err != nil { - return err - } + KMSKeyID, err := setting.Setting{ + Description: "KMS Key ID is used by Secrets Manager to encrypt and decrypt secrets. Any role or user accessing a secret must also have access to the KMS key. When pushing updates, the default setting will preserve existing KMS keys. The aws/ssm key is the default Systems Manager KMS key.", + Prop: awsStoreKMSKeyID, + DefaultValue: s.clog.GetDataByStore(s.Name(), awsStoreKMSKeyID, defaultSMKMSKey), + Prompt: s.uo.Prompt, + Silent: s.uo.Silent, + AutoSave: false, + Vault: file, + }.Get(s.clog.Context, s.io) + if err != nil { + return err + } - if value != defaultSMKMSKey { - KMSKeyID = value - } + if KMSKeyID == defaultSMKMSKey { + KMSKeyID = "" } //------------------------------------------ @@ -214,7 +198,7 @@ func (s AWSSecretsManagerStore) Push(file *catalog.File, fileData []byte, versio continue } - key := formatSecretToken(s.context, file.Path, name) + key := formatSecretToken(s.clog.Context, file.Path, name) removed := true for k := range params { @@ -237,7 +221,7 @@ func (s AWSSecretsManagerStore) Push(file *catalog.File, fileData []byte, versio for name, value := range params { - key := formatSecretToken(s.context, file.Path, name) + key := formatSecretToken(s.clog.Context, file.Path, name) storedProps, err := getSecret(key, svc) @@ -324,7 +308,7 @@ func hasSecretChanged(existing secret, name, value string, data map[string]strin return true } - keyID, _ := data["AWS_STORE_KMS_KEY_ID"] + keyID, _ := data[awsStoreKMSKeyID] return v != value || (keyID != "" && existing.keyID != keyID) } @@ -410,7 +394,7 @@ func (s AWSSecretsManagerStore) Pull(file *catalog.File, version string) ([]byte svc := secretsmanager.New(s.Session) - storedSecrets, err := getSecrets(s.context, file.Path, file.Data, svc) + storedSecrets, err := getSecrets(s.clog.Context, file.Path, file.Data, svc) if err != nil { return []byte{}, contract.Attributes{}, err } @@ -438,7 +422,7 @@ func (s AWSSecretsManagerStore) Purge(file *catalog.File, version string) error continue } - key := formatSecretToken(s.context, file.Path, name) + key := formatSecretToken(s.clog.Context, file.Path, name) if _, err := svc.DeleteSecret(&secretsmanager.DeleteSecretInput{ SecretId: aws.String(key), @@ -462,7 +446,7 @@ func (s AWSSecretsManagerStore) Changed(file *catalog.File, fileData []byte, ver continue } - secret, err := describeSecret(formatSecretToken(s.context, file.Path, name), svc) + secret, err := describeSecret(formatSecretToken(s.clog.Context, file.Path, name), svc) if err != nil { return time.Time{}, err } diff --git a/components/store/harbor_store.go b/components/store/harbor_store.go deleted file mode 100644 index 075649b..0000000 --- a/components/store/harbor_store.go +++ /dev/null @@ -1,449 +0,0 @@ -package store - -// import ( -// "bytes" -// "encoding/json" -// "errors" -// "fmt" -// "net/http" -// "time" - -// "github.com/subosito/gotenv" -// "github.com/turnerlabs/cstore/components/catalog" -// "github.com/turnerlabs/cstore/components/contract" -// "github.com/turnerlabs/cstore/components/models" -// "github.com/turnerlabs/cstore/components/prompt" -// "github.com/turnerlabs/cstore/components/token" -// harborauth "github.com/turnerlabs/harbor-auth-client" -// ) - -// const ( -// authURL = "http://auth.services.dmtio.net" -// shipURL = "http://shipit.services.dmtio.net" - -// tokenToken = "HARBOR_TOKEN" -// userToken = "HARBOR_USER" -// passToken = "HARBOR_PASS" - -// shipmentToken = "HARBOR_SHIPMENT" -// containerToken = "HARBOR_CONTAINER" -// envToken = "HARBOR_ENV" - -// modifiedToken = "CSTORE_MODIFIED" -// modifiedLayout = "2006-01-02 15:04:05.999999999 -0700 MST" - -// envTypeBasic = "basic" -// envTypeDiscover = "discover" -// envTypeHidden = "hidden" -// ) - -// // HarborStore ... -// type HarborStore struct { -// Vault contract.IVault - -// Auth HarborAuth -// Shipment HarborShipment - -// io models.IO -// } - -// // HarborAuth ... -// type HarborAuth struct { -// User string -// Token string -// } - -// // HarborShipment ... -// type HarborShipment struct { -// Name string -// Container string -// Env string -// } - -// // Name ... -// func (s HarborStore) Name() string { -// return "harbor" -// } - -// // Supports ... -// func (s HarborStore) Supports(feature string) bool { -// return false -// } - -// // Description ... -// func (s HarborStore) Description() string { -// return `Environment variables listed in a .env file can be stored in Harbor at the shipment container level. - -// When pushing a .env file, a user will be prompted for NT credentails. When the temporary access token expires, the user will be prompted for credentials again. - -// A shipment, environment, and container are required when using this store to identify which container will store the environment variables. -// ` -// } - -// // Pre ... -// func (s *HarborStore) Pre(contextID string, file catalog.File, cv contract.IVault, promptUser bool, io models.IO) error { -// s.io = io - -// client, err := harborauth.NewAuthClient(authURL) -// if err != nil { -// return err -// } - -// s.Shipment = HarborShipment{} -// s.Auth = HarborAuth{} - -// isAuth := false - -// // Argonauts Login ID -// if value, err := cv.Get(contextID, userToken, ""); err == nil { -// s.Auth.User = value -// } - -// if value, err := cv.Get(contextID, tokenToken, ""); err == nil { -// s.Auth.Token = value -// } - -// if len(s.Auth.Token) > 0 && len(s.Auth.User) > 0 { -// isAuth, _ = client.IsAuthenticated(s.Auth.User, s.Auth.Token) -// } - -// if !isAuth { -// // Argonauts Login ID -// s.Auth.User = prompt.GetValFromUser(userToken, prompt.Options{}, io) -// pass := prompt.GetValFromUser(passToken, prompt.Options{HideInput: true}, s.io) - -// token, success, err := client.Login(s.Auth.User, pass) -// if err != nil || !success { -// return err -// } - -// err = cv.Set(contextID, userToken, "", s.Auth.User) -// if err != nil { -// return err -// } - -// err = cv.Set(contextID, tokenToken, "", token) -// if err != nil { -// return err -// } - -// s.Auth.Token = token -// } - -// if shipment, found := file.Data[shipmentToken]; found { -// s.Shipment.Name = shipment -// } else { -// s.Shipment.Name = prompt.GetValFromUser(shipmentToken, prompt.Options{}, s.io) -// } - -// if container, found := file.Data[containerToken]; found { -// s.Shipment.Container = container -// } else { -// s.Shipment.Container = prompt.GetValFromUser(containerToken, prompt.Options{}, s.io) -// } - -// if env, found := file.Data[envToken]; found { -// s.Shipment.Env = env -// } else { -// s.Shipment.Env = prompt.GetValFromUser(envToken, prompt.Options{}, s.io) -// } - -// return nil -// } - -// // Push ... -// func (s HarborStore) Push(file catalog.File, fileData []byte, version string) (map[string]string, error) { - -// if !file.IsEnv { -// return map[string]string{}, fmt.Errorf("cannot store file %s", file.Path) -// } - -// data := map[string]string{ -// shipmentToken: s.Shipment.Name, -// containerToken: s.Shipment.Container, -// envToken: s.Shipment.Env, -// } - -// localKeys := gotenv.Parse(bytes.NewReader(fileData)) -// localKeys[modifiedToken] = time.Now().UTC().String() - -// url := buildURL(s.Shipment) - -// for key, value := range localKeys { - -// prefixedKey := addEnvVarPrefix(key) - -// keyType := envTypeHidden -// if storedKeyType, found := file.Data[prefixedKey]; found { -// if storedKeyType != envVarType { -// keyType = storedKeyType -// } -// } - -// p := pair{ -// Name: key, -// Value: value, -// Type: keyType, -// } - -// if err := createKey(p, url, s.Auth); err != nil { -// if err := updateKey(p, url, s.Auth); err != nil { -// return data, err -// } -// } - -// data[prefixedKey] = keyType -// } - -// harborKeys, err := getHarborKeys(s.Shipment, s.Auth) -// if err != nil { -// return data, err -// } - -// for key := range harborKeys { -// prefixedKey := addEnvVarPrefix(key) - -// if _, found := file.Data[prefixedKey]; found { -// if _, found := localKeys[key]; !found { -// if err := deleteKey(key, url, s.Auth); err != nil { -// return data, err -// } -// } -// } -// } - -// return data, nil -// } - -// // Pull ... -// func (s HarborStore) Pull(file catalog.File, version string) ([]byte, error) { - -// keys, err := getHarborKeys(s.Shipment, s.Auth) -// if err != nil { -// return []byte{}, err -// } - -// var buffer bytes.Buffer - -// for key, contents := range keys { -// if key == modifiedToken { -// continue -// } - -// if _, found := file.Data[addEnvVarPrefix(key)]; found { -// buffer.WriteString(fmt.Sprintf("%s=%s\n", key, contents.value)) -// } -// } - -// // if modified, found := keys[modifiedToken]; found { -// // m, err := time.Parse(modifiedLayout, modified.value) -// // if err == nil { -// // attr.LastModified = m -// // } -// // } - -// return buffer.Bytes(), nil -// } - -// // Purge ... -// func (s HarborStore) Purge(file catalog.File, version string) error { - -// url := buildURL(s.Shipment) - -// for key, value := range file.Data { -// if isEnvVarType(value) { -// if err := deleteKey(key, url, s.Auth); err != nil { -// return err -// } -// } -// } - -// return nil -// } - -// // GetTokenValues ... -// func (s HarborStore) GetTokenValues(tokens map[string]token.Token, contextID string) (map[string]token.Token, error) { -// return map[string]token.Token{}, nil -// } - -// // SaveTokenValues ... -// func (s HarborStore) SaveTokenValues(tokens map[string]token.Token, contextID string) (map[string]token.Token, error) { -// return map[string]token.Token{}, nil -// } - -// func isEnvVarType(envVarType string) bool { -// switch envVarType { -// case envTypeBasic: -// return true -// case envTypeDiscover: -// return true -// case envTypeHidden: -// return true -// default: -// return false -// } -// } - -// type pair struct { -// Name string `json:"name"` -// Value string `json:"value"` -// Type string `json:"type"` -// } - -// func createKey(p pair, url string, auth HarborAuth) error { -// client := &http.Client{} - -// b, err := json.Marshal(p) -// if err != nil { -// return err -// } - -// url = fmt.Sprintf("%s/envVars", url) - -// r := bytes.NewReader(b) - -// req, err := http.NewRequest("POST", url, r) -// req.Header.Add("x-token", auth.Token) -// req.Header.Add("x-username", auth.User) -// req.Header.Add("Content-Type", "application/json") - -// resp, err := client.Do(req) -// if err != nil { -// return err -// } - -// if resp.StatusCode != http.StatusCreated { -// return errors.New(resp.Status) -// } - -// return nil -// } - -// func updateKey(p pair, url string, auth HarborAuth) error { - -// client := &http.Client{} - -// b, err := json.Marshal(p) -// if err != nil { -// return err -// } - -// url = fmt.Sprintf("%s/envVar/%s", url, p.Name) - -// r := bytes.NewReader(b) - -// req, err := http.NewRequest("PUT", url, r) -// req.Header.Add("x-token", auth.Token) -// req.Header.Add("x-username", auth.User) -// req.Header.Add("Content-Type", "application/json") - -// resp, err := client.Do(req) -// if err != nil { -// return err -// } - -// if resp.StatusCode != http.StatusOK { -// return errors.New(resp.Status) -// } - -// return nil -// } - -// func deleteKey(key, url string, auth HarborAuth) error { - -// client := &http.Client{} - -// url = fmt.Sprintf("%s/envVar/%s", url, key) - -// req, err := http.NewRequest("DELETE", url, nil) -// req.Header.Add("x-token", auth.Token) -// req.Header.Add("x-username", auth.User) -// req.Header.Add("Content-Type", "application/json") - -// resp, err := client.Do(req) -// if err != nil { -// return err -// } - -// if resp.StatusCode != http.StatusOK { -// return errors.New(resp.Status) -// } - -// return nil -// } - -// type harborKey struct { -// value string -// vType string -// } - -// func getHarborKeys(shipment HarborShipment, auth HarborAuth) (map[string]harborKey, error) { - -// client := &http.Client{} - -// url := fmt.Sprintf("%s/v1/shipment/%s/environment/%s", shipURL, shipment.Name, shipment.Env) - -// req, err := http.NewRequest("GET", url, nil) -// req.Header.Add("x-token", auth.Token) -// req.Header.Add("x-username", auth.User) -// req.Header.Add("Content-Type", "application/json") - -// resp, err := client.Do(req) -// if err != nil { -// return nil, err -// } - -// if resp.StatusCode != http.StatusOK { -// return nil, errors.New(resp.Status) -// } - -// s := new(HShipment) -// if err = json.NewDecoder(resp.Body).Decode(s); err != nil { -// return nil, err -// } - -// envVars := map[string]harborKey{} - -// for _, c := range s.Containers { -// if c.Name == shipment.Container { -// for _, envVar := range c.EnvVars { -// envVars[envVar.Name] = harborKey{ -// value: envVar.Value, -// vType: envVar.Type, -// } -// } -// } -// } - -// return envVars, nil -// } - -// // HShipment ... -// type HShipment struct { -// Containers []HContainers `json:"containers"` -// } - -// // HContainers ... -// type HContainers struct { -// Name string `json:"name"` -// EnvVars []HEnvVar `json:"envVars"` -// } - -// // HEnvVar ... -// type HEnvVar struct { -// Name string `json:"name"` -// Value string `json:"value"` -// Type string `json:"type"` -// } - -// func buildURL(shipment HarborShipment) string { -// return fmt.Sprintf("%s/v1/shipment/%s/environment/%s/container/%s", shipURL, shipment.Name, shipment.Env, shipment.Container) -// } - -// func init() { -// //-------------------------------- -// //- Disabled until converted to v2 -// //-------------------------------- -// //s := new(HarborStore) -// //stores[s.Name()] = s -// } diff --git a/components/store/source_control_store.go b/components/store/source_control_store.go index 9f701cd..17e1a27 100644 --- a/components/store/source_control_store.go +++ b/components/store/source_control_store.go @@ -10,18 +10,11 @@ import ( "github.com/turnerlabs/cstore/components/contract" localFile "github.com/turnerlabs/cstore/components/file" "github.com/turnerlabs/cstore/components/models" - "github.com/turnerlabs/cstore/components/setting" ) // SourceControlStore ... type SourceControlStore struct { clog catalog.Catalog - - settings map[string]setting.Setting - - uo cfg.UserOptions - - io models.IO } // Name ... @@ -55,11 +48,6 @@ func (s SourceControlStore) Description() string { // Pre ... func (s *SourceControlStore) Pre(clog catalog.Catalog, file *catalog.File, access contract.IVault, uo cfg.UserOptions, io models.IO) error { - s.settings = map[string]setting.Setting{} - - s.uo = uo - s.io = io - s.clog = clog return nil diff --git a/components/vault/aws.go b/components/vault/aws.go new file mode 100644 index 0000000..86f4ca1 --- /dev/null +++ b/components/vault/aws.go @@ -0,0 +1,13 @@ +package vault + +const ( + awsRegion = "AWS_REGION" + awsProfile = "AWS_PROFILE" + awsSecretAccessKey = "AWS_SECRET_ACCESS_KEY" + awsAccessKeyID = "AWS_ACCESS_KEY_ID" + + awsVaultKMSKeyID = "AWS_VAULT_KMS_KEY_ID" + + awsDefaultRegion = "us-east-1" + awsDefaultProfile = "default" +) diff --git a/components/vault/aws_secrets_manager_vault.go b/components/vault/aws_secrets_manager_vault.go index 909aba2..ac7ce96 100644 --- a/components/vault/aws_secrets_manager_vault.go +++ b/components/vault/aws_secrets_manager_vault.go @@ -5,6 +5,8 @@ import ( "errors" "fmt" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/session" @@ -27,8 +29,11 @@ type vaultSettings struct { type AWSSecretsManagerVault struct { Session *session.Session - settings vaultSettings - io models.IO + clog catalog.Catalog + fileEntry *catalog.File + + uo cfg.UserOptions + io models.IO } // Name ... @@ -57,30 +62,85 @@ func (v AWSSecretsManagerVault) BuildKey(contextID, group, prop string) string { } // Pre ... -func (v *AWSSecretsManagerVault) Pre(clog catalog.Catalog, fileEntry *catalog.File, uo cfg.UserOptions, io models.IO) error { +func (v *AWSSecretsManagerVault) Pre(clog catalog.Catalog, fileEntry *catalog.File, access contract.IVault, uo cfg.UserOptions, io models.IO) error { + v.uo = uo v.io = io - v.settings = vaultSettings{ - KMSKeyID: setting.Setting{ - Description: "KMS Key ID is used by Secrets Manager to encrypt and decrypt secrets. Any role or user accessing a secret must also have access to the KMS key. The aws/secretsmanager is the default Secrets Manager KMS key.", - Group: "AWS", - Prop: "VAULT_KMS_KEY_ID", - Prompt: uo.Prompt, - Silent: uo.Silent, - AutoSave: false, - DefaultValue: clog.GetDataByVault(v.Name(), "AWS_VAULT_KMS_KEY_ID", defaultKMSKey), - Vault: fileEntry, - }, + v.fileEntry = fileEntry + + //------------------------------------------ + //- Get AWS Region + //------------------------------------------ + region, err := setting.Setting{ + Description: fmt.Sprintf("Silence this %s vault prompt by setting environment variable.", v.Name()), + Group: clog.Context, + Prop: "AWS_REGION", + Prompt: uo.Prompt, + Silent: uo.Silent, + AutoSave: true, + DefaultValue: awsDefaultRegion, + Vault: EnvVault{}, + }.Get(clog.Context, io) + + //------------------------------------------ + //- Get AWS Credentials from Environment + //------------------------------------------ + if _, ok := access.(EnvVault); ok { + v.Session, err = session.NewSession(&aws.Config{ + Region: aws.String(region), + }) + + return err } - sess, err := session.NewSession() + //------------------------------------------ + //- Get AWS Credentials from Vault + //------------------------------------------ + id, err := setting.Setting{ + Description: fmt.Sprintf("Store credential for %s in %s.", v.Name(), access.Name()), + Group: clog.Context, + Prop: "AWS_ACCESS_KEY_ID", + Prompt: uo.Prompt, + Silent: uo.Silent, + AutoSave: true, + Vault: access, + }.Get(clog.Context, io) if err != nil { return err } - v.Session = sess + secret, err := setting.Setting{ + Description: fmt.Sprintf("Store credential for %s in %s.", v.Name(), access.Name()), + Group: clog.Context, + Prop: "AWS_SECRET_ACCESS_KEY", + Prompt: uo.Prompt, + Silent: uo.Silent, + AutoSave: true, + Vault: access, + }.Get(clog.Context, io) + if err != nil { + return err + } - return nil + token, err := setting.Setting{ + Description: fmt.Sprintf("Store credential for %s in %s.", v.Name(), access.Name()), + Group: clog.Context, + Prop: "AWS_SESSION_TOKEN", + Prompt: uo.Prompt, + Silent: uo.Silent, + AutoSave: true, + Vault: access, + }.Get(clog.Context, io) + if err != nil { + return err + } + + v.Session, err = session.NewSession(&aws.Config{ + Region: aws.String(region), + Credentials: credentials.NewStaticCredentials(id, secret, token), + }) + + return err } // Set ... @@ -90,7 +150,15 @@ func (v AWSSecretsManagerVault) Set(contextID, group, prop, value string) error svc := secretsmanager.New(v.Session) - KMSKeyID, err := v.settings.KMSKeyID.Get(contextID, v.io) + KMSKeyID, err := setting.Setting{ + Description: "KMS Key ID is used by Secrets Manager to encrypt and decrypt secrets. Any role or user accessing a secret must also have access to the KMS key. The aws/secretsmanager is the default Secrets Manager KMS key.", + Prop: awsVaultKMSKeyID, + Prompt: v.uo.Prompt, + Silent: v.uo.Silent, + AutoSave: false, + DefaultValue: v.clog.GetDataByVault(v.Name(), awsVaultKMSKeyID, defaultKMSKey), + Vault: v.fileEntry, + }.Get(contextID, v.io) if err != nil { return err } diff --git a/components/vault/env_vault.go b/components/vault/env_vault.go index d931ba7..4ae27fb 100644 --- a/components/vault/env_vault.go +++ b/components/vault/env_vault.go @@ -1,7 +1,6 @@ package vault import ( - "fmt" "os" "strings" @@ -30,15 +29,11 @@ When using this vault, users are prompted for any required environment variables // BuildKey ... func (v EnvVault) BuildKey(contextID, group, prop string) string { - if len(prop) > 0 { - return strings.ToUpper(fmt.Sprintf("%s_%s", group, prop)) - } - - return strings.ToUpper(group) + return strings.ToUpper(prop) } // Pre ... -func (v EnvVault) Pre(clog catalog.Catalog, fileEntry *catalog.File, uo cfg.UserOptions, io models.IO) error { +func (v EnvVault) Pre(clog catalog.Catalog, fileEntry *catalog.File, access contract.IVault, uo cfg.UserOptions, io models.IO) error { return nil } diff --git a/components/vault/file_vault.go b/components/vault/file_vault.go index a88849d..9d9a607 100644 --- a/components/vault/file_vault.go +++ b/components/vault/file_vault.go @@ -43,7 +43,7 @@ func (v FileVault) BuildKey(contextID, group, prop string) string { } // Pre ... -func (v FileVault) Pre(clog catalog.Catalog, fileEntry *catalog.File, uo cfg.UserOptions, io models.IO) error { +func (v FileVault) Pre(clog catalog.Catalog, fileEntry *catalog.File, access contract.IVault, uo cfg.UserOptions, io models.IO) error { return nil } diff --git a/components/vault/keychain_vault_darwin.go b/components/vault/keychain_vault_darwin.go index 8019ecf..c1997bf 100644 --- a/components/vault/keychain_vault_darwin.go +++ b/components/vault/keychain_vault_darwin.go @@ -29,14 +29,14 @@ func (v KeychainVault) Description() string { // BuildKey ... func (v KeychainVault) BuildKey(contextID, group, prop string) string { if len(prop) > 0 { - return fmt.Sprintf("%s-%s", group, prop) + return fmt.Sprintf("%s: %s", group, prop) } return group } // Pre ... -func (v KeychainVault) Pre(clog catalog.Catalog, fileEntry *catalog.File, uo cfg.UserOptions, io models.IO) error { +func (v KeychainVault) Pre(clog catalog.Catalog, fileEntry *catalog.File, access contract.IVault, uo cfg.UserOptions, io models.IO) error { return nil } diff --git a/components/vault/vault.go b/components/vault/vault.go index 429a993..c109195 100644 --- a/components/vault/vault.go +++ b/components/vault/vault.go @@ -18,14 +18,14 @@ func Get() map[string]contract.IVault { } // GetBy ... -func GetBy(name, defaultVault string, clog catalog.Catalog, fileEntry *catalog.File, uo cfg.UserOptions, io models.IO) (contract.IVault, error) { +func GetBy(name, defaultVault string, clog catalog.Catalog, fileEntry *catalog.File, access contract.IVault, uo cfg.UserOptions, io models.IO) (contract.IVault, error) { if len(name) == 0 { v := vaults[defaultVault] - return v, v.Pre(clog, fileEntry, uo, io) + return v, v.Pre(clog, fileEntry, access, uo, io) } if v, found := vaults[name]; found { - return v, v.Pre(clog, fileEntry, uo, io) + return v, v.Pre(clog, fileEntry, access, uo, io) } return nil, errors.New("vault not found") }