From b543fbdd600043ba28b0869d3aec3a81236e2364 Mon Sep 17 00:00:00 2001 From: kendavis2 Date: Thu, 30 Apr 2020 14:24:08 -0400 Subject: [PATCH] tokenize file path --- cli/cmd/clean.go | 6 ++--- cli/cmd/init.go | 3 ++- cli/cmd/list.go | 2 +- cli/cmd/pull.go | 22 ++++++++-------- cli/cmd/purge.go | 18 +++++++------ cli/cmd/push.go | 22 ++++++++-------- components/catalog/model.go | 11 +++++--- components/path/substitute.go | 25 +++++++++++++++++++ components/store/aws_param_store.go | 18 ++++++------- components/store/aws_s3_store.go | 8 +++--- components/store/aws_secret_manager_store.go | 10 ++++---- components/store/aws_secrets_manager_store.go | 12 ++++----- components/store/source_control_store.go | 4 +-- components/store/store.go | 2 +- docs/CATALOG.md | 2 +- pull.go | 2 +- 16 files changed, 100 insertions(+), 67 deletions(-) create mode 100644 components/path/substitute.go diff --git a/cli/cmd/clean.go b/cli/cmd/clean.go index c05729d..3bdefeb 100644 --- a/cli/cmd/clean.go +++ b/cli/cmd/clean.go @@ -44,7 +44,7 @@ func cleanCatalog(catalogPath string) { for _, f := range files { if f.IsRef { - cleanCatalog(path.BuildPath(root, f.Path)) + cleanCatalog(path.BuildPath(root, f.ActualPath())) } remoteComp, err := remote.InitComponents(&f, clog, uo, ioStreams) @@ -54,7 +54,7 @@ func cleanCatalog(catalogPath string) { } if !remoteComp.Store.SupportsFeature(store.SourceControlFeature) { - file := clog.GetFullPath(f.Path) + file := clog.GetFullPath(f.ActualPath()) if err := os.Remove(file); err != nil { if !os.IsNotExist(err) { display.Error(fmt.Errorf("failed to delete %s (%s)", file, err), ioStreams.UserOutput) @@ -69,7 +69,7 @@ func cleanCatalog(catalogPath string) { } } - secretsFile := fmt.Sprintf("%s.secrets", clog.GetFullPath(f.Path)) + secretsFile := fmt.Sprintf("%s.secrets", clog.GetFullPath(f.ActualPath())) if err := os.Remove(secretsFile); err != nil { if !os.IsNotExist(err) { display.Error(fmt.Errorf("failed to delete %s (%s)", secretsFile, err), ioStreams.UserOutput) diff --git a/cli/cmd/init.go b/cli/cmd/init.go index b810859..472c8ee 100644 --- a/cli/cmd/init.go +++ b/cli/cmd/init.go @@ -3,6 +3,7 @@ package cmd import ( "os" + "github.com/turnerlabs/cstore/v4/components/path" "github.com/turnerlabs/cstore/v4/components/remote" "github.com/spf13/cobra" @@ -29,7 +30,7 @@ var initCmd = &cobra.Command{ for _, filePath := range getFilePathsToPush(clog, uo) { - file, err := localFile.GetBy(clog.GetFullPath(filePath)) + file, err := localFile.GetBy(clog.GetFullPath(path.SubstituteTokens(filePath))) if err != nil { display.Error(err, ioStreams.UserOutput) continue diff --git a/cli/cmd/list.go b/cli/cmd/list.go index ff02f4c..d1e58a3 100644 --- a/cli/cmd/list.go +++ b/cli/cmd/list.go @@ -51,7 +51,7 @@ func listFilesFor(catalogPath string, opt cfg.UserOptions, io models.IO) (int, e total := 0 for _, fileEntry := range clog.FilesBy(opt.GetPaths(clog.CWD), opt.TagList, opt.AllTags, opt.Version) { - fullPath := path.BuildPath(basePath, fileEntry.Path) + fullPath := path.BuildPath(basePath, fileEntry.ActualPath()) //------------------------------------------------- //- If entry is catalog, print child entries. diff --git a/cli/cmd/pull.go b/cli/cmd/pull.go index 36ce710..c5a9af9 100644 --- a/cli/cmd/pull.go +++ b/cli/cmd/pull.go @@ -93,7 +93,7 @@ func Pull(catalogPath string, opt cfg.UserOptions, io models.IO) (int, int, erro //- Check for a linked catalog with child files. //---------------------------------------------------- if fileEntry.IsRef { - c, t, err := Pull(path.BuildPath(root, fileEntry.Path), opt, io) + c, t, err := Pull(path.BuildPath(root, fileEntry.ActualPath()), opt, io) if err != nil { return 0, 0, err } @@ -111,7 +111,7 @@ func Pull(catalogPath string, opt cfg.UserOptions, io models.IO) (int, int, erro fileEntryTemp := fileEntry remoteComp, err := remote.InitComponents(&fileEntryTemp, clog, opt, io) if err != nil { - display.Error(fmt.Errorf("PullFailedException3: %s (%s)", getPath(root, fileEntry.Path, opt.Version), err), io.UserOutput) + display.Error(fmt.Errorf("PullFailedException3: %s (%s)", getPath(root, fileEntry.ActualPath(), opt.Version), err), io.UserOutput) continue } @@ -120,7 +120,7 @@ func Pull(catalogPath string, opt cfg.UserOptions, io models.IO) (int, int, erro //---------------------------------------------------- file, _, err := remoteComp.Store.Pull(&fileEntry, opt.Version) if err != nil { - display.Error(fmt.Errorf("PullFailedException4: %s (%s)", getPath(root, fileEntry.Path, opt.Version), err), io.UserOutput) + display.Error(fmt.Errorf("PullFailedException4: %s (%s)", getPath(root, fileEntry.ActualPath(), opt.Version), err), io.UserOutput) continue } @@ -138,20 +138,20 @@ func Pull(catalogPath string, opt cfg.UserOptions, io models.IO) (int, int, erro if opt.InjectSecrets || opt.ModifySecrets { if !fileEntry.SupportsSecrets() { - display.Error(fmt.Errorf("IncompatibleFileError: %s secrets not supported", fileEntry.Path), io.UserOutput) + display.Error(fmt.Errorf("IncompatibleFileError: %s secrets not supported", fileEntry.ActualPath()), io.UserOutput) continue } tokens, err := token.Find(fileWithSecrets, fileEntry.Type, false) if err != nil { - display.Error(fmt.Errorf("MissingTokensError: failed to find tokens in file %s (%s)", fileEntry.Path, err), io.UserOutput) + display.Error(fmt.Errorf("MissingTokensError: failed to find tokens in file %s (%s)", fileEntry.ActualPath(), err), io.UserOutput) } for k, t := range tokens { value, err := remoteComp.Secrets.Get(clog.Context, t.Secret(), t.Prop) if err != nil { - display.Error(fmt.Errorf("GetSecretValueError: failed to get value for %s/%s for %s (%s)", t.Secret(), t.Prop, path.BuildPath(root, fileEntry.Path), err), io.UserOutput) + display.Error(fmt.Errorf("GetSecretValueError: failed to get value for %s/%s for %s (%s)", t.Secret(), t.Prop, path.BuildPath(root, fileEntry.ActualPath()), err), io.UserOutput) continue } @@ -163,14 +163,14 @@ func Pull(catalogPath string, opt cfg.UserOptions, io models.IO) (int, int, erro file, err = token.Replace(file, fileEntry.Type, tokens, true) if err != nil { - display.Error(fmt.Errorf("TokenReplacementError: failed to replace tokens in file %s (%s)", fileEntry.Path, err), io.UserOutput) + display.Error(fmt.Errorf("TokenReplacementError: failed to replace tokens in file %s (%s)", fileEntry.ActualPath(), err), io.UserOutput) } } if opt.InjectSecrets { fileWithSecrets, err = token.Replace(fileWithSecrets, fileEntry.Type, tokens, false) if err != nil { - display.Error(fmt.Errorf("TokenReplacementError: failed to replace tokens in file %s (%s)", fileEntry.Path, err), io.UserOutput) + display.Error(fmt.Errorf("TokenReplacementError: failed to replace tokens in file %s (%s)", fileEntry.ActualPath(), err), io.UserOutput) } } } @@ -181,7 +181,7 @@ func Pull(catalogPath string, opt cfg.UserOptions, io models.IO) (int, int, erro if opt.ExportEnv || len(opt.ExportFormat) > 0 { if !compatibleFormat(opt.ExportFormat, fileEntry.Type) { - display.Error(fmt.Errorf("IncompatibleExportFormat: file %s is incompatible with export format %s", fileEntry.Path, opt.ExportFormat), io.UserOutput) + display.Error(fmt.Errorf("IncompatibleExportFormat: file %s is incompatible with export format %s", fileEntry.ActualPath(), opt.ExportFormat), io.UserOutput) continue } @@ -196,7 +196,7 @@ func Pull(catalogPath string, opt cfg.UserOptions, io models.IO) (int, int, erro //----------------------------------------------------- //- Save editable, secret, and alternate files locally. //----------------------------------------------------- - fullPath := clog.GetFullPath(path.BuildPath(root, fileEntry.Path)) + fullPath := clog.GetFullPath(path.BuildPath(root, fileEntry.ActualPath())) if len(opt.AlternateRestorePath) == 0 { if err = localFile.Save(fullPath, file); err != nil { @@ -219,7 +219,7 @@ func Pull(catalogPath string, opt cfg.UserOptions, io models.IO) (int, int, erro } fmt.Fprint(io.UserOutput, "Retrieving [") - color.New(color.FgBlue).Fprintf(io.UserOutput, path.BuildPath(root, fileEntry.Path)) + color.New(color.FgBlue).Fprintf(io.UserOutput, path.BuildPath(root, fileEntry.ActualPath())) fmt.Fprint(io.UserOutput, "]") if len(opt.Version) > 0 { diff --git a/cli/cmd/purge.go b/cli/cmd/purge.go index b0e0bf5..22d978f 100644 --- a/cli/cmd/purge.go +++ b/cli/cmd/purge.go @@ -60,15 +60,16 @@ func Purge(opt cfg.UserOptions, io models.IO) error { } fileList := "" + for _, f := range files { if len(opt.Version) > 0 { - fileList = fmt.Sprintf("%sDelete [%s](%s) from [%s]\n", fileList, f.Path, opt.Version, f.Store) + fileList = fmt.Sprintf("%sDelete [%s](%s) from [%s]\n", fileList, f.ActualPath(), opt.Version, f.Store) } else { - fileList = fmt.Sprintf("%sDelete [%s] from [%s]\n", fileList, f.Path, f.Store) + fileList = fmt.Sprintf("%sDelete [%s] from [%s]\n", fileList, f.ActualPath(), f.Store) } } - if !prompt.Confirm(fmt.Sprintf("File data will be permanently deleted from remote storage! Local files and secrets stored in AWS Secrets Manager will not be affected.\n\n%s \nContinue?", fileList), prompt.Danger, io) { + if !prompt.Confirm(fmt.Sprintf("File data will be permanently deleted from remote storage! Local files will not be affected.\n\n%s \nContinue?", fileList), prompt.Danger, io) { color.New(color.Bold, color.FgRed).Fprint(ioStreams.UserOutput, "\nOperation Aborted!\n") os.Exit(0) } @@ -98,7 +99,7 @@ func Purge(opt cfg.UserOptions, io models.IO) error { //---------------------------------------------------- remoteComp, err := remote.InitComponents(&fileEntryTemp, clog, opt, io) if err != nil { - display.Error(fmt.Errorf("Purge aborted for %s! (%s)", fileEntry.Path, err), ioStreams.UserOutput) + display.Error(fmt.Errorf("Purge aborted for %s! (%s)", fileEntry.ActualPath(), err), ioStreams.UserOutput) continue } @@ -132,7 +133,7 @@ func Purge(opt cfg.UserOptions, io models.IO) error { for _, version := range fileEntry.Versions { if err = remoteComp.Store.Purge(&fileEntry, version); err != nil { - display.Error(fmt.Errorf("Purge aborted for %s (%s). (%s)", fileEntry.Path, version, err), io.UserOutput) + display.Error(fmt.Errorf("Purge aborted for %s (%s). (%s)", fileEntry.ActualPath(), version, err), io.UserOutput) undeletedVersions = append(undeletedVersions, version) continue } @@ -147,7 +148,7 @@ func Purge(opt cfg.UserOptions, io models.IO) error { //---------------------------------------------------- if len(undeletedVersions) == 0 { if err = remoteComp.Store.Purge(&fileEntry, none); err != nil { - display.ErrorText(fmt.Sprintf("Purge aborted for %s (%s)", fileEntry.Path, err), io.UserOutput) + display.ErrorText(fmt.Sprintf("Purge aborted for %s (%s)", fileEntry.ActualPath(), err), io.UserOutput) continue } @@ -158,10 +159,11 @@ func Purge(opt cfg.UserOptions, io models.IO) error { //---------------------------------------------------- //- Delete the ghost .cstore reference file. //---------------------------------------------------- - fullPath := clog.GetFullPath(path.RemoveFileName(fileEntry.Path)) + fullPath := clog.GetFullPath(path.RemoveFileName(fileEntry.ActualPath())) + if len(fullPath) > 0 && !clog.AnyFilesIn(path.RemoveFileName(fileEntry.Path)) { if err := os.Remove(fmt.Sprintf("%s%s", fullPath, catalog.GhostFile)); err != nil { - display.Error(fmt.Errorf(".cstore file could not be removed for %s! (%s)", fileEntry.Path, err), io.UserOutput) + display.Error(fmt.Errorf(".cstore file could not be removed for %s! (%s)", fileEntry.ActualPath(), err), io.UserOutput) } } diff --git a/cli/cmd/push.go b/cli/cmd/push.go index 65abaf8..df9d619 100644 --- a/cli/cmd/push.go +++ b/cli/cmd/push.go @@ -55,7 +55,7 @@ func Push(opt cfg.UserOptions, io models.IO) error { fmt.Fprintln(io.UserOutput) for _, filePath := range getFilePathsToPush(clog, opt) { - file, err := localFile.GetBy(clog.GetFullPath(filePath)) + file, err := localFile.GetBy(clog.GetFullPath(path.SubstituteTokens(filePath))) if err != nil { display.Error(err, io.UserOutput) continue @@ -77,7 +77,7 @@ func Push(opt cfg.UserOptions, io models.IO) error { //------------------------------------------------- if fileEntry.IsRef { fmt.Fprint(io.UserOutput, "Linking [") - color.New(color.FgBlue).Fprintf(io.UserOutput, fileEntry.Path) + color.New(color.FgBlue).Fprintf(io.UserOutput, fileEntry.ActualPath()) fmt.Fprintln(io.UserOutput, "]") if err := clog.UpdateEntry(fileEntry); err != nil { display.Error(err, io.UserOutput) @@ -100,7 +100,7 @@ func Push(opt cfg.UserOptions, io models.IO) error { //- Begin push process. //-------------------------------------------------- fmt.Fprint(io.UserOutput, "Pushing [") - color.New(color.FgBlue).Fprintf(io.UserOutput, fileEntry.Path) + color.New(color.FgBlue).Fprintf(io.UserOutput, fileEntry.ActualPath()) fmt.Fprint(io.UserOutput, "]") if len(opt.Version) > 0 { @@ -124,7 +124,7 @@ func Push(opt cfg.UserOptions, io models.IO) error { } else { if !prompt.Confirm(fmt.Sprintf("Overwrite %s with %s?", formatVersion(opt.Version), formatVersion(version)), prompt.Warn, io) { fmt.Fprint(io.UserOutput, "Skipping [") - color.New(color.FgBlue).Fprintf(io.UserOutput, fileEntry.Path) + color.New(color.FgBlue).Fprintf(io.UserOutput, fileEntry.ActualPath()) fmt.Fprintln(io.UserOutput, "]") continue } @@ -134,7 +134,7 @@ func Push(opt cfg.UserOptions, io models.IO) error { if !current { if !prompt.Confirm(fmt.Sprintf("Remotely stored [%s] was modified %s. Overwrite?", filePath, lastModified.Format("01/02/06")), prompt.Warn, io) { fmt.Fprint(io.UserOutput, "Skipping [") - color.New(color.FgBlue).Fprintf(io.UserOutput, fileEntry.Path) + color.New(color.FgBlue).Fprintf(io.UserOutput, fileEntry.ActualPath()) fmt.Fprintln(io.UserOutput, "]") continue } @@ -162,7 +162,7 @@ func Push(opt cfg.UserOptions, io models.IO) error { file = token.RemoveSecrets(file) - if err = localFile.Save(clog.GetFullPath(fileEntry.Path), file); err != nil { + if err = localFile.Save(clog.GetFullPath(fileEntry.ActualPath()), file); err != nil { logger.L.Print(err) } } @@ -207,9 +207,9 @@ func Push(opt cfg.UserOptions, io models.IO) error { } //--------------------------------------------------------------------- - //- Create the ghost .cstore reference file when not in cStore.yml dir. + //- Create the ghost .cstore reference file when not in cstore.yml dir. //--------------------------------------------------------------------- - justThePath := path.RemoveFileName(filePath) + justThePath := path.RemoveFileName(fileEntry.ActualPath()) if len(clog.GetFullPath(justThePath)) > 0 { if err := catalog.WriteGhost(clog.GetFullPath(justThePath), catalog.Ghost{ @@ -219,14 +219,14 @@ func Push(opt cfg.UserOptions, io models.IO) error { } } - filesPushed = append(filesPushed, fileEntry.Path) + filesPushed = append(filesPushed, fileEntry.ActualPath()) //------------------------------------------------- //- If user specified, delete local files. //------------------------------------------------- if fileEntry.DeleteAfterPush { - os.Remove(clog.GetFullPath(fileEntry.Path)) - os.Remove(fmt.Sprintf("%s.secrets", clog.GetFullPath(fileEntry.Path))) + os.Remove(clog.GetFullPath(fileEntry.ActualPath())) + os.Remove(fmt.Sprintf("%s.secrets", clog.GetFullPath(fileEntry.ActualPath()))) } } diff --git a/components/catalog/model.go b/components/catalog/model.go index b8a935b..455f547 100644 --- a/components/catalog/model.go +++ b/components/catalog/model.go @@ -118,6 +118,11 @@ type File struct { Versions []string `ymal:"versions,omitempty"` } +// ActualPath ... +func (f File) ActualPath() string { + return path.SubstituteTokens(f.Path) +} + // Key ... func (f File) Key() string { return hashPath(f.Path) @@ -125,7 +130,7 @@ func (f File) Key() string { // ContextKey ... func (f File) ContextKey(context string) string { - return buildKey(context, hashPath(f.Path)) + return buildKey(context, hashPath(f.ActualPath())) } // SupportsSecrets ... @@ -324,7 +329,7 @@ func (c Catalog) FilesBy(paths, tags []string, allTags bool, version string) map // AnyFilesIn ... func (c Catalog) AnyFilesIn(dir string) bool { for _, f := range c.Files { - if path.RemoveFileName(f.Path) == dir { + if path.RemoveFileName(f.ActualPath()) == dir { return true } } @@ -357,7 +362,7 @@ func (c *Catalog) UpdateEntry(newFile File) error { if oldFile, found := c.Files[key]; found { if len(newFile.Store) > 0 && newFile.Store != oldFile.Store { - return fmt.Errorf("AreadyStoredException: Purge %s from %s before pushing to %s", newFile.Path, oldFile.Store, newFile.Store) + return fmt.Errorf("AreadyStoredException: Purge %s from %s before pushing to %s", newFile.ActualPath(), oldFile.Store, newFile.Store) } } diff --git a/components/path/substitute.go b/components/path/substitute.go new file mode 100644 index 0000000..7dce6ff --- /dev/null +++ b/components/path/substitute.go @@ -0,0 +1,25 @@ +package path + +import ( + "os" + "strings" + "regexp" +) + +// SubstituteTokens ... +func SubstituteTokens(path string) string { + + re := regexp.MustCompile(`\${([A-Z_]+)}`) + + matches := re.FindAllStringSubmatch(path, -1) + + for _, m := range matches { + env, found := os.LookupEnv(m[1]) + + if found { + path = strings.Replace(path, m[0], env, -1) + } + } + + return path +} \ No newline at end of file diff --git a/components/store/aws_param_store.go b/components/store/aws_param_store.go index a11cc81..3385280 100644 --- a/components/store/aws_param_store.go +++ b/components/store/aws_param_store.go @@ -177,7 +177,7 @@ func (s AWSParameterStore) Push(file *catalog.File, fileData []byte, version str //- Get encryption key //------------------------------------------ value, err := setting.Setting{ - Description: "KMS Key ID is used by Parameter Store to encrypt and decrypt secrets. Any role or user accessing a secret must also have access to the KMS key. When pushing updates, the default setting will preserve existing KMS keys. The aws/ssm key is the default Systems Manager KMS key.", + Description: "KMS Key ID is used by Parameter Store to encrypt and decrypt secrets. Any role or user accessing a secret must also have access to the KMS key. When pushing updates, the default setting will preserve existing KMS keys. The aws/ssm key is the default Secrets Manager KMS key.", Prop: awsStoreKMSKeyID, DefaultValue: s.clog.GetDataByStore(s.Name(), awsStoreKMSKeyID, defaultPSKMSKey), Prompt: s.uo.Prompt, @@ -203,13 +203,13 @@ func (s AWSParameterStore) Push(file *catalog.File, fileData []byte, version str svc := ssm.New(s.Session) - storedParams, err := getStoredParamsWithMetaData(s.clog.Context, file.Path, version, svc) + storedParams, err := getStoredParamsWithMetaData(s.clog.Context, file.ActualPath(), version, svc) if err != nil { return err } for name, value := range newParams { - remoteKey := buildRemoteKey(s.clog.Context, file.Path, name, version) + remoteKey := buildRemoteKey(s.clog.Context, file.ActualPath(), name, version) newParam := param{ name: remoteKey, @@ -244,7 +244,7 @@ func (s AWSParameterStore) Push(file *catalog.File, fileData []byte, version str //------------------------------------------ for _, remoteParam := range storedParams { - param := strings.Replace(remoteParam.name, buildRemotePath(s.clog.Context, file.Path, version)+"/", "", 1) + param := strings.Replace(remoteParam.name, buildRemotePath(s.clog.Context, file.ActualPath(), version)+"/", "", 1) if !isParamIn(param, newParams) { if _, err := svc.DeleteParameter(&ssm.DeleteParameterInput{ @@ -264,7 +264,7 @@ func (s AWSParameterStore) Pull(file *catalog.File, version string) ([]byte, con svc := ssm.New(s.Session) - storedParams, err := getStoredParams(s.clog.Context, file.Path, version, svc) + storedParams, err := getStoredParams(s.clog.Context, file.ActualPath(), version, svc) if err != nil { return []byte{}, contract.Attributes{}, err } @@ -280,7 +280,7 @@ func (s AWSParameterStore) Pull(file *catalog.File, version string) ([]byte, con v := value if s.uo.StoreCommand == cmdRefFormat { - buffer.WriteString(fmt.Sprintf("%s=%s\n", name, buildRemoteKey(s.clog.Context, file.Path, name, version))) + buffer.WriteString(fmt.Sprintf("%s=%s\n", name, buildRemoteKey(s.clog.Context, file.ActualPath(), name, version))) } else { buffer.WriteString(fmt.Sprintf("%s=%s\n", name, v)) } @@ -294,7 +294,7 @@ func (s AWSParameterStore) Purge(file *catalog.File, version string) error { svc := ssm.New(s.Session) - storedParams, err := getStoredParams(s.clog.Context, file.Path, version, svc) + storedParams, err := getStoredParams(s.clog.Context, file.ActualPath(), version, svc) if err != nil { return err } @@ -327,7 +327,7 @@ func (s AWSParameterStore) Changed(file *catalog.File, fileData []byte, version svc := ssm.New(s.Session) - storedParams, err := getStoredParams(s.clog.Context, file.Path, version, svc) + storedParams, err := getStoredParams(s.clog.Context, file.ActualPath(), version, svc) if err != nil { return time.Time{}, err } @@ -336,7 +336,7 @@ func (s AWSParameterStore) Changed(file *catalog.File, fileData []byte, version for _, p := range storedParams { for name, value := range config { - remoteKey := buildRemoteKey(s.clog.Context, file.Path, name, version) + remoteKey := buildRemoteKey(s.clog.Context, file.ActualPath(), name, version) if remoteKey == p.name && value != p.value { changedParams = append(changedParams, p) diff --git a/components/store/aws_s3_store.go b/components/store/aws_s3_store.go index c13e770..4daf56e 100644 --- a/components/store/aws_s3_store.go +++ b/components/store/aws_s3_store.go @@ -161,7 +161,7 @@ func (s *S3Store) Pre(clog catalog.Catalog, file *catalog.File, access contract. // Purge ... func (s S3Store) Purge(file *catalog.File, version string) error { - contextKey := s.key(file.Path, version) + contextKey := s.key(file.ActualPath(), version) bucket, err := s.bucket.Get(s.clog.Context, s.io) if err != nil { @@ -185,7 +185,7 @@ func (s S3Store) Purge(file *catalog.File, version string) error { // Push ... func (s S3Store) Push(file *catalog.File, fileData []byte, version string) error { - contextKey := s.key(file.Path, version) + contextKey := s.key(file.ActualPath(), version) bucket, err := s.bucket.Get(s.clog.Context, s.io) if err != nil { @@ -232,7 +232,7 @@ func (s S3Store) Push(file *catalog.File, fileData []byte, version string) error // Pull ... func (s S3Store) Pull(file *catalog.File, version string) ([]byte, contract.Attributes, error) { - contextKey := s.key(file.Path, version) + contextKey := s.key(file.ActualPath(), version) setting := s.bucket setting.Prompt = false @@ -266,7 +266,7 @@ func (s S3Store) Pull(file *catalog.File, version string) ([]byte, contract.Attr // Changed ... func (s S3Store) Changed(file *catalog.File, fileData []byte, version string) (time.Time, error) { - contextKey := s.key(file.Path, version) + contextKey := s.key(file.ActualPath(), version) setting := s.bucket setting.Prompt = false diff --git a/components/store/aws_secret_manager_store.go b/components/store/aws_secret_manager_store.go index 59c20bb..cfd10d7 100644 --- a/components/store/aws_secret_manager_store.go +++ b/components/store/aws_secret_manager_store.go @@ -162,7 +162,7 @@ func (s AWSSecretManagerStore) Push(file *catalog.File, fileData []byte, version //- Get encryption key //------------------------------------------ value, err := setting.Setting{ - Description: "KMS Key ID is used by Secrets Manager to encrypt and decrypt secrets. Any role or user accessing a secret must also have access to the KMS key. When pushing updates, the default setting will preserve existing KMS keys. The aws/ssm key is the default Systems Manager KMS key.", + Description: "KMS Key ID is used by Secrets Manager to encrypt and decrypt secrets. Any role or user accessing a secret must also have access to the KMS key. When pushing updates, the default setting will preserve existing KMS keys. The aws/ssm key is the default Secrets Manager KMS key.", Prop: awsStoreKMSKeyID, DefaultValue: s.clog.GetDataByStore(s.Name(), awsStoreKMSKeyID, defaultSMKMSKey), Prompt: s.uo.Prompt, @@ -207,7 +207,7 @@ func (s AWSSecretManagerStore) pushBlob(file *catalog.File, fileData []byte, KMS svc := secretsmanager.New(s.Session) - key := fmt.Sprintf("%s/%s", s.clog.Context, file.Path) + key := fmt.Sprintf("%s/%s", s.clog.Context, file.ActualPath()) sv, err := svc.GetSecretValue(&secretsmanager.GetSecretValueInput{ SecretId: aws.String(key), @@ -269,7 +269,7 @@ func (s AWSSecretManagerStore) Pull(file *catalog.File, version string) ([]byte, svc := secretsmanager.New(s.Session) - key := fmt.Sprintf("%s/%s", s.clog.Context, file.Path) + key := fmt.Sprintf("%s/%s", s.clog.Context, file.ActualPath()) sv, err := svc.GetSecretValue(&secretsmanager.GetSecretValueInput{ SecretId: aws.String(key), @@ -296,7 +296,7 @@ func (s AWSSecretManagerStore) Purge(file *catalog.File, version string) error { svc := secretsmanager.New(s.Session) - key := fmt.Sprintf("%s/%s", s.clog.Context, file.Path) + key := fmt.Sprintf("%s/%s", s.clog.Context, file.ActualPath()) if _, err := svc.DeleteSecret(&secretsmanager.DeleteSecretInput{ SecretId: aws.String(key), @@ -311,7 +311,7 @@ func (s AWSSecretManagerStore) Purge(file *catalog.File, version string) error { func (s AWSSecretManagerStore) Changed(file *catalog.File, fileData []byte, version string) (time.Time, error) { svc := secretsmanager.New(s.Session) - key := fmt.Sprintf("%s/%s", s.clog.Context, file.Path) + key := fmt.Sprintf("%s/%s", s.clog.Context, file.ActualPath()) secret, err := describeSecret(key, svc) if err != nil { diff --git a/components/store/aws_secrets_manager_store.go b/components/store/aws_secrets_manager_store.go index 0f04da1..6f63db1 100644 --- a/components/store/aws_secrets_manager_store.go +++ b/components/store/aws_secrets_manager_store.go @@ -162,7 +162,7 @@ func (s AWSSecretsManagerStore) Push(file *catalog.File, fileData []byte, versio //- Get encryption key //------------------------------------------ value, err := setting.Setting{ - Description: "KMS Key ID is used by Secrets Manager to encrypt and decrypt secrets. Any role or user accessing a secret must also have access to the KMS key. When pushing updates, the default setting will preserve existing KMS keys. The aws/ssm key is the default Systems Manager KMS key.", + Description: "KMS Key ID is used by Secrets Manager to encrypt and decrypt secrets. Any role or user accessing a secret must also have access to the KMS key. When pushing updates, the default setting will preserve existing KMS keys. The aws/ssm key is the default Secrets Manager KMS key.", Prop: awsStoreKMSKeyID, DefaultValue: s.clog.GetDataByStore(s.Name(), awsStoreKMSKeyID, defaultSMKMSKey), Prompt: s.uo.Prompt, @@ -239,7 +239,7 @@ func (s AWSSecretsManagerStore) pushFile(file *catalog.File, fileData []byte, KM continue } - key := formatSecretToken(s.clog.Context, file.Path, name) + key := formatSecretToken(s.clog.Context, file.ActualPath(), name) removed := true for k := range params { @@ -267,7 +267,7 @@ func (s AWSSecretsManagerStore) pushFile(file *catalog.File, fileData []byte, KM keyID: KMSKeyID.value, } - key := formatSecretToken(s.clog.Context, file.Path, name) + key := formatSecretToken(s.clog.Context, file.ActualPath(), name) storedProps, err := getSecret(key, svc) @@ -441,7 +441,7 @@ func (s AWSSecretsManagerStore) Pull(file *catalog.File, version string) ([]byte svc := secretsmanager.New(s.Session) - storedSecrets, err := getSecrets(s.clog.Context, file.Path, file.Data, svc) + storedSecrets, err := getSecrets(s.clog.Context, file.ActualPath(), file.Data, svc) if err != nil { return []byte{}, contract.Attributes{}, err } @@ -500,7 +500,7 @@ func (s AWSSecretsManagerStore) Purge(file *catalog.File, version string) error continue } - key := formatSecretToken(s.clog.Context, file.Path, name) + key := formatSecretToken(s.clog.Context, file.ActualPath(), name) if _, err := svc.DeleteSecret(&secretsmanager.DeleteSecretInput{ SecretId: aws.String(key), @@ -524,7 +524,7 @@ func (s AWSSecretsManagerStore) Changed(file *catalog.File, fileData []byte, ver continue } - secret, err := describeSecret(formatSecretToken(s.clog.Context, file.Path, name), svc) + secret, err := describeSecret(formatSecretToken(s.clog.Context, file.ActualPath(), name), svc) if err != nil { if err.Error() == contract.ErrSecretNotFound.Error() { return time.Time{}, nil diff --git a/components/store/source_control_store.go b/components/store/source_control_store.go index d240c22..5fcf45e 100644 --- a/components/store/source_control_store.go +++ b/components/store/source_control_store.go @@ -66,7 +66,7 @@ func (s SourceControlStore) Push(file *catalog.File, fileData []byte, version st // Pull ... func (s SourceControlStore) Pull(file *catalog.File, version string) ([]byte, contract.Attributes, error) { - b, err := localFile.GetBy(s.clog.GetFullPath(file.Path)) + b, err := localFile.GetBy(s.clog.GetFullPath(file.ActualPath())) if err != nil { return b, contract.Attributes{}, err } @@ -76,7 +76,7 @@ func (s SourceControlStore) Pull(file *catalog.File, version string) ([]byte, co // Purge ... func (s SourceControlStore) Purge(file *catalog.File, version string) error { - return os.Remove(s.clog.GetFullPath(file.Path)) + return os.Remove(s.clog.GetFullPath(file.ActualPath())) } // Changed ... diff --git a/components/store/store.go b/components/store/store.go index ec75c7d..643ece7 100644 --- a/components/store/store.go +++ b/components/store/store.go @@ -56,7 +56,7 @@ func Select(file *catalog.File, clog catalog.Catalog, v contract.IVault, uo cfg. } val := prompt.GetValFromUser("Remote Store", prompt.Options{ - Description: fmt.Sprintf("The remote storage solution where %s data will be pushed. (%s)", file.Path, supportedStores), + Description: fmt.Sprintf("The remote storage solution where %s data will be pushed. (%s)", file.ActualPath(), supportedStores), DefaultValue: GetDefaultStoreFor(file.Type), }, io) diff --git a/docs/CATALOG.md b/docs/CATALOG.md index 44f26f8..cd9cbc1 100644 --- a/docs/CATALOG.md +++ b/docs/CATALOG.md @@ -8,7 +8,7 @@ A list of fields tracked in the `cstore.yml` file to make managing configuration |-|-|-|-|-| | version | `string` | v1, v2, v3, v4 | yes | Catalog version used by the CLI to determine the format of the catalog. | | context | `string` || yes | The unique id used in the remote store or vault to link the catalog to the remotely stored data. This vault is often the key prefix for remotely stored data. | -| file.path | `string` || yes | The local file path of the remotely stored data relative to the catalog. | +| file.path | `string` || yes | The local file path of the remotely stored data relative to the catalog. This field can be tokenized after the initial push by editing the cstore.yml directly. When the file is pushed or pulled, cStore will look for an environment variables that match and replace any tokens. (e.g. `service/${ENV}/.env` => `service/dev/.env`) | | file.alternatePath | `string` || no | An alternate local file path to restore the data to when a file is retrieved.| | file.store | `string` | `aws-secret`, `aws-secrets`, `aws-s3`, `aws-parameter`, `source-control` | yes | The CLI key identifying the current remote storage solution. | | file.isRef| `bool` | `true`,`false` | yes | A flag indicating when the file is referencing another linked catalog or remotely stored data. | diff --git a/pull.go b/pull.go index 5c3ac84..48c887e 100644 --- a/pull.go +++ b/pull.go @@ -39,7 +39,7 @@ func Pull(catalogPath string, o Options) ([]byte, error) { } if len(files) == 0 { - return data, fmt.Errorf("FileNotFoundError: file not found in %s", opt.Catalog) + return data, fmt.Errorf("FileNotFoundError: file not found in %s", catalogPath) } for _, fileEntry := range files {