From 31d9e30474787eda199ee66a7297406e57b7fe78 Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Sun, 6 Feb 2022 10:39:41 -0500 Subject: [PATCH] add helper to extract regular files Signed-off-by: Alex Goodman --- internal/err_helper.go | 7 ++++--- syft/file/all_regular_files.go | 30 +++++++++++++++++++++++++++ syft/file/classification_cataloger.go | 2 +- syft/file/contents_cataloger.go | 2 +- syft/file/digest_cataloger.go | 7 ++----- syft/file/secrets_cataloger.go | 7 ++----- syft/source/directory_resolver.go | 2 ++ syft/source/file_metadata.go | 1 + test/cli/packages_cmd_test.go | 28 ++++++++++++------------- test/cli/power_user_cmd_test.go | 2 +- 10 files changed, 58 insertions(+), 30 deletions(-) create mode 100644 syft/file/all_regular_files.go diff --git a/internal/err_helper.go b/internal/err_helper.go index dad5f9c3dcd..237d071c00a 100644 --- a/internal/err_helper.go +++ b/internal/err_helper.go @@ -16,12 +16,13 @@ func CloseAndLogError(closer io.Closer, location string) { } type ErrPath struct { - Path string - Err error + Cataloger string + Path string + Err error } func (e ErrPath) Error() string { - return fmt.Sprintf("unable to observe contents of %+v: %v", e.Path, e.Err) + return fmt.Sprintf("%s unable to observe contents of %+v: %v", e.Cataloger, e.Path, e.Err) } func IsErrPath(err error) bool { diff --git a/syft/file/all_regular_files.go b/syft/file/all_regular_files.go new file mode 100644 index 00000000000..e7612c8b242 --- /dev/null +++ b/syft/file/all_regular_files.go @@ -0,0 +1,30 @@ +package file + +import ( + "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/syft/source" +) + +func allRegularFiles(resolver source.FileResolver) (locations []source.Location) { + for location := range resolver.AllLocations() { + resolvedLocations, err := resolver.FilesByPath(location.RealPath) + if err != nil { + log.Warnf("unable to resolve %+v: %+v", location, err) + continue + } + + for _, resolvedLocation := range resolvedLocations { + metadata, err := resolver.FileMetadataByLocation(resolvedLocation) + if err != nil { + log.Warnf("unable to get metadata for %+v: %+v", location, err) + continue + } + + if metadata.Type != source.RegularFile { + continue + } + locations = append(locations, resolvedLocation) + } + } + return locations +} diff --git a/syft/file/classification_cataloger.go b/syft/file/classification_cataloger.go index 3c23da3702c..01a0685efe2 100644 --- a/syft/file/classification_cataloger.go +++ b/syft/file/classification_cataloger.go @@ -19,7 +19,7 @@ func (i *ClassificationCataloger) Catalog(resolver source.FileResolver) (map[sou results := make(map[source.Coordinates][]Classification) numResults := 0 - for location := range resolver.AllLocations() { + for _, location := range allRegularFiles(resolver) { for _, classifier := range i.classifiers { result, err := classifier.Classify(resolver, location) if err != nil { diff --git a/syft/file/contents_cataloger.go b/syft/file/contents_cataloger.go index 93ede0a9698..5a43e9661ed 100644 --- a/syft/file/contents_cataloger.go +++ b/syft/file/contents_cataloger.go @@ -65,7 +65,7 @@ func (i *ContentsCataloger) catalogLocation(resolver source.FileResolver, locati buf := &bytes.Buffer{} if _, err = io.Copy(base64.NewEncoder(base64.StdEncoding, buf), contentReader); err != nil { - return "", internal.ErrPath{Path: location.RealPath, Err: err} + return "", internal.ErrPath{Cataloger: "contents-cataloger", Path: location.RealPath, Err: err} } return buf.String(), nil diff --git a/syft/file/digest_cataloger.go b/syft/file/digest_cataloger.go index 8660410405e..5b395b8f814 100644 --- a/syft/file/digest_cataloger.go +++ b/syft/file/digest_cataloger.go @@ -34,10 +34,7 @@ func NewDigestsCataloger(hashes []crypto.Hash) (*DigestsCataloger, error) { func (i *DigestsCataloger) Catalog(resolver source.FileResolver) (map[source.Coordinates][]Digest, error) { results := make(map[source.Coordinates][]Digest) - var locations []source.Location - for location := range resolver.AllLocations() { - locations = append(locations, location) - } + locations := allRegularFiles(resolver) stage, prog := digestsCatalogingProgress(int64(len(locations))) for _, location := range locations { stage.Current = location.RealPath @@ -90,7 +87,7 @@ func (i *DigestsCataloger) catalogLocation(resolver source.FileResolver, locatio size, err := io.Copy(io.MultiWriter(writers...), contentReader) if err != nil { - return nil, internal.ErrPath{Path: location.RealPath, Err: err} + return nil, internal.ErrPath{Cataloger: "digests-cataloger", Path: location.RealPath, Err: err} } if size == 0 { diff --git a/syft/file/secrets_cataloger.go b/syft/file/secrets_cataloger.go index ed0a4a95db7..b91a730532c 100644 --- a/syft/file/secrets_cataloger.go +++ b/syft/file/secrets_cataloger.go @@ -42,10 +42,7 @@ func NewSecretsCataloger(patterns map[string]*regexp.Regexp, revealValues bool, func (i *SecretsCataloger) Catalog(resolver source.FileResolver) (map[source.Coordinates][]SearchResult, error) { results := make(map[source.Coordinates][]SearchResult) - var locations []source.Location - for location := range resolver.AllLocations() { - locations = append(locations, location) - } + locations := allRegularFiles(resolver) stage, prog, secretsDiscovered := secretsCatalogingProgress(int64(len(locations))) for _, location := range locations { stage.Current = location.RealPath @@ -86,7 +83,7 @@ func (i *SecretsCataloger) catalogLocation(resolver source.FileResolver, locatio // TODO: in the future we can swap out search strategies here secrets, err := catalogLocationByLine(resolver, location, i.patterns) if err != nil { - return nil, internal.ErrPath{Path: location.RealPath, Err: err} + return nil, internal.ErrPath{Cataloger: "secrets-cataloger", Path: location.RealPath, Err: err} } if i.revealValues { diff --git a/syft/source/directory_resolver.go b/syft/source/directory_resolver.go index b6c155d222a..6c6f051d157 100644 --- a/syft/source/directory_resolver.go +++ b/syft/source/directory_resolver.go @@ -233,7 +233,9 @@ func (r directoryResolver) addSymlinkToIndex(p string, info os.FileInfo) (string } location := NewLocationFromDirectory(p, *ref) + location.VirtualPath = p metadata := fileMetadataFromPath(p, usedInfo, r.isInIndex(location)) + metadata.LinkDestination = linkTarget r.addFileMetadataToIndex(ref, metadata) return targetAbsPath, nil diff --git a/syft/source/file_metadata.go b/syft/source/file_metadata.go index 37c97d81819..432a1d8f4e0 100644 --- a/syft/source/file_metadata.go +++ b/syft/source/file_metadata.go @@ -63,6 +63,7 @@ func fileMetadataFromPath(path string, info os.FileInfo, withMIMEType bool) File // unsupported across platforms UserID: uid, GroupID: gid, + Size: info.Size(), MIMEType: mimeType, } } diff --git a/test/cli/packages_cmd_test.go b/test/cli/packages_cmd_test.go index 93bba2f8ef9..6225eb7c35b 100644 --- a/test/cli/packages_cmd_test.go +++ b/test/cli/packages_cmd_test.go @@ -209,55 +209,55 @@ func TestRegistryAuth(t *testing.T) { }{ { name: "fallback to keychain", - args: []string{"packages", "-vv", "registry:localhost:5000/something:latest"}, + args: []string{"packages", "-vv", "registry:localhost:17/something:latest"}, assertions: []traitAssertion{ assertInOutput("source=OciRegistry"), - assertInOutput("localhost:5000/something:latest"), + assertInOutput("localhost:17/something:latest"), assertInOutput("no registry credentials configured, using the default keychain"), }, }, { name: "use creds", - args: []string{"packages", "-vv", "registry:localhost:5000/something:latest"}, + args: []string{"packages", "-vv", "registry:localhost:17/something:latest"}, env: map[string]string{ - "SYFT_REGISTRY_AUTH_AUTHORITY": "localhost:5000", + "SYFT_REGISTRY_AUTH_AUTHORITY": "localhost:17", "SYFT_REGISTRY_AUTH_USERNAME": "username", "SYFT_REGISTRY_AUTH_PASSWORD": "password", }, assertions: []traitAssertion{ assertInOutput("source=OciRegistry"), - assertInOutput("localhost:5000/something:latest"), - assertInOutput(`using basic auth for registry "localhost:5000"`), + assertInOutput("localhost:17/something:latest"), + assertInOutput(`using basic auth for registry "localhost:17"`), }, }, { name: "use token", - args: []string{"packages", "-vv", "registry:localhost:5000/something:latest"}, + args: []string{"packages", "-vv", "registry:localhost:17/something:latest"}, env: map[string]string{ - "SYFT_REGISTRY_AUTH_AUTHORITY": "localhost:5000", + "SYFT_REGISTRY_AUTH_AUTHORITY": "localhost:17", "SYFT_REGISTRY_AUTH_TOKEN": "token", }, assertions: []traitAssertion{ assertInOutput("source=OciRegistry"), - assertInOutput("localhost:5000/something:latest"), - assertInOutput(`using token for registry "localhost:5000"`), + assertInOutput("localhost:17/something:latest"), + assertInOutput(`using token for registry "localhost:17"`), }, }, { name: "not enough info fallsback to keychain", - args: []string{"packages", "-vv", "registry:localhost:5000/something:latest"}, + args: []string{"packages", "-vv", "registry:localhost:17/something:latest"}, env: map[string]string{ - "SYFT_REGISTRY_AUTH_AUTHORITY": "localhost:5000", + "SYFT_REGISTRY_AUTH_AUTHORITY": "localhost:17", }, assertions: []traitAssertion{ assertInOutput("source=OciRegistry"), - assertInOutput("localhost:5000/something:latest"), + assertInOutput("localhost:17/something:latest"), assertInOutput(`no registry credentials configured, using the default keychain`), }, }, { name: "allows insecure http flag", - args: []string{"packages", "-vv", "registry:localhost:5000/something:latest"}, + args: []string{"packages", "-vv", "registry:localhost:17/something:latest"}, env: map[string]string{ "SYFT_REGISTRY_INSECURE_USE_HTTP": "true", }, diff --git a/test/cli/power_user_cmd_test.go b/test/cli/power_user_cmd_test.go index 81a928fcdea..393e28e051d 100644 --- a/test/cli/power_user_cmd_test.go +++ b/test/cli/power_user_cmd_test.go @@ -83,7 +83,7 @@ func TestPowerUserCmdFlags(t *testing.T) { }, }, { - name: "defaut-secrets-dir-results-w-reveal-values", + name: "default-secrets-dir-results-w-reveal-values", env: map[string]string{ "SYFT_SECRETS_REVEAL_VALUES": "true", },