Resolve symlinks when fetching file contents (#782)

go.mod

@@ -13,7 +13,7 @@ require (
 	github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04
 	github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b
 	github.com/anchore/packageurl-go v0.0.0-20210922164639-b3fa992ebd29
-	github.com/anchore/stereoscope v0.0.0-20220214165125-25ebd49a842b
+	github.com/anchore/stereoscope v0.0.0-20220217141419-c6f02aed9ed2
 	github.com/antihax/optional v1.0.0
 	github.com/bmatcuk/doublestar/v4 v4.0.2
 	github.com/docker/docker v20.10.12+incompatible

go.sum

@@ -282,8 +282,8 @@ github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b h1:e1bmaoJfZV
 github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b/go.mod h1:Bkc+JYWjMCF8OyZ340IMSIi2Ebf3uwByOk6ho4wne1E=
 github.com/anchore/packageurl-go v0.0.0-20210922164639-b3fa992ebd29 h1:K9LfnxwhqvihqU0+MF325FNy7fsKV9EGaUxdfR4gnWk=
 github.com/anchore/packageurl-go v0.0.0-20210922164639-b3fa992ebd29/go.mod h1:Oc1UkGaJwY6ND6vtAqPSlYrptKRJngHwkwB6W7l1uP0=
-github.com/anchore/stereoscope v0.0.0-20220214165125-25ebd49a842b h1:PMMXpTEHVVLErrXQ6mH9ocLAQyvQu/LUhdstrhx7AC4=
-github.com/anchore/stereoscope v0.0.0-20220214165125-25ebd49a842b/go.mod h1:QpDHHV2h1NNfu7klzU75XC8RvSlaPK6HHgi0dy8A6sk=
+github.com/anchore/stereoscope v0.0.0-20220217141419-c6f02aed9ed2 h1:QuvMG+rqqJmtFRL+jqj5pFgjQcJSnEHEbtj1lKowLLQ=
+github.com/anchore/stereoscope v0.0.0-20220217141419-c6f02aed9ed2/go.mod h1:QpDHHV2h1NNfu7klzU75XC8RvSlaPK6HHgi0dy8A6sk=
 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
 github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=
 github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=

internal/err_helper.go

@@ -16,12 +16,13 @@ func CloseAndLogError(closer io.Closer, location string) {
 }
 
 type ErrPath struct {
-	Path string
-	Err  error
+	Context string
+	Path    string
+	Err     error
 }
 
 func (e ErrPath) Error() string {
-	return fmt.Sprintf("unable to observe contents of %+v: %v", e.Path, e.Err)
+	return fmt.Sprintf("%s unable to observe contents of %+v: %v", e.Context, e.Path, e.Err)
 }
 
 func IsErrPath(err error) bool {

syft/file/all_regular_files.go

@@ -0,0 +1,30 @@
+package file
+
+import (
+	"github.com/anchore/syft/internal/log"
+	"github.com/anchore/syft/syft/source"
+)
+
+func allRegularFiles(resolver source.FileResolver) (locations []source.Location) {
+	for location := range resolver.AllLocations() {
+		resolvedLocations, err := resolver.FilesByPath(location.RealPath)
+		if err != nil {
+			log.Warnf("unable to resolve %+v: %+v", location, err)
+			continue
+		}
+
+		for _, resolvedLocation := range resolvedLocations {
+			metadata, err := resolver.FileMetadataByLocation(resolvedLocation)
+			if err != nil {
+				log.Warnf("unable to get metadata for %+v: %+v", location, err)
+				continue
+			}
+
+			if metadata.Type != source.RegularFile {
+				continue
+			}
+			locations = append(locations, resolvedLocation)
+		}
+	}
+	return locations
+}

syft/file/all_regular_files_test.go

@@ -0,0 +1,74 @@
+package file
+
+import (
+	"github.com/anchore/stereoscope/pkg/imagetest"
+	"github.com/anchore/syft/syft/source"
+	"github.com/scylladb/go-set/strset"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"testing"
+)
+
+func Test_allRegularFiles(t *testing.T) {
+	type access struct {
+		realPath    string
+		virtualPath string
+	}
+	tests := []struct {
+		name             string
+		setup            func() source.FileResolver
+		wantRealPaths    *strset.Set
+		wantVirtualPaths *strset.Set
+	}{
+		{
+			name: "image",
+			setup: func() source.FileResolver {
+				testImage := "image-file-type-mix"
+
+				if *updateImageGoldenFiles {
+					imagetest.UpdateGoldenFixtureImage(t, testImage)
+				}
+
+				img := imagetest.GetGoldenFixtureImage(t, testImage)
+
+				s, err := source.NewFromImage(img, "---")
+				require.NoError(t, err)
+
+				r, err := s.FileResolver(source.SquashedScope)
+				require.NoError(t, err)
+
+				return r
+			},
+			wantRealPaths:    strset.New("/file-1.txt"),
+			wantVirtualPaths: strset.New("/file-1.txt", "/symlink-1", "/hardlink-1"),
+		},
+		{
+			name: "directory",
+			setup: func() source.FileResolver {
+				s, err := source.NewFromDirectory("test-fixtures/symlinked-root/nested/link-root")
+				require.NoError(t, err)
+				r, err := s.FileResolver(source.SquashedScope)
+				require.NoError(t, err)
+				return r
+			},
+			wantRealPaths:    strset.New("file1.txt", "nested/file2.txt"),
+			wantVirtualPaths: strset.New("nested/linked-file1.txt"),
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			resolver := tt.setup()
+			locations := allRegularFiles(resolver)
+			realLocations := strset.New()
+			virtualLocations := strset.New()
+			for _, l := range locations {
+				realLocations.Add(l.RealPath)
+				if l.VirtualPath != "" {
+					virtualLocations.Add(l.VirtualPath)
+				}
+			}
+			assert.ElementsMatch(t, tt.wantRealPaths.List(), realLocations.List(), "mismatched real paths")
+			assert.ElementsMatch(t, tt.wantVirtualPaths.List(), virtualLocations.List(), "mismatched virtual paths")
+		})
+	}
+}

syft/file/classification_cataloger.go

@@ -19,11 +19,12 @@ func (i *ClassificationCataloger) Catalog(resolver source.FileResolver) (map[sou
 	results := make(map[source.Coordinates][]Classification)
 
 	numResults := 0
-	for location := range resolver.AllLocations() {
+	for _, location := range allRegularFiles(resolver) {
 		for _, classifier := range i.classifiers {
 			result, err := classifier.Classify(resolver, location)
 			if err != nil {
-				return nil, err
+				log.Warnf("file classification cataloger failed with class=%q at location=%+v: %+v", classifier.Class, location, err)
+				continue
 			}
 			if result != nil {
 				results[location.Coordinates] = append(results[location.Coordinates], *result)

syft/file/classification_cataloger_test.go

@@ -1,6 +1,7 @@
 package file
 
 import (
+	"github.com/anchore/stereoscope/pkg/imagetest"
 	"testing"
 
 	"github.com/anchore/syft/syft/source"
@@ -88,7 +89,7 @@ func TestClassifierCataloger_DefaultClassifiers_PositiveCases(t *testing.T) {
 		{
 			name:       "positive-busybox",
 			fixtureDir: "test-fixtures/classifiers/positive",
-			location:   "busybox",
+			location:   "[", // note: busybox is a link to [
 			expected: []Classification{
 				{
 					Class: "busybox-binary",
@@ -116,13 +117,67 @@ func TestClassifierCataloger_DefaultClassifiers_PositiveCases(t *testing.T) {
 			actualResults, err := c.Catalog(resolver)
 			test.expectedErr(t, err)
 
-			loc := source.NewLocation(test.location)
+			ok := false
+			for actualLoc, actualClassification := range actualResults {
+				if test.location == actualLoc.RealPath {
+					ok = true
+					assert.Equal(t, test.expected, actualClassification)
+				}
+			}
+
+			if !ok {
+				t.Fatalf("could not find test location=%q", test.location)
+			}
+
+		})
+	}
+}
+
+func TestClassifierCataloger_DefaultClassifiers_PositiveCases_Image(t *testing.T) {
+	tests := []struct {
+		name         string
+		fixtureImage string
+		location     string
+		expected     []Classification
+		expectedErr  func(assert.TestingT, error, ...interface{}) bool
+	}{
+		{
+			name:         "busybox-regression",
+			fixtureImage: "image-busybox",
+			location:     "/bin/[",
+			expected: []Classification{
+				{
+					Class: "busybox-binary",
+					Metadata: map[string]string{
+						"version": "1.35.0",
+					},
+				},
+			},
+			expectedErr: assert.NoError,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+
+			c, err := NewClassificationCataloger(DefaultClassifiers)
+			test.expectedErr(t, err)
+
+			img := imagetest.GetFixtureImage(t, "docker-archive", test.fixtureImage)
+			src, err := source.NewFromImage(img, "test-img")
+			test.expectedErr(t, err)
+
+			resolver, err := src.FileResolver(source.SquashedScope)
+			test.expectedErr(t, err)
+
+			actualResults, err := c.Catalog(resolver)
+			test.expectedErr(t, err)
 
 			ok := false
-			for actual_loc, actual_classification := range actualResults {
-				if loc.RealPath == actual_loc.RealPath {
+			for actuaLoc, actualClassification := range actualResults {
+				if actuaLoc.RealPath == test.location {
 					ok = true
-					assert.Equal(t, test.expected, actual_classification)
+					assert.Equal(t, test.expected, actualClassification)
 				}
 			}
 

syft/file/contents_cataloger.go

@@ -3,6 +3,7 @@ package file
 import (
 	"bytes"
 	"encoding/base64"
+	"fmt"
 	"io"
 
 	"github.com/anchore/syft/internal"
@@ -66,9 +67,12 @@ func (i *ContentsCataloger) catalogLocation(resolver source.FileResolver, locati
 	buf := &bytes.Buffer{}
 	encoder := base64.NewEncoder(base64.StdEncoding, buf)
 	if _, err = io.Copy(encoder, contentReader); err != nil {
-		return "", internal.ErrPath{Path: location.RealPath, Err: err}
+		return "", internal.ErrPath{Context: "contents-cataloger", Path: location.RealPath, Err: err}
+	}
+	// note: it's important to close the reader before reading from the buffer since closing will flush the remaining bytes
+	if err := encoder.Close(); err != nil {
+		return "", fmt.Errorf("unable to close base64 encoder: %w", err)
 	}
-	encoder.Close()
 
 	return buf.String(), nil
 }

syft/file/digest_cataloger.go

@@ -2,6 +2,7 @@ package file
 
 import (
 	"crypto"
+	"errors"
 	"fmt"
 	"hash"
 	"io"
@@ -19,6 +20,8 @@ import (
 	"github.com/anchore/syft/syft/source"
 )
 
+var errUndigestableFile = errors.New("undigestable file")
+
 type DigestsCataloger struct {
 	hashes []crypto.Hash
 }
@@ -31,16 +34,18 @@ func NewDigestsCataloger(hashes []crypto.Hash) (*DigestsCataloger, error) {
 
 func (i *DigestsCataloger) Catalog(resolver source.FileResolver) (map[source.Coordinates][]Digest, error) {
 	results := make(map[source.Coordinates][]Digest)
-	var locations []source.Location
-	for location := range resolver.AllLocations() {
-		locations = append(locations, location)
-	}
+	locations := allRegularFiles(resolver)
 	stage, prog := digestsCatalogingProgress(int64(len(locations)))
 	for _, location := range locations {
 		stage.Current = location.RealPath
 		result, err := i.catalogLocation(resolver, location)
+
+		if errors.Is(err, errUndigestableFile) {
+			continue
+		}
+
 		if internal.IsErrPathPermission(err) {
-			log.Debugf("file digests cataloger skipping - %+v", err)
+			log.Debugf("file digests cataloger skipping %q: %+v", location.RealPath, err)
 			continue
 		}
 
@@ -56,6 +61,16 @@ func (i *DigestsCataloger) Catalog(resolver source.FileResolver) (map[source.Coo
 }
 
 func (i *DigestsCataloger) catalogLocation(resolver source.FileResolver, location source.Location) ([]Digest, error) {
+	meta, err := resolver.FileMetadataByLocation(location)
+	if err != nil {
+		return nil, err
+	}
+
+	// we should only attempt to report digests for files that are regular files (don't attempt to resolve links)
+	if meta.Type != source.RegularFile {
+		return nil, errUndigestableFile
+	}
+
 	contentReader, err := resolver.FileContentsByLocation(location)
 	if err != nil {
 		return nil, err
@@ -72,7 +87,7 @@ func (i *DigestsCataloger) catalogLocation(resolver source.FileResolver, locatio
 
 	size, err := io.Copy(io.MultiWriter(writers...), contentReader)
 	if err != nil {
-		return nil, internal.ErrPath{Path: location.RealPath, Err: err}
+		return nil, internal.ErrPath{Context: "digests-cataloger", Path: location.RealPath, Err: err}
 	}
 
 	if size == 0 {