fix(sbom): add checksum to files (#3888)

Co-authored-by: knqyf263 <[email protected]>

integration/testdata/conda-spdx.json.golden

@@ -1,7 +1,7 @@
 {
 	"SPDXID": "SPDXRef-DOCUMENT",
 	"creationInfo": {
-		"created": "2023-01-08T23:58:16.700785648Z",
+		"created": "2023-03-29T19:07:18Z",
 		"creators": [
 			"Tool: trivy-dev",
 			"Organization: aquasecurity"
@@ -11,14 +11,26 @@
 	"documentDescribes": [
 		"SPDXRef-Filesystem-6e0ac6a0fab50ab4"
 	],
-	"documentNamespace": "http://aquasecurity.github.io/trivy/filesystem/testdata/fixtures/fs/conda-3be0d21e-5711-451e-8b1b-2ac8775a3abb",
+	"documentNamespace": "http://aquasecurity.github.io/trivy/filesystem/testdata/fixtures/fs/conda-d872c7e3-4c6c-4fa1-a9b6-3e69dc71ff3b",
 	"files": [
 		{
 			"SPDXID": "SPDXRef-File-600e5e0110a84891",
+			"checksums": [
+				{
+					"algorithm": "SHA1",
+					"checksumValue": "237db0da53131e4548cb1181337fa0f420299e1f"
+				}
+			],
 			"fileName": "miniconda3/envs/testenv/conda-meta/openssl-1.1.1q-h7f8727e_0.json"
 		},
 		{
 			"SPDXID": "SPDXRef-File-7eb62e2a3edddc0a",
+			"checksums": [
+				{
+					"algorithm": "SHA1",
+					"checksumValue": "a6a2db7668f1ad541d704369fc66c96a4415aa24"
+				}
+			],
 			"fileName": "miniconda3/envs/testenv/conda-meta/pip-22.2.2-py38h06a4308_0.json"
 		}
 	],
@@ -33,50 +45,50 @@
 		},
 		{
 			"SPDXID": "SPDXRef-Filesystem-6e0ac6a0fab50ab4",
-			"downloadLocation": "NONE",
 			"attributionTexts": [
 				"SchemaVersion: 2"
 			],
+			"downloadLocation": "NONE",
 			"filesAnalyzed": false,
 			"name": "testdata/fixtures/fs/conda"
 		},
 		{
-			"SPDXID": "SPDXRef-Package-2984084f02572600",
+			"SPDXID": "SPDXRef-Package-6b677e82217fb5bd",
 			"downloadLocation": "NONE",
 			"externalRefs": [
 				{
 					"referenceCategory": "PACKAGE-MANAGER",
-					"referenceLocator": "pkg:conda/[email protected]",
+					"referenceLocator": "pkg:conda/[email protected]",
 					"referenceType": "purl"
 				}
 			],
 			"filesAnalyzed": false,
 			"hasFiles": [
-				"SPDXRef-File-600e5e0110a84891"
+				"SPDXRef-File-7eb62e2a3edddc0a"
 			],
-			"licenseConcluded": "OpenSSL",
-			"licenseDeclared": "OpenSSL",
-			"name": "openssl",
-			"versionInfo": "1.1.1q"
+			"licenseConcluded": "MIT",
+			"licenseDeclared": "MIT",
+			"name": "pip",
+			"versionInfo": "22.2.2"
 		},
 		{
-			"SPDXID": "SPDXRef-Package-ac33eb699b3aa81d",
+			"SPDXID": "SPDXRef-Package-b1088cb4090e3a55",
 			"downloadLocation": "NONE",
 			"externalRefs": [
 				{
 					"referenceCategory": "PACKAGE-MANAGER",
-					"referenceLocator": "pkg:conda/[email protected]",
+					"referenceLocator": "pkg:conda/[email protected]",
 					"referenceType": "purl"
 				}
 			],
 			"filesAnalyzed": false,
 			"hasFiles": [
-				"SPDXRef-File-7eb62e2a3edddc0a"
+				"SPDXRef-File-600e5e0110a84891"
 			],
-			"licenseConcluded": "MIT",
-			"licenseDeclared": "MIT",
-			"name": "pip",
-			"versionInfo": "22.2.2"
+			"licenseConcluded": "OpenSSL",
+			"licenseDeclared": "OpenSSL",
+			"name": "openssl",
+			"versionInfo": "1.1.1q"
 		}
 	],
 	"relationships": [
@@ -91,12 +103,12 @@
 			"spdxElementId": "SPDXRef-Filesystem-6e0ac6a0fab50ab4"
 		},
 		{
-			"relatedSpdxElement": "SPDXRef-Package-2984084f02572600",
+			"relatedSpdxElement": "SPDXRef-Package-b1088cb4090e3a55",
 			"relationshipType": "CONTAINS",
 			"spdxElementId": "SPDXRef-Application-ee5ef1aa4ac89125"
 		},
 		{
-			"relatedSpdxElement": "SPDXRef-Package-ac33eb699b3aa81d",
+			"relatedSpdxElement": "SPDXRef-Package-6b677e82217fb5bd",
 			"relationshipType": "CONTAINS",
 			"spdxElementId": "SPDXRef-Application-ee5ef1aa4ac89125"
 		}

pkg/commands/artifact/run.go

@@ -609,6 +609,12 @@ func initScannerConfig(opts flag.Options, cacheClient cache.Cache) (ScannerConfi
 		}
 	}
 
+	// SPDX needs to calculate digests for package files
+	var fileChecksum bool
+	if opts.Format == report.FormatSPDXJSON || opts.Format == report.FormatSPDX {
+		fileChecksum = true
+	}
+
 	remoteOpts := opts.Remote()
 
 	return ScannerConfig{
@@ -635,6 +641,7 @@ func initScannerConfig(opts flag.Options, cacheClient cache.Cache) (ScannerConfi
 			Platform:          opts.Platform,
 			Slow:              opts.Slow,
 			AWSRegion:         opts.Region,
+			FileChecksum:      fileChecksum,
 
 			// For OCI registries
 			RemoteOptions: remoteOpts,

pkg/digest/digest.go

@@ -0,0 +1,78 @@
+package digest
+
+import (
+	"crypto/sha1" // nolint
+	"crypto/sha256"
+	"fmt"
+	"hash"
+	"io"
+	"strings"
+
+	"golang.org/x/xerrors"
+)
+
+type Algorithm string
+
+func (a Algorithm) String() string {
+	return string(a)
+}
+
+// supported digest types
+const (
+	SHA1   Algorithm = "sha1"   // sha1 with hex encoding (lower case only)
+	SHA256 Algorithm = "sha256" // sha256 with hex encoding (lower case only)
+)
+
+// Digest allows simple protection of hex formatted digest strings, prefixed by their algorithm.
+//
+// The following is an example of the contents of Digest types:
+//
+//	sha256:7173b809ca12ec5dee4506cd86be934c4596dd234ee82c0662eac04a8c2c71dc
+type Digest string
+
+// NewDigest returns a Digest from alg and a hash.Hash object.
+func NewDigest(alg Algorithm, h hash.Hash) Digest {
+	return Digest(fmt.Sprintf("%s:%x", alg, h.Sum(nil)))
+}
+
+func (d Digest) Algorithm() Algorithm {
+	return Algorithm(d[:d.sepIndex()])
+}
+
+func (d Digest) Encoded() string {
+	return string(d[d.sepIndex()+1:])
+}
+
+func (d Digest) String() string {
+	return string(d)
+}
+
+func (d Digest) sepIndex() int {
+	i := strings.Index(string(d), ":")
+	if i < 0 {
+		i = 0
+	}
+	return i
+}
+
+func CalcSHA1(r io.ReadSeeker) (Digest, error) {
+	defer r.Seek(0, io.SeekStart)
+
+	h := sha1.New() // nolint
+	if _, err := io.Copy(h, r); err != nil {
+		return "", xerrors.Errorf("unable to calculate sha1 digest: %w", err)
+	}
+
+	return NewDigest(SHA1, h), nil
+}
+
+func CalcSHA256(r io.ReadSeeker) (Digest, error) {
+	defer r.Seek(0, io.SeekStart)
+
+	h := sha256.New()
+	if _, err := io.Copy(h, r); err != nil {
+		return "", xerrors.Errorf("unable to calculate sha256 digest: %w", err)
+	}
+
+	return NewDigest(SHA256, h), nil
+}

pkg/fanal/analyzer/analyzer.go

@@ -143,7 +143,8 @@ type PostAnalysisInput struct {
 }
 
 type AnalysisOptions struct {
-	Offline bool
+	Offline      bool
+	FileChecksum bool
 }
 
 type AnalysisResult struct {

pkg/fanal/analyzer/executable/executable.go

@@ -2,13 +2,11 @@ package executable
 
 import (
 	"context"
-	"crypto/sha256"
-	"encoding/hex"
-	"io"
 	"os"
 
 	"golang.org/x/xerrors"
 
+	"github.com/aquasecurity/trivy/pkg/digest"
 	"github.com/aquasecurity/trivy/pkg/fanal/analyzer"
 	"github.com/aquasecurity/trivy/pkg/fanal/utils"
 )
@@ -30,15 +28,14 @@ func (a executableAnalyzer) Analyze(_ context.Context, input analyzer.AnalysisIn
 		return nil, nil
 	}
 
-	h := sha256.New()
-	if _, err = io.Copy(h, input.Content); err != nil {
+	dig, err := digest.CalcSHA256(input.Content)
+	if err != nil {
 		return nil, xerrors.Errorf("sha256 error: %w", err)
 	}
-	s := hex.EncodeToString(h.Sum(nil))
 
 	return &analyzer.AnalysisResult{
 		Digests: map[string]string{
-			input.FilePath: "sha256:" + s,
+			input.FilePath: dig.String(),
 		},
 	}, nil
 }

pkg/fanal/analyzer/language/analyze.go

@@ -1,33 +1,88 @@
 package language
 
 import (
+	"io"
 	"strings"
 
 	"golang.org/x/xerrors"
 
 	dio "github.com/aquasecurity/go-dep-parser/pkg/io"
 	godeptypes "github.com/aquasecurity/go-dep-parser/pkg/types"
+	"github.com/aquasecurity/trivy/pkg/digest"
 	"github.com/aquasecurity/trivy/pkg/fanal/analyzer"
 	"github.com/aquasecurity/trivy/pkg/fanal/types"
 	"github.com/aquasecurity/trivy/pkg/licensing"
+	"github.com/aquasecurity/trivy/pkg/log"
 )
 
+// Analyze returns an analysis result of the lock file
 func Analyze(fileType, filePath string, r dio.ReadSeekerAt, parser godeptypes.Parser) (*analyzer.AnalysisResult, error) {
+	app, err := Parse(fileType, filePath, r, parser)
+	if err != nil {
+		return nil, xerrors.Errorf("failed to parse %s: %w", filePath, err)
+	}
+
+	if app == nil {
+		return nil, nil
+	}
+
+	return &analyzer.AnalysisResult{Applications: []types.Application{*app}}, nil
+}
+
+// AnalyzePackage returns an analysis result of the package file other than lock files
+func AnalyzePackage(fileType, filePath string, r dio.ReadSeekerAt, parser godeptypes.Parser, checksum bool) (*analyzer.AnalysisResult, error) {
+	app, err := ParsePackage(fileType, filePath, r, parser, checksum)
+	if err != nil {
+		return nil, xerrors.Errorf("failed to parse %s: %w", filePath, err)
+	}
+
+	if app == nil {
+		return nil, nil
+	}
+
+	return &analyzer.AnalysisResult{Applications: []types.Application{*app}}, nil
+}
+
+// Parse returns a parsed result of the lock file
+func Parse(fileType, filePath string, r dio.ReadSeekerAt, parser godeptypes.Parser) (*types.Application, error) {
 	parsedLibs, parsedDependencies, err := parser.Parse(r)
 	if err != nil {
 		return nil, xerrors.Errorf("failed to parse %s: %w", filePath, err)
 	}
 
 	// The file path of each library should be empty in case of dependency list such as lock file
 	// since they all will be the same path.
-	return ToAnalysisResult(fileType, filePath, "", parsedLibs, parsedDependencies), nil
+	return toApplication(fileType, filePath, "", nil, parsedLibs, parsedDependencies), nil
 }
 
-func ToApplication(fileType, filePath, libFilePath string, libs []godeptypes.Library, depGraph []godeptypes.Dependency) *types.Application {
+// ParsePackage returns a parsed result of the package file
+func ParsePackage(fileType, filePath string, r dio.ReadSeekerAt, parser godeptypes.Parser, checksum bool) (*types.Application, error) {
+	parsedLibs, parsedDependencies, err := parser.Parse(r)
+	if err != nil {
+		return nil, xerrors.Errorf("failed to parse %s: %w", filePath, err)
+	}
+
+	// The reader is not passed if the checksum is not necessarily calculated.
+	if !checksum {
+		r = nil
+	}
+
+	// The file path of each library should be empty in case of dependency list such as lock file
+	// since they all will be the same path.
+	return toApplication(fileType, filePath, filePath, r, parsedLibs, parsedDependencies), nil
+}
+
+func toApplication(fileType, filePath, libFilePath string, r dio.ReadSeekerAt, libs []godeptypes.Library, depGraph []godeptypes.Dependency) *types.Application {
 	if len(libs) == 0 {
 		return nil
 	}
 
+	// Calculate the file digest when one of `spdx` formats is selected
+	d, err := calculateDigest(r)
+	if err != nil {
+		log.Logger.Warnf("Unable to get checksum for %s: %s", filePath, err)
+	}
+
 	deps := make(map[string][]string)
 	for _, dep := range depGraph {
 		deps[dep.ID] = dep.DependsOn
@@ -59,6 +114,7 @@ func ToApplication(fileType, filePath, libFilePath string, libs []godeptypes.Lib
 			Licenses:  licenses,
 			DependsOn: deps[lib.ID],
 			Locations: locs,
+			Digest:    d,
 		})
 	}
 
@@ -69,11 +125,14 @@ func ToApplication(fileType, filePath, libFilePath string, libs []godeptypes.Lib
 	}
 }
 
-func ToAnalysisResult(fileType, filePath, libFilePath string, libs []godeptypes.Library, depGraph []godeptypes.Dependency) *analyzer.AnalysisResult {
-	app := ToApplication(fileType, filePath, libFilePath, libs, depGraph)
-	if app == nil {
-		return nil
+func calculateDigest(r dio.ReadSeekerAt) (digest.Digest, error) {
+	if r == nil {
+		return "", nil
+	}
+	// return reader to start after it has been read in analyzer
+	if _, err := r.Seek(0, io.SeekStart); err != nil {
+		return "", xerrors.Errorf("unable to seek: %w", err)
 	}
 
-	return &analyzer.AnalysisResult{Applications: []types.Application{*app}}
+	return digest.CalcSHA1(r)
 }