Merge pull request #2096 from puerco/yamlconf

Fixes to SBOM libraries

cmd/bom/cmd/generate.go

@@ -74,6 +74,7 @@ type generateOptions struct {
 	noGoTransient  bool
 	namespace      string
 	outputFile     string
+	configFile     string
 	images         []string
 	tarballs       []string
 	files          []string
@@ -83,17 +84,22 @@ type generateOptions struct {
 
 // Validate verify options consistency
 func (opts *generateOptions) Validate() error {
-	if len(opts.images) == 0 && len(opts.files) == 0 && len(opts.tarballs) == 0 && len(opts.directories) == 0 {
+	if opts.configFile == "" &&
+		len(opts.images) == 0 &&
+		len(opts.files) == 0 &&
+		len(opts.tarballs) == 0 &&
+		len(opts.directories) == 0 {
 		return errors.New("to generate a SPDX BOM you have to provide at least one image or file")
 	}
 
 	// A namespace URL is required
-	if opts.namespace == "" {
-		msg := "\nYou did not specify a namespace for your document. This is an error.\n"
+	if opts.configFile == "" && opts.namespace == "" {
+		msg := "Error. No namespace defined\n\n"
+		msg += "You did not specify a namespace for your document. This is an error.\n"
 		msg += "To produce a valid SPDX SBOM, the document has to have an URI as its\n"
-		msg += "namespace.\n\nYou can use http://example.com/ for now if you are testing but your\n"
-		msg += "final document must have the namespace URI pointing to the location where\n"
-		msg += "you SBOM will be referenced in the future.\n\n"
+		msg += "namespace.\n\nIf you are testing, you can use http://example.com/ for now but your\n"
+		msg += "final document must have a namespace URI pointing to the location where\n"
+		msg += "your SBOM will be referenced in the future.\n\n"
 		msg += "For more details, check the SPDX documentation here:\n"
 		msg += "https://spdx.github.io/spdx-spec/2-document-creation-information/#25-spdx-document-namespace\n\n"
 		msg += "Hint: --namespace is your friend here\n\n"
@@ -102,7 +108,7 @@ func (opts *generateOptions) Validate() error {
 		return errors.New("A namespace URI must be defined to have a compliant SPDX BOM")
 	}
 
-	// CHeck namespace is a valid URL
+	// Check namespace is a valid URL
 	if _, err := url.Parse(opts.namespace); err != nil {
 		return errors.Wrap(err, "parsing the namespace URL")
 	}
@@ -118,6 +124,7 @@ func init() {
 		[]string{},
 		"list of images",
 	)
+
 	generateCmd.PersistentFlags().StringSliceVarP(
 		&genOpts.files,
 		"file",
@@ -157,11 +164,12 @@ func init() {
 	)
 
 	generateCmd.PersistentFlags().BoolVar(
-		&genOpts.noGitignore,
+		&genOpts.noGoModules,
 		"no-gomod",
 		false,
 		"don't perform go.mod analysis, sbom will not include data about go packages",
 	)
+
 	generateCmd.PersistentFlags().BoolVar(
 		&genOpts.noGoTransient,
 		"no-transient",
@@ -192,6 +200,14 @@ func init() {
 		false,
 		"go deeper into images using the available analyzers",
 	)
+
+	generateCmd.PersistentFlags().StringVarP(
+		&genOpts.configFile,
+		"config",
+		"c",
+		"",
+		"path to yaml SBOM configuration file",
+	)
 }
 
 func generateBOM(opts *generateOptions) error {
@@ -211,6 +227,7 @@ func generateBOM(opts *generateOptions) error {
 		AnalyseLayers:    opts.analyze,
 		ProcessGoModules: !opts.noGoModules,
 		OnlyDirectDeps:   !opts.noGoTransient,
+		ConfigFile:       opts.configFile,
 	}
 
 	// We only replace the ignore patterns one or more where defined

pkg/license/catalog.go

@@ -17,11 +17,13 @@ limitations under the License.
 package license
 
 import (
+	"os"
 	"path/filepath"
 	"sync"
 
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
+	"sigs.k8s.io/release-utils/util"
 )
 
 // CatalogOptions are the spdx settings
@@ -80,6 +82,11 @@ func (catalog *Catalog) WriteLicensesAsText(targetDir string) error {
 	if catalog.List.Licenses == nil {
 		return errors.New("unable to write licenses, they have not been loaded yet")
 	}
+	if !util.Exists(targetDir) {
+		if err := os.MkdirAll(targetDir, os.FileMode(0o755)); err != nil {
+			return errors.Wrap(err, "creating license data dir")
+		}
+	}
 	wg := sync.WaitGroup{}
 	var err error
 	for _, l := range catalog.List.Licenses {

pkg/license/download.go

@@ -150,7 +150,11 @@ func (ddi *DefaultDownloaderImpl) GetLicenses() (licenses *List, err error) {
 
 	// Create a new Throttler that will get `parallelDownloads` urls at a time
 	t := throttler.New(ddi.Options.parallelDownloads, len(licenseList.LicenseData))
-	for _, l := range licenseList.LicenseData {
+	for i, l := range licenseList.LicenseData {
+		// Plus signs in the license list come as xml entity
+		licenseList.LicenseData[i].LicenseID = strings.ReplaceAll(
+			licenseList.LicenseData[i].LicenseID, `+`, `+`,
+		)
 		licURL := l.DetailsURL
 		// If the license URLs have a local reference
 		if strings.HasPrefix(licURL, "./") {

pkg/license/implementation.go

@@ -61,7 +61,8 @@ func (d *ReaderDefaultImpl) ClassifyFile(path string) (licenseTag string, moreTa
 
 // ClassifyLicenseFiles takes a list of paths and tries to find return all licenses found in it
 func (d *ReaderDefaultImpl) ClassifyLicenseFiles(paths []string) (
-	licenseList []ClassifyResult, unrecognizedPaths []string, err error) {
+	licenseList []*ClassifyResult, unrecognizedPaths []string, err error) {
+	licenseList = []*ClassifyResult{}
 	// Run the files through the clasifier
 	for _, f := range paths {
 		label, _, err := d.ClassifyFile(f)
@@ -78,12 +79,16 @@ func (d *ReaderDefaultImpl) ClassifyLicenseFiles(paths []string) (
 			return nil, unrecognizedPaths,
 				errors.New(fmt.Sprintf("ID does not correspond to a valid license: '%s'", label))
 		}
+		licenseText, err := os.ReadFile(f)
+		if err != nil {
+			return nil, nil, errors.Wrap(err, "reading license text")
+		}
 		// Apend to the return results
-		licenseList = append(licenseList, ClassifyResult{f, license})
+		licenseList = append(licenseList, &ClassifyResult{f, string(licenseText), license})
 	}
-	if len(paths) > 0 {
+	if len(paths) != len(licenseList) {
 		logrus.Infof(
-			"License classifier recognized %d/%d (%d%%) os the files",
+			"License classifier recognized %d/%d (%d%%) of the license files",
 			len(licenseList), len(paths), (len(licenseList)/len(paths))*100,
 		)
 	}
@@ -172,13 +177,13 @@ func (d *ReaderDefaultImpl) Initialize(opts *ReaderOptions) error {
 	logrus.Infof("Writing license data to %s", opts.CachePath())
 
 	// Write the licenses to disk as th classifier will need them
-	if err := catalog.WriteLicensesAsText(opts.CachePath()); err != nil {
+	if err := catalog.WriteLicensesAsText(opts.LicensesPath()); err != nil {
 		return errors.Wrap(err, "writing license data to disk")
 	}
 
 	// Create the implementation's classifier
 	d.lc = licenseclassifier.NewClassifier(opts.ConfidenceThreshold)
-	return errors.Wrap(d.lc.LoadLicenses(opts.CachePath()), "loading licenses at init")
+	return errors.Wrap(d.lc.LoadLicenses(opts.LicensesPath()), "loading licenses at init")
 }
 
 // Classifier returns the license classifier

pkg/license/license.go

@@ -28,6 +28,7 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
+	"sigs.k8s.io/release-utils/util"
 )
 
 const (
@@ -160,8 +161,79 @@ func (r *Reader) LicenseFromFile(filePath string) (license *License, err error)
 	return license, err
 }
 
+// ReadTopLicense returns the topmost license file in a directory
+func (r *Reader) ReadTopLicense(path string) (*ClassifyResult, error) {
+	licenseFilePath := ""
+	// First, if we have a topmost license, we use that one
+	commonNames := []string{"LICENSE", "LICENSE.txt", "COPYING", "COPYRIGHT"}
+	for _, f := range commonNames {
+		if util.Exists(filepath.Join(path, f)) {
+			licenseFilePath = filepath.Join(path, f)
+			break
+		}
+	}
+
+	if licenseFilePath != "" {
+		result, _, err := r.impl.ClassifyLicenseFiles([]string{licenseFilePath})
+		if err != nil {
+			return nil, errors.Wrap(err, "scanning topmost license file")
+		}
+		if len(result) != 0 {
+			logrus.Infof("Concluded license %s from %s", result[0].License.LicenseID, licenseFilePath)
+			return result[0], nil
+		}
+	}
+
+	// If the standard file did not work, then we try to
+	// find the license file at the highest dir in the FS tree
+	var res *ClassifyResult
+	bestPartsN := 0
+	licenseFiles, err := r.impl.FindLicenseFiles(path)
+	if err != nil {
+		return nil, errors.Wrap(err, "finding license files in path")
+	}
+	for _, fileName := range licenseFiles {
+		try := false
+		dir := filepath.Dir(fileName)
+		parts := strings.Split(dir, string(filepath.Separator))
+		// If this file is higher in the fstree, we use it
+		if bestPartsN == 0 || len(parts) < bestPartsN {
+			try = true
+		}
+		// If this file in the same level but the path is shorter, we try it
+		if len(parts) == bestPartsN && len(fileName) < len(licenseFilePath) {
+			try = true
+		}
+
+		// If this file is not a better candidate, skip to the next one
+		if !try {
+			continue
+		}
+
+		result, _, err := r.impl.ClassifyLicenseFiles([]string{fileName})
+		if err != nil {
+			return nil, errors.Wrap(err, "scanning topmost license file")
+		}
+
+		// If the file is a license, use it
+		if len(result) > 0 {
+			bestPartsN = len(parts)
+			licenseFilePath = fileName
+			res = result[0]
+		}
+	}
+	if res == nil {
+		logrus.Warnf("Could not find any licensing information in %s", path)
+	} else {
+		logrus.Infof("Concluded license %s from %s", res.License.LicenseID, licenseFilePath)
+	}
+	return res, nil
+}
+
 // ReadLicenses returns an array of all licenses found in the specified path
-func (r *Reader) ReadLicenses(path string) (licenseList []ClassifyResult, unknownPaths []string, err error) {
+func (r *Reader) ReadLicenses(path string) (
+	licenseList []*ClassifyResult, unknownPaths []string, err error,
+) {
 	licenseFiles, err := r.impl.FindLicenseFiles(path)
 	if err != nil {
 		return nil, nil, errors.Wrap(err, "searching for license files")
@@ -177,6 +249,7 @@ func (r *Reader) ReadLicenses(path string) (licenseList []ClassifyResult, unknow
 // ClassifyResult abstracts the data resulting from a file classification
 type ClassifyResult struct {
 	File    string
+	Text    string
 	License *License
 }
 
@@ -186,7 +259,7 @@ type ClassifyResult struct {
 // initializes -> finds license files to scan -> classifies them to a SPDX license
 type ReaderImplementation interface {
 	Initialize(*ReaderOptions) error
-	ClassifyLicenseFiles([]string) ([]ClassifyResult, []string, error)
+	ClassifyLicenseFiles([]string) ([]*ClassifyResult, []string, error)
 	ClassifyFile(string) (string, []string, error)
 	LicenseFromFile(string) (*License, error)
 	LicenseFromLabel(string) *License