Skip to content

Commit

Permalink
Merge pull request #2147 from puerco/spdx-multiarch-images
Browse files Browse the repository at this point in the history
SPDX bug fixes found during v1.22.0-beta.0
k8s-ci-robot authored Jul 2, 2021

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
2 parents 1815950 + 17700c4 commit 6e3b30b
Showing 13 changed files with 1,152 additions and 461 deletions.
6 changes: 1 addition & 5 deletions pkg/license/download.go
Original file line number Diff line number Diff line change
@@ -150,11 +150,7 @@ func (ddi *DefaultDownloaderImpl) GetLicenses() (licenses *List, err error) {

// Create a new Throttler that will get `parallelDownloads` urls at a time
t := throttler.New(ddi.Options.parallelDownloads, len(licenseList.LicenseData))
for i, l := range licenseList.LicenseData {
// Plus signs in the license list come as xml entity
licenseList.LicenseData[i].LicenseID = strings.ReplaceAll(
licenseList.LicenseData[i].LicenseID, `+`, `+`,
)
for _, l := range licenseList.LicenseData {
licURL := l.DetailsURL
// If the license URLs have a local reference
if strings.HasPrefix(licURL, "./") {
43 changes: 7 additions & 36 deletions pkg/spdx/builder.go
Original file line number Diff line number Diff line change
@@ -21,8 +21,6 @@ import (
"os"
"path/filepath"

"github.com/google/go-containerregistry/pkg/name"
"github.com/google/uuid"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"gopkg.in/yaml.v2"
@@ -161,12 +159,6 @@ func (builder *defaultDocBuilderImpl) GenerateDoc(
}
}

tmpdir, err := os.MkdirTemp(opts.WorkDir, "doc-build-")
if err != nil {
return nil, errors.Wrapf(err, "creating temporary workdir in %s", opts.WorkDir)
}
defer os.RemoveAll(tmpdir)

// Create the new document
doc = NewDocument()
doc.Name = genopts.Name
@@ -189,44 +181,22 @@ func (builder *defaultDocBuilderImpl) GenerateDoc(
}
}

// Process all image references from registries
for _, i := range genopts.Images {
logrus.Infof("Processing image: %s", i)
tararchive := filepath.Join(tmpdir, uuid.New().String()+".tar")
if err := spdx.PullImagesToArchive(i, tararchive); err != nil {
return nil, errors.Wrapf(err, "writing image %s to file", i)
}
p, err := spdx.PackageFromImageTarball(tararchive, &TarballOptions{})
logrus.Infof("Processing image reference: %s", i)
p, err := spdx.ImageRefToPackage(i)
if err != nil {
return nil, errors.Wrap(err, "generating tarball package")
}
ref, err := name.ParseReference(i)
if err != nil {
return nil, errors.Wrapf(err, "parsing image reference %q", i)
}

// Grab the package data from wither the tag or, if it's a digest,
// from parsing the digest
tag, ok := ref.(name.Tag)
if ok {
p.Name = tag.RepositoryStr()
p.DownloadLocation = tag.Name()
p.Version = tag.Identifier()
} else {
dgst, ok := ref.(name.Digest)
if ok {
p.Version = dgst.DigestStr()
p.Name = dgst.RepositoryStr()
p.DownloadLocation = dgst.Name()
}
return nil, errors.Wrapf(err, "generating SPDX package from image ref %s", i)
}
if err := doc.AddPackage(p); err != nil {
return nil, errors.Wrap(err, "adding package to document")
}
}

// Porcess OCI image archives
for _, tb := range genopts.Tarballs {
logrus.Infof("Processing tarball %s", tb)
p, err := spdx.PackageFromImageTarball(tb, &TarballOptions{})
p, err := spdx.PackageFromImageTarball(tb)
if err != nil {
return nil, errors.Wrap(err, "generating tarball package")
}
@@ -235,6 +205,7 @@ func (builder *defaultDocBuilderImpl) GenerateDoc(
}
}

// Process single files, not part of a package
for _, f := range genopts.Files {
logrus.Infof("Processing file %s", f)
f, err := spdx.FileFromPath(f)
20 changes: 7 additions & 13 deletions pkg/spdx/document.go
Original file line number Diff line number Diff line change
@@ -23,7 +23,6 @@ import (
"html/template"
"log"
"os"
"regexp"
"time"

"github.com/google/uuid"
@@ -92,22 +91,17 @@ func (d *Document) AddPackage(pkg *Package) error {
d.Packages = map[string]*Package{}
}

if pkg.ID == "" {
// If we so not have an ID but have a name generate it fro there
reg := regexp.MustCompile("[^a-zA-Z0-9-]+")
id := reg.ReplaceAllString(pkg.Name, "")
if id != "" {
pkg.ID = "SPDXRef-Package-" + id
}
if pkg.SPDXID() == "" {
pkg.BuildID(pkg.Name)
}
if pkg.ID == "" {
if pkg.SPDXID() == "" {
return errors.New("package id is needed to add a new package")
}
if _, ok := d.Packages[pkg.ID]; ok {
return errors.New("a package named " + pkg.ID + " already exists in the document")
if _, ok := d.Packages[pkg.SPDXID()]; ok {
return errors.New("a package named " + pkg.SPDXID() + " already exists in the document")
}

d.Packages[pkg.ID] = pkg
d.Packages[pkg.SPDXID()] = pkg
return nil
}

@@ -129,7 +123,7 @@ func (d *Document) Render() (doc string, err error) {
var buf bytes.Buffer
funcMap := template.FuncMap{
// The name "title" is what the function will be called in the template text.
"dateFormat": func(t time.Time) string { return t.UTC().Format("2006-02-01T15:04:05Z") },
"dateFormat": func(t time.Time) string { return t.UTC().Format("2006-01-02T15:04:05Z") },
}

if d.Name == "" {
86 changes: 7 additions & 79 deletions pkg/spdx/file.go
Original file line number Diff line number Diff line change
@@ -18,16 +18,10 @@ package spdx

import (
"bytes"
"crypto/sha1"
"html/template"
"os"
"path/filepath"
"strings"
"text/template"

"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"sigs.k8s.io/release-utils/hash"
"sigs.k8s.io/release-utils/util"
)

var fileTemplate = `{{ if .Name }}FileName: {{ .Name }}
@@ -49,68 +43,16 @@ FileCopyrightText: {{ if .CopyrightText }}<text>{{ .CopyrightText }}

// File abstracts a file contained in a package
type File struct {
Name string // string /Makefile
FileName string // Name of the file
ID string // SPDXRef-Makefile
LicenseConcluded string // GPL-3.0-or-later
Entity
LicenseInfoInFile string // GPL-3.0-or-later
CopyrightText string // NOASSERTION
SourceFile string // Source file to read from (not part of the spec)
Checksum map[string]string

options *FileOptions // Options
}

func NewFile() (f *File) {
f = &File{
options: &FileOptions{},
}
f = &File{}
f.Entity.Opts = &ObjectOptions{}
return f
}

func (f *File) Options() *FileOptions {
return f.options
}

// FileOptions
type FileOptions struct {
WorkDir string
}

// ReadChecksums receives a path to a file and calculates its checksums
func (f *File) ReadChecksums(filePath string) error {
if f.Checksum == nil {
f.Checksum = map[string]string{}
}
file, err := os.Open(filePath)
if err != nil {
return errors.Wrap(err, "opening file for reading: "+filePath)
}
defer file.Close()
// TODO: Make this line like the others once this PR is
// included in a k-sigs/release-util release:
// https://github.com/kubernetes-sigs/release-utils/pull/16
s1, err := hash.ForFile(filePath, sha1.New())
if err != nil {
return errors.Wrap(err, "getting sha1 sum for file")
}
s256, err := hash.SHA256ForFile(filePath)
if err != nil {
return errors.Wrap(err, "getting file checksums")
}
s512, err := hash.SHA512ForFile(filePath)
if err != nil {
return errors.Wrap(err, "getting file checksums")
}

f.Checksum = map[string]string{
"SHA1": s1,
"SHA256": s256,
"SHA512": s512,
}
return nil
}

// Render renders the document fragment of a file
func (f *File) Render() (docFragment string, err error) {
// If we have not yet checksummed the file, do it now:
@@ -140,21 +82,7 @@ func (f *File) Render() (docFragment string, err error) {
return docFragment, nil
}

// ReadSourceFile reads the source file for the package and populates
// the fields derived from it (Checksums and FileName)
func (f *File) ReadSourceFile(path string) error {
if !util.Exists(path) {
return errors.New("unable to find package source file")
}

if err := f.ReadChecksums(path); err != nil {
return errors.Wrap(err, "reading file checksums")
}

f.SourceFile = path
f.Name = strings.TrimPrefix(
path, f.Options().WorkDir+string(filepath.Separator),
)
f.ID = "SPDXRef-File-" + f.Checksum["SHA256"][0:15]
return nil
// BuildID sets the file ID, optionally from a series of strings
func (f *File) BuildID(seeds ...string) {
f.Entity.BuildID(append([]string{"SPDXRef-File"}, seeds...)...)
}
45 changes: 36 additions & 9 deletions pkg/spdx/gomod.go
Original file line number Diff line number Diff line change
@@ -94,7 +94,11 @@ func (pkg *GoPackage) ToSPDXPackage() (*Package, error) {
return nil, errors.Wrap(err, "building repository from package import path")
}
spdxPackage := NewPackage()
spdxPackage.Name = pkg.ImportPath + "@" + strings.TrimSuffix(pkg.Revision, "+incompatible")
spdxPackage.Name = pkg.ImportPath
if pkg.Revision != "" {
spdxPackage.Name += "@" + strings.TrimSuffix(pkg.Revision, "+incompatible")
}
spdxPackage.BuildID()
spdxPackage.DownloadLocation = repo.Repo
spdxPackage.LicenseConcluded = pkg.LicenseID
spdxPackage.Version = strings.TrimSuffix(pkg.Revision, "+incompatible")
@@ -164,7 +168,7 @@ func (mod *GoModule) ScanLicenses() error {
return errors.Wrap(err, "creating license scanner")
}

// Create a new Throttler that will get `parallelDownloads` urls at a time
// Create a new Throttler that will get parallelDownloads urls at a time
t := throttler.New(10, len(mod.Packages))
// Do a quick re-check for missing downloads
// todo: paralelize this. urgently.
@@ -247,14 +251,31 @@ func (mod *GoModule) BuildFullPackageList(g *modfile.File) (packageList []*GoPac
list := map[string]map[string]*ModEntry{}
for dec.More() {
m := &ModEntry{}
// Decode the json stream as we get "Module" blocks from go:
if err := dec.Decode(m); err != nil {
return nil, errors.Wrap(err, "decoding module list")
}
if m.Module.Path != "" {
if _, ok := list[m.Module.Path]; !ok {
list[m.Module.Path] = map[string]*ModEntry{}
}
list[m.Module.Path][m.Module.Version] = m

// Go list will return modules with a specific version
// and sometime duplicate entries, generic for the module
// witjout version. We try to handle both cases here:
if m.Module.Version == "" {
// If we got a generic module entry, add it to the list
// but only if we do not have a more specific (versioned)
// entry
if len(list[m.Module.Path]) == 0 {
list[m.Module.Path][m.Module.Version] = m
}
} else {
// If we got a specific version, but previously had a
// generic entry for the module, delete it
list[m.Module.Path][m.Module.Version] = m
delete(list[m.Module.Path], "")
}
}
}
logrus.Info("Adding full list of dependencies:")
@@ -330,11 +351,11 @@ func (di *GoModDefaultImpl) BuildPackageList(gomod *modfile.File) ([]*GoPackage,
// the download dir in the LocalDir field
func (di *GoModDefaultImpl) DownloadPackage(pkg *GoPackage, opts *GoModuleOptions, force bool) error {
if pkg.LocalDir != "" && util.Exists(pkg.LocalDir) && !force {
logrus.Infof("Not downloading %s as it already has local data", pkg.ImportPath)
logrus.WithField("package", pkg.ImportPath).Infof("Not downloading %s as it already has local data", pkg.ImportPath)
return nil
}

logrus.Infof("Downloading package %s@%s", pkg.ImportPath, pkg.Revision)
logrus.WithField("package", pkg.ImportPath).Infof("Downloading package %s@%s", pkg.ImportPath, pkg.Revision)
repo, err := vcs.RepoRootForImportPath(pkg.ImportPath, true)
if err != nil {
return errors.Wrapf(err, "Fetching package %s from %s", pkg.ImportPath, repo.Repo)
@@ -363,13 +384,19 @@ func (di *GoModDefaultImpl) DownloadPackage(pkg *GoPackage, opts *GoModuleOption
m := goModRevRe.FindStringSubmatch(pkg.Revision)
if len(m) > 1 {
rev = m[1]
logrus.Infof("Using commit %s as revision for download", rev)
logrus.WithField("package", pkg.ImportPath).Infof("Using commit %s as revision for download", rev)
}
if err := repo.VCS.CreateAtRev(tmpDir, repo.Repo, rev); err != nil {
return errors.Wrapf(err, "creating local clone of %s", repo.Repo)
if rev == "" {
if err := repo.VCS.Create(tmpDir, repo.Repo); err != nil {
return errors.Wrapf(err, "creating local clone of %s", repo.Repo)
}
} else {
if err := repo.VCS.CreateAtRev(tmpDir, repo.Repo, rev); err != nil {
return errors.Wrapf(err, "creating local clone of %s", repo.Repo)
}
}

logrus.Infof("Go Package %s (rev %s) downloaded to %s", pkg.ImportPath, pkg.Revision, tmpDir)
logrus.WithField("package", pkg.ImportPath).Infof("Go Package %s (rev %s) downloaded to %s", pkg.ImportPath, pkg.Revision, tmpDir)
pkg.LocalDir = tmpDir
pkg.TmpDir = true
return nil
357 changes: 327 additions & 30 deletions pkg/spdx/implementation.go

Large diffs are not rendered by default.

141 changes: 141 additions & 0 deletions pkg/spdx/object.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,141 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package spdx

import (
"crypto/sha1"
"os"
"path/filepath"
"strings"

"github.com/pkg/errors"
"sigs.k8s.io/release-utils/hash"
"sigs.k8s.io/release-utils/util"
)

// Object is an interface that dictates the common methods of spdx
// objects. Currently this includes files and packages.
type Object interface {
SPDXID() string
ReadSourceFile(string) error
Render() (string, error)
BuildID(seeds ...string)
}

type Entity struct {
ID string // Identifier string for the object in the doc
SourceFile string // Local file to read for information
Name string // Name of the package
DownloadLocation string // Download point for the entity
CopyrightText string // NOASSERTION
FileName string // Name of the file
LicenseConcluded string // LicenseID o NOASSERTION
Opts *ObjectOptions // Entity options
Relationships []*Relationship // List of objects that have a relationship woth this package
Checksum map[string]string // Colection of source file checksums
}

type ObjectOptions struct {
WorkDir string
}

func (e *Entity) Options() *ObjectOptions {
return e.Opts
}

// SPDXID returns the SPDX reference string for the object
func (e *Entity) SPDXID() string {
return e.ID
}

// BuildID sets the file ID, optionally from a series of strings
func (e *Entity) BuildID(seeds ...string) {
if len(seeds) <= 1 {
seeds = append(seeds, e.Name)
}
e.ID = buildIDString(seeds...)
}

// AddRelated this adds a related object to the file to be rendered
// on the document. The exact output depends on the related obj options
func (e *Entity) AddRelationship(rel *Relationship) {
e.Relationships = append(e.Relationships, rel)
}

// ReadChecksums receives a path to a file and calculates its checksums
func (e *Entity) ReadChecksums(filePath string) error {
if e.Checksum == nil {
e.Checksum = map[string]string{}
}
file, err := os.Open(filePath)
if err != nil {
return errors.Wrap(err, "opening file for reading: "+filePath)
}
defer file.Close()
// TODO: Make this line like the others once this PR is
// included in a k-sigs/release-util release:
// https://github.com/kubernetes-sigs/release-utils/pull/16
s1, err := hash.ForFile(filePath, sha1.New())
if err != nil {
return errors.Wrap(err, "getting sha1 sum for file")
}
s256, err := hash.SHA256ForFile(filePath)
if err != nil {
return errors.Wrap(err, "getting file checksums")
}
s512, err := hash.SHA512ForFile(filePath)
if err != nil {
return errors.Wrap(err, "getting file checksums")
}

e.Checksum = map[string]string{
"SHA1": s1,
"SHA256": s256,
"SHA512": s512,
}
return nil
}

// ReadSourceFile reads the source file for the package and populates
// the fields derived from it (Checksums and FileName)
func (e *Entity) ReadSourceFile(path string) error {
if !util.Exists(path) {
return errors.New("unable to find package source file")
}

if err := e.ReadChecksums(path); err != nil {
return errors.Wrap(err, "reading file checksums")
}

e.SourceFile = path

// If the entity name is blank, we set it to the file path
e.FileName = strings.TrimPrefix(
path, e.Options().WorkDir+string(filepath.Separator),
)

if e.Name == "" {
e.Name = e.FileName
}

return nil
}

// Render is overridden by Package and File with their own variants
func (e *Entity) Render() (string, error) {
return "", nil
}
199 changes: 68 additions & 131 deletions pkg/spdx/package.go
Original file line number Diff line number Diff line change
@@ -20,16 +20,13 @@ import (
"bytes"
"crypto/sha1"
"fmt"
"html/template"
"path/filepath"
"regexp"
"sort"
"strings"
"sync"
"text/template"

"github.com/pkg/errors"
"sigs.k8s.io/release-utils/hash"
"sigs.k8s.io/release-utils/util"
"github.com/sirupsen/logrus"
)

var packageTemplate = `##### Package: {{ .Name }}
@@ -64,20 +61,14 @@ PackageCopyrightText: {{ if .CopyrightText }}<text>{{ .CopyrightText }}

// Package groups a set of files
type Package struct {
Entity
sync.RWMutex
FilesAnalyzed bool // true
Name string // hello-go-src
ID string // SPDXRef-Package-hello-go-src
DownloadLocation string // git@github.com:swinslow/spdx-examples.git#example6/content/src
VerificationCode string // 6486e016b01e9ec8a76998cefd0705144d869234
LicenseConcluded string // LicenseID o NOASSERTION
LicenseInfoFromFiles []string // GPL-3.0-or-later
LicenseDeclared string // GPL-3.0-or-later
LicenseComments string // record any relevant background information or analysis that went in to arriving at the Concluded License
CopyrightText string // string NOASSERTION
Version string // Package version
FileName string // Name of the package
SourceFile string // Source file for the package (taball for images, rpm, deb, etc)

// Supplier: the actual distribution source for the package/directory
Supplier struct {
@@ -90,60 +81,19 @@ type Package struct {
Person string // person name and optional (<email>)
Organization string // organization name and optional (<email>)
}
// Subpackages contained
Packages map[string]*Package // Sub packages conatined in this pkg
Files map[string]*File // List of files
Checksum map[string]string // Checksum of the package
Dependencies map[string]*Package // Packages marked as dependencies

options *PackageOptions // Options
}

func NewPackage() (p *Package) {
p = &Package{
options: &PackageOptions{},
}
p = &Package{}
p.Entity.Opts = &ObjectOptions{}
return p
}

type PackageOptions struct {
WorkDir string // Working directory to read files from
}

func (p *Package) Options() *PackageOptions {
return p.options
}

// ReadSourceFile reads the source file for the package and populates
// the package fields derived from it (Checksums and FileName)
func (p *Package) ReadSourceFile(path string) error {
if !util.Exists(path) {
return errors.New("unable to find package source file")
}
s256, err := hash.SHA256ForFile(path)
if err != nil {
return errors.Wrap(err, "getting source file sha256")
}
s512, err := hash.SHA512ForFile(path)
if err != nil {
return errors.Wrap(err, "getting source file sha512")
}
p.Checksum = map[string]string{
"SHA256": s256,
"SHA512": s512,
}
p.SourceFile = path
p.FileName = strings.TrimPrefix(path, p.Options().WorkDir+string(filepath.Separator))
return nil
}

// AddFile adds a file contained in the package
func (p *Package) AddFile(file *File) error {
p.Lock()
defer p.Unlock()
if p.Files == nil {
p.Files = map[string]*File{}
}

// If file does not have an ID, we try to build one
// by hashing the file name
if file.ID == "" {
@@ -157,68 +107,62 @@ func (p *Package) AddFile(file *File) error {
if _, err := h.Write([]byte(p.Name + ":" + file.Name)); err != nil {
return errors.Wrap(err, "getting sha1 of filename")
}
file.ID = "SPDXRef-File-" + fmt.Sprintf("%x", h.Sum(nil))
}
p.Files[file.ID] = file
return nil
}

// preProcessSubPackage performs a basic check on a package
// to ensure it can be added as a subpackage, trying to infer
// missing data when possible
func (p *Package) preProcessSubPackage(pkg *Package) error {
if pkg.ID == "" {
// If we so not have an ID but have a name generate it fro there
reg := regexp.MustCompile(validNameCharsRe)
id := reg.ReplaceAllString(pkg.Name, "")
if id != "" {
pkg.ID = "SPDXRef-Package-" + id
}
}
if pkg.ID == "" {
return errors.New("package name is needed to add a new package")
}
if _, ok := p.Packages[pkg.ID]; ok {
return errors.New("a package named " + pkg.ID + " already exists as a subpackage")
file.BuildID(fmt.Sprintf("%x", h.Sum(nil)))
}

if _, ok := p.Dependencies[pkg.ID]; ok {
return errors.New("a package named " + pkg.ID + " already exists as a dependency")
}
// Add the file to the package's relationships
p.AddRelationship(&Relationship{
FullRender: true,
Type: CONTAINS,
Peer: file,
})

return nil
}

// AddPackage adds a new subpackage to a package
func (p *Package) AddPackage(pkg *Package) error {
if p.Packages == nil {
p.Packages = map[string]*Package{}
}

if err := p.preProcessSubPackage(pkg); err != nil {
return errors.Wrap(err, "performing subpackage preprocessing")
}

p.Packages[pkg.ID] = pkg
p.AddRelationship(&Relationship{
Peer: pkg,
Type: CONTAINS,
FullRender: true,
})
return nil
}

// AddDependency adds a new subpackage as a dependency
func (p *Package) AddDependency(pkg *Package) error {
if p.Dependencies == nil {
p.Dependencies = map[string]*Package{}
}
p.AddRelationship(&Relationship{
Peer: pkg,
Type: DEPENDS_ON,
FullRender: true,
})
return nil
}

if err := p.preProcessSubPackage(pkg); err != nil {
return errors.Wrap(err, "performing subpackage preprocessing")
// Files returns all contained files in the package
func (p *Package) Files() []*File {
ret := []*File{}
for _, rel := range p.Relationships {
if rel.Peer != nil {
if p, ok := rel.Peer.(*File); ok {
ret = append(ret, p)
}
}
}

p.Dependencies[pkg.ID] = pkg
return nil
return ret
}

// Render renders the document fragment of the package
func (p *Package) Render() (docFragment string, err error) {
// First thing, check all relationships
if len(p.Relationships) > 0 {
logrus.Infof("Package %s has %d relationships defined", p.SPDXID(), len(p.Relationships))
if err := p.CheckRelationships(); err != nil {
return "", errors.Wrap(err, "checking package relationships")
}
}

var buf bytes.Buffer
tmpl, err := template.New("package").Parse(packageTemplate)
if err != nil {
@@ -233,11 +177,12 @@ func (p *Package) Render() (docFragment string, err error) {
// entry of the SPDX package:
filesTagList := []string{}
if p.FilesAnalyzed {
if len(p.Files) == 0 {
files := p.Files()
if len(files) == 0 {
return docFragment, errors.New("unable to get package verification code, package has no files")
}
shaList := []string{}
for _, f := range p.Files {
for _, f := range files {
if f.Checksum == nil {
return docFragment, errors.New("unable to render package, file has no checksums")
}
@@ -273,9 +218,8 @@ func (p *Package) Render() (docFragment string, err error) {
}
}

// If no license tags where collected from files, then
// the BOM has to express "NONE" in the LicenseInfoFromFiles
// section to be compliant:
// If no license tags where collected from files, then the BOM has
// to express "NONE" in the LicenseInfoFromFiles section to be compliant:
if len(filesTagList) == 0 {
p.LicenseInfoFromFiles = append(p.LicenseInfoFromFiles, NONE)
}
@@ -288,39 +232,32 @@ func (p *Package) Render() (docFragment string, err error) {

docFragment = buf.String()

for _, f := range p.Files {
fileFragment, err := f.Render()
// Add the output from all related files
for _, rel := range p.Relationships {
fragment, err := rel.Render(p)
if err != nil {
return "", errors.Wrap(err, "rendering file "+f.Name)
return "", errors.Wrap(err, "rendering relationship")
}
docFragment += fileFragment
docFragment += fmt.Sprintf("Relationship: %s CONTAINS %s\n\n", p.ID, f.ID)
docFragment += fragment
}
docFragment += "\n"
return docFragment, nil
}

// Print the contained sub packages
if p.Packages != nil {
for _, pkg := range p.Packages {
pkgDoc, err := pkg.Render()
if err != nil {
return "", errors.Wrap(err, "rendering pkg "+pkg.Name)
// CheckRelationships ensures al linked relationships are complete
// before rendering.
func (p *Package) CheckRelationships() error {
for _, related := range p.Relationships {
if related.Peer != nil {
if related.Peer.SPDXID() == "" {
related.Peer.BuildID()
}

docFragment += pkgDoc
docFragment += fmt.Sprintf("Relationship: %s CONTAINS %s\n\n", p.ID, pkg.ID)
}
}
return nil
}

// Print the contained dependencies
if p.Dependencies != nil {
for _, pkg := range p.Dependencies {
pkgDoc, err := pkg.Render()
if err != nil {
return "", errors.Wrap(err, "rendering pkg "+pkg.Name)
}

docFragment += pkgDoc
docFragment += fmt.Sprintf("Relationship: %s DEPENDS_ON %s\n\n", p.ID, pkg.ID)
}
}
return docFragment, nil
// BuildID sets the file ID, optionally from a series of strings
func (p *Package) BuildID(seeds ...string) {
p.Entity.BuildID(append([]string{"SPDXRef-Package"}, seeds...)...)
}
101 changes: 101 additions & 0 deletions pkg/spdx/relationship.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package spdx

import (
"fmt"

"github.com/pkg/errors"
)

type RelationshipType string

// nolint
const (
DESCRIBES RelationshipType = "DESCRIBES"
DESCRIBED_BY RelationshipType = "DESCRIBED_BY"
CONTAINS RelationshipType = "CONTAINS"
CONTAINED_BY RelationshipType = "CONTAINED_BY"
DEPENDS_ON RelationshipType = "DEPENDS_ON"
DEPENDENCY_OF RelationshipType = "DEPENDENCY_OF"
DEPENDENCY_MANIFEST_OF RelationshipType = "DEPENDENCY_MANIFEST_OF"
BUILD_DEPENDENCY_OF RelationshipType = "BUILD_DEPENDENCY_OF"
DEV_DEPENDENCY_OF RelationshipType = "DEV_DEPENDENCY_OF"
OPTIONAL_DEPENDENCY_OF RelationshipType = "OPTIONAL_DEPENDENCY_OF"
PROVIDED_DEPENDENCY_OF RelationshipType = "PROVIDED_DEPENDENCY_OF"
TEST_DEPENDENCY_OF RelationshipType = "TEST_DEPENDENCY_OF"
RUNTIME_DEPENDENCY_OF RelationshipType = "RUNTIME_DEPENDENCY_OF"
EXAMPLE_OF RelationshipType = "EXAMPLE_OF"
GENERATES RelationshipType = "GENERATES"
GENERATED_FROM RelationshipType = "GENERATED_FROM"
ANCESTOR_OF RelationshipType = "ANCESTOR_OF"
DESCENDANT_OF RelationshipType = "DESCENDANT_OF"
VARIANT_OF RelationshipType = "VARIANT_OF"
DISTRIBUTION_ARTIFACT RelationshipType = "DISTRIBUTION_ARTIFACT"
PATCH_FOR RelationshipType = "PATCH_FOR"
PATCH_APPLIED RelationshipType = "PATCH_APPLIED"
COPY_OF RelationshipType = "COPY_OF"
FILE_ADDED RelationshipType = "FILE_ADDED"
FILE_DELETED RelationshipType = "FILE_DELETED"
FILE_MODIFIED RelationshipType = "FILE_MODIFIED"
EXPANDED_FROM_ARCHIVE RelationshipType = "EXPANDED_FROM_ARCHIVE"
DYNAMIC_LINK RelationshipType = "DYNAMIC_LINK"
STATIC_LINK RelationshipType = "STATIC_LINK"
DATA_FILE_OF RelationshipType = "DATA_FILE_OF"
TEST_CASE_OF RelationshipType = "TEST_CASE_OF"
BUILD_TOOL_OF RelationshipType = "BUILD_TOOL_OF"
DEV_TOOL_OF RelationshipType = "DEV_TOOL_OF"
TEST_OF RelationshipType = "TEST_OF"
TEST_TOOL_OF RelationshipType = "TEST_TOOL_OF"
DOCUMENTATION_OF RelationshipType = "DOCUMENTATION_OF"
OPTIONAL_COMPONENT_OF RelationshipType = "OPTIONAL_COMPONENT_OF"
METAFILE_OF RelationshipType = "METAFILE_OF"
PACKAGE_OF RelationshipType = "PACKAGE_OF"
AMENDS RelationshipType = "AMENDS"
PREREQUISITE_FOR RelationshipType = "PREREQUISITE_FOR"
HAS_PREREQUISITE RelationshipType = "HAS_PREREQUISITE"
OTHER RelationshipType = "OTHER"
)

type Relationship struct {
FullRender bool // Flag, then true the package will be rendered in the doc
PeerReference string // SPDX Ref of the peer object. Will override the ID of provided package if set
Comment string // Relationship ship commnet
Type RelationshipType // Relationship of the specified package
Peer Object //
}

func (ro *Relationship) Render(hostObject Object) (string, error) {
if ro.Peer.SPDXID() == "" {
return "", errors.New("unable to render relationship, peer object has no SPDX ID")
}

if hostObject.SPDXID() == "" {
return "", errors.New("Unable to rennder relationship, hostObject has no ID")
}

docFragment := ""
if ro.FullRender {
objDoc, err := ro.Peer.Render()
if err != nil {
return "", errors.Wrapf(err, "rendering related object %s", hostObject.SPDXID())
}
docFragment += objDoc
}
docFragment += fmt.Sprintf("Relationship: %s %s %s\n", hostObject.SPDXID(), ro.Type, ro.Peer.SPDXID())
return docFragment, nil
}
128 changes: 58 additions & 70 deletions pkg/spdx/spdx.go
Original file line number Diff line number Diff line change
@@ -37,7 +37,7 @@ const (
spdxLicenseData = spdxTempDir + "/licenses"
spdxLicenseDlCache = spdxTempDir + "/downloadCache"
gitIgnoreFile = ".gitignore"
validNameCharsRe = `[^a-zA-Z0-9-]+`
validIDCharsRe = `[^a-zA-Z0-9-.]+` // https://spdx.github.io/spdx-spec/3-package-information/#32-package-spdx-identifier

// Consts of some SPDX expressions
NONE = "NONE"
@@ -95,6 +95,42 @@ type TarballOptions struct {
ExtractDir string // Directory where the docker tar archive will be extracted
}

// buildIDString takes a list of seed strings and builds a
// valid SPDX ID string from them. If none is supplied, an
// ID using an UUID will be returned
func buildIDString(seeds ...string) string {
validSeeds := []string{}
numValidSeeds := 0
reg := regexp.MustCompile(validIDCharsRe)
for _, s := range seeds {
// Replace some chars with - to keep the sense of the ID
for _, r := range []string{"/", ":"} {
s = strings.ReplaceAll(s, r, "-")
}
s = reg.ReplaceAllString(s, "")
if s != "" {
validSeeds = append(validSeeds, s)
if !strings.HasPrefix(s, "SPDXRef-") {
numValidSeeds++
}
}
}

// If we did not get any seeds, use an UUID
if numValidSeeds == 0 {
validSeeds = append(validSeeds, uuid.New().String())
}

id := ""
for _, s := range validSeeds {
if id != "" {
id += "-"
}
id += s
}
return id
}

// PackageFromDirectory indexes all files in a directory and builds a
// SPDX package describing its contents
func (spdx *SPDX) PackageFromDirectory(dirPath string) (pkg *Package, err error) {
@@ -137,9 +173,7 @@ func (spdx *SPDX) PackageFromDirectory(dirPath string) (pkg *Package, err error)
pkg = NewPackage()
pkg.FilesAnalyzed = true
pkg.Name = filepath.Base(dirPath)
// If the package file will result in an empty ID, generate one
reg := regexp.MustCompile(validNameCharsRe)
if reg.ReplaceAllString(pkg.Name, "") == "" {
if pkg.Name == "" {
pkg.Name = uuid.NewString()
}
pkg.LicenseConcluded = licenseTag
@@ -203,70 +237,8 @@ func (spdx *SPDX) PackageFromDirectory(dirPath string) (pkg *Package, err error)
}

// PackageFromImageTarball returns a SPDX package from a tarball
func (spdx *SPDX) PackageFromImageTarball(
tarPath string, opts *TarballOptions,
) (imagePackage *Package, err error) {
logrus.Infof("Generating SPDX package from image tarball %s", tarPath)

// Extract all files from tarfile
opts.ExtractDir, err = spdx.impl.ExtractTarballTmp(tarPath)
if err != nil {
return nil, errors.Wrap(err, "extracting tarball to temp dir")
}
defer os.RemoveAll(opts.ExtractDir)

// Read the archive manifest json:
manifest, err := spdx.impl.ReadArchiveManifest(
filepath.Join(opts.ExtractDir, archiveManifestFilename),
)
if err != nil {
return nil, errors.Wrap(err, "while reading docker archive manifest")
}

if len(manifest.RepoTags) == 0 {
return nil, errors.New("No RepoTags found in manifest")
}

if manifest.RepoTags[0] == "" {
return nil, errors.New(
"unable to add tar archive, manifest does not have a RepoTags entry",
)
}

logrus.Infof("Package describes %s image", manifest.RepoTags[0])

// Create the new SPDX package
imagePackage = NewPackage()
imagePackage.Options().WorkDir = opts.ExtractDir
imagePackage.Name = manifest.RepoTags[0]

logrus.Infof("Image manifest lists %d layers", len(manifest.LayerFiles))

// Cycle all the layers from the manifest and add them as packages
for _, layerFile := range manifest.LayerFiles {
// Generate a package from a layer
pkg, err := spdx.impl.PackageFromLayerTarBall(layerFile, opts)
if err != nil {
return nil, errors.Wrap(err, "building package from layer")
}

// If the option is enabled, scan the container layers
if spdx.options.AnalyzeLayers {
if err := spdx.AnalyzeImageLayer(filepath.Join(opts.ExtractDir, layerFile), pkg); err != nil {
return nil, errors.Wrap(err, "scanning layer "+pkg.ID)
}
} else {
logrus.Info("Not performing deep image analysis (opts.AnalyzeLayers = false)")
}

// Add the layer package to the image package
if err := imagePackage.AddPackage(pkg); err != nil {
return nil, errors.Wrap(err, "adding layer to image package")
}
}

// return the finished package
return imagePackage, nil
func (spdx *SPDX) PackageFromImageTarball(tarPath string) (imagePackage *Package, err error) {
return spdx.impl.PackageFromImageTarball(tarPath, spdx.Options())
}

// FileFromPath creates a File object from a path
@@ -285,7 +257,7 @@ func (spdx *SPDX) FileFromPath(filePath string) (*File, error) {
// it matches a known image from which a spdx package can be
// enriched with more information
func (spdx *SPDX) AnalyzeImageLayer(layerPath string, pkg *Package) error {
return NewImageAnalyzer().AnalyzeLayer(layerPath, pkg)
return spdx.impl.AnalyzeImageLayer(layerPath, pkg)
}

// ExtractTarballTmp extracts a tarball to a temp file
@@ -294,6 +266,22 @@ func (spdx *SPDX) ExtractTarballTmp(tarPath string) (tmpDir string, err error) {
}

// PullImagesToArchive
func (spdx *SPDX) PullImagesToArchive(reference, path string) error {
func (spdx *SPDX) PullImagesToArchive(reference, path string) ([]struct {
Reference string
Archive string
Arch string
OS string
}, error) {
return spdx.impl.PullImagesToArchive(reference, path)
}

// ImageRefToPackage gets an image reference (tag or digest) and returns
// a spdx package describing it. It can take two forms:
// - When the reference is a digest (or single image), a single package
// describing the layers is returned
// - When the reference is an image index, the returned package is a
// package referencing each of the images, each in its own packages.
// All subpackages are returned with a relationship of VARIANT_OF
func (spdx *SPDX) ImageRefToPackage(reference string) (pkg *Package, err error) {
return spdx.impl.ImageRefToPackage(reference, spdx.Options())
}
43 changes: 10 additions & 33 deletions pkg/spdx/spdx_test.go
Original file line number Diff line number Diff line change
@@ -25,14 +25,7 @@ import (
"k8s.io/release/pkg/spdx/spdxfakes"
)

var (
err = errors.New("synthetic error")
manifest = &spdx.ArchiveManifest{
ConfigFilename: "9283479287498237498.json",
RepoTags: []string{"image-test:latest"},
LayerFiles: []string{"ksjdhfkjsdhfkjsdhf/layer.tar"},
}
)
var err = errors.New("synthetic error")

func TestPackageFromImageTarball(t *testing.T) {
for _, tc := range []struct {
@@ -41,29 +34,13 @@ func TestPackageFromImageTarball(t *testing.T) {
}{
{ // success
prepare: func(mock *spdxfakes.FakeSpdxImplementation) {
mock.ExtractTarballTmpReturns("/mock/path", nil)
mock.ReadArchiveManifestReturns(manifest, nil)
mock.PackageFromLayerTarBallReturns(&spdx.Package{Name: "test"}, nil)
mock.PackageFromImageTarballReturns(&spdx.Package{Entity: spdx.Entity{Name: "test"}}, nil)
},
shouldError: false,
},
{
prepare: func(mock *spdxfakes.FakeSpdxImplementation) {
mock.ReadArchiveManifestReturns(manifest, nil)
mock.ExtractTarballTmpReturns("", err)
},
shouldError: true,
},
{
prepare: func(mock *spdxfakes.FakeSpdxImplementation) {
mock.ReadArchiveManifestReturns(nil, err)
},
shouldError: true,
},
{
{ // PackageFromImageTarball fails
prepare: func(mock *spdxfakes.FakeSpdxImplementation) {
mock.ReadArchiveManifestReturns(manifest, nil)
mock.PackageFromLayerTarBallReturns(nil, err)
mock.PackageFromImageTarballReturns(nil, err)
},
shouldError: true,
},
@@ -73,13 +50,13 @@ func TestPackageFromImageTarball(t *testing.T) {
mock := &spdxfakes.FakeSpdxImplementation{}
tc.prepare(mock)
sut.SetImplementation(mock)

dir, err := sut.PackageFromImageTarball("mock.tar", &spdx.TarballOptions{})
// Run the test function
pkg, err := sut.PackageFromImageTarball("mock.tar")
if tc.shouldError {
require.NotNil(t, err)
} else {
require.Nil(t, err)
require.NotNil(t, dir)
require.NotNil(t, pkg)
}
}
}
@@ -124,13 +101,13 @@ func TestPullImagesToArchive(t *testing.T) {
}{
{ // success
prepare: func(mock *spdxfakes.FakeSpdxImplementation) {
mock.PullImagesToArchiveReturns(nil)
mock.PullImagesToArchiveReturns(nil, nil)
},
shouldError: false,
},
{ // success
prepare: func(mock *spdxfakes.FakeSpdxImplementation) {
mock.PullImagesToArchiveReturns(err)
mock.PullImagesToArchiveReturns(nil, err)
},
shouldError: true,
},
@@ -141,7 +118,7 @@ func TestPullImagesToArchive(t *testing.T) {
tc.prepare(mock)
sut.SetImplementation(mock)

err := sut.PullImagesToArchive("mock-image:latest", "/tmp")
_, err := sut.PullImagesToArchive("mock-image:latest", "/tmp")
if tc.shouldError {
require.NotNil(t, err)
} else {
29 changes: 27 additions & 2 deletions pkg/spdx/spdx_unit_test.go
Original file line number Diff line number Diff line change
@@ -29,6 +29,31 @@ import (
"sigs.k8s.io/release-utils/util"
)

func TestBuildIDString(t *testing.T) {
cases := []struct {
seeds []string
expected string
}{
{[]string{"1234"}, "1234"},
{[]string{"abc"}, "abc"},
{[]string{"ABC"}, "ABC"},
{[]string{"ABC", "123"}, "ABC-123"},
{[]string{"Hello:bye", "123"}, "Hello-bye-123"},
{[]string{"Hello^bye", "123"}, "Hellobye-123"},
{[]string{"Hello:bye", "123", "&^%&$"}, "Hello-bye-123"},
}
for _, tc := range cases {
require.Equal(t, tc.expected, buildIDString(tc.seeds...))
}

// If we do not pass any seeds, func should return an UUID
// which is 36 chars long
require.Len(t, buildIDString(), 36)

// Same thing for only invalid chars
require.Len(t, buildIDString("&^$&^%"), 36)
}

func TestUnitExtractTarballTmp(t *testing.T) {
tar := writeTestTarball(t)
require.NotNil(t, tar)
@@ -82,9 +107,9 @@ func TestPackageFromLayerTarBall(t *testing.T) {
defer os.Remove(tar.Name())

sut := spdxDefaultImplementation{}
_, err := sut.PackageFromLayerTarBall("lsdkjflksdjflk", &TarballOptions{})
_, err := sut.PackageFromLayerTarball("lsdkjflksdjflk", &TarballOptions{})
require.NotNil(t, err)
pkg, err := sut.PackageFromLayerTarBall(tar.Name(), &TarballOptions{})
pkg, err := sut.PackageFromLayerTarball(tar.Name(), &TarballOptions{})
require.Nil(t, err)
require.NotNil(t, pkg)

415 changes: 362 additions & 53 deletions pkg/spdx/spdxfakes/fake_spdx_implementation.go

Large diffs are not rendered by default.

0 comments on commit 6e3b30b

Please sign in to comment.