From 4edb72f7e54219d28fa72cfa0ffdaf5fad8b7310 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adolfo=20Garc=C3=ADa=20Veytia=20=28Puerco=29?= Date: Sat, 2 Oct 2021 02:21:10 -0500 Subject: [PATCH 01/12] Add startTime to anago state MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The anago state now registers the beggining of stage/release as it is needed to generate the provenance metadata. Signed-off-by: Adolfo García Veytia (Puerco) --- pkg/anago/anago.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pkg/anago/anago.go b/pkg/anago/anago.go index f178c8ed826..34ee8d8f19d 100644 --- a/pkg/anago/anago.go +++ b/pkg/anago/anago.go @@ -18,6 +18,7 @@ package anago import ( "fmt" + "time" "github.com/blang/semver" "github.com/pkg/errors" @@ -151,13 +152,18 @@ type State struct { // Indicates if we're going to create a new release branch. createReleaseBranch bool + + // startTime is the time when stage/release starts + startTime time.Time } // DefaultState returns a new empty State func DefaultState() *State { // The default state is empty, it will be initialized after ValidateOptions() - // runs in Stage/Releas. It will change as the satege/release processes move forward - return &State{} + // runs in Stage/Release. It will change as the satege/release processes move forward + return &State{ + startTime: time.Now(), + } } func (s *State) SetCreateReleaseBranch(createReleaseBranch bool) { From 566e81e7ab4d716ed11da632b9c8cfee138e2c3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adolfo=20Garc=C3=ADa=20Veytia=20=28Puerco=29?= Date: Sat, 2 Oct 2021 22:12:10 -0500 Subject: [PATCH 02/12] Add provenance file const MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a constant for the provenance filename `provenance.json` Signed-off-by: Adolfo García Veytia (Puerco) --- pkg/release/release.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/release/release.go b/pkg/release/release.go index 425a00140aa..1817df64327 100644 --- a/pkg/release/release.go +++ b/pkg/release/release.go @@ -128,6 +128,8 @@ const ( DockerHubEnvKey = "DOCKERHUB_TOKEN" // Env var containing the docker key DockerHubUserName = "k8sreleng" // Docker Hub username + + ProvenanceFilename = "provenance.json" // Name of the SLSA provenance file (used in stage and release) ) var ( From dc93a10571e702c9ae1369f5c763d5464b9dda57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adolfo=20Garc=C3=ADa=20Veytia=20=28Puerco=29?= Date: Sat, 2 Oct 2021 22:36:06 -0500 Subject: [PATCH 03/12] Add BUILD_ID to GCB environment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We now make the GCB BUILD_ID identifier available to krel as an envvar to include it in the provenance metadata. Signed-off-by: Adolfo García Veytia (Puerco) --- gcb/stage/cloudbuild.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/gcb/stage/cloudbuild.yaml b/gcb/stage/cloudbuild.yaml index fdd9b822f14..5cbdf1da935 100644 --- a/gcb/stage/cloudbuild.yaml +++ b/gcb/stage/cloudbuild.yaml @@ -49,6 +49,7 @@ steps: - "TOOL_ORG=${_TOOL_ORG}" - "TOOL_REPO=${_TOOL_REPO}" - "TOOL_REF=${_TOOL_REF}" + - "BUILD_ID=${BUILD_ID}" secretEnv: - GITHUB_TOKEN - DOCKERHUB_TOKEN From b817a96e7e044510e519932b1b57e6d2d3266845 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adolfo=20Garc=C3=ADa=20Veytia=20=28Puerco=29?= Date: Sat, 2 Oct 2021 22:38:33 -0500 Subject: [PATCH 04/12] go pkg: provenance MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit adds a new go package: provenance This code contains the types and functions to generate provenance metadata. It currently includes code to generate SLSA compliant predicates and automatically scan direcories to add files as subjects in a provenance attestation. Signing is currently not supported but the envelope type is already in the package for the next itearation, the push towards SLSA2 in the Kubernetes release process. Signed-off-by: Adolfo García Veytia (Puerco) --- go.mod | 7 +- go.sum | 9 +- pkg/provenance/predicate.go | 76 ++++++++++++++++ pkg/provenance/provenance.go | 70 ++++++++++++++ pkg/provenance/statement.go | 172 +++++++++++++++++++++++++++++++++++ 5 files changed, 332 insertions(+), 2 deletions(-) create mode 100644 pkg/provenance/predicate.go create mode 100644 pkg/provenance/provenance.go create mode 100644 pkg/provenance/statement.go diff --git a/go.mod b/go.mod index 0ae56c9edce..baaaa5100ac 100644 --- a/go.mod +++ b/go.mod @@ -81,6 +81,7 @@ require ( github.com/hashicorp/go-multierror v1.0.0 // indirect github.com/hashicorp/go-retryablehttp v0.6.4 // indirect github.com/imdario/mergo v0.3.12 // indirect + github.com/in-toto/in-toto-golang v0.3.2 github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect @@ -104,7 +105,6 @@ require ( golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e // indirect golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f // indirect golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect - golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf // indirect golang.org/x/text v0.3.6 // indirect golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect @@ -116,3 +116,8 @@ require ( gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) + +require ( + github.com/shibumi/go-pathspec v1.2.0 // indirect + golang.org/x/sys v0.0.0-20210909193231-528a39cd75f3 // indirect +) diff --git a/go.sum b/go.sum index 08010ade51a..263171ea18f 100644 --- a/go.sum +++ b/go.sum @@ -170,6 +170,8 @@ github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnht github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb h1:EDmT6Q9Zs+SbUoc7Ik9EfrFqcylYqgPZ9ANSbTAntnE= +github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb/go.mod h1:ZjrT6AXHbDs86ZSdt/osfBi5qfexBrKUdONk989Wnk4= github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE= github.com/containerd/aufs v0.0.0-20201003224125-76a6863f2989/go.mod h1:AkGGQs9NM2vtYHaUen+NljV0/baGCAPELGm2q9ZXpWU= github.com/containerd/aufs v0.0.0-20210316121734-20793ff83c97/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU= @@ -578,6 +580,8 @@ github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/in-toto/in-toto-golang v0.3.2 h1:8qaEsqLzRpdV+XPA1nFCWI2hrE9x+og7QwXhyfOxhVA= +github.com/in-toto/in-toto-golang v0.3.2/go.mod h1:xhKHGL6hqxBTdADHOnoxyhY5AiKuXfTtN+8SUs7LHTE= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA= @@ -850,6 +854,8 @@ github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAm github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/shibumi/go-pathspec v1.2.0 h1:KVKEDHYk7bQolRMs7nfzjT3SBOCgcXFJzccnj9bsGbA= +github.com/shibumi/go-pathspec v1.2.0/go.mod h1:bDxCftD0fST3qXIlHoQ/fChsU4mWMVklXp1yPErQaaY= github.com/shirou/gopsutil v0.0.0-20190901111213-e4ec7b275ada h1:WokF3GuxBeL+n4Lk4Fa8v9mbdjlrl7bHuneF4N1bk2I= github.com/shirou/gopsutil v0.0.0-20190901111213-e4ec7b275ada/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc= github.com/shirou/gopsutil/v3 v3.21.8 h1:nKct+uP0TV8DjjNiHanKf8SAuub+GNsbrOtM9Nl9biA= @@ -1247,8 +1253,9 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf h1:2ucpDCmfkl8Bd/FsLtiD653Wf96cW37s+iGx93zsu4k= golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210909193231-528a39cd75f3 h1:3Ad41xy2WCESpufXwgs7NpDSu+vjxqLt2UFqUV+20bI= +golang.org/x/sys v0.0.0-20210909193231-528a39cd75f3/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/pkg/provenance/predicate.go b/pkg/provenance/predicate.go new file mode 100644 index 00000000000..1c4ce5cd3d9 --- /dev/null +++ b/pkg/provenance/predicate.go @@ -0,0 +1,76 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package provenance + +import ( + "encoding/json" + "os" + + intoto "github.com/in-toto/in-toto-golang/in_toto" + "github.com/pkg/errors" +) + +type Predicate struct { + intoto.ProvenancePredicate + impl PredicateImplementation +} + +// setImplementation sets the predicate implementation +func (p *Predicate) SetImplementation(impl PredicateImplementation) { + p.impl = impl +} + +// AddMaterial adds an entry to the listo of materials +func (p *Predicate) AddMaterial(uri string, ds intoto.DigestSet) { + p.impl.AddMaterial(p, uri, ds) +} + +// Write outputs the predicate as JSON to a file +func (p *Predicate) Write(path string) error { + return p.impl.Write(p, path) +} + +//counterfeiter:generate . PredicateImplementation +type PredicateImplementation interface { + AddMaterial(*Predicate, string, intoto.DigestSet) + Write(*Predicate, string) error +} + +type defaultPredicateImplementation struct{} + +// Write dumps the predicate data into a JSON file +func (pi *defaultPredicateImplementation) Write(p *Predicate, path string) error { + jsonData, err := json.Marshal(p) + if err != nil { + return errors.Wrap(err, "marshalling predicate to json") + } + return errors.Wrap( + os.WriteFile(path, jsonData, os.FileMode(0o644)), + "writing predicate file", + ) +} + +// AddMaterial adds a material to the entry +func (pi *defaultPredicateImplementation) AddMaterial(p *Predicate, uri string, ds intoto.DigestSet) { + if p.Materials == nil { + p.Materials = []intoto.ProvenanceMaterial{} + } + p.Materials = append(p.Materials, intoto.ProvenanceMaterial{ + URI: uri, + Digest: ds, + }) +} diff --git a/pkg/provenance/provenance.go b/pkg/provenance/provenance.go new file mode 100644 index 00000000000..4df8c697a06 --- /dev/null +++ b/pkg/provenance/provenance.go @@ -0,0 +1,70 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -generate + +package provenance + +import ( + intoto "github.com/in-toto/in-toto-golang/in_toto" +) + +// NewStatement creates a new attestation +func NewSLSAStatement() *Statement { + return &Statement{ + + StatementHeader: intoto.StatementHeader{ + Type: intoto.StatementInTotoV01, + PredicateType: intoto.PredicateSLSAProvenanceV01, + Subject: []intoto.Subject{}, + }, + Predicate: NewSLSAPredicate(), + + impl: &defaultStatementImplementation{}, + } +} + +// NewSLSAPredicate returns a new SLSA provenance predicate +func NewSLSAPredicate() Predicate { + return Predicate{ + ProvenancePredicate: intoto.ProvenancePredicate{ + Builder: intoto.ProvenanceBuilder{ + ID: "", + }, + Recipe: intoto.ProvenanceRecipe{ + Type: "", + DefinedInMaterial: new(int), + EntryPoint: "", + Arguments: nil, + Environment: nil, + }, + Metadata: &intoto.ProvenanceMetadata{ + Completeness: intoto.ProvenanceComplete{}, + }, + Materials: []intoto.ProvenanceMaterial{}, + }, + impl: &defaultPredicateImplementation{}, + } +} + +// Envelope is the outermost layer of the attestation, handling authentication and +// serialization. The format and protocol are defined in DSSE and adopted by in-toto in ITE-5. +// https://github.com/in-toto/attestation/blob/main/spec/README.md#envelope +type Envelope struct { + PayloadType string `json:"payloadType"` + Payload string `json:"payload"` + Signatures []interface{} `json:"signatures"` +} diff --git a/pkg/provenance/statement.go b/pkg/provenance/statement.go new file mode 100644 index 00000000000..ed1eddf740f --- /dev/null +++ b/pkg/provenance/statement.go @@ -0,0 +1,172 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* + Useful links: + SLSA Provenance predicate spec: https://slsa.dev/provenance/v0.1 + In-toto attestation spec: https://github.com/in-toto/attestation/blob/main/spec/README.md +*/ + +package provenance + +import ( + "encoding/json" + "io/fs" + "os" + "path/filepath" + + "github.com/pkg/errors" + "sigs.k8s.io/release-utils/hash" + + intoto "github.com/in-toto/in-toto-golang/in_toto" +) + +// Statement is the middle layer of the attestation, binding it to a particular subject and +// unambiguously identifying the types of the predicate. +// https://github.com/in-toto/attestation/blob/main/spec/README.md#statement +type Statement struct { + intoto.StatementHeader + Predicate Predicate `json:"predicate"` + impl StatementImplementation +} + +func (s *Statement) SetImplementation(si StatementImplementation) { + s.impl = si +} + +// Write outputs the predicate as JSON to a file +func (s *Statement) Write(path string) error { + return s.impl.Write(s, path) +} + +// ReadSubjectsFromDir reads a directory and adds every file as a subject +// to the statement. +func (s *Statement) ReadSubjectsFromDir(path string) (err error) { + return s.impl.ReadSubjectsFromDir(s, path) +} + +// AddSubject adds an entry to the listo of materials +func (s *Statement) AddSubject(uri string, ds intoto.DigestSet) { + s.impl.AddSubject(s, uri, ds) +} + +// AddSubjectFromFile adds a subject to the list by checking a file in the filesystem +func (s *Statement) AddSubjectFromFile(filePath string) error { + subject, err := s.impl.SubjectFromFile(filePath) + if err != nil { + return errors.Wrapf(err, "creating subject from file %s", filePath) + } + s.impl.AddSubject(s, subject.Name, subject.Digest) + return nil +} + +// LoadPredicate loads a predicate from a json file +func (s *Statement) LoadPredicate(path string) error { + data, err := os.ReadFile(path) + if err != nil { + return errors.Wrap(err, "opening predicate file") + } + p := Predicate{} + if err := json.Unmarshal(data, &p); err != nil { + return errors.Wrap(err, "unmarshalling predicate json") + } + s.Predicate = p + return nil +} + +//counterfeiter:generate . StatementImplementation +type StatementImplementation interface { + AddSubject(*Statement, string, intoto.DigestSet) + ReadSubjectsFromDir(*Statement, string) error + SubjectFromFile(string) (intoto.Subject, error) + Write(*Statement, string) error +} + +type defaultStatementImplementation struct{} + +// AddSubject adds a material to the entry +func (si *defaultStatementImplementation) AddSubject( + s *Statement, name string, ds intoto.DigestSet, +) { + if s.Subject == nil { + s.Subject = []intoto.Subject{} + } + s.Subject = append(s.Subject, intoto.Subject{ + Name: name, + Digest: ds, + }) +} + +// ReadSubjectsFromDir reads a directory and adds all files found as +// subjects of the statement. +func (si *defaultStatementImplementation) ReadSubjectsFromDir( + s *Statement, dirPath string) (err error) { + // Traverse the directory + if err := fs.WalkDir(os.DirFS(dirPath), ".", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + + if d.Type() == os.ModeSymlink { + return nil + } + hashVal, err := hash.SHA256ForFile(filepath.Join(dirPath, path)) + if err != nil { + return errors.Wrapf(err, "hashing file %s", path) + } + s.AddSubject(path, intoto.DigestSet{"sha256": hashVal}) + return nil + }); err != nil { + return errors.Wrap(err, "buiding directory tree") + } + return nil +} + +// SubjectFromFile reads a file and return an in-toto subject describing it +func (si *defaultStatementImplementation) SubjectFromFile(filePath string) (subject intoto.Subject, err error) { + subject = intoto.Subject{ + Name: filePath, + Digest: map[string]string{}, + } + h256, err := hash.SHA256ForFile(filePath) + if err != nil { + return subject, errors.Wrapf(err, "getting sha256 for file %s", filePath) + } + h512, err := hash.SHA512ForFile(filePath) + if err != nil { + return subject, errors.Wrapf(err, "getting sha512 for %s", filePath) + } + subject.Digest = map[string]string{ + "sha256": h256, + "sha512": h512, + } + return subject, nil +} + +// Write dumps the statement data to disk in json +func (si *defaultStatementImplementation) Write(s *Statement, path string) error { + jsonData, err := json.Marshal(s) + if err != nil { + return errors.Wrap(err, "marshalling statement to json") + } + return errors.Wrap( + os.WriteFile(path, jsonData, os.FileMode(0o644)), + "writing predicate file", + ) +} From d1938d6a83e430cc07f5166dd9e2ff11269e811e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adolfo=20Garc=C3=ADa=20Veytia=20=28Puerco=29?= Date: Sat, 2 Oct 2021 22:43:09 -0500 Subject: [PATCH 05/12] generate: provenance fakes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit counterfeiter provenance fakes for tests Signed-off-by: Adolfo García Veytia (Puerco) --- .../fake_predicate_implementation.go | 157 +++++++++ .../fake_statement_implementation.go | 312 ++++++++++++++++++ 2 files changed, 469 insertions(+) create mode 100644 pkg/provenance/provenancefakes/fake_predicate_implementation.go create mode 100644 pkg/provenance/provenancefakes/fake_statement_implementation.go diff --git a/pkg/provenance/provenancefakes/fake_predicate_implementation.go b/pkg/provenance/provenancefakes/fake_predicate_implementation.go new file mode 100644 index 00000000000..495e99d7f58 --- /dev/null +++ b/pkg/provenance/provenancefakes/fake_predicate_implementation.go @@ -0,0 +1,157 @@ +// Code generated by counterfeiter. DO NOT EDIT. +package provenancefakes + +import ( + "sync" + + "github.com/in-toto/in-toto-golang/in_toto" + "k8s.io/release/pkg/provenance" +) + +type FakePredicateImplementation struct { + AddMaterialStub func(*provenance.Predicate, string, in_toto.DigestSet) + addMaterialMutex sync.RWMutex + addMaterialArgsForCall []struct { + arg1 *provenance.Predicate + arg2 string + arg3 in_toto.DigestSet + } + WriteStub func(*provenance.Predicate, string) error + writeMutex sync.RWMutex + writeArgsForCall []struct { + arg1 *provenance.Predicate + arg2 string + } + writeReturns struct { + result1 error + } + writeReturnsOnCall map[int]struct { + result1 error + } + invocations map[string][][]interface{} + invocationsMutex sync.RWMutex +} + +func (fake *FakePredicateImplementation) AddMaterial(arg1 *provenance.Predicate, arg2 string, arg3 in_toto.DigestSet) { + fake.addMaterialMutex.Lock() + fake.addMaterialArgsForCall = append(fake.addMaterialArgsForCall, struct { + arg1 *provenance.Predicate + arg2 string + arg3 in_toto.DigestSet + }{arg1, arg2, arg3}) + stub := fake.AddMaterialStub + fake.recordInvocation("AddMaterial", []interface{}{arg1, arg2, arg3}) + fake.addMaterialMutex.Unlock() + if stub != nil { + fake.AddMaterialStub(arg1, arg2, arg3) + } +} + +func (fake *FakePredicateImplementation) AddMaterialCallCount() int { + fake.addMaterialMutex.RLock() + defer fake.addMaterialMutex.RUnlock() + return len(fake.addMaterialArgsForCall) +} + +func (fake *FakePredicateImplementation) AddMaterialCalls(stub func(*provenance.Predicate, string, in_toto.DigestSet)) { + fake.addMaterialMutex.Lock() + defer fake.addMaterialMutex.Unlock() + fake.AddMaterialStub = stub +} + +func (fake *FakePredicateImplementation) AddMaterialArgsForCall(i int) (*provenance.Predicate, string, in_toto.DigestSet) { + fake.addMaterialMutex.RLock() + defer fake.addMaterialMutex.RUnlock() + argsForCall := fake.addMaterialArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 +} + +func (fake *FakePredicateImplementation) Write(arg1 *provenance.Predicate, arg2 string) error { + fake.writeMutex.Lock() + ret, specificReturn := fake.writeReturnsOnCall[len(fake.writeArgsForCall)] + fake.writeArgsForCall = append(fake.writeArgsForCall, struct { + arg1 *provenance.Predicate + arg2 string + }{arg1, arg2}) + stub := fake.WriteStub + fakeReturns := fake.writeReturns + fake.recordInvocation("Write", []interface{}{arg1, arg2}) + fake.writeMutex.Unlock() + if stub != nil { + return stub(arg1, arg2) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakePredicateImplementation) WriteCallCount() int { + fake.writeMutex.RLock() + defer fake.writeMutex.RUnlock() + return len(fake.writeArgsForCall) +} + +func (fake *FakePredicateImplementation) WriteCalls(stub func(*provenance.Predicate, string) error) { + fake.writeMutex.Lock() + defer fake.writeMutex.Unlock() + fake.WriteStub = stub +} + +func (fake *FakePredicateImplementation) WriteArgsForCall(i int) (*provenance.Predicate, string) { + fake.writeMutex.RLock() + defer fake.writeMutex.RUnlock() + argsForCall := fake.writeArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakePredicateImplementation) WriteReturns(result1 error) { + fake.writeMutex.Lock() + defer fake.writeMutex.Unlock() + fake.WriteStub = nil + fake.writeReturns = struct { + result1 error + }{result1} +} + +func (fake *FakePredicateImplementation) WriteReturnsOnCall(i int, result1 error) { + fake.writeMutex.Lock() + defer fake.writeMutex.Unlock() + fake.WriteStub = nil + if fake.writeReturnsOnCall == nil { + fake.writeReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.writeReturnsOnCall[i] = struct { + result1 error + }{result1} +} + +func (fake *FakePredicateImplementation) Invocations() map[string][][]interface{} { + fake.invocationsMutex.RLock() + defer fake.invocationsMutex.RUnlock() + fake.addMaterialMutex.RLock() + defer fake.addMaterialMutex.RUnlock() + fake.writeMutex.RLock() + defer fake.writeMutex.RUnlock() + copiedInvocations := map[string][][]interface{}{} + for key, value := range fake.invocations { + copiedInvocations[key] = value + } + return copiedInvocations +} + +func (fake *FakePredicateImplementation) recordInvocation(key string, args []interface{}) { + fake.invocationsMutex.Lock() + defer fake.invocationsMutex.Unlock() + if fake.invocations == nil { + fake.invocations = map[string][][]interface{}{} + } + if fake.invocations[key] == nil { + fake.invocations[key] = [][]interface{}{} + } + fake.invocations[key] = append(fake.invocations[key], args) +} + +var _ provenance.PredicateImplementation = new(FakePredicateImplementation) diff --git a/pkg/provenance/provenancefakes/fake_statement_implementation.go b/pkg/provenance/provenancefakes/fake_statement_implementation.go new file mode 100644 index 00000000000..76bfb3b7683 --- /dev/null +++ b/pkg/provenance/provenancefakes/fake_statement_implementation.go @@ -0,0 +1,312 @@ +// Code generated by counterfeiter. DO NOT EDIT. +package provenancefakes + +import ( + "sync" + + "github.com/in-toto/in-toto-golang/in_toto" + "k8s.io/release/pkg/provenance" +) + +type FakeStatementImplementation struct { + AddSubjectStub func(*provenance.Statement, string, in_toto.DigestSet) + addSubjectMutex sync.RWMutex + addSubjectArgsForCall []struct { + arg1 *provenance.Statement + arg2 string + arg3 in_toto.DigestSet + } + ReadSubjectsFromDirStub func(*provenance.Statement, string) error + readSubjectsFromDirMutex sync.RWMutex + readSubjectsFromDirArgsForCall []struct { + arg1 *provenance.Statement + arg2 string + } + readSubjectsFromDirReturns struct { + result1 error + } + readSubjectsFromDirReturnsOnCall map[int]struct { + result1 error + } + SubjectFromFileStub func(string) (in_toto.Subject, error) + subjectFromFileMutex sync.RWMutex + subjectFromFileArgsForCall []struct { + arg1 string + } + subjectFromFileReturns struct { + result1 in_toto.Subject + result2 error + } + subjectFromFileReturnsOnCall map[int]struct { + result1 in_toto.Subject + result2 error + } + WriteStub func(*provenance.Statement, string) error + writeMutex sync.RWMutex + writeArgsForCall []struct { + arg1 *provenance.Statement + arg2 string + } + writeReturns struct { + result1 error + } + writeReturnsOnCall map[int]struct { + result1 error + } + invocations map[string][][]interface{} + invocationsMutex sync.RWMutex +} + +func (fake *FakeStatementImplementation) AddSubject(arg1 *provenance.Statement, arg2 string, arg3 in_toto.DigestSet) { + fake.addSubjectMutex.Lock() + fake.addSubjectArgsForCall = append(fake.addSubjectArgsForCall, struct { + arg1 *provenance.Statement + arg2 string + arg3 in_toto.DigestSet + }{arg1, arg2, arg3}) + stub := fake.AddSubjectStub + fake.recordInvocation("AddSubject", []interface{}{arg1, arg2, arg3}) + fake.addSubjectMutex.Unlock() + if stub != nil { + fake.AddSubjectStub(arg1, arg2, arg3) + } +} + +func (fake *FakeStatementImplementation) AddSubjectCallCount() int { + fake.addSubjectMutex.RLock() + defer fake.addSubjectMutex.RUnlock() + return len(fake.addSubjectArgsForCall) +} + +func (fake *FakeStatementImplementation) AddSubjectCalls(stub func(*provenance.Statement, string, in_toto.DigestSet)) { + fake.addSubjectMutex.Lock() + defer fake.addSubjectMutex.Unlock() + fake.AddSubjectStub = stub +} + +func (fake *FakeStatementImplementation) AddSubjectArgsForCall(i int) (*provenance.Statement, string, in_toto.DigestSet) { + fake.addSubjectMutex.RLock() + defer fake.addSubjectMutex.RUnlock() + argsForCall := fake.addSubjectArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 +} + +func (fake *FakeStatementImplementation) ReadSubjectsFromDir(arg1 *provenance.Statement, arg2 string) error { + fake.readSubjectsFromDirMutex.Lock() + ret, specificReturn := fake.readSubjectsFromDirReturnsOnCall[len(fake.readSubjectsFromDirArgsForCall)] + fake.readSubjectsFromDirArgsForCall = append(fake.readSubjectsFromDirArgsForCall, struct { + arg1 *provenance.Statement + arg2 string + }{arg1, arg2}) + stub := fake.ReadSubjectsFromDirStub + fakeReturns := fake.readSubjectsFromDirReturns + fake.recordInvocation("ReadSubjectsFromDir", []interface{}{arg1, arg2}) + fake.readSubjectsFromDirMutex.Unlock() + if stub != nil { + return stub(arg1, arg2) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeStatementImplementation) ReadSubjectsFromDirCallCount() int { + fake.readSubjectsFromDirMutex.RLock() + defer fake.readSubjectsFromDirMutex.RUnlock() + return len(fake.readSubjectsFromDirArgsForCall) +} + +func (fake *FakeStatementImplementation) ReadSubjectsFromDirCalls(stub func(*provenance.Statement, string) error) { + fake.readSubjectsFromDirMutex.Lock() + defer fake.readSubjectsFromDirMutex.Unlock() + fake.ReadSubjectsFromDirStub = stub +} + +func (fake *FakeStatementImplementation) ReadSubjectsFromDirArgsForCall(i int) (*provenance.Statement, string) { + fake.readSubjectsFromDirMutex.RLock() + defer fake.readSubjectsFromDirMutex.RUnlock() + argsForCall := fake.readSubjectsFromDirArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeStatementImplementation) ReadSubjectsFromDirReturns(result1 error) { + fake.readSubjectsFromDirMutex.Lock() + defer fake.readSubjectsFromDirMutex.Unlock() + fake.ReadSubjectsFromDirStub = nil + fake.readSubjectsFromDirReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeStatementImplementation) ReadSubjectsFromDirReturnsOnCall(i int, result1 error) { + fake.readSubjectsFromDirMutex.Lock() + defer fake.readSubjectsFromDirMutex.Unlock() + fake.ReadSubjectsFromDirStub = nil + if fake.readSubjectsFromDirReturnsOnCall == nil { + fake.readSubjectsFromDirReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.readSubjectsFromDirReturnsOnCall[i] = struct { + result1 error + }{result1} +} + +func (fake *FakeStatementImplementation) SubjectFromFile(arg1 string) (in_toto.Subject, error) { + fake.subjectFromFileMutex.Lock() + ret, specificReturn := fake.subjectFromFileReturnsOnCall[len(fake.subjectFromFileArgsForCall)] + fake.subjectFromFileArgsForCall = append(fake.subjectFromFileArgsForCall, struct { + arg1 string + }{arg1}) + stub := fake.SubjectFromFileStub + fakeReturns := fake.subjectFromFileReturns + fake.recordInvocation("SubjectFromFile", []interface{}{arg1}) + fake.subjectFromFileMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeStatementImplementation) SubjectFromFileCallCount() int { + fake.subjectFromFileMutex.RLock() + defer fake.subjectFromFileMutex.RUnlock() + return len(fake.subjectFromFileArgsForCall) +} + +func (fake *FakeStatementImplementation) SubjectFromFileCalls(stub func(string) (in_toto.Subject, error)) { + fake.subjectFromFileMutex.Lock() + defer fake.subjectFromFileMutex.Unlock() + fake.SubjectFromFileStub = stub +} + +func (fake *FakeStatementImplementation) SubjectFromFileArgsForCall(i int) string { + fake.subjectFromFileMutex.RLock() + defer fake.subjectFromFileMutex.RUnlock() + argsForCall := fake.subjectFromFileArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeStatementImplementation) SubjectFromFileReturns(result1 in_toto.Subject, result2 error) { + fake.subjectFromFileMutex.Lock() + defer fake.subjectFromFileMutex.Unlock() + fake.SubjectFromFileStub = nil + fake.subjectFromFileReturns = struct { + result1 in_toto.Subject + result2 error + }{result1, result2} +} + +func (fake *FakeStatementImplementation) SubjectFromFileReturnsOnCall(i int, result1 in_toto.Subject, result2 error) { + fake.subjectFromFileMutex.Lock() + defer fake.subjectFromFileMutex.Unlock() + fake.SubjectFromFileStub = nil + if fake.subjectFromFileReturnsOnCall == nil { + fake.subjectFromFileReturnsOnCall = make(map[int]struct { + result1 in_toto.Subject + result2 error + }) + } + fake.subjectFromFileReturnsOnCall[i] = struct { + result1 in_toto.Subject + result2 error + }{result1, result2} +} + +func (fake *FakeStatementImplementation) Write(arg1 *provenance.Statement, arg2 string) error { + fake.writeMutex.Lock() + ret, specificReturn := fake.writeReturnsOnCall[len(fake.writeArgsForCall)] + fake.writeArgsForCall = append(fake.writeArgsForCall, struct { + arg1 *provenance.Statement + arg2 string + }{arg1, arg2}) + stub := fake.WriteStub + fakeReturns := fake.writeReturns + fake.recordInvocation("Write", []interface{}{arg1, arg2}) + fake.writeMutex.Unlock() + if stub != nil { + return stub(arg1, arg2) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeStatementImplementation) WriteCallCount() int { + fake.writeMutex.RLock() + defer fake.writeMutex.RUnlock() + return len(fake.writeArgsForCall) +} + +func (fake *FakeStatementImplementation) WriteCalls(stub func(*provenance.Statement, string) error) { + fake.writeMutex.Lock() + defer fake.writeMutex.Unlock() + fake.WriteStub = stub +} + +func (fake *FakeStatementImplementation) WriteArgsForCall(i int) (*provenance.Statement, string) { + fake.writeMutex.RLock() + defer fake.writeMutex.RUnlock() + argsForCall := fake.writeArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeStatementImplementation) WriteReturns(result1 error) { + fake.writeMutex.Lock() + defer fake.writeMutex.Unlock() + fake.WriteStub = nil + fake.writeReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeStatementImplementation) WriteReturnsOnCall(i int, result1 error) { + fake.writeMutex.Lock() + defer fake.writeMutex.Unlock() + fake.WriteStub = nil + if fake.writeReturnsOnCall == nil { + fake.writeReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.writeReturnsOnCall[i] = struct { + result1 error + }{result1} +} + +func (fake *FakeStatementImplementation) Invocations() map[string][][]interface{} { + fake.invocationsMutex.RLock() + defer fake.invocationsMutex.RUnlock() + fake.addSubjectMutex.RLock() + defer fake.addSubjectMutex.RUnlock() + fake.readSubjectsFromDirMutex.RLock() + defer fake.readSubjectsFromDirMutex.RUnlock() + fake.subjectFromFileMutex.RLock() + defer fake.subjectFromFileMutex.RUnlock() + fake.writeMutex.RLock() + defer fake.writeMutex.RUnlock() + copiedInvocations := map[string][][]interface{}{} + for key, value := range fake.invocations { + copiedInvocations[key] = value + } + return copiedInvocations +} + +func (fake *FakeStatementImplementation) recordInvocation(key string, args []interface{}) { + fake.invocationsMutex.Lock() + defer fake.invocationsMutex.Unlock() + if fake.invocations == nil { + fake.invocations = map[string][][]interface{}{} + } + if fake.invocations[key] == nil { + fake.invocations[key] = [][]interface{}{} + } + fake.invocations[key] = append(fake.invocations[key], args) +} + +var _ provenance.StatementImplementation = new(FakeStatementImplementation) From d5366eae1bd7b53e66776677e974261930c28cc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adolfo=20Garc=C3=ADa=20Veytia=20=28Puerco=29?= Date: Sat, 2 Oct 2021 22:41:28 -0500 Subject: [PATCH 06/12] pkg/provenance tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commits adds testing for the provenance package. It depends on the next commit which generates the fakes. Signed-off-by: Adolfo García Veytia (Puerco) --- pkg/provenance/predicate_test.go | 66 ++++++++++++ pkg/provenance/predicate_unit_test.go | 47 +++++++++ pkg/provenance/statement_test.go | 132 ++++++++++++++++++++++++ pkg/provenance/statement_unit_test.go | 141 ++++++++++++++++++++++++++ 4 files changed, 386 insertions(+) create mode 100644 pkg/provenance/predicate_test.go create mode 100644 pkg/provenance/predicate_unit_test.go create mode 100644 pkg/provenance/statement_test.go create mode 100644 pkg/provenance/statement_unit_test.go diff --git a/pkg/provenance/predicate_test.go b/pkg/provenance/predicate_test.go new file mode 100644 index 00000000000..e57e72580af --- /dev/null +++ b/pkg/provenance/predicate_test.go @@ -0,0 +1,66 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package provenance_test + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/require" + "k8s.io/release/pkg/provenance" + "k8s.io/release/pkg/provenance/provenancefakes" +) + +// getPredicateSUT returns a predicate loaded with the test implementation +func getPredicateSUT() *provenance.Predicate { + p := provenance.NewSLSAPredicate() + p.SetImplementation(&provenancefakes.FakePredicateImplementation{}) + return &p +} + +func TestWrite(t *testing.T) { + for _, tc := range []struct { + prepare func(*provenancefakes.FakePredicateImplementation) + shouldError bool + }{ + { + // Write errors + prepare: func(mock *provenancefakes.FakePredicateImplementation) { + mock.WriteReturns(errors.New("Fake error")) + }, + shouldError: true, + }, + { + // Write succeeds + prepare: func(mock *provenancefakes.FakePredicateImplementation) { + mock.WriteReturns(nil) + }, + shouldError: false, + }, + } { + p := getPredicateSUT() + mock := &provenancefakes.FakePredicateImplementation{} + tc.prepare(mock) + p.SetImplementation(mock) + res := p.Write("/tmp/mock") + if tc.shouldError { + require.NotNil(t, res) + } else { + require.Nil(t, res) + } + } +} diff --git a/pkg/provenance/predicate_unit_test.go b/pkg/provenance/predicate_unit_test.go new file mode 100644 index 00000000000..6189a3f8b22 --- /dev/null +++ b/pkg/provenance/predicate_unit_test.go @@ -0,0 +1,47 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package provenance + +import ( + "os" + "testing" + + intoto "github.com/in-toto/in-toto-golang/in_toto" + "github.com/stretchr/testify/require" +) + +func TestWrite(t *testing.T) { + p := NewSLSAPredicate() + tmp, err := os.CreateTemp("", "predicate-test") + require.Nil(t, err) + defer os.Remove(tmp.Name()) + + res := p.Write(tmp.Name()) + require.Nil(t, res) + require.FileExists(t, tmp.Name()) + s, err := os.Stat(tmp.Name()) + require.Nil(t, err) + require.Greater(t, s.Size(), int64(0)) +} + +func TestAddMaterial(t *testing.T) { + p := NewSLSAPredicate() + sha1 := "c91cc89922941ace4f79113227a0166f24b8a98b" + p.AddMaterial("https://www.example.com/", intoto.DigestSet{"sha1": sha1}) + require.Equal(t, 1, len(p.Materials)) + require.Equal(t, sha1, p.Materials[0].Digest["sha1"]) +} diff --git a/pkg/provenance/statement_test.go b/pkg/provenance/statement_test.go new file mode 100644 index 00000000000..299c99ee8f0 --- /dev/null +++ b/pkg/provenance/statement_test.go @@ -0,0 +1,132 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package provenance_test + +import ( + "errors" + "testing" + + intoto "github.com/in-toto/in-toto-golang/in_toto" + "github.com/stretchr/testify/require" + "k8s.io/release/pkg/provenance" + "k8s.io/release/pkg/provenance/provenancefakes" +) + +func getStatementSUT() *provenance.Statement { + sut := provenance.NewSLSAStatement() + sut.SetImplementation(&provenancefakes.FakeStatementImplementation{}) + return sut +} + +func TestReadSubjectsFromDir(t *testing.T) { + for _, tc := range []struct { + prepare func(*provenancefakes.FakeStatementImplementation) + shouldError bool + }{ + { + // Read errors + prepare: func(mock *provenancefakes.FakeStatementImplementation) { + mock.ReadSubjectsFromDirReturns(errors.New("mock error")) + }, + shouldError: true, + }, + { + // Read succeeds + prepare: func(mock *provenancefakes.FakeStatementImplementation) { + mock.ReadSubjectsFromDirReturns(nil) + }, + shouldError: false, + }, + } { + s := getStatementSUT() + mock := &provenancefakes.FakeStatementImplementation{} + tc.prepare(mock) + s.SetImplementation(mock) + res := s.ReadSubjectsFromDir("/tmp/mock/") + if tc.shouldError { + require.NotNil(t, res) + } else { + require.Nil(t, res) + } + } +} + +func TestAddSubjectFromFile(t *testing.T) { + for _, tc := range []struct { + prepare func(*provenancefakes.FakeStatementImplementation) + shouldError bool + }{ + { + // Read errors + prepare: func(mock *provenancefakes.FakeStatementImplementation) { + mock.SubjectFromFileReturns(intoto.Subject{}, errors.New("mock error")) + }, + shouldError: true, + }, + { + // Read succeeds + prepare: func(mock *provenancefakes.FakeStatementImplementation) { + mock.SubjectFromFileReturns(intoto.Subject{}, nil) + }, + shouldError: false, + }, + } { + s := getStatementSUT() + mock := &provenancefakes.FakeStatementImplementation{} + tc.prepare(mock) + s.SetImplementation(mock) + res := s.AddSubjectFromFile("/tmp/mock/") + if tc.shouldError { + require.NotNil(t, res) + } else { + require.Nil(t, res) + } + } +} + +func TestWriteStatement(t *testing.T) { + for _, tc := range []struct { + prepare func(*provenancefakes.FakeStatementImplementation) + shouldError bool + }{ + { + // Write errors + prepare: func(mock *provenancefakes.FakeStatementImplementation) { + mock.WriteReturns(errors.New("Fake error")) + }, + shouldError: true, + }, + { + // Write succeeds + prepare: func(mock *provenancefakes.FakeStatementImplementation) { + mock.WriteReturns(nil) + }, + shouldError: false, + }, + } { + p := getStatementSUT() + mock := &provenancefakes.FakeStatementImplementation{} + tc.prepare(mock) + p.SetImplementation(mock) + res := p.Write("/tmp/mock") + if tc.shouldError { + require.NotNil(t, res) + } else { + require.Nil(t, res) + } + } +} diff --git a/pkg/provenance/statement_unit_test.go b/pkg/provenance/statement_unit_test.go new file mode 100644 index 00000000000..21d37297bb1 --- /dev/null +++ b/pkg/provenance/statement_unit_test.go @@ -0,0 +1,141 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package provenance + +import ( + "os" + "path/filepath" + "testing" + + intoto "github.com/in-toto/in-toto-golang/in_toto" + "github.com/stretchr/testify/require" + "sigs.k8s.io/release-utils/util" +) + +func TestReadSubjectsFromDir(t *testing.T) { + s := NewSLSAStatement() + testdata := []struct { + filename string + content string + hash string + }{ + {"en.txt", "Hello world", "64ec88ca00b268e5ba1a35678a1b5316d212f4f366b2477232534a8aeca37f3c"}, + {"es.txt", "Hola mundo", "ca8f60b2cc7f05837d98b208b57fb6481553fc5f1219d59618fd025002a66f5c"}, + {"es/mx.txt", "Quiobos", "0ff2872124d43e90de9221ec849c94f3c797d8daf9254230055c8ebe41fc8b47"}, + {"de.txt", "Hallo Welt", "2d2da19605a34e037dbe82173f98a992a530a5fdd53dad882f570d4ba204ef30"}, + {"de/ch.txt", "Salü", "d64d0c924abb7b5bbc9352cab90676f69d36170deefc2f224b17fe3de71e6a53"}, + } + + // Create a directory with some files + dir, err := os.MkdirTemp("", "") + require.Nil(t, err) + defer os.RemoveAll(dir) + + for _, testfile := range testdata { + path := filepath.Join(dir, testfile.filename) + if !util.Exists(filepath.Dir(path)) { + require.Nil(t, os.Mkdir(filepath.Dir(path), os.FileMode(0o755))) + } + require.Nil(t, os.WriteFile( + path, []byte(testfile.content), os.FileMode(0o644)), + "writing test file", + ) + } + + // Read the files as subjects of the predicate + require.Nil(t, s.ReadSubjectsFromDir(dir), "Reading subjects") + require.Equal(t, len(testdata), len(s.Subject)) + + // Cycle all subjects and check the hashes match + for _, subject := range s.Subject { + seen := false + for _, data := range testdata { + if data.filename == subject.Name { + seen = true + require.Equal(t, data.hash, subject.Digest["sha256"], "invalid subject hash: "+subject.Name) + } + } + require.True(t, seen, "file not found in subjects: "+subject.Name) + } +} + +func TestAddSubject(t *testing.T) { + s := NewSLSAStatement() + sha1 := "cd7f2fdcbd859060732c8a9677d9e838babfa6b9" + s.AddSubject("https://www.example.com/", intoto.DigestSet{"sha1": sha1}) + require.Equal(t, 1, len(s.Subject)) + require.Equal(t, sha1, s.Subject[0].Digest["sha1"]) +} + +func TestLoadPredicate(t *testing.T) { + prData := `{"builder":{"id":"Test@1.0"},"metadata":{"buildInvocationId":"CICD1234","completeness":{"arguments":false,"environment":false,"materials":false},"reproducible":false,"BuildStartedOn":null,"buildFinishedOn":null},"recipe":{"type":"","definedInMaterial":0,"entryPoint":"","arguments":null,"environment":null},"materials":[]}` + + file, err := os.CreateTemp("", "predicate") + require.Nil(t, err) + defer os.Remove(file.Name()) + require.Nil(t, os.WriteFile(file.Name(), []byte(prData), os.FileMode(0o644))) + + s := NewSLSAStatement() + require.Nil(t, s.LoadPredicate(file.Name()), "loading predicate from file") + + require.Equal(t, "Test@1.0", s.Predicate.Builder.ID) +} + +func TestSubjectFromFile(t *testing.T) { + // Create a test file + f, err := os.CreateTemp("", "") + require.Nil(t, err) + defer os.Remove(f.Name()) + require.Nil(t, os.WriteFile(f.Name(), []byte("Hello world"), os.FileMode(0o644))) + + // Create a subject from the temporary file + si := defaultStatementImplementation{} + subject, err := si.SubjectFromFile(f.Name()) + require.Nil(t, err, "creating subject from file") + + // Check the filename + require.Equal(t, f.Name(), subject.Name) + + // Verify the hashes match the expected values + require.Equal( + t, "64ec88ca00b268e5ba1a35678a1b5316d212f4f366b2477232534a8aeca37f3c", + subject.Digest["sha256"], + ) + require.Equal( + t, "b7f783baed8297f0db917462184ff4f08e69c2d5e5f79a942600f9725f58ce1f29c18139bf80b06c0fff2bdd34738452ecf40c488c22a7e3d80cdf6f9c1c0d47", + subject.Digest["sha512"], + ) + + // Attempting a subject from a directory must fail + _, err = si.SubjectFromFile(filepath.Dir(f.Name())) + require.NotNil(t, err, "should err trying to create a subject from a dir") +} + +func TestWriteStatement(t *testing.T) { + s := NewSLSAStatement() + s.Predicate.Builder.ID = "asd" + tmp, err := os.CreateTemp("", "statement-test") + require.Nil(t, err) + defer os.Remove(tmp.Name()) + + res := s.Write(tmp.Name()) + require.Nil(t, res) + require.FileExists(t, tmp.Name()) + st, err := os.Stat(tmp.Name()) + require.Nil(t, err) + require.Greater(t, st.Size(), int64(0)) +} From 47399bb429e8e4ce0ac351ef8b33b9ff3e51d7bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adolfo=20Garc=C3=ADa=20Veytia=20=28Puerco=29?= Date: Sun, 3 Oct 2021 17:43:38 -0500 Subject: [PATCH 07/12] anago/stage: Delete archived sources after staging artifacts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To be able to work with the source tarball, anago/stage now deletes the source tarball after staging the rest of the artifacts. Signed-off-by: Adolfo García Veytia (Puerco) --- pkg/anago/anagofakes/fake_stage_impl.go | 92 ++++++++++++++++++++----- pkg/anago/stage.go | 10 +++ pkg/anago/stage_test.go | 6 ++ pkg/build/push.go | 9 ++- 4 files changed, 100 insertions(+), 17 deletions(-) diff --git a/pkg/anago/anagofakes/fake_stage_impl.go b/pkg/anago/anagofakes/fake_stage_impl.go index 5d7b39ed7b2..a3170fcb107 100644 --- a/pkg/anago/anagofakes/fake_stage_impl.go +++ b/pkg/anago/anagofakes/fake_stage_impl.go @@ -1,19 +1,3 @@ -/* -Copyright The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - // Code generated by counterfeiter. DO NOT EDIT. package anagofakes @@ -141,6 +125,18 @@ type FakeStageImpl struct { result1 string result2 error } + DeleteLocalSourceTarballStub func(*build.Options, string) error + deleteLocalSourceTarballMutex sync.RWMutex + deleteLocalSourceTarballArgsForCall []struct { + arg1 *build.Options + arg2 string + } + deleteLocalSourceTarballReturns struct { + result1 error + } + deleteLocalSourceTarballReturnsOnCall map[int]struct { + result1 error + } DockerHubLoginStub func() error dockerHubLoginMutex sync.RWMutex dockerHubLoginArgsForCall []struct { @@ -982,6 +978,68 @@ func (fake *FakeStageImpl) CurrentBranchReturnsOnCall(i int, result1 string, res }{result1, result2} } +func (fake *FakeStageImpl) DeleteLocalSourceTarball(arg1 *build.Options, arg2 string) error { + fake.deleteLocalSourceTarballMutex.Lock() + ret, specificReturn := fake.deleteLocalSourceTarballReturnsOnCall[len(fake.deleteLocalSourceTarballArgsForCall)] + fake.deleteLocalSourceTarballArgsForCall = append(fake.deleteLocalSourceTarballArgsForCall, struct { + arg1 *build.Options + arg2 string + }{arg1, arg2}) + stub := fake.DeleteLocalSourceTarballStub + fakeReturns := fake.deleteLocalSourceTarballReturns + fake.recordInvocation("DeleteLocalSourceTarball", []interface{}{arg1, arg2}) + fake.deleteLocalSourceTarballMutex.Unlock() + if stub != nil { + return stub(arg1, arg2) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeStageImpl) DeleteLocalSourceTarballCallCount() int { + fake.deleteLocalSourceTarballMutex.RLock() + defer fake.deleteLocalSourceTarballMutex.RUnlock() + return len(fake.deleteLocalSourceTarballArgsForCall) +} + +func (fake *FakeStageImpl) DeleteLocalSourceTarballCalls(stub func(*build.Options, string) error) { + fake.deleteLocalSourceTarballMutex.Lock() + defer fake.deleteLocalSourceTarballMutex.Unlock() + fake.DeleteLocalSourceTarballStub = stub +} + +func (fake *FakeStageImpl) DeleteLocalSourceTarballArgsForCall(i int) (*build.Options, string) { + fake.deleteLocalSourceTarballMutex.RLock() + defer fake.deleteLocalSourceTarballMutex.RUnlock() + argsForCall := fake.deleteLocalSourceTarballArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeStageImpl) DeleteLocalSourceTarballReturns(result1 error) { + fake.deleteLocalSourceTarballMutex.Lock() + defer fake.deleteLocalSourceTarballMutex.Unlock() + fake.DeleteLocalSourceTarballStub = nil + fake.deleteLocalSourceTarballReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeStageImpl) DeleteLocalSourceTarballReturnsOnCall(i int, result1 error) { + fake.deleteLocalSourceTarballMutex.Lock() + defer fake.deleteLocalSourceTarballMutex.Unlock() + fake.DeleteLocalSourceTarballStub = nil + if fake.deleteLocalSourceTarballReturnsOnCall == nil { + fake.deleteLocalSourceTarballReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.deleteLocalSourceTarballReturnsOnCall[i] = struct { + result1 error + }{result1} +} + func (fake *FakeStageImpl) DockerHubLogin() error { fake.dockerHubLoginMutex.Lock() ret, specificReturn := fake.dockerHubLoginReturnsOnCall[len(fake.dockerHubLoginArgsForCall)] @@ -2398,6 +2456,8 @@ func (fake *FakeStageImpl) Invocations() map[string][][]interface{} { defer fake.commitEmptyMutex.RUnlock() fake.currentBranchMutex.RLock() defer fake.currentBranchMutex.RUnlock() + fake.deleteLocalSourceTarballMutex.RLock() + defer fake.deleteLocalSourceTarballMutex.RUnlock() fake.dockerHubLoginMutex.RLock() defer fake.dockerHubLoginMutex.RUnlock() fake.generateChangelogMutex.RLock() diff --git a/pkg/anago/stage.go b/pkg/anago/stage.go index e4030012997..c5b22562e5e 100644 --- a/pkg/anago/stage.go +++ b/pkg/anago/stage.go @@ -151,6 +151,7 @@ type stageImpl interface { StageLocalSourceTree( options *build.Options, workDir, buildVersion string, ) error + DeleteLocalSourceTarball(*build.Options, string) error StageLocalArtifacts(options *build.Options) error PushReleaseArtifacts( options *build.Options, srcPath, gcsPath string, @@ -255,6 +256,10 @@ func (d *defaultStageImpl) StageLocalSourceTree( return build.NewInstance(options).StageLocalSourceTree(workDir, buildVersion) } +func (d *defaultStageImpl) DeleteLocalSourceTarball(options *build.Options, workDir string) error { + return build.NewInstance(options).DeleteLocalSourceTarball(workDir) +} + func (d *defaultStageImpl) StageLocalArtifacts( options *build.Options, ) error { @@ -798,6 +803,11 @@ func (d *DefaultStage) StageArtifacts() error { if err := d.impl.PushContainerImages(pushBuildOptions); err != nil { return errors.Wrap(err, "pushing container images") } + + // Delete the local tarball + if err := d.impl.DeleteLocalSourceTarball(pushBuildOptions, workspaceDir); err != nil { + return errors.Wrap(err, "delete source tarball") + } } args := "" diff --git a/pkg/anago/stage_test.go b/pkg/anago/stage_test.go index 26410051b47..b07d56834be 100644 --- a/pkg/anago/stage_test.go +++ b/pkg/anago/stage_test.go @@ -512,6 +512,12 @@ func TestStageArtifacts(t *testing.T) { }, shouldError: true, }, + { // DeleteLocalSourceTarball fails + prepare: func(mock *anagofakes.FakeStageImpl) { + mock.DeleteLocalSourceTarballReturns(err) + }, + shouldError: true, + }, } { opts := anago.DefaultStageOptions() sut := anago.NewDefaultStage(opts) diff --git a/pkg/build/push.go b/pkg/build/push.go index 78bbaeb4e49..fbfd440c4e2 100644 --- a/pkg/build/push.go +++ b/pkg/build/push.go @@ -513,6 +513,13 @@ func (bi *Instance) StageLocalSourceTree(workDir, buildVersion string) error { return errors.Wrap(err, "copy tarball to GCS") } - logrus.Infof("Removing local source tree tarball") + return nil +} + +// DeleteLocalSourceTarball the deletion of the tarball is now decoupled from +// StageLocalSourceTree to be able to use it during the anago.stage function +func (bi *Instance) DeleteLocalSourceTarball(workDir string) error { + tarballPath := filepath.Join(workDir, release.SourcesTar) + logrus.Infof("Removing local source tree tarball " + tarballPath) return errors.Wrap(os.RemoveAll(tarballPath), "remove local source tarball") } From b1f92c461912d96fd386e650628944cfb96c2284 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adolfo=20Garc=C3=ADa=20Veytia=20=28Puerco=29?= Date: Sun, 3 Oct 2021 18:22:57 -0500 Subject: [PATCH 08/12] anago/stage: Produce only one source tarball MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When running a multi-tag release (eg official + rc), krel would compress and upload the whole kubernetes source for each version. Now, we only do it once as the tarball is exactly the same for all release in a single run. This should speed the release process. Signed-off-by: Adolfo García Veytia (Puerco) --- pkg/anago/stage.go | 52 +++++++++++++++++++++++----------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/pkg/anago/stage.go b/pkg/anago/stage.go index c5b22562e5e..ada593097fa 100644 --- a/pkg/anago/stage.go +++ b/pkg/anago/stage.go @@ -745,33 +745,33 @@ func (d *DefaultStage) GenerateBillOfMaterials() error { } func (d *DefaultStage) StageArtifacts() error { + pushBuildOptions := &build.Options{ + Bucket: d.options.Bucket(), + Registry: d.options.ContainerRegistry(), + AllowDup: true, + ValidateRemoteImageDigests: true, + } + if err := d.impl.CheckReleaseBucket(pushBuildOptions); err != nil { + return errors.Wrap(err, "check release bucket access") + } + + // Stage the local source tree + if err := d.impl.StageLocalSourceTree( + pushBuildOptions, + workspaceDir, + d.options.BuildVersion, + ); err != nil { + return errors.Wrap(err, "staging local source tree") + } + for _, version := range d.state.versions.Ordered() { logrus.Infof("Staging artifacts for version %s", version) buildDir := filepath.Join( gitRoot, fmt.Sprintf("%s-%s", release.BuildDir, version), ) - bucket := d.options.Bucket() - containerRegistry := d.options.ContainerRegistry() - pushBuildOptions := &build.Options{ - Bucket: bucket, - BuildDir: buildDir, - Registry: containerRegistry, - Version: version, - AllowDup: true, - ValidateRemoteImageDigests: true, - } - if err := d.impl.CheckReleaseBucket(pushBuildOptions); err != nil { - return errors.Wrap(err, "check release bucket access") - } - - // Stage the local source tree - if err := d.impl.StageLocalSourceTree( - pushBuildOptions, - workspaceDir, - d.options.BuildVersion, - ); err != nil { - return errors.Wrap(err, "staging local source tree") - } + // Set the version-specific option for the push + pushBuildOptions.Version = version + pushBuildOptions.BuildDir = buildDir // Stage local artifacts and write checksums if err := d.impl.StageLocalArtifacts(pushBuildOptions); err != nil { @@ -803,11 +803,11 @@ func (d *DefaultStage) StageArtifacts() error { if err := d.impl.PushContainerImages(pushBuildOptions); err != nil { return errors.Wrap(err, "pushing container images") } + } - // Delete the local tarball - if err := d.impl.DeleteLocalSourceTarball(pushBuildOptions, workspaceDir); err != nil { - return errors.Wrap(err, "delete source tarball") - } + // Delete the local source tarball + if err := d.impl.DeleteLocalSourceTarball(pushBuildOptions, workspaceDir); err != nil { + return errors.Wrap(err, "delete source tarball") } args := "" From 4c5191cd54c13a0e9899449ab7d8e8e9ac8778e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adolfo=20Garc=C3=ADa=20Veytia=20=28Puerco=29?= Date: Sun, 3 Oct 2021 18:34:55 -0500 Subject: [PATCH 09/12] anago/stage: Generate the staging provenance metadata MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit anago/stage will now generate the provenance metadata of the artifacts produced after building. While staging the files, an in-toto attestation is built and uploaded with the release. Signed-off-by: Adolfo García Veytia (Puerco) --- pkg/anago/stage.go | 175 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 174 insertions(+), 1 deletion(-) diff --git a/pkg/anago/stage.go b/pkg/anago/stage.go index ada593097fa..bde8358d288 100644 --- a/pkg/anago/stage.go +++ b/pkg/anago/stage.go @@ -20,18 +20,23 @@ import ( "fmt" "os" "path/filepath" + "regexp" "strings" + "time" "github.com/blang/semver" "github.com/pkg/errors" "github.com/sirupsen/logrus" + intoto "github.com/in-toto/in-toto-golang/in_toto" "k8s.io/release/pkg/build" "k8s.io/release/pkg/changelog" "k8s.io/release/pkg/gcp/gcb" + "k8s.io/release/pkg/provenance" "k8s.io/release/pkg/release" "k8s.io/release/pkg/spdx" "sigs.k8s.io/release-sdk/git" + "sigs.k8s.io/release-sdk/object" "sigs.k8s.io/release-utils/log" ) @@ -167,6 +172,9 @@ type stageImpl interface { AddBinariesToSBOM(*spdx.Document, string) error AddTarfilesToSBOM(*spdx.Document, string) error VerifyArtifacts([]string) error + GenerateAttestation(*StageState, *StageOptions) (*provenance.Statement, error) + PushAttestation(*provenance.Statement, *StageOptions) error + AddProvenanceSubject(*provenance.Statement, string, bool) error } func (d *defaultStageImpl) Submit(options *gcb.Options) error { @@ -745,6 +753,12 @@ func (d *DefaultStage) GenerateBillOfMaterials() error { } func (d *DefaultStage) StageArtifacts() error { + // Generat the intoto attestation, reloaded with the current run data + statement, err := d.impl.GenerateAttestation(d.state, d.options) + if err != nil { + return errors.Wrap(err, "generating the provenance attestation") + } + // Init a the push options we will use pushBuildOptions := &build.Options{ Bucket: d.options.Bucket(), Registry: d.options.ContainerRegistry(), @@ -764,6 +778,13 @@ func (d *DefaultStage) StageArtifacts() error { return errors.Wrap(err, "staging local source tree") } + // Add the sources tarball to the attestation + if err := d.impl.AddProvenanceSubject( + statement, filepath.Join(workspaceDir, release.SourcesTar), false, + ); err != nil { + return errors.Wrap(err, "adding sources tarball to provenance attestation") + } + for _, version := range d.state.versions.Ordered() { logrus.Infof("Staging artifacts for version %s", version) buildDir := filepath.Join( @@ -778,7 +799,7 @@ func (d *DefaultStage) StageArtifacts() error { return errors.Wrap(err, "staging local artifacts") } gcsPath := filepath.Join( - d.options.Bucket(), "stage", d.options.BuildVersion, version, + d.options.Bucket(), release.StagePath, d.options.BuildVersion, version, ) // Push gcs-stage to GCS @@ -803,6 +824,19 @@ func (d *DefaultStage) StageArtifacts() error { if err := d.impl.PushContainerImages(pushBuildOptions); err != nil { return errors.Wrap(err, "pushing container images") } + + // Add artifacts to the attestation, this should get both release-images + // and gcs-stage directories in one call. + if err := d.impl.AddProvenanceSubject( + statement, filepath.Join(buildDir), true, + ); err != nil { + return errors.Wrapf(err, "adding provenance of release-images for version %s", version) + } + } + + // Push the attestation metadata file to the bucket + if err := d.impl.PushAttestation(statement, d.options); err != nil { + return errors.Wrap(err, "writing provenance metadata to disk") } // Delete the local source tarball @@ -827,3 +861,142 @@ func (d *DefaultStage) StageArtifacts() error { ) return nil } + +// GenerateAttestation creates a provenance attestation with its predicate +// preloaded with the current krel run information +func (d *defaultStageImpl) GenerateAttestation(state *StageState, options *StageOptions) (attestation *provenance.Statement, err error) { + // Build the arguments RawMessage: + arguments := map[string]string{ + "--type=": options.ReleaseType, + "--branch=": options.ReleaseBranch, + "--build-version=": options.BuildVersion, + } + if options.NoMock { + arguments["--nomock"] = "true" + } + + // Fetch the last commit: + repo, err := git.OpenRepo(gitRoot) + if err != nil { + return nil, errors.Wrap(err, "opening repository to check commit hash") + } + // TODO: When this PR merges and the commit is part of a release: + // https://github.com/kubernetes-sigs/release-sdk/pull/6 + // and k/release is bumped, replace the commit logic with this line: + // commitSHA, err := repo.LastCommitSha() + logData, err := repo.ShowLastCommit() + if err != nil { + return nil, errors.Wrap(err, "getting last commit data") + } + re := regexp.MustCompile(`commit\s+([a-f0-9]{40})`) + commitSHA := re.FindString(logData) + if commitSHA == "" { + return nil, errors.New("Unable to find last commit sha in git output") + } + + // Create the predicate to populate it with the current + // run metadata: + p := provenance.NewSLSAPredicate() + + // TODO: In regular runs, this will insert "master", we should + // record the git sha of the commit in k/release we are using. + p.Builder.ID = fmt.Sprintf( + "pkg:github/%s/%s@%s", os.Getenv("TOOL_ORG"), + os.Getenv("TOOL_REPO"), os.Getenv("TOOL_REF"), + ) + // Some of these fields have yet to be checked to assign the + // correct values to them + // This is commented as the in-toto go port does not have it + // p.Metadata.BuildInvocationID: os.Getenv("BUILD_ID"), + p.Metadata.Completeness.Arguments = true // The arguments are complete as we know the from GCB + p.Metadata.Completeness.Materials = true // The materials are complete as we only use the github repo + startTime := state.startTime.UTC() + endTime := time.Now().UTC() + p.Metadata.BuildStartedOn = &startTime + p.Metadata.BuildFinishedOn = &endTime + + p.Recipe.Type = "https://cloudbuild.googleapis.com/CloudBuildYaml@v1" + p.Recipe.EntryPoint = "https://github.com/kubernetes/release/blob/master/gcb/stage/cloudbuild.yaml" + p.Recipe.Arguments = arguments + + p.AddMaterial("git+https://github.com/kubernetes/kubernetes", intoto.DigestSet{"sha1": commitSHA}) + + // Create the new attestation and attach the predicate + attestation = provenance.NewSLSAStatement() + attestation.Predicate = p + + return attestation, nil +} + +// PushAttestation writes the provenance metadata to the staging location in +// the Google Cloud Bucket. +func (d *defaultStageImpl) PushAttestation(attestation *provenance.Statement, options *StageOptions) (err error) { + gcsPath := filepath.Join(options.Bucket(), release.StagePath, options.BuildVersion) + // Clean up the subkects list to normalize and only keep the ones + // we are interested in: + newSubjects := []intoto.Subject{} + for _, sub := range attestation.Subject { + // The sources tar is special: + if sub.Name == filepath.Join(workspaceDir, release.SourcesTar) { + sub.Name = object.GcsPrefix + filepath.Join(gcsPath, release.SourcesTar) + newSubjects = append(newSubjects, sub) + continue + } + + // If the artifact is not in the images or gcs-stage dir, skip + if !strings.HasPrefix(sub.Name, release.ImagesPath) && + !strings.HasPrefix(sub.Name, release.GCSStagePath) { + continue + } + + // Now the tricky part. We need to re-append the version tag, ie + // gcs-stage/v1.23.0-alpha.4/file.txt shoud be + // v1.23.0-alpha.4/gcs-stage/v1.23.0-alpha.4/file.txt shoud be + parts := strings.Split(sub.Name, string(filepath.Separator)) + if len(parts) < 3 { + logrus.Warnf("Unkwn path found in provenance subjects: %s", sub.Name) + continue + } + sub.Name = object.GcsPrefix + filepath.Join(gcsPath, parts[1], sub.Name) + newSubjects = append(newSubjects, sub) + } + + attestation.Subject = newSubjects + + // Create a temporary file: + f, err := os.CreateTemp("", "provenance-") + if err != nil { + return errors.Wrap(err, "creating temp file for provenance metadata") + } + // Write the provenance statement to disk: + if err := attestation.Write(f.Name()); err != nil { + return errors.Wrap(err, "writing provenance attestation to disk") + } + + // TODO for SLSA2: Sign the attestation + // Upload the metadata file to the staging bucket + pushBuildOptions := &build.Options{ + Bucket: options.Bucket(), + AllowDup: true, + } + + if err := d.CheckReleaseBucket(pushBuildOptions); err != nil { + return errors.Wrap(err, "check release bucket access") + } + + // Push the provenance file to GCS + return errors.Wrap( + d.PushReleaseArtifacts(pushBuildOptions, f.Name(), filepath.Join(gcsPath, release.ProvenanceFilename)), + "pushing provenance manifest", + ) +} + +// AddProvenanceSubject adds artifacts to the provenance metadata. +func (d *defaultStageImpl) AddProvenanceSubject(statement *provenance.Statement, path string, isDir bool) (err error) { + if isDir { + err = statement.ReadSubjectsFromDir(path) + } else { + err = statement.AddSubjectFromFile(path) + } + return errors.Wrapf(err, "adding %s to provenance metadata", path) +} From 1313e9f063dad019b4630f0042af41e19da65cae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adolfo=20Garc=C3=ADa=20Veytia=20=28Puerco=29?= Date: Sun, 3 Oct 2021 18:56:43 -0500 Subject: [PATCH 10/12] generate: anago/stage implementation fakes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Regenerate the stage implementation fakes with the new provenance functions Signed-off-by: Adolfo García Veytia (Puerco) --- pkg/anago/anagofakes/fake_stage_impl.go | 253 ++++++++++++++++++++++++ 1 file changed, 253 insertions(+) diff --git a/pkg/anago/anagofakes/fake_stage_impl.go b/pkg/anago/anagofakes/fake_stage_impl.go index a3170fcb107..dec36bb2a51 100644 --- a/pkg/anago/anagofakes/fake_stage_impl.go +++ b/pkg/anago/anagofakes/fake_stage_impl.go @@ -1,3 +1,19 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + // Code generated by counterfeiter. DO NOT EDIT. package anagofakes @@ -5,9 +21,11 @@ import ( "sync" "github.com/blang/semver" + "k8s.io/release/pkg/anago" "k8s.io/release/pkg/build" "k8s.io/release/pkg/changelog" "k8s.io/release/pkg/gcp/gcb" + "k8s.io/release/pkg/provenance" "k8s.io/release/pkg/release" "k8s.io/release/pkg/spdx" "sigs.k8s.io/release-sdk/git" @@ -26,6 +44,19 @@ type FakeStageImpl struct { addBinariesToSBOMReturnsOnCall map[int]struct { result1 error } + AddProvenanceSubjectStub func(*provenance.Statement, string, bool) error + addProvenanceSubjectMutex sync.RWMutex + addProvenanceSubjectArgsForCall []struct { + arg1 *provenance.Statement + arg2 string + arg3 bool + } + addProvenanceSubjectReturns struct { + result1 error + } + addProvenanceSubjectReturnsOnCall map[int]struct { + result1 error + } AddTarfilesToSBOMStub func(*spdx.Document, string) error addTarfilesToSBOMMutex sync.RWMutex addTarfilesToSBOMArgsForCall []struct { @@ -147,6 +178,20 @@ type FakeStageImpl struct { dockerHubLoginReturnsOnCall map[int]struct { result1 error } + GenerateAttestationStub func(*anago.StageState, *anago.StageOptions) (*provenance.Statement, error) + generateAttestationMutex sync.RWMutex + generateAttestationArgsForCall []struct { + arg1 *anago.StageState + arg2 *anago.StageOptions + } + generateAttestationReturns struct { + result1 *provenance.Statement + result2 error + } + generateAttestationReturnsOnCall map[int]struct { + result1 *provenance.Statement + result2 error + } GenerateChangelogStub func(*changelog.Options) error generateChangelogMutex sync.RWMutex generateChangelogArgsForCall []struct { @@ -283,6 +328,18 @@ type FakeStageImpl struct { prepareWorkspaceStageReturnsOnCall map[int]struct { result1 error } + PushAttestationStub func(*provenance.Statement, *anago.StageOptions) error + pushAttestationMutex sync.RWMutex + pushAttestationArgsForCall []struct { + arg1 *provenance.Statement + arg2 *anago.StageOptions + } + pushAttestationReturns struct { + result1 error + } + pushAttestationReturnsOnCall map[int]struct { + result1 error + } PushContainerImagesStub func(*build.Options) error pushContainerImagesMutex sync.RWMutex pushContainerImagesArgsForCall []struct { @@ -483,6 +540,69 @@ func (fake *FakeStageImpl) AddBinariesToSBOMReturnsOnCall(i int, result1 error) }{result1} } +func (fake *FakeStageImpl) AddProvenanceSubject(arg1 *provenance.Statement, arg2 string, arg3 bool) error { + fake.addProvenanceSubjectMutex.Lock() + ret, specificReturn := fake.addProvenanceSubjectReturnsOnCall[len(fake.addProvenanceSubjectArgsForCall)] + fake.addProvenanceSubjectArgsForCall = append(fake.addProvenanceSubjectArgsForCall, struct { + arg1 *provenance.Statement + arg2 string + arg3 bool + }{arg1, arg2, arg3}) + stub := fake.AddProvenanceSubjectStub + fakeReturns := fake.addProvenanceSubjectReturns + fake.recordInvocation("AddProvenanceSubject", []interface{}{arg1, arg2, arg3}) + fake.addProvenanceSubjectMutex.Unlock() + if stub != nil { + return stub(arg1, arg2, arg3) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeStageImpl) AddProvenanceSubjectCallCount() int { + fake.addProvenanceSubjectMutex.RLock() + defer fake.addProvenanceSubjectMutex.RUnlock() + return len(fake.addProvenanceSubjectArgsForCall) +} + +func (fake *FakeStageImpl) AddProvenanceSubjectCalls(stub func(*provenance.Statement, string, bool) error) { + fake.addProvenanceSubjectMutex.Lock() + defer fake.addProvenanceSubjectMutex.Unlock() + fake.AddProvenanceSubjectStub = stub +} + +func (fake *FakeStageImpl) AddProvenanceSubjectArgsForCall(i int) (*provenance.Statement, string, bool) { + fake.addProvenanceSubjectMutex.RLock() + defer fake.addProvenanceSubjectMutex.RUnlock() + argsForCall := fake.addProvenanceSubjectArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 +} + +func (fake *FakeStageImpl) AddProvenanceSubjectReturns(result1 error) { + fake.addProvenanceSubjectMutex.Lock() + defer fake.addProvenanceSubjectMutex.Unlock() + fake.AddProvenanceSubjectStub = nil + fake.addProvenanceSubjectReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeStageImpl) AddProvenanceSubjectReturnsOnCall(i int, result1 error) { + fake.addProvenanceSubjectMutex.Lock() + defer fake.addProvenanceSubjectMutex.Unlock() + fake.AddProvenanceSubjectStub = nil + if fake.addProvenanceSubjectReturnsOnCall == nil { + fake.addProvenanceSubjectReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.addProvenanceSubjectReturnsOnCall[i] = struct { + result1 error + }{result1} +} + func (fake *FakeStageImpl) AddTarfilesToSBOM(arg1 *spdx.Document, arg2 string) error { fake.addTarfilesToSBOMMutex.Lock() ret, specificReturn := fake.addTarfilesToSBOMReturnsOnCall[len(fake.addTarfilesToSBOMArgsForCall)] @@ -1093,6 +1213,71 @@ func (fake *FakeStageImpl) DockerHubLoginReturnsOnCall(i int, result1 error) { }{result1} } +func (fake *FakeStageImpl) GenerateAttestation(arg1 *anago.StageState, arg2 *anago.StageOptions) (*provenance.Statement, error) { + fake.generateAttestationMutex.Lock() + ret, specificReturn := fake.generateAttestationReturnsOnCall[len(fake.generateAttestationArgsForCall)] + fake.generateAttestationArgsForCall = append(fake.generateAttestationArgsForCall, struct { + arg1 *anago.StageState + arg2 *anago.StageOptions + }{arg1, arg2}) + stub := fake.GenerateAttestationStub + fakeReturns := fake.generateAttestationReturns + fake.recordInvocation("GenerateAttestation", []interface{}{arg1, arg2}) + fake.generateAttestationMutex.Unlock() + if stub != nil { + return stub(arg1, arg2) + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeStageImpl) GenerateAttestationCallCount() int { + fake.generateAttestationMutex.RLock() + defer fake.generateAttestationMutex.RUnlock() + return len(fake.generateAttestationArgsForCall) +} + +func (fake *FakeStageImpl) GenerateAttestationCalls(stub func(*anago.StageState, *anago.StageOptions) (*provenance.Statement, error)) { + fake.generateAttestationMutex.Lock() + defer fake.generateAttestationMutex.Unlock() + fake.GenerateAttestationStub = stub +} + +func (fake *FakeStageImpl) GenerateAttestationArgsForCall(i int) (*anago.StageState, *anago.StageOptions) { + fake.generateAttestationMutex.RLock() + defer fake.generateAttestationMutex.RUnlock() + argsForCall := fake.generateAttestationArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeStageImpl) GenerateAttestationReturns(result1 *provenance.Statement, result2 error) { + fake.generateAttestationMutex.Lock() + defer fake.generateAttestationMutex.Unlock() + fake.GenerateAttestationStub = nil + fake.generateAttestationReturns = struct { + result1 *provenance.Statement + result2 error + }{result1, result2} +} + +func (fake *FakeStageImpl) GenerateAttestationReturnsOnCall(i int, result1 *provenance.Statement, result2 error) { + fake.generateAttestationMutex.Lock() + defer fake.generateAttestationMutex.Unlock() + fake.GenerateAttestationStub = nil + if fake.generateAttestationReturnsOnCall == nil { + fake.generateAttestationReturnsOnCall = make(map[int]struct { + result1 *provenance.Statement + result2 error + }) + } + fake.generateAttestationReturnsOnCall[i] = struct { + result1 *provenance.Statement + result2 error + }{result1, result2} +} + func (fake *FakeStageImpl) GenerateChangelog(arg1 *changelog.Options) error { fake.generateChangelogMutex.Lock() ret, specificReturn := fake.generateChangelogReturnsOnCall[len(fake.generateChangelogArgsForCall)] @@ -1744,6 +1929,68 @@ func (fake *FakeStageImpl) PrepareWorkspaceStageReturnsOnCall(i int, result1 err }{result1} } +func (fake *FakeStageImpl) PushAttestation(arg1 *provenance.Statement, arg2 *anago.StageOptions) error { + fake.pushAttestationMutex.Lock() + ret, specificReturn := fake.pushAttestationReturnsOnCall[len(fake.pushAttestationArgsForCall)] + fake.pushAttestationArgsForCall = append(fake.pushAttestationArgsForCall, struct { + arg1 *provenance.Statement + arg2 *anago.StageOptions + }{arg1, arg2}) + stub := fake.PushAttestationStub + fakeReturns := fake.pushAttestationReturns + fake.recordInvocation("PushAttestation", []interface{}{arg1, arg2}) + fake.pushAttestationMutex.Unlock() + if stub != nil { + return stub(arg1, arg2) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeStageImpl) PushAttestationCallCount() int { + fake.pushAttestationMutex.RLock() + defer fake.pushAttestationMutex.RUnlock() + return len(fake.pushAttestationArgsForCall) +} + +func (fake *FakeStageImpl) PushAttestationCalls(stub func(*provenance.Statement, *anago.StageOptions) error) { + fake.pushAttestationMutex.Lock() + defer fake.pushAttestationMutex.Unlock() + fake.PushAttestationStub = stub +} + +func (fake *FakeStageImpl) PushAttestationArgsForCall(i int) (*provenance.Statement, *anago.StageOptions) { + fake.pushAttestationMutex.RLock() + defer fake.pushAttestationMutex.RUnlock() + argsForCall := fake.pushAttestationArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeStageImpl) PushAttestationReturns(result1 error) { + fake.pushAttestationMutex.Lock() + defer fake.pushAttestationMutex.Unlock() + fake.PushAttestationStub = nil + fake.pushAttestationReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeStageImpl) PushAttestationReturnsOnCall(i int, result1 error) { + fake.pushAttestationMutex.Lock() + defer fake.pushAttestationMutex.Unlock() + fake.PushAttestationStub = nil + if fake.pushAttestationReturnsOnCall == nil { + fake.pushAttestationReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.pushAttestationReturnsOnCall[i] = struct { + result1 error + }{result1} +} + func (fake *FakeStageImpl) PushContainerImages(arg1 *build.Options) error { fake.pushContainerImagesMutex.Lock() ret, specificReturn := fake.pushContainerImagesReturnsOnCall[len(fake.pushContainerImagesArgsForCall)] @@ -2440,6 +2687,8 @@ func (fake *FakeStageImpl) Invocations() map[string][][]interface{} { defer fake.invocationsMutex.RUnlock() fake.addBinariesToSBOMMutex.RLock() defer fake.addBinariesToSBOMMutex.RUnlock() + fake.addProvenanceSubjectMutex.RLock() + defer fake.addProvenanceSubjectMutex.RUnlock() fake.addTarfilesToSBOMMutex.RLock() defer fake.addTarfilesToSBOMMutex.RUnlock() fake.branchNeedsCreationMutex.RLock() @@ -2460,6 +2709,8 @@ func (fake *FakeStageImpl) Invocations() map[string][][]interface{} { defer fake.deleteLocalSourceTarballMutex.RUnlock() fake.dockerHubLoginMutex.RLock() defer fake.dockerHubLoginMutex.RUnlock() + fake.generateAttestationMutex.RLock() + defer fake.generateAttestationMutex.RUnlock() fake.generateChangelogMutex.RLock() defer fake.generateChangelogMutex.RUnlock() fake.generateReleaseVersionMutex.RLock() @@ -2480,6 +2731,8 @@ func (fake *FakeStageImpl) Invocations() map[string][][]interface{} { defer fake.openRepoMutex.RUnlock() fake.prepareWorkspaceStageMutex.RLock() defer fake.prepareWorkspaceStageMutex.RUnlock() + fake.pushAttestationMutex.RLock() + defer fake.pushAttestationMutex.RUnlock() fake.pushContainerImagesMutex.RLock() defer fake.pushContainerImagesMutex.RUnlock() fake.pushReleaseArtifactsMutex.RLock() From 71f3cde32362679694817bd60712de8957a1f939 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adolfo=20Garc=C3=ADa=20Veytia=20=28Puerco=29?= Date: Sun, 3 Oct 2021 19:07:04 -0500 Subject: [PATCH 11/12] Add provenance functions to stage integration tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit adds the new provenance functions in anago/stage to the integration tests in TestStageArtifacts() Signed-off-by: Adolfo García Veytia (Puerco) --- pkg/anago/stage_test.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/pkg/anago/stage_test.go b/pkg/anago/stage_test.go index b07d56834be..42681accbbe 100644 --- a/pkg/anago/stage_test.go +++ b/pkg/anago/stage_test.go @@ -518,6 +518,24 @@ func TestStageArtifacts(t *testing.T) { }, shouldError: true, }, + { // GenerateAttestation fails + prepare: func(mock *anagofakes.FakeStageImpl) { + mock.GenerateAttestationReturns(nil, err) + }, + shouldError: true, + }, + { // PushAttestation fails + prepare: func(mock *anagofakes.FakeStageImpl) { + mock.PushAttestationReturns(err) + }, + shouldError: true, + }, + { // Add AddProvenanceSubject fails + prepare: func(mock *anagofakes.FakeStageImpl) { + mock.AddProvenanceSubjectReturns(err) + }, + shouldError: true, + }, } { opts := anago.DefaultStageOptions() sut := anago.NewDefaultStage(opts) From 1998bdf87aa59bc4cee63db6d87a6b8853a33463 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adolfo=20Garc=C3=ADa=20Veytia=20=28Puerco=29?= Date: Sun, 3 Oct 2021 20:16:33 -0500 Subject: [PATCH 12/12] Support pushing single files in PushReleaseArtifacts() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit modifies the function Instance.PushReleaseArtifacts() in the build package to support pushing single files to the release buckets. Before, PushReleaseArtifacts() would fail if applied to a single file. Signed-off-by: Adolfo García Veytia (Puerco) --- pkg/build/push.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pkg/build/push.go b/pkg/build/push.go index fbfd440c4e2..9c9a6d56ef9 100644 --- a/pkg/build/push.go +++ b/pkg/build/push.go @@ -399,6 +399,18 @@ func (bi *Instance) PushReleaseArtifacts(srcPath, gcsPath string) error { logrus.Infof("Pushing release artifacts from %s to %s", srcPath, dstPath) + finfo, err := os.Stat(srcPath) + if err != nil { + return errors.Wrap(err, "checking if source path is a directory") + } + + // If we are handling a single file copy instead of rsync + if !finfo.IsDir() { + return errors.Wrap( + bi.objStore.CopyToRemote(srcPath, dstPath), "copying file to GCS", + ) + } + return errors.Wrap( bi.objStore.RsyncRecursive(srcPath, dstPath), "rsync artifacts to GCS", )