diff --git a/cmd/gh2gcs/cmd/root.go b/cmd/gh2gcs/cmd/root.go index db410176a10..a55a4a6fbea 100644 --- a/cmd/gh2gcs/cmd/root.go +++ b/cmd/gh2gcs/cmd/root.go @@ -27,9 +27,9 @@ import ( "github.com/spf13/pflag" "gopkg.in/yaml.v2" - "k8s.io/release/pkg/gcp" "k8s.io/release/pkg/gh2gcs" "k8s.io/release/pkg/github" + "sigs.k8s.io/release-sdk/gcli" "sigs.k8s.io/release-utils/log" ) @@ -199,7 +199,7 @@ func run(opts *options) error { return errors.Wrap(err, "validating gh2gcs options") } - if err := gcp.PreCheck(); err != nil { + if err := gcli.PreCheck(); err != nil { return errors.Wrap(err, "pre-checking for GCP package usage") } diff --git a/cmd/krel/cmd/testgrid.go b/cmd/krel/cmd/testgrid.go index 891e9ea349c..e993b4693ab 100644 --- a/cmd/krel/cmd/testgrid.go +++ b/cmd/krel/cmd/testgrid.go @@ -34,7 +34,7 @@ import ( "k8s.io/release/pkg/git" "k8s.io/release/pkg/github" - "k8s.io/release/pkg/object" + "sigs.k8s.io/release-sdk/object" "sigs.k8s.io/release-utils/http" ) diff --git a/go.mod b/go.mod index c38b64a8546..c815ad4c705 100644 --- a/go.mod +++ b/go.mod @@ -42,6 +42,7 @@ require ( k8s.io/utils v0.0.0-20210305010621-2afb4311ab10 sigs.k8s.io/k8s-container-image-promoter v1.337.0 sigs.k8s.io/mdtoc v1.0.1 + sigs.k8s.io/release-sdk v0.1.0 sigs.k8s.io/release-utils v0.3.0 sigs.k8s.io/yaml v1.2.0 sigs.k8s.io/zeitgeist v0.3.0 diff --git a/go.sum b/go.sum index 4ab4665828c..2572ae17dac 100644 --- a/go.sum +++ b/go.sum @@ -1621,6 +1621,8 @@ sigs.k8s.io/k8s-container-image-promoter v1.337.0 h1:gs4zv5TSQzlJAapX1Tkm5FuLcX1 sigs.k8s.io/k8s-container-image-promoter v1.337.0/go.mod h1:wdNhLd/SlIs+rgTw8D6RaaUWSdTnM0OEhGfXU3kI50o= sigs.k8s.io/mdtoc v1.0.1 h1:6ECKhQnbetwZBR6R2IeT2LH+1w+2Zsip0iXjikgaXIk= sigs.k8s.io/mdtoc v1.0.1/go.mod h1:COYBtOjsaCg7o7SC4eaLwEXPuVRSuiVuLLRrHd7kShw= +sigs.k8s.io/release-sdk v0.1.0 h1:gesMqsV1W1r0PS9dbxeX0nuW45ZoRsIzILA23+WFjYU= +sigs.k8s.io/release-sdk v0.1.0/go.mod h1:XhDgjf0bu2Vuyosp0Cc/mc3UUJiWS171S7WsNxA9iY0= sigs.k8s.io/release-utils v0.2.0/go.mod h1:9O5livl2h3Q56jUkoZ7UnV22XVRB6MuD4l/51C2vAPg= sigs.k8s.io/release-utils v0.2.1/go.mod h1:J9xpziRNRI4mAeMZxPRryDodQMoMudMu6yC1aViFHU4= sigs.k8s.io/release-utils v0.3.0 h1:cyNeXvm+2lPn67f4MWmq9xapZDAI5hekpT7iQPRxta4= diff --git a/pkg/anago/anagofakes/fake_release_impl.go b/pkg/anago/anagofakes/fake_release_impl.go index 9d85e38eb2e..0b06b0e670c 100644 --- a/pkg/anago/anagofakes/fake_release_impl.go +++ b/pkg/anago/anagofakes/fake_release_impl.go @@ -24,8 +24,8 @@ import ( "k8s.io/release/pkg/announce" "k8s.io/release/pkg/build" "k8s.io/release/pkg/gcp/gcb" - "k8s.io/release/pkg/object" "k8s.io/release/pkg/release" + "sigs.k8s.io/release-sdk/object" ) type FakeReleaseImpl struct { diff --git a/pkg/anago/release.go b/pkg/anago/release.go index fb997af9b3c..b7e71da1f38 100644 --- a/pkg/anago/release.go +++ b/pkg/anago/release.go @@ -29,8 +29,8 @@ import ( "k8s.io/release/pkg/build" "k8s.io/release/pkg/gcp/gcb" "k8s.io/release/pkg/git" - "k8s.io/release/pkg/object" "k8s.io/release/pkg/release" + "sigs.k8s.io/release-sdk/object" "sigs.k8s.io/release-utils/log" "sigs.k8s.io/release-utils/util" ) diff --git a/pkg/api/files/validation.go b/pkg/api/files/validation.go index 94d8b146a4e..47ff4ab1214 100644 --- a/pkg/api/files/validation.go +++ b/pkg/api/files/validation.go @@ -21,7 +21,7 @@ import ( "fmt" "strings" - "k8s.io/release/pkg/object" + "sigs.k8s.io/release-sdk/object" ) // Validate checks for semantic errors in the yaml fields (the structure of the diff --git a/pkg/build/build.go b/pkg/build/build.go index 10d75e85878..06c8fea0417 100644 --- a/pkg/build/build.go +++ b/pkg/build/build.go @@ -20,8 +20,8 @@ import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" - "k8s.io/release/pkg/object" "k8s.io/release/pkg/release" + "sigs.k8s.io/release-sdk/object" ) var DefaultExtraVersionMarkers = []string{} diff --git a/pkg/changelog/impl.go b/pkg/changelog/impl.go index a241907836f..e55fb54a76a 100644 --- a/pkg/changelog/impl.go +++ b/pkg/changelog/impl.go @@ -36,7 +36,7 @@ import ( "k8s.io/release/pkg/notes" "k8s.io/release/pkg/notes/document" "k8s.io/release/pkg/notes/options" - "k8s.io/release/pkg/object" + "sigs.k8s.io/release-sdk/object" "sigs.k8s.io/release-utils/http" "sigs.k8s.io/release-utils/util" ) diff --git a/pkg/cve/client.go b/pkg/cve/client.go index b3cbf849ffa..29e00941c7b 100644 --- a/pkg/cve/client.go +++ b/pkg/cve/client.go @@ -22,8 +22,8 @@ import ( "github.com/pkg/errors" - "k8s.io/release/pkg/object" "k8s.io/release/pkg/release" + "sigs.k8s.io/release-sdk/object" ) const ( diff --git a/pkg/cve/impl.go b/pkg/cve/impl.go index 2b1893893de..7f9a23dc828 100644 --- a/pkg/cve/impl.go +++ b/pkg/cve/impl.go @@ -30,7 +30,7 @@ import ( "gopkg.in/yaml.v2" "k8s.io/release/pkg/notes" - "k8s.io/release/pkg/object" + "sigs.k8s.io/release-sdk/object" ) //counterfeiter:generate . ClientImplementation diff --git a/pkg/filepromoter/filestore.go b/pkg/filepromoter/filestore.go index 7fdf64754a0..ba5a9431ad7 100644 --- a/pkg/filepromoter/filestore.go +++ b/pkg/filepromoter/filestore.go @@ -28,7 +28,7 @@ import ( "google.golang.org/api/option" api "k8s.io/release/pkg/api/files" - "k8s.io/release/pkg/object" + "sigs.k8s.io/release-sdk/object" ) // FilestorePromoter manages the promotion of files. diff --git a/pkg/filepromoter/gcs.go b/pkg/filepromoter/gcs.go index 1f3aa93fd7a..9330298d213 100644 --- a/pkg/filepromoter/gcs.go +++ b/pkg/filepromoter/gcs.go @@ -30,7 +30,7 @@ import ( "google.golang.org/api/iterator" api "k8s.io/release/pkg/api/files" - "k8s.io/release/pkg/object" + "sigs.k8s.io/release-sdk/object" ) type gcsSyncFilestore struct { diff --git a/pkg/gcp/auth/auth.go b/pkg/gcp/auth/auth.go index 9b63bd4a501..908ecaa599e 100644 --- a/pkg/gcp/auth/auth.go +++ b/pkg/gcp/auth/auth.go @@ -20,12 +20,11 @@ import ( "strings" "github.com/pkg/errors" - - "k8s.io/release/pkg/gcp" + "sigs.k8s.io/release-sdk/gcli" ) func GetCurrentGCPUser() (string, error) { - gcpUser, err := gcp.GCloudOutput( + gcpUser, err := gcli.GCloudOutput( "auth", "list", "--filter=status:ACTIVE", @@ -46,7 +45,7 @@ func GetCurrentGCPUser() (string, error) { } func ConfigureDocker() error { - err := gcp.GCloud( + err := gcli.GCloud( "auth", "configure-docker", ) diff --git a/pkg/gcp/build/build.go b/pkg/gcp/build/build.go index 8a2365181f0..81448a111bd 100644 --- a/pkg/gcp/build/build.go +++ b/pkg/gcp/build/build.go @@ -34,9 +34,9 @@ import ( "google.golang.org/api/cloudbuild/v1" "google.golang.org/api/option" - "k8s.io/release/pkg/gcp" - "k8s.io/release/pkg/object" "k8s.io/release/pkg/release" + "sigs.k8s.io/release-sdk/gcli" + "sigs.k8s.io/release-sdk/object" "sigs.k8s.io/release-utils/command" "sigs.k8s.io/release-utils/tar" "sigs.k8s.io/yaml" @@ -237,7 +237,7 @@ func RunSingleJob(o *Options, jobName, uploaded, version string, subs map[string args = append(args, diskSizeArg) } - cmd := command.New(gcp.GCloudExecutable, args...) + cmd := command.New(gcli.GCloudExecutable, args...) if o.LogDir != "" { p := path.Join(o.LogDir, strings.ReplaceAll(jobName, "/", "-")+".log") diff --git a/pkg/gcp/gcb/gcb.go b/pkg/gcp/gcb/gcb.go index 1dc4618d6cf..6ed35c2a78a 100644 --- a/pkg/gcp/gcb/gcb.go +++ b/pkg/gcp/gcb/gcb.go @@ -29,12 +29,12 @@ import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" - "k8s.io/release/pkg/gcp" "k8s.io/release/pkg/gcp/auth" "k8s.io/release/pkg/gcp/build" "k8s.io/release/pkg/git" "k8s.io/release/pkg/kubecross" "k8s.io/release/pkg/release" + "sigs.k8s.io/release-sdk/gcli" "sigs.k8s.io/release-utils/util" ) @@ -189,7 +189,7 @@ func (g *GCB) Submit() error { toolRepo := release.GetToolRepo() toolRef := release.GetToolRef() - if err := gcp.PreCheck(); err != nil { + if err := gcli.PreCheck(); err != nil { return errors.Wrap(err, "pre-checking for GCP package usage") } diff --git a/pkg/gcp/gcp.go b/pkg/gcp/gcp.go deleted file mode 100644 index c67d0094fac..00000000000 --- a/pkg/gcp/gcp.go +++ /dev/null @@ -1,78 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package gcp - -import ( - "github.com/pkg/errors" - - "sigs.k8s.io/release-utils/command" -) - -const ( - GCloudExecutable = "gcloud" - gsutilExecutable = "gsutil" -) - -// PreCheck checks if all requirements are fulfilled to run this package and -// all sub-packages -func PreCheck() error { - for _, e := range []string{ - GCloudExecutable, - gsutilExecutable, - } { - if !command.Available(e) { - return errors.Errorf( - "%s executable is not available in $PATH", e, - ) - } - } - - return nil -} - -// GCloud can be used to run a 'gcloud' command -func GCloud(args ...string) error { - return command.New(GCloudExecutable, args...).RunSilentSuccess() -} - -// GCloudOutput can be used to run a 'gcloud' command while capturing its output -func GCloudOutput(args ...string) (string, error) { - stream, err := command.New(GCloudExecutable, args...).RunSilentSuccessOutput() - if err != nil { - return "", errors.Wrapf(err, "executing %s", GCloudExecutable) - } - return stream.OutputTrimNL(), nil -} - -// GSUtil can be used to run a 'gsutil' command -func GSUtil(args ...string) error { - return command.New(gsutilExecutable, args...).RunSilentSuccess() -} - -// GSUtilOutput can be used to run a 'gsutil' command while capturing its output -func GSUtilOutput(args ...string) (string, error) { - stream, err := command.New(gsutilExecutable, args...).RunSilentSuccessOutput() - if err != nil { - return "", errors.Wrapf(err, "executing %s", gsutilExecutable) - } - return stream.OutputTrimNL(), nil -} - -// GSUtilStatus can be used to run a 'gsutil' command while capturing its status -func GSUtilStatus(args ...string) (*command.Status, error) { - return command.New(gsutilExecutable, args...).Run() -} diff --git a/pkg/gh2gcs/gh2gcs.go b/pkg/gh2gcs/gh2gcs.go index c84a1fe95f8..c513a708e00 100644 --- a/pkg/gh2gcs/gh2gcs.go +++ b/pkg/gh2gcs/gh2gcs.go @@ -23,7 +23,7 @@ import ( "github.com/sirupsen/logrus" "k8s.io/release/pkg/github" - "k8s.io/release/pkg/object" + "sigs.k8s.io/release-sdk/object" ) // Config contains a slice of `ReleaseConfig` to be used when unmarshalling a diff --git a/pkg/notes/notes_map.go b/pkg/notes/notes_map.go index 309363ff5f2..8d7bd64bd67 100644 --- a/pkg/notes/notes_map.go +++ b/pkg/notes/notes_map.go @@ -25,7 +25,7 @@ import ( "github.com/sirupsen/logrus" "gopkg.in/yaml.v2" - "k8s.io/release/pkg/object" + "sigs.k8s.io/release-sdk/object" ) // MapProvider interface that obtains release notes maps from a source diff --git a/pkg/object/gcs.go b/pkg/object/gcs.go deleted file mode 100644 index 242a6160103..00000000000 --- a/pkg/object/gcs.go +++ /dev/null @@ -1,419 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package object - -import ( - "os" - "path/filepath" - "strings" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - - "k8s.io/release/pkg/gcp" -) - -type GCS struct { - // gsutil options - concurrent bool - recursive bool - noClobber bool - - // local options - // AllowMissing allows a copy operation to be skipped if the source or - // destination does not exist. This is useful for scenarios where copy - // operations happen in a loop/channel, so a single "failure" does not block - // the entire operation. - allowMissing bool -} - -func NewGCS() *GCS { - return &GCS{ - concurrent: true, - recursive: true, - noClobber: true, - allowMissing: true, - } -} - -func (g *GCS) SetOptions(opts ...OptFn) { - for _, f := range opts { - f(g) - } -} - -func (g *GCS) WithConcurrent(concurrent bool) OptFn { - return func(Store) { - g.concurrent = concurrent - } -} - -func (g *GCS) WithRecursive(recursive bool) OptFn { - return func(Store) { - g.recursive = recursive - } -} - -func (g *GCS) WithNoClobber(noClobber bool) OptFn { - return func(Store) { - g.noClobber = noClobber - } -} - -func (g *GCS) WithAllowMissing(allowMissing bool) OptFn { - return func(Store) { - g.allowMissing = allowMissing - } -} - -func (g *GCS) Concurrent() bool { - return g.concurrent -} - -func (g *GCS) Recursive() bool { - return g.recursive -} - -func (g *GCS) NoClobber() bool { - return g.noClobber -} - -func (g *GCS) AllowMissing() bool { - return g.allowMissing -} - -var ( - // GcsPrefix url prefix for google cloud storage buckets - GcsPrefix = "gs://" - concurrentFlag = "-m" - recursiveFlag = "-r" - noClobberFlag = "-n" -) - -// CopyToGCS copies a local directory to the specified GCS path -func (g *GCS) CopyToRemote(src, gcsPath string) error { - logrus.Infof("Copying %s to GCS (%s)", src, gcsPath) - gcsPath, gcsPathErr := g.NormalizePath(gcsPath) - if gcsPathErr != nil { - return errors.Wrap(gcsPathErr, "normalize GCS path") - } - - _, err := os.Stat(src) - if err != nil { - logrus.Info("Unable to get local source directory info") - - if g.allowMissing { - logrus.Infof("Source directory (%s) does not exist. Skipping GCS upload.", src) - return nil - } - - return errors.New("source directory does not exist") - } - - return g.bucketCopy(src, gcsPath) -} - -// CopyToLocal copies a GCS path to the specified local directory -func (g *GCS) CopyToLocal(gcsPath, dst string) error { - logrus.Infof("Copying GCS (%s) to %s", gcsPath, dst) - gcsPath, gcsPathErr := g.NormalizePath(gcsPath) - if gcsPathErr != nil { - return errors.Wrap(gcsPathErr, "normalize GCS path") - } - - return g.bucketCopy(gcsPath, dst) -} - -// CopyBucketToBucket copies between two GCS paths. -func (g *GCS) CopyBucketToBucket(src, dst string) error { - logrus.Infof("Copying %s to %s", src, dst) - - src, srcErr := g.NormalizePath(src) - if srcErr != nil { - return errors.Wrap(srcErr, "normalize GCS path") - } - - dst, dstErr := g.NormalizePath(dst) - if dstErr != nil { - return errors.Wrap(dstErr, "normalize GCS path") - } - - return g.bucketCopy(src, dst) -} - -func (g *GCS) bucketCopy(src, dst string) error { - args := []string{} - - if g.concurrent { - logrus.Debug("Setting GCS copy to run concurrently") - args = append(args, concurrentFlag) - } - - args = append(args, "cp") - if g.recursive { - logrus.Debug("Setting GCS copy to run recursively") - args = append(args, recursiveFlag) - } - if g.noClobber { - logrus.Debug("Setting GCS copy to not clobber existing files") - args = append(args, noClobberFlag) - } - - args = append(args, src, dst) - - if err := gcp.GSUtil(args...); err != nil { - return errors.Wrap(err, "gcs copy") - } - - return nil -} - -// GetReleasePath returns a GCS path to retrieve builds from or push builds to -// -// Expected destination format: -// gs:///[/fast][/] -func (g *GCS) GetReleasePath( - bucket, gcsRoot, version string, - fast bool) (string, error) { - gcsPath, err := g.getPath( - bucket, - gcsRoot, - version, - "release", - fast, - ) - if err != nil { - return "", errors.Wrap(err, "normalize GCS path") - } - - logrus.Infof("Release path is %s", gcsPath) - return gcsPath, nil -} - -// GetMarkerPath returns a GCS path where version markers should be stored -// -// Expected destination format: -// gs:/// -func (g *GCS) GetMarkerPath( - bucket, gcsRoot string) (string, error) { - gcsPath, err := g.getPath( - bucket, - gcsRoot, - "", - "marker", - false, - ) - if err != nil { - return "", errors.Wrap(err, "normalize GCS path") - } - - logrus.Infof("Version marker path is %s", gcsPath) - return gcsPath, nil -} - -// GetReleasePath returns a GCS path to retrieve builds from or push builds to -// -// Expected destination format: -// gs:///[/fast][/] -// TODO: Support "release" buildType -func (g *GCS) getPath( - bucket, gcsRoot, version, pathType string, - fast bool) (string, error) { - if gcsRoot == "" { - return "", errors.New("GCS root must be specified") - } - - gcsPathParts := []string{} - - gcsPathParts = append(gcsPathParts, bucket, gcsRoot) - - if pathType == "release" { - if fast { - gcsPathParts = append(gcsPathParts, "fast") - } - - if version != "" { - gcsPathParts = append(gcsPathParts, version) - } - } else if pathType == "marker" { - } else { - return "", errors.New("a GCS path type must be specified") - } - - // Ensure any constructed GCS path is prefixed with `gs://` - return g.NormalizePath(gcsPathParts...) -} - -// NormalizeGCSPath takes a GCS path and ensures that the `GcsPrefix` is -// prepended to it. -// TODO: Should there be an append function for paths to prevent multiple calls -// like in build.checkBuildExists()? -func (g *GCS) NormalizePath(gcsPathParts ...string) (string, error) { - gcsPath := "" - - // Ensure there is at least one element in the gcsPathParts slice before - // trying to construct a path - if len(gcsPathParts) == 0 { - return "", errors.New("must contain at least one path part") - } else if len(gcsPathParts) == 1 { - if gcsPathParts[0] == "" { - return "", errors.New("path should not be an empty string") - } - - gcsPath = gcsPathParts[0] - } else { - var emptyParts int - - for i, part := range gcsPathParts { - if part == "" { - emptyParts++ - } - - if i == 0 { - continue - } - - if strings.Contains(part, "gs:/") { - return "", errors.New("one of the GCS path parts contained a `gs:/`, which may suggest a filepath.Join() error in the caller") - } - - if i == len(gcsPathParts)-1 && emptyParts == len(gcsPathParts) { - return "", errors.New("all paths provided were empty") - } - } - - gcsPath = filepath.Join(gcsPathParts...) - } - - // Strip `gs://` if it was included in gcsPathParts - gcsPath = strings.TrimPrefix(gcsPath, GcsPrefix) - - // Strip `gs:/` if: - // - `gs://` was included in gcsPathParts - // - gcsPathParts had more than element - // - filepath.Join() was called somewhere in a caller's logic - gcsPath = strings.TrimPrefix(gcsPath, "gs:/") - - // Strip `/` - // This scenario may never happen, but let's catch it, just in case - gcsPath = strings.TrimPrefix(gcsPath, "/") - - gcsPath = GcsPrefix + gcsPath - - isNormalized := g.IsPathNormalized(gcsPath) - if !isNormalized { - return gcsPath, errors.New("unknown error while trying to normalize GCS path") - } - - return gcsPath, nil -} - -// IsPathNormalized determines if a GCS path is prefixed with `gs://`. -// Use this function as pre-check for any gsutil/GCS functions that manipulate -// GCS bucket contents. -func (g *GCS) IsPathNormalized(gcsPath string) bool { - var errCount int - - if !strings.HasPrefix(gcsPath, GcsPrefix) { - logrus.Errorf( - "GCS path (%s) should be prefixed with `%s`", gcsPath, GcsPrefix, - ) - errCount++ - } - - strippedPath := strings.TrimPrefix(gcsPath, GcsPrefix) - if strings.Contains(strippedPath, "gs:/") { - logrus.Errorf("GCS path (%s) should be prefixed with `gs:/`", gcsPath) - errCount++ - } - - // TODO: Add logic to handle invalid path characters - - if errCount > 0 { - return false - } - - return true -} - -// RsyncRecursive runs `gsutil rsync` in recursive mode. The caller of this -// function has to ensure that the provided paths are prefixed with gs:// if -// necessary (see `NormalizePath()`). -func (g *GCS) RsyncRecursive(src, dst string) error { - return errors.Wrap( - gcp.GSUtil(concurrentFlag, "rsync", recursiveFlag, src, dst), - "running gsutil rsync", - ) -} - -// PathExists returns true if the specified GCS path exists. -func (g *GCS) PathExists(gcsPath string) (bool, error) { - if !g.IsPathNormalized(gcsPath) { - return false, errors.Errorf( - "cannot run `gsutil ls` GCS path does not begin with `%s`", - GcsPrefix, - ) - } - - // Do an ls with gsutil to check if the file exists: - if err := gcp.GSUtil( - "ls", - gcsPath, - ); err != nil { - // .. but check the message because if not found - // it will exit with an error - if strings.Contains(err.Error(), "One or more URLs matched no objects") { - return false, nil - } - // Anything else we treat as error - return false, err - } - - logrus.Infof("Found %s", gcsPath) - return true, nil -} - -// DeletePath deletes a bucket location recursevly -func (g *GCS) DeletePath(path string) error { - path, err := g.NormalizePath(path) - if err != nil { - return errors.Wrap(err, "normalizing GCS path") - } - - // Build the command arguments - args := []string{"-q"} - - if g.concurrent { - logrus.Debug("Setting GCS copy to run concurrently") - args = append(args, concurrentFlag) - } - - args = append(args, "rm") - - if g.recursive { - logrus.Debug("Setting GCS copy to run recursively") - args = append(args, recursiveFlag) - } - - args = append(args, path) - - // Cal gsutil to remove the path - if err = gcp.GSUtil(args...); err != nil { - return errors.Wrap(err, "while calling gsutil to remove path") - } - return nil -} diff --git a/pkg/object/gcs_test.go b/pkg/object/gcs_test.go deleted file mode 100644 index 7d1367d94cc..00000000000 --- a/pkg/object/gcs_test.go +++ /dev/null @@ -1,234 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package object_test - -import ( - "testing" - - "github.com/stretchr/testify/require" - - "k8s.io/release/pkg/object" -) - -var testGCS = object.NewGCS() - -func TestGCSSetOptions(t *testing.T) { - for _, tc := range []struct { - name string - gcs *object.GCS - opt bool - expected bool - }{ - { - name: "should be false", - gcs: testGCS, - opt: false, - expected: false, - }, - } { - t.Logf("test case: %v", tc.name) - - testGCS.SetOptions( - testGCS.WithConcurrent(tc.opt), - testGCS.WithRecursive(tc.opt), - testGCS.WithNoClobber(tc.opt), - testGCS.WithAllowMissing(tc.opt), - ) - - require.Equal(t, tc.expected, testGCS.Concurrent()) - require.Equal(t, tc.expected, testGCS.Recursive()) - require.Equal(t, tc.expected, testGCS.NoClobber()) - require.Equal(t, tc.expected, testGCS.AllowMissing()) - } -} - -// TODO: Add production use cases -func TestGetReleasePath(t *testing.T) { - for _, tc := range []struct { - bucket, gcsRoot, version string - expected string - fast bool - shouldError bool - }{ - { // default CI build - bucket: "k8s-release-dev", - gcsRoot: "ci", - expected: "gs://k8s-release-dev/ci", - shouldError: false, - }, - { // fast CI build - bucket: "k8s-release-dev", - gcsRoot: "ci", - version: "", - fast: true, - expected: "gs://k8s-release-dev/ci/fast", - shouldError: false, - }, - { // has version - bucket: "k8s-release-dev", - gcsRoot: "ci", - version: "42", - fast: true, - expected: "gs://k8s-release-dev/ci/fast/42", - shouldError: false, - }, - } { - actual, err := testGCS.GetReleasePath( - tc.bucket, - tc.gcsRoot, - tc.version, - tc.fast, - ) - - require.Equal(t, tc.expected, actual) - - if tc.shouldError { - require.NotNil(t, err) - } else { - require.Nil(t, err) - } - } -} - -// TODO: Add production use cases -func TestGetMarkerPath(t *testing.T) { - for _, tc := range []struct { - bucket, gcsRoot string - expected string - shouldError bool - }{ - { // default CI build - bucket: "k8s-release-dev", - gcsRoot: "ci", - expected: "gs://k8s-release-dev/ci", - shouldError: false, - }, - } { - actual, err := testGCS.GetMarkerPath( - tc.bucket, - tc.gcsRoot, - ) - - require.Equal(t, tc.expected, actual) - - if tc.shouldError { - require.NotNil(t, err) - } else { - require.Nil(t, err) - } - } -} - -func TestNormalizePath(t *testing.T) { - for _, tc := range []struct { - gcsPathParts []string - expected string - shouldError bool - }{ - { // empty parts - gcsPathParts: []string{}, - expected: "", - shouldError: true, - }, - { // one element, but empty string - gcsPathParts: []string{ - "", - }, - expected: "", - shouldError: true, - }, - { // multiple elements, all empty strings - gcsPathParts: []string{ - "", - "", - }, - expected: "", - shouldError: true, - }, - { // strip `gs://` properly - gcsPathParts: []string{ - "gs://foo", - }, - expected: "gs://foo", - shouldError: false, - }, - { // strip `gs:/` properly - gcsPathParts: []string{ - "gs://foo/bar", - }, - expected: "gs://foo/bar", - shouldError: false, - }, - { // strip `/` properly - gcsPathParts: []string{ - "/foo/bar", - }, - expected: "gs://foo/bar", - shouldError: false, - }, - { // multiple parts - gcsPathParts: []string{ - "foo", - "bar", - }, - expected: "gs://foo/bar", - shouldError: false, - }, - { // one of the non-zero parts already contains the `gs://` prefix - gcsPathParts: []string{ - "k8s-release-dev", - "gs://k8s-release-dev/ci-no-bootstrap/fast/v1.20.0-beta.1.655+d20e3246bade17", - }, - expected: "", - shouldError: true, - }, - } { - actual, err := testGCS.NormalizePath(tc.gcsPathParts...) - - require.Equal(t, tc.expected, actual) - - if tc.shouldError { - require.NotNil(t, err) - } else { - require.Nil(t, err) - } - } -} - -func TestIsPathNormalized(t *testing.T) { - for _, tc := range []struct { - gcsPath string - expected bool - }{ - { // GCS path (%s) should be prefixed with `gs://` - gcsPath: "k8s-release-dev/g", - expected: false, - }, - { // filepath.Join() caused an additional `gs:/` to be included in the path - gcsPath: "gs://k8s-release-dev/gs:/k8s-release-dev/ci-no-bootstrap/fast/v1.20.0-beta.1.655+d20e3246bade17", - expected: false, - }, - { // fast CI build - gcsPath: "gs://k8s-release-dev/ci/fast", - expected: true, - }, - } { - actual := testGCS.IsPathNormalized(tc.gcsPath) - - require.Equal(t, tc.expected, actual) - } -} diff --git a/pkg/object/objectfakes/fake_store.go b/pkg/object/objectfakes/fake_store.go deleted file mode 100644 index d2981280873..00000000000 --- a/pkg/object/objectfakes/fake_store.go +++ /dev/null @@ -1,794 +0,0 @@ -/* -Copyright The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by counterfeiter. DO NOT EDIT. -package objectfakes - -import ( - "sync" - - "k8s.io/release/pkg/object" -) - -type FakeStore struct { - CopyBucketToBucketStub func(string, string) error - copyBucketToBucketMutex sync.RWMutex - copyBucketToBucketArgsForCall []struct { - arg1 string - arg2 string - } - copyBucketToBucketReturns struct { - result1 error - } - copyBucketToBucketReturnsOnCall map[int]struct { - result1 error - } - CopyToLocalStub func(string, string) error - copyToLocalMutex sync.RWMutex - copyToLocalArgsForCall []struct { - arg1 string - arg2 string - } - copyToLocalReturns struct { - result1 error - } - copyToLocalReturnsOnCall map[int]struct { - result1 error - } - CopyToRemoteStub func(string, string) error - copyToRemoteMutex sync.RWMutex - copyToRemoteArgsForCall []struct { - arg1 string - arg2 string - } - copyToRemoteReturns struct { - result1 error - } - copyToRemoteReturnsOnCall map[int]struct { - result1 error - } - GetMarkerPathStub func(string, string) (string, error) - getMarkerPathMutex sync.RWMutex - getMarkerPathArgsForCall []struct { - arg1 string - arg2 string - } - getMarkerPathReturns struct { - result1 string - result2 error - } - getMarkerPathReturnsOnCall map[int]struct { - result1 string - result2 error - } - GetReleasePathStub func(string, string, string, bool) (string, error) - getReleasePathMutex sync.RWMutex - getReleasePathArgsForCall []struct { - arg1 string - arg2 string - arg3 string - arg4 bool - } - getReleasePathReturns struct { - result1 string - result2 error - } - getReleasePathReturnsOnCall map[int]struct { - result1 string - result2 error - } - IsPathNormalizedStub func(string) bool - isPathNormalizedMutex sync.RWMutex - isPathNormalizedArgsForCall []struct { - arg1 string - } - isPathNormalizedReturns struct { - result1 bool - } - isPathNormalizedReturnsOnCall map[int]struct { - result1 bool - } - NormalizePathStub func(...string) (string, error) - normalizePathMutex sync.RWMutex - normalizePathArgsForCall []struct { - arg1 []string - } - normalizePathReturns struct { - result1 string - result2 error - } - normalizePathReturnsOnCall map[int]struct { - result1 string - result2 error - } - PathExistsStub func(string) (bool, error) - pathExistsMutex sync.RWMutex - pathExistsArgsForCall []struct { - arg1 string - } - pathExistsReturns struct { - result1 bool - result2 error - } - pathExistsReturnsOnCall map[int]struct { - result1 bool - result2 error - } - RsyncRecursiveStub func(string, string) error - rsyncRecursiveMutex sync.RWMutex - rsyncRecursiveArgsForCall []struct { - arg1 string - arg2 string - } - rsyncRecursiveReturns struct { - result1 error - } - rsyncRecursiveReturnsOnCall map[int]struct { - result1 error - } - SetOptionsStub func(...object.OptFn) - setOptionsMutex sync.RWMutex - setOptionsArgsForCall []struct { - arg1 []object.OptFn - } - invocations map[string][][]interface{} - invocationsMutex sync.RWMutex -} - -func (fake *FakeStore) CopyBucketToBucket(arg1 string, arg2 string) error { - fake.copyBucketToBucketMutex.Lock() - ret, specificReturn := fake.copyBucketToBucketReturnsOnCall[len(fake.copyBucketToBucketArgsForCall)] - fake.copyBucketToBucketArgsForCall = append(fake.copyBucketToBucketArgsForCall, struct { - arg1 string - arg2 string - }{arg1, arg2}) - stub := fake.CopyBucketToBucketStub - fakeReturns := fake.copyBucketToBucketReturns - fake.recordInvocation("CopyBucketToBucket", []interface{}{arg1, arg2}) - fake.copyBucketToBucketMutex.Unlock() - if stub != nil { - return stub(arg1, arg2) - } - if specificReturn { - return ret.result1 - } - return fakeReturns.result1 -} - -func (fake *FakeStore) CopyBucketToBucketCallCount() int { - fake.copyBucketToBucketMutex.RLock() - defer fake.copyBucketToBucketMutex.RUnlock() - return len(fake.copyBucketToBucketArgsForCall) -} - -func (fake *FakeStore) CopyBucketToBucketCalls(stub func(string, string) error) { - fake.copyBucketToBucketMutex.Lock() - defer fake.copyBucketToBucketMutex.Unlock() - fake.CopyBucketToBucketStub = stub -} - -func (fake *FakeStore) CopyBucketToBucketArgsForCall(i int) (string, string) { - fake.copyBucketToBucketMutex.RLock() - defer fake.copyBucketToBucketMutex.RUnlock() - argsForCall := fake.copyBucketToBucketArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2 -} - -func (fake *FakeStore) CopyBucketToBucketReturns(result1 error) { - fake.copyBucketToBucketMutex.Lock() - defer fake.copyBucketToBucketMutex.Unlock() - fake.CopyBucketToBucketStub = nil - fake.copyBucketToBucketReturns = struct { - result1 error - }{result1} -} - -func (fake *FakeStore) CopyBucketToBucketReturnsOnCall(i int, result1 error) { - fake.copyBucketToBucketMutex.Lock() - defer fake.copyBucketToBucketMutex.Unlock() - fake.CopyBucketToBucketStub = nil - if fake.copyBucketToBucketReturnsOnCall == nil { - fake.copyBucketToBucketReturnsOnCall = make(map[int]struct { - result1 error - }) - } - fake.copyBucketToBucketReturnsOnCall[i] = struct { - result1 error - }{result1} -} - -func (fake *FakeStore) CopyToLocal(arg1 string, arg2 string) error { - fake.copyToLocalMutex.Lock() - ret, specificReturn := fake.copyToLocalReturnsOnCall[len(fake.copyToLocalArgsForCall)] - fake.copyToLocalArgsForCall = append(fake.copyToLocalArgsForCall, struct { - arg1 string - arg2 string - }{arg1, arg2}) - stub := fake.CopyToLocalStub - fakeReturns := fake.copyToLocalReturns - fake.recordInvocation("CopyToLocal", []interface{}{arg1, arg2}) - fake.copyToLocalMutex.Unlock() - if stub != nil { - return stub(arg1, arg2) - } - if specificReturn { - return ret.result1 - } - return fakeReturns.result1 -} - -func (fake *FakeStore) CopyToLocalCallCount() int { - fake.copyToLocalMutex.RLock() - defer fake.copyToLocalMutex.RUnlock() - return len(fake.copyToLocalArgsForCall) -} - -func (fake *FakeStore) CopyToLocalCalls(stub func(string, string) error) { - fake.copyToLocalMutex.Lock() - defer fake.copyToLocalMutex.Unlock() - fake.CopyToLocalStub = stub -} - -func (fake *FakeStore) CopyToLocalArgsForCall(i int) (string, string) { - fake.copyToLocalMutex.RLock() - defer fake.copyToLocalMutex.RUnlock() - argsForCall := fake.copyToLocalArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2 -} - -func (fake *FakeStore) CopyToLocalReturns(result1 error) { - fake.copyToLocalMutex.Lock() - defer fake.copyToLocalMutex.Unlock() - fake.CopyToLocalStub = nil - fake.copyToLocalReturns = struct { - result1 error - }{result1} -} - -func (fake *FakeStore) CopyToLocalReturnsOnCall(i int, result1 error) { - fake.copyToLocalMutex.Lock() - defer fake.copyToLocalMutex.Unlock() - fake.CopyToLocalStub = nil - if fake.copyToLocalReturnsOnCall == nil { - fake.copyToLocalReturnsOnCall = make(map[int]struct { - result1 error - }) - } - fake.copyToLocalReturnsOnCall[i] = struct { - result1 error - }{result1} -} - -func (fake *FakeStore) CopyToRemote(arg1 string, arg2 string) error { - fake.copyToRemoteMutex.Lock() - ret, specificReturn := fake.copyToRemoteReturnsOnCall[len(fake.copyToRemoteArgsForCall)] - fake.copyToRemoteArgsForCall = append(fake.copyToRemoteArgsForCall, struct { - arg1 string - arg2 string - }{arg1, arg2}) - stub := fake.CopyToRemoteStub - fakeReturns := fake.copyToRemoteReturns - fake.recordInvocation("CopyToRemote", []interface{}{arg1, arg2}) - fake.copyToRemoteMutex.Unlock() - if stub != nil { - return stub(arg1, arg2) - } - if specificReturn { - return ret.result1 - } - return fakeReturns.result1 -} - -func (fake *FakeStore) CopyToRemoteCallCount() int { - fake.copyToRemoteMutex.RLock() - defer fake.copyToRemoteMutex.RUnlock() - return len(fake.copyToRemoteArgsForCall) -} - -func (fake *FakeStore) CopyToRemoteCalls(stub func(string, string) error) { - fake.copyToRemoteMutex.Lock() - defer fake.copyToRemoteMutex.Unlock() - fake.CopyToRemoteStub = stub -} - -func (fake *FakeStore) CopyToRemoteArgsForCall(i int) (string, string) { - fake.copyToRemoteMutex.RLock() - defer fake.copyToRemoteMutex.RUnlock() - argsForCall := fake.copyToRemoteArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2 -} - -func (fake *FakeStore) CopyToRemoteReturns(result1 error) { - fake.copyToRemoteMutex.Lock() - defer fake.copyToRemoteMutex.Unlock() - fake.CopyToRemoteStub = nil - fake.copyToRemoteReturns = struct { - result1 error - }{result1} -} - -func (fake *FakeStore) CopyToRemoteReturnsOnCall(i int, result1 error) { - fake.copyToRemoteMutex.Lock() - defer fake.copyToRemoteMutex.Unlock() - fake.CopyToRemoteStub = nil - if fake.copyToRemoteReturnsOnCall == nil { - fake.copyToRemoteReturnsOnCall = make(map[int]struct { - result1 error - }) - } - fake.copyToRemoteReturnsOnCall[i] = struct { - result1 error - }{result1} -} - -func (fake *FakeStore) GetMarkerPath(arg1 string, arg2 string) (string, error) { - fake.getMarkerPathMutex.Lock() - ret, specificReturn := fake.getMarkerPathReturnsOnCall[len(fake.getMarkerPathArgsForCall)] - fake.getMarkerPathArgsForCall = append(fake.getMarkerPathArgsForCall, struct { - arg1 string - arg2 string - }{arg1, arg2}) - stub := fake.GetMarkerPathStub - fakeReturns := fake.getMarkerPathReturns - fake.recordInvocation("GetMarkerPath", []interface{}{arg1, arg2}) - fake.getMarkerPathMutex.Unlock() - if stub != nil { - return stub(arg1, arg2) - } - if specificReturn { - return ret.result1, ret.result2 - } - return fakeReturns.result1, fakeReturns.result2 -} - -func (fake *FakeStore) GetMarkerPathCallCount() int { - fake.getMarkerPathMutex.RLock() - defer fake.getMarkerPathMutex.RUnlock() - return len(fake.getMarkerPathArgsForCall) -} - -func (fake *FakeStore) GetMarkerPathCalls(stub func(string, string) (string, error)) { - fake.getMarkerPathMutex.Lock() - defer fake.getMarkerPathMutex.Unlock() - fake.GetMarkerPathStub = stub -} - -func (fake *FakeStore) GetMarkerPathArgsForCall(i int) (string, string) { - fake.getMarkerPathMutex.RLock() - defer fake.getMarkerPathMutex.RUnlock() - argsForCall := fake.getMarkerPathArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2 -} - -func (fake *FakeStore) GetMarkerPathReturns(result1 string, result2 error) { - fake.getMarkerPathMutex.Lock() - defer fake.getMarkerPathMutex.Unlock() - fake.GetMarkerPathStub = nil - fake.getMarkerPathReturns = struct { - result1 string - result2 error - }{result1, result2} -} - -func (fake *FakeStore) GetMarkerPathReturnsOnCall(i int, result1 string, result2 error) { - fake.getMarkerPathMutex.Lock() - defer fake.getMarkerPathMutex.Unlock() - fake.GetMarkerPathStub = nil - if fake.getMarkerPathReturnsOnCall == nil { - fake.getMarkerPathReturnsOnCall = make(map[int]struct { - result1 string - result2 error - }) - } - fake.getMarkerPathReturnsOnCall[i] = struct { - result1 string - result2 error - }{result1, result2} -} - -func (fake *FakeStore) GetReleasePath(arg1 string, arg2 string, arg3 string, arg4 bool) (string, error) { - fake.getReleasePathMutex.Lock() - ret, specificReturn := fake.getReleasePathReturnsOnCall[len(fake.getReleasePathArgsForCall)] - fake.getReleasePathArgsForCall = append(fake.getReleasePathArgsForCall, struct { - arg1 string - arg2 string - arg3 string - arg4 bool - }{arg1, arg2, arg3, arg4}) - stub := fake.GetReleasePathStub - fakeReturns := fake.getReleasePathReturns - fake.recordInvocation("GetReleasePath", []interface{}{arg1, arg2, arg3, arg4}) - fake.getReleasePathMutex.Unlock() - if stub != nil { - return stub(arg1, arg2, arg3, arg4) - } - if specificReturn { - return ret.result1, ret.result2 - } - return fakeReturns.result1, fakeReturns.result2 -} - -func (fake *FakeStore) GetReleasePathCallCount() int { - fake.getReleasePathMutex.RLock() - defer fake.getReleasePathMutex.RUnlock() - return len(fake.getReleasePathArgsForCall) -} - -func (fake *FakeStore) GetReleasePathCalls(stub func(string, string, string, bool) (string, error)) { - fake.getReleasePathMutex.Lock() - defer fake.getReleasePathMutex.Unlock() - fake.GetReleasePathStub = stub -} - -func (fake *FakeStore) GetReleasePathArgsForCall(i int) (string, string, string, bool) { - fake.getReleasePathMutex.RLock() - defer fake.getReleasePathMutex.RUnlock() - argsForCall := fake.getReleasePathArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4 -} - -func (fake *FakeStore) GetReleasePathReturns(result1 string, result2 error) { - fake.getReleasePathMutex.Lock() - defer fake.getReleasePathMutex.Unlock() - fake.GetReleasePathStub = nil - fake.getReleasePathReturns = struct { - result1 string - result2 error - }{result1, result2} -} - -func (fake *FakeStore) GetReleasePathReturnsOnCall(i int, result1 string, result2 error) { - fake.getReleasePathMutex.Lock() - defer fake.getReleasePathMutex.Unlock() - fake.GetReleasePathStub = nil - if fake.getReleasePathReturnsOnCall == nil { - fake.getReleasePathReturnsOnCall = make(map[int]struct { - result1 string - result2 error - }) - } - fake.getReleasePathReturnsOnCall[i] = struct { - result1 string - result2 error - }{result1, result2} -} - -func (fake *FakeStore) IsPathNormalized(arg1 string) bool { - fake.isPathNormalizedMutex.Lock() - ret, specificReturn := fake.isPathNormalizedReturnsOnCall[len(fake.isPathNormalizedArgsForCall)] - fake.isPathNormalizedArgsForCall = append(fake.isPathNormalizedArgsForCall, struct { - arg1 string - }{arg1}) - stub := fake.IsPathNormalizedStub - fakeReturns := fake.isPathNormalizedReturns - fake.recordInvocation("IsPathNormalized", []interface{}{arg1}) - fake.isPathNormalizedMutex.Unlock() - if stub != nil { - return stub(arg1) - } - if specificReturn { - return ret.result1 - } - return fakeReturns.result1 -} - -func (fake *FakeStore) IsPathNormalizedCallCount() int { - fake.isPathNormalizedMutex.RLock() - defer fake.isPathNormalizedMutex.RUnlock() - return len(fake.isPathNormalizedArgsForCall) -} - -func (fake *FakeStore) IsPathNormalizedCalls(stub func(string) bool) { - fake.isPathNormalizedMutex.Lock() - defer fake.isPathNormalizedMutex.Unlock() - fake.IsPathNormalizedStub = stub -} - -func (fake *FakeStore) IsPathNormalizedArgsForCall(i int) string { - fake.isPathNormalizedMutex.RLock() - defer fake.isPathNormalizedMutex.RUnlock() - argsForCall := fake.isPathNormalizedArgsForCall[i] - return argsForCall.arg1 -} - -func (fake *FakeStore) IsPathNormalizedReturns(result1 bool) { - fake.isPathNormalizedMutex.Lock() - defer fake.isPathNormalizedMutex.Unlock() - fake.IsPathNormalizedStub = nil - fake.isPathNormalizedReturns = struct { - result1 bool - }{result1} -} - -func (fake *FakeStore) IsPathNormalizedReturnsOnCall(i int, result1 bool) { - fake.isPathNormalizedMutex.Lock() - defer fake.isPathNormalizedMutex.Unlock() - fake.IsPathNormalizedStub = nil - if fake.isPathNormalizedReturnsOnCall == nil { - fake.isPathNormalizedReturnsOnCall = make(map[int]struct { - result1 bool - }) - } - fake.isPathNormalizedReturnsOnCall[i] = struct { - result1 bool - }{result1} -} - -func (fake *FakeStore) NormalizePath(arg1 ...string) (string, error) { - fake.normalizePathMutex.Lock() - ret, specificReturn := fake.normalizePathReturnsOnCall[len(fake.normalizePathArgsForCall)] - fake.normalizePathArgsForCall = append(fake.normalizePathArgsForCall, struct { - arg1 []string - }{arg1}) - stub := fake.NormalizePathStub - fakeReturns := fake.normalizePathReturns - fake.recordInvocation("NormalizePath", []interface{}{arg1}) - fake.normalizePathMutex.Unlock() - if stub != nil { - return stub(arg1...) - } - if specificReturn { - return ret.result1, ret.result2 - } - return fakeReturns.result1, fakeReturns.result2 -} - -func (fake *FakeStore) NormalizePathCallCount() int { - fake.normalizePathMutex.RLock() - defer fake.normalizePathMutex.RUnlock() - return len(fake.normalizePathArgsForCall) -} - -func (fake *FakeStore) NormalizePathCalls(stub func(...string) (string, error)) { - fake.normalizePathMutex.Lock() - defer fake.normalizePathMutex.Unlock() - fake.NormalizePathStub = stub -} - -func (fake *FakeStore) NormalizePathArgsForCall(i int) []string { - fake.normalizePathMutex.RLock() - defer fake.normalizePathMutex.RUnlock() - argsForCall := fake.normalizePathArgsForCall[i] - return argsForCall.arg1 -} - -func (fake *FakeStore) NormalizePathReturns(result1 string, result2 error) { - fake.normalizePathMutex.Lock() - defer fake.normalizePathMutex.Unlock() - fake.NormalizePathStub = nil - fake.normalizePathReturns = struct { - result1 string - result2 error - }{result1, result2} -} - -func (fake *FakeStore) NormalizePathReturnsOnCall(i int, result1 string, result2 error) { - fake.normalizePathMutex.Lock() - defer fake.normalizePathMutex.Unlock() - fake.NormalizePathStub = nil - if fake.normalizePathReturnsOnCall == nil { - fake.normalizePathReturnsOnCall = make(map[int]struct { - result1 string - result2 error - }) - } - fake.normalizePathReturnsOnCall[i] = struct { - result1 string - result2 error - }{result1, result2} -} - -func (fake *FakeStore) PathExists(arg1 string) (bool, error) { - fake.pathExistsMutex.Lock() - ret, specificReturn := fake.pathExistsReturnsOnCall[len(fake.pathExistsArgsForCall)] - fake.pathExistsArgsForCall = append(fake.pathExistsArgsForCall, struct { - arg1 string - }{arg1}) - stub := fake.PathExistsStub - fakeReturns := fake.pathExistsReturns - fake.recordInvocation("PathExists", []interface{}{arg1}) - fake.pathExistsMutex.Unlock() - if stub != nil { - return stub(arg1) - } - if specificReturn { - return ret.result1, ret.result2 - } - return fakeReturns.result1, fakeReturns.result2 -} - -func (fake *FakeStore) PathExistsCallCount() int { - fake.pathExistsMutex.RLock() - defer fake.pathExistsMutex.RUnlock() - return len(fake.pathExistsArgsForCall) -} - -func (fake *FakeStore) PathExistsCalls(stub func(string) (bool, error)) { - fake.pathExistsMutex.Lock() - defer fake.pathExistsMutex.Unlock() - fake.PathExistsStub = stub -} - -func (fake *FakeStore) PathExistsArgsForCall(i int) string { - fake.pathExistsMutex.RLock() - defer fake.pathExistsMutex.RUnlock() - argsForCall := fake.pathExistsArgsForCall[i] - return argsForCall.arg1 -} - -func (fake *FakeStore) PathExistsReturns(result1 bool, result2 error) { - fake.pathExistsMutex.Lock() - defer fake.pathExistsMutex.Unlock() - fake.PathExistsStub = nil - fake.pathExistsReturns = struct { - result1 bool - result2 error - }{result1, result2} -} - -func (fake *FakeStore) PathExistsReturnsOnCall(i int, result1 bool, result2 error) { - fake.pathExistsMutex.Lock() - defer fake.pathExistsMutex.Unlock() - fake.PathExistsStub = nil - if fake.pathExistsReturnsOnCall == nil { - fake.pathExistsReturnsOnCall = make(map[int]struct { - result1 bool - result2 error - }) - } - fake.pathExistsReturnsOnCall[i] = struct { - result1 bool - result2 error - }{result1, result2} -} - -func (fake *FakeStore) RsyncRecursive(arg1 string, arg2 string) error { - fake.rsyncRecursiveMutex.Lock() - ret, specificReturn := fake.rsyncRecursiveReturnsOnCall[len(fake.rsyncRecursiveArgsForCall)] - fake.rsyncRecursiveArgsForCall = append(fake.rsyncRecursiveArgsForCall, struct { - arg1 string - arg2 string - }{arg1, arg2}) - stub := fake.RsyncRecursiveStub - fakeReturns := fake.rsyncRecursiveReturns - fake.recordInvocation("RsyncRecursive", []interface{}{arg1, arg2}) - fake.rsyncRecursiveMutex.Unlock() - if stub != nil { - return stub(arg1, arg2) - } - if specificReturn { - return ret.result1 - } - return fakeReturns.result1 -} - -func (fake *FakeStore) RsyncRecursiveCallCount() int { - fake.rsyncRecursiveMutex.RLock() - defer fake.rsyncRecursiveMutex.RUnlock() - return len(fake.rsyncRecursiveArgsForCall) -} - -func (fake *FakeStore) RsyncRecursiveCalls(stub func(string, string) error) { - fake.rsyncRecursiveMutex.Lock() - defer fake.rsyncRecursiveMutex.Unlock() - fake.RsyncRecursiveStub = stub -} - -func (fake *FakeStore) RsyncRecursiveArgsForCall(i int) (string, string) { - fake.rsyncRecursiveMutex.RLock() - defer fake.rsyncRecursiveMutex.RUnlock() - argsForCall := fake.rsyncRecursiveArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2 -} - -func (fake *FakeStore) RsyncRecursiveReturns(result1 error) { - fake.rsyncRecursiveMutex.Lock() - defer fake.rsyncRecursiveMutex.Unlock() - fake.RsyncRecursiveStub = nil - fake.rsyncRecursiveReturns = struct { - result1 error - }{result1} -} - -func (fake *FakeStore) RsyncRecursiveReturnsOnCall(i int, result1 error) { - fake.rsyncRecursiveMutex.Lock() - defer fake.rsyncRecursiveMutex.Unlock() - fake.RsyncRecursiveStub = nil - if fake.rsyncRecursiveReturnsOnCall == nil { - fake.rsyncRecursiveReturnsOnCall = make(map[int]struct { - result1 error - }) - } - fake.rsyncRecursiveReturnsOnCall[i] = struct { - result1 error - }{result1} -} - -func (fake *FakeStore) SetOptions(arg1 ...object.OptFn) { - fake.setOptionsMutex.Lock() - fake.setOptionsArgsForCall = append(fake.setOptionsArgsForCall, struct { - arg1 []object.OptFn - }{arg1}) - stub := fake.SetOptionsStub - fake.recordInvocation("SetOptions", []interface{}{arg1}) - fake.setOptionsMutex.Unlock() - if stub != nil { - fake.SetOptionsStub(arg1...) - } -} - -func (fake *FakeStore) SetOptionsCallCount() int { - fake.setOptionsMutex.RLock() - defer fake.setOptionsMutex.RUnlock() - return len(fake.setOptionsArgsForCall) -} - -func (fake *FakeStore) SetOptionsCalls(stub func(...object.OptFn)) { - fake.setOptionsMutex.Lock() - defer fake.setOptionsMutex.Unlock() - fake.SetOptionsStub = stub -} - -func (fake *FakeStore) SetOptionsArgsForCall(i int) []object.OptFn { - fake.setOptionsMutex.RLock() - defer fake.setOptionsMutex.RUnlock() - argsForCall := fake.setOptionsArgsForCall[i] - return argsForCall.arg1 -} - -func (fake *FakeStore) Invocations() map[string][][]interface{} { - fake.invocationsMutex.RLock() - defer fake.invocationsMutex.RUnlock() - fake.copyBucketToBucketMutex.RLock() - defer fake.copyBucketToBucketMutex.RUnlock() - fake.copyToLocalMutex.RLock() - defer fake.copyToLocalMutex.RUnlock() - fake.copyToRemoteMutex.RLock() - defer fake.copyToRemoteMutex.RUnlock() - fake.getMarkerPathMutex.RLock() - defer fake.getMarkerPathMutex.RUnlock() - fake.getReleasePathMutex.RLock() - defer fake.getReleasePathMutex.RUnlock() - fake.isPathNormalizedMutex.RLock() - defer fake.isPathNormalizedMutex.RUnlock() - fake.normalizePathMutex.RLock() - defer fake.normalizePathMutex.RUnlock() - fake.pathExistsMutex.RLock() - defer fake.pathExistsMutex.RUnlock() - fake.rsyncRecursiveMutex.RLock() - defer fake.rsyncRecursiveMutex.RUnlock() - fake.setOptionsMutex.RLock() - defer fake.setOptionsMutex.RUnlock() - copiedInvocations := map[string][][]interface{}{} - for key, value := range fake.invocations { - copiedInvocations[key] = value - } - return copiedInvocations -} - -func (fake *FakeStore) recordInvocation(key string, args []interface{}) { - fake.invocationsMutex.Lock() - defer fake.invocationsMutex.Unlock() - if fake.invocations == nil { - fake.invocations = map[string][][]interface{}{} - } - if fake.invocations[key] == nil { - fake.invocations[key] = [][]interface{}{} - } - fake.invocations[key] = append(fake.invocations[key], args) -} - -var _ object.Store = new(FakeStore) diff --git a/pkg/object/store.go b/pkg/object/store.go deleted file mode 100644 index 5730ef04402..00000000000 --- a/pkg/object/store.go +++ /dev/null @@ -1,44 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package object - -//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -generate - -//counterfeiter:generate . Store -type Store interface { - // Configure options - SetOptions(opts ...OptFn) - - // Path operations - NormalizePath(pathParts ...string) (string, error) - IsPathNormalized(path string) bool - PathExists(path string) (bool, error) - - // Copy operations - // TODO: Determine if these methods should even be part of the interface - // TODO: Maybe overly specific. Consider reducing these down to Copy() - CopyToRemote(local, remote string) error - CopyToLocal(remote, local string) error - CopyBucketToBucket(src, dst string) error - RsyncRecursive(src, dst string) error - - // TODO: Overly specific. We should only care these methods during a release. - GetReleasePath(bucket, gcsRoot, version string, fast bool) (string, error) - GetMarkerPath(bucket, gcsRoot string) (string, error) -} - -type OptFn func(Store) diff --git a/pkg/release/archive.go b/pkg/release/archive.go index 78e76183648..f52bb46c4d9 100644 --- a/pkg/release/archive.go +++ b/pkg/release/archive.go @@ -25,8 +25,8 @@ import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" - "k8s.io/release/pkg/gcp" - "k8s.io/release/pkg/object" + "sigs.k8s.io/release-sdk/gcli" + "sigs.k8s.io/release-sdk/object" "sigs.k8s.io/release-utils/command" "sigs.k8s.io/release-utils/tar" @@ -206,7 +206,7 @@ func (a *defaultArchiverImpl) MakeFilesPrivate(archiveBucketPath string) error { return errors.Wrap(err, "normalizing gcs path to modify ACL") } // logrun -s $GSUTIL acl ch -d AllUsers "$archive_bucket/$build_dir/${LOGFILE##*/}*" || true - if err := gcp.GSUtil("acl", "ch", "-d", "AllUsers", logsPath); err != nil { + if err := gcli.GSUtil("acl", "ch", "-d", "AllUsers", logsPath); err != nil { return errors.Wrapf(err, "removing public access from files in %s", archiveBucketPath) } return nil @@ -349,7 +349,7 @@ func (a *defaultArchiverImpl) CleanStagedBuilds(bucketPath, buildVersion string) } // Get all staged build that match the pattern - output, err := gcp.GSUtilOutput("ls", "-d", path) + output, err := gcli.GSUtilOutput("ls", "-d", path) if err != nil { return errors.Wrap(err, "listing bucket contents") } diff --git a/pkg/release/archive_unit_test.go b/pkg/release/archive_unit_test.go index e3a0d19b7a3..b374f501ffc 100644 --- a/pkg/release/archive_unit_test.go +++ b/pkg/release/archive_unit_test.go @@ -23,7 +23,7 @@ import ( "github.com/stretchr/testify/require" - "k8s.io/release/pkg/object" + "sigs.k8s.io/release-sdk/object" ) const fictionalTestBucketName = "kubernetes-test-name" diff --git a/pkg/release/prerequisites.go b/pkg/release/prerequisites.go index 9e1ab8b5530..e899bba4968 100644 --- a/pkg/release/prerequisites.go +++ b/pkg/release/prerequisites.go @@ -23,9 +23,9 @@ import ( "github.com/shirou/gopsutil/v3/disk" "github.com/sirupsen/logrus" - "k8s.io/release/pkg/gcp" "k8s.io/release/pkg/git" "k8s.io/release/pkg/github" + "sigs.k8s.io/release-sdk/gcli" "sigs.k8s.io/release-utils/command" "sigs.k8s.io/release-utils/env" ) @@ -85,7 +85,7 @@ func (*defaultPrerequisitesChecker) CommandAvailable( func (*defaultPrerequisitesChecker) GCloudOutput( args ...string, ) (string, error) { - return gcp.GCloudOutput(args...) + return gcli.GCloudOutput(args...) } func (*defaultPrerequisitesChecker) IsEnvSet(key string) bool { diff --git a/pkg/release/publish.go b/pkg/release/publish.go index 5e37336e9e4..a22c6d594fc 100644 --- a/pkg/release/publish.go +++ b/pkg/release/publish.go @@ -26,8 +26,8 @@ import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" - "k8s.io/release/pkg/gcp" - "k8s.io/release/pkg/object" + "sigs.k8s.io/release-sdk/gcli" + "sigs.k8s.io/release-sdk/object" "sigs.k8s.io/release-utils/http" "sigs.k8s.io/release-utils/util" ) @@ -77,15 +77,15 @@ type defaultPublisher struct { } func (*defaultPublisher) GSUtil(args ...string) error { - return gcp.GSUtil(args...) + return gcli.GSUtil(args...) } func (*defaultPublisher) GSUtilOutput(args ...string) (string, error) { - return gcp.GSUtilOutput(args...) + return gcli.GSUtilOutput(args...) } func (*defaultPublisher) GSUtilStatus(args ...string) (bool, error) { - status, err := gcp.GSUtilStatus(args...) + status, err := gcli.GSUtilStatus(args...) if err != nil { return false, err } diff --git a/pkg/release/release.go b/pkg/release/release.go index 8992469e36a..e231a249ce9 100644 --- a/pkg/release/release.go +++ b/pkg/release/release.go @@ -35,7 +35,7 @@ import ( "k8s.io/release/pkg/git" "k8s.io/release/pkg/github" - "k8s.io/release/pkg/object" + "sigs.k8s.io/release-sdk/object" "sigs.k8s.io/release-utils/command" "sigs.k8s.io/release-utils/env" rhash "sigs.k8s.io/release-utils/hash" @@ -80,7 +80,7 @@ const ( // GCEPath is the directory where GCE scripts are created. GCEPath = ReleaseStagePath + "/full/kubernetes/cluster/gce" - // GCIPath is the path for the container optimized OS for GCP. + // GCIPath is the path for the container optimized OS for gcli. GCIPath = ReleaseStagePath + "/full/kubernetes/cluster/gce/gci" // ReleaseTarsPath is the directory where release artifacts are created. diff --git a/pkg/release/workspace.go b/pkg/release/workspace.go index 7fea3675249..7bce3c4afc0 100644 --- a/pkg/release/workspace.go +++ b/pkg/release/workspace.go @@ -29,8 +29,8 @@ import ( "k8s.io/release/pkg/git" "k8s.io/release/pkg/github" "k8s.io/release/pkg/license" - "k8s.io/release/pkg/object" "k8s.io/release/pkg/spdx" + "sigs.k8s.io/release-sdk/object" "sigs.k8s.io/release-utils/tar" "sigs.k8s.io/release-utils/util" )