From f6be5dfcb645efd7e11e80d3328fe7c713aa96df Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Mon, 6 Apr 2020 18:05:43 -0400 Subject: [PATCH 1/2] mantle/ore: gcloud: clean up a lot of CL specific bits in upload.go - Don't try to detect or set defaults for bucket, name, or file arguments. - Don't try to set a default prefix/path to use in GS buckets --- mantle/cmd/ore/gcloud/upload.go | 33 ++++++++------------------------- src/cosalib/gcp.py | 1 - 2 files changed, 8 insertions(+), 26 deletions(-) diff --git a/mantle/cmd/ore/gcloud/upload.go b/mantle/cmd/ore/gcloud/upload.go index 0bdd4b36ef..ced04177d7 100644 --- a/mantle/cmd/ore/gcloud/upload.go +++ b/mantle/cmd/ore/gcloud/upload.go @@ -19,7 +19,6 @@ import ( "io/ioutil" "net/url" "os" - "path/filepath" "strings" "github.com/spf13/cobra" @@ -27,7 +26,6 @@ import ( "google.golang.org/api/storage/v1" "github.com/coreos/mantle/platform/api/gcloud" - "github.com/coreos/mantle/sdk" ) var ( @@ -40,7 +38,6 @@ var ( uploadBucket string uploadImageName string - uploadBoard string uploadFile string uploadFedora bool uploadForce bool @@ -48,13 +45,12 @@ var ( ) func init() { - build := sdk.BuildRoot() - cmdUpload.Flags().StringVar(&uploadBucket, "bucket", "gs://users.developer.core-os.net", "gs://bucket/prefix/ prefix defaults to $USER") - cmdUpload.Flags().StringVar(&uploadImageName, "name", "", "name for uploaded image, defaults to COREOS_VERSION") - cmdUpload.Flags().StringVar(&uploadBoard, "board", "amd64-usr", "board used for naming with default prefix only") - cmdUpload.Flags().StringVar(&uploadFile, "file", - build+"/images/amd64-usr/latest/coreos_production_gce.tar.gz", - "path_to_coreos_image (build with: ./image_to_vm.sh --format=gce ...)") + cmdUpload.Flags().StringVar(&uploadBucket, "bucket", "", "gs://bucket/prefix/") + cmdUpload.MarkFlagRequired("bucket") + cmdUpload.Flags().StringVar(&uploadImageName, "name", "", "name for uploaded image") + cmdUpload.MarkFlagRequired("name") + cmdUpload.Flags().StringVar(&uploadFile, "file", "", "path to image .tar.gz file to upload") + cmdUpload.MarkFlagRequired("file") cmdUpload.Flags().BoolVar(&uploadFedora, "fcos", false, "Flag this is Fedora CoreOS (or a derivative); currently enables SECURE_BOOT and UEFI_COMPATIBLE") cmdUpload.Flags().BoolVar(&uploadForce, "force", false, "overwrite existing GS and GCE images without prompt") cmdUpload.Flags().StringVar(&uploadWriteUrl, "write-url", "", "output the uploaded URL to the named file") @@ -67,16 +63,6 @@ func runUpload(cmd *cobra.Command, args []string) { os.Exit(2) } - // if an image name is unspecified try to use version.txt - if uploadImageName == "" { - ver, err := sdk.VersionsFromDir(filepath.Dir(uploadFile)) - if err != nil { - fmt.Fprintf(os.Stderr, "Unable to get version from image directory, provide a -name flag or include a version.txt in the image directory: %v\n", err) - os.Exit(1) - } - uploadImageName = ver.Version - } - gsURL, err := url.Parse(uploadBucket) if err != nil { fmt.Fprintf(os.Stderr, "%v\n", err) @@ -90,12 +76,9 @@ func runUpload(cmd *cobra.Command, args []string) { fmt.Fprintf(os.Stderr, "URL missing bucket name %v\n", uploadBucket) os.Exit(1) } - // if prefix not specified default name to gs://bucket/$USER/$BOARD/$VERSION if gsURL.Path == "" { - if user := os.Getenv("USER"); user != "" { - gsURL.Path = "/" + os.Getenv("USER") - gsURL.Path += "/" + uploadBoard - } + fmt.Fprint(os.Stderr, "prefix not specified. Refusing to upload in root directory of bucket\n") + os.Exit(1) } uploadBucket = gsURL.Host diff --git a/src/cosalib/gcp.py b/src/cosalib/gcp.py index dd443446c4..d8ed6227d1 100644 --- a/src/cosalib/gcp.py +++ b/src/cosalib/gcp.py @@ -54,7 +54,6 @@ def gcp_run_ore(build, args): '--basename', build.build_name, 'upload', '--force', # We want to support restarting the pipeline - '--board=""', '--bucket', f'gs://{args.bucket}/{build.build_name}', '--json-key', args.json_key, '--name', gcp_name, From 32b0062401f50be913a91252dee3afc94e340894 Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Mon, 6 Apr 2020 18:07:55 -0400 Subject: [PATCH 2/2] mantle/ore: gcloud: remove bucket subpath from GCE image name Before this commit if I try to upload to gs://mybucket/fedora-coreos my image name would get rewritten to append the path `fedora-coreos` into the image name. For example: - `fedora-coreos-31-20200404-dev-1-gcp-x86-64` turns into - `fedora-coreos-fedora-coreos-31-20200404-dev-1-gcp-x86-64` This was introduced in d98ec2e and doesn't seem to be needed. Let's rework it so that transformation doesn't happen. --- mantle/cmd/ore/gcloud/upload.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mantle/cmd/ore/gcloud/upload.go b/mantle/cmd/ore/gcloud/upload.go index ced04177d7..c3fabf250c 100644 --- a/mantle/cmd/ore/gcloud/upload.go +++ b/mantle/cmd/ore/gcloud/upload.go @@ -82,10 +82,10 @@ func runUpload(cmd *cobra.Command, args []string) { } uploadBucket = gsURL.Host - uploadImageName = strings.TrimPrefix(gsURL.Path+"/"+uploadImageName, "/") - // create equivalent image names for GS and GCE + imageNameGS := strings.TrimPrefix(gsURL.Path+"/"+uploadImageName, "/") + ".tar.gz" + + // Sanitize the image name for GCE imageNameGCE := gceSanitize(uploadImageName) - imageNameGS := uploadImageName + ".tar.gz" storageAPI, err := storage.New(api.Client()) if err != nil {