From d98ec2e34114b2fae917f0d1fc490c7be3f6bd95 Mon Sep 17 00:00:00 2001 From: Patrick Baxter Date: Thu, 2 Apr 2015 20:02:02 -0400 Subject: [PATCH] cmd/plume: upload cmd also creates GCE image This unifies uploading to Google Storage and creating an image in GCE into one step. The Google Storage filename is mapped into an image name for simplicity. --- auth/google.go | 6 +- cmd/plume/upload.go | 154 ++++++++++++++++++++++++++++++++++++++------ 2 files changed, 137 insertions(+), 23 deletions(-) diff --git a/auth/google.go b/auth/google.go index ebb3ffb5bd..0a35e53ca8 100644 --- a/auth/google.go +++ b/auth/google.go @@ -35,7 +35,8 @@ var conf = oauth2.Config{ TokenURL: "https://accounts.google.com/o/oauth2/token", }, RedirectURL: "urn:ietf:wg:oauth:2.0:oob", - Scopes: []string{"https://www.googleapis.com/auth/devstorage.full_control"}, + Scopes: []string{"https://www.googleapis.com/auth/devstorage.full_control", + "https://www.googleapis.com/auth/compute"}, } func writeCache(cachePath string, tok *oauth2.Token) error { @@ -76,9 +77,10 @@ func getToken() (*oauth2.Token, error) { if err != nil { log.Printf("Error reading google token cache file: %v", err) } - if tok == nil { + if tok == nil || !tok.Valid() { url := conf.AuthCodeURL("state", oauth2.AccessTypeOffline) fmt.Printf("Visit the URL for the auth dialog: %v\n", url) + fmt.Print("Enter token: ") var code string if _, err := fmt.Scan(&code); err != nil { diff --git a/cmd/plume/upload.go b/cmd/plume/upload.go index 88ee8b577b..c90329090c 100644 --- a/cmd/plume/upload.go +++ b/cmd/plume/upload.go @@ -29,6 +29,7 @@ import ( "github.com/coreos/mantle/Godeps/_workspace/src/google.golang.org/cloud/storage" "github.com/coreos/mantle/auth" "github.com/coreos/mantle/cli" + "google.golang.org/api/compute/v1" ) var ( @@ -36,24 +37,26 @@ var ( Name: "upload", Summary: "Upload os image", - Usage: "-bucket gs://bucket/prefix/ -image filepath", - Description: "Upload os image to Google Storage bucket", + Usage: "plume upload", + Description: "Upload os image to Google Storage bucket and create image in GCE. Intended for use in SDK.", Flags: *flag.NewFlagSet("upload", flag.ExitOnError), Run: runUpload, } - bucket string - image string - imageName string - projectID string + bucket string + imagePath string + imageName string + gceProjectID string + board string ) func init() { - cmdUpload.Flags.StringVar(&bucket, "bucket", "gs://coreos-plume", "gs://bucket/prefix/") - cmdUpload.Flags.StringVar(&projectID, "projectID", "coreos-gce-testing", "found in developers console") - cmdUpload.Flags.StringVar(&imageName, "name", "", "filename for uploaded image, defaults to COREOS_VERSION") - cmdUpload.Flags.StringVar(&image, "image", + cmdUpload.Flags.StringVar(&bucket, "bucket", "gs://users.developer.core-os.net", "gs://bucket/prefix/ prefix defaults to $USER") + cmdUpload.Flags.StringVar(&gceProjectID, "gce-project", "coreos-gce-testing", "Google Compute project ID") + cmdUpload.Flags.StringVar(&imageName, "name", "", "name for uploaded image, defaults to COREOS_VERSION") + cmdUpload.Flags.StringVar(&imagePath, "image", "/mnt/host/source/src/build/images/amd64-usr/latest/coreos_production_gce.tar.gz", "path_to_coreos_image (build with: ./image_to_vm.sh --format=gce ...)") + cmdUpload.Flags.StringVar(&board, "board", "amd64-usr", "board used for naming with default prefix only") cli.Register(cmdUpload) } @@ -65,13 +68,12 @@ func runUpload(args []string) int { // if an image name is unspecified try to use version.txt if imageName == "" { - imageName = getImageVersion(image) + imageName = getImageVersion(imagePath) if imageName == "" { - fmt.Fprintf(os.Stderr, "Unable to get version from image directory, provide a -name flag or include version.txt in the image directory\n") + fmt.Fprintf(os.Stderr, "Unable to get version from image directory, provide a -name flag or include a version.txt in the image directory\n") return 1 } } - imageName += ".tar.gz" gsURL, err := url.Parse(bucket) if err != nil { @@ -86,8 +88,19 @@ func runUpload(args []string) int { fmt.Fprintf(os.Stderr, "URL missing bucket name %v\n", bucket) return 1 } + // if prefix not specified default name to gs://bucket/$USER/$BOARD/$VERSION + if gsURL.Path == "" { + if user := os.Getenv("USER"); user != "" { + gsURL.Path = "/" + os.Getenv("USER") + gsURL.Path += "/" + board + } + } + bucket = gsURL.Host imageName = strings.TrimPrefix(gsURL.Path+"/"+imageName, "/") + // create equivalent image names for GS and GCE + imageNameGCE := gceSanitize(imageName) + imageNameGS := imageName + ".tar.gz" client, err := auth.GoogleClient() if err != nil { @@ -95,17 +108,74 @@ func runUpload(args []string) int { return 1 } - fmt.Printf("Writing %v to %v\n", imageName, bucket) + fmt.Printf("Writing %v to gs://%v ...\n", imageNameGS, bucket) + fmt.Printf("(Sometimes this takes a few mintues)\n") - if err := writeFile(client, imageName); err != nil { + if err := writeFile(client, imagePath, imageNameGS); err != nil { fmt.Fprintf(os.Stderr, "Uploading image failed: %v\n", err) return 1 } - fmt.Printf("Update successful!\n") + fmt.Printf("Upload successful!\n") + fmt.Printf("Creating image in GCE: %v...\n", imageNameGCE) + + // create image on gce + storageSrc := fmt.Sprintf("https://storage.googleapis.com/%v/%v", bucket, imageNameGS) + err = createImage(client, gceProjectID, imageNameGCE, storageSrc) + + // if image already exists ask to delete and try again + if err != nil && strings.HasSuffix(err.Error(), "alreadyExists") { + var ans string + fmt.Printf("Image %v already exists on GCE. Overwrite? (y/n):", imageNameGCE) + if _, err = fmt.Scan(&ans); err != nil { + fmt.Fprintf(os.Stderr, "Scanning overwrite input: %v", err) + return 1 + } + switch ans { + case "y", "Y", "yes": + fmt.Println("Overriding existing image...") + err = forceCreateImage(client, gceProjectID, imageNameGCE, storageSrc) + default: + fmt.Println("Skipped GCE image creation") + return 0 + } + } + if err != nil { + fmt.Fprintf(os.Stderr, "Creating GCE image failed: %v\n", err) + return 1 + } + fmt.Printf("Image %v sucessfully created in GCE\n", imageNameGCE) + return 0 } +// Converts an image name from Google Storage to an equivalent GCE image +// name. NOTE: Not a fully generlized sanitizer for GCE. Designed for +// the default version.txt name (ex: 633.1.0+2015-03-31-1538). See: +// https://godoc.org/google.golang.org/api/compute/v1#Image +func gceSanitize(name string) string { + if name == "" { + return name + } + + // remove incompatible chars from version.txt + name = strings.Replace(name, ".", "-", -1) + name = strings.Replace(name, "+", "-", -1) + + // remove forward slashes likely from prefix + name = strings.Replace(name, "/", "-", -1) + + // ensure name starts with [a-z] + char := name[0] + if char >= 'a' && char <= 'z' { + return name + } + if char >= 'A' && char <= 'Z' { + return strings.ToLower(name[:1]) + name[1:] + } + return "v" + name +} + // Attempt to get version.txt from image build directory. Return "" if // unable to retrieve version.txt from directory. func getImageVersion(imagePath string) string { @@ -126,17 +196,22 @@ func getImageVersion(imagePath string) string { return version } -func writeFile(client *http.Client, filename string) error { - ctx := cloud.NewContext(projectID, client) - wc := storage.NewWriter(ctx, bucket, filename) +// Write file to Google Storage +func writeFile(client *http.Client, filename, destname string) error { + // dummy value is used since a project name isn't necessary unless + // we are creating new buckets + ctx := cloud.NewContext("dummy", client) + wc := storage.NewWriter(ctx, bucket, destname) wc.ContentType = "application/x-gzip" wc.ACL = []storage.ACLRule{{storage.AllAuthenticatedUsers, storage.RoleReader}} - imageFile, err := os.Open(image) + file, err := os.Open(filename) if err != nil { return err } - _, err = io.Copy(wc, imageFile) + defer file.Close() + + _, err = io.Copy(wc, file) if err != nil { return err } @@ -146,3 +221,40 @@ func writeFile(client *http.Client, filename string) error { return nil } + +// Create image on GCE and return. Will not overwrite existing image. +func createImage(client *http.Client, proj, name, source string) error { + computeService, err := compute.New(client) + if err != nil { + return err + } + imageService := compute.NewImagesService(computeService) + image := &compute.Image{ + Name: name, + RawDisk: &compute.ImageRawDisk{ + Source: source, + }, + } + _, err = imageService.Insert(proj, image).Do() + if err != nil { + return err + } + return nil +} + +// Delete image on GCE and then recreate it. +func forceCreateImage(client *http.Client, proj, name, source string) error { + // delete + computeService, err := compute.New(client) + if err != nil { + return fmt.Errorf("deleting image: %v", err) + } + imageService := compute.NewImagesService(computeService) + _, err = imageService.Delete(proj, name).Do() + if err != nil { + return fmt.Errorf("deleting image: %v", err) + } + + // create + return createImage(client, proj, name, source) +}