Skip to content

Commit

Permalink
cmd/plume: upload cmd also creates GCE image
Browse files Browse the repository at this point in the history 
This unifies uploading to Google Storage and creating an image in GCE
into one step. The Google Storage filename is mapped into an image name
for simplicity.
  • Loading branch information
Patrick Baxter committed Apr 3, 2015
1 parent 1616c61 commit d98ec2e
Show file tree
Hide file tree
Showing 2 changed files with 137 additions and 23 deletions.
6 changes: 4 additions & 2 deletions auth/google.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,8 @@ var conf = oauth2.Config{
TokenURL: "https://accounts.google.com/o/oauth2/token",
},
RedirectURL: "urn:ietf:wg:oauth:2.0:oob",
Scopes: []string{"https://www.googleapis.com/auth/devstorage.full_control"},
Scopes: []string{"https://www.googleapis.com/auth/devstorage.full_control",
"https://www.googleapis.com/auth/compute"},
}

func writeCache(cachePath string, tok *oauth2.Token) error {
Expand Down Expand Up @@ -76,9 +77,10 @@ func getToken() (*oauth2.Token, error) {
if err != nil {
log.Printf("Error reading google token cache file: %v", err)
}
if tok == nil {
if tok == nil || !tok.Valid() {
url := conf.AuthCodeURL("state", oauth2.AccessTypeOffline)
fmt.Printf("Visit the URL for the auth dialog: %v\n", url)
fmt.Print("Enter token: ")

var code string
if _, err := fmt.Scan(&code); err != nil {
Expand Down
154 changes: 133 additions & 21 deletions cmd/plume/upload.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,31 +29,34 @@ import (
"github.com/coreos/mantle/Godeps/_workspace/src/google.golang.org/cloud/storage"
"github.com/coreos/mantle/auth"
"github.com/coreos/mantle/cli"
"google.golang.org/api/compute/v1"
)

var (
cmdUpload = &cli.Command{

Name: "upload",
Summary: "Upload os image",
Usage: "-bucket gs://bucket/prefix/ -image filepath",
Description: "Upload os image to Google Storage bucket",
Usage: "plume upload",
Description: "Upload os image to Google Storage bucket and create image in GCE. Intended for use in SDK.",
Flags: *flag.NewFlagSet("upload", flag.ExitOnError),
Run: runUpload,
}
bucket string
image string
imageName string
projectID string
bucket string
imagePath string
imageName string
gceProjectID string
board string
)

func init() {
cmdUpload.Flags.StringVar(&bucket, "bucket", "gs://coreos-plume", "gs://bucket/prefix/")
cmdUpload.Flags.StringVar(&projectID, "projectID", "coreos-gce-testing", "found in developers console")
cmdUpload.Flags.StringVar(&imageName, "name", "", "filename for uploaded image, defaults to COREOS_VERSION")
cmdUpload.Flags.StringVar(&image, "image",
cmdUpload.Flags.StringVar(&bucket, "bucket", "gs://users.developer.core-os.net", "gs://bucket/prefix/ prefix defaults to $USER")
cmdUpload.Flags.StringVar(&gceProjectID, "gce-project", "coreos-gce-testing", "Google Compute project ID")
cmdUpload.Flags.StringVar(&imageName, "name", "", "name for uploaded image, defaults to COREOS_VERSION")
cmdUpload.Flags.StringVar(&imagePath, "image",
"/mnt/host/source/src/build/images/amd64-usr/latest/coreos_production_gce.tar.gz",
"path_to_coreos_image (build with: ./image_to_vm.sh --format=gce ...)")
cmdUpload.Flags.StringVar(&board, "board", "amd64-usr", "board used for naming with default prefix only")
cli.Register(cmdUpload)
}

Expand All @@ -65,13 +68,12 @@ func runUpload(args []string) int {

// if an image name is unspecified try to use version.txt
if imageName == "" {
imageName = getImageVersion(image)
imageName = getImageVersion(imagePath)
if imageName == "" {
fmt.Fprintf(os.Stderr, "Unable to get version from image directory, provide a -name flag or include version.txt in the image directory\n")
fmt.Fprintf(os.Stderr, "Unable to get version from image directory, provide a -name flag or include a version.txt in the image directory\n")
return 1
}
}
imageName += ".tar.gz"

gsURL, err := url.Parse(bucket)
if err != nil {
Expand All @@ -86,26 +88,94 @@ func runUpload(args []string) int {
fmt.Fprintf(os.Stderr, "URL missing bucket name %v\n", bucket)
return 1
}
// if prefix not specified default name to gs://bucket/$USER/$BOARD/$VERSION
if gsURL.Path == "" {
if user := os.Getenv("USER"); user != "" {
gsURL.Path = "/" + os.Getenv("USER")
gsURL.Path += "/" + board
}
}

bucket = gsURL.Host
imageName = strings.TrimPrefix(gsURL.Path+"/"+imageName, "/")
// create equivalent image names for GS and GCE
imageNameGCE := gceSanitize(imageName)
imageNameGS := imageName + ".tar.gz"

client, err := auth.GoogleClient()
if err != nil {
fmt.Fprintf(os.Stderr, "Authentication failed: %v\n", err)
return 1
}

fmt.Printf("Writing %v to %v\n", imageName, bucket)
fmt.Printf("Writing %v to gs://%v ...\n", imageNameGS, bucket)
fmt.Printf("(Sometimes this takes a few mintues)\n")

if err := writeFile(client, imageName); err != nil {
if err := writeFile(client, imagePath, imageNameGS); err != nil {
fmt.Fprintf(os.Stderr, "Uploading image failed: %v\n", err)
return 1
}

fmt.Printf("Update successful!\n")
fmt.Printf("Upload successful!\n")
fmt.Printf("Creating image in GCE: %v...\n", imageNameGCE)

// create image on gce
storageSrc := fmt.Sprintf("https://storage.googleapis.com/%v/%v", bucket, imageNameGS)
err = createImage(client, gceProjectID, imageNameGCE, storageSrc)

// if image already exists ask to delete and try again
if err != nil && strings.HasSuffix(err.Error(), "alreadyExists") {
var ans string
fmt.Printf("Image %v already exists on GCE. Overwrite? (y/n):", imageNameGCE)
if _, err = fmt.Scan(&ans); err != nil {
fmt.Fprintf(os.Stderr, "Scanning overwrite input: %v", err)
return 1
}
switch ans {
case "y", "Y", "yes":
fmt.Println("Overriding existing image...")
err = forceCreateImage(client, gceProjectID, imageNameGCE, storageSrc)
default:
fmt.Println("Skipped GCE image creation")
return 0
}
}
if err != nil {
fmt.Fprintf(os.Stderr, "Creating GCE image failed: %v\n", err)
return 1
}
fmt.Printf("Image %v sucessfully created in GCE\n", imageNameGCE)

return 0
}

// Converts an image name from Google Storage to an equivalent GCE image
// name. NOTE: Not a fully generlized sanitizer for GCE. Designed for
// the default version.txt name (ex: 633.1.0+2015-03-31-1538). See:
// https://godoc.org/google.golang.org/api/compute/v1#Image
func gceSanitize(name string) string {
if name == "" {
return name
}

// remove incompatible chars from version.txt
name = strings.Replace(name, ".", "-", -1)
name = strings.Replace(name, "+", "-", -1)

// remove forward slashes likely from prefix
name = strings.Replace(name, "/", "-", -1)

// ensure name starts with [a-z]
char := name[0]
if char >= 'a' && char <= 'z' {
return name
}
if char >= 'A' && char <= 'Z' {
return strings.ToLower(name[:1]) + name[1:]
}
return "v" + name
}

// Attempt to get version.txt from image build directory. Return "" if
// unable to retrieve version.txt from directory.
func getImageVersion(imagePath string) string {
Expand All @@ -126,17 +196,22 @@ func getImageVersion(imagePath string) string {
return version
}

func writeFile(client *http.Client, filename string) error {
ctx := cloud.NewContext(projectID, client)
wc := storage.NewWriter(ctx, bucket, filename)
// Write file to Google Storage
func writeFile(client *http.Client, filename, destname string) error {
// dummy value is used since a project name isn't necessary unless
// we are creating new buckets
ctx := cloud.NewContext("dummy", client)
wc := storage.NewWriter(ctx, bucket, destname)
wc.ContentType = "application/x-gzip"
wc.ACL = []storage.ACLRule{{storage.AllAuthenticatedUsers, storage.RoleReader}}

imageFile, err := os.Open(image)
file, err := os.Open(filename)
if err != nil {
return err
}
_, err = io.Copy(wc, imageFile)
defer file.Close()

_, err = io.Copy(wc, file)
if err != nil {
return err
}
Expand All @@ -146,3 +221,40 @@ func writeFile(client *http.Client, filename string) error {

return nil
}

// Create image on GCE and return. Will not overwrite existing image.
func createImage(client *http.Client, proj, name, source string) error {
computeService, err := compute.New(client)
if err != nil {
return err
}
imageService := compute.NewImagesService(computeService)
image := &compute.Image{
Name: name,
RawDisk: &compute.ImageRawDisk{
Source: source,
},
}
_, err = imageService.Insert(proj, image).Do()
if err != nil {
return err
}
return nil
}

// Delete image on GCE and then recreate it.
func forceCreateImage(client *http.Client, proj, name, source string) error {
// delete
computeService, err := compute.New(client)
if err != nil {
return fmt.Errorf("deleting image: %v", err)
}
imageService := compute.NewImagesService(computeService)
_, err = imageService.Delete(proj, name).Do()
if err != nil {
return fmt.Errorf("deleting image: %v", err)
}

// create
return createImage(client, proj, name, source)
}

0 comments on commit d98ec2e

Please sign in to comment.