From 3d0b393fd4647834c9d549b6c4bf8a83069ddd09 Mon Sep 17 00:00:00 2001 From: alexwo Date: Wed, 2 Jan 2019 21:34:03 +0200 Subject: [PATCH 1/5] Fix full stemcell create flow 1.Fixed part size too small issue when uploading a 3.5GB image 2.Fixed - Extract stemcell image to get raw image --- src/bosh-alicloud-cpi/action/create_stemcell.go | 1 + 1 file changed, 1 insertion(+) diff --git a/src/bosh-alicloud-cpi/action/create_stemcell.go b/src/bosh-alicloud-cpi/action/create_stemcell.go index 6cf4d52e..1a2d25ac 100644 --- a/src/bosh-alicloud-cpi/action/create_stemcell.go +++ b/src/bosh-alicloud-cpi/action/create_stemcell.go @@ -21,6 +21,7 @@ const ( AlicloudImageNamePrefix = "stemcell" MinImageDiskSize = 5 //in GB OSS_BUCKET_NAME_MAX_LENGTH = 64 + PART_SIZE = 5*1024*1024 ) type StemcellProps struct { From 4bf3486a7821a28abe8812ac38299ea9513f7cd1 Mon Sep 17 00:00:00 2001 From: alexwo Date: Wed, 2 Jan 2019 21:38:55 +0200 Subject: [PATCH 2/5] fix create stemcell from tarball flow --- .../action/create_stemcell.go | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/src/bosh-alicloud-cpi/action/create_stemcell.go b/src/bosh-alicloud-cpi/action/create_stemcell.go index 1a2d25ac..020f56ea 100644 --- a/src/bosh-alicloud-cpi/action/create_stemcell.go +++ b/src/bosh-alicloud-cpi/action/create_stemcell.go @@ -187,26 +187,37 @@ func (a CreateStemcellMethod) importImage(props StemcellProps) (string, error) { func (a CreateStemcellMethod) CreateFromTarball(imagePath string, props StemcellProps) (string, error) { imageName := fmt.Sprintf("%s-%s.raw", AlicloudImageNamePrefix, a.getUUIDName(props)) bucketName := fmt.Sprintf("%s-%s", alicloud.AlicloudDefaultImageName, uuid.New().String()) + if len(bucketName) > OSS_BUCKET_NAME_MAX_LENGTH { bucketName = bucketName[0:OSS_BUCKET_NAME_MAX_LENGTH] } + + var out bytes.Buffer + var stderr bytes.Buffer + + defer a.osses.DeleteBucket(bucketName) + if err := a.osses.CreateBucket(bucketName, oss.ACL(oss.ACLPublicRead)); err != nil { return "", bosherr.WrapErrorf(err, "Creating Alicloud OSS Bucket") } - defer a.osses.DeleteBucket(bucketName) bucket, err := a.osses.GetBucket(bucketName) + if err != nil { return "", bosherr.WrapErrorf(err, "Geting oss bucket") } - imageFile, err := a.stemcells.OpenLocalFile(imagePath) + cmd := exec.Command("tar", "-xf", imagePath) + cmd.Dir = path.Dir(imagePath) + cmd.Stdout = &out + cmd.Stderr = &stderr + err = cmd.Run() + if err != nil { - return "", bosherr.WrapErrorf(err, "Reading stemcell image file from local") + return "", bosherr.WrapErrorf(err, fmt.Sprintf("%s-(%s)-(%s)", "Unable to extract image", out.String(),stderr.String())) } - defer imageFile.Close() - err = a.osses.UploadFile(*bucket, imageName, imagePath, 100*1024, oss.Routines(5)) + err = a.osses.UploadFile(*bucket, imageName, fmt.Sprintf("%s/%s", path.Dir(imagePath), "root.img"), PART_SIZE, oss.Routines(5)) if err != nil { return "", bosherr.WrapErrorf(err, "Uploading stemcell image file to oss") } @@ -218,6 +229,7 @@ func (a CreateStemcellMethod) CreateFromTarball(imagePath string, props Stemcell if err != nil { return "", bosherr.WrapErrorf(err, "Creating Alicloud Image from Tarball") } + return image, err } From fde8e9c2fa074003c049feb393c4bb9c07444b85 Mon Sep 17 00:00:00 2001 From: alexwo Date: Thu, 3 Jan 2019 11:04:19 +0200 Subject: [PATCH 3/5] adding missing imports --- src/bosh-alicloud-cpi/action/create_stemcell.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/bosh-alicloud-cpi/action/create_stemcell.go b/src/bosh-alicloud-cpi/action/create_stemcell.go index 020f56ea..f754b124 100644 --- a/src/bosh-alicloud-cpi/action/create_stemcell.go +++ b/src/bosh-alicloud-cpi/action/create_stemcell.go @@ -15,6 +15,11 @@ import ( bosherr "github.com/cloudfoundry/bosh-utils/errors" "github.com/cppforlife/bosh-cpi-go/apiv1" "github.com/google/uuid" + "os" + "io" + "os/exec" + "bytes" + "path" ) const ( From c313c8cd33b8ffda18378b4e64d8de8f86ce514f Mon Sep 17 00:00:00 2001 From: alexwo Date: Thu, 3 Jan 2019 11:25:18 +0200 Subject: [PATCH 4/5] remove unused imports --- src/bosh-alicloud-cpi/action/create_stemcell.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/bosh-alicloud-cpi/action/create_stemcell.go b/src/bosh-alicloud-cpi/action/create_stemcell.go index f754b124..c5dd0ac9 100644 --- a/src/bosh-alicloud-cpi/action/create_stemcell.go +++ b/src/bosh-alicloud-cpi/action/create_stemcell.go @@ -15,8 +15,6 @@ import ( bosherr "github.com/cloudfoundry/bosh-utils/errors" "github.com/cppforlife/bosh-cpi-go/apiv1" "github.com/google/uuid" - "os" - "io" "os/exec" "bytes" "path" From cfedb3205eacf616ba323b72f1b409e23012d367 Mon Sep 17 00:00:00 2001 From: alexwo Date: Thu, 3 Jan 2019 12:12:42 +0200 Subject: [PATCH 5/5] reformat code --- src/bosh-alicloud-cpi/action/create_stemcell.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/bosh-alicloud-cpi/action/create_stemcell.go b/src/bosh-alicloud-cpi/action/create_stemcell.go index c5dd0ac9..2df42574 100644 --- a/src/bosh-alicloud-cpi/action/create_stemcell.go +++ b/src/bosh-alicloud-cpi/action/create_stemcell.go @@ -10,13 +10,13 @@ import ( "strconv" "strings" + "bytes" "github.com/aliyun/alibaba-cloud-sdk-go/services/ecs" "github.com/aliyun/aliyun-oss-go-sdk/oss" bosherr "github.com/cloudfoundry/bosh-utils/errors" "github.com/cppforlife/bosh-cpi-go/apiv1" "github.com/google/uuid" "os/exec" - "bytes" "path" ) @@ -24,7 +24,7 @@ const ( AlicloudImageNamePrefix = "stemcell" MinImageDiskSize = 5 //in GB OSS_BUCKET_NAME_MAX_LENGTH = 64 - PART_SIZE = 5*1024*1024 + PART_SIZE = 5 * 1024 * 1024 ) type StemcellProps struct { @@ -33,10 +33,10 @@ type StemcellProps struct { Disk interface{} `json:"disk"` DiskFormat string `json:"disk_format"` diskGB int - Hypervisor string `json:"hypervisor"` - Name string `json:"name"` - OsDistro string `json:"os_distro"` - OsType string `json:"os_type"` + Hypervisor string `json:"hypervisor"` + Name string `json:"name"` + OsDistro string `json:"os_distro"` + OsType string `json:"os_type"` //RootDeviceName string `json:"root_device_name"` SourceUrl string `json:"source_url"` //SourceSha1 string `json:"raw_disk_sha1,omitempty"` @@ -217,7 +217,7 @@ func (a CreateStemcellMethod) CreateFromTarball(imagePath string, props Stemcell err = cmd.Run() if err != nil { - return "", bosherr.WrapErrorf(err, fmt.Sprintf("%s-(%s)-(%s)", "Unable to extract image", out.String(),stderr.String())) + return "", bosherr.WrapErrorf(err, fmt.Sprintf("%s-(%s)-(%s)", "Unable to extract image", out.String(), stderr.String())) } err = a.osses.UploadFile(*bucket, imageName, fmt.Sprintf("%s/%s", path.Dir(imagePath), "root.img"), PART_SIZE, oss.Routines(5))