From ca18de7614262b031e33ce326710796fd7251d6c Mon Sep 17 00:00:00 2001 From: SandeepPissay Date: Thu, 23 Mar 2017 11:24:23 -0700 Subject: [PATCH] Implemented AttachISO task (#4) AttachISO task creates the user-data/meta-data cloud init files and creates cloud-init.iso file using "genisoimage" tool. It then uploads it to the datastore where the master/worker VM resides and inserts it into the cd-rom device of the master/worker VM. When the master/worker VM powers on, the cloud-init package in it runs the bootstrap script that downloads nodeup and runs it. Also removed redundant VirtualMachineModelBuilder that does nothing. Testing done: 1. Tested end to end that the master and worker VMs executes the cloud-init script successfully. 2, "make ci" is successful. --- pkg/model/vspheremodel/autoscalinggroup.go | 7 +- upup/pkg/fi/cloudup/apply_cluster.go | 6 - upup/pkg/fi/cloudup/vsphere/vsphere_cloud.go | 79 ++++++++++++- upup/pkg/fi/cloudup/vspheretasks/attachiso.go | 110 +++++++++++++++++- .../pkg/fi/cloudup/vspheretasks/cloud_init.go | 35 +++--- upup/pkg/fi/cloudup/vspheretasks/vmpoweron.go | 15 ++- 6 files changed, 217 insertions(+), 35 deletions(-) rename pkg/model/vspheremodel/virtualmachine.go => upup/pkg/fi/cloudup/vspheretasks/cloud_init.go (57%) diff --git a/pkg/model/vspheremodel/autoscalinggroup.go b/pkg/model/vspheremodel/autoscalinggroup.go index f4c0679b8eb67..51d11bdd32943 100644 --- a/pkg/model/vspheremodel/autoscalinggroup.go +++ b/pkg/model/vspheremodel/autoscalinggroup.go @@ -43,12 +43,15 @@ func (b *AutoscalingGroupModelBuilder) Build(c *fi.ModelBuilderContext) error { Name: &name, VMTemplateName: fi.String(defaultVmTemplateName), } + c.AddTask(createVmTask) attachISOTaskName := "AttachISO-" + name attachISOTask := &vspheretasks.AttachISO{ - Name: &attachISOTaskName, - VM: createVmTask, + Name: &attachISOTaskName, + VM: createVmTask, + IG: ig, + BootstrapScript: b.BootstrapScript, } c.AddTask(attachISOTask) diff --git a/upup/pkg/fi/cloudup/apply_cluster.go b/upup/pkg/fi/cloudup/apply_cluster.go index 6891f48a4c1ea..d216c62f82421 100644 --- a/upup/pkg/fi/cloudup/apply_cluster.go +++ b/upup/pkg/fi/cloudup/apply_cluster.go @@ -456,12 +456,6 @@ func (c *ApplyClusterCmd) Run() error { //&model.SSHKeyModelBuilder{KopsModelContext: modelContext}, ) case fi.CloudProviderVSphere: - vsphereModelContext := &vspheremodel.VSphereModelContext{ - KopsModelContext: modelContext, - } - - l.Builders = append(l.Builders, - &vspheremodel.VirtualMachineModelBuilder{VSphereModelContext: vsphereModelContext}) default: return fmt.Errorf("unknown cloudprovider %q", cluster.Spec.CloudProvider) diff --git a/upup/pkg/fi/cloudup/vsphere/vsphere_cloud.go b/upup/pkg/fi/cloudup/vsphere/vsphere_cloud.go index 43c1a9bc860d7..a0b197fe2d4c3 100644 --- a/upup/pkg/fi/cloudup/vsphere/vsphere_cloud.go +++ b/upup/pkg/fi/cloudup/vsphere/vsphere_cloud.go @@ -25,7 +25,10 @@ import ( "github.com/vmware/govmomi" "github.com/vmware/govmomi/find" "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/property" "github.com/vmware/govmomi/vim25" + "github.com/vmware/govmomi/vim25/mo" + "github.com/vmware/govmomi/vim25/soap" "github.com/vmware/govmomi/vim25/types" "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/upup/pkg/fi" @@ -49,9 +52,10 @@ type VSphereCloud struct { } const ( - snapshotName string = "LinkCloneSnapshotPoint" - snapshotDesc string = "Snapshot created by kops" - privateDNS string = "coredns" + snapshotName string = "LinkCloneSnapshotPoint" + snapshotDesc string = "Snapshot created by kops" + privateDNS string = "coredns" + cloudInitFile string = "cloud-init.iso" ) var _ fi.Cloud = &VSphereCloud{} @@ -221,3 +225,72 @@ func (c *VSphereCloud) PowerOn(vm string) error { task.Wait(ctx) return nil } + +func (c *VSphereCloud) UploadAndAttachISO(vm *string, isoFile string) error { + f := find.NewFinder(c.Client.Client, true) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + dc, err := f.Datacenter(ctx, c.Datacenter) + if err != nil { + return err + } + f.SetDatacenter(dc) + + vmRef, err := f.VirtualMachine(ctx, *vm) + if err != nil { + return err + } + + var refs []types.ManagedObjectReference + refs = append(refs, vmRef.Reference()) + var vmResult mo.VirtualMachine + + pc := property.DefaultCollector(c.Client.Client) + err = pc.RetrieveOne(ctx, vmRef.Reference(), []string{"datastore"}, &vmResult) + if err != nil { + glog.Fatalf("Unable to retrieve VM summary for VM %s", *vm) + } + glog.V(4).Infof("vm property collector result :%+v\n", vmResult) + + // We expect the VM to be on only 1 datastore + dsRef := vmResult.Datastore[0].Reference() + var dsResult mo.Datastore + err = pc.RetrieveOne(ctx, dsRef, []string{"summary"}, &dsResult) + if err != nil { + glog.Fatalf("Unable to retrieve datastore summary for datastore %s", dsRef) + } + glog.V(4).Infof("datastore property collector result :%+v\n", dsResult) + dsObj, err := f.Datastore(ctx, dsResult.Summary.Name) + if err != nil { + return err + } + p := soap.DefaultUpload + dstIsoFile := getCloudInitFileName(*vm) + glog.V(2).Infof("Uploading ISO file %s to datastore %+v, destination iso is %s\n", isoFile, dsObj, dstIsoFile) + err = dsObj.UploadFile(ctx, isoFile, dstIsoFile, &p) + if err != nil { + return err + } + glog.V(2).Infof("Uploaded ISO file %s", isoFile) + + // Find the cd-rom devide and insert the cloud init iso file into it. + devices, err := vmRef.Device(ctx) + if err != nil { + return err + } + + // passing empty cd-rom name so that the first one gets returned + cdrom, err := devices.FindCdrom("") + if err != nil { + return err + } + iso := dsObj.Path(dstIsoFile) + glog.V(2).Infof("Inserting ISO file %s into cd-rom", iso) + return vmRef.EditDevice(ctx, devices.InsertIso(cdrom, iso)) + +} + +func getCloudInitFileName(vmName string) string { + return vmName + "/" + cloudInitFile +} diff --git a/upup/pkg/fi/cloudup/vspheretasks/attachiso.go b/upup/pkg/fi/cloudup/vspheretasks/attachiso.go index ce34aa4e800a9..7cd416031504e 100644 --- a/upup/pkg/fi/cloudup/vspheretasks/attachiso.go +++ b/upup/pkg/fi/cloudup/vspheretasks/attachiso.go @@ -17,19 +17,43 @@ limitations under the License. package vspheretasks import ( + "bytes" + "fmt" "github.com/golang/glog" + "github.com/pborman/uuid" + "io/ioutil" + "k8s.io/kops/pkg/apis/kops" + "k8s.io/kops/pkg/model" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/cloudup/vsphere" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" ) // AttachISO represents the cloud-init ISO file attached to a VMware VM //go:generate fitask -type=AttachISO type AttachISO struct { - Name *string - VM *VirtualMachine + Name *string + VM *VirtualMachine + IG *kops.InstanceGroup + BootstrapScript *model.BootstrapScript } var _ fi.HasName = &AttachISO{} +var _ fi.HasDependencies = &AttachISO{} + +func (o *AttachISO) GetDependencies(tasks map[string]fi.Task) []fi.Task { + var deps []fi.Task + vmCreateTask := tasks["VirtualMachine/"+*o.VM.Name] + if vmCreateTask == nil { + glog.Fatalf("Unable to find create VM task %s dependency for AttachISO %s", *o.VM.Name, *o.Name) + } + deps = append(deps, vmCreateTask) + return deps +} // GetName returns the Name of the object, implementing fi.HasName func (o *AttachISO) GetName() *string { @@ -56,7 +80,85 @@ func (_ *AttachISO) CheckChanges(a, e, changes *AttachISO) error { return nil } -func (_ *AttachISO) RenderVC(t *vsphere.VSphereAPITarget, a, e, changes *AttachISO) error { - glog.Info("AttachISO.RenderVC invoked!") +func (_ *AttachISO) RenderVSphere(t *vsphere.VSphereAPITarget, a, e, changes *AttachISO) error { + startupScript, err := changes.BootstrapScript.ResourceNodeUp(changes.IG) + startupStr, err := startupScript.AsString() + if err != nil { + return fmt.Errorf("error rendering startup script: %v", err) + } + dir, err := ioutil.TempDir("", *changes.VM.Name) + defer os.RemoveAll(dir) + + isoFile := createISO(changes, startupStr, dir) + err = t.Cloud.UploadAndAttachISO(changes.VM.Name, isoFile) + if err != nil { + return err + } + return nil } + +func createUserData(startupStr string, dir string) { + // Update the startup script to add the extra spaces for + // indentation when copied to the user-data file. + strArray := strings.Split(startupStr, "\n") + for i, str := range strArray { + if len(str) > 0 { + strArray[i] = " " + str + } + } + startupStr = strings.Join(strArray, "\n") + + data := strings.Replace(userDataTemplate, "$SCRIPT", startupStr, -1) + userDataFile := filepath.Join(dir, "user-data") + glog.V(4).Infof("User data file content: %s", data) + + if err := ioutil.WriteFile(userDataFile, []byte(data), 0644); err != nil { + glog.Fatalf("Unable to write user-data into file %s", userDataFile) + } + return +} + +func createMetaData(dir string, vmName string) { + data := strings.Replace(metaDataTemplate, "$INSTANCE_ID", uuid.NewUUID().String(), -1) + data = strings.Replace(data, "$LOCAL_HOST_NAME", vmName, -1) + + glog.V(4).Infof("Meta data file content: %s", string(data)) + + metaDataFile := filepath.Join(dir, "meta-data") + if err := ioutil.WriteFile(metaDataFile, []byte(data), 0644); err != nil { + glog.Fatalf("Unable to write meta-data into file %s", metaDataFile) + } + return +} + +func createISO(changes *AttachISO, startupStr string, dir string) string { + createUserData(startupStr, dir) + createMetaData(dir, *changes.VM.Name) + + isoFile := filepath.Join(dir, *changes.VM.Name+".iso") + var commandName string + + switch os := runtime.GOOS; os { + case "darwin": + commandName = "mkisofs" + case "linux": + commandName = "genisoimage" + + default: + glog.Fatalf("Cannot generate ISO file %s. Unsupported operation system (%s)!!!", isoFile, os) + } + cmd := exec.Command(commandName, "-o", isoFile, "-volid", "cidata", "-joliet", "-rock", dir) + var out bytes.Buffer + cmd.Stdout = &out + var stderr bytes.Buffer + cmd.Stderr = &stderr + + err := cmd.Run() + if err != nil { + glog.Fatalf("Error %s occurred while executing command %+v", err, cmd) + } + glog.V(4).Infof("%s std output : %s\n", commandName, out.String()) + glog.V(4).Infof("%s std error : %s\n", commandName, stderr.String()) + return isoFile +} diff --git a/pkg/model/vspheremodel/virtualmachine.go b/upup/pkg/fi/cloudup/vspheretasks/cloud_init.go similarity index 57% rename from pkg/model/vspheremodel/virtualmachine.go rename to upup/pkg/fi/cloudup/vspheretasks/cloud_init.go index e6a646c71cdbd..661527ef4fb1e 100644 --- a/pkg/model/vspheremodel/virtualmachine.go +++ b/upup/pkg/fi/cloudup/vspheretasks/cloud_init.go @@ -14,21 +14,20 @@ See the License for the specific language governing permissions and limitations under the License. */ -package vspheremodel - -import ( - "fmt" - "k8s.io/kops/upup/pkg/fi" -) - -// Do we need this model builder? - -// AutoscalingGroupModelBuilder configures AutoscalingGroup objects -type VirtualMachineModelBuilder struct { - *VSphereModelContext -} - -func (b *VirtualMachineModelBuilder) Build(c *fi.ModelBuilderContext) error { - fmt.Print("In VirtualMachineModelBuilder.Build function!!") - return nil -} +package vspheretasks + +// Template for user-data file in the cloud-init ISO +const userDataTemplate = `#cloud-config +write_files: + - content: | +$SCRIPT + owner: root:root + path: /root/script.sh + permissions: "0644" + +runcmd: + - bash /root/script.sh 2>&1 > /var/log/script.log` + +// Template for meta-data file in the cloud-init ISO +const metaDataTemplate = `instance-id: $INSTANCE_ID +local-hostname: $LOCAL_HOST_NAME` diff --git a/upup/pkg/fi/cloudup/vspheretasks/vmpoweron.go b/upup/pkg/fi/cloudup/vspheretasks/vmpoweron.go index 4778e90901c48..2cf15019fa0b9 100644 --- a/upup/pkg/fi/cloudup/vspheretasks/vmpoweron.go +++ b/upup/pkg/fi/cloudup/vspheretasks/vmpoweron.go @@ -30,6 +30,17 @@ type VMPowerOn struct { } var _ fi.HasName = &VMPowerOn{} +var _ fi.HasDependencies = &VMPowerOn{} + +func (o *VMPowerOn) GetDependencies(tasks map[string]fi.Task) []fi.Task { + var deps []fi.Task + attachISOTask := tasks["AttachISO/"+*o.AttachISO.Name] + if attachISOTask == nil { + glog.Fatalf("Unable to find attachISO task %s dependency for VMPowerOn %s", *o.AttachISO.Name, *o.Name) + } + deps = append(deps, attachISOTask) + return deps +} // GetName returns the Name of the object, implementing fi.HasName func (o *VMPowerOn) GetName() *string { @@ -56,8 +67,8 @@ func (_ *VMPowerOn) CheckChanges(a, e, changes *VMPowerOn) error { return nil } -func (_ *VMPowerOn) RenderVC(t *vsphere.VSphereAPITarget, a, e, changes *VMPowerOn) error { - glog.V(2).Infof("VMPowerOn.RenderVC invoked for vm %s", *changes.AttachISO.VM.Name) +func (_ *VMPowerOn) RenderVSphere(t *vsphere.VSphereAPITarget, a, e, changes *VMPowerOn) error { + glog.V(2).Infof("VMPowerOn.RenderVSphere invoked for vm %s", *changes.AttachISO.VM.Name) err := t.Cloud.PowerOn(*changes.AttachISO.VM.Name) return err }