diff --git a/pkg/model/vspheremodel/autoscalinggroup.go b/pkg/model/vspheremodel/autoscalinggroup.go index e4ce7c2efab9f..958ba1de829b1 100644 --- a/pkg/model/vspheremodel/autoscalinggroup.go +++ b/pkg/model/vspheremodel/autoscalinggroup.go @@ -17,7 +17,6 @@ limitations under the License. package vspheremodel import ( - "github.com/golang/glog" "k8s.io/kops/pkg/model" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/cloudup/vspheretasks" @@ -32,13 +31,16 @@ type AutoscalingGroupModelBuilder struct { var _ fi.ModelBuilder = &AutoscalingGroupModelBuilder{} +const defaultVmTemplateName = "Ubuntu_16_10" + func (b *AutoscalingGroupModelBuilder) Build(c *fi.ModelBuilderContext) error { - glog.Warning("AutoscalingGroupModelBuilder.Build not implemented for vsphere") + // Note that we are creating a VM per instance group. Instance group represents a group of VMs. + // This logic should change once we add support for multiple master and worker nodes. for _, ig := range b.InstanceGroups { name := b.AutoscalingGroupName(ig) createVmTask := &vspheretasks.VirtualMachine{ Name: &name, - VMTemplateName: fi.String("dummyVmTemplate"), + VMTemplateName: fi.String(defaultVmTemplateName), } c.AddTask(createVmTask) diff --git a/upup/pkg/fi/cloudup/vsphere/vsphere_cloud.go b/upup/pkg/fi/cloudup/vsphere/vsphere_cloud.go index 842affe359aac..4c294589eb12c 100644 --- a/upup/pkg/fi/cloudup/vsphere/vsphere_cloud.go +++ b/upup/pkg/fi/cloudup/vsphere/vsphere_cloud.go @@ -17,12 +17,20 @@ limitations under the License. package vsphere import ( + "context" "fmt" "github.com/golang/glog" + "github.com/pkg/errors" + "github.com/vmware/govmomi" + "github.com/vmware/govmomi/find" + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25" + "github.com/vmware/govmomi/vim25/types" "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/upup/pkg/fi" "k8s.io/kubernetes/federation/pkg/dnsprovider" k8sroute53 "k8s.io/kubernetes/federation/pkg/dnsprovider/providers/aws/route53" + "net/url" "os" ) @@ -32,8 +40,14 @@ type VSphereCloud struct { Cluster string Username string Password string + Client *govmomi.Client } +const ( + snapshotName string = "LinkCloneSnapshotPoint" + snapshotDesc string = "Snapshot created by kops" +) + var _ fi.Cloud = &VSphereCloud{} func (c *VSphereCloud) ProviderID() fi.CloudProviderID { @@ -44,15 +58,32 @@ func NewVSphereCloud(spec *kops.ClusterSpec) (*VSphereCloud, error) { server := *spec.CloudConfig.VSphereServer datacenter := *spec.CloudConfig.VSphereDatacenter cluster := *spec.CloudConfig.VSphereResourcePool + glog.V(2).Infof("Creating vSphere Cloud with server(%s), datacenter(%s), cluster(%s)", server, datacenter, cluster) + username := os.Getenv("VSPHERE_USERNAME") password := os.Getenv("VSPHERE_PASSWORD") if username == "" || password == "" { return nil, fmt.Errorf("Failed to detect vSphere username and password. Please set env variables: VSPHERE_USERNAME and VSPHERE_PASSWORD accordingly.") } - c := &VSphereCloud{Server: server, Datacenter: datacenter, Cluster: cluster, Username: username, Password: password} - // TODO: create a client of govmomi here? - return c, nil + u, err := url.Parse(fmt.Sprintf("https://%s/sdk", server)) + if err != nil { + return nil, err + } + glog.V(2).Infof("Creating vSphere Cloud URL is %s", u) + + // set username and password in URL + u.User = url.UserPassword(username, password) + + c, err := govmomi.NewClient(context.TODO(), u, true) + if err != nil { + return nil, err + } + // Add retry functionality + c.RoundTripper = vim25.Retry(c.RoundTripper, vim25.TemporaryNetworkError(5)) + vsphereCloud := &VSphereCloud{Server: server, Datacenter: datacenter, Cluster: cluster, Username: username, Password: password, Client: c} + glog.V(2).Infof("Created vSphere Cloud successfully: %+v", vsphereCloud) + return vsphereCloud, nil } func (c *VSphereCloud) DNS() (dnsprovider.Interface, error) { @@ -66,6 +97,79 @@ func (c *VSphereCloud) DNS() (dnsprovider.Interface, error) { } func (c *VSphereCloud) FindVPCInfo(id string) (*fi.VPCInfo, error) { - glog.Warningf("FindVPCInfo not (yet) implemented on VSphere") + glog.Warning("FindVPCInfo not (yet) implemented on VSphere") return nil, nil } + +func (c *VSphereCloud) CreateLinkClonedVm(vmName, vmImage *string) (string, error) { + f := find.NewFinder(c.Client.Client, true) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + dc, err := f.Datacenter(ctx, c.Datacenter) + if err != nil { + return "", err + } + f.SetDatacenter(dc) + + templateVm, err := f.VirtualMachine(ctx, *vmImage) + if err != nil { + return "", err + } + + glog.V(2).Infof("Template VM ref is %+v", templateVm) + datacenterFolders, err := dc.Folders(ctx) + if err != nil { + return "", err + } + + // Create snapshot of the template VM if not already snapshotted. + snapshot, err := createSnapshot(ctx, templateVm, snapshotName, snapshotDesc) + if err != nil { + return "", err + } + + clsComputeRes, err := f.ClusterComputeResource(ctx, c.Cluster) + glog.V(4).Infof("Cluster compute resource is %+v", clsComputeRes) + if err != nil { + return "", err + } + + resPool, err := clsComputeRes.ResourcePool(ctx) + glog.V(4).Infof("Cluster resource pool is %+v", resPool) + if err != nil { + return "", err + } + + if resPool == nil { + return "", errors.New(fmt.Sprintf("No resource pool found for cluster %s", c.Cluster)) + } + + resPoolRef := resPool.Reference() + snapshotRef := snapshot.Reference() + + cloneSpec := &types.VirtualMachineCloneSpec{ + Config: &types.VirtualMachineConfigSpec{}, + Location: types.VirtualMachineRelocateSpec{ + Pool: &resPoolRef, + DiskMoveType: "createNewChildDiskBacking", + }, + Snapshot: &snapshotRef, + } + + // Create a link cloned VM from the template VM's snapshot + clonedVmTask, err := templateVm.Clone(ctx, datacenterFolders.VmFolder, *vmName, *cloneSpec) + if err != nil { + return "", err + } + + clonedVmTaskInfo, err := clonedVmTask.WaitForResult(ctx, nil) + if err != nil { + return "", err + } + + clonedVm := clonedVmTaskInfo.Result.(object.Reference) + glog.V(2).Infof("Created VM %s successfully", clonedVm) + + return clonedVm.Reference().Value, nil +} diff --git a/upup/pkg/fi/cloudup/vsphere/vsphere_utils.go b/upup/pkg/fi/cloudup/vsphere/vsphere_utils.go new file mode 100644 index 0000000000000..6fade3eedb772 --- /dev/null +++ b/upup/pkg/fi/cloudup/vsphere/vsphere_utils.go @@ -0,0 +1,103 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vsphere + +import ( + "context" + "github.com/golang/glog" + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25/mo" + "github.com/vmware/govmomi/vim25/types" + "path" + "sync" +) + +var snapshotLock sync.Mutex + +func createSnapshot(ctx context.Context, vm *object.VirtualMachine, snapshotName string, snapshotDesc string) (object.Reference, error) { + snapshotLock.Lock() + defer snapshotLock.Unlock() + + snapshotRef, err := findSnapshot(vm, ctx, snapshotName) + if err != nil { + return nil, err + } + glog.V(4).Infof("Template VM is %s and snapshot is %s", vm, snapshotRef) + if snapshotRef != nil { + return snapshotRef, nil + } + + task, err := vm.CreateSnapshot(ctx, snapshotName, snapshotDesc, false, false) + if err != nil { + return nil, err + } + + taskInfo, err := task.WaitForResult(ctx, nil) + if err != nil { + return nil, err + } + glog.Infof("taskInfo.Result is %s", taskInfo.Result) + return taskInfo.Result.(object.Reference), nil +} + +type snapshotMap map[string][]object.Reference + +func (m snapshotMap) add(parent string, tree []types.VirtualMachineSnapshotTree) { + for i, st := range tree { + sname := st.Name + names := []string{sname, st.Snapshot.Value} + + if parent != "" { + sname = path.Join(parent, sname) + // Add full path as an option to resolve duplicate names + names = append(names, sname) + } + + for _, name := range names { + m[name] = append(m[name], &tree[i].Snapshot) + } + + m.add(sname, st.ChildSnapshotList) + } +} + +func findSnapshot(v *object.VirtualMachine, ctx context.Context, name string) (object.Reference, error) { + var o mo.VirtualMachine + + err := v.Properties(ctx, v.Reference(), []string{"snapshot"}, &o) + if err != nil { + return nil, err + } + + if o.Snapshot == nil || len(o.Snapshot.RootSnapshotList) == 0 { + return nil, nil + } + + m := make(snapshotMap) + m.add("", o.Snapshot.RootSnapshotList) + + s := m[name] + switch len(s) { + case 0: + return nil, nil + case 1: + return s[0], nil + default: + glog.Warningf("VM %s seems to have more than one snapshots with name %s. Using a random snapshot.", v, name) + return s[0], nil + } +} diff --git a/upup/pkg/fi/cloudup/vspheretasks/virtualmachine.go b/upup/pkg/fi/cloudup/vspheretasks/virtualmachine.go index 4fc9bb1a09176..bdcb3f32e6ace 100644 --- a/upup/pkg/fi/cloudup/vspheretasks/virtualmachine.go +++ b/upup/pkg/fi/cloudup/vspheretasks/virtualmachine.go @@ -48,26 +48,30 @@ func (o *VirtualMachine) String() string { } func (e *VirtualMachine) CompareWithID() *string { - glog.Info("VirtualMachine.CompareWithID invoked!") + glog.V(4).Info("VirtualMachine.CompareWithID invoked!") return e.Name } func (e *VirtualMachine) Find(c *fi.Context) (*VirtualMachine, error) { - glog.Info("VirtualMachine.Find invoked!") + glog.V(4).Info("VirtualMachine.Find invoked!") return nil, nil } func (e *VirtualMachine) Run(c *fi.Context) error { - glog.Info("VirtualMachine.Run invoked!") + glog.V(4).Info("VirtualMachine.Run invoked!") return fi.DefaultDeltaRunMethod(e, c) } func (_ *VirtualMachine) CheckChanges(a, e, changes *VirtualMachine) error { - glog.Info("VirtualMachine.CheckChanges invoked!") + glog.V(4).Info("VirtualMachine.CheckChanges invoked!") return nil } func (_ *VirtualMachine) RenderVSphere(t *vsphere.VSphereAPITarget, a, e, changes *VirtualMachine) error { - glog.Info("VirtualMachine.RenderVSphere invoked!") + glog.V(4).Infof("VirtualMachine.RenderVSphere invoked with a(%+v) e(%+v) and changes(%+v)", a, e, changes) + _, err := t.Cloud.CreateLinkClonedVm(changes.Name, changes.VMTemplateName) + if err != nil { + return err + } return nil }