Skip to content

Commit

Permalink
Add a provision for handling image based bootstrap (#406)
Browse files Browse the repository at this point in the history
* Add a provision for handling image based bootstrap

AHV has a limit of 32KB for cloud-init userdata. In Openshift, the
ignition can be rather large (a magnitude over the limit). In
order to support larger userdata files, we allow mounting the customization
as an image.

* Only set guestcustomization explicitly when bootstrap ref is secret

* Use lowercase for data_source_reference kind.

---------

Co-authored-by: Sid Shukla <[email protected]>
  • Loading branch information
yanhua121 and thunderboltsid committed Apr 11, 2024
1 parent 2591f82 commit 99adbe3
Show file tree
Hide file tree
Showing 4 changed files with 126 additions and 51 deletions.
10 changes: 10 additions & 0 deletions api/v1beta1/nutanixmachine_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,16 @@ const (
// resources associated with NutanixMachine before removing it from the
// API Server.
NutanixMachineFinalizer = "nutanixmachine.infrastructure.cluster.x-k8s.io"

// NutanixMachineBootstrapRefKindSecret represents the Kind of Secret
// referenced by NutanixMachine's BootstrapRef.
NutanixMachineBootstrapRefKindSecret = "Secret"

// NutanixMachineBootstrapRefKindImage represents the Kind of Image
// referenced by NutanixMachine's BootstrapRef. If the BootstrapRef.Kind is set
// to Image, the NutanixMachine will be created with the image mounted
// as a CD-ROM.
NutanixMachineBootstrapRefKindImage = "Image"
)

// NutanixMachineSpec defines the desired state of NutanixMachine
Expand Down
159 changes: 112 additions & 47 deletions controllers/nutanixmachine_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import (
apitypes "k8s.io/apimachinery/pkg/types"
kerrors "k8s.io/apimachinery/pkg/util/errors"
coreinformers "k8s.io/client-go/informers/core/v1"
"k8s.io/utils/ptr"
capiv1 "sigs.k8s.io/cluster-api/api/v1beta1"
capierrors "sigs.k8s.io/cluster-api/errors"
capiutil "sigs.k8s.io/cluster-api/util"
Expand All @@ -56,6 +57,9 @@ import (

const (
projectKind = "project"

deviceTypeCDROM = "CDROM"
adapterTypeIDE = "IDE"
)

var (
Expand Down Expand Up @@ -518,31 +522,6 @@ func (r *NutanixMachineReconciler) getOrCreateVM(rctx *nctx.MachineContext) (*nu
return nil, err
}

// Get Image UUID
imageUUID, err := GetImageUUID(ctx, nc, rctx.NutanixMachine.Spec.Image.Name, rctx.NutanixMachine.Spec.Image.UUID)
if err != nil {
errorMsg := fmt.Errorf("failed to get the image UUID to create the VM %s. %v", vmName, err)
rctx.SetFailureStatus(capierrors.CreateMachineError, errorMsg)
return nil, err
}

// Get the bootstrapData from the referenced secret
bootstrapData, err := r.getBootstrapData(rctx)
if err != nil {
log.Error(err, fmt.Sprintf("failed to get the bootstrap data to create the VM %s", vmName))
return nil, err
}
// Encode the bootstrapData by base64
bsdataEncoded := base64.StdEncoding.EncodeToString(bootstrapData)
log.V(1).Info(fmt.Sprintf("Retrieved the bootstrap data from secret %s (before encoding size: %d, encoded string size:%d)",
rctx.NutanixMachine.Spec.BootstrapRef.Name, len(bootstrapData), len(bsdataEncoded)))

// Generate metadata for the VM
vmUUID := uuid.New()
metadata := fmt.Sprintf("{\"hostname\": \"%s\", \"uuid\": \"%s\"}", rctx.Machine.Name, vmUUID)
// Encode the metadata by base64
metadataEncoded := base64.StdEncoding.EncodeToString([]byte(metadata))

vmInput := &nutanixClientV3.VMIntentInput{}
vmSpec := &nutanixClientV3.VM{Name: utils.StringPtr(vmName)}

Expand All @@ -556,19 +535,6 @@ func (r *NutanixMachineReconciler) getOrCreateVM(rctx *nctx.MachineContext) (*nu
}
}

// Create Disk Spec for systemdisk to be set later in VM Spec
diskSize := rctx.NutanixMachine.Spec.SystemDiskSize
diskSizeMib := GetMibValueOfQuantity(diskSize)
systemDisk, err := CreateSystemDiskSpec(imageUUID, diskSizeMib)
if err != nil {
errorMsg := fmt.Errorf("error occurred while creating system disk spec: %v", err)
rctx.SetFailureStatus(capierrors.CreateMachineError, errorMsg)
return nil, errorMsg
}
diskList := []*nutanixClientV3.VMDisk{
systemDisk,
}

// Set Categories to VM Sepc before creating VM
categories, err := GetCategoryVMSpec(ctx, nc, r.getMachineCategoryIdentifiers(rctx))
if err != nil {
Expand Down Expand Up @@ -598,8 +564,14 @@ func (r *NutanixMachineReconciler) getOrCreateVM(rctx *nctx.MachineContext) (*nu
return nil, err
}

memorySize := rctx.NutanixMachine.Spec.MemorySize
memorySizeMib := GetMibValueOfQuantity(memorySize)
diskList, err := getDiskList(rctx)
if err != nil {
errorMsg := fmt.Errorf("failed to get the disk list to create the VM %s. %v", vmName, err)
rctx.SetFailureStatus(capierrors.CreateMachineError, errorMsg)
return nil, err

Check warning on line 571 in controllers/nutanixmachine_controller.go

View check run for this annotation

Codecov / codecov/patch

controllers/nutanixmachine_controller.go#L567-L571

Added lines #L567 - L571 were not covered by tests
}

memorySizeMib := GetMibValueOfQuantity(rctx.NutanixMachine.Spec.MemorySize)

Check warning on line 574 in controllers/nutanixmachine_controller.go

View check run for this annotation

Codecov / codecov/patch

controllers/nutanixmachine_controller.go#L574

Added line #L574 was not covered by tests
vmSpec.Resources = &nutanixClientV3.VMResources{
PowerState: utils.StringPtr("ON"),
HardwareClockTimezone: utils.StringPtr("UTC"),
Expand All @@ -609,19 +581,18 @@ func (r *NutanixMachineReconciler) getOrCreateVM(rctx *nctx.MachineContext) (*nu
NicList: nicList,
DiskList: diskList,
GpuList: gpuList,
GuestCustomization: &nutanixClientV3.GuestCustomization{
IsOverridable: utils.BoolPtr(true),
CloudInit: &nutanixClientV3.GuestCustomizationCloudInit{
UserData: utils.StringPtr(bsdataEncoded),
MetaData: utils.StringPtr(metadataEncoded),
},
},
}
vmSpec.ClusterReference = &nutanixClientV3.Reference{
Kind: utils.StringPtr("cluster"),
UUID: utils.StringPtr(peUUID),
}

if err := r.addGuestCustomizationToVM(rctx, vmSpec); err != nil {
errorMsg := fmt.Errorf("error occurred while adding guest customization to vm spec: %v", err)
rctx.SetFailureStatus(capierrors.CreateMachineError, errorMsg)
return nil, err

Check warning on line 593 in controllers/nutanixmachine_controller.go

View check run for this annotation

Codecov / codecov/patch

controllers/nutanixmachine_controller.go#L590-L593

Added lines #L590 - L593 were not covered by tests
}

// Set BootType in VM Spec before creating VM
err = r.addBootTypeToVM(rctx, vmSpec)
if err != nil {
Expand Down Expand Up @@ -685,6 +656,100 @@ func (r *NutanixMachineReconciler) getOrCreateVM(rctx *nctx.MachineContext) (*nu
return vm, nil
}

func (r *NutanixMachineReconciler) addGuestCustomizationToVM(rctx *nctx.MachineContext, vmSpec *nutanixClientV3.VM) error {
// Get the bootstrapData
bootstrapRef := rctx.NutanixMachine.Spec.BootstrapRef
if bootstrapRef.Kind == infrav1.NutanixMachineBootstrapRefKindSecret {
bootstrapData, err := r.getBootstrapData(rctx)
if err != nil {
return err

Check warning on line 665 in controllers/nutanixmachine_controller.go

View check run for this annotation

Codecov / codecov/patch

controllers/nutanixmachine_controller.go#L661-L665

Added lines #L661 - L665 were not covered by tests
}

// Encode the bootstrapData by base64
bsdataEncoded := base64.StdEncoding.EncodeToString(bootstrapData)
metadata := fmt.Sprintf("{\"hostname\": \"%s\", \"uuid\": \"%s\"}", rctx.Machine.Name, uuid.New())
metadataEncoded := base64.StdEncoding.EncodeToString([]byte(metadata))

Check warning on line 671 in controllers/nutanixmachine_controller.go

View check run for this annotation

Codecov / codecov/patch

controllers/nutanixmachine_controller.go#L669-L671

Added lines #L669 - L671 were not covered by tests

vmSpec.Resources.GuestCustomization = &nutanixClientV3.GuestCustomization{

Check warning on line 673 in controllers/nutanixmachine_controller.go

View check run for this annotation

Codecov / codecov/patch

controllers/nutanixmachine_controller.go#L673

Added line #L673 was not covered by tests
IsOverridable: utils.BoolPtr(true),
CloudInit: &nutanixClientV3.GuestCustomizationCloudInit{
UserData: utils.StringPtr(bsdataEncoded),
MetaData: utils.StringPtr(metadataEncoded),
},
}
}

return nil

Check warning on line 682 in controllers/nutanixmachine_controller.go

View check run for this annotation

Codecov / codecov/patch

controllers/nutanixmachine_controller.go#L682

Added line #L682 was not covered by tests
}

func getDiskList(rctx *nctx.MachineContext) ([]*nutanixClientV3.VMDisk, error) {
diskList := make([]*nutanixClientV3.VMDisk, 0)

Check warning on line 686 in controllers/nutanixmachine_controller.go

View check run for this annotation

Codecov / codecov/patch

controllers/nutanixmachine_controller.go#L686

Added line #L686 was not covered by tests

systemDisk, err := getSystemDisk(rctx)
if err != nil {
return nil, err

Check warning on line 690 in controllers/nutanixmachine_controller.go

View check run for this annotation

Codecov / codecov/patch

controllers/nutanixmachine_controller.go#L688-L690

Added lines #L688 - L690 were not covered by tests
}
diskList = append(diskList, systemDisk)

Check warning on line 692 in controllers/nutanixmachine_controller.go

View check run for this annotation

Codecov / codecov/patch

controllers/nutanixmachine_controller.go#L692

Added line #L692 was not covered by tests

bootstrapRef := rctx.NutanixMachine.Spec.BootstrapRef
if bootstrapRef.Kind == infrav1.NutanixMachineBootstrapRefKindImage {
bootstrapDisk, err := getBootstrapDisk(rctx)
if err != nil {
return nil, err

Check warning on line 698 in controllers/nutanixmachine_controller.go

View check run for this annotation

Codecov / codecov/patch

controllers/nutanixmachine_controller.go#L694-L698

Added lines #L694 - L698 were not covered by tests
}

diskList = append(diskList, bootstrapDisk)

Check warning on line 701 in controllers/nutanixmachine_controller.go

View check run for this annotation

Codecov / codecov/patch

controllers/nutanixmachine_controller.go#L701

Added line #L701 was not covered by tests
}

return diskList, nil

Check warning on line 704 in controllers/nutanixmachine_controller.go

View check run for this annotation

Codecov / codecov/patch

controllers/nutanixmachine_controller.go#L704

Added line #L704 was not covered by tests
}

func getSystemDisk(rctx *nctx.MachineContext) (*nutanixClientV3.VMDisk, error) {
nodeOSImageName := rctx.NutanixMachine.Spec.Image.Name
nodeOSImageUUID, err := GetImageUUID(rctx.Context, rctx.NutanixClient, nodeOSImageName, rctx.NutanixMachine.Spec.Image.UUID)
if err != nil {
errorMsg := fmt.Errorf("failed to get the image UUID for image named %q: %w", *nodeOSImageName, err)
rctx.SetFailureStatus(capierrors.CreateMachineError, errorMsg)
return nil, err

Check warning on line 713 in controllers/nutanixmachine_controller.go

View check run for this annotation

Codecov / codecov/patch

controllers/nutanixmachine_controller.go#L708-L713

Added lines #L708 - L713 were not covered by tests
}

systemDiskSizeMib := GetMibValueOfQuantity(rctx.NutanixMachine.Spec.SystemDiskSize)
systemDisk, err := CreateSystemDiskSpec(nodeOSImageUUID, systemDiskSizeMib)
if err != nil {
errorMsg := fmt.Errorf("error occurred while creating system disk spec: %w", err)
rctx.SetFailureStatus(capierrors.CreateMachineError, errorMsg)
return nil, err

Check warning on line 721 in controllers/nutanixmachine_controller.go

View check run for this annotation

Codecov / codecov/patch

controllers/nutanixmachine_controller.go#L716-L721

Added lines #L716 - L721 were not covered by tests
}

return systemDisk, nil

Check warning on line 724 in controllers/nutanixmachine_controller.go

View check run for this annotation

Codecov / codecov/patch

controllers/nutanixmachine_controller.go#L724

Added line #L724 was not covered by tests
}

func getBootstrapDisk(rctx *nctx.MachineContext) (*nutanixClientV3.VMDisk, error) {
bootstrapImageName := rctx.NutanixMachine.Spec.BootstrapRef.Name
bootstrapImageUUID, err := GetImageUUID(rctx.Context, rctx.NutanixClient, &bootstrapImageName, nil)
if err != nil {
errorMsg := fmt.Errorf("failed to get the image UUID for image named %q: %w", bootstrapImageName, err)
rctx.SetFailureStatus(capierrors.CreateMachineError, errorMsg)
return nil, err

Check warning on line 733 in controllers/nutanixmachine_controller.go

View check run for this annotation

Codecov / codecov/patch

controllers/nutanixmachine_controller.go#L728-L733

Added lines #L728 - L733 were not covered by tests
}

bootstrapDisk := &nutanixClientV3.VMDisk{

Check warning on line 736 in controllers/nutanixmachine_controller.go

View check run for this annotation

Codecov / codecov/patch

controllers/nutanixmachine_controller.go#L736

Added line #L736 was not covered by tests
DeviceProperties: &nutanixClientV3.VMDiskDeviceProperties{
DeviceType: ptr.To(deviceTypeCDROM),
DiskAddress: &nutanixClientV3.DiskAddress{
AdapterType: ptr.To(adapterTypeIDE),
DeviceIndex: ptr.To(int64(0)),
},
},
DataSourceReference: &nutanixClientV3.Reference{
Kind: ptr.To(strings.ToLower(infrav1.NutanixMachineBootstrapRefKindImage)),
UUID: ptr.To(bootstrapImageUUID),
},
}

return bootstrapDisk, nil

Check warning on line 750 in controllers/nutanixmachine_controller.go

View check run for this annotation

Codecov / codecov/patch

controllers/nutanixmachine_controller.go#L750

Added line #L750 was not covered by tests
}

// getBootstrapData returns the Bootstrap data from the ref secret
func (r *NutanixMachineReconciler) getBootstrapData(rctx *nctx.MachineContext) ([]byte, error) {
if rctx.NutanixMachine.Spec.BootstrapRef == nil {
Expand Down
4 changes: 2 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ go 1.21

require (
github.com/blang/semver v3.5.1+incompatible
github.com/blang/semver/v4 v4.0.0
github.com/google/uuid v1.3.0
github.com/nutanix-cloud-native/prism-go-client v0.3.4
github.com/onsi/ginkgo/v2 v2.6.0
Expand All @@ -15,7 +16,7 @@ require (
k8s.io/apiextensions-apiserver v0.25.2
k8s.io/apimachinery v0.25.2
k8s.io/client-go v0.25.2
k8s.io/utils v0.0.0-20220823124924-e9cbc92d1a73
k8s.io/utils v0.0.0-20240310230437-4693a0247e57
sigs.k8s.io/cluster-api v1.3.5
sigs.k8s.io/cluster-api/test v1.3.5
sigs.k8s.io/controller-runtime v0.13.1
Expand Down Expand Up @@ -43,7 +44,6 @@ require (
github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20220418222510-f25a4f6275ed // indirect
github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/blang/semver/v4 v4.0.0 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/coredns/caddy v1.1.0 // indirect
github.com/coredns/corefile-migration v1.0.20 // indirect
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -1357,8 +1357,8 @@ k8s.io/kube-openapi v0.0.0-20220803164354-a70c9af30aea h1:3QOH5+2fGsY8e1qf+GIFpg
k8s.io/kube-openapi v0.0.0-20220803164354-a70c9af30aea/go.mod h1:C/N6wCaBHeBHkHUesQOQy2/MZqGgMAFPqGsGQLdbZBU=
k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
k8s.io/utils v0.0.0-20220823124924-e9cbc92d1a73 h1:H9TCJUUx+2VA0ZiD9lvtaX8fthFsMoD+Izn93E/hm8U=
k8s.io/utils v0.0.0-20220823124924-e9cbc92d1a73/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
k8s.io/utils v0.0.0-20240310230437-4693a0247e57 h1:gbqbevonBh57eILzModw6mrkbwM0gQBEuevE/AaBsHY=
k8s.io/utils v0.0.0-20240310230437-4693a0247e57/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
Expand Down

0 comments on commit 99adbe3

Please sign in to comment.