From c5fef72cf5e88f292ceea910e6f3114b6aa36ef5 Mon Sep 17 00:00:00 2001 From: Thomas Tendyck Date: Wed, 12 Apr 2023 13:54:36 +0200 Subject: [PATCH] split util package to reduce dependencies of premain --- cli/internal/cmd/install.go | 10 +++---- cli/internal/cmd/install_test.go | 10 +++---- cli/internal/cmd/precheck.go | 12 ++++----- cli/internal/cmd/precheck_test.go | 16 +++++------ cli/internal/helm/client.go | 20 +++++++------- cli/internal/helm/client_test.go | 44 +++++++++++++++---------------- injector/injector.go | 12 ++++----- util/k8sutil/k8sutil.go | 34 ++++++++++++++++++++++++ util/util.go | 26 ------------------ 9 files changed, 96 insertions(+), 88 deletions(-) create mode 100644 util/k8sutil/k8sutil.go diff --git a/cli/internal/cmd/install.go b/cli/internal/cmd/install.go index dfbbafff..2a2819dc 100644 --- a/cli/internal/cmd/install.go +++ b/cli/internal/cmd/install.go @@ -16,7 +16,7 @@ import ( "github.com/edgelesssys/marblerun/cli/internal/helm" "github.com/edgelesssys/marblerun/cli/internal/kube" - "github.com/edgelesssys/marblerun/util" + "github.com/edgelesssys/marblerun/util/k8sutil" "github.com/spf13/cobra" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -225,18 +225,18 @@ func getSGXResourceKey(ctx context.Context, kubeClient kubernetes.Interface) (st for _, node := range nodes.Items { if nodeHasAlibabaDevPlugin(node.Status.Capacity) { - return util.AlibabaEpc.String(), nil + return k8sutil.AlibabaEpc.String(), nil } if nodeHasAzureDevPlugin(node.Status.Capacity) { - return util.AzureEpc.String(), nil + return k8sutil.AzureEpc.String(), nil } if nodeHasIntelDevPlugin(node.Status.Capacity) { - return util.IntelEpc.String(), nil + return k8sutil.IntelEpc.String(), nil } } // assume cluster has the intel SGX device plugin by default - return util.IntelEpc.String(), nil + return k8sutil.IntelEpc.String(), nil } // errorAndCleanup returns the given error and deletes resources which might have been created previously. diff --git a/cli/internal/cmd/install_test.go b/cli/internal/cmd/install_test.go index bdd9060f..a59a9623 100644 --- a/cli/internal/cmd/install_test.go +++ b/cli/internal/cmd/install_test.go @@ -16,7 +16,7 @@ import ( "testing" "github.com/edgelesssys/marblerun/cli/internal/helm" - "github.com/edgelesssys/marblerun/util" + "github.com/edgelesssys/marblerun/util/k8sutil" "github.com/spf13/cobra" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -151,9 +151,9 @@ func TestGetSGXResourceKey(t *testing.T) { }, Status: corev1.NodeStatus{ Capacity: corev1.ResourceList{ - util.IntelEnclave: resource.MustParse("10"), - util.IntelEpc: resource.MustParse("500"), - util.IntelProvision: resource.MustParse("10"), + k8sutil.IntelEnclave: resource.MustParse("10"), + k8sutil.IntelEpc: resource.MustParse("500"), + k8sutil.IntelProvision: resource.MustParse("10"), }, }, } @@ -162,7 +162,7 @@ func TestGetSGXResourceKey(t *testing.T) { resourceKey, err := getSGXResourceKey(ctx, testClient) assert.NoError(err) - assert.Equal(util.IntelEpc.String(), resourceKey) + assert.Equal(k8sutil.IntelEpc.String(), resourceKey) } func TestErrorAndCleanup(t *testing.T) { diff --git a/cli/internal/cmd/precheck.go b/cli/internal/cmd/precheck.go index 497ef6c8..772dc0c3 100644 --- a/cli/internal/cmd/precheck.go +++ b/cli/internal/cmd/precheck.go @@ -8,7 +8,7 @@ package cmd import ( "github.com/edgelesssys/marblerun/cli/internal/kube" - "github.com/edgelesssys/marblerun/util" + "github.com/edgelesssys/marblerun/util/k8sutil" "github.com/spf13/cobra" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -75,20 +75,20 @@ func nodeSupportsSGX(capacityInfo corev1.ResourceList) bool { // nodeHasAlibabaDevPlugin checks if a node has the Alibaba device plugin installed (https://github.com/AliyunContainerService/sgx-device-plugin). func nodeHasAlibabaDevPlugin(capacityInfo corev1.ResourceList) bool { - epcQuant := capacityInfo[util.AlibabaEpc] + epcQuant := capacityInfo[k8sutil.AlibabaEpc] return epcQuant.Value() != 0 } // nodeHasAzureDevPlugin checks if a node has the Azures SGX device plugin installed (https://github.com/Azure/aks-engine/blob/master/docs/topics/sgx.md#deploying-the-sgx-device-plugin). func nodeHasAzureDevPlugin(capacityInfo corev1.ResourceList) bool { - epcQuant := capacityInfo[util.AzureEpc] + epcQuant := capacityInfo[k8sutil.AzureEpc] return epcQuant.Value() != 0 } // nodeHasIntelDevPlugin checks if a node has the Intel SGX device plugin installed (https://github.com/intel/intel-device-plugins-for-kubernetes#sgx-device-plugin). func nodeHasIntelDevPlugin(capacityInfo corev1.ResourceList) bool { - epcQuant := capacityInfo[util.IntelEpc] - enclaveQuant := capacityInfo[util.IntelEnclave] - provisionQuant := capacityInfo[util.IntelProvision] + epcQuant := capacityInfo[k8sutil.IntelEpc] + enclaveQuant := capacityInfo[k8sutil.IntelEnclave] + provisionQuant := capacityInfo[k8sutil.IntelProvision] return !(epcQuant.Value() == 0 || enclaveQuant.Value() == 0 || provisionQuant.Value() == 0) } diff --git a/cli/internal/cmd/precheck_test.go b/cli/internal/cmd/precheck_test.go index c2f4b870..24d4c1f5 100644 --- a/cli/internal/cmd/precheck_test.go +++ b/cli/internal/cmd/precheck_test.go @@ -11,7 +11,7 @@ import ( "context" "testing" - "github.com/edgelesssys/marblerun/util" + "github.com/edgelesssys/marblerun/util/k8sutil" "github.com/spf13/cobra" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -53,9 +53,9 @@ func TestNodeSupportsSGX(t *testing.T) { }, Status: corev1.NodeStatus{ Capacity: corev1.ResourceList{ - util.IntelEnclave: resource.MustParse("10"), - util.IntelEpc: resource.MustParse("500"), - util.IntelProvision: resource.MustParse("10"), + k8sutil.IntelEnclave: resource.MustParse("10"), + k8sutil.IntelEpc: resource.MustParse("500"), + k8sutil.IntelProvision: resource.MustParse("10"), }, }, } @@ -78,7 +78,7 @@ func TestNodeSupportsSGX(t *testing.T) { }, Status: corev1.NodeStatus{ Capacity: corev1.ResourceList{ - util.AzureEpc: resource.MustParse("500"), + k8sutil.AzureEpc: resource.MustParse("500"), }, }, } @@ -126,9 +126,9 @@ func TestCliCheckSGXSupport(t *testing.T) { }, Status: corev1.NodeStatus{ Capacity: corev1.ResourceList{ - util.IntelEnclave: resource.MustParse("10"), - util.IntelEpc: resource.MustParse("500"), - util.IntelProvision: resource.MustParse("10"), + k8sutil.IntelEnclave: resource.MustParse("10"), + k8sutil.IntelEpc: resource.MustParse("500"), + k8sutil.IntelProvision: resource.MustParse("10"), }, }, } diff --git a/cli/internal/helm/client.go b/cli/internal/helm/client.go index 4ea1b723..69a107ee 100644 --- a/cli/internal/helm/client.go +++ b/cli/internal/helm/client.go @@ -16,7 +16,7 @@ import ( "strings" "time" - "github.com/edgelesssys/marblerun/util" + "github.com/edgelesssys/marblerun/util/k8sutil" "github.com/gofrs/flock" "gopkg.in/yaml.v2" "helm.sh/helm/v3/pkg/action" @@ -277,7 +277,7 @@ func setSGXValues(resourceKey string, values, chartValues map[string]interface{} } var needNewLimit bool - limit := util.GetEPCResourceLimit(resourceKey) + limit := k8sutil.GetEPCResourceLimit(resourceKey) // remove all previously set sgx resource limits if presetLimits, ok := chartValues["coordinator"].(map[string]interface{})["resources"].(map[string]interface{})["limits"].(map[string]interface{}); ok { @@ -306,9 +306,9 @@ func setSGXValues(resourceKey string, values, chartValues map[string]interface{} } // Make sure provision and enclave bit is set if the Intel plugin is used - if resourceKey == util.IntelEpc.String() { - values["coordinator"].(map[string]interface{})["resources"].(map[string]interface{})["limits"].(map[string]interface{})[util.IntelProvision.String()] = 1 - values["coordinator"].(map[string]interface{})["resources"].(map[string]interface{})["limits"].(map[string]interface{})[util.IntelEnclave.String()] = 1 + if resourceKey == k8sutil.IntelEpc.String() { + values["coordinator"].(map[string]interface{})["resources"].(map[string]interface{})["limits"].(map[string]interface{})[k8sutil.IntelProvision.String()] = 1 + values["coordinator"].(map[string]interface{})["resources"].(map[string]interface{})["limits"].(map[string]interface{})[k8sutil.IntelEnclave.String()] = 1 } } @@ -316,18 +316,18 @@ func setSGXValues(resourceKey string, values, chartValues map[string]interface{} // Choice is based on the resource key of the used SGX device plugin. func needsDeletion(existingKey, sgxKey string) bool { sgxResources := []string{ - util.AlibabaEpc.String(), util.AzureEpc.String(), util.IntelEpc.String(), - util.IntelProvision.String(), util.IntelEnclave.String(), + k8sutil.AlibabaEpc.String(), k8sutil.AzureEpc.String(), k8sutil.IntelEpc.String(), + k8sutil.IntelProvision.String(), k8sutil.IntelEnclave.String(), } switch sgxKey { - case util.AlibabaEpc.String(), util.AzureEpc.String(): + case k8sutil.AlibabaEpc.String(), k8sutil.AzureEpc.String(): // Delete all non Alibaba/Azure SGX resources depending on the used SGX device plugin return sgxKey != existingKey && keyInList(existingKey, sgxResources) - case util.IntelEpc.String(): + case k8sutil.IntelEpc.String(): // Delete all non Intel SGX resources depending on the used SGX device plugin // Keep Intel provision and enclave bit - return keyInList(existingKey, []string{util.AlibabaEpc.String(), util.AzureEpc.String()}) + return keyInList(existingKey, []string{k8sutil.AlibabaEpc.String(), k8sutil.AzureEpc.String()}) default: // Either no SGX plugin or a custom SGX plugin is used // Delete all known SGX resources diff --git a/cli/internal/helm/client_test.go b/cli/internal/helm/client_test.go index 8b524705..05e602fb 100644 --- a/cli/internal/helm/client_test.go +++ b/cli/internal/helm/client_test.go @@ -9,7 +9,7 @@ package helm import ( "testing" - "github.com/edgelesssys/marblerun/util" + "github.com/edgelesssys/marblerun/util/k8sutil" "github.com/stretchr/testify/assert" ) @@ -20,62 +20,62 @@ func TestNeedsDeletion(t *testing.T) { wantDeletion bool }{ "intel key with azure plugin": { - existingKey: util.IntelEpc.String(), - sgxKey: util.AzureEpc.String(), + existingKey: k8sutil.IntelEpc.String(), + sgxKey: k8sutil.AzureEpc.String(), wantDeletion: true, }, "intel key with alibaba plugin": { - existingKey: util.IntelEpc.String(), - sgxKey: util.AlibabaEpc.String(), + existingKey: k8sutil.IntelEpc.String(), + sgxKey: k8sutil.AlibabaEpc.String(), wantDeletion: true, }, "azure key with intel plugin": { - existingKey: util.AzureEpc.String(), - sgxKey: util.IntelEpc.String(), + existingKey: k8sutil.AzureEpc.String(), + sgxKey: k8sutil.IntelEpc.String(), wantDeletion: true, }, "azure key with alibaba plugin": { - existingKey: util.AzureEpc.String(), - sgxKey: util.AlibabaEpc.String(), + existingKey: k8sutil.AzureEpc.String(), + sgxKey: k8sutil.AlibabaEpc.String(), wantDeletion: true, }, "alibaba key with intel plugin": { - existingKey: util.AlibabaEpc.String(), - sgxKey: util.IntelEpc.String(), + existingKey: k8sutil.AlibabaEpc.String(), + sgxKey: k8sutil.IntelEpc.String(), wantDeletion: true, }, "alibaba key with azure plugin": { - existingKey: util.AlibabaEpc.String(), - sgxKey: util.AzureEpc.String(), + existingKey: k8sutil.AlibabaEpc.String(), + sgxKey: k8sutil.AzureEpc.String(), wantDeletion: true, }, "same key": { - existingKey: util.IntelEpc.String(), - sgxKey: util.IntelEpc.String(), + existingKey: k8sutil.IntelEpc.String(), + sgxKey: k8sutil.IntelEpc.String(), wantDeletion: false, }, "intel provision with intel plugin": { - existingKey: util.IntelProvision.String(), - sgxKey: util.IntelEpc.String(), + existingKey: k8sutil.IntelProvision.String(), + sgxKey: k8sutil.IntelEpc.String(), wantDeletion: false, }, "intel enclave with intel plugin": { - existingKey: util.IntelEnclave.String(), - sgxKey: util.IntelEpc.String(), + existingKey: k8sutil.IntelEnclave.String(), + sgxKey: k8sutil.IntelEpc.String(), wantDeletion: false, }, "regular resource with intel plugin": { existingKey: "cpu", - sgxKey: util.IntelEpc.String(), + sgxKey: k8sutil.IntelEpc.String(), wantDeletion: false, }, "custom resource with intel plugin": { existingKey: "custom-sgx-resource", - sgxKey: util.IntelEpc.String(), + sgxKey: k8sutil.IntelEpc.String(), wantDeletion: false, }, "intel provision with custom plugin": { - existingKey: util.IntelProvision.String(), + existingKey: k8sutil.IntelProvision.String(), sgxKey: "custom-sgx-resource", wantDeletion: true, }, diff --git a/injector/injector.go b/injector/injector.go index fb4f60e4..ded1053e 100644 --- a/injector/injector.go +++ b/injector/injector.go @@ -15,7 +15,7 @@ import ( "net/http" "strings" - "github.com/edgelesssys/marblerun/util" + "github.com/edgelesssys/marblerun/util/k8sutil" v1 "k8s.io/api/admission/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -184,18 +184,18 @@ func mutate(body []byte, coordAddr, domainName, resourceKey string) ([]byte, err container.Resources.Limits = make(map[corev1.ResourceName]resource.Quantity) } switch resourceKey { - case util.IntelEpc.String(): + case k8sutil.IntelEpc.String(): // Intels device plugin offers 3 resources: // epc : sets EPC for the container // enclave : provides a handle to /dev/sgx_enclave // provision : provides a handle to /dev/sgx_provision, this is not needed when the Marble utilises out-of-process quote-generation - setResourceLimit(container.Resources.Limits, util.IntelEpc, util.GetEPCResourceLimit(resourceKey)) - setResourceLimit(container.Resources.Limits, util.IntelEnclave, "1") - setResourceLimit(container.Resources.Limits, util.IntelProvision, "1") + setResourceLimit(container.Resources.Limits, k8sutil.IntelEpc, k8sutil.GetEPCResourceLimit(resourceKey)) + setResourceLimit(container.Resources.Limits, k8sutil.IntelEnclave, "1") + setResourceLimit(container.Resources.Limits, k8sutil.IntelProvision, "1") default: // Azure and Alibaba Cloud plugins offer only 1 resource // for custom plugins we can only inject the resource provided by the `resourceKey` - setResourceLimit(container.Resources.Limits, corev1.ResourceName(resourceKey), util.GetEPCResourceLimit(resourceKey)) + setResourceLimit(container.Resources.Limits, corev1.ResourceName(resourceKey), k8sutil.GetEPCResourceLimit(resourceKey)) } } diff --git a/util/k8sutil/k8sutil.go b/util/k8sutil/k8sutil.go new file mode 100644 index 00000000..dc4222be --- /dev/null +++ b/util/k8sutil/k8sutil.go @@ -0,0 +1,34 @@ +// Copyright (c) Edgeless Systems GmbH. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +package k8sutil + +import corev1 "k8s.io/api/core/v1" + +const ( + IntelEpc corev1.ResourceName = "sgx.intel.com/epc" + IntelEnclave corev1.ResourceName = "sgx.intel.com/enclave" + IntelProvision corev1.ResourceName = "sgx.intel.com/provision" + AzureEpc corev1.ResourceName = "kubernetes.azure.com/sgx_epc_mem_in_MiB" + AlibabaEpc corev1.ResourceName = "alibabacloud.com/sgx_epc_MiB" +) + +// GetEPCResourceLimit returns the amount of EPC to set for k8s deployments depending on the used sgx device plugin. +func GetEPCResourceLimit(resourceKey string) string { + switch resourceKey { + case AzureEpc.String(): + // azure device plugin expects epc in MiB + return "10" + case AlibabaEpc.String(): + // alibaba device plugin expects epc in MiB + return "10" + case IntelEpc.String(): + // intels device plugin expects epc as a k8s resource quantity + return "10Mi" + default: + return "10" + } +} diff --git a/util/util.go b/util/util.go index 16e5a5c3..e351f62a 100644 --- a/util/util.go +++ b/util/util.go @@ -17,15 +17,6 @@ import ( "os" "golang.org/x/crypto/hkdf" - corev1 "k8s.io/api/core/v1" -) - -const ( - IntelEpc corev1.ResourceName = "sgx.intel.com/epc" - IntelEnclave corev1.ResourceName = "sgx.intel.com/enclave" - IntelProvision corev1.ResourceName = "sgx.intel.com/provision" - AzureEpc corev1.ResourceName = "kubernetes.azure.com/sgx_epc_mem_in_MiB" - AlibabaEpc corev1.ResourceName = "alibabacloud.com/sgx_epc_MiB" ) // DefaultCertificateIPAddresses defines a placeholder value used for automated x509 certificate generation. @@ -114,20 +105,3 @@ func MustGetwd() string { } panic(err) } - -// GetEPCResorceLimit returns the amount of EPC to set for k8s deployments depending on the used sgx device plugin. -func GetEPCResourceLimit(resourceKey string) string { - switch resourceKey { - case AzureEpc.String(): - // azure device plugin expects epc in MiB - return "10" - case AlibabaEpc.String(): - // alibaba device plugin expects epc in MiB - return "10" - case IntelEpc.String(): - // intels device plugin expects epc as a k8s resource quantity - return "10Mi" - default: - return "10" - } -}