Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

split util package to reduce dependencies of premain #387

Merged
merged 1 commit into from
Apr 12, 2023
Merged

split util package to reduce dependencies of premain #387

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions cli/internal/cmd/install.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ import (

"github.com/edgelesssys/marblerun/cli/internal/helm"
"github.com/edgelesssys/marblerun/cli/internal/kube"
"github.com/edgelesssys/marblerun/util"
"github.com/edgelesssys/marblerun/util/k8sutil"
"github.com/spf13/cobra"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
Expand Down Expand Up @@ -225,18 +225,18 @@ func getSGXResourceKey(ctx context.Context, kubeClient kubernetes.Interface) (st

for _, node := range nodes.Items {
if nodeHasAlibabaDevPlugin(node.Status.Capacity) {
return util.AlibabaEpc.String(), nil
return k8sutil.AlibabaEpc.String(), nil
}
if nodeHasAzureDevPlugin(node.Status.Capacity) {
return util.AzureEpc.String(), nil
return k8sutil.AzureEpc.String(), nil
}
if nodeHasIntelDevPlugin(node.Status.Capacity) {
return util.IntelEpc.String(), nil
return k8sutil.IntelEpc.String(), nil
}
}

// assume cluster has the intel SGX device plugin by default
return util.IntelEpc.String(), nil
return k8sutil.IntelEpc.String(), nil
}

// errorAndCleanup returns the given error and deletes resources which might have been created previously.
Expand Down
10 changes: 5 additions & 5 deletions cli/internal/cmd/install_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ import (
"testing"

"github.com/edgelesssys/marblerun/cli/internal/helm"
"github.com/edgelesssys/marblerun/util"
"github.com/edgelesssys/marblerun/util/k8sutil"
"github.com/spf13/cobra"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
Expand Down Expand Up @@ -151,9 +151,9 @@ func TestGetSGXResourceKey(t *testing.T) {
},
Status: corev1.NodeStatus{
Capacity: corev1.ResourceList{
util.IntelEnclave: resource.MustParse("10"),
util.IntelEpc: resource.MustParse("500"),
util.IntelProvision: resource.MustParse("10"),
k8sutil.IntelEnclave: resource.MustParse("10"),
k8sutil.IntelEpc: resource.MustParse("500"),
k8sutil.IntelProvision: resource.MustParse("10"),
},
},
}
Expand All @@ -162,7 +162,7 @@ func TestGetSGXResourceKey(t *testing.T) {

resourceKey, err := getSGXResourceKey(ctx, testClient)
assert.NoError(err)
assert.Equal(util.IntelEpc.String(), resourceKey)
assert.Equal(k8sutil.IntelEpc.String(), resourceKey)
}

func TestErrorAndCleanup(t *testing.T) {
Expand Down
12 changes: 6 additions & 6 deletions cli/internal/cmd/precheck.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ package cmd

import (
"github.com/edgelesssys/marblerun/cli/internal/kube"
"github.com/edgelesssys/marblerun/util"
"github.com/edgelesssys/marblerun/util/k8sutil"
"github.com/spf13/cobra"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
Expand Down Expand Up @@ -75,20 +75,20 @@ func nodeSupportsSGX(capacityInfo corev1.ResourceList) bool {

// nodeHasAlibabaDevPlugin checks if a node has the Alibaba device plugin installed (https://github.com/AliyunContainerService/sgx-device-plugin).
func nodeHasAlibabaDevPlugin(capacityInfo corev1.ResourceList) bool {
epcQuant := capacityInfo[util.AlibabaEpc]
epcQuant := capacityInfo[k8sutil.AlibabaEpc]
return epcQuant.Value() != 0
}

// nodeHasAzureDevPlugin checks if a node has the Azures SGX device plugin installed (https://github.com/Azure/aks-engine/blob/master/docs/topics/sgx.md#deploying-the-sgx-device-plugin).
func nodeHasAzureDevPlugin(capacityInfo corev1.ResourceList) bool {
epcQuant := capacityInfo[util.AzureEpc]
epcQuant := capacityInfo[k8sutil.AzureEpc]
return epcQuant.Value() != 0
}

// nodeHasIntelDevPlugin checks if a node has the Intel SGX device plugin installed (https://github.com/intel/intel-device-plugins-for-kubernetes#sgx-device-plugin).
func nodeHasIntelDevPlugin(capacityInfo corev1.ResourceList) bool {
epcQuant := capacityInfo[util.IntelEpc]
enclaveQuant := capacityInfo[util.IntelEnclave]
provisionQuant := capacityInfo[util.IntelProvision]
epcQuant := capacityInfo[k8sutil.IntelEpc]
enclaveQuant := capacityInfo[k8sutil.IntelEnclave]
provisionQuant := capacityInfo[k8sutil.IntelProvision]
return !(epcQuant.Value() == 0 || enclaveQuant.Value() == 0 || provisionQuant.Value() == 0)
}
16 changes: 8 additions & 8 deletions cli/internal/cmd/precheck_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import (
"context"
"testing"

"github.com/edgelesssys/marblerun/util"
"github.com/edgelesssys/marblerun/util/k8sutil"
"github.com/spf13/cobra"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
Expand Down Expand Up @@ -53,9 +53,9 @@ func TestNodeSupportsSGX(t *testing.T) {
},
Status: corev1.NodeStatus{
Capacity: corev1.ResourceList{
util.IntelEnclave: resource.MustParse("10"),
util.IntelEpc: resource.MustParse("500"),
util.IntelProvision: resource.MustParse("10"),
k8sutil.IntelEnclave: resource.MustParse("10"),
k8sutil.IntelEpc: resource.MustParse("500"),
k8sutil.IntelProvision: resource.MustParse("10"),
},
},
}
Expand All @@ -78,7 +78,7 @@ func TestNodeSupportsSGX(t *testing.T) {
},
Status: corev1.NodeStatus{
Capacity: corev1.ResourceList{
util.AzureEpc: resource.MustParse("500"),
k8sutil.AzureEpc: resource.MustParse("500"),
},
},
}
Expand Down Expand Up @@ -126,9 +126,9 @@ func TestCliCheckSGXSupport(t *testing.T) {
},
Status: corev1.NodeStatus{
Capacity: corev1.ResourceList{
util.IntelEnclave: resource.MustParse("10"),
util.IntelEpc: resource.MustParse("500"),
util.IntelProvision: resource.MustParse("10"),
k8sutil.IntelEnclave: resource.MustParse("10"),
k8sutil.IntelEpc: resource.MustParse("500"),
k8sutil.IntelProvision: resource.MustParse("10"),
},
},
}
Expand Down
20 changes: 10 additions & 10 deletions cli/internal/helm/client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ import (
"strings"
"time"

"github.com/edgelesssys/marblerun/util"
"github.com/edgelesssys/marblerun/util/k8sutil"
"github.com/gofrs/flock"
"gopkg.in/yaml.v2"
"helm.sh/helm/v3/pkg/action"
Expand Down Expand Up @@ -277,7 +277,7 @@ func setSGXValues(resourceKey string, values, chartValues map[string]interface{}
}

var needNewLimit bool
limit := util.GetEPCResourceLimit(resourceKey)
limit := k8sutil.GetEPCResourceLimit(resourceKey)

// remove all previously set sgx resource limits
if presetLimits, ok := chartValues["coordinator"].(map[string]interface{})["resources"].(map[string]interface{})["limits"].(map[string]interface{}); ok {
Expand Down Expand Up @@ -306,28 +306,28 @@ func setSGXValues(resourceKey string, values, chartValues map[string]interface{}
}

// Make sure provision and enclave bit is set if the Intel plugin is used
if resourceKey == util.IntelEpc.String() {
values["coordinator"].(map[string]interface{})["resources"].(map[string]interface{})["limits"].(map[string]interface{})[util.IntelProvision.String()] = 1
values["coordinator"].(map[string]interface{})["resources"].(map[string]interface{})["limits"].(map[string]interface{})[util.IntelEnclave.String()] = 1
if resourceKey == k8sutil.IntelEpc.String() {
values["coordinator"].(map[string]interface{})["resources"].(map[string]interface{})["limits"].(map[string]interface{})[k8sutil.IntelProvision.String()] = 1
values["coordinator"].(map[string]interface{})["resources"].(map[string]interface{})["limits"].(map[string]interface{})[k8sutil.IntelEnclave.String()] = 1
}
}

// needsDeletion checks if an existing key of a helm chart should be deleted.
// Choice is based on the resource key of the used SGX device plugin.
func needsDeletion(existingKey, sgxKey string) bool {
sgxResources := []string{
util.AlibabaEpc.String(), util.AzureEpc.String(), util.IntelEpc.String(),
util.IntelProvision.String(), util.IntelEnclave.String(),
k8sutil.AlibabaEpc.String(), k8sutil.AzureEpc.String(), k8sutil.IntelEpc.String(),
k8sutil.IntelProvision.String(), k8sutil.IntelEnclave.String(),
}

switch sgxKey {
case util.AlibabaEpc.String(), util.AzureEpc.String():
case k8sutil.AlibabaEpc.String(), k8sutil.AzureEpc.String():
// Delete all non Alibaba/Azure SGX resources depending on the used SGX device plugin
return sgxKey != existingKey && keyInList(existingKey, sgxResources)
case util.IntelEpc.String():
case k8sutil.IntelEpc.String():
// Delete all non Intel SGX resources depending on the used SGX device plugin
// Keep Intel provision and enclave bit
return keyInList(existingKey, []string{util.AlibabaEpc.String(), util.AzureEpc.String()})
return keyInList(existingKey, []string{k8sutil.AlibabaEpc.String(), k8sutil.AzureEpc.String()})
default:
// Either no SGX plugin or a custom SGX plugin is used
// Delete all known SGX resources
Expand Down
44 changes: 22 additions & 22 deletions cli/internal/helm/client_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ package helm
import (
"testing"

"github.com/edgelesssys/marblerun/util"
"github.com/edgelesssys/marblerun/util/k8sutil"
"github.com/stretchr/testify/assert"
)

Expand All @@ -20,62 +20,62 @@ func TestNeedsDeletion(t *testing.T) {
wantDeletion bool
}{
"intel key with azure plugin": {
existingKey: util.IntelEpc.String(),
sgxKey: util.AzureEpc.String(),
existingKey: k8sutil.IntelEpc.String(),
sgxKey: k8sutil.AzureEpc.String(),
wantDeletion: true,
},
"intel key with alibaba plugin": {
existingKey: util.IntelEpc.String(),
sgxKey: util.AlibabaEpc.String(),
existingKey: k8sutil.IntelEpc.String(),
sgxKey: k8sutil.AlibabaEpc.String(),
wantDeletion: true,
},
"azure key with intel plugin": {
existingKey: util.AzureEpc.String(),
sgxKey: util.IntelEpc.String(),
existingKey: k8sutil.AzureEpc.String(),
sgxKey: k8sutil.IntelEpc.String(),
wantDeletion: true,
},
"azure key with alibaba plugin": {
existingKey: util.AzureEpc.String(),
sgxKey: util.AlibabaEpc.String(),
existingKey: k8sutil.AzureEpc.String(),
sgxKey: k8sutil.AlibabaEpc.String(),
wantDeletion: true,
},
"alibaba key with intel plugin": {
existingKey: util.AlibabaEpc.String(),
sgxKey: util.IntelEpc.String(),
existingKey: k8sutil.AlibabaEpc.String(),
sgxKey: k8sutil.IntelEpc.String(),
wantDeletion: true,
},
"alibaba key with azure plugin": {
existingKey: util.AlibabaEpc.String(),
sgxKey: util.AzureEpc.String(),
existingKey: k8sutil.AlibabaEpc.String(),
sgxKey: k8sutil.AzureEpc.String(),
wantDeletion: true,
},
"same key": {
existingKey: util.IntelEpc.String(),
sgxKey: util.IntelEpc.String(),
existingKey: k8sutil.IntelEpc.String(),
sgxKey: k8sutil.IntelEpc.String(),
wantDeletion: false,
},
"intel provision with intel plugin": {
existingKey: util.IntelProvision.String(),
sgxKey: util.IntelEpc.String(),
existingKey: k8sutil.IntelProvision.String(),
sgxKey: k8sutil.IntelEpc.String(),
wantDeletion: false,
},
"intel enclave with intel plugin": {
existingKey: util.IntelEnclave.String(),
sgxKey: util.IntelEpc.String(),
existingKey: k8sutil.IntelEnclave.String(),
sgxKey: k8sutil.IntelEpc.String(),
wantDeletion: false,
},
"regular resource with intel plugin": {
existingKey: "cpu",
sgxKey: util.IntelEpc.String(),
sgxKey: k8sutil.IntelEpc.String(),
wantDeletion: false,
},
"custom resource with intel plugin": {
existingKey: "custom-sgx-resource",
sgxKey: util.IntelEpc.String(),
sgxKey: k8sutil.IntelEpc.String(),
wantDeletion: false,
},
"intel provision with custom plugin": {
existingKey: util.IntelProvision.String(),
existingKey: k8sutil.IntelProvision.String(),
sgxKey: "custom-sgx-resource",
wantDeletion: true,
},
Expand Down
12 changes: 6 additions & 6 deletions injector/injector.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ import (
"net/http"
"strings"

"github.com/edgelesssys/marblerun/util"
"github.com/edgelesssys/marblerun/util/k8sutil"
v1 "k8s.io/api/admission/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
Expand Down Expand Up @@ -184,18 +184,18 @@ func mutate(body []byte, coordAddr, domainName, resourceKey string) ([]byte, err
container.Resources.Limits = make(map[corev1.ResourceName]resource.Quantity)
}
switch resourceKey {
case util.IntelEpc.String():
case k8sutil.IntelEpc.String():
// Intels device plugin offers 3 resources:
// epc : sets EPC for the container
// enclave : provides a handle to /dev/sgx_enclave
// provision : provides a handle to /dev/sgx_provision, this is not needed when the Marble utilises out-of-process quote-generation
setResourceLimit(container.Resources.Limits, util.IntelEpc, util.GetEPCResourceLimit(resourceKey))
setResourceLimit(container.Resources.Limits, util.IntelEnclave, "1")
setResourceLimit(container.Resources.Limits, util.IntelProvision, "1")
setResourceLimit(container.Resources.Limits, k8sutil.IntelEpc, k8sutil.GetEPCResourceLimit(resourceKey))
setResourceLimit(container.Resources.Limits, k8sutil.IntelEnclave, "1")
setResourceLimit(container.Resources.Limits, k8sutil.IntelProvision, "1")
default:
// Azure and Alibaba Cloud plugins offer only 1 resource
// for custom plugins we can only inject the resource provided by the `resourceKey`
setResourceLimit(container.Resources.Limits, corev1.ResourceName(resourceKey), util.GetEPCResourceLimit(resourceKey))
setResourceLimit(container.Resources.Limits, corev1.ResourceName(resourceKey), k8sutil.GetEPCResourceLimit(resourceKey))
}
}

Expand Down
34 changes: 34 additions & 0 deletions util/k8sutil/k8sutil.go
View file
Open in desktop
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Missing license header

Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
// Copyright (c) Edgeless Systems GmbH.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at https://mozilla.org/MPL/2.0/.

package k8sutil

import corev1 "k8s.io/api/core/v1"

const (
IntelEpc corev1.ResourceName = "sgx.intel.com/epc"
IntelEnclave corev1.ResourceName = "sgx.intel.com/enclave"
IntelProvision corev1.ResourceName = "sgx.intel.com/provision"
AzureEpc corev1.ResourceName = "kubernetes.azure.com/sgx_epc_mem_in_MiB"
AlibabaEpc corev1.ResourceName = "alibabacloud.com/sgx_epc_MiB"
)

// GetEPCResourceLimit returns the amount of EPC to set for k8s deployments depending on the used sgx device plugin.
func GetEPCResourceLimit(resourceKey string) string {
switch resourceKey {
case AzureEpc.String():
// azure device plugin expects epc in MiB
return "10"
case AlibabaEpc.String():
// alibaba device plugin expects epc in MiB
return "10"
case IntelEpc.String():
// intels device plugin expects epc as a k8s resource quantity
return "10Mi"
default:
return "10"
}
}
26 changes: 0 additions & 26 deletions util/util.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,6 @@ import (
"os"

"golang.org/x/crypto/hkdf"
corev1 "k8s.io/api/core/v1"
)

const (
IntelEpc corev1.ResourceName = "sgx.intel.com/epc"
IntelEnclave corev1.ResourceName = "sgx.intel.com/enclave"
IntelProvision corev1.ResourceName = "sgx.intel.com/provision"
AzureEpc corev1.ResourceName = "kubernetes.azure.com/sgx_epc_mem_in_MiB"
AlibabaEpc corev1.ResourceName = "alibabacloud.com/sgx_epc_MiB"
)

// DefaultCertificateIPAddresses defines a placeholder value used for automated x509 certificate generation.
Expand Down Expand Up @@ -114,20 +105,3 @@ func MustGetwd() string {
}
panic(err)
}

// GetEPCResorceLimit returns the amount of EPC to set for k8s deployments depending on the used sgx device plugin.
func GetEPCResourceLimit(resourceKey string) string {
switch resourceKey {
case AzureEpc.String():
// azure device plugin expects epc in MiB
return "10"
case AlibabaEpc.String():
// alibaba device plugin expects epc in MiB
return "10"
case IntelEpc.String():
// intels device plugin expects epc as a k8s resource quantity
return "10Mi"
default:
return "10"
}
}