From ad7ca9791bc8e80fb3d3734ebcdd7cc919d94548 Mon Sep 17 00:00:00 2001 From: Moritz Sanft <58110325+msanft@users.noreply.github.com> Date: Fri, 19 Jul 2024 17:27:45 +0200 Subject: [PATCH 1/3] cli: embed multiple reference values This adds support for embedding a more versatile format of reference values (i.e. a structured type) into the Contrast binaries. This will allow us to embed all reference values at build-time from a single source (the Nix build file) rather than having SVNs in Go code and inserting trusted measurements via the go build commandline. It will now embed a JSON file containing the reference values, which is unmarshaled at first default manifest generation. --- cli/cmd/common.go | 5 - cli/cmd/generate.go | 115 ++++++------ cli/cmd/verify.go | 2 +- cli/main.go | 11 +- coordinator/internal/authority/authority.go | 2 +- .../internal/authority/authority_test.go | 6 +- .../internal/authority/userapi_test.go | 29 +-- e2e/genpolicy/genpolicy_test.go | 10 +- e2e/getdents/getdents_test.go | 11 +- e2e/openssl/openssl_test.go | 11 +- e2e/policy/policy_test.go | 12 +- e2e/servicemesh/servicemesh_test.go | 11 ++ internal/kuberesource/constants.go | 3 - internal/kuberesource/mutators.go | 12 ++ internal/kuberesource/mutators_test.go | 8 +- internal/kuberesource/parts.go | 16 +- internal/kuberesource/parts_test.go | 4 +- internal/kuberesource/resourcegen/main.go | 16 +- internal/kuberesource/sets.go | 27 ++- .../manifest/assets/reference-values.json | 1 + internal/manifest/constants.go | 70 +++++--- internal/manifest/manifest.go | 138 ++++++-------- internal/manifest/manifest_test.go | 168 +++++------------- internal/manifest/referencevalues.go | 96 ++++++++++ internal/manifest/referencevalues_test.go | 115 ++++++++++++ packages/by-name/contrast/package.nix | 37 ++-- packages/by-name/kata/kata-image/package.nix | 15 ++ .../microsoft/runtime-class-files/package.nix | 3 +- 28 files changed, 587 insertions(+), 367 deletions(-) create mode 100755 internal/manifest/assets/reference-values.json create mode 100644 internal/manifest/referencevalues.go create mode 100644 internal/manifest/referencevalues_test.go diff --git a/cli/cmd/common.go b/cli/cmd/common.go index 0b862eecf2..317f0868cd 100644 --- a/cli/cmd/common.go +++ b/cli/cmd/common.go @@ -6,7 +6,6 @@ package cmd import ( "context" _ "embed" - "fmt" "os" "path/filepath" "time" @@ -46,10 +45,6 @@ var ( DefaultCoordinatorPolicyHash = "" ) -func runtimeHandler(digest string) string { - return fmt.Sprintf("contrast-cc-%s", digest[:32]) -} - func cachedir(subdir string) (string, error) { dir := os.Getenv(cacheDirEnv) if dir == "" { diff --git a/cli/cmd/generate.go b/cli/cmd/generate.go index e9184b7f5b..52e8ae816e 100644 --- a/cli/cmd/generate.go +++ b/cli/cmd/generate.go @@ -88,7 +88,7 @@ subcommands.`, func runGenerate(cmd *cobra.Command, args []string) error { flags, err := parseGenerateFlags(cmd) if err != nil { - return fmt.Errorf("failed to parse flags: %w", err) + return fmt.Errorf("parse flags: %w", err) } log, err := newCLILogger(cmd) @@ -101,23 +101,48 @@ func runGenerate(cmd *cobra.Command, args []string) error { return err } - if err := patchTargets(paths, flags.imageReplacementsFile, flags.skipInitializer, log); err != nil { - return fmt.Errorf("failed to patch targets: %w", err) + // generate a manifest by checking if a manifest exists and using that, + // or otherwise using a default. + var mnf *manifest.Manifest + existingManifest, err := os.ReadFile(flags.manifestPath) + if errors.Is(err, fs.ErrNotExist) { + // Manifest does not exist, create a new one + mnf, err = manifest.Default(flags.referenceValuesPlatform) + if err != nil { + return fmt.Errorf("create default manifest: %w", err) + } + } else if err != nil { + // Manifest exists but could not be read, return error + return fmt.Errorf("read existing manifest: %w", err) + } else { + // Manifest exists and was read successfully, unmarshal it + if err := json.Unmarshal(existingManifest, &mnf); err != nil { + return fmt.Errorf("unmarshal existing manifest: %w", err) + } + } + + runtimeHandler, err := mnf.RuntimeHandler(flags.referenceValuesPlatform) + if err != nil { + return fmt.Errorf("get runtime handler: %w", err) + } + + if err := patchTargets(paths, flags.imageReplacementsFile, runtimeHandler, flags.skipInitializer, log); err != nil { + return fmt.Errorf("patch targets: %w", err) } fmt.Fprintln(cmd.OutOrStdout(), "✔️ Patched targets") if err := generatePolicies(cmd.Context(), flags.policyPath, flags.settingsPath, flags.genpolicyCachePath, paths, log); err != nil { - return fmt.Errorf("failed to generate policies: %w", err) + return fmt.Errorf("generate policies: %w", err) } fmt.Fprintln(cmd.OutOrStdout(), "✔️ Generated workload policy annotations") policies, err := policiesFromKubeResources(paths) if err != nil { - return fmt.Errorf("failed to find kube resources with policy: %w", err) + return fmt.Errorf("find kube resources with policy: %w", err) } policyMap, err := manifestPolicyMapFromPolicies(policies) if err != nil { - return fmt.Errorf("failed to create policy map: %w", err) + return fmt.Errorf("create policy map: %w", err) } if err := generateWorkloadOwnerKey(flags); err != nil { @@ -127,53 +152,35 @@ func runGenerate(cmd *cobra.Command, args []string) error { return fmt.Errorf("generating seedshare owner key: %w", err) } - defaultManifest := manifest.Default() - switch flags.referenceValuesPlatform { - case platforms.AKSCloudHypervisorSNP: - defaultManifest = manifest.DefaultAKS() - } - - defaultManifestData, err := json.MarshalIndent(&defaultManifest, "", " ") - if err != nil { - return fmt.Errorf("marshaling default manifest: %w", err) - } - manifestData, err := readFileOrDefault(flags.manifestPath, defaultManifestData) - if err != nil { - return fmt.Errorf("failed to read manifest file: %w", err) - } - var manifest *manifest.Manifest - if err := json.Unmarshal(manifestData, &manifest); err != nil { - return fmt.Errorf("failed to unmarshal manifest: %w", err) - } - manifest.Policies = policyMap - if err := manifest.Validate(); err != nil { + mnf.Policies = policyMap + if err := mnf.Validate(); err != nil { return fmt.Errorf("validating manifest: %w", err) } if flags.disableUpdates { - manifest.WorkloadOwnerKeyDigests = nil + mnf.WorkloadOwnerKeyDigests = nil } else { for _, keyPath := range flags.workloadOwnerKeys { - if err := addWorkloadOwnerKeyToManifest(manifest, keyPath); err != nil { + if err := addWorkloadOwnerKeyToManifest(mnf, keyPath); err != nil { return fmt.Errorf("adding workload owner key to manifest: %w", err) } } } - slices.Sort(manifest.WorkloadOwnerKeyDigests) + slices.Sort(mnf.WorkloadOwnerKeyDigests) for _, keyPath := range flags.seedshareOwnerKeys { - if err := addSeedshareOwnerKeyToManifest(manifest, keyPath); err != nil { + if err := addSeedshareOwnerKeyToManifest(mnf, keyPath); err != nil { return fmt.Errorf("adding seedshare owner key to manifest: %w", err) } } - slices.Sort(manifest.SeedshareOwnerPubKeys) + slices.Sort(mnf.SeedshareOwnerPubKeys) - manifestData, err = json.MarshalIndent(manifest, "", " ") + manifestData, err := json.MarshalIndent(mnf, "", " ") if err != nil { - return fmt.Errorf("failed to marshal manifest: %w", err) + return fmt.Errorf("marshal manifest: %w", err) } if err := os.WriteFile(flags.manifestPath, append(manifestData, '\n'), 0o644); err != nil { - return fmt.Errorf("failed to write manifest: %w", err) + return fmt.Errorf("write manifest: %w", err) } fmt.Fprintf(cmd.OutOrStdout(), "✔️ Updated manifest %s\n", flags.manifestPath) @@ -181,7 +188,7 @@ func runGenerate(cmd *cobra.Command, args []string) error { if hash := getCoordinatorPolicyHash(policies, log); hash != "" { coordHashPath := filepath.Join(flags.workspaceDir, coordHashFilename) if err := os.WriteFile(coordHashPath, []byte(hash), 0o644); err != nil { - return fmt.Errorf("failed to write coordinator policy hash: %w", err) + return fmt.Errorf("write coordinator policy hash: %w", err) } } @@ -207,7 +214,7 @@ func findGenerateTargets(args []string, logger *slog.Logger) ([]string, error) { return nil }) if err != nil { - return nil, fmt.Errorf("failed to walk %s: %w", path, err) + return nil, fmt.Errorf("walk %s: %w", path, err) } } if len(paths) == 0 { @@ -227,7 +234,7 @@ func filterNonCoCoRuntime(runtimeClassNamePrefix string, paths []string, logger for _, path := range paths { data, err := os.ReadFile(path) if err != nil { - logger.Warn("Failed to read file", "path", path, "err", err) + logger.Warn("read file", "path", path, "err", err) continue } if !bytes.Contains(data, []byte(runtimeClassNamePrefix)) { @@ -248,21 +255,21 @@ func generatePolicies(ctx context.Context, regoRulesPath, policySettingsPath, ge } binaryInstallDir, err := installDir() if err != nil { - return fmt.Errorf("failed to get install dir: %w", err) + return fmt.Errorf("get install dir: %w", err) } genpolicyInstall, err := embedbin.New().Install(binaryInstallDir, genpolicyBin) if err != nil { - return fmt.Errorf("failed to install genpolicy: %w", err) + return fmt.Errorf("install genpolicy: %w", err) } defer func() { if err := genpolicyInstall.Uninstall(); err != nil { - logger.Warn("Failed to uninstall genpolicy tool", "err", err) + logger.Warn("uninstall genpolicy tool", "err", err) } }() for _, yamlPath := range yamlPaths { policyHash, err := generatePolicyForFile(ctx, genpolicyInstall.Path(), regoRulesPath, policySettingsPath, yamlPath, genpolicyCachePath, logger) if err != nil { - return fmt.Errorf("failed to generate policy for %s: %w", yamlPath, err) + return fmt.Errorf("generate policy for %s: %w", yamlPath, err) } if policyHash == [32]byte{} { continue @@ -273,7 +280,7 @@ func generatePolicies(ctx context.Context, regoRulesPath, policySettingsPath, ge return nil } -func patchTargets(paths []string, imageReplacementsFile string, skipInitializer bool, logger *slog.Logger) error { +func patchTargets(paths []string, imageReplacementsFile, runtimeHandler string, skipInitializer bool, logger *slog.Logger) error { var replacements map[string]string var err error if imageReplacementsFile != "" { @@ -296,11 +303,11 @@ func patchTargets(paths []string, imageReplacementsFile string, skipInitializer for _, path := range paths { data, err := os.ReadFile(path) if err != nil { - return fmt.Errorf("failed to read %s: %w", path, err) + return fmt.Errorf("read %s: %w", path, err) } kubeObjs, err := kuberesource.UnmarshalApplyConfigurations(data) if err != nil { - return fmt.Errorf("failed to unmarshal %s: %w", path, err) + return fmt.Errorf("unmarshal %s: %w", path, err) } if !skipInitializer { @@ -314,7 +321,7 @@ func patchTargets(paths []string, imageReplacementsFile string, skipInitializer kubeObjs = kuberesource.PatchImages(kubeObjs, replacements) - replaceRuntimeClassName := runtimeClassNamePatcher() + replaceRuntimeClassName := runtimeClassNamePatcher(runtimeHandler) for i := range kubeObjs { kubeObjs[i] = kuberesource.MapPodSpec(kubeObjs[i], replaceRuntimeClassName) } @@ -325,7 +332,7 @@ func patchTargets(paths []string, imageReplacementsFile string, skipInitializer return err } if err := os.WriteFile(path, resource, os.ModePerm); err != nil { - return fmt.Errorf("failed to write %s: %w", path, err) + return fmt.Errorf("write %s: %w", path, err) } } return nil @@ -362,8 +369,7 @@ func injectServiceMesh(resources []any) error { return nil } -func runtimeClassNamePatcher() func(*applycorev1.PodSpecApplyConfiguration) *applycorev1.PodSpecApplyConfiguration { - handler := runtimeHandler(manifest.TrustedMeasurement) +func runtimeClassNamePatcher(handler string) func(*applycorev1.PodSpecApplyConfiguration) *applycorev1.PodSpecApplyConfiguration { return func(spec *applycorev1.PodSpecApplyConfiguration) *applycorev1.PodSpecApplyConfiguration { if spec.RuntimeClassName == nil || *spec.RuntimeClassName == handler { return spec @@ -633,19 +639,6 @@ func parseGenerateFlags(cmd *cobra.Command) (*generateFlags, error) { }, nil } -// readFileOrDefault reads the file at path, -// or returns the default value if the file doesn't exist. -func readFileOrDefault(path string, deflt []byte) ([]byte, error) { - data, err := os.ReadFile(path) - if err == nil { - return data, nil - } - if !os.IsNotExist(err) { - return nil, err - } - return deflt, nil -} - // createFileWithDefault creates the file at path with the default value, // if it doesn't exist. func createFileWithDefault(path string, perm fs.FileMode, dflt func() ([]byte, error)) error { diff --git a/cli/cmd/verify.go b/cli/cmd/verify.go index c8ee86ae84..adb06568c5 100644 --- a/cli/cmd/verify.go +++ b/cli/cmd/verify.go @@ -175,7 +175,7 @@ func parseVerifyFlags(cmd *cobra.Command) (*verifyFlags, error) { } func newCoordinatorValidateOptsGen(mnfst manifest.Manifest, hostData []byte) (*snp.StaticValidateOptsGenerator, error) { - validateOpts, err := mnfst.SNPValidateOpts() + validateOpts, err := mnfst.AKSValidateOpts() if err != nil { return nil, err } diff --git a/cli/main.go b/cli/main.go index 3b6345b9a5..0c38efa517 100644 --- a/cli/main.go +++ b/cli/main.go @@ -5,6 +5,7 @@ package main import ( "context" + "encoding/json" "fmt" "os" "os/signal" @@ -14,7 +15,6 @@ import ( "github.com/edgelesssys/contrast/cli/cmd" "github.com/edgelesssys/contrast/cli/constants" "github.com/edgelesssys/contrast/internal/manifest" - "github.com/edgelesssys/contrast/node-installer/platforms" "github.com/spf13/cobra" ) @@ -43,11 +43,10 @@ func buildVersionString() string { fmt.Fprintf(versionsWriter, "\t%s\n", image) } } - fmt.Fprint(versionsWriter, "\n") - fmt.Fprintf(versionsWriter, "reference values for %s platform:\n", platforms.AKSCloudHypervisorSNP.String()) - fmt.Fprintf(versionsWriter, "\truntime handler:\tcontrast-cc-%s\n", manifest.TrustedMeasurement[:32]) - fmt.Fprintf(versionsWriter, "\tlaunch digest:\t%s\n", manifest.TrustedMeasurement) - fmt.Fprintf(versionsWriter, "\tgenpolicy version:\t%s\n", constants.GenpolicyVersion) + if refValues, err := json.MarshalIndent(manifest.EmbeddedReferenceValues(), "\t", " "); err == nil { + fmt.Fprintf(versionsWriter, "embedded reference values:\t%s\n", refValues) + } + fmt.Fprintf(versionsWriter, "genpolicy version:\t%s\n", constants.GenpolicyVersion) versionsWriter.Flush() return versionsBuilder.String() } diff --git a/coordinator/internal/authority/authority.go b/coordinator/internal/authority/authority.go index 32a09e7b1b..d6435f9130 100644 --- a/coordinator/internal/authority/authority.go +++ b/coordinator/internal/authority/authority.go @@ -94,7 +94,7 @@ func (m *Authority) SNPValidateOpts(report *sevsnp.Report) (*validate.Options, e return nil, fmt.Errorf("hostdata %s not found in manifest", hostData) } - return mnfst.SNPValidateOpts() + return mnfst.AKSValidateOpts() } // ValidateCallback creates a certificate bundle for the verified client. diff --git a/coordinator/internal/authority/authority_test.go b/coordinator/internal/authority/authority_test.go index 564431252d..3cb2ca6e83 100644 --- a/coordinator/internal/authority/authority_test.go +++ b/coordinator/internal/authority/authority_test.go @@ -15,6 +15,7 @@ import ( "github.com/edgelesssys/contrast/coordinator/history" "github.com/edgelesssys/contrast/internal/manifest" "github.com/edgelesssys/contrast/internal/userapi" + "github.com/edgelesssys/contrast/node-installer/platforms" "github.com/google/go-sev-guest/proto/sevsnp" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/testutil" @@ -79,12 +80,13 @@ func newManifest(t *testing.T) (*manifest.Manifest, []byte, [][]byte) { policyHash := sha256.Sum256(policy) policyHashHex := manifest.NewHexString(policyHash[:]) - mnfst := manifest.DefaultAKS() + mnfst, err := manifest.Default(platforms.AKSCloudHypervisorSNP) + require.NoError(t, err) mnfst.Policies = map[manifest.HexString][]string{policyHashHex: {"test"}} mnfst.WorkloadOwnerKeyDigests = []manifest.HexString{keyDigest} mnfstBytes, err := json.Marshal(mnfst) require.NoError(t, err) - return &mnfst, mnfstBytes, [][]byte{policy} + return mnfst, mnfstBytes, [][]byte{policy} } func requireGauge(t *testing.T, reg *prometheus.Registry, val int) { diff --git a/coordinator/internal/authority/userapi_test.go b/coordinator/internal/authority/userapi_test.go index caa7ce7130..07865f7ee1 100644 --- a/coordinator/internal/authority/userapi_test.go +++ b/coordinator/internal/authority/userapi_test.go @@ -21,6 +21,7 @@ import ( "github.com/edgelesssys/contrast/coordinator/history" "github.com/edgelesssys/contrast/internal/manifest" "github.com/edgelesssys/contrast/internal/userapi" + "github.com/edgelesssys/contrast/node-installer/platforms" "github.com/prometheus/client_golang/prometheus" "github.com/spf13/afero" "github.com/stretchr/testify/assert" @@ -32,13 +33,15 @@ import ( ) func TestManifestSet(t *testing.T) { - newBaseManifest := func() manifest.Manifest { - return manifest.Default() + newBaseManifest := func() *manifest.Manifest { + mnf, err := manifest.Default(platforms.AKSCloudHypervisorSNP) + require.NoError(t, err) + return mnf } newManifestBytes := func(f func(*manifest.Manifest)) []byte { m := newBaseManifest() if f != nil { - f(&m) + f(m) } b, err := json.Marshal(m) require.NoError(t, err) @@ -221,7 +224,8 @@ func TestGetManifests(t *testing.T) { require.Equal(codes.FailedPrecondition, status.Code(err)) assert.Nil(resp) - m := manifest.Default() + m, err := manifest.Default(platforms.AKSCloudHypervisorSNP) + require.NoError(err) m.Policies = map[manifest.HexString][]string{ manifest.HexString("ca978112ca1bbdcafac231b39a23dc4da786eff8147c4e72b9807785afee48bb"): {"a1", "a2"}, manifest.HexString("3e23e8160039594a33894f6564e1b1348bbd7a0088d42c4acb73eeaed59c009d"): {"b1", "b2"}, @@ -373,13 +377,15 @@ func TestRecoveryFlow(t *testing.T) { // TestUserAPIConcurrent tests potential synchronization problems between the different // gRPCs of the server. func TestUserAPIConcurrent(t *testing.T) { - newBaseManifest := func() manifest.Manifest { - return manifest.Default() + newBaseManifest := func() *manifest.Manifest { + mnf, err := manifest.Default(platforms.AKSCloudHypervisorSNP) + require.NoError(t, err) + return mnf } newManifestBytes := func(f func(*manifest.Manifest)) []byte { m := newBaseManifest() if f != nil { - f(&m) + f(m) } b, err := json.Marshal(m) require.NoError(t, err) @@ -459,9 +465,12 @@ func rpcContext(key *ecdsa.PrivateKey) context.Context { } func manifestWithWorkloadOwnerKey(key *ecdsa.PrivateKey) (*manifest.Manifest, error) { - m := manifest.Default() + m, err := manifest.Default(platforms.AKSCloudHypervisorSNP) + if err != nil { + return nil, err + } if key == nil { - return &m, nil + return m, nil } pubKey, err := x509.MarshalPKIXPublicKey(&key.PublicKey) if err != nil { @@ -470,7 +479,7 @@ func manifestWithWorkloadOwnerKey(key *ecdsa.PrivateKey) (*manifest.Manifest, er ownerKeyHash := sha256.Sum256(pubKey) ownerKeyHex := manifest.NewHexString(ownerKeyHash[:]) m.WorkloadOwnerKeyDigests = []manifest.HexString{ownerKeyHex} - return &m, nil + return m, nil } func parsePEMCertificate(t *testing.T, pemCert []byte) *x509.Certificate { diff --git a/e2e/genpolicy/genpolicy_test.go b/e2e/genpolicy/genpolicy_test.go index 0d54cb93e8..72b9bec506 100644 --- a/e2e/genpolicy/genpolicy_test.go +++ b/e2e/genpolicy/genpolicy_test.go @@ -19,6 +19,8 @@ import ( "github.com/edgelesssys/contrast/e2e/internal/contrasttest" "github.com/edgelesssys/contrast/e2e/internal/kubeclient" "github.com/edgelesssys/contrast/internal/kuberesource" + "github.com/edgelesssys/contrast/internal/manifest" + "github.com/edgelesssys/contrast/node-installer/platforms" "github.com/stretchr/testify/require" ) @@ -29,13 +31,19 @@ var ( // TestGenpolicy runs regression tests for generated policies. func TestGenpolicy(t *testing.T) { + // TODO(msanft): Make this configurable + platform := platforms.AKSCloudHypervisorSNP + testCases := kuberesource.GenpolicyRegressionTests() + runtimeHandler, err := manifest.DefaultPlatformHandler(platform) + require.NoError(t, err) + for name, deploy := range testCases { t.Run(name, func(t *testing.T) { ct := contrasttest.New(t, imageReplacementsFile, namespaceFile, skipUndeploy) - ct.Init(t, []any{deploy}) + ct.Init(t, kuberesource.PatchRuntimeHandlers([]any{deploy}, runtimeHandler)) require.True(t, t.Run("generate", func(t *testing.T) { require := require.New(t) diff --git a/e2e/getdents/getdents_test.go b/e2e/getdents/getdents_test.go index 1a8bff1969..a4724eaa9b 100644 --- a/e2e/getdents/getdents_test.go +++ b/e2e/getdents/getdents_test.go @@ -19,6 +19,8 @@ import ( "github.com/edgelesssys/contrast/e2e/internal/contrasttest" "github.com/edgelesssys/contrast/e2e/internal/kubeclient" "github.com/edgelesssys/contrast/internal/kuberesource" + "github.com/edgelesssys/contrast/internal/manifest" + "github.com/edgelesssys/contrast/node-installer/platforms" "github.com/stretchr/testify/require" ) @@ -34,9 +36,16 @@ var ( func TestGetDEnts(t *testing.T) { ct := contrasttest.New(t, imageReplacementsFile, namespaceFile, skipUndeploy) - resources, err := kuberesource.GetDEnts() + // TODO(msanft): Make this configurable + platform := platforms.AKSCloudHypervisorSNP + + runtimeHandler, err := manifest.DefaultPlatformHandler(platform) require.NoError(t, err) + resources := kuberesource.GetDEnts() + + resources = kuberesource.PatchRuntimeHandlers(resources, runtimeHandler) + ct.Init(t, resources) // Call generate to patch the runtime class. diff --git a/e2e/openssl/openssl_test.go b/e2e/openssl/openssl_test.go index 2060e1563a..32381b5b37 100644 --- a/e2e/openssl/openssl_test.go +++ b/e2e/openssl/openssl_test.go @@ -21,6 +21,7 @@ import ( "github.com/edgelesssys/contrast/e2e/internal/kubeclient" "github.com/edgelesssys/contrast/internal/kuberesource" "github.com/edgelesssys/contrast/internal/manifest" + "github.com/edgelesssys/contrast/node-installer/platforms" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -43,11 +44,19 @@ var ( func TestOpenSSL(t *testing.T) { ct := contrasttest.New(t, imageReplacementsFile, namespaceFile, skipUndeploy) - resources := kuberesource.OpenSSL() + // TODO(msanft): Make this configurable + platform := platforms.AKSCloudHypervisorSNP + + runtimeHandler, err := manifest.DefaultPlatformHandler(platform) + require.NoError(t, err) + resources := kuberesource.OpenSSL() coordinator := kuberesource.CoordinatorBundle() + resources = append(resources, coordinator...) + resources = kuberesource.PatchRuntimeHandlers(resources, runtimeHandler) + resources = kuberesource.AddPortForwarders(resources) ct.Init(t, resources) diff --git a/e2e/policy/policy_test.go b/e2e/policy/policy_test.go index 0fd02696ad..028fc1a833 100644 --- a/e2e/policy/policy_test.go +++ b/e2e/policy/policy_test.go @@ -21,6 +21,7 @@ import ( "github.com/edgelesssys/contrast/internal/kubeapi" "github.com/edgelesssys/contrast/internal/kuberesource" "github.com/edgelesssys/contrast/internal/manifest" + "github.com/edgelesssys/contrast/node-installer/platforms" "github.com/prometheus/common/expfmt" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -40,10 +41,19 @@ var ( func TestPolicy(t *testing.T) { ct := contrasttest.New(t, imageReplacementsFile, namespaceFile, skipUndeploy) - resources := kuberesource.OpenSSL() + // TODO(msanft): Make this configurable + platform := platforms.AKSCloudHypervisorSNP + + runtimeHandler, err := manifest.DefaultPlatformHandler(platform) + require.NoError(t, err) + resources := kuberesource.OpenSSL() coordinatorBundle := kuberesource.CoordinatorBundle() + resources = append(resources, coordinatorBundle...) + + resources = kuberesource.PatchRuntimeHandlers(resources, runtimeHandler) + resources = kuberesource.AddPortForwarders(resources) ct.Init(t, resources) diff --git a/e2e/servicemesh/servicemesh_test.go b/e2e/servicemesh/servicemesh_test.go index 555b9cfdce..af992a8db4 100644 --- a/e2e/servicemesh/servicemesh_test.go +++ b/e2e/servicemesh/servicemesh_test.go @@ -20,6 +20,8 @@ import ( "github.com/edgelesssys/contrast/e2e/internal/contrasttest" "github.com/edgelesssys/contrast/e2e/internal/kubeclient" "github.com/edgelesssys/contrast/internal/kuberesource" + "github.com/edgelesssys/contrast/internal/manifest" + "github.com/edgelesssys/contrast/node-installer/platforms" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -33,11 +35,20 @@ var ( func TestIngressEgress(t *testing.T) { ct := contrasttest.New(t, imageReplacementsFile, namespaceFile, skipUndeploy) + // TODO(msanft): Make this configurable + platform := platforms.AKSCloudHypervisorSNP + + runtimeHandler, err := manifest.DefaultPlatformHandler(platform) + require.NoError(t, err) + resources := kuberesource.Emojivoto(kuberesource.ServiceMeshIngressEgress) coordinator := kuberesource.CoordinatorBundle() + resources = append(resources, coordinator...) + resources = kuberesource.PatchRuntimeHandlers(resources, runtimeHandler) + resources = kuberesource.AddPortForwarders(resources) ct.Init(t, resources) diff --git a/internal/kuberesource/constants.go b/internal/kuberesource/constants.go index f1813a2fcd..a5c6d49b0a 100644 --- a/internal/kuberesource/constants.go +++ b/internal/kuberesource/constants.go @@ -2,6 +2,3 @@ // SPDX-License-Identifier: AGPL-3.0-only package kuberesource - -// This value is injected at build time. -var runtimeHandler = "contrast-cc" diff --git a/internal/kuberesource/mutators.go b/internal/kuberesource/mutators.go index 9ef10f34a4..43a9da52a1 100644 --- a/internal/kuberesource/mutators.go +++ b/internal/kuberesource/mutators.go @@ -236,6 +236,18 @@ func PatchImages(resources []any, replacements map[string]string) []any { return out } +// PatchRuntimeHandlers replaces runtime handlers in a set of resources. +func PatchRuntimeHandlers(resources []any, runtimeHandler string) []any { + var out []any + for _, resource := range resources { + out = append(out, MapPodSpec(resource, func(spec *applycorev1.PodSpecApplyConfiguration) *applycorev1.PodSpecApplyConfiguration { + spec.RuntimeClassName = &runtimeHandler + return spec + })) + } + return out +} + // PatchNamespaces replaces namespaces in a set of resources. func PatchNamespaces(resources []any, namespace string) []any { var nsPtr *string diff --git a/internal/kuberesource/mutators_test.go b/internal/kuberesource/mutators_test.go index 8ad74ecafd..4c010b17c6 100644 --- a/internal/kuberesource/mutators_test.go +++ b/internal/kuberesource/mutators_test.go @@ -12,6 +12,8 @@ import ( ) func TestPatchNamespaces(t *testing.T) { + require := require.New(t) + coordinator := CoordinatorBundle() openssl := OpenSSL() emojivoto := Emojivoto(ServiceMeshIngressEgress) @@ -33,8 +35,7 @@ func TestPatchNamespaces(t *testing.T) { set: emojivoto, }, } { - t.Run(tc.name, func(t *testing.T) { - require := require.New(t) + t.Run(tc.name, func(_ *testing.T) { expectedNamespace := "right-namespace" set := PatchNamespaces(tc.set, expectedNamespace) u, err := ResourcesToUnstructured(set) @@ -44,8 +45,7 @@ func TestPatchNamespaces(t *testing.T) { require.Equal(expectedNamespace, obj.GetNamespace()) } }) - t.Run(tc.name+"-empty-namespace", func(t *testing.T) { - require := require.New(t) + t.Run(tc.name+"-empty-namespace", func(_ *testing.T) { set := PatchNamespaces(tc.set, "some-namespace") set = PatchNamespaces(set, "") u, err := ResourcesToUnstructured(set) diff --git a/internal/kuberesource/parts.go b/internal/kuberesource/parts.go index 5ba6d4a819..71352b77d7 100644 --- a/internal/kuberesource/parts.go +++ b/internal/kuberesource/parts.go @@ -7,6 +7,7 @@ import ( "fmt" "strconv" + "github.com/edgelesssys/contrast/internal/manifest" "github.com/edgelesssys/contrast/node-installer/platforms" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -17,7 +18,12 @@ import ( ) // ContrastRuntimeClass creates a new RuntimeClassConfig. -func ContrastRuntimeClass(platform platforms.Platform) *RuntimeClassConfig { +func ContrastRuntimeClass(platform platforms.Platform) (*RuntimeClassConfig, error) { + runtimeHandler, err := manifest.DefaultPlatformHandler(platform) + if err != nil { + return nil, fmt.Errorf("getting default runtime handler: %w", err) + } + r := RuntimeClass(runtimeHandler). WithHandler(runtimeHandler). WithLabels(map[string]string{"addonmanager.kubernetes.io/mode": "Reconcile"}). @@ -27,7 +33,7 @@ func ContrastRuntimeClass(platform platforms.Platform) *RuntimeClassConfig { r.WithScheduling(Scheduling(map[string]string{"kubernetes.azure.com/kata-cc-isolation": "true"})) } - return &RuntimeClassConfig{r} + return &RuntimeClassConfig{r}, nil } // NodeInstallerConfig wraps a DaemonSetApplyConfiguration for a node installer. @@ -39,6 +45,11 @@ type NodeInstallerConfig struct { func NodeInstaller(namespace string, platform platforms.Platform) (*NodeInstallerConfig, error) { name := "contrast-node-installer" + runtimeHandler, err := manifest.DefaultPlatformHandler(platform) + if err != nil { + return nil, fmt.Errorf("getting default runtime handler: %w", err) + } + var nodeInstallerImageURL string switch platform { case platforms.AKSCloudHypervisorSNP: @@ -232,7 +243,6 @@ func Coordinator(namespace string) *CoordinatorConfig { WithLabels(map[string]string{"app.kubernetes.io/name": "coordinator"}). WithAnnotations(map[string]string{"contrast.edgeless.systems/pod-role": "coordinator"}). WithSpec(PodSpec(). - WithRuntimeClassName(runtimeHandler). WithContainers( Container(). WithName("coordinator"). diff --git a/internal/kuberesource/parts_test.go b/internal/kuberesource/parts_test.go index edc25ab83f..7793b422a5 100644 --- a/internal/kuberesource/parts_test.go +++ b/internal/kuberesource/parts_test.go @@ -24,9 +24,7 @@ func TestNewPortForwarder(t *testing.T) { func TestCoordinator(t *testing.T) { require := require.New(t) - config := Coordinator("default") - - b, err := EncodeResources(config) + b, err := EncodeResources(Coordinator("default")) require.NoError(err) t.Log("\n" + string(b)) } diff --git a/internal/kuberesource/resourcegen/main.go b/internal/kuberesource/resourcegen/main.go index 59e6f1e65a..ac7ecb0ad3 100644 --- a/internal/kuberesource/resourcegen/main.go +++ b/internal/kuberesource/resourcegen/main.go @@ -10,6 +10,7 @@ import ( "os" "github.com/edgelesssys/contrast/internal/kuberesource" + "github.com/edgelesssys/contrast/internal/manifest" "github.com/edgelesssys/contrast/node-installer/platforms" ) @@ -28,12 +29,18 @@ func main() { flag.Parse() var platform platforms.Platform + var runtimeHandler string if *rawPlatform != "" { var err error platform, err = platforms.FromString(*rawPlatform) if err != nil { log.Fatalf("Error parsing platform: %v", err) } + + runtimeHandler, err = manifest.DefaultPlatformHandler(platform) + if err != nil { + log.Fatalf("Error getting default runtime handler: %v", err) + } } var resources []any @@ -42,18 +49,17 @@ func main() { var err error switch set { case "coordinator": - subResources = kuberesource.CoordinatorBundle() + subResources = kuberesource.PatchRuntimeHandlers(kuberesource.CoordinatorBundle(), runtimeHandler) case "runtime": subResources, err = kuberesource.Runtime(platform) - if err != nil { - log.Fatalf("Error generating runtime: %v", err) - } case "openssl": - subResources = kuberesource.OpenSSL() + subResources = kuberesource.PatchRuntimeHandlers(kuberesource.OpenSSL(), runtimeHandler) case "emojivoto": subResources = kuberesource.Emojivoto(kuberesource.ServiceMeshDisabled) + subResources = kuberesource.PatchRuntimeHandlers(subResources, runtimeHandler) case "emojivoto-sm-ingress": subResources = kuberesource.Emojivoto(kuberesource.ServiceMeshIngressEgress) + subResources = kuberesource.PatchRuntimeHandlers(subResources, runtimeHandler) default: log.Fatalf("Error: unknown set: %s\n", set) } diff --git a/internal/kuberesource/sets.go b/internal/kuberesource/sets.go index d6e76d0ba0..3caa0f93e2 100644 --- a/internal/kuberesource/sets.go +++ b/internal/kuberesource/sets.go @@ -15,12 +15,12 @@ import ( // CoordinatorBundle returns the Coordinator and a matching Service. func CoordinatorBundle() []any { - coordinator := Coordinator("").StatefulSetApplyConfiguration - coordinatorService := ServiceForStatefulSet(coordinator). + coordinatorSfSets := Coordinator("").StatefulSetApplyConfiguration + coordinatorService := ServiceForStatefulSet(coordinatorSfSets). WithAnnotations(map[string]string{exposeServiceAnnotation: "true"}) resources := []any{ - coordinator, + coordinatorSfSets, coordinatorService, } @@ -31,14 +31,19 @@ func CoordinatorBundle() []any { func Runtime(platform platforms.Platform) ([]any, error) { ns := "" - runtimeClass := ContrastRuntimeClass(platform).RuntimeClassApplyConfiguration + runtimeClass, err := ContrastRuntimeClass(platform) + if err != nil { + return nil, fmt.Errorf("creating runtime class: %w", err) + } + + runtimeClassApplyConfig := runtimeClass.RuntimeClassApplyConfiguration nodeInstaller, err := NodeInstaller(ns, platform) if err != nil { return nil, fmt.Errorf("creating node installer: %w", err) } return []any{ - runtimeClass, + runtimeClassApplyConfig, nodeInstaller.DaemonSetApplyConfiguration, }, nil } @@ -46,7 +51,6 @@ func Runtime(platform platforms.Platform) ([]any, error) { // OpenSSL returns a set of resources for testing with OpenSSL. func OpenSSL() []any { ns := "" - backend := Deployment("openssl-backend", ns). WithSpec(DeploymentSpec(). WithReplicas(1). @@ -56,7 +60,6 @@ func OpenSSL() []any { WithTemplate(PodTemplateSpec(). WithLabels(map[string]string{"app.kubernetes.io/name": "openssl-backend"}). WithSpec(PodSpec(). - WithRuntimeClassName(runtimeHandler). WithContainers( Container(). WithName("openssl-backend"). @@ -92,7 +95,6 @@ func OpenSSL() []any { WithTemplate(PodTemplateSpec(). WithLabels(map[string]string{"app.kubernetes.io/name": "openssl-frontend"}). WithSpec(PodSpec(). - WithRuntimeClassName(runtimeHandler). WithContainers( Container(). WithName("openssl-frontend"). @@ -130,7 +132,7 @@ func OpenSSL() []any { } // GetDEnts returns a set of resources for testing getdents entry limits. -func GetDEnts() ([]any, error) { +func GetDEnts() []any { tester := Deployment("getdents-tester", ""). WithSpec(DeploymentSpec(). WithReplicas(1). @@ -140,7 +142,6 @@ func GetDEnts() ([]any, error) { WithTemplate(PodTemplateSpec(). WithLabels(map[string]string{"app.kubernetes.io/name": "getdents-tester"}). WithSpec(PodSpec(). - WithRuntimeClassName(runtimeHandler). WithContainers( Container(). WithName("getdents-tester"). @@ -154,7 +155,7 @@ func GetDEnts() ([]any, error) { ), ) - return []any{tester}, nil + return []any{tester} } // GenpolicyRegressionTests returns deployments for regression testing genpolicy. @@ -172,7 +173,6 @@ func GenpolicyRegressionTests() map[string]*applyappsv1.DeploymentApplyConfigura WithTemplate(PodTemplateSpec(). WithLabels(map[string]string{"app.kubernetes.io/name": badLayer}). WithSpec(PodSpec(). - WithRuntimeClassName(runtimeHandler). WithContainers( Container(). WithName(badLayer). @@ -234,7 +234,6 @@ func Emojivoto(smMode serviceMeshMode) []any { "version": "v11", }). WithSpec(PodSpec(). - WithRuntimeClassName(runtimeHandler). WithServiceAccountName("emoji"). WithContainers( Container(). @@ -340,7 +339,6 @@ func Emojivoto(smMode serviceMeshMode) []any { "version": "v11", }). WithSpec(PodSpec(). - WithRuntimeClassName(runtimeHandler). WithServiceAccountName("voting"). WithContainers( Container(). @@ -412,7 +410,6 @@ func Emojivoto(smMode serviceMeshMode) []any { "version": "v11", }). WithSpec(PodSpec(). - WithRuntimeClassName(runtimeHandler). WithServiceAccountName("web"). WithContainers( Container(). diff --git a/internal/manifest/assets/reference-values.json b/internal/manifest/assets/reference-values.json new file mode 100755 index 0000000000..eda2960cec --- /dev/null +++ b/internal/manifest/assets/reference-values.json @@ -0,0 +1 @@ +"THIS FILE IS REPLACED DURING BUILD AND ONLY HERE TO SATISFY GO TOOLING" diff --git a/internal/manifest/constants.go b/internal/manifest/constants.go index 1e0900b18e..8b4572f80c 100644 --- a/internal/manifest/constants.go +++ b/internal/manifest/constants.go @@ -3,32 +3,58 @@ package manifest -// TrustedMeasurement contains the expected launch digest and is injected at build time. -var TrustedMeasurement = "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" - -// Default returns a default manifest. -func Default() Manifest { - return Manifest{ - ReferenceValues: ReferenceValues{ - TrustedMeasurement: HexString(TrustedMeasurement), - }, +import ( + "encoding/json" + "fmt" + + "github.com/edgelesssys/contrast/node-installer/platforms" +) + +// Default returns a default manifest with reference values for the given platform. +func Default(platform platforms.Platform) (*Manifest, error) { + refValues := setReferenceValuesIfUninitialized() + + mnfst := Manifest{} + switch platform { + case platforms.AKSCloudHypervisorSNP: + return &Manifest{ + ReferenceValues: ReferenceValues{ + AKS: refValues.AKS, + }, + }, nil + case platforms.RKE2QEMUTDX, platforms.K3sQEMUTDX: + return &Manifest{ + ReferenceValues: ReferenceValues{ + BareMetalTDX: refValues.BareMetalTDX, + }, + }, nil } + return &mnfst, nil } -// DefaultAKS returns a default manifest with AKS reference values. -func DefaultAKS() Manifest { - mnfst := Default() - mnfst.ReferenceValues.SNP = SNPReferenceValues{ - MinimumTCB: SNPTCB{ - BootloaderVersion: toPtr(SVN(3)), - TEEVersion: toPtr(SVN(0)), - SNPVersion: toPtr(SVN(8)), - MicrocodeVersion: toPtr(SVN(115)), - }, +// DefaultPlatformHandler is a short-hand for getting the default runtime handler for a platform. +func DefaultPlatformHandler(platform platforms.Platform) (string, error) { + mnf, err := Default(platform) + if err != nil { + return "", fmt.Errorf("generating manifest: %w", err) } - return mnfst + return mnf.RuntimeHandler(platform) +} + +// EmbeddedReferenceValues returns the reference values embedded in the binary. +func EmbeddedReferenceValues() ReferenceValues { + return setReferenceValuesIfUninitialized() } -func toPtr[T any](t T) *T { - return &t +// EmbeddedReferenceValuesIfUninitialized returns the reference values embedded in the binary. +func setReferenceValuesIfUninitialized() ReferenceValues { + var embeddedReferenceValues *ReferenceValues + + if err := json.Unmarshal(EmbeddedReferenceValuesJSON, &embeddedReferenceValues); err != nil { + // As this relies on a constant, predictable value (i.e. the embedded JSON), which -- in a correctly built binary -- should + // unmarshal safely into the [ReferenceValues], it's acceptable to panic here. + panic(fmt.Errorf("failed to unmarshal embedded reference values: %w", err)) + } + + return *embeddedReferenceValues } diff --git a/internal/manifest/manifest.go b/internal/manifest/manifest.go index 6132a9ef7c..10bfbf885a 100644 --- a/internal/manifest/manifest.go +++ b/internal/manifest/manifest.go @@ -6,11 +6,9 @@ package manifest import ( "crypto/sha256" "encoding/base64" - "encoding/hex" - "encoding/json" "fmt" - "strconv" + "github.com/edgelesssys/contrast/node-installer/platforms" "github.com/google/go-sev-guest/abi" "github.com/google/go-sev-guest/kds" "github.com/google/go-sev-guest/validate" @@ -25,72 +23,6 @@ type Manifest struct { SeedshareOwnerPubKeys []HexString } -// ReferenceValues contains the workload independent reference values. -type ReferenceValues struct { - SNP SNPReferenceValues - // TrustedMeasurement is the hash of the trusted launch digest. - TrustedMeasurement HexString -} - -// SNPReferenceValues contains reference values for the SNP report. -type SNPReferenceValues struct { - MinimumTCB SNPTCB -} - -// SNPTCB represents a set of SNP TCB values. -type SNPTCB struct { - BootloaderVersion *SVN - TEEVersion *SVN - SNPVersion *SVN - MicrocodeVersion *SVN -} - -// SVN is a SNP secure version number. -type SVN uint8 - -// UInt8 returns the uint8 value of the SVN. -func (s *SVN) UInt8() uint8 { - return uint8(*s) -} - -// MarshalJSON marshals the SVN to JSON. -func (s SVN) MarshalJSON() ([]byte, error) { - return []byte(strconv.Itoa(int(s))), nil -} - -// UnmarshalJSON unmarshals the SVN from a JSON. -func (s *SVN) UnmarshalJSON(data []byte) error { - var value float64 - if err := json.Unmarshal(data, &value); err != nil { - return err - } - - if value < 0 || value > 255 { // Ensure the value fits into uint8 range - return fmt.Errorf("value out of range for uint8") - } - - *s = SVN(value) - return nil -} - -// HexString is a hex encoded string. -type HexString string - -// NewHexString creates a new HexString from a byte slice. -func NewHexString(b []byte) HexString { - return HexString(hex.EncodeToString(b)) -} - -// String returns the string representation of the HexString. -func (h HexString) String() string { - return string(h) -} - -// Bytes returns the byte slice representation of the HexString. -func (h HexString) Bytes() ([]byte, error) { - return hex.DecodeString(string(h)) -} - // HexStrings is a slice of HexString. type HexStrings []HexString @@ -128,6 +60,26 @@ func (p Policy) Hash() HexString { // Validate checks the validity of all fields in the reference values. func (r ReferenceValues) Validate() error { + if r.AKS != nil { + if err := r.AKS.Validate(); err != nil { + return fmt.Errorf("validating AKS reference values: %w", err) + } + } + if r.BareMetalTDX != nil { + if err := r.BareMetalTDX.Validate(); err != nil { + return fmt.Errorf("validating bare metal TDX reference values: %w", err) + } + } + + if r.BareMetalTDX == nil && r.AKS == nil { + return fmt.Errorf("reference values in manifest cannot be empty. Is the chosen platform supported?") + } + + return nil +} + +// Validate checks the validity of all fields in the AKS reference values. +func (r AKSReferenceValues) Validate() error { if r.SNP.MinimumTCB.BootloaderVersion == nil { return fmt.Errorf("field BootloaderVersion in manifest cannot be empty") } else if r.SNP.MinimumTCB.TEEVersion == nil { @@ -145,6 +97,14 @@ func (r ReferenceValues) Validate() error { return nil } +// Validate checks the validity of all fields in the bare metal TDX reference values. +func (r BareMetalTDXReferenceValues) Validate() error { + if r.TrustedMeasurement == "" { + return fmt.Errorf("field TrustedMeasurement in manifest cannot be empty") + } + return nil +} + // Validate checks the validity of all fields in the manifest. func (m *Manifest) Validate() error { for policyHash := range m.Policies { @@ -175,13 +135,17 @@ func (m *Manifest) Validate() error { return nil } -// SNPValidateOpts returns validate options populated with the manifest's -// SNP reference values and trusted measurement. -func (m *Manifest) SNPValidateOpts() (*validate.Options, error) { +// AKSValidateOpts returns validate options populated with the manifest's +// AKS reference values and trusted measurement. +func (m *Manifest) AKSValidateOpts() (*validate.Options, error) { + if m.ReferenceValues.AKS == nil { + return nil, fmt.Errorf("no AKS reference values present in manifest") + } + if err := m.Validate(); err != nil { return nil, fmt.Errorf("validating manifest: %w", err) } - trustedMeasurement, err := m.ReferenceValues.TrustedMeasurement.Bytes() + trustedMeasurement, err := m.ReferenceValues.AKS.TrustedMeasurement.Bytes() if err != nil { return nil, fmt.Errorf("failed to convert TrustedMeasurement from manifest to byte slices: %w", err) } @@ -194,17 +158,29 @@ func (m *Manifest) SNPValidateOpts() (*validate.Options, error) { }, VMPL: new(int), // VMPL0 MinimumTCB: kds.TCBParts{ - BlSpl: m.ReferenceValues.SNP.MinimumTCB.BootloaderVersion.UInt8(), - TeeSpl: m.ReferenceValues.SNP.MinimumTCB.TEEVersion.UInt8(), - SnpSpl: m.ReferenceValues.SNP.MinimumTCB.SNPVersion.UInt8(), - UcodeSpl: m.ReferenceValues.SNP.MinimumTCB.MicrocodeVersion.UInt8(), + BlSpl: m.ReferenceValues.AKS.SNP.MinimumTCB.BootloaderVersion.UInt8(), + TeeSpl: m.ReferenceValues.AKS.SNP.MinimumTCB.TEEVersion.UInt8(), + SnpSpl: m.ReferenceValues.AKS.SNP.MinimumTCB.SNPVersion.UInt8(), + UcodeSpl: m.ReferenceValues.AKS.SNP.MinimumTCB.MicrocodeVersion.UInt8(), }, MinimumLaunchTCB: kds.TCBParts{ - BlSpl: m.ReferenceValues.SNP.MinimumTCB.BootloaderVersion.UInt8(), - TeeSpl: m.ReferenceValues.SNP.MinimumTCB.TEEVersion.UInt8(), - SnpSpl: m.ReferenceValues.SNP.MinimumTCB.SNPVersion.UInt8(), - UcodeSpl: m.ReferenceValues.SNP.MinimumTCB.MicrocodeVersion.UInt8(), + BlSpl: m.ReferenceValues.AKS.SNP.MinimumTCB.BootloaderVersion.UInt8(), + TeeSpl: m.ReferenceValues.AKS.SNP.MinimumTCB.TEEVersion.UInt8(), + SnpSpl: m.ReferenceValues.AKS.SNP.MinimumTCB.SNPVersion.UInt8(), + UcodeSpl: m.ReferenceValues.AKS.SNP.MinimumTCB.MicrocodeVersion.UInt8(), }, PermitProvisionalFirmware: true, }, nil } + +// RuntimeHandler returns the runtime handler for the given platform. +func (m *Manifest) RuntimeHandler(platform platforms.Platform) (string, error) { + switch platform { + case platforms.AKSCloudHypervisorSNP: + return fmt.Sprintf("contrast-cc-%s", m.ReferenceValues.AKS.TrustedMeasurement[:32]), nil + case platforms.K3sQEMUTDX, platforms.RKE2QEMUTDX: + return fmt.Sprintf("contrast-cc-%s", m.ReferenceValues.BareMetalTDX.TrustedMeasurement[:32]), nil + default: + return "", fmt.Errorf("unsupported platform %s", platform) + } +} diff --git a/internal/manifest/manifest_test.go b/internal/manifest/manifest_test.go index aa91e7ebb3..2c7143ee91 100644 --- a/internal/manifest/manifest_test.go +++ b/internal/manifest/manifest_test.go @@ -5,91 +5,15 @@ package manifest import ( "encoding/base64" - "encoding/json" "strconv" "testing" + "github.com/edgelesssys/contrast/node-installer/platforms" "github.com/google/go-sev-guest/kds" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) -func TestSVN(t *testing.T) { - testCases := []struct { - enc string - dec SVN - wantErr bool - }{ - {enc: "0", dec: 0}, - {enc: "1", dec: 1}, - {enc: "255", dec: 255}, - {enc: "256", dec: 0, wantErr: true}, - {enc: "-1", dec: 0, wantErr: true}, - } - - t.Run("MarshalJSON", func(t *testing.T) { - for _, tc := range testCases { - if tc.wantErr { - continue - } - t.Run(tc.enc, func(t *testing.T) { - assert := assert.New(t) - enc, err := json.Marshal(tc.dec) - assert.NoError(err) - assert.Equal(tc.enc, string(enc)) - }) - } - }) - - t.Run("UnmarshalJSON", func(t *testing.T) { - for _, tc := range testCases { - t.Run(tc.enc, func(t *testing.T) { - assert := assert.New(t) - var dec SVN - err := json.Unmarshal([]byte(tc.enc), &dec) - if tc.wantErr { - assert.Error(err) - return - } - assert.NoError(err) - assert.Equal(tc.dec, dec) - }) - } - }) -} - -func TestHexString(t *testing.T) { - testCases := []struct { - b []byte - s string - }{ - {b: []byte{0x00}, s: "00"}, - {b: []byte{0x01}, s: "01"}, - {b: []byte{0x0f}, s: "0f"}, - {b: []byte{0x10}, s: "10"}, - {b: []byte{0x11}, s: "11"}, - {b: []byte{0xff}, s: "ff"}, - {b: []byte{0x00, 0x01}, s: "0001"}, - } - - for _, tc := range testCases { - t.Run(tc.s, func(t *testing.T) { - assert := assert.New(t) - hexString := NewHexString(tc.b) - assert.Equal(tc.s, hexString.String()) - b, err := hexString.Bytes() - assert.NoError(err) - assert.Equal(tc.b, b) - }) - } - - t.Run("invalid hexstring", func(t *testing.T) { - assert := assert.New(t) - hexString := HexString("invalid") - _, err := hexString.Bytes() - assert.Error(err) - }) -} - func TestHexStrings(t *testing.T) { testCases := []struct { hs HexStrings @@ -148,37 +72,38 @@ func TestPolicy(t *testing.T) { } func TestValidate(t *testing.T) { + mnf, err := Default(platforms.AKSCloudHypervisorSNP) + require.NoError(t, err) + testCases := []struct { - m Manifest + m *Manifest wantErr bool }{ { - m: DefaultAKS(), - }, - { - m: Default(), - wantErr: true, + m: mnf, }, { - m: Manifest{ + m: &Manifest{ Policies: map[HexString][]string{HexString(""): {}}, - ReferenceValues: DefaultAKS().ReferenceValues, + ReferenceValues: mnf.ReferenceValues, }, wantErr: true, }, { - m: Manifest{ + m: &Manifest{ Policies: map[HexString][]string{HexString(""): {}}, ReferenceValues: ReferenceValues{ - SNP: Default().ReferenceValues.SNP, - TrustedMeasurement: "", + AKS: &AKSReferenceValues{ + SNP: mnf.ReferenceValues.AKS.SNP, + TrustedMeasurement: "", + }, }, }, wantErr: true, }, { - m: Manifest{ - ReferenceValues: Default().ReferenceValues, + m: &Manifest{ + ReferenceValues: mnf.ReferenceValues, WorkloadOwnerKeyDigests: []HexString{HexString("")}, }, wantErr: true, @@ -197,44 +122,31 @@ func TestValidate(t *testing.T) { } } -func TestSNPValidateOpts(t *testing.T) { - testCases := []struct { - m Manifest - wantErr bool - }{ - {m: DefaultAKS()}, - {m: Default(), wantErr: true}, - } +func TestAKSValidateOpts(t *testing.T) { + assert := assert.New(t) - for i, tc := range testCases { - t.Run(strconv.Itoa(i), func(t *testing.T) { - assert := assert.New(t) + m, err := Default(platforms.AKSCloudHypervisorSNP) + require.NoError(t, err) - opts, err := tc.m.SNPValidateOpts() - if tc.wantErr { - assert.Error(err) - return - } - assert.NoError(err) - - tcb := tc.m.ReferenceValues.SNP.MinimumTCB - assert.NotNil(tcb.BootloaderVersion) - assert.NotNil(tcb.TEEVersion) - assert.NotNil(tcb.SNPVersion) - assert.NotNil(tcb.MicrocodeVersion) - - trustedMeasurement, err := tc.m.ReferenceValues.TrustedMeasurement.Bytes() - assert.NoError(err) - assert.Equal(trustedMeasurement, opts.Measurement) - - tcbParts := kds.TCBParts{ - BlSpl: tcb.BootloaderVersion.UInt8(), - TeeSpl: tcb.TEEVersion.UInt8(), - SnpSpl: tcb.SNPVersion.UInt8(), - UcodeSpl: tcb.MicrocodeVersion.UInt8(), - } - assert.Equal(tcbParts, opts.MinimumTCB) - assert.Equal(tcbParts, opts.MinimumLaunchTCB) - }) + opts, err := m.AKSValidateOpts() + assert.NoError(err) + + tcb := m.ReferenceValues.AKS.SNP.MinimumTCB + assert.NotNil(tcb.BootloaderVersion) + assert.NotNil(tcb.TEEVersion) + assert.NotNil(tcb.SNPVersion) + assert.NotNil(tcb.MicrocodeVersion) + + trustedMeasurement, err := m.ReferenceValues.AKS.TrustedMeasurement.Bytes() + assert.NoError(err) + assert.Equal(trustedMeasurement, opts.Measurement) + + tcbParts := kds.TCBParts{ + BlSpl: tcb.BootloaderVersion.UInt8(), + TeeSpl: tcb.TEEVersion.UInt8(), + SnpSpl: tcb.SNPVersion.UInt8(), + UcodeSpl: tcb.MicrocodeVersion.UInt8(), } + assert.Equal(tcbParts, opts.MinimumTCB) + assert.Equal(tcbParts, opts.MinimumLaunchTCB) } diff --git a/internal/manifest/referencevalues.go b/internal/manifest/referencevalues.go new file mode 100644 index 0000000000..8399be0ee3 --- /dev/null +++ b/internal/manifest/referencevalues.go @@ -0,0 +1,96 @@ +// Copyright 2024 Edgeless Systems GmbH +// SPDX-License-Identifier: AGPL-3.0-only + +package manifest + +import ( + _ "embed" + "encoding/hex" + "encoding/json" + "fmt" + "strconv" +) + +// EmbeddedReferenceValuesJSON contains the embedded reference values in JSON format. +// At startup, they are unmarshaled into a globally-shared ReferenceValues struct. +// +//go:embed assets/reference-values.json +var EmbeddedReferenceValuesJSON []byte + +// ReferenceValues contains the workload-independent reference values for each platform. +type ReferenceValues struct { + // AKS holds the reference values for AKS. + AKS *AKSReferenceValues `json:"aks,omitempty"` + // BareMetalTDX holds the reference values for TDX on bare metal. + BareMetalTDX *BareMetalTDXReferenceValues `json:"bareMetalTDX,omitempty"` +} + +// AKSReferenceValues contains reference values for AKS. +type AKSReferenceValues struct { + SNP SNPReferenceValues + TrustedMeasurement HexString +} + +// BareMetalTDXReferenceValues contains reference values for BareMetalTDX. +type BareMetalTDXReferenceValues struct { + TrustedMeasurement HexString +} + +// SNPReferenceValues contains reference values for the SNP report. +type SNPReferenceValues struct { + MinimumTCB SNPTCB +} + +// SNPTCB represents a set of SNP TCB values. +type SNPTCB struct { + BootloaderVersion *SVN + TEEVersion *SVN + SNPVersion *SVN + MicrocodeVersion *SVN +} + +// SVN is a SNP secure version number. +type SVN uint8 + +// UInt8 returns the uint8 value of the SVN. +func (s *SVN) UInt8() uint8 { + return uint8(*s) +} + +// MarshalJSON marshals the SVN to JSON. +func (s SVN) MarshalJSON() ([]byte, error) { + return []byte(strconv.Itoa(int(s))), nil +} + +// UnmarshalJSON unmarshals the SVN from a JSON. +func (s *SVN) UnmarshalJSON(data []byte) error { + var value float64 + if err := json.Unmarshal(data, &value); err != nil { + return err + } + + if value < 0 || value > 255 { // Ensure the value fits into uint8 range + return fmt.Errorf("value out of range for uint8") + } + + *s = SVN(value) + return nil +} + +// HexString is a hex encoded string. +type HexString string + +// NewHexString creates a new HexString from a byte slice. +func NewHexString(b []byte) HexString { + return HexString(hex.EncodeToString(b)) +} + +// String returns the string representation of the HexString. +func (h HexString) String() string { + return string(h) +} + +// Bytes returns the byte slice representation of the HexString. +func (h HexString) Bytes() ([]byte, error) { + return hex.DecodeString(string(h)) +} diff --git a/internal/manifest/referencevalues_test.go b/internal/manifest/referencevalues_test.go new file mode 100644 index 0000000000..9ddf56bc3f --- /dev/null +++ b/internal/manifest/referencevalues_test.go @@ -0,0 +1,115 @@ +// Copyright 2024 Edgeless Systems GmbH +// SPDX-License-Identifier: AGPL-3.0-only + +package manifest + +import ( + "encoding/json" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSVN(t *testing.T) { + testCases := []struct { + enc string + dec SVN + wantErr bool + }{ + {enc: "0", dec: 0}, + {enc: "1", dec: 1}, + {enc: "255", dec: 255}, + {enc: "256", dec: 0, wantErr: true}, + {enc: "-1", dec: 0, wantErr: true}, + } + + t.Run("MarshalJSON", func(t *testing.T) { + for _, tc := range testCases { + if tc.wantErr { + continue + } + t.Run(tc.enc, func(t *testing.T) { + assert := assert.New(t) + enc, err := json.Marshal(tc.dec) + assert.NoError(err) + assert.Equal(tc.enc, string(enc)) + }) + } + }) + + t.Run("UnmarshalJSON", func(t *testing.T) { + for _, tc := range testCases { + t.Run(tc.enc, func(t *testing.T) { + assert := assert.New(t) + var dec SVN + err := json.Unmarshal([]byte(tc.enc), &dec) + if tc.wantErr { + assert.Error(err) + return + } + assert.NoError(err) + assert.Equal(tc.dec, dec) + }) + } + }) +} + +func TestHexString(t *testing.T) { + testCases := []struct { + b []byte + s string + }{ + {b: []byte{0x00}, s: "00"}, + {b: []byte{0x01}, s: "01"}, + {b: []byte{0x0f}, s: "0f"}, + {b: []byte{0x10}, s: "10"}, + {b: []byte{0x11}, s: "11"}, + {b: []byte{0xff}, s: "ff"}, + {b: []byte{0x00, 0x01}, s: "0001"}, + } + + t.Run("Bytes", func(t *testing.T) { + for _, tc := range testCases { + t.Run(tc.s, func(t *testing.T) { + assert := assert.New(t) + hexString := NewHexString(tc.b) + assert.Equal(tc.s, hexString.String()) + b, err := hexString.Bytes() + assert.NoError(err) + assert.Equal(tc.b, b) + }) + } + }) + + t.Run("MarshalJSON", func(t *testing.T) { + for _, tc := range testCases { + t.Run(tc.s, func(t *testing.T) { + assert := assert.New(t) + hexString := NewHexString(tc.b) + enc, err := json.Marshal(hexString) + assert.NoError(err) + assert.Equal(fmt.Sprintf("\"%s\"", tc.s), string(enc)) + }) + } + }) + + t.Run("UnmarshalJSON", func(t *testing.T) { + for _, tc := range testCases { + t.Run(tc.s, func(t *testing.T) { + assert := assert.New(t) + var hexString HexString + err := json.Unmarshal([]byte(fmt.Sprintf("\"%s\"", tc.s)), &hexString) + assert.NoError(err) + assert.Equal(tc.s, hexString.String()) + }) + } + }) + + t.Run("invalid hexstring", func(t *testing.T) { + assert := assert.New(t) + hexString := HexString("invalid") + _, err := hexString.Bytes() + assert.Error(err) + }) +} diff --git a/packages/by-name/contrast/package.nix b/packages/by-name/contrast/package.nix index 9fff661a1a..88abc63cb3 100644 --- a/packages/by-name/contrast/package.nix +++ b/packages/by-name/contrast/package.nix @@ -6,6 +6,7 @@ buildGoModule, buildGoTest, microsoft, + kata, genpolicy ? microsoft.genpolicy, contrast, installShellFiles, @@ -25,11 +26,7 @@ let tags = [ "e2e" ]; - ldflags = [ - "-s" - "-X github.com/edgelesssys/contrast/internal/manifest.TrustedMeasurement=${launchDigest}" - "-X github.com/edgelesssys/contrast/internal/kuberesource.runtimeHandler=${runtimeHandler}" - ]; + ldflags = [ "-s" ]; subPackages = [ "e2e/genpolicy" @@ -41,10 +38,29 @@ let ]; }; - launchDigest = builtins.readFile "${microsoft.runtime-class-files}/launch-digest.hex"; - - runtimeHandler = lib.removeSuffix "\n" ( - builtins.readFile "${microsoft.runtime-class-files}/runtime-handler" + # Reference values that we embed into the Contrast CLI for + # deployment generation and attestation. + embeddedReferenceValues = builtins.toFile "reference-values.json" ( + builtins.toJSON { + aks = { + snp = { + minimumTCB = { + bootloaderVersion = 3; + teeVersion = 0; + snpVersion = 8; + microcodeVersion = 115; + }; + }; + trustedMeasurement = lib.removeSuffix "\n" ( + builtins.readFile "${microsoft.runtime-class-files}/launch-digest.hex" + ); + }; + bareMetalTDX = { + trustedMeasurement = lib.removeSuffix "\n" ( + builtins.readFile "${kata.runtime-class-files}/launch-digest.hex" + ); + }; + } ); packageOutputs = [ @@ -93,6 +109,7 @@ buildGoModule rec { install -D ${lib.getExe genpolicy} cli/cmd/assets/genpolicy install -D ${genpolicy.settings-dev}/genpolicy-settings.json cli/cmd/assets/genpolicy-settings.json install -D ${genpolicy.rules}/genpolicy-rules.rego cli/cmd/assets/genpolicy-rules.rego + install -D ${embeddedReferenceValues} internal/manifest/assets/reference-values.json ''; CGO_ENABLED = 0; @@ -101,8 +118,6 @@ buildGoModule rec { "-w" "-X github.com/edgelesssys/contrast/cli/constants.Version=${version}" "-X github.com/edgelesssys/contrast/cli/constants.GenpolicyVersion=${genpolicy.version}" - "-X github.com/edgelesssys/contrast/internal/manifest.TrustedMeasurement=${launchDigest}" - "-X github.com/edgelesssys/contrast/internal/kuberesource.runtimeHandler=${runtimeHandler}" ]; preCheck = '' diff --git a/packages/by-name/kata/kata-image/package.nix b/packages/by-name/kata/kata-image/package.nix index 6df8337cd1..802e9d7f46 100644 --- a/packages/by-name/kata/kata-image/package.nix +++ b/packages/by-name/kata/kata-image/package.nix @@ -146,6 +146,21 @@ stdenv.mkDerivation rec { buildPhase = '' runHook preBuild + # Check if filesystem is ext.* + fstype=$(stat -f -c %T .) + if [[ $fstye == "ext4" || $fstype == "ext2/ext3" ]]; then + echo "Due to a bug in the image build, kata-image can unfortunately not be built on $fstype filesystems." + echo "As a workaround, you can build the derivation on a different filesystem with the following:" + echo "systemctl edit nix-daemon" + echo "Then, when editing the unit, enter:" + echo "[Service]" + echo 'Environment=TMPDIR=/some-non-ext*-filesystem' + echo "Then restart the nix-daemon with:" + echo "systemctl restart nix-daemon" + echo "Then rebuild the derivation." + exit 1 + fi + # use a fakeroot environment to build the rootfs as a tar # this is required to create files with the correct ownership and permissions # including suid diff --git a/packages/by-name/microsoft/runtime-class-files/package.nix b/packages/by-name/microsoft/runtime-class-files/package.nix index 19aa1a9b4d..98303f1a9a 100644 --- a/packages/by-name/microsoft/runtime-class-files/package.nix +++ b/packages/by-name/microsoft/runtime-class-files/package.nix @@ -24,8 +24,7 @@ stdenvNoCC.mkDerivation { buildPhase = '' mkdir -p $out igvmmeasure -b ${igvm} | dd conv=lcase > $out/launch-digest.hex - echo -n "contrast-cc-" > $out/runtime-handler - cat $out/launch-digest.hex | head -c 32 >> $out/runtime-handler + printf "contrast-cc-%s" "$(cat $out/launch-digest.hex | head -c 32)" > $out/runtime-handler ''; passthru = { From de7d8fa4f346b212768d3cbab63992408558e17c Mon Sep 17 00:00:00 2001 From: Moritz Sanft <58110325+msanft@users.noreply.github.com> Date: Fri, 19 Jul 2024 17:30:53 +0200 Subject: [PATCH 2/3] justfile: supply platform when generating deployment --- justfile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/justfile b/justfile index f4cfba89b0..586d0525cb 100644 --- a/justfile +++ b/justfile @@ -1,5 +1,5 @@ # Undeploy, rebuild, deploy. -default target=default_deploy_target platform=default_platform cli=default_cli: soft-clean coordinator initializer openssl port-forwarder service-mesh-proxy (node-installer platform) (runtime platform) (apply "runtime") (deploy target cli) set verify (wait-for-workload target) +default target=default_deploy_target platform=default_platform cli=default_cli: soft-clean coordinator initializer openssl port-forwarder service-mesh-proxy (node-installer platform) (runtime platform) (apply "runtime") (deploy target cli platform) set verify (wait-for-workload target) # Build and push a container image. push target: @@ -57,7 +57,7 @@ e2e target=default_deploy_target: soft-clean coordinator initializer openssl por --skip-undeploy=true # Generate policies, apply Kubernetes manifests. -deploy target=default_deploy_target cli=default_cli: (populate target) (generate cli) (apply target) +deploy target=default_deploy_target cli=default_cli platform=default_platform: (populate target) (generate cli platform) (apply target) # Populate the workspace with a runtime class deployment runtime platform=default_platform: @@ -69,24 +69,24 @@ runtime platform=default_platform: runtime > ./{{ workspace_dir }}/runtime/runtime.yml # Populate the workspace with a Kubernetes deployment -populate target=default_deploy_target: +populate target=default_deploy_target platform=default_platform: #!/usr/bin/env bash set -euo pipefail mkdir -p ./{{ workspace_dir }} mkdir -p ./{{ workspace_dir }}/deployment nix shell .#contrast --command resourcegen \ --image-replacements ./{{ workspace_dir }}/just.containerlookup --namespace {{ target }}${namespace_suffix-} \ - --add-namespace-object --add-port-forwarders \ + --add-namespace-object --add-port-forwarders --platform {{ platform }} \ {{ target }} coordinator > ./{{ workspace_dir }}/deployment/deployment.yml echo "{{ target }}${namespace_suffix-}" > ./{{ workspace_dir }}/just.namespace # Generate policies, update manifest. -generate cli=default_cli: +generate cli=default_cli platform=default_platform: #!/usr/bin/env bash nix run .#{{ cli }} -- generate \ --workspace-dir ./{{ workspace_dir }} \ --image-replacements ./{{ workspace_dir }}/just.containerlookup \ - --reference-values aks-clh-snp \ + --reference-values {{ platform }}\ ./{{ workspace_dir }}/deployment/*.yml # Apply Kubernetes manifests from /deployment From fda28a4a1623b7b9fd9c638bde814e427a585939 Mon Sep 17 00:00:00 2001 From: Moritz Sanft <58110325+msanft@users.noreply.github.com> Date: Thu, 25 Jul 2024 14:45:41 +0200 Subject: [PATCH 3/3] ci: run ext*-depending builds in tmpfs This puts the Nix builds requiring a non-ext filesystem into a tmpfs. This is a workaround until the upstream bug is reported and resolved to / in fakeroot. --- .github/actions/nix_tmpfs/action.yml | 13 +++++++++++++ .github/workflows/e2e_openssl.yml | 1 + .github/workflows/e2e_policy.yml | 1 + .github/workflows/e2e_regression.yml | 1 + .github/workflows/e2e_servicemesh.yml | 3 ++- 5 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 .github/actions/nix_tmpfs/action.yml diff --git a/.github/actions/nix_tmpfs/action.yml b/.github/actions/nix_tmpfs/action.yml new file mode 100644 index 0000000000..1b194cfd74 --- /dev/null +++ b/.github/actions/nix_tmpfs/action.yml @@ -0,0 +1,13 @@ +name: use tmpfs for nix builds +description: Set Nix' build directory to a tmpfs to fix builds that don't work in the runner-provisioned ext2/ext3 filesystem + +runs: + using: "composite" + steps: + - name: use tmpfs for nix builds + shell: bash + run: | + sudo mkdir -p /etc/systemd/system/nix-daemon.service.d + echo -e "[Service]\nEnvironment=TMPDIR=/dev/shm" | sudo tee /etc/systemd/system/nix-daemon.service.d/tmpfs.conf + sudo systemctl daemon-reload + sudo systemctl restart nix-daemon diff --git a/.github/workflows/e2e_openssl.yml b/.github/workflows/e2e_openssl.yml index 343e4e2a60..294c29457f 100644 --- a/.github/workflows/e2e_openssl.yml +++ b/.github/workflows/e2e_openssl.yml @@ -59,6 +59,7 @@ jobs: echo "SYNC_ENDPOINT=http://$sync_ip:8080" | tee -a "$GITHUB_ENV" sync_uuid=$(kubectl get configmap sync-server-fifo -o jsonpath='{.data.uuid}') echo "SYNC_FIFO_UUID=$sync_uuid" | tee -a "$GITHUB_ENV" + - uses: ./.github/actions/nix_tmpfs - name: Build and prepare deployments run: | just coordinator initializer openssl port-forwarder node-installer diff --git a/.github/workflows/e2e_policy.yml b/.github/workflows/e2e_policy.yml index 53990fe7c5..19a1e379b2 100644 --- a/.github/workflows/e2e_policy.yml +++ b/.github/workflows/e2e_policy.yml @@ -59,6 +59,7 @@ jobs: echo "SYNC_ENDPOINT=http://$sync_ip:8080" | tee -a "$GITHUB_ENV" sync_uuid=$(kubectl get configmap sync-server-fifo -o jsonpath='{.data.uuid}') echo "SYNC_FIFO_UUID=$sync_uuid" | tee -a "$GITHUB_ENV" + - uses: ./.github/actions/nix_tmpfs - name: Build and prepare deployments run: | just coordinator initializer openssl port-forwarder node-installer diff --git a/.github/workflows/e2e_regression.yml b/.github/workflows/e2e_regression.yml index 741537dd2b..3716522f9a 100644 --- a/.github/workflows/e2e_regression.yml +++ b/.github/workflows/e2e_regression.yml @@ -58,6 +58,7 @@ jobs: - name: Get credentials for CI cluster run: | just get-credentials + - uses: ./.github/actions/nix_tmpfs - name: Build and prepare deployments run: | just node-installer diff --git a/.github/workflows/e2e_servicemesh.yml b/.github/workflows/e2e_servicemesh.yml index 7af86572c1..1d5b64604e 100644 --- a/.github/workflows/e2e_servicemesh.yml +++ b/.github/workflows/e2e_servicemesh.yml @@ -53,12 +53,13 @@ jobs: - name: Get credentials for CI cluster run: | just get-credentials - - name: Set sync environemnt + - name: Set sync environment run: | sync_ip=$(kubectl get svc sync -o jsonpath='{.status.loadBalancer.ingress[0].ip}') echo "SYNC_ENDPOINT=http://$sync_ip:8080" | tee -a "$GITHUB_ENV" sync_uuid=$(kubectl get configmap sync-server-fifo -o jsonpath='{.data.uuid}') echo "SYNC_FIFO_UUID=$sync_uuid" | tee -a "$GITHUB_ENV" + - uses: ./.github/actions/nix_tmpfs - name: Build and prepare deployments run: | just coordinator initializer port-forwarder service-mesh-proxy node-installer