DV Setup: Charon Integration with Sedge (#368)

* charon poc * fix tppo * add distributed flag to vc if Distributed * Update configs/client_images.yaml Co-authored-by: Oisín Kyne <[email protected]> * fix:remove "-" * fix: validator-blocker * add Distributed Client Test * Update types, fix bugs, reviews * fix types for distributed validator image defaults * Update templates/services/merge/distributedValidator/charon.tmpl Co-authored-by: Oisín Kyne <[email protected]> * Make DV service generic * fix tests * fix dv tests * allow import of distributed validator keystores * updates to Teku key import * updates to Teku validator init * fixed lighthouse imports * use DefaultAbsSedgeDataPath for charon key imports * Document DV Setup Process with Sedge * Update docs/docs/commands/importKey.mdx Co-authored-by: Oisín Kyne <[email protected]> * Update docs/docs/quickstart/charon.mdx Co-authored-by: Oisín Kyne <[email protected]> * Update docs/docs/quickstart/charon.mdx Co-authored-by: Oisín Kyne <[email protected]> * Update docs/docs/quickstart/charon.mdx Co-authored-by: Oisín Kyne <[email protected]> * Update docs/docs/quickstart/charon.mdx Co-authored-by: Oisín Kyne <[email protected]> * Update docs/docs/quickstart/charon.mdx Co-authored-by: Oisín Kyne <[email protected]> * Update docs/docs/quickstart/charon.mdx Co-authored-by: Oisín Kyne <[email protected]> * Update docs/docs/quickstart/charon.mdx Co-authored-by: Oisín Kyne <[email protected]> * Update docs/docs/quickstart/charon.mdx Co-authored-by: Oisín Kyne <[email protected]> * Update docs/docs/quickstart/charon.mdx Co-authored-by: Oisín Kyne <[email protected]> * Fixed defaultKeystorePath for Charon key imports * import key tests from distributed option * bug fixes * add dv extra flags feature * Allow custom dv images, and few review fixes * Add script to fetch and update Charon to the latest version * lighthouse import-key tests * teku import-key tests * run formatter * lodestar import keys test in distributed mode * debug tests * fix file permissions for distributed key imports * teku import key tests * skip key-import tests * Revert last commit * remove error logs * fix missing test data * add Nimbus client to DV * remove redundant methods * Update Changelog * Update CHANGELOG.md Co-authored-by: Miguel Tenorio <[email protected]> --------- Co-authored-by: Oisín Kyne <[email protected]> Co-authored-by: xin <[email protected]> Co-authored-by: Miguel Tenorio <[email protected]>
NethermindEth · Nov 14, 2024 · d04b0b4 · d04b0b4
1 parent a204f33
commit d04b0b4
Show file tree

Hide file tree

Showing 59 changed files with 1,557 additions and 269 deletions.
diff --git a/.gitignore b/.gitignore
@@ -19,3 +19,8 @@ courtney/
 mocks/
 
 build/lido-exporter
+.charon/
+node*/
+
+keystore*
+!cli/actions/testdata/charon/validator_keys/keystore*
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+- New cli flag --distributed for running cluster with Charon distributed validator
+
 ## [v1.7.1] - 2024-11-1
 
 ### Added
@@ -23,6 +26,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [v1.6.0] - 2024-10-18
 
+
 ### Added
 - New command `lido-status` to display data of Lido Node Operator.
 - New command `monitoring` to run monitoring stack setup with Grafana, Prometheus, Node Exporter and Lido Exporter.

diff --git a/README.md b/README.md
@@ -136,6 +136,11 @@ friends, to amateur operators — to operate validators by providing an ETH-base
 Sedge supports the Lido CSM, allowing users to generate validator keys and set up their full nodes with ease. You can 
 read more about it in [our documentation](https://docs.sedge.nethermind.io/docs/quickstart/staking-with-lido)! 
 
+## Charon DV integration
+Charon is used by stakers to distribute the responsibility of running Ethereum Validators across a number of different instances and client implementations. Setting up and running a full ethereum node with charon, needs some learning curve and compatibility knowledge, in order for the setup to be fully compliant with the charon configuration requirements for different BN-VC combinations. We want to provide a better and guided user experience for setting up a DV with Charon.
+
+Integrating Charon with Sedge would make it easy for stakers to setup and run a DV with Charon without having to go through each individual client setup docs and their compatibility with DVT.
+
 ## Supported networks and clients
 
 ### Mainnet

diff --git a/cli/actions/generation_test.go b/cli/actions/generation_test.go
@@ -95,6 +95,13 @@ func TestGenerateDockerCompose(t *testing.T) {
 		if err != nil {
 			t.Errorf("SupportedClients(\"validator\") failed: %v", err)
 		}
+		var distributedValidatorClients []string
+		if network == "holesky" {
+			distributedValidatorClients, err = c.SupportedClients("distributedValidator")
+			if err != nil {
+				t.Errorf("SupportedClients(\"distributedValidator\") failed: %v", err)
+			}
+		}
 
 		rNum, err := rand.Int(rand.Reader, big.NewInt(int64(100)))
 		if err != nil {
@@ -265,6 +272,25 @@ func TestGenerateDockerCompose(t *testing.T) {
 						},
 					)
 				}
+
+				// For distributedValidator
+				if utils.Contains(distributedValidatorClients, "charon") {
+					tests = append(tests,
+						genTestData{
+							name: fmt.Sprintf("execution: %s, consensus: %s, validator: %s,distributedValidator: %s, network: %s,  all, with distributedValidator", executionCl, consensusCl, consensusCl, distributedValidatorClients, network),
+							genData: generate.GenData{
+								Distributed:                true,
+								DistributedValidatorClient: &clients.Client{Name: "charon", Type: "distributedValidator"},
+								ExecutionClient:            &clients.Client{Name: executionCl, Type: "execution"},
+								ConsensusClient:            &clients.Client{Name: consensusCl, Type: "consensus"},
+								ValidatorClient:            &clients.Client{Name: consensusCl, Type: "validator"},
+								Services:                   []string{"execution", "consensus", "validator", "distributedValidator"},
+								Network:                    network,
+							},
+						},
+					)
+				}
+
 			}
 		}
 	}
@@ -287,6 +313,9 @@ func TestGenerateDockerCompose(t *testing.T) {
 			if tc.genData.ValidatorClient != nil {
 				tc.genData.ValidatorClient.SetImageOrDefault("")
 			}
+			if tc.genData.DistributedValidatorClient != nil {
+				tc.genData.DistributedValidatorClient.SetImageOrDefault("")
+			}
 
 			_, err := sedgeAction.Generate(actions.GenerateOptions{
 				GenerationData: tc.genData,
@@ -474,6 +503,14 @@ func TestGenerateDockerCompose(t *testing.T) {
 				}
 			}
 
+			// Validate that Distributed Validator Client info matches the sample data
+			if tc.genData.DistributedValidatorClient != nil {
+				// Check that the distributed-validator service is set.
+				assert.NotNil(t, cmpData.Services.DistributedValidator)
+				// Check that the distributed-validator container Volume is set.
+				assert.Equal(t, "${DV_DATA_DIR}:/opt/charon/.charon", cmpData.Services.DistributedValidator.Volumes[0])
+			}
+
 			if tc.genData.ValidatorClient == nil {
 				// Check validator blocker is not set if validator is not set
 				assert.Nil(t, cmpData.Services.ValidatorBlocker)

diff --git a/cli/actions/importKeys.go b/cli/actions/importKeys.go
@@ -27,6 +27,7 @@ import (
 
 	"github.com/NethermindEth/sedge/configs"
 	"github.com/NethermindEth/sedge/internal/images/validator-import/lighthouse"
+	"github.com/NethermindEth/sedge/internal/images/validator-import/prysm"
 	"github.com/NethermindEth/sedge/internal/images/validator-import/teku"
 	"github.com/NethermindEth/sedge/internal/pkg/commands"
 	"github.com/NethermindEth/sedge/internal/pkg/services"
@@ -51,6 +52,7 @@ type ImportValidatorKeysOptions struct {
 	GenerationPath  string
 	ContainerTag    string
 	CustomConfig    ImportValidatorKeysCustomOptions
+	Distributed     bool
 }
 type ImportValidatorKeysCustomOptions struct {
 	NetworkConfigPath string
@@ -100,19 +102,95 @@ func (s *sedgeActions) ImportValidatorKeys(options ImportValidatorKeysOptions) e
 	options.GenerationPath = absGenerationPath
 
 	if !isDefaultKeysPath(options.GenerationPath, options.From) {
-		defaultKeystorePath := filepath.Join(options.GenerationPath, "keystore")
-		log.Warnf("The keys path is not the default one, copying the keys to the default path %s", defaultKeystorePath)
-		copy.Copy(options.From, defaultKeystorePath)
+		if !options.Distributed {
+			defaultKeystorePath := filepath.Join(options.GenerationPath, "keystore")
+			log.Warnf("The keys path is not the default one, copying the keys to the default path %s", defaultKeystorePath)
+			copy.Copy(options.From, defaultKeystorePath)
+		}
+	}
+
+	if options.Distributed {
+		cwd, _ := os.Getwd()
+		charonPath := filepath.Join(cwd, ".charon")
+
+		if !isDefaultKeysPath(options.GenerationPath, options.From) {
+			charonPath = options.From
+			log.Infof("Copying the keys from %s", charonPath)
+			options.From = filepath.Join(options.GenerationPath, "keystore")
+		}
+		defaultCharonPath := filepath.Join(configs.DefaultAbsSedgeDataPath, ".charon")
+		// Copy the folder from charonPath to defaultCharonPath
+		log.Infof("Copying Charon contents to the default path %s", defaultCharonPath)
+		if err := os.MkdirAll(defaultCharonPath, 0o755); err != nil {
+			return err
+		}
+		if err := copy.Copy(charonPath, defaultCharonPath); err != nil {
+			return err
+		}
+		charonValidatorKeysPath := filepath.Join(charonPath, "validator_keys")
+		defaultKeystorePath := filepath.Join(configs.DefaultAbsSedgeDataPath, "keystore")
+		log.Infof("Copying the keys to the default path %s", defaultKeystorePath)
+		if err := os.MkdirAll(defaultKeystorePath, 0o755); err != nil {
+			return err
+		}
+
+		validatorKeysPath := filepath.Join(defaultKeystorePath, "validator_keys")
+		if err := os.MkdirAll(validatorKeysPath, 0o755); err != nil {
+			return err
+		}
+
+		depositDataPath := filepath.Join(charonPath, "deposit-data.json")
+		depositDataPathDest := filepath.Join(defaultKeystorePath, "deposit-data.json")
+		if err := copy.Copy(depositDataPath, depositDataPathDest); err != nil {
+			return err
+		}
+
+		files, err := os.ReadDir(charonValidatorKeysPath)
+		if err != nil {
+			log.Fatal(err)
+		}
+		len := len(files)
+		for i := 0; i < len/2; i++ {
+			keystorePath := filepath.Join(charonValidatorKeysPath, fmt.Sprintf("keystore-%d.json", i))
+			validatorPath := filepath.Join(validatorKeysPath, fmt.Sprintf("keystore-%d.json", i))
+			if err := copy.Copy(keystorePath, validatorPath); err != nil {
+				return err
+			}
+
+			keystoreTxtPath := filepath.Join(charonValidatorKeysPath, fmt.Sprintf("keystore-%d.txt", i))
+			keystorePasswordPath := filepath.Join(defaultKeystorePath, fmt.Sprintf("keystore-%d.txt", i))
+			if err := copy.Copy(keystoreTxtPath, keystorePasswordPath); err != nil {
+				return err
+			}
+		}
+		if options.ValidatorClient == "prysm" {
+			keystorePasswordPath := filepath.Join(defaultKeystorePath, "keystore_password.txt")
+			f, err := os.Create(keystorePasswordPath)
+			if err != nil {
+				return err
+			}
+			f.WriteString("prysm-validator-secret")
+			defer f.Close()
+		}
 	}
 
 	var ctID string
 	switch options.ValidatorClient {
 	case "prysm":
-		prysmCtID, err := setupPrysmValidatorImportContainer(s.dockerClient, s.dockerServiceManager, options)
-		if err != nil {
-			return err
+		prysmCtID := ""
+		if options.Distributed {
+			prysmCtID, err = setupPrysmValidatorImportContainerDV(s.dockerClient, s.commandRunner, s.dockerServiceManager, options)
+			if err != nil {
+				return err
+			}
+			ctID = prysmCtID
+		} else {
+			prysmCtID, err := setupPrysmValidatorImportContainer(s.dockerClient, s.dockerServiceManager, options)
+			if err != nil {
+				return err
+			}
+			ctID = prysmCtID
 		}
-		ctID = prysmCtID
 	case "nimbus":
 		nimbusCtID, err := setupNimbusValidatorImport(s.dockerClient, s.dockerServiceManager, options)
 		if err != nil {
@@ -143,7 +221,11 @@ func (s *sedgeActions) ImportValidatorKeys(options ImportValidatorKeysOptions) e
 	log.Info("Importing validator keys")
 	var runErr error
 	if options.ValidatorClient == "nimbus" {
-		runErr = runAndWaitImportKeysNimbus(s.dockerClient, s.dockerServiceManager, ctID)
+		if !options.Distributed {
+			runErr = runAndWaitImportKeysNimbus(s.dockerClient, s.dockerServiceManager, ctID)
+		} else {
+			runErr = runAndWaitImportKeys(s.dockerClient, s.dockerServiceManager, ctID)
+		}
 	} else {
 		runErr = runAndWaitImportKeys(s.dockerClient, s.dockerServiceManager, ctID)
 	}
@@ -260,17 +342,47 @@ func setupNimbusValidatorImport(dockerClient client.APIClient, dockerServiceMana
 	} else {
 		cmd = append(cmd, "--network="+options.Network)
 	}
+	containerConfig := &container.Config{
+		Image:        validatorImage,
+		Cmd:          cmd,
+		AttachStdin:  true,
+		AttachStderr: true,
+		AttachStdout: true,
+		OpenStdin:    true,
+		Tty:          true,
+	}
+	if options.Distributed {
+		containerConfig = &container.Config{
+			Image: validatorImage,
+			Entrypoint: []string{
+				"sh", "-c", `
+				#!/usr/bin/env bash
+				set -e
+				tmpkeys="/keystore/validator_keys/tmpkeys"
+				mkdir -p ${tmpkeys}
+				for f in /keystore/validator_keys/keystore-*.json; do
+					echo "Importing key ${f}"
+					pwdfile="/keystore/$(basename "$f" .json).txt"
+					password=$(cat ${pwdfile})
+					echo "Using password file ${pwdfile}"
+					echo "Using password ${password}"
+					cp "${f}" "${tmpkeys}"
+					# Import keystore with password.
+						echo "$password" | \
+						/home/user/nimbus_beacon_node deposits import \
+						--data-dir=/data \
+						${tmpkeys}
+					filename="$(basename ${f})"
+  					rm "${tmpkeys}/${filename}"
+				done
+			`,
+			},
+		}
+	}
+
 	log.Debugf("Creating %s container", validatorImportCtName)
 	ct, err := dockerClient.ContainerCreate(context.Background(),
-		&container.Config{
-			Image:        validatorImage,
-			Cmd:          cmd,
-			AttachStdin:  true,
-			AttachStderr: true,
-			AttachStdout: true,
-			OpenStdin:    true,
-			Tty:          true,
-		},
+		containerConfig,
 		&container.HostConfig{
 			Mounts:      mounts,
 			VolumesFrom: []string{consensusCtName, validatorCtName},
@@ -330,11 +442,33 @@ func setupLodestarValidatorImport(dockerClient client.APIClient, dockerServiceMa
 		cmd = append(cmd, "--preset", preset)
 	}
 	log.Debugf("Creating %s container", validatorImportCtName)
-	ct, err := dockerClient.ContainerCreate(context.Background(),
-		&container.Config{
+	containerConfig := &container.Config{
+		Image: validatorImage,
+		Cmd:   cmd,
+	}
+	if options.Distributed {
+		containerConfig = &container.Config{
 			Image: validatorImage,
-			Cmd:   cmd,
-		},
+			Entrypoint: []string{
+				"sh", "-c", `
+				#!/bin/sh
+				set -e
+				for f in /keystore/validator_keys/keystore-*.json; do
+					echo "Importing key ${f}"
+					pwdfile="/keystore/$(basename "$f" .json).txt"
+					echo "Using password file ${pwdfile}"
+					# Import keystore with password.
+					node /usr/app/packages/cli/bin/lodestar validator import \
+						--dataDir="/data" \
+						--importKeystores="$f" \
+						--importKeystoresPassword="${pwdfile}"
+				done
+			`,
+			},
+		}
+	}
+	ct, err := dockerClient.ContainerCreate(context.Background(),
+		containerConfig,
 		&container.HostConfig{
 			Mounts:      mounts,
 			VolumesFrom: []string{validatorCtName},
@@ -515,6 +649,57 @@ func runAndWaitImportKeys(dockerClient client.APIClient, dockerServiceManager Do
 	}
 }
 
+func setupPrysmValidatorImportContainerDV(dockerClient client.APIClient, commandRunner commands.CommandRunner, serviceManager DockerServiceManager, options ImportValidatorKeysOptions) (string, error) {
+	var (
+		validatorCtName       = services.ContainerNameWithTag(services.DefaultSedgeValidatorClient, options.ContainerTag)
+		validatorImportCtName = services.ContainerNameWithTag(services.ServiceCtValidatorImport, options.ContainerTag)
+	)
+	// Init build context
+	contextDir, err := prysm.InitContext()
+	if err != nil {
+		return "", err
+	}
+	// Build image
+	buildCmd := commandRunner.BuildDockerBuildCMD(commands.DockerBuildOptions{
+		Path: contextDir,
+		Tag:  "sedge/prysm-import-teku",
+		Args: map[string]string{
+			"NETWORK":       options.Network,
+			"PRYSM_VERSION": configs.ClientImages.Validator.Prysm.String(),
+		},
+	})
+	log.Infof(configs.RunningCommand, buildCmd.Cmd)
+	if _, _, err := commandRunner.RunCMD(buildCmd); err != nil {
+		return "", err
+	}
+	// Mounts
+	mounts := []mount.Mount{
+		{
+			Type:   mount.TypeBind,
+			Source: options.From,
+			Target: "/keystore",
+		},
+	}
+	log.Debugf("Creating %s container", validatorImportCtName)
+	containerConfig := &container.Config{
+		Image: "sedge/prysm-import-teku",
+	}
+	ct, err := dockerClient.ContainerCreate(context.Background(),
+		containerConfig,
+		&container.HostConfig{
+			Mounts:      mounts,
+			VolumesFrom: []string{validatorCtName},
+		},
+		&network.NetworkingConfig{},
+		&v1.Platform{},
+		validatorImportCtName,
+	)
+	if err != nil {
+		return "", err
+	}
+	return ct.ID, nil
+}
+
 // runAndWaitImportKeysNimbus starts the container in interactive mode and waits for it to finish.
 func runAndWaitImportKeysNimbus(dockerClient client.APIClient, dockerServiceManager DockerServiceManager, ctID string) error {
 	log.Debugf("Starting interactive container with id: %s", ctID)