Merge pull request #361 from OffchainLabs/validator

node can act as Validator

Dockerfile

@@ -67,11 +67,6 @@ COPY --from=contracts-builder workspace/solgen/build/contracts/src/precompiles/
 COPY --from=contracts-builder workspace/.make/ .make/
 RUN PATH="$PATH:/usr/local/go/bin" make build-wasm-bin
 
-FROM scratch as machine-exporter
-COPY --from=wasm-libs-builder /workspace/target/machine/ machine/
-COPY --from=wasm-bin-builder /workspace/target/machine/ machine/
-
-
 FROM rust:1.57-slim-bullseye as prover-header-builder
 WORKDIR /workspace
 RUN export DEBIAN_FRONTEND=noninteractive && \
@@ -104,30 +99,35 @@ RUN touch -a -m arbitrator/prover/src/lib.rs && \
 FROM scratch as prover-export
 COPY --from=prover-builder /workspace/target/ /
 
-FROM golang:1.17-bullseye as replay-env-builder
+FROM debian:bullseye-slim as module-root-calc
 WORKDIR /workspace
 RUN export DEBIAN_FRONTEND=noninteractive && \
     apt-get update && \
     apt-get install -y wabt
-COPY go.mod go.sum ./
-COPY go-ethereum/go.mod go-ethereum/go.sum go-ethereum/
-COPY fastcache/go.mod fastcache/go.sum fastcache/
-RUN go mod download
-COPY . ./
 COPY --from=prover-export / target/
 COPY --from=wasm-bin-builder /workspace/target/ target/
 COPY --from=wasm-bin-builder /workspace/.make/ .make/
 COPY --from=wasm-libs-builder /workspace/target/ target/
 COPY --from=wasm-libs-builder /workspace/arbitrator/wasm-libraries/ arbitrator/wasm-libraries/
 COPY --from=wasm-libs-builder /workspace/.make/ .make/
-RUN target/bin/prover target/machine/replay.wasm --output-module-root -l target/machine/wasi_stub.wasm -l target/machine/host_io.wasm -l target/machine/soft-float.wasm -l target/machine/go_stub.wasm -l target/machine/brotli.wasm  > target/machine/module_root
+RUN target/bin/prover target/machine/replay.wasm --output-module-root -l target/machine/wasi_stub.wasm -l target/machine/soft-float.wasm -l target/machine/go_stub.wasm -l target/machine/host_io.wasm -l target/machine/brotli.wasm  > target/machine/module_root
+
+FROM scratch as machine-export
+COPY --from=module-root-calc /workspace/target/machine/ /machine
 
-FROM replay-env-builder as node-builder
+
+FROM golang:1.17-bullseye as node-builder
+WORKDIR /workspace
 RUN export DEBIAN_FRONTEND=noninteractive && \
     apt-get update && \
-    apt-get install -y protobuf-compiler
+    apt-get install -y protobuf-compiler wabt
 RUN go install google.golang.org/protobuf/cmd/[email protected] && \
     go install google.golang.org/grpc/cmd/[email protected]
+COPY go.mod go.sum ./
+COPY go-ethereum/go.mod go-ethereum/go.sum go-ethereum/
+COPY fastcache/go.mod fastcache/go.sum fastcache/
+RUN go mod download
+COPY . ./
 COPY --from=contracts-builder workspace/solgen/build/ solgen/build/
 COPY --from=contracts-builder workspace/.make/ .make/
 COPY --from=prover-header-export / target/
@@ -140,5 +140,10 @@ RUN go build -o ./target/bin/node ./cmd/node
 RUN go build -o ./target/bin/deploy ./cmd/deploy
 
 FROM debian:bullseye-slim as nitro-node
+WORKDIR /workspace
+RUN export DEBIAN_FRONTEND=noninteractive && \
+    apt-get update && \
+    apt-get install -y wabt
 COPY --from=node-builder /workspace/target/ target/
+COPY --from=machine-export / target/
 ENTRYPOINT [ "./target/bin/node" ]

cmd/node/node.go

@@ -12,7 +12,10 @@ import (
 	"io/ioutil"
 	"math/big"
 	"os"
+	"path"
+	"path/filepath"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/ethereum/go-ethereum/accounts"
@@ -33,6 +36,7 @@ import (
 	"github.com/offchainlabs/nitro/das"
 	"github.com/offchainlabs/nitro/statetransfer"
 	"github.com/offchainlabs/nitro/util"
+	"github.com/offchainlabs/nitro/validator"
 	"github.com/offchainlabs/nitro/wsbroadcastserver"
 )
 
@@ -44,7 +48,6 @@ func main() {
 	l1conn := flag.String("l1conn", "", "l1 connection (required unless no l1 listener)")
 
 	l1sequencer := flag.Bool("l1sequencer", false, "act and post to l1 as sequencer")
-	l1role := flag.String("l1role", "none", "either sequencer, listener, or none")
 	l1keystore := flag.String("l1keystore", "", "l1 private key store (required if l1role == sequencer)")
 	l1Account := flag.String("l1Account", "", "l1 seq account to use (default is first account in keystore)")
 	l1passphrase := flag.String("l1passphrase", "passphrase", "l1 private key file passphrase (1required if l1role == sequencer)")
@@ -87,8 +90,12 @@ func main() {
 	validatorstrategy := flag.String("validatorstrategy", "watchtower", "L1 validator strategy, either watchtower, defensive, stakeLatest, or makeNodes (requires l1role=validator)")
 	l1validatorwithoutblockvalidator := flag.Bool("UNSAFEl1validatorwithoutblockvalidator", false, "DANGEROUS! allows running an L1 validator without a block validator")
 	stakerinterval := flag.Duration("stakerinterval", time.Minute, "how often the L1 validator should check the status of the L1 rollup and maybe take action with its stake")
+	wasmrootpath := flag.String("wasmrootpath", "", "path to wasm files (replay.wasm, wasi_stub.wasm, soft-float.wasm, go_stub.wasm, host_io.wasm, brotli.wasm)")
+	wasmmoduleroot := flag.String("wasmmoduleroot", "", "wasm module root (if empty, read from <wasmrootpath>/module_root)")
+	wasmcachepath := flag.String("wasmcachepath", "", "path for cache of wasm machines")
 
 	flag.Parse()
+	ctx := context.Background()
 
 	glogger := log.NewGlogHandler(log.StreamHandler(os.Stderr, log.TerminalFormat(false)))
 	glogger.Verbosity(log.Lvl(*loglevel))
@@ -98,7 +105,6 @@ func main() {
 	l2ChainId := new(big.Int).SetUint64(*l2ChainIdUint)
 
 	nodeConf := arbnode.NodeConfigDefault
-	log.Info("Running Arbitrum node with", "role", *l1role)
 	if *nol1Listener {
 		nodeConf.L1Reader = false
 		nodeConf.Sequencer = true // we sequence messages, but not to l1
@@ -145,22 +151,57 @@ func main() {
 		panic("dataavailability.mode not recognized")
 	}
 
-	if *l1validator {
-		if !nodeConf.L1Reader {
-			flag.Usage()
-			panic("l1validator requires l1role other than \"none\"")
+	if *wasmrootpath != "" {
+		validator.StaticNitroMachineConfig.RootPath = *wasmrootpath
+	} else {
+		execfile, err := os.Executable()
+		if err != nil {
+			panic(err)
 		}
+		targetDir := filepath.Dir(filepath.Dir(execfile))
+		validator.StaticNitroMachineConfig.RootPath = filepath.Join(targetDir, "machine")
+	}
+
+	wasmModuleRootString := *wasmmoduleroot
+	if wasmModuleRootString == "" {
+		fileToRead := path.Join(validator.StaticNitroMachineConfig.RootPath, "module_root")
+		fileBytes, err := ioutil.ReadFile(fileToRead)
+		if err != nil {
+			if *l1deploy || (*l1validator && !*l1validatorwithoutblockvalidator) {
+				panic(fmt.Errorf("failed reading wasmModuleRoot from file, err %w", err))
+			}
+		}
+		wasmModuleRootString = strings.TrimSpace(string(fileBytes))
+		if len(wasmModuleRootString) > 64 {
+			wasmModuleRootString = wasmModuleRootString[0:64]
+		}
+	}
+	wasmModuleRoot := common.HexToHash(wasmModuleRootString)
+
+	if *l1validator {
 		nodeConf.L1Validator = true
 		nodeConf.L1ValidatorConfig.Strategy = *validatorstrategy
 		nodeConf.L1ValidatorConfig.StakerInterval = *stakerinterval
-		if !nodeConf.BlockValidator && !*l1validatorwithoutblockvalidator {
-			flag.Usage()
-			panic("L1 validator requires block validator to safely function")
+		if !*l1validatorwithoutblockvalidator {
+			nodeConf.BlockValidator = true
+			if *wasmcachepath != "" {
+				validator.StaticNitroMachineConfig.InitialMachineCachePath = *wasmcachepath
+			}
+			go func() {
+				expectedRoot := wasmModuleRoot
+				foundRoot, err := validator.GetInitialModuleRoot(ctx)
+				if err != nil {
+					panic(fmt.Errorf("failed reading wasmModuleRoot from machine: %w", err))
+				}
+				if foundRoot != expectedRoot {
+					panic(fmt.Errorf("incompatible wasmModuleRoot expected: %v found %v", expectedRoot, foundRoot))
+				} else {
+					log.Info("loaded wasm machine", "wasmModuleRoot", foundRoot)
+				}
+			}()
 		}
 	}
 
-	ctx := context.Background()
-
 	var l1client *ethclient.Client
 	var deployInfo arbnode.RollupAddresses
 	var l1TransactionOpts *bind.TransactOpts
@@ -185,7 +226,6 @@ func main() {
 				flag.Usage()
 				panic("deploy but not sequencer")
 			}
-			var wasmModuleRoot common.Hash
 			if nodeConf.BlockValidator {
 				// TODO actually figure out the wasmModuleRoot
 				panic("deploy as validator not yet supported")

docker-compose.yaml

@@ -20,11 +20,39 @@ services:
       - "seqdata:/data"
       - "l1keystore:/l1keystore"
       - "deploydata:/deploydata"
-    command: -datadir /data -dev -l1role sequencer -l1conn ws://geth:8546 -l1keystore /l1keystore -l1deployment /deploydata/deployment.json -httphost 0.0.0.0 -wshost 0.0.0.0 -UNSAFEl1validatorwithoutblockvalidator -l1validator -validatorstrategy MakeNodes -stakerinterval 10s -l1sequencer
+    command: -datadir /data -dev -l1conn ws://geth:8546 -l1keystore /l1keystore -l1deployment /deploydata/deployment.json -httphost 0.0.0.0 -wshost 0.0.0.0 -l1sequencer
     depends_on:
       - geth
+  staker-unsafe:
+    pid: host # allow debugging
+    build: .
+    ports:
+      - "7545"
+      - "7546"
+    volumes:
+      - "unsafestaker-data:/data"
+      - "l1keystore:/l1keystore"
+      - "deploydata:/deploydata"
+    command: -datadir /data -dev -l1conn ws://geth:8546 -l1keystore /l1keystore -l1deployment /deploydata/deployment.json -httphost 0.0.0.0 -wshost 0.0.0.0 -l1validator -validatorstrategy MakeNodes -stakerinterval 10s -forwardingtarget null -UNSAFEl1validatorwithoutblockvalidator
+    depends_on:
+      - sequencer
+  validator:
+    pid: host # allow debugging
+    build: .
+    ports:
+      - "7545"
+      - "7546"
+    volumes:
+      - "validator-data:/data"
+      - "l1keystore:/l1keystore"
+      - "deploydata:/deploydata"
+    command: -datadir /data -dev -l1conn ws://geth:8546 -l1keystore /l1keystore -l1deployment /deploydata/deployment.json -httphost 0.0.0.0 -wshost 0.0.0.0 -l1validator -validatorstrategy MakeNodes -stakerinterval 10s -forwardingtarget null
+    depends_on:
+      - sequencer
 volumes:
   l1data:
   l1keystore:
   seqdata:
+  unsafestaker-data:
+  validator-data:
   deploydata:

test-node.bash

@@ -12,11 +12,6 @@ if ! which docker-compose > /dev/null; then
     exit 1
 fi
 
-if [[ $# -gt 1 ]]; then
-    echo Error! One parameter max!
-    exit 1
-fi
-
 num_volumes=`docker volume ls --filter label=com.docker.compose.project=nitro -q | wc -l`
 
 if [[ $num_volumes -eq 0 ]]; then
@@ -26,6 +21,40 @@ else
 fi
 
 force_build=false
+validate=false
+while [[ $# -gt 0 ]]; do
+    case $1 in
+        --init)
+            if ! $force_init; then
+                echo == Warning! this will remove all previous data
+                read -p "are you sure? [y/n]" -n 1 response
+                if [[ $response == "y" ]] || [[ $response == "Y" ]]; then
+                    force_init=true
+                    echo
+                else
+                    exit 0
+                fi
+            fi
+            shift
+            ;;
+        --build)
+            force_build=true
+            shift
+            ;;
+        --validate)
+            validate=true
+            shift
+            ;;
+        *)
+            echo Usage: $0 \[OPTIONS..]
+            echo
+            echo OPTIONS:
+            echo --build:           rebuild docker image
+            echo --init:            remove all data, rebuild, deploy new rollup
+            echo --validate:        heavy computation, validating all blocks in WASM
+            exit 0
+    esac
+done
 
 if [[ $# -eq 1 ]]; then
     if [[ $1 == "--init" ]]; then
@@ -72,4 +101,9 @@ fi
 echo == Launching Sequencer
 echo if things go wrong - use --init to create a new chain
 echo
-docker-compose up sequencer
+if $validate; then
+    STAKER_NODE="validator"
+else
+    STAKER_NODE="staker-unsafe"
+fi
+docker-compose up sequencer $STAKER_NODE

validator/machine.go

@@ -42,6 +42,9 @@ func freeMachine(mach *ArbitratorMachine) {
 }
 
 func machineFromPointer(ptr *C.struct_Machine) *ArbitratorMachine {
+	if ptr == nil {
+		return nil
+	}
 	mach := &ArbitratorMachine{ptr: ptr}
 	runtime.SetFinalizer(mach, freeMachine)
 	return mach

validator/nitro_machine.go

@@ -43,10 +43,10 @@ type NitroMachineConfig struct {
 }
 
 var StaticNitroMachineConfig = NitroMachineConfig{
-	RootPath:                "./target/",
-	ProverBinPath:           "machine/replay.wasm",
-	ModulePaths:             []string{"machine/wasi_stub.wasm", "machine/soft-float.wasm", "machine/go_stub.wasm", "machine/host_io.wasm", "machine/brotli.wasm"},
-	InitialMachineCachePath: "etc/initial-machine-cache",
+	RootPath:                "./target/machine/",
+	ProverBinPath:           "replay.wasm",
+	ModulePaths:             []string{"wasi_stub.wasm", "soft-float.wasm", "go_stub.wasm", "host_io.wasm", "brotli.wasm"},
+	InitialMachineCachePath: "./target/etc/initial-machine-cache",
 }
 
 var zeroStepMachine staticMachineData
@@ -55,7 +55,7 @@ var hostIoMachine staticMachineData
 func init() {
 	_, thisfile, _, _ := runtime.Caller(0)
 	projectDir := filepath.Dir(filepath.Dir(thisfile))
-	StaticNitroMachineConfig.RootPath = filepath.Join(projectDir, "target")
+	StaticNitroMachineConfig.RootPath = filepath.Join(projectDir, "target/machine")
 
 	zeroStepMachine.chanSignal = make(chan struct{})
 	hostIoMachine.chanSignal = make(chan struct{})
@@ -66,10 +66,14 @@ func createZeroStepMachineInternal() {
 	for _, module := range StaticNitroMachineConfig.ModulePaths {
 		moduleList = append(moduleList, filepath.Join(StaticNitroMachineConfig.RootPath, module))
 	}
+	binPath := filepath.Join(StaticNitroMachineConfig.RootPath, StaticNitroMachineConfig.ProverBinPath)
 	cModuleList := CreateCStringList(moduleList)
-	cBinPath := C.CString(filepath.Join(StaticNitroMachineConfig.RootPath, StaticNitroMachineConfig.ProverBinPath))
-
+	cBinPath := C.CString(binPath)
+	log.Info("creating nitro machine", "binpath", binPath, "moduleList", moduleList)
 	baseMachine := C.arbitrator_load_machine(cBinPath, cModuleList, C.intptr_t(len(moduleList)))
+	if baseMachine == nil {
+		panic("failed to create base machine")
+	}
 	FreeCStringList(cModuleList, len(moduleList))
 	C.free(unsafe.Pointer(cBinPath))
 	zeroStepMachine.machine = machineFromPointer(baseMachine)
@@ -90,7 +94,7 @@ func createHostIoMachineInternal() {
 	machine := zerostep.Clone()
 	hash := machine.Hash()
 	expectedName := hash.String() + ".bin"
-	cacheDir := path.Join(StaticNitroMachineConfig.RootPath, StaticNitroMachineConfig.InitialMachineCachePath)
+	cacheDir := StaticNitroMachineConfig.InitialMachineCachePath
 	foundInCache := false
 	saveStateToFile := true
 	err = os.MkdirAll(cacheDir, 0o755)
@@ -195,12 +199,15 @@ func waitForMachine(ctx context.Context, machine *staticMachineData) (*Arbitrato
 	if machine.err != nil {
 		return nil, machine.err
 	}
-	if machine.ready {
-		return machine.machine, nil
-	}
 	if ctx.Err() != nil {
 		return nil, ctx.Err()
 	}
+	if machine.machine == nil {
+		return nil, errors.New("machine is nill")
+	}
+	if machine.ready {
+		return machine.machine, nil
+	}
 	return nil, errors.New("failed to get machine")
 }