Support go-rapidsnark in addition to snarkjs with a flag (Part 1)

Dockerfile

@@ -1,20 +1,14 @@
 # Build ES node in a stock Go builder container
-FROM golang:1.20-alpine as builder
-
-RUN apk add --no-cache gcc musl-dev linux-headers make
-
-# build
+FROM golang:1.21 as builder
 ADD . /es-node
 WORKDIR /es-node
 RUN make
 
 # Pull ES node into a second stage deploy alpine container
-FROM node:16-alpine
+FROM alpine:latest
 COPY --from=builder /es-node/build/ /es-node/build/
-
-# For zk proof
+RUN apk add --no-cache bash libstdc++ gcompat libgomp nodejs npm
 RUN npm install -g snarkjs
-RUN apk add --no-cache bash curl grep
 
 # Entrypoint
 COPY --from=builder /es-node/run.sh /es-node/

Makefile

@@ -9,11 +9,11 @@ LDFLAGSSTRING +=-X 'main.BuildTime=$(BUILDDATE)'
 LDFLAGS := -ldflags "$(LDFLAGSSTRING)"
 
 es-node: build
-	cp -r ethstorage/prover/snarkjs build/bin
+	cp -r ethstorage/prover/snark_lib build/bin
 	mkdir -p build/bin/snarkbuild
 
 build:
-	env GO111MODULE=on CGO_ENABLED=0 GOOS=$(TARGETOS) GOARCH=$(TARGETARCH) go build -v $(LDFLAGS) -o build/bin/es-node ./cmd/es-node/
+	env GO111MODULE=on CGO_ENABLED=0 GOOS=$(TARGETOS) GOARCH=$(TARGETARCH) go build -v $(LDFLAGS) -o build/bin/es-node -tags rapidsnark_asm ./cmd/es-node/
 
 clean:
 	rm -r build

README.md

@@ -19,4 +19,4 @@ Please join our [Discord](https://discord.com/invite/xhCwaMp7ps) for any technic
 
 # License
 
-This repository is licensed under [BLS](/LICENSE).
+This repository is licensed under [BLS](/LICENSE).

docker-compose.yaml

@@ -11,6 +11,6 @@ services:
     environment:
       - ES_NODE_STORAGE_MINER=${ES_NODE_STORAGE_MINER}
       - ES_NODE_SIGNER_PRIVATE_KEY=${ES_NODE_SIGNER_PRIVATE_KEY}
-    command: /es-node/run.sh
+    command:  ["bash", "-c", "/es-node/run.sh --miner.zk-prover-impl=2"]
     container_name: es
     tty: true

ethstorage/miner/cli.go

@@ -10,6 +10,7 @@ import (
 	"path/filepath"
 
 	"github.com/ethstorage/go-ethstorage/ethstorage/flags/types"
+	"github.com/ethstorage/go-ethstorage/ethstorage/prover"
 	"github.com/ethstorage/go-ethstorage/ethstorage/rollup"
 	"github.com/urfave/cli"
 )
@@ -21,6 +22,7 @@ const (
 	ZKeyFileNameFlagName     = "miner.zkey"
 	ZKWorkingDirFlagName     = "miner.zk-working-dir"
 	ZKProverModeFlagName     = "miner.zk-prover-mode"
+	ZKProverImplFlagName     = "miner.zk-prover-impl"
 	ThreadsPerShardFlagName  = "miner.threads-per-shard"
 	MinimumProfitFlagName    = "miner.min-profit"
 )
@@ -47,27 +49,33 @@ func CLIFlags(envPrefix string) []cli.Flag {
 		},
 		&types.BigFlag{
 			Name:   MinimumProfitFlagName,
-			Usage:  "Minimum profit for mining transactions",
+			Usage:  "Minimum profit for mining transactions in wei",
 			Value:  DefaultConfig.MinimumProfit,
 			EnvVar: rollup.PrefixEnvVar(envPrefix, "MIN_PROFIT"),
 		},
 		cli.StringFlag{
 			Name:   ZKeyFileNameFlagName,
-			Usage:  "zkey file name which should be put in the snarkjs folder",
+			Usage:  "zkey file name which should be put in the snark_lib folder",
 			Value:  DefaultConfig.ZKeyFileName,
 			EnvVar: rollup.PrefixEnvVar(envPrefix, "ZKEY_FILE"),
 		},
 		cli.StringFlag{
 			Name:   ZKWorkingDirFlagName,
-			Usage:  "Path to the snarkjs folder",
+			Usage:  "Path to the snark_lib folder",
 			Value:  DefaultConfig.ZKWorkingDir,
 			EnvVar: rollup.PrefixEnvVar(envPrefix, "ZK_WORKING_DIR"),
 		},
 		cli.Uint64Flag{
 			Name:   ZKProverModeFlagName,
-			Usage:  "ZK prover mode, 1: one proof per sample, 2: one proof for multiple samples. Default: 2",
+			Usage:  "ZK prover mode, 1: one proof per sample, 2: one proof for multiple samples",
 			Value:  DefaultConfig.ZKProverMode,
-			EnvVar: rollup.PrefixEnvVar(envPrefix, "ZK_PROVER_Mode"),
+			EnvVar: rollup.PrefixEnvVar(envPrefix, "ZK_PROVER_MODE"),
+		},
+		cli.Uint64Flag{
+			Name:   ZKProverImplFlagName,
+			Usage:  "ZK prover implementation, 1: snarkjs, 2: go-rapidsnark",
+			Value:  DefaultConfig.ZKProverImpl,
+			EnvVar: rollup.PrefixEnvVar(envPrefix, "ZK_PROVER_IMPL"),
 		},
 		cli.Uint64Flag{
 			Name:   ThreadsPerShardFlagName,
@@ -87,14 +95,15 @@ type CLIConfig struct {
 	ZKeyFileName     string
 	ZKWorkingDir     string
 	ZKProverMode     uint64
+	ZKProverImpl     uint64
 	ThreadsPerShard  uint64
 }
 
 func (c CLIConfig) Check() error {
-	info, err := os.Stat(filepath.Join(c.ZKWorkingDir, "snarkjs"))
+	info, err := os.Stat(filepath.Join(c.ZKWorkingDir, prover.SnarkLib))
 	if err != nil {
 		if os.IsNotExist(err) || !info.IsDir() {
-			return fmt.Errorf("snarkjs folder not found in ZKWorkingDir: %v", err)
+			return fmt.Errorf("%s folder not found in ZKWorkingDir: %v", prover.SnarkLib, err)
 		}
 	}
 	return nil
@@ -116,6 +125,7 @@ func (c CLIConfig) ToMinerConfig() (Config, error) {
 	cfg.MinimumProfit = c.MinimumProfit
 	cfg.ZKeyFileName = c.ZKeyFileName
 	cfg.ZKProverMode = c.ZKProverMode
+	cfg.ZKProverImpl = c.ZKProverImpl
 	cfg.ThreadsPerShard = c.ThreadsPerShard
 	return cfg, nil
 }
@@ -129,6 +139,7 @@ func ReadCLIConfig(ctx *cli.Context) CLIConfig {
 		ZKeyFileName:     ctx.GlobalString(ZKeyFileNameFlagName),
 		ZKWorkingDir:     ctx.GlobalString(ZKWorkingDirFlagName),
 		ZKProverMode:     ctx.GlobalUint64(ZKProverModeFlagName),
+		ZKProverImpl:     ctx.GlobalUint64(ZKProverImplFlagName),
 		ThreadsPerShard:  ctx.GlobalUint64(ThreadsPerShardFlagName),
 	}
 	return cfg

ethstorage/miner/config.go

@@ -32,6 +32,7 @@ type Config struct {
 	ZKeyFileName     string
 	ZKWorkingDir     string
 	ZKProverMode     uint64
+	ZKProverImpl     uint64
 	ThreadsPerShard  uint64
 	SignerFnFactory  signer.SignerFactory
 	SignerAddr       common.Address
@@ -50,6 +51,7 @@ var DefaultConfig = Config{
 	ZKeyFileName:     "blob_poseidon2.zkey",
 	ZKWorkingDir:     filepath.Join("build", "bin"),
 	ZKProverMode:     2,
+	ZKProverImpl:     1,
 	ThreadsPerShard:  uint64(2 * runtime.NumCPU()),
 	MinimumProfit:    common.Big0,
 }

ethstorage/miner/l1_mining_api.go

@@ -203,7 +203,7 @@ func (m *l1MiningAPI) SubmitMinedResult(ctx context.Context, contract common.Add
 	}
 	err = m.SendTransaction(ctx, signedTx)
 	if err != nil {
-		m.lg.Error("Send tx failed", "error", err)
+		m.lg.Error("Send tx failed", "txNonce", nonce, "gasPrice", gasPrice, "error", err)
 		return common.Hash{}, err
 	}
 	m.lg.Info("Submit mined result done", "shard", rst.startShardId, "block", rst.blockNumber,

ethstorage/miner/miner.go

@@ -24,6 +24,7 @@ type L1API interface {
 	GetMiningInfo(ctx context.Context, contract common.Address, shardIdx uint64) (*miningInfo, error)
 	SubmitMinedResult(ctx context.Context, contract common.Address, rst result, config Config) (common.Hash, error)
 	GetDataHashes(ctx context.Context, contract common.Address, kvIdxes []uint64) ([]common.Hash, error)
+	BlockNumber(ctx context.Context) (uint64, error)
 }
 
 type MiningProver interface {

ethstorage/miner/miner_test.go

@@ -61,11 +61,12 @@ func newMiner(t *testing.T, storageMgr *es.StorageManager, client *eth.PollingCl
 		PriorityGasPrice: new(big.Int).SetUint64(10),
 		ThreadsPerShard:  1,
 		ZKProverMode:     2,
+		ZKProverImpl:     1,
 		ZKeyFileName:     "blob_poseidon2.zkey",
 	}
 	l1api := NewL1MiningAPI(client, lg)
 	zkWorkingDir, _ := filepath.Abs("../prover")
-	pvr := prover.NewKZGPoseidonProver(zkWorkingDir, defaultConfig.ZKeyFileName, defaultConfig.ZKProverMode, lg)
+	pvr := prover.NewKZGPoseidonProver(zkWorkingDir, defaultConfig.ZKeyFileName, defaultConfig.ZKProverMode, defaultConfig.ZKProverImpl, lg)
 	fd := new(event.Feed)
 	miner := New(defaultConfig, storageMgr, l1api, &pvr, fd, lg)
 	return miner

ethstorage/miner/worker.go

@@ -21,13 +21,12 @@ import (
 )
 
 const (
-	chainHeadChanSize = 1
-	taskQueueSize     = 1
-	resultQueueSize   = 10
-	sampleSizeBits    = 5 // 32 bytes
-	// always use new block hash to mine for each slot
-	mineTimeOut              = 12 // seconds
-	miningTransactionTimeout = 25 // seconds
+	chainHeadChanSize        = 1
+	taskQueueSize            = 1
+	resultQueueSize          = 10
+	sampleSizeBits           = 5  // 32 bytes
+	slot                     = 12 // seconds
+	miningTransactionTimeout = 50 // seconds
 )
 
 var (
@@ -184,7 +183,7 @@ func (w *worker) newWorkLoop() {
 			if !w.isRunning() {
 				break
 			}
-			w.lg.Info("Updating tasks with L1 new head", "blockNumber", block.Number, "blockTime", block.Time, "now", uint64(time.Now().Unix()))
+			w.lg.Info("Updating tasks with L1 new head", "blockNumber", block.Number, "blockTime", block.Time, "blockHash", block.Hash, "now", uint64(time.Now().Unix()))
 			// TODO suspend mining if:
 			// 1) a mining tx is already submitted; or
 			// 2) if the last mining time is too close (the reward is not enough).
@@ -255,7 +254,7 @@ func (w *worker) updateDifficulty(shardIdx, blockTime uint64) (*big.Int, error)
 		w.lg.Warn("Failed to get es mining info", "error", err.Error())
 		return nil, err
 	}
-	w.lg.Info("Mining info retrieved", "shard", shardIdx, "LastMineTime", info.LastMineTime, "Difficulty", info.Difficulty, "proofsSubmitted", info.BlockMined)
+	w.lg.Info("Mining info retrieved", "shard", shardIdx, "lastMineTime", info.LastMineTime, "difficulty", info.Difficulty, "proofsSubmitted", info.BlockMined)
 	reqDiff := new(big.Int).Div(maxUint256, expectedDiff(
 		info.LastMineTime,
 		blockTime,
@@ -329,6 +328,19 @@ func (w *worker) resultLoop() {
 				continue
 			}
 			w.lg.Info("Mining result loop get result", "shard", result.startShardId, "block", result.blockNumber, "nonce", result.nonce)
+			// check block number to avoid `blockNumber == block.number` in the contract, loop for 5 times in case of empty slots
+			for i := 0; i < 5; i++ {
+				latest, err := w.l1API.BlockNumber(context.Background())
+				if err != nil {
+					w.lg.Warn("Failed to get L1 block number", "error", err.Error())
+					continue
+				}
+				if latest > result.blockNumber.Uint64() {
+					break
+				}
+				w.lg.Info("Will wait a slot for the block number to increase")
+				time.Sleep(slot * time.Second)
+			}
 			txHash, err := w.l1API.SubmitMinedResult(
 				context.Background(),
 				w.storageMgr.ContractAddress(),
@@ -444,7 +456,8 @@ func (w *worker) mineTask(t *taskItem) (bool, error) {
 	nonce := t.nonceStart
 	w.lg.Debug("Mining task started", "shard", t.shardIdx, "thread", t.thread, "block", t.blockNumber, "nonces", fmt.Sprintf("%d~%d", t.nonceStart, t.nonceEnd))
 	for w.isRunning() {
-		if time.Since(startTime).Seconds() > mineTimeOut {
+		// always use new randao to mine for each slot
+		if time.Since(startTime).Seconds() > slot {
 			if t.thread == 0 {
 				nonceTriedTotal := (nonce - t.nonceStart) * w.config.ThreadsPerShard
 				w.lg.Warn("Mining tasks timed out", "shard", t.shardIdx, "block", t.blockNumber,

ethstorage/node/node.go

@@ -252,6 +252,7 @@ func (n *EsNode) initMiner(ctx context.Context, cfg *Config) error {
 		cfg.Mining.ZKWorkingDir,
 		cfg.Mining.ZKeyFileName,
 		cfg.Mining.ZKProverMode,
+		cfg.Mining.ZKProverImpl,
 		n.log,
 	)
 	n.miner = miner.New(cfg.Mining, n.storageManager, l1api, &pvr, n.feed, n.log)

ethstorage/prover/README.md

@@ -27,4 +27,50 @@ Go-rapidsnark also provides the go version of the witness calculator, so that ou
 
 We also need to investigate the gnark to decide whether we want to choose it or rapidsnark, but at least we have a better alternative than snarkjs
 
-source: https://github.com/ethstorage/go-ethstorage/pull/14#issuecomment-1590816080
+source: https://github.com/ethstorage/go-ethstorage/pull/14#issuecomment-1590816080
+
+
+
+### Performance updates
+
+1. Tests on MacOS amd64 show no performance gain of go-rapidsnark.
+
+ Implementation  | Proof per sample (mode 1) | Proof for 2 samples (mode 2) 
+-- | -- | -- 
+snarkjs | 43.8  |  60.1
+go-rapidsnark + wazero.Circom2WZWitnessCalculator |  47.1 | 82.3
+go-rapidsnark + wasmer.Circom2WitnessCalculator | 49.5 | 101.8
+
+Note rapidsnark supports 2 witness calculator implementations which `wazero` wins, so `wazero` is used with go-rapidsnark in below tests.
+
+2. Tests on MacOS arm64 (m3) show a slight performance gain of go-rapidsnark.
+
+ Implementation  | Proof per sample (mode 1) | Proof for 2 samples (mode 2) 
+-- | -- | -- 
+snarkjs |  16.2 | 31.7
+go-rapidsnark| 15.1 | 28.0 
+
+3. Ubuntu (AX101)
+
+ Implementation  | Proof per sample (mode 1) | Proof for 2 samples (mode 2) 
+-- | -- | -- 
+snarkjs | 12.6  |  24.4
+go-rapidsnark |  5.7 | 14.1
+
+4. With `-tags rapidsnark_asm` AX101 has an obvious improvement, esp. mode2：
+
+ Implementation  | Proof per sample (mode 1) | Proof for 2 samples (mode 2) 
+-- | -- | -- 
+go-rapidsnark |  5.7 | 14.1
+go-rapidsnark + `-tags rapidsnark_asm` |  4.6 | 8.2
+
+But this is not the case for Mac. MacOS build has the optimization enabled according to [the go-rapidsnark doc](https://github.com/iden3/go-rapidsnark/tree/main/prover#performance-optimization-on-x86_64-hardware).
+
+5. Tests on MacOS amd64, comparing to rapidsnark (c++)
+
+Implementation|Proof per sample (mode 1)|Proof for 2 samples (mode 2)
+-- | -- | --
+snarkjs | 43.8 | 60.1 |
+go-rapidsnark| 47.1 | 82.3 | 
+rapidsnark| 8.2 | 16.7 |
+