From 8fbb779a7b7f90344d50ec2b7dc46338b82f1728 Mon Sep 17 00:00:00 2001 From: syntrust Date: Fri, 12 Jan 2024 16:53:49 +0800 Subject: [PATCH 01/33] test 2 shards --- integration_tests/common.go | 2 +- integration_tests/node_mine_test.go | 78 +++++++++++------------------ 2 files changed, 29 insertions(+), 51 deletions(-) diff --git a/integration_tests/common.go b/integration_tests/common.go index 002a7f87..10128736 100644 --- a/integration_tests/common.go +++ b/integration_tests/common.go @@ -21,7 +21,7 @@ var ( privateKey = os.Getenv("ES_NODE_SIGNER_PRIVATE_KEY") minerAddr = common.HexToAddress(os.Getenv("ES_NODE_STORAGE_MINER")) lg = esLog.NewLogger(esLog.CLIConfig{ - Level: "debug", + Level: "info", Format: "text", Color: term.IsTerminal(int(os.Stdout.Fd())), }) diff --git a/integration_tests/node_mine_test.go b/integration_tests/node_mine_test.go index 55315bc4..bf9e2ef3 100644 --- a/integration_tests/node_mine_test.go +++ b/integration_tests/node_mine_test.go @@ -8,11 +8,9 @@ package integration import ( "context" "fmt" - "io/ioutil" "math/big" "os" "path/filepath" - "strings" "sync" "testing" "time" @@ -35,8 +33,7 @@ const ( dataFileName = "shard-%d.dat" ) -// TODO: test 2 shards -var shardIds = []uint64{0} +var shardIds = []uint64{0, 1} func TestMining(t *testing.T) { contract := l1Contract @@ -63,7 +60,6 @@ func TestMining(t *testing.T) { storConfig.Filenames = files miningConfig := initMiningConfig(t, contract, pClient) lg.Info("Initialzed mining config", "miningConfig", fmt.Sprintf("%+v", miningConfig)) - defer cleanFiles(miningConfig.ZKWorkingDir) shardManager, err := initShardManager(*storConfig) if err != nil { t.Fatalf("init shard manager error: %v", err) @@ -98,12 +94,24 @@ func TestMining(t *testing.T) { lg.Error("L1 heads subscription error", "err", err) }() prepareData(t, pClient, storageManager, miningConfig.StorageCost.String()) - // fillEmpty(t, pClient, storageManager) + fillEmpty(t, pClient, storageManager) mnr.Start() var wg sync.WaitGroup + minedShardSig := make(chan uint64, len(shardIds)) minedShardCh := make(chan uint64) - for _, s := range shardIds { + minedShards := make(map[uint64]bool) + go func() { + for minedShard := range minedShardCh { + minedShardSig <- minedShard + lg.Info("Mined shard", "shard", minedShard) + if !minedShards[minedShard] { + minedShards[minedShard] = true + wg.Done() + } + } + }() + for i, s := range shardIds { feed.Send(protocol.EthStorageSyncDone{ DoneType: protocol.SingleShardDone, ShardId: s, @@ -118,54 +126,24 @@ func TestMining(t *testing.T) { } go waitForMined(l1api, contract, mnr.ChainHeadCh, s, info.BlockMined.Uint64(), minedShardCh) wg.Add(1) - time.Sleep(360 * time.Second) - } - - go func() { - minedShards := make(map[uint64]bool) - for minedShard := range minedShardCh { - if !minedShards[minedShard] { - lg.Info("Mined shard", "shard", minedShard) - minedShards[minedShard] = true - wg.Done() - lg.Info("wait group done") + // defer next shard mining so that the started shard can be mined for a while + if i != len(shardIds)-1 { + var miningTime time.Duration = 600 + timeout := time.After(miningTime * time.Second) + select { + case minedShard := <-minedShardSig: + lg.Info(fmt.Sprintf("Shard %d successfully mined, will start next: shard %d", minedShard, i+1)) + case <-timeout: + lg.Info(fmt.Sprintf("Shard %d has been mined for %ds, will start next shard", i, miningTime)) } } - }() - lg.Info("wait group waiting") + } wg.Wait() l1HeadsSub.Unsubscribe() mnr.Close() close() } -func cleanFiles(proverDir string) { - for _, shardId := range shardIds { - fileName := fmt.Sprintf(dataFileName, shardId) - if _, err := os.Stat(fileName); !os.IsNotExist(err) { - err = os.Remove(fileName) - if err != nil { - fmt.Println(err) - } - } - } - - folderPath := filepath.Join(proverDir, "snarkbuild") - files, err := ioutil.ReadDir(folderPath) - if err != nil { - fmt.Println(err) - return - } - for _, file := range files { - if !strings.HasPrefix(file.Name(), ".") { - err = os.RemoveAll(filepath.Join(folderPath, file.Name())) - if err != nil { - fmt.Println(err) - } - } - } -} - func waitForMined(l1api miner.L1API, contract common.Address, chainHeadCh chan eth.L1BlockRef, shardIdx, lastMined uint64, exitCh chan uint64) { for range chainHeadCh { info, err := l1api.GetMiningInfo( @@ -230,11 +208,11 @@ func initShardManager(storConfig storage.StorageConfig) (*ethstorage.ShardManage func fillEmpty(t *testing.T, l1Client *eth.PollingClient, storageMgr *ethstorage.StorageManager) { lg.Info("Filling empty started") - totalBlobs := storageMgr.KvEntries() * uint64(len(shardIds)) + totalEntries := storageMgr.KvEntries() * uint64(len(shardIds)) lastKvIdx := storageMgr.LastKvIndex() - lg.Info("Filling empty", "lastBlobIdx", lastKvIdx, "totalBlobs", totalBlobs) + lg.Info("Filling empty", "lastKvIdx", lastKvIdx, "totalBlobs", totalEntries) - inserted, next, err := storageMgr.CommitEmptyBlobs(lastKvIdx, totalBlobs-1) + inserted, next, err := storageMgr.CommitEmptyBlobs(lastKvIdx, totalEntries-1) if err != nil { t.Fatalf("Commit empty blobs failed %v", err) } From 881fd786146d6016dcb07a1ec67d6220702eb897 Mon Sep 17 00:00:00 2001 From: syntrust Date: Fri, 12 Jan 2024 17:18:44 +0800 Subject: [PATCH 02/33] merge --- ethstorage/metrics/metrics.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/ethstorage/metrics/metrics.go b/ethstorage/metrics/metrics.go index 9cedfffe..bc0a4184 100644 --- a/ethstorage/metrics/metrics.go +++ b/ethstorage/metrics/metrics.go @@ -422,6 +422,15 @@ func NewMetrics(procName string) *Metrics { Help: "1 if the es node has finished starting up", }), + BandwidthTotal: factory.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: ns, + Subsystem: "p2p", + Name: "bandwidth_bytes_total", + Help: "P2P bandwidth by direction", + }, []string{ + "direction", + }), + registry: registry, factory: factory, From e46f6e5bcdecdfd9cb4ae8832192e337094f4002 Mon Sep 17 00:00:00 2001 From: syntrust Date: Mon, 15 Jan 2024 11:59:23 +0800 Subject: [PATCH 03/33] start miner early --- integration_tests/node_mine_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/integration_tests/node_mine_test.go b/integration_tests/node_mine_test.go index bf9e2ef3..430919ce 100644 --- a/integration_tests/node_mine_test.go +++ b/integration_tests/node_mine_test.go @@ -93,9 +93,10 @@ func TestMining(t *testing.T) { } lg.Error("L1 heads subscription error", "err", err) }() + mnr.Start() + prepareData(t, pClient, storageManager, miningConfig.StorageCost.String()) fillEmpty(t, pClient, storageManager) - mnr.Start() var wg sync.WaitGroup minedShardSig := make(chan uint64, len(shardIds)) From f706d4dbb5d7944f052d84813533dc9b9e5c93eb Mon Sep 17 00:00:00 2001 From: syntrust Date: Wed, 17 Jan 2024 10:14:12 +0800 Subject: [PATCH 04/33] use unique key --- integration_tests/kzg_prover_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/integration_tests/kzg_prover_test.go b/integration_tests/kzg_prover_test.go index 41cab3a0..353c5fa6 100644 --- a/integration_tests/kzg_prover_test.go +++ b/integration_tests/kzg_prover_test.go @@ -11,6 +11,7 @@ import ( "math/big" "os" "testing" + "time" "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/accounts/abi" @@ -98,7 +99,7 @@ func uploadBlob(t *testing.T, data []byte) common.Hash { if err != nil { t.Fatalf("Error getting nonce: %v", err) } - blbKey := "0x0000000000000000000000000000000000000000000000000000000000000001" + blbKey := crypto.Keccak256Hash(new(big.Int).SetInt64(time.Now().UnixNano()).Bytes()) blbIdx := common.Big0 length := new(big.Int).SetInt64(128 * 1024) @@ -111,7 +112,7 @@ func uploadBlob(t *testing.T, data []byte) common.Hash { {Type: uint256Type}, {Type: uint256Type}, } - values := []interface{}{common.HexToHash(blbKey), blbIdx, length} + values := []interface{}{blbKey, blbIdx, length} dataField, err := args.Pack(values...) if err != nil { t.Fatalf("Error getting calldata: %v", err) From f76b3f44ad1d07c6ac858e9c379c06cffbf62d53 Mon Sep 17 00:00:00 2001 From: syntrust Date: Mon, 22 Jan 2024 12:16:30 +0800 Subject: [PATCH 05/33] remove useless code --- integration_tests/zk_prover_test.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/integration_tests/zk_prover_test.go b/integration_tests/zk_prover_test.go index 182eaa86..638141e1 100644 --- a/integration_tests/zk_prover_test.go +++ b/integration_tests/zk_prover_test.go @@ -29,9 +29,8 @@ import ( var zkp1Contract = common.HexToAddress(os.Getenv("ES_NODE_STORAGE_L1CONTRACT_ZKP1")) const ( - l1ContractV1 = "0xc3208C27285ed9516F21a89053326Bb895DD78F7" - prPath = "../ethstorage/prover" - zkeyName = "blob_poseidon.zkey" + prPath = "../ethstorage/prover" + zkeyName = "blob_poseidon.zkey" ) func TestZKProver_GenerateZKProofPerSample(t *testing.T) { From c8b8888a5dc7cbecabcc05c78d980443ea0bce95 Mon Sep 17 00:00:00 2001 From: syntrust Date: Tue, 23 Jan 2024 15:06:53 +0800 Subject: [PATCH 06/33] mock mining process --- integration_tests/node_mine_test.go | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/integration_tests/node_mine_test.go b/integration_tests/node_mine_test.go index dc41a1d4..131f202c 100644 --- a/integration_tests/node_mine_test.go +++ b/integration_tests/node_mine_test.go @@ -94,8 +94,6 @@ func TestMining(t *testing.T) { lg.Error("L1 heads subscription error", "err", err) }() mnr.Start() - prepareData(t, pClient, storageManager, miningConfig.StorageCost.String()) - fillEmpty(t, pClient, storageManager) var wg sync.WaitGroup minedShardSig := make(chan uint64, len(shardIds)) minedShardCh := make(chan uint64) @@ -111,6 +109,14 @@ func TestMining(t *testing.T) { } }() for i, s := range shardIds { + kvs := shardManager.KvEntries() + go func() { + fillEmpty(t, storageManager, kvs) + for j := 0; j < 3; j++ { + time.Sleep(3 * time.Minute) + prepareData(t, pClient, storageManager, miningConfig.StorageCost.String(), kvs/3) + } + }() feed.Send(protocol.EthStorageSyncDone{ DoneType: protocol.SingleShardDone, ShardId: s, @@ -131,9 +137,9 @@ func TestMining(t *testing.T) { timeout := time.After(miningTime * time.Second) select { case minedShard := <-minedShardSig: - lg.Info(fmt.Sprintf("Shard %d successfully mined, will start next: shard %d", minedShard, i+1)) + lg.Info(fmt.Sprintf("Shard %d successfully mined, will start next shard: %d", minedShard, i+1)) case <-timeout: - lg.Info(fmt.Sprintf("Shard %d has been mined for %ds, will start next shard", i, miningTime)) + lg.Info(fmt.Sprintf("Shard %d has been mined for %ds, will start next shard: %d", i, miningTime, i+1)) } } } @@ -205,21 +211,18 @@ func initShardManager(storConfig storage.StorageConfig) (*ethstorage.ShardManage return shardManager, nil } -func fillEmpty(t *testing.T, l1Client *eth.PollingClient, storageMgr *ethstorage.StorageManager) { - lg.Info("Filling empty started") - totalEntries := storageMgr.KvEntries() * uint64(len(shardIds)) +func fillEmpty(t *testing.T, storageMgr *ethstorage.StorageManager, entriesToFill uint64) { lastKvIdx := storageMgr.LastKvIndex() - lg.Info("Filling empty", "lastKvIdx", lastKvIdx, "totalBlobs", totalEntries) - - inserted, next, err := storageMgr.CommitEmptyBlobs(lastKvIdx, totalEntries-1) + lg.Info("Filling empty", "lastKvIdx", lastKvIdx, "entriesToFill", entriesToFill) + inserted, next, err := storageMgr.CommitEmptyBlobs(lastKvIdx, entriesToFill+lastKvIdx-1) if err != nil { t.Fatalf("Commit empty blobs failed %v", err) } lg.Info("Filling empty done", "inserted", inserted, "next", next) } -func prepareData(t *testing.T, l1Client *eth.PollingClient, storageMgr *ethstorage.StorageManager, value string) { - data := generateRandomContent(128 * 10) +func prepareData(t *testing.T, l1Client *eth.PollingClient, storageMgr *ethstorage.StorageManager, value string, blobLen uint64) { + data := generateRandomContent(128 * int(blobLen)) blobs := utils.EncodeBlobs(data) t.Logf("Blobs len %d \n", len(blobs)) var hashs []common.Hash From a2ac36c1337bb8f2e48be75eb341a5c38fc685c5 Mon Sep 17 00:00:00 2001 From: syntrust Date: Tue, 7 May 2024 17:17:41 +0800 Subject: [PATCH 07/33] use lastMineTime to check shard exist --- cmd/es-node/main.go | 8 ++++---- cmd/es-node/utils.go | 20 ++++++++++++++++++-- integration_tests/common.go | 2 +- 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/cmd/es-node/main.go b/cmd/es-node/main.go index 75743e08..d88a9d18 100644 --- a/cmd/es-node/main.go +++ b/cmd/es-node/main.go @@ -235,13 +235,13 @@ func EsNodeInit(ctx *cli.Context) error { for i := 0; i < len(shardIndexes); i++ { shard := uint64(shardIndexes[i]) if shard > 0 { - diff, err := getDifficulty(cctx, client, l1Contract, shard) + timeStamp, err := getLastMineTime(cctx, client, l1Contract, shard) if err != nil { log.Error("Failed to get shard info from contract", "error", err) return err } - if diff != nil && diff.Cmp(big.NewInt(0)) == 0 { - return fmt.Errorf("Shard not exist: %d", shard) + if timeStamp != nil && timeStamp.Cmp(big.NewInt(0)) == 0 { + return fmt.Errorf("shard not exist: %d", shard) } } shardIdxList = append(shardIdxList, shard) @@ -254,7 +254,7 @@ func EsNodeInit(ctx *cli.Context) error { return err } if len(shardList) == 0 { - return fmt.Errorf("No shard indexes found") + return fmt.Errorf("no shard indexes found") } shardIdxList = shardList } diff --git a/cmd/es-node/utils.go b/cmd/es-node/utils.go index e47c29e7..501fad4a 100644 --- a/cmd/es-node/utils.go +++ b/cmd/es-node/utils.go @@ -117,6 +117,22 @@ func getShardList(ctx context.Context, client *ethclient.Client, contract common } func getDifficulty(ctx context.Context, client *ethclient.Client, contract common.Address, shardIdx uint64) (*big.Int, error) { + res, err := getMiningInfo(ctx, client, contract, shardIdx) + if err != nil { + return nil, err + } + return res[1].(*big.Int), nil +} + +func getLastMineTime(ctx context.Context, client *ethclient.Client, contract common.Address, shardIdx uint64) (*big.Int, error) { + res, err := getMiningInfo(ctx, client, contract, shardIdx) + if err != nil { + return nil, err + } + return res[0].(*big.Int), nil +} + +func getMiningInfo(ctx context.Context, client *ethclient.Client, contract common.Address, shardIdx uint64) ([]interface{}, error) { uint256Type, _ := abi.NewType("uint256", "", nil) dataField, _ := abi.Arguments{{Type: uint256Type}}.Pack(new(big.Int).SetUint64(shardIdx)) h := crypto.Keccak256Hash([]byte(`infos(uint256)`)) @@ -136,10 +152,10 @@ func getDifficulty(ctx context.Context, client *ethclient.Client, contract commo {Type: uint256Type}, }.UnpackValues(bs) if res == nil || len(res) < 3 { - log.Error("Query difficulty by shard", "error", "invalid result", "result", res) + log.Error("Query mining info by shard", "error", "invalid result", "result", res) return nil, fmt.Errorf("invalid result: %v", res) } - return res[1].(*big.Int), nil + return res, nil } func createDataFile(cfg *storage.StorageConfig, shardIdxList []uint64, datadir string, encodingType int) ([]string, error) { diff --git a/integration_tests/common.go b/integration_tests/common.go index 9985c84b..d1e2d8a6 100644 --- a/integration_tests/common.go +++ b/integration_tests/common.go @@ -22,7 +22,7 @@ var ( minerAddr = common.HexToAddress(os.Getenv("ES_NODE_STORAGE_MINER")) prPath = "../ethstorage/prover" lg = esLog.NewLogger(esLog.CLIConfig{ - Level: "info", + Level: "debug", Format: "text", Color: term.IsTerminal(int(os.Stdout.Fd())), }) From 5e8b9668dcf6b6843cad82e8acccfaea60c2ad0e Mon Sep 17 00:00:00 2001 From: syntrust Date: Thu, 9 May 2024 14:57:36 +0800 Subject: [PATCH 08/33] support multiple shards --- run.sh | 212 +++++++++++++++++++++++++++++++++------------------------ 1 file changed, 122 insertions(+), 90 deletions(-) diff --git a/run.sh b/run.sh index ba83d855..b316f12b 100755 --- a/run.sh +++ b/run.sh @@ -25,99 +25,78 @@ if [ ${#ES_NODE_SIGNER_PRIVATE_KEY} -ne 64 ]; then exit 1 fi -# ZK prover mode, 1: one proof per sample, 2: one proof for multiple samples. -zkp_mode= -i=1 -while [ $i -le $# ]; do - if [ "${!i}" = "--miner.zk-prover-mode" ]; then - j=$((i+1)) - zkp_mode="${!j}" - break - else - if echo "${!i}" | grep -qE -- "--miner\.zk-prover-mode=([0-9]+)"; then - zkp_mode=$(echo "${!i}" | sed -E 's/.*=([0-9]+)/\1/') +# function to extract a value from command-line arguments +extract_value() { + local arg_name="$1" + local arg_value="" + local i=1 + local skip=0 + + while [ $i -le $# ]; do + if [ "${!i}" = "$arg_name" ]; then + if [ $skip = 0 ]; then + skip=1 + else + j=$((i+1)) + arg_value="${!j}" break + fi + elif echo "${!i}" | grep -qE -- "$arg_name=(.*)"; then + arg_value=$(echo ${!i} | cut -d'=' -f2) + break fi - fi - i=$((i+1)) -done - -if [ -n "$zkp_mode" ] && [ "$zkp_mode" != 1 ] && [ "$zkp_mode" != 2 ]; then - echo "Error: zk prover mode can only be 1 or 2." - exit 1 -fi - -if [ -n "$zkp_mode" ]; then - echo "The zk prover mode has been overridden to $zkp_mode" -fi - -# download zkey if not yet -zkey_name="blob_poseidon2.zkey" -zkey_size=560301223 -zkey_url="https://es-node-zkey.s3.us-west-1.amazonaws.com/blob_poseidon2_testnet1.zkey" -if [ "$zkp_mode" = 1 ]; then - zkey_name="blob_poseidon.zkey" - zkey_size=280151245 - zkey_url="https://drive.usercontent.google.com/download?id=1ZLfhYeCXMnbk6wUiBADRAn1mZ8MI_zg-&export=download&confirm=t&uuid=16ddcd58-2498-4d65-8931-934df3d0065c" -fi -zkey_file="./build/bin/snark_lib/$zkey_name" -if [ ! -e ${zkey_file} ] || [ $(wc -c < ${zkey_file}) -ne ${zkey_size} ]; then - echo "Start downloading ${zkey_file}..." - curl $zkey_url -o ${zkey_file} - if [ ! -e ${zkey_file} ]; then - echo "Error: The zkey file was not downloaded. Please try again." - exit 1 - fi - if [ $(wc -c < ${zkey_file}) -ne ${zkey_size} ]; then - echo "Error: The zkey file was not downloaded correctly. You can check the file content for more information." - exit 1 - fi -fi + i=$((i+1)) + done + + echo "$arg_value" +} + +# function to remove a specified flag from command-line arguments +remove_flag() { + local flag="$1" + local args=("$@") + local new_args=() + + for arg in "${args[@]}"; do + if [[ "$arg" == "$flag" ]]; then + i=$((i+1)) + continue + elif [[ "$arg" == "$flag"=* ]]; then + continue + else + new_args+=("$arg") + fi + done + echo "${new_args[@]}" +} # ZK prover implementation, 1: snarkjs, 2: go-rapidsnark. -zkp_impl= -i=1 -while [ $i -le $# ]; do - if [ "${!i}" = "--miner.zk-prover-impl" ]; then - j=$((i+1)) - zkp_impl="${!j}" - break - else - if echo "${!i}" | grep -qE -- "--miner\.zk-prover-impl=([0-9]+)"; then - zkp_impl=$(echo "${!i}" | sed -E 's/.*=([0-9]+)/\1/') - break - fi - fi - i=$((i+1)) -done - +zkp_impl=$(extract_value "--miner.zk-prover-impl" "$@") if [ -n "$zkp_impl" ] && [ "$zkp_impl" != 1 ] && [ "$zkp_impl" != 2 ]; then - echo "miner.zk-prover-impl can only be 1 or 2" - exit 1 -fi + echo "Error: zk prover implementation can only be 1 or 2." + exit 1 +fi if [ -n "$zkp_impl" ]; then - echo "The zk prover implementation has been overridden to $zkp_impl" -fi + echo "The zk prover implementation has been overridden to $zkp_impl" + else + zkp_impl=1 +fi if [ "$zkp_impl" = 1 ]; then - if ! [ -x "$(command -v node)" ]; then echo 'Error: Node.js is not installed.' exit 1 fi - # check node js version node_version=$(node -v) major_version=$(echo $node_version | cut -d'v' -f2 | cut -d'.' -f1) - if [ "$major_version" -lt 16 ]; then echo "Error: Node.js version is too old." exit 1 fi - # install snarkjs if not if ! [ "$(command -v snarkjs)" ]; then echo "snarkjs not found, start installing..." @@ -130,12 +109,67 @@ if [ "$zkp_impl" = 1 ]; then exit 1 fi fi +fi + +# ZK prover mode, 1: one proof per sample, 2: one proof for multiple samples. +zkp_mode=$(extract_value "--miner.zk-prover-mode" "$@") +if [ -n "$zkp_mode" ] && [ "$zkp_mode" != 1 ] && [ "$zkp_mode" != 2 ]; then + echo "Error: zk prover mode can only be 1 or 2." + exit 1 +fi + +if [ -n "$zkp_mode" ]; then + echo "The zk prover mode has been overridden to $zkp_mode" + else + zkp_mode=2 +fi + +# download zkey if not yet +zkey_name="blob_poseidon2.zkey" +zkey_size=560301223 +zkey_url="https://es-node-zkey.s3.us-west-1.amazonaws.com/blob_poseidon2_testnet1.zkey" +if [ "$zkp_mode" = 1 ]; then + zkey_name="blob_poseidon.zkey" + zkey_size=280151245 + zkey_url="https://drive.usercontent.google.com/download?id=1ZLfhYeCXMnbk6wUiBADRAn1mZ8MI_zg-&export=download&confirm=t&uuid=16ddcd58-2498-4d65-8931-934df3d0065c" +fi +zkey_file="./build/bin/snark_lib/$zkey_name" +if [ ! -e ${zkey_file} ] || [ $(wc -c < ${zkey_file}) -ne ${zkey_size} ]; then + echo "Start downloading ${zkey_file}..." + curl $zkey_url -o ${zkey_file} + if [ ! -e ${zkey_file} ]; then + echo "Error: The zkey file was not downloaded. Please try again." + exit 1 + fi + if [ $(wc -c < ${zkey_file}) -ne ${zkey_size} ]; then + echo "Error: The zkey file was not downloaded correctly. You can check the file content for more information." + exit 1 + fi fi -executable="./build/bin/es-node" data_dir="./es-data" -storage_file_0="$data_dir/shard-0.dat" +init_flags="init" +shard_array=(0) +files_to_init=() +storage_files_flag=() +additional_args=$@ +shards=$(extract_value "--shards" "$@") + +if [ -n "$shards" ]; then + IFS=',' read -ra shard_array <<< "$shards" + additional_args=$(remove_flag "--shards" "$@") +fi + +for shard in "${shard_array[@]}"; do + storage_file="$data_dir/shard-$shard.dat" + if [ ! -e $storage_file ]; then + files_to_init+=("$storage_file") + init_flags+=" --shard_index $shard" + fi + storage_files_flag+=("--storage.files $storage_file") +done + common_flags=" --datadir $data_dir \ --l1.rpc http://88.99.30.186:8545 \ @@ -143,15 +177,23 @@ common_flags=" --datadir $data_dir \ --storage.miner $ES_NODE_STORAGE_MINER \ " -# init shard 0 -es_node_init="init --shard_index 0" +executable="./build/bin/es-node" + +# create data files (init) +if [ ${#files_to_init[@]} -gt 0 ]; then + if $executable $init_flags $common_flags ; then + echo "Initialized ${files_to_init[@]} successfully" + else + echo "Error: failed to initialize storage files: ${files_to_init[@]}" + exit 1 + fi +fi -# start node # TODO remove --network -es_node_start=" --network devnet \ +start_flags=" --network devnet \ --miner.enabled \ --miner.zkey $zkey_name \ - --storage.files $storage_file_0 \ + ${storage_files_flag[@]} \ --signer.private-key $ES_NODE_SIGNER_PRIVATE_KEY \ --l1.beacon http://88.99.30.186:3500 \ --l1.beacon-based-time 1706684472 \ @@ -162,17 +204,7 @@ es_node_start=" --network devnet \ --p2p.max.request.size 4194304 \ --p2p.sync.concurrency 32 \ --p2p.bootnodes enr:-Li4QF3vBkkDQYNLHlVjW5NcEpXAsfNtE1lUVb_LgUQ_Ot2afS8jbDfnYQBDABJud_5Hd1hX_1cNeGVU6Tem06WDlfaGAY1e3vNvimV0aHN0b3JhZ2XbAYDY15SATFINPAhMgF43o16QBXrDKDH5b8GAgmlkgnY0gmlwhEFtP5qJc2VjcDI1NmsxoQK8XODtSv0IsrhBxZmTZBZEoLssb7bTX0YOVl6S0yLxuYN0Y3CCJAaDdWRwgnZh \ -$@" - -# create data file for shard 0 if not yet -if [ ! -e $storage_file_0 ]; then - if $executable $es_node_init $common_flags ; then - echo "Initialized ${storage_file_0} successfully" - else - echo "Error: failed to initialize ${storage_file_0}" - exit 1 - fi -fi +$additional_args" # start es-node -exec $executable $common_flags $es_node_start +exec $executable $common_flags $start_flags From 798e9577e23062922c3807cf0f796f31f665e380 Mon Sep 17 00:00:00 2001 From: syntrust Date: Thu, 9 May 2024 14:59:52 +0800 Subject: [PATCH 09/33] skip existed file --- cmd/es-node/main.go | 4 ++-- cmd/es-node/utils.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/es-node/main.go b/cmd/es-node/main.go index 75743e08..494d299f 100644 --- a/cmd/es-node/main.go +++ b/cmd/es-node/main.go @@ -241,7 +241,7 @@ func EsNodeInit(ctx *cli.Context) error { return err } if diff != nil && diff.Cmp(big.NewInt(0)) == 0 { - return fmt.Errorf("Shard not exist: %d", shard) + return fmt.Errorf("shard not exist: %d", shard) } } shardIdxList = append(shardIdxList, shard) @@ -254,7 +254,7 @@ func EsNodeInit(ctx *cli.Context) error { return err } if len(shardList) == 0 { - return fmt.Errorf("No shard indexes found") + return fmt.Errorf("no shard indexes found") } shardIdxList = shardList } diff --git a/cmd/es-node/utils.go b/cmd/es-node/utils.go index e47c29e7..6b850523 100644 --- a/cmd/es-node/utils.go +++ b/cmd/es-node/utils.go @@ -154,8 +154,8 @@ func createDataFile(cfg *storage.StorageConfig, shardIdxList []uint64, datadir s for _, shardIdx := range shardIdxList { dataFile := filepath.Join(datadir, fmt.Sprintf(fileName, shardIdx)) if _, err := os.Stat(dataFile); err == nil { - log.Error("Creating data file", "error", "file already exists, will not overwrite", "file", dataFile) - return nil, err + log.Info("Data file exists, skiping", "shard", shardIdx, "file", dataFile) + continue } if cfg.ChunkSize == 0 { return nil, fmt.Errorf("chunk size should not be 0") From c3f22675c9bda11f33da953a8968d280432904e0 Mon Sep 17 00:00:00 2001 From: syntrust Date: Fri, 10 May 2024 10:43:59 +0800 Subject: [PATCH 10/33] updates --- integration_tests/node_mine_test.go | 71 ++++++++++++++--------------- 1 file changed, 35 insertions(+), 36 deletions(-) diff --git a/integration_tests/node_mine_test.go b/integration_tests/node_mine_test.go index 0743ea90..48e8b87c 100644 --- a/integration_tests/node_mine_test.go +++ b/integration_tests/node_mine_test.go @@ -44,13 +44,12 @@ func TestMining(t *testing.T) { if err != nil { t.Fatalf("Failed to connect to the Ethereum client: %v", err) } - + // verify it is an empty contract lastKv, err := pClient.GetStorageLastBlobIdx(rpc.LatestBlockNumber.Int64()) if err != nil { - lg.Error("Failed to get lastKvIdx", "error", err) - } else { - lg.Info("lastKv", "lastKv", lastKv) + t.Fatalf("Failed to get lastKvIdx: %v", err) } + lg.Info("lastKv", "lastKv", lastKv) if lastKv != 0 { t.Fatalf("A newly deployed storage contract is required") } @@ -123,7 +122,7 @@ func TestMining(t *testing.T) { fillEmpty(t, storageManager, kvs) for j := 0; j < 3; j++ { time.Sleep(3 * time.Minute) - prepareData(t, pClient, storageManager, miningConfig.StorageCost.String(), kvs/3) + prepareData(t, pClient, storageManager, miningConfig.StorageCost.String()) } }() feed.Send(protocol.EthStorageSyncDone{ @@ -158,33 +157,6 @@ func TestMining(t *testing.T) { close() } -func cleanFiles(proverDir string) { - for _, shardId := range shardIds { - fileName := fmt.Sprintf(dataFileName, shardId) - if _, err := os.Stat(fileName); !os.IsNotExist(err) { - err = os.Remove(fileName) - if err != nil { - fmt.Println(err) - } - } - } - - folderPath := filepath.Join(proverDir, "snarkbuild") - files, err := os.ReadDir(folderPath) - if err != nil { - fmt.Println(err) - return - } - for _, file := range files { - if !strings.HasPrefix(file.Name(), ".") { - err = os.RemoveAll(filepath.Join(folderPath, file.Name())) - if err != nil { - fmt.Println(err) - } - } - } -} - func waitForMined(l1api miner.L1API, contract common.Address, chainHeadCh chan eth.L1BlockRef, shardIdx, lastMined uint64, exitCh chan uint64) { for range chainHeadCh { info, err := l1api.GetMiningInfo( @@ -277,11 +249,11 @@ func prepareData(t *testing.T, l1Client *eth.PollingClient, storageMgr *ethstora t.Fatalf("Get chain id failed %v", err) } for i := 0; i < txs; i++ { - max := maxBlobsPerTx + blobsPerTx := maxBlobsPerTx if i == txs-1 { - max = last + blobsPerTx = last } - blobGroup := blobs[i*maxBlobsPerTx : i*maxBlobsPerTx+max] + blobGroup := blobs[i*maxBlobsPerTx : i*maxBlobsPerTx+blobsPerTx] var blobData []byte for _, bd := range blobGroup { blobData = append(blobData, bd[:]...) @@ -424,8 +396,35 @@ func initMiningConfig(t *testing.T, l1Contract common.Address, client *eth.Polli proverPath, _ := filepath.Abs(prPath) miningConfig.ZKWorkingDir = proverPath miningConfig.ZKProverMode = 2 - miningConfig.ZKProverImpl = 1 + miningConfig.ZKProverImpl = 2 miningConfig.ThreadsPerShard = 2 miningConfig.MinimumProfit = new(big.Int).SetInt64(-1e18) return miningConfig } + +func cleanFiles(proverDir string) { + for _, shardId := range shardIds { + fileName := fmt.Sprintf(dataFileName, shardId) + if _, err := os.Stat(fileName); !os.IsNotExist(err) { + err = os.Remove(fileName) + if err != nil { + fmt.Println(err) + } + } + } + + folderPath := filepath.Join(proverDir, "snarkbuild") + files, err := os.ReadDir(folderPath) + if err != nil { + fmt.Println(err) + return + } + for _, file := range files { + if !strings.HasPrefix(file.Name(), ".") { + err = os.RemoveAll(filepath.Join(folderPath, file.Name())) + if err != nil { + fmt.Println(err) + } + } + } +} From 320e115d198458d8b3237a98c668c5982b2eff28 Mon Sep 17 00:00:00 2001 From: syntrust Date: Tue, 14 May 2024 14:03:54 +0800 Subject: [PATCH 11/33] add comments --- cmd/es-node/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/es-node/main.go b/cmd/es-node/main.go index 422eddd3..6c8bbfb6 100644 --- a/cmd/es-node/main.go +++ b/cmd/es-node/main.go @@ -240,6 +240,7 @@ func EsNodeInit(ctx *cli.Context) error { log.Error("Failed to get shard info from contract", "error", err) return err } + // lastMineTime will be set to non-zero when opening a new shard if timeStamp != nil && timeStamp.Cmp(big.NewInt(0)) == 0 { return fmt.Errorf("shard not exist: %d", shard) } From bf8c2bcc28de98f295f6a992ebddefddebfac74a Mon Sep 17 00:00:00 2001 From: syntrust Date: Wed, 15 May 2024 11:42:28 +0800 Subject: [PATCH 12/33] dynamic blob fee --- cmd/es-utils/utils/utils.go | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/cmd/es-utils/utils/utils.go b/cmd/es-utils/utils/utils.go index 8859752f..30b0c5a3 100644 --- a/cmd/es-utils/utils/utils.go +++ b/cmd/es-utils/utils/utils.go @@ -16,6 +16,7 @@ import ( "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus/misc/eip4844" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/crypto/kzg4844" @@ -130,6 +131,17 @@ func SendBlobTx( if err != nil { log.Crit("Invalid max_fee_per_data_gas", "error", err) } + if maxFeePerDataGas256 == nil { + blobBaseFee, err := queryBlobBaseFee(client) + if err != nil { + log.Crit("Error getting blob base fee", "error", err) + } + blobBaseFee256, ok := uint256.FromBig(blobBaseFee) + if !ok { + log.Crit("Error converting blob base fee to uint256") + } + maxFeePerDataGas256 = blobBaseFee256 + } var blobs []kzg4844.Blob if needEncoding { blobs = EncodeBlobs(data) @@ -325,7 +337,7 @@ func UploadBlobs( 5000000, "", "", - "300000000", + "", chainID, calldata, ) @@ -374,6 +386,16 @@ func UploadBlobs( return getKvInfo(pc, contractAddr, len(blobs)) } +func queryBlobBaseFee(l1 *ethclient.Client) (*big.Int, error) { + block, err := l1.BlockByNumber(context.Background(), new(big.Int).SetInt64(rpc.LatestBlockNumber.Int64())) + if err != nil { + return nil, err + } + excessBlobGas := eip4844.CalcExcessBlobGas(*block.ExcessBlobGas(), *block.BlobGasUsed()) + blobBaseFee := eip4844.CalcBlobFee(excessBlobGas) + return blobBaseFee, nil +} + func getKvInfo(pc *eth.PollingClient, contractAddr common.Address, blobLen int) ([]uint64, []common.Hash, error) { lastIdx, err := pc.GetStorageLastBlobIdx(rpc.LatestBlockNumber.Int64()) if err != nil { From 9669615c3d38938afb5192327915e3f5139189f0 Mon Sep 17 00:00:00 2001 From: syntrust Date: Wed, 15 May 2024 14:09:03 +0800 Subject: [PATCH 13/33] fix blob price --- cmd/es-utils/utils/utils.go | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/cmd/es-utils/utils/utils.go b/cmd/es-utils/utils/utils.go index 30b0c5a3..e99e6e42 100644 --- a/cmd/es-utils/utils/utils.go +++ b/cmd/es-utils/utils/utils.go @@ -127,11 +127,14 @@ func SendBlobTx( } } - maxFeePerDataGas256, err := DecodeUint256String(maxFeePerDataGas) - if err != nil { - log.Crit("Invalid max_fee_per_data_gas", "error", err) - } - if maxFeePerDataGas256 == nil { + var blobPrice *uint256.Int + if maxFeePerDataGas != "" { + maxFeePerDataGas256, err := DecodeUint256String(maxFeePerDataGas) + if err != nil { + log.Crit("Invalid max_fee_per_data_gas", "error", err) + } + blobPrice = maxFeePerDataGas256 + } else { blobBaseFee, err := queryBlobBaseFee(client) if err != nil { log.Crit("Error getting blob base fee", "error", err) @@ -140,7 +143,7 @@ func SendBlobTx( if !ok { log.Crit("Error converting blob base fee to uint256") } - maxFeePerDataGas256 = blobBaseFee256 + blobPrice = blobBaseFee256 } var blobs []kzg4844.Blob if needEncoding { @@ -171,7 +174,7 @@ func SendBlobTx( To: to, Value: value256, Data: calldataBytes, - BlobFeeCap: maxFeePerDataGas256, + BlobFeeCap: blobPrice, BlobHashes: versionedHashes, Sidecar: sideCar, } From 4c169c3b4035a890a62b42c1197fb119d22977ad Mon Sep 17 00:00:00 2001 From: syntrust Date: Wed, 15 May 2024 14:20:50 +0800 Subject: [PATCH 14/33] fix blob price --- cmd/es-utils/utils/utils.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/es-utils/utils/utils.go b/cmd/es-utils/utils/utils.go index e99e6e42..f101b87b 100644 --- a/cmd/es-utils/utils/utils.go +++ b/cmd/es-utils/utils/utils.go @@ -139,9 +139,10 @@ func SendBlobTx( if err != nil { log.Crit("Error getting blob base fee", "error", err) } + log.Info("Query blob base fee done", "blobBaseFee", blobBaseFee) blobBaseFee256, ok := uint256.FromBig(blobBaseFee) if !ok { - log.Crit("Error converting blob base fee to uint256") + log.Crit("Error converting blob base fee to uint256", "blobBaseFee", blobBaseFee) } blobPrice = blobBaseFee256 } From 63cfc947925077df92de9ffee0bc8ef53bfba45a Mon Sep 17 00:00:00 2001 From: syntrust Date: Wed, 15 May 2024 14:57:15 +0800 Subject: [PATCH 15/33] fixes --- cmd/es-utils/utils/utils.go | 4 ++-- integration_tests/node_mine_test.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/es-utils/utils/utils.go b/cmd/es-utils/utils/utils.go index f101b87b..684b597c 100644 --- a/cmd/es-utils/utils/utils.go +++ b/cmd/es-utils/utils/utils.go @@ -140,8 +140,8 @@ func SendBlobTx( log.Crit("Error getting blob base fee", "error", err) } log.Info("Query blob base fee done", "blobBaseFee", blobBaseFee) - blobBaseFee256, ok := uint256.FromBig(blobBaseFee) - if !ok { + blobBaseFee256, nok := uint256.FromBig(blobBaseFee) + if nok { log.Crit("Error converting blob base fee to uint256", "blobBaseFee", blobBaseFee) } blobPrice = blobBaseFee256 diff --git a/integration_tests/node_mine_test.go b/integration_tests/node_mine_test.go index 48e8b87c..f851d654 100644 --- a/integration_tests/node_mine_test.go +++ b/integration_tests/node_mine_test.go @@ -121,7 +121,7 @@ func TestMining(t *testing.T) { go func() { fillEmpty(t, storageManager, kvs) for j := 0; j < 3; j++ { - time.Sleep(3 * time.Minute) + time.Sleep(1 * time.Minute) prepareData(t, pClient, storageManager, miningConfig.StorageCost.String()) } }() From 6556bba6891872acc7b009779a69c846f092ccce Mon Sep 17 00:00:00 2001 From: syntrust Date: Wed, 15 May 2024 19:07:43 +0800 Subject: [PATCH 16/33] fix --- integration_tests/node_mine_test.go | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/integration_tests/node_mine_test.go b/integration_tests/node_mine_test.go index f851d654..8d8cae29 100644 --- a/integration_tests/node_mine_test.go +++ b/integration_tests/node_mine_test.go @@ -59,7 +59,7 @@ func TestMining(t *testing.T) { t.Fatalf("Create data files error: %v", err) } storConfig.Filenames = files - miningConfig := initMiningConfig(t, contract, pClient) + miningConfig := initMiningConfig(t, pClient) lg.Info("Initialzed mining config", "miningConfig", fmt.Sprintf("%+v", miningConfig)) shardManager, err := initShardManager(*storConfig) if err != nil { @@ -279,21 +279,12 @@ func prepareData(t *testing.T, l1Client *eth.PollingClient, storageMgr *ethstora if err != nil { t.Fatalf("Download all metas failed %v", err) } - totalKvs := len(shardIds) * int(storageMgr.KvEntries()) - limit := totalKvs - if limit > len(ids) { - limit = len(ids) - } - for i := 0; i < limit; i++ { + for i := 0; i < len(ids); i++ { err := storageMgr.CommitBlob(ids[i], blobs[i][:], hashs[i]) if err != nil { t.Fatalf("Failed to commit blob: i=%d, id=%d, error: %v", i, ids[i], err) } } - _, _, err = storageMgr.CommitEmptyBlobs(uint64(limit), uint64(totalKvs)-1) - if err != nil { - t.Fatalf("Commit empty blobs failed %v", err) - } } func createDataFiles(cfg *storage.StorageConfig) ([]string, error) { @@ -325,7 +316,7 @@ func createDataFiles(cfg *storage.StorageConfig) ([]string, error) { return files, nil } -func initMiningConfig(t *testing.T, l1Contract common.Address, client *eth.PollingClient) *miner.Config { +func initMiningConfig(t *testing.T, client *eth.PollingClient) *miner.Config { miningConfig := &miner.Config{} factory, addrFrom, err := signer.SignerFactoryFromConfig(signer.CLIConfig{ PrivateKey: privateKey, From d8ffc96ed98fc58df77f47d10803ba4c41a8403f Mon Sep 17 00:00:00 2001 From: syntrust Date: Thu, 16 May 2024 15:08:25 +0800 Subject: [PATCH 17/33] fix --- integration_tests/node_mine_test.go | 23 +++++------------------ 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/integration_tests/node_mine_test.go b/integration_tests/node_mine_test.go index 8d8cae29..bf4a5ae3 100644 --- a/integration_tests/node_mine_test.go +++ b/integration_tests/node_mine_test.go @@ -31,7 +31,7 @@ import ( ) const ( - maxBlobsPerTx = 4 + maxBlobsPerTx = 3 dataFileName = "shard-%d.dat" ) @@ -118,13 +118,8 @@ func TestMining(t *testing.T) { }() for i, s := range shardIds { kvs := shardManager.KvEntries() - go func() { - fillEmpty(t, storageManager, kvs) - for j := 0; j < 3; j++ { - time.Sleep(1 * time.Minute) - prepareData(t, pClient, storageManager, miningConfig.StorageCost.String()) - } - }() + fillEmpty(t, storageManager, kvs) + prepareData(t, pClient, storageManager, miningConfig.StorageCost.String()) feed.Send(protocol.EthStorageSyncDone{ DoneType: protocol.SingleShardDone, ShardId: s, @@ -230,7 +225,7 @@ func fillEmpty(t *testing.T, storageMgr *ethstorage.StorageManager, entriesToFil } func prepareData(t *testing.T, l1Client *eth.PollingClient, storageMgr *ethstorage.StorageManager, value string) { - data := generateRandomContent(124 * 10) + data := generateRandomContent(124 * 3) blobs := utils.EncodeBlobs(data) t.Logf("Blobs len %d \n", len(blobs)) var hashs []common.Hash @@ -270,20 +265,12 @@ func prepareData(t *testing.T, l1Client *eth.PollingClient, storageMgr *ethstora hashs = append(hashs, dataHashes...) ids = append(ids, kvIdxes...) } - block, err := l1Client.BlockNumber(context.Background()) - if err != nil { - t.Fatalf("Failed to get block number %v", err) - } - storageMgr.Reset(int64(block)) - err = storageMgr.DownloadAllMetas(context.Background(), 1) - if err != nil { - t.Fatalf("Download all metas failed %v", err) - } for i := 0; i < len(ids); i++ { err := storageMgr.CommitBlob(ids[i], blobs[i][:], hashs[i]) if err != nil { t.Fatalf("Failed to commit blob: i=%d, id=%d, error: %v", i, ids[i], err) } + t.Logf("Committed blob: i=%d, id=%d, hash=%x", i, ids[i], hashs[i]) } } From b6eede9295c36d04f22c1cd8182c3b940f9f64fc Mon Sep 17 00:00:00 2001 From: syntrust Date: Tue, 21 May 2024 11:01:58 +0800 Subject: [PATCH 18/33] remove useless --- integration_tests/node_mine_test.go | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/integration_tests/node_mine_test.go b/integration_tests/node_mine_test.go index bf4a5ae3..9942744f 100644 --- a/integration_tests/node_mine_test.go +++ b/integration_tests/node_mine_test.go @@ -11,7 +11,6 @@ import ( "math/big" "os" "path/filepath" - "strings" "sync" "testing" "time" @@ -379,30 +378,3 @@ func initMiningConfig(t *testing.T, client *eth.PollingClient) *miner.Config { miningConfig.MinimumProfit = new(big.Int).SetInt64(-1e18) return miningConfig } - -func cleanFiles(proverDir string) { - for _, shardId := range shardIds { - fileName := fmt.Sprintf(dataFileName, shardId) - if _, err := os.Stat(fileName); !os.IsNotExist(err) { - err = os.Remove(fileName) - if err != nil { - fmt.Println(err) - } - } - } - - folderPath := filepath.Join(proverDir, "snarkbuild") - files, err := os.ReadDir(folderPath) - if err != nil { - fmt.Println(err) - return - } - for _, file := range files { - if !strings.HasPrefix(file.Name(), ".") { - err = os.RemoveAll(filepath.Join(folderPath, file.Name())) - if err != nil { - fmt.Println(err) - } - } - } -} From 83f851379faeedb039c7a3af5df59b9aeef53114 Mon Sep 17 00:00:00 2001 From: syntrust Date: Tue, 21 May 2024 11:14:37 +0800 Subject: [PATCH 19/33] fix test --- integration_tests/node_mine_test.go | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/integration_tests/node_mine_test.go b/integration_tests/node_mine_test.go index 9942744f..10ba1786 100644 --- a/integration_tests/node_mine_test.go +++ b/integration_tests/node_mine_test.go @@ -100,6 +100,8 @@ func TestMining(t *testing.T) { } lg.Error("L1 heads subscription error", "err", err) }() + fillEmpty(t, storageManager) + prepareData(t, pClient, storageManager, miningConfig.StorageCost.String()) mnr.Start() var wg sync.WaitGroup minedShardSig := make(chan uint64, len(shardIds)) @@ -116,9 +118,6 @@ func TestMining(t *testing.T) { } }() for i, s := range shardIds { - kvs := shardManager.KvEntries() - fillEmpty(t, storageManager, kvs) - prepareData(t, pClient, storageManager, miningConfig.StorageCost.String()) feed.Send(protocol.EthStorageSyncDone{ DoneType: protocol.SingleShardDone, ShardId: s, @@ -135,7 +134,7 @@ func TestMining(t *testing.T) { wg.Add(1) // defer next shard mining so that the started shard can be mined for a while if i != len(shardIds)-1 { - var miningTime time.Duration = 600 + var miningTime time.Duration = 60 timeout := time.After(miningTime * time.Second) select { case minedShard := <-minedShardSig: @@ -213,10 +212,11 @@ func initShardManager(storConfig storage.StorageConfig) (*ethstorage.ShardManage return shardManager, nil } -func fillEmpty(t *testing.T, storageMgr *ethstorage.StorageManager, entriesToFill uint64) { +func fillEmpty(t *testing.T, storageMgr *ethstorage.StorageManager) { lastKvIdx := storageMgr.LastKvIndex() - lg.Info("Filling empty", "lastKvIdx", lastKvIdx, "entriesToFill", entriesToFill) - inserted, next, err := storageMgr.CommitEmptyBlobs(lastKvIdx, entriesToFill+lastKvIdx-1) + totalEntries := storageMgr.KvEntries() * uint64(len(shardIds)) + lg.Info("Filling empty", "lastKvIdx", lastKvIdx) + inserted, next, err := storageMgr.CommitEmptyBlobs(lastKvIdx, totalEntries-1) if err != nil { t.Fatalf("Commit empty blobs failed %v", err) } @@ -224,7 +224,7 @@ func fillEmpty(t *testing.T, storageMgr *ethstorage.StorageManager, entriesToFil } func prepareData(t *testing.T, l1Client *eth.PollingClient, storageMgr *ethstorage.StorageManager, value string) { - data := generateRandomContent(124 * 3) + data := generateRandomContent(124 * 5) blobs := utils.EncodeBlobs(data) t.Logf("Blobs len %d \n", len(blobs)) var hashs []common.Hash @@ -264,10 +264,19 @@ func prepareData(t *testing.T, l1Client *eth.PollingClient, storageMgr *ethstora hashs = append(hashs, dataHashes...) ids = append(ids, kvIdxes...) } + block, err := l1Client.BlockNumber(context.Background()) + if err != nil { + t.Fatalf("Failed to get block number %v", err) + } + storageMgr.Reset(int64(block)) + err = storageMgr.DownloadAllMetas(context.Background(), 1) + if err != nil { + t.Fatalf("Download all metas failed %v", err) + } for i := 0; i < len(ids); i++ { err := storageMgr.CommitBlob(ids[i], blobs[i][:], hashs[i]) if err != nil { - t.Fatalf("Failed to commit blob: i=%d, id=%d, error: %v", i, ids[i], err) + t.Fatalf("Failed to commit blob: i=%d, id=%d, hash=%x, error: %v", i, ids[i], hashs[i], err) } t.Logf("Committed blob: i=%d, id=%d, hash=%x", i, ids[i], hashs[i]) } From 360566a2ed5a6e3752480fc363cb11e356557406 Mon Sep 17 00:00:00 2001 From: syntrust Date: Fri, 2 Aug 2024 18:23:32 +0800 Subject: [PATCH 20/33] fix comments - add shard ingore status in contract --- cmd/es-node/main.go | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/cmd/es-node/main.go b/cmd/es-node/main.go index 6c8bbfb6..5c677f1a 100644 --- a/cmd/es-node/main.go +++ b/cmd/es-node/main.go @@ -6,7 +6,6 @@ package main import ( "context" "fmt" - "math/big" "net" "os" "os/signal" @@ -231,21 +230,8 @@ func EsNodeInit(ctx *cli.Context) error { log.Info("Storage config loaded", "storageCfg", storageCfg) var shardIdxList []uint64 if len(shardIndexes) > 0 { - // check existense of shard indexes but add shard 0 anyway for i := 0; i < len(shardIndexes); i++ { - shard := uint64(shardIndexes[i]) - if shard > 0 { - timeStamp, err := getLastMineTime(cctx, client, l1Contract, shard) - if err != nil { - log.Error("Failed to get shard info from contract", "error", err) - return err - } - // lastMineTime will be set to non-zero when opening a new shard - if timeStamp != nil && timeStamp.Cmp(big.NewInt(0)) == 0 { - return fmt.Errorf("shard not exist: %d", shard) - } - } - shardIdxList = append(shardIdxList, shard) + shardIdxList = append(shardIdxList, uint64(shardIndexes[i])) } } else { // get shard indexes of length shardLen from contract From a00efd1f1da05abb75ad771396cb5972e4db73f7 Mon Sep 17 00:00:00 2001 From: syntrust Date: Fri, 2 Aug 2024 19:24:20 +0800 Subject: [PATCH 21/33] update init --- cmd/es-node/main.go | 10 +++++++++- cmd/es-node/utils.go | 4 ++-- init-l2.sh | 17 ++++++----------- init.sh | 17 ++++++----------- 4 files changed, 23 insertions(+), 25 deletions(-) diff --git a/cmd/es-node/main.go b/cmd/es-node/main.go index 5c677f1a..e4542b95 100644 --- a/cmd/es-node/main.go +++ b/cmd/es-node/main.go @@ -230,8 +230,16 @@ func EsNodeInit(ctx *cli.Context) error { log.Info("Storage config loaded", "storageCfg", storageCfg) var shardIdxList []uint64 if len(shardIndexes) > 0 { + out: for i := 0; i < len(shardIndexes); i++ { - shardIdxList = append(shardIdxList, uint64(shardIndexes[i])) + new := uint64(shardIndexes[i]) + // prevent duplicated + for _, s := range shardIdxList { + if s == new { + continue out + } + } + shardIdxList = append(shardIdxList, new) } } else { // get shard indexes of length shardLen from contract diff --git a/cmd/es-node/utils.go b/cmd/es-node/utils.go index 501fad4a..9965ece3 100644 --- a/cmd/es-node/utils.go +++ b/cmd/es-node/utils.go @@ -170,8 +170,8 @@ func createDataFile(cfg *storage.StorageConfig, shardIdxList []uint64, datadir s for _, shardIdx := range shardIdxList { dataFile := filepath.Join(datadir, fmt.Sprintf(fileName, shardIdx)) if _, err := os.Stat(dataFile); err == nil { - log.Error("Creating data file", "error", "file already exists, will not overwrite", "file", dataFile) - return nil, err + log.Warn("Creating data file", "error", "file already exists, will not overwrite", "file", dataFile) + continue } if cfg.ChunkSize == 0 { return nil, fmt.Errorf("chunk size should not be 0") diff --git a/init-l2.sh b/init-l2.sh index a16810f5..968b6fe5 100755 --- a/init-l2.sh +++ b/init-l2.sh @@ -114,7 +114,6 @@ if [ "$zkp_impl" = 1 ]; then fi data_dir="./es-data" -storage_file_0="$data_dir/shard-0.dat" es_node_init="$executable init --shard_index 0 \ --datadir $data_dir \ @@ -123,14 +122,10 @@ es_node_init="$executable init --shard_index 0 \ --storage.miner $ES_NODE_STORAGE_MINER \ $remaining_args" -# create data file for shard 0 if not yet -if [ ! -e $storage_file_0 ]; then - if $es_node_init ; then - echo "√ Initialized ${storage_file_0} successfully" - else - echo "Error: failed to initialize ${storage_file_0}" - exit 1 - fi -else - echo "Warning: storage file ${storage_file_0} already exists, skip initialization." +# es-node will skip init if data files already exist +if $es_node_init ; then + echo "√ Initialized data files successfully." +else + echo "Error: failed to initialize data files." + exit 1 fi \ No newline at end of file diff --git a/init.sh b/init.sh index bd04304e..f5100741 100755 --- a/init.sh +++ b/init.sh @@ -114,7 +114,6 @@ if [ "$zkp_impl" = 1 ]; then fi data_dir="./es-data" -storage_file_0="$data_dir/shard-0.dat" es_node_init="$executable init --shard_index 0 \ --datadir $data_dir \ @@ -123,14 +122,10 @@ es_node_init="$executable init --shard_index 0 \ --storage.miner $ES_NODE_STORAGE_MINER \ $remaining_args" -# create data file for shard 0 if not yet -if [ ! -e $storage_file_0 ]; then - if $es_node_init ; then - echo "√ Initialized ${storage_file_0} successfully" - else - echo "Error: failed to initialize ${storage_file_0}" - exit 1 - fi -else - echo "Warning: storage file ${storage_file_0} already exists, skip initialization." +# es-node will skip init if data files already exist +if $es_node_init ; then + echo "√ Initialized data files successfully." +else + echo "Error: failed to initialize data files." + exit 1 fi \ No newline at end of file From b5f1eff23985030fbda88f7676198dcf20bec901 Mon Sep 17 00:00:00 2001 From: syntrust Date: Tue, 6 Aug 2024 17:17:23 +0800 Subject: [PATCH 22/33] handle --storage.files --- run-l2.sh | 15 ++++++++------- run.sh | 15 ++++++++------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/run-l2.sh b/run-l2.sh index a958a7ee..7eb2cfb3 100755 --- a/run-l2.sh +++ b/run-l2.sh @@ -32,17 +32,18 @@ $executable --version echo "========================================" data_dir="./es-data" -storage_file_0="$data_dir/shard-0.dat" - -if [ ! -e $storage_file_0 ]; then - echo "Error: storage file not found: ${storage_file_0}. Please run 'init.sh' first." - exit 1 -fi +file_flags="" + +for file in ${data_dir}/shard-[0-9]*.dat; do + if [ -f "$file" ]; then + file_flags+=" --storage.files $file" + fi +done start_flags=" --network devnet \ --datadir $data_dir \ + $file_flags \ --storage.l1contract 0x64003adbdf3014f7E38FC6BE752EB047b95da89A \ - --storage.files $storage_file_0 \ --storage.miner $ES_NODE_STORAGE_MINER \ --l1.rpc http://65.109.20.29:8545 \ --l1.block_time 2 \ diff --git a/run.sh b/run.sh index c3809c72..2aa2751b 100755 --- a/run.sh +++ b/run.sh @@ -34,17 +34,18 @@ $executable --version echo "========================================" data_dir="./es-data" -storage_file_0="$data_dir/shard-0.dat" - -if [ ! -e $storage_file_0 ]; then - echo "Error: storage file not found: ${storage_file_0}. Please run 'init.sh' first." - exit 1 -fi +file_flags="" + +for file in ${data_dir}/shard-[0-9]*.dat; do + if [ -f "$file" ]; then + file_flags+=" --storage.files $file" + fi +done start_flags=" --network devnet \ --datadir $data_dir \ + $file_flags \ --storage.l1contract 0x804C520d3c084C805E37A35E90057Ac32831F96f \ - --storage.files $storage_file_0 \ --storage.miner $ES_NODE_STORAGE_MINER \ --l1.rpc http://88.99.30.186:8545 \ --l1.beacon http://88.99.30.186:3500 \ From eecf5f598f43f4a818d437f265e358f141bbc286 Mon Sep 17 00:00:00 2001 From: syntrust Date: Wed, 7 Aug 2024 17:07:51 +0800 Subject: [PATCH 23/33] minor --- cmd/es-node/utils.go | 8 -------- integration_tests/run_tests.sh | 16 ++++++++-------- integration_tests/zk_prover_test.go | 2 ++ 3 files changed, 10 insertions(+), 16 deletions(-) diff --git a/cmd/es-node/utils.go b/cmd/es-node/utils.go index 9965ece3..8b395062 100644 --- a/cmd/es-node/utils.go +++ b/cmd/es-node/utils.go @@ -124,14 +124,6 @@ func getDifficulty(ctx context.Context, client *ethclient.Client, contract commo return res[1].(*big.Int), nil } -func getLastMineTime(ctx context.Context, client *ethclient.Client, contract common.Address, shardIdx uint64) (*big.Int, error) { - res, err := getMiningInfo(ctx, client, contract, shardIdx) - if err != nil { - return nil, err - } - return res[0].(*big.Int), nil -} - func getMiningInfo(ctx context.Context, client *ethclient.Client, contract common.Address, shardIdx uint64) ([]interface{}, error) { uint256Type, _ := abi.NewType("uint256", "", nil) dataField, _ := abi.Arguments{{Type: uint256Type}}.Pack(new(big.Int).SetUint64(shardIdx)) diff --git a/integration_tests/run_tests.sh b/integration_tests/run_tests.sh index b163164f..7101f60d 100755 --- a/integration_tests/run_tests.sh +++ b/integration_tests/run_tests.sh @@ -18,7 +18,7 @@ if [ -z "$ES_NODE_STORAGE_MINER" ]; then fi # A contract that will be update with new blob uploaded for the KZG test if [ -z "$ES_NODE_STORAGE_L1CONTRACT_KZG" ]; then - export ES_NODE_STORAGE_L1CONTRACT_KZG=0xB6e01Ca0c33B2bAbd2eccf008F0759131FC284dB + export ES_NODE_STORAGE_L1CONTRACT_KZG=0xFA067778264A219011D27081483D9930733fb6FC fi # A contract address that clef server checks against before signing the miner transaction if [ -z "$ES_NODE_STORAGE_L1CONTRACT_CLEF" ]; then @@ -34,7 +34,7 @@ if [ -z "$ES_NODE_STORAGE_L1CONTRACT_ZKP1" ]; then fi # The commonly used l1 eth rpc endpoint if [ -z "$ES_NODE_L1_ETH_RPC" ]; then - export ES_NODE_L1_ETH_RPC=http://88.99.30.186:8545 + export ES_NODE_L1_ETH_RPC="http://65.109.20.29:8545" # L2 fi # The clef endpoint that the miner will use to sign the transaction if [ -z "$ES_NODE_CLEF_RPC" ]; then @@ -46,12 +46,12 @@ echo ES_NODE_STORAGE_L1CONTRACT = $ES_NODE_STORAGE_L1CONTRACT echo ES_NODE_STORAGE_MINER = $ES_NODE_STORAGE_MINER # download zkeys if not yet -zkey_file="./ethstorage/prover/snark_lib/blob_poseidon.zkey" -if [ ! -e ${zkey_file} ]; then - echo "${zkey_file} not found, start downloading..." - zkey_url="https://drive.usercontent.google.com/download?id=1ZLfhYeCXMnbk6wUiBADRAn1mZ8MI_zg-&export=download&confirm=t&uuid=16ddcd58-2498-4d65-8931-934df3d0065c" - curl $zkey_url -o ${zkey_file} -fi +# zkey_file="./ethstorage/prover/snark_lib/blob_poseidon.zkey" +# if [ ! -e ${zkey_file} ]; then +# echo "${zkey_file} not found, start downloading..." +# zkey_url="https://drive.usercontent.google.com/download?id=1ZLfhYeCXMnbk6wUiBADRAn1mZ8MI_zg-&export=download&confirm=t&uuid=16ddcd58-2498-4d65-8931-934df3d0065c" +# curl $zkey_url -o ${zkey_file} +# fi zkey_file="./ethstorage/prover/snark_lib/blob_poseidon2.zkey" if [ ! -e ${zkey_file} ]; then echo "${zkey_file} not found, start downloading..." diff --git a/integration_tests/zk_prover_test.go b/integration_tests/zk_prover_test.go index d5746b9a..42ef8c58 100644 --- a/integration_tests/zk_prover_test.go +++ b/integration_tests/zk_prover_test.go @@ -31,6 +31,8 @@ var zkp1Contract = common.HexToAddress(os.Getenv("ES_NODE_STORAGE_L1CONTRACT_ZKP const zkeyName = "blob_poseidon.zkey" func TestZKProver_GenerateZKProofPerSample(t *testing.T) { + // skip now as zkey is not ready for mode 1 + t.SkipNow() proverPath, _ := filepath.Abs(prPath) zkeyFull := filepath.Join(proverPath, prover.SnarkLib, zkeyName) if _, err := os.Stat(zkeyFull); os.IsNotExist(err) { From 757a6d398dd7a145e9441511521a32d32fe64ecf Mon Sep 17 00:00:00 2001 From: syntrust Date: Wed, 14 Aug 2024 16:12:29 +0800 Subject: [PATCH 24/33] fix mining test --- cmd/es-utils/utils/utils.go | 23 ++++++-- ethstorage/storage_manager.go | 4 +- integration_tests/gen_blob.go | 3 + integration_tests/node_mine_test.go | 86 ++++++++++++++++++++-------- integration_tests/run_tests.sh | 8 ++- integration_tests/zk_prover2_test.go | 4 +- 6 files changed, 94 insertions(+), 34 deletions(-) diff --git a/cmd/es-utils/utils/utils.go b/cmd/es-utils/utils/utils.go index 684b597c..f0601e3a 100644 --- a/cmd/es-utils/utils/utils.go +++ b/cmd/es-utils/utils/utils.go @@ -316,6 +316,8 @@ func UploadBlobs( } signer := crypto.PubkeyToAddress(key.PublicKey) var keys []common.Hash + var blobIndex []*big.Int + var lengthes []*big.Int var blobs []kzg4844.Blob if needEncoding { @@ -325,10 +327,23 @@ func UploadBlobs( } for i, blob := range blobs { keys = append(keys, genKey(signer, i, blob[:])) + blobIndex = append(blobIndex, new(big.Int).SetUint64(uint64(i))) + lengthes = append(lengthes, new(big.Int).SetUint64(BlobSize)) } bytes32Array, _ := abi.NewType("bytes32[]", "", nil) - dataField, _ := abi.Arguments{{Type: bytes32Array}}.Pack(keys) - h := crypto.Keccak256Hash([]byte("putBlobs(bytes32[])")) + uint256Array, _ := abi.NewType("uint256[]", "", nil) + args := abi.Arguments{ + {Type: bytes32Array}, + {Type: uint256Array}, + {Type: uint256Array}, + } + fmt.Println("blobIndex", blobIndex, "lengthes", lengthes) + dataField, err := args.Pack(keys, blobIndex, lengthes) + if err != nil { + log.Error("Failed to pack data", "err", err) + return nil, nil, err + } + h := crypto.Keccak256Hash([]byte("putBlobs(bytes32[],uint256[],uint256[])")) calldata := "0x" + common.Bytes2Hex(append(h[0:4], dataField...)) tx := SendBlobTx( rpc, @@ -391,11 +406,11 @@ func UploadBlobs( } func queryBlobBaseFee(l1 *ethclient.Client) (*big.Int, error) { - block, err := l1.BlockByNumber(context.Background(), new(big.Int).SetInt64(rpc.LatestBlockNumber.Int64())) + block, err := l1.HeaderByNumber(context.Background(), nil) if err != nil { return nil, err } - excessBlobGas := eip4844.CalcExcessBlobGas(*block.ExcessBlobGas(), *block.BlobGasUsed()) + excessBlobGas := eip4844.CalcExcessBlobGas(*block.ExcessBlobGas, *block.BlobGasUsed) blobBaseFee := eip4844.CalcBlobFee(excessBlobGas) return blobBaseFee, nil } diff --git a/ethstorage/storage_manager.go b/ethstorage/storage_manager.go index 9d9be6ac..bf2270b7 100644 --- a/ethstorage/storage_manager.go +++ b/ethstorage/storage_manager.go @@ -186,7 +186,7 @@ func (s *StorageManager) CommitBlobs(kvIndices []uint64, blobs [][]byte, commits for i := 0; i < len(kvIndices); i++ { encodedBlob, success, err := s.shardManager.TryEncodeKV(kvIndices[i], blobs[i], commits[i]) if !success || err != nil { - log.Warn("Blob encode failed", "index", kvIndices[i], "err", err.Error()) + log.Warn("Blob encode failed", "index", kvIndices[i], "err", err) continue } encodedBlobs[i] = encodedBlob @@ -230,7 +230,7 @@ func (s *StorageManager) CommitEmptyBlobs(start, limit uint64) (uint64, uint64, for i := start; i <= limit; i++ { encodedBlob, success, err := s.shardManager.TryEncodeKV(i, emptyBs, hash) if !success || err != nil { - log.Warn("Blob encode failed", "index", i, "err", err.Error()) + log.Warn("Blob encode failed", "index", i, "err", err) break } encodedBlobs = append(encodedBlobs, encodedBlob) diff --git a/integration_tests/gen_blob.go b/integration_tests/gen_blob.go index 75b74bb3..9d96c18a 100644 --- a/integration_tests/gen_blob.go +++ b/integration_tests/gen_blob.go @@ -47,3 +47,6 @@ func generateRandomContent(sizeInKB int) []byte { } return []byte(content) } +func generateRandomBlobs(blobLen int) []byte { + return generateRandomContent(128 * 31 / 32 * blobLen) +} diff --git a/integration_tests/node_mine_test.go b/integration_tests/node_mine_test.go index bef0771e..261e5bae 100644 --- a/integration_tests/node_mine_test.go +++ b/integration_tests/node_mine_test.go @@ -31,31 +31,40 @@ import ( "github.com/ethstorage/go-ethstorage/ethstorage/storage" ) +var ( + datadir string + randaoSourceURL = os.Getenv("ES_NODE_RANDAO_RPC") +) + const ( maxBlobsPerTx = 3 dataFileName = "shard-%d.dat" ) -var shardIds = []uint64{0, 1} - func TestMining(t *testing.T) { + setup(t) + t.Cleanup(func() { + teardown(t) + }) + contract := l1Contract lg.Info("Test mining", "l1Endpoint", l1Endpoint, "contract", contract) - pClient, err := eth.Dial(l1Endpoint, contract, 12, lg) + pClient, err := eth.Dial(l1Endpoint, contract, 2, lg) if err != nil { t.Fatalf("Failed to connect to the Ethereum client: %v", err) } - // verify it is an empty contract + storConfig := initStorageConfig(t, pClient, contract, minerAddr) + lastKv, err := pClient.GetStorageLastBlobIdx(rpc.LatestBlockNumber.Int64()) if err != nil { t.Fatalf("Failed to get lastKvIdx: %v", err) } lg.Info("lastKv", "lastKv", lastKv) - if lastKv != 0 { - t.Fatalf("A newly deployed storage contract is required") - } - storConfig := initStorageConfig(t, pClient, contract, minerAddr) - files, err := createDataFiles(storConfig) + curShard := lastKv / storConfig.KvEntriesPerShard + lg.Info("Current shard", "shardId", curShard) + shardIds := []uint64{curShard + 1, curShard + 2} + lg.Info("Shards to mine", "shardIds", shardIds) + files, err := createDataFiles(storConfig, shardIds) if err != nil { t.Fatalf("Create data files error: %v", err) } @@ -71,7 +80,12 @@ func TestMining(t *testing.T) { resourcesCtx, close := context.WithCancel(context.Background()) feed := new(event.Feed) - l1api := miner.NewL1MiningAPI(pClient, nil, lg) + rc, err := eth.DialRandaoSource(resourcesCtx, randaoSourceURL, l1Endpoint, 2, lg) + if err != nil { + t.Fatalf("Failed to connect to the randao source: %v", err) + } + + l1api := miner.NewL1MiningAPI(pClient, rc, lg) pvr := prover.NewKZGPoseidonProver( miningConfig.ZKWorkingDir, miningConfig.ZKeyFile, @@ -103,8 +117,10 @@ func TestMining(t *testing.T) { } lg.Error("L1 heads subscription error", "err", err) }() - fillEmpty(t, storageManager) - prepareData(t, pClient, storageManager, miningConfig.StorageCost.String()) + if err := fillEmpty(storageManager, lastKv); err != nil { + t.Fatalf("Failed to fill empty: %v", err) + } + prepareData(t, pClient, storageManager, miningConfig.StorageCost) mnr.Start() var wg sync.WaitGroup minedShardSig := make(chan uint64, len(shardIds)) @@ -215,19 +231,20 @@ func initShardManager(storConfig storage.StorageConfig) (*ethstorage.ShardManage return shardManager, nil } -func fillEmpty(t *testing.T, storageMgr *ethstorage.StorageManager) { - lastKvIdx := storageMgr.LastKvIndex() - totalEntries := storageMgr.KvEntries() * uint64(len(shardIds)) +func fillEmpty(storageMgr *ethstorage.StorageManager, lastKvIdx uint64) error { + totalEntries := storageMgr.KvEntries() * 2 lg.Info("Filling empty", "lastKvIdx", lastKvIdx) - inserted, next, err := storageMgr.CommitEmptyBlobs(lastKvIdx, totalEntries-1) + inserted, next, err := storageMgr.CommitEmptyBlobs(lastKvIdx+1, lastKvIdx+totalEntries) if err != nil { - t.Fatalf("Commit empty blobs failed %v", err) + return err } lg.Info("Filling empty done", "inserted", inserted, "next", next) + return nil } -func prepareData(t *testing.T, l1Client *eth.PollingClient, storageMgr *ethstorage.StorageManager, value string) { - data := generateRandomContent(124 * 5) +// fill contract with 2 shards of blobs +func prepareData(t *testing.T, l1Client *eth.PollingClient, storageMgr *ethstorage.StorageManager, value *big.Int) { + data := generateRandomBlobs(int(storageMgr.KvEntries() * 2)) blobs := utils.EncodeBlobs(data) t.Logf("Blobs len %d \n", len(blobs)) var hashs []common.Hash @@ -258,7 +275,8 @@ func prepareData(t *testing.T, l1Client *eth.PollingClient, storageMgr *ethstora if len(blobData) == 0 { break } - kvIdxes, dataHashes, err := utils.UploadBlobs(l1Client, l1Endpoint, privateKey, chainID.String(), storageMgr.ContractAddress(), blobData, false, value) + totalValue := new(big.Int).Mul(value, big.NewInt(int64(blobsPerTx))) + kvIdxes, dataHashes, err := utils.UploadBlobs(l1Client, l1Endpoint, privateKey, chainID.String(), storageMgr.ContractAddress(), blobData, false, totalValue.String()) if err != nil { t.Fatalf("Upload blobs failed %v", err) } @@ -285,10 +303,10 @@ func prepareData(t *testing.T, l1Client *eth.PollingClient, storageMgr *ethstora } } -func createDataFiles(cfg *storage.StorageConfig) ([]string, error) { +func createDataFiles(cfg *storage.StorageConfig, shardIds []uint64) ([]string, error) { var files []string for _, shardIdx := range shardIds { - fileName := fmt.Sprintf(dataFileName, shardIdx) + fileName := filepath.Join(datadir, fmt.Sprintf(dataFileName, shardIdx)) if _, err := os.Stat(fileName); err == nil { lg.Warn("Creating data file: file already exists, will be overwritten", "file", fileName) } @@ -375,14 +393,18 @@ func initMiningConfig(t *testing.T, client *eth.PollingClient) *miner.Config { t.Fatal("get storageCost", err) } miningConfig.StorageCost = new(big.Int).SetBytes(result) + t.Logf("storageCost=%d\n", miningConfig.StorageCost.Int64()) result, err = client.ReadContractField("prepaidAmount", nil) if err != nil { t.Fatal("get prepaidAmount", err) } miningConfig.PrepaidAmount = new(big.Int).SetBytes(result) - - miningConfig.ZKeyFile = zkey2Name proverPath, _ := filepath.Abs(prPath) + zkeyFull := filepath.Join(proverPath, prover.SnarkLib, zkey2Name) + if _, err := os.Stat(zkeyFull); os.IsNotExist(err) { + t.Fatalf("%s not found", zkeyFull) + } + miningConfig.ZKeyFile = zkeyFull miningConfig.ZKWorkingDir = proverPath miningConfig.ZKProverMode = 2 miningConfig.ZKProverImpl = 2 @@ -390,3 +412,19 @@ func initMiningConfig(t *testing.T, client *eth.PollingClient) *miner.Config { miningConfig.MinimumProfit = new(big.Int).SetInt64(-1e18) return miningConfig } + +func setup(t *testing.T) { + datadir = t.TempDir() + err := os.MkdirAll(datadir, 0700) + if err != nil { + t.Fatalf("Failed to create datadir: %v", err) + } + t.Logf("datadir %s", datadir) +} + +func teardown(t *testing.T) { + err := os.RemoveAll(datadir) + if err != nil { + t.Errorf("Failed to remove datadir: %v", err) + } +} diff --git a/integration_tests/run_tests.sh b/integration_tests/run_tests.sh index 7101f60d..cfe4f6c0 100755 --- a/integration_tests/run_tests.sh +++ b/integration_tests/run_tests.sh @@ -26,7 +26,7 @@ if [ -z "$ES_NODE_STORAGE_L1CONTRACT_CLEF" ]; then fi # A newly deployed contract is required for each run for miner test, with zkp verifier of mode 2 if [ -z "$ES_NODE_STORAGE_L1CONTRACT" ]; then - export ES_NODE_STORAGE_L1CONTRACT=0x458FF419F57050a6ec2d3Eb049a28f4F6937c11E + export ES_NODE_STORAGE_L1CONTRACT=0x035342b5eAba858fd8C8b91658Bd712fA7C74C7d fi # A contract with zkp verifier of mode 1 (one proof per sample) if [ -z "$ES_NODE_STORAGE_L1CONTRACT_ZKP1" ]; then @@ -41,6 +41,10 @@ if [ -z "$ES_NODE_CLEF_RPC" ]; then export ES_NODE_CLEF_RPC="http://65.108.236.27:8550" fi +if [ -z "$ES_NODE_RANDAO_RPC" ]; then + export ES_NODE_RANDAO_RPC="http://88.99.30.186:8545" +fi + echo ES_NODE_L1_ETH_RPC = $ES_NODE_L1_ETH_RPC echo ES_NODE_STORAGE_L1CONTRACT = $ES_NODE_STORAGE_L1CONTRACT echo ES_NODE_STORAGE_MINER = $ES_NODE_STORAGE_MINER @@ -59,5 +63,5 @@ if [ ! -e ${zkey_file} ]; then curl $zkey_url -o ${zkey_file} fi -go test -tags rapidsnark_asm -timeout 0 github.com/ethstorage/go-ethstorage/integration_tests -v +go test -tags rapidsnark_asm -run ^TestMining$ -timeout 0 github.com/ethstorage/go-ethstorage/integration_tests -v \ No newline at end of file diff --git a/integration_tests/zk_prover2_test.go b/integration_tests/zk_prover2_test.go index 775bc6dd..167daa87 100644 --- a/integration_tests/zk_prover2_test.go +++ b/integration_tests/zk_prover2_test.go @@ -47,8 +47,8 @@ func TestZKProver_GenerateZKProof(t *testing.T) { "12199007973319674300030596965685270475268514105269206407619072166392043015767", } libDir := filepath.Join(proverPath, prover.SnarkLib) - pjs := prover.NewZKProver(proverPath, zkey2Name, prover.Wasm2Name, lg) - pgo, err := prover.NewZKProverGo(libDir, zkey2Name, prover.Wasm2Name, lg) + pjs := prover.NewZKProver(proverPath, zkeyFull, prover.Wasm2Name, lg) + pgo, err := prover.NewZKProverGo(libDir, zkeyFull, prover.Wasm2Name, lg) if err != nil { t.Errorf("NewZKProverGo() error = %v", err) return From 14bcb0f520352279dfc346c26f4664b0936fa200 Mon Sep 17 00:00:00 2001 From: syntrust Date: Thu, 15 Aug 2024 16:44:09 +0800 Subject: [PATCH 25/33] print history err --- ethstorage/miner/worker.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ethstorage/miner/worker.go b/ethstorage/miner/worker.go index 80010026..2ef1308e 100644 --- a/ethstorage/miner/worker.go +++ b/ethstorage/miner/worker.go @@ -456,6 +456,9 @@ func (w *worker) resultLoop() { errorCache = append(errorCache, err) case <-w.exitCh: w.lg.Warn("Worker is exiting from result loop...") + for _, e := range errorCache { + w.lg.Error("Mining error since es-node launched", "err", e) + } return } } From 4b8c608f3bb52279350a5bfd76c9c48863d512d1 Mon Sep 17 00:00:00 2001 From: syntrust Date: Thu, 15 Aug 2024 16:44:44 +0800 Subject: [PATCH 26/33] use eth_blobBaseFee --- cmd/es-utils/utils/utils.go | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/cmd/es-utils/utils/utils.go b/cmd/es-utils/utils/utils.go index f0601e3a..be4db2d2 100644 --- a/cmd/es-utils/utils/utils.go +++ b/cmd/es-utils/utils/utils.go @@ -16,7 +16,6 @@ import ( "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/consensus/misc/eip4844" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/crypto/kzg4844" @@ -330,6 +329,7 @@ func UploadBlobs( blobIndex = append(blobIndex, new(big.Int).SetUint64(uint64(i))) lengthes = append(lengthes, new(big.Int).SetUint64(BlobSize)) } + log.Info("blobs", "keys", keys, "blobIndexes", blobIndex, "sizes", lengthes) bytes32Array, _ := abi.NewType("bytes32[]", "", nil) uint256Array, _ := abi.NewType("uint256[]", "", nil) args := abi.Arguments{ @@ -337,7 +337,6 @@ func UploadBlobs( {Type: uint256Array}, {Type: uint256Array}, } - fmt.Println("blobIndex", blobIndex, "lengthes", lengthes) dataField, err := args.Pack(keys, blobIndex, lengthes) if err != nil { log.Error("Failed to pack data", "err", err) @@ -402,20 +401,23 @@ func UploadBlobs( log.Info("Timed out for receipt, query contract for data hash...") } // if wait receipt timed out or failed, query contract for data hash - return getKvInfo(pc, contractAddr, len(blobs)) + return getKvInfo(pc, len(blobs)) } func queryBlobBaseFee(l1 *ethclient.Client) (*big.Int, error) { - block, err := l1.HeaderByNumber(context.Background(), nil) + var hex string + err := l1.Client().CallContext(context.Background(), &hex, "eth_blobBaseFee") if err != nil { return nil, err } - excessBlobGas := eip4844.CalcExcessBlobGas(*block.ExcessBlobGas, *block.BlobGasUsed) - blobBaseFee := eip4844.CalcBlobFee(excessBlobGas) + blobBaseFee, ok := new(big.Int).SetString(hex, 0) + if !ok { + return nil, errors.New("invalid blob base fee") + } return blobBaseFee, nil } -func getKvInfo(pc *eth.PollingClient, contractAddr common.Address, blobLen int) ([]uint64, []common.Hash, error) { +func getKvInfo(pc *eth.PollingClient, blobLen int) ([]uint64, []common.Hash, error) { lastIdx, err := pc.GetStorageLastBlobIdx(rpc.LatestBlockNumber.Int64()) if err != nil { return nil, nil, err From a20627d417a312c904dff878351ddb91ac0b8292 Mon Sep 17 00:00:00 2001 From: syntrust Date: Thu, 15 Aug 2024 16:46:03 +0800 Subject: [PATCH 27/33] fix test --- integration_tests/node_mine_test.go | 61 +++++++++++++++++------------ 1 file changed, 37 insertions(+), 24 deletions(-) diff --git a/integration_tests/node_mine_test.go b/integration_tests/node_mine_test.go index 261e5bae..362849bf 100644 --- a/integration_tests/node_mine_test.go +++ b/integration_tests/node_mine_test.go @@ -37,7 +37,7 @@ var ( ) const ( - maxBlobsPerTx = 3 + maxBlobsPerTx = 6 dataFileName = "shard-%d.dat" ) @@ -54,7 +54,6 @@ func TestMining(t *testing.T) { t.Fatalf("Failed to connect to the Ethereum client: %v", err) } storConfig := initStorageConfig(t, pClient, contract, minerAddr) - lastKv, err := pClient.GetStorageLastBlobIdx(rpc.LatestBlockNumber.Int64()) if err != nil { t.Fatalf("Failed to get lastKvIdx: %v", err) @@ -98,11 +97,12 @@ func TestMining(t *testing.T) { mnr := miner.New(miningConfig, db, storageManager, l1api, br, &pvr, feed, lg) lg.Info("Initialized miner") - l1HeadsSub := event.ResubscribeErr(time.Second*10, func(ctx context.Context, err error) (event.Subscription, error) { + randaoHeadsSub := event.ResubscribeErr(time.Second*10, func(ctx context.Context, err error) (event.Subscription, error) { if err != nil { - lg.Warn("Resubscribing after failed L1 subscription", "err", err) + lg.Warn("Resubscribing after failed randao head subscription", "err", err) } - return eth.WatchHeadChanges(resourcesCtx, pClient, func(ctx context.Context, sig eth.L1BlockRef) { + return eth.WatchHeadChanges(resourcesCtx, rc, func(ctx context.Context, sig eth.L1BlockRef) { + lg.Debug("OnNewRandaoSourceHead", "blockNumber", sig.Number) select { case mnr.ChainHeadCh <- sig: default: @@ -110,17 +110,19 @@ func TestMining(t *testing.T) { } }) }) + lg.Info("Randao head subscribed") go func() { - err, ok := <-l1HeadsSub.Err() + err, ok := <-randaoHeadsSub.Err() if !ok { return } - lg.Error("L1 heads subscription error", "err", err) + lg.Error("Randao heads subscription error", "err", err) }() - if err := fillEmpty(storageManager, lastKv); err != nil { + + if err := fillEmpty(storageManager, shardIds); err != nil { t.Fatalf("Failed to fill empty: %v", err) } - prepareData(t, pClient, storageManager, miningConfig.StorageCost) + prepareData(t, pClient, storageManager) mnr.Start() var wg sync.WaitGroup minedShardSig := make(chan uint64, len(shardIds)) @@ -157,14 +159,14 @@ func TestMining(t *testing.T) { timeout := time.After(miningTime * time.Second) select { case minedShard := <-minedShardSig: - lg.Info(fmt.Sprintf("Shard %d successfully mined, will start next shard: %d", minedShard, i+1)) + lg.Info(fmt.Sprintf("Shard %d successfully mined, will start next shard: %d", minedShard, shardIds[i+1])) case <-timeout: - lg.Info(fmt.Sprintf("Shard %d has been mined for %ds, will start next shard: %d", i, miningTime, i+1)) + lg.Info(fmt.Sprintf("Shard %d has been mined for %ds, will start next shard: %d", shardIds[i], miningTime, shardIds[i+1])) } } } wg.Wait() - l1HeadsSub.Unsubscribe() + randaoHeadsSub.Unsubscribe() mnr.Close() close() } @@ -231,10 +233,11 @@ func initShardManager(storConfig storage.StorageConfig) (*ethstorage.ShardManage return shardManager, nil } -func fillEmpty(storageMgr *ethstorage.StorageManager, lastKvIdx uint64) error { - totalEntries := storageMgr.KvEntries() * 2 - lg.Info("Filling empty", "lastKvIdx", lastKvIdx) - inserted, next, err := storageMgr.CommitEmptyBlobs(lastKvIdx+1, lastKvIdx+totalEntries) +func fillEmpty(storageMgr *ethstorage.StorageManager, shards []uint64) error { + start := shards[0] * storageMgr.KvEntries() + totalEntries := storageMgr.KvEntries()*uint64(len(shards)) - 1 + lg.Info("Filling empty to shards", "start", start, "end", start+totalEntries) + inserted, next, err := storageMgr.CommitEmptyBlobs(start, start+totalEntries) if err != nil { return err } @@ -242,9 +245,9 @@ func fillEmpty(storageMgr *ethstorage.StorageManager, lastKvIdx uint64) error { return nil } -// fill contract with 2 shards of blobs -func prepareData(t *testing.T, l1Client *eth.PollingClient, storageMgr *ethstorage.StorageManager, value *big.Int) { - data := generateRandomBlobs(int(storageMgr.KvEntries() * 2)) +func prepareData(t *testing.T, l1Client *eth.PollingClient, storageMgr *ethstorage.StorageManager) { + // fill contract with almost 2 shards of blobs + data := generateRandomBlobs(int(storageMgr.KvEntries()*2) - 1) blobs := utils.EncodeBlobs(data) t.Logf("Blobs len %d \n", len(blobs)) var hashs []common.Hash @@ -262,6 +265,12 @@ func prepareData(t *testing.T, l1Client *eth.PollingClient, storageMgr *ethstora if err != nil { t.Fatalf("Get chain id failed %v", err) } + result, err := l1Client.ReadContractField("upfrontPayment", nil) + if err != nil { + t.Fatal("get upfrontPayment", err) + } + payment := new(big.Int).SetBytes(result) + lg.Info("Query upfront payment", "upfrontPayment", payment) for i := 0; i < txs; i++ { blobsPerTx := maxBlobsPerTx if i == txs-1 { @@ -275,10 +284,11 @@ func prepareData(t *testing.T, l1Client *eth.PollingClient, storageMgr *ethstora if len(blobData) == 0 { break } - totalValue := new(big.Int).Mul(value, big.NewInt(int64(blobsPerTx))) + totalValue := new(big.Int).Mul(payment, big.NewInt(int64(blobsPerTx))) + totalValue = totalValue.Div(totalValue.Mul(totalValue, big.NewInt(11)), big.NewInt(10)) kvIdxes, dataHashes, err := utils.UploadBlobs(l1Client, l1Endpoint, privateKey, chainID.String(), storageMgr.ContractAddress(), blobData, false, totalValue.String()) if err != nil { - t.Fatalf("Upload blobs failed %v", err) + t.Fatalf("Upload blobs failed: %v", err) } t.Logf("kvIdxes=%v \n", kvIdxes) t.Logf("dataHashes=%x \n", dataHashes) @@ -294,12 +304,16 @@ func prepareData(t *testing.T, l1Client *eth.PollingClient, storageMgr *ethstora if err != nil { t.Fatalf("Download all metas failed %v", err) } + startKv := storageMgr.Shards()[0] * storageMgr.KvEntries() for i := 0; i < len(ids); i++ { + if ids[i] < startKv { + continue + } err := storageMgr.CommitBlob(ids[i], blobs[i][:], hashs[i]) if err != nil { - t.Fatalf("Failed to commit blob: i=%d, id=%d, hash=%x, error: %v", i, ids[i], hashs[i], err) + t.Fatalf("Failed to commit blob: i=%d, kvIndex=%d, hash=%x, error: %v", i, ids[i], hashs[i], err) } - t.Logf("Committed blob: i=%d, id=%d, hash=%x", i, ids[i], hashs[i]) + t.Logf("Committed blob: i=%d, kvIndex=%d, hash=%x", i, ids[i], hashs[i]) } } @@ -393,7 +407,6 @@ func initMiningConfig(t *testing.T, client *eth.PollingClient) *miner.Config { t.Fatal("get storageCost", err) } miningConfig.StorageCost = new(big.Int).SetBytes(result) - t.Logf("storageCost=%d\n", miningConfig.StorageCost.Int64()) result, err = client.ReadContractField("prepaidAmount", nil) if err != nil { t.Fatal("get prepaidAmount", err) From 46ad0c8c4c5096712f0c916e049e3e0cf555f22d Mon Sep 17 00:00:00 2001 From: syntrust Date: Mon, 19 Aug 2024 18:55:44 +0800 Subject: [PATCH 28/33] use upfrontPaymentInBatch --- integration_tests/node_mine_test.go | 50 +++++++++++++++++------------ integration_tests/run_tests.sh | 6 ++-- 2 files changed, 33 insertions(+), 23 deletions(-) diff --git a/integration_tests/node_mine_test.go b/integration_tests/node_mine_test.go index 362849bf..23c60a5c 100644 --- a/integration_tests/node_mine_test.go +++ b/integration_tests/node_mine_test.go @@ -15,8 +15,11 @@ import ( "testing" "time" + "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/rpc" "github.com/ethstorage/go-ethstorage/cmd/es-utils/utils" @@ -143,15 +146,7 @@ func TestMining(t *testing.T) { DoneType: protocol.SingleShardDone, ShardId: s, }) - info, err := l1api.GetMiningInfo( - context.Background(), - contract, - s, - ) - if err != nil { - t.Fatalf("Failed to get es mining info for shard %d: %v", s, err) - } - go waitForMined(l1api, contract, mnr.ChainHeadCh, s, info.BlockMined.Uint64(), minedShardCh) + go waitForMined(l1api, contract, mnr.ChainHeadCh, s, minedShardCh) wg.Add(1) // defer next shard mining so that the started shard can be mined for a while if i != len(shardIds)-1 { @@ -171,7 +166,7 @@ func TestMining(t *testing.T) { close() } -func waitForMined(l1api miner.L1API, contract common.Address, chainHeadCh chan eth.L1BlockRef, shardIdx, lastMined uint64, exitCh chan uint64) { +func waitForMined(l1api miner.L1API, contract common.Address, chainHeadCh chan eth.L1BlockRef, shardIdx uint64, exitCh chan uint64) { for range chainHeadCh { info, err := l1api.GetMiningInfo( context.Background(), @@ -182,8 +177,9 @@ func waitForMined(l1api miner.L1API, contract common.Address, chainHeadCh chan e lg.Warn("Failed to get es mining info", "error", err.Error()) continue } - if info.BlockMined.Uint64() > lastMined { - lg.Info("Mined new", "shard", shardIdx, "lastMined", lastMined, "justMined", info.BlockMined) + lg.Info("Starting shard mining", "shard", shardIdx, "lastMined", info.BlockMined, "blockNumber", info.LastMineTime) + if info.BlockMined.Uint64() > 0 { + lg.Info("Mined new", "shard", shardIdx, "justMined", info.BlockMined) exitCh <- shardIdx return } @@ -245,6 +241,23 @@ func fillEmpty(storageMgr *ethstorage.StorageManager, shards []uint64) error { return nil } +func getPayment(l1Client *eth.PollingClient, contract common.Address, batch uint64) (*big.Int, error) { + uint256Type, _ := abi.NewType("uint256", "", nil) + dataField, _ := abi.Arguments{{Type: uint256Type}}.Pack(new(big.Int).SetUint64(batch)) + h := crypto.Keccak256Hash([]byte(`upfrontPaymentInBatch(uint256)`)) + calldata := append(h[0:4], dataField...) + msg := ethereum.CallMsg{ + To: &contract, + Data: calldata, + } + bs, err := l1Client.CallContract(context.Background(), msg, nil) + if err != nil { + lg.Error("Failed to call contract", "error", err.Error()) + return nil, err + } + return new(big.Int).SetBytes(bs), nil +} + func prepareData(t *testing.T, l1Client *eth.PollingClient, storageMgr *ethstorage.StorageManager) { // fill contract with almost 2 shards of blobs data := generateRandomBlobs(int(storageMgr.KvEntries()*2) - 1) @@ -265,12 +278,6 @@ func prepareData(t *testing.T, l1Client *eth.PollingClient, storageMgr *ethstora if err != nil { t.Fatalf("Get chain id failed %v", err) } - result, err := l1Client.ReadContractField("upfrontPayment", nil) - if err != nil { - t.Fatal("get upfrontPayment", err) - } - payment := new(big.Int).SetBytes(result) - lg.Info("Query upfront payment", "upfrontPayment", payment) for i := 0; i < txs; i++ { blobsPerTx := maxBlobsPerTx if i == txs-1 { @@ -284,8 +291,11 @@ func prepareData(t *testing.T, l1Client *eth.PollingClient, storageMgr *ethstora if len(blobData) == 0 { break } - totalValue := new(big.Int).Mul(payment, big.NewInt(int64(blobsPerTx))) - totalValue = totalValue.Div(totalValue.Mul(totalValue, big.NewInt(11)), big.NewInt(10)) + totalValue, err := getPayment(l1Client, l1Contract, uint64(blobsPerTx)) + if err != nil { + t.Fatalf("Get payment failed %v", err) + } + t.Logf("Get payment totalValue=%v \n", totalValue) kvIdxes, dataHashes, err := utils.UploadBlobs(l1Client, l1Endpoint, privateKey, chainID.String(), storageMgr.ContractAddress(), blobData, false, totalValue.String()) if err != nil { t.Fatalf("Upload blobs failed: %v", err) diff --git a/integration_tests/run_tests.sh b/integration_tests/run_tests.sh index cfe4f6c0..708f47dc 100755 --- a/integration_tests/run_tests.sh +++ b/integration_tests/run_tests.sh @@ -18,7 +18,7 @@ if [ -z "$ES_NODE_STORAGE_MINER" ]; then fi # A contract that will be update with new blob uploaded for the KZG test if [ -z "$ES_NODE_STORAGE_L1CONTRACT_KZG" ]; then - export ES_NODE_STORAGE_L1CONTRACT_KZG=0xFA067778264A219011D27081483D9930733fb6FC + export ES_NODE_STORAGE_L1CONTRACT_KZG=0xe8F0898cbA701E677970DB33404A817Ff42D4499 fi # A contract address that clef server checks against before signing the miner transaction if [ -z "$ES_NODE_STORAGE_L1CONTRACT_CLEF" ]; then @@ -26,7 +26,7 @@ if [ -z "$ES_NODE_STORAGE_L1CONTRACT_CLEF" ]; then fi # A newly deployed contract is required for each run for miner test, with zkp verifier of mode 2 if [ -z "$ES_NODE_STORAGE_L1CONTRACT" ]; then - export ES_NODE_STORAGE_L1CONTRACT=0x035342b5eAba858fd8C8b91658Bd712fA7C74C7d + export ES_NODE_STORAGE_L1CONTRACT=0xe8F0898cbA701E677970DB33404A817Ff42D4499 fi # A contract with zkp verifier of mode 1 (one proof per sample) if [ -z "$ES_NODE_STORAGE_L1CONTRACT_ZKP1" ]; then @@ -63,5 +63,5 @@ if [ ! -e ${zkey_file} ]; then curl $zkey_url -o ${zkey_file} fi -go test -tags rapidsnark_asm -run ^TestMining$ -timeout 0 github.com/ethstorage/go-ethstorage/integration_tests -v +go test -tags rapidsnark_asm -run ^TestMining$ -timeout 0 github.com/ethstorage/go-ethstorage/integration_tests -v -count=1 \ No newline at end of file From 462f069b3fe22b9e842ddb8921f0d9d022d930de Mon Sep 17 00:00:00 2001 From: syntrust Date: Mon, 19 Aug 2024 19:05:44 +0800 Subject: [PATCH 29/33] fix merge err --- run.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/run.sh b/run.sh index e8ae1647..5cab5994 100755 --- a/run.sh +++ b/run.sh @@ -29,7 +29,6 @@ start_flags=" --network devnet \ --datadir $data_dir \ $file_flags \ --storage.l1contract 0x804C520d3c084C805E37A35E90057Ac32831F96f \ - --storage.miner $ES_NODE_STORAGE_MINER \ --l1.rpc http://88.99.30.186:8545 \ --l1.beacon http://88.99.30.186:3500 \ --l1.beacon-based-time 1706684472 \ From 4b3cbed70ef9e33eccee5919e98a9abef000a685 Mon Sep 17 00:00:00 2001 From: syntrust Date: Wed, 21 Aug 2024 11:24:00 +0800 Subject: [PATCH 30/33] minor --- ethstorage/miner/worker.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethstorage/miner/worker.go b/ethstorage/miner/worker.go index 2ef1308e..c37f92e7 100644 --- a/ethstorage/miner/worker.go +++ b/ethstorage/miner/worker.go @@ -352,7 +352,7 @@ func (w *worker) taskLoop(taskCh chan *taskItem) { w.lg.Info("Mine task success", "shard", ti.shardIdx, "thread", ti.thread, "block", ti.blockNumber) } case <-w.exitCh: - w.lg.Warn("Worker is exiting from task loop...") + w.lg.Debug("Worker is exiting from task loop...") return } } From e320111af7931abe527237752b38478a7ff89337 Mon Sep 17 00:00:00 2001 From: syntrust Date: Mon, 26 Aug 2024 10:06:10 +0800 Subject: [PATCH 31/33] useful log --- ethstorage/miner/worker.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ethstorage/miner/worker.go b/ethstorage/miner/worker.go index c37f92e7..5ef9c502 100644 --- a/ethstorage/miner/worker.go +++ b/ethstorage/miner/worker.go @@ -548,19 +548,19 @@ func (w *worker) mineTask(t *taskItem) (bool, error) { return false, err } if t.requiredDiff.Cmp(new(big.Int).SetBytes(hash1.Bytes())) >= 0 { - w.lg.Info("Calculated a valid hash", "shard", t.shardIdx, "thread", t.thread, "block", t.blockNumber, "nonce", nonce) + w.lg.Info("Calculated a valid hash", "shard", t.shardIdx, "block", t.blockNumber, "timestamp", t.mineTime, "randao", t.mixHash, "nonce", nonce, "hash0", hash0, "hash1", hash1, "sampleIdxs", sampleIdxs) dataSet, kvIdxs, sampleIdxsInKv, encodingKeys, encodedSamples, err := w.getMiningData(t.task, sampleIdxs) if err != nil { w.lg.Error("Get sample data failed", "kvIdxs", kvIdxs, "sampleIdxsInKv", sampleIdxsInKv, "err", err.Error()) return false, err } - w.lg.Info("Got sample data", "shard", t.shardIdx, "thread", t.thread, "block", t.blockNumber, "kvIdxs", kvIdxs, "sampleIdxsInKv", sampleIdxsInKv) + w.lg.Info("Got sample data", "shard", t.shardIdx, "block", t.blockNumber, "encodedSamples", encodedSamples) masks, decodeProof, inclusiveProofs, err := w.prover.GetStorageProof(dataSet, encodingKeys, sampleIdxsInKv) if err != nil { w.lg.Error("Get storage proof error", "kvIdx", kvIdxs, "sampleIdxsInKv", sampleIdxsInKv, "error", err.Error()) return false, fmt.Errorf("get proof err: %v", err) } - w.lg.Info("Got storage proof", "shard", t.shardIdx, "thread", t.thread, "block", t.blockNumber, "kvIdx", kvIdxs, "sampleIdxsInKv", sampleIdxsInKv) + w.lg.Info("Got storage proof", "shard", t.shardIdx, "block", t.blockNumber, "kvIdx", kvIdxs, "sampleIdxsInKv", sampleIdxsInKv) newResult := &result{ blockNumber: t.blockNumber, startShardId: t.shardIdx, From 179012104f411e8cfa482967f29d226fb34952e5 Mon Sep 17 00:00:00 2001 From: syntrust Date: Fri, 30 Aug 2024 15:50:09 +0800 Subject: [PATCH 32/33] support not mining shard0 --- init.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/init.sh b/init.sh index 856bcca5..b99abdbe 100755 --- a/init.sh +++ b/init.sh @@ -15,6 +15,7 @@ zkp_impl=1 zkp_mode=2 remaining_args="" +shards="--shard_index 0" while [ $# -gt 0 ]; do if [[ $1 == --miner.zk-prover-impl ]]; then @@ -24,6 +25,9 @@ while [ $# -gt 0 ]; do zkp_mode=$2 shift 2 else + if [[ $1 == --shard_index ]]; then + shards="" + fi remaining_args="$remaining_args $1" shift fi @@ -101,7 +105,7 @@ fi data_dir="./es-data" -es_node_init="$executable init --shard_index 0 \ +es_node_init="$executable init $shards \ --datadir $data_dir \ --l1.rpc http://88.99.30.186:8545 \ --storage.l1contract 0x804C520d3c084C805E37A35E90057Ac32831F96f \ From 2da0c0dbbbe6529b227ba66aadebb865239926cd Mon Sep 17 00:00:00 2001 From: syntrust Date: Tue, 3 Sep 2024 10:36:46 +0800 Subject: [PATCH 33/33] fix comments --- integration_tests/run_tests.sh | 7 ------- 1 file changed, 7 deletions(-) diff --git a/integration_tests/run_tests.sh b/integration_tests/run_tests.sh index 708f47dc..0b5950fa 100755 --- a/integration_tests/run_tests.sh +++ b/integration_tests/run_tests.sh @@ -49,13 +49,6 @@ echo ES_NODE_L1_ETH_RPC = $ES_NODE_L1_ETH_RPC echo ES_NODE_STORAGE_L1CONTRACT = $ES_NODE_STORAGE_L1CONTRACT echo ES_NODE_STORAGE_MINER = $ES_NODE_STORAGE_MINER -# download zkeys if not yet -# zkey_file="./ethstorage/prover/snark_lib/blob_poseidon.zkey" -# if [ ! -e ${zkey_file} ]; then -# echo "${zkey_file} not found, start downloading..." -# zkey_url="https://drive.usercontent.google.com/download?id=1ZLfhYeCXMnbk6wUiBADRAn1mZ8MI_zg-&export=download&confirm=t&uuid=16ddcd58-2498-4d65-8931-934df3d0065c" -# curl $zkey_url -o ${zkey_file} -# fi zkey_file="./ethstorage/prover/snark_lib/blob_poseidon2.zkey" if [ ! -e ${zkey_file} ]; then echo "${zkey_file} not found, start downloading..."