Feat/rebasehive

.circleci/config.yml

@@ -1,79 +1,14 @@
 version: 2.1
-orbs:
-  go: circleci/[email protected]
-
-jobs:
-  # This job builds the hive executable and stores it in the workspace.
-  build:
-    docker:
-      - image: cimg/go:1.19
-    steps:
-      # Build it.
-      - checkout
-      - go/load-cache
-      - go/mod-download
-      - go/save-cache
-      - run: {command: 'go build -ldflags="-extldflags=-static" .'}
-      # Store the executable.
-      - persist_to_workspace:
-          root: .
-          paths: ["hive"]
 
-  # This job runs the smoke test simulations. This requires a virtual
-  # machine instead of the container-based build environment because
-  # hive needs to be able to talk to the docker containers it creates.
-  smoke-tests:
-    machine:
-      image: ubuntu-2004:202201-02
-    steps:
-      - checkout
-      - attach_workspace: {at: "/tmp/build"}
-      - run:
-          command: "/tmp/build/hive --sim=smoke/genesis --client=go-ethereum"
-      - run:
-          command: "/tmp/build/hive --sim=smoke/network --client=go-ethereum"
+setup: true
 
-  # This job also runs the smoke test simulations, but against a remote dockerd.
-  smoke-tests-remote-docker:
-    docker:
-      - image: cimg/base:2022.04
-    steps:
-      - checkout
-      - attach_workspace: {at: "/tmp/build"}
-      - setup_remote_docker: {version: 20.10.14}
-      - run:
-          command: "/tmp/build/hive --sim=smoke/genesis --client=go-ethereum --loglevel 5"
-      - run:
-          command: "/tmp/build/hive --sim=smoke/network --client=go-ethereum --loglevel 5"
-
-  # This job runs the go unit tests.
-  go-test:
-    docker:
-      - image: cimg/go:1.19
-    steps:
-      # Get the source.
-      - checkout
-      - go/load-cache
-      - go/mod-download
-      - go/save-cache
-      # Run the tests.
-      - run:
-          name: "hive module tests"
-          command: "go test -cover ./..."
-      - run:
-          name: "hiveproxy module tests"
-          command: "go test -cover ./..."
-          working_directory: "./hiveproxy"
-      - run:
-          name: "Compile Go simulators"
-          command: ".circleci/compile-simulators.sh"
+orbs:
+  path-filtering: circleci/[email protected]
 
 workflows:
-  main:
+  setup-workflow:
     jobs:
-      - go-test
-      - build
-      - smoke-tests:
-          requires: ["build"]
-      - smoke-tests-remote-docker:
-          requires: ["build"]
+      - path-filtering/filter:
+          mapping: |
+            simulators/portal/.* rust-ci true
+          base-revision: origin/master

.circleci/continue_config.yml

@@ -0,0 +1,106 @@
+version: 2.1
+orbs:
+  go: circleci/[email protected]
+
+parameters:
+  rust-ci:
+    type: boolean
+    default: false  
+
+jobs:
+  # This job builds the hive executable and stores it in the workspace.
+  build:
+    docker:
+      - image: cimg/go:1.21
+    steps:
+      # Build it.
+      - checkout
+      - go/load-cache
+      - go/mod-download
+      - go/save-cache
+      - run: {command: 'go build -ldflags="-s -extldflags=-static" -tags "osusergo netgo static_build" .'}
+      # Store the executable.
+      - persist_to_workspace:
+          root: .
+          paths: ["hive"]
+
+  # This job runs the smoke test simulations. This requires a virtual
+  # machine instead of the container-based build environment because
+  # hive needs to be able to talk to the docker containers it creates.
+  smoke-tests:
+    machine:
+      image: ubuntu-2004:202201-02
+    steps:
+      - checkout
+      - attach_workspace: {at: "/tmp/build"}
+      - run:
+          command: "/tmp/build/hive --sim=smoke/genesis --client=go-ethereum"
+      - run:
+          command: "/tmp/build/hive --sim=smoke/network --client=go-ethereum"
+
+  # This job also runs the smoke test simulations, but against a remote dockerd.
+  smoke-tests-remote-docker:
+    docker:
+      - image: cimg/base:2022.04
+    steps:
+      - checkout
+      - attach_workspace: {at: "/tmp/build"}
+      - setup_remote_docker: {version: 20.10.14}
+      - run:
+          command: "/tmp/build/hive --sim=smoke/genesis --client=go-ethereum --loglevel 5"
+      - run:
+          command: "/tmp/build/hive --sim=smoke/network --client=go-ethereum --loglevel 5"
+
+  # This job runs the go unit tests.
+  go-test:
+    docker:
+      - image: cimg/go:1.21
+    steps:
+      # Get the source.
+      - checkout
+      - go/load-cache
+      - go/mod-download
+      - go/save-cache
+      # Run the tests.
+      - run:
+          name: "hive module tests"
+          command: "go test -cover ./..."
+      - run:
+          name: "hiveproxy module tests"
+          command: "go test -cover ./..."
+          working_directory: "./hiveproxy"
+      - run:
+          name: "Compile Go simulators"
+          command: ".circleci/compile-simulators.sh"
+  # this makes sure the rust code is good
+  rust-simulators:
+    docker:
+      - image: cimg/rust:1.75.0
+    steps:
+      - checkout
+      - run:
+          name: Install rustfmt
+          command: rustup component add rustfmt
+      - run:
+          name: Install Clippy
+          command: rustup component add clippy
+      - run:
+          name: Install Clang
+          command: sudo apt update && sudo apt-get install clang -y
+      - run:
+          name: "Lint, build, test Rust simulators"
+          command: ".circleci/rust-simulators.sh"          
+
+workflows:
+  main:
+    jobs:
+      - go-test
+      - build
+      - smoke-tests:
+          requires: ["build"]
+      - smoke-tests-remote-docker:
+          requires: ["build"]
+  rust-jobs:
+    when: << pipeline.parameters.rust-ci >>
+    jobs:
+      - rust-simulators

.circleci/rust-simulators.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+# This causes the bash script to exit immediately if any commands errors out
+set -e
+
+failed=""
+sims=$(find simulators -name Cargo.toml)
+for d in $sims; do
+    d="$(dirname "$d")"
+    echo "Lint, build, test $d"
+    ( cd "$d" || exit 1;
+    cargo fmt --all -- --check;
+    cargo clippy --all --all-targets --all-features --no-deps -- --deny warnings;
+    cargo test --workspace -- --nocapture;
+    )
+done

.gitignore

@@ -3,3 +3,7 @@ workspace
 .idea/
 # build output
 /hive
+
+# build output for rust simulators files
+simulators/**/Cargo.lock
+simulators/**/target

clients/besu/Dockerfile

@@ -17,7 +17,6 @@ COPY genesis.json /genesis.json
 COPY mapper.jq /mapper.jq
 COPY besu.sh /opt/besu/bin/besu-hive.sh
 COPY enode.sh /hive-bin/enode.sh
-COPY trusted_setup.txt /trusted_setup.txt
 
 # Set execute permissions for scripts
 RUN chmod +x /opt/besu/bin/besu-hive.sh /hive-bin/enode.sh

clients/besu/Dockerfile.git

@@ -1,24 +1,28 @@
 ### Build Besu From Git:
 
 ## Builder stage: Compiles besu from a git repository
-FROM openjdk:17-jdk-slim as builder
+FROM ubuntu:22.04 as builder
 
 ARG tag=main
 ARG github=hyperledger/besu
 
-RUN echo "Cloning: $github - $tag" && \
-    apt-get update && apt-get install -y git libsodium-dev libnss3-dev \
+RUN echo "installing java on ubuntu base image" \
+    && apt-get update && apt-get install -y git libsodium-dev libnss3-dev \
+    && apt-get install --no-install-recommends -q --assume-yes ca-certificates-java=20190909* \
+    && apt-get install --no-install-recommends -q --assume-yes openjdk-17-jre-headless=17* libjemalloc-dev=5.* \
+    && echo "Cloning: $github - $tag" \
     && git clone --depth 1 --branch $tag https://github.com/$github \
     && cd besu && ./gradlew installDist
 
 ## Final stage: Sets up the environment for running besu
-FROM openjdk:17-jdk-slim
+FROM ubuntu:22.04
 
 # Copy compiled binary from builder
 COPY --from=builder /besu/build/install/besu /opt/besu
 
 RUN apt-get update && apt-get install -y curl jq libsodium23 libnss3-dev \
-    && apt-get clean && rm -rf /var/lib/apt/lists/*
+    && apt-get install --no-install-recommends -q --assume-yes openjdk-17-jre-headless=17* libjemalloc-dev=5.*  \
+    && apt-get clean && rm -rf /var/lib/apt/lists/* 
 
 # Create version.txt
 RUN /opt/besu/bin/besu --version > /version.txt
@@ -28,7 +32,6 @@ COPY genesis.json /genesis.json
 COPY mapper.jq /mapper.jq
 COPY besu.sh /opt/besu/bin/besu-hive.sh
 COPY enode.sh /hive-bin/enode.sh
-COPY trusted_setup.txt /trusted_setup.txt
 
 # Set execute permissions for scripts
 RUN chmod +x /opt/besu/bin/besu-hive.sh /hive-bin/enode.sh

clients/besu/Dockerfile.local

@@ -1,22 +1,26 @@
 ### Build Besu Locally:
 
 ## Builder stage: Compiles besu from a local directory
-FROM openjdk:17-jdk-slim as builder
+FROM ubuntu:22.04 as builder
 
 # Default local client path: clients/besu/<besu>
 ARG local_path=besu
 COPY $local_path besu
 
 RUN apt-get update && apt-get install -y git libsodium-dev libnss3-dev \
+    && apt-get install --no-install-recommends -q --assume-yes ca-certificates-java=20190909 \
+    && apt-get install --no-install-recommends -q --assume-yes openjdk-17-jre-headless=17* libjemalloc-dev=5.* \
     && cd besu && ./gradlew installDist
 
 ## Final stage: Sets up the environment for running besu
-FROM openjdk:17-jdk-slim
+FROM ubuntu:22.04
 
 # Copy compiled binary from builder
 COPY --from=builder /besu/build/install/besu /opt/besu
 
 RUN apt-get update && apt-get install -y curl jq libsodium23 libnss3-dev \
+    && apt-get install --no-install-recommends -q --assume-yes ca-certificates-java=20190909 \
+    && apt-get install --no-install-recommends -q --assume-yes openjdk-17-jre-headless=17* libjemalloc-dev=5.* \
     && apt-get clean && rm -rf /var/lib/apt/lists/*
 
 # Create version.txt
@@ -27,7 +31,6 @@ COPY genesis.json /genesis.json
 COPY mapper.jq /mapper.jq
 COPY besu.sh /opt/besu/bin/besu-hive.sh
 COPY enode.sh /hive-bin/enode.sh
-COPY trusted_setup.txt /trusted_setup.txt
 
 # Set execute permissions for scripts
 RUN chmod +x /opt/besu/bin/besu-hive.sh /hive-bin/enode.sh

clients/besu/besu.sh

@@ -35,13 +35,11 @@
 #
 #  - HIVE_MINER                enables mining. value is coinbase.
 #  - HIVE_MINER_EXTRA          extra-data field to set for newly minted blocks
-#  - HIVE_SKIP_POW             If set, skip PoW verification
 #  - HIVE_LOGLEVEL             Client log level
 #  - HIVE_GRAPHQL_ENABLED      If set, GraphQL is enabled on port 8545 and RPC is disabled
 #
 # These flags are not supported by the Besu hive client
 #
-#  - HIVE_TESTNET              whether testnet nonces (2^20) are needed
 #  - HIVE_FORK_DAO_VOTE        whether the node support (or opposes) the DAO fork
 
 set -e
@@ -63,9 +61,19 @@ esac
 FLAGS="--logging=$LOG --data-storage-format=BONSAI"
 
 # Configure the chain.
-jq -f /mapper.jq /genesis.json > /besugenesis.json
-echo -n "Genesis: "; cat /besugenesis.json
-FLAGS="$FLAGS --genesis-file=/besugenesis.json "
+mv /genesis.json /genesis-input.json
+jq -f /mapper.jq /genesis-input.json > /genesis.json
+FLAGS="$FLAGS --genesis-file=/genesis.json "
+
+# Dump genesis. 
+if [ "$HIVE_LOGLEVEL" -lt 4 ]; then
+    echo "Supplied genesis state (trimmed, use --sim.loglevel 4 or 5 for full output):"
+    jq 'del(.alloc[] | select(.balance == "0x123450000000000000000"))' /genesis.json
+else
+    echo "Supplied genesis state:"
+    cat /genesis.json
+fi
+
 
 # Enable experimental 'berlin' hard-fork features if configured.
 #if [ -n "$HIVE_FORK_BERLIN" ]; then
@@ -75,10 +83,8 @@ FLAGS="$FLAGS --genesis-file=/besugenesis.json "
 # The client should start after loading the blocks, this option configures it.
 IMPORTFLAGS="--run"
 
-# Disable PoW check if requested.
-if [ -n "$HIVE_SKIP_POW" ]; then
-    IMPORTFLAGS="$IMPORTFLAGS --skip-pow-validation-enabled"
-fi
+# Skip PoW checks on import.
+IMPORTFLAGS="$IMPORTFLAGS --skip-pow-validation-enabled"
 
 # Load chain.rlp if present.
 if [ -f /chain.rlp ]; then
@@ -113,7 +119,7 @@ fi
 if [ "$HIVE_MINER_EXTRA" != "" ]; then
     FLAGS="$FLAGS --miner-extra-data=$HIVE_MINER_EXTRA"
 fi
-FLAGS="$FLAGS --min-gas-price=1 --tx-pool-price-bump=0 --tx-pool-limit-by-account-percentage=1"
+FLAGS="$FLAGS --min-gas-price=1 --tx-pool-price-bump=0"
 
 # Configure peer-to-peer networking.
 if [ "$HIVE_BOOTNODE" != "" ]; then
@@ -150,12 +156,7 @@ RPCFLAGS="$RPCFLAGS --rpc-ws-enabled --rpc-ws-api=DEBUG,ETH,NET,WEB3,ADMIN --rpc
 # Enable merge support if needed
 if [ "$HIVE_TERMINAL_TOTAL_DIFFICULTY" != "" ]; then
     echo "0x7365637265747365637265747365637265747365637265747365637265747365" > /jwtsecret
-    RPCFLAGS="$RPCFLAGS --engine-host-allowlist=* --engine-jwt-enabled --engine-jwt-secret /jwtsecret"
-fi
-
-# Enable KZG trusted setup if cancun timestamp is set, needed for custom genesis on Besu wtih Cancun
-if [ "$HIVE_CANCUN_TIMESTAMP" != "" ]; then
-    FLAGS="$FLAGS --kzg-trusted-setup=/trusted_setup.txt"
+    RPCFLAGS="$RPCFLAGS --engine-host-allowlist=* --engine-jwt-secret /jwtsecret"
 fi
 
 # Start Besu.

clients/besu/mapper.jq

@@ -43,7 +43,9 @@ def to_int:
     "muirGlacierBlock": env.HIVE_FORK_MUIR_GLACIER|to_int,
     "berlinBlock": env.HIVE_FORK_BERLIN|to_int,
     "londonBlock": env.HIVE_FORK_LONDON|to_int,
-    "parisBlock": env.HIVE_MERGE_BLOCK_ID|to_int,
+    "arrowGlacierBlock": env.HIVE_FORK_ARROW_GLACIER|to_int,
+    "grayGlacierBlock": env.HIVE_FORK_GRAY_GLACIER|to_int,
+    "mergeNetsplitBlock": env.HIVE_MERGE_BLOCK_ID|to_int,
     "terminalTotalDifficulty": env.HIVE_TERMINAL_TOTAL_DIFFICULTY|to_int,
     "shanghaiTime": env.HIVE_SHANGHAI_TIMESTAMP|to_int,
     "cancunTime": env.HIVE_CANCUN_TIMESTAMP|to_int,