Merge branch 'develop' into chore/libzkp-v0.11.0

Dockerfile

@@ -2,7 +2,6 @@
 ARG COMMIT=""
 ARG VERSION=""
 ARG BUILDNUM=""
-ARG LIBSCROLL_ZSTD_VERSION=v0.1.0-rc0-ubuntu20.04
 ARG SCROLL_LIB_PATH=/scroll/lib
 
 # Build libzkp dependency
@@ -27,13 +26,11 @@ FROM scrolltech/go-rust-builder:go-1.20-rust-nightly-2022-12-10 as builder
 
 ADD . /go-ethereum
 
-ARG LIBSCROLL_ZSTD_VERSION
 ARG SCROLL_LIB_PATH
 
 RUN mkdir -p $SCROLL_LIB_PATH
 
 COPY --from=zkp-builder /app/target/release/libzkp.so $SCROLL_LIB_PATH
-RUN wget -O $SCROLL_LIB_PATH/libscroll_zstd.so https://github.com/scroll-tech/da-codec/releases/download/$LIBSCROLL_ZSTD_VERSION/libscroll_zstd.so
 
 ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$SCROLL_LIB_PATH
 ENV CGO_LDFLAGS="-L$SCROLL_LIB_PATH -Wl,-rpath,$SCROLL_LIB_PATH"
@@ -44,20 +41,18 @@ RUN cd /go-ethereum && env GO111MODULE=on go run build/ci.go install -buildtags
 FROM ubuntu:20.04
 
 RUN apt-get -qq update \
-    && apt-get -qq install -y --no-install-recommends ca-certificates wget
+    && apt-get -qq install -y --no-install-recommends ca-certificates
 
 COPY --from=builder /go-ethereum/build/bin/geth /usr/local/bin/
 
-ARG LIBSCROLL_ZSTD_VERSION
 ARG SCROLL_LIB_PATH
 
 RUN mkdir -p $SCROLL_LIB_PATH
 
 COPY --from=zkp-builder /app/target/release/libzkp.so $SCROLL_LIB_PATH
-RUN wget -O $SCROLL_LIB_PATH/libscroll_zstd.so https://github.com/scroll-tech/da-codec/releases/download/$LIBSCROLL_ZSTD_VERSION/libscroll_zstd.so
 
 ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$SCROLL_LIB_PATH
-ENV CGO_LDFLAGS="-L$SCROLL_LIB_PATH -Wl,-rpath,$SCROLL_LIB_PATH"
+ENV CGO_LDFLAGS="-ldl -L$SCROLL_LIB_PATH -Wl,-rpath,$SCROLL_LIB_PATH"
 
 EXPOSE 8545 8546 30303 30303/udp
 ENTRYPOINT ["geth"]

Dockerfile.mockccc

@@ -2,40 +2,22 @@
 ARG COMMIT=""
 ARG VERSION=""
 ARG BUILDNUM=""
-ARG LIBSCROLL_ZSTD_VERSION=v0.1.0-rc0-ubuntu20.04
 ARG SCROLL_LIB_PATH=/scroll/lib
 
 # Build Geth in a stock Go builder container
 FROM scrolltech/go-rust-builder:go-1.20-rust-nightly-2022-12-10 as builder
 
 ADD . /go-ethereum
+RUN cd /go-ethereum && env GO111MODULE=on go run build/ci.go install ./cmd/geth
 
-ARG LIBSCROLL_ZSTD_VERSION
-ARG SCROLL_LIB_PATH
-
-RUN mkdir -p $SCROLL_LIB_PATH
-
-RUN apt-get -qq update && apt-get -qq install -y wget
-RUN wget -O $SCROLL_LIB_PATH/libscroll_zstd.so https://github.com/scroll-tech/da-codec/releases/download/$LIBSCROLL_ZSTD_VERSION/libscroll_zstd.so
-
-ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$SCROLL_LIB_PATH
-ENV CGO_LDFLAGS="-L$SCROLL_LIB_PATH -Wl,-rpath,$SCROLL_LIB_PATH"
+RUN cd /go-ethereum && env GO111MODULE=on go run build/ci.go install ./cmd/geth
 
 # Pull Geth into a second stage deploy alpine container
 FROM ubuntu:20.04
 
-COPY --from=builder /go-ethereum/build/bin/geth /usr/local/bin/
-
-ARG LIBSCROLL_ZSTD_VERSION
-ARG SCROLL_LIB_PATH
-
-RUN mkdir -p $SCROLL_LIB_PATH
+ENV CGO_LDFLAGS="-ldl"
 
-RUN apt-get -qq update && apt-get -qq install -y wget
-RUN wget -O $SCROLL_LIB_PATH/libscroll_zstd.so https://github.com/scroll-tech/da-codec/releases/download/$LIBSCROLL_ZSTD_VERSION/libscroll_zstd.so
-
-ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$SCROLL_LIB_PATH
-ENV CGO_LDFLAGS="-L$SCROLL_LIB_PATH -Wl,-rpath,$SCROLL_LIB_PATH"
+COPY --from=builder /go-ethereum/build/bin/geth /usr/local/bin/
 
 EXPOSE 8545 8546 30303 30303/udp
 ENTRYPOINT ["geth"]

Makefile

@@ -2,29 +2,22 @@
 # with Go source code. If you know what GOPATH is then you probably
 # don't need to bother with make.
 
-.PHONY: geth android ios evm all test clean libzkp libzstd
+.PHONY: geth android ios evm all test clean libzkp
 
 GOBIN = ./build/bin
 GO ?= latest
-GORUN = go run
-LIBSCROLL_ZSTD_VERSION = v0.1.0-rc0-ubuntu20.04
-SCROLL_LIB_PATH = /scroll/lib
+GORUN = env GO111MODULE=on go run
 
 libzkp:
 	cd $(PWD)/rollup/circuitcapacitychecker/libzkp && make libzkp
 
-libzstd:
-	@sudo mkdir -p $(SCROLL_LIB_PATH)
-	@sudo wget -O $(SCROLL_LIB_PATH)/libscroll_zstd.so https://github.com/scroll-tech/da-codec/releases/download/$(LIBSCROLL_ZSTD_VERSION)/libscroll_zstd.so
-
-nccc_geth: libzstd ## geth without circuit capacity checker
-	@LD_LIBRARY_PATH="$(LD_LIBRARY_PATH):$(SCROLL_LIB_PATH)" CGO_LDFLAGS="-L$(SCROLL_LIB_PATH) -Wl,-rpath,$(SCROLL_LIB_PATH)" $(GORUN) build/ci.go install ./cmd/geth
+nccc_geth: ## geth without circuit capacity checker
+	$(GORUN) build/ci.go install ./cmd/geth
 	@echo "Done building."
 	@echo "Run \"$(GOBIN)/geth\" to launch geth."
 
-geth: libzkp libzstd
-	@sudo cp $(PWD)/rollup/circuitcapacitychecker/libzkp/libzkp.so $(SCROLL_LIB_PATH)
-	@LD_LIBRARY_PATH="$(LD_LIBRARY_PATH):$(SCROLL_LIB_PATH)" CGO_LDFLAGS="-L$(SCROLL_LIB_PATH) -Wl,-rpath,$(SCROLL_LIB_PATH)" $(GORUN) build/ci.go install -buildtags circuit_capacity_checker ./cmd/geth
+geth: libzkp
+	$(GORUN) build/ci.go install -buildtags circuit_capacity_checker ./cmd/geth
 	@echo "Done building."
 	@echo "Run \"$(GOBIN)/geth\" to launch geth."
 
@@ -43,14 +36,17 @@ ios:
 	@echo "Done building."
 	@echo "Import \"$(GOBIN)/Geth.framework\" to use the library."
 
-test:
-	./run_test.sh
+test: all
+	# genesis test
+	cd ${PWD}/cmd/geth; go test -test.run TestCustomGenesis
+	# module test
+	$(GORUN) build/ci.go test ./consensus ./core ./eth ./miner ./node ./trie ./rollup/...
 
 lint: ## Run linters.
 	$(GORUN) build/ci.go lint
 
 clean:
-	go clean -cache
+	env GO111MODULE=on go clean -cache
 	rm -fr build/_workspace/pkg/ $(GOBIN)/*
 
 # The devtools target installs tools required for 'go generate'.

build/ci.go

@@ -232,6 +232,9 @@ func doInstall(cmdline []string) {
 	// Show packages during build.
 	gobuild.Args = append(gobuild.Args, "-v")
 
+	// Add -ldl flags for libscroll_zstd.a.
+	gobuild.Args = append(gobuild.Args, "-ldflags", "-extldflags -ldl")
+
 	// Now we choose what we're even building.
 	// Default: collect all 'main' packages in cmd/ and build those.
 	packages := flag.Args()

cmd/geth/main.go

@@ -159,6 +159,8 @@ var (
 		utils.GpoPercentileFlag,
 		utils.GpoMaxGasPriceFlag,
 		utils.GpoIgnoreGasPriceFlag,
+		utils.GpoCongestionThresholdFlag,
+
 		utils.MinerNotifyFullFlag,
 		configFileFlag,
 		utils.CatalystFlag,

cmd/geth/usage.go

@@ -204,6 +204,7 @@ var AppHelpFlagGroups = []flags.FlagGroup{
 			utils.GpoPercentileFlag,
 			utils.GpoMaxGasPriceFlag,
 			utils.GpoIgnoreGasPriceFlag,
+			utils.GpoCongestionThresholdFlag,
 		},
 	},
 	{

cmd/utils/flags.go

@@ -735,6 +735,11 @@ var (
 		Usage: "Gas price below which gpo will ignore transactions",
 		Value: ethconfig.Defaults.GPO.IgnorePrice.Int64(),
 	}
+	GpoCongestionThresholdFlag = cli.IntFlag{
+		Name:  "gpo.congestionthreshold",
+		Usage: "Number of pending transactions to consider the network congested and suggest a minimum tip cap",
+		Value: ethconfig.Defaults.GPO.CongestedThreshold,
+	}
 
 	// Metrics flags
 	MetricsEnabledFlag = cli.BoolFlag{
@@ -1429,6 +1434,9 @@ func setGPO(ctx *cli.Context, cfg *gasprice.Config, light bool) {
 	if ctx.GlobalIsSet(GpoIgnoreGasPriceFlag.Name) {
 		cfg.IgnorePrice = big.NewInt(ctx.GlobalInt64(GpoIgnoreGasPriceFlag.Name))
 	}
+	if ctx.GlobalIsSet(GpoCongestionThresholdFlag.Name) {
+		cfg.CongestedThreshold = ctx.GlobalInt(GpoCongestionThresholdFlag.Name)
+	}
 }
 
 func setTxPool(ctx *cli.Context, cfg *core.TxPoolConfig) {

eth/backend.go

@@ -264,6 +264,7 @@ func New(stack *node.Node, config *ethconfig.Config, l1Client sync_service.EthCl
 	if gpoParams.Default == nil {
 		gpoParams.Default = config.Miner.GasPrice
 	}
+	gpoParams.DefaultBasePrice = new(big.Int).SetUint64(config.TxPool.PriceLimit)
 	eth.APIBackend.gpo = gasprice.NewOracle(eth.APIBackend, gpoParams)
 
 	// Setup DNS discovery iterators.

eth/gasprice/feehistory_test.go

@@ -56,7 +56,7 @@ func TestFeeHistory(t *testing.T) {
 			MaxHeaderHistory: c.maxHeader,
 			MaxBlockHistory:  c.maxBlock,
 		}
-		backend := newTestBackend(t, big.NewInt(16), c.pending)
+		backend := newTestBackend(t, big.NewInt(16), c.pending, 0)
 		oracle := NewOracle(backend, config)
 
 		first, reward, baseFee, ratio, err := oracle.FeeHistory(context.Background(), c.count, c.last, c.percent)

eth/gasprice/gasprice.go

@@ -39,16 +39,19 @@ const sampleNumber = 3 // Number of transactions sampled in a block
 var (
 	DefaultMaxPrice    = big.NewInt(500 * params.GWei)
 	DefaultIgnorePrice = big.NewInt(2 * params.Wei)
+	DefaultBasePrice   = big.NewInt(0)
 )
 
 type Config struct {
-	Blocks           int
-	Percentile       int
-	MaxHeaderHistory int
-	MaxBlockHistory  int
-	Default          *big.Int `toml:",omitempty"`
-	MaxPrice         *big.Int `toml:",omitempty"`
-	IgnorePrice      *big.Int `toml:",omitempty"`
+	Blocks             int
+	Percentile         int
+	MaxHeaderHistory   int
+	MaxBlockHistory    int
+	Default            *big.Int `toml:",omitempty"`
+	MaxPrice           *big.Int `toml:",omitempty"`
+	IgnorePrice        *big.Int `toml:",omitempty"`
+	CongestedThreshold int      // Number of pending transactions to consider the network congested and suggest a minimum tip cap.
+	DefaultBasePrice   *big.Int `toml:",omitempty"` // Base price to set when CongestedThreshold is reached before Curie (EIP 1559).
 }
 
 // OracleBackend includes all necessary background APIs for oracle.
@@ -60,6 +63,7 @@ type OracleBackend interface {
 	ChainConfig() *params.ChainConfig
 	SubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) event.Subscription
 	StateAt(root common.Hash) (*state.StateDB, error)
+	Stats() (pending int, queued int)
 }
 
 // Oracle recommends gas prices based on the content of recent
@@ -75,6 +79,8 @@ type Oracle struct {
 
 	checkBlocks, percentile           int
 	maxHeaderHistory, maxBlockHistory int
+	congestedThreshold                int      // Number of pending transactions to consider the network congested and suggest a minimum tip cap.
+	defaultBasePrice                  *big.Int // Base price to set when CongestedThreshold is reached before Curie (EIP 1559).
 	historyCache                      *lru.Cache
 }
 
@@ -116,6 +122,16 @@ func NewOracle(backend OracleBackend, params Config) *Oracle {
 		maxBlockHistory = 1
 		log.Warn("Sanitizing invalid gasprice oracle max block history", "provided", params.MaxBlockHistory, "updated", maxBlockHistory)
 	}
+	congestedThreshold := params.CongestedThreshold
+	if congestedThreshold < 0 {
+		congestedThreshold = 0
+		log.Warn("Sanitizing invalid gasprice oracle congested threshold", "provided", params.CongestedThreshold, "updated", congestedThreshold)
+	}
+	defaultBasePrice := params.DefaultBasePrice
+	if defaultBasePrice == nil || defaultBasePrice.Int64() < 0 {
+		defaultBasePrice = DefaultBasePrice
+		log.Warn("Sanitizing invalid gasprice oracle default base price", "provided", params.DefaultBasePrice, "updated", defaultBasePrice)
+	}
 
 	cache, _ := lru.New(2048)
 	headEvent := make(chan core.ChainHeadEvent, 1)
@@ -131,15 +147,17 @@ func NewOracle(backend OracleBackend, params Config) *Oracle {
 	}()
 
 	return &Oracle{
-		backend:          backend,
-		lastPrice:        params.Default,
-		maxPrice:         maxPrice,
-		ignorePrice:      ignorePrice,
-		checkBlocks:      blocks,
-		percentile:       percent,
-		maxHeaderHistory: maxHeaderHistory,
-		maxBlockHistory:  maxBlockHistory,
-		historyCache:     cache,
+		backend:            backend,
+		lastPrice:          params.Default,
+		maxPrice:           maxPrice,
+		ignorePrice:        ignorePrice,
+		checkBlocks:        blocks,
+		percentile:         percent,
+		maxHeaderHistory:   maxHeaderHistory,
+		maxBlockHistory:    maxBlockHistory,
+		congestedThreshold: congestedThreshold,
+		defaultBasePrice:   defaultBasePrice,
+		historyCache:       cache,
 	}
 }
 
@@ -170,6 +188,28 @@ func (oracle *Oracle) SuggestTipCap(ctx context.Context) (*big.Int, error) {
 	if headHash == lastHead {
 		return new(big.Int).Set(lastPrice), nil
 	}
+
+	// If pending txs are less than oracle.congestedThreshold, we consider the network to be non-congested and suggest
+	// a minimal tip cap. This is to prevent users from overpaying for gas when the network is not congested and a few
+	// high-priced txs are causing the suggested tip cap to be high.
+	pendingTxCount, _ := oracle.backend.Stats()
+	if pendingTxCount < oracle.congestedThreshold {
+		// Before Curie (EIP-1559), we need to return the total suggested gas price. After Curie we return 1 wei as the tip cap,
+		// as the base fee is set separately or added manually for legacy transactions.
+		// Set price to 1 as otherwise tx with a 0 tip might be filtered out by the default mempool config.
+		price := big.NewInt(1)
+		if !oracle.backend.ChainConfig().IsCurie(head.Number) {
+			price = oracle.defaultBasePrice
+		}
+
+		oracle.cacheLock.Lock()
+		oracle.lastHead = headHash
+		oracle.lastPrice = price
+		oracle.cacheLock.Unlock()
+
+		return new(big.Int).Set(price), nil
+	}
+
 	var (
 		sent, exp int
 		number    = head.Number.Uint64()