merge master (#496)

* Check message address against signature (#477)

* Check signing validator's address matches msg address

* Add comments about use of sig data in tests

* Try fix Circle build test failures

* Try fix Circle build test failures, take 2

* tx price heap fix (#471)

* contract_comm/currency/currency.go

* fixed the txn price-sorted min-heap

* merge master (#490)

* Add precompiles to access validator set (#441)

* set max gas to double of the charged gas for the 'intrinsic' smart contract calls (#472)

* set max gas to double of the charged gas for the 'intrinsic' evm operations

* addressed PR comments

* addressed pr comment

* Adds Prepared Certificates to ensure Istanbul liveness (#366)

* Check message address against signature (#477)

* Check signing validator's address matches msg address

* Add comments about use of sig data in tests

* Try fix Circle build test failures

* Try fix Circle build test failures, take 2

* added new option --use-in-memory-discovery-table (#479)

* added new option --use-in-memory-discovery-table

* merge master (#489)

* Adds Prepared Certificates to ensure Istanbul liveness (#366)

* Check message address against signature (#477)

* Check signing validator's address matches msg address

* Add comments about use of sig data in tests

* Try fix Circle build test failures

* Try fix Circle build test failures, take 2

* Allow v4/v5 on a bootnode simultaneously, allow mobile to use discv5 (#454)

* changes for isolating celo networks

* changes to get unit tests working

* changes to add salt in the discovery packets

* removed checking for the ip address when handling a reply

* added ping-ip-from-packet option to bootnode

* for handling expected replies, don't filter on expected sender ip address if --pingIPFromPacket is used

* Add v4 flag

* Add unhandled and quicken docker builds

* Add salt & logs

* Add v4 flag

* Add PeerDiscovery to mobile node config

* Remove logs

* Remove hardcoded bootnodes

* Add salt & turn on discv5

* Delete hardcoded eth bootnodes

* Make Discoveryv5 configurable

* Lint

* Add comment to bootnode v4/v5 handling

* Change PeerDiscovery -> NoDiscovery

* Remove mobile geth no discovery

* Reduce istanbul default timeout, cap exp backoff (#475)

* Reduce istanbul default timeout, cap exp backoff

* Ensure round 0 timeout factors in block period

* Sanitize logs (#495)

* Change registry lookup and infrastructure lookup error logs to debug level

* Sanitize logs regarding registry deployment

* Change empty abi logging and comment

* Lower log level from error to warning for potentially outdated istanbul messages

* Change back to an error message

* Add input length checks for precompiled contracts (#476)

* add input length checks

* check exact input length. add a new error for input length. check input in a few more places

* add tests that verify the input-length checks for contracts that don't require an evm instance

* fix formatting

* add comments to explain input length checks

* run the formatter

* e2e transfer test was failing because it passes in a transaction options object, making the input larger than 96 bytes

* e2e tests have revealed that our precompiled contracts need to be tolerant of inputs that are longer than the bytes that are actually read

.circleci/config.yml

@@ -70,18 +70,13 @@ jobs:
             set -euo pipefail
             export CELO_MONOREPO_DIR="$PWD"
             git clone --depth 1 https://github.com/celo-org/celo-monorepo.git ${CELO_MONOREPO_DIR} -b master
-            cd ${CELO_MONOREPO_DIR}/packages
-            # TODO(ashishb): Delete unnecessary packages to speed up build time.
-            # It would be better whitelist certain packages and delete the rest.
-            # Deletion does not work right now and yarn fails with weird errors.
-            # This will be enabled and resolved later.
-            # rm -rf analytics blockchain-api cli docs faucet helm-charts mobile notification-service react-components transaction-metrics-exporter verification-pool-api verifier web
-            cd ${CELO_MONOREPO_DIR}/packages/celotool
-            yarn || yarn
-            yarn --cwd=${CELO_MONOREPO_DIR}/packages/utils build
-            yarn --cwd=${CELO_MONOREPO_DIR}/packages/walletkit build
-            yarn --cwd=${CELO_MONOREPO_DIR}/packages/celotool build
-
+            yarn install || yarn install
+            # separate build to avoid ENOMEM in CI :(
+            yarn build --scope @celo/utils
+            yarn build --scope @celo/protocol
+            yarn build --scope docs
+            yarn build --scope @celo/walletkit
+            yarn build --ignore @celo/protocol --ignore docs --ignore @celo/walletkit --ignore @celo/web --ignore @celo/mobile --ignore @celo/react-components
       - run:
           name: Setup Go language
           command: |

Dockerfile.alltools

@@ -1,8 +1,8 @@
 FROM ubuntu:16.04 as rustbuilder
 RUN apt update && apt install -y curl musl-tools
-ADD . /go-ethereum
 RUN curl https://sh.rustup.rs -sSf | sh -s -- -y
 RUN $HOME/.cargo/bin/rustup install 1.36.0 && $HOME/.cargo/bin/rustup default 1.36.0 && $HOME/.cargo/bin/rustup target add x86_64-unknown-linux-musl
+ADD ./vendor /go-ethereum/vendor
 RUN cd /go-ethereum/vendor/github.com/celo-org/bls-zexe/bls && $HOME/.cargo/bin/cargo build --target x86_64-unknown-linux-musl --release
 
 # Build Geth in a stock Go builder container

cmd/bootnode/main.go

@@ -27,6 +27,7 @@ import (
 	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/p2p"
 	"github.com/ethereum/go-ethereum/p2p/discover"
 	"github.com/ethereum/go-ethereum/p2p/discv5"
 	"github.com/ethereum/go-ethereum/p2p/enode"
@@ -43,6 +44,7 @@ func main() {
 		nodeKeyHex       = flag.String("nodekeyhex", "", "private key as hex (for testing)")
 		natdesc          = flag.String("nat", "none", "port mapping mechanism (any|none|upnp|pmp|extip:<IP>)")
 		netrestrict      = flag.String("netrestrict", "", "restrict network communication to the given IP networks (CIDR masks)")
+		runv4            = flag.Bool("v4", true, "run a v4 topic discovery bootnode")
 		runv5            = flag.Bool("v5", false, "run a v5 topic discovery bootnode")
 		verbosity        = flag.Int("verbosity", int(log.LvlInfo), "log verbosity (0-9)")
 		vmodule          = flag.String("vmodule", "", "log verbosity pattern")
@@ -64,6 +66,8 @@ func main() {
 		utils.Fatalf("-nat: %v", err)
 	}
 	switch {
+	case !*runv4 && !*runv5:
+		utils.Fatalf("Must specify at least one discovery protocol (v4 or v5)")
 	case *genKey != "":
 		nodeKey, err = crypto.GenerateKey()
 		if err != nil {
@@ -124,22 +128,41 @@ func main() {
 		}
 	}
 
-	if *runv5 {
-		if _, err := discv5.ListenUDP(nodeKey, conn, "", restrictList); err != nil {
-			utils.Fatalf("%v", err)
+	// If v4 and v5 are both enabled, a packet is first tried as v4
+	// and then v5 if v4 decoding fails, following the same pattern as full
+	// nodes that use v4 and v5:
+	// https://github.com/celo-org/celo-blockchain/blob/7fbd6f3574f1c1c1e657c152fc63fb771adab3af/p2p/server.go#L588
+	var unhandled chan discover.ReadPacket
+	var sconn *p2p.SharedUDPConn
+	if *runv4 {
+		if *runv5 {
+			unhandled = make(chan discover.ReadPacket, 100)
+			sconn = &p2p.SharedUDPConn{conn, unhandled}
 		}
-	} else {
 		db, _ := enode.OpenDB("")
 		ln := enode.NewLocalNode(db, nodeKey, *networkId)
 		cfg := discover.Config{
 			PrivateKey:       nodeKey,
 			NetRestrict:      restrictList,
 			PingIPFromPacket: *pingIPFromPacket,
+			Unhandled:        unhandled,
 		}
 		if _, err := discover.ListenUDP(conn, ln, cfg); err != nil {
 			utils.Fatalf("%v", err)
 		}
 	}
 
+	if *runv5 {
+		var err error
+		if sconn != nil {
+			_, err = discv5.ListenUDP(nodeKey, sconn, "", restrictList)
+		} else {
+			_, err = discv5.ListenUDP(nodeKey, conn, "", restrictList)
+		}
+		if err != nil {
+			utils.Fatalf("%v", err)
+		}
+	}
+
 	select {}
 }

cmd/geth/main.go

@@ -139,6 +139,7 @@ var (
 		utils.IstanbulRequestTimeoutFlag,
 		utils.IstanbulBlockPeriodFlag,
 		utils.PingIPFromPacketFlag,
+		utils.UseInMemoryDiscoverTable,
 	}
 
 	rpcFlags = []cli.Flag{

cmd/geth/usage.go

@@ -183,6 +183,8 @@ var AppHelpFlagGroups = []flagGroup{
 			utils.NetrestrictFlag,
 			utils.NodeKeyFileFlag,
 			utils.NodeKeyHexFlag,
+			utils.PingIPFromPacketFlag,
+			utils.UseInMemoryDiscoverTable,
 		},
 	},
 	{

cmd/utils/flags.go

@@ -563,6 +563,10 @@ var (
 		Name:  "ping-ip-from-packet",
 		Usage: "Has the discovery protocol use the IP address given by a ping packet",
 	}
+	UseInMemoryDiscoverTable = cli.BoolFlag{
+		Name:  "use-in-memory-discovery-table",
+		Usage: "Specifies whether to use an in memory discovery table",
+	}
 
 	// ATM the url is left to the user and deployment to
 	JSpathFlag = cli.StringFlag{
@@ -995,6 +999,9 @@ func SetP2PConfig(ctx *cli.Context, cfg *p2p.Config) {
 	if ctx.GlobalIsSet(PingIPFromPacketFlag.Name) {
 		cfg.PingIPFromPacket = true
 	}
+	if ctx.GlobalIsSet(UseInMemoryDiscoverTable.Name) {
+		cfg.UseInMemoryNodeDatabase = true
+	}
 
 	// if we're running a light client or server, force enable the v5 peer discovery
 	// unless it is explicitly disabled with --nodiscover note that explicitly specifying

consensus/istanbul/backend/announce.go

@@ -314,7 +314,7 @@ func (sb *Backend) handleIstAnnounce(payload []byte) error {
 
 		newValEnode := &validatorEnode{enodeURL: enodeUrl, view: msg.View}
 		if err := sb.valEnodeTable.upsert(msg.Address, newValEnode, valSet, sb.Address()); err != nil {
-			sb.logger.Error("Error in upserting a valenode entry", "AnnounceMsg", msg, "error", err)
+			sb.logger.Warn("Error in upserting a valenode entry", "AnnounceMsg", msg, "error", err)
 			return err
 		}
 	}

consensus/istanbul/backend/engine.go

@@ -456,14 +456,15 @@ func (sb *Backend) IsLastBlockOfEpoch(header *types.Header) bool {
 func (sb *Backend) Finalize(chain consensus.ChainReader, header *types.Header, state *state.StateDB, txs []*types.Transaction, uncles []*types.Header, receipts []*types.Receipt, randomness *types.Randomness) (*types.Block, error) {
 	// Trigger an update to the gas price minimum in the GasPriceMinimum contract based on block congestion
 	updatedGasPriceMinimum, err := gpm.UpdateGasPriceMinimum(header, state)
-
-	if err != nil {
+	if err == contract_errors.ErrRegistryContractNotDeployed {
+		log.Debug("Error in updating gas price minimum", "error", err, "updatedGasPriceMinimum", updatedGasPriceMinimum)
+	} else if err != nil {
 		log.Error("Error in updating gas price minimum", "error", err, "updatedGasPriceMinimum", updatedGasPriceMinimum)
 	}
 
 	goldTokenAddress, err := contract_comm.GetRegisteredAddress(params.GoldTokenRegistryId, header, state)
-	if err == contract_errors.ErrSmartContractNotDeployed {
-		log.Warn("Registry address lookup failed", "err", err)
+	if err == contract_errors.ErrSmartContractNotDeployed || err == contract_errors.ErrRegistryContractNotDeployed {
+		log.Debug("Registry address lookup failed", "err", err)
 	} else if err != nil {
 		log.Error(err.Error())
 	}
@@ -474,7 +475,7 @@ func (sb *Backend) Finalize(chain consensus.ChainReader, header *types.Header, s
 		infrastructureBlockReward := big.NewInt(params.Ether)
 		governanceAddress, err := contract_comm.GetRegisteredAddress(params.GovernanceRegistryId, header, state)
 		if err == contract_errors.ErrSmartContractNotDeployed {
-			log.Warn("Registry address lookup failed", "err", err)
+			log.Debug("Registry address lookup failed", "err", err)
 		} else if err != nil {
 			log.Error(err.Error())
 		}
@@ -487,7 +488,7 @@ func (sb *Backend) Finalize(chain consensus.ChainReader, header *types.Header, s
 		stakerBlockReward := big.NewInt(params.Ether)
 		lockedGoldAddress, err := contract_comm.GetRegisteredAddress(params.LockedGoldRegistryId, header, state)
 		if err == contract_errors.ErrSmartContractNotDeployed {
-			log.Warn("Registry address lookup failed", "err", err, "contract id", params.LockedGoldRegistryId)
+			log.Debug("Registry address lookup failed", "err", err, "contract id", params.LockedGoldRegistryId)
 		} else if err != nil {
 			log.Error(err.Error())
 		}

consensus/istanbul/config.go

@@ -31,7 +31,7 @@ type Config struct {
 }
 
 var DefaultConfig = &Config{
-	RequestTimeout: 10000,
+	RequestTimeout: 3000,
 	BlockPeriod:    1,
 	ProposerPolicy: RoundRobin,
 	Epoch:          30000,

consensus/istanbul/core/core.go

@@ -396,11 +396,14 @@ func (c *core) newRoundChangeTimer() {
 func (c *core) newRoundChangeTimerForView(view *istanbul.View) {
 	c.stopTimer()
 
-	// set timeout based on the round number
 	timeout := time.Duration(c.config.RequestTimeout) * time.Millisecond
 	round := view.Round.Uint64()
-	if round > 0 {
-		timeout += time.Duration(math.Pow(2, float64(round))) * time.Second
+	if round == 0 {
+		// timeout for first round takes into account expected block period
+		timeout += time.Duration(c.config.BlockPeriod) * time.Second
+	} else {
+		// timeout for subsequent rounds adds an exponential backup, capped at 2**5 = 32s
+		timeout += time.Duration(math.Pow(2, math.Min(float64(round), 5.))) * time.Second
 	}
 
 	c.roundChangeTimer = time.AfterFunc(timeout, func() {

consensus/istanbul/core/handler.go

@@ -108,7 +108,7 @@ func (c *core) handleEvents() {
 			case backlogEvent:
 				// No need to check signature for internal messages
 				if err := c.handleCheckedMsg(ev.msg, ev.src); err != nil {
-					c.logger.Error("Error in handling istanbul message that was sent from a backlog event", "err", err)
+					c.logger.Warn("Error in handling istanbul message that was sent from a backlog event", "err", err)
 				}
 			}
 		case event, ok := <-c.timeoutSub.Chan():

consensus/istanbul/core/testbackend_test.go

@@ -100,7 +100,6 @@ func (self *testSystemBackend) Broadcast(valSet istanbul.ValidatorSet, message [
 	return nil
 }
 func (self *testSystemBackend) Gossip(valSet istanbul.ValidatorSet, message []byte, msgCode uint64, ignoreCache bool) error {
-	testLogger.Warn("not sign any data")
 	return nil
 }
 

consensus/istanbul/core/types_test.go

@@ -122,12 +122,15 @@ func testSubjectWithSignature(t *testing.T) {
 	}
 	expectedSig := []byte{0x01}
 
+	correctAddress := common.HexToAddress("0x1")
+	spooferAddress := common.HexToAddress("0x2")
+
 	subjectPayload, _ := Encode(s)
 	// 1. Encode test
 	m := &istanbul.Message{
 		Code:          istanbul.MsgPreprepare,
 		Msg:           subjectPayload,
-		Address:       common.HexToAddress("0x1234567890"),
+		Address:       correctAddress,
 		Signature:     expectedSig,
 		CommittedSeal: []byte{},
 	}
@@ -141,7 +144,7 @@ func testSubjectWithSignature(t *testing.T) {
 	// 2.1 Test normal validate func
 	decodedMsg := new(istanbul.Message)
 	err = decodedMsg.FromPayload(msgPayload, func(data []byte, sig []byte) (common.Address, error) {
-		return common.Address{}, nil
+		return correctAddress, nil
 	})
 	if err != nil {
 		t.Errorf("error mismatch: have %v, want nil", err)
@@ -170,6 +173,15 @@ func testSubjectWithSignature(t *testing.T) {
 	if err != istanbul.ErrUnauthorizedAddress {
 		t.Errorf("error mismatch: have %v, want %v", err, istanbul.ErrUnauthorizedAddress)
 	}
+
+	// 2.4 Test spoofing signature by another validator validate func
+	decodedMsg = new(istanbul.Message)
+	err = decodedMsg.FromPayload(msgPayload, func(data []byte, sig []byte) (common.Address, error) {
+		return spooferAddress, nil
+	})
+	if err != istanbul.ErrInvalidSigner {
+		t.Errorf("error mismatch: have %v, want ErrInvalidSigner", err)
+	}
 }
 
 func TestMessageEncodeDecode(t *testing.T) {

consensus/istanbul/errors.go

@@ -22,6 +22,8 @@ var (
 	// ErrUnauthorizedAddress is returned when given address cannot be found in
 	// current validator set.
 	ErrUnauthorizedAddress = errors.New("unauthorized address")
+	// ErrInvalidSigner is returned if a message's signature does not correspond to the address in msg.Address
+	ErrInvalidSigner = errors.New("signed by incorrect validator")
 	// ErrStoppedEngine is returned if the engine is stopped
 	ErrStoppedEngine = errors.New("stopped engine")
 	// ErrStartedEngine is returned if the engine is already started

consensus/istanbul/types.go

@@ -324,10 +324,15 @@ func (m *Message) FromPayload(b []byte, validateFn func([]byte, []byte) (common.
 			return err
 		}
 
-		_, err = validateFn(payload, m.Signature)
+		signed_val_addr, err := validateFn(payload, m.Signature)
+		if err != nil {
+			return err
+		}
+		if signed_val_addr != m.Address {
+			return ErrInvalidSigner
+		}
 	}
-	// Still return the message even the err is not nil
-	return err
+	return nil
 }
 
 func (m *Message) Payload() ([]byte, error) {

contract_comm/evm.go

@@ -224,7 +224,7 @@ func makeCallWithContractId(registryId [32]byte, abi abi.ABI, funcName string, a
 			log.Warn("Contract not yet deployed", "contractId", registryId)
 			return 0, err
 		} else if err == errors.ErrRegistryContractNotDeployed {
-			log.Warn("Contract Address Registry not yet deployed")
+			log.Debug("Contract Address Registry not yet deployed")
 			return 0, err
 		} else {
 			log.Error("Error in contract communication", "contract id", registryId, "error", err)

contract_comm/random/random.go

@@ -101,8 +101,8 @@ func commitmentDbLocation(commitment common.Hash) []byte {
 
 func address() *common.Address {
 	randomAddress, err := contract_comm.GetRegisteredAddress(params.RandomRegistryId, nil, nil)
-	if err == errors.ErrSmartContractNotDeployed {
-		log.Warn("Registry address lookup failed", "err", err, "contract id", params.RandomRegistryId)
+	if err == errors.ErrSmartContractNotDeployed || err == errors.ErrRegistryContractNotDeployed {
+		log.Debug("Registry address lookup failed", "err", err, "contract id", params.RandomRegistryId)
 	} else if err != nil {
 		log.Error(err.Error())
 	}

core/state_transition.go

@@ -381,7 +381,7 @@ func (st *StateTransition) TransitionDb() (ret []byte, usedGas uint64, failed bo
 		recipientTxFee = new(big.Int).Sub(totalTxFee, infraTxFee)
 		err = st.creditGas(*st.infrastructureAccountAddress, infraTxFee, msg.GasCurrency())
 	} else {
-		log.Error("no infrastructure account address found - sending entire txFee to fee recipient")
+		log.Debug("no infrastructure account address found - sending entire txFee to fee recipient")
 		recipientTxFee = totalTxFee
 	}
 

core/tx_list.go

@@ -578,7 +578,7 @@ func (l *txPricedList) getHeapWithMinHead() (*priceHeap, *types.Transaction) {
 				cheapestTxn = []*types.Transaction(*cheapestHeap)[0]
 			} else {
 				txn := []*types.Transaction(*priceHeap)[0]
-				if currency.Cmp(cheapestTxn.GasPrice(), cheapestTxn.GasCurrency(), txn.GasPrice(), txn.GasCurrency()) < 0 {
+				if currency.Cmp(txn.GasPrice(), txn.GasCurrency(), cheapestTxn.GasPrice(), cheapestTxn.GasCurrency()) < 0 {
 					cheapestHeap = priceHeap
 				}
 			}