Fix evidence submission (#3612)

* Fix evidence submission by using fix in custom branch tendermint-rs

* Check that evidence command saw the evidence in the block

* Skip submitting evidence if client is already frozen or expired

* Skip frozen clients

* Add more delay in standard misbehaviour test

* Use latest tendermint-rs

Cargo.toml

@@ -23,10 +23,17 @@ overflow-checks = true
 
 [patch.crates-io]
 ibc-proto                        = { git = "https://github.com/cosmos/ibc-proto-rs.git", branch = "ccv-protos" }
-# tendermint                       = { git = "https://github.com/informalsystems/tendermint-rs.git", branch = "main" }
-# tendermint-rpc                   = { git = "https://github.com/informalsystems/tendermint-rs.git", branch = "main" }
-# tendermint-proto                 = { git = "https://github.com/informalsystems/tendermint-rs.git", branch = "main" }
-# tendermint-light-client          = { git = "https://github.com/informalsystems/tendermint-rs.git", branch = "main" }
-# tendermint-light-client-verifier = { git = "https://github.com/informalsystems/tendermint-rs.git", branch = "main" }
-# tendermint-light-client-detector = { git = "https://github.com/informalsystems/tendermint-rs.git", branch = "main" }
-# tendermint-testgen               = { git = "https://github.com/informalsystems/tendermint-rs.git", branch = "main" }
+# tendermint                       = { path = "../tendermint-rs/tendermint" }
+# tendermint-rpc                   = { path = "../tendermint-rs/rpc" }
+# tendermint-proto                 = { path = "../tendermint-rs/proto" }
+# tendermint-light-client          = { path = "../tendermint-rs/light-client" }
+# tendermint-light-client-verifier = { path = "../tendermint-rs/light-client-verifier" }
+# tendermint-light-client-detector = { path = "../tendermint-rs/light-client-detector" }
+# tendermint-testgen               = { path = "../tendermint-rs/testgen" }
+# tendermint                       = { git = "https://github.com/informalsystems/tendermint-rs.git", branch = "fix-evidence" }
+# tendermint-rpc                   = { git = "https://github.com/informalsystems/tendermint-rs.git", branch = "fix-evidence" }
+# tendermint-proto                 = { git = "https://github.com/informalsystems/tendermint-rs.git", branch = "fix-evidence" }
+# tendermint-light-client          = { git = "https://github.com/informalsystems/tendermint-rs.git", branch = "fix-evidence" }
+# tendermint-light-client-verifier = { git = "https://github.com/informalsystems/tendermint-rs.git", branch = "fix-evidence" }
+# tendermint-light-client-detector = { git = "https://github.com/informalsystems/tendermint-rs.git", branch = "fix-evidence" }
+# tendermint-testgen               = { git = "https://github.com/informalsystems/tendermint-rs.git", branch = "fix-evidence" }

ci/misbehaviour-ics/double_sign_test.sh

@@ -1,7 +1,13 @@
 #!/bin/bash
 # shellcheck disable=2086,2004
 
-set -eux 
+set -eu
+
+DEBUG=${DEBUG:-false}
+
+if [ "$DEBUG" = true ]; then
+    set -x
+fi
 
 # User balance of stake tokens 
 USER_COINS="100000000000stake"
@@ -13,10 +19,16 @@ NODE_IP="127.0.0.1"
 # Home directory
 HOME_DIR="/tmp/hermes-ics-double-sign"
 
+# Hermes debug
+if [ "$DEBUG" = true ]; then
+    HERMES_DEBUG="--debug=rpc"
+else
+    HERMES_DEBUG=""
+fi
 # Hermes config
 HERMES_CONFIG="$HOME_DIR/hermes.toml"
 # Hermes binary
-HERMES_BIN="cargo run -q --bin hermes -- --config $HERMES_CONFIG"
+HERMES_BIN="cargo run -q --bin hermes -- $HERMES_DEBUG --config $HERMES_CONFIG"
 
 # Validator moniker
 MONIKERS=("coordinator" "alice" "bob")

ci/misbehaviour-ics/light_client_attack_test.sh

@@ -1,7 +1,13 @@
 #!/bin/bash
 # shellcheck disable=2086
-set -eux 
 
+set -eu
+
+DEBUG=${DEBUG:-false}
+
+if [ "$DEBUG" = true ]; then
+    set -x
+fi
 
 diag() {
   echo ">>
@@ -22,12 +28,19 @@ NODE_IP="127.0.0.1"
 # Home directory
 HOME_DIR="/tmp/hermes-ics-misbehaviour"
 
+# Hermes debug
+if [ "$DEBUG" = true ]; then
+    HERMES_DEBUG="--debug=rpc"
+else
+    HERMES_DEBUG=""
+fi
+
 # Hermes config
 HERMES_CONFIG="$HOME_DIR/hermes.toml"
 HERMES_CONFIG_FORK="$HOME_DIR/hermes-fork.toml"
 # Hermes binary
-HERMES_BIN="cargo run -q --bin hermes -- --config $HERMES_CONFIG"
-HERMES_BIN_FORK="cargo run -q --bin hermes -- --config $HERMES_CONFIG_FORK"
+HERMES_BIN="cargo run -q --bin hermes -- $HERMES_DEBUG --config $HERMES_CONFIG"
+HERMES_BIN_FORK="cargo run -q --bin hermes -- $HERMES_DEBUG --config $HERMES_CONFIG_FORK"
 
 # Validator moniker
 MONIKER="coordinator"
@@ -469,3 +482,11 @@ else
 fi
 
 sleep 10
+
+if grep -q "found light client attack evidence" ${HOME_DIR}/hermes-evidence-logs.txt; then
+    diag "Evidence found, success!"
+else
+    diag "Evidence not found, aborting."
+    exit 1
+fi
+

ci/misbehaviour/misbehaviour_test.sh

@@ -42,7 +42,7 @@ info "Update client on ibc-0 against the forked chain ibc-1-f"
 $HERMES --config config_fork.toml update client --client 07-tendermint-0 --host-chain ibc-0
 
 info "Wait for chain ibc-1 to stop..."
-sleep 5
+sleep 10
 
 info "Killing Hermes"
 kill -9 "$HERMES_PID"

crates/chain-registry/Cargo.toml

@@ -14,7 +14,7 @@ description  = """
 [dependencies]
 ibc-proto         = { version = "0.33.0" }
 ibc-relayer-types = { version = "0.25.0", path = "../relayer-types" }
-tendermint-rpc = { version = "0.33.0", features = [
+tendermint-rpc = { version = "0.33.2", features = [
     "http-client",
     "websocket-client",
 ] }

crates/relayer-cli/Cargo.toml

@@ -56,15 +56,15 @@ tracing                  = "0.1.36"
 tracing-subscriber       = { version = "0.3.14", features = ["fmt", "env-filter", "json"]}
 time                     = "0.3"
 [dependencies.tendermint]
-version = "0.33.0"
+version = "0.33.2"
 features = ["secp256k1"]
 
 [dependencies.tendermint-rpc]
-version = "0.33.0"
+version = "0.33.2"
 features = ["http-client", "websocket-client"]
 
 [dependencies.tendermint-light-client-verifier]
-version = "0.33.0"
+version = "0.33.2"
 
 [dependencies.abscissa_core]
 version = "=0.6.0"

crates/relayer-cli/src/commands/evidence.rs

@@ -23,6 +23,7 @@ use ibc_relayer_types::applications::ics28_ccv::msgs::ccv_double_voting::MsgSubm
 use ibc_relayer_types::applications::ics28_ccv::msgs::ccv_misbehaviour::MsgSubmitIcsConsumerMisbehaviour;
 use ibc_relayer_types::clients::ics07_tendermint::header::Header as TendermintHeader;
 use ibc_relayer_types::clients::ics07_tendermint::misbehaviour::Misbehaviour as TendermintMisbehaviour;
+use ibc_relayer_types::core::ics02_client::client_state::ClientState;
 use ibc_relayer_types::core::ics02_client::height::Height;
 use ibc_relayer_types::core::ics02_client::msgs::misbehaviour::MsgSubmitMisbehaviour;
 use ibc_relayer_types::core::ics24_host::identifier::{ChainId, ClientId};
@@ -128,7 +129,7 @@ fn monitor_misbehaviours(
     // target_height: 1
     // iterations: 200
 
-    debug!(
+    info!(
         "checking past {check_past_blocks} blocks for misbehaviour evidence: {}..{}",
         latest_height, target_height
     );
@@ -195,13 +196,13 @@ fn check_misbehaviour_at(
         match evidence {
             tendermint::evidence::Evidence::DuplicateVote(dv) => {
                 warn!("found duplicate vote evidence");
-                debug!("{dv:#?}");
+                trace!("{dv:#?}");
 
                 handle_duplicate_vote(rt.clone(), chain, key_name, *dv)?;
             }
             tendermint::evidence::Evidence::LightClientAttack(lc) => {
                 warn!("found light client attack evidence");
-                debug!("{lc:#?}");
+                trace!("{lc:#?}");
 
                 handle_light_client_attack(rt.clone(), chain, key_name, *lc)?;
             }
@@ -426,25 +427,36 @@ fn submit_light_client_attack_evidence(
     );
 
     let signer = counterparty.get_signer()?;
-
     let common_height = Height::from_tm(evidence.common_height, chain.id());
+
+    if counterparty_client.is_frozen() {
+        warn!(
+            "skipping client `{}` on chain `{}` as it is already frozen",
+            counterparty_client_id,
+            counterparty.id()
+        );
+
+        return Ok(());
+    }
+
     let mut msgs = match counterparty_client.wait_and_build_update_client(common_height) {
         Ok(msgs) => msgs,
+
         Err(e) => {
             warn!(
-                "failed to build update client message for client `{}` on chain `{}`: {e}",
+                "skipping update client message for client `{}` on chain `{}`",
                 counterparty_client_id,
                 counterparty.id()
             );
+            warn!("reason: failed to build update client message: {e}");
 
-            warn!("skipping update client message");
             vec![]
         }
     };
 
     if is_counterparty_provider(chain, counterparty) {
         info!(
-            "submitting CCV misbehaviour for client `{}` on provider chain `{}`",
+            "will submit CCV misbehaviour for client `{}` on provider chain `{}`",
             counterparty_client_id,
             counterparty.id(),
         );
@@ -459,7 +471,7 @@ fn submit_light_client_attack_evidence(
     };
 
     info!(
-        "submitting IBC misbehaviour for client `{}` on chain `{}`",
+        "will submit IBC misbehaviour for client `{}` on chain `{}`",
         counterparty_client_id,
         counterparty.id(),
     );
@@ -561,17 +573,22 @@ fn fetch_all_counterparty_clients(
         let client_state = match client_state {
             Ok((client_state, _)) => client_state,
             Err(e) => {
-                warn!(
+                error!(
                     "failed to fetch client state for counterparty client {client_id}, skipping..."
                 );
-                warn!("reason: {e}");
+                error!("reason: {e}");
 
                 continue;
             }
         };
 
         let counterparty_chain_id = client_state.chain_id();
-        debug!("found counterparty client with id {client_id} on counterparty chain {counterparty_chain_id}");
+        info!("found counterparty client with id {client_id} on counterparty chain {counterparty_chain_id}");
+
+        if client_state.is_frozen() {
+            info!("counterparty client {client_id} is already frozen, skipping...");
+            continue;
+        }
 
         counterparty_clients.push((counterparty_chain_id, client_id.clone()));
     }

crates/relayer-types/Cargo.toml

@@ -44,24 +44,24 @@ num-rational = "0.4.1"
 regex = "1"
 
 [dependencies.tendermint]
-version = "0.33.0"
+version = "0.33.2"
 features = ["clock"]
 
 [dependencies.tendermint-proto]
-version = "0.33.0"
+version = "0.33.2"
 
 [dependencies.tendermint-light-client-verifier]
-version = "0.33.0"
+version = "0.33.2"
 features = ["rust-crypto"]
 
 [dependencies.tendermint-testgen]
-version = "0.33.0"
+version = "0.33.2"
 optional = true
 
 [dev-dependencies]
 env_logger = "0.10.0"
 tracing = { version = "0.1.36", default-features = false }
 tracing-subscriber = { version = "0.3.14", features = ["fmt", "env-filter", "json"] }
 test-log = { version = "0.2.10", features = ["trace"] }
-tendermint-rpc = { version = "0.33.0", features = ["http-client", "websocket-client"] }
-tendermint-testgen = { version = "0.33.0" } # Needed for generating (synthetic) light blocks.
+tendermint-rpc = { version = "0.33.2", features = ["http-client", "websocket-client"] }
+tendermint-testgen = { version = "0.33.2" } # Needed for generating (synthetic) light blocks.