Merge branch 'audit-q3-2024' into kunal/agg-hook-check-msgValue

.changeset/dirty-clocks-repeat.md

@@ -0,0 +1,5 @@
+---
+'@hyperlane-xyz/utils': patch
+---
+
+Fix objMerge implementation

.changeset/dry-foxes-battle.md

@@ -0,0 +1,6 @@
+---
+'@hyperlane-xyz/cli': minor
+---
+
+Add strategyUrl detect and validation in the beginning of `warp apply`
+Remove yaml transactions print from `warp apply`

.changeset/few-goats-add.md

@@ -0,0 +1,5 @@
+---
+'@hyperlane-xyz/sdk': minor
+---
+
+Enroll new validators. Add tx overrides when deploying ICA accounts. Core checker now surfaces owner violations for defaultHook and requiredHook. App checker temporarily ignores bytecode mismatch violations.

.changeset/flat-swans-help.md

@@ -0,0 +1,7 @@
+---
+'@hyperlane-xyz/utils': minor
+'@hyperlane-xyz/cli': minor
+'@hyperlane-xyz/sdk': minor
+---
+
+Adds new `core check` command to compare local configuration and on chain deployments. Adds memoization to the EvmHookReader to avoid repeating configuration derivation

.changeset/fresh-donkeys-smile.md

@@ -0,0 +1,6 @@
+---
+'@hyperlane-xyz/cli': minor
+'@hyperlane-xyz/sdk': minor
+---
+
+Enable concurrency for IsmModule

.changeset/green-kangaroos-whisper.md

@@ -0,0 +1,5 @@
+---
+'@hyperlane-xyz/core': patch
+---
+
+Refactor TokenRouter internal amount accounting for use in scaling Warp Routes

.changeset/grumpy-ears-relate.md

@@ -0,0 +1,6 @@
+---
+'@hyperlane-xyz/cli': minor
+'@hyperlane-xyz/sdk': minor
+---
+
+Adds the warp check command to compare warp routes config files with on chain warp route deployments

.changeset/itchy-singers-hang.md

@@ -0,0 +1,5 @@
+---
+'@hyperlane-xyz/core': patch
+---
+
+Patched OPL2ToL1Ism to check for correct messageId for external call in verify

.changeset/long-swans-drive.md

@@ -0,0 +1,5 @@
+---
+'@hyperlane-xyz/cli': minor
+---
+
+Add prompt in `warp init` command to choose if a trusted relayer should be used instead of making the choice by default for the user and enable the `--yes` flag to default to a trusted ISM

.changeset/plenty-chicken-clean.md

@@ -0,0 +1,6 @@
+---
+'@hyperlane-xyz/cli': minor
+'@hyperlane-xyz/sdk': minor
+---
+
+Add rebasing yield route support into CLI/SDK

.changeset/red-actors-shop.md

@@ -0,0 +1,5 @@
+---
+'@hyperlane-xyz/core': patch
+---
+
+Added nonce to HypERC4626

.changeset/silver-dancers-rhyme.md

@@ -0,0 +1,5 @@
+---
+'@hyperlane-xyz/infra': minor
+---
+
+Updates the warpIds for Renzo's latest deployment to Sei and Taiko to be used by the Checker

.changeset/sweet-humans-argue.md

@@ -0,0 +1,5 @@
+---
+'@hyperlane-xyz/core': minor
+---
+
+Added PRECISION and rateUpdateNonce to ensure compatibility of HypERC4626

.changeset/tender-spiders-deny.md

@@ -0,0 +1,5 @@
+---
+'@hyperlane-xyz/cli': minor
+---
+
+Update `warp apply` such that it updates in place AND extends in a single call

.changeset/tidy-hornets-beam.md

@@ -0,0 +1,5 @@
+---
+'@hyperlane-xyz/sdk': minor
+---
+
+Add deployments for mainnets: flow, metall2, polynomial

.changeset/tricky-mangos-sin.md

@@ -0,0 +1,5 @@
+---
+'@hyperlane-xyz/cli': minor
+---
+
+updates the multi chain selection prompt by adding search functionality and an optional confirmation prompt for the current selection

.changeset/wise-camels-repair.md

@@ -0,0 +1,5 @@
+---
+'@hyperlane-xyz/sdk': patch
+---
+
+Set transaction overrides and add 10% gas limit buffer when sending message through HyperlaneCore.

.github/workflows/codespell.yml

@@ -21,15 +21,13 @@ jobs:
       - name: Checkout the repository
         uses: actions/checkout@v4
 
-      - name: pip cache
-        uses: buildjet/cache@v4
+      - name: Setup python
+        uses: actions/setup-python@v5
         with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
-          restore-keys: ${{ runner.os }}-pip-
+          python-version: '3.x'
 
-      - name: Install prerequisites
-        run: sudo pip install -r ./.codespell/requirements.txt
+      - name: Install codespell requirements
+        run: pip install -r ./.codespell/requirements.txt
 
       - name: Spell check
         run: codespell --config=./.codespell/.codespellrc

.github/workflows/test.yml

@@ -234,40 +234,6 @@ jobs:
           RUST_BACKTRACE: 'full'
           SEALEVEL_ENABLED: ${{ steps.check-rust-changes.outputs.rust_changes }}
 
-  cli-advanced-e2e:
-    runs-on: ubuntu-latest
-    if: github.event_name == 'push' || (github.event_name == 'pull_request' && github.base_ref == 'main' || github.base_ref == 'cli-2.0') || github.event_name == 'merge_group'
-    needs: [yarn-install]
-    strategy:
-      matrix:
-        include:
-          - test-type: preset_hook_enabled
-          - test-type: configure_hook_enabled
-          - test-type: pi_with_core_chain
-    steps:
-      - uses: actions/setup-node@v4
-        with:
-          node-version: 18
-
-      - uses: actions/checkout@v4
-        with:
-          ref: ${{ github.event.pull_request.head.sha || github.sha }}
-          submodules: recursive
-
-      - name: foundry-install
-        uses: foundry-rs/foundry-toolchain@v1
-
-      - name: yarn-build
-        uses: ./.github/actions/yarn-build-with-cache
-        with:
-          ref: ${{ github.event.pull_request.head.sha || github.sha }}
-
-      - name: Checkout registry
-        uses: ./.github/actions/checkout-registry
-
-      - name: cli e2e tests
-        run: ./typescript/cli/ci-advanced-test.sh ${{ matrix.test-type }}
-
   env-test:
     runs-on: ubuntu-latest
     env:

.registryrc

@@ -1 +1 @@
-3b9f67add3338f151154bc5db76bf4e296f018dd
+8583d0841615313c8c880e765eba760378e061cd

rust/main/agents/relayer/src/msg/gas_payment/mod.rs

@@ -105,7 +105,7 @@ impl GasPaymentEnforcer {
         for (policy, whitelist) in &self.policies {
             if !whitelist.msg_matches(message, true) {
                 trace!(
-                    msg=%message,
+                    hyp_message=%message,
                     ?policy,
                     ?whitelist,
                     "Message did not match whitelist for policy"
@@ -114,13 +114,13 @@ impl GasPaymentEnforcer {
             }
 
             trace!(
-                msg=%message,
+                hyp_message=%message,
                 ?policy,
                 ?whitelist,
                 "Message matched whitelist for policy"
             );
             debug!(
-                msg=%message,
+                hyp_message=%message,
                 ?policy,
                 ?current_payment,
                 ?current_expenditure,
@@ -149,7 +149,7 @@ impl GasPaymentEnforcer {
         }
 
         error!(
-            msg=%message,
+            hyp_message=%message,
             policies=?self.policies,
             "No gas payment policy matched for message; consider adding a default policy to the end of the policies array which uses a wildcard whitelist."
         );
@@ -159,7 +159,7 @@ impl GasPaymentEnforcer {
     pub fn record_tx_outcome(&self, message: &HyperlaneMessage, outcome: TxOutcome) -> Result<()> {
         // This log is required in E2E, hence the use of a `const`
         debug!(
-            msg=%message,
+            hyp_message=%message,
             ?outcome,
             "{}",
             GAS_EXPENDITURE_LOG_MESSAGE,

rust/main/agents/relayer/src/msg/metadata/multisig/base.rs

@@ -126,11 +126,11 @@ impl<T: MultisigIsmMetadataBuilder> MetadataBuilder for T {
             .await
             .context(CTX)?
         {
-            debug!(?message, ?metadata.checkpoint, "Found checkpoint with quorum");
+            debug!(hyp_message=?message, ?metadata.checkpoint, "Found checkpoint with quorum");
             Ok(Some(self.format_metadata(metadata)?))
         } else {
             info!(
-                ?message, ?validators, threshold, ism=%multisig_ism.address(),
+                hyp_message=?message, ?validators, threshold, ism=%multisig_ism.address(),
                 "Could not fetch metadata: Unable to reach quorum"
             );
             Ok(None)

rust/main/agents/relayer/src/msg/metadata/multisig/merkle_root_multisig.rs

@@ -45,7 +45,7 @@ impl MultisigIsmMetadataBuilder for MerkleRootMultisigMetadataBuilder {
                 .await
                 .context(CTX)?,
             debug!(
-                ?message,
+                hyp_message=?message,
                 "No merkle leaf found for message id, must have not been enqueued in the tree"
             )
         );

rust/main/agents/relayer/src/msg/metadata/multisig/message_id_multisig.rs

@@ -42,7 +42,7 @@ impl MultisigIsmMetadataBuilder for MessageIdMultisigMetadataBuilder {
                 .await
                 .context(CTX)?,
             debug!(
-                ?message,
+                hyp_message=?message,
                 "No merkle leaf found for message id, must have not been enqueued in the tree"
             )
         );

rust/main/agents/relayer/src/msg/op_queue.rs

@@ -29,11 +29,10 @@ impl OpQueue {
     /// it's very likely that its status has just changed, so this forces the caller to consider the new status
     #[instrument(skip(self), ret, fields(queue_label=%self.queue_metrics_label), level = "trace")]
     pub async fn push(&self, mut op: QueueOperation, new_status: Option<PendingOperationStatus>) {
-        if let Some(new_status) = new_status {
-            op.set_status(new_status);
-        }
-        // increment the metric before pushing onto the queue, because we lose ownership afterwards
-        self.get_operation_metric(op.as_ref()).inc();
+        op.set_status_and_update_metrics(
+            new_status,
+            Arc::new(self.get_operation_metric(op.as_ref())),
+        );
 
         self.queue.lock().await.push(Reverse(op));
     }
@@ -52,9 +51,6 @@ impl OpQueue {
         let mut queue = self.queue.lock().await;
         let mut popped = vec![];
         while let Some(Reverse(op)) = queue.pop() {
-            // even if the metric is decremented here, the operation may fail to process and be re-added to the queue.
-            // in those cases, the queue length will look like it has spikes whose sizes are at most `limit`
-            self.get_operation_metric(op.as_ref()).dec();
             popped.push(op);
             if popped.len() >= limit {
                 break;
@@ -242,6 +238,12 @@ pub mod test {
         fn set_retries(&mut self, _retries: u32) {
             todo!()
         }
+
+        fn get_metric(&self) -> Option<Arc<IntGauge>> {
+            None
+        }
+
+        fn set_metric(&mut self, _metric: Arc<IntGauge>) {}
     }
 
     pub fn dummy_metrics_and_label() -> (IntGaugeVec, String) {

rust/main/agents/relayer/src/msg/op_submitter.rs

@@ -293,6 +293,7 @@ async fn prepare_task(
                 }
                 PendingOperationResult::Drop => {
                     metrics.ops_dropped.inc();
+                    op.decrement_metric_if_exists();
                 }
                 PendingOperationResult::Confirm(reason) => {
                     debug!(?op, "Pushing operation to confirm queue");
@@ -341,7 +342,7 @@ async fn submit_task(
     }
 }
 
-#[instrument(skip(confirm_queue, metrics), ret, level = "debug")]
+#[instrument(skip(prepare_queue, confirm_queue, metrics), ret, level = "debug")]
 async fn submit_single_operation(
     mut op: QueueOperation,
     prepare_queue: &mut OpQueue,
@@ -369,6 +370,7 @@ async fn submit_single_operation(
         }
         PendingOperationResult::Drop => {
             // Not expected to hit this case in `submit`, but it's here for completeness
+            op.decrement_metric_if_exists();
         }
         PendingOperationResult::Success | PendingOperationResult::Confirm(_) => {
             confirm_op(op, confirm_queue, metrics).await
@@ -457,6 +459,7 @@ async fn confirm_operation(
         PendingOperationResult::Success => {
             debug!(?op, "Operation confirmed");
             metrics.ops_confirmed.inc();
+            op.decrement_metric_if_exists();
         }
         PendingOperationResult::NotReady => {
             confirm_queue.push(op, None).await;
@@ -475,6 +478,7 @@ async fn confirm_operation(
         }
         PendingOperationResult::Drop => {
             metrics.ops_dropped.inc();
+            op.decrement_metric_if_exists();
         }
     }
     operation_result

rust/main/agents/relayer/src/msg/pending_message.rs

@@ -82,6 +82,9 @@ pub struct PendingMessage {
     #[new(default)]
     #[serde(skip_serializing)]
     metadata: Option<Vec<u8>>,
+    #[new(default)]
+    #[serde(skip_serializing)]
+    metric: Option<Arc<IntGauge>>,
 }
 
 impl Debug for PendingMessage {
@@ -481,6 +484,14 @@ impl PendingOperation for PendingMessage {
     fn try_get_mailbox(&self) -> Option<Arc<dyn Mailbox>> {
         Some(self.ctx.destination_mailbox.clone())
     }
+
+    fn get_metric(&self) -> Option<Arc<IntGauge>> {
+        self.metric.clone()
+    }
+
+    fn set_metric(&mut self, metric: Arc<IntGauge>) {
+        self.metric = Some(metric);
+    }
 }
 
 impl PendingMessage {

rust/main/agents/relayer/src/msg/processor.rs

@@ -163,7 +163,7 @@ impl DirectionalNonceIterator {
         if let Some(message) = self.indexed_message_with_nonce()? {
             Self::update_max_nonce_gauge(&message, metrics);
             if !self.is_message_processed()? {
-                debug!(?message, iterator=?self, "Found processable message");
+                debug!(hyp_message=?message, iterator=?self, "Found processable message");
                 return Ok(MessageStatus::Processable(message));
             } else {
                 return Ok(MessageStatus::Processed);

rust/main/chains/hyperlane-cosmos/src/error.rs

@@ -53,9 +53,24 @@ pub enum HyperlaneCosmosError {
     /// Public key error
     #[error("{0}")]
     PublicKeyError(String),
+    /// Address error
+    #[error("{0}")]
+    AddressError(String),
     /// Signer info error
     #[error("{0}")]
     SignerInfoError(String),
+    /// Serde error
+    #[error("{0}")]
+    SerdeError(#[from] serde_json::Error),
+    /// Empty error
+    #[error("{0}")]
+    UnparsableEmptyField(String),
+    /// Parsing error
+    #[error("{0}")]
+    ParsingFailed(String),
+    /// Parsing attempt failed
+    #[error("Parsing attempt failed. (Errors: {0:?})")]
+    ParsingAttemptsFailed(Vec<HyperlaneCosmosError>),
 }
 
 impl From<HyperlaneCosmosError> for ChainCommunicationError {