add retr logic to see if helps

.github/workflows/integration-tests.yaml

@@ -17,7 +17,7 @@ on:
 
 jobs:
   Integration-tests:
-    runs-on: ubuntu-latest  # Ensure the correct runner is used
+    runs-on: runs-on,runner=2cpu-linux-x64,run-id=${{ github.run_id }}
 
     steps:
       - name: Checkout code
@@ -66,5 +66,5 @@ jobs:
 
       # Run Integration Tests
       - name: Run Integration Tests
-        run: cargo test --manifest-path integration-tests/Cargo.toml
+        run: cargo test --manifest-path integration-tests/Cargo.toml -- --nocapture
         working-directory: rust

rust/integration-tests/src/diff_test_helper/default_processor.rs

@@ -1,14 +1,11 @@
-use crate::models::default_models::{BlockMetadataTransaction, CurrentTableItem, TableItem};
+use crate::models::default_models::{BlockMetadataTransaction, TableItem};
 use anyhow::Result;
 use diesel::{
     pg::PgConnection,
     query_dsl::methods::{FilterDsl, ThenOrderDsl},
     ExpressionMethods, RunQueryDsl,
 };
-use processor::schema::{
-    block_metadata_transactions::dsl as bmt_dsl, current_table_items::dsl as cti_dsl,
-    table_items::dsl as ti_dsl,
-};
+use processor::schema::{block_metadata_transactions::dsl as bmt_dsl, table_items::dsl as ti_dsl};
 use serde_json::Value;
 use std::collections::HashMap;
 
@@ -22,6 +19,7 @@ pub fn load_data(
     let bmt_result = bmt_dsl::block_metadata_transactions
         .filter(bmt_dsl::version.eq_any(&txn_versions))
         .then_order_by(bmt_dsl::version.asc())
+        .then_order_by(bmt_dsl::block_height.asc())
         .load::<BlockMetadataTransaction>(conn)?;
     result_map.insert(
         "block_metadata_transactions".to_string(),
@@ -34,14 +32,14 @@ pub fn load_data(
         .load::<TableItem>(conn)?;
     result_map.insert("table_items".to_string(), serde_json::to_value(&ti_result)?);
 
-    let cti_result = cti_dsl::current_table_items
-        .filter(cti_dsl::last_transaction_version.eq_any(&txn_versions))
-        .then_order_by(cti_dsl::last_transaction_version.asc())
-        .load::<CurrentTableItem>(conn)?;
-    result_map.insert(
-        "current_table_items".to_string(),
-        serde_json::to_value(&cti_result)?,
-    );
+    // let cti_result = cti_dsl::current_table_items
+    //     .filter(cti_dsl::last_transaction_version.eq_any(&txn_versions))
+    //     .then_order_by(cti_dsl::last_transaction_version.asc())
+    //     .load::<CurrentTableItem>(conn)?;
+    // result_map.insert(
+    //     "current_table_items".to_string(),
+    //     serde_json::to_value(&cti_result)?,
+    // );
 
     // TODO: revisit we need to find out a way to find correct table metadata
     // let tm_result = tm_dsl::table_metadatas

rust/integration-tests/src/diff_test_helper/fungible_asset_processor.rs

@@ -50,6 +50,7 @@ pub fn load_data(
     let fungible_asset_balances_result = fab_dsl::fungible_asset_balances
         .filter(fab_dsl::transaction_version.eq_any(&txn_versions))
         .then_order_by(fab_dsl::transaction_version.asc())
+        .then_order_by(fab_dsl::write_set_change_index.asc())
         .load::<FungibleAssetBalance>(conn);
     let all_fungible_asset_balances = fungible_asset_balances_result?;
     let fungible_asset_balances_json = serde_json::to_string_pretty(&all_fungible_asset_balances)?;

rust/integration-tests/src/lib.rs

@@ -19,6 +19,12 @@ mod scenarios_tests;
 #[cfg(test)]
 mod sdk_tests;
 
+use std::time::Duration;
+use tokio::time::sleep; // You can use tokio's async sleep for delay
+
+const MAX_RETRIES: u32 = 3;
+const RETRY_DELAY: Duration = Duration::from_secs(2);
+
 /// The test context struct holds the test name and the transaction batches.
 pub struct TestContext {
     pub transaction_batches: Vec<Transaction>,
@@ -125,19 +131,64 @@ impl TestContext {
 
             last_version = Some(version);
 
-            // For DiffTest, run verification after each transaction
+            // For DiffTest, run verification after each transaction with retry logic
             if matches!(test_type, TestType::Diff(_)) {
-                test_type.run_verification(&mut conn, &version.to_string(), &verification_f)?;
+                let mut attempts = 0;
+                loop {
+                    attempts += 1;
+                    match test_type.run_verification(
+                        &mut conn,
+                        &version.to_string(),
+                        &verification_f,
+                    ) {
+                        Ok(_) => break,
+                        Err(e) if attempts < MAX_RETRIES => {
+                            eprintln!(
+                                "Verification failed on attempt {}. Retrying... Error: {:?}",
+                                attempts, e
+                            );
+                            sleep(RETRY_DELAY).await;
+                        },
+                        Err(e) => {
+                            return Err(anyhow::anyhow!(
+                                "Verification failed after {} attempts: {:?}",
+                                MAX_RETRIES,
+                                e
+                            ));
+                        },
+                    }
+                }
             }
         }
-        // For ScenarioTest, use the last transaction version if needed
+
+        // For ScenarioTest, use the last transaction version if needed with retry logic
         if matches!(test_type, TestType::Scenario(_)) {
             if let Some(last_version) = last_version {
-                test_type.run_verification(
-                    &mut conn,
-                    &last_version.to_string(),
-                    &verification_f,
-                )?;
+                let mut attempts = 0;
+                loop {
+                    attempts += 1;
+                    match test_type.run_verification(
+                        &mut conn,
+                        &last_version.to_string(),
+                        &verification_f,
+                    ) {
+                        Ok(_) => break,
+                        Err(e) if attempts < MAX_RETRIES => {
+                            eprintln!(
+                                "Verification failed on attempt {}. Retrying... Error: {:?}",
+                                attempts, e
+                            );
+                            sleep(RETRY_DELAY).await;
+                        },
+                        Err(e) => {
+                            return Err(anyhow::anyhow!(
+                                "Verification failed after {} attempts: {:?}",
+                                MAX_RETRIES,
+                                e
+                            ));
+                        },
+                    }
+                }
             } else {
                 return Err(anyhow::anyhow!(
                     "No transactions found to get the last version"