Cardano transaction sets & proof in client cli

docs/website/root/manual/developer-docs/nodes/mithril-client.md

@@ -119,14 +119,15 @@ Display the help menu:
 You should see:
 
 ```bash
-This program shows, downloads, and verifies certified blockchain artifacts.
+This program shows, downloads and verifies certified blockchain artifacts.
 
-Usage: mithril-client-cli [OPTIONS] <COMMAND>
+Usage: mithril-client [OPTIONS] <COMMAND>
 
 Commands:
-  snapshot                    Snapshot commands
-  mithril-stake-distribution  Mithril stake distribution management (alias: msd)
-  help                        Print this message or the help for the given subcommand(s)
+  snapshot                    Snapshot management
+  mithril-stake-distribution  Mithril Stake Distribution management (alias: msd)
+  cardano-transaction         [unstable] Cardano transactions management (alias: ctx)
+  help                        Print this message or the help of the given subcommand(s)
 
 Options:
       --run-mode <RUN_MODE>
@@ -136,17 +137,18 @@ Options:
       --config-directory <CONFIG_DIRECTORY>
           Directory where configuration file is located [default: ./config]
       --aggregator-endpoint <AGGREGATOR_ENDPOINT>
-          Override configuration Aggregator endpoint URL
+          Override configuration Aggregator endpoint URL [env: AGGREGATOR_ENDPOINT=]
       --log-format-json
           Enable JSON output for logs displayed according to verbosity level
       --log-output <LOG_OUTPUT>
           Redirect the logs to a file
+      --unstable
+          Enable unstable commands (such as Cardano Transactions)
   -h, --help
           Print help
   -V, --version
           Print version
 
-
 ```
 
 Run in release mode with the default configuration:
@@ -246,6 +248,15 @@ mithril_client mithril-stake-distribution list
 
 # 6- Download and verify the given Mithril stake distribution
 mithril_client mithril-stake-distribution download $MITHRIL_STAKE_DISTRIBUTION_ARTIFACT_HASH
+
+# 7- List Cardano transaction sets
+mithril_client --unstable cardano-transaction sets list
+
+# 8- Show detailed information about a Cardano transaction sets
+mithril_client --unstable cardano-transaction sets show $CARDANO_TRANSACTION_SETS_HASH 
+
+# 9- Certify that given list of transactions hashes are included in the Cardano transactions set
+mithril_client --unstable cardano-transaction certify $TRANSACTION_HASH_1,$TRANSACTION_HASH_2
 ```
 
 ### Local image
@@ -283,6 +294,15 @@ Here are the subcommands available:
 | **help** | Prints this message or the help for the given subcommand(s)|
 | **list** | Lists available Mithril stake distributions|
 
+### Cardano transactions
+
+| Subcommand | Performed action |
+|------------|------------------|
+| **certify** | Certifies that given list of transactions hashes are included in the Cardano transactions set|
+| **help** | Prints this message or the help for the given subcommand(s)|
+| **sets list** | Lists available Cardano transactions sets|
+| **sets show** | Shows information about a Cardano transaction sets|
+
 ## Configuration parameters
 
 The configuration parameters can be set in either of the following ways:
@@ -296,6 +316,7 @@ Here is a list of the available parameters:
 | Parameter | Command line (long) |  Command line (short) | Environment variable | Description | Default value | Example | Mandatory |
 |-----------|---------------------|:---------------------:|----------------------|-------------|---------------|---------|:---------:|
 | `verbose` | `--verbose` | `-v` | `VERBOSE` | Verbosity level | - | Parsed from the number of occurrences: `-v` for `Warning`, `-vv` for `Info`, `-vvv` for `Debug` and `-vvvv` for `Trace` | :heavy_check_mark: |
+| `unstable` | `--unstable` | - | - | Enable unstable commands | - | - | - |
 | `run_mode` | `--run-mode` | - | `RUN_MODE` | Runtime mode | `dev` | - | :heavy_check_mark: |
 | `network` | - | - | `NETWORK` | Cardano network | - | `testnet` or `mainnet` or `devnet` | :heavy_check_mark: |
 | `aggregator_endpoint` | `--aggregator-endpoint` | - | `AGGREGATOR_ENDPOINT` | Aggregator node endpoint | - | `https://aggregator.pre-release-preview.api.mithril.network/aggregator` | :heavy_check_mark: |
@@ -336,3 +357,23 @@ Here is a list of the available parameters:
 |-----------|---------------------|:---------------------:|----------------------|-------------|---------------|---------|:---------:|
 | `artifact_hash` | `--artifact-hash` | - | - | Hash of the Mithril stake distribution artifact or `latest` for the latest artifact | - | - | :heavy_check_mark: |
 | `download_dir` | `--download-dir` | - | - | Directory where the Mithril stake distribution will be downloaded | . | - | - |
+
+`cardano-transaction --unstable sets show` command:
+
+| Parameter | Command line (long) |  Command line (short) | Environment variable | Description | Default value | Example | Mandatory |
+|-----------|---------------------|:---------------------:|----------------------|-------------|---------------|---------|:---------:|
+| `hash` | `--hash` | - | `HASH` | Cardano transaction sets hash or `latest` for the latest Cardano transaction sets | - | - | :heavy_check_mark: |
+| `json` | `--json` | - | - | Enable JSON output for command results | - | - | - |
+
+`cardano-transaction --unstable sets list` command:
+
+| Parameter | Command line (long) |  Command line (short) | Environment variable | Description | Default value | Example | Mandatory |
+|-----------|---------------------|:---------------------:|----------------------|-------------|---------------|---------|:---------:|
+| `json` | `--json` | - | - | Enable JSON output for command results | - | - | - |
+
+`cardano-transaction --unstable download` command:
+
+| Parameter | Command line (long) |  Command line (short) | Environment variable | Description | Default value | Example | Mandatory |
+|-----------|---------------------|:---------------------:|----------------------|-------------|---------------|---------|:---------:|
+| `transactions_hashes` | `--transactions_hashes` | - | `TRANSACTIONS_HASHES` | Cardano transactions hashes separated by commas | - | - | :heavy_check_mark: |
+| `json` | `--json` | - | - | Enable JSON output for progress logs | - | - | - |

mithril-aggregator/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "mithril-aggregator"
-version = "0.4.36"
+version = "0.4.37"
 description = "A Mithril Aggregator server"
 authors = { workspace = true }
 edition = { workspace = true }

mithril-aggregator/src/database/cardano_transaction_migration.rs

@@ -20,6 +20,21 @@ create table cardano_tx (
 );
 
 create unique index cardano_tx_immutable_file_number_index on cardano_tx(immutable_file_number);
+"#,
+        ),
+        // Migration 2
+        // Fix the `cardano_tx` table index on immutable_file number, incorrectly marked as unique.
+        SqlMigration::new(
+            2,
+            r#"
+-- remove all data from the cardano tx table since a lot of transactions where missing for each
+-- block and we rely on their insert order.
+delete from cardano_tx;
+
+drop index cardano_tx_immutable_file_number_index;
+create index cardano_tx_immutable_file_number_index on cardano_tx(immutable_file_number);
+
+vacuum;
 "#,
         ),
     ]

mithril-aggregator/src/database/provider/cardano_transaction.rs

@@ -122,9 +122,7 @@ impl<'client> Provider<'client> for CardanoTransactionProvider<'client> {
         let aliases = SourceAlias::new(&[("{:cardano_tx:}", "cardano_tx")]);
         let projection = Self::Entity::get_projection().expand(aliases);
 
-        format!(
-            "select {projection} from cardano_tx where {condition} order by transaction_hash desc"
-        )
+        format!("select {projection} from cardano_tx where {condition} order by rowid")
     }
 }
 
@@ -205,7 +203,7 @@ impl CardanoTransactionRepository {
         Self { connection }
     }
 
-    /// Return all the [CardanoTransactionRecord]s in the database.
+    /// Return all the [CardanoTransactionRecord]s in the database using chronological order.
     pub async fn get_all_transactions(&self) -> StdResult<Vec<CardanoTransactionRecord>> {
         let provider = CardanoTransactionProvider::new(&self.connection);
         let filters = WhereCondition::default();
@@ -214,7 +212,8 @@ impl CardanoTransactionRepository {
         Ok(transactions.collect())
     }
 
-    /// Return all the [CardanoTransactionRecord]s in the database up to the given beacon.
+    /// Return all the [CardanoTransactionRecord]s in the database up to the given beacon using
+    /// chronological order.
     pub async fn get_transactions_up_to(
         &self,
         beacon: &Beacon,
@@ -551,22 +550,15 @@ mod tests {
             .await
             .unwrap();
 
-        assert_eq!(
-            cardano_transactions[0..=14]
-                .iter()
-                .cloned()
-                .rev()
-                .collect::<Vec<_>>(),
-            transaction_result
-        );
+        assert_eq!(cardano_transactions[0..=14].to_vec(), transaction_result);
 
         let transaction_result = repository
             .get_up_to(&Beacon::new("".to_string(), 1, 300))
             .await
             .unwrap();
 
         assert_eq!(
-            cardano_transactions.into_iter().rev().collect::<Vec<_>>(),
+            cardano_transactions.into_iter().collect::<Vec<_>>(),
             transaction_result
         );
 
@@ -603,7 +595,6 @@ mod tests {
         let transactions_result = repository.get_all_transactions().await.unwrap();
         let transactions_expected: Vec<CardanoTransactionRecord> = cardano_transactions
             .iter()
-            .rev()
             .map(|tx| tx.clone().into())
             .collect();
 

mithril-aggregator/src/services/prover.rs

@@ -28,7 +28,7 @@ pub trait ProverService: Sync + Send {
 #[cfg_attr(test, automock)]
 #[async_trait]
 pub trait TransactionsRetriever: Sync + Send {
-    /// Get transactions up to given beacon
+    /// Get transactions up to given beacon using chronological order
     async fn get_up_to(&self, beacon: &Beacon) -> StdResult<Vec<CardanoTransaction>>;
 }
 

mithril-client-cli/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "mithril-client-cli"
-version = "0.5.19"
+version = "0.6.0"
 description = "A Mithril Client"
 authors = { workspace = true }
 edition = { workspace = true }
@@ -32,8 +32,7 @@ fs2 = "0.4.3"
 futures = "0.3.28"
 human_bytes = { version = "0.4.3", features = ["fast"] }
 indicatif = { version = "0.17.7", features = ["tokio"] }
-mithril-client = { path = "../mithril-client", features = ["fs"] }
-mithril-common = { path = "../mithril-common", features = ["full"] }
+mithril-client = { path = "../mithril-client", features = ["fs", "unstable"] }
 openssl = { version = "0.10.63", features = ["vendored"], optional = true }
 openssl-probe = { version = "0.1.5", optional = true }
 serde = { version = "1.0.196", features = ["derive"] }
@@ -49,9 +48,6 @@ slog-term = "2.9.0"
 thiserror = "1.0.56"
 tokio = { version = "1.35.1", features = ["full"] }
 
-[dev-dependencies]
-mithril-common = { path = "../mithril-common", features = ["test_http_server"] }
-
 [features]
-portable = ["mithril-common/portable"]
+portable = ["mithril-client/portable"]
 bundle_openssl = ["dep:openssl", "dep:openssl-probe"]