[ISSUE #313]🚀Read Commit log file from cmd

rocketmq-cli/Cargo.toml

@@ -11,3 +11,12 @@ readme.workspace = true
 description.workspace = true
 
 [dependencies]
+rocketmq-common = { version = "0.2.0", path = "../rocketmq-common" }
+rocketmq-store = { version = "0.2.0", path = "../rocketmq-store" }
+
+
+clap = { version = "4.2.7", features = ["derive"] }
+tabled = "0.15.0"
+[[bin]]
+name = "rocketmq-cli-rust"
+path = "src/bin/rocketmq_cli.rs"

rocketmq-cli/README.md

@@ -0,0 +1,104 @@
+# Rocketmq-rust cli
+
+## Overview
+
+Provide some command-line tools to read data from RocketMQ files.
+
+## Getting Started
+
+### Requirements
+
+1. rust toolchain MSRV is 1.75.(stable,nightly)
+
+## Run rocketmq-rust cli
+
+**Run the following command to see usage：**
+
+- **windows platform**
+
+  ```cmd
+  cargo run --bin rocketmq-cli-rust -- --help
+
+  RocketMQ CLI(Rust)
+
+  Usage: rocketmq-cli-rust.exe <COMMAND>
+
+  Commands:
+    read-message-log  read message log file
+    help              Print this message or the help of the given subcommand(s)
+
+  Options:
+    -h, --help     Print help
+    -V, --version  Print version
+
+
+  cargo run --bin rocketmq-cli-rust help read-message-log
+  read message log file
+
+  Usage: rocketmq-cli-rust.exe read-message-log [OPTIONS]
+
+  Options:
+    -c, --config <FILE>  message log file path
+    -f, --from <FROM>    The number of data started to be read, default to read from the beginning. start from 0
+    -t, --to <TO>        The position of the data for ending the reading, defaults to reading until the end of the file.
+    -h, --help           Print help
+    -V, --version        Print version
+  ```
+
+- **Linux platform**
+
+  ```shell
+  $ cargo run --bin rocketmq-cli-rust -- --help
+
+  RocketMQ CLI(Rust)
+
+  Usage: rocketmq-cli-rust.exe <COMMAND>
+
+  Commands:
+    read-message-log  read message log file
+    help              Print this message or the help of the given subcommand(s)
+
+  Options:
+    -h, --help     Print help
+    -V, --version  Print version
+
+
+  $ cargo run --bin rocketmq-cli-rust help read-message-log
+  read message log file
+
+  Usage: rocketmq-cli-rust.exe read-message-log [OPTIONS]
+
+  Options:
+    -c, --config <FILE>  message log file path
+    -f, --from <FROM>    The number of data started to be read, default to read from the beginning. start from 0
+    -t, --to <TO>        The position of the data for ending the reading, defaults to reading until the end of the file.
+    -h, --help           Print help
+    -V, --version        Print version$ cargo run --bin rocketmq-namesrv-rust -- --help
+
+  RocketMQ Name server(Rust)
+
+  Usage: rocketmq-namesrv-rust [OPTIONS]
+
+  Options:
+    -p, --port <PORT>  rocketmq name server port [default: 9876]
+    -i, --ip <IP>      rocketmq name server ip [default: 127.0.0.1]
+    -h, --help         Print help
+    -V, --version      Print version
+  ```
+
+### read-message-log Command
+
+example for **`read-message-log`** (Linux platform)
+
+```bash
+$ ./rocketmq-cli-rust read-message-log -c /mnt/c/Users/ljbmx/store/commitlog/00000000000000000000 -f 0 -t 2
+file size: 1073741824B
++----------------------------------+
+| message_id                       |
++----------------------------------+
+| AC16B00100002A9F0000000000000000 |
++----------------------------------+
+| AC16B00100002A9F000000000000032A |
++----------------------------------+
+```
+

rocketmq-cli/src/bin/rocketmq_cli.rs

@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+use clap::Parser;
+use rocketmq_cli::{
+    command_line::{Commands, RootCli},
+    content_show::print_content,
+};
+
+fn main() {
+    let cli = RootCli::parse();
+    match cli.command {
+        Commands::ReadMessageLog { config, from, to } => {
+            print_content(from, to, config);
+        }
+    }
+}

rocketmq-cli/src/command_line.rs

@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+use std::path::PathBuf;
+
+use clap::{Parser, Subcommand};
+
+#[derive(Parser, Debug)]
+#[command(author = "mxsm", version = "0.2.0", about = "RocketMQ CLI(Rust)")]
+pub struct RootCli {
+    #[command(subcommand)]
+    pub command: Commands,
+}
+
+#[derive(Debug, Subcommand)]
+pub enum Commands {
+    #[command(
+        arg_required_else_help = true,
+        author = "mxsm",
+        version = "0.2.0",
+        about = "read message log file"
+    )]
+    ReadMessageLog {
+        #[arg(
+            short,
+            long,
+            value_name = "FILE",
+            default_missing_value = "None",
+            help = "message log file path"
+        )]
+        config: Option<PathBuf>,
+
+        #[arg(
+            short = 'f',
+            long,
+            value_name = "FROM",
+            default_missing_value = "None",
+            help = "The number of data started to be read, default to read from the beginning. \
+                    start from 0"
+        )]
+        from: Option<u32>,
+
+        #[arg(
+            short = 't',
+            long,
+            value_name = "TO",
+            default_missing_value = "None",
+            help = "The position of the data for ending the reading, defaults to reading until \
+                    the end of the file."
+        )]
+        to: Option<u32>,
+    },
+}

rocketmq-cli/src/content_show.rs

@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+use std::{fs, path::PathBuf};
+
+use rocketmq_common::common::message::message_decoder;
+use rocketmq_store::log_file::mapped_file::default_impl_refactor::LocalMappedFile;
+use tabled::{Table, Tabled};
+
+pub fn print_content(from: Option<u32>, to: Option<u32>, path: Option<PathBuf>) {
+    if path.is_none() {
+        println!("path is none");
+        return;
+    }
+    let path_buf = path.unwrap().into_os_string();
+    let file_metadata = fs::metadata(path_buf.clone()).unwrap();
+    println!("file size: {}B", file_metadata.len());
+    let mut mapped_file = LocalMappedFile::new(
+        path_buf.to_os_string().to_string_lossy().to_string(),
+        file_metadata.len(),
+    );
+    // read message number
+    let mut counter = 0;
+    let form = from.unwrap_or_default();
+    let to = match to {
+        None => u32::MAX,
+        Some(value) => value,
+    };
+    let mut current_pos = 0usize;
+    let mut table = vec![];
+    loop {
+        if counter >= to {
+            break;
+        }
+        let bytes = mapped_file.get_bytes(current_pos, 4);
+        if bytes.is_none() {
+            break;
+        }
+        let size_bytes = bytes.unwrap();
+        let size = i32::from_be_bytes(size_bytes[0..4].try_into().unwrap());
+        if size <= 0 {
+            break;
+        }
+        counter += 1;
+        if counter < form {
+            current_pos += size as usize;
+            continue;
+        }
+        let mut msg_bytes = mapped_file.get_bytes(current_pos, size as usize);
+        current_pos += size as usize;
+        if msg_bytes.is_none() {
+            break;
+        }
+        let message =
+            message_decoder::decode(msg_bytes.as_mut().unwrap(), true, false, false, false, true);
+        //parse message bytes and print it
+        match message {
+            None => {}
+            Some(value) => {
+                table.push(MessagePrint {
+                    message_id: value.msg_id.clone(),
+                });
+            }
+        }
+    }
+    println!("{}", Table::new(table));
+}
+
+#[derive(Tabled)]
+struct MessagePrint {
+    message_id: String,
+}
+
+#[cfg(test)]
+mod tests {
+    use std::path::PathBuf;
+
+    use crate::content_show::print_content;
+
+    #[test]
+    pub fn test_print_content() {
+        print_content(
+            Some(0),
+            Some(1),
+            Some(PathBuf::from(
+                "C:\\Users\\ljbmx\\store\\commitlog\\00000000000000000000",
+            )),
+        );
+    }
+}

rocketmq-cli/src/lib.rs

@@ -15,17 +15,5 @@
  * limitations under the License.
  */
 
-pub fn add(left: usize, right: usize) -> usize {
-    left + right
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn it_works() {
-        let result = add(2, 2);
-        assert_eq!(result, 4);
-    }
-}
+pub mod command_line;
+pub mod content_show;

rocketmq-common/Cargo.toml

@@ -24,7 +24,7 @@ serde_json.workspace = true
 num_cpus.workspace = true
 
 #futures
-futures = {version = "0.3",features = ["default","thread-pool"]}
+futures = { version = "0.3", features = ["default", "thread-pool"] }
 futures-util = "0.3"
 futures-core = "0.3"
 futures-executor = "0.3"
@@ -33,7 +33,14 @@ futures-task = "0.3"
 bytes.workspace = true
 lazy_static.workspace = true
 
+
 config.workspace = true
 
 #tools
-dirs.workspace = true
+dirs.workspace = true
+
+byteorder = "1.5.0"
+
+flate2 = { version = "1.0.17", features = ["zlib"], default-features = false }
+lz4_flex = { version = "0.11", default-features = false }
+zstd = "0.13"