From 7402f430b80602cc30ae4cdf07bf8ccb7a818978 Mon Sep 17 00:00:00 2001 From: Piotr Gankiewicz Date: Mon, 23 Dec 2024 10:16:16 +0100 Subject: [PATCH] Cleanup errors, update dependencies (#1396) --- Cargo.lock | 164 +++--- bench/src/consumer.rs | 10 +- cli/Cargo.toml | 6 +- cli/src/credentials.rs | 34 +- configs/server.toml | 6 - .../server/scenarios/message_size_scenario.rs | 13 +- sdk/Cargo.toml | 3 +- sdk/src/binary/mapper.rs | 477 ++++++++++++++---- sdk/src/client.rs | 6 +- .../consumer_groups/create_consumer_group.rs | 11 +- .../consumer_offsets/get_consumer_offset.rs | 6 +- .../consumer_offsets/store_consumer_offset.rs | 12 +- sdk/src/error.rs | 124 +++-- sdk/src/http/client.rs | 23 +- sdk/src/http/consumer_groups.rs | 15 +- sdk/src/http/consumer_offsets.rs | 5 +- sdk/src/http/messages.rs | 5 +- sdk/src/http/personal_access_tokens.rs | 15 +- sdk/src/http/streams.rs | 15 +- sdk/src/http/system.rs | 20 +- sdk/src/http/topics.rs | 15 +- sdk/src/http/users.rs | 20 +- sdk/src/messages/flush_unsaved_buffer.rs | 6 +- sdk/src/messages/poll_messages.rs | 24 +- sdk/src/messages/send_messages.rs | 16 +- sdk/src/models/header.rs | 67 ++- sdk/src/partitions/create_partitions.rs | 6 +- sdk/src/partitions/delete_partitions.rs | 6 +- .../create_personal_access_token.rs | 10 +- .../delete_personal_access_token.rs | 4 +- .../login_with_personal_access_token.rs | 4 +- sdk/src/quic/client.rs | 74 ++- sdk/src/streams/create_stream.rs | 10 +- sdk/src/streams/update_stream.rs | 5 +- sdk/src/system/get_client.rs | 7 +- sdk/src/tcp/client.rs | 68 ++- sdk/src/topics/create_topic.rs | 29 +- sdk/src/topics/update_topic.rs | 17 +- sdk/src/users/change_password.rs | 9 +- sdk/src/users/create_user.rs | 15 +- sdk/src/users/login_user.rs | 25 +- sdk/src/users/update_user.rs | 5 +- sdk/src/utils/byte_size.rs | 4 +- server/Cargo.toml | 29 +- server/src/command.rs | 6 +- server/src/compat/mod.rs | 1 - .../compat/storage_conversion/converter.rs | 238 --------- server/src/compat/storage_conversion/mod.rs | 299 ----------- .../storage_conversion/persistency/mod.rs | 7 - .../persistency/partitions.rs | 332 ------------ .../persistency/personal_access_tokens.rs | 56 -- .../storage_conversion/persistency/streams.rs | 127 ----- .../storage_conversion/persistency/topics.rs | 197 -------- .../storage_conversion/persistency/users.rs | 57 --- server/src/configs/config_provider.rs | 6 +- server/src/configs/defaults.rs | 1 - server/src/configs/system.rs | 7 - server/src/http/error.rs | 6 - server/src/http/jwt/storage.rs | 31 +- server/src/log/logger.rs | 93 ++-- server/src/quic/quic_sender.rs | 30 +- server/src/state/file.rs | 64 ++- server/src/state/models.rs | 28 +- server/src/streaming/models/messages.rs | 18 +- server/src/streaming/partitions/storage.rs | 31 +- server/src/streaming/persistence/persister.rs | 40 +- server/src/streaming/segments/storage.rs | 205 +++++--- server/src/streaming/streams/topics.rs | 4 +- server/src/streaming/systems/storage.rs | 22 +- server/src/streaming/systems/streams.rs | 21 +- server/src/streaming/systems/system.rs | 12 +- server/src/streaming/topics/messages.rs | 21 +- server/src/streaming/topics/storage.rs | 9 +- server/src/tcp/sender.rs | 21 +- server/src/versioning.rs | 18 +- 75 files changed, 1378 insertions(+), 2075 deletions(-) delete mode 100644 server/src/compat/storage_conversion/converter.rs delete mode 100644 server/src/compat/storage_conversion/mod.rs delete mode 100644 server/src/compat/storage_conversion/persistency/mod.rs delete mode 100644 server/src/compat/storage_conversion/persistency/partitions.rs delete mode 100644 server/src/compat/storage_conversion/persistency/personal_access_tokens.rs delete mode 100644 server/src/compat/storage_conversion/persistency/streams.rs delete mode 100644 server/src/compat/storage_conversion/persistency/topics.rs delete mode 100644 server/src/compat/storage_conversion/persistency/users.rs diff --git a/Cargo.lock b/Cargo.lock index e09ee727c..48f4cf698 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "addr2line" @@ -506,9 +506,9 @@ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "bcrypt" -version = "0.15.1" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e65938ed058ef47d92cf8b346cc76ef48984572ade631927e9937b5ffc7662c7" +checksum = "2b1866ecef4f2d06a0bb77880015fdf2b89e25a1c2e5addacb87e459c86dc67e" dependencies = [ "base64 0.22.1", "blowfish", @@ -769,7 +769,7 @@ dependencies = [ "semver", "serde", "serde_json", - "thiserror 2.0.4", + "thiserror 2.0.9", ] [[package]] @@ -1490,9 +1490,9 @@ dependencies = [ [[package]] name = "error_set" -version = "0.6.6" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "035e6bd945d30eca06493ab2ff1b2c6b72ec71a77ef2572efd2507920d4901e3" +checksum = "f6ad3479a6b04587258cc631600172b97e955d63109f85bc84c78a935960f135" dependencies = [ "error_set_impl", "tracing", @@ -1500,9 +1500,9 @@ dependencies = [ [[package]] name = "error_set_impl" -version = "0.6.6" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfa19096b48e6696de0e2b9b4cb2cc232277f93cfa317f163ff1751a2e4baaea" +checksum = "a43c6448a712f52898b4b25a4efe8a802fa933d6ba0d4e8695346a562dc7aa53" dependencies = [ "indices", "proc-macro2", @@ -1625,16 +1625,6 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "fs2" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" -dependencies = [ - "libc", - "winapi", -] - [[package]] name = "fs_extra" version = "1.3.0" @@ -1736,15 +1726,6 @@ dependencies = [ "slab", ] -[[package]] -name = "fxhash" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" -dependencies = [ - "byteorder", -] - [[package]] name = "generic-array" version = "0.14.7" @@ -2265,7 +2246,7 @@ dependencies = [ [[package]] name = "iggy" -version = "0.6.51" +version = "0.6.60" dependencies = [ "aes-gcm", "ahash 0.8.11", @@ -2292,7 +2273,6 @@ dependencies = [ "keyring", "lazy_static", "passterm", - "pem", "quinn", "regex", "reqwest", @@ -2304,7 +2284,7 @@ dependencies = [ "serde_json", "serde_with", "strum", - "thiserror 2.0.4", + "thiserror 2.0.9", "tokio", "tokio-rustls", "toml", @@ -2315,7 +2295,7 @@ dependencies = [ [[package]] name = "iggy-cli" -version = "0.8.3" +version = "0.8.4" dependencies = [ "ahash 0.8.11", "anyhow", @@ -2325,7 +2305,7 @@ dependencies = [ "iggy", "keyring", "passterm", - "thiserror 1.0.69", + "thiserror 2.0.9", "tokio", "tracing", "tracing-appender", @@ -2988,23 +2968,23 @@ dependencies = [ [[package]] name = "opentelemetry" -version = "0.26.0" +version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "570074cc999d1a58184080966e5bd3bf3a9a4af650c3b05047c2621e7405cd17" +checksum = "ab70038c28ed37b97d8ed414b6429d343a8bbf44c9f79ec854f3a643029ba6d7" dependencies = [ "futures-core", "futures-sink", "js-sys", - "once_cell", "pin-project-lite", "thiserror 1.0.69", + "tracing", ] [[package]] name = "opentelemetry-appender-tracing" -version = "0.26.0" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14f62d9a23c680ab91c74605f5006110768eb67600bb654937fef5c852fb8ec7" +checksum = "ab5feffc321035ad94088a7e5333abb4d84a8726e54a802e736ce9dd7237e85b" dependencies = [ "log", "opentelemetry", @@ -3015,9 +2995,9 @@ dependencies = [ [[package]] name = "opentelemetry-http" -version = "0.26.0" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6351496aeaa49d7c267fb480678d85d1cd30c5edb20b497c48c56f62a8c14b99" +checksum = "10a8a7f5f6ba7c1b286c2fbca0454eaba116f63bbe69ed250b642d36fbb04d80" dependencies = [ "async-trait", "bytes", @@ -3028,9 +3008,9 @@ dependencies = [ [[package]] name = "opentelemetry-otlp" -version = "0.26.0" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29e1f9c8b032d4f635c730c0efcf731d5e2530ea13fa8bef7939ddc8420696bd" +checksum = "91cf61a1868dacc576bf2b2a1c3e9ab150af7272909e80085c3173384fe11f76" dependencies = [ "async-trait", "futures-core", @@ -3044,13 +3024,14 @@ dependencies = [ "thiserror 1.0.69", "tokio", "tonic", + "tracing", ] [[package]] name = "opentelemetry-proto" -version = "0.26.1" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9d3968ce3aefdcca5c27e3c4ea4391b37547726a70893aab52d3de95d5f8b34" +checksum = "a6e05acbfada5ec79023c85368af14abd0b307c015e9064d249b2a950ef459a6" dependencies = [ "opentelemetry", "opentelemetry_sdk", @@ -3060,22 +3041,21 @@ dependencies = [ [[package]] name = "opentelemetry-semantic-conventions" -version = "0.26.0" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db945c1eaea8ac6a9677185357480d215bb6999faa9f691d0c4d4d641eab7a09" +checksum = "bc1b6902ff63b32ef6c489e8048c5e253e2e4a803ea3ea7e783914536eb15c52" [[package]] name = "opentelemetry_sdk" -version = "0.26.0" +version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2c627d9f4c9cdc1f21a29ee4bfbd6028fcb8bcf2a857b43f3abdf72c9c862f3" +checksum = "231e9d6ceef9b0b2546ddf52335785ce41252bc7474ee8ba05bfad277be13ab8" dependencies = [ "async-trait", "futures-channel", "futures-executor", "futures-util", "glob", - "once_cell", "opentelemetry", "percent-encoding", "rand", @@ -3083,6 +3063,7 @@ dependencies = [ "thiserror 1.0.69", "tokio", "tokio-stream", + "tracing", ] [[package]] @@ -3498,7 +3479,7 @@ dependencies = [ "rustc-hash 2.1.0", "rustls", "socket2", - "thiserror 2.0.4", + "thiserror 2.0.9", "tokio", "tracing", ] @@ -3518,7 +3499,7 @@ dependencies = [ "rustls-pki-types", "rustls-platform-verifier", "slab", - "thiserror 2.0.4", + "thiserror 2.0.9", "tinyvec", "tracing", "web-time", @@ -3846,28 +3827,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "rmp" -version = "0.8.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "228ed7c16fa39782c3b3468e974aec2795e9089153cd08ee2e9aefb3613334c4" -dependencies = [ - "byteorder", - "num-traits", - "paste", -] - -[[package]] -name = "rmp-serde" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52e599a477cf9840e92f2cde9a7189e67b42c57532749bf90aea6ec10facd4db" -dependencies = [ - "byteorder", - "rmp", - "serde", -] - [[package]] name = "rust-ini" version = "0.19.0" @@ -4186,9 +4145,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.133" +version = "1.0.134" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377" +checksum = "d00f4175c42ee48b15416f6193a959ba3a0d67fc699a0db9ad12df9f83991c7d" dependencies = [ "itoa", "memchr", @@ -4284,7 +4243,7 @@ dependencies = [ [[package]] name = "server" -version = "0.4.86" +version = "0.4.90" dependencies = [ "ahash 0.8.11", "anyhow", @@ -4309,7 +4268,6 @@ dependencies = [ "futures", "iggy", "jsonwebtoken", - "log", "moka", "openssl", "opentelemetry", @@ -4322,19 +4280,17 @@ dependencies = [ "rcgen", "reqwest", "ring", - "rmp-serde", "rust-s3", "rustls", "rustls-pemfile", "serde", "serde_json", "serde_with", - "sled", "static-toml", "strip-ansi-escapes", "strum", - "sysinfo", - "thiserror 1.0.69", + "sysinfo 0.33.0", + "thiserror 2.0.9", "tikv-jemallocator", "tokio", "tokio-native-tls", @@ -4430,22 +4386,6 @@ dependencies = [ "autocfg", ] -[[package]] -name = "sled" -version = "0.34.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f96b4737c2ce5987354855aed3797279def4ebf734436c6aa4552cf8e169935" -dependencies = [ - "crc32fast", - "crossbeam-epoch", - "crossbeam-utils", - "fs2", - "fxhash", - "libc", - "log", - "parking_lot 0.11.2", -] - [[package]] name = "smallvec" version = "1.13.2" @@ -4613,6 +4553,20 @@ dependencies = [ "windows", ] +[[package]] +name = "sysinfo" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "948512566b1895f93b1592c7574baeb2de842f224f2aab158799ecadb8ebbb46" +dependencies = [ + "core-foundation-sys", + "libc", + "memchr", + "ntapi", + "rayon", + "windows", +] + [[package]] name = "system-configuration" version = "0.6.1" @@ -4676,11 +4630,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.4" +version = "2.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f49a1853cf82743e3b7950f77e0f4d622ca36cf4317cba00c767838bac8d490" +checksum = "f072643fd0190df67a8bab670c20ef5d8737177d6ac6b2e9a236cb096206b2cc" dependencies = [ - "thiserror-impl 2.0.4", + "thiserror-impl 2.0.9", ] [[package]] @@ -4696,9 +4650,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.4" +version = "2.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8381894bb3efe0c4acac3ded651301ceee58a15d47c2e34885ed1908ad667061" +checksum = "7b50fa271071aae2e6ee85f842e2e28ba8cd2c5fb67f11fcb1fd70b276f9e7d4" dependencies = [ "proc-macro2", "quote", @@ -5075,9 +5029,9 @@ dependencies = [ [[package]] name = "tracing-opentelemetry" -version = "0.27.0" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc58af5d3f6c5811462cabb3289aec0093f7338e367e5a33d28c0433b3c7360b" +checksum = "97a971f6058498b5c0f1affa23e7ea202057a7301dbff68e968b2d578bcbd053" dependencies = [ "js-sys", "once_cell", @@ -5257,7 +5211,7 @@ dependencies = [ "regex", "rustc_version", "rustversion", - "sysinfo", + "sysinfo 0.32.1", "time", "vergen-lib", ] @@ -5952,7 +5906,7 @@ dependencies = [ "pbkdf2", "rand", "sha1", - "thiserror 2.0.4", + "thiserror 2.0.9", "time", "zeroize", "zopfli", diff --git a/bench/src/consumer.rs b/bench/src/consumer.rs index 2f916c20c..d6cb86726 100644 --- a/bench/src/consumer.rs +++ b/bench/src/consumer.rs @@ -164,12 +164,10 @@ impl Consumer { let latency = before_poll.elapsed(); if let Err(e) = polled_messages { - if let IggyError::InvalidResponse(code, _, _) = e { - if code == 2010 { - topic_not_found_counter += 1; - if topic_not_found_counter > 1000 { - return Err(e); - } + if matches!(e, IggyError::TopicIdNotFound(_, _)) { + topic_not_found_counter += 1; + if topic_not_found_counter > 1000 { + return Err(e); } } else { return Err(e); diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 60517d696..4e7c15b42 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "iggy-cli" -version = "0.8.3" +version = "0.8.4" edition = "2021" authors = ["bartosz.ciesla@gmail.com"] repository = "https://github.com/iggy-rs/iggy" @@ -20,10 +20,10 @@ anyhow = "1.0.86" clap = { version = "4.5.17", features = ["derive"] } clap_complete = "4.5.26" figlet-rs = "0.1.5" -iggy = { path = "../sdk", features = ["iggy-cli"], version = "0.6.50" } +iggy = { path = "../sdk", features = ["iggy-cli"], version = "0.6.60" } keyring = { version = "3.2.0", features = ["sync-secret-service", "vendored"], optional = true } passterm = "2.0.1" -thiserror = "1.0.61" +thiserror = "2.0.9" tokio = { version = "1.40.0", features = ["full"] } tracing = "0.1.37" tracing-appender = "0.2.2" diff --git a/cli/src/credentials.rs b/cli/src/credentials.rs index e345f20ef..2c8dc4606 100644 --- a/cli/src/credentials.rs +++ b/cli/src/credentials.rs @@ -159,29 +159,17 @@ impl<'a> IggyCredentials<'a> { let login_result = client.login_with_personal_access_token(token_value).await; if let Err(err) = login_result { - match err { - IggyError::InvalidResponse(code, _, _) => { - // Check what does the code means and if that's one of the cases - // when we should delete the session and inform user to try login again - // TODO: improve this part when we have more information about the codes - if code == IggyError::ResourceNotFound(String::new()).as_code() - || code == IggyError::Unauthenticated.as_code() - || code - == IggyError::PersonalAccessTokenExpired( - String::new(), - 0, - ) - .as_code() - { - let server_session = - ServerSession::new(server_address.clone()); - server_session.delete()?; - bail!("Login session expired for Iggy server: {server_address}, please login again or use other authentication method"); - } - } - _ => { - bail!("Problem with server login with token: {token_value}"); - } + if matches!( + err, + IggyError::Unauthenticated + | IggyError::ResourceNotFound(_) + | IggyError::PersonalAccessTokenExpired(_, _) + ) { + let server_session = ServerSession::new(server_address.clone()); + server_session.delete()?; + bail!("Login session expired for Iggy server: {server_address}, please login again or use other authentication method"); + } else { + bail!("Problem with server login with token: {token_value}"); } } } diff --git a/configs/server.toml b/configs/server.toml index 91aa361ca..c034a9f7e 100644 --- a/configs/server.toml +++ b/configs/server.toml @@ -332,12 +332,6 @@ path = "backup" # Subpath of the backup directory where converted segment data is stored after compatibility conversion. path = "compatibility" -# Legacy database configuration - used only for the migration purposes. -#[system.database] -## Path for storing database files. -## Specifies the directory where database files are stored, relative to `system.path`. -#path = "database" - [system.state] # Determines whether to enforce file synchronization on state updates (boolean). # `true` ensures immediate writing of data to disk for durability. diff --git a/integration/tests/server/scenarios/message_size_scenario.rs b/integration/tests/server/scenarios/message_size_scenario.rs index 0b885ad4e..4612f44df 100644 --- a/integration/tests/server/scenarios/message_size_scenario.rs +++ b/integration/tests/server/scenarios/message_size_scenario.rs @@ -3,7 +3,6 @@ use iggy::client::{MessageClient, StreamClient, TopicClient}; use iggy::clients::client::IggyClient; use iggy::consumer::Consumer; use iggy::error::IggyError; -use iggy::error::IggyError::InvalidResponse; use iggy::messages::poll_messages::PollingStrategy; use iggy::messages::send_messages::{Message, Partitioning}; use iggy::models::header::{HeaderKey, HeaderValue}; @@ -65,21 +64,13 @@ pub async fn run(client_factory: &dyn ClientFactory) { send_message_and_check_result( &client, MessageToSend::OfSizeWithHeaders(100_001, 10_000_000), - Err(InvalidResponse( - 4017, - 23, - "Too big headers payload".to_owned(), - )), + Err(IggyError::TooBigHeadersPayload), ) .await; send_message_and_check_result( &client, MessageToSend::OfSizeWithHeaders(100_000, 10_000_001), - Err(InvalidResponse( - 4022, - 23, - "Too big message payload".to_owned(), - )), + Err(IggyError::TooBigMessagePayload), ) .await; diff --git a/sdk/Cargo.toml b/sdk/Cargo.toml index 570c859cb..210caae4e 100644 --- a/sdk/Cargo.toml +++ b/sdk/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "iggy" -version = "0.6.51" +version = "0.6.60" description = "Iggy is the persistent message streaming platform written in Rust, supporting QUIC, TCP and HTTP transport protocols, capable of processing millions of messages per second." edition = "2021" license = "MIT" @@ -40,7 +40,6 @@ humantime = "2.1.0" keyring = { version = "3.2.0", optional = true, features = ["sync-secret-service", "vendored"] } lazy_static = "1.4.0" passterm = { version = "2.0.1", optional = true } -pem = { version = "3.0.4" } quinn = { version = "0.11.5" } regex = "1.10.4" reqwest = { version = "0.12.9", default-features = false, features = ["json", "rustls-tls"] } diff --git a/sdk/src/binary/mapper.rs b/sdk/src/binary/mapper.rs index c1636a8c7..04a56eb9a 100644 --- a/sdk/src/binary/mapper.rs +++ b/sdk/src/binary/mapper.rs @@ -30,46 +30,142 @@ const EMPTY_PERSONAL_ACCESS_TOKENS: Vec = vec![]; const EMPTY_CONSUMER_GROUPS: Vec = vec![]; pub fn map_stats(payload: Bytes) -> Result { - let process_id = u32::from_le_bytes(payload[..4].try_into()?); - let cpu_usage = f32::from_le_bytes(payload[4..8].try_into()?); - let total_cpu_usage = f32::from_le_bytes(payload[8..12].try_into()?); - let memory_usage = u64::from_le_bytes(payload[12..20].try_into()?).into(); - let total_memory = u64::from_le_bytes(payload[20..28].try_into()?).into(); - let available_memory = u64::from_le_bytes(payload[28..36].try_into()?).into(); - let run_time = u64::from_le_bytes(payload[36..44].try_into()?).into(); - let start_time = u64::from_le_bytes(payload[44..52].try_into()?).into(); - let read_bytes = u64::from_le_bytes(payload[52..60].try_into()?).into(); - let written_bytes = u64::from_le_bytes(payload[60..68].try_into()?).into(); - let messages_size_bytes = u64::from_le_bytes(payload[68..76].try_into()?).into(); - let streams_count = u32::from_le_bytes(payload[76..80].try_into()?); - let topics_count = u32::from_le_bytes(payload[80..84].try_into()?); - let partitions_count = u32::from_le_bytes(payload[84..88].try_into()?); - let segments_count = u32::from_le_bytes(payload[88..92].try_into()?); - let messages_count = u64::from_le_bytes(payload[92..100].try_into()?); - let clients_count = u32::from_le_bytes(payload[100..104].try_into()?); - let consumer_groups_count = u32::from_le_bytes(payload[104..108].try_into()?); + let process_id = u32::from_le_bytes( + payload[..4] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); + let cpu_usage = f32::from_le_bytes( + payload[4..8] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); + let total_cpu_usage = f32::from_le_bytes( + payload[8..12] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); + let memory_usage = u64::from_le_bytes( + payload[12..20] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ) + .into(); + let total_memory = u64::from_le_bytes( + payload[20..28] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ) + .into(); + let available_memory = u64::from_le_bytes( + payload[28..36] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ) + .into(); + let run_time = u64::from_le_bytes( + payload[36..44] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ) + .into(); + let start_time = u64::from_le_bytes( + payload[44..52] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ) + .into(); + let read_bytes = u64::from_le_bytes( + payload[52..60] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ) + .into(); + let written_bytes = u64::from_le_bytes( + payload[60..68] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ) + .into(); + let messages_size_bytes = u64::from_le_bytes( + payload[68..76] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ) + .into(); + let streams_count = u32::from_le_bytes( + payload[76..80] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); + let topics_count = u32::from_le_bytes( + payload[80..84] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); + let partitions_count = u32::from_le_bytes( + payload[84..88] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); + let segments_count = u32::from_le_bytes( + payload[88..92] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); + let messages_count = u64::from_le_bytes( + payload[92..100] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); + let clients_count = u32::from_le_bytes( + payload[100..104] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); + let consumer_groups_count = u32::from_le_bytes( + payload[104..108] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); let mut current_position = 108; - let hostname_length = - u32::from_le_bytes(payload[current_position..current_position + 4].try_into()?) as usize; + let hostname_length = u32::from_le_bytes( + payload[current_position..current_position + 4] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ) as usize; let hostname = - from_utf8(&payload[current_position + 4..current_position + 4 + hostname_length])? + from_utf8(&payload[current_position + 4..current_position + 4 + hostname_length]) + .map_err(|_| IggyError::InvalidUtf8)? .to_string(); current_position += 4 + hostname_length; - let os_name_length = - u32::from_le_bytes(payload[current_position..current_position + 4].try_into()?) as usize; - let os_name = from_utf8(&payload[current_position + 4..current_position + 4 + os_name_length])? + let os_name_length = u32::from_le_bytes( + payload[current_position..current_position + 4] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ) as usize; + let os_name = from_utf8(&payload[current_position + 4..current_position + 4 + os_name_length]) + .map_err(|_| IggyError::InvalidUtf8)? .to_string(); current_position += 4 + os_name_length; - let os_version_length = - u32::from_le_bytes(payload[current_position..current_position + 4].try_into()?) as usize; + let os_version_length = u32::from_le_bytes( + payload[current_position..current_position + 4] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ) as usize; let os_version = - from_utf8(&payload[current_position + 4..current_position + 4 + os_version_length])? + from_utf8(&payload[current_position + 4..current_position + 4 + os_version_length]) + .map_err(|_| IggyError::InvalidUtf8)? .to_string(); current_position += 4 + os_version_length; - let kernel_version_length = - u32::from_le_bytes(payload[current_position..current_position + 4].try_into()?) as usize; + let kernel_version_length = u32::from_le_bytes( + payload[current_position..current_position + 4] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ) as usize; let kernel_version = - from_utf8(&payload[current_position + 4..current_position + 4 + kernel_version_length])? + from_utf8(&payload[current_position + 4..current_position + 4 + kernel_version_length]) + .map_err(|_| IggyError::InvalidUtf8)? .to_string(); Ok(Stats { @@ -99,9 +195,21 @@ pub fn map_stats(payload: Bytes) -> Result { } pub fn map_consumer_offset(payload: Bytes) -> Result { - let partition_id = u32::from_le_bytes(payload[..4].try_into()?); - let current_offset = u64::from_le_bytes(payload[4..12].try_into()?); - let stored_offset = u64::from_le_bytes(payload[12..20].try_into()?); + let partition_id = u32::from_le_bytes( + payload[..4] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); + let current_offset = u64::from_le_bytes( + payload[4..12] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); + let stored_offset = u64::from_le_bytes( + payload[12..20] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); Ok(ConsumerOffsetInfo { partition_id, current_offset, @@ -113,8 +221,11 @@ pub fn map_user(payload: Bytes) -> Result { let (user, position) = map_to_user_info(payload.clone(), 0)?; let has_permissions = payload[position]; let permissions = if has_permissions == 1 { - let permissions_length = - u32::from_le_bytes(payload[position + 1..position + 5].try_into()?) as usize; + let permissions_length = u32::from_le_bytes( + payload[position + 1..position + 5] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ) as usize; let permissions = payload.slice(position + 5..position + 5 + permissions_length); Some(Permissions::from_bytes(permissions)?) } else { @@ -168,7 +279,11 @@ pub fn map_personal_access_tokens( } pub fn map_identity_info(payload: Bytes) -> Result { - let user_id = u32::from_le_bytes(payload[..4].try_into()?); + let user_id = u32::from_le_bytes( + payload[..4] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); Ok(IdentityInfo { user_id, access_token: None, @@ -177,7 +292,9 @@ pub fn map_identity_info(payload: Bytes) -> Result { pub fn map_raw_pat(payload: Bytes) -> Result { let token_length = payload[0]; - let token = from_utf8(&payload[1..1 + token_length as usize])?.to_string(); + let token = from_utf8(&payload[1..1 + token_length as usize]) + .map_err(|_| IggyError::InvalidUtf8)? + .to_string(); Ok(RawPersonalAccessToken { token }) } @@ -187,9 +304,21 @@ pub fn map_client(payload: Bytes) -> Result { let length = payload.len(); while position < length { for _ in 0..client.consumer_groups_count { - let stream_id = u32::from_le_bytes(payload[position..position + 4].try_into()?); - let topic_id = u32::from_le_bytes(payload[position + 4..position + 8].try_into()?); - let group_id = u32::from_le_bytes(payload[position + 8..position + 12].try_into()?); + let stream_id = u32::from_le_bytes( + payload[position..position + 4] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); + let topic_id = u32::from_le_bytes( + payload[position + 4..position + 8] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); + let group_id = u32::from_le_bytes( + payload[position + 8..position + 12] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); let consumer_group = ConsumerGroupInfo { stream_id, topic_id, @@ -239,19 +368,51 @@ pub fn map_polled_messages(payload: Bytes) -> Result } let length = payload.len(); - let partition_id = u32::from_le_bytes(payload[..4].try_into()?); - let current_offset = u64::from_le_bytes(payload[4..12].try_into()?); + let partition_id = u32::from_le_bytes( + payload[..4] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); + let current_offset = u64::from_le_bytes( + payload[4..12] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); // Currently ignored - let _messages_count = u32::from_le_bytes(payload[12..16].try_into()?); + let _messages_count = u32::from_le_bytes( + payload[12..16] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); let mut position = 16; let mut messages = Vec::new(); while position < length { - let offset = u64::from_le_bytes(payload[position..position + 8].try_into()?); + let offset = u64::from_le_bytes( + payload[position..position + 8] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); let state = MessageState::from_code(payload[position + 8])?; - let timestamp = u64::from_le_bytes(payload[position + 9..position + 17].try_into()?); - let id = u128::from_le_bytes(payload[position + 17..position + 33].try_into()?); - let checksum = u32::from_le_bytes(payload[position + 33..position + 37].try_into()?); - let headers_length = u32::from_le_bytes(payload[position + 37..position + 41].try_into()?); + let timestamp = u64::from_le_bytes( + payload[position + 9..position + 17] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); + let id = u128::from_le_bytes( + payload[position + 17..position + 33] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); + let checksum = u32::from_le_bytes( + payload[position + 33..position + 37] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); + let headers_length = u32::from_le_bytes( + payload[position + 37..position + 41] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); let headers = if headers_length > 0 { let headers_payload = payload.slice(position + 41..position + 41 + headers_length as usize); @@ -260,7 +421,11 @@ pub fn map_polled_messages(payload: Bytes) -> Result None }; position += headers_length as usize; - let message_length = u32::from_le_bytes(payload[position + 41..position + 45].try_into()?); + let message_length = u32::from_le_bytes( + payload[position + 41..position + 45] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); let payload_range = position + 45..position + 45 + message_length as usize; if payload_range.start > length || payload_range.end > length { break; @@ -334,14 +499,37 @@ pub fn map_stream(payload: Bytes) -> Result { } fn map_to_stream(payload: Bytes, position: usize) -> Result<(Stream, usize), IggyError> { - let id = u32::from_le_bytes(payload[position..position + 4].try_into()?); - let created_at = u64::from_le_bytes(payload[position + 4..position + 12].try_into()?).into(); - let topics_count = u32::from_le_bytes(payload[position + 12..position + 16].try_into()?); - let size_bytes = u64::from_le_bytes(payload[position + 16..position + 24].try_into()?).into(); - let messages_count = u64::from_le_bytes(payload[position + 24..position + 32].try_into()?); + let id = u32::from_le_bytes( + payload[position..position + 4] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); + let created_at = u64::from_le_bytes( + payload[position + 4..position + 12] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ) + .into(); + let topics_count = u32::from_le_bytes( + payload[position + 12..position + 16] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); + let size_bytes = u64::from_le_bytes( + payload[position + 16..position + 24] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ) + .into(); + let messages_count = u64::from_le_bytes( + payload[position + 24..position + 32] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); let name_length = payload[position + 32]; - let name = - from_utf8(&payload[position + 33..position + 33 + name_length as usize])?.to_string(); + let name = from_utf8(&payload[position + 33..position + 33 + name_length as usize]) + .map_err(|_| IggyError::InvalidUtf8)? + .to_string(); let read_bytes = 4 + 8 + 4 + 8 + 8 + 1 + name_length as usize; Ok(( Stream { @@ -402,26 +590,52 @@ pub fn map_topic(payload: Bytes) -> Result { } fn map_to_topic(payload: Bytes, position: usize) -> Result<(Topic, usize), IggyError> { - let id = u32::from_le_bytes(payload[position..position + 4].try_into()?); - let created_at = u64::from_le_bytes(payload[position + 4..position + 12].try_into()?); + let id = u32::from_le_bytes( + payload[position..position + 4] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); + let created_at = u64::from_le_bytes( + payload[position + 4..position + 12] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); let created_at = created_at.into(); - let partitions_count = u32::from_le_bytes(payload[position + 12..position + 16].try_into()?); - let message_expiry = match u64::from_le_bytes(payload[position + 16..position + 24].try_into()?) - { + let partitions_count = u32::from_le_bytes( + payload[position + 12..position + 16] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); + let message_expiry = match u64::from_le_bytes( + payload[position + 16..position + 24] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ) { 0 => IggyExpiry::NeverExpire, message_expiry => message_expiry.into(), }; let compression_algorithm = CompressionAlgorithm::from_code(payload[position + 24])?; - let max_topic_size = u64::from_le_bytes(payload[position + 25..position + 33].try_into()?); + let max_topic_size = u64::from_le_bytes( + payload[position + 25..position + 33] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); let max_topic_size: MaxTopicSize = max_topic_size.into(); let replication_factor = payload[position + 33]; let size_bytes = IggyByteSize::from(u64::from_le_bytes( - payload[position + 34..position + 42].try_into()?, + payload[position + 34..position + 42] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, )); - let messages_count = u64::from_le_bytes(payload[position + 42..position + 50].try_into()?); + let messages_count = u64::from_le_bytes( + payload[position + 42..position + 50] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); let name_length = payload[position + 50]; - let name = - from_utf8(&payload[position + 51..position + 51 + name_length as usize])?.to_string(); + let name = from_utf8(&payload[position + 51..position + 51 + name_length as usize]) + .map_err(|_| IggyError::InvalidUtf8)? + .to_string(); let read_bytes = 4 + 8 + 4 + 8 + 8 + 8 + 8 + 1 + 1 + 1 + name_length as usize; Ok(( Topic { @@ -441,13 +655,38 @@ fn map_to_topic(payload: Bytes, position: usize) -> Result<(Topic, usize), IggyE } fn map_to_partition(payload: Bytes, position: usize) -> Result<(Partition, usize), IggyError> { - let id = u32::from_le_bytes(payload[position..position + 4].try_into()?); - let created_at = u64::from_le_bytes(payload[position + 4..position + 12].try_into()?); + let id = u32::from_le_bytes( + payload[position..position + 4] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); + let created_at = u64::from_le_bytes( + payload[position + 4..position + 12] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); let created_at = created_at.into(); - let segments_count = u32::from_le_bytes(payload[position + 12..position + 16].try_into()?); - let current_offset = u64::from_le_bytes(payload[position + 16..position + 24].try_into()?); - let size_bytes = u64::from_le_bytes(payload[position + 24..position + 32].try_into()?).into(); - let messages_count = u64::from_le_bytes(payload[position + 32..position + 40].try_into()?); + let segments_count = u32::from_le_bytes( + payload[position + 12..position + 16] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); + let current_offset = u64::from_le_bytes( + payload[position + 16..position + 24] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); + let size_bytes = u64::from_le_bytes( + payload[position + 24..position + 32] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ) + .into(); + let messages_count = u64::from_le_bytes( + payload[position + 32..position + 40] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); let read_bytes = 4 + 8 + 4 + 8 + 8 + 8; Ok(( Partition { @@ -503,12 +742,25 @@ fn map_to_consumer_group( payload: Bytes, position: usize, ) -> Result<(ConsumerGroup, usize), IggyError> { - let id = u32::from_le_bytes(payload[position..position + 4].try_into()?); - let partitions_count = u32::from_le_bytes(payload[position + 4..position + 8].try_into()?); - let members_count = u32::from_le_bytes(payload[position + 8..position + 12].try_into()?); + let id = u32::from_le_bytes( + payload[position..position + 4] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); + let partitions_count = u32::from_le_bytes( + payload[position + 4..position + 8] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); + let members_count = u32::from_le_bytes( + payload[position + 8..position + 12] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); let name_length = payload[position + 12]; - let name = - from_utf8(&payload[position + 13..position + 13 + name_length as usize])?.to_string(); + let name = from_utf8(&payload[position + 13..position + 13 + name_length as usize]) + .map_err(|_| IggyError::InvalidUtf8)? + .to_string(); let read_bytes = 13 + name_length as usize; Ok(( ConsumerGroup { @@ -525,13 +777,22 @@ fn map_to_consumer_group_member( payload: Bytes, position: usize, ) -> Result<(ConsumerGroupMember, usize), IggyError> { - let id = u32::from_le_bytes(payload[position..position + 4].try_into()?); - let partitions_count = u32::from_le_bytes(payload[position + 4..position + 8].try_into()?); + let id = u32::from_le_bytes( + payload[position..position + 4] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); + let partitions_count = u32::from_le_bytes( + payload[position + 4..position + 8] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); let mut partitions = Vec::new(); for i in 0..partitions_count { let partition_id = u32::from_le_bytes( payload[position + 8 + (i * 4) as usize..position + 8 + ((i + 1) * 4) as usize] - .try_into()?, + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, ); partitions.push(partition_id); } @@ -552,8 +813,16 @@ fn map_to_client_info( mut position: usize, ) -> Result<(ClientInfo, usize), IggyError> { let mut read_bytes; - let client_id = u32::from_le_bytes(payload[position..position + 4].try_into()?); - let user_id = u32::from_le_bytes(payload[position + 4..position + 8].try_into()?); + let client_id = u32::from_le_bytes( + payload[position..position + 4] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); + let user_id = u32::from_le_bytes( + payload[position + 4..position + 8] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); let user_id = match user_id { 0 => None, _ => Some(user_id), @@ -567,12 +836,21 @@ fn map_to_client_info( } .to_string(); - let address_length = - u32::from_le_bytes(payload[position + 9..position + 13].try_into()?) as usize; - let address = from_utf8(&payload[position + 13..position + 13 + address_length])?.to_string(); + let address_length = u32::from_le_bytes( + payload[position + 9..position + 13] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ) as usize; + let address = from_utf8(&payload[position + 13..position + 13 + address_length]) + .map_err(|_| IggyError::InvalidUtf8)? + .to_string(); read_bytes = 4 + 4 + 1 + 4 + address_length; position += read_bytes; - let consumer_groups_count = u32::from_le_bytes(payload[position..position + 4].try_into()?); + let consumer_groups_count = u32::from_le_bytes( + payload[position..position + 4] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); read_bytes += 4; Ok(( ClientInfo { @@ -587,14 +865,23 @@ fn map_to_client_info( } fn map_to_user_info(payload: Bytes, position: usize) -> Result<(UserInfo, usize), IggyError> { - let id = u32::from_le_bytes(payload[position..position + 4].try_into()?); - let created_at = u64::from_le_bytes(payload[position + 4..position + 12].try_into()?); + let id = u32::from_le_bytes( + payload[position..position + 4] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); + let created_at = u64::from_le_bytes( + payload[position + 4..position + 12] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); let created_at = created_at.into(); let status = payload[position + 12]; let status = UserStatus::from_code(status)?; let username_length = payload[position + 13]; - let username = - from_utf8(&payload[position + 14..position + 14 + username_length as usize])?.to_string(); + let username = from_utf8(&payload[position + 14..position + 14 + username_length as usize]) + .map_err(|_| IggyError::InvalidUtf8)? + .to_string(); let read_bytes = 4 + 8 + 1 + 1 + username_length as usize; Ok(( @@ -613,9 +900,15 @@ fn map_to_pat_info( position: usize, ) -> Result<(PersonalAccessTokenInfo, usize), IggyError> { let name_length = payload[position]; - let name = from_utf8(&payload[position + 1..position + 1 + name_length as usize])?.to_string(); + let name = from_utf8(&payload[position + 1..position + 1 + name_length as usize]) + .map_err(|_| IggyError::InvalidUtf8)? + .to_string(); let position = position + 1 + name_length as usize; - let expiry_at = u64::from_le_bytes(payload[position..position + 8].try_into()?); + let expiry_at = u64::from_le_bytes( + payload[position..position + 8] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); let expiry_at = match expiry_at { 0 => None, value => Some(value.into()), diff --git a/sdk/src/client.rs b/sdk/src/client.rs index 9711b8df9..bd866e796 100644 --- a/sdk/src/client.rs +++ b/sdk/src/client.rs @@ -558,7 +558,11 @@ impl ConnectionString { enabled: true, max_retries: match reconnection_retries.as_str() { "unlimited" => None, - _ => Some(reconnection_retries.parse()?), + _ => Some( + reconnection_retries + .parse() + .map_err(|_| IggyError::InvalidNumberValue)?, + ), }, interval: IggyDuration::from_str(reconnection_interval.as_str()) .map_err(|_| IggyError::InvalidConnectionString)?, diff --git a/sdk/src/consumer_groups/create_consumer_group.rs b/sdk/src/consumer_groups/create_consumer_group.rs index a449e5c1e..288b5881f 100644 --- a/sdk/src/consumer_groups/create_consumer_group.rs +++ b/sdk/src/consumer_groups/create_consumer_group.rs @@ -94,11 +94,16 @@ impl BytesSerializable for CreateConsumerGroup { position += stream_id.get_size_bytes().as_bytes_usize(); let topic_id = Identifier::from_bytes(bytes.slice(position..))?; position += topic_id.get_size_bytes().as_bytes_usize(); - let group_id = u32::from_le_bytes(bytes[position..position + 4].try_into()?); + let group_id = u32::from_le_bytes( + bytes[position..position + 4] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); let group_id = if group_id == 0 { None } else { Some(group_id) }; let name_length = bytes[position + 4]; - let name = - from_utf8(&bytes[position + 5..position + 5 + name_length as usize])?.to_string(); + let name = from_utf8(&bytes[position + 5..position + 5 + name_length as usize]) + .map_err(|_| IggyError::InvalidUtf8)? + .to_string(); let command = CreateConsumerGroup { stream_id, topic_id, diff --git a/sdk/src/consumer_offsets/get_consumer_offset.rs b/sdk/src/consumer_offsets/get_consumer_offset.rs index c9b773e02..9d5913bb3 100644 --- a/sdk/src/consumer_offsets/get_consumer_offset.rs +++ b/sdk/src/consumer_offsets/get_consumer_offset.rs @@ -94,7 +94,11 @@ impl BytesSerializable for GetConsumerOffset { position += stream_id.get_size_bytes().as_bytes_usize(); let topic_id = Identifier::from_bytes(bytes.slice(position..))?; position += topic_id.get_size_bytes().as_bytes_usize(); - let partition_id = u32::from_le_bytes(bytes[position..position + 4].try_into()?); + let partition_id = u32::from_le_bytes( + bytes[position..position + 4] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); let partition_id = if partition_id == 0 { None } else { diff --git a/sdk/src/consumer_offsets/store_consumer_offset.rs b/sdk/src/consumer_offsets/store_consumer_offset.rs index a76dfb539..a9c7cd4bc 100644 --- a/sdk/src/consumer_offsets/store_consumer_offset.rs +++ b/sdk/src/consumer_offsets/store_consumer_offset.rs @@ -94,13 +94,21 @@ impl BytesSerializable for StoreConsumerOffset { position += stream_id.get_size_bytes().as_bytes_usize(); let topic_id = Identifier::from_bytes(bytes.slice(position..))?; position += topic_id.get_size_bytes().as_bytes_usize(); - let partition_id = u32::from_le_bytes(bytes[position..position + 4].try_into()?); + let partition_id = u32::from_le_bytes( + bytes[position..position + 4] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); let partition_id = if partition_id == 0 { None } else { Some(partition_id) }; - let offset = u64::from_le_bytes(bytes[position + 4..position + 12].try_into()?); + let offset = u64::from_le_bytes( + bytes[position + 4..position + 12] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); let command = StoreConsumerOffset { consumer, stream_id, diff --git a/sdk/src/error.rs b/sdk/src/error.rs index ca5a53dcc..0fc56abab 100644 --- a/sdk/src/error.rs +++ b/sdk/src/error.rs @@ -3,7 +3,7 @@ use crate::utils::topic_size::MaxTopicSize; use strum::{EnumDiscriminants, FromRepr, IntoStaticStr}; use thiserror::Error; -#[derive(Debug, Error, EnumDiscriminants, IntoStaticStr)] +#[derive(Debug, Error, EnumDiscriminants, IntoStaticStr, FromRepr)] #[repr(u32)] #[strum(serialize_all = "snake_case")] #[strum_discriminants( @@ -48,18 +48,16 @@ pub enum IggyError { CannotOpenDatabase(String) = 19, #[error("Resource with key: {0} was not found.")] ResourceNotFound(String) = 20, - #[error("Cannot load resource. Reason: {0:#}")] - CannotLoadResource(#[source] anyhow::Error) = 21, - #[error("Cannot save resource. Reason: {0:#}")] - CannotSaveResource(#[source] anyhow::Error) = 22, - #[error("Cannot delete resource. Reason: {0:#}")] - CannotDeleteResource(#[source] anyhow::Error) = 23, - #[error("Cannot serialize resource. Reason: {0:#}")] - CannotSerializeResource(#[source] anyhow::Error) = 24, - #[error("Cannot deserialize resource. Reason: {0:#}")] - CannotDeserializeResource(#[source] anyhow::Error) = 25, #[error("Stale client")] StaleClient = 30, + #[error("TCP error")] + TcpError = 31, + #[error("QUIC error")] + QuicError = 32, + #[error("Invalid server address")] + InvalidServerAddress = 33, + #[error("Invalid client address")] + InvalidClientAddress = 34, #[error("Unauthenticated")] Unauthenticated = 40, #[error("Unauthorized")] @@ -94,8 +92,6 @@ pub enum IggyError { UsersLimitReached = 55, #[error("Not connected")] NotConnected = 61, - #[error("Request error")] - RequestError(#[from] reqwest::Error) = 62, #[error("Client shutdown")] ClientShutdown = 63, #[error("Invalid TLS domain")] @@ -124,52 +120,38 @@ pub enum IggyError { AccessTokenMissing = 77, #[error("Invalid access token")] InvalidAccessToken = 78, + #[error("Invalid size bytes")] + InvalidSizeBytes = 80, + #[error("Invalid UTF-8")] + InvalidUtf8 = 81, + #[error("Invalid number encoding")] + InvalidNumberEncoding = 82, + #[error("Invalid boolean value")] + InvalidBooleanValue, + #[error("Invalid number value")] + InvalidNumberValue, #[error("Client with ID: {0} was not found.")] ClientNotFound(u32) = 100, #[error("Invalid client ID")] InvalidClientId = 101, - #[error("IO error")] - IoError(#[from] std::io::Error) = 200, - #[error("Write error")] - WriteError(#[from] quinn::WriteError) = 201, - #[error("Cannot parse UTF8")] - CannotParseUtf8(#[from] std::str::Utf8Error) = 202, - #[error("Cannot parse integer")] - CannotParseInt(#[from] std::num::ParseIntError) = 203, - #[error("Cannot parse integer")] - CannotParseSlice(#[from] std::array::TryFromSliceError) = 204, - #[error("Cannot parse byte unit")] - CannotParseByteUnit(#[from] byte_unit::ParseError) = 205, #[error("Connection closed")] ConnectionClosed = 206, - #[error("Cannot parse float")] - CannotParseFloat(#[from] std::num::ParseFloatError) = 207, - #[error("Cannot parse bool")] - CannotParseBool(#[from] std::str::ParseBoolError) = 208, #[error("Cannot parse header kind from {0}")] CannotParseHeaderKind(String) = 209, #[error("HTTP response error, status: {0}, body: {1}")] HttpResponseError(u16, String) = 300, - #[error("Request middleware error")] - RequestMiddlewareError(#[from] reqwest_middleware::Error) = 301, + #[error("Invalid HTTP request")] + InvalidHttpRequest = 301, + #[error("Invalid JSON response")] + InvalidJsonResponse = 302, + #[error("Invalid bytes response")] + InvalidBytesResponse = 303, + #[error("Empty response")] + EmptyResponse = 304, #[error("Cannot create endpoint")] - CannotCreateEndpoint = 302, + CannotCreateEndpoint = 305, #[error("Cannot parse URL")] - CannotParseUrl = 303, - #[error("Invalid response: {0}: {2})")] - InvalidResponse(u32, u32, String) = 304, - #[error("Empty response")] - EmptyResponse = 305, - #[error("Cannot parse address")] - CannotParseAddress(#[from] std::net::AddrParseError) = 306, - #[error("Read error")] - ReadError(#[from] quinn::ReadError) = 307, - #[error("Connection error")] - ConnectionError(#[from] quinn::ConnectionError) = 308, - #[error("Read to end error")] - ReadToEndError(#[from] quinn::ReadToEndError) = 309, - #[error("Closed error")] - ClosedError(#[from] quinn::ClosedStream) = 310, + CannotParseUrl = 306, #[error("Cannot create streams directory, Path: {0}")] CannotCreateStreamsDirectory(String) = 1000, #[error("Cannot create stream with ID: {0} directory, Path: {1}")] @@ -260,8 +242,8 @@ pub enum IggyError { CannotCreatePartitionDirectory(u32, u32, u32) = 3002, #[error("Cannot open partition log file")] CannotOpenPartitionLogFile = 3003, - #[error("Cannot read partitions directories. Reason: {0:#}")] - CannotReadPartitions(#[source] anyhow::Error) = 3004, + #[error("Cannot read partitions directories.")] + CannotReadPartitions = 3004, #[error( "Failed to delete partition with ID: {0} for stream with ID: {1} and topic with ID: {2}" )] @@ -296,12 +278,12 @@ pub enum IggyError { CannotCreateSegmentIndexFile(String) = 4004, #[error("Failed to create segment time index file for Path: {0}.")] CannotCreateSegmentTimeIndexFile(String) = 4005, - #[error("Cannot save messages to segment. Reason: {0:#}")] - CannotSaveMessagesToSegment(#[source] anyhow::Error) = 4006, - #[error("Cannot save index to segment. Reason: {0:#}")] - CannotSaveIndexToSegment(#[source] anyhow::Error) = 4007, - #[error("Cannot save time index to segment. Reason: {0:#}")] - CannotSaveTimeIndexToSegment(#[source] anyhow::Error) = 4008, + #[error("Cannot save messages to segment.")] + CannotSaveMessagesToSegment = 4006, + #[error("Cannot save index to segment.")] + CannotSaveIndexToSegment = 4007, + #[error("Cannot save time index to segment.")] + CannotSaveTimeIndexToSegment = 4008, #[error("Invalid messages count")] InvalidMessagesCount = 4009, #[error("Cannot append message")] @@ -386,8 +368,34 @@ pub enum IggyError { CannotReadBatchPayload = 7004, #[error("Invalid connection string")] InvalidConnectionString = 8000, - #[error("Snaphot file completion failed")] + #[error("Snapshot file completion failed")] SnapshotFileCompletionFailed = 9000, + #[error("Cannot serialize resource")] + CannotSerializeResource = 10000, + #[error("Cannot deserialize resource")] + CannotDeserializeResource = 10001, + #[error("Cannot read file")] + CannotReadFile = 10002, + #[error("Cannot read file metadata")] + CannotReadFileMetadata = 10003, + #[error("Cannot seek file")] + CannotSeekFile = 10004, + #[error("Cannot append to file")] + CannotAppendToFile = 10005, + #[error("Cannot write to file")] + CannotWriteToFile = 10006, + #[error("Cannot overwrite file")] + CannotOverwriteFile = 10007, + #[error("Cannot delete file")] + CannotDeleteFile = 10008, + #[error("Cannot sync file")] + CannotSyncFile = 10009, + #[error("Cannot read index offset")] + CannotReadIndexOffset = 10010, + #[error("Cannot read index position")] + CannotReadIndexPosition = 10011, + #[error("Cannot read index timestamp")] + CannotReadIndexTimestamp = 10012, } impl IggyError { @@ -401,6 +409,10 @@ impl IggyError { self.into() } + pub fn from_code(code: u32) -> Self { + IggyError::from_repr(code).unwrap_or(IggyError::Error) + } + pub fn from_code_as_string(code: u32) -> &'static str { IggyErrorDiscriminants::from_repr(code) .map(|discriminant| discriminant.into()) diff --git a/sdk/src/http/client.rs b/sdk/src/http/client.rs index db3298367..c5b007270 100644 --- a/sdk/src/http/client.rs +++ b/sdk/src/http/client.rs @@ -85,7 +85,8 @@ impl HttpTransport for HttpClient { .get(url) .bearer_auth(token.deref()) .send() - .await?; + .await + .map_err(|_| IggyError::InvalidHttpRequest)?; Self::handle_response(response).await } @@ -104,7 +105,8 @@ impl HttpTransport for HttpClient { .bearer_auth(token.deref()) .query(query) .send() - .await?; + .await + .map_err(|_| IggyError::InvalidHttpRequest)?; Self::handle_response(response).await } @@ -123,7 +125,8 @@ impl HttpTransport for HttpClient { .bearer_auth(token.deref()) .json(payload) .send() - .await?; + .await + .map_err(|_| IggyError::InvalidHttpRequest)?; Self::handle_response(response).await } @@ -142,7 +145,8 @@ impl HttpTransport for HttpClient { .bearer_auth(token.deref()) .json(payload) .send() - .await?; + .await + .map_err(|_| IggyError::InvalidHttpRequest)?; Self::handle_response(response).await } @@ -156,7 +160,8 @@ impl HttpTransport for HttpClient { .delete(url) .bearer_auth(token.deref()) .send() - .await?; + .await + .map_err(|_| IggyError::InvalidHttpRequest)?; Self::handle_response(response).await } @@ -175,7 +180,8 @@ impl HttpTransport for HttpClient { .bearer_auth(token.deref()) .query(query) .send() - .await?; + .await + .map_err(|_| IggyError::InvalidHttpRequest)?; Self::handle_response(response).await } @@ -196,7 +202,10 @@ impl HttpTransport for HttpClient { token: token.to_owned(), }; let response = self.post("/users/refresh-token", &command).await?; - let identity_info: IdentityInfo = response.json().await?; + let identity_info: IdentityInfo = response + .json() + .await + .map_err(|_| IggyError::InvalidJsonResponse)?; if identity_info.access_token.is_none() { return Err(IggyError::JwtMissing); } diff --git a/sdk/src/http/consumer_groups.rs b/sdk/src/http/consumer_groups.rs index 03dfeb435..462522d1f 100644 --- a/sdk/src/http/consumer_groups.rs +++ b/sdk/src/http/consumer_groups.rs @@ -26,7 +26,10 @@ impl ConsumerGroupClient for HttpClient { return Ok(None); } - let consumer_group = response.json().await?; + let consumer_group = response + .json() + .await + .map_err(|_| IggyError::InvalidJsonResponse)?; Ok(Some(consumer_group)) } @@ -38,7 +41,10 @@ impl ConsumerGroupClient for HttpClient { let response = self .get(&get_path(&stream_id.as_cow_str(), &topic_id.as_cow_str())) .await?; - let consumer_groups = response.json().await?; + let consumer_groups = response + .json() + .await + .map_err(|_| IggyError::InvalidJsonResponse)?; Ok(consumer_groups) } @@ -60,7 +66,10 @@ impl ConsumerGroupClient for HttpClient { }, ) .await?; - let consumer_group = response.json().await?; + let consumer_group = response + .json() + .await + .map_err(|_| IggyError::InvalidJsonResponse)?; Ok(consumer_group) } diff --git a/sdk/src/http/consumer_offsets.rs b/sdk/src/http/consumer_offsets.rs index 2bda15f97..f4fa4afd7 100644 --- a/sdk/src/http/consumer_offsets.rs +++ b/sdk/src/http/consumer_offsets.rs @@ -55,7 +55,10 @@ impl ConsumerOffsetClient for HttpClient { return Ok(None); } - let offset = response.json().await?; + let offset = response + .json() + .await + .map_err(|_| IggyError::InvalidJsonResponse)?; Ok(Some(offset)) } } diff --git a/sdk/src/http/messages.rs b/sdk/src/http/messages.rs index 09b04954d..b14602e3a 100644 --- a/sdk/src/http/messages.rs +++ b/sdk/src/http/messages.rs @@ -36,7 +36,10 @@ impl MessageClient for HttpClient { }, ) .await?; - let messages = response.json().await?; + let messages = response + .json() + .await + .map_err(|_| IggyError::InvalidJsonResponse)?; Ok(messages) } diff --git a/sdk/src/http/personal_access_tokens.rs b/sdk/src/http/personal_access_tokens.rs index 787c1ba08..5b316c808 100644 --- a/sdk/src/http/personal_access_tokens.rs +++ b/sdk/src/http/personal_access_tokens.rs @@ -15,7 +15,10 @@ const PATH: &str = "/personal-access-tokens"; impl PersonalAccessTokenClient for HttpClient { async fn get_personal_access_tokens(&self) -> Result, IggyError> { let response = self.get(PATH).await?; - let personal_access_tokens = response.json().await?; + let personal_access_tokens = response + .json() + .await + .map_err(|_| IggyError::InvalidJsonResponse)?; Ok(personal_access_tokens) } @@ -33,7 +36,10 @@ impl PersonalAccessTokenClient for HttpClient { }, ) .await?; - let personal_access_token = response.json().await?; + let personal_access_token = response + .json() + .await + .map_err(|_| IggyError::InvalidJsonResponse)?; Ok(personal_access_token) } @@ -54,7 +60,10 @@ impl PersonalAccessTokenClient for HttpClient { }, ) .await?; - let identity_info: IdentityInfo = response.json().await?; + let identity_info: IdentityInfo = response + .json() + .await + .map_err(|_| IggyError::InvalidJsonResponse)?; self.set_token_from_identity(&identity_info).await?; Ok(identity_info) } diff --git a/sdk/src/http/streams.rs b/sdk/src/http/streams.rs index ae113f9bc..38f413fd8 100644 --- a/sdk/src/http/streams.rs +++ b/sdk/src/http/streams.rs @@ -18,13 +18,19 @@ impl StreamClient for HttpClient { return Ok(None); } - let stream = response.json().await?; + let stream = response + .json() + .await + .map_err(|_| IggyError::InvalidJsonResponse)?; Ok(Some(stream)) } async fn get_streams(&self) -> Result, IggyError> { let response = self.get(PATH).await?; - let streams = response.json().await?; + let streams = response + .json() + .await + .map_err(|_| IggyError::InvalidJsonResponse)?; Ok(streams) } @@ -42,7 +48,10 @@ impl StreamClient for HttpClient { }, ) .await?; - let stream = response.json().await?; + let stream = response + .json() + .await + .map_err(|_| IggyError::InvalidJsonResponse)?; Ok(stream) } diff --git a/sdk/src/http/system.rs b/sdk/src/http/system.rs index a87c468ab..56763fa52 100644 --- a/sdk/src/http/system.rs +++ b/sdk/src/http/system.rs @@ -19,7 +19,10 @@ const SNAPSHOT: &str = "/snapshot"; impl SystemClient for HttpClient { async fn get_stats(&self) -> Result { let response = self.get(STATS).await?; - let stats = response.json().await?; + let stats = response + .json() + .await + .map_err(|_| IggyError::InvalidJsonResponse)?; Ok(stats) } @@ -33,13 +36,19 @@ impl SystemClient for HttpClient { return Ok(None); } - let client = response.json().await?; + let client = response + .json() + .await + .map_err(|_| IggyError::InvalidJsonResponse)?; Ok(Some(client)) } async fn get_clients(&self) -> Result, IggyError> { let response = self.get(CLIENTS).await?; - let clients = response.json().await?; + let clients = response + .json() + .await + .map_err(|_| IggyError::InvalidJsonResponse)?; Ok(clients) } @@ -66,7 +75,10 @@ impl SystemClient for HttpClient { }, ) .await?; - let file = response.bytes().await?; + let file = response + .bytes() + .await + .map_err(|_| IggyError::InvalidBytesResponse)?; let snapshot = Snapshot::new(file.to_vec()); Ok(snapshot) } diff --git a/sdk/src/http/topics.rs b/sdk/src/http/topics.rs index ac02a56cb..62247d87a 100644 --- a/sdk/src/http/topics.rs +++ b/sdk/src/http/topics.rs @@ -28,13 +28,19 @@ impl TopicClient for HttpClient { return Ok(None); } - let topic = response.json().await?; + let topic = response + .json() + .await + .map_err(|_| IggyError::InvalidJsonResponse)?; Ok(Some(topic)) } async fn get_topics(&self, stream_id: &Identifier) -> Result, IggyError> { let response = self.get(&get_path(&stream_id.as_cow_str())).await?; - let topics = response.json().await?; + let topics = response + .json() + .await + .map_err(|_| IggyError::InvalidJsonResponse)?; Ok(topics) } @@ -64,7 +70,10 @@ impl TopicClient for HttpClient { }, ) .await?; - let topic = response.json().await?; + let topic = response + .json() + .await + .map_err(|_| IggyError::InvalidJsonResponse)?; Ok(topic) } diff --git a/sdk/src/http/users.rs b/sdk/src/http/users.rs index 20ed47b30..3307919de 100644 --- a/sdk/src/http/users.rs +++ b/sdk/src/http/users.rs @@ -24,13 +24,19 @@ impl UserClient for HttpClient { return Ok(None); } - let user = response.json().await?; + let user = response + .json() + .await + .map_err(|_| IggyError::InvalidJsonResponse)?; Ok(Some(user)) } async fn get_users(&self) -> Result, IggyError> { let response = self.get(PATH).await?; - let users = response.json().await?; + let users = response + .json() + .await + .map_err(|_| IggyError::InvalidJsonResponse)?; Ok(users) } @@ -52,7 +58,10 @@ impl UserClient for HttpClient { }, ) .await?; - let user = response.json().await?; + let user = response + .json() + .await + .map_err(|_| IggyError::InvalidJsonResponse)?; Ok(user) } @@ -126,7 +135,10 @@ impl UserClient for HttpClient { }, ) .await?; - let identity_info = response.json().await?; + let identity_info = response + .json() + .await + .map_err(|_| IggyError::InvalidJsonResponse)?; self.set_token_from_identity(&identity_info).await?; Ok(identity_info) } diff --git a/sdk/src/messages/flush_unsaved_buffer.rs b/sdk/src/messages/flush_unsaved_buffer.rs index 952db9fd6..80f69e034 100644 --- a/sdk/src/messages/flush_unsaved_buffer.rs +++ b/sdk/src/messages/flush_unsaved_buffer.rs @@ -75,7 +75,11 @@ impl BytesSerializable for FlushUnsavedBuffer { position += stream_id.to_bytes().len(); let topic_id = Identifier::from_bytes(bytes.slice(position..))?; position += topic_id.to_bytes().len(); - let partition_id = u32::from_le_bytes(bytes[position..position + 4].try_into()?); + let partition_id = u32::from_le_bytes( + bytes[position..position + 4] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); position += 4; let fsync = bytes[position] == 1; Ok(FlushUnsavedBuffer { diff --git a/sdk/src/messages/poll_messages.rs b/sdk/src/messages/poll_messages.rs index 58cd31b23..901027f12 100644 --- a/sdk/src/messages/poll_messages.rs +++ b/sdk/src/messages/poll_messages.rs @@ -268,19 +268,31 @@ impl BytesSerializable for PollMessages { position += stream_id.get_size_bytes().as_bytes_usize(); let topic_id = Identifier::from_bytes(bytes.slice(position..))?; position += topic_id.get_size_bytes().as_bytes_usize(); - let partition_id = u32::from_le_bytes(bytes[position..position + 4].try_into()?); + let partition_id = u32::from_le_bytes( + bytes[position..position + 4] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); let partition_id = match partition_id { 0 => None, partition_id => Some(partition_id), }; let polling_kind = PollingKind::from_code(bytes[position + 4])?; position += 5; - let value = u64::from_le_bytes(bytes[position..position + 8].try_into()?); + let value = u64::from_le_bytes( + bytes[position..position + 8] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); let strategy = PollingStrategy { kind: polling_kind, value, }; - let count = u32::from_le_bytes(bytes[position + 8..position + 12].try_into()?); + let count = u32::from_le_bytes( + bytes[position + 8..position + 12] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); let auto_commit = bytes[position + 12]; let auto_commit = matches!(auto_commit, 1); let command = PollMessages { @@ -379,7 +391,11 @@ impl BytesSerializable for PollingStrategy { } let kind = PollingKind::from_code(bytes[0])?; - let value = u64::from_le_bytes(bytes[1..9].try_into()?); + let value = u64::from_le_bytes( + bytes[1..9] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); let strategy = PollingStrategy { kind, value }; Ok(strategy) } diff --git a/sdk/src/messages/send_messages.rs b/sdk/src/messages/send_messages.rs index 01d954bc4..d26639b51 100644 --- a/sdk/src/messages/send_messages.rs +++ b/sdk/src/messages/send_messages.rs @@ -381,11 +381,19 @@ impl BytesSerializable for Message { return Err(IggyError::InvalidCommand); } - let mut id = u128::from_le_bytes(bytes[..16].try_into()?); + let mut id = u128::from_le_bytes( + bytes[..16] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); if id == 0 { id = Uuid::now_v7().to_u128_le(); } - let headers_length = u32::from_le_bytes(bytes[16..20].try_into()?); + let headers_length = u32::from_le_bytes( + bytes[16..20] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); let headers = if headers_length > 0 { Some(HashMap::from_bytes( bytes.slice(20..20 + headers_length as usize), @@ -395,7 +403,9 @@ impl BytesSerializable for Message { }; let payload_length = u32::from_le_bytes( - bytes[20 + headers_length as usize..24 + headers_length as usize].try_into()?, + bytes[20 + headers_length as usize..24 + headers_length as usize] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, ); if payload_length == 0 { return Err(IggyError::EmptyMessagePayload); diff --git a/sdk/src/models/header.rs b/sdk/src/models/header.rs index bf17c06f0..fd3c9ccc2 100644 --- a/sdk/src/models/header.rs +++ b/sdk/src/models/header.rs @@ -209,19 +209,45 @@ impl HeaderValue { match kind { HeaderKind::Raw => Self::from_raw(value.as_bytes()), HeaderKind::String => Self::from_str(value), - HeaderKind::Bool => Self::from_bool(value.parse()?), - HeaderKind::Int8 => Self::from_int8(value.parse()?), - HeaderKind::Int16 => Self::from_int16(value.parse()?), - HeaderKind::Int32 => Self::from_int32(value.parse()?), - HeaderKind::Int64 => Self::from_int64(value.parse()?), - HeaderKind::Int128 => Self::from_int128(value.parse()?), - HeaderKind::Uint8 => Self::from_uint8(value.parse()?), - HeaderKind::Uint16 => Self::from_uint16(value.parse()?), - HeaderKind::Uint32 => Self::from_uint32(value.parse()?), - HeaderKind::Uint64 => Self::from_uint64(value.parse()?), - HeaderKind::Uint128 => Self::from_uint128(value.parse()?), - HeaderKind::Float32 => Self::from_float32(value.parse()?), - HeaderKind::Float64 => Self::from_float64(value.parse()?), + HeaderKind::Bool => { + Self::from_bool(value.parse().map_err(|_| IggyError::InvalidBooleanValue)?) + } + HeaderKind::Int8 => { + Self::from_int8(value.parse().map_err(|_| IggyError::InvalidNumberValue)?) + } + HeaderKind::Int16 => { + Self::from_int16(value.parse().map_err(|_| IggyError::InvalidNumberValue)?) + } + HeaderKind::Int32 => { + Self::from_int32(value.parse().map_err(|_| IggyError::InvalidNumberValue)?) + } + HeaderKind::Int64 => { + Self::from_int64(value.parse().map_err(|_| IggyError::InvalidNumberValue)?) + } + HeaderKind::Int128 => { + Self::from_int128(value.parse().map_err(|_| IggyError::InvalidNumberValue)?) + } + HeaderKind::Uint8 => { + Self::from_uint8(value.parse().map_err(|_| IggyError::InvalidNumberValue)?) + } + HeaderKind::Uint16 => { + Self::from_uint16(value.parse().map_err(|_| IggyError::InvalidNumberValue)?) + } + HeaderKind::Uint32 => { + Self::from_uint32(value.parse().map_err(|_| IggyError::InvalidNumberValue)?) + } + HeaderKind::Uint64 => { + Self::from_uint64(value.parse().map_err(|_| IggyError::InvalidNumberValue)?) + } + HeaderKind::Uint128 => { + Self::from_uint128(value.parse().map_err(|_| IggyError::InvalidNumberValue)?) + } + HeaderKind::Float32 => { + Self::from_float32(value.parse().map_err(|_| IggyError::InvalidNumberValue)?) + } + HeaderKind::Float64 => { + Self::from_float64(value.parse().map_err(|_| IggyError::InvalidNumberValue)?) + } } } /// Creates a new header value from the specified raw bytes. @@ -244,7 +270,7 @@ impl HeaderValue { return Err(IggyError::InvalidHeaderValue); } - Ok(std::str::from_utf8(&self.value)?) + std::str::from_utf8(&self.value).map_err(|_| IggyError::InvalidUtf8) } /// Creates a new header value from the specified string. @@ -589,7 +615,11 @@ impl BytesSerializable for HashMap { let mut headers = Self::new(); let mut position = 0; while position < bytes.len() { - let key_length = u32::from_le_bytes(bytes[position..position + 4].try_into()?) as usize; + let key_length = u32::from_le_bytes( + bytes[position..position + 4] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ) as usize; if key_length == 0 || key_length > 255 { return Err(IggyError::InvalidHeaderKey); } @@ -602,8 +632,11 @@ impl BytesSerializable for HashMap { position += key_length; let kind = HeaderKind::from_code(bytes[position])?; position += 1; - let value_length = - u32::from_le_bytes(bytes[position..position + 4].try_into()?) as usize; + let value_length = u32::from_le_bytes( + bytes[position..position + 4] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ) as usize; if value_length == 0 || value_length > 255 { return Err(IggyError::InvalidHeaderValue); } diff --git a/sdk/src/partitions/create_partitions.rs b/sdk/src/partitions/create_partitions.rs index 3206b0a8d..dbb0e0ca7 100644 --- a/sdk/src/partitions/create_partitions.rs +++ b/sdk/src/partitions/create_partitions.rs @@ -73,7 +73,11 @@ impl BytesSerializable for CreatePartitions { position += stream_id.get_size_bytes().as_bytes_usize(); let topic_id = Identifier::from_bytes(bytes.slice(position..))?; position += topic_id.get_size_bytes().as_bytes_usize(); - let partitions_count = u32::from_le_bytes(bytes[position..position + 4].try_into()?); + let partitions_count = u32::from_le_bytes( + bytes[position..position + 4] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); let command = CreatePartitions { stream_id, topic_id, diff --git a/sdk/src/partitions/delete_partitions.rs b/sdk/src/partitions/delete_partitions.rs index 9d5e1f64a..82edcebcb 100644 --- a/sdk/src/partitions/delete_partitions.rs +++ b/sdk/src/partitions/delete_partitions.rs @@ -73,7 +73,11 @@ impl BytesSerializable for DeletePartitions { position += stream_id.get_size_bytes().as_bytes_usize(); let topic_id = Identifier::from_bytes(bytes.slice(position..))?; position += topic_id.get_size_bytes().as_bytes_usize(); - let partitions_count = u32::from_le_bytes(bytes[position..position + 4].try_into()?); + let partitions_count = u32::from_le_bytes( + bytes[position..position + 4] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); let command = DeletePartitions { stream_id, topic_id, diff --git a/sdk/src/personal_access_tokens/create_personal_access_token.rs b/sdk/src/personal_access_tokens/create_personal_access_token.rs index 0c90328ba..99495c198 100644 --- a/sdk/src/personal_access_tokens/create_personal_access_token.rs +++ b/sdk/src/personal_access_tokens/create_personal_access_token.rs @@ -70,13 +70,19 @@ impl BytesSerializable for CreatePersonalAccessToken { } let name_length = bytes[0]; - let name = from_utf8(&bytes.slice(1..1 + name_length as usize))?.to_string(); + let name = from_utf8(&bytes.slice(1..1 + name_length as usize)) + .map_err(|_| IggyError::InvalidUtf8)? + .to_string(); if name.len() != name_length as usize { return Err(IggyError::InvalidCommand); } let position = 1 + name_length as usize; - let expiry = u64::from_le_bytes(bytes[position..position + 8].try_into()?); + let expiry = u64::from_le_bytes( + bytes[position..position + 8] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); let expiry: IggyExpiry = expiry.into(); let command = CreatePersonalAccessToken { name, expiry }; diff --git a/sdk/src/personal_access_tokens/delete_personal_access_token.rs b/sdk/src/personal_access_tokens/delete_personal_access_token.rs index 6cd464cfc..ee711d02e 100644 --- a/sdk/src/personal_access_tokens/delete_personal_access_token.rs +++ b/sdk/src/personal_access_tokens/delete_personal_access_token.rs @@ -64,7 +64,9 @@ impl BytesSerializable for DeletePersonalAccessToken { } let name_length = bytes[0]; - let name = from_utf8(&bytes[1..1 + name_length as usize])?.to_string(); + let name = from_utf8(&bytes[1..1 + name_length as usize]) + .map_err(|_| IggyError::InvalidUtf8)? + .to_string(); if name.len() != name_length as usize { return Err(IggyError::InvalidCommand); } diff --git a/sdk/src/personal_access_tokens/login_with_personal_access_token.rs b/sdk/src/personal_access_tokens/login_with_personal_access_token.rs index 833a90e50..b52b06d85 100644 --- a/sdk/src/personal_access_tokens/login_with_personal_access_token.rs +++ b/sdk/src/personal_access_tokens/login_with_personal_access_token.rs @@ -56,7 +56,9 @@ impl BytesSerializable for LoginWithPersonalAccessToken { } let token_length = bytes[0]; - let token = from_utf8(&bytes[1..1 + token_length as usize])?.to_string(); + let token = from_utf8(&bytes[1..1 + token_length as usize]) + .map_err(|_| IggyError::InvalidUtf8)? + .to_string(); if token.len() != token_length as usize { return Err(IggyError::InvalidCommand); } diff --git a/sdk/src/quic/client.rs b/sdk/src/quic/client.rs index 6a281ee3d..011f901ca 100644 --- a/sdk/src/quic/client.rs +++ b/sdk/src/quic/client.rs @@ -144,7 +144,13 @@ impl QuicClient { /// Create a new QUIC client for the provided configuration. pub fn create(config: Arc) -> Result { - let server_address = config.server_address.parse::()?; + let server_address = config + .server_address + .parse::() + .map_err(|error| { + error!("Invalid server address: {error}"); + IggyError::InvalidServerAddress + })?; let client_address = if server_address.is_ipv6() && config.client_address == QuicClientConfig::default().client_address { @@ -152,7 +158,11 @@ impl QuicClient { } else { &config.client_address } - .parse::()?; + .parse::() + .map_err(|error| { + error!("Invalid client address: {error}"); + IggyError::InvalidClientAddress + })?; let quic_config = configure(&config)?; let endpoint = Endpoint::client(client_address); @@ -178,12 +188,20 @@ impl QuicClient { async fn handle_response(&self, recv: &mut RecvStream) -> Result { let buffer = recv .read_to_end(self.config.response_buffer_size as usize) - .await?; + .await + .map_err(|error| { + error!("Failed to read response data: {error}"); + IggyError::QuicError + })?; if buffer.is_empty() { return Err(IggyError::EmptyResponse); } - let status = u32::from_le_bytes(buffer[..4].try_into().unwrap()); + let status = u32::from_le_bytes( + buffer[..4] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); if status != 0 { error!( "Received an invalid response with status: {} ({}).", @@ -191,19 +209,14 @@ impl QuicClient { IggyError::from_code_as_string(status) ); - let length = - u32::from_le_bytes(buffer[4..RESPONSE_INITIAL_BYTES_LENGTH].try_into().unwrap()); - let error_message = String::from_utf8_lossy( - &buffer[RESPONSE_INITIAL_BYTES_LENGTH + 4 - ..RESPONSE_INITIAL_BYTES_LENGTH + length as usize], - ) - .to_string(); - - return Err(IggyError::InvalidResponse(status, length, error_message)); + return Err(IggyError::from_code(status)); } - let length = - u32::from_le_bytes(buffer[4..RESPONSE_INITIAL_BYTES_LENGTH].try_into().unwrap()); + let length = u32::from_le_bytes( + buffer[4..RESPONSE_INITIAL_BYTES_LENGTH] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); trace!("Status: OK. Response length: {}", length); if length <= 1 { return Ok(Bytes::new()); @@ -296,7 +309,10 @@ impl QuicClient { return Err(IggyError::CannotEstablishConnection); } - connection = connection_result?; + connection = connection_result.map_err(|error| { + error!("Failed to establish QUIC connection: {error}"); + IggyError::CannotEstablishConnection + })?; remote_address = connection.remote_address(); break; } @@ -405,13 +421,29 @@ impl QuicClient { let connection = self.connection.lock().await; if let Some(connection) = connection.as_ref() { let payload_length = payload.len() + REQUEST_INITIAL_BYTES_LENGTH; - let (mut send, mut recv) = connection.open_bi().await?; + let (mut send, mut recv) = connection.open_bi().await.map_err(|error| { + error!("Failed to open a bidirectional stream: {error}"); + IggyError::QuicError + })?; trace!("Sending a QUIC request with code: {code}"); send.write_all(&(payload_length as u32).to_le_bytes()) - .await?; - send.write_all(&code.to_le_bytes()).await?; - send.write_all(&payload).await?; - send.finish()?; + .await + .map_err(|error| { + error!("Failed to write payload length: {error}"); + IggyError::QuicError + })?; + send.write_all(&code.to_le_bytes()).await.map_err(|error| { + error!("Failed to write payload code: {error}"); + IggyError::QuicError + })?; + send.write_all(&payload).await.map_err(|error| { + error!("Failed to write payload: {error}"); + IggyError::QuicError + })?; + send.finish().map_err(|error| { + error!("Failed to finish sending data: {error}"); + IggyError::QuicError + })?; trace!("Sent a QUIC request with code: {code}, waiting for a response..."); return self.handle_response(&mut recv).await; } diff --git a/sdk/src/streams/create_stream.rs b/sdk/src/streams/create_stream.rs index fb7dab111..ed990d25e 100644 --- a/sdk/src/streams/create_stream.rs +++ b/sdk/src/streams/create_stream.rs @@ -71,14 +71,20 @@ impl BytesSerializable for CreateStream { return Err(IggyError::InvalidCommand); } - let stream_id = u32::from_le_bytes(bytes[..4].try_into()?); + let stream_id = u32::from_le_bytes( + bytes[..4] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); let stream_id = if stream_id == 0 { None } else { Some(stream_id) }; let name_length = bytes[4]; - let name = from_utf8(&bytes[5..5 + name_length as usize])?.to_string(); + let name = from_utf8(&bytes[5..5 + name_length as usize]) + .map_err(|_| IggyError::InvalidUtf8)? + .to_string(); if name.len() != name_length as usize { return Err(IggyError::InvalidCommand); } diff --git a/sdk/src/streams/update_stream.rs b/sdk/src/streams/update_stream.rs index 33d6bd240..c363400d2 100644 --- a/sdk/src/streams/update_stream.rs +++ b/sdk/src/streams/update_stream.rs @@ -73,8 +73,9 @@ impl BytesSerializable for UpdateStream { let stream_id = Identifier::from_bytes(bytes.clone())?; position += stream_id.get_size_bytes().as_bytes_usize(); let name_length = bytes[position]; - let name = - from_utf8(&bytes[position + 1..position + 1 + name_length as usize])?.to_string(); + let name = from_utf8(&bytes[position + 1..position + 1 + name_length as usize]) + .map_err(|_| IggyError::InvalidUtf8)? + .to_string(); if name.len() != name_length as usize { return Err(IggyError::InvalidCommand); } diff --git a/sdk/src/system/get_client.rs b/sdk/src/system/get_client.rs index c053824d8..0fcf1b7a8 100644 --- a/sdk/src/system/get_client.rs +++ b/sdk/src/system/get_client.rs @@ -49,7 +49,12 @@ impl BytesSerializable for GetClient { return Err(IggyError::InvalidCommand); } - let client_id = u32::from_le_bytes(bytes.as_ref().try_into()?); + let client_id = u32::from_le_bytes( + bytes + .as_ref() + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); let command = GetClient { client_id }; Ok(command) } diff --git a/sdk/src/tcp/client.rs b/sdk/src/tcp/client.rs index da8458863..46188f86f 100644 --- a/sdk/src/tcp/client.rs +++ b/sdk/src/tcp/client.rs @@ -11,7 +11,7 @@ use crate::utils::duration::IggyDuration; use crate::utils::timestamp::IggyTimestamp; use async_broadcast::{broadcast, Receiver, Sender}; use async_trait::async_trait; -use bytes::{Buf, BufMut, Bytes, BytesMut}; +use bytes::{BufMut, Bytes, BytesMut}; use rustls::pki_types::{pem::PemObject, CertificateDer, ServerName}; use std::fmt::Debug; use std::net::SocketAddr; @@ -99,7 +99,7 @@ impl ConnectionStream for TcpConnectionStream { "Failed to read data by client: {} from the TCP connection: {error}", self.client_address ); - IggyError::from(error) + IggyError::TcpError }) } @@ -109,7 +109,7 @@ impl ConnectionStream for TcpConnectionStream { "Failed to write data by client: {} to the TCP connection: {error}", self.client_address ); - IggyError::from(error) + IggyError::TcpError }) } @@ -119,7 +119,7 @@ impl ConnectionStream for TcpConnectionStream { "Failed to flush data by client: {} to the TCP connection: {error}", self.client_address ); - IggyError::from(error) + IggyError::TcpError }) } @@ -129,7 +129,7 @@ impl ConnectionStream for TcpConnectionStream { "Failed to shutdown the TCP connection by client: {} to the TCP connection: {error}", self.client_address ); - IggyError::from(error) + IggyError::TcpError }) } } @@ -142,7 +142,7 @@ impl ConnectionStream for TcpTlsConnectionStream { "Failed to read data by client: {} from the TCP TLS connection: {error}", self.client_address ); - IggyError::from(error) + IggyError::TcpError }) } @@ -152,7 +152,7 @@ impl ConnectionStream for TcpTlsConnectionStream { "Failed to write data by client: {} to the TCP TLS connection: {error}", self.client_address ); - IggyError::from(error) + IggyError::TcpError }) } @@ -162,7 +162,7 @@ impl ConnectionStream for TcpTlsConnectionStream { "Failed to flush data by client: {} to the TCP TLS connection: {error}", self.client_address ); - IggyError::from(error) + IggyError::TcpError }) } @@ -172,7 +172,7 @@ impl ConnectionStream for TcpTlsConnectionStream { "Failed to shutdown the TCP TLS connection by client: {} to the TCP TLS connection: {error}", self.client_address ); - IggyError::from(error) + IggyError::TcpError }) } } @@ -346,18 +346,7 @@ impl TcpClient { ); } - let mut error_details_buffer = BytesMut::with_capacity(length as usize); - error_details_buffer.put_bytes(0, length as usize); - stream.read(&mut error_details_buffer).await?; - - let string_length = error_details_buffer.get_u32_le(); - let error_message = String::from_utf8_lossy(&error_details_buffer); - - return Err(IggyError::InvalidResponse( - status, - string_length, - error_message.to_string(), - )); + return Err(IggyError::from_code(status)); } trace!("Status: OK. Response length: {}", length); @@ -452,9 +441,18 @@ impl TcpClient { return Err(IggyError::CannotEstablishConnection); } - let stream = connection?; - client_address = stream.local_addr()?; - remote_address = stream.peer_addr()?; + let stream = connection.map_err(|error| { + error!("Failed to establish TCP connection to the server: {error}",); + IggyError::CannotEstablishConnection + })?; + client_address = stream.local_addr().map_err(|error| { + error!("Failed to get the local address of the client: {error}",); + IggyError::CannotEstablishConnection + })?; + remote_address = stream.peer_addr().map_err(|error| { + error!("Failed to get the remote address of the server: {error}",); + IggyError::CannotEstablishConnection + })?; self.client_address.lock().await.replace(client_address); if !tls_enabled { @@ -489,13 +487,19 @@ impl TcpClient { .with_root_certificates(root_cert_store) .with_no_client_auth(); let connector = TlsConnector::from(Arc::new(config)); - let stream = TcpStream::connect(client_address).await?; + let stream = TcpStream::connect(client_address).await.map_err(|error| { + error!("Failed to establish TCP connection to the server: {error}",); + IggyError::CannotEstablishConnection + })?; let tls_domain = self.config.tls_domain.to_owned(); let domain = ServerName::try_from(tls_domain).map_err(|error| { error!("Failed to create a server name from the domain. {error}",); IggyError::InvalidTlsDomain })?; - let stream = connector.connect(domain, stream).await?; + let stream = connector.connect(domain, stream).await.map_err(|error| { + error!("Failed to establish a TLS connection to the server: {error}",); + IggyError::CannotEstablishConnection + })?; connection_stream = Box::new(TcpTlsConnectionStream::new( client_address, TlsStream::Client(stream), @@ -609,8 +613,16 @@ impl TcpClient { return Err(IggyError::EmptyResponse); } - let status = u32::from_le_bytes(response_buffer[..4].try_into()?); - let length = u32::from_le_bytes(response_buffer[4..].try_into()?); + let status = u32::from_le_bytes( + response_buffer[..4] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); + let length = u32::from_le_bytes( + response_buffer[4..] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); return self.handle_response(status, length, stream.as_mut()).await; } diff --git a/sdk/src/topics/create_topic.rs b/sdk/src/topics/create_topic.rs index 746861c1c..047d733da 100644 --- a/sdk/src/topics/create_topic.rs +++ b/sdk/src/topics/create_topic.rs @@ -123,21 +123,38 @@ impl BytesSerializable for CreateTopic { let mut position = 0; let stream_id = Identifier::from_bytes(bytes.clone())?; position += stream_id.get_size_bytes().as_bytes_usize(); - let topic_id = u32::from_le_bytes(bytes[position..position + 4].try_into()?); + let topic_id = u32::from_le_bytes( + bytes[position..position + 4] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); let topic_id = if topic_id == 0 { None } else { Some(topic_id) }; - let partitions_count = u32::from_le_bytes(bytes[position + 4..position + 8].try_into()?); + let partitions_count = u32::from_le_bytes( + bytes[position + 4..position + 8] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); let compression_algorithm = CompressionAlgorithm::from_code(bytes[position + 8])?; - let message_expiry = u64::from_le_bytes(bytes[position + 9..position + 17].try_into()?); + let message_expiry = u64::from_le_bytes( + bytes[position + 9..position + 17] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); let message_expiry: IggyExpiry = message_expiry.into(); - let max_topic_size = u64::from_le_bytes(bytes[position + 17..position + 25].try_into()?); + let max_topic_size = u64::from_le_bytes( + bytes[position + 17..position + 25] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); let max_topic_size: MaxTopicSize = max_topic_size.into(); let replication_factor = match bytes[position + 25] { 0 => None, factor => Some(factor), }; let name_length = bytes[position + 26]; - let name = - from_utf8(&bytes[position + 27..(position + 27 + name_length as usize)])?.to_string(); + let name = from_utf8(&bytes[position + 27..(position + 27 + name_length as usize)]) + .map_err(|_| IggyError::InvalidUtf8)? + .to_string(); if name.len() != name_length as usize { return Err(IggyError::InvalidCommand); } diff --git a/sdk/src/topics/update_topic.rs b/sdk/src/topics/update_topic.rs index 1b935c4b1..b3c8c0347 100644 --- a/sdk/src/topics/update_topic.rs +++ b/sdk/src/topics/update_topic.rs @@ -117,17 +117,26 @@ impl BytesSerializable for UpdateTopic { position += topic_id.get_size_bytes().as_bytes_usize(); let compression_algorithm = CompressionAlgorithm::from_code(bytes[position])?; position += 1; - let message_expiry = u64::from_le_bytes(bytes[position..position + 8].try_into()?); + let message_expiry = u64::from_le_bytes( + bytes[position..position + 8] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); let message_expiry: IggyExpiry = message_expiry.into(); - let max_topic_size = u64::from_le_bytes(bytes[position + 8..position + 16].try_into()?); + let max_topic_size = u64::from_le_bytes( + bytes[position + 8..position + 16] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); let max_topic_size: MaxTopicSize = max_topic_size.into(); let replication_factor = match bytes[position + 16] { 0 => None, factor => Some(factor), }; let name_length = bytes[position + 17]; - let name = - from_utf8(&bytes[position + 18..(position + 18 + name_length as usize)])?.to_string(); + let name = from_utf8(&bytes[position + 18..(position + 18 + name_length as usize)]) + .map_err(|_| IggyError::InvalidUtf8)? + .to_string(); if name.len() != name_length as usize { return Err(IggyError::InvalidCommand); } diff --git a/sdk/src/users/change_password.rs b/sdk/src/users/change_password.rs index 4e67d2034..cb10ad03f 100644 --- a/sdk/src/users/change_password.rs +++ b/sdk/src/users/change_password.rs @@ -86,12 +86,15 @@ impl BytesSerializable for ChangePassword { let current_password_length = bytes[position]; position += 1; let current_password = - from_utf8(&bytes[position..position + current_password_length as usize])?.to_string(); + from_utf8(&bytes[position..position + current_password_length as usize]) + .map_err(|_| IggyError::InvalidUtf8)? + .to_string(); position += current_password_length as usize; let new_password_length = bytes[position]; position += 1; - let new_password = - from_utf8(&bytes[position..position + new_password_length as usize])?.to_string(); + let new_password = from_utf8(&bytes[position..position + new_password_length as usize]) + .map_err(|_| IggyError::InvalidUtf8)? + .to_string(); let command = ChangePassword { user_id, diff --git a/sdk/src/users/create_user.rs b/sdk/src/users/create_user.rs index 187a8cb8e..b2b3b265f 100644 --- a/sdk/src/users/create_user.rs +++ b/sdk/src/users/create_user.rs @@ -98,7 +98,9 @@ impl BytesSerializable for CreateUser { } let username_length = bytes[0]; - let username = from_utf8(&bytes[1..1 + username_length as usize])?.to_string(); + let username = from_utf8(&bytes[1..1 + username_length as usize]) + .map_err(|_| IggyError::InvalidUtf8)? + .to_string(); if username.len() != username_length as usize { return Err(IggyError::InvalidCommand); } @@ -106,8 +108,9 @@ impl BytesSerializable for CreateUser { let mut position = 1 + username_length as usize; let password_length = bytes[position]; position += 1; - let password = - from_utf8(&bytes[position..position + password_length as usize])?.to_string(); + let password = from_utf8(&bytes[position..position + password_length as usize]) + .map_err(|_| IggyError::InvalidUtf8)? + .to_string(); if password.len() != password_length as usize { return Err(IggyError::InvalidCommand); } @@ -122,7 +125,11 @@ impl BytesSerializable for CreateUser { position += 1; let permissions = if has_permissions == 1 { - let permissions_length = u32::from_le_bytes(bytes[position..position + 4].try_into()?); + let permissions_length = u32::from_le_bytes( + bytes[position..position + 4] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); position += 4; Some(Permissions::from_bytes( bytes.slice(position..position + permissions_length as usize), diff --git a/sdk/src/users/login_user.rs b/sdk/src/users/login_user.rs index 8af47bd64..e182b001b 100644 --- a/sdk/src/users/login_user.rs +++ b/sdk/src/users/login_user.rs @@ -102,7 +102,9 @@ impl BytesSerializable for LoginUser { } let username_length = bytes[0]; - let username = from_utf8(&bytes[1..=(username_length as usize)])?.to_string(); + let username = from_utf8(&bytes[1..=(username_length as usize)]) + .map_err(|_| IggyError::InvalidUtf8)? + .to_string(); if username.len() != username_length as usize { return Err(IggyError::InvalidCommand); } @@ -111,30 +113,41 @@ impl BytesSerializable for LoginUser { let password = from_utf8( &bytes[2 + username_length as usize ..2 + username_length as usize + password_length as usize], - )? + ) + .map_err(|_| IggyError::InvalidUtf8)? .to_string(); if password.len() != password_length as usize { return Err(IggyError::InvalidCommand); } let position = 2 + username_length as usize + password_length as usize; - let version_length = u32::from_le_bytes(bytes[position..position + 4].try_into()?); + let version_length = u32::from_le_bytes( + bytes[position..position + 4] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); let version = match version_length { 0 => None, _ => { let version = - from_utf8(&bytes[position + 4..position + 4 + version_length as usize])? + from_utf8(&bytes[position + 4..position + 4 + version_length as usize]) + .map_err(|_| IggyError::InvalidUtf8)? .to_string(); Some(version) } }; let position = position + 4 + version_length as usize; - let context_length = u32::from_le_bytes(bytes[position..position + 4].try_into()?); + let context_length = u32::from_le_bytes( + bytes[position..position + 4] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); let context = match context_length { 0 => None, _ => { let context = - from_utf8(&bytes[position + 4..position + 4 + context_length as usize])? + from_utf8(&bytes[position + 4..position + 4 + context_length as usize]) + .map_err(|_| IggyError::InvalidUtf8)? .to_string(); Some(context) } diff --git a/sdk/src/users/update_user.rs b/sdk/src/users/update_user.rs index 03bd3acda..37cbdacc0 100644 --- a/sdk/src/users/update_user.rs +++ b/sdk/src/users/update_user.rs @@ -92,8 +92,9 @@ impl BytesSerializable for UpdateUser { let username = if has_username == 1 { let username_length = bytes[position]; position += 1; - let username = - from_utf8(&bytes[position..position + username_length as usize])?.to_string(); + let username = from_utf8(&bytes[position..position + username_length as usize]) + .map_err(|_| IggyError::InvalidUtf8)? + .to_string(); position += username_length as usize; Some(username) } else { diff --git a/sdk/src/utils/byte_size.rs b/sdk/src/utils/byte_size.rs index e2a783f9a..be915bc56 100644 --- a/sdk/src/utils/byte_size.rs +++ b/sdk/src/utils/byte_size.rs @@ -104,7 +104,9 @@ impl FromStr for IggyByteSize { if matches!(s, "0" | "unlimited" | "Unlimited" | "none" | "None") { Ok(IggyByteSize(Byte::from_u64(0))) } else { - Ok(IggyByteSize(Byte::from_str(s)?)) + Ok(IggyByteSize( + Byte::from_str(s).map_err(|_| IggyError::InvalidSizeBytes)?, + )) } } } diff --git a/server/Cargo.toml b/server/Cargo.toml index 69cf3d117..0944d4d7d 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "server" -version = "0.4.86" +version = "0.4.90" edition = "2021" build = "src/build.rs" @@ -16,7 +16,7 @@ async-trait = "0.1.82" atone = "0.3.7" axum = "0.7.5" axum-server = { version = "0.7.1", features = ["tls-rustls"] } -bcrypt = "0.15.1" +bcrypt = "0.16.0" bincode = "1.3.3" blake3 = "1.5.4" bytes = "1.6.0" @@ -26,19 +26,18 @@ console-subscriber = { version = "0.4.0", optional = true } dashmap = "6.0.1" derive_more = "1.0.0" dotenvy = { version = "0.15.7" } -error_set = { version = "0.6.4", features = ["tracing"] } +error_set = { version = "0.8.3", features = ["tracing"] } figlet-rs = "0.1.5" -figment = { version = "0.10.18", features = ["toml", "env"] } +figment = { version = "0.10.19", features = ["toml", "env"] } flume = "0.11.0" futures = "0.3.30" iggy = { path = "../sdk" } jsonwebtoken = "9.3.0" -log = "0.4.20" moka = { version = "0.12.5", features = ["future"] } openssl = { version = "0.10.66", features = ["vendored"] } -opentelemetry = { version = "0.26.0", features = ["trace", "logs"] } -opentelemetry-appender-tracing = { version = "0.26.0", features = ["log"] } -opentelemetry-otlp = { version = "0.26.0", features = [ +opentelemetry = { version = "0.27.0", features = ["trace", "logs"] } +opentelemetry-appender-tracing = { version = "0.27.0", features = ["log"] } +opentelemetry-otlp = { version = "0.27.0", features = [ "logs", "trace", "grpc-tonic", @@ -47,8 +46,8 @@ opentelemetry-otlp = { version = "0.26.0", features = [ "reqwest-client", "tokio", ] } -opentelemetry-semantic-conventions = { version = "0.26.0" } -opentelemetry_sdk = { version = "0.26.0", features = [ +opentelemetry-semantic-conventions = { version = "0.27.0" } +opentelemetry_sdk = { version = "0.27.0", features = [ "rt-tokio", "logs", "trace", @@ -62,19 +61,17 @@ reqwest = { version = "0.12.4", features = [ "rustls-tls-no-provider", ] } ring = "0.17.8" -rmp-serde = "1.3.0" rust-s3 = { version = "0.34.0", features = ["default"] } rustls = { version = "0.23.10" } rustls-pemfile = "2.1.2" serde = { version = "1.0.210", features = ["derive", "rc"] } serde_json = "1.0.127" serde_with = { version = "3.8.1", features = ["base64", "macros"] } -sled = "0.34.7" static-toml = "1.2.0" strip-ansi-escapes = "0.2.0" strum = { version = "0.26.2", features = ["derive"] } -sysinfo = "0.32.0" -thiserror = "1.0.61" +sysinfo = "0.33.0" +thiserror = "2.0.9" tokio = { version = "1.40.0", features = ["full"] } tokio-native-tls = "0.3.1" toml = "0.8.14" @@ -85,7 +82,7 @@ tower-http = { version = "0.6.1", features = [ ] } tracing = { version = "0.1.40" } tracing-appender = "0.2.3" -tracing-opentelemetry = { version = "0.27.0" } +tracing-opentelemetry = { version = "0.28.0" } tracing-subscriber = { version = "0.3.18", features = ["fmt", "env-filter"] } ulid = "1.1.2" uuid = { version = "1.1.0", features = ["v7", "fast-rng", "zerocopy"] } @@ -97,7 +94,7 @@ tikv-jemallocator = { version = "0.6", optional = true } [build-dependencies] figment = { version = "0.10.18", features = ["json", "toml", "env"] } -serde_json = "1.0.127" +serde_json = "1.0.134" vergen-git2 = { version = "1.0.0", features = [ "build", "cargo", diff --git a/server/src/command.rs b/server/src/command.rs index 8ab7e103b..b34c20888 100644 --- a/server/src/command.rs +++ b/server/src/command.rs @@ -150,7 +150,11 @@ impl BytesSerializable for ServerCommand { } fn from_bytes(bytes: Bytes) -> Result { - let code = u32::from_le_bytes(bytes[..4].try_into()?); + let code = u32::from_le_bytes( + bytes[..4] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); let payload = bytes.slice(4..); match code { PING_CODE => Ok(ServerCommand::Ping(Ping::from_bytes(payload)?)), diff --git a/server/src/compat/mod.rs b/server/src/compat/mod.rs index b8badb778..c25a93c7f 100644 --- a/server/src/compat/mod.rs +++ b/server/src/compat/mod.rs @@ -1,2 +1 @@ pub mod index_conversion; -pub mod storage_conversion; diff --git a/server/src/compat/storage_conversion/converter.rs b/server/src/compat/storage_conversion/converter.rs deleted file mode 100644 index 98f5dea85..000000000 --- a/server/src/compat/storage_conversion/converter.rs +++ /dev/null @@ -1,238 +0,0 @@ -use crate::compat::storage_conversion::COMPONENT; -use crate::state::command::EntryCommand; -use crate::state::models::CreatePersonalAccessTokenWithHash; -use crate::state::State; -use crate::streaming::personal_access_tokens::personal_access_token::PersonalAccessToken; -use crate::streaming::storage::SystemStorage; -use crate::streaming::streams::stream::Stream; -use crate::streaming::users::user::User; -use error_set::ResultContext; -use iggy::consumer_groups::create_consumer_group::CreateConsumerGroup; -use iggy::error::IggyError; -use iggy::locking::IggySharedMutFn; -use iggy::personal_access_tokens::create_personal_access_token::CreatePersonalAccessToken; -use iggy::streams::create_stream::CreateStream; -use iggy::topics::create_topic::CreateTopic; -use iggy::users::create_user::CreateUser; -use iggy::utils::expiry::IggyExpiry; -use iggy::utils::timestamp::IggyTimestamp; -use std::path::Path; -use std::sync::Arc; -use tokio::fs::create_dir; -use tracing::{error, info}; - -pub async fn convert( - state: Arc, - storage: Arc, - mut streams: Vec, - mut users: Vec, - personal_access_tokens: Vec, -) -> Result<(), IggyError> { - info!("Converting storage to new format"); - state.init().await?; - streams.sort_by(|a, b| a.stream_id.cmp(&b.stream_id)); - users.sort_by(|a, b| a.id.cmp(&b.id)); - info!("Converting {} users", users.len()); - for user in users { - state - .apply( - 0, - EntryCommand::CreateUser(CreateUser { - username: user.username.clone(), - password: user.password, - status: user.status, - permissions: user.permissions.clone(), - }), - ) - .await - .with_error(|_| { - format!( - "{COMPONENT} - failed to apply create user with username: {}", - user.username, - ) - })?; - } - - info!( - "Converting {} personal access tokens", - personal_access_tokens.len() - ); - for personal_access_token in personal_access_tokens { - let now = IggyTimestamp::now(); - let mut expiry = IggyExpiry::NeverExpire; - if let Some(expiry_at) = personal_access_token.expiry_at { - if expiry_at.as_micros() <= now.as_micros() { - continue; - } - expiry = IggyExpiry::ExpireDuration((expiry_at.as_micros() - now.as_micros()).into()); - } - - state - .apply( - personal_access_token.user_id, - EntryCommand::CreatePersonalAccessToken(CreatePersonalAccessTokenWithHash { - command: CreatePersonalAccessToken { - name: personal_access_token.name.clone(), - expiry, - }, - hash: personal_access_token.token, - }), - ) - .await - .with_error(|_| { - format!( - "{COMPONENT} - failed to apply create personal access token, user ID: {}, personal_access_token_name: {}", - personal_access_token.user_id, personal_access_token.name, - ) - })?; - } - - info!("Converting {} streams", streams.len()); - for stream in streams { - state - .apply( - 0, - EntryCommand::CreateStream(CreateStream { - stream_id: Some(stream.stream_id), - name: stream.name, - }), - ) - .await?; - - info!( - "Converting {} topics for stream with ID: {}", - stream.topics.len(), - stream.stream_id - ); - for topic in stream.topics.into_values() { - state - .apply( - 0, - EntryCommand::CreateTopic(CreateTopic { - stream_id: topic.stream_id.try_into()?, - topic_id: Some(topic.topic_id), - partitions_count: topic.partitions.len() as u32, - compression_algorithm: topic.compression_algorithm, - message_expiry: topic.message_expiry, - max_topic_size: topic.max_topic_size, - replication_factor: if topic.replication_factor > 0 { - Some(topic.replication_factor) - } else { - None - }, - name: topic.name, - }), - ) - .await - .with_error(|_| { - format!( - "{COMPONENT} - failed to create topic, stream ID: {}, topic ID: {}", - topic.stream_id, topic.topic_id, - ) - })?; - - info!( - "Converting {} consumer groups for topic with ID: {}", - topic.consumer_groups.len(), - topic.topic_id, - ); - for group in topic.consumer_groups.into_values() { - let group = group.read().await; - state - .apply( - 0, - EntryCommand::CreateConsumerGroup(CreateConsumerGroup { - stream_id: stream.stream_id.try_into()?, - topic_id: topic.topic_id.try_into()?, - group_id: Some(group.group_id), - name: group.name.to_owned(), - }), - ) - .await - .with_error(|_| format!( - "{COMPONENT} - failed to create consumer group, stream ID: {}, topic ID: {}, group ID: {}", - stream.stream_id, topic.topic_id, group.group_id, - ))?; - } - - info!( - "Converting {} partitions for topic with ID: {}", - topic.partitions.len(), - topic.topic_id, - ); - for partition in topic.partitions.into_values() { - let partition = partition.read().await; - - if !Path::new(&partition.offsets_path).exists() - && create_dir(&partition.offsets_path).await.is_err() - { - error!( - "Failed to create offsets directory for partition with ID: {} for stream with ID: {} and topic with ID: {}.", - partition.partition_id, partition.stream_id, partition.topic_id - ); - return Err(IggyError::CannotCreatePartition( - partition.partition_id, - partition.stream_id, - partition.topic_id, - )); - } - - info!("Creating consumer offsets directory for partition with ID: {} for stream with ID: {} and topic with ID: {}, path: {}", - partition.partition_id, partition.stream_id, partition.topic_id, partition.consumer_offsets_path); - if !Path::new(&partition.consumer_offsets_path).exists() - && create_dir(&partition.consumer_offsets_path).await.is_err() - { - error!( - "Failed to create consumer offsets directory for partition with ID: {} for stream with ID: {} and topic with ID: {}.", - partition.partition_id, partition.stream_id, partition.topic_id - ); - return Err(IggyError::CannotCreatePartition( - partition.partition_id, - partition.stream_id, - partition.topic_id, - )); - } - - info!("Creating consumer group offsets directory for partition with ID: {} for stream with ID: {} and topic with ID: {}, path: {}", - partition.partition_id, partition.stream_id, partition.topic_id, partition.consumer_group_offsets_path); - if !Path::new(&partition.consumer_group_offsets_path).exists() - && create_dir(&partition.consumer_group_offsets_path) - .await - .is_err() - { - error!( - "Failed to create consumer group offsets directory for partition with ID: {} for stream with ID: {} and topic with ID: {}.", - partition.partition_id, partition.stream_id, partition.topic_id - ); - return Err(IggyError::CannotCreatePartition( - partition.partition_id, - partition.stream_id, - partition.topic_id, - )); - } - - info!("Converting {} consumer offsets for partition with ID: {} for stream with ID: {} and topic with ID: {}", - partition.consumer_offsets.len(), partition.partition_id, partition.stream_id, partition.topic_id); - for offset in partition.consumer_offsets.iter() { - storage.partition.save_consumer_offset(&offset) - .await - .with_error(|_| format!("{COMPONENT} - failed to save consumer offsets, stream ID: {}, topic ID: {}, partition ID: {}", - partition.stream_id, partition.topic_id, partition.partition_id, - ))?; - } - - info!("Converting {} consumer group offsets for partition with ID: {} for stream with ID: {} and topic with ID: {}", - partition.consumer_group_offsets.len(), partition.partition_id, partition.stream_id, partition.topic_id); - for offset in partition.consumer_group_offsets.iter() { - storage.partition.save_consumer_offset(&offset) - .await - .with_error(|_| format!("{COMPONENT} - failed to save consumer group offsets, stream ID: {}, topic ID: {}, partition ID: {}", - partition.stream_id, partition.topic_id, partition.partition_id, - ))?; - } - } - } - } - info!("Conversion completed"); - Ok(()) -} diff --git a/server/src/compat/storage_conversion/mod.rs b/server/src/compat/storage_conversion/mod.rs deleted file mode 100644 index 58f8941ac..000000000 --- a/server/src/compat/storage_conversion/mod.rs +++ /dev/null @@ -1,299 +0,0 @@ -mod converter; -mod persistency; - -use crate::compat::storage_conversion::persistency::{personal_access_tokens, streams, users}; -use crate::configs::system::SystemConfig; -use crate::state::system::{PartitionState, StreamState, TopicState}; -use crate::state::State; -use crate::streaming::batching::message_batch::RetainedMessageBatch; -use crate::streaming::partitions::partition::{ConsumerOffset, Partition}; -use crate::streaming::persistence::persister::Persister; -use crate::streaming::segments::index::{Index, IndexRange}; -use crate::streaming::segments::segment::Segment; -use crate::streaming::storage::{ - PartitionStorage, SegmentStorage, StreamStorage, SystemInfoStorage, SystemStorage, TopicStorage, -}; -use crate::streaming::streams::stream::Stream; -use crate::streaming::systems::info::SystemInfo; -use crate::streaming::topics::topic::Topic; -use async_trait::async_trait; -use error_set::ResultContext; -use iggy::consumer::ConsumerKind; -use iggy::error::IggyError; -use iggy::utils::byte_size::IggyByteSize; -use std::path::Path; -use std::sync::Arc; -use tokio::fs::{read_dir, rename}; -use tracing::{error, info}; - -pub const COMPONENT: &str = "STORAGE_CONVERSION"; - -pub async fn init( - config: Arc, - metadata: Arc, - storage: Arc, -) -> Result<(), IggyError> { - if Path::new(&config.get_state_log_path()).exists() { - info!("State log already exists, skipping storage migration"); - return Ok(()); - } - - let path = config.get_database_path(); - if path.is_none() { - info!("No database path configured, skipping storage migration"); - return Ok(()); - } - - let database_path = path.unwrap(); - if !Path::new(&database_path).exists() { - error!("Database directory: {database_path} does not exist - cannot migrate storage."); - return Err(IggyError::CannotOpenDatabase(database_path)); - } - - let db_file = format!("{database_path}/db"); - if !Path::new(&db_file).exists() { - error!("Database file at path: {db_file} does not exist - cannot migrate storage."); - return Err(IggyError::CannotOpenDatabase(db_file)); - } - - info!("Starting storage migration, database path: {database_path}"); - let db = sled::open(&database_path); - if db.is_err() { - panic!("Cannot open database at: {database_path}"); - } - let db = db.unwrap(); - let mut streams = Vec::new(); - let dir_entries = read_dir(&config.get_streams_path()).await; - if let Err(error) = dir_entries { - error!("Cannot read streams directory: {}", error); - return Err(IggyError::CannotReadStreams); - } - - let noop_storage = SystemStorage { - info: Arc::new(NoopSystemInfoStorage {}), - stream: Arc::new(NoopStreamStorage {}), - topic: Arc::new(NoopTopicStorage {}), - partition: Arc::new(NoopPartitionStorage {}), - segment: Arc::new(NoopSegmentStorage {}), - persister: Arc::new(NoopPersister {}), - }; - let noop_storage = Arc::new(noop_storage); - let mut dir_entries = dir_entries.unwrap(); - while let Some(dir_entry) = dir_entries.next_entry().await.unwrap_or(None) { - let name = dir_entry.file_name().into_string().unwrap(); - let stream_id = name.parse::(); - if stream_id.is_err() { - error!("Invalid stream ID file with name: '{}'.", name); - continue; - } - - let stream_id = stream_id.unwrap(); - let mut stream = Stream::empty(stream_id, "stream", config.clone(), noop_storage.clone()); - streams::load(&config, &db, &mut stream) - .await - .with_error(|_| { - format!( - "STORAGE_CONVERSION - failed to load stream, stream ID: {}", - stream.stream_id - ) - })?; - streams.push(stream); - } - - let users = users::load_all(&db) - .await - .with_error(|_| "STORAGE_CONVERSION - failed to load users".to_string())?; - let personal_access_tokens = personal_access_tokens::load_all(&db) - .await - .with_error(|_| "STORAGE_CONVERSION - failed to load personal access tokens".to_string())?; - converter::convert(metadata, storage, streams, users, personal_access_tokens) - .await - .with_error(|_| { - "STORAGE_CONVERSION - failed to convert storage to new format".to_string() - })?; - let old_database_path = format!("{database_path}_old"); - rename(&database_path, &old_database_path) - .await - .with_error(|_| { - format!( - "STORAGE_CONVERSION - failed to rename database from {} to {}", - database_path, old_database_path - ) - })?; - info!("Storage migration has completed, new state log was cacreated and old database was moved to: {old_database_path} (now it can be safely deleted)."); - Ok(()) -} - -struct NoopPersister {} -struct NoopSystemInfoStorage {} -struct NoopStreamStorage {} -struct NoopTopicStorage {} -struct NoopPartitionStorage {} -struct NoopSegmentStorage {} - -#[async_trait] -impl Persister for NoopPersister { - async fn append(&self, _path: &str, _bytes: &[u8]) -> Result<(), IggyError> { - Ok(()) - } - - async fn overwrite(&self, _path: &str, _bytes: &[u8]) -> Result<(), IggyError> { - Ok(()) - } - - async fn delete(&self, _path: &str) -> Result<(), IggyError> { - Ok(()) - } -} - -#[async_trait] -impl SystemInfoStorage for NoopSystemInfoStorage { - async fn load(&self) -> Result { - Ok(SystemInfo::default()) - } - - async fn save(&self, _system_info: &SystemInfo) -> Result<(), IggyError> { - Ok(()) - } -} - -#[async_trait] -impl StreamStorage for NoopStreamStorage { - async fn load(&self, _stream: &mut Stream, _state: StreamState) -> Result<(), IggyError> { - Ok(()) - } - - async fn save(&self, _stream: &Stream) -> Result<(), IggyError> { - Ok(()) - } - - async fn delete(&self, _stream: &Stream) -> Result<(), IggyError> { - Ok(()) - } -} - -#[async_trait] -impl TopicStorage for NoopTopicStorage { - async fn load(&self, _topic: &mut Topic, _state: TopicState) -> Result<(), IggyError> { - Ok(()) - } - - async fn save(&self, _topic: &Topic) -> Result<(), IggyError> { - Ok(()) - } - - async fn delete(&self, _topic: &Topic) -> Result<(), IggyError> { - Ok(()) - } -} - -#[async_trait] -impl PartitionStorage for NoopPartitionStorage { - async fn load( - &self, - _partition: &mut Partition, - _state: PartitionState, - ) -> Result<(), IggyError> { - Ok(()) - } - - async fn save(&self, _partition: &Partition) -> Result<(), IggyError> { - Ok(()) - } - - async fn delete(&self, _partition: &Partition) -> Result<(), IggyError> { - Ok(()) - } - - async fn save_consumer_offset(&self, _offset: &ConsumerOffset) -> Result<(), IggyError> { - Ok(()) - } - - async fn load_consumer_offsets( - &self, - _kind: ConsumerKind, - _path: &str, - ) -> Result, IggyError> { - Ok(vec![]) - } - - async fn delete_consumer_offsets(&self, _path: &str) -> Result<(), IggyError> { - Ok(()) - } - - async fn delete_consumer_offset(&self, _path: &str) -> Result<(), IggyError> { - Ok(()) - } -} - -#[async_trait] -impl SegmentStorage for NoopSegmentStorage { - async fn load(&self, _segment: &mut Segment) -> Result<(), IggyError> { - Ok(()) - } - - async fn save(&self, _segment: &Segment) -> Result<(), IggyError> { - Ok(()) - } - - async fn delete(&self, _segment: &Segment) -> Result<(), IggyError> { - Ok(()) - } - - async fn load_message_batches( - &self, - _segment: &Segment, - _index_range: &IndexRange, - ) -> Result, IggyError> { - Ok(vec![]) - } - - async fn load_newest_batches_by_size( - &self, - _segment: &Segment, - _size: u64, - ) -> Result, IggyError> { - Ok(vec![]) - } - - async fn save_batches( - &self, - _segment: &Segment, - _batch: RetainedMessageBatch, - ) -> Result { - Ok(IggyByteSize::default()) - } - - async fn load_message_ids(&self, _segment: &Segment) -> Result, IggyError> { - Ok(vec![]) - } - - async fn load_checksums(&self, _segment: &Segment) -> Result<(), IggyError> { - Ok(()) - } - - async fn load_all_indexes(&self, _segment: &Segment) -> Result, IggyError> { - Ok(vec![]) - } - - async fn load_index_range( - &self, - _segment: &Segment, - _index_start_offset: u64, - _index_end_offset: u64, - ) -> Result, IggyError> { - Ok(None) - } - - async fn save_index(&self, _index_path: &str, _index: Index) -> Result<(), IggyError> { - Ok(()) - } - - async fn try_load_index_for_timestamp( - &self, - _segment: &Segment, - _timestamp: u64, - ) -> Result, IggyError> { - Ok(None) - } -} diff --git a/server/src/compat/storage_conversion/persistency/mod.rs b/server/src/compat/storage_conversion/persistency/mod.rs deleted file mode 100644 index b06df1164..000000000 --- a/server/src/compat/storage_conversion/persistency/mod.rs +++ /dev/null @@ -1,7 +0,0 @@ -pub mod partitions; -pub mod personal_access_tokens; -pub mod streams; -pub mod topics; -pub mod users; - -pub const COMPONENT: &str = "STORAGE_CONVERSION_PERSISTENCY"; diff --git a/server/src/compat/storage_conversion/persistency/partitions.rs b/server/src/compat/storage_conversion/persistency/partitions.rs deleted file mode 100644 index 19c74275e..000000000 --- a/server/src/compat/storage_conversion/persistency/partitions.rs +++ /dev/null @@ -1,332 +0,0 @@ -use crate::compat::storage_conversion::persistency::COMPONENT; -use crate::configs::system::SystemConfig; -use crate::streaming::batching::batch_accumulator::BatchAccumulator; -use crate::streaming::partitions::partition::{ConsumerOffset, Partition}; -use crate::streaming::segments::segment::{Segment, LOG_EXTENSION}; -use anyhow::Context; -use error_set::ResultContext; -use iggy::consumer::ConsumerKind; -use iggy::error::IggyError; -use iggy::utils::timestamp::IggyTimestamp; -use serde::{Deserialize, Serialize}; -use sled::Db; -use std::sync::atomic::Ordering; -use tokio::fs; -use tracing::{info, warn}; - -pub async fn load_consumer_offsets( - db: &Db, - config: &SystemConfig, - kind: ConsumerKind, - stream_id: u32, - topic_id: u32, - partition_id: u32, -) -> Result, IggyError> { - let mut consumer_offsets = Vec::new(); - let key_prefix = format!( - "{}:", - get_key_prefix(kind, stream_id, topic_id, partition_id) - ); - for data in db.scan_prefix(&key_prefix) { - let consumer_offset = match data.with_context(|| { - format!( - "Failed to load consumer offset, when searching by key: {}", - key_prefix - ) - }) { - Ok((key, value)) => { - let key = String::from_utf8(key.to_vec()).unwrap(); - let offset = u64::from_be_bytes(value.as_ref().try_into().unwrap()); - let consumer_id = key.split(':').last().unwrap().parse::().unwrap(); - ConsumerOffsetCompat { - key, - kind, - consumer_id, - offset, - } - } - Err(err) => { - return Err(IggyError::CannotLoadResource(err)); - } - }; - consumer_offsets.push(consumer_offset); - } - - consumer_offsets.sort_by(|a, b| a.consumer_id.cmp(&b.consumer_id)); - let consumer_offsets = consumer_offsets - .into_iter() - .map(|consumer_offset| { - let path = match kind { - ConsumerKind::Consumer => { - config.get_consumer_offsets_path(stream_id, topic_id, partition_id) - } - ConsumerKind::ConsumerGroup => { - config.get_consumer_group_offsets_path(stream_id, topic_id, partition_id) - } - }; - let path = format!("{path}/{}", consumer_offset.consumer_id); - ConsumerOffset { - kind: consumer_offset.kind, - consumer_id: consumer_offset.consumer_id, - offset: consumer_offset.offset, - path, - } - }) - .collect::>(); - - Ok(consumer_offsets) -} - -pub async fn load( - config: &SystemConfig, - db: &Db, - partition: &mut Partition, -) -> Result<(), IggyError> { - info!( - "Loading partition with ID: {} for stream with ID: {} and topic with ID: {}, for path: {} from disk...", - partition.partition_id, partition.stream_id, partition.topic_id, partition.partition_path - ); - let dir_entries = fs::read_dir(&partition.partition_path).await; - if let Err(err) = fs::read_dir(&partition.partition_path) - .await - .with_context(|| format!("Failed to read partition with ID: {} for stream with ID: {} and topic with ID: {} and path: {}", partition.partition_id, partition.stream_id, partition.topic_id, partition.partition_path)) - { - return Err(IggyError::CannotReadPartitions(err)); - } - - let key = get_partition_key( - partition.stream_id, - partition.topic_id, - partition.partition_id, - ); - let partition_data = match db - .get(&key) - .with_context(|| format!("Failed to load partition with key: {}", key)) - { - Ok(partition_data) => { - if let Some(partition_data) = partition_data { - let partition_data = rmp_serde::from_slice::(&partition_data) - .with_context(|| format!("Failed to deserialize partition with key: {}", key)); - if let Err(err) = partition_data { - return Err(IggyError::CannotDeserializeResource(err)); - } else { - partition_data.unwrap() - } - } else { - return Err(IggyError::ResourceNotFound(key)); - } - } - Err(err) => { - return Err(IggyError::CannotLoadResource(err)); - } - }; - - partition.created_at = partition_data.created_at; - - let consumer_offsets_for_consumer = load_consumer_offsets( - db, - config, - ConsumerKind::Consumer, - partition.stream_id, - partition.topic_id, - partition.partition_id, - ) - .await - .with_error(|_| format!( - "{COMPONENT} - failed to load consumer offsets for cosumer, stream ID: {}, topic ID: {}, partition ID: {}", - partition.stream_id, - partition.topic_id, - partition.partition_id), - )?; - - let consumer_offsets_for_group = load_consumer_offsets( - db, - config, - ConsumerKind::ConsumerGroup, - partition.stream_id, - partition.topic_id, - partition.partition_id, - ) - .await - .with_error(|_| format!( - "{COMPONENT} - failed to load consumer offsets for group, stream ID: {}, topic ID: {}, partition ID: {}", - partition.stream_id, - partition.topic_id, - partition.partition_id), - )?; - - for consumer_offset in consumer_offsets_for_consumer { - partition - .consumer_offsets - .insert(consumer_offset.consumer_id, consumer_offset); - } - - for consumer_offset in consumer_offsets_for_group { - partition - .consumer_group_offsets - .insert(consumer_offset.consumer_id, consumer_offset); - } - - let mut dir_entries = dir_entries.unwrap(); - while let Some(dir_entry) = dir_entries.next_entry().await.unwrap_or(None) { - let metadata = dir_entry.metadata().await.unwrap(); - if metadata.is_dir() { - continue; - } - - let path = dir_entry.path(); - let extension = path.extension(); - if extension.is_none() || extension.unwrap() != LOG_EXTENSION { - continue; - } - - let log_file_name = dir_entry - .file_name() - .into_string() - .unwrap() - .replace(&format!(".{}", LOG_EXTENSION), ""); - - let start_offset = log_file_name.parse::().unwrap(); - let mut segment = Segment::create( - partition.stream_id, - partition.topic_id, - partition.partition_id, - start_offset, - partition.config.clone(), - partition.storage.clone(), - partition.message_expiry, - partition.size_of_parent_stream.clone(), - partition.size_of_parent_topic.clone(), - partition.size_bytes.clone(), - partition.messages_count_of_parent_stream.clone(), - partition.messages_count_of_parent_topic.clone(), - partition.messages_count.clone(), - ); - - segment.load().await.with_error(|_| format!( - "{COMPONENT} - failed to load segment with stream ID: {}, topic ID: {}, partition ID: {}", - segment.stream_id, - segment.topic_id, - segment.partition_id, - ))?; - let capacity = partition.config.partition.messages_required_to_save; - if !segment.is_closed { - segment.unsaved_messages = Some(BatchAccumulator::new( - segment.current_offset, - capacity as usize, - )) - } - - // If the first segment has at least a single message, we should increment the offset. - if !partition.should_increment_offset { - partition.should_increment_offset = segment.size_bytes > 0; - } - - if partition.config.partition.validate_checksum { - info!("Validating messages checksum for partition with ID: {} and segment with start offset: {}...", partition.partition_id, segment.start_offset); - segment.storage.segment.load_checksums(&segment).await.with_error(|_| format!( - "{COMPONENT} - failed to load checksums for segment with stream ID: {}, topic ID: {}, partition ID: {}", - segment.stream_id, - segment.topic_id, - segment.partition_id, - ))?; - info!("Validated messages checksum for partition with ID: {} and segment with start offset: {}.", partition.partition_id, segment.start_offset); - } - - // Load the unique message IDs for the partition if the deduplication feature is enabled. - let mut unique_message_ids_count = 0; - if let Some(message_deduplicator) = &partition.message_deduplicator { - info!("Loading unique message IDs for partition with ID: {} and segment with start offset: {}...", partition.partition_id, segment.start_offset); - let message_ids = segment.storage.segment.load_message_ids(&segment).await.with_error(|_| format!( - "{COMPONENT} - failed to load message ids for segment with stream ID: {}, topic ID: {}, partition ID: {}", - segment.stream_id, - segment.topic_id, - segment.partition_id, - ))?; - for message_id in message_ids { - if message_deduplicator.try_insert(&message_id).await { - unique_message_ids_count += 1; - } else { - warn!("Duplicated message ID: {} for partition with ID: {} and segment with start offset: {}.", message_id, partition.partition_id, segment.start_offset); - } - } - info!("Loaded: {} unique message IDs for partition with ID: {} and segment with start offset: {}...", unique_message_ids_count, partition.partition_id, segment.start_offset); - } - - partition - .segments_count_of_parent_stream - .fetch_add(1, Ordering::SeqCst); - partition.segments.push(segment); - } - - partition - .segments - .sort_by(|a, b| a.start_offset.cmp(&b.start_offset)); - - let end_offsets = partition - .segments - .iter() - .skip(1) - .map(|segment| segment.start_offset - 1) - .collect::>(); - - let segments_count = partition.segments.len(); - for (end_offset_index, segment) in partition.get_segments_mut().iter_mut().enumerate() { - if end_offset_index == segments_count - 1 { - break; - } - - segment.end_offset = end_offsets[end_offset_index]; - } - - if !partition.segments.is_empty() { - let last_segment = partition.segments.last_mut().unwrap(); - if last_segment.is_closed { - last_segment.end_offset = last_segment.current_offset; - } - - partition.current_offset = last_segment.current_offset; - } - - partition.load_consumer_offsets().await.with_error(|_| format!( - "{COMPONENT} - failed to load consumere offsets for stream ID: {}, topic ID: {}, partiton ID: {}", - partition.stream_id, - partition.topic_id, - partition.partition_id, - ))?; - info!( - "Loaded partition with ID: {} for stream with ID: {} and topic with ID: {}, current offset: {}.", - partition.partition_id, partition.stream_id, partition.topic_id, partition.current_offset - ); - - Ok(()) -} - -#[derive(Debug, Serialize, Deserialize)] -struct PartitionData { - created_at: IggyTimestamp, -} - -#[derive(Debug, PartialEq, Clone)] -struct ConsumerOffsetCompat { - pub key: String, - pub kind: ConsumerKind, - pub consumer_id: u32, - pub offset: u64, -} - -fn get_partition_key(stream_id: u32, topic_id: u32, partition_id: u32) -> String { - format!( - "streams:{}:topics:{}:partitions:{}", - stream_id, topic_id, partition_id - ) -} - -pub fn get_key_prefix( - kind: ConsumerKind, - stream_id: u32, - topic_id: u32, - partition_id: u32, -) -> String { - format!("{kind}_offsets:{stream_id}:{topic_id}:{partition_id}") -} diff --git a/server/src/compat/storage_conversion/persistency/personal_access_tokens.rs b/server/src/compat/storage_conversion/persistency/personal_access_tokens.rs deleted file mode 100644 index abfcd46e7..000000000 --- a/server/src/compat/storage_conversion/persistency/personal_access_tokens.rs +++ /dev/null @@ -1,56 +0,0 @@ -use crate::streaming::personal_access_tokens::personal_access_token::PersonalAccessToken; -use anyhow::Context; -use iggy::error::IggyError; -use iggy::models::user_info::UserId; -use serde::{Deserialize, Serialize}; -use sled::Db; - -const KEY_PREFIX: &str = "personal_access_token"; - -pub async fn load_all(db: &Db) -> Result, IggyError> { - let mut personal_access_tokens = Vec::new(); - for data in db.scan_prefix(format!("{}:token:", KEY_PREFIX)) { - let personal_access_token_data = match data.with_context(|| { - format!( - "Failed to load personal access token, when searching by key: {}", - KEY_PREFIX - ) - }) { - Ok((_, value)) => match rmp_serde::from_slice::(&value) - .with_context(|| { - format!( - "Failed to deserialize personal access token, when searching by key: {}", - KEY_PREFIX - ) - }) { - Ok(personal_access_token) => personal_access_token, - Err(err) => { - return Err(IggyError::CannotDeserializeResource(err)); - } - }, - Err(err) => { - return Err(IggyError::CannotLoadResource(err)); - } - }; - - let personal_access_token = PersonalAccessToken::raw( - personal_access_token_data.user_id, - &personal_access_token_data.name, - &personal_access_token_data.token, - personal_access_token_data - .expiry - .map(|expiry| expiry.into()), - ); - personal_access_tokens.push(personal_access_token); - } - - Ok(personal_access_tokens) -} - -#[derive(Debug, Serialize, Deserialize)] -struct PersonalAccessTokenData { - pub user_id: UserId, - pub name: String, - pub token: String, - pub expiry: Option, -} diff --git a/server/src/compat/storage_conversion/persistency/streams.rs b/server/src/compat/storage_conversion/persistency/streams.rs deleted file mode 100644 index f2944b0f8..000000000 --- a/server/src/compat/storage_conversion/persistency/streams.rs +++ /dev/null @@ -1,127 +0,0 @@ -use crate::compat::storage_conversion::persistency::topics; -use crate::compat::storage_conversion::persistency::COMPONENT; -use crate::configs::system::SystemConfig; -use crate::streaming::streams::stream::Stream; -use crate::streaming::topics::topic::Topic; -use anyhow::Context; -use error_set::ResultContext; -use iggy::error::IggyError; -use iggy::utils::timestamp::IggyTimestamp; -use serde::{Deserialize, Serialize}; -use sled::Db; -use std::path::Path; -use tokio::fs; -use tracing::{error, info}; - -#[derive(Debug, Serialize, Deserialize)] -struct StreamData { - name: String, - created_at: IggyTimestamp, -} - -pub async fn load(config: &SystemConfig, db: &Db, stream: &mut Stream) -> Result<(), IggyError> { - info!("Loading stream with ID: {} from disk...", stream.stream_id); - if !Path::new(&stream.path).exists() { - return Err(IggyError::StreamIdNotFound(stream.stream_id)); - } - - let key = get_key(stream.stream_id); - let stream_data = match db.get(&key).with_context(|| { - format!( - "Failed to load stream with ID: {}, key: {}", - stream.stream_id, key - ) - }) { - Ok(stream_data) => { - if let Some(stream_data) = stream_data { - let stream_data = - rmp_serde::from_slice::(&stream_data).with_context(|| { - format!( - "Failed to deserialize stream with ID: {}, key: {}", - stream.stream_id, key - ) - }); - match stream_data { - Ok(stream_data) => stream_data, - Err(err) => { - return Err(IggyError::CannotDeserializeResource(err)); - } - } - } else { - return Err(IggyError::ResourceNotFound(key)); - } - } - Err(err) => { - return Err(IggyError::CannotLoadResource(err)); - } - }; - - stream.name = stream_data.name; - stream.created_at = stream_data.created_at; - let mut topics = Vec::new(); - let dir_entries = fs::read_dir(&stream.topics_path).await; - if dir_entries.is_err() { - return Err(IggyError::CannotReadTopics(stream.stream_id)); - } - - let mut dir_entries = dir_entries.unwrap(); - while let Some(dir_entry) = dir_entries.next_entry().await.unwrap_or(None) { - let name = dir_entry.file_name().into_string().unwrap(); - let topic_id = name.parse::(); - if topic_id.is_err() { - error!("Invalid topic ID file with name: '{}'.", name); - continue; - } - - let topic_id = topic_id.unwrap(); - let mut topic = Topic::empty( - stream.stream_id, - topic_id, - "topic", - stream.size_bytes.clone(), - stream.messages_count.clone(), - stream.segments_count.clone(), - stream.config.clone(), - stream.storage.clone(), - ); - topics::load(config, db, &mut topic).await.with_error(|_| { - format!( - "{COMPONENT} - failed to load topic, stream ID: {}, topic ID: {}", - topic.stream_id, topic.topic_id, - ) - })?; - topics.push(topic); - } - - for topic in topics.into_iter() { - if stream.topics.contains_key(&topic.topic_id) { - error!( - "Topic with ID: '{}' already exists for stream with ID: {}.", - &topic.topic_id, &stream.stream_id - ); - continue; - } - - if stream.topics_ids.contains_key(&topic.name) { - error!( - "Topic with name: '{}' already exists for stream with ID: {}.", - &topic.name, &stream.stream_id - ); - continue; - } - - stream.topics_ids.insert(topic.name.clone(), topic.topic_id); - stream.topics.insert(topic.topic_id, topic); - } - - info!( - "Loaded stream: '{}' with ID: {} from disk.", - &stream.name, &stream.stream_id - ); - - Ok(()) -} - -fn get_key(stream_id: u32) -> String { - format!("streams:{}", stream_id) -} diff --git a/server/src/compat/storage_conversion/persistency/topics.rs b/server/src/compat/storage_conversion/persistency/topics.rs deleted file mode 100644 index 072b02b61..000000000 --- a/server/src/compat/storage_conversion/persistency/topics.rs +++ /dev/null @@ -1,197 +0,0 @@ -use crate::compat::storage_conversion::persistency::partitions; -use crate::compat::storage_conversion::persistency::COMPONENT; -use crate::configs::system::SystemConfig; -use crate::streaming::partitions::partition::Partition; -use crate::streaming::topics::consumer_group::ConsumerGroup; -use crate::streaming::topics::topic::Topic; -use anyhow::Context; -use error_set::ResultContext; -use iggy::compression::compression_algorithm::CompressionAlgorithm; -use iggy::error::IggyError; -use iggy::locking::IggySharedMut; -use iggy::locking::IggySharedMutFn; -use iggy::utils::byte_size::IggyByteSize; -use iggy::utils::duration::{IggyDuration, SEC_IN_MICRO}; -use iggy::utils::expiry::IggyExpiry; -use iggy::utils::timestamp::IggyTimestamp; -use serde::{Deserialize, Serialize}; -use sled::Db; -use std::path::Path; -use tokio::fs; -use tokio::sync::RwLock; -use tracing::{error, info}; - -#[derive(Debug, Serialize, Deserialize)] -struct ConsumerGroupData { - id: u32, - name: String, -} - -pub async fn load(config: &SystemConfig, db: &Db, topic: &mut Topic) -> Result<(), IggyError> { - info!("Loading topic {} from disk...", topic); - if !Path::new(&topic.path).exists() { - return Err(IggyError::TopicIdNotFound(topic.topic_id, topic.stream_id)); - } - - let key = get_topic_key(topic.stream_id, topic.topic_id); - let topic_data = match db - .get(&key) - .with_context(|| format!("Failed to load topic with key: {}", key)) - { - Ok(data) => { - if let Some(topic_data) = data { - let topic_data = rmp_serde::from_slice::(&topic_data) - .with_context(|| format!("Failed to deserialize topic with key: {}", key)); - if let Err(err) = topic_data { - return Err(IggyError::CannotDeserializeResource(err)); - } else { - topic_data.unwrap() - } - } else { - return Err(IggyError::ResourceNotFound(key)); - } - } - Err(err) => { - return Err(IggyError::CannotLoadResource(err)); - } - }; - - topic.name = topic_data.name; - topic.created_at = topic_data.created_at; - topic.message_expiry = match topic_data.message_expiry { - Some(expiry) => { - IggyExpiry::ExpireDuration(IggyDuration::from(expiry as u64 * SEC_IN_MICRO)) - } - None => IggyExpiry::NeverExpire, - }; - topic.compression_algorithm = topic_data.compression_algorithm; - topic.max_topic_size = topic_data.max_topic_size.into(); - topic.replication_factor = topic_data.replication_factor; - - let dir_entries = fs::read_dir(&topic.partitions_path).await - .with_context(|| format!("Failed to read partition with ID: {} for stream with ID: {} for topic with ID: {} and path: {}", - topic.topic_id, topic.stream_id, topic.topic_id, &topic.partitions_path)); - if let Err(err) = dir_entries { - return Err(IggyError::CannotReadPartitions(err)); - } - - let mut dir_entries = dir_entries.unwrap(); - while let Some(dir_entry) = dir_entries.next_entry().await.unwrap_or(None) { - let metadata = dir_entry.metadata().await; - if metadata.is_err() || metadata.unwrap().is_file() { - continue; - } - - let name = dir_entry.file_name().into_string().unwrap(); - let partition_id = name.parse::(); - if partition_id.is_err() { - error!("Invalid partition ID file with name: '{}'.", name); - continue; - } - - let partition_id = partition_id.unwrap(); - let mut partition = Partition::create( - topic.stream_id, - topic.topic_id, - partition_id, - false, - topic.config.clone(), - topic.storage.clone(), - topic.message_expiry, - topic.messages_count_of_parent_stream.clone(), - topic.messages_count.clone(), - topic.size_of_parent_stream.clone(), - topic.size_bytes.clone(), - topic.segments_count_of_parent_stream.clone(), - IggyTimestamp::zero(), - ); - partitions::load(config, db, &mut partition) - .await - .with_error(|_| { - format!( - "{COMPONENT} - failed to load partition, stream ID: {}, topic ID: {}, partition ID: {}", - partition.stream_id, - partition.topic_id, - partition.partition_id, - ) - })?; - topic - .partitions - .insert(partition.partition_id, IggySharedMut::new(partition)); - } - - let consumer_groups = load_consumer_groups(db, topic).await.with_error(|_| { - format!( - "{COMPONENT} - failed to load consumer groups, stream ID: {}, topic_id: {}", - topic.stream_id, topic.topic_id, - ) - })?; - topic.consumer_groups = consumer_groups - .into_iter() - .map(|group| (group.group_id, RwLock::new(group))) - .collect(); - info!("Loaded topic {topic}"); - Ok(()) -} - -pub async fn load_consumer_groups(db: &Db, topic: &Topic) -> Result, IggyError> { - info!("Loading consumer groups for topic {} from disk...", topic); - let key_prefix = get_consumer_groups_key_prefix(topic.stream_id, topic.topic_id); - let mut consumer_groups = Vec::new(); - for data in db.scan_prefix(format!("{}:", key_prefix)) { - let consumer_group = match data.with_context(|| { - format!( - "Failed to load consumer group when searching for key: {}", - key_prefix - ) - }) { - Ok((_, value)) => { - match rmp_serde::from_slice::(&value).with_context(|| { - format!( - "Failed to deserialize consumer group with key: {}", - key_prefix - ) - }) { - Ok(user) => user, - Err(err) => { - return Err(IggyError::CannotDeserializeResource(err)); - } - } - } - Err(err) => { - return Err(IggyError::CannotLoadResource(err)); - } - }; - let consumer_group = ConsumerGroup::new( - topic.topic_id, - consumer_group.id, - &consumer_group.name, - topic.get_partitions_count(), - ); - consumer_groups.push(consumer_group); - } - info!( - "Loaded {} consumer groups for topic {}", - consumer_groups.len(), - topic - ); - Ok(consumer_groups) -} - -#[derive(Debug, Serialize, Deserialize)] -struct TopicData { - name: String, - created_at: IggyTimestamp, - message_expiry: Option, - compression_algorithm: CompressionAlgorithm, - max_topic_size: Option, - replication_factor: u8, -} - -fn get_topic_key(stream_id: u32, topic_id: u32) -> String { - format!("streams:{}:topics:{}", stream_id, topic_id) -} - -fn get_consumer_groups_key_prefix(stream_id: u32, topic_id: u32) -> String { - format!("streams:{stream_id}:topics:{topic_id}:consumer_groups") -} diff --git a/server/src/compat/storage_conversion/persistency/users.rs b/server/src/compat/storage_conversion/persistency/users.rs deleted file mode 100644 index a561e12e0..000000000 --- a/server/src/compat/storage_conversion/persistency/users.rs +++ /dev/null @@ -1,57 +0,0 @@ -use crate::streaming::users::user::User; -use anyhow::Context; -use iggy::error::IggyError; -use iggy::models::permissions::Permissions; -use iggy::models::user_info::UserId; -use iggy::models::user_status::UserStatus; -use iggy::utils::timestamp::IggyTimestamp; -use serde::{Deserialize, Serialize}; -use sled::Db; - -const KEY_PREFIX: &str = "users"; - -pub async fn load_all(db: &Db) -> Result, IggyError> { - let mut users = Vec::new(); - for data in db.scan_prefix(format!("{}:", KEY_PREFIX)) { - let user_data = match data.with_context(|| { - format!( - "Failed to load user, when searching for key: {}", - KEY_PREFIX - ) - }) { - Ok((_, value)) => match rmp_serde::from_slice::(&value).with_context(|| { - format!( - "Failed to deserialize user, when searching for key: {}", - KEY_PREFIX - ) - }) { - Ok(user) => user, - Err(err) => { - return Err(IggyError::CannotDeserializeResource(err)); - } - }, - Err(err) => { - return Err(IggyError::CannotLoadResource(err)); - } - }; - let mut user = User::empty(user_data.id); - user.status = user_data.status; - user.username = user_data.username; - user.password = user_data.password; - user.created_at = user_data.created_at; - user.permissions = user_data.permissions; - users.push(user); - } - - Ok(users) -} - -#[derive(Debug, Serialize, Deserialize)] -struct UserData { - pub id: UserId, - pub status: UserStatus, - pub username: String, - pub password: String, - pub created_at: IggyTimestamp, - pub permissions: Option, -} diff --git a/server/src/configs/config_provider.rs b/server/src/configs/config_provider.rs index 4b41c8f57..5b2c713cb 100644 --- a/server/src/configs/config_provider.rs +++ b/server/src/configs/config_provider.rs @@ -122,7 +122,6 @@ impl CustomEnvProvider { combined_keys.clear(); return; } - _ => { continue; } @@ -152,6 +151,11 @@ impl CustomEnvProvider { } fn try_parse_value(value: &str) -> FigmentValue { + if value.starts_with('[') && value.ends_with(']') { + let value = value.trim_start_matches('[').trim_end_matches(']'); + let values: Vec = value.split(',').map(Self::try_parse_value).collect(); + return FigmentValue::from(values); + } if value == "true" { return FigmentValue::from(true); } diff --git a/server/src/configs/defaults.rs b/server/src/configs/defaults.rs index 46e72538e..d643cde19 100644 --- a/server/src/configs/defaults.rs +++ b/server/src/configs/defaults.rs @@ -294,7 +294,6 @@ impl Default for SystemConfig { SystemConfig { path: SERVER_CONFIG.system.path.parse().unwrap(), backup: BackupConfig::default(), - database: None, runtime: RuntimeConfig::default(), logging: LoggingConfig::default(), cache: CacheConfig::default(), diff --git a/server/src/configs/system.rs b/server/src/configs/system.rs index 3ebcf3f3c..a1025138e 100644 --- a/server/src/configs/system.rs +++ b/server/src/configs/system.rs @@ -13,7 +13,6 @@ use serde_with::DisplayFromStr; pub struct SystemConfig { pub path: String, pub backup: BackupConfig, - pub database: Option, pub state: StateConfig, pub runtime: RuntimeConfig, pub logging: LoggingConfig, @@ -135,12 +134,6 @@ impl SystemConfig { self.path.to_string() } - pub fn get_database_path(&self) -> Option { - self.database - .as_ref() - .map(|database| format!("{}/{}", self.get_system_path(), database.path)) - } - pub fn get_state_path(&self) -> String { format!("{}/state", self.get_system_path()) } diff --git a/server/src/http/error.rs b/server/src/http/error.rs index cd1eed50d..86a94be0e 100644 --- a/server/src/http/error.rs +++ b/server/src/http/error.rs @@ -36,13 +36,7 @@ impl IntoResponse for CustomError { IggyError::ConsumerGroupIdNotFound(_, _) => StatusCode::NOT_FOUND, IggyError::ConsumerGroupNameNotFound(_, _) => StatusCode::NOT_FOUND, IggyError::ConsumerGroupMemberNotFound(_, _, _) => StatusCode::NOT_FOUND, - IggyError::CannotLoadResource(_) => StatusCode::NOT_FOUND, IggyError::ResourceNotFound(_) => StatusCode::NOT_FOUND, - IggyError::IoError(_) => StatusCode::INTERNAL_SERVER_ERROR, - IggyError::WriteError(_) => StatusCode::INTERNAL_SERVER_ERROR, - IggyError::CannotParseInt(_) => StatusCode::INTERNAL_SERVER_ERROR, - IggyError::CannotParseSlice(_) => StatusCode::INTERNAL_SERVER_ERROR, - IggyError::CannotParseUtf8(_) => StatusCode::INTERNAL_SERVER_ERROR, IggyError::Unauthenticated => StatusCode::UNAUTHORIZED, IggyError::Unauthorized => StatusCode::FORBIDDEN, _ => StatusCode::BAD_REQUEST, diff --git a/server/src/http/jwt/storage.rs b/server/src/http/jwt/storage.rs index 5249f3589..dccf6b047 100644 --- a/server/src/http/jwt/storage.rs +++ b/server/src/http/jwt/storage.rs @@ -9,7 +9,7 @@ use iggy::error::IggyError; use std::collections::HashMap; use std::sync::Arc; use tokio::io::AsyncReadExt; -use tracing::info; +use tracing::{error, info}; #[derive(Debug)] pub struct TokenStorage { @@ -35,7 +35,10 @@ impl TokenStorage { } info!("Loading revoked access tokens from: {}", self.path); - let mut file = file.unwrap(); + let mut file = file.map_err(|error| { + error!("Cannot open revoked access tokens file: {error}"); + IggyError::CannotReadFile + })?; let file_size = file .metadata() .await @@ -44,20 +47,24 @@ impl TokenStorage { "{COMPONENT} - failed to read file metadata, path: {}", self.path ) - })? + }) + .map_err(|_| IggyError::CannotReadFileMetadata)? .len() as usize; let mut buffer = BytesMut::with_capacity(file_size); buffer.put_bytes(0, file_size); - file.read_exact(&mut buffer).await.with_error(|_| { - format!( - "{COMPONENT} - failed to read file into buffer, path: {}", - self.path - ) - })?; + file.read_exact(&mut buffer) + .await + .with_error(|_| { + format!( + "{COMPONENT} - failed to read file into buffer, path: {}", + self.path + ) + }) + .map_err(|_| IggyError::CannotReadFile)?; let tokens: HashMap = bincode::deserialize(&buffer) .with_context(|| "Failed to deserialize revoked access tokens") - .map_err(IggyError::CannotDeserializeResource)?; + .map_err(|_| IggyError::CannotDeserializeResource)?; let tokens = tokens .into_iter() @@ -80,7 +87,7 @@ impl TokenStorage { map.insert(token.id.to_owned(), token.expiry); let bytes = bincode::serialize(&map) .with_context(|| "Failed to serialize revoked access tokens") - .map_err(IggyError::CannotSerializeResource)?; + .map_err(|_| IggyError::CannotSerializeResource)?; self.persister .overwrite(&self.path, &bytes) .await @@ -112,7 +119,7 @@ impl TokenStorage { let bytes = bincode::serialize(&map) .with_context(|| "Failed to serialize revoked access tokens") - .map_err(IggyError::CannotSerializeResource)?; + .map_err(|_| IggyError::CannotSerializeResource)?; self.persister .overwrite(&self.path, &bytes) .await diff --git a/server/src/log/logger.rs b/server/src/log/logger.rs index 12ff477ab..a2599a221 100644 --- a/server/src/log/logger.rs +++ b/server/src/log/logger.rs @@ -1,11 +1,10 @@ use crate::configs::server::{TelemetryConfig, TelemetryTransport}; use crate::configs::system::LoggingConfig; use crate::server_error::LogError; -use opentelemetry::logs::LoggerProvider as _; -use opentelemetry::trace::TracerProvider as _; +use opentelemetry::trace::TracerProvider; use opentelemetry::{global, KeyValue}; use opentelemetry_appender_tracing::layer::OpenTelemetryTracingBridge; -use opentelemetry_otlp::WithExportConfig; +use opentelemetry_otlp::{WithExportConfig, WithHttpConfig}; use opentelemetry_sdk::propagation::TraceContextPropagator; use opentelemetry_sdk::runtime::Tokio; use opentelemetry_sdk::Resource; @@ -175,75 +174,67 @@ impl Logging { ]); let logger_provider = match self.telemetry_config.logs.transport { - TelemetryTransport::GRPC => opentelemetry_otlp::new_pipeline() - .logging() + TelemetryTransport::GRPC => opentelemetry_sdk::logs::LoggerProvider::builder() .with_resource(resource.clone()) - .with_exporter( - opentelemetry_otlp::new_exporter() - .tonic() - .with_endpoint(self.telemetry_config.logs.endpoint.clone()), + .with_batch_exporter( + opentelemetry_otlp::LogExporter::builder() + .with_tonic() + .with_endpoint(self.telemetry_config.logs.endpoint.clone()) + .build() + .expect("Failed to initialize gRPC logger."), + Tokio, ) - .install_batch(Tokio) - .expect("Failed to initialize gRPC logger."), - TelemetryTransport::HTTP => opentelemetry_otlp::new_pipeline() - .logging() + .build(), + TelemetryTransport::HTTP => opentelemetry_sdk::logs::LoggerProvider::builder() .with_resource(resource.clone()) - .with_exporter( - opentelemetry_otlp::new_exporter() - .http() + .with_batch_exporter( + opentelemetry_otlp::LogExporter::builder() + .with_http() .with_http_client(reqwest::Client::new()) + .with_endpoint(self.telemetry_config.logs.endpoint.clone()) .with_protocol(opentelemetry_otlp::Protocol::HttpBinary) - .with_endpoint(self.telemetry_config.logs.endpoint.clone()), + .build() + .expect("Failed to initialize HTTP logger."), + Tokio, ) - .install_batch(Tokio) - .expect("Failed to initialize HTTP logger."), + .build(), }; - let logger = logger_provider - .logger_builder(service_name.to_owned()) - .with_version(VERSION) - .build(); - let tracer_provider = match self.telemetry_config.traces.transport { - TelemetryTransport::GRPC => opentelemetry_otlp::new_pipeline() - .tracing() - .with_exporter( - opentelemetry_otlp::new_exporter() - .tonic() - .with_endpoint(self.telemetry_config.traces.endpoint.clone()), - ) - .with_trace_config( - opentelemetry_sdk::trace::Config::default().with_resource(resource), + TelemetryTransport::GRPC => opentelemetry_sdk::trace::TracerProvider::builder() + .with_resource(resource.clone()) + .with_batch_exporter( + opentelemetry_otlp::SpanExporter::builder() + .with_tonic() + .with_endpoint(self.telemetry_config.traces.endpoint.clone()) + .build() + .expect("Failed to initialize gRPC tracer."), + Tokio, ) - .install_batch(Tokio) - .expect("Failed to initialize gRPC tracer."), - TelemetryTransport::HTTP => opentelemetry_otlp::new_pipeline() - .tracing() - .with_exporter( - opentelemetry_otlp::new_exporter() - .http() + .build(), + TelemetryTransport::HTTP => opentelemetry_sdk::trace::TracerProvider::builder() + .with_resource(resource.clone()) + .with_batch_exporter( + opentelemetry_otlp::SpanExporter::builder() + .with_http() .with_http_client(reqwest::Client::new()) + .with_endpoint(self.telemetry_config.traces.endpoint.clone()) .with_protocol(opentelemetry_otlp::Protocol::HttpBinary) - .with_endpoint(self.telemetry_config.traces.endpoint.clone()), - ) - .with_trace_config( - opentelemetry_sdk::trace::Config::default().with_resource(resource), + .build() + .expect("Failed to initialize HTTP tracer."), + Tokio, ) - .install_batch(Tokio) - .expect("Failed to initialize HTTP tracer."), + .build(), }; - let tracer = tracer_provider - .tracer_builder(service_name.to_owned()) - .with_version(VERSION) - .build(); + let tracer = tracer_provider.tracer(service_name); global::set_tracer_provider(tracer_provider.clone()); global::set_text_map_propagator(TraceContextPropagator::new()); global::shutdown_tracer_provider(); Registry::default() .with(layers) - .with(OpenTelemetryTracingBridge::new(logger.provider())) + .with(OpenTelemetryTracingBridge::new(&logger_provider)) .with(OpenTelemetryLayer::new(tracer)) .with(EnvFilter::try_from_default_env().unwrap_or(EnvFilter::new("INFO"))) .init(); diff --git a/server/src/quic/quic_sender.rs b/server/src/quic/quic_sender.rs index a325995c4..cf77524a5 100644 --- a/server/src/quic/quic_sender.rs +++ b/server/src/quic/quic_sender.rs @@ -1,12 +1,10 @@ use crate::binary::sender::Sender; use crate::quic::COMPONENT; use async_trait::async_trait; -use bytes::{BufMut, BytesMut}; use error_set::ResultContext; use iggy::error::IggyError; use quinn::{RecvStream, SendStream}; -use std::mem::size_of; -use tracing::debug; +use tracing::{debug, error}; const STATUS_OK: &[u8] = &[0; 4]; @@ -22,12 +20,12 @@ unsafe impl Sync for QuicSender {} #[async_trait] impl Sender for QuicSender { async fn read(&mut self, buffer: &mut [u8]) -> Result { - let read_bytes = self.recv.read(buffer).await; - if let Err(error) = read_bytes { - return Err(IggyError::from(error)); - } + let read_bytes = self.recv.read(buffer).await.map_err(|error| { + error!("Failed to read from the stream: {:?}", error); + IggyError::QuicError + })?; - Ok(read_bytes.unwrap().unwrap()) + Ok(read_bytes.ok_or(IggyError::QuicError)?) } async fn send_empty_ok_response(&mut self) -> Result<(), IggyError> { @@ -39,15 +37,7 @@ impl Sender for QuicSender { } async fn send_error_response(&mut self, error: IggyError) -> Result<(), IggyError> { - let error_message = error.to_string(); - let length = error_message.len() as u32; - - let mut error_details_buffer = - BytesMut::with_capacity(error_message.len() + size_of::()); - error_details_buffer.put_u32_le(length); - error_details_buffer.put_slice(error_message.as_bytes()); - - self.send_response(&error.as_code().to_le_bytes(), &error_details_buffer) + self.send_response(&error.as_code().to_le_bytes(), &[]) .await } } @@ -59,10 +49,12 @@ impl QuicSender { self.send .write_all(&[status, &length, payload].as_slice().concat()) .await - .with_error(|_| format!("{COMPONENT} - failed to write buffer to the stream"))?; + .with_error(|_| format!("{COMPONENT} - failed to write buffer to the stream")) + .map_err(|_| IggyError::QuicError)?; self.send .finish() - .with_error(|_| format!("{COMPONENT} - failed to finish send stream"))?; + .with_error(|_| format!("{COMPONENT} - failed to finish send stream")) + .map_err(|_| IggyError::QuicError)?; debug!("Sent response with status: {:?}", status); Ok(()) } diff --git a/server/src/state/file.rs b/server/src/state/file.rs index 20055f5bd..5d118d9a7 100644 --- a/server/src/state/file.rs +++ b/server/src/state/file.rs @@ -11,13 +11,12 @@ use iggy::error::IggyError; use iggy::utils::byte_size::IggyByteSize; use iggy::utils::crypto::Encryptor; use iggy::utils::timestamp::IggyTimestamp; -use log::debug; use std::fmt::Debug; use std::path::Path; use std::sync::atomic::{AtomicU32, AtomicU64, Ordering}; use std::sync::Arc; use tokio::io::{AsyncReadExt, BufReader}; -use tracing::{error, info}; +use tracing::{debug, error, info}; const BUF_READER_CAPACITY_BYTES: usize = 512 * 1000; const FILE_STATE_PARSE_ERROR: &str = "STATE - failed to parse file state"; @@ -103,12 +102,15 @@ impl State for FileState { return Err(IggyError::StateFileNotFound); } - let file = file::open(&self.path).await.with_error(|_| { - format!( - "{COMPONENT} - failed to open state file, path: {}", - self.path - ) - })?; + let file = file::open(&self.path) + .await + .with_error(|_| { + format!( + "{COMPONENT} - failed to open state file, path: {}", + self.path + ) + }) + .map_err(|_| IggyError::CannotReadFile)?; let file_size = file .metadata() .await @@ -117,7 +119,8 @@ impl State for FileState { "{COMPONENT} - failed to load state file metadata, path: {}", self.path ) - })? + }) + .map_err(|_| IggyError::CannotReadFileMetadata)? .len(); if file_size == 0 { info!("State file is empty"); @@ -137,7 +140,8 @@ impl State for FileState { let index = reader .read_u64_le() .await - .with_error(|_| format!("{FILE_STATE_PARSE_ERROR} index"))?; + .with_error(|_| format!("{FILE_STATE_PARSE_ERROR} index")) + .map_err(|_| IggyError::InvalidNumberEncoding)?; total_size += 8; if entries_count > 0 && index != current_index + 1 { error!( @@ -153,65 +157,81 @@ impl State for FileState { let term = reader .read_u64_le() .await - .with_error(|_| format!("{FILE_STATE_PARSE_ERROR} term"))?; + .with_error(|_| format!("{FILE_STATE_PARSE_ERROR} term")) + .map_err(|_| IggyError::InvalidNumberEncoding)?; total_size += 8; let leader_id = reader .read_u32_le() .await - .with_error(|_| format!("{FILE_STATE_PARSE_ERROR} leader_id"))?; + .with_error(|_| format!("{FILE_STATE_PARSE_ERROR} leader_id")) + .map_err(|_| IggyError::InvalidNumberEncoding)?; total_size += 4; let version = reader .read_u32_le() .await - .with_error(|_| format!("{FILE_STATE_PARSE_ERROR} version"))?; + .with_error(|_| format!("{FILE_STATE_PARSE_ERROR} version")) + .map_err(|_| IggyError::InvalidNumberEncoding)?; total_size += 4; let flags = reader .read_u64_le() .await - .with_error(|_| format!("{FILE_STATE_PARSE_ERROR} flags"))?; + .with_error(|_| format!("{FILE_STATE_PARSE_ERROR} flags")) + .map_err(|_| IggyError::InvalidNumberEncoding)?; total_size += 8; let timestamp = IggyTimestamp::from( reader .read_u64_le() .await - .with_error(|_| format!("{FILE_STATE_PARSE_ERROR} timestamp"))?, + .with_error(|_| format!("{FILE_STATE_PARSE_ERROR} timestamp")) + .map_err(|_| IggyError::InvalidNumberEncoding)?, ); total_size += 8; let user_id = reader .read_u32_le() .await - .with_error(|_| format!("{FILE_STATE_PARSE_ERROR} user_id"))?; + .with_error(|_| format!("{FILE_STATE_PARSE_ERROR} user_id")) + .map_err(|_| IggyError::InvalidNumberEncoding)?; total_size += 4; let checksum = reader .read_u32_le() .await - .with_error(|_| format!("{FILE_STATE_PARSE_ERROR} checksum"))?; + .with_error(|_| format!("{FILE_STATE_PARSE_ERROR} checksum")) + .map_err(|_| IggyError::InvalidNumberEncoding)?; total_size += 4; let context_length = reader .read_u32_le() .await - .with_error(|_| format!("{FILE_STATE_PARSE_ERROR} context context_length"))? + .with_error(|_| format!("{FILE_STATE_PARSE_ERROR} context context_length")) + .map_err(|_| IggyError::InvalidNumberEncoding)? as usize; total_size += 4; let mut context = BytesMut::with_capacity(context_length); context.put_bytes(0, context_length); - reader.read_exact(&mut context).await?; + reader + .read_exact(&mut context) + .await + .map_err(|_| IggyError::CannotReadFile)?; let context = context.freeze(); total_size += context_length as u64; let code = reader .read_u32_le() .await - .with_error(|_| format!("{FILE_STATE_PARSE_ERROR} code"))?; + .with_error(|_| format!("{FILE_STATE_PARSE_ERROR} code")) + .map_err(|_| IggyError::InvalidNumberEncoding)?; total_size += 4; let mut command_length = reader .read_u32_le() .await - .with_error(|_| format!("{FILE_STATE_PARSE_ERROR} command_length"))? + .with_error(|_| format!("{FILE_STATE_PARSE_ERROR} command_length")) + .map_err(|_| IggyError::InvalidNumberEncoding)? as usize; total_size += 4; let mut command = BytesMut::with_capacity(command_length); command.put_bytes(0, command_length); - reader.read_exact(&mut command).await?; + reader + .read_exact(&mut command) + .await + .map_err(|_| IggyError::CannotReadFile)?; total_size += command_length as u64; let command_payload; if let Some(encryptor) = &self.encryptor { diff --git a/server/src/state/models.rs b/server/src/state/models.rs index b544e0d9a..35155029f 100644 --- a/server/src/state/models.rs +++ b/server/src/state/models.rs @@ -45,22 +45,32 @@ impl BytesSerializable for CreatePersonalAccessTokenWithHash { Self: Sized, { let mut position = 0; - let command_length = - u32::from_le_bytes(bytes[position..position + 4].try_into().with_error(|_| { - format!("{COMPONENT} - failed to parse personal access token command length") - })?); + let command_length = u32::from_le_bytes( + bytes[position..position + 4] + .try_into() + .with_error(|_| { + format!("{COMPONENT} - failed to parse personal access token command length") + }) + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); position += 4; let command_bytes = bytes.slice(position..position + command_length as usize); position += command_length as usize; let command = CreatePersonalAccessToken::from_bytes(command_bytes).with_error(|_| { format!("{COMPONENT} - failed to parse personal access token command") })?; - let hash_length = - u32::from_le_bytes(bytes[position..position + 4].try_into().with_error(|_| { - format!("{COMPONENT} - failed to parse personal access token hash length") - })?); + let hash_length = u32::from_le_bytes( + bytes[position..position + 4] + .try_into() + .with_error(|_| { + format!("{COMPONENT} - failed to parse personal access token hash length") + }) + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); position += 4; - let hash = from_utf8(&bytes[position..position + hash_length as usize])?.to_string(); + let hash = from_utf8(&bytes[position..position + hash_length as usize]) + .map_err(|_| IggyError::InvalidUtf8)? + .to_string(); Ok(Self { command, hash }) } } diff --git a/server/src/streaming/models/messages.rs b/server/src/streaming/models/messages.rs index 0aa897bff..0f174c230 100644 --- a/server/src/streaming/models/messages.rs +++ b/server/src/streaming/models/messages.rs @@ -1,3 +1,4 @@ +use crate::streaming::local_sizeable::LocalSizeable; use crate::streaming::models::COMPONENT; use bytes::{BufMut, Bytes, BytesMut}; use error_set::ResultContext; @@ -13,8 +14,6 @@ use std::collections::HashMap; use std::ops::Deref; use std::sync::Arc; -use crate::streaming::local_sizeable::LocalSizeable; - // It's the same as PolledMessages from Iggy models, but with the Arc instead of Message. #[derive(Debug, Serialize, Deserialize)] pub struct PolledMessages { @@ -94,29 +93,34 @@ impl RetainedMessage { let offset = u64::from_le_bytes( bytes[..8] .try_into() - .with_error(|_| format!("{COMPONENT} - failed to parse message offset"))?, + .with_error(|_| format!("{COMPONENT} - failed to parse message offset")) + .map_err(|_| IggyError::InvalidNumberEncoding)?, ); let message_state = MessageState::from_code(bytes[8]) .with_error(|_| format!("{COMPONENT} - failed to parse message state"))?; let timestamp = u64::from_le_bytes( bytes[9..17] .try_into() - .with_error(|_| format!("{COMPONENT} - failed to parse message timestamp"))?, + .with_error(|_| format!("{COMPONENT} - failed to parse message timestamp")) + .map_err(|_| IggyError::InvalidNumberEncoding)?, ); let id = u128::from_le_bytes( bytes[17..33] .try_into() - .with_error(|_| format!("{COMPONENT} - failed to parse message id"))?, + .with_error(|_| format!("{COMPONENT} - failed to parse message id")) + .map_err(|_| IggyError::InvalidNumberEncoding)?, ); let checksum = u32::from_le_bytes( bytes[33..37] .try_into() - .with_error(|_| format!("{COMPONENT} - failed to parse message checksum"))?, + .with_error(|_| format!("{COMPONENT} - failed to parse message checksum")) + .map_err(|_| IggyError::InvalidNumberEncoding)?, ); let headers_length = u32::from_le_bytes( bytes[37..41] .try_into() - .with_error(|_| format!("{COMPONENT} - failed to parse message headers_length"))?, + .with_error(|_| format!("{COMPONENT} - failed to parse message headers_length")) + .map_err(|_| IggyError::InvalidNumberEncoding)?, ); let headers = if headers_length > 0 { Some(bytes.slice(41..41 + headers_length as usize)) diff --git a/server/src/streaming/partitions/storage.rs b/server/src/streaming/partitions/storage.rs index a679a0c68..939847131 100644 --- a/server/src/streaming/partitions/storage.rs +++ b/server/src/streaming/partitions/storage.rs @@ -46,14 +46,20 @@ impl PartitionStorage for FilePartitionStorage { ); partition.created_at = state.created_at; let dir_entries = fs::read_dir(&partition.partition_path).await; - if let Err(err) = fs::read_dir(&partition.partition_path) + if fs::read_dir(&partition.partition_path) .await + .inspect_err(|err| { + error!( + "Failed to read partition with ID: {} for stream with ID: {} and topic with ID: {} and path: {}. Error: {}", + partition.partition_id, partition.stream_id, partition.topic_id, partition.partition_path, err + ); + }) .with_context(|| format!( - "{COMPONENT} - failed to read partition with ID: {} for stream with ID: {} and topic with ID: {} and path: {}", + "{COMPONENT} - failed to read partition with ID: {} for stream with ID: {} and topic with ID: {} and path: {}.", partition.partition_id, partition.stream_id, partition.topic_id, partition.partition_path, - )) + )).is_err() { - return Err(IggyError::CannotReadPartitions(err)); + return Err(IggyError::CannotReadPartitions); } let mut dir_entries = dir_entries.unwrap(); @@ -376,12 +382,17 @@ impl PartitionStorage for FilePartitionStorage { let path = path.unwrap().to_string(); let consumer_id = consumer_id.unwrap(); - let mut file = file::open(&path).await.with_error(|_| { - format!("{COMPONENT} - failed to open offset file, path: {path}") - })?; - let offset = file.read_u64_le().await.with_error(|_| { - format!("{COMPONENT} - failed to read consumer offset from file, path: {path}") - })?; + let mut file = file::open(&path) + .await + .with_error(|_| format!("{COMPONENT} - failed to open offset file, path: {path}")) + .map_err(|_| IggyError::CannotReadFile)?; + let offset = file + .read_u64_le() + .await + .with_error(|_| { + format!("{COMPONENT} - failed to read consumer offset from file, path: {path}") + }) + .map_err(|_| IggyError::CannotReadFile)?; consumer_offsets.push(ConsumerOffset { kind, diff --git a/server/src/streaming/persistence/persister.rs b/server/src/streaming/persistence/persister.rs index 65041a3b7..143b91f61 100644 --- a/server/src/streaming/persistence/persister.rs +++ b/server/src/streaming/persistence/persister.rs @@ -39,27 +39,32 @@ impl Persister for FilePersister { async fn append(&self, path: &str, bytes: &[u8]) -> Result<(), IggyError> { let mut file = file::append(path) .await - .with_error(|_| format!("{COMPONENT} - failed to append to file: {path}"))?; + .with_error(|_| format!("{COMPONENT} - failed to append to file: {path}")) + .map_err(|_| IggyError::CannotAppendToFile)?; file.write_all(bytes) .await - .with_error(|_| format!("{COMPONENT} - failed to write data to file: {path}"))?; + .with_error(|_| format!("{COMPONENT} - failed to write data to file: {path}")) + .map_err(|_| IggyError::CannotWriteToFile)?; Ok(()) } async fn overwrite(&self, path: &str, bytes: &[u8]) -> Result<(), IggyError> { let mut file = file::overwrite(path) .await - .with_error(|_| format!("{COMPONENT} - failed to overwrite file: {path}"))?; + .with_error(|_| format!("{COMPONENT} - failed to overwrite file: {path}")) + .map_err(|_| IggyError::CannotOverwriteFile)?; file.write_all(bytes) .await - .with_error(|_| format!("{COMPONENT} - failed to write data to file: {path}"))?; + .with_error(|_| format!("{COMPONENT} - failed to write data to file: {path}")) + .map_err(|_| IggyError::CannotWriteToFile)?; Ok(()) } async fn delete(&self, path: &str) -> Result<(), IggyError> { fs::remove_file(path) .await - .with_error(|_| format!("{COMPONENT} - failed to delete file: {path}"))?; + .with_error(|_| format!("{COMPONENT} - failed to delete file: {path}")) + .map_err(|_| IggyError::CannotDeleteFile)?; Ok(()) } } @@ -69,33 +74,40 @@ impl Persister for FileWithSyncPersister { async fn append(&self, path: &str, bytes: &[u8]) -> Result<(), IggyError> { let mut file = file::append(path) .await - .with_error(|_| format!("{COMPONENT} - failed to append to file: {path}"))?; + .with_error(|_| format!("{COMPONENT} - failed to append to file: {path}")) + .map_err(|_| IggyError::CannotAppendToFile)?; file.write_all(bytes) .await - .with_error(|_| format!("{COMPONENT} - failed to write data to file: {path}"))?; + .with_error(|_| format!("{COMPONENT} - failed to write data to file: {path}")) + .map_err(|_| IggyError::CannotWriteToFile)?; file.sync_all() .await - .with_error(|_| format!("{COMPONENT} - failed to sync file after appending: {path}"))?; + .with_error(|_| format!("{COMPONENT} - failed to sync file after appending: {path}")) + .map_err(|_| IggyError::CannotSyncFile)?; Ok(()) } async fn overwrite(&self, path: &str, bytes: &[u8]) -> Result<(), IggyError> { let mut file = file::overwrite(path) .await - .with_error(|_| format!("{COMPONENT} - failed to overwrite file: {path}"))?; + .with_error(|_| format!("{COMPONENT} - failed to overwrite file: {path}")) + .map_err(|_| IggyError::CannotOverwriteFile)?; file.write_all(bytes) .await - .with_error(|_| format!("{COMPONENT} - failed to write data to file: {path}"))?; - file.sync_all().await.with_error(|_| { - format!("{COMPONENT} - failed to sync file after overwriting: {path}") - })?; + .with_error(|_| format!("{COMPONENT} - failed to write data to file: {path}")) + .map_err(|_| IggyError::CannotWriteToFile)?; + file.sync_all() + .await + .with_error(|_| format!("{COMPONENT} - failed to sync file after overwriting: {path}")) + .map_err(|_| IggyError::CannotSyncFile)?; Ok(()) } async fn delete(&self, path: &str) -> Result<(), IggyError> { fs::remove_file(path) .await - .with_error(|_| format!("{COMPONENT} - failed to delete file: {path}"))?; + .with_error(|_| format!("{COMPONENT} - failed to delete file: {path}")) + .map_err(|_| IggyError::CannotDeleteFile)?; Ok(()) } } diff --git a/server/src/streaming/segments/storage.rs b/server/src/streaming/segments/storage.rs index e8f08ebbd..8660d62d3 100644 --- a/server/src/streaming/segments/storage.rs +++ b/server/src/streaming/segments/storage.rs @@ -48,12 +48,15 @@ impl SegmentStorage for FileSegmentStorage { "Loading segment from disk for start offset: {} and partition with ID: {} for topic with ID: {} and stream with ID: {} ...", segment.start_offset, segment.partition_id, segment.topic_id, segment.stream_id ); - let log_file = file::open(&segment.log_path).await.with_error(|_| { - format!( - "{COMPONENT} - failed to open log_file, path: {}", - segment.log_path - ) - })?; + let log_file = file::open(&segment.log_path) + .await + .with_error(|_| { + format!( + "{COMPONENT} - failed to open log_file, path: {}", + segment.log_path + ) + }) + .map_err(|_| IggyError::CannotReadFile)?; let file_size = log_file.metadata().await.unwrap().len() as u64; segment.size_bytes = IggyByteSize::from(file_size); segment.last_index_position = file_size as _; @@ -261,13 +264,14 @@ impl SegmentStorage for FileSegmentStorage { let mut bytes = BytesMut::with_capacity(batch_size.as_bytes_usize()); batch.extend(&mut bytes); - if let Err(err) = self + if self .persister .append(&segment.log_path, &bytes) .await .with_context(|| format!("Failed to save messages to segment: {}", segment.log_path)) + .is_err() { - return Err(IggyError::CannotSaveMessagesToSegment(err)); + return Err(IggyError::CannotSaveMessagesToSegment); } Ok(batch_size) @@ -320,12 +324,15 @@ impl SegmentStorage for FileSegmentStorage { async fn load_all_indexes(&self, segment: &Segment) -> Result, IggyError> { trace!("Loading indexes from file..."); - let file = file::open(&segment.index_path).await.with_error(|_| { - format!( - "{COMPONENT} - failed to open index file, path: {}", - segment.index_path - ) - })?; + let file = file::open(&segment.index_path) + .await + .with_error(|_| { + format!( + "{COMPONENT} - failed to open index file, path: {}", + segment.index_path + ) + }) + .map_err(|_| IggyError::CannotReadFile)?; let file_size = file .metadata() .await @@ -334,7 +341,8 @@ impl SegmentStorage for FileSegmentStorage { "{COMPONENT} - failed to load index file metadata, path: {}", segment.index_path ) - })? + }) + .map_err(|_| IggyError::CannotReadFileMetadata)? .len() as usize; if file_size == 0 { trace!("Index file is empty."); @@ -345,24 +353,36 @@ impl SegmentStorage for FileSegmentStorage { let mut indexes = Vec::with_capacity(indexes_count); let mut reader = BufReader::with_capacity(BUF_READER_CAPACITY_BYTES, file); for idx_num in 0..indexes_count { - let offset = reader.read_u32_le().await.inspect_err(|error| { - error!( - "Cannot read offset from index file for index number: {}. Error: {}", - idx_num, &error - ) - })?; - let position = reader.read_u32_le().await.inspect_err(|error| { - error!( - "Cannot read position from index file for offset: {}. Error: {}", - offset, &error - ) - })?; - let timestamp = reader.read_u64_le().await.inspect_err(|error| { - error!( - "Cannot read timestamp from index file for offset: {}. Error: {}", - offset, &error - ) - })?; + let offset = reader + .read_u32_le() + .await + .inspect_err(|error| { + error!( + "Cannot read offset from index file for index number: {}. Error: {}", + idx_num, &error + ) + }) + .map_err(|_| IggyError::CannotReadIndexOffset)?; + let position = reader + .read_u32_le() + .await + .inspect_err(|error| { + error!( + "Cannot read position from index file for offset: {}. Error: {}", + offset, &error + ) + }) + .map_err(|_| IggyError::CannotReadIndexPosition)?; + let timestamp = reader + .read_u64_le() + .await + .inspect_err(|error| { + error!( + "Cannot read timestamp from index file for offset: {}. Error: {}", + offset, &error + ) + }) + .map_err(|_| IggyError::CannotReadIndexTimestamp)?; indexes.push(Index { offset, position, @@ -404,12 +424,15 @@ impl SegmentStorage for FileSegmentStorage { return Ok(None); } - let file = file::open(&segment.index_path).await.with_error(|_| { - format!( - "{COMPONENT} - failed to open segment's index path, path: {}", - segment.index_path - ) - })?; + let file = file::open(&segment.index_path) + .await + .with_error(|_| { + format!( + "{COMPONENT} - failed to open segment's index path, path: {}", + segment.index_path + ) + }) + .map_err(|_| IggyError::CannotReadFile)?; let file_length = file .metadata() .await @@ -418,7 +441,8 @@ impl SegmentStorage for FileSegmentStorage { "{COMPONENT} - failed to load index's metadata, path: {}", segment.index_path ) - })? + }) + .map_err(|_| IggyError::CannotReadFileMetadata)? .len() as u32; if file_length == 0 { trace!("Index file is empty."); @@ -442,15 +466,18 @@ impl SegmentStorage for FileSegmentStorage { let offset = reader .read_u32_le() .await - .with_error(|_| format!("{COMPONENT} - failed to load index's offset"))?; + .with_error(|_| format!("{COMPONENT} - failed to load index's offset")) + .map_err(|_| IggyError::CannotReadIndexOffset)?; let position = reader .read_u32_le() .await - .with_error(|_| format!("{COMPONENT} - failed to load index's position"))?; + .with_error(|_| format!("{COMPONENT} - failed to load index's position")) + .map_err(|_| IggyError::CannotReadIndexPosition)?; let timestamp = reader .read_u64_le() .await - .with_error(|_| format!("{COMPONENT} - failed to load index's timestamp"))?; + .with_error(|_| format!("{COMPONENT} - failed to load index's timestamp")) + .map_err(|_| IggyError::CannotReadIndexTimestamp)?; read_bytes += INDEX_SIZE; let idx = Index { offset, @@ -487,16 +514,16 @@ impl SegmentStorage for FileSegmentStorage { bytes.put_u32_le(index.offset); bytes.put_u32_le(index.position); bytes.put_u64_le(index.timestamp); - if let Err(err) = self - .persister + self.persister .append(index_path, &bytes) .await - .with_context(|| format!("Failed to save index to segment: {}", index_path)) - { - return Err(IggyError::CannotSaveIndexToSegment(err)); - } - - Ok(()) + .map_err(|error| { + error!( + "Failed to save index to segment: {}, error: {}", + index_path, error + ); + IggyError::CannotSaveIndexToSegment + }) } async fn try_load_index_for_timestamp( @@ -505,12 +532,15 @@ impl SegmentStorage for FileSegmentStorage { timestamp: u64, ) -> Result, IggyError> { trace!("Loading time indexes from file..."); - let file = file::open(&segment.index_path).await.with_error(|_| { - format!( - "{COMPONENT} - failed to open segment's index file, path: {}", - segment.index_path - ) - })?; + let file = file::open(&segment.index_path) + .await + .with_error(|_| { + format!( + "{COMPONENT} - failed to open segment's index file, path: {}", + segment.index_path + ) + }) + .map_err(|_| IggyError::CannotReadFile)?; let file_size = file .metadata() .await @@ -519,7 +549,8 @@ impl SegmentStorage for FileSegmentStorage { "{COMPONENT} - failed to load index's metadata, path: {}", segment.index_path ) - })? + }) + .map_err(|_| IggyError::CannotReadFileMetadata)? .len() as usize; if file_size == 0 { trace!("Time index file is empty."); @@ -530,9 +561,18 @@ impl SegmentStorage for FileSegmentStorage { let mut read_bytes = 0; let mut idx_pred = HeadTailBuffer::new(); loop { - let offset = reader.read_u32_le().await?; - let position = reader.read_u32_le().await?; - let time = reader.read_u64_le().await?; + let offset = reader + .read_u32_le() + .await + .map_err(|_| IggyError::CannotReadIndexOffset)?; + let position = reader + .read_u32_le() + .await + .map_err(|_| IggyError::CannotReadIndexPosition)?; + let time = reader + .read_u64_le() + .await + .map_err(|_| IggyError::CannotReadIndexTimestamp)?; let idx = Index { offset, position, @@ -555,8 +595,24 @@ async fn load_batches_by_range( index_range: &IndexRange, mut on_batch: impl FnMut(RetainedMessageBatch) -> Result<(), IggyError>, ) -> Result<(), IggyError> { - let file = file::open(&segment.log_path).await?; - let file_size = file.metadata().await?.len(); + let file = file::open(&segment.log_path).await.map_err(|error| { + error!( + "Cannot open segment's log file: {}, error: {}", + segment.log_path, error + ); + IggyError::CannotReadFile + })?; + let file_size = file + .metadata() + .await + .map_err(|error| { + error!( + "Cannot read segment's log file metadata: {}, error: {}", + segment.log_path, error + ); + IggyError::CannotReadFileMetadata + })? + .len(); if file_size == 0 { return Ok(()); } @@ -578,7 +634,8 @@ async fn load_batches_by_range( "{COMPONENT} - failed to seek to position {}", index_range.start.position ) - })?; + }) + .map_err(|_| IggyError::CannotSeekFile)?; let mut read_bytes = index_range.start.position as u64; let mut last_batch_to_read = false; @@ -635,8 +692,24 @@ async fn load_messages_by_size( size_bytes: u64, mut on_batch: impl FnMut(RetainedMessageBatch) -> Result<(), IggyError>, ) -> Result<(), IggyError> { - let file = file::open(&segment.log_path).await?; - let file_size = file.metadata().await?.len(); + let file = file::open(&segment.log_path).await.map_err(|error| { + error!( + "Cannot open segment's log file: {}, error: {}", + segment.log_path, error + ); + IggyError::CannotReadFile + })?; + let file_size = file + .metadata() + .await + .map_err(|error| { + error!( + "Cannot read segment's log file metadata: {}, error: {}", + segment.log_path, error + ); + IggyError::CannotReadFileMetadata + })? + .len(); if file_size == 0 { return Ok(()); } diff --git a/server/src/streaming/streams/topics.rs b/server/src/streaming/streams/topics.rs index 283d2dc52..d447a8497 100644 --- a/server/src/streaming/streams/topics.rs +++ b/server/src/streaming/streams/topics.rs @@ -203,7 +203,7 @@ impl Stream { let topic = self .topics .remove(&id) - .ok_or_else(|| IggyError::TopicIdNotFound(id, self.stream_id))?; + .ok_or(IggyError::TopicIdNotFound(id, self.stream_id))?; self.topics_ids .remove(&topic.name) @@ -219,7 +219,7 @@ impl Stream { self.topics .remove(&topic_id) - .ok_or_else(|| IggyError::TopicIdNotFound(topic_id, self.stream_id)) + .ok_or(IggyError::TopicIdNotFound(topic_id, self.stream_id)) } pub async fn delete_topic(&mut self, id: &Identifier) -> Result { diff --git a/server/src/streaming/systems/storage.rs b/server/src/streaming/systems/storage.rs index 21435d1f4..32982ebf7 100644 --- a/server/src/streaming/systems/storage.rs +++ b/server/src/streaming/systems/storage.rs @@ -43,25 +43,29 @@ impl SystemInfoStorage for FileSystemInfoStorage { "{COMPONENT} - failed to retrieve metadata for file at path: {}", self.path ) - })? + }) + .map_err(|_| IggyError::CannotReadFileMetadata)? .len() as usize; let mut buffer = BytesMut::with_capacity(file_size); buffer.put_bytes(0, file_size); - file.read_exact(&mut buffer).await.with_error(|_| { - format!( - "{COMPONENT} - failed to read file content from path: {}", - self.path - ) - })?; + file.read_exact(&mut buffer) + .await + .with_error(|_| { + format!( + "{COMPONENT} - failed to read file content from path: {}", + self.path + ) + }) + .map_err(|_| IggyError::CannotReadFile)?; bincode::deserialize(&buffer) .with_context(|| "Failed to deserialize system info") - .map_err(IggyError::CannotDeserializeResource) + .map_err(|_| IggyError::CannotDeserializeResource) } async fn save(&self, system_info: &SystemInfo) -> Result<(), IggyError> { let data = bincode::serialize(&system_info) .with_context(|| "Failed to serialize system info") - .map_err(IggyError::CannotSerializeResource)?; + .map_err(|_| IggyError::CannotSerializeResource)?; self.persister .overwrite(&self.path, &data) .await diff --git a/server/src/streaming/systems/streams.rs b/server/src/streaming/systems/streams.rs index a9c0f8975..8a39de8c1 100644 --- a/server/src/streaming/systems/streams.rs +++ b/server/src/streaming/systems/streams.rs @@ -25,22 +25,19 @@ impl System { ) -> Result<(), IggyError> { info!("Loading streams from disk..."); let mut unloaded_streams = Vec::new(); - let dir_entries = read_dir(&self.config.get_streams_path()).await; - if let Err(error) = dir_entries { - error!("Cannot read streams directory: {}", error); - return Err(IggyError::CannotReadStreams); - } + let mut dir_entries = read_dir(&self.config.get_streams_path()) + .await + .map_err(|error| { + error!("Cannot read streams directory: {error}"); + IggyError::CannotReadStreams + })?; - let mut dir_entries = dir_entries?; while let Some(dir_entry) = dir_entries.next_entry().await.unwrap_or(None) { let name = dir_entry.file_name().into_string().unwrap(); - let stream_id = name.parse::(); - if stream_id.is_err() { + let stream_id = name.parse::().map_err(|_| { error!("Invalid stream ID file with name: '{name}'."); - continue; - } - - let stream_id = stream_id?; + IggyError::InvalidNumberValue + })?; let stream_state = streams.iter().find(|s| s.id == stream_id); if stream_state.is_none() { error!("Stream with ID: '{stream_id}' was not found in state, but exists on disk and will be removed."); diff --git a/server/src/streaming/systems/system.rs b/server/src/streaming/systems/system.rs index 73ae283dc..b0e9d3e46 100644 --- a/server/src/streaming/systems/system.rs +++ b/server/src/streaming/systems/system.rs @@ -23,12 +23,12 @@ use tracing::{error, info, instrument, trace}; use crate::archiver::disk::DiskArchiver; use crate::archiver::s3::S3Archiver; use crate::archiver::{Archiver, ArchiverKind}; +use crate::map_toggle_str; use crate::state::file::FileState; use crate::state::system::SystemState; use crate::state::State; use crate::streaming::users::user::User; use crate::versioning::SemanticVersion; -use crate::{compat, map_toggle_str}; use iggy::locking::IggySharedMut; use iggy::locking::IggySharedMutFn; use iggy::models::user_info::UserId; @@ -208,16 +208,6 @@ impl System { self.config.get_system_path() ); - if self.config.database.is_some() { - compat::storage_conversion::init( - self.config.clone(), - self.state.clone(), - self.storage.clone(), - ) - .await - .with_error(|_| format!("{COMPONENT} - failed to initialize storage conversion"))?; - } - let state_entries = self .state .init() diff --git a/server/src/streaming/topics/messages.rs b/server/src/streaming/topics/messages.rs index 68632b47c..39a9ae9d4 100644 --- a/server/src/streaming/topics/messages.rs +++ b/server/src/streaming/topics/messages.rs @@ -92,9 +92,11 @@ impl Topic { let partition_id = match partitioning.kind { PartitioningKind::Balanced => self.get_next_partition_id(), - PartitioningKind::PartitionId => { - u32::from_le_bytes(partitioning.value[..partitioning.length as usize].try_into()?) - } + PartitioningKind::PartitionId => u32::from_le_bytes( + partitioning.value[..partitioning.length as usize] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ), PartitioningKind::MessagesKey => { self.calculate_partition_id_by_messages_key_hash(&partitioning.value) } @@ -112,9 +114,11 @@ impl Topic { ) -> Result<(), IggyError> { let partition = self.partitions.get(&partition_id); partition - .ok_or_else(|| { - IggyError::PartitionNotFound(partition_id, self.stream_id, self.stream_id) - })? + .ok_or(IggyError::PartitionNotFound( + partition_id, + self.stream_id, + self.stream_id, + ))? .write() .await .flush_unsaved_buffer(fsync) @@ -128,7 +132,7 @@ impl Topic { ) -> Result<(), IggyError> { let partition = self.partitions.get(&appendable_batch_info.partition_id); partition - .ok_or_else(|| { + .ok_or({ IggyError::PartitionNotFound( appendable_batch_info.partition_id, self.stream_id, @@ -181,7 +185,8 @@ impl Topic { // TODO: load data from database instead of calculating the size on disk let total_size_on_disk_bytes = folder_size(&path) .await - .with_error(|_| format!("{COMPONENT} - failed to get folder size, path: {path}"))?; + .with_error(|_| format!("{COMPONENT} - failed to get folder size, path: {path}")) + .map_err(|_| IggyError::InvalidSizeBytes)?; for partition_lock in self.partitions.values_mut() { let mut partition = partition_lock.write().await; diff --git a/server/src/streaming/topics/storage.rs b/server/src/streaming/topics/storage.rs index e983521e7..cafde573e 100644 --- a/server/src/streaming/topics/storage.rs +++ b/server/src/streaming/topics/storage.rs @@ -48,15 +48,12 @@ impl TopicStorage for FileTopicStorage { topic.compression_algorithm = state.compression_algorithm; topic.replication_factor = state.replication_factor.unwrap_or(1); - let dir_entries = fs::read_dir(&topic.partitions_path).await + let mut dir_entries = fs::read_dir(&topic.partitions_path).await .with_context(|| format!("Failed to read partition with ID: {} for stream with ID: {} for topic with ID: {} and path: {}", - topic.topic_id, topic.stream_id, topic.topic_id, &topic.partitions_path)); - if let Err(err) = dir_entries { - return Err(IggyError::CannotReadPartitions(err)); - } + topic.topic_id, topic.stream_id, topic.topic_id, &topic.partitions_path)) + .map_err(|_| IggyError::CannotReadPartitions)?; let mut unloaded_partitions = Vec::new(); - let mut dir_entries = dir_entries.unwrap(); while let Some(dir_entry) = dir_entries.next_entry().await.unwrap_or(None) { let metadata = dir_entry.metadata().await; if metadata.is_err() || metadata.unwrap().is_file() { diff --git a/server/src/tcp/sender.rs b/server/src/tcp/sender.rs index 890a3e69d..999365478 100644 --- a/server/src/tcp/sender.rs +++ b/server/src/tcp/sender.rs @@ -1,6 +1,4 @@ -use bytes::{BufMut, BytesMut}; use iggy::error::IggyError; -use std::mem::size_of; use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; use tracing::debug; @@ -17,7 +15,7 @@ where if error.kind() == std::io::ErrorKind::UnexpectedEof { Err(IggyError::ConnectionClosed) } else { - Err(IggyError::from(error)) + Err(IggyError::TcpError) } } } @@ -44,19 +42,7 @@ pub(crate) async fn send_error_response( where T: AsyncRead + AsyncWrite + Unpin, { - let error_message = error.to_string(); - let length = error_message.len() as u32; - - let mut error_details_buffer = BytesMut::with_capacity(error_message.len() + size_of::()); - error_details_buffer.put_u32_le(length); - error_details_buffer.put_slice(error_message.as_bytes()); - - send_response( - stream, - &error.as_code().to_le_bytes(), - &error_details_buffer, - ) - .await + send_response(stream, &error.as_code().to_le_bytes(), &[]).await } pub(crate) async fn send_response( @@ -71,7 +57,8 @@ where let length = (payload.len() as u32).to_le_bytes(); stream .write_all(&[status, &length, payload].as_slice().concat()) - .await?; + .await + .map_err(|_| IggyError::TcpError)?; debug!("Sent response with status: {:?}", status); Ok(()) } diff --git a/server/src/versioning.rs b/server/src/versioning.rs index edb07f5e8..6cf83b694 100644 --- a/server/src/versioning.rs +++ b/server/src/versioning.rs @@ -15,9 +15,21 @@ impl FromStr for SemanticVersion { type Err = IggyError; fn from_str(s: &str) -> Result { let mut version = s.split('.'); - let major = version.next().unwrap().parse::()?; - let minor = version.next().unwrap().parse::()?; - let patch = version.next().unwrap().parse::()?; + let major = version + .next() + .unwrap() + .parse::() + .map_err(|_| IggyError::InvalidNumberValue)?; + let minor = version + .next() + .unwrap() + .parse::() + .map_err(|_| IggyError::InvalidNumberValue)?; + let patch = version + .next() + .unwrap() + .parse::() + .map_err(|_| IggyError::InvalidNumberValue)?; Ok(SemanticVersion { major, minor,