From 0f5eef941b2962b3a2093e782941b5d9d2f31b66 Mon Sep 17 00:00:00 2001 From: Ivan Enderlin Date: Mon, 8 Jul 2024 12:11:08 +0200 Subject: [PATCH] bench: Add the `room_list` bench. Example of the output: ``` room_list/sync/10 time: [3.5673 ms 3.5899 ms 3.6181 ms] thrpt: [2.7639 Kelem/s 2.7856 Kelem/s 2.8032 Kelem/s] Found 12 outliers among 100 measurements (12.00%) 1 (1.00%) low severe 3 (3.00%) high mild 8 (8.00%) high severe room_list/sync/100 time: [3.6342 ms 3.6667 ms 3.7037 ms] thrpt: [27.000 Kelem/s 27.273 Kelem/s 27.516 Kelem/s] Found 12 outliers among 100 measurements (12.00%) 5 (5.00%) high mild 7 (7.00%) high severe room_list/sync/1000 time: [30.426 ms 30.611 ms 30.810 ms] thrpt: [32.457 Kelem/s 32.668 Kelem/s 32.867 Kelem/s] Found 20 outliers among 100 measurements (20.00%) 13 (13.00%) high mild 7 (7.00%) high severe Benchmarking room_list/sync/10000: Warming up for 3.0000 s Warning: Unable to complete 100 samples in 5.0s. You may wish to increase target time to 35.7s, or reduce sample count to 10. room_list/sync/10000 time: [353.56 ms 355.14 ms 356.97 ms] thrpt: [28.014 Kelem/s 28.158 Kelem/s 28.284 Kelem/s] Found 5 outliers among 100 measurements (5.00%) 3 (3.00%) high mild 2 (2.00%) high severe ``` --- Cargo.lock | 3 + benchmarks/Cargo.toml | 7 + benchmarks/benches/room_list_bench.rs | 176 ++++++++++++++++++++++++++ 3 files changed, 186 insertions(+) create mode 100644 benchmarks/benches/room_list_bench.rs diff --git a/Cargo.lock b/Cargo.lock index 27318477b35..a006db8b073 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -531,17 +531,20 @@ name = "benchmarks" version = "1.0.0" dependencies = [ "criterion", + "futures-util", "matrix-sdk", "matrix-sdk-base", "matrix-sdk-crypto", "matrix-sdk-sqlite", "matrix-sdk-test", + "matrix-sdk-ui", "pprof", "ruma", "serde", "serde_json", "tempfile", "tokio", + "wiremock", ] [[package]] diff --git a/benchmarks/Cargo.toml b/benchmarks/Cargo.toml index 93b359b8422..2f7e14f4a13 100644 --- a/benchmarks/Cargo.toml +++ b/benchmarks/Cargo.toml @@ -9,16 +9,19 @@ publish = false [dependencies] criterion = { version = "0.5.1", features = ["async", "async_tokio", "html_reports"] } +futures-util = { workspace = true } matrix-sdk-base = { workspace = true } matrix-sdk-crypto = { workspace = true } matrix-sdk-sqlite = { workspace = true, features = ["crypto-store"] } matrix-sdk-test = { workspace = true } matrix-sdk = { workspace = true, features = ["native-tls", "e2e-encryption", "sqlite"] } +matrix-sdk-ui = { workspace = true } ruma = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } tempfile = "3.3.0" tokio = { version = "1.24.2", default-features = false, features = ["rt-multi-thread"] } +wiremock = { workspace = true } [target.'cfg(target_os = "linux")'.dependencies] pprof = { version = "0.13.0", features = ["flamegraph", "criterion"] } @@ -34,3 +37,7 @@ harness = false [[bench]] name = "room_bench" harness = false + +[[bench]] +name = "room_list_bench" +harness = false diff --git a/benchmarks/benches/room_list_bench.rs b/benchmarks/benches/room_list_bench.rs new file mode 100644 index 00000000000..cbce7a3fc83 --- /dev/null +++ b/benchmarks/benches/room_list_bench.rs @@ -0,0 +1,176 @@ +use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; +use futures_util::{pin_mut, StreamExt}; +use matrix_sdk::test_utils::logged_in_client_with_server; +use matrix_sdk_ui::{room_list_service::filters::new_filter_non_left, RoomListService}; +use serde::Deserialize; +use serde_json::{json, Map, Value}; +use tokio::runtime::Builder; +use wiremock::{http::Method, Match, Mock, Request, ResponseTemplate}; + +struct SlidingSyncMatcher; + +impl Match for SlidingSyncMatcher { + fn matches(&self, request: &Request) -> bool { + request.url.path() == "/_matrix/client/unstable/org.matrix.msc3575/sync" + && request.method == Method::POST + } +} + +#[derive(Deserialize)] +pub(crate) struct PartialSlidingSyncRequest { + pub txn_id: Option, +} + +fn criterion() -> Criterion { + #[cfg(target_os = "linux")] + let criterion = Criterion::default().with_profiler(pprof::criterion::PProfProfiler::new( + 100, + pprof::criterion::Output::Flamegraph(None), + )); + + #[cfg(not(target_os = "linux"))] + let criterion = Criterion::default(); + + criterion +} + +pub fn room_list(c: &mut Criterion) { + let runtime = Builder::new_multi_thread() + .enable_time() + .enable_io() + .build() + .expect("Can't create runtime"); + + let mut benchmark = c.benchmark_group("room_list"); + + for number_of_rooms in [10, 100, 1000, 10_000] { + benchmark.throughput(Throughput::Elements(number_of_rooms)); + benchmark.bench_with_input( + BenchmarkId::new("sync", number_of_rooms), + &number_of_rooms, + |benchmark, maximum_number_of_rooms| { + benchmark.to_async(&runtime).iter(|| async { + // Create a fresh new `Client` and a mocked server. + let (client, server) = logged_in_client_with_server().await; + + // Create a new `RoomListService`. + let room_list_service = RoomListService::new(client.clone()) + .await + .expect("Failed to create the `RoomListService`"); + + // Get the `RoomListService` sync stream. + let room_list_stream = room_list_service.sync(); + pin_mut!(room_list_stream); + + // Get the `RoomList` itself with a default filter. + let room_list = room_list_service + .all_rooms() + .await + .expect("Failed to fetch the `all_rooms` room list"); + let (room_list_entries, room_list_entries_controller) = room_list + .entries_with_dynamic_adapters(100, client.roominfo_update_receiver()); + pin_mut!(room_list_entries); + room_list_entries_controller.set_filter(Box::new(new_filter_non_left())); + room_list_entries.next().await; + + // “Send” (mocked) and receive rooms. + { + const ROOMS_BATCH_SIZE: u64 = 100; + let maximum_number_of_rooms = *maximum_number_of_rooms; + let mut number_of_sent_rooms = 0; + let mut room_nth = 0; + + while number_of_sent_rooms < maximum_number_of_rooms { + let number_of_rooms_to_send = u64::max( + number_of_sent_rooms + ROOMS_BATCH_SIZE, + maximum_number_of_rooms, + ) - number_of_sent_rooms; + + number_of_sent_rooms += number_of_rooms_to_send; + + let rooms_as_json = Value::Object( + (0..number_of_rooms_to_send) + .map(|_| { + let room_id = format!("!r{room_nth}:matrix.org"); + + let room = json!({ + // The recency timestamp is different, so that it triggers a re-ordering of the + // room list every time, just to stress the APIs. + "timestamp": room_nth % 10, + // One state event to activate the associated code path. + "required_state": [ + { + "content": { + "name": format!("Room #{room_nth}"), + }, + "event_id": format!("$s{room_nth}"), + "origin_server_ts": 1, + "sender": "@example:matrix.org", + "state_key": "", + "type": "m.room.name" + }, + ], + // One room event to active the associated code path. + "timeline": [ + { + "event_id": format!("$t{room_nth}"), + "sender": "@example:matrix.org", + "type": "m.room.message", + "content": { + "body": "foo", + "msgtype": "m.text", + }, + "origin_server_ts": 1, + } + ], + }); + + room_nth += 1; + + (room_id, room) + }) + .collect::>(), + ); + + // Mock the response from the server. + let _mock_guard = Mock::given(SlidingSyncMatcher) + .respond_with(move |request: &Request| { + let partial_request: PartialSlidingSyncRequest = + request.body_json().unwrap(); + + ResponseTemplate::new(200).set_body_json(json!({ + "txn_id": partial_request.txn_id, + "pos": room_nth.to_string(), + "rooms": rooms_as_json, + })) + }) + .mount_as_scoped(&server) + .await; + + // Sync the room list service. + assert!( + room_list_stream.next().await.is_some(), + "`room_list_stream` has stopped" + ); + + // Sync the room list entries. + assert!( + room_list_entries.next().await.is_some(), + "`room_list_entries` has stopped" + ); + } + } + }) + }, + ); + } + + benchmark.finish(); +} + +criterion_group! { + name = benches; + config = criterion(); + targets = room_list +} +criterion_main!(benches);