Skip to content

Commit

Permalink
bench: Add the room_list bench.
Browse files Browse the repository at this point in the history
Example of the output:

```
room_list/sync/10       time:   [3.5673 ms 3.5899 ms 3.6181 ms]
                        thrpt:  [2.7639 Kelem/s 2.7856 Kelem/s 2.8032 Kelem/s]
Found 12 outliers among 100 measurements (12.00%)
  1 (1.00%) low severe
  3 (3.00%) high mild
  8 (8.00%) high severe
room_list/sync/100      time:   [3.6342 ms 3.6667 ms 3.7037 ms]
                        thrpt:  [27.000 Kelem/s 27.273 Kelem/s 27.516 Kelem/s]
Found 12 outliers among 100 measurements (12.00%)
  5 (5.00%) high mild
  7 (7.00%) high severe
room_list/sync/1000     time:   [30.426 ms 30.611 ms 30.810 ms]
                        thrpt:  [32.457 Kelem/s 32.668 Kelem/s 32.867 Kelem/s]
Found 20 outliers among 100 measurements (20.00%)
  13 (13.00%) high mild
  7 (7.00%) high severe
Benchmarking room_list/sync/10000: Warming up for 3.0000 s
Warning: Unable to complete 100 samples in 5.0s. You may wish to increase target time to 35.7s, or reduce sample count to 10.
room_list/sync/10000    time:   [353.56 ms 355.14 ms 356.97 ms]
                        thrpt:  [28.014 Kelem/s 28.158 Kelem/s 28.284 Kelem/s]
Found 5 outliers among 100 measurements (5.00%)
  3 (3.00%) high mild
  2 (2.00%) high severe
```
  • Loading branch information
Hywan committed Jul 8, 2024
1 parent 11cbf84 commit 0f5eef9
Show file tree
Hide file tree
Showing 3 changed files with 186 additions and 0 deletions.
3 changes: 3 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

7 changes: 7 additions & 0 deletions benchmarks/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -9,16 +9,19 @@ publish = false

[dependencies]
criterion = { version = "0.5.1", features = ["async", "async_tokio", "html_reports"] }
futures-util = { workspace = true }
matrix-sdk-base = { workspace = true }
matrix-sdk-crypto = { workspace = true }
matrix-sdk-sqlite = { workspace = true, features = ["crypto-store"] }
matrix-sdk-test = { workspace = true }
matrix-sdk = { workspace = true, features = ["native-tls", "e2e-encryption", "sqlite"] }
matrix-sdk-ui = { workspace = true }
ruma = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
tempfile = "3.3.0"
tokio = { version = "1.24.2", default-features = false, features = ["rt-multi-thread"] }
wiremock = { workspace = true }

[target.'cfg(target_os = "linux")'.dependencies]
pprof = { version = "0.13.0", features = ["flamegraph", "criterion"] }
Expand All @@ -34,3 +37,7 @@ harness = false
[[bench]]
name = "room_bench"
harness = false

[[bench]]
name = "room_list_bench"
harness = false
176 changes: 176 additions & 0 deletions benchmarks/benches/room_list_bench.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,176 @@
use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput};
use futures_util::{pin_mut, StreamExt};
use matrix_sdk::test_utils::logged_in_client_with_server;
use matrix_sdk_ui::{room_list_service::filters::new_filter_non_left, RoomListService};
use serde::Deserialize;
use serde_json::{json, Map, Value};
use tokio::runtime::Builder;
use wiremock::{http::Method, Match, Mock, Request, ResponseTemplate};

struct SlidingSyncMatcher;

impl Match for SlidingSyncMatcher {
fn matches(&self, request: &Request) -> bool {
request.url.path() == "/_matrix/client/unstable/org.matrix.msc3575/sync"
&& request.method == Method::POST
}
}

#[derive(Deserialize)]
pub(crate) struct PartialSlidingSyncRequest {
pub txn_id: Option<String>,
}

fn criterion() -> Criterion {
#[cfg(target_os = "linux")]
let criterion = Criterion::default().with_profiler(pprof::criterion::PProfProfiler::new(
100,
pprof::criterion::Output::Flamegraph(None),
));

#[cfg(not(target_os = "linux"))]
let criterion = Criterion::default();

criterion
}

pub fn room_list(c: &mut Criterion) {
let runtime = Builder::new_multi_thread()
.enable_time()
.enable_io()
.build()
.expect("Can't create runtime");

let mut benchmark = c.benchmark_group("room_list");

for number_of_rooms in [10, 100, 1000, 10_000] {
benchmark.throughput(Throughput::Elements(number_of_rooms));
benchmark.bench_with_input(
BenchmarkId::new("sync", number_of_rooms),
&number_of_rooms,
|benchmark, maximum_number_of_rooms| {
benchmark.to_async(&runtime).iter(|| async {
// Create a fresh new `Client` and a mocked server.
let (client, server) = logged_in_client_with_server().await;

// Create a new `RoomListService`.
let room_list_service = RoomListService::new(client.clone())
.await
.expect("Failed to create the `RoomListService`");

// Get the `RoomListService` sync stream.
let room_list_stream = room_list_service.sync();
pin_mut!(room_list_stream);

// Get the `RoomList` itself with a default filter.
let room_list = room_list_service
.all_rooms()
.await
.expect("Failed to fetch the `all_rooms` room list");
let (room_list_entries, room_list_entries_controller) = room_list
.entries_with_dynamic_adapters(100, client.roominfo_update_receiver());
pin_mut!(room_list_entries);
room_list_entries_controller.set_filter(Box::new(new_filter_non_left()));
room_list_entries.next().await;

// “Send” (mocked) and receive rooms.
{
const ROOMS_BATCH_SIZE: u64 = 100;
let maximum_number_of_rooms = *maximum_number_of_rooms;
let mut number_of_sent_rooms = 0;
let mut room_nth = 0;

while number_of_sent_rooms < maximum_number_of_rooms {
let number_of_rooms_to_send = u64::max(
number_of_sent_rooms + ROOMS_BATCH_SIZE,
maximum_number_of_rooms,
) - number_of_sent_rooms;

number_of_sent_rooms += number_of_rooms_to_send;

let rooms_as_json = Value::Object(
(0..number_of_rooms_to_send)
.map(|_| {
let room_id = format!("!r{room_nth}:matrix.org");

let room = json!({
// The recency timestamp is different, so that it triggers a re-ordering of the
// room list every time, just to stress the APIs.
"timestamp": room_nth % 10,
// One state event to activate the associated code path.
"required_state": [
{
"content": {
"name": format!("Room #{room_nth}"),
},
"event_id": format!("$s{room_nth}"),
"origin_server_ts": 1,
"sender": "@example:matrix.org",
"state_key": "",
"type": "m.room.name"
},
],
// One room event to active the associated code path.
"timeline": [
{
"event_id": format!("$t{room_nth}"),
"sender": "@example:matrix.org",
"type": "m.room.message",
"content": {
"body": "foo",
"msgtype": "m.text",
},
"origin_server_ts": 1,
}
],
});

room_nth += 1;

(room_id, room)
})
.collect::<Map<_, _>>(),
);

// Mock the response from the server.
let _mock_guard = Mock::given(SlidingSyncMatcher)
.respond_with(move |request: &Request| {
let partial_request: PartialSlidingSyncRequest =
request.body_json().unwrap();

ResponseTemplate::new(200).set_body_json(json!({
"txn_id": partial_request.txn_id,
"pos": room_nth.to_string(),
"rooms": rooms_as_json,
}))
})
.mount_as_scoped(&server)
.await;

// Sync the room list service.
assert!(
room_list_stream.next().await.is_some(),
"`room_list_stream` has stopped"
);

// Sync the room list entries.
assert!(
room_list_entries.next().await.is_some(),
"`room_list_entries` has stopped"
);
}
}
})
},
);
}

benchmark.finish();
}

criterion_group! {
name = benches;
config = criterion();
targets = room_list
}
criterion_main!(benches);

0 comments on commit 0f5eef9

Please sign in to comment.