Merge pull request #500 from ocaml-multicore/current-bench

Run benchmarks with current-bench

.github/workflows/main.yml

@@ -52,7 +52,8 @@ jobs:
       - run: | 
           opam pin -yn eio.dev .
           opam pin -yn eio_windows.dev .
-          opam install eio eio_windows --deps-only --with-test
+          opam pin -yn eio_main.dev .
+          opam install eio eio_windows eio_main --deps-only --with-test
       - run: opam exec -- dune build
       - run: opam exec -- dune runtest
       - run: opam exec -- dune exec -- ./examples/net/main.exe

Makefile

@@ -4,16 +4,7 @@ all:
 	dune build @runtest @all
 
 bench:
-	dune exec -- ./bench/bench_http.exe
-	dune exec -- ./bench/bench_condition.exe
-	dune exec -- ./bench/bench_buf_read.exe
-	dune exec -- ./bench/bench_mutex.exe
-	dune exec -- ./bench/bench_yield.exe
-	dune exec -- ./bench/bench_promise.exe
-	dune exec -- ./bench/bench_stream.exe
-	dune exec -- ./bench/bench_semaphore.exe
-	dune exec -- ./bench/bench_cancel.exe
-	if ocamlc -config | grep -q '^system: linux'; then dune exec -- ./lib_eio_linux/tests/bench_noop.exe; fi
+	dune exec -- ./bench/main.exe
 
 test_posix:
 	EIO_BACKEND=posix dune runtest

bench/bench_buf_read.ml

@@ -2,15 +2,18 @@ module R = Eio.Buf_read
 
 let test_data = String.init 100_000_000 (fun _ -> 'x')
 
-let () =
+let run _env =
   let r = R.of_string test_data in
   let t0 = Unix.gettimeofday () in
   let i = ref 0 in
   try
     while true do
       assert (R.any_char r = 'x');
       incr i
-    done
+    done;
+    assert false
   with End_of_file ->
     let t1 = Unix.gettimeofday () in
-    Eio.traceln "Read %d bytes in %.3fs" !i (t1 -. t0)
+    let time = t1 -. t0 in
+    let bytes_per_second = float (String.length test_data) /. time in
+    [Metric.create "any_char" (`Float bytes_per_second) "bytes/s" "Parsing a long string one character at a time"]

bench/bench_cancel.ml

@@ -20,8 +20,12 @@ let run_bench ?domain_mgr ~clock () =
     | Some dm -> Eio.Domain_manager.run dm (fun () -> run_sender stream)
     | None -> run_sender stream
   in
+  let name str =
+    match domain_mgr with
+    | Some _ -> str ^ "/separate domains"
+    | None -> str ^ "/single domain"
+  in
   Gc.full_major ();
-  let _minor0, prom0, _major0 = Gc.counters () in
   let t0 = Eio.Time.now clock in
   try
     Switch.run (fun sw ->
@@ -39,17 +43,17 @@ let run_bench ?domain_mgr ~clock () =
     let t1 = Eio.Time.now clock in
     let time_total = t1 -. t0 in
     let time_per_iter = time_total /. float n_iters in
-    let _minor1, prom1, _major1 = Gc.counters () in
-    let prom = prom1 -. prom0 in
-    Printf.printf "%11b, %7.2f, %13.4f\n%!" (domain_mgr <> None) (1e9 *. time_per_iter) (prom /. float n_iters)
+    Metric.create
+      (name "take-first")
+      (`Float (1e9 *. time_per_iter)) "ns"
+      "Time to take from one of two streams"
 
 let main ~domain_mgr ~clock =
-  Printf.printf "use_domains,  ns/iter, promoted/iter\n%!";
-  run_bench ~clock ();
-  run_bench ~domain_mgr ~clock ()
+  let m1 = run_bench ~clock () in
+  let m2 = run_bench ~domain_mgr ~clock () in
+  [m1; m2]
 
-let () =
-  Eio_main.run @@ fun env ->
+let run env =
   main
     ~domain_mgr:(Eio.Stdenv.domain_mgr env)
     ~clock:(Eio.Stdenv.clock env)

bench/bench_condition.ml

@@ -14,7 +14,6 @@ let run_publisher cond v =
     Eio.Condition.broadcast cond
   done
 
-
 let run_consumer cond v =
   try
     while true do
@@ -36,8 +35,12 @@ let run_bench ?domain_mgr ~clock () =
     | Some dm -> Eio.Domain_manager.run dm (fun () -> run_consumer cond v)
     | None -> run_consumer cond v
   in
+  let name str =
+    match domain_mgr with
+    | Some _ -> str ^ "_domain"
+    | None -> str
+  in
   Gc.full_major ();
-  let _minor0, prom0, _major0 = Gc.counters () in
   let t0 = Eio.Time.now clock in
   for _ = 1 to n_iters do
     Fiber.all [
@@ -49,17 +52,14 @@ let run_bench ?domain_mgr ~clock () =
   let t1 = Eio.Time.now clock in
   let time_total = t1 -. t0 in
   let time_per_iter = time_total /. float n_iters in
-  let _minor1, prom1, _major1 = Gc.counters () in
-  let prom = prom1 -. prom0 in
-  Printf.printf "%11b, %7.2f, %13.4f\n%!" (domain_mgr <> None) (1e3 *. time_per_iter) (prom /. float n_iters)
+  Metric.create (name "broadcast") (`Float (1e3 *. time_per_iter)) "ms" "Time to signal a new value"
 
-let main ~domain_mgr ~clock =
-  Printf.printf "use_domains,  ms/iter, promoted/iter\n%!";
+let main ~domain_mgr ~clock = [
   run_bench ~clock ();
-  run_bench ~domain_mgr ~clock ()
+  run_bench ~domain_mgr ~clock ();
+]
 
-let () =
-  Eio_main.run @@ fun env ->
+let run env =
   main
     ~domain_mgr:(Eio.Stdenv.domain_mgr env)
     ~clock:(Eio.Stdenv.clock env)

bench/bench_fd.ml

@@ -4,7 +4,10 @@ let time label len fn =
   let t0 = Unix.gettimeofday () in
   fn ();
   let t1 = Unix.gettimeofday () in
-  Fmt.pr "%9s, %.1f@." label (float len /. (t1 -. t0) /. (2. ** 30.))
+  Metric.create
+    label
+    (`Float (float len /. (t1 -. t0) /. (2. ** 30.))) "GB/s"
+    "Reading from /dev/zero using a single FD"
 
 let main ~domain_mgr zero =
   let iters = 100_000 in
@@ -15,23 +18,21 @@ let main ~domain_mgr zero =
   let run1 () =
     for _ = 1 to iters do Eio.Flow.read_exact zero buf done
   in
-  print_endline "     test, GB/s";
-  time "1 fiber" (iters * len) run1;
-  time (Fmt.str "%d fibers" n_fibers) (iters * n_fibers * len) (fun () ->
+  [time "fibers:1" (iters * len) run1;
+  time (Fmt.str "fibers:%d" n_fibers) (iters * n_fibers * len) (fun () ->
       Switch.run @@ fun sw ->
       for _ = 1 to n_fibers do
         Fiber.fork ~sw run1
       done
     );
-  time (Fmt.str "%d domains" n_domains) (iters * n_domains * len) (fun () ->
+  time (Fmt.str "domains:%d" n_domains) (iters * n_domains * len) (fun () ->
       Switch.run @@ fun sw ->
       for _ = 1 to n_domains do
         Fiber.fork ~sw (fun () -> Eio.Domain_manager.run domain_mgr run1)
       done
-    )
+    )]
 
 let ( / ) = Eio.Path.( / )
 
-let () =
-  Eio_main.run @@ fun env ->
+let run env =
   Eio.Path.with_open_in (env#fs / "/dev/zero") (main ~domain_mgr:env#domain_mgr)

bench/bench_http.ml

@@ -88,22 +88,20 @@ let main net domain_mgr ~n_client_domains ~n_server_domains ~n_connections_per_d
       done
     );
   let t1 = Unix.gettimeofday () in
-  Fmt.pr "clients, servers, requests, requests/s@.";
+  (* Fmt.pr "clients, servers, requests, requests/s@."; *)
   let requests = n_connections_per_domain * n_client_domains * n_requests_per_connection in
   assert (requests = Atomic.get total);
   let req_per_s = float requests /. (t1 -. t0) in
-  Fmt.pr "%7d, %7d, %8d, %.1f@." n_client_domains n_server_domains requests req_per_s
+  Metric.create
+    (Printf.sprintf "requests:%d client-domains:%d server-domains:%d" requests n_client_domains n_server_domains)
+   (`Float req_per_s) "requests/s" "Request rate of a HTTP client/server system"
 
-let () =
-  print_endline "";     (* work around dune bug *)
-  Eio_main.run @@ fun env ->
-  let main = main env#net env#domain_mgr in
-  match Sys.argv with
-  | [| _ |] -> main ~n_client_domains:4 ~n_server_domains:4 ~n_connections_per_domain:25 ~n_requests_per_connection:1000
-  | [| _; n_client_domains; n_server_domains; n_connections_per_domain; n_requests_per_connection |] ->
-    main
-      ~n_client_domains:(int_of_string n_client_domains)
-      ~n_server_domains:(int_of_string n_server_domains)
-      ~n_connections_per_domain:(int_of_string n_connections_per_domain)
-      ~n_requests_per_connection:(int_of_string n_requests_per_connection)
-  | _ -> Fmt.failwith "usage: bench_http [clients servers connections_per_domain requests_per_connection]"
+let run env =
+  let metrics =
+    main env#net env#domain_mgr
+      ~n_client_domains:4
+      ~n_server_domains:4
+      ~n_connections_per_domain:25
+      ~n_requests_per_connection:1000
+  in
+  [metrics]

bench/bench_mutex.ml

@@ -2,7 +2,7 @@ open Eio.Std
 
 let v = ref 0
 
-let run_sender ~iters_per_thread mutex =
+let run_worker ~iters_per_thread mutex =
   for _ = 1 to iters_per_thread do
     Eio.Mutex.lock mutex;
     let x = !v in
@@ -16,16 +16,15 @@ let run_sender ~iters_per_thread mutex =
 let run_bench ~domain_mgr ~clock ~use_domains ~iters_per_thread ~threads =
   let mutex = Eio.Mutex.create () in
   Gc.full_major ();
-  let _minor0, prom0, _major0 = Gc.counters () in
   let t0 = Eio.Time.now clock in
   Switch.run (fun sw ->
       for _ = 1 to threads do
         Fiber.fork ~sw (fun () ->
             if use_domains then (
               Eio.Domain_manager.run domain_mgr @@ fun () ->
-              run_sender ~iters_per_thread mutex
+              run_worker ~iters_per_thread mutex
             ) else (
-              run_sender ~iters_per_thread mutex
+              run_worker ~iters_per_thread mutex
             )
           )
       done
@@ -35,24 +34,21 @@ let run_bench ~domain_mgr ~clock ~use_domains ~iters_per_thread ~threads =
   let time_total = t1 -. t0 in
   let n_iters = iters_per_thread * threads in
   let time_per_iter = time_total /. float n_iters in
-  let _minor1, prom1, _major1 = Gc.counters () in
-  let prom = prom1 -. prom0 in
-  Printf.printf "%11b, %12d, %8d, %8.2f, %13.4f\n%!" use_domains n_iters threads (1e9 *. time_per_iter) (prom /. float n_iters)
+  Metric.create
+    (Printf.sprintf "iterations=%d threads=%d" n_iters threads)
+    (`Float (1e9 *. time_per_iter)) "ns" "Time to update a shared counter"
 
 let main ~domain_mgr ~clock =
-  Printf.printf "use_domains, iters/thread,  threads,  ns/iter, promoted/iter\n%!";
   [false, 1_000_000, 1;
    false, 1_000_000, 2;
    false,   100_000, 8;
    true,    100_000, 1;
    true,     10_000, 2;
    true,     10_000, 8]
-  |> List.iter (fun (use_domains, iters_per_thread, threads) ->
-      run_bench ~domain_mgr ~clock ~use_domains ~iters_per_thread ~threads
-    )
+  |> List.map (fun (use_domains, iters_per_thread, threads) ->
+      run_bench ~domain_mgr ~clock ~use_domains ~iters_per_thread ~threads)
 
-let () =
-  Eio_main.run @@ fun env ->
+let run env =
   main
     ~domain_mgr:(Eio.Stdenv.domain_mgr env)
     ~clock:(Eio.Stdenv.clock env)

bench/bench_promise.ml

@@ -52,8 +52,11 @@ let bench_resolved ~clock ~n_iters =
     t := !t + Promise.await p;
   done;
   let t1 = Eio.Time.now clock in
-  Printf.printf "Reading a resolved promise: %.3f ns\n%!" (1e9 *. (t1 -. t0) /. float n_iters);
-  assert (!t = n_iters)
+  assert (!t = n_iters);
+  Metric.create
+    "read-resolved"
+    (`Float (1e9 *. (t1 -. t0) /. float n_iters)) "ns"
+    "Time to read a resolved promise"
 
 let maybe_spin v fn =
   if v then Fiber.first spin fn
@@ -62,7 +65,6 @@ let maybe_spin v fn =
 let run_bench ~domain_mgr ~spin ~clock ~use_domains ~n_iters =
   let init_p, init_r = Promise.create () in
   Gc.full_major ();
-  let _minor0, prom0, _major0 = Gc.counters () in
   let t0 = Eio.Time.now clock in
   Fiber.both
     (fun () ->
@@ -79,23 +81,28 @@ let run_bench ~domain_mgr ~spin ~clock ~use_domains ~n_iters =
   let t1 = Eio.Time.now clock in
   let time_total = t1 -. t0 in
   let time_per_iter = time_total /. float n_iters in
-  let _minor1, prom1, _major1 = Gc.counters () in
-  let prom = prom1 -. prom0 in
-  let domains = Printf.sprintf "%b/%b" use_domains spin in
-  Printf.printf "%11s, %8d, %8.2f, %13.4f\n%!" domains n_iters (1e9 *. time_per_iter) (prom /. float n_iters)
+  let domains_label =
+    if use_domains then
+      if spin then "with-spin"
+      else "without-spin"
+    else "no"
+  in
+  Metric.create
+    (Printf.sprintf "iterations:%d domains:%s" n_iters domains_label)
+    (`Float (1e9 *. time_per_iter)) "ns"
+    "Time to round-trip a request/reply"
 
 let main ~domain_mgr ~clock =
-  bench_resolved ~clock ~n_iters:(10_000_000);
-  Printf.printf "domains/spin, n_iters,  ns/iter, promoted/iter\n%!";
-  [false, false, 1_000_000;
+  let resolved = bench_resolved ~clock ~n_iters:(10_000_000) in
+  let metrics = [false, false, 1_000_000;
    true,  true,    100_000;
    true,  false,   100_000]
-  |> List.iter (fun (use_domains, spin, n_iters) ->
+  |> List.map (fun (use_domains, spin, n_iters) ->
       run_bench ~domain_mgr ~spin ~clock ~use_domains ~n_iters
-    )
+  ) in
+  resolved :: metrics
 
-let () =
-  Eio_main.run @@ fun env ->
+let run env =
   main
     ~domain_mgr:(Eio.Stdenv.domain_mgr env)
     ~clock:(Eio.Stdenv.clock env)

bench/bench_semaphore.ml

@@ -36,29 +36,27 @@ let run_bench ~domain_mgr ~clock ~use_domains ~n_iters ~n_resources =
     )
   in
   Gc.full_major ();
-  let _minor0, prom0, _major0 = Gc.counters () in
   Fiber.all (List.init n_workers (Fun.const run));
   let t1 = Eio.Time.now clock in
   let time_total = t1 -. !t0 in
   let time_per_iter = time_total /. float n_iters in
-  let _minor1, prom1, _major1 = Gc.counters () in
-  let prom = prom1 -. prom0 in
-  Printf.printf "%11b, %8d, %11d, %8.2f, %13.4f\n%!" use_domains n_iters n_resources (1e9 *. time_per_iter) (prom /. float n_iters)
+  Metric.create
+    (Printf.sprintf "iterations:%d resources:%d" n_iters n_resources)
+    (`Float (1e9 *. time_per_iter)) "ns"
+    "Time to acquire a semaphore, yeild, and release it"
 
 let main ~domain_mgr ~clock =
-  Printf.printf "use_domains,   n_iters,  resources, ns/iter, promoted/iter\n%!";
   [false, 100_000, 2;
    false, 100_000, 3;
    false, 100_000, 4;
    true,   10_000, 2;
    true,   10_000, 3;
    true,   10_000, 4]
-  |> List.iter (fun (use_domains, n_iters, n_resources) ->
+  |> List.map (fun (use_domains, n_iters, n_resources) ->
       run_bench ~domain_mgr ~clock ~use_domains ~n_iters ~n_resources
     )
 
-let () =
-  Eio_main.run @@ fun env ->
+let run env =
   main
     ~domain_mgr:(Eio.Stdenv.domain_mgr env)
     ~clock:(Eio.Stdenv.clock env)