Add mixed read/write tests to the benchmark. (#1130)

## Description of change

As we make changes that might impact mixed read/write workloads it is
useful to have some examples of these in our benchmarks. This change
adds a 20/80, 50/50, and 80/20 read/write workload to the benchmarks.

The results reporting is updated to support multiple job types in a
single benchmark. This works by continuing to average over iterations as
before, but averages each job separately then sums the averages to
produce the final throughput number for that benchmark (i.e. (avg(read
throughput) + avg(write throughput)) for the mixed benchmarks).

## Does this change impact existing behavior?

No.

## Does this change need a changelog entry in any of the crates?

No.

---

By submitting this pull request, I confirm that my contribution is made
under the terms of the Apache 2.0 license and I agree to the terms of
the [Developer Certificate of Origin
(DCO)](https://developercertificate.org/).

Signed-off-by: Andrew Peace <[email protected]>

mountpoint-s3/scripts/fio/mix/mix_1r4w.fio

@@ -0,0 +1,24 @@
+[global]
+name=fs_bench
+bs=256k
+runtime=30s
+time_based
+group_reporting
+
+[sequential_read]
+size=100G
+rw=read
+ioengine=sync
+fallocate=none
+numjobs=1
+
+[sequential_write_four_threads]
+new_group
+numjobs=4
+size=100G
+rw=write
+ioengine=sync
+fallocate=none
+create_on_open=1
+fsync_on_close=1
+unlink=1

mountpoint-s3/scripts/fio/mix/mix_2r2w.fio

@@ -0,0 +1,24 @@
+[global]
+name=fs_bench
+bs=256k
+runtime=30s
+time_based
+group_reporting
+
+[sequential_read_two_threads]
+size=100G
+rw=read
+ioengine=sync
+fallocate=none
+numjobs=2
+
+[sequential_write_two_threads]
+new_group
+numjobs=2
+size=100G
+rw=write
+ioengine=sync
+fallocate=none
+create_on_open=1
+fsync_on_close=1
+unlink=1

mountpoint-s3/scripts/fio/mix/mix_4r1w.fio

@@ -0,0 +1,24 @@
+[global]
+name=fs_bench
+bs=256k
+runtime=30s
+time_based
+group_reporting
+
+[sequential_read_four_threads]
+size=100G
+rw=read
+ioengine=sync
+fallocate=none
+numjobs=4
+
+[sequential_write]
+new_group
+numjobs=1
+size=100G
+rw=write
+ioengine=sync
+fallocate=none
+create_on_open=1
+fsync_on_close=1
+unlink=1

mountpoint-s3/scripts/fs_bench.sh

@@ -37,7 +37,7 @@ cd ${project_dir}
 results_dir=results
 runtime_seconds=30
 startdelay_seconds=30
-iterations=10
+: ${iterations:=10}
 
 rm -rf ${results_dir}
 mkdir -p ${results_dir}
@@ -74,13 +74,28 @@ run_fio_job() {
   done
   echo "done"
 
-  # combine the results and find an average value
-  jq -n 'reduce inputs.jobs[] as $job (null; .name = $job.jobname | .len += 1 | .value += (if ($job."job options".rw == "read")
-      then $job.read.bw / 1024
-      elif ($job."job options".rw == "randread") then $job.read.bw / 1024
-      elif ($job."job options".rw == "randwrite") then $job.write.bw / 1024
-      elif ($job."job options".rw | startswith("read")) then $job.read.bw / 1024
-      else $job.write.bw / 1024 end)) | {name: .name, value: (.value / .len), unit: "MiB/s"}' ${results_dir}/${job_name}_iter*.json | tee ${results_dir}/${job_name}_parsed.json
+  # combine the results and find an average value.  If the test was a single job, take the
+  # average across all iterations.  If it was multiple jobs, take the average across all
+  # iterations of each job, then sum the averages to get the final results.  For example,
+  # for a mixed read/write test, this would average all read jobs, all write jobs, and
+  # combine the two averages into a throughput.
+  jq -s '[
+      [.[].jobs] | add | group_by(.jobname)[] |
+          {
+              name: .[0].jobname,
+              value: ((map(
+                  if .["job options"].rw | test("^(rand)?read")
+                  then .read.bw
+                  else .write.bw
+                  end
+              ) | add) / (length * 1024))
+          }
+      ] | {
+          name: map(.name) | unique | join(","),
+          value: map(.value) | add,
+          unit: "MiB/s"
+      }' \
+      ${results_dir}/${job_name}_iter*.json | tee ${results_dir}/${job_name}_parsed.json
 }
 
 should_run_job() {
@@ -179,6 +194,7 @@ run_benchmarks() {
 
 run_benchmarks read
 run_benchmarks write
+run_benchmarks mix
 
 # combine all bench results into one json file
 echo "Throughput:"