benchmarks.yml

---
common_options: &common_options
  data:
    throughput:
      regexp: 'throughput: *(.*?) samples\/sec,'
    latency:
      regexp: 'Per-batch latency avg: *(.*?) ms,'
  output:
    - [samples/sec, "throughput"]
    - [latency(ms), "latency"]

tf2_efficientdet_d0_infer_gen_pod4:
  <<: *common_options
  description: EfficientDet-D0 batch-sizes 1 to 3 inference on 4 IPUs.
  parameters:
    batchsize: 1,2,3
  cmd: >-
    poprun
      --vv
      --num-instances 4
      --num-replicas 4
    python ipu_inference.py
      --model-name efficientdet-d0
      --micro-batch-size {batchsize}
      --dataset-type generated
      --random-weights

tf2_efficientdet_d0_max_batch_size_infer_gen_pod4:
  <<: *common_options
  description: EfficientDet-D0-D4 max batch size for 4 IPUs.
  parameters:
    modelname: d0,d1,d2,d3,d4
  cmd: >-
    poprun
      --vv
      --num-instances 4
      --num-replicas 4
    python ipu_inference.py
      --model-name efficientdet-{modelname}
      --dataset-type generated
      --random-weights

tf2_efficientdet_d0_low_latency_infer_gen_pod4:
  <<: *common_options
  description: |
    EfficientDet-D0 batch-sizes 1 low latency using the TF embedded
    application runtime.
  parameters:
    modelname: d0,d1,d2,d3,d4
  cmd: >-
    python ipu_embedded_inference.py
      --config efficientdet-low-latency
      --model-name efficientdet-{modelname}
      --dataset-type generated
      --random-weights
      --num-repeats 500