From 1146d54a13ce2ba865eb59c3c8c45dd1a47fb6e6 Mon Sep 17 00:00:00 2001 From: tony92151 Date: Sun, 16 Feb 2025 12:38:18 +0800 Subject: [PATCH 1/6] tmp only test on linux py311 --- .github/workflows/github-actions.yml | 408 --------------------------- 1 file changed, 408 deletions(-) diff --git a/.github/workflows/github-actions.yml b/.github/workflows/github-actions.yml index 41d65fe..7fce7ac 100644 --- a/.github/workflows/github-actions.yml +++ b/.github/workflows/github-actions.yml @@ -19,438 +19,30 @@ jobs: toxpython: 'python3.11' tox_env: 'docs' os: 'ubuntu-latest' - - name: 'py39-pytest83-nodist-cover (ubuntu)' - python: '3.9' - toxpython: 'python3.9' - python_arch: 'x64' - tox_env: 'py39-pytest83-nodist-cover' - os: 'ubuntu-latest' - - name: 'py39-pytest83-nodist-cover (windows)' - python: '3.9' - toxpython: 'python3.9' - python_arch: 'x64' - tox_env: 'py39-pytest83-nodist-cover' - os: 'windows-latest' - - name: 'py39-pytest83-nodist-cover (macos)' - python: '3.9' - toxpython: 'python3.9' - python_arch: 'arm64' - tox_env: 'py39-pytest83-nodist-cover' - os: 'macos-latest' - - name: 'py39-pytest83-nodist-nocov (ubuntu)' - python: '3.9' - toxpython: 'python3.9' - python_arch: 'x64' - tox_env: 'py39-pytest83-nodist-nocov' - os: 'ubuntu-latest' - - name: 'py39-pytest83-nodist-nocov (windows)' - python: '3.9' - toxpython: 'python3.9' - python_arch: 'x64' - tox_env: 'py39-pytest83-nodist-nocov' - os: 'windows-latest' - - name: 'py39-pytest83-nodist-nocov (macos)' - python: '3.9' - toxpython: 'python3.9' - python_arch: 'arm64' - tox_env: 'py39-pytest83-nodist-nocov' - os: 'macos-latest' - - name: 'py39-pytest83-xdist-cover (ubuntu)' - python: '3.9' - toxpython: 'python3.9' - python_arch: 'x64' - tox_env: 'py39-pytest83-xdist-cover' - os: 'ubuntu-latest' - - name: 'py39-pytest83-xdist-cover (windows)' - python: '3.9' - toxpython: 'python3.9' - python_arch: 'x64' - tox_env: 'py39-pytest83-xdist-cover' - os: 'windows-latest' - - name: 'py39-pytest83-xdist-cover (macos)' - python: '3.9' - toxpython: 'python3.9' - python_arch: 'arm64' - tox_env: 'py39-pytest83-xdist-cover' - os: 'macos-latest' - - name: 'py39-pytest83-xdist-nocov (ubuntu)' - python: '3.9' - toxpython: 'python3.9' - python_arch: 'x64' - tox_env: 'py39-pytest83-xdist-nocov' - os: 'ubuntu-latest' - - name: 'py39-pytest83-xdist-nocov (windows)' - python: '3.9' - toxpython: 'python3.9' - python_arch: 'x64' - tox_env: 'py39-pytest83-xdist-nocov' - os: 'windows-latest' - - name: 'py39-pytest83-xdist-nocov (macos)' - python: '3.9' - toxpython: 'python3.9' - python_arch: 'arm64' - tox_env: 'py39-pytest83-xdist-nocov' - os: 'macos-latest' - - name: 'py310-pytest83-nodist-cover (ubuntu)' - python: '3.10' - toxpython: 'python3.10' - python_arch: 'x64' - tox_env: 'py310-pytest83-nodist-cover' - os: 'ubuntu-latest' - - name: 'py310-pytest83-nodist-cover (windows)' - python: '3.10' - toxpython: 'python3.10' - python_arch: 'x64' - tox_env: 'py310-pytest83-nodist-cover' - os: 'windows-latest' - - name: 'py310-pytest83-nodist-cover (macos)' - python: '3.10' - toxpython: 'python3.10' - python_arch: 'arm64' - tox_env: 'py310-pytest83-nodist-cover' - os: 'macos-latest' - - name: 'py310-pytest83-nodist-nocov (ubuntu)' - python: '3.10' - toxpython: 'python3.10' - python_arch: 'x64' - tox_env: 'py310-pytest83-nodist-nocov' - os: 'ubuntu-latest' - - name: 'py310-pytest83-nodist-nocov (windows)' - python: '3.10' - toxpython: 'python3.10' - python_arch: 'x64' - tox_env: 'py310-pytest83-nodist-nocov' - os: 'windows-latest' - - name: 'py310-pytest83-nodist-nocov (macos)' - python: '3.10' - toxpython: 'python3.10' - python_arch: 'arm64' - tox_env: 'py310-pytest83-nodist-nocov' - os: 'macos-latest' - - name: 'py310-pytest83-xdist-cover (ubuntu)' - python: '3.10' - toxpython: 'python3.10' - python_arch: 'x64' - tox_env: 'py310-pytest83-xdist-cover' - os: 'ubuntu-latest' - - name: 'py310-pytest83-xdist-cover (windows)' - python: '3.10' - toxpython: 'python3.10' - python_arch: 'x64' - tox_env: 'py310-pytest83-xdist-cover' - os: 'windows-latest' - - name: 'py310-pytest83-xdist-cover (macos)' - python: '3.10' - toxpython: 'python3.10' - python_arch: 'arm64' - tox_env: 'py310-pytest83-xdist-cover' - os: 'macos-latest' - - name: 'py310-pytest83-xdist-nocov (ubuntu)' - python: '3.10' - toxpython: 'python3.10' - python_arch: 'x64' - tox_env: 'py310-pytest83-xdist-nocov' - os: 'ubuntu-latest' - - name: 'py310-pytest83-xdist-nocov (windows)' - python: '3.10' - toxpython: 'python3.10' - python_arch: 'x64' - tox_env: 'py310-pytest83-xdist-nocov' - os: 'windows-latest' - - name: 'py310-pytest83-xdist-nocov (macos)' - python: '3.10' - toxpython: 'python3.10' - python_arch: 'arm64' - tox_env: 'py310-pytest83-xdist-nocov' - os: 'macos-latest' - name: 'py311-pytest83-nodist-cover (ubuntu)' python: '3.11' toxpython: 'python3.11' python_arch: 'x64' tox_env: 'py311-pytest83-nodist-cover' os: 'ubuntu-latest' - - name: 'py311-pytest83-nodist-cover (windows)' - python: '3.11' - toxpython: 'python3.11' - python_arch: 'x64' - tox_env: 'py311-pytest83-nodist-cover' - os: 'windows-latest' - - name: 'py311-pytest83-nodist-cover (macos)' - python: '3.11' - toxpython: 'python3.11' - python_arch: 'arm64' - tox_env: 'py311-pytest83-nodist-cover' - os: 'macos-latest' - name: 'py311-pytest83-nodist-nocov (ubuntu)' python: '3.11' toxpython: 'python3.11' python_arch: 'x64' tox_env: 'py311-pytest83-nodist-nocov' os: 'ubuntu-latest' - - name: 'py311-pytest83-nodist-nocov (windows)' - python: '3.11' - toxpython: 'python3.11' - python_arch: 'x64' - tox_env: 'py311-pytest83-nodist-nocov' - os: 'windows-latest' - - name: 'py311-pytest83-nodist-nocov (macos)' - python: '3.11' - toxpython: 'python3.11' - python_arch: 'arm64' - tox_env: 'py311-pytest83-nodist-nocov' - os: 'macos-latest' - name: 'py311-pytest83-xdist-cover (ubuntu)' python: '3.11' toxpython: 'python3.11' python_arch: 'x64' tox_env: 'py311-pytest83-xdist-cover' os: 'ubuntu-latest' - - name: 'py311-pytest83-xdist-cover (windows)' - python: '3.11' - toxpython: 'python3.11' - python_arch: 'x64' - tox_env: 'py311-pytest83-xdist-cover' - os: 'windows-latest' - - name: 'py311-pytest83-xdist-cover (macos)' - python: '3.11' - toxpython: 'python3.11' - python_arch: 'arm64' - tox_env: 'py311-pytest83-xdist-cover' - os: 'macos-latest' - name: 'py311-pytest83-xdist-nocov (ubuntu)' python: '3.11' toxpython: 'python3.11' python_arch: 'x64' tox_env: 'py311-pytest83-xdist-nocov' os: 'ubuntu-latest' - - name: 'py311-pytest83-xdist-nocov (windows)' - python: '3.11' - toxpython: 'python3.11' - python_arch: 'x64' - tox_env: 'py311-pytest83-xdist-nocov' - os: 'windows-latest' - - name: 'py311-pytest83-xdist-nocov (macos)' - python: '3.11' - toxpython: 'python3.11' - python_arch: 'arm64' - tox_env: 'py311-pytest83-xdist-nocov' - os: 'macos-latest' - - name: 'py312-pytest83-nodist-cover (ubuntu)' - python: '3.12' - toxpython: 'python3.12' - python_arch: 'x64' - tox_env: 'py312-pytest83-nodist-cover' - os: 'ubuntu-latest' - - name: 'py312-pytest83-nodist-cover (windows)' - python: '3.12' - toxpython: 'python3.12' - python_arch: 'x64' - tox_env: 'py312-pytest83-nodist-cover' - os: 'windows-latest' - - name: 'py312-pytest83-nodist-cover (macos)' - python: '3.12' - toxpython: 'python3.12' - python_arch: 'arm64' - tox_env: 'py312-pytest83-nodist-cover' - os: 'macos-latest' - - name: 'py312-pytest83-nodist-nocov (ubuntu)' - python: '3.12' - toxpython: 'python3.12' - python_arch: 'x64' - tox_env: 'py312-pytest83-nodist-nocov' - os: 'ubuntu-latest' - - name: 'py312-pytest83-nodist-nocov (windows)' - python: '3.12' - toxpython: 'python3.12' - python_arch: 'x64' - tox_env: 'py312-pytest83-nodist-nocov' - os: 'windows-latest' - - name: 'py312-pytest83-nodist-nocov (macos)' - python: '3.12' - toxpython: 'python3.12' - python_arch: 'arm64' - tox_env: 'py312-pytest83-nodist-nocov' - os: 'macos-latest' - - name: 'py312-pytest83-xdist-cover (ubuntu)' - python: '3.12' - toxpython: 'python3.12' - python_arch: 'x64' - tox_env: 'py312-pytest83-xdist-cover' - os: 'ubuntu-latest' - - name: 'py312-pytest83-xdist-cover (windows)' - python: '3.12' - toxpython: 'python3.12' - python_arch: 'x64' - tox_env: 'py312-pytest83-xdist-cover' - os: 'windows-latest' - - name: 'py312-pytest83-xdist-cover (macos)' - python: '3.12' - toxpython: 'python3.12' - python_arch: 'arm64' - tox_env: 'py312-pytest83-xdist-cover' - os: 'macos-latest' - - name: 'py312-pytest83-xdist-nocov (ubuntu)' - python: '3.12' - toxpython: 'python3.12' - python_arch: 'x64' - tox_env: 'py312-pytest83-xdist-nocov' - os: 'ubuntu-latest' - - name: 'py312-pytest83-xdist-nocov (windows)' - python: '3.12' - toxpython: 'python3.12' - python_arch: 'x64' - tox_env: 'py312-pytest83-xdist-nocov' - os: 'windows-latest' - - name: 'py312-pytest83-xdist-nocov (macos)' - python: '3.12' - toxpython: 'python3.12' - python_arch: 'arm64' - tox_env: 'py312-pytest83-xdist-nocov' - os: 'macos-latest' - - name: 'pypy39-pytest83-nodist-cover (ubuntu)' - python: 'pypy-3.9' - toxpython: 'pypy3.9' - python_arch: 'x64' - tox_env: 'pypy39-pytest83-nodist-cover' - os: 'ubuntu-latest' - - name: 'pypy39-pytest83-nodist-cover (windows)' - python: 'pypy-3.9' - toxpython: 'pypy3.9' - python_arch: 'x64' - tox_env: 'pypy39-pytest83-nodist-cover' - os: 'windows-latest' - - name: 'pypy39-pytest83-nodist-cover (macos)' - python: 'pypy-3.9' - toxpython: 'pypy3.9' - python_arch: 'arm64' - tox_env: 'pypy39-pytest83-nodist-cover' - os: 'macos-latest' - - name: 'pypy39-pytest83-nodist-nocov (ubuntu)' - python: 'pypy-3.9' - toxpython: 'pypy3.9' - python_arch: 'x64' - tox_env: 'pypy39-pytest83-nodist-nocov' - os: 'ubuntu-latest' - - name: 'pypy39-pytest83-nodist-nocov (windows)' - python: 'pypy-3.9' - toxpython: 'pypy3.9' - python_arch: 'x64' - tox_env: 'pypy39-pytest83-nodist-nocov' - os: 'windows-latest' - - name: 'pypy39-pytest83-nodist-nocov (macos)' - python: 'pypy-3.9' - toxpython: 'pypy3.9' - python_arch: 'arm64' - tox_env: 'pypy39-pytest83-nodist-nocov' - os: 'macos-latest' - - name: 'pypy39-pytest83-xdist-cover (ubuntu)' - python: 'pypy-3.9' - toxpython: 'pypy3.9' - python_arch: 'x64' - tox_env: 'pypy39-pytest83-xdist-cover' - os: 'ubuntu-latest' - - name: 'pypy39-pytest83-xdist-cover (windows)' - python: 'pypy-3.9' - toxpython: 'pypy3.9' - python_arch: 'x64' - tox_env: 'pypy39-pytest83-xdist-cover' - os: 'windows-latest' - - name: 'pypy39-pytest83-xdist-cover (macos)' - python: 'pypy-3.9' - toxpython: 'pypy3.9' - python_arch: 'arm64' - tox_env: 'pypy39-pytest83-xdist-cover' - os: 'macos-latest' - - name: 'pypy39-pytest83-xdist-nocov (ubuntu)' - python: 'pypy-3.9' - toxpython: 'pypy3.9' - python_arch: 'x64' - tox_env: 'pypy39-pytest83-xdist-nocov' - os: 'ubuntu-latest' - - name: 'pypy39-pytest83-xdist-nocov (windows)' - python: 'pypy-3.9' - toxpython: 'pypy3.9' - python_arch: 'x64' - tox_env: 'pypy39-pytest83-xdist-nocov' - os: 'windows-latest' - - name: 'pypy39-pytest83-xdist-nocov (macos)' - python: 'pypy-3.9' - toxpython: 'pypy3.9' - python_arch: 'arm64' - tox_env: 'pypy39-pytest83-xdist-nocov' - os: 'macos-latest' - - name: 'pypy310-pytest83-nodist-cover (ubuntu)' - python: 'pypy-3.10' - toxpython: 'pypy3.10' - python_arch: 'x64' - tox_env: 'pypy310-pytest83-nodist-cover' - os: 'ubuntu-latest' - - name: 'pypy310-pytest83-nodist-cover (windows)' - python: 'pypy-3.10' - toxpython: 'pypy3.10' - python_arch: 'x64' - tox_env: 'pypy310-pytest83-nodist-cover' - os: 'windows-latest' - - name: 'pypy310-pytest83-nodist-cover (macos)' - python: 'pypy-3.10' - toxpython: 'pypy3.10' - python_arch: 'arm64' - tox_env: 'pypy310-pytest83-nodist-cover' - os: 'macos-latest' - - name: 'pypy310-pytest83-nodist-nocov (ubuntu)' - python: 'pypy-3.10' - toxpython: 'pypy3.10' - python_arch: 'x64' - tox_env: 'pypy310-pytest83-nodist-nocov' - os: 'ubuntu-latest' - - name: 'pypy310-pytest83-nodist-nocov (windows)' - python: 'pypy-3.10' - toxpython: 'pypy3.10' - python_arch: 'x64' - tox_env: 'pypy310-pytest83-nodist-nocov' - os: 'windows-latest' - - name: 'pypy310-pytest83-nodist-nocov (macos)' - python: 'pypy-3.10' - toxpython: 'pypy3.10' - python_arch: 'arm64' - tox_env: 'pypy310-pytest83-nodist-nocov' - os: 'macos-latest' - - name: 'pypy310-pytest83-xdist-cover (ubuntu)' - python: 'pypy-3.10' - toxpython: 'pypy3.10' - python_arch: 'x64' - tox_env: 'pypy310-pytest83-xdist-cover' - os: 'ubuntu-latest' - - name: 'pypy310-pytest83-xdist-cover (windows)' - python: 'pypy-3.10' - toxpython: 'pypy3.10' - python_arch: 'x64' - tox_env: 'pypy310-pytest83-xdist-cover' - os: 'windows-latest' - - name: 'pypy310-pytest83-xdist-cover (macos)' - python: 'pypy-3.10' - toxpython: 'pypy3.10' - python_arch: 'arm64' - tox_env: 'pypy310-pytest83-xdist-cover' - os: 'macos-latest' - - name: 'pypy310-pytest83-xdist-nocov (ubuntu)' - python: 'pypy-3.10' - toxpython: 'pypy3.10' - python_arch: 'x64' - tox_env: 'pypy310-pytest83-xdist-nocov' - os: 'ubuntu-latest' - - name: 'pypy310-pytest83-xdist-nocov (windows)' - python: 'pypy-3.10' - toxpython: 'pypy3.10' - python_arch: 'x64' - tox_env: 'pypy310-pytest83-xdist-nocov' - os: 'windows-latest' - - name: 'pypy310-pytest83-xdist-nocov (macos)' - python: 'pypy-3.10' - toxpython: 'pypy3.10' - python_arch: 'arm64' - tox_env: 'pypy310-pytest83-xdist-nocov' - os: 'macos-latest' steps: - uses: actions/checkout@v4 with: From f835e51f684bd04c7576671c3111567697f3e3e2 Mon Sep 17 00:00:00 2001 From: tony92151 Date: Sun, 16 Feb 2025 13:59:15 +0800 Subject: [PATCH 2/6] add pytest_compare_scale_unit in hook --- src/pytest_benchmark/cli.py | 6 +++- src/pytest_benchmark/hookspec.py | 21 +++++++++++ src/pytest_benchmark/plugin.py | 61 +++++++++++++++++++------------- 3 files changed, 63 insertions(+), 25 deletions(-) diff --git a/src/pytest_benchmark/cli.py b/src/pytest_benchmark/cli.py index 8e29625..f7d9bb2 100644 --- a/src/pytest_benchmark/cli.py +++ b/src/pytest_benchmark/cli.py @@ -3,6 +3,7 @@ from _pytest import pathlib from _pytest._io import TerminalWriter +from _pytest.config import Config from _pytest.config.findpaths import locate_config from pytest_benchmark.csv import CSVResults @@ -148,7 +149,10 @@ def main(): histogram=first_or_value(args.histogram, False), name_format=NAME_FORMATTERS[args.name], logger=logger, - scale_unit=partial(hook.pytest_benchmark_scale_unit, config=None), + scale_unit=partial( + hook.pytest_compare_scale_unit, + config=Config.fromdictargs({"time_unit": args.time_unit}, []), + ), ) groups = hook.pytest_benchmark_group_stats( benchmarks=storage.load_benchmarks(*args.glob_or_file), diff --git a/src/pytest_benchmark/hookspec.py b/src/pytest_benchmark/hookspec.py index bc57627..c5440b0 100644 --- a/src/pytest_benchmark/hookspec.py +++ b/src/pytest_benchmark/hookspec.py @@ -22,6 +22,27 @@ def pytest_benchmark_scale_unit(config, unit, benchmarks, best, worst, sort): """ +@pytest.hookspec(firstresult=True) +def pytest_compare_scale_unit(config, unit, benchmarks, best, worst, sort): + """ + To have custom time scaling do something like this: + + .. sourcecode:: python + + def pytest_compare_scale_unit(config, unit, benchmarks, best, worst, sort): + if unit == 'seconds': + prefix = '' + scale = 1.0 + elif unit == 'operations': + prefix = 'K' + scale = 0.001 + else: + raise RuntimeError("Unexpected measurement unit %r" % unit) + + return prefix, scale + """ + + @pytest.hookspec(firstresult=True) def pytest_benchmark_generate_machine_info(config): """ diff --git a/src/pytest_benchmark/plugin.py b/src/pytest_benchmark/plugin.py index 8ce16e2..1c3af3b 100644 --- a/src/pytest_benchmark/plugin.py +++ b/src/pytest_benchmark/plugin.py @@ -85,6 +85,13 @@ def add_display_options(addoption, prefix='benchmark-'): default='normal', help="How to format names in results. Can be one of 'short', 'normal', 'long', or 'trial'. Default: %(default)r", ) + addoption( + f'--{prefix}time-unit', + metavar='COLUMN', + default=None, + choices=['ns', 'us', 'ms', 's', 'auto'], + help="Unit to scale the results to. Available units: 'ns', 'us', 'ms', 's'. Default: 'auto'.", + ) def add_histogram_options(addoption, prefix='benchmark-'): @@ -298,13 +305,6 @@ def pytest_addoption(parser): help='Save cprofile dumps as FILENAME-PREFIX-test_name.prof. If FILENAME-PREFIX contains' f" slashes ('/') then directories will be created. Default: {cprofile_dump_prefix!r}", ) - group.addoption( - '--benchmark-time-unit', - metavar='COLUMN', - default=None, - choices=['ns', 'us', 'ms', 's', 'auto'], - help="Unit to scale the results to. Available units: 'ns', 'us', 'ms', 's'. Default: 'auto'.", - ) add_global_options(group.addoption) add_display_options(group.addoption) add_histogram_options(group.addoption) @@ -394,26 +394,39 @@ def get_cpu_info(): return cpuinfo.get_cpu_info() or {} -def pytest_benchmark_scale_unit(config, unit, benchmarks, best, worst, sort): - config_time_unit = config.getoption('benchmark_time_unit', None) if config else None - if config_time_unit == 'ns': - return 'n', 1e9 - elif config_time_unit == 'us': - return 'u', 1e6 - elif config_time_unit == 'ms': - return 'm', 1e3 - elif config_time_unit == 's': - return '', 1.0 - assert config_time_unit in ('auto', None) - if unit == 'seconds': +def _scale_unit(config_time_unit, unit, benchmarks, best, worst, sort): + if config_time_unit == "ns": + return "n", 1e9 + elif config_time_unit == "us": + return "u", 1e6 + elif config_time_unit == "ms": + return "m", 1e3 + elif config_time_unit == "s": + return "", 1.0 + assert config_time_unit in ("auto", None) + if unit == "seconds": time_unit_key = sort - if sort in ('name', 'fullname'): - time_unit_key = 'min' + if sort in ("name", "fullname"): + time_unit_key = "min" return time_unit(best.get(sort, benchmarks[0][time_unit_key])) - elif unit == 'operations': - return operations_unit(worst.get('ops', benchmarks[0]['ops'])) + elif unit == "operations": + return operations_unit(worst.get("ops", benchmarks[0]["ops"])) else: - raise RuntimeError(f'Unexpected measurement unit {unit!r}') + raise RuntimeError(f"Unexpected measurement unit {unit!r}") + + +def pytest_compare_scale_unit(config, unit, benchmarks, best, worst, sort): + config_time_unit = config.getoption("time_unit", None) if config else None + scale_result = _scale_unit(config_time_unit, unit, benchmarks, best, worst, sort) + if scale_result is not None: + return scale_result + + +def pytest_benchmark_scale_unit(config, unit, benchmarks, best, worst, sort): + config_time_unit = config.getoption("benchmark_time_unit", None) if config else None + scale_result = _scale_unit(config_time_unit, unit, benchmarks, best, worst, sort) + if scale_result is not None: + return scale_result def pytest_benchmark_generate_machine_info(): From 7409ed923edd12d10727bfbc91cabe939b353a98 Mon Sep 17 00:00:00 2001 From: tony92151 Date: Sun, 16 Feb 2025 14:12:51 +0800 Subject: [PATCH 3/6] reuse pytest_benchmark_scale_unit --- src/pytest_benchmark/cli.py | 4 +-- src/pytest_benchmark/hookspec.py | 21 -------------- src/pytest_benchmark/plugin.py | 47 ++++++++++++-------------------- 3 files changed, 19 insertions(+), 53 deletions(-) diff --git a/src/pytest_benchmark/cli.py b/src/pytest_benchmark/cli.py index f7d9bb2..5a0aa02 100644 --- a/src/pytest_benchmark/cli.py +++ b/src/pytest_benchmark/cli.py @@ -150,8 +150,8 @@ def main(): name_format=NAME_FORMATTERS[args.name], logger=logger, scale_unit=partial( - hook.pytest_compare_scale_unit, - config=Config.fromdictargs({"time_unit": args.time_unit}, []), + hook.pytest_benchmark_scale_unit, + config=Config.fromdictargs({"benchmark_time_unit": args.time_unit}, []), ), ) groups = hook.pytest_benchmark_group_stats( diff --git a/src/pytest_benchmark/hookspec.py b/src/pytest_benchmark/hookspec.py index c5440b0..bc57627 100644 --- a/src/pytest_benchmark/hookspec.py +++ b/src/pytest_benchmark/hookspec.py @@ -22,27 +22,6 @@ def pytest_benchmark_scale_unit(config, unit, benchmarks, best, worst, sort): """ -@pytest.hookspec(firstresult=True) -def pytest_compare_scale_unit(config, unit, benchmarks, best, worst, sort): - """ - To have custom time scaling do something like this: - - .. sourcecode:: python - - def pytest_compare_scale_unit(config, unit, benchmarks, best, worst, sort): - if unit == 'seconds': - prefix = '' - scale = 1.0 - elif unit == 'operations': - prefix = 'K' - scale = 0.001 - else: - raise RuntimeError("Unexpected measurement unit %r" % unit) - - return prefix, scale - """ - - @pytest.hookspec(firstresult=True) def pytest_benchmark_generate_machine_info(config): """ diff --git a/src/pytest_benchmark/plugin.py b/src/pytest_benchmark/plugin.py index 1c3af3b..9a2daf7 100644 --- a/src/pytest_benchmark/plugin.py +++ b/src/pytest_benchmark/plugin.py @@ -394,39 +394,26 @@ def get_cpu_info(): return cpuinfo.get_cpu_info() or {} -def _scale_unit(config_time_unit, unit, benchmarks, best, worst, sort): - if config_time_unit == "ns": - return "n", 1e9 - elif config_time_unit == "us": - return "u", 1e6 - elif config_time_unit == "ms": - return "m", 1e3 - elif config_time_unit == "s": - return "", 1.0 - assert config_time_unit in ("auto", None) - if unit == "seconds": +def pytest_benchmark_scale_unit(config, unit, benchmarks, best, worst, sort): + config_time_unit = config.getoption('benchmark_time_unit', None) if config else None + if config_time_unit == 'ns': + return 'n', 1e9 + elif config_time_unit == 'us': + return 'u', 1e6 + elif config_time_unit == 'ms': + return 'm', 1e3 + elif config_time_unit == 's': + return '', 1.0 + assert config_time_unit in ('auto', None) + if unit == 'seconds': time_unit_key = sort - if sort in ("name", "fullname"): - time_unit_key = "min" + if sort in ('name', 'fullname'): + time_unit_key = 'min' return time_unit(best.get(sort, benchmarks[0][time_unit_key])) - elif unit == "operations": - return operations_unit(worst.get("ops", benchmarks[0]["ops"])) + elif unit == 'operations': + return operations_unit(worst.get('ops', benchmarks[0]['ops'])) else: - raise RuntimeError(f"Unexpected measurement unit {unit!r}") - - -def pytest_compare_scale_unit(config, unit, benchmarks, best, worst, sort): - config_time_unit = config.getoption("time_unit", None) if config else None - scale_result = _scale_unit(config_time_unit, unit, benchmarks, best, worst, sort) - if scale_result is not None: - return scale_result - - -def pytest_benchmark_scale_unit(config, unit, benchmarks, best, worst, sort): - config_time_unit = config.getoption("benchmark_time_unit", None) if config else None - scale_result = _scale_unit(config_time_unit, unit, benchmarks, best, worst, sort) - if scale_result is not None: - return scale_result + raise RuntimeError(f'Unexpected measurement unit {unit!r}') def pytest_benchmark_generate_machine_info(): From 804d8753c7f93e99c686a6b94533f40b38705be9 Mon Sep 17 00:00:00 2001 From: tony92151 Date: Sun, 16 Feb 2025 14:28:16 +0800 Subject: [PATCH 4/6] rollback tmp workflow --- .github/workflows/github-actions.yml | 408 +++++++++++++++++++++++++++ 1 file changed, 408 insertions(+) diff --git a/.github/workflows/github-actions.yml b/.github/workflows/github-actions.yml index 7fce7ac..41d65fe 100644 --- a/.github/workflows/github-actions.yml +++ b/.github/workflows/github-actions.yml @@ -19,30 +19,438 @@ jobs: toxpython: 'python3.11' tox_env: 'docs' os: 'ubuntu-latest' + - name: 'py39-pytest83-nodist-cover (ubuntu)' + python: '3.9' + toxpython: 'python3.9' + python_arch: 'x64' + tox_env: 'py39-pytest83-nodist-cover' + os: 'ubuntu-latest' + - name: 'py39-pytest83-nodist-cover (windows)' + python: '3.9' + toxpython: 'python3.9' + python_arch: 'x64' + tox_env: 'py39-pytest83-nodist-cover' + os: 'windows-latest' + - name: 'py39-pytest83-nodist-cover (macos)' + python: '3.9' + toxpython: 'python3.9' + python_arch: 'arm64' + tox_env: 'py39-pytest83-nodist-cover' + os: 'macos-latest' + - name: 'py39-pytest83-nodist-nocov (ubuntu)' + python: '3.9' + toxpython: 'python3.9' + python_arch: 'x64' + tox_env: 'py39-pytest83-nodist-nocov' + os: 'ubuntu-latest' + - name: 'py39-pytest83-nodist-nocov (windows)' + python: '3.9' + toxpython: 'python3.9' + python_arch: 'x64' + tox_env: 'py39-pytest83-nodist-nocov' + os: 'windows-latest' + - name: 'py39-pytest83-nodist-nocov (macos)' + python: '3.9' + toxpython: 'python3.9' + python_arch: 'arm64' + tox_env: 'py39-pytest83-nodist-nocov' + os: 'macos-latest' + - name: 'py39-pytest83-xdist-cover (ubuntu)' + python: '3.9' + toxpython: 'python3.9' + python_arch: 'x64' + tox_env: 'py39-pytest83-xdist-cover' + os: 'ubuntu-latest' + - name: 'py39-pytest83-xdist-cover (windows)' + python: '3.9' + toxpython: 'python3.9' + python_arch: 'x64' + tox_env: 'py39-pytest83-xdist-cover' + os: 'windows-latest' + - name: 'py39-pytest83-xdist-cover (macos)' + python: '3.9' + toxpython: 'python3.9' + python_arch: 'arm64' + tox_env: 'py39-pytest83-xdist-cover' + os: 'macos-latest' + - name: 'py39-pytest83-xdist-nocov (ubuntu)' + python: '3.9' + toxpython: 'python3.9' + python_arch: 'x64' + tox_env: 'py39-pytest83-xdist-nocov' + os: 'ubuntu-latest' + - name: 'py39-pytest83-xdist-nocov (windows)' + python: '3.9' + toxpython: 'python3.9' + python_arch: 'x64' + tox_env: 'py39-pytest83-xdist-nocov' + os: 'windows-latest' + - name: 'py39-pytest83-xdist-nocov (macos)' + python: '3.9' + toxpython: 'python3.9' + python_arch: 'arm64' + tox_env: 'py39-pytest83-xdist-nocov' + os: 'macos-latest' + - name: 'py310-pytest83-nodist-cover (ubuntu)' + python: '3.10' + toxpython: 'python3.10' + python_arch: 'x64' + tox_env: 'py310-pytest83-nodist-cover' + os: 'ubuntu-latest' + - name: 'py310-pytest83-nodist-cover (windows)' + python: '3.10' + toxpython: 'python3.10' + python_arch: 'x64' + tox_env: 'py310-pytest83-nodist-cover' + os: 'windows-latest' + - name: 'py310-pytest83-nodist-cover (macos)' + python: '3.10' + toxpython: 'python3.10' + python_arch: 'arm64' + tox_env: 'py310-pytest83-nodist-cover' + os: 'macos-latest' + - name: 'py310-pytest83-nodist-nocov (ubuntu)' + python: '3.10' + toxpython: 'python3.10' + python_arch: 'x64' + tox_env: 'py310-pytest83-nodist-nocov' + os: 'ubuntu-latest' + - name: 'py310-pytest83-nodist-nocov (windows)' + python: '3.10' + toxpython: 'python3.10' + python_arch: 'x64' + tox_env: 'py310-pytest83-nodist-nocov' + os: 'windows-latest' + - name: 'py310-pytest83-nodist-nocov (macos)' + python: '3.10' + toxpython: 'python3.10' + python_arch: 'arm64' + tox_env: 'py310-pytest83-nodist-nocov' + os: 'macos-latest' + - name: 'py310-pytest83-xdist-cover (ubuntu)' + python: '3.10' + toxpython: 'python3.10' + python_arch: 'x64' + tox_env: 'py310-pytest83-xdist-cover' + os: 'ubuntu-latest' + - name: 'py310-pytest83-xdist-cover (windows)' + python: '3.10' + toxpython: 'python3.10' + python_arch: 'x64' + tox_env: 'py310-pytest83-xdist-cover' + os: 'windows-latest' + - name: 'py310-pytest83-xdist-cover (macos)' + python: '3.10' + toxpython: 'python3.10' + python_arch: 'arm64' + tox_env: 'py310-pytest83-xdist-cover' + os: 'macos-latest' + - name: 'py310-pytest83-xdist-nocov (ubuntu)' + python: '3.10' + toxpython: 'python3.10' + python_arch: 'x64' + tox_env: 'py310-pytest83-xdist-nocov' + os: 'ubuntu-latest' + - name: 'py310-pytest83-xdist-nocov (windows)' + python: '3.10' + toxpython: 'python3.10' + python_arch: 'x64' + tox_env: 'py310-pytest83-xdist-nocov' + os: 'windows-latest' + - name: 'py310-pytest83-xdist-nocov (macos)' + python: '3.10' + toxpython: 'python3.10' + python_arch: 'arm64' + tox_env: 'py310-pytest83-xdist-nocov' + os: 'macos-latest' - name: 'py311-pytest83-nodist-cover (ubuntu)' python: '3.11' toxpython: 'python3.11' python_arch: 'x64' tox_env: 'py311-pytest83-nodist-cover' os: 'ubuntu-latest' + - name: 'py311-pytest83-nodist-cover (windows)' + python: '3.11' + toxpython: 'python3.11' + python_arch: 'x64' + tox_env: 'py311-pytest83-nodist-cover' + os: 'windows-latest' + - name: 'py311-pytest83-nodist-cover (macos)' + python: '3.11' + toxpython: 'python3.11' + python_arch: 'arm64' + tox_env: 'py311-pytest83-nodist-cover' + os: 'macos-latest' - name: 'py311-pytest83-nodist-nocov (ubuntu)' python: '3.11' toxpython: 'python3.11' python_arch: 'x64' tox_env: 'py311-pytest83-nodist-nocov' os: 'ubuntu-latest' + - name: 'py311-pytest83-nodist-nocov (windows)' + python: '3.11' + toxpython: 'python3.11' + python_arch: 'x64' + tox_env: 'py311-pytest83-nodist-nocov' + os: 'windows-latest' + - name: 'py311-pytest83-nodist-nocov (macos)' + python: '3.11' + toxpython: 'python3.11' + python_arch: 'arm64' + tox_env: 'py311-pytest83-nodist-nocov' + os: 'macos-latest' - name: 'py311-pytest83-xdist-cover (ubuntu)' python: '3.11' toxpython: 'python3.11' python_arch: 'x64' tox_env: 'py311-pytest83-xdist-cover' os: 'ubuntu-latest' + - name: 'py311-pytest83-xdist-cover (windows)' + python: '3.11' + toxpython: 'python3.11' + python_arch: 'x64' + tox_env: 'py311-pytest83-xdist-cover' + os: 'windows-latest' + - name: 'py311-pytest83-xdist-cover (macos)' + python: '3.11' + toxpython: 'python3.11' + python_arch: 'arm64' + tox_env: 'py311-pytest83-xdist-cover' + os: 'macos-latest' - name: 'py311-pytest83-xdist-nocov (ubuntu)' python: '3.11' toxpython: 'python3.11' python_arch: 'x64' tox_env: 'py311-pytest83-xdist-nocov' os: 'ubuntu-latest' + - name: 'py311-pytest83-xdist-nocov (windows)' + python: '3.11' + toxpython: 'python3.11' + python_arch: 'x64' + tox_env: 'py311-pytest83-xdist-nocov' + os: 'windows-latest' + - name: 'py311-pytest83-xdist-nocov (macos)' + python: '3.11' + toxpython: 'python3.11' + python_arch: 'arm64' + tox_env: 'py311-pytest83-xdist-nocov' + os: 'macos-latest' + - name: 'py312-pytest83-nodist-cover (ubuntu)' + python: '3.12' + toxpython: 'python3.12' + python_arch: 'x64' + tox_env: 'py312-pytest83-nodist-cover' + os: 'ubuntu-latest' + - name: 'py312-pytest83-nodist-cover (windows)' + python: '3.12' + toxpython: 'python3.12' + python_arch: 'x64' + tox_env: 'py312-pytest83-nodist-cover' + os: 'windows-latest' + - name: 'py312-pytest83-nodist-cover (macos)' + python: '3.12' + toxpython: 'python3.12' + python_arch: 'arm64' + tox_env: 'py312-pytest83-nodist-cover' + os: 'macos-latest' + - name: 'py312-pytest83-nodist-nocov (ubuntu)' + python: '3.12' + toxpython: 'python3.12' + python_arch: 'x64' + tox_env: 'py312-pytest83-nodist-nocov' + os: 'ubuntu-latest' + - name: 'py312-pytest83-nodist-nocov (windows)' + python: '3.12' + toxpython: 'python3.12' + python_arch: 'x64' + tox_env: 'py312-pytest83-nodist-nocov' + os: 'windows-latest' + - name: 'py312-pytest83-nodist-nocov (macos)' + python: '3.12' + toxpython: 'python3.12' + python_arch: 'arm64' + tox_env: 'py312-pytest83-nodist-nocov' + os: 'macos-latest' + - name: 'py312-pytest83-xdist-cover (ubuntu)' + python: '3.12' + toxpython: 'python3.12' + python_arch: 'x64' + tox_env: 'py312-pytest83-xdist-cover' + os: 'ubuntu-latest' + - name: 'py312-pytest83-xdist-cover (windows)' + python: '3.12' + toxpython: 'python3.12' + python_arch: 'x64' + tox_env: 'py312-pytest83-xdist-cover' + os: 'windows-latest' + - name: 'py312-pytest83-xdist-cover (macos)' + python: '3.12' + toxpython: 'python3.12' + python_arch: 'arm64' + tox_env: 'py312-pytest83-xdist-cover' + os: 'macos-latest' + - name: 'py312-pytest83-xdist-nocov (ubuntu)' + python: '3.12' + toxpython: 'python3.12' + python_arch: 'x64' + tox_env: 'py312-pytest83-xdist-nocov' + os: 'ubuntu-latest' + - name: 'py312-pytest83-xdist-nocov (windows)' + python: '3.12' + toxpython: 'python3.12' + python_arch: 'x64' + tox_env: 'py312-pytest83-xdist-nocov' + os: 'windows-latest' + - name: 'py312-pytest83-xdist-nocov (macos)' + python: '3.12' + toxpython: 'python3.12' + python_arch: 'arm64' + tox_env: 'py312-pytest83-xdist-nocov' + os: 'macos-latest' + - name: 'pypy39-pytest83-nodist-cover (ubuntu)' + python: 'pypy-3.9' + toxpython: 'pypy3.9' + python_arch: 'x64' + tox_env: 'pypy39-pytest83-nodist-cover' + os: 'ubuntu-latest' + - name: 'pypy39-pytest83-nodist-cover (windows)' + python: 'pypy-3.9' + toxpython: 'pypy3.9' + python_arch: 'x64' + tox_env: 'pypy39-pytest83-nodist-cover' + os: 'windows-latest' + - name: 'pypy39-pytest83-nodist-cover (macos)' + python: 'pypy-3.9' + toxpython: 'pypy3.9' + python_arch: 'arm64' + tox_env: 'pypy39-pytest83-nodist-cover' + os: 'macos-latest' + - name: 'pypy39-pytest83-nodist-nocov (ubuntu)' + python: 'pypy-3.9' + toxpython: 'pypy3.9' + python_arch: 'x64' + tox_env: 'pypy39-pytest83-nodist-nocov' + os: 'ubuntu-latest' + - name: 'pypy39-pytest83-nodist-nocov (windows)' + python: 'pypy-3.9' + toxpython: 'pypy3.9' + python_arch: 'x64' + tox_env: 'pypy39-pytest83-nodist-nocov' + os: 'windows-latest' + - name: 'pypy39-pytest83-nodist-nocov (macos)' + python: 'pypy-3.9' + toxpython: 'pypy3.9' + python_arch: 'arm64' + tox_env: 'pypy39-pytest83-nodist-nocov' + os: 'macos-latest' + - name: 'pypy39-pytest83-xdist-cover (ubuntu)' + python: 'pypy-3.9' + toxpython: 'pypy3.9' + python_arch: 'x64' + tox_env: 'pypy39-pytest83-xdist-cover' + os: 'ubuntu-latest' + - name: 'pypy39-pytest83-xdist-cover (windows)' + python: 'pypy-3.9' + toxpython: 'pypy3.9' + python_arch: 'x64' + tox_env: 'pypy39-pytest83-xdist-cover' + os: 'windows-latest' + - name: 'pypy39-pytest83-xdist-cover (macos)' + python: 'pypy-3.9' + toxpython: 'pypy3.9' + python_arch: 'arm64' + tox_env: 'pypy39-pytest83-xdist-cover' + os: 'macos-latest' + - name: 'pypy39-pytest83-xdist-nocov (ubuntu)' + python: 'pypy-3.9' + toxpython: 'pypy3.9' + python_arch: 'x64' + tox_env: 'pypy39-pytest83-xdist-nocov' + os: 'ubuntu-latest' + - name: 'pypy39-pytest83-xdist-nocov (windows)' + python: 'pypy-3.9' + toxpython: 'pypy3.9' + python_arch: 'x64' + tox_env: 'pypy39-pytest83-xdist-nocov' + os: 'windows-latest' + - name: 'pypy39-pytest83-xdist-nocov (macos)' + python: 'pypy-3.9' + toxpython: 'pypy3.9' + python_arch: 'arm64' + tox_env: 'pypy39-pytest83-xdist-nocov' + os: 'macos-latest' + - name: 'pypy310-pytest83-nodist-cover (ubuntu)' + python: 'pypy-3.10' + toxpython: 'pypy3.10' + python_arch: 'x64' + tox_env: 'pypy310-pytest83-nodist-cover' + os: 'ubuntu-latest' + - name: 'pypy310-pytest83-nodist-cover (windows)' + python: 'pypy-3.10' + toxpython: 'pypy3.10' + python_arch: 'x64' + tox_env: 'pypy310-pytest83-nodist-cover' + os: 'windows-latest' + - name: 'pypy310-pytest83-nodist-cover (macos)' + python: 'pypy-3.10' + toxpython: 'pypy3.10' + python_arch: 'arm64' + tox_env: 'pypy310-pytest83-nodist-cover' + os: 'macos-latest' + - name: 'pypy310-pytest83-nodist-nocov (ubuntu)' + python: 'pypy-3.10' + toxpython: 'pypy3.10' + python_arch: 'x64' + tox_env: 'pypy310-pytest83-nodist-nocov' + os: 'ubuntu-latest' + - name: 'pypy310-pytest83-nodist-nocov (windows)' + python: 'pypy-3.10' + toxpython: 'pypy3.10' + python_arch: 'x64' + tox_env: 'pypy310-pytest83-nodist-nocov' + os: 'windows-latest' + - name: 'pypy310-pytest83-nodist-nocov (macos)' + python: 'pypy-3.10' + toxpython: 'pypy3.10' + python_arch: 'arm64' + tox_env: 'pypy310-pytest83-nodist-nocov' + os: 'macos-latest' + - name: 'pypy310-pytest83-xdist-cover (ubuntu)' + python: 'pypy-3.10' + toxpython: 'pypy3.10' + python_arch: 'x64' + tox_env: 'pypy310-pytest83-xdist-cover' + os: 'ubuntu-latest' + - name: 'pypy310-pytest83-xdist-cover (windows)' + python: 'pypy-3.10' + toxpython: 'pypy3.10' + python_arch: 'x64' + tox_env: 'pypy310-pytest83-xdist-cover' + os: 'windows-latest' + - name: 'pypy310-pytest83-xdist-cover (macos)' + python: 'pypy-3.10' + toxpython: 'pypy3.10' + python_arch: 'arm64' + tox_env: 'pypy310-pytest83-xdist-cover' + os: 'macos-latest' + - name: 'pypy310-pytest83-xdist-nocov (ubuntu)' + python: 'pypy-3.10' + toxpython: 'pypy3.10' + python_arch: 'x64' + tox_env: 'pypy310-pytest83-xdist-nocov' + os: 'ubuntu-latest' + - name: 'pypy310-pytest83-xdist-nocov (windows)' + python: 'pypy-3.10' + toxpython: 'pypy3.10' + python_arch: 'x64' + tox_env: 'pypy310-pytest83-xdist-nocov' + os: 'windows-latest' + - name: 'pypy310-pytest83-xdist-nocov (macos)' + python: 'pypy-3.10' + toxpython: 'pypy3.10' + python_arch: 'arm64' + tox_env: 'pypy310-pytest83-xdist-nocov' + os: 'macos-latest' steps: - uses: actions/checkout@v4 with: From 000aea324c14f9b8063735eb6273239eb0c200cc Mon Sep 17 00:00:00 2001 From: tony92151 Date: Sun, 16 Feb 2025 14:43:49 +0800 Subject: [PATCH 5/6] add unit test for --time-unit --- tests/test_cli.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_cli.py b/tests/test_cli.py index 98cdcaa..f534803 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -93,6 +93,7 @@ def test_help_compare(testdir, args): [ 'usage: py.test-benchmark compare [-h] [--sort COL] [--group-by LABEL]', ' [--columns LABELS] [--name FORMAT]', + ' [--time-unit COLUMN]', ' [--histogram [FILENAME-PREFIX]]', ' [--csv [FILENAME]]', ' [[]glob_or_file *[]]', @@ -116,6 +117,8 @@ def test_help_compare(testdir, args): " outliers, ops, rounds, iterations'", " --name FORMAT How to format names in results. Can be one of 'short',", " 'normal', 'long', or 'trial'. Default: 'normal'", + " --time-unit COLUMN Unit to scale the results to. Available units: 'ns',", + " 'us', 'ms', 's'. Default: 'auto'.", ' --histogram [FILENAME-PREFIX]', ' Plot graphs of min/max/avg/stddev over time in', ' FILENAME-PREFIX-test_name.svg. If FILENAME-PREFIX', From 97bdb5e3db214dfb7f52f3f7a52bb72c21648480 Mon Sep 17 00:00:00 2001 From: tony92151 Date: Sun, 16 Feb 2025 15:05:50 +0800 Subject: [PATCH 6/6] add test_compare_with_unit_scale --- tests/test_cli.py | 50 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/tests/test_cli.py b/tests/test_cli.py index f534803..d96ca6e 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -289,6 +289,56 @@ def test_compare(testdir, name, name_pattern_generator): assert result.ret == 0 +@pytest.mark.parametrize( + ('name', 'name_pattern_generator', 'unit'), + [ + ('short', lambda n: '*xfast_parametrized[[]0[]] ' '(%.4d*)' % n, "s"), + ('short', lambda n: '*xfast_parametrized[[]0[]] ' '(%.4d*)' % n, "ms"), + ('short', lambda n: '*xfast_parametrized[[]0[]] ' '(%.4d*)' % n, "us"), + ('short', lambda n: '*xfast_parametrized[[]0[]] ' '(%.4d*)' % n, "ns"), + ], +) +def test_compare_with_unit_scale(testdir, name, name_pattern_generator, unit): + result = testdir.run( + 'py.test-benchmark', + '--storage', + STORAGE, + 'compare', + '0001', + '0002', + '0003', + '--sort', + 'min', + '--columns', + 'min,max', + '--name', + name, + '--histogram', + 'foobar', + '--csv', + 'foobar', + '--time-unit', + unit + ) + result.stderr.fnmatch_lines(['Generated csv: *foobar.csv']) + LineMatcher(testdir.tmpdir.join('foobar.csv').readlines(cr=0)).fnmatch_lines( + [ + 'name,min,max', + 'tests/test_normal.py::test_xfast_parametrized[[]0[]],2.15628567*e-07,1.03186158*e-05', + 'tests/test_normal.py::test_xfast_parametrized[[]0[]],2.16902756*e-07,7.73929968*e-06', + 'tests/test_normal.py::test_xfast_parametrized[[]0[]],2.17314542*e-07,1.14473891*e-05', + '', + ] + ) + result.stdout.fnmatch_lines( + [ + '---*--- benchmark: 3 tests ---*---', + f'Name (time in {unit}) * Min * Max ', + ] + ) + assert result.ret == 0 + + def test_compare_csv(testdir): test = testdir.makepyfile(""" import pytest