diff --git a/.coveragerc b/.coveragerc index 13bb18d98..3ababd737 100644 --- a/.coveragerc +++ b/.coveragerc @@ -7,6 +7,7 @@ branch = True parallel = True [paths] +data_file = ${BB_COVERAGE_PATH-default .}/.coverage source = benchbuild/ */site-packages/benchbuild/ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 93715eb2e..43c301594 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -54,6 +54,7 @@ jobs: strategy: matrix: python-version: ["3.9", "3.10"] + db_support: [true, false] steps: - uses: actions/checkout@v2 @@ -76,8 +77,13 @@ jobs: - name: Run integration tests env: - BB_CONTAINER_ROOT: '/tmp' - BB_CONTAINER_RUNROOT: '/tmp' + BB_DB_ENABLED: ${{ matrix.db_support }} + BB_DB_CONNECT_STRING: "sqlite://" + BB_COVERAGE_COLLECT: true + BB_COVERAGE_PATH: ${{ github.workspace }} + BB_COVERAGE_CONFIG: ${{ github.workspace }}/.coveragerc + BB_CONTAINER_ROOT: ${{ runner.temp }} + BB_CONTAINER_RUNROOT: ${{ runner.temp }} BB_VERBOSITY: 5 run: | coverage run -p `which benchbuild` bootstrap -s @@ -93,6 +99,7 @@ jobs: coverage run -p `which benchbuild` container bases --import -E raw bzip2/benchbuild coverage run -p `which benchbuild` container rmi --with-projects -E raw bzip2/benchbuild coverage combine + coverage report -m - uses: actions/upload-artifact@master with: diff --git a/benchbuild/res/wrapping/run_compiler.py.inc b/benchbuild/res/wrapping/run_compiler.py.inc index 6b45852e5..50d6135b7 100644 --- a/benchbuild/res/wrapping/run_compiler.py.inc +++ b/benchbuild/res/wrapping/run_compiler.py.inc @@ -2,6 +2,11 @@ # import os import sys + +{% if collect_coverage %} +import coverage +{% endif %} + os.environ["OPENBLAS_NUM_THREADS"] = "4" from plumbum import TEE, local @@ -57,4 +62,18 @@ def main(argv): if __name__ == "__main__": - sys.exit(main(sys.argv)) +{% if collect_coverage %} + cov = coverage.Coverage( + config_file="{{ coverage_config }}", + data_file="{{ coverage_path }}/.coverage", + data_suffix=True, + branch=True + ) + cov.start() +{% endif %} + ret = main(sys.argv) +{% if collect_coverage %} + cov.stop() + cov.save() +{% endif %} + sys.exit(ret) diff --git a/benchbuild/res/wrapping/run_dynamic.py.inc b/benchbuild/res/wrapping/run_dynamic.py.inc index 30939ff4b..e6d0aa1a1 100644 --- a/benchbuild/res/wrapping/run_dynamic.py.inc +++ b/benchbuild/res/wrapping/run_dynamic.py.inc @@ -6,6 +6,10 @@ import sys from plumbum import TEE, local +{% if collect_coverage %} +import coverage +{% endif %} + # Performance optimization for benchbuild: don't import any experiments or # projects. Everything necessary should be imported when loading (unpickling) # the project. @@ -53,4 +57,18 @@ def main(argv): if __name__ == "__main__": - sys.exit(main(sys.argv)) +{% if collect_coverage %} + cov = coverage.Coverage( + config_file="{{ coverage_config }}", + data_file="{{ coverage_path }}/.coverage", + data_suffix=True, + branch=True + ) + cov.start() +{% endif %} + ret = main(sys.argv) +{% if collect_coverage %} + cov.stop() + cov.save() +{% endif %} + sys.exit(ret) diff --git a/benchbuild/res/wrapping/run_static.py.inc b/benchbuild/res/wrapping/run_static.py.inc index ba8cc105a..bd5f1b3a7 100644 --- a/benchbuild/res/wrapping/run_static.py.inc +++ b/benchbuild/res/wrapping/run_static.py.inc @@ -3,6 +3,10 @@ import os import sys +{% if collect_coverage %} +import coverage +{% endif %} + from plumbum import TEE, local # Performance optimization for benchbuild: don't import any experiments or @@ -38,4 +42,18 @@ def main(argv): if __name__ == "__main__": - sys.exit(main(sys.argv)) +{% if collect_coverage %} + cov = coverage.Coverage( + config_file="{{ coverage_config }}", + data_file="{{ coverage_path }}/.coverage", + data_suffix=True, + branch=True + ) + cov.start() +{% endif %} + ret = main(sys.argv) +{% if collect_coverage %} + cov.stop() + cov.save() +{% endif %} + sys.exit(ret) diff --git a/benchbuild/settings.py b/benchbuild/settings.py index 9024aab36..f44c1e9de 100644 --- a/benchbuild/settings.py +++ b/benchbuild/settings.py @@ -151,7 +151,7 @@ }, "connect_string": { "desc": "sqlalchemy connect string", - "default": "" + "default": "sqlite://" }, "rollback": { "desc": "Rollback all operations after benchbuild completes.", @@ -513,5 +513,20 @@ } } +CFG["coverage"] = { + "collect": { + "desc": "Should benchuild collect coverage inside wrapped binaries.", + "default": False + }, + "config": { + "desc": "Where is the coverage config?", + "default": ".coveragerc" + }, + "path": { + "desc": "Where should the coverage files be placed?", + "default": None + } +} + s.setup_config(CFG) s.update_env(CFG) diff --git a/benchbuild/utils/wrapping.py b/benchbuild/utils/wrapping.py index 9797c7eda..29d3c1d05 100644 --- a/benchbuild/utils/wrapping.py +++ b/benchbuild/utils/wrapping.py @@ -133,6 +133,10 @@ def wrap( env = CFG["env"].value + collect_coverage = bool(CFG["coverage"]["collect"]) + coverage_config = str(CFG["coverage"]["config"]) + coverage_path = str(CFG["coverage"]["path"]) + bin_path = list_to_path(env.get("PATH", [])) bin_path = list_to_path([bin_path, os.environ["PATH"]]) @@ -149,6 +153,9 @@ def wrap( ld_library_path=str(bin_lib_path), home=str(home), python=python, + collect_coverage=collect_coverage, + coverage_config=coverage_config, + coverage_path=coverage_path ) ) @@ -208,6 +215,9 @@ def wrap_dynamic( project_file = persist(project, suffix=".project") cfg_env = CFG["env"].value + collect_coverage = bool(CFG["coverage"]["collect"]) + coverage_config = str(CFG["coverage"]["config"]) + coverage_path = str(CFG["coverage"]["path"]) bin_path = list_to_path(cfg_env.get("PATH", [])) bin_path = list_to_path([bin_path, os.environ["PATH"]]) @@ -226,6 +236,9 @@ def wrap_dynamic( home=str(home), python=python, name_filters=name_filters, + collect_coverage=collect_coverage, + coverage_config=coverage_config, + coverage_path=coverage_path ) ) @@ -266,6 +279,10 @@ def wrap_cc( project_file = persist(project, suffix=".project") + collect_coverage = bool(CFG["coverage"]["collect"]) + coverage_config = str(CFG["coverage"]["config"]) + coverage_path = str(CFG["coverage"]["path"]) + with open(filepath, "w") as wrapper: wrapper.write( template.render( @@ -273,6 +290,9 @@ def wrap_cc( project_file=str(project_file), python=python, detect_project=detect_project, + collect_coverage=collect_coverage, + coverage_config=coverage_config, + coverage_path=coverage_path ) )