From 3df01ea25d81c17056281edcf771c8164bd81881 Mon Sep 17 00:00:00 2001 From: Marc Bresson <50196352+MarcBresson@users.noreply.github.com> Date: Mon, 8 Jul 2024 15:26:59 +0200 Subject: [PATCH] New docs about CI/BUILD_NUMBER env vars (#12578) Closes #12577 Co-authored-by: Marc Bresson Co-authored-by: Bruno Oliveira (cherry picked from commit 6933bef0b0f193a32a3716a72a26e7184542376b) --- changelog/12577.doc.rst | 3 + doc/en/explanation/ci.rst | 70 ++++++++++++ doc/en/explanation/index.rst | 3 +- doc/en/reference/reference.rst | 203 +++++++++++++++++---------------- src/_pytest/helpconfig.py | 6 + 5 files changed, 183 insertions(+), 102 deletions(-) create mode 100644 changelog/12577.doc.rst create mode 100644 doc/en/explanation/ci.rst diff --git a/changelog/12577.doc.rst b/changelog/12577.doc.rst new file mode 100644 index 00000000000..0bd427e177d --- /dev/null +++ b/changelog/12577.doc.rst @@ -0,0 +1,3 @@ +`CI` and `BUILD_NUMBER` environment variables role is discribed in +the reference doc. They now also appears when doing `pytest -h` +-- by :user:`MarcBresson`. diff --git a/doc/en/explanation/ci.rst b/doc/en/explanation/ci.rst new file mode 100644 index 00000000000..45fe658d14f --- /dev/null +++ b/doc/en/explanation/ci.rst @@ -0,0 +1,70 @@ +.. _`ci-pipelines`: + +CI Pipelines +============ + +Rationale +--------- + +The goal of testing in a CI pipeline is different from testing locally. Indeed, +you can quickly edit some code and run your tests again on your computer, but +it is not possible with CI pipeline. They run on a separate server and are +triggered by specific actions. + +From that observation, pytest can detect when it is in a CI environment and +adapt some of its behaviours. + +How CI is detected +------------------ + +Pytest knows it is in a CI environment when either one of these environment variables are set, +regardless of their value: + +* `CI`: used by many CI systems. +* `BUILD_NUMBER`: used by Jenkins. + +Effects on CI +------------- + +For now, the effects on pytest of being in a CI environment are limited. + +When a CI environment is detected, the output of the short test summary info is no longer truncated to the terminal size i.e. the entire message will be shown. + + .. code-block:: python + + # content of test_ci.py + import pytest + + + def test_db_initialized(): + pytest.fail( + "deliberately failing for demo purpose, Lorem ipsum dolor sit amet, " + "consectetur adipiscing elit. Cras facilisis, massa in suscipit " + "dignissim, mauris lacus molestie nisi, quis varius metus nulla ut ipsum." + ) + + +Running this locally, without any extra options, will output: + + .. code-block:: pytest + + $ pytest test_ci.py + ... + ========================= short test summary info ========================== + FAILED test_backends.py::test_db_initialized[d2] - Failed: deliberately f... + +*(Note the truncated text)* + + +While running this on CI will output: + + .. code-block:: pytest + + $ export CI=true + $ pytest test_ci.py + ... + ========================= short test summary info ========================== + FAILED test_backends.py::test_db_initialized[d2] - Failed: deliberately failing + for demo purpose, Lorem ipsum dolor sit amet, consectetur adipiscing elit. Cras + facilisis, massa in suscipit dignissim, mauris lacus molestie nisi, quis varius + metus nulla ut ipsum. diff --git a/doc/en/explanation/index.rst b/doc/en/explanation/index.rst index 53910f1eb7b..2edf60a5d8b 100644 --- a/doc/en/explanation/index.rst +++ b/doc/en/explanation/index.rst @@ -11,5 +11,6 @@ Explanation anatomy fixtures goodpractices - flaky pythonpath + ci + flaky diff --git a/doc/en/reference/reference.rst b/doc/en/reference/reference.rst index 3675c7cb2ee..2628c1e9a04 100644 --- a/doc/en/reference/reference.rst +++ b/doc/en/reference/reference.rst @@ -1120,11 +1120,11 @@ Environment variables that can be used to change pytest's behavior. .. envvar:: CI -When set (regardless of value), pytest acknowledges that is running in a CI process. Alternative to ``BUILD_NUMBER`` variable. +When set (regardless of value), pytest acknowledges that is running in a CI process. Alternative to ``BUILD_NUMBER`` variable. See also :ref:`ci-pipelines`. .. envvar:: BUILD_NUMBER -When set (regardless of value), pytest acknowledges that is running in a CI process. Alternative to CI variable. +When set (regardless of value), pytest acknowledges that is running in a CI process. Alternative to CI variable. See also :ref:`ci-pipelines`. .. envvar:: PYTEST_ADDOPTS @@ -1938,19 +1938,18 @@ All the command-line flags can be obtained by running ``pytest --help``:: general: -k EXPRESSION Only run tests which match the given substring expression. An expression is a Python evaluable - expression where all names are substring-matched - against test names and their parent classes. - Example: -k 'test_method or test_other' matches all - test functions and classes whose name contains - 'test_method' or 'test_other', while -k 'not - test_method' matches those that don't contain - 'test_method' in their names. -k 'not test_method - and not test_other' will eliminate the matches. - Additionally keywords are matched to classes and - functions containing extra names in their - 'extra_keyword_matches' set, as well as functions - which have names assigned directly to them. The - matching is case-insensitive. + expression where all names are substring-matched against + test names and their parent classes. Example: -k + 'test_method or test_other' matches all test functions + and classes whose name contains 'test_method' or + 'test_other', while -k 'not test_method' matches those + that don't contain 'test_method' in their names. -k 'not + test_method and not test_other' will eliminate the + matches. Additionally keywords are matched to classes + and functions containing extra names in their + 'extra_keyword_matches' set, as well as functions which + have names assigned directly to them. The matching is + case-insensitive. -m MARKEXPR Only run tests matching given mark expression. For example: -m 'mark1 and not mark2'. --markers show markers (builtin, plugin and per-project ones). @@ -1968,28 +1967,28 @@ All the command-line flags can be obtained by running ``pytest --help``:: --trace Immediately break when running each test --capture=method Per-test capturing method: one of fd|sys|no|tee-sys -s Shortcut for --capture=no - --runxfail Report the results of xfail tests as if they were - not marked - --lf, --last-failed Rerun only the tests that failed at the last run (or - all if none failed) - --ff, --failed-first Run all tests, but run the last failures first. This - may re-order tests and thus lead to repeated fixture + --runxfail Report the results of xfail tests as if they were not + marked + --lf, --last-failed Rerun only the tests that failed at the last run (or all + if none failed) + --ff, --failed-first Run all tests, but run the last failures first. This may + re-order tests and thus lead to repeated fixture setup/teardown. --nf, --new-first Run tests from new files first, then the rest of the tests sorted by file mtime --cache-show=[CACHESHOW] - Show cache contents, don't perform collection or - tests. Optional argument: glob (default: '*'). + Show cache contents, don't perform collection or tests. + Optional argument: glob (default: '*'). --cache-clear Remove all cache contents at start of test run --lfnf={all,none}, --last-failed-no-failures={all,none} - With ``--lf``, determines whether to execute tests - when there are no previously (known) failures or - when no cached ``lastfailed`` data was found. - ``all`` (the default) runs the full test suite - again. ``none`` just emits a message about no known - failures and exits successfully. - --sw, --stepwise Exit on test failure and continue from last failing - test next time + With ``--lf``, determines whether to execute tests when + there are no previously (known) failures or when no + cached ``lastfailed`` data was found. ``all`` (the + default) runs the full test suite again. ``none`` just + emits a message about no known failures and exits + successfully. + --sw, --stepwise Exit on test failure and continue from last failing test + next time --sw-skip, --stepwise-skip Ignore the first failing test but stop on the next failing test. Implicitly enables --stepwise. @@ -2006,55 +2005,53 @@ All the command-line flags can be obtained by running ``pytest --help``:: -r chars Show extra test summary info as specified by chars: (f)ailed, (E)rror, (s)kipped, (x)failed, (X)passed, (p)assed, (P)assed with output, (a)ll except passed - (p/P), or (A)ll. (w)arnings are enabled by default - (see --disable-warnings), 'N' can be used to reset - the list. (default: 'fE'). + (p/P), or (A)ll. (w)arnings are enabled by default (see + --disable-warnings), 'N' can be used to reset the list. + (default: 'fE'). --disable-warnings, --disable-pytest-warnings Disable warnings summary -l, --showlocals Show locals in tracebacks (disabled by default) - --no-showlocals Hide locals in tracebacks (negate --showlocals - passed through addopts) - --tb=style Traceback print mode - (auto/long/short/line/native/no) + --no-showlocals Hide locals in tracebacks (negate --showlocals passed + through addopts) + --tb=style Traceback print mode (auto/long/short/line/native/no) + --xfail-tb Show tracebacks for xfail (as long as --tb != no) --show-capture={no,stdout,stderr,log,all} Controls how captured stdout/stderr/log is shown on failed tests. Default: all. --full-trace Don't cut any tracebacks (default is to cut) --color=color Color terminal output (yes/no/auto) --code-highlight={yes,no} - Whether code should be highlighted (only if --color - is also enabled). Default: yes. + Whether code should be highlighted (only if --color is + also enabled). Default: yes. --pastebin=mode Send failed|all info to bpaste.net pastebin service --junit-xml=path Create junit-xml style report file at given path --junit-prefix=str Prepend prefix to classnames in junit-xml output pytest-warnings: -W PYTHONWARNINGS, --pythonwarnings=PYTHONWARNINGS - Set which warnings to report, see -W option of - Python itself + Set which warnings to report, see -W option of Python + itself --maxfail=num Exit after first num failures or errors --strict-config Any warnings encountered while parsing the `pytest` section of the configuration file raise errors - --strict-markers Markers not registered in the `markers` section of - the configuration file raise errors + --strict-markers Markers not registered in the `markers` section of the + configuration file raise errors --strict (Deprecated) alias to --strict-markers -c FILE, --config-file=FILE Load configuration from `FILE` instead of trying to locate one of the implicit configuration files. --continue-on-collection-errors Force test execution even if collection errors occur - --rootdir=ROOTDIR Define root directory for tests. Can be relative - path: 'root_dir', './root_dir', - 'root_dir/another_dir/'; absolute path: - '/home/user/root_dir'; path with variables: - '$HOME/root_dir'. + --rootdir=ROOTDIR Define root directory for tests. Can be relative path: + 'root_dir', './root_dir', 'root_dir/another_dir/'; + absolute path: '/home/user/root_dir'; path with + variables: '$HOME/root_dir'. collection: --collect-only, --co Only collect tests, don't execute them --pyargs Try to interpret all arguments as Python packages --ignore=path Ignore path during collection (multi-allowed) - --ignore-glob=path Ignore path pattern during collection (multi- - allowed) + --ignore-glob=path Ignore path pattern during collection (multi-allowed) --deselect=nodeid_prefix Deselect item (via node id prefix) during collection (multi-allowed) @@ -2064,8 +2061,8 @@ All the command-line flags can be obtained by running ``pytest --help``:: --collect-in-virtualenv Don't ignore tests in a local virtualenv directory --import-mode={prepend,append,importlib} - Prepend/append to sys.path when importing test - modules and conftest files. Default: prepend. + Prepend/append to sys.path when importing test modules + and conftest files. Default: prepend. --doctest-modules Run doctests in all .py modules --doctest-report={none,cdiff,ndiff,udiff,only_first_failure} Choose another output format for diffs on doctest @@ -2078,38 +2075,37 @@ All the command-line flags can be obtained by running ``pytest --help``:: failure test session debugging and configuration: - --basetemp=dir Base temporary directory for this test run. - (Warning: this directory is removed if it exists.) - -V, --version Display pytest version and information about - plugins. When given twice, also display information - about plugins. + --basetemp=dir Base temporary directory for this test run. (Warning: + this directory is removed if it exists.) + -V, --version Display pytest version and information about plugins. + When given twice, also display information about + plugins. -h, --help Show help message and configuration info -p name Early-load given plugin module name or entry point - (multi-allowed). To avoid loading of plugins, use - the `no:` prefix, e.g. `no:doctest`. + (multi-allowed). To avoid loading of plugins, use the + `no:` prefix, e.g. `no:doctest`. --trace-config Trace considerations of conftest.py files --debug=[DEBUG_FILE_NAME] Store internal tracing debug information in this log - file. This file is opened with 'w' and truncated as - a result, care advised. Default: pytestdebug.log. + file. This file is opened with 'w' and truncated as a + result, care advised. Default: pytestdebug.log. -o OVERRIDE_INI, --override-ini=OVERRIDE_INI - Override ini option with "option=value" style, e.g. - `-o xfail_strict=True -o cache_dir=cache`. + Override ini option with "option=value" style, e.g. `-o + xfail_strict=True -o cache_dir=cache`. --assert=MODE Control assertion debugging tools. 'plain' performs no assertion debugging. - 'rewrite' (the default) rewrites assert statements - in test modules on import to provide assert - expression information. + 'rewrite' (the default) rewrites assert statements in + test modules on import to provide assert expression + information. --setup-only Only setup fixtures, do not execute tests --setup-show Show setup of fixtures while executing tests - --setup-plan Show what fixtures and tests would be executed but - don't execute anything + --setup-plan Show what fixtures and tests would be executed but don't + execute anything logging: - --log-level=LEVEL Level of messages to catch/display. Not set by - default, so it depends on the root/parent log - handler's effective level, where it is "WARNING" by - default. + --log-level=LEVEL Level of messages to catch/display. Not set by default, + so it depends on the root/parent log handler's effective + level, where it is "WARNING" by default. --log-format=LOG_FORMAT Log format used by the logging module --log-date-format=LOG_DATE_FORMAT @@ -2133,8 +2129,13 @@ All the command-line flags can be obtained by running ``pytest --help``:: Auto-indent multiline messages passed to the logging module. Accepts true|on, false|off or an integer. --log-disable=LOGGER_DISABLE - Disable a logger by name. Can be passed multiple - times. + Disable a logger by name. Can be passed multiple times. + + Custom options: + --lsof Run FD checks if lsof is available + --runpytest={inprocess,subprocess} + Run pytest sub runs in tests using an 'inprocess' or + 'subprocess' (python -m main) method [pytest] ini-options in the first pytest.ini|tox.ini|setup.cfg|pyproject.toml file found: @@ -2149,37 +2150,33 @@ All the command-line flags can be obtained by running ``pytest --help``:: warnings.filterwarnings. Processed after -W/--pythonwarnings. consider_namespace_packages (bool): - Consider namespace packages when resolving module - names during import - usefixtures (args): List of default fixtures to be used with this - project + Consider namespace packages when resolving module names + during import + usefixtures (args): List of default fixtures to be used with this project python_files (args): Glob-style file patterns for Python test module discovery python_classes (args): - Prefixes or glob names for Python test class - discovery + Prefixes or glob names for Python test class discovery python_functions (args): Prefixes or glob names for Python test function and method discovery disable_test_id_escaping_and_forfeit_all_rights_to_community_support (bool): - Disable string escape non-ASCII characters, might - cause unwanted side effects(use at your own risk) + Disable string escape non-ASCII characters, might cause + unwanted side effects(use at your own risk) console_output_style (string): - Console output: "classic", or with additional - progress information ("progress" (percentage) | - "count" | "progress-even-when-capture-no" (forces - progress even when capture=no) + Console output: "classic", or with additional progress + information ("progress" (percentage) | "count" | + "progress-even-when-capture-no" (forces progress even + when capture=no) verbosity_test_cases (string): Specify a verbosity level for test case execution, - overriding the main level. Higher levels will - provide more detailed information about each test - case executed. - xfail_strict (bool): Default for the strict parameter of xfail markers - when not given explicitly (default: False) + overriding the main level. Higher levels will provide + more detailed information about each test case executed. + xfail_strict (bool): Default for the strict parameter of xfail markers when + not given explicitly (default: False) tmp_path_retention_count (string): How many sessions should we keep the `tmp_path` - directories, according to - `tmp_path_retention_policy`. + directories, according to `tmp_path_retention_policy`. tmp_path_retention_policy (string): Controls which directories created by the `tmp_path` fixture are kept around, based on test outcome. @@ -2188,9 +2185,9 @@ All the command-line flags can be obtained by running ``pytest --help``:: Enables the pytest_assertion_pass hook. Make sure to delete any previously generated pyc cache files. verbosity_assertions (string): - Specify a verbosity level for assertions, overriding - the main level. Higher levels will provide more - detailed explanation when an assertion fails. + Specify a verbosity level for assertions, overriding the + main level. Higher levels will provide more detailed + explanation when an assertion fails. junit_suite_name (string): Test suite name for JUnit report junit_logging (string): @@ -2212,8 +2209,8 @@ All the command-line flags can be obtained by running ``pytest --help``:: log_format (string): Default value for --log-format log_date_format (string): Default value for --log-date-format - log_cli (bool): Enable log display during test run (also known as - "live logging") + log_cli (bool): Enable log display during test run (also known as "live + logging") log_cli_level (string): Default value for --log-cli-level log_cli_format (string): @@ -2233,14 +2230,18 @@ All the command-line flags can be obtained by running ``pytest --help``:: Default value for --log-auto-indent pythonpath (paths): Add paths to sys.path faulthandler_timeout (string): - Dump the traceback of all threads if a test takes - more than TIMEOUT seconds to finish + Dump the traceback of all threads if a test takes more + than TIMEOUT seconds to finish addopts (args): Extra command line options minversion (string): Minimally required pytest version required_plugins (args): Plugins that must be present for pytest to run + pytester_example_dir (string): + Directory to take the pytester example files from Environment variables: + CI When set (regardless of value), pytest knows it is running in a CI process and does not truncate summary info + BUILD_NUMBER equivalent to CI PYTEST_ADDOPTS Extra command line options PYTEST_PLUGINS Comma-separated plugins to load during startup PYTEST_DISABLE_PLUGIN_AUTOLOAD Set to disable plugin auto-loading diff --git a/src/_pytest/helpconfig.py b/src/_pytest/helpconfig.py index fa7c322cf8e..bb8b27a2390 100644 --- a/src/_pytest/helpconfig.py +++ b/src/_pytest/helpconfig.py @@ -212,6 +212,12 @@ def showhelp(config: Config) -> None: tw.line() tw.line("Environment variables:") vars = [ + ( + "CI", + "When set (regardless of value), pytest knows it is running in a " + "CI process and does not truncate summary info", + ), + ("BUILD_NUMBER", "Equivalent to CI"), ("PYTEST_ADDOPTS", "Extra command line options"), ("PYTEST_PLUGINS", "Comma-separated plugins to load during startup"), ("PYTEST_DISABLE_PLUGIN_AUTOLOAD", "Set to disable plugin auto-loading"),