Skip to content

Commit

Permalink
[internal] Switch internal lockfile usage to Pex (#14771)
Browse files Browse the repository at this point in the history 
We start with only changing our internal lockfiles—not default tool lockfiles—to avoid risking impacting users in any way until we have a little more dogfooding.

## File name convention

We still need to figure out what we want to call lockfiles by defaults. Pants and Pex both don't care at all, but the convention we establish will be followed by many users.

For now, I go with `3rdparty/python/black.lockfile` to mirror what we do with JVM. I figure we can rename this once we settle on a convention before we possibly rename the default value of `[python].resolves` - Git can handle the file rename fine.

## Resolve benchmark

Found by `rm -rf ~/.pants/cache .pids`, then running `build-support/bin/generate_all_lockfiles.sh --internal` with `--no-dynamic-ui` set and comparing start vs completed log messages for each resolve.

| Resolve        | Poetry | Pex | Improvement |
|----------------|--------|-----|-------------|
| flake8         | 4.7    | 1.9 | 2.8         |
| mypy           | 3.1    | 1.8 | 1.3         |
| pytest         | 9.1    | 8.8 | 0.3         |
| python-default | 13.4   | 6.4 | 7.0         |

## Lockfile installation benchmark

Found by `rm -rf ~/.pants/cache .pids`, then running `./pants --no-dynamic-ui lint check test src/python/pants/util/strutil_test.py` and comparing start vs completed log messages for building each PEX.

| PEX                | Poetry | Pex | Improvement |
|--------------------|--------|-----|-------------|
| flake8.pex         | 3.5    | 2.4 | 1.1         |
| mypy.pex           | 7.1    | 6.1 | 1.0           |
| pytest.pex         | 6.6    | 3.9 | 2.7         |
| python_default.pex | 12.9   | 8.4 | 4.5         |

[ci skip-rust]
  • Loading branch information
@Eric-Arellano
Eric-Arellano authored Mar 16, 2022
1 parent 6e0ee75 commit 612bd84
Show file tree
Hide file tree
Showing 13 changed files with 3,340 additions and 667 deletions.
1 change: 1 addition & 0 deletions .github/workflows/test.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -282,6 +282,7 @@ jobs:
build_wheels_linux:
container: quay.io/pypa/manylinux2014_x86_64:latest
env:
PANTS_CA_CERTS_PATH: /opt/_internal/certs.pem
PANTS_REMOTE_CACHE_READ: 'false'
PANTS_REMOTE_CACHE_WRITE: 'false'
if: ${{ github.repository_owner == 'pantsbuild' }}
Expand Down
7 changes: 5 additions & 2 deletions 3rdparty/python/BUILD
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,11 @@
# Copyright 2014 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

# See requirements.txt in this directory to change deps.
python_requirements()
python_requirements(
overrides={
"humbug": {"dependencies": ["#setuptools"]},
},
)

# Useful when using IntelliJ/PyCharm remote debugging. Importing `pydevd_pycharm` at
# the breakpoint will cause dep inference to add this dep on the remote debugger client.
Expand Down
340 changes: 340 additions & 0 deletions 3rdparty/python/flake8.lockfile
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,340 @@
// This lockfile was autogenerated by Pants. To regenerate, run:
//
// build-support/bin/generate_all_lockfiles.sh
//
// --- BEGIN PANTS LOCKFILE METADATA: DO NOT EDIT OR REMOVE ---
// {
// "version": 2,
// "valid_for_interpreter_constraints": [
// "CPython<3.10,>=3.7"
// ],
// "generated_with_requirements": [
// "flake8-2020<1.7.0,>=1.6.0",
// "flake8-comprehensions<4.0,>=3.8.0",
// "flake8-no-implicit-concat",
// "flake8-pantsbuild<3,>=2.0",
// "flake8<4.0,>=3.9.2"
// ]
// }
// --- END PANTS LOCKFILE METADATA ---

{
"allow_builds": true,
"allow_prereleases": false,
"allow_wheels": true,
"build_isolation": true,
"constraints": [],
"locked_resolves": [
{
"locked_requirements": [
{
"artifacts": [
{
"algorithm": "sha256",
"hash": "bf8fd333346d844f616e8d47905ef3a3384edae6b4e9beb0c5101e25e3110907",
"url": "https://files.pythonhosted.org/packages/fc/80/35a0716e5d5101e643404dabd20f07f5528a21f3ef4032d31a49c913237b/flake8-3.9.2-py2.py3-none-any.whl"
},
{
"algorithm": "sha256",
"hash": "07528381786f2a6237b061f6e96610a4167b226cb926e2aa2b6b1d78057c576b",
"url": "https://files.pythonhosted.org/packages/9e/47/15b267dfe7e03dca4c4c06e7eadbd55ef4dfd368b13a0bab36d708b14366/flake8-3.9.2.tar.gz"
}
],
"project_name": "flake8",
"requires_dists": [
"configparser; python_version < \"3.2\"",
"enum34; python_version < \"3.4\"",
"functools32; python_version < \"3.2\"",
"importlib-metadata; python_version < \"3.8\"",
"mccabe<0.7.0,>=0.6.0",
"pycodestyle<2.8.0,>=2.7.0",
"pyflakes<2.4.0,>=2.3.0",
"typing; python_version < \"3.5\""
],
"requires_python": "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7",
"version": "3.9.2"
},
{
"artifacts": [
{
"algorithm": "sha256",
"hash": "efcc056fb723e1ea5307e3b663c7c328f1c23a5ff0a0fd3be695a918d8245c3a",
"url": "https://files.pythonhosted.org/packages/0e/f0/255eb37e80770b9123ae6519b442b6bf8e1769c57b7f7ecdb449d8c1c70f/flake8_2020-1.6.1-py2.py3-none-any.whl"
},
{
"algorithm": "sha256",
"hash": "db523e3383befc17c895219551ff6c9b2f6e0a5cae4c7739ea65a2238bdc6f74",
"url": "https://files.pythonhosted.org/packages/9c/06/c76162b9130320346b427120a247b8d1d08240e9b3190800818248c176a9/flake8_2020-1.6.1.tar.gz"
}
],
"project_name": "flake8-2020",
"requires_dists": [
"flake8>=3.7",
"importlib-metadata>=0.9; python_version < \"3.8\""
],
"requires_python": ">=3.6.1",
"version": "1.6.1"
},
{
"artifacts": [
{
"algorithm": "sha256",
"hash": "9406314803abe1193c064544ab14fdc43c58424c0882f6ff8a581eb73fc9bb58",
"url": "https://files.pythonhosted.org/packages/dc/65/5e9e256930fbe186074daf00d68e0cd17facfcb6c2523ade810ea9cd344f/flake8_comprehensions-3.8.0-py3-none-any.whl"
},
{
"algorithm": "sha256",
"hash": "8e108707637b1d13734f38e03435984f6b7854fa6b5a4e34f93e69534be8e521",
"url": "https://files.pythonhosted.org/packages/8c/56/ba0c2878c2b64237afd978b4f176bcb9d671419d9b71ece454f0117fb943/flake8-comprehensions-3.8.0.tar.gz"
}
],
"project_name": "flake8-comprehensions",
"requires_dists": [
"flake8!=3.2.0,>=3.0",
"importlib-metadata; python_version < \"3.8\""
],
"requires_python": ">=3.7",
"version": "3.8"
},
{
"artifacts": [
{
"algorithm": "sha256",
"hash": "ae5c17d0bce1a1c5f1117786c111616ffe9f81bf4fe98bcdd715d44a7371b370",
"url": "https://files.pythonhosted.org/packages/32/04/d09613b15ed7aa893886650376ee89306fbf0948ca240bd5c7bdd1cb5366/flake8_no_implicit_concat-0.3.3-py3-none-any.whl"
},
{
"algorithm": "sha256",
"hash": "b68ff39c5620b0c9fd412c22e6dd98e56ca255eb9cc007135dbc26033dc5e07d",
"url": "https://files.pythonhosted.org/packages/f3/a9/1621072f5d24aaca00a7df5c966414685686a54d496c8b717f344949c1cf/flake8-no-implicit-concat-0.3.3.tar.gz"
}
],
"project_name": "flake8-no-implicit-concat",
"requires_dists": [
"black; extra == \"dev\"",
"codecov; extra == \"dev\"",
"coverage; extra == \"dev\"",
"darglint; extra == \"dev\"",
"flake8",
"flake8-2020; extra == \"dev\"",
"flake8-black; extra == \"dev\"",
"flake8-broken-line; extra == \"dev\"",
"flake8-builtins; extra == \"dev\"",
"flake8-docstrings; extra == \"dev\"",
"flake8-isort; extra == \"dev\"",
"flake8-rst-docstrings; extra == \"dev\"",
"flake8; extra == \"dev\"",
"hacking>=4; extra == \"dev\"",
"isort; extra == \"dev\"",
"more-itertools; python_version < \"3.10\"",
"mypy; extra == \"dev\"",
"pep8-naming; extra == \"dev\"",
"typing; python_version < \"3.5\""
],
"requires_python": ">=3.3",
"version": "0.3.3"
},
{
"artifacts": [
{
"algorithm": "sha256",
"hash": "8d45cf26a55fe7d66de944a6c93b64456852cfd3a7bea81640553081469129e7",
"url": "https://files.pythonhosted.org/packages/ac/bd/96927daa8c58b14da8c6c4809c028d5a21eb9cb4e0cf887447c45898a4df/flake8_pantsbuild-2.0.0-py3-none-any.whl"
},
{
"algorithm": "sha256",
"hash": "1b72558db6d718c33f4410eff80f7afc2bc0300190a733d92b3cf05b231c2450",
"url": "https://files.pythonhosted.org/packages/41/a7/c8f796a5856e019e030035c0fc12d561ae27638b38c49a7128a8956f418a/flake8-pantsbuild-2.0.0.tar.gz"
}
],
"project_name": "flake8-pantsbuild",
"requires_dists": [
"flake8>=3.7",
"importlib_metadata>=1.3.0; python_version < \"3.8\""
],
"requires_python": ">=3.6",
"version": "2"
},
{
"artifacts": [
{
"algorithm": "sha256",
"hash": "1208431ca90a8cca1a6b8af391bb53c1a2db74e5d1cef6ddced95d4b2062edc6",
"url": "https://files.pythonhosted.org/packages/92/f2/c48787ca7d1e20daa185e1b6b2d4e16acd2fb5e0320bc50ffc89b91fa4d7/importlib_metadata-4.11.3-py3-none-any.whl"
},
{
"algorithm": "sha256",
"hash": "ea4c597ebf37142f827b8f39299579e31685c31d3a438b59f469406afd0f2539",
"url": "https://files.pythonhosted.org/packages/3e/1d/964b27278cfa369fbe9041f604ab09c6e99556f8b7910781b4584b428c2f/importlib_metadata-4.11.3.tar.gz"
}
],
"project_name": "importlib-metadata",
"requires_dists": [
"flufl.flake8; extra == \"testing\"",
"importlib-resources>=1.3; python_version < \"3.9\" and extra == \"testing\"",
"ipython; extra == \"perf\"",
"jaraco.packaging>=9; extra == \"docs\"",
"packaging; extra == \"testing\"",
"pyfakefs; extra == \"testing\"",
"pytest-black>=0.3.7; platform_python_implementation != \"PyPy\" and extra == \"testing\"",
"pytest-checkdocs>=2.4; extra == \"testing\"",
"pytest-cov; extra == \"testing\"",
"pytest-enabler>=1.0.1; extra == \"testing\"",
"pytest-flake8; extra == \"testing\"",
"pytest-mypy>=0.9.1; platform_python_implementation != \"PyPy\" and extra == \"testing\"",
"pytest-perf>=0.9.2; extra == \"testing\"",
"pytest>=6; extra == \"testing\"",
"rst.linker>=1.9; extra == \"docs\"",
"sphinx; extra == \"docs\"",
"typing-extensions>=3.6.4; python_version < \"3.8\"",
"zipp>=0.5"
],
"requires_python": ">=3.7",
"version": "4.11.3"
},
{
"artifacts": [
{
"algorithm": "sha256",
"hash": "ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42",
"url": "https://files.pythonhosted.org/packages/87/89/479dc97e18549e21354893e4ee4ef36db1d237534982482c3681ee6e7b57/mccabe-0.6.1-py2.py3-none-any.whl"
},
{
"algorithm": "sha256",
"hash": "dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f",
"url": "https://files.pythonhosted.org/packages/06/18/fa675aa501e11d6d6ca0ae73a101b2f3571a565e0f7d38e062eec18a91ee/mccabe-0.6.1.tar.gz"
}
],
"project_name": "mccabe",
"requires_dists": [],
"requires_python": null,
"version": "0.6.1"
},
{
"artifacts": [
{
"algorithm": "sha256",
"hash": "43e6dd9942dffd72661a2c4ef383ad7da1e6a3e968a927ad7a6083ab410a688b",
"url": "https://files.pythonhosted.org/packages/e5/c3/48e2c81038f57e8caab9a6e6fb6c2fc23536c59b092abefc447e6b5d1903/more_itertools-8.12.0-py3-none-any.whl"
},
{
"algorithm": "sha256",
"hash": "7dc6ad46f05f545f900dd59e8dfb4e84a4827b97b3cfecb175ea0c7d247f6064",
"url": "https://files.pythonhosted.org/packages/dc/b5/c216ffeace7b89b7387fe08e1b39a07c6da38ea82c60e2e630dd5883813b/more-itertools-8.12.0.tar.gz"
}
],
"project_name": "more-itertools",
"requires_dists": [],
"requires_python": ">=3.5",
"version": "8.12"
},
{
"artifacts": [
{
"algorithm": "sha256",
"hash": "514f76d918fcc0b55c6680472f0a37970994e07bbb80725808c17089be302068",
"url": "https://files.pythonhosted.org/packages/de/cc/227251b1471f129bc35e966bb0fceb005969023926d744139642d847b7ae/pycodestyle-2.7.0-py2.py3-none-any.whl"
},
{
"algorithm": "sha256",
"hash": "c389c1d06bf7904078ca03399a4816f974a1d590090fecea0c63ec26ebaf1cef",
"url": "https://files.pythonhosted.org/packages/02/b3/c832123f2699892c715fcdfebb1a8fdeffa11bb7b2350e46ecdd76b45a20/pycodestyle-2.7.0.tar.gz"
}
],
"project_name": "pycodestyle",
"requires_dists": [],
"requires_python": "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7",
"version": "2.7"
},
{
"artifacts": [
{
"algorithm": "sha256",
"hash": "7893783d01b8a89811dd72d7dfd4d84ff098e5eed95cfa8905b22bbffe52efc3",
"url": "https://files.pythonhosted.org/packages/6c/11/2a745612f1d3cbbd9c69ba14b1b43a35a2f5c3c81cd0124508c52c64307f/pyflakes-2.3.1-py2.py3-none-any.whl"
},
{
"algorithm": "sha256",
"hash": "f5bc8ecabc05bb9d291eb5203d6810b49040f6ff446a756326104746cc00c1db",
"url": "https://files.pythonhosted.org/packages/a8/0f/0dc480da9162749bf629dca76570972dd9cce5bedc60196a3c912875c87d/pyflakes-2.3.1.tar.gz"
}
],
"project_name": "pyflakes",
"requires_dists": [],
"requires_python": "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7",
"version": "2.3.1"
},
{
"artifacts": [
{
"algorithm": "sha256",
"hash": "21c85e0fe4b9a155d0799430b0ad741cdce7e359660ccbd8b530613e8df88ce2",
"url": "https://files.pythonhosted.org/packages/45/6b/44f7f8f1e110027cf88956b59f2fad776cca7e1704396d043f89effd3a0e/typing_extensions-4.1.1-py3-none-any.whl"
},
{
"algorithm": "sha256",
"hash": "1a9462dcc3347a79b1f1c0271fbe79e844580bb598bafa1ed208b94da3cdcd42",
"url": "https://files.pythonhosted.org/packages/b1/5a/8b5fbb891ef3f81fc923bf3cb4a578c0abf9471eb50ce0f51c74212182ab/typing_extensions-4.1.1.tar.gz"
}
],
"project_name": "typing-extensions",
"requires_dists": [],
"requires_python": ">=3.6",
"version": "4.1.1"
},
{
"artifacts": [
{
"algorithm": "sha256",
"hash": "b47250dd24f92b7dd6a0a8fc5244da14608f3ca90a5efcd37a3b1642fac9a375",
"url": "https://files.pythonhosted.org/packages/52/c5/df7953fe6065185af5956265e3b16f13c2826c2b1ba23d43154f3af453bc/zipp-3.7.0-py3-none-any.whl"
},
{
"algorithm": "sha256",
"hash": "9f50f446828eb9d45b267433fd3e9da8d801f614129124863f9c51ebceafb87d",
"url": "https://files.pythonhosted.org/packages/94/64/3115548d41cb001378099cb4fc6a6889c64ef43ac1b0e68c9e80b55884fa/zipp-3.7.0.tar.gz"
}
],
"project_name": "zipp",
"requires_dists": [
"func-timeout; extra == \"testing\"",
"jaraco.itertools; extra == \"testing\"",
"jaraco.packaging>=8.2; extra == \"docs\"",
"pytest-black>=0.3.7; platform_python_implementation != \"PyPy\" and extra == \"testing\"",
"pytest-checkdocs>=2.4; extra == \"testing\"",
"pytest-cov; extra == \"testing\"",
"pytest-enabler>=1.0.1; extra == \"testing\"",
"pytest-flake8; extra == \"testing\"",
"pytest-mypy; platform_python_implementation != \"PyPy\" and extra == \"testing\"",
"pytest>=6; extra == \"testing\"",
"rst.linker>=1.9; extra == \"docs\"",
"sphinx; extra == \"docs\""
],
"requires_python": ">=3.7",
"version": "3.7"
}
],
"platform_tag": [
"cp39",
"cp39",
"macosx_11_0_arm64"
]
}
],
"pex_version": "2.1.72",
"prefer_older_binary": false,
"requirements": [
"flake8-2020<1.7.0,>=1.6.0",
"flake8-comprehensions<4.0,>=3.8.0",
"flake8-no-implicit-concat",
"flake8-pantsbuild<3,>=2.0",
"flake8<4.0,>=3.9.2"
],
"requires_python": [
"<3.10,>=3.7"
],
"resolver_version": "pip-2020-resolver",
"style": "universal",
"transitive": true,
"use_pep517": null
}
Loading

0 comments on commit 612bd84

Please sign in to comment.