From e044fe84d85eee9e1c0ea014fb403efc963cbbbe Mon Sep 17 00:00:00 2001 From: Aaron Miller Date: Thu, 16 May 2024 23:51:51 -0700 Subject: [PATCH] [SymForce] Use uv for requirements Topic: sf-uv --- .github/workflows/solve_requirements.yml | 44 +++++ ...uirements.txt => dev_requirements_py38.txt | 152 ++++++++---------- test/symforce_requirements_test.py | 35 ++-- 3 files changed, 121 insertions(+), 110 deletions(-) create mode 100644 .github/workflows/solve_requirements.yml rename dev_requirements.txt => dev_requirements_py38.txt (72%) diff --git a/.github/workflows/solve_requirements.yml b/.github/workflows/solve_requirements.yml new file mode 100644 index 000000000..c049d2732 --- /dev/null +++ b/.github/workflows/solve_requirements.yml @@ -0,0 +1,44 @@ +name: Solve Pip Requirements + +on: + workflow_dispatch: + +jobs: + solve: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + py_version: [8, 9, 10, 11, 12] + steps: + - name: Checkout + uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: '3.${{ matrix.py_version }}' + + - run: | + pip install uv + uv pip install -r dev_requirements_py3${{ matrix.py_version }}.txt + PYTHONPATH=$(pwd) python test/symforce_requirements_test.py --update --piptools_upgrade + + - uses: actions/upload-artifact@v4 + with: + path: dev_requirements_py3${{ matrix.py_version }}.txt + + publish: + runs-on: ubuntu-latest + needs: solve + steps: + - uses: actions/checkout@v4 + + - uses: actions/download-artifact@v4 + with: + merge-multiple: true + + - uses: peter-evans/create-pull-request@v6 + with: + commit-message: Update Requirements + title: Update Requirements + body: "" \ No newline at end of file diff --git a/dev_requirements.txt b/dev_requirements_py38.txt similarity index 72% rename from dev_requirements.txt rename to dev_requirements_py38.txt index bfe9c388c..c0b842474 100644 --- a/dev_requirements.txt +++ b/dev_requirements_py38.txt @@ -1,21 +1,12 @@ -# -# This file is autogenerated by pip-compile with Python 3.8 -# by the following command: -# -# python test/symforce_requirements_test.py --update -# ---index-url https://pypi.python.org/simple - -# Create a requirement incompatible with python < 3.8 -symforce_requires_python_38_or_higher___your_python_version_is_incompatible; python_version < '3.8' - +# This file was autogenerated by uv via the following command: +# uv pip compile --all-extras --output-file=dev_requirements_py38.txt pyproject.toml alabaster==0.7.13 # via sphinx argh==0.31.2 # via + # symforce (pyproject.toml) # skymarshal - # symforce (setup.py) -astroid==3.1.0 +astroid==3.2.1 # via pylint asttokens==2.4.1 # via stack-data @@ -23,7 +14,7 @@ attrs==23.2.0 # via # jsonschema # referencing -babel==2.14.0 +babel==2.15.0 # via sphinx backcall==0.2.0 # via ipython @@ -34,29 +25,25 @@ beautifulsoup4==4.12.3 bleach==6.1.0 # via nbconvert breathe==4.35.0 - # via symforce (setup.py) + # via symforce (pyproject.toml) build==1.2.1 # via pip-tools certifi==2024.2.2 # via requests charset-normalizer==3.3.2 # via requests -clang-format==18.1.4 - # via symforce (setup.py) +clang-format==18.1.5 + # via symforce (pyproject.toml) click==8.1.7 # via pip-tools -cmake==3.26.4 - # via symforce (setup.py) comm==0.2.2 # via ipykernel contourpy==1.1.1 # via matplotlib -coverage==7.5.0 - # via symforce (setup.py) +coverage==7.5.1 + # via symforce (pyproject.toml) cycler==0.12.1 # via matplotlib -cython==0.29.37 - # via symforce (setup.py) debugpy==1.8.1 # via ipykernel decorator==5.1.1 @@ -77,10 +64,10 @@ fastjsonschema==2.19.1 # via nbformat fonttools==4.51.0 # via matplotlib -furo==2024.1.29 - # via symforce (setup.py) +furo==2024.5.6 + # via symforce (pyproject.toml) graphviz==0.20.3 - # via symforce (setup.py) + # via symforce (pyproject.toml) idna==3.7 # via requests imagesize==1.4.1 @@ -98,24 +85,24 @@ importlib-resources==6.4.0 # jsonschema-specifications # matplotlib ipykernel==6.29.4 - # via symforce (setup.py) + # via symforce (pyproject.toml) ipython==8.12.3 # via ipykernel ipython-genutils==0.2.0 - # via symforce (setup.py) + # via symforce (pyproject.toml) isort==5.13.2 # via pylint jedi==0.19.1 # via ipython -jinja2==3.1.3 +jinja2==3.1.4 # via + # symforce (pyproject.toml) # myst-parser # nbconvert # nbsphinx # skymarshal # sphinx - # symforce (setup.py) -jsonschema==4.21.1 +jsonschema==4.22.0 # via nbformat jsonschema-specifications==2023.12.1 # via jsonschema @@ -135,7 +122,7 @@ jupyterlab-pygments==0.3.0 kiwisolver==1.4.5 # via matplotlib lazy-object-proxy==1.10.0 - # via symforce (setup.py) + # via symforce (pyproject.toml) llvmlite==0.41.1 # via numba markdown-it-py==3.0.0 @@ -147,14 +134,14 @@ markupsafe==2.1.5 # jinja2 # nbconvert matplotlib==3.7.5 - # via symforce (setup.py) + # via symforce (pyproject.toml) matplotlib-inline==0.1.7 # via # ipykernel # ipython mccabe==0.7.0 # via pylint -mdit-py-plugins==0.4.0 +mdit-py-plugins==0.4.1 # via myst-parser mdurl==0.1.2 # via markdown-it-py @@ -163,14 +150,14 @@ mistune==3.0.2 mpmath==1.3.0 # via sympy mypy==1.8.0 - # via symforce (setup.py) + # via symforce (pyproject.toml) mypy-extensions==1.0.0 # via mypy -myst-parser==3.0.0 - # via symforce (setup.py) +myst-parser==3.0.1 + # via symforce (pyproject.toml) nbclient==0.10.0 # via nbconvert -nbconvert==7.16.3 +nbconvert==7.16.4 # via nbsphinx nbformat==5.10.4 # via @@ -178,23 +165,23 @@ nbformat==5.10.4 # nbconvert # nbsphinx # nbstripout -nbsphinx==0.9.3 - # via symforce (setup.py) +nbsphinx==0.9.4 + # via symforce (pyproject.toml) nbstripout==0.7.1 - # via symforce (setup.py) + # via symforce (pyproject.toml) nest-asyncio==1.6.0 # via ipykernel numba==0.58.1 - # via symforce (setup.py) + # via symforce (pyproject.toml) numpy==1.24.4 # via + # symforce (pyproject.toml) # contourpy # matplotlib # numba # pandas # scipy # skymarshal - # symforce (setup.py) # symforce-sym packaging==24.0 # via @@ -205,7 +192,7 @@ packaging==24.0 # plotly # sphinx pandas==2.0.3 - # via symforce (setup.py) + # via symforce (pyproject.toml) pandocfilters==1.5.1 # via nbconvert parso==0.8.4 @@ -216,16 +203,18 @@ pickleshare==0.7.5 # via ipython pillow==10.3.0 # via matplotlib +pip==24.0 + # via pip-tools pip-tools==7.4.1 - # via symforce (setup.py) + # via symforce (pyproject.toml) pkgutil-resolve-name==1.3.10 # via jsonschema -platformdirs==4.2.1 +platformdirs==4.2.2 # via # jupyter-core # pylint -plotly==5.21.0 - # via symforce (setup.py) +plotly==5.22.0 + # via symforce (pyproject.toml) ply==3.11 # via skymarshal prompt-toolkit==3.0.43 @@ -237,18 +226,18 @@ ptyprocess==0.7.0 pure-eval==0.2.2 # via stack-data pybind11-stubgen==0.16.2 - # via symforce (setup.py) -pygments==2.17.2 + # via symforce (pyproject.toml) +pygments==2.18.0 # via # furo # ipython # nbconvert # sphinx -pylint==3.1.0 - # via symforce (setup.py) +pylint==3.2.0 + # via symforce (pyproject.toml) pyparsing==3.1.2 # via matplotlib -pyproject-hooks==1.0.0 +pyproject-hooks==1.1.0 # via # build # pip-tools @@ -263,37 +252,40 @@ pytz==2024.1 # pandas pyyaml==6.0.1 # via myst-parser -pyzmq==26.0.2 +pyzmq==26.0.3 # via # ipykernel # jupyter-client -referencing==0.34.0 +referencing==0.35.1 # via # jsonschema # jsonschema-specifications requests==2.31.0 # via sphinx -rpds-py==0.18.0 +rpds-py==0.18.1 # via # jsonschema # referencing ruff==0.3.7 - # via symforce (setup.py) + # via symforce (pyproject.toml) scipy==1.10.1 - # via symforce (setup.py) + # via symforce (pyproject.toml) +setuptools==69.5.1 + # via pip-tools six==1.16.0 # via # asttokens # bleach # python-dateutil -file:./third_party/skymarshal - # via symforce (setup.py) +skymarshal @ file:///home/aaron/symforce/third_party/skymarshal + # via symforce (pyproject.toml) snowballstemmer==2.2.0 # via sphinx soupsieve==2.5 # via beautifulsoup4 sphinx==7.1.2 # via + # symforce (pyproject.toml) # breathe # furo # myst-parser @@ -301,11 +293,10 @@ sphinx==7.1.2 # sphinx-basic-ng # sphinx-copybutton # sphinxext-opengraph - # symforce (setup.py) sphinx-basic-ng==1.0.0b2 # via furo sphinx-copybutton==0.5.2 - # via symforce (setup.py) + # via symforce (pyproject.toml) sphinxcontrib-applehelp==1.0.4 # via sphinx sphinxcontrib-devhelp==1.0.2 @@ -319,14 +310,14 @@ sphinxcontrib-qthelp==1.0.3 sphinxcontrib-serializinghtml==1.1.5 # via sphinx sphinxext-opengraph==0.9.1 - # via symforce (setup.py) + # via symforce (pyproject.toml) stack-data==0.6.3 # via ipython -file:./gen/python - # via symforce (setup.py) +symforce-sym @ file:///home/aaron/symforce/gen/python + # via symforce (pyproject.toml) sympy==1.11.1 - # via symforce (setup.py) -tenacity==8.2.3 + # via symforce (pyproject.toml) +tenacity==8.3.0 # via plotly tinycss2==1.3.0 # via nbconvert @@ -336,8 +327,7 @@ tomli==2.0.1 # mypy # pip-tools # pylint - # pyproject-hooks -tomlkit==0.12.4 +tomlkit==0.12.5 # via pylint tornado==6.4 # via @@ -356,13 +346,13 @@ traitlets==5.14.3 # nbformat # nbsphinx types-jinja2==2.11.9 - # via symforce (setup.py) + # via symforce (pyproject.toml) types-markupsafe==1.1.10 # via types-jinja2 types-requests==2.31.0.20240406 - # via symforce (setup.py) -types-setuptools==69.5.0.20240423 - # via symforce (setup.py) + # via symforce (pyproject.toml) +types-setuptools==69.5.0.20240513 + # via symforce (pyproject.toml) typing-extensions==4.11.0 # via # astroid @@ -382,20 +372,8 @@ webencodings==0.5.1 # bleach # tinycss2 wheel==0.43.0 - # via - # pip-tools - # symforce (setup.py) -zipp==3.18.1 + # via pip-tools +zipp==3.18.2 # via # importlib-metadata # importlib-resources - -# The following packages are considered to be unsafe in a requirements file: -pip==24.0 - # via - # pip-tools - # symforce (setup.py) -setuptools==69.5.1 - # via - # pip-tools - # symforce (setup.py) diff --git a/test/symforce_requirements_test.py b/test/symforce_requirements_test.py index 65098ff6b..cafb4680b 100644 --- a/test/symforce_requirements_test.py +++ b/test/symforce_requirements_test.py @@ -7,13 +7,12 @@ import os import re import sys -import textwrap from symforce import path_util from symforce import python_util from symforce import typing as T from symforce.test_util import TestCase -from symforce.test_util import symengine_only +from symforce.test_util import sympy_only class SymforceRequirementsTest(TestCase): @@ -35,12 +34,14 @@ def main(*args: T.Any, **kwargs: T.Any) -> None: TestCase.main(*args, **kwargs) - @symengine_only + @sympy_only def test_requirements(self) -> None: output_dir = self.make_output_dir("sf_requirements_test_") - output_requirements_file = output_dir / "dev_requirements.txt" - symforce_requirements_file = path_util.symforce_root() / "dev_requirements.txt" + version = sys.version_info.minor + + output_requirements_file = output_dir / f"dev_requirements_py3{version}.txt" + symforce_requirements_file = path_util.symforce_root() / f"dev_requirements_py3{version}.txt" local_requirements_map = { "skymarshal @ file://localhost/{}/third_party/skymarshal": "file:./third_party/skymarshal", @@ -68,15 +69,13 @@ def test_requirements(self) -> None: asyncio.run( python_util.execute_subprocess( [ - sys.executable, - "-m", - "piptools", + "uv", + "pip", "compile", + "--all-extras", f"--output-file={output_requirements_file}", "--index-url=https://pypi.python.org/simple", - "--allow-unsafe", - "--extra=dev", - "--extra=_setup", + "pyproject.toml", ] + maybe_piptools_upgrade, cwd=path_util.symforce_root(), @@ -93,22 +92,12 @@ def test_requirements(self) -> None: for key, value in local_requirements_map.items(): requirements_contents = re.sub(key.format(".*"), value, requirements_contents) - # Inject the python version requirement - sentinel = "--index-url https://pypi.python.org/simple\n" - version_requirement = textwrap.dedent( - """ - # Create a requirement incompatible with python < 3.8 - symforce_requires_python_38_or_higher___your_python_version_is_incompatible; python_version < '3.8' - """ - ) - requirements_contents = requirements_contents.replace( - sentinel, sentinel + version_requirement - ) + requirements_contents = requirements_contents.replace(str(output_dir) + "/", "") output_requirements_file.write_text(requirements_contents) self.compare_or_update_file( - path_util.symforce_data_root() / "dev_requirements.txt", output_requirements_file + path_util.symforce_data_root() / f"dev_requirements_py3{version}.txt", output_requirements_file )