From 778046426eb0f2101414805f5beb4843b4f5c0aa Mon Sep 17 00:00:00 2001 From: Patrick Dallaire Date: Thu, 23 Dec 2021 23:08:01 -0500 Subject: [PATCH] Add automatic expansion of --requirement list Related to issue kivy/python-for-android#2529 --- pythonforandroid/toolchain.py | 138 +++++++++++++++++++++++++++++++--- setup.py | 2 +- tests/test_toolchain.py | 88 ++++++++++++++++++++++ 3 files changed, 218 insertions(+), 10 deletions(-) diff --git a/pythonforandroid/toolchain.py b/pythonforandroid/toolchain.py index aa242a4170..9266645e9b 100644 --- a/pythonforandroid/toolchain.py +++ b/pythonforandroid/toolchain.py @@ -5,7 +5,6 @@ This module defines the entry point for command line and programmatic use. """ - from os import environ from pythonforandroid import __version__ from pythonforandroid.pythonpackage import get_dep_names_of_package @@ -651,9 +650,12 @@ def add_parser(subparsers, *args, **kwargs): "pyproject.toml"))): have_setup_py_or_similar = True - # Process requirements and put version in environ - if hasattr(args, 'requirements'): - requirements = [] + # Process requirements and put version in environ: + if hasattr(args, 'requirements') and args.requirements: + all_recipes = [ + recipe.lower() for recipe in + set(Recipe.list_recipes(self.ctx)) + ] # Add dependencies from setup.py, but only if they are recipes # (because otherwise, setup.py itself will install them later) @@ -672,10 +674,6 @@ def add_parser(subparsers, *args, **kwargs): ) ] info("Dependencies obtained: " + str(dependencies)) - all_recipes = [ - recipe.lower() for recipe in - set(Recipe.list_recipes(self.ctx)) - ] dependencies = set(dependencies).intersection( set(all_recipes) ) @@ -691,7 +689,126 @@ def add_parser(subparsers, *args, **kwargs): "package? Will continue WITHOUT setup.py deps." ) - # Parse --requirements argument list: + non_recipe_requirements = [] + for requirement in args.requirements.split(','): + requirement_name = re.sub(r'==\d+(\.\d+)*', '', requirement) + if requirement_name not in all_recipes: + non_recipe_requirements.append(requirement) + args.requirements = re.sub( + r',?{}'.format(requirement), '', args.requirements) + + # Compile "non-recipe" requirements' dependencies and add to list. + # Otherwise, only recipe requirements' dependencies get installed. + # More info https://github.com/kivy/python-for-android/issues/2529 + if non_recipe_requirements: + info("Compiling dependencies for: " + "{}".format(non_recipe_requirements)) + + output = shprint( + sh.bash, '-c', + "echo -e '{}' > requirements.in && " + "pip-compile -v --dry-run --annotation-style=line && " + "rm requirements.in".format( + '\n'.join(non_recipe_requirements))) + + # Parse pip-compile output + parsed_requirement_info_list = [] + for line in output.splitlines(): + match_data = re.match( + r'^([\w.-]+)==(\d+(\.\d+)*).*' + r'#\s+via\s+([\w\s,.-]+)', line) + + if match_data: + parent_requirements = match_data.group(4).split(', ') + requirement_name = match_data.group(1) + requirement_version = match_data.group(2) + + # Requirement is a "non-recipe" one we started with. + if '-r requirements.in' in parent_requirements: + parent_requirements.remove('-r requirements.in') + + parsed_requirement_info_list.append([ + requirement_name, + requirement_version, + parent_requirements]) + + info("Requirements obtained from pip-compile: " + "{}".format(["{}=={}".format(x[0], x[1]) + for x in parsed_requirement_info_list])) + + # Remove indirect requirements ultimately installed by a recipe + original_parsed_requirement_count = -1 + while len(parsed_requirement_info_list) != \ + original_parsed_requirement_count: + + original_parsed_requirement_count = \ + len(parsed_requirement_info_list) + + for i, parsed_requirement_info in \ + enumerate(reversed(parsed_requirement_info_list)): + + index = original_parsed_requirement_count - i - 1 + requirement_name, requirement_version, \ + parent_requirements = parsed_requirement_info + + # If any parent requirement has a recipe, this + # requirement ought also to be installed by it. + # Hence, it's better not to add this requirement the + # expanded list. + parent_requirements_with_recipe = list( + set(parent_requirements).intersection( + set(all_recipes))) + + # Any parent requirement removed for the expanded list + # implies that it and its own requirements (including + # this requirement) will be installed by a recipe. + # Hence, it's better not to add this requirement the + # expanded list. + requirement_name_list = \ + [x[0] for x in parsed_requirement_info_list] + parent_requirements_still_in_list = list( + set(parent_requirements).intersection( + set(requirement_name_list))) + + is_ultimately_installed_by_a_recipe = \ + len(parent_requirements) and \ + (parent_requirements_with_recipe or + len(parent_requirements_still_in_list) != + len(parent_requirements)) + + if is_ultimately_installed_by_a_recipe: + info( + '{} will be installed by a recipe. Removing ' + 'it from requirement list expansion.'.format( + requirement_name)) + del parsed_requirement_info_list[index] + + for parsed_requirement_info in parsed_requirement_info_list: + requirement_name, requirement_version, \ + parent_requirements = parsed_requirement_info + + # If the requirement has a recipe, don't use specific + # version constraints determined by pip-compile. Some + # recipes may not support the specified version. Therefor, + # it's probably safer to just let them use their default + # version. User can still force the usage of specific + # version by explicitly declaring it with --requirements. + requirement_has_recipe = requirement_name in all_recipes + requirement_str = \ + requirement_name if requirement_has_recipe else \ + '{}=={}'.format(requirement_name, requirement_version) + + requirement_names_arg = re.sub( + r'==\d+(\.\d+)*', '', args.requirements).split(',') + + # This expansion was carried out based on "non-recipe" + # requirements. Hence,the counter-part, requirements + # with a recipe, may already be part of list. + if requirement_name not in requirement_names_arg: + args.requirements += ',' + requirement_str + + # Handle specific version requirement constraints (e.g. foo==x.y) + requirements = [] for requirement in split_argument_list(args.requirements): if "==" in requirement: requirement, version = requirement.split(u"==", 1) @@ -701,6 +818,9 @@ def add_parser(subparsers, *args, **kwargs): requirements.append(requirement) args.requirements = u",".join(requirements) + info('Expanded Requirements List: ' + '{}'.format(args.requirements.split(','))) + self.warn_on_deprecated_args(args) self.storage_dir = args.storage_dir diff --git a/setup.py b/setup.py index ef9ee0cfb2..c200027801 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ install_reqs = [ 'appdirs', 'colorama>=0.3.3', 'jinja2', 'six', 'enum34; python_version<"3.4"', 'sh>=1.10; sys_platform!="nt"', - 'pep517<0.7.0', 'toml', + 'pep517<0.7.0', 'toml', 'pip-tools' ] # (pep517 and toml are used by pythonpackage.py) diff --git a/tests/test_toolchain.py b/tests/test_toolchain.py index d7c73c6319..10ad5bb0ef 100644 --- a/tests/test_toolchain.py +++ b/tests/test_toolchain.py @@ -1,4 +1,5 @@ import io +import os import sys import pytest from unittest import mock @@ -124,6 +125,93 @@ def test_create_no_sdk_dir(self): assert ex_info.value.message == ( 'Android SDK dir was not specified, exiting.') + def test_create_with_complex_requirements(self): + requirements = [ + 'python3==3.8.10', # direct requirement with recipe using version constraint + 'pandas', # direct requirement with recipe (no version constraint) + 'mfpymake==1.2.2', # direct requirement without recipe using version constraint + 'telenium', # direct requirement without recipe (no version constraint) + 'numpy==1.21.4', # indirect requirement with recipe using version constraint + 'mako==1.1.5', # indirect requirement without recipe using version constraint + # There's no reason to specify an indirect requirement unless we want to install a specific version. + ] + argv = [ + 'toolchain.py', + 'create', + '--sdk-dir=/tmp/android-sdk', + '--ndk-dir=/tmp/android-ndk', + '--bootstrap=service_only', + '--requirements={}'.format(','.join(requirements)), + '--dist-name=test_toolchain', + '--activity-class-name=abc.myapp.android.CustomPythonActivity', + '--service-class-name=xyz.myapp.android.CustomPythonService', + ] + with patch_sys_argv(argv), mock.patch( + 'pythonforandroid.build.get_available_apis' + ) as m_get_available_apis, mock.patch( + 'pythonforandroid.build.get_toolchain_versions' + ) as m_get_toolchain_versions, mock.patch( + 'pythonforandroid.build.get_ndk_platform_dir' + ) as m_get_ndk_platform_dir, mock.patch( + 'pythonforandroid.toolchain.build_recipes' + ) as m_build_recipes, mock.patch( + 'pythonforandroid.bootstraps.service_only.' + 'ServiceOnlyBootstrap.assemble_distribution' + ) as m_run_distribute: + m_get_available_apis.return_value = [27] + m_get_toolchain_versions.return_value = (['4.9'], True) + m_get_ndk_platform_dir.return_value = ( + '/tmp/android-ndk/platforms/android-21/arch-arm', True) + tchain = ToolchainCL() + assert tchain.ctx.activity_class_name == 'abc.myapp.android.CustomPythonActivity' + assert tchain.ctx.service_class_name == 'xyz.myapp.android.CustomPythonService' + assert m_get_available_apis.call_args_list in [ + [mock.call('/tmp/android-sdk')], # linux case + [mock.call('/private/tmp/android-sdk')] # macos case + ] + assert m_get_toolchain_versions.call_args_list in [ + [mock.call('/tmp/android-ndk', mock.ANY)], # linux case + [mock.call('/private/tmp/android-ndk', mock.ANY)], # macos case + ] + build_order = [ + 'android', 'cython', 'genericndkbuild', 'hostpython3', 'libbz2', + 'libffi', 'liblzma', 'numpy', 'openssl', 'pandas', 'pyjnius', + 'python3', 'pytz', 'setuptools', 'six', 'sqlite3' + ] + python_modules = [ + 'certifi', 'charset-normalizer', 'cheroot', 'cherrypy', + 'idna', 'importlib-resources', 'jaraco.classes', + 'jaraco.collections', 'jaraco.functools', 'jaraco.text', + 'json-rpc', 'mako', 'markupsafe', 'mfpymake', 'more-itertools', + 'networkx', 'portend', 'python-dateutil', 'requests', 'telenium', + 'tempora', 'urllib3', 'werkzeug', 'ws4py', 'zc.lockfile', 'zipp' + ] + context = mock.ANY + project_dir = None + # The pip-compile tool used to expanded the list of requirements + # doesn't always return results in the same order. + # _Call object and its properties are immutable, so we create a + # new one with the same values. + m_build_recipes_call_args = mock.call( + sorted(m_build_recipes.call_args_list[0][0][0]), + sorted(m_build_recipes.call_args_list[0][0][1]), + m_build_recipes.call_args_list[0][0][2], + m_build_recipes.call_args_list[0][0][3], + ignore_project_setup_py=m_build_recipes.call_args_list[0][1]['ignore_project_setup_py'] + ) + assert m_build_recipes_call_args == mock.call( + sorted(build_order), + sorted(python_modules), + context, + project_dir, + ignore_project_setup_py=False + ) + assert m_run_distribute.call_args_list == [mock.call()] + assert 'VERSION_python3' in os.environ and os.environ['VERSION_python3'] == '3.8.10' + assert 'VERSION_mfpymake' in os.environ and os.environ['VERSION_mfpymake'] == '1.2.2' + assert 'VERSION_numpy' in os.environ and os.environ['VERSION_numpy'] == '1.21.4' + assert 'VERSION_mako' in os.environ and os.environ['VERSION_mako'] == '1.1.5' + @pytest.mark.skipif(sys.version_info < (3, 0), reason="requires python3") def test_recipes(self): """