Skip to content

Commit

Permalink
Add automatic expansion of --requirement list
Browse files Browse the repository at this point in the history 
Related to issue kivy#2529
  • Loading branch information
@pdallair
pdallair committed Dec 27, 2021
1 parent 4563f3e commit 7780464
Show file tree
Hide file tree
Showing 3 changed files with 218 additions and 10 deletions.
138 changes: 129 additions & 9 deletions pythonforandroid/toolchain.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
This module defines the entry point for command line and programmatic use.
"""

from os import environ
from pythonforandroid import __version__
from pythonforandroid.pythonpackage import get_dep_names_of_package
Expand Down Expand Up @@ -651,9 +650,12 @@ def add_parser(subparsers, *args, **kwargs):
"pyproject.toml"))):
have_setup_py_or_similar = True

# Process requirements and put version in environ
if hasattr(args, 'requirements'):
requirements = []
# Process requirements and put version in environ:
if hasattr(args, 'requirements') and args.requirements:
all_recipes = [
recipe.lower() for recipe in
set(Recipe.list_recipes(self.ctx))
]

# Add dependencies from setup.py, but only if they are recipes
# (because otherwise, setup.py itself will install them later)
Expand All @@ -672,10 +674,6 @@ def add_parser(subparsers, *args, **kwargs):
)
]
info("Dependencies obtained: " + str(dependencies))
all_recipes = [
recipe.lower() for recipe in
set(Recipe.list_recipes(self.ctx))
]
dependencies = set(dependencies).intersection(
set(all_recipes)
)
Expand All @@ -691,7 +689,126 @@ def add_parser(subparsers, *args, **kwargs):
"package? Will continue WITHOUT setup.py deps."
)

# Parse --requirements argument list:
non_recipe_requirements = []
for requirement in args.requirements.split(','):
requirement_name = re.sub(r'==\d+(\.\d+)*', '', requirement)
if requirement_name not in all_recipes:
non_recipe_requirements.append(requirement)
args.requirements = re.sub(
r',?{}'.format(requirement), '', args.requirements)

# Compile "non-recipe" requirements' dependencies and add to list.
# Otherwise, only recipe requirements' dependencies get installed.
# More info https://github.com/kivy/python-for-android/issues/2529
if non_recipe_requirements:
info("Compiling dependencies for: "
"{}".format(non_recipe_requirements))

output = shprint(
sh.bash, '-c',
"echo -e '{}' > requirements.in && "
"pip-compile -v --dry-run --annotation-style=line && "
"rm requirements.in".format(
'\n'.join(non_recipe_requirements)))

# Parse pip-compile output
parsed_requirement_info_list = []
for line in output.splitlines():
match_data = re.match(
r'^([\w.-]+)==(\d+(\.\d+)*).*'
r'#\s+via\s+([\w\s,.-]+)', line)

if match_data:
parent_requirements = match_data.group(4).split(', ')
requirement_name = match_data.group(1)
requirement_version = match_data.group(2)

# Requirement is a "non-recipe" one we started with.
if '-r requirements.in' in parent_requirements:
parent_requirements.remove('-r requirements.in')

parsed_requirement_info_list.append([
requirement_name,
requirement_version,
parent_requirements])

info("Requirements obtained from pip-compile: "
"{}".format(["{}=={}".format(x[0], x[1])
for x in parsed_requirement_info_list]))

# Remove indirect requirements ultimately installed by a recipe
original_parsed_requirement_count = -1
while len(parsed_requirement_info_list) != \
original_parsed_requirement_count:

original_parsed_requirement_count = \
len(parsed_requirement_info_list)

for i, parsed_requirement_info in \
enumerate(reversed(parsed_requirement_info_list)):

index = original_parsed_requirement_count - i - 1
requirement_name, requirement_version, \
parent_requirements = parsed_requirement_info

# If any parent requirement has a recipe, this
# requirement ought also to be installed by it.
# Hence, it's better not to add this requirement the
# expanded list.
parent_requirements_with_recipe = list(
set(parent_requirements).intersection(
set(all_recipes)))

# Any parent requirement removed for the expanded list
# implies that it and its own requirements (including
# this requirement) will be installed by a recipe.
# Hence, it's better not to add this requirement the
# expanded list.
requirement_name_list = \
[x[0] for x in parsed_requirement_info_list]
parent_requirements_still_in_list = list(
set(parent_requirements).intersection(
set(requirement_name_list)))

is_ultimately_installed_by_a_recipe = \
len(parent_requirements) and \
(parent_requirements_with_recipe or
len(parent_requirements_still_in_list) !=
len(parent_requirements))

if is_ultimately_installed_by_a_recipe:
info(
'{} will be installed by a recipe. Removing '
'it from requirement list expansion.'.format(
requirement_name))
del parsed_requirement_info_list[index]

for parsed_requirement_info in parsed_requirement_info_list:
requirement_name, requirement_version, \
parent_requirements = parsed_requirement_info

# If the requirement has a recipe, don't use specific
# version constraints determined by pip-compile. Some
# recipes may not support the specified version. Therefor,
# it's probably safer to just let them use their default
# version. User can still force the usage of specific
# version by explicitly declaring it with --requirements.
requirement_has_recipe = requirement_name in all_recipes
requirement_str = \
requirement_name if requirement_has_recipe else \
'{}=={}'.format(requirement_name, requirement_version)

requirement_names_arg = re.sub(
r'==\d+(\.\d+)*', '', args.requirements).split(',')

# This expansion was carried out based on "non-recipe"
# requirements. Hence,the counter-part, requirements
# with a recipe, may already be part of list.
if requirement_name not in requirement_names_arg:
args.requirements += ',' + requirement_str

# Handle specific version requirement constraints (e.g. foo==x.y)
requirements = []
for requirement in split_argument_list(args.requirements):
if "==" in requirement:
requirement, version = requirement.split(u"==", 1)
Expand All @@ -701,6 +818,9 @@ def add_parser(subparsers, *args, **kwargs):
requirements.append(requirement)
args.requirements = u",".join(requirements)

info('Expanded Requirements List: '
'{}'.format(args.requirements.split(',')))

self.warn_on_deprecated_args(args)

self.storage_dir = args.storage_dir
Expand Down
2 changes: 1 addition & 1 deletion setup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
install_reqs = [
'appdirs', 'colorama>=0.3.3', 'jinja2', 'six',
'enum34; python_version<"3.4"', 'sh>=1.10; sys_platform!="nt"',
'pep517<0.7.0', 'toml',
'pep517<0.7.0', 'toml', 'pip-tools'
]
# (pep517 and toml are used by pythonpackage.py)

Expand Down
88 changes: 88 additions & 0 deletions tests/test_toolchain.py
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import io
import os
import sys
import pytest
from unittest import mock
Expand Down Expand Up @@ -124,6 +125,93 @@ def test_create_no_sdk_dir(self):
assert ex_info.value.message == (
'Android SDK dir was not specified, exiting.')

def test_create_with_complex_requirements(self):
requirements = [
'python3==3.8.10', # direct requirement with recipe using version constraint
'pandas', # direct requirement with recipe (no version constraint)
'mfpymake==1.2.2', # direct requirement without recipe using version constraint
'telenium', # direct requirement without recipe (no version constraint)
'numpy==1.21.4', # indirect requirement with recipe using version constraint
'mako==1.1.5', # indirect requirement without recipe using version constraint
# There's no reason to specify an indirect requirement unless we want to install a specific version.
]
argv = [
'toolchain.py',
'create',
'--sdk-dir=/tmp/android-sdk',
'--ndk-dir=/tmp/android-ndk',
'--bootstrap=service_only',
'--requirements={}'.format(','.join(requirements)),
'--dist-name=test_toolchain',
'--activity-class-name=abc.myapp.android.CustomPythonActivity',
'--service-class-name=xyz.myapp.android.CustomPythonService',
]
with patch_sys_argv(argv), mock.patch(
'pythonforandroid.build.get_available_apis'
) as m_get_available_apis, mock.patch(
'pythonforandroid.build.get_toolchain_versions'
) as m_get_toolchain_versions, mock.patch(
'pythonforandroid.build.get_ndk_platform_dir'
) as m_get_ndk_platform_dir, mock.patch(
'pythonforandroid.toolchain.build_recipes'
) as m_build_recipes, mock.patch(
'pythonforandroid.bootstraps.service_only.'
'ServiceOnlyBootstrap.assemble_distribution'
) as m_run_distribute:
m_get_available_apis.return_value = [27]
m_get_toolchain_versions.return_value = (['4.9'], True)
m_get_ndk_platform_dir.return_value = (
'/tmp/android-ndk/platforms/android-21/arch-arm', True)
tchain = ToolchainCL()
assert tchain.ctx.activity_class_name == 'abc.myapp.android.CustomPythonActivity'
assert tchain.ctx.service_class_name == 'xyz.myapp.android.CustomPythonService'
assert m_get_available_apis.call_args_list in [
[mock.call('/tmp/android-sdk')], # linux case
[mock.call('/private/tmp/android-sdk')] # macos case
]
assert m_get_toolchain_versions.call_args_list in [
[mock.call('/tmp/android-ndk', mock.ANY)], # linux case
[mock.call('/private/tmp/android-ndk', mock.ANY)], # macos case
]
build_order = [
'android', 'cython', 'genericndkbuild', 'hostpython3', 'libbz2',
'libffi', 'liblzma', 'numpy', 'openssl', 'pandas', 'pyjnius',
'python3', 'pytz', 'setuptools', 'six', 'sqlite3'
]
python_modules = [
'certifi', 'charset-normalizer', 'cheroot', 'cherrypy',
'idna', 'importlib-resources', 'jaraco.classes',
'jaraco.collections', 'jaraco.functools', 'jaraco.text',
'json-rpc', 'mako', 'markupsafe', 'mfpymake', 'more-itertools',
'networkx', 'portend', 'python-dateutil', 'requests', 'telenium',
'tempora', 'urllib3', 'werkzeug', 'ws4py', 'zc.lockfile', 'zipp'
]
context = mock.ANY
project_dir = None
# The pip-compile tool used to expanded the list of requirements
# doesn't always return results in the same order.
# _Call object and its properties are immutable, so we create a
# new one with the same values.
m_build_recipes_call_args = mock.call(
sorted(m_build_recipes.call_args_list[0][0][0]),
sorted(m_build_recipes.call_args_list[0][0][1]),
m_build_recipes.call_args_list[0][0][2],
m_build_recipes.call_args_list[0][0][3],
ignore_project_setup_py=m_build_recipes.call_args_list[0][1]['ignore_project_setup_py']
)
assert m_build_recipes_call_args == mock.call(
sorted(build_order),
sorted(python_modules),
context,
project_dir,
ignore_project_setup_py=False
)
assert m_run_distribute.call_args_list == [mock.call()]
assert 'VERSION_python3' in os.environ and os.environ['VERSION_python3'] == '3.8.10'
assert 'VERSION_mfpymake' in os.environ and os.environ['VERSION_mfpymake'] == '1.2.2'
assert 'VERSION_numpy' in os.environ and os.environ['VERSION_numpy'] == '1.21.4'
assert 'VERSION_mako' in os.environ and os.environ['VERSION_mako'] == '1.1.5'

@pytest.mark.skipif(sys.version_info < (3, 0), reason="requires python3")
def test_recipes(self):
"""
Expand Down

0 comments on commit 7780464

Please sign in to comment.