Skip to content

Commit

Permalink
Add automatic expansion of --requirement list
Browse files Browse the repository at this point in the history 
Related to issue #2529
  • Loading branch information
@pdallair
pdallair committed Dec 29, 2021
1 parent 06dd1e0 commit e0a6786
Show file tree
Hide file tree
Showing 3 changed files with 391 additions and 48 deletions.
298 changes: 251 additions & 47 deletions pythonforandroid/toolchain.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
This module defines the entry point for command line and programmatic use.
"""

from os import environ
from pythonforandroid import __version__
from pythonforandroid.pythonpackage import get_dep_names_of_package
Expand Down Expand Up @@ -227,6 +226,246 @@ def split_argument_list(arg_list):
return re.split(r'[ ,]+', arg_list)


def __expand_requirements_arg_from_project_files(ctx, args):
"""Parse additional requirements from setup.py or pyproject.toml file
and add to --requirements arg if --use_setup_py argument was specified"""
all_recipes = [
recipe.lower() for recipe in
set(Recipe.list_recipes(ctx))
]

has_setup_py_or_toml = False
if getattr(args, "private", None) is not None:
project_dir = getattr(args, "private")
has_setup_py = os.path.exists(
os.path.join(project_dir, "setup.py"))
has_toml = os.path.exists(
os.path.join(project_dir, "pyproject.toml"))
has_setup_py_or_toml = has_setup_py or has_toml

# Add dependencies from setup.py, but only if they are recipes
# (because otherwise, setup.py itself will install them later)
if has_setup_py_or_toml and getattr(args, "use_setup_py", False):
try:
info("Analyzing package dependencies. MAY TAKE A WHILE.")
# Get all the dependencies corresponding to a recipe:
dependencies = [
dep.lower() for dep in
get_dep_names_of_package(
args.private,
keep_version_pins=True,
recursive=True,
verbose=True,
)
]
info("Dependencies obtained: " + str(dependencies))
dependencies = set(dependencies).intersection(
set(all_recipes)
)

# Add dependencies to argument list:
if len(dependencies) > 0:
if len(args.requirements) > 0:
args.requirements += u","
args.requirements += u",".join(dependencies)

except ValueError:
# Not a python package, apparently.
warning(
"Processing failed, is this project a valid "
"package? Will continue WITHOUT setup.py deps."
)


def has_a_recipe(ctx, requirement):
all_recipes = [
recipe.lower() for recipe in
set(Recipe.list_recipes(ctx))
]
requirement_name = re.sub(r'==\d+(\.\d+)*', '', requirement)
return requirement_name in all_recipes


def __run_pip_compile(requirements):
return shprint(
sh.bash, '-c',
"echo -e '{}' > requirements.in && "
"pip-compile --dry-run --annotation-style=line && "
"rm requirements.in".format(
'\n'.join(requirements)))


def __parse_pip_compile_output(output):
parsed_requirement_info_list = []
for line in output.splitlines():
match_data = re.match(
r'^([\w.-]+)==(\d+(\.\d+)*).*'
r'#\s+via\s+([\w\s,.-]+)', line)

if match_data:
parent_requirements = match_data.group(4).split(', ')
requirement_name = match_data.group(1)
requirement_version = match_data.group(2)

# Requirement is a "non-recipe" one we started with.
if '-r requirements.in' in parent_requirements:
parent_requirements.remove('-r requirements.in')

parsed_requirement_info_list.append([
requirement_name,
requirement_version,
parent_requirements])

return parsed_requirement_info_list


def __run_pip_compile_and_parse_output(requirements):
return __parse_pip_compile_output(__run_pip_compile(requirements))


def __is_requirement_installed_by_recipe(
ctx, current_requirement_info, remaining_requirement_names):

requirement_name, requirement_version, \
parent_requirements = current_requirement_info

# If any parent requirement has a recipe, this
# requirement ought also to be installed by it.
# Hence, it's better not to add this requirement the
# expanded list.
parent_requirements_with_recipe = [
parent_requirement
for parent_requirement in parent_requirements
if has_a_recipe(ctx, parent_requirement)
]

# Any parent requirement removed for the expanded list
# implies that it and its own requirements (including
# this requirement) will be installed by a recipe.
# Hence, it's better not to add this requirement the
# expanded list.
parent_requirements_still_in_list = [
parent_requirement
for parent_requirement in parent_requirements
if parent_requirement in remaining_requirement_names
]

is_installed_by_a_recipe = \
len(parent_requirements) and \
(parent_requirements_with_recipe or
len(parent_requirements_still_in_list) !=
len(parent_requirements))

if is_installed_by_a_recipe:
info('{}\n\t{}\n\t{}\n\t{}'.format(
requirement_name,
parent_requirements,
parent_requirements_with_recipe,
parent_requirements_still_in_list))

return is_installed_by_a_recipe


def __prune_requirements_installed_by_recipe(ctx, requirement_info_list):
original_requirement_count = -1

while len(requirement_info_list) != original_requirement_count:
original_requirement_count = len(requirement_info_list)

for i, requirement_info in enumerate(reversed(requirement_info_list)):
index = original_requirement_count - i - 1

remaining_requirement_names = \
[x[0] for x in requirement_info_list]

if __is_requirement_installed_by_recipe(
ctx, requirement_info, remaining_requirement_names):
info('{} will be installed by a recipe. Removing '
'it from requirement list expansion.'.format(
requirement_info[0]))

del requirement_info_list[index]


def __add_compiled_requirements_to_args(
ctx, args, compiled_requirment_info_list):

for requirement_info in compiled_requirment_info_list:
requirement_name, requirement_version, \
parent_requirements = requirement_info

# If the requirement has a recipe, don't use specific
# version constraints determined by pip-compile. Some
# recipes may not support the specified version. Therefor,
# it's probably safer to just let them use their default
# version. User can still force the usage of specific
# version by explicitly declaring it with --requirements.
requirement_str = \
requirement_name if has_a_recipe(ctx, requirement_name) else \
'{}=={}'.format(requirement_name, requirement_version)

requirement_names_arg = split_argument_list(re.sub(
r'==\d+(\.\d+)*', '', args.requirements))

# This expansion was carried out based on "non-recipe"
# requirements. Hence,the counter-part, requirements
# with a recipe, may already be part of list.
if requirement_name not in requirement_names_arg:
args.requirements += ',' + requirement_str


def __expand_requirements_arg_from_pip_compile(ctx, args):
"""Use pip-compile to generate requirement dependencies and add to
--requirements command line argument."""

non_recipe_requirements = [
requirement for requirement in split_argument_list(args.requirements)
if not has_a_recipe(ctx, requirement)
]
non_recipe_requirements_regex = \
r',?\s+' + r'|,?\s+'.join(non_recipe_requirements)
args.requirements = \
re.sub(non_recipe_requirements_regex, '', args.requirements)

# Compile "non-recipe" requirements' dependencies and add to
# args.requirement. Otherwise, only recipe requirements'
# dependencies would get installed.
# More info https://github.com/kivy/python-for-android/issues/2529
if non_recipe_requirements:
info("Compiling dependencies for: "
"{}".format(non_recipe_requirements))

parsed_requirement_info_list = \
__run_pip_compile_and_parse_output(non_recipe_requirements)

info("Requirements obtained from pip-compile: "
"{}".format(["{}=={}".format(x[0], x[1])
for x in parsed_requirement_info_list]))

__prune_requirements_installed_by_recipe(
ctx, parsed_requirement_info_list)

info("Requirements remaining after recipe dependency \"prunage\": "
"{}".format(["{}=={}".format(x[0], x[1])
for x in parsed_requirement_info_list]))

__add_compiled_requirements_to_args(
ctx, args, parsed_requirement_info_list)


def expand_requirements_args(ctx, args):
"""Expand --requirements arg value to include what may have not
been specified by the user, such as:
* requirements specified in local project setup.py or pyproject.toml
(if --use_setup_py was used)
* indirect requirements (i.e., the requirements of our requirements).
(e.g., if user specifies beautifulsoup4, the appropriate version of
soupsieve is added).
"""
__expand_requirements_arg_from_project_files(ctx, args)
__expand_requirements_arg_from_pip_compile(ctx, args)


class NoAbbrevParser(argparse.ArgumentParser):
"""We want to disable argument abbreviation so as not to interfere
with passing through arguments to build.py, but in python2 argparse
Expand Down Expand Up @@ -646,55 +885,20 @@ def add_parser(subparsers, *args, **kwargs):
args, "with_debug_symbols", False
)

have_setup_py_or_similar = False
if getattr(args, "private", None) is not None:
project_dir = getattr(args, "private")
if (os.path.exists(os.path.join(project_dir, "setup.py")) or
os.path.exists(os.path.join(project_dir,
"pyproject.toml"))):
have_setup_py_or_similar = True
# Process requirements and put version in environ:
if getattr(args, 'requirements', []):
all_recipes = [
recipe.lower() for recipe in
set(Recipe.list_recipes(self.ctx))
]

# Process requirements and put version in environ
if hasattr(args, 'requirements'):
requirements = []
expand_requirements_args(self.ctx, args)

# Add dependencies from setup.py, but only if they are recipes
# (because otherwise, setup.py itself will install them later)
if (have_setup_py_or_similar and
getattr(args, "use_setup_py", False)):
try:
info("Analyzing package dependencies. MAY TAKE A WHILE.")
# Get all the dependencies corresponding to a recipe:
dependencies = [
dep.lower() for dep in
get_dep_names_of_package(
args.private,
keep_version_pins=True,
recursive=True,
verbose=True,
)
]
info("Dependencies obtained: " + str(dependencies))
all_recipes = [
recipe.lower() for recipe in
set(Recipe.list_recipes(self.ctx))
]
dependencies = set(dependencies).intersection(
set(all_recipes)
)
# Add dependencies to argument list:
if len(dependencies) > 0:
if len(args.requirements) > 0:
args.requirements += u","
args.requirements += u",".join(dependencies)
except ValueError:
# Not a python package, apparently.
warning(
"Processing failed, is this project a valid "
"package? Will continue WITHOUT setup.py deps."
)
info('Expanded Requirements List: '
'{}'.format(split_argument_list(args.requirements)))

# Parse --requirements argument list:
# Handle specific version requirement constraints (e.g. foo==x.y)
requirements = []
for requirement in split_argument_list(args.requirements):
if "==" in requirement:
requirement, version = requirement.split(u"==", 1)
Expand Down
2 changes: 1 addition & 1 deletion setup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
install_reqs = [
'appdirs', 'colorama>=0.3.3', 'jinja2', 'six',
'enum34; python_version<"3.4"', 'sh>=1.10; sys_platform!="nt"',
'pep517<0.7.0', 'toml',
'pep517<0.7.0', 'toml', 'pip-tools'
]
# (pep517 and toml are used by pythonpackage.py)

Expand Down
Loading

0 comments on commit e0a6786

Please sign in to comment.