Add automatic expansion of --requirement list

Related to issue kivy#2529

pythonforandroid/toolchain.py

@@ -5,7 +5,6 @@
 
 This module defines the entry point for command line and programmatic use.
 """
-
 from os import environ
 from pythonforandroid import __version__
 from pythonforandroid.pythonpackage import get_dep_names_of_package
@@ -651,9 +650,12 @@ def add_parser(subparsers, *args, **kwargs):
                                                 "pyproject.toml"))):
                 have_setup_py_or_similar = True
 
-        # Process requirements and put version in environ
-        if hasattr(args, 'requirements'):
-            requirements = []
+        # Process requirements and put version in environ:
+        if hasattr(args, 'requirements') and args.requirements:
+            all_recipes = [
+                recipe.lower() for recipe in
+                set(Recipe.list_recipes(self.ctx))
+            ]
 
             # Add dependencies from setup.py, but only if they are recipes
             # (because otherwise, setup.py itself will install them later)
@@ -672,10 +674,6 @@ def add_parser(subparsers, *args, **kwargs):
                         )
                     ]
                     info("Dependencies obtained: " + str(dependencies))
-                    all_recipes = [
-                        recipe.lower() for recipe in
-                        set(Recipe.list_recipes(self.ctx))
-                    ]
                     dependencies = set(dependencies).intersection(
                         set(all_recipes)
                     )
@@ -691,7 +689,101 @@ def add_parser(subparsers, *args, **kwargs):
                         "package? Will continue WITHOUT setup.py deps."
                     )
 
-            # Parse --requirements argument list:
+            non_recipe_requirements = []
+            for requirement in args.requirements.split(','):
+                if re.sub(r'==\d+(\.\d+)*', '', requirement) not in all_recipes:
+                    non_recipe_requirements.append(requirement)
+                    args.requirements = re.sub(
+                        r',?{}'.format(requirement), '', args.requirements)
+
+            # Compile "non-recipe" requirements' dependencies and add to list.
+            # Otherwise, only recipe requirements' dependencies get installed.
+            if non_recipe_requirements:
+                info("Compiling dependencies for: "
+                     "{}".format(non_recipe_requirements))
+
+                output = shprint(
+                    sh.bash, '-c',
+                    "echo -e '{}' > requirements.in && " 
+                    "pip-compile -v --dry-run --annotation-style=line && " 
+                    "rm requirements.in".format(
+                        '\n'.join(non_recipe_requirements)))
+
+                # Parse pip-compile output
+                parsed_requirement_info_list = []
+                for line in output.splitlines():
+                    match_data = re.match(
+                        r'^([\w.-]+)==(\d+(\.\d+)*).*'
+                        r'#\s+via\s+([\w\s,.-]+)', line)
+
+                    if match_data:
+                        parent_requirements = match_data.group(4).split(', ')
+                        requirement_name = match_data.group(1)
+                        requirement_version = match_data.group(2)
+
+                        if '-r requirements.in' in parent_requirements:
+                            parent_requirements.remove('-r requirements.in')
+
+                        parsed_requirement_info_list.append([
+                            requirement_name,
+                            requirement_version,
+                            parent_requirements])
+
+                # Some requirements discovered might have recipes.
+                # Remove the other requirements that derive from it to
+                # avoid installing them twice.
+                original_parsed_requirement_count = -1
+                while len(parsed_requirement_info_list) != \
+                        original_parsed_requirement_count:
+
+                    original_parsed_requirement_count = \
+                        len(parsed_requirement_info_list)
+
+                    for i, parsed_requirement_info in \
+                            enumerate(reversed(parsed_requirement_info_list)):
+
+                        index = original_parsed_requirement_count - i - 1
+                        requirement_name, requirement_version, \
+                            parent_requirements = parsed_requirement_info
+
+                        parent_requirements_with_recipe = list(
+                            set(parent_requirements).intersection(
+                                set(all_recipes)))
+
+                        # If there are parent requirements and they're not in
+                        # the list anymore, it's because they were removed
+                        # during a previous iteration.  Hence, this requirement
+                        # can be removed as well.
+                        requirement_name_list = \
+                            [x[0] for x in parsed_requirement_info_list]
+                        parent_requirements_still_in_list = list(
+                            set(parent_requirements).intersection(
+                                set(requirement_name_list)))
+
+                        is_ultimately_installed_by_a_recipe = \
+                            len(parent_requirements) and \
+                            (parent_requirements_with_recipe or not
+                                parent_requirements_still_in_list)
+
+                        if is_ultimately_installed_by_a_recipe:
+                            del parsed_requirement_info_list[index]
+
+                for parsed_requirement_info in parsed_requirement_info_list:
+                    requirement_name, requirement_version, \
+                        parent_requirements = parsed_requirement_info
+
+                    # Don't use version restriction if it's a recipe,
+                    # some recipes may not support the specified version.
+                    # Let them use their default version instead.
+                    requirement_has_recipe = requirement_name in all_recipes
+                    requirement_name = \
+                        '{}=={}'.format(requirement_name, requirement_version)
+                        # requirement_name if requirement_has_recipe else \
+
+                    args.requirements += ',' + requirement_name
+
+            # Handle specific version requirement constraints (e.g. foo==x.y)
+            requirements = []
             for requirement in split_argument_list(args.requirements):
                 if "==" in requirement:
                     requirement, version = requirement.split(u"==", 1)
@@ -701,6 +793,9 @@ def add_parser(subparsers, *args, **kwargs):
                 requirements.append(requirement)
             args.requirements = u",".join(requirements)
 
+            info('Expanded Requirements List: '
+                 '{}'.format(args.requirements.split(',')))
+
         self.warn_on_deprecated_args(args)
 
         self.storage_dir = args.storage_dir

setup.py

@@ -22,7 +22,7 @@
 install_reqs = [
     'appdirs', 'colorama>=0.3.3', 'jinja2', 'six',
     'enum34; python_version<"3.4"', 'sh>=1.10; sys_platform!="nt"',
-    'pep517<0.7.0', 'toml',
+    'pep517<0.7.0', 'toml', 'pip-tools'
 ]
 # (pep517 and toml are used by pythonpackage.py)