Add automatic expansion of --requirement list

Related to issue kivy#2529

pythonforandroid/toolchain.py

@@ -5,7 +5,6 @@
 
 This module defines the entry point for command line and programmatic use.
 """
-
 from os import environ
 from pythonforandroid import __version__
 from pythonforandroid.pythonpackage import get_dep_names_of_package
@@ -651,9 +650,12 @@ def add_parser(subparsers, *args, **kwargs):
                                                 "pyproject.toml"))):
                 have_setup_py_or_similar = True
 
-        # Process requirements and put version in environ
-        if hasattr(args, 'requirements'):
-            requirements = []
+        # Process requirements and put version in environ:
+        if hasattr(args, 'requirements') and args.requirements:
+            all_recipes = [
+                recipe.lower() for recipe in
+                set(Recipe.list_recipes(self.ctx))
+            ]
 
             # Add dependencies from setup.py, but only if they are recipes
             # (because otherwise, setup.py itself will install them later)
@@ -672,10 +674,6 @@ def add_parser(subparsers, *args, **kwargs):
                         )
                     ]
                     info("Dependencies obtained: " + str(dependencies))
-                    all_recipes = [
-                        recipe.lower() for recipe in
-                        set(Recipe.list_recipes(self.ctx))
-                    ]
                     dependencies = set(dependencies).intersection(
                         set(all_recipes)
                     )
@@ -691,7 +689,48 @@ def add_parser(subparsers, *args, **kwargs):
                         "package? Will continue WITHOUT setup.py deps."
                     )
 
-            # Parse --requirements argument list:
+            non_recipe_requirements = []
+            for requirement in args.requirements.split(','):
+                if re.sub(r'==\d+(\.\d+)*', '', requirement) not in all_recipes:
+                    non_recipe_requirements.append(requirement)
+                    args.requirements = re.sub(
+                        r',?{}'.format(requirement), '', args.requirements)
+
+            # Compile "non-recipe" requirements' dependencies and add to list.
+            if non_recipe_requirements:
+                info("Compiling dependencies for: "
+                     "{}".format(non_recipe_requirements))
+
+                output = shprint(
+                    sh.bash, '-c',
+                    "echo '{}' | sed -e 's/,/\\n/g' > requirements.in && " 
+                    "pip-compile -v --dry-run --annotation-style line && " 
+                    "rm requirements.in".format(
+                        ','.join(non_recipe_requirements)))
+
+                for line in output.splitlines():
+                    match_data = re.match(
+                        r'^([\w-]+)==\d+(\.\d+).*#\s+via\s+([\w\s,-]+|'
+                        r'-r\s+requirements\.in)', line)
+
+                    if match_data:
+                        # One of the original "non-recipe" requirements
+                        if 'requirement.in' in match_data.group(3):
+                            args.requirements += ',' + match_data.group(1)
+
+                        else:
+                            dependents = match_data.group(3).split(', ')
+                            dependents_with_recipe = \
+                                set(dependents).intersection(set(all_recipes))
+
+                            # Some dependencies discovered might have recipes.
+                            # Only include those that aren't installed by said
+                            # recipes to avoid installing them twice.
+                            if not len(dependents_with_recipe):
+                                args.requirements += ',' + match_data.group(1)
+
+            # Handle specific version requirement constraints (e.g. foo==x.y)
+            requirements = []
             for requirement in split_argument_list(args.requirements):
                 if "==" in requirement:
                     requirement, version = requirement.split(u"==", 1)
@@ -701,6 +740,9 @@ def add_parser(subparsers, *args, **kwargs):
                 requirements.append(requirement)
             args.requirements = u",".join(requirements)
 
+            info('Final Requirements List: '
+                 '{}'.format(args.requirements.split(',')))
+
         self.warn_on_deprecated_args(args)
 
         self.storage_dir = args.storage_dir

setup.py

@@ -22,7 +22,7 @@
 install_reqs = [
     'appdirs', 'colorama>=0.3.3', 'jinja2', 'six',
     'enum34; python_version<"3.4"', 'sh>=1.10; sys_platform!="nt"',
-    'pep517<0.7.0', 'toml',
+    'pep517<0.7.0', 'toml', 'pip-tools'
 ]
 # (pep517 and toml are used by pythonpackage.py)