Skip to content

Commit

Permalink
Fix various setup.py processing errors
Browse files Browse the repository at this point in the history 
- fix file litter of constraint.txt
- fix --private relative paths during build.py's setup.py install
- fix "" and "." not being recognized as folder by pythonpackage.py
  • Loading branch information
Jonas Thiem committed Jun 12, 2019
1 parent 4e67d23 commit 6ae074e
Show file tree
Hide file tree
Showing 4 changed files with 131 additions and 106 deletions.
226 changes: 123 additions & 103 deletions pythonforandroid/build.py
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
from __future__ import print_function

from os.path import (join, realpath, dirname, expanduser, exists,
split, isdir)
from os.path import (
abspath, join, realpath, dirname, expanduser, exists,
split, isdir
)
from os import environ
import copy
import os
Expand Down Expand Up @@ -590,6 +592,120 @@ def project_has_setup_py(project_dir):
return False


def run_setuppy_install(ctx, project_dir, env=None):
if env is None:
env = dict()

with current_directory(project_dir):
info('got setup.py or similar, running project install. ' +
'(disable this behavior with --ignore-setup-py)')

# Compute & output the constraints we will use:
info('Contents that will be used for constraints.txt:')
constraints = subprocess.check_output([
join(
ctx.build_dir, "venv", "bin", "pip"
),
"freeze"
], env=copy.copy(env))
try:
constraints = constraints.decode("utf-8", "replace")
except AttributeError:
pass
info(constraints)

# Make sure all packages found are fixed in version
# by writing a constraint file, to avoid recipes being
# upgraded & reinstalled:
with open('._tmp_p4a_recipe_constraints.txt', 'wb') as fileh:
fileh.write(constraints.encode("utf-8", "replace"))
try:

info('Populating venv\'s site-packages with '
'ctx.get_site_packages_dir()...')

# Copy dist contents into site-packages for discovery.
# Why this is needed:
# --target is somewhat evil and messes with discovery of
# packages in PYTHONPATH if that also includes the target
# folder. So we need to use the regular virtualenv
# site-packages folder instead.
# Reference:
# https://github.com/pypa/pip/issues/6223
ctx_site_packages_dir = os.path.normpath(
os.path.abspath(ctx.get_site_packages_dir())
)
venv_site_packages_dir = os.path.normpath(os.path.join(
ctx.build_dir, "venv", "lib", [
f for f in os.listdir(os.path.join(
ctx.build_dir, "venv", "lib"
)) if f.startswith("python")
][0], "site-packages"
))
copied_over_contents = []
for f in os.listdir(ctx_site_packages_dir):
full_path = os.path.join(ctx_site_packages_dir, f)
if not os.path.exists(os.path.join(
venv_site_packages_dir, f
)):
if os.path.isdir(full_path):
shutil.copytree(full_path, os.path.join(
venv_site_packages_dir, f
))
else:
shutil.copy2(full_path, os.path.join(
venv_site_packages_dir, f
))
copied_over_contents.append(f)

# Get listing of virtualenv's site-packages, to see the
# newly added things afterwards & copy them back into
# the distribution folder / build context site-packages:
previous_venv_contents = os.listdir(
venv_site_packages_dir
)

# Actually run setup.py:
info('Launching package install...')
shprint(sh.bash, '-c', (
"'" + join(
ctx.build_dir, "venv", "bin", "pip"
).replace("'", "'\"'\"'") + "' " +
"install -c ._tmp_p4a_recipe_constraints.txt -v ."
).format(ctx.get_site_packages_dir().
replace("'", "'\"'\"'")),
_env=copy.copy(env))

# Go over all new additions and copy them back:
info('Copying additions resulting from setup.py back '
'into ctx.get_site_packages_dir()...')
new_venv_additions = []
for f in (set(os.listdir(venv_site_packages_dir)) -
set(previous_venv_contents)):
new_venv_additions.append(f)
full_path = os.path.join(venv_site_packages_dir, f)
if os.path.isdir(full_path):
shutil.copytree(full_path, os.path.join(
ctx_site_packages_dir, f
))
else:
shutil.copy2(full_path, os.path.join(
ctx_site_packages_dir, f
))

# Undo all the changes we did to the venv-site packages:
info('Reverting additions to '
'virtualenv\'s site-packages...')
for f in set(copied_over_contents + new_venv_additions):
full_path = os.path.join(venv_site_packages_dir, f)
if os.path.isdir(full_path):
shutil.rmtree(full_path)
else:
os.remove(full_path)
finally:
os.remove("._tmp_p4a_recipe_constraints.txt")


def run_pymodules_install(ctx, modules, project_dir=None,
ignore_setup_py=False):
""" This function will take care of all non-recipe things, by:
Expand All @@ -605,6 +721,10 @@ def run_pymodules_install(ctx, modules, project_dir=None,
info('*** PYTHON PACKAGE / PROJECT INSTALL STAGE ***')
modules = list(filter(ctx.not_has_package, modules))

# We change current working directory later, so this
# has to be an absolute path:
project_dir = abspath(project_dir)

# Bail out if no python deps and no setup.py to process:
if not modules and (
ignore_setup_py or
Expand Down Expand Up @@ -697,107 +817,7 @@ def run_pymodules_install(ctx, modules, project_dir=None,
if project_dir is not None and (
project_has_setup_py(project_dir) and not ignore_setup_py
):
with current_directory(project_dir):
info('got setup.py or similar, running project install. ' +
'(disable this behavior with --ignore-setup-py)')

# Compute & output the constraints we will use:
info('Contents that will be used for constraints.txt:')
constraints = subprocess.check_output([
join(
ctx.build_dir, "venv", "bin", "pip"
),
"freeze"
], env=copy.copy(env))
try:
constraints = constraints.decode("utf-8", "replace")
except AttributeError:
pass
info(constraints)

# Make sure all packages found are fixed in version
# by writing a constraint file, to avoid recipes being
# upgraded & reinstalled:
with open('constraints.txt', 'wb') as fileh:
fileh.write(constraints.encode("utf-8", "replace"))

info('Populating venv\'s site-packages with '
'ctx.get_site_packages_dir()...')

# Copy dist contents into site-packages for discovery.
# Why this is needed:
# --target is somewhat evil and messes with discovery of
# packages in PYTHONPATH if that also includes the target
# folder. So we need to use the regular virtualenv
# site-packages folder instead.
# Reference:
# https://github.com/pypa/pip/issues/6223
ctx_site_packages_dir = os.path.normpath(
os.path.abspath(ctx.get_site_packages_dir())
)
venv_site_packages_dir = os.path.normpath(os.path.join(
ctx.build_dir, "venv", "lib", [
f for f in os.listdir(os.path.join(
ctx.build_dir, "venv", "lib"
)) if f.startswith("python")
][0], "site-packages"
))
copied_over_contents = []
for f in os.listdir(ctx_site_packages_dir):
full_path = os.path.join(ctx_site_packages_dir, f)
if not os.path.exists(os.path.join(
venv_site_packages_dir, f
)):
if os.path.isdir(full_path):
shutil.copytree(full_path, os.path.join(
venv_site_packages_dir, f
))
else:
shutil.copy2(full_path, os.path.join(
venv_site_packages_dir, f
))
copied_over_contents.append(f)

# Get listing of virtualenv's site-packages, to see the
# newly added things afterwards & copy them back into
# the distribution folder / build context site-packages:
previous_venv_contents = os.listdir(venv_site_packages_dir)

# Actually run setup.py:
info('Launching package install...')
shprint(sh.bash, '-c', (
"'" + join(
ctx.build_dir, "venv", "bin", "pip"
).replace("'", "'\"'\"'") + "' " +
"install -c constraints.txt -v ."
).format(ctx.get_site_packages_dir().replace("'", "'\"'\"'")),
_env=copy.copy(env))

# Go over all new additions and copy them back:
info('Copying additions resulting from setup.py back ' +
'into ctx.get_site_packages_dir()...')
new_venv_additions = []
for f in (set(os.listdir(venv_site_packages_dir)) -
set(previous_venv_contents)):
new_venv_additions.append(f)
full_path = os.path.join(venv_site_packages_dir, f)
if os.path.isdir(full_path):
shutil.copytree(full_path, os.path.join(
ctx_site_packages_dir, f
))
else:
shutil.copy2(full_path, os.path.join(
ctx_site_packages_dir, f
))

# Undo all the changes we did to the venv-site packages:
info('Reverting additions to virtualenv\'s site-packages...')
for f in set(copied_over_contents + new_venv_additions):
full_path = os.path.join(venv_site_packages_dir, f)
if os.path.isdir(full_path):
shutil.rmtree(full_path)
else:
os.remove(full_path)
run_setuppy_install(ctx, project_dir, env)
elif not ignore_setup_py:
info("No setup.py found in project directory: " +
str(project_dir)
Expand Down
4 changes: 2 additions & 2 deletions pythonforandroid/pythonpackage.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -559,7 +559,7 @@ def parse_as_folder_reference(dep):
# Check if this is either not an url, or a file URL:
if dep.startswith(("/", "file://")) or (
dep.find("/") > 0 and
dep.find("://") < 0):
dep.find("://") < 0) or (dep in ["", "."]):
if dep.startswith("file://"):
dep = urlunquote(urlparse(dep).path)
return dep
Expand Down Expand Up @@ -689,7 +689,7 @@ def get_package_dependencies(package,
for package_dep in current_queue:
new_reqs = set()
if verbose:
print("get_package_dependencies: resolving dependecy "
print("get_package_dependencies: resolving dependency "
"to package name: ".format(package_dep))
package = get_package_name(package_dep)
if package.lower() in packages_processed:
Expand Down
5 changes: 4 additions & 1 deletion pythonforandroid/toolchain.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -631,7 +631,10 @@ def add_parser(subparsers, *args, **kwargs):
args.requirements += u",".join(dependencies)
except ValueError:
# Not a python package, apparently.
pass
warning(
"Processing failed, is this project a valid "
"package? Will continue WITHOUT setup.py deps."
)

# Parse --requirements argument list:
for requirement in split_argument_list(args.requirements):
Expand Down
2 changes: 2 additions & 0 deletions tests/test_pythonpackage_basic.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,8 @@ def test_is_filesystem_path():
assert not is_filesystem_path("test @ bla")
assert is_filesystem_path("/abc/c@d")
assert not is_filesystem_path("https://user:pw@host/")
assert is_filesystem_path(".")
assert is_filesystem_path("")


def test_parse_as_folder_reference():
Expand Down

0 comments on commit 6ae074e

Please sign in to comment.