Reintroduces the original python2's recipe by @tito as python2legacy

(This branch and commit  has been created as a sample to help decide if it is worth it to maintain the old python2 recipe as a new recipe named python2legacy. here we will see the minimal changes that we should make to make it work.)

In order to maintain compatibility to pygame bootstrap and old sdl libraries, we perform several operations at once, this way can be reverted easily:
  - add python2legacy compatibility to basic recipes needed for pygame and bootstraps
  - solve the conflicts between python recipes
  - cause the pygame bootstrap has troubles to successfully build with the new python2's build, we enforce to build with python2legacy
  - update the core files to the new python2legacy
  - add compatibility python2legacy to all bootstraps
  - add some python2legacy tests

Note: this new addition should be properly documented and add more recipes compatibility for python2legacy.

References: Pull Request kivy#1460

doc/source/buildoptions.rst

@@ -198,6 +198,13 @@ This bootstrap will eventually be deprecated in favour of sdl2, but
 not before the sdl2 bootstrap includes all the features that would be
 lost.
 
+.. warning:: In order to successfully build a pygame bootstrap you will need
+    to use an android ndk previous to the unified headers feature, otherwise
+    the build will fail, because pygame depends on sdl 1.2.x and the jpeg
+    library included into the sdl libs needs some headers that are removed
+    since the unified headers feature was introduced. The recommended ndk
+    version to be used is r13b.
+
 Build options
 %%%%%%%%%%%%%
 

pythonforandroid/bootstrap.py

@@ -243,10 +243,15 @@ def strip_libraries(self, arch):
             return
         strip = sh.Command(strip)
 
-        filens = shprint(sh.find, join(self.dist_dir, '_python_bundle',
-                                       '_python_bundle', 'modules'),
-                         join(self.dist_dir, 'libs'),
-                         '-iname', '*.so', _env=env).stdout.decode('utf-8')
+        if self.ctx.python_recipe.name == 'python2legacy':
+            filens = shprint(sh.find, join(self.dist_dir, 'private'),
+                             join(self.dist_dir, 'libs'),
+                             '-iname', '*.so', _env=env).stdout.decode('utf-8')
+        else:
+            filens = shprint(sh.find, join(self.dist_dir, '_python_bundle',
+                                           '_python_bundle', 'modules'),
+                             join(self.dist_dir, 'libs'),
+                             '-iname', '*.so', _env=env).stdout.decode('utf-8')
 
         logger.info('Stripping libraries in private dir')
         for filen in filens.split('\n'):

pythonforandroid/bootstraps/pygame/__init__.py

@@ -1,15 +1,14 @@
 from pythonforandroid.toolchain import Bootstrap, current_directory, info, info_main, shprint
+from pythonforandroid.util import ensure_dir
 from os.path import join, exists
-from os import walk
-import glob
 import sh
 
 
 class PygameBootstrap(Bootstrap):
     name = 'pygame'
 
-    recipe_depends = ['hostpython2', 'python2', 'pyjnius', 'sdl', 'pygame',
-                      'android', 'kivy']
+    recipe_depends = ['hostpython2legacy', 'python2legacy', 'pyjnius',
+                      'sdl', 'pygame', 'android', 'kivy']
 
     def run_distribute(self):
         info_main('# Creating Android project from build and {} bootstrap'.format(
@@ -45,53 +44,30 @@ def run_distribute(self):
                 fileh.write('sdk.dir={}'.format(self.ctx.sdk_dir))
 
             info('Copying python distribution')
-            hostpython = sh.Command(self.ctx.hostpython)
-            try:
-                shprint(hostpython, '-OO', '-m', 'compileall', self.ctx.get_python_install_dir(),
-                        _tail=10, _filterout="^Listing")
-            except sh.ErrorReturnCode:
-                pass
-            if not exists('python-install'):
-                shprint(sh.cp, '-a', self.ctx.get_python_install_dir(), './python-install')
-
-            self.distribute_libs(arch, [join(self.build_dir, 'libs', arch.arch), self.ctx.get_libs_dir(arch.arch)])
+
+            python_bundle_dir = join('_python_bundle', '_python_bundle')
+            if 'python2legacy' in self.ctx.recipe_build_order:
+                # a special case with its own packaging location
+                python_bundle_dir = 'private'
+                # And also must had an install directory, make sure of that
+                self.ctx.python_recipe.create_python_install(self.dist_dir)
+
+            self.distribute_libs(
+                arch, [join(self.build_dir, 'libs', arch.arch),
+                       self.ctx.get_libs_dir(arch.arch)])
             self.distribute_aars(arch)
             self.distribute_javaclasses(self.ctx.javaclass_dir)
 
-            info('Filling private directory')
-            if not exists(join('private', 'lib')):
-                shprint(sh.cp, '-a', join('python-install', 'lib'), 'private')
-            shprint(sh.mkdir, '-p', join('private', 'include', 'python2.7'))
-
-            shprint(sh.mv, join('libs', arch.arch, 'libpymodules.so'), 'private/')
-            shprint(sh.cp, join('python-install', 'include', 'python2.7', 'pyconfig.h'), join('private', 'include', 'python2.7/'))
-
-            info('Removing some unwanted files')
-            shprint(sh.rm, '-f', join('private', 'lib', 'libpython2.7.so'))
-            shprint(sh.rm, '-rf', join('private', 'lib', 'pkgconfig'))
-
-            with current_directory(join(self.dist_dir, 'private', 'lib', 'python2.7')):
-                # shprint(sh.xargs, 'rm', sh.grep('-E', '*\.(py|pyx|so\.o|so\.a|so\.libs)$', sh.find('.')))
-                removes = []
-                for dirname, something, filens in walk('.'):
-                    for filename in filens:
-                        for suffix in ('py', 'pyc', 'so.o', 'so.a', 'so.libs'):
-                            if filename.endswith(suffix):
-                                removes.append(filename)
-                shprint(sh.rm, '-f', *removes)
-
-                info('Deleting some other stuff not used on android')
-                # To quote the original distribute.sh, 'well...'
-                # shprint(sh.rm, '-rf', 'ctypes')
-                shprint(sh.rm, '-rf', 'lib2to3')
-                shprint(sh.rm, '-rf', 'idlelib')
-                for filename in glob.glob('config/libpython*.a'):
-                    shprint(sh.rm, '-f', filename)
-                shprint(sh.rm, '-rf', 'config/python.o')
-                shprint(sh.rm, '-rf', 'lib-dynload/_ctypes_test.so')
-                shprint(sh.rm, '-rf', 'lib-dynload/_testcapi.so')
+            ensure_dir(python_bundle_dir)
+            site_packages_dir = self.ctx.python_recipe.create_python_bundle(
+                join(self.dist_dir, python_bundle_dir), arch)
+
+            if 'sqlite3' not in self.ctx.recipe_build_order:
+                with open('blacklist.txt', 'a') as fileh:
+                    fileh.write('\nsqlite3/*\nlib-dynload/_sqlite3.so\n')
 
         self.strip_libraries(arch)
+        self.fry_eggs(site_packages_dir)
         super(PygameBootstrap, self).run_distribute()
 
 

pythonforandroid/bootstraps/pygame/build/build.py

@@ -215,6 +215,7 @@ def make_package(args):
     default_icon = 'templates/kivy-icon.png'
     default_presplash = 'templates/kivy-presplash.jpg'
     default_ouya_icon = 'templates/kivy-ouya-icon.png'
+    default_ant_properties = 'templates/ant.properties'
     # Figure out the version code, if necessary.
     if not args.numeric_version:
         for i in args.version.split('.'):
@@ -345,6 +346,9 @@ def make_package(args):
     shutil.copy(args.presplash or default_presplash,
                 'res/drawable/presplash.jpg')
 
+    # Copy over the ant.properties file.
+    shutil.copy(default_ant_properties, 'ant.properties')
+
     # If OUYA support was requested, copy over the OUYA icon
     if args.ouya_category:
         if not os.path.isdir('res/drawable-xhdpi'):

pythonforandroid/bootstraps/pygame/build/jni/Application.mk

@@ -8,8 +8,7 @@ APP_PROJECT_PATH := $(call my-dir)/..
 APP_MODULES := application sdl sdl_main tremor png jpeg freetype sdl_ttf sdl_image sqlite3
 
 APP_ABI := $(ARCH)
-# AND: I have no idea why I have to specify app_platform when distribute.sh seems to just set the sysroot cflag
-# AND: Either way, this has to *at least* be configurable
-APP_PLATFORM := android-14
+APP_PLATFORM := $(NDK_API)
 APP_STL := gnustl_static
 APP_CFLAGS += $(OFLAG)
+NDK_TOOLCHAIN_VERSION := 4.9

pythonforandroid/bootstraps/pygame/build/templates/ant.properties

@@ -0,0 +1,2 @@
+java.target=1.7
+java.source=1.7

pythonforandroid/bootstraps/sdl2/__init__.py

@@ -8,7 +8,8 @@
 class SDL2GradleBootstrap(Bootstrap):
     name = 'sdl2'
 
-    recipe_depends = ['sdl2', ('python2', 'python3', 'python3crystax')]
+    recipe_depends = [
+        'sdl2', ('python2', 'python2legacy', 'python3', 'python3crystax')]
 
     def run_distribute(self):
         info_main("# Creating Android project ({})".format(self.name))
@@ -33,13 +34,18 @@ def run_distribute(self):
         with current_directory(self.dist_dir):
             info("Copying Python distribution")
 
+            python_bundle_dir = join('_python_bundle', '_python_bundle')
+            if 'python2legacy' in self.ctx.recipe_build_order:
+                # a special case with its own packaging location
+                python_bundle_dir = 'private'
+                # And also must had an install directory, make sure of that
+                self.ctx.python_recipe.create_python_install(self.dist_dir)
+
             self.distribute_libs(arch, [self.ctx.get_libs_dir(arch.arch)])
             self.distribute_javaclasses(self.ctx.javaclass_dir,
                                         dest_dir=join("src", "main", "java"))
 
-            python_bundle_dir = join('_python_bundle', '_python_bundle')
             ensure_dir(python_bundle_dir)
-
             site_packages_dir = self.ctx.python_recipe.create_python_bundle(
                 join(self.dist_dir, python_bundle_dir), arch)
 

pythonforandroid/bootstraps/sdl2/build/build.py

@@ -129,6 +129,7 @@ def make_python_zip():
         print('No compiled python is present to zip, skipping.')
         return
 
+    print('This should only be the case if you are using the python2legacy')
     global python_files
     d = realpath(join('private', 'lib', 'python2.7'))
 

pythonforandroid/bootstraps/sdl2/build/jni/src/start.c

@@ -176,6 +176,18 @@ int main(int argc, char *argv[]) {
    * replace sys.path with our path
    */
   PyRun_SimpleString("import sys, posix\n");
+  if (dir_exists("lib")) {
+    /* If we are in python2legacy, set up the paths correctly */
+    LOGP("Setting up python from ANDROID_PRIVATE");
+    PyRun_SimpleString("private = posix.environ['ANDROID_APP_PATH']\n"
+                       "argument = posix.environ['ANDROID_ARGUMENT']\n"
+                       "sys.path[:] = [ \n"
+                       "    private + '/lib/python27.zip', \n"
+                       "    private + '/lib/python2.7/', \n"
+                       "    private + '/lib/python2.7/lib-dynload/', \n"
+                       "    private + '/lib/python2.7/site-packages/', \n"
+                       "    argument ]\n");
+  }
 
   char add_site_packages_dir[256];
   if (dir_exists(crystax_python_dir)) {

pythonforandroid/bootstraps/service_only/__init__.py

@@ -9,7 +9,8 @@ class ServiceOnlyBootstrap(Bootstrap):
 
     name = 'service_only'
 
-    recipe_depends = ['genericndkbuild', ('python2', 'python3', 'python3crystax')]
+    recipe_depends = ['genericndkbuild',
+                      ('python2', 'python2legacy', 'python3', 'python3crystax')]
 
     def run_distribute(self):
         info_main('# Creating Android project from build and {} bootstrap'.format(
@@ -31,11 +32,17 @@ def run_distribute(self):
         with current_directory(self.dist_dir):
             info('Copying python distribution')
 
+            python_bundle_dir = join('_python_bundle', '_python_bundle')
+            if 'python2legacy' in self.ctx.recipe_build_order:
+                # a special case with its own packaging location
+                python_bundle_dir = 'private'
+                # And also must had an install directory, make sure of that
+                self.ctx.python_recipe.create_python_install(self.dist_dir)
+
             self.distribute_libs(arch, [self.ctx.get_libs_dir(arch.arch)])
             self.distribute_aars(arch)
             self.distribute_javaclasses(self.ctx.javaclass_dir)
 
-            python_bundle_dir = join('_python_bundle', '_python_bundle')
             ensure_dir(python_bundle_dir)
             site_packages_dir = self.ctx.python_recipe.create_python_bundle(
                 join(self.dist_dir, python_bundle_dir), arch)

pythonforandroid/bootstraps/service_only/build/jni/src/start.c

@@ -175,6 +175,18 @@ int main(int argc, char *argv[]) {
    * replace sys.path with our path
    */
   PyRun_SimpleString("import sys, posix\n");
+  if (dir_exists("lib")) {
+    /* If we are in python2legacy, set up the paths correctly */
+    LOGP("Setting up python from ANDROID_PRIVATE");
+    PyRun_SimpleString("private = posix.environ['ANDROID_APP_PATH']\n"
+                       "argument = posix.environ['ANDROID_ARGUMENT']\n"
+                       "sys.path[:] = [ \n"
+                       "    private + '/lib/python27.zip', \n"
+                       "    private + '/lib/python2.7/', \n"
+                       "    private + '/lib/python2.7/lib-dynload/', \n"
+                       "    private + '/lib/python2.7/site-packages/', \n"
+                       "    argument ]\n");
+  }
 
   char add_site_packages_dir[256];
   if (dir_exists(crystax_python_dir)) {

pythonforandroid/bootstraps/webview/__init__.py

@@ -7,7 +7,8 @@
 class WebViewBootstrap(Bootstrap):
     name = 'webview'
 
-    recipe_depends = ['genericndkbuild', ('python2', 'python3', 'python3crystax')]
+    recipe_depends = ['genericndkbuild',
+                      ('python2', 'python2legacy', 'python3', 'python3crystax')]
 
     def run_distribute(self):
         info_main('# Creating Android project from build and {} bootstrap'.format(
@@ -27,11 +28,17 @@ def run_distribute(self):
         with current_directory(self.dist_dir):
             info('Copying python distribution')
 
+            python_bundle_dir = join('_python_bundle', '_python_bundle')
+            if 'python2legacy' in self.ctx.recipe_build_order:
+                # a special case with its own packaging location
+                python_bundle_dir = 'private'
+                # And also must had an install directory, make sure of that
+                self.ctx.python_recipe.create_python_install(self.dist_dir)
+
             self.distribute_libs(arch, [self.ctx.get_libs_dir(arch.arch)])
             self.distribute_aars(arch)
             self.distribute_javaclasses(self.ctx.javaclass_dir)
 
-            python_bundle_dir = join('_python_bundle', '_python_bundle')
             ensure_dir(python_bundle_dir)
             site_packages_dir = self.ctx.python_recipe.create_python_bundle(
                 join(self.dist_dir, python_bundle_dir), arch)

pythonforandroid/bootstraps/webview/build/jni/src/start.c

@@ -175,6 +175,18 @@ int main(int argc, char *argv[]) {
    * replace sys.path with our path
    */
   PyRun_SimpleString("import sys, posix\n");
+  if (dir_exists("lib")) {
+    /* If we are in python2legacy, set up the paths correctly */
+    LOGP("Setting up python from ANDROID_PRIVATE");
+    PyRun_SimpleString("private = posix.environ['ANDROID_APP_PATH']\n"
+                       "argument = posix.environ['ANDROID_ARGUMENT']\n"
+                       "sys.path[:] = [ \n"
+                       "    private + '/lib/python27.zip', \n"
+                       "    private + '/lib/python2.7/', \n"
+                       "    private + '/lib/python2.7/lib-dynload/', \n"
+                       "    private + '/lib/python2.7/site-packages/', \n"
+                       "    argument ]\n");
+  }
 
   char add_site_packages_dir[256];
   if (dir_exists(crystax_python_dir)) {

pythonforandroid/build.py

@@ -541,6 +541,12 @@ def get_site_packages_dir(self, arch=None):
         '''Returns the location of site-packages in the python-install build
         dir.
         '''
+        if self.python_recipe.name == 'python2legacy':
+            return join(self.get_python_install_dir(),
+                        'lib', 'python2.7', 'site-packages')
+
+        # Only python2legacy is a special case, other
+        # python recipes use the python install dir
         return self.get_python_install_dir()
 
     def get_libs_dir(self, arch):