From 0de935425bbe1280bd57daf34e1194be0f52913d Mon Sep 17 00:00:00 2001 From: Mirko Galimberti Date: Sun, 1 May 2022 15:39:48 +0200 Subject: [PATCH 1/2] Introduces pythonforandroid/prerequisites.py (Experimental). This allows a more granular check and install process for dependencies on both CI jobs and users installation. --- .github/workflows/push.yml | 31 ++++ ci/makefiles/osx.mk | 12 +- pythonforandroid/prerequisites.py | 244 ++++++++++++++++++++++++++++++ 3 files changed, 276 insertions(+), 11 deletions(-) create mode 100644 pythonforandroid/prerequisites.py diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 4b16482c57..feff6bac88 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -5,6 +5,7 @@ on: ['push', 'pull_request'] env: APK_ARTIFACT_FILENAME: bdist_unit_tests_app-debug-1.1-.apk AAB_ARTIFACT_FILENAME: bdist_unit_tests_app-release-1.1-.aab + PYTHONFORANDROID_PREREQUISITES_INSTALL_INTERACTIVE: 0 jobs: @@ -107,6 +108,16 @@ jobs: steps: - name: Checkout python-for-android uses: actions/checkout@v2 + - name: Install python-for-android + run: | + source ci/osx_ci.sh + arm64_set_path_and_python_version 3.9.7 + python3 -m pip install -e . + - name: Install prerequisites via pythonforandroid/prerequisites.py (Experimental) + run: | + source ci/osx_ci.sh + arm64_set_path_and_python_version 3.9.7 + python3 pythonforandroid/prerequisites.py - name: Install dependencies run: | source ci/osx_ci.sh @@ -182,6 +193,16 @@ jobs: steps: - name: Checkout python-for-android uses: actions/checkout@v2 + - name: Install python-for-android + run: | + source ci/osx_ci.sh + arm64_set_path_and_python_version 3.9.7 + python3 -m pip install -e . + - name: Install prerequisites via pythonforandroid/prerequisites.py (Experimental) + run: | + source ci/osx_ci.sh + arm64_set_path_and_python_version 3.9.7 + python3 pythonforandroid/prerequisites.py - name: Install dependencies run: | source ci/osx_ci.sh @@ -258,6 +279,16 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 + - name: Install python-for-android + run: | + source ci/osx_ci.sh + arm64_set_path_and_python_version 3.9.7 + python3 -m pip install -e . + - name: Install prerequisites via pythonforandroid/prerequisites.py (Experimental) + run: | + source ci/osx_ci.sh + arm64_set_path_and_python_version 3.9.7 + python3 pythonforandroid/prerequisites.py - name: Install dependencies run: | source ci/osx_ci.sh diff --git a/ci/makefiles/osx.mk b/ci/makefiles/osx.mk index 8a04db154c..9395053a6e 100644 --- a/ci/makefiles/osx.mk +++ b/ci/makefiles/osx.mk @@ -3,12 +3,7 @@ # The following variable/s can be override when running the file ANDROID_HOME ?= $(HOME)/.android -all: install_java upgrade_cython install_android_ndk_sdk install_p4a - -install_java: - brew tap adoptopenjdk/openjdk - brew install --cask adoptopenjdk13 - /usr/libexec/java_home -V +all: upgrade_cython install_android_ndk_sdk upgrade_cython: pip3 install --upgrade Cython @@ -16,8 +11,3 @@ upgrade_cython: install_android_ndk_sdk: mkdir -p $(ANDROID_HOME) make -f ci/makefiles/android.mk JAVA_HOME=`/usr/libexec/java_home -v 13` - -install_p4a: - # check python version and install p4a - python3 --version - pip3 install -e . diff --git a/pythonforandroid/prerequisites.py b/pythonforandroid/prerequisites.py new file mode 100644 index 0000000000..4b01898716 --- /dev/null +++ b/pythonforandroid/prerequisites.py @@ -0,0 +1,244 @@ +#!/usr/bin/env python3 + +import sys +import platform +import os +import subprocess +from pythonforandroid.logger import info, warning, error + + +class Prerequisite(object): + name = "Default" + mandatory = True + darwin_installer_is_supported = False + linux_installer_is_supported = False + + def is_valid(self): + if self.checker(): + info(f"Prerequisite {self.name} is met") + return (True, "") + elif not self.mandatory: + warning( + f"Prerequisite {self.name} is not met, but is marked as non-mandatory" + ) + else: + error(f"Prerequisite {self.name} is not met") + + def checker(self): + if sys.platform == "darwin": + return self.darwin_checker() + elif sys.platform == "linux": + return self.linux_checker() + else: + raise Exception("Unsupported platform") + + def ask_to_install(self): + if ( + os.environ.get("PYTHONFORANDROID_PREREQUISITES_INSTALL_INTERACTIVE", "1") + == "1" + ): + res = input( + f"Do you want automatically install prerequisite {self.name}? [y/N] " + ) + if res.lower() == "y": + return True + else: + return False + else: + info( + "Session is not interactive (usually this happens during a CI run), so let's consider it as a YES" + ) + return True + + def install(self): + info(f"python-for-android can automatically install prerequisite: {self.name}") + if self.ask_to_install(): + if sys.platform == "darwin": + self.darwin_installer() + elif sys.platform == "linux": + self.linux_installer() + else: + raise Exception("Unsupported platform") + else: + info( + f"Skipping installation of prerequisite {self.name} as per user request" + ) + + def show_helper(self): + if sys.platform == "darwin": + self.darwin_helper() + elif sys.platform == "linux": + self.linux_helper() + else: + raise Exception("Unsupported platform") + + def install_is_supported(self): + if sys.platform == "darwin": + return self.darwin_installer_is_supported + elif sys.platform == "linux": + return self.linux_installer_is_supported + + def linux_checker(self): + raise Exception(f"Unsupported prerequisite check on linux for {self.name}") + + def darwin_checker(self): + raise Exception(f"Unsupported prerequisite check on macOS for {self.name}") + + def linux_installer(self): + raise Exception(f"Unsupported prerequisite installer on linux for {self.name}") + + def darwin_installer(self): + raise Exception(f"Unsupported prerequisite installer on macOS for {self.name}") + + def darwin_helper(self): + info(f"No helper available for prerequisite: {self.name} on macOS") + + def linux_helper(self): + info(f"No helper available for prerequisite: {self.name} on linux") + + +class JDKPrerequisite(Prerequisite): + name = "JDK" + mandatory = True + darwin_installer_is_supported = True + min_supported_version = 11 + + def darwin_checker(self): + if "JAVA_HOME" in os.environ: + info("Found JAVA_HOME environment variable, using it") + jdk_path = os.environ["JAVA_HOME"] + else: + jdk_path = self._darwin_get_libexec_jdk_path(version=None) + return self._darwin_jdk_is_supported(jdk_path) + + def _darwin_get_libexec_jdk_path(self, version=None): + version_args = [] + if version is not None: + version_args = ["-v", version] + return ( + subprocess.run( + ["/usr/libexec/java_home", *version_args], + stdout=subprocess.PIPE, + ) + .stdout.strip() + .decode() + ) + + def _darwin_jdk_is_supported(self, jdk_path): + if not jdk_path: + return False + + javac_bin = os.path.join(jdk_path, "bin", "javac") + if not os.path.exists(javac_bin): + return False + + p = subprocess.Popen( + [javac_bin, "-version"], stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + _stdout_res, _stderr_res = p.communicate() + + if p.returncode != 0: + error("Failed to run javac to check JDK version") + return False + + if not _stdout_res: + _stdout_res = _stderr_res + + res = _stdout_res.strip().decode() + + major_version = int(res.split(" ")[-1].split(".")[0]) + if major_version >= self.min_supported_version: + info(f"Found a valid JDK at {jdk_path}") + return True + else: + error(f"JDK {self.min_supported_version} or higher is required") + return False + + def darwin_helper(self): + info( + "python-for-android requires a JDK 11 or higher to be installed on macOS," + "but seems like you don't have one installed." + ) + info( + "If you think that a valid JDK is already installed, please verify that " + "you have a JDK 11 or higher installed and that `/usr/libexec/java_home` " + "shows the correct path." + ) + info( + "If you have multiple JDK installations, please make sure that you have " + "`JAVA_HOME` environment variable set to the correct JDK installation." + ) + + def darwin_installer(self): + info( + "Looking for a JDK 11 or higher installation which is not the default one ..." + ) + jdk_path = self._darwin_get_libexec_jdk_path(version="11+") + + if not self._darwin_jdk_is_supported(jdk_path): + info("We're unlucky, there's no JDK 11 or higher installation available") + + base_url = "https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.2%2B8/" + if platform.machine() == "arm64": + filename = "OpenJDK17U-jdk_aarch64_mac_hotspot_17.0.2_8.tar.gz" + else: + filename = "OpenJDK17U-jdk_x64_mac_hotspot_17.0.2_8.tar.gz" + + info(f"Downloading {filename} from {base_url}") + subprocess.check_output( + [ + "curl", + "-L", + f"{base_url}{filename}", + "-o", + f"/tmp/{filename}", + ] + ) + + user_library_java_path = os.path.expanduser( + "~/Library/Java/JavaVirtualMachines" + ) + info(f"Extracting {filename} to {user_library_java_path}") + subprocess.check_output( + [ + "mkdir", + "-p", + user_library_java_path, + ], + ) + subprocess.check_output( + ["tar", "xzf", f"/tmp/{filename}", "-C", user_library_java_path], + ) + + jdk_path = self._darwin_get_libexec_jdk_path(version="17.0.2+8") + + info(f"Setting JAVA_HOME to {jdk_path}") + os.environ["JAVA_HOME"] = jdk_path + + +if __name__ == "__main__": + DEFAULT_PREREQUISITES = dict(darwin=[JDKPrerequisite()], linux=[], all_platforms=[]) + + required_prerequisites = ( + DEFAULT_PREREQUISITES["all_platforms"] + DEFAULT_PREREQUISITES[sys.platform] + ) + + prerequisites_not_met = [] + + warning( + "prerequisites.py is experimental and does not support all prerequisites yet." + ) + warning("Please report any issues to the python-for-android issue tracker.") + + # Phase 1: Check if all prerequisites are met and add the ones + # which are not to `prerequisites_not_met` + for prerequisite in required_prerequisites: + if not prerequisite.is_valid(): + prerequisites_not_met.append(prerequisite) + + # Phase 2: Setup/Install all prerequisites that are not met + # (where possible), otherwise show an helper. + for prerequisite in prerequisites_not_met: + prerequisite.show_helper() + if prerequisite.install_is_supported(): + prerequisite.install() From 72e97207a6a491736f4a62d784b85762270ca539 Mon Sep 17 00:00:00 2001 From: Mirko Galimberti Date: Sat, 7 May 2022 17:36:31 +0200 Subject: [PATCH 2/2] Add check_and_install_default_prerequisites so ENV vars are set into the subprocess --- pythonforandroid/prerequisites.py | 6 +++++- pythonforandroid/toolchain.py | 2 ++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/pythonforandroid/prerequisites.py b/pythonforandroid/prerequisites.py index 4b01898716..f9cbb2c993 100644 --- a/pythonforandroid/prerequisites.py +++ b/pythonforandroid/prerequisites.py @@ -216,7 +216,7 @@ def darwin_installer(self): os.environ["JAVA_HOME"] = jdk_path -if __name__ == "__main__": +def check_and_install_default_prerequisites(): DEFAULT_PREREQUISITES = dict(darwin=[JDKPrerequisite()], linux=[], all_platforms=[]) required_prerequisites = ( @@ -242,3 +242,7 @@ def darwin_installer(self): prerequisite.show_helper() if prerequisite.install_is_supported(): prerequisite.install() + + +if __name__ == "__main__": + check_and_install_default_prerequisites() diff --git a/pythonforandroid/toolchain.py b/pythonforandroid/toolchain.py index 2636eaca5b..520012d55b 100644 --- a/pythonforandroid/toolchain.py +++ b/pythonforandroid/toolchain.py @@ -13,6 +13,7 @@ RECOMMENDED_NDK_API, RECOMMENDED_TARGET_API, print_recommendations) from pythonforandroid.util import BuildInterruptingException, load_source from pythonforandroid.entrypoints import main +from pythonforandroid.prerequisites import check_and_install_default_prerequisites def check_python_dependencies(): @@ -66,6 +67,7 @@ def check_python_dependencies(): exit(1) +check_and_install_default_prerequisites() check_python_dependencies()