diff --git a/.github/workflows/build_apk.yml b/.github/workflows/build_apk.yml new file mode 100644 index 00000000..46d9e788 --- /dev/null +++ b/.github/workflows/build_apk.yml @@ -0,0 +1,96 @@ +name: Build Android App + +on: + workflow_dispatch: + # Inputs the workflow accepts. + inputs: + whl-url: + description: 'URL for Kolibri whl file' + required: true + arch: + description: 'Architecture of the build' + required: true + default: '32bit' + type: choice + options: + - 32bit + - 64bit + workflow_call: + inputs: + whl-file-name: + required: true + type: string + arch: + description: 'Architecture of the build: 32bit or 64bit' + required: true + type: string + ref: + description: 'A ref for this workflow to check out its own repo' + required: true + type: string + outputs: + apk-file-name: + description: "APK file name" + value: ${{ jobs.build_apk.outputs.apk-file-name }} + +jobs: + build_apk: + runs-on: ubuntu-latest + outputs: + apk-file-name: ${{ steps.get-apk-filename.outputs.apk-file-name }} + steps: + - uses: actions/checkout@v2 + if: ${{ !inputs.ref }} + - uses: actions/checkout@v2 + if: ${{ inputs.ref }} + with: + repository: learningequality/kolibri-android-installer + ref: ${{ inputs.ref }} + - name: Set up Python 3.9 + uses: actions/setup-python@v2 + with: + python-version: 3.9 + - uses: actions/cache@v2 + with: + # This is where python for android puts its intermediary build + # files - we cache this to improve build performance, but be + # aggressive in clearing the cache whenever any file changes + # in the repository. + path: ~/.local + key: ${{ runner.os }}-local-${{ github.event.inputs.arch || inputs.arch }}-${{ hashFiles('*') }} + restore-keys: | + ${{ runner.os }}-local-${{ github.event.inputs.arch || inputs.arch }}- + - uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + - name: Download the whlfile from URL + if: ${{ github.event.inputs.whl-url }} + run: make get-whl whl=${{ github.event.inputs.whl-url }} + - name: Download the whlfile from artifacts + if: ${{ inputs.whl-file-name }} + uses: actions/download-artifact@v2 + with: + name: ${{ inputs.whl-file-name }} + path: whl + - name: Install dependencies + run: pip install -r requirements.txt + - name: Build the app + env: + ARCH: ${{ github.event.inputs.arch || inputs.arch }} + # No need to set the ANDROID_HOME environment variable here that is used for + # setting up the ANDROID SDK and ANDROID NDK, as the github actions images + # have these SDKs and NDKs already installed. + # https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md#environment-variables-3 + ANDROIDSDK: ${{ env.ANDROID_SDK_ROOT }} + ANDROIDNDK: ${{ env.ANDROID_NDK_ROOT }} + run: make kolibri.apk.unsigned + - name: Get APK filename + id: get-apk-filename + run: echo "::set-output name=apk-file-name::$(ls dist | grep .apk | cat)" + - uses: actions/upload-artifact@v2 + with: + name: ${{ steps.get-apk-filename.outputs.apk-file-name }} + path: dist/${{ steps.get-apk-filename.outputs.apk-file-name }} diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml new file mode 100644 index 00000000..eee2234a --- /dev/null +++ b/.github/workflows/pre-commit.yml @@ -0,0 +1,28 @@ +name: Linting + +on: [push, pull_request] + +jobs: + pre_job: + name: Path match check + runs-on: ubuntu-latest + # Map a step output to a job output + outputs: + should_skip: ${{ steps.skip_check.outputs.should_skip }} + steps: + - id: skip_check + uses: fkirc/skip-duplicate-actions@master + with: + github_token: ${{ github.token }} + paths_ignore: '["**.po", "**.json"]' + linting: + name: All file linting + needs: pre_job + if: ${{ needs.pre_job.outputs.should_skip != 'true' }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: 3.9 + - uses: pre-commit/action@v2.0.0 diff --git a/.gitignore b/.gitignore index b6dfa32e..24bc08a3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,22 +1,20 @@ src/kolibri -src/preseeded_kolibri_home -src/extra-packages tmpenv whl/* # File format for signing key *.jks -# Generated for pew -project_info.json -# pew output folder +# output folder dist/ __pycache__ *.pyc build_docker -build/ bin/ build.log tmphome/ -static/ \ No newline at end of file +android_root/ + +*.apk +.env diff --git a/.p4a b/.p4a new file mode 100644 index 00000000..1c114717 --- /dev/null +++ b/.p4a @@ -0,0 +1,22 @@ +--window +--bootstrap "webview" +--package "org.learningequality.Kolibri" +--name "Kolibri" +--dist_name "kolibri" +--private "src" +--orientation sensor +--requirements python3,android,pyjnius,genericndkbuild,sqlite3,cryptography,twisted,attrs,bcrypt,service_identity,pyasn1,pyasn1_modules,pyopenssl,openssl,six +--android-api 30 +--minsdk 21 +--permission WRITE_EXTERNAL_STORAGE +--permission ACCESS_NETWORK_STATE +--permission FOREGROUND_SERVICE +--service server:server.py +--service remoteshell:remoteshell.py +--presplash assets/icon.png +--presplash-color #FFFFFF +--icon assets/icon.png +--fileprovider-paths src/fileprovider_paths.xml +--add-asset assets/_load.html:_load.html +--whitelist ./allowlist.txt +--blacklist ./blocklist.txt diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..d5d936a9 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,30 @@ +repos: + - repo: https://github.com/python/black + rev: 22.3.0 + hooks: + - id: black + types_or: [ python, pyi ] + exclude: '^.+?\.template$' + - repo: https://github.com/pycqa/flake8 + rev: 4.0.1 + hooks: + - id: flake8 + exclude: '^.+?\.template$' + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.1.0 + hooks: + - id: trailing-whitespace + exclude: '^.+?\.template$' + - id: check-yaml + exclude: '^.+?\.template$' + - id: check-added-large-files + exclude: '^.+?\.template$' + - id: debug-statements + exclude: '^.+?\.template$' + - id: end-of-file-fixer + exclude: '^.+?(\.json|\.template)$' + + - repo: https://github.com/asottile/reorder_python_imports + rev: v2.6.0 + hooks: + - id: reorder-python-imports diff --git a/Dockerfile b/Dockerfile index 4cefe632..851ef90e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -49,22 +49,8 @@ ENV PATH /usr/local/bin:$PATH RUN cd /usr/local/bin && \ ln -s $(which python3) python -ENV PEW_BRANCH=p4a_update -ENV P4A_BRANCH=pew_webview - -# Allows us to invalidate cache if those repos update. -# Intentionally not pinning for dev velocity. -ADD https://github.com/learningequality/pyeverywhere/archive/$PEW_BRANCH.zip pew.zip -ADD https://github.com/learningequality/python-for-android/archive/$P4A_BRANCH.zip p4a.zip - -# clean up the pew cache if the source repos changed -RUN rm -r /home/kivy/.pyeverywhere || true - # install python dependencies -RUN pip install cython virtualenv pbxproj && \ - # get custom packages - pip install -e git+https://github.com/learningequality/pyeverywhere@$PEW_BRANCH#egg=pyeverywhere && \ - pip install -e git+https://github.com/learningequality/python-for-android@$P4A_BRANCH#egg=python-for-android && \ +RUN pip install -r requirements.txt && \ useradd -lm kivy RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \ @@ -78,10 +64,14 @@ WORKDIR /home/kivy ARG ARCH=$ARCH # Initializes the directory, owned by new user. Volume mounts adopt existing permissions, etc. -RUN mkdir ~/.local ~/.pyeverywhere +RUN mkdir ~/.local COPY --chown=kivy:kivy . . +RUN make setup + +RUN set -a; source .env; set +a + ENTRYPOINT [ "make" ] CMD [ "kolibri.apk" ] diff --git a/Makefile b/Makefile index 635ec56c..08437d2e 100644 --- a/Makefile +++ b/Makefile @@ -1,48 +1,101 @@ # run with envvar `ARCH=64bit` to build for v8a ifeq (${ARCH}, 64bit) - ARM_VER=v8a + ARM_VER := v8a + P4A_ARCH := arm64-v8a else - ARM_VER=v7a + ARM_VER := v7a + P4A_ARCH := armeabi-v7a endif +OSNAME := $(shell uname -s) + +ifeq ($(OSNAME), Darwin) + PLATFORM := macosx +else + PLATFORM := linux +endif + +ANDROID_API := 30 +ANDROIDNDKVER := 21.4.7075529 + +SDK := ${ANDROID_HOME}/android-sdk-$(PLATFORM) + +# This checks if an environment variable with a specific name +# exists. If it doesn't, it prints an error message and exits. +# For example to check for the presence of the ANDROIDSDK environment +# variable, you could use: +# make guard-ANDROIDSDK +guard-%: + @ if [ "${${*}}" = "" ]; then \ + echo "Environment variable $* not set"; \ + exit 1; \ + fi + +needs-android-dirs: + $(MAKE) guard-ANDROIDSDK + $(MAKE) guard-ANDROIDNDK + # Clear out apks clean: - - rm -rf dist/android/*.apk project_info.json ./src/kolibri + - rm -rf dist/*.apk src/kolibri tmpenv deepclean: clean python-for-android clean_dists - rm -r build || true rm -r dist || true yes y | docker system prune -a || true rm build_docker 2> /dev/null +.PHONY: clean-whl +clean-whl: + rm -rf whl + mkdir whl + +.PHONY: get-whl +get-whl: clean-whl +# The eval and shell commands here are evaluated when the recipe is parsed, so we put the cleanup +# into a prerequisite make step, in order to ensure they happen prior to the download. + $(eval DLFILE = $(shell wget --content-disposition -P whl/ "${whl}" 2>&1 | grep "Saving to: " | sed 's/Saving to: ‘//' | sed 's/’//')) + $(eval WHLFILE = $(shell echo "${DLFILE}" | sed "s/\?.*//")) + [ "${DLFILE}" = "${WHLFILE}" ] || mv "${DLFILE}" "${WHLFILE}" + # Extract the whl file src/kolibri: clean rm -r src/kolibri 2> /dev/null || true - unzip -qo "whl/kolibri*.whl" "kolibri/*" -x "kolibri/dist/cext*" -d src/ + unzip -qo "whl/kolibri*.whl" "kolibri/*" -x "kolibri/dist/py2only*" -d src/ # patch Django to allow migrations to be pyc files, as p4a compiles and deletes the originals sed -i 's/if name.endswith(".py"):/if name.endswith(".py") or name.endswith(".pyc"):/g' src/kolibri/dist/django/db/migrations/loader.py - ./delete_kolibri_blacklist.sh - -# Generate the project info file -project_info.json: project_info.template src/kolibri scripts/create_project_info.py - python ./scripts/create_project_info.py .PHONY: p4a_android_distro -p4a_android_distro: whitelist.txt project_info.json - pew init android ${ARCH} +p4a_android_distro: needs-android-dirs + p4a create --arch=$(P4A_ARCH) -ifdef P4A_RELEASE_KEYSTORE_PASSWD -pew_release_flag = --release -endif +.PHONY: needs-version +needs-version: + $(eval APK_VERSION ?= $(shell python3 scripts/version.py apk_version)) + $(eval BUILD_NUMBER ?= $(shell python3 scripts/version.py build_number)) .PHONY: kolibri.apk -# Build the debug version of the apk -kolibri.apk: p4a_android_distro preseeded_kolibri_home - echo "--- :android: Build APK" - pew build $(pew_release_flag) android ${ARCH} - +# Build the signed version of the apk +# For some reason, p4a defauls to adding a final '-' to the filename, so we remove it in the final step. +kolibri.apk: p4a_android_distro src/kolibri needs-version + $(MAKE) guard-P4A_RELEASE_KEYSTORE + $(MAKE) guard-P4A_RELEASE_KEYALIAS + $(MAKE) guard-P4A_RELEASE_KEYSTORE_PASSWD + $(MAKE) guard-P4A_RELEASE_KEYALIAS_PASSWD + @echo "--- :android: Build APK" + p4a apk --release --sign --arch=$(P4A_ARCH) --version=$(APK_VERSION) --numeric-version=$(BUILD_NUMBER) + mkdir -p dist + mv kolibri__$(P4A_ARCH)-$(APK_VERSION)-.apk dist/kolibri__$(P4A_ARCH)-$(APK_VERSION).apk + +.PHONY: kolibri.apk.unsigned +# Build the unsigned debug version of the apk +# For some reason, p4a defauls to adding a final '-' to the filename, so we remove it in the final step. +kolibri.apk.unsigned: p4a_android_distro src/kolibri needs-version + @echo "--- :android: Build APK (unsigned)" + p4a apk --arch=$(P4A_ARCH) --version=$(APK_VERSION) --numeric-version=$(BUILD_NUMBER) + mkdir -p dist + mv kolibri__$(P4A_ARCH)-debug-$(APK_VERSION)-.apk dist/kolibri__$(P4A_ARCH)-debug-$(APK_VERSION).apk # DOCKER BUILD @@ -52,39 +105,45 @@ kolibri.apk: p4a_android_distro preseeded_kolibri_home build_docker: Dockerfile docker build -t android_kolibri . -preseeded_kolibri_home: export KOLIBRI_HOME := src/preseeded_kolibri_home -preseeded_kolibri_home: export PYTHONPATH := tmpenv -preseeded_kolibri_home: - rm -r tmpenv 2> /dev/null || true - rm -r src/preseeded_kolibri_home 2> /dev/null || true - pip uninstall kolibri 2> /dev/null || true - pip install --target tmpenv whl/*.whl - tmpenv/bin/kolibri plugin enable kolibri.plugins.app - tmpenv/bin/kolibri start --port=0 - sleep 1 - tmpenv/bin/kolibri stop - sleep 1 - yes yes | tmpenv/bin/kolibri manage deprovision - rm -r src/preseeded_kolibri_home/logs 2> /dev/null || true - rm -r src/preseeded_kolibri_home/sessions 2> /dev/null || true - rm -r src/preseeded_kolibri_home/process_cache 2> /dev/null || true - touch src/preseeded_kolibri_home/was_preseeded - # Run the docker image. # TODO Would be better to just specify the file here? run_docker: build_docker ./scripts/rundocker.sh -softbuild: project_info.json - pew build $(pew_release_flag) android ${ARCH} - install: adb uninstall org.learningequality.Kolibri || true 2> /dev/null - adb install dist/android/*$(ARM_VER)-debug-*.apk + adb install dist/*$(ARM_VER)-debug-*.apk -run: install - adb shell am start -n org.learningequality.Kolibri/org.kivy.android.PythonActivity - sleep 1 +logcat: adb logcat | grep -i -E "python|kolibr| `adb shell ps | grep ' org.learningequality.Kolibri$$' | tr -s [:space:] ' ' | cut -d' ' -f2` " | grep -E -v "WifiTrafficPoller|localhost:5000|NetworkManagementSocketTagger|No jobs to start" -launch: softbuild run +$(SDK)/cmdline-tools: + @echo "Downloading Android SDK build tools" + wget https://dl.google.com/android/repository/commandlinetools-$(PLATFORM)-7583922_latest.zip + unzip commandlinetools-$(PLATFORM)-7583922_latest.zip -d $(SDK) + rm commandlinetools-$(PLATFORM)-7583922_latest.zip + +sdk: + yes y | $(SDK)/cmdline-tools/bin/sdkmanager "platform-tools" --sdk_root=$(SDK) + yes y | $(SDK)/cmdline-tools/bin/sdkmanager "platforms;android-$(ANDROID_API)" --sdk_root=$(SDK) + yes y | $(SDK)/cmdline-tools/bin/sdkmanager "system-images;android-$(ANDROID_API);default;x86_64" --sdk_root=$(SDK) + yes y | $(SDK)/cmdline-tools/bin/sdkmanager "build-tools;30.0.3" --sdk_root=$(SDK) + yes y | $(SDK)/cmdline-tools/bin/sdkmanager "ndk;$(ANDROIDNDKVER)" --sdk_root=$(SDK) + @echo "Accepting all licenses" + yes | $(SDK)/cmdline-tools/bin/sdkmanager --licenses --sdk_root=$(SDK) + +# All of these commands are non-destructive, so if the cmdline-tools are already installed, make will skip +# based on the directory existing. +# The SDK installations will take a little time, but will not attempt to redownload if already installed. +setup: + $(MAKE) guard-ANDROID_HOME + mkdir -p $(SDK) + $(MAKE) $(SDK)/cmdline-tools + $(MAKE) sdk + @echo "Make sure to set the necessary environment variables" + @echo "export ANDROIDSDK=$(SDK)" + @echo "export ANDROIDNDK=$(SDK)/ndk/$(ANDROIDNDKVER)" + @echo "ANDROIDSDK=$(SDK)\nANDROIDNDK=$(SDK)/ndk/$(ANDROIDNDKVER)" > .env + +clean-tools: + rm -rf ${ANDROID_HOME} diff --git a/README.md b/README.md index 8f166a07..40bafeb8 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Kolibri Android Installer -Wraps Kolibri in an android-compatibility layer. Uses PyEverywhere to automate build, and relies on Python-For-Android for compatibility on the Android platform. +Wraps Kolibri in an android-compatibility layer. Relies on Python-For-Android to build the APK and for compatibility on the Android platform. ## Build on Docker @@ -14,27 +14,34 @@ This project was primarily developed on Docker, so this method is more rigorousl 4. The generated APK will end up in the `bin/` folder. -## Build on Host +## Building for Development -1. Install [PyEverywhere](https://github.com/learningequality/pyeverywhere) and its build dependencies (for building Python). For Ubuntu, you can find the list of dependencies in the [Dockerfile](./Dockerfile). +1. Install the Android SDK and Android NDK. -2. Build or download a Kolibri WHL file, and place it in the `whl/` directory. +Run `make setup`. +Follow the instructions from the command to set your environment variables. -3. Run `make Kolibri*.apk` to set up the build environment (downloads depenedencies and sets up project template) on first run, then build the apk. Watch for success at the end, or errors, which might indicate missing build dependencies or build errors. If successful, there should be an APK in the `bin/` directory. +2. Install the Python dependencies: -## Installing the apk -1. Connect your Android device over USB, with USB Debugging enabled. +`pip install -r requirements.txt` -2. Ensure that `adb devices` brings up your device. Afterward, run `adb install bin/Kolibri-*-debug.apk` to install onto the device. (Assuming you built the apk in `debug` mode, the default. +3. Ensure you have all [necessary packages for Python for Android](https://python-for-android.readthedocs.io/en/latest/quickstart/#installing-dependencies). +4. Build or download a Kolibri WHL file, and place it in the `whl/` directory. -## Running the apk from the terminal +To download a Kolibri WHL file, you can use `make whl=` from the command line. It will download it and put it in the correct directory. -### If you have `pew` installed +5. If you need a 64bit build, set the `ARCH` environment variable to `64bit`. -1. Run `pew run android`. You will be able to monitor the output in the terminal. The app should show a black screen and then a loading screen. +6. Run `make kolibri.apk.unsigned` to build the apk. Watch for success at the end, or errors, which might indicate missing build dependencies or build errors. If successful, there should be an APK in the `dist/` directory. -### If you only have `adb` installed +## Installing the apk +1. Connect your Android device over USB, with USB Debugging enabled. + +2. Ensure that `adb devices` brings up your device. Afterward, run `make install` to install onto the device. + + +## Running the apk from the terminal 1. Run `adb shell am start -n org.learningequality.Kolibri/org.kivy.android.PythonActivity` diff --git a/allowlist.txt b/allowlist.txt new file mode 100644 index 00000000..fc0fa20e --- /dev/null +++ b/allowlist.txt @@ -0,0 +1,9 @@ +# Ensure that Django's SQLite backend module is included +sqlite3/* +lib-dynload/_sqlite3.so +unittest/* +wsgiref/* +lib-dynload/_csv.so +lib-dynload/_json.so +# Django REST framework has a dependency on the Django test module +django/test/* diff --git a/assets/_load.html b/assets/_load.html index f2823a66..d29f561e 100644 --- a/assets/_load.html +++ b/assets/_load.html @@ -299,4 +299,4 @@

Starting Kolibri

- \ No newline at end of file + diff --git a/assets/icon.png b/assets/icon.png new file mode 100644 index 00000000..9b29237c Binary files /dev/null and b/assets/icon.png differ diff --git a/assets/launch-image.png b/assets/launch-image.png deleted file mode 100644 index 586e6f8b..00000000 Binary files a/assets/launch-image.png and /dev/null differ diff --git a/blocklist.txt b/blocklist.txt new file mode 100644 index 00000000..e392c2d5 --- /dev/null +++ b/blocklist.txt @@ -0,0 +1,49 @@ +# remove some assorted additional plugins +kolibri/plugins/demo_server/* + +# remove python2-only stuff +kolibri/dist/py2only/* + +# Remove cextensions +kolibri/dist/cext* + +# remove source maps +*.js.map + +# remove unused translation files from django and other apps +kolibri/dist/rest_framework/locale/* +kolibri/dist/django_filters/locale/* +kolibri/dist/mptt/locale/* + +kolibri/dist/django/contrib/admindocs/locale/* +kolibri/dist/django/contrib/auth/locale/* +kolibri/dist/django/contrib/sites/locale/* +kolibri/dist/django/contrib/contenttypes/locale/* +kolibri/dist/django/contrib/flatpages/locale/* +kolibri/dist/django/contrib/sessions/locale/* +kolibri/dist/django/contrib/humanize/locale/* +kolibri/dist/django/contrib/admin/locale/* + +# remove some django components entirely +kolibri/dist/django/contrib/gis/* +kolibri/dist/django/contrib/redirects/* +kolibri/dist/django/conf/app_template/* +kolibri/dist/django/conf/project_template/* +kolibri/dist/django/db/backends/postgresql_psycopg2/* +kolibri/dist/django/db/backends/postgresql/* +kolibri/dist/django/db/backends/mysql/* +kolibri/dist/django/db/backends/oracle/* +kolibri/dist/django/contrib/postgres/* + +# remove bigger chunks of django admin (may not want to do this) +kolibri/dist/django/contrib/admin/static/* +kolibri/dist/django/contrib/admin/templates/* + +# other assorted testing stuff +*/test/* +*/tests/* +kolibri/dist/tzlocal/test_data/* + +# remove some unnecessary apps +kolibri/dist/redis_cache/* +kolibri/dist/redis/* diff --git a/delete_kolibri_blacklist.sh b/delete_kolibri_blacklist.sh deleted file mode 100755 index 9c31d380..00000000 --- a/delete_kolibri_blacklist.sh +++ /dev/null @@ -1,53 +0,0 @@ -# remove some assorted additional plugins -rm -r src/kolibri/plugins/demo_server - -# remove python2-only stuff -rm -r src/kolibri/dist/py2only - -# remove pycountry and replace with stub -# (only used by getlang_by_alpha2 in le-utils, which Kolibri doesn't call) -rm -r src/kolibri/dist/pycountry/* -touch src/kolibri/dist/pycountry/__init__.py - -# remove source maps -find src/kolibri -name "*.js.map" -type f -delete - -# remove node_modules (contains only core-js) -rm -r src/kolibri/core/node_modules - -# remove unused translation files from django and other apps -rm -r src/kolibri/dist/rest_framework/locale -rm -r src/kolibri/dist/django_filters/locale -rm -r src/kolibri/dist/mptt/locale - -rm -r src/kolibri/dist/django/contrib/admindocs/locale -rm -r src/kolibri/dist/django/contrib/auth/locale -rm -r src/kolibri/dist/django/contrib/sites/locale -rm -r src/kolibri/dist/django/contrib/contenttypes/locale -rm -r src/kolibri/dist/django/contrib/flatpages/locale -rm -r src/kolibri/dist/django/contrib/sessions/locale -rm -r src/kolibri/dist/django/contrib/humanize/locale -rm -r src/kolibri/dist/django/contrib/admin/locale - -# remove some django components entirely -rm -r src/kolibri/dist/django/contrib/gis -rm -r src/kolibri/dist/django/contrib/redirects -rm -r src/kolibri/dist/django/conf/app_template -rm -r src/kolibri/dist/django/conf/project_template -rm -r src/kolibri/dist/django/db/backends/postgresql_psycopg2 -rm -r src/kolibri/dist/django/db/backends/postgresql -rm -r src/kolibri/dist/django/db/backends/mysql -rm -r src/kolibri/dist/django/db/backends/oracle -rm -r src/kolibri/dist/django/contrib/postgres - -# remove bigger chunks of django admin (may not want to do this) -rm -r src/kolibri/dist/django/contrib/admin/static -rm -r src/kolibri/dist/django/contrib/admin/templates - -# other assorted testing stuff -find src/kolibri -wholename "*/test/*" -not -wholename "*/django/test/*" -delete -rm -r src/kolibri/dist/tzlocal/test_data - -# remove some unnecessary apps -rm -r src/kolibri/dist/redis_cache -rm -r src/kolibri/dist/redis \ No newline at end of file diff --git a/icon.png b/icon.png deleted file mode 100644 index 5bbbab8f..00000000 Binary files a/icon.png and /dev/null differ diff --git a/large_icon.png b/large_icon.png deleted file mode 100644 index 4d1bfda1..00000000 Binary files a/large_icon.png and /dev/null differ diff --git a/logcat.sh b/logcat.sh index bedb3d57..3b7cb79a 100755 --- a/logcat.sh +++ b/logcat.sh @@ -1,10 +1,9 @@ main_pid=$(adb shell ps | grep ' org.learningequality.Kolibri$' | tr -s [:space:] ' ' | cut -d' ' -f2) server_pid=$(adb shell ps | grep ' org.learningequality.Kolibri:service_kolibri$' | tr -s [:space:] ' ' | cut -d' ' -f2) -exclusion="NetworkManagementSocketTagger|Could not ping|No jobs|port:5000" if [ -z "$server_pid" ]; then echo "Searching for: $main_pid" adb logcat | grep -E " $main_pid " else echo "Searching for: $main_pid | $server_pid " | egrep -v "$exclusion" adb logcat | grep -E " $main_pid | $server_pid " | egrep -v "$exclusion" -fi \ No newline at end of file +fi diff --git a/project_info.template b/project_info.template deleted file mode 100644 index 1cd2516a..00000000 --- a/project_info.template +++ /dev/null @@ -1,20 +0,0 @@ -{ - "name": "Kolibri", - "version": "${apk_version}", - "build_number": "${build_number}", - "identifier": "org.learningequality.Kolibri", - "requirements": {"android": ["python3", "android", "pyjnius", "genericndkbuild", "sqlite3", "cryptography", "twisted", "attrs", "bcrypt", "service_identity", "pyasn1", "pyasn1_modules", "pyopenssl", "openssl", "six", "flask"]}, - "whitelist_file": {"android": "whitelist.txt"}, - "icons": {"android": "icon.png"}, - "launch_images": {"android": "assets/launch-image.png"}, - "asset_dirs": ["assets"], - "extra_build_options": { - "android": { - "services": ["kolibri:android_service.py"], - "extra_permissions": ["FOREGROUND_SERVICE"], - "sdk": 30, - "minsdk": 21, - "fileprovider_paths_filename": "fileprovider_paths.xml" - } - } -} diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..cd4a7931 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +cython +virtualenv +git+https://github.com/learningequality/python-for-android@350d7158a0a35f578a95d50de969101579bbdc4f#egg=python-for-android diff --git a/scripts/create_dummy_project_info.py b/scripts/create_dummy_project_info.py deleted file mode 100644 index 0dc354e2..00000000 --- a/scripts/create_dummy_project_info.py +++ /dev/null @@ -1,6 +0,0 @@ -from string import Template - -with open('project_info.template', 'r') as pi_template_file, open('./project_info.json', 'w') as pi_file: - pi_template = Template(pi_template_file.read()) - pi = pi_template.substitute(apk_version='0', build_number='0') - pi_file.write(pi) diff --git a/scripts/rundocker.sh b/scripts/rundocker.sh index 80810558..d16d4b19 100755 --- a/scripts/rundocker.sh +++ b/scripts/rundocker.sh @@ -6,7 +6,6 @@ CONTAINER_HOME=/home/kivy # Specifies the name of the docker volume used to store p4a cache P4A_CACHE=p4a_cached_dir_$ARCH -PEW_CACHE=pew_cache_dir CID_FILE=kolibri-android-app-container-id.cid.txt @@ -14,13 +13,13 @@ CID_FILE=kolibri-android-app-container-id.cid.txt # creates a volume for reuse between builds, holding p4a's android distro docker create -it \ --mount type=volume,src=${P4A_CACHE},dst=${CONTAINER_HOME}/.local \ - --mount type=volume,src=${PEW_CACHE},dst=${CONTAINER_HOME}/.pyeverywhere \ --env BUILDKITE_BUILD_NUMBER \ --env P4A_RELEASE_KEYSTORE=${CONTAINER_HOME}/$(basename "${P4A_RELEASE_KEYSTORE}") \ --env P4A_RELEASE_KEYALIAS \ --env P4A_RELEASE_KEYSTORE_PASSWD \ --env P4A_RELEASE_KEYALIAS_PASSWD \ --env ARCH \ + --env ANDROID_HOME=${CONTAINER_HOME}/.local/android \ --cidfile ${CID_FILE} \ android_kolibri diff --git a/scripts/create_project_info.py b/scripts/version.py similarity index 59% rename from scripts/create_project_info.py rename to scripts/version.py index ef8fbc95..454e789d 100644 --- a/scripts/create_project_info.py +++ b/scripts/version.py @@ -1,22 +1,21 @@ import os -import re import subprocess +import sys from datetime import datetime -from string import Template def kolibri_version(): """ Returns the major.minor version of Kolibri if it exists """ - with open('./src/kolibri/VERSION', 'r') as version_file: + with open("./src/kolibri/VERSION", "r") as version_file: # p4a only likes digits and decimals return version_file.read().strip() + def commit_hash(): """ Returns the number of commits of the Kolibri Android repo. Returns 0 if something fails. - TODO hash, unless there's a tag. Use alias to annotate """ repo_dir = os.path.dirname(os.path.abspath(__file__)) @@ -26,10 +25,11 @@ def commit_hash(): stderr=subprocess.PIPE, shell=True, cwd=repo_dir, - universal_newlines=True + universal_newlines=True, ) return p.communicate()[0].rstrip() + def git_tag(): repo_dir = os.path.dirname(os.path.abspath(__file__)) p = subprocess.Popen( @@ -38,25 +38,28 @@ def git_tag(): stderr=subprocess.PIPE, shell=True, cwd=repo_dir, - universal_newlines=True + universal_newlines=True, ) return p.communicate()[0].rstrip() + def build_type(): - key_alias = os.getenv('P4A_RELEASE_KEYALIAS', 'unknown') - if key_alias == 'LE_DEV_KEY': - return 'dev' - if key_alias == 'LE_RELEASE_KEY': - return 'official' + key_alias = os.getenv("P4A_RELEASE_KEYALIAS", "unknown") + if key_alias == "LE_DEV_KEY": + return "dev" + if key_alias == "LE_RELEASE_KEY": + return "official" return key_alias + def apk_version(): """ Returns the version to be used for the Kolibri Android app. Schema: [kolibri version]-[android installer version or githash]-[build signature type] """ android_version_indicator = git_tag() or commit_hash() - return '{}-{}-{}'.format(kolibri_version(), android_version_indicator, build_type()) + return "{}-{}-{}".format(kolibri_version(), android_version_indicator, build_type()) + def build_number(): """ @@ -70,28 +73,23 @@ def build_number(): # We can't go backwards. So we're adding to the one submitted at first. build_base_number = 2008998000 - buildkite_build_number = os.getenv('BUILDKITE_BUILD_NUMBER') - increment_for_64bit = 1 if os.getenv('ARCH', '') == '64bit' else 0 - - print('--- Assigning Build Number') + buildkite_build_number = os.getenv("BUILDKITE_BUILD_NUMBER") + increment_for_64bit = 1 if os.getenv("ARCH", "") == "64bit" else 0 if buildkite_build_number is not None: - build_number = build_base_number + 2 * int(buildkite_build_number) + increment_for_64bit - print(build_number) + build_number = ( + build_base_number + 2 * int(buildkite_build_number) + increment_for_64bit + ) return str(build_number) - print('Buildkite build # not found, using dev alternative') - alt_build_number = (int(datetime.now().strftime('%y%m%d%H%M')) - build_base_number) * 2 + increment_for_64bit - print(alt_build_number) + alt_build_number = ( + int(datetime.now().strftime("%y%m%d%H%M")) - build_base_number + ) * 2 + increment_for_64bit return alt_build_number -def create_project_info(): - """ - Generates project_info.json based on project_info.template - """ - with open('project_info.template', 'r') as pi_template_file, open('./project_info.json', 'w') as pi_file: - pi_template = Template(pi_template_file.read()) - pi = pi_template.substitute(apk_version=apk_version(), build_number=build_number()) - pi_file.write(pi) -create_project_info() +if __name__ == "__main__": + if sys.argv[1] == "apk_version": + print(apk_version()) + elif sys.argv[1] == "build_number": + print(build_number()) diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 00000000..a38353a6 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,4 @@ +[flake8] +ignore = E226,E203,E41,W503,E741 +max-line-length = 160 +max-complexity = 10 diff --git a/src/android_service.py b/src/android_service.py deleted file mode 100644 index d7ba0919..00000000 --- a/src/android_service.py +++ /dev/null @@ -1,65 +0,0 @@ -import initialization # keep this first, to ensure we're set up for other imports - -import flask -import logging -import os -import pew.ui -import shutil -import time - -from config import FLASK_PORT - -from android_utils import share_by_intent -from kolibri_utils import get_content_file_path - -# initialize logging before loading any third-party modules, as they may cause logging to get configured. -logging.basicConfig(level=logging.DEBUG) - - -logging.info("Entering android_service.py...") - -from android_utils import get_service_args, make_service_foreground -from kolibri_utils import start_kolibri_server - -# load the arguments passed into the service into environment variables -args = get_service_args() -for arg, val in args.items(): - print("setting envvar '{}' to '{}'".format(arg, val)) - os.environ[arg] = str(val) - -# move in a templated Kolibri data directory, including pre-migrated DB, to speed up startup -HOME_TEMPLATE_PATH = "preseeded_kolibri_home" -HOME_PATH = os.environ["KOLIBRI_HOME"] -if not os.path.exists(HOME_PATH) and os.path.exists(HOME_TEMPLATE_PATH): - shutil.move(HOME_TEMPLATE_PATH, HOME_PATH) - -# ensure the service stays running by "foregrounding" it with a persistent notification -make_service_foreground("Kolibri is running...", "Click here to resume.") - -# start the kolibri server as a thread -thread = pew.ui.PEWThread(target=start_kolibri_server) -thread.daemon = True -thread.start() - -# start a parallel Flask server as a backchannel for triggering events -flaskapp = flask.Flask(__name__) - -@flaskapp.route('/share_by_intent') -def do_share_by_intent(): - - args = flask.request.args - allowed_args = ["filename", "path", "msg", "app", "mimetype"] - kwargs = {key: args[key] for key in args if key in allowed_args} - - if "filename" in kwargs: - kwargs["path"] = get_content_file_path(kwargs.pop("filename")) - - logging.error("Sharing: {}".format(kwargs)) - - share_by_intent(**kwargs) - - return "OK, boomer" - - -if __name__ == "__main__": - flaskapp.run(host="localhost", port=FLASK_PORT) diff --git a/src/android_utils.py b/src/android_utils.py index 6af767a9..a83db783 100644 --- a/src/android_utils.py +++ b/src/android_utils.py @@ -1,18 +1,19 @@ import json -import logging import os import re + from cryptography import x509 from cryptography.hazmat.backends import default_backend -from jnius import autoclass, cast, jnius +from jnius import autoclass +from jnius import cast +from jnius import jnius -logging.basicConfig(level=logging.DEBUG) AndroidString = autoclass("java.lang.String") Context = autoclass("android.content.Context") Environment = autoclass("android.os.Environment") File = autoclass("java.io.File") -FileProvider = autoclass('android.support.v4.content.FileProvider') +FileProvider = autoclass("android.support.v4.content.FileProvider") Intent = autoclass("android.content.Intent") NotificationBuilder = autoclass("android.app.Notification$Builder") NotificationManager = autoclass("android.app.NotificationManager") @@ -31,7 +32,9 @@ def is_service_context(): def get_service(): - assert is_service_context(), "Cannot get service, as we are not in a service context." + assert ( + is_service_context() + ), "Cannot get service, as we are not in a service context." PythonService = autoclass("org.kivy.android.PythonService") return PythonService.mService @@ -40,13 +43,18 @@ def get_timezone_name(): return Timezone.getDefault().getDisplayName() -def start_service(service_name, service_args): - service = autoclass("org.learningequality.Kolibri.Service{}".format(service_name.title())) +def start_service(service_name, service_args=None): + service_args = service_args or {} + service = autoclass( + "org.learningequality.Kolibri.Service{}".format(service_name.title()) + ) service.start(PythonActivity.mActivity, json.dumps(dict(service_args))) def get_service_args(): - assert is_service_context(), "Cannot get service args, as we are not in a service context." + assert ( + is_service_context() + ), "Cannot get service args, as we are not in a service context." return json.loads(os.environ.get("PYTHON_SERVICE_ARGUMENT") or "{}") @@ -71,7 +79,7 @@ def is_app_installed(app_id): try: manager.getPackageInfo(app_id, PackageManager.GET_ACTIVITIES) - except jnius.JavaException as e: + except jnius.JavaException: return False return True @@ -84,12 +92,14 @@ def get_home_folder(): def send_whatsapp_message(msg): - share_by_intent(msg=msg, app="com.whatsapp") + share_by_intent(message=msg, app="com.whatsapp") def share_by_intent(path=None, filename=None, message=None, app=None, mimetype=None): - assert path or message or filename, "Must provide either a path, a filename, or a msg to share" + assert ( + path or message or filename + ), "Must provide either a path, a filename, or a msg to share" sendIntent = Intent() sendIntent.setAction(Intent.ACTION_SEND) @@ -97,7 +107,7 @@ def share_by_intent(path=None, filename=None, message=None, app=None, mimetype=N uri = FileProvider.getUriForFile( Context.getApplicationContext(), "org.learningequality.Kolibri.fileprovider", - File(path) + File(path), ) parcelable = cast("android.os.Parcelable", uri) sendIntent.putExtra(Intent.EXTRA_STREAM, parcelable) @@ -120,9 +130,16 @@ def make_service_foreground(title, message): if SDK_INT >= 26: NotificationChannel = autoclass("android.app.NotificationChannel") - notification_service = cast(NotificationManager, get_activity().getSystemService(Context.NOTIFICATION_SERVICE)) + notification_service = cast( + NotificationManager, + get_activity().getSystemService(Context.NOTIFICATION_SERVICE), + ) channel_id = get_activity().getPackageName() - app_channel = NotificationChannel(channel_id, "Kolibri Background Server", NotificationManager.IMPORTANCE_DEFAULT) + app_channel = NotificationChannel( + channel_id, + "Kolibri Background Server", + NotificationManager.IMPORTANCE_DEFAULT, + ) notification_service.createNotificationChannel(app_channel) notification_builder = NotificationBuilder(app_context, channel_id) else: @@ -131,7 +148,11 @@ def make_service_foreground(title, message): notification_builder.setContentTitle(AndroidString(title)) notification_builder.setContentText(AndroidString(message)) notification_intent = Intent(app_context, PythonActivity) - notification_intent.setFlags(Intent.FLAG_ACTIVITY_CLEAR_TOP | Intent.FLAG_ACTIVITY_SINGLE_TOP | Intent.FLAG_ACTIVITY_NEW_TASK) + notification_intent.setFlags( + Intent.FLAG_ACTIVITY_CLEAR_TOP + | Intent.FLAG_ACTIVITY_SINGLE_TOP + | Intent.FLAG_ACTIVITY_NEW_TASK + ) notification_intent.setAction(Intent.ACTION_MAIN) notification_intent.addCategory(Intent.CATEGORY_LAUNCHER) intent = PendingIntent.getActivity(service, 0, notification_intent, 0) @@ -144,7 +165,9 @@ def make_service_foreground(title, message): def get_signature_key_issuer(): signature = get_package_info(flags=PackageManager.GET_SIGNATURES).signatures[0] - cert = x509.load_der_x509_certificate(signature.toByteArray().tostring(), default_backend()) + cert = x509.load_der_x509_certificate( + signature.toByteArray().tostring(), default_backend() + ) return cert.issuer.rfc4514_string() diff --git a/src/config.py b/src/config.py deleted file mode 100644 index a9c1af87..00000000 --- a/src/config.py +++ /dev/null @@ -1,2 +0,0 @@ -KOLIBRI_PORT = 8080 -FLASK_PORT = 5226 \ No newline at end of file diff --git a/src/fileprovider_paths.xml b/src/fileprovider_paths.xml index 3514378c..1a14190b 100644 --- a/src/fileprovider_paths.xml +++ b/src/fileprovider_paths.xml @@ -2,4 +2,4 @@ - \ No newline at end of file + diff --git a/src/initialization.py b/src/initialization.py index 5c52b87f..977fbdd7 100644 --- a/src/initialization.py +++ b/src/initialization.py @@ -1,44 +1,48 @@ +import logging import os -import pew.ui import re import sys -from jnius import autoclass from android_utils import get_activity +from android_utils import get_home_folder +from android_utils import get_signature_key_issuing_organization +from android_utils import get_timezone_name +from android_utils import get_version_name +from jnius import autoclass + +# initialize logging before loading any third-party modules, as they may cause logging to get configured. +logging.basicConfig(level=logging.DEBUG) script_dir = os.path.dirname(os.path.abspath(__file__)) sys.path.append(script_dir) sys.path.append(os.path.join(script_dir, "kolibri", "dist")) sys.path.append(os.path.join(script_dir, "extra-packages")) +signing_org = get_signature_key_issuing_organization() +if signing_org == "Learning Equality": + runmode = "android-testing" +elif signing_org == "Android": + runmode = "android-debug" +elif signing_org == "Google Inc.": + runmode = "" # Play Store! +else: + runmode = "android-" + re.sub(r"[^a-z ]", "", signing_org.lower()).replace(" ", "-") +os.environ["KOLIBRI_RUN_MODE"] = runmode + +os.environ["TZ"] = get_timezone_name() +os.environ["LC_ALL"] = "en_US.UTF-8" + +os.environ["KOLIBRI_HOME"] = get_home_folder() +os.environ["KOLIBRI_APK_VERSION_NAME"] = get_version_name() os.environ["DJANGO_SETTINGS_MODULE"] = "kolibri_app_settings" -Secure = autoclass('android.provider.Settings$Secure') -node_id = Secure.getString( - get_activity().getContentResolver(), - Secure.ANDROID_ID -) +os.environ["KOLIBRI_CHERRYPY_THREAD_POOL"] = "2" + +Secure = autoclass("android.provider.Settings$Secure") + +node_id = Secure.getString(get_activity().getContentResolver(), Secure.ANDROID_ID) # Don't set this if the retrieved id is falsy, too short, or a specific # id that is known to be hardcoded in many devices. if node_id and len(node_id) >= 16 and node_id != "9774d56d682e549c": os.environ["MORANGO_NODE_ID"] = node_id - -if pew.ui.platform == "android": - # initialize some system environment variables needed to run smoothly on Android - - from android_utils import get_timezone_name, get_signature_key_issuing_organization - - signing_org = get_signature_key_issuing_organization() - if signing_org == "Learning Equality": - runmode = "android-testing" - elif signing_org == "Android": - runmode = "android-debug" - elif signing_org == "Google Inc.": - runmode = "" # Play Store! - else: - runmode = "android-" + re.sub(r"[^a-z ]", "", signing_org.lower()).replace(" ", "-") - os.environ["KOLIBRI_RUN_MODE"] = runmode - - os.environ["TZ"] = get_timezone_name() - os.environ["LC_ALL"] = "en_US.UTF-8" diff --git a/src/kolibri_app_settings.py b/src/kolibri_app_settings.py index 745979b7..149f51d2 100644 --- a/src/kolibri_app_settings.py +++ b/src/kolibri_app_settings.py @@ -2,10 +2,7 @@ from __future__ import print_function from __future__ import unicode_literals -import os -import tempfile - -from kolibri.deployment.default.settings.base import * +from kolibri.deployment.default.settings.base import * # noqa E402 SESSION_EXPIRE_AT_BROWSER_CLOSE = False SESSION_COOKIE_AGE = 52560000 diff --git a/src/kolibri_utils.py b/src/kolibri_utils.py deleted file mode 100644 index 2ebd9924..00000000 --- a/src/kolibri_utils.py +++ /dev/null @@ -1,27 +0,0 @@ -import os -import pew.ui -import sys - -import android_utils - - -def start_kolibri_server(): - from kolibri.utils.cli import main - # activate app mode - from kolibri.plugins.utils import enable_plugin - enable_plugin('kolibri.plugins.app') - - # register app capabilities - from kolibri.plugins.app.utils import interface - interface.register(share_file=android_utils.share_by_intent) - - print("Starting Kolibri server...") - print("Port: {}".format(os.environ.get("KOLIBRI_HTTP_PORT", "(default)"))) - print("Home folder: {}".format(os.environ.get("KOLIBRI_HOME", "(default)"))) - print("Timezone: {}".format(os.environ.get("TZ", "(default)"))) - main(["start", "--foreground"]) - - -def get_content_file_path(filename): - from kolibri.core.content.utils.paths import get_content_storage_file_path - return get_content_storage_file_path(filename) \ No newline at end of file diff --git a/src/main.py b/src/main.py index 2462a528..a0c1b956 100644 --- a/src/main.py +++ b/src/main.py @@ -1,148 +1,45 @@ -import initialization # keep this first, to ensure we're set up for other imports - import logging -import os import time -import urllib.error -import urllib.request - -# initialize logging before loading any third-party modules, as they may cause logging to get configured. -logging.basicConfig(level=logging.DEBUG) - -import pew -import pew.ui - -from config import KOLIBRI_PORT - -pew.set_app_name("Kolibri") -logging.info("Entering main.py...") - - -if pew.ui.platform == "android": - - from android_utils import get_home_folder, get_version_name - - os.environ["KOLIBRI_HOME"] = get_home_folder() - os.environ["KOLIBRI_APK_VERSION_NAME"] = get_version_name() - # We can't use symlinks as at least some Android devices have the user storage - # and app data directories on different mount points. - os.environ['KOLIBRI_STATIC_USE_SYMLINKS'] = "False" - - -def get_init_url(next_url='/'): - # we need to initialize Kolibri to allow us to access the app key - from kolibri.utils.cli import initialize - initialize(skip_update=True) - - from kolibri.plugins.app.utils import interface - return interface.get_initialize_url(next_url=next_url) - - -def start_kolibri(port): - - os.environ["KOLIBRI_HTTP_PORT"] = str(port) - - if pew.ui.platform == "android": - - logging.info("Starting kolibri server via Android service...") - - from android_utils import start_service - start_service("kolibri", dict(os.environ)) - - else: - - logging.info("Starting kolibri server directly as thread...") - - from kolibri_utils import start_kolibri_server - - thread = pew.ui.PEWThread(target=start_kolibri_server) - thread.daemon = True - thread.start() - - -class Application(pew.ui.PEWApp): - - def setUp(self): - """ - Start your UI and app run loop here. - """ - - # Set loading screen - self.loader_url = "file:///android_asset/_load.html" - self.kolibri_loaded = False - self.view = pew.ui.WebUIView("Kolibri", self.loader_url, delegate=self) - - # start kolibri server - start_kolibri(KOLIBRI_PORT) - self.load_thread = pew.ui.PEWThread(target=self.wait_for_server) - self.load_thread.daemon = True - self.load_thread.start() +import initialization # noqa: F401 keep this first, to ensure we're set up for other imports +from android_utils import start_service +from jnius import autoclass +from kolibri.main import enable_plugin +from kolibri.plugins.app.utils import interface +from kolibri.utils.cli import initialize +from kolibri.utils.server import _read_pid_file +from kolibri.utils.server import PID_FILE +from kolibri.utils.server import STATUS_RUNNING +from kolibri.utils.server import wait_for_status +from runnable import Runnable - # make sure we show the UI before run completes, as otherwise - # it is possible the run can complete before the UI is shown, - # causing the app to shut down early - self.view.show() - return 0 +PythonActivity = autoclass("org.kivy.android.PythonActivity") - def page_loaded(self, url): - """ - This is a PyEverywhere delegate method to let us know the WebView is ready to use. - """ +loadUrl = Runnable(PythonActivity.mWebView.loadUrl) - # On Android, there is a system back button, that works like the browser back button. Make sure we clear the - # history after first load so that the user cannot go back to the loading screen. We cannot clear the history - # during load, so we do it here. - # For more info, see: https://stackoverflow.com/questions/8103532/how-to-clear-webview-history-in-android - if ( - pew.ui.platform == "android" - and not self.kolibri_loaded - and url != self.loader_url - ): - # FIXME: Change pew to reference the native webview as webview.native_webview rather than webview.webview - # for clarity. - self.kolibri_loaded = True - self.view.webview.webview.clearHistory() +logging.info("Initializing Kolibri and running any upgrade routines") - def wait_for_server(self): - home_url = "http://localhost:{port}".format(port=KOLIBRI_PORT) - # test url to see if server has started - def running(): - try: - with urllib.request.urlopen(home_url) as response: - response.read() - return True - except urllib.error.URLError: - return False +loadUrl("file:///android_asset/_load.html") - # Tie up this thread until the server is running - while not running(): - logging.info( - "Kolibri server not yet started, checking again in one second..." - ) - time.sleep(1) +# activate app mode +enable_plugin("kolibri.plugins.app") - # Check for saved URL, which exists when the app was put to sleep last time it ran - saved_state = self.view.get_view_state() - logging.debug("Persisted View State: {}".format(self.view.get_view_state())) +# we need to initialize Kolibri to allow us to access the app key +initialize() - next_url = '/' - if "URL" in saved_state and saved_state["URL"].startswith(home_url): - next_url = saved_state["URL"].replace(home_url, '') +# start kolibri server +logging.info("Starting kolibri server via Android service...") +start_service("server") - start_url = home_url + get_init_url(next_url) - pew.ui.run_on_main_thread(self.view.load_url, start_url) +# Tie up this thread until the server is running +wait_for_status(STATUS_RUNNING, timeout=120) - if pew.ui.platform == "android": - from remoteshell import launch_remoteshell - self.remoteshell_thread = pew.ui.PEWThread(target=launch_remoteshell) - self.remoteshell_thread.daemon = True - self.remoteshell_thread.start() +_, port, _, _ = _read_pid_file(PID_FILE) - def get_main_window(self): - return self.view +start_url = "http://127.0.0.1:{port}".format(port=port) + interface.get_initialize_url() +loadUrl(start_url) +start_service("remoteshell") -if __name__ == "__main__": - app = Application() - app.run() +while True: + time.sleep(0.05) diff --git a/src/remoteshell.py b/src/remoteshell.py index 60d57a53..8dd3142f 100644 --- a/src/remoteshell.py +++ b/src/remoteshell.py @@ -1,15 +1,19 @@ -# import initialization - -import django import os +import initialization # noqa: F401 keep this first, to ensure we're set up for other imports +from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization from cryptography.hazmat.primitives.asymmetric import rsa -from cryptography.hazmat.backends import default_backend -from twisted.internet import reactor, defer -from twisted.cred import portal, checkers, error, credentials -from twisted.conch import manhole, manhole_ssh +from kolibri.main import initialize +from twisted.conch import manhole +from twisted.conch import manhole_ssh from twisted.conch.ssh import keys +from twisted.cred import checkers +from twisted.cred import credentials +from twisted.cred import error +from twisted.cred import portal +from twisted.internet import defer +from twisted.internet import reactor from zope.interface import implementer @@ -18,7 +22,7 @@ def get_key_pair(refresh=False): # calculate paths where we'll store our SSH server keys KEYPATH = os.path.join(os.environ.get("KOLIBRI_HOME", "."), "ssh_host_key") PUBKEYPATH = KEYPATH + ".pub" - + # check whether we already have keys there, and use them if so if os.path.isfile(KEYPATH) and os.path.isfile(PUBKEYPATH) and not refresh: with open(KEYPATH) as f, open(PUBKEYPATH) as pf: @@ -26,18 +30,20 @@ def get_key_pair(refresh=False): # otherwise, generate a new key pair and serialize it key = rsa.generate_private_key( - backend=default_backend(), - public_exponent=65537, - key_size=2048 + backend=default_backend(), public_exponent=65537, key_size=2048 ) private_key = key.private_bytes( serialization.Encoding.PEM, serialization.PrivateFormat.TraditionalOpenSSL, - serialization.NoEncryption()).decode() - public_key = key.public_key().public_bytes( - serialization.Encoding.OpenSSH, - serialization.PublicFormat.OpenSSH + serialization.NoEncryption(), ).decode() + public_key = ( + key.public_key() + .public_bytes( + serialization.Encoding.OpenSSH, serialization.PublicFormat.OpenSSH + ) + .decode() + ) # store the keys to disk for use again later with open(KEYPATH, "w") as f, open(PUBKEYPATH, "w") as pf: @@ -53,13 +59,16 @@ class KolibriSuperAdminCredentialsChecker(object): Check that the device is unprovisioned, or the credentials are for a super admin, or the password matches the temp password set over ADB. """ + credentialInterfaces = (credentials.IUsernamePassword,) def requestAvatarId(self, creds): from kolibri.core.auth.models import FacilityUser # if a temporary password was set over ADB, allow login with it - TEMP_ADMIN_PASS_PATH = os.path.join(os.environ.get("KOLIBRI_HOME", "."), "temp_admin_pass") + TEMP_ADMIN_PASS_PATH = os.path.join( + os.environ.get("KOLIBRI_HOME", "."), "temp_admin_pass" + ) if os.path.isfile(TEMP_ADMIN_PASS_PATH): with open(TEMP_ADMIN_PASS_PATH) as f: provided_password = creds.password.decode() @@ -83,22 +92,23 @@ def requestAvatarId(self, creds): def _get_manhole_factory(namespace): # ensure django has been set up so we can use the ORM etc in the shell - django.setup() + initialize(skip_update=True) # set up the twisted manhole with Kolibri-based authentication def get_manhole(_): return manhole.Manhole(namespace) + realm = manhole_ssh.TerminalRealm() realm.chainedProtocolFactory.protocolFactory = get_manhole p = portal.Portal(realm) p.registerChecker(KolibriSuperAdminCredentialsChecker()) f = manhole_ssh.ConchFactory(p) - + # get the SSH server key pair to use private_rsa, public_rsa = get_key_pair() f.publicKeys[b"ssh-rsa"] = keys.Key.fromString(public_rsa) f.privateKeys[b"ssh-rsa"] = keys.Key.fromString(private_rsa) - + return f diff --git a/src/runnable.py b/src/runnable.py new file mode 100644 index 00000000..8552c480 --- /dev/null +++ b/src/runnable.py @@ -0,0 +1,41 @@ +""" +Runnable +======== + +""" +from jnius import autoclass +from jnius import java_method +from jnius import PythonJavaClass + +# reference to the activity +_PythonActivity = autoclass("org.kivy.android.PythonActivity") + + +class Runnable(PythonJavaClass): + """Wrapper around Java Runnable class. This class can be used to schedule a + call of a Python function into the PythonActivity thread. + """ + + __javainterfaces__ = ["java/lang/Runnable"] + __runnables__ = [] + + def __init__(self, func): + super(Runnable, self).__init__() + self.func = func + + def __call__(self, *args, **kwargs): + self.args = args + self.kwargs = kwargs + Runnable.__runnables__.append(self) + _PythonActivity.mActivity.runOnUiThread(self) + + @java_method("()V") + def run(self): + try: + self.func(*self.args, **self.kwargs) + except Exception: + import traceback + + traceback.print_exc() + + Runnable.__runnables__.remove(self) diff --git a/src/server.py b/src/server.py new file mode 100644 index 00000000..5a63b2e2 --- /dev/null +++ b/src/server.py @@ -0,0 +1,25 @@ +import logging +import os + +import initialization # noqa: F401 keep this first, to ensure we're set up for other imports +from android_utils import make_service_foreground +from android_utils import share_by_intent +from kolibri.main import initialize +from kolibri.main import start +from kolibri.plugins.app.utils import interface +from kolibri.utils.conf import KOLIBRI_HOME + +logging.info("Entering Kolibri server service") + +# ensure the service stays running by "foregrounding" it with a persistent notification +make_service_foreground("Kolibri is running...", "Click here to resume.") + +initialize(skip_update=True) + +# register app capabilities +interface.register(share_file=share_by_intent) + +logging.info("Home folder: {}".format(KOLIBRI_HOME)) +logging.info("Timezone: {}".format(os.environ.get("TZ", "(default)"))) +# start the kolibri server +start() diff --git a/whitelist.txt b/whitelist.txt deleted file mode 100644 index 481830cb..00000000 --- a/whitelist.txt +++ /dev/null @@ -1,6 +0,0 @@ -sqlite3/* -lib-dynload/_sqlite3.so -unittest/* -wsgiref/* -lib-dynload/_csv.so -lib-dynload/_json.so