Switch Kolibri server start to a separate persistent foreground service

.gitignore

@@ -9,6 +9,11 @@ project_info.json
 # pew output folder
 dist/
 
+__pycache__
 *.pyc
 build_docker
+build/
 bin/
+build.log
+tmphome/
+static/

Dockerfile

@@ -30,6 +30,7 @@ RUN dpkg --add-architecture i386 && \
     zlib1g-dev \
     zlib1g:i386 \
     python-wxgtk3.0 \
+    libgtk-3-dev \
     && apt-get clean
 
 

Makefile

@@ -2,14 +2,14 @@
 clean:
 	- rm -rf dist/android/*.apk project_info.json ./src/kolibri
 
-# Replace the default loading page, so that it will be replaced with our own version
-replaceloadingpage:
-	rm -f .buildozer/android/platform/build/dists/kolibri/webview_includes/_load.html
-	cp ./assets/_load.html .buildozer/android/platform/build/dists/kolibri/webview_includes/
-	cp ./assets/loading-spinner.gif .buildozer/android/platform/build/dists/kolibri/webview_includes/
+deepclean: clean
+	rm -r ~/.local/share/python-for-android
+	rm -r build
+	yes y | docker system prune -a
+	rm build_docker 2> /dev/null
 
 # Extract the whl file
-src/kolibri:
+src/kolibri: clean
 	unzip -qo "whl/kolibri*.whl" "kolibri/*" -x "kolibri/dist/cext*" -d src/
 
 # Generate the project info file
@@ -37,7 +37,20 @@ kolibri.apk: p4a_android_distro
 build_docker: Dockerfile
 	docker build -t android_kolibri .
 
+create_premigrated_db:
+	pip install whl/*.whl
+	KOLIBRI_HOME=tmphome kolibri manage migrate
+	cp tmphome/db.sqlite3 src/db.sqlite3.empty
+
 # Run the docker image.
 # TODO Would be better to just specify the file here?
 run_docker: build_docker
 	./scripts/rundocker.sh
+
+launch:
+	pew build android $(pew_release_flag)
+	adb uninstall org.learningequality.Kolibri || true 2> /dev/null
+	rm dist/android/Kolibri-0-debug.apk || true 2> /dev/null
+	adb install dist/android/*-debug.apk
+	adb shell am start -n org.learningequality.Kolibri/org.kivy.android.PythonActivity
+	adb logcat | grep -E "python|Python| server "

README.md

@@ -49,9 +49,9 @@ Run `adb logcat -v brief python:D *:F` to get all debug logs from the Kolibri se
   - If your device doesn't aggressively kill the server, you can open Chrome and use remote debugging tools to see the logs on your desktop.
   - You can also leave the app open and port forward the Android device's Kolibri port using [adb](https://developer.android.com/studio/command-line/adb#forwardports):
   ```
-  adb forward tcp:5000 tcp:5001
+  adb forward tcp:8080 tcp:8081
   ```
-  then going into your desktop's browser and accessing `localhost:5001`. Note that you can map to any port on the host machine, the second argument.
+  then going into your desktop's browser and accessing `localhost:8081`. Note that you can map to any port on the host machine, the second argument.
 
 Alternatively, you can debug the webview directly. Modern Android versions should let you do so from the developer settings.
 

project_info.template

@@ -3,9 +3,10 @@
   "version": "${apk_version}",
   "build_number": "${build_number}",
   "identifier": "org.learningequality.Kolibri",
-  "requirements": {"android": ["python2","pyjnius","genericndkbuild", "sqlite3", "cryptography", "pyopenssl", "openssl", "six"]},
+  "requirements": {"android": ["python2", "pyjnius", "genericndkbuild", "sqlite3", "cryptography", "pyopenssl", "openssl", "six"]},
   "whitelist_file": {"android": "whitelist.txt"},
   "icons": {"android": "icon.png"},
   "launch_images": {"android": "assets/launch-image.png"},
-  "asset_dirs": ["assets"]
+  "asset_dirs": ["assets"],
+  "extra_build_options": {"android": {"services": ["server:server.py"], "extra_permissions": [], "sdk": 17}}
 }

src/config.py

@@ -0,0 +1 @@
+PORT = 8080

src/kolibri_app_settings.py

@@ -0,0 +1,11 @@
+from __future__ import absolute_import
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import os
+import tempfile
+
+from kolibri.deployment.default.settings.base import *
+
+SESSION_EXPIRE_AT_BROWSER_CLOSE = False
+SESSION_COOKIE_AGE = 52560000

src/main.py

@@ -1,6 +1,6 @@
+import json
 import logging
 import os
-import sys
 import time
 import urllib2
 
@@ -10,53 +10,47 @@
 import pew
 import pew.ui
 
+from config import PORT
+
 pew.set_app_name("Kolibri")
 logging.info("Entering main.py...")
 
 
-if pew.ui.platform == "android":
-    from jnius import autoclass
-
-    PythonActivity = autoclass("org.kivy.android.PythonActivity")
-    File = autoclass("java.io.File")
-    Timezone = autoclass("java.util.TimeZone")
-
+def start_kolibri_server(port):
 
-# TODO check for storage availibility, allow user to chose sd card or internal
-def get_home_folder():
-    kolibri_home_file = PythonActivity.getExternalFilesDir(None)
-    return kolibri_home_file.toString()
+    if pew.ui.platform == "android":
 
+        from jnius import autoclass
 
-script_dir = os.path.dirname(os.path.abspath(__file__))
-sys.path.append(script_dir)
-sys.path.append(os.path.join(script_dir, "kolibri", "dist"))
+        PythonActivity = autoclass("org.kivy.android.PythonActivity")
+        service = autoclass('org.learningequality.Kolibri.ServiceServer')
 
-os.environ["DJANGO_SETTINGS_MODULE"] = "kolibri.deployment.default.settings.base"
+        # TODO: check for storage availability, allow user to chose sd card or internal
+        def get_home_folder():
+            kolibri_home_file = PythonActivity.mActivity.getExternalFilesDir(None)
+            return kolibri_home_file.toString()
 
-if pew.ui.platform == "android":
-    os.environ["KOLIBRI_HOME"] = get_home_folder()
-    os.environ["TZ"] = Timezone.getDefault().getDisplayName()
-    os.environ["LC_ALL"] = "en_US.UTF-8"
+        logging.info("Starting kolibri server via Android service...")
 
-    logging.info("Home folder: {}".format(os.environ["KOLIBRI_HOME"]))
-    logging.info("Timezone: {}".format(os.environ["TZ"]))
+        service.start(PythonActivity.mActivity, json.dumps({
+            "HOME": get_home_folder(),
+            "PORT": port,
+            "VERSION": PythonActivity.getPackageManager().getPackageInfo(PythonActivity.getPackageName(), 0).versionName,
+        }))
 
+    else:
 
-def start_django():
+        from server import start_django
 
-    # remove this after Kolibri no longer needs it
-    if sys.version[0] == "2":
-        reload(sys)
-        sys.setdefaultencoding("utf8")
+        logging.info("Starting kolibri server directly as thread...")
 
-    logging.info("Starting server...")
-    from kolibri.utils.cli import main
-
-    main(["start", "--foreground", "--port=5000"])
+        thread = pew.ui.PEWThread(target=start_django, args=(port,))
+        thread.daemon = True
+        thread.start()
 
 
 class Application(pew.ui.PEWApp):
+
     def setUp(self):
         """
         Start your UI and app run loop here.
@@ -69,9 +63,7 @@ def setUp(self):
         self.view = pew.ui.WebUIView("Kolibri", self.loader_url, delegate=self)
 
         # start thread
-        self.thread = pew.ui.PEWThread(target=start_django)
-        self.thread.daemon = True
-        self.thread.start()
+        start_kolibri_server(PORT)
 
         self.load_thread = pew.ui.PEWThread(target=self.wait_for_server)
         self.load_thread.daemon = True
@@ -103,11 +95,10 @@ def page_loaded(self, url):
             self.view.webview.webview.clearHistory()
 
     def wait_for_server(self):
-        from kolibri.utils import server
 
-        home_url = "http://localhost:5000"
+        home_url = "http://localhost:{port}".format(port=PORT)
 
-        # test url to see if servr has started
+        # test url to see if server has started
         def running():
             try:
                 urllib2.urlopen(home_url)
@@ -127,10 +118,11 @@ def running():
         logging.debug("Persisted View State: {}".format(self.view.get_view_state()))
 
         if "URL" in saved_state and saved_state["URL"].startswith(home_url):
-            pew.ui.run_on_main_thread(self.view.load_url(saved_state["URL"]))
-            return
+            start_url = saved_state["URL"]
+        else:
+            start_url = home_url
 
-        pew.ui.run_on_main_thread(self.view.load_url(home_url))
+        pew.ui.run_on_main_thread(self.view.load_url, start_url)
 
     def get_main_window(self):
         return self.view

src/server.py

@@ -0,0 +1,88 @@
+import json
+import logging
+import os
+import shutil
+import sys
+
+# initialize logging before loading any third-party modules, as they may cause logging to get configured.
+logging.basicConfig(level=logging.DEBUG)
+
+import pew
+import pew.ui
+
+logging.info("Entering server.py...")
+
+script_dir = os.path.dirname(os.path.abspath(__file__))
+sys.path.append(script_dir)
+sys.path.append(os.path.join(script_dir, "kolibri", "dist"))
+
+os.environ["DJANGO_SETTINGS_MODULE"] = "kolibri_app_settings"
+
+# TODO: before shipping the app, make this contingent on debug vs production mode
+os.environ["KOLIBRI_RUN_MODE"] = "pew-dev"
+
+def start_django(port):
+
+    from kolibri.utils.cli import main
+
+    logging.info("Starting server...")
+
+    main(["start", "--foreground", "--port={port}".format(port=port)])
+
+
+if pew.ui.platform == "android":
+
+    from jnius import autoclass
+
+    service_args = json.loads(os.environ.get("PYTHON_SERVICE_ARGUMENT") or "{}")
+
+    service = autoclass('org.kivy.android.PythonService').mService
+    File = autoclass("java.io.File")
+    Timezone = autoclass("java.util.TimeZone")
+    AndroidString = autoclass('java.lang.String')
+    Drawable = autoclass("{}.R$drawable".format(service.getPackageName()))
+    Context = autoclass('android.content.Context')
+    Intent = autoclass('android.content.Intent')
+    PendingIntent = autoclass('android.app.PendingIntent')
+    NotificationBuilder = autoclass('android.app.Notification$Builder')
+    Notification = autoclass('android.app.Notification')
+
+    def make_service_foreground(title, message):
+        PythonActivity = autoclass('org.kivy.android.PythonActivity')
+        app_context = service.getApplication().getApplicationContext()
+        notification_builder = NotificationBuilder(app_context)
+        notification_builder.setContentTitle(AndroidString(title))
+        notification_builder.setContentText(AndroidString(message))
+        notification_intent = Intent(app_context, PythonActivity)
+        notification_intent.setFlags(Intent.FLAG_ACTIVITY_CLEAR_TOP | Intent.FLAG_ACTIVITY_SINGLE_TOP | Intent.FLAG_ACTIVITY_NEW_TASK)
+        notification_intent.setAction(Intent.ACTION_MAIN)
+        notification_intent.addCategory(Intent.CATEGORY_LAUNCHER)
+        intent = PendingIntent.getActivity(service, 0, notification_intent, 0)
+        notification_builder.setContentIntent(intent)
+        notification_builder.setSmallIcon(Drawable.icon)
+        notification_builder.setAutoCancel(True)
+        new_notification = notification_builder.getNotification()
+        service.startForeground(1, new_notification)
+
+    os.environ["KOLIBRI_HOME"] = service_args["HOME"]
+
+    # copy an empty pre-migrated database into the Kolibri data directory to speed up startup
+    DB_TEMPLATE_PATH = "db.sqlite3.empty"
+    DB_PATH = os.path.join(os.environ["KOLIBRI_HOME"], "db.sqlite3")
+    if not os.path.exists(DB_PATH) and os.path.exists(DB_TEMPLATE_PATH):
+        if not os.path.exists(os.environ["KOLIBRI_HOME"]):
+            os.makedirs(os.environ["KOLIBRI_HOME"])
+        shutil.copyfile(DB_TEMPLATE_PATH, DB_PATH)
+
+    # store the version name into an envvar to be picked up by Kolibri
+    os.environ["KOLIBRI_APK_VERSION_NAME"] = service_args["VERSION"]
+
+    os.environ["TZ"] = Timezone.getDefault().getDisplayName()
+    os.environ["LC_ALL"] = "en_US.UTF-8"
+
+    logging.info("Home folder: {}".format(os.environ["KOLIBRI_HOME"]))
+    logging.info("Timezone: {}".format(os.environ["TZ"]))
+
+    make_service_foreground("Kolibri is running...", "Click here to resume.")
+
+    start_django(service_args["PORT"])