Merge pull request #53 from jamalex/develop

Additional optimizations (pre-seed entire kolibri data directory, and blacklist unneeded source files)

.buildkite/pipeline.yaml

@@ -1,5 +1,16 @@
 steps:
-  - block: "Build :android:?"
-    if: build.source == "trigger_job"
-  - label: "Build Android App"
+  - block: "Build dev APK?"
+    # Always run if this is a Kolibri release
+    if: build.env("LE_KOLIBRI_RELEASE") != "true"
+
+  - label: "Build dev APK"
     command: ".buildkite/build.sh"
+    env: KOLIBRI_ANDROID_BUILD_MODE=dev
+
+  - block: "Build release APK?"
+    # Always run if this is a Kolibri release
+    if: build.env("LE_KOLIBRI_RELEASE") == "true"
+
+  - label: "Build Release APK :tada:"
+    command: ".buildkite/build.sh"
+    env: KOLIBRI_ANDROID_BUILD_MODE=release

.gitignore

@@ -1,4 +1,7 @@
 src/kolibri
+src/preseeded_kolibri_home
+src/extra-packages
+tmpenv
 whl/*
 
 # File format for signing key

Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:16.04 as build
+FROM ubuntu:bionic as build
 LABEL maintainer="Learning Equality <[email protected]>" tag="kolibrikivy"
 ENV DEBIAN_FRONTEND noninteractive
 
@@ -14,11 +14,13 @@ RUN dpkg --add-architecture i386 && \
     cython \
     gcc \
     git \
+    iproute2 \
     libffi-dev \
     libltdl-dev\
     libncurses5:i386 \
     libstdc++6:i386 \
     libtool \
+    locales \
     lsb-release \
     openjdk-8-jdk \
     python-dev \
@@ -27,27 +29,43 @@ RUN dpkg --add-architecture i386 && \
     wget \
     xclip \
     xsel \
+    zip \
     zlib1g-dev \
     zlib1g:i386 \
     python-wxgtk3.0 \
     libgtk-3-dev \
+    python3 \
     && apt-get clean
 
+# Use java 1.8 because Ubuntu's gradle version doesn't support 1.11
+ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
+ENV PATH=$PATH:$JAVA_HOME
+
+RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \
+  python3 get-pip.py
+
+# Ensure that python is using python3
+# copying approach from official python images
+ENV PATH /usr/local/bin:$PATH
+RUN cd /usr/local/bin && \
+  ln -s $(which python3) python
 
 # Allows us to invalidate cache if those repos update.
 # Intentionally not pinning for dev velocity.
 ADD https://github.com/kollivier/python-for-android/archive/webview_plus.zip p4a.zip
 ADD https://github.com/kollivier/pyeverywhere/archive/dev.zip pew.zip
 
 # install python dependencies
-RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \
-  python get-pip.py && \
-  pip install cython virtualenv && \
+RUN pip install cython virtualenv && \
   # get kevin's custom packages
-  pip install -e git+https://github.com/kollivier/pyeverywhere@dev#egg=pyeverywhere && \
-  pip install -e git+https://github.com/kollivier/python-for-android@webview_plus#egg=python-for-android && \
+  pip install -e git+https://github.com/kollivier/pyeverywhere@p4a_update#egg=pyeverywhere && \
+  pip install -e git+https://github.com/kollivier/python-for-android@pew_webview#egg=python-for-android && \
   useradd -lm kivy
 
+RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \
+    locale-gen
+ENV LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8
+
 USER kivy:kivy
 WORKDIR /home/kivy
 

Makefile

@@ -1,18 +1,28 @@
+# run with `make ... ARCH=64bit` to build for v8a
 ARCH=
 
+ifeq ($(ARCH), 64bit)
+  ARM_VER=v8a
+else
+  ARM_VER=v7a
+endif
+
 # Clear out apks
 clean:
 	- rm -rf dist/android/*.apk project_info.json ./src/kolibri
 
 deepclean: clean
-	rm -r ~/.local/share/python-for-android
-	rm -r build
-	yes y | docker system prune -a
+	python-for-android clean_dists
+	rm -r build || true
+	rm -r dist || true
+	yes y | docker system prune -a || true
 	rm build_docker 2> /dev/null
 
 # Extract the whl file
 src/kolibri: clean
+	rm -r src/kolibri 2> /dev/null || true
 	unzip -qo "whl/kolibri*.whl" "kolibri/*" -x "kolibri/dist/cext*" -d src/
+	./delete_kolibri_blacklist.sh
 
 # Generate the project info file
 project_info.json: project_info.template src/kolibri scripts/create_project_info.py
@@ -27,8 +37,8 @@ pew_release_flag = --release
 endif
 
 .PHONY: kolibri.apk
-# Buld the debug version of the apk
-kolibri.apk: p4a_android_distro
+# Build the debug version of the apk
+kolibri.apk: p4a_android_distro preseeded_kolibri_home
 	pew build android $(pew_release_flag) $(ARCH)
 
 # DOCKER BUILD
@@ -39,20 +49,38 @@ kolibri.apk: p4a_android_distro
 build_docker: Dockerfile
 	docker build -t android_kolibri .
 
-create_premigrated_db:
-	pip install whl/*.whl
-	KOLIBRI_HOME=tmphome kolibri manage migrate
-	cp tmphome/db.sqlite3 src/db.sqlite3.empty
+preseeded_kolibri_home: export KOLIBRI_HOME := src/preseeded_kolibri_home
+preseeded_kolibri_home: export PYTHONPATH := tmpenv
+preseeded_kolibri_home:
+	rm -r tmpenv 2> /dev/null || true
+	rm -r src/preseeded_kolibri_home 2> /dev/null || true
+	pip uninstall kolibri 2> /dev/null || true
+	pip install --target tmpenv whl/*.whl
+	tmpenv/bin/kolibri start --port=16294
+	sleep 1
+	tmpenv/bin/kolibri stop
+	sleep 1
+	yes yes | tmpenv/bin/kolibri manage deprovision
+	rm -r src/preseeded_kolibri_home/logs 2> /dev/null || true
+	rm -r src/preseeded_kolibri_home/sessions 2> /dev/null || true
+	rm -r src/preseeded_kolibri_home/process_cache 2> /dev/null || true
+	touch src/preseeded_kolibri_home/was_preseeded
 
 # Run the docker image.
 # TODO Would be better to just specify the file here?
 run_docker: build_docker
 	./scripts/rundocker.sh
 
-launch:
+softbuild: project_info.json
 	pew build android $(pew_release_flag) $(ARCH)
+
+install:
 	adb uninstall org.learningequality.Kolibri || true 2> /dev/null
-	rm dist/android/Kolibri-0-debug.apk || true 2> /dev/null
-	adb install dist/android/*-debug.apk
+	adb install dist/android/*$(ARM_VER)-debug-*.apk
+
+run: install
 	adb shell am start -n org.learningequality.Kolibri/org.kivy.android.PythonActivity
-	adb logcat | grep -E "python|Python| server "
+	sleep 1
+	adb logcat | grep -i -E "python|kolibr| `adb shell ps | grep ' org.learningequality.Kolibri$$' | tr -s [:space:] ' ' | cut -d' ' -f2` " | grep -E -v "WifiTrafficPoller|localhost:5000|NetworkManagementSocketTagger|No jobs to start"
+
+launch: softbuild run