Access kolibri's job storage database to reconcile tasks

learningequality · Dec 12, 2023 · e1f8fb5 · e1f8fb5
1 parent 017faa3
commit e1f8fb5
Show file tree

Hide file tree

Showing 13 changed files with 701 additions and 30 deletions.
diff --git a/python-for-android/dists/kolibri/build.gradle b/python-for-android/dists/kolibri/build.gradle
@@ -109,4 +109,5 @@ dependencies {
     implementation 'androidx.concurrent:concurrent-futures:1.1.0'
     implementation 'androidx.work:work-runtime:2.7.1'
     implementation 'androidx.work:work-multiprocess:2.7.1'
+    implementation 'net.sourceforge.streamsupport:java9-concurrent-backport:2.0.5'
 }
diff --git a/python-for-android/dists/kolibri/src/main/java/org/kivy/android/PythonWorker.java b/python-for-android/dists/kolibri/src/main/java/org/kivy/android/PythonWorker.java
@@ -3,6 +3,7 @@
 import android.annotation.SuppressLint;
 import android.content.Context;
 import android.util.Log;
+import android.os.Process;
 
 import androidx.annotation.NonNull;
 import androidx.concurrent.futures.CallbackToFutureAdapter;
@@ -80,11 +81,13 @@ protected Result doWork() {
 
         Log.d(TAG, id + " Running with python worker argument: " + arg);
 
+        String serializedArg = String.join(",", id, arg, Integer.toString(Process.myPid()), Long.toString(Thread.currentThread().getId()));
+
         int res = nativeStart(
                 androidPrivate, androidArgument,
                 workerEntrypoint, pythonName,
                 pythonHome, pythonPath,
-                arg
+                serializedArg
         );
         Log.d(TAG, id + " Finished remote python work: " + res);
 

diff --git a/python-for-android/dists/kolibri/src/main/java/org/learningequality/ContextUtil.java b/python-for-android/dists/kolibri/src/main/java/org/learningequality/ContextUtil.java
@@ -3,20 +3,19 @@
 import android.content.Context;
 
 import org.kivy.android.PythonActivity;
-import org.kivy.android.PythonService;
-import org.kivy.android.PythonWorker;
-import org.learningequality.NotificationRef;
+import org.learningequality.Kolibri.TaskworkerWorker;
+import org.learningequality.Kolibri.TaskworkerWorkerService;
 
 public class ContextUtil {
     public static Context getApplicationContext() {
-        if (isActivityContext()) {
-            return PythonActivity.mActivity.getApplicationContext();
+        if (isWorkerContext()) {
+            return TaskworkerWorker.mWorker.getApplicationContext();
         }
         if (isServiceContext()) {
-            return PythonService.mService.getApplicationContext();
+            return TaskworkerWorkerService.mService.getApplicationContext();
         }
-        if (isWorkerContext()) {
-            return PythonWorker.mWorker.getApplicationContext();
+        if (isActivityContext()) {
+            return PythonActivity.mActivity.getApplicationContext();
         }
         return null;
     }
@@ -26,10 +25,10 @@ public static boolean isActivityContext() {
     }
 
     public static boolean isServiceContext() {
-        return PythonService.mService != null;
+        return TaskworkerWorkerService.mService != null;
     }
 
     public static boolean isWorkerContext() {
-        return PythonWorker.mWorker != null;
+        return TaskworkerWorker.mWorker != null;
     }
 }
diff --git a/python-for-android/dists/kolibri/src/main/java/org/learningequality/Kolibri/App.java b/python-for-android/dists/kolibri/src/main/java/org/learningequality/Kolibri/App.java
@@ -3,22 +3,29 @@
 import android.app.Application;
 import android.content.Context;
 import android.os.Build;
+import android.util.Log;
 
 import androidx.annotation.NonNull;
 import androidx.core.app.NotificationManagerCompat;
 import androidx.core.app.NotificationChannelCompat;
 import androidx.work.Configuration;
+import androidx.work.PeriodicWorkRequest;
+import androidx.work.WorkManager;
 
 import org.learningequality.NotificationRef;
+import org.learningequality.Task;
 
 import java.util.concurrent.Executors;
 
+import java9.util.concurrent.CompletableFuture;
+
 public class App extends Application implements Configuration.Provider {
     @Override
     public void onCreate() {
         super.onCreate();
         NotificationRef.initialize(this);
         createNotificationChannels();
+        reconcileTasks();
     }
 
     @NonNull
@@ -27,8 +34,6 @@ public Configuration getWorkManagerConfiguration() {
         String processName = getApplicationContext().getPackageName();
         processName += getApplicationContext().getString(R.string.task_worker_process);
 
-
-
         // Using the same quantity of worker threads as Kolibri's python side:
         // https://github.com/learningequality/kolibri/blob/release-v0.16.x/kolibri/utils/options.py#L683
         return new Configuration.Builder()
@@ -64,4 +69,16 @@ private void createNotificationChannels() {
             notificationManager.createNotificationChannel(taskChannel);
         }
     }
+
+    private void reconcileTasks() {
+        // Reconcile tasks on startup, in this main thread (blocking!)
+        CompletableFuture<Boolean> f = Task.reconcile(this, null);
+        f.whenCompleteAsync((result, throwable) -> {
+            if (throwable != null) {
+                Log.e("Kolibri", "Main thread task reconciliation failed", throwable);
+            } else {
+                Log.i("Kolibri", "Main thread task reconciliation completed");
+            }
+        });
+    }
 }
diff --git a/...for-android/dists/kolibri/src/main/java/org/learningequality/Kolibri/ReconcileWorker.java b/...for-android/dists/kolibri/src/main/java/org/learningequality/Kolibri/ReconcileWorker.java
@@ -0,0 +1,47 @@
+package org.learningequality.Kolibri;
+
+import android.annotation.SuppressLint;
+import android.content.Context;
+import android.util.Log;
+
+import androidx.annotation.NonNull;
+import androidx.work.ListenableWorker;
+import androidx.work.WorkerParameters;
+import androidx.work.impl.utils.futures.SettableFuture;
+
+import com.google.common.util.concurrent.ListenableFuture;
+
+import java9.util.concurrent.CompletableFuture;
+import java.util.concurrent.Executor;
+
+import org.learningequality.Task;
+
+public class ReconcileWorker extends ListenableWorker {
+    public static final String TAG = "Kolibri.ReconcileWorker";
+
+    public ReconcileWorker(@NonNull Context appContext, @NonNull WorkerParameters workerParams) {
+        super(appContext, workerParams);
+    }
+
+    @SuppressLint("RestrictedApi")
+    @NonNull
+    public ListenableFuture<Result> startWork() {
+        Log.i(TAG, "Starting reconcile task");
+        SettableFuture<Result> future = SettableFuture.create();
+        Executor executor = getBackgroundExecutor();
+
+        CompletableFuture<Boolean> f = Task.reconcile(getApplicationContext(), executor);
+
+        f.whenCompleteAsync((result, throwable) -> {
+            if (throwable != null) {
+                Log.e(TAG, "Reconcile task failed", throwable);
+                future.setException(throwable);
+            } else {
+                Log.i(TAG, "Reconcile task completed: " + (result ? "success" : "failure"));
+                future.set(result ? Result.success() : Result.failure());
+            }
+        }, executor);
+
+        return future;
+    }
+}
diff --git a/...oid/dists/kolibri/src/main/java/org/learningequality/Kolibri/TaskworkerWorkerService.java b/...oid/dists/kolibri/src/main/java/org/learningequality/Kolibri/TaskworkerWorkerService.java
@@ -3,6 +3,7 @@
 import android.content.Context;
 import android.util.Log;
 
+import androidx.work.PeriodicWorkRequest;
 import androidx.work.multiprocess.RemoteWorkerService;
 import androidx.work.WorkManager;
 
@@ -27,6 +28,7 @@ public void onCreate() {
         );
         // Initialize the work manager
         WorkManager.getInstance(getApplicationContext());
+        enqueueTaskReconciliation();
         super.onCreate();
         // We could potentially remove this and leave the notification up to long-running workers
         // bound to the service
@@ -43,4 +45,19 @@ public void onDestroy() {
     public NotificationRef getNotificationRef() {
         return new NotificationRef(NotificationRef.REF_CHANNEL_SERVICE);
     }
+
+    private void enqueueTaskReconciliation() {
+        WorkManager workManager = WorkManager.getInstance(this);
+        PeriodicWorkRequest reconcileRequest = new PeriodicWorkRequest.Builder(
+                ReconcileWorker.class,
+                60,
+                java.util.concurrent.TimeUnit.MINUTES
+        ).build();
+
+        workManager.enqueueUniquePeriodicWork(
+                "task_reconciliation",
+                androidx.work.ExistingPeriodicWorkPolicy.KEEP,
+                reconcileRequest
+        );
+    }
 }
diff --git a/...r-android/dists/kolibri/src/main/java/org/learningequality/Kolibri/sqlite/JobStorage.java b/...r-android/dists/kolibri/src/main/java/org/learningequality/Kolibri/sqlite/JobStorage.java
@@ -0,0 +1,39 @@
+package org.learningequality.Kolibri.sqlite;
+
+import org.learningequality.sqlite.schema.DatabaseTable;
+
+public class JobStorage {
+    public static final String DATABASE_NAME = "job_storage.sqlite3";
+
+    public static class Jobs implements DatabaseTable {
+        public static final String TABLE_NAME = "jobs";
+
+        public String getTableName() {
+            return TABLE_NAME;
+        }
+
+        public static final StringColumn id = new StringColumn("id");
+        public static final StringColumn worker_process = new StringColumn("worker_process");
+        public static final StringColumn worker_thread = new StringColumn("worker_thread");
+        public static final StringColumn worker_extra = new StringColumn("worker_extra");
+        public static final StringColumn time_updated = new StringColumn("time_updated");
+        public static final StringColumn state = new StringColumn("state");
+
+        public enum State implements ColumnEnum<String> {
+            PENDING,
+            QUEUED,
+            SCHEDULED,
+            SELECTED,
+            RUNNING,
+            CANCELING,
+            CANCELED,
+            FAILED,
+            COMPLETED
+            ;
+
+            public StringColumn getColumn() {
+                return state;
+            }
+        }
+    }
+}