Merge pull request #1154 from akvo/feature/1153-download-the-JSON-fil…

…e-with-queue [#1153] Add sync button to manage form
akvo · Feb 14, 2024 · 02fe5dd · 02fe5dd
2 parents 308ddf3 + c89ab26
commit 02fe5dd
Show file tree

Hide file tree

Showing 18 changed files with 275 additions and 32 deletions.
diff --git a/app/src/components/SyncService.js b/app/src/components/SyncService.js
@@ -1,9 +1,14 @@
 import { useCallback, useEffect } from 'react';
-import { BuildParamsState, UIState } from '../store';
+import { BuildParamsState, DatapointSyncState, UIState } from '../store';
 import { backgroundTask } from '../lib';
-import crudJobs, { jobStatus, MAX_ATTEMPT } from '../database/crud/crud-jobs';
+import crudJobs, {
+  jobStatus,
+  MAX_ATTEMPT,
+  SYNC_DATAPOINT_JOB_NAME,
+} from '../database/crud/crud-jobs';
 import { SYNC_FORM_SUBMISSION_TASK_NAME, syncStatus } from '../lib/background-task';
 import { crudDataPoints } from '../database/crud';
+import { downloadDatapointsJson, fetchDatapoints } from '../lib/sync-datapoints';
 /**
  * This sync only works in the foreground service
  */
@@ -94,6 +99,60 @@ const SyncService = () => {
     };
   }, [syncInSecond, isOnline, onSync]);
 
+  const onSyncDataPoint = useCallback(async () => {
+    const activeJob = await crudJobs.getActiveJob(SYNC_DATAPOINT_JOB_NAME);
+
+    DatapointSyncState.update((s) => {
+      s.added = false;
+      s.inProgress = activeJob ? true : false;
+    });
+
+    if (activeJob && activeJob.status === jobStatus.PENDING && activeJob.attempt < MAX_ATTEMPT) {
+      await crudJobs.updateJob(activeJob.id, {
+        status: jobStatus.ON_PROGRESS,
+      });
+
+      try {
+        const allData = await fetchDatapoints();
+        const urls = allData.map(({ url, form_id: formId }) => ({ url, formId }));
+        await Promise.all(urls.map(({ formId, url }) => downloadDatapointsJson(formId, url)));
+        await crudJobs.deleteJob(activeJob.id);
+
+        DatapointSyncState.update((s) => {
+          s.inProgress = false;
+        });
+      } catch (error) {
+        DatapointSyncState.update((s) => {
+          s.added = true;
+        });
+        await crudJobs.updateJob(activeJob.id, {
+          status: jobStatus.PENDING,
+          attempt: activeJob.attempt + 1,
+          info: String(error),
+        });
+      }
+    }
+
+    if (activeJob && activeJob.status === jobStatus.PENDING && activeJob.attempt === MAX_ATTEMPT) {
+      await crudJobs.deleteJob(activeJob.id);
+    }
+  }, []);
+
+  useEffect(() => {
+    const unsubsDataSync = DatapointSyncState.subscribe(
+      (s) => s.added,
+      (added) => {
+        if (added) {
+          onSyncDataPoint();
+        }
+      },
+    );
+
+    return () => {
+      unsubsDataSync();
+    };
+  }, [onSyncDataPoint]);
+
   return null; // This is a service component, no rendering is needed
 };
 

diff --git a/app/src/database/conn.js b/app/src/database/conn.js
@@ -76,7 +76,7 @@ const removeDB = async () => {
       /**
        * Check user session before deletion
        */
-      const db = openDatabase();
+      const db = openDatabase('db.db');
       const { rows } = await tx(db, 'SELECT * FROM users where active = ?', [1]);
       if (rows.length === 0) {
         /**

diff --git a/app/src/database/crud/crud-forms.js b/app/src/database/crud/crud-forms.js
@@ -65,10 +65,13 @@ const formsQuery = () => {
       });
       return await conn.tx(db, insertQuery, []);
     },
-    updateForm: async ({ id: formId, latest = 0 }) => {
-      // update latest to false
-      const updateQuery = query.update('forms', { formId }, { latest: latest });
-      return await conn.tx(db, updateQuery, [formId]);
+    updateForm: async ({ userId, formId, version, formJSON, latest = 1 }) => {
+      const updateQuery = query.update(
+        'forms',
+        { userId, formId },
+        { version, latest, json: formJSON ? JSON.stringify(formJSON).replace(/'/g, "''") : null },
+      );
+      return await conn.tx(db, updateQuery, [userId, formId]);
     },
     getMyForms: async () => {
       const session = await crudUsers.getActiveUser();

diff --git a/app/src/database/crud/crud-jobs.js b/app/src/database/crud/crud-jobs.js
@@ -13,6 +13,8 @@ export const jobStatus = {
 
 export const MAX_ATTEMPT = 3;
 
+export const SYNC_DATAPOINT_JOB_NAME = 'sync-form-datapoints';
+
 const tableName = 'jobs';
 const jobsQuery = () => {
   return {
@@ -23,11 +25,12 @@ const jobsQuery = () => {
           /**
            * Make sure the app only gets active jobs from current user
            */
-          const where = { active: 1, type, user: session.id };
+          const where = { type, user: session.id };
+          const params = [type, session.id];
           const nocase = false;
           const order_by = 'createdAt';
           const readQuery = query.read(tableName, where, nocase, order_by);
-          const { rows } = await conn.tx(db, readQuery, [1, type, session.id]);
+          const { rows } = await conn.tx(db, readQuery, params);
           if (!rows.length) {
             return null;
           }
@@ -44,16 +47,15 @@ const jobsQuery = () => {
         const insertQuery = query.insert(tableName, {
           ...data,
           createdAt,
-          uuid: Crypto.randomUUID(), // TODO: Remove if not needed
         });
         return await conn.tx(db, insertQuery, []);
       } catch (error) {
-        return null;
+        return Promise.reject(error);
       }
     },
     updateJob: async (id, data) => {
       try {
-        const updateQuery = query.update(tableName, { id }, { ...data });
+        const updateQuery = query.update(tableName, { id }, data);
         return await conn.tx(db, updateQuery, [id]);
       } catch {
         return null;

diff --git a/app/src/database/crud/crud-monitoring.js b/app/src/database/crud/crud-monitoring.js
@@ -8,13 +8,37 @@ const monitoringQuery = () => {
       const insertQuery = query.insert('monitoring', {
         formId: formId,
         uuid: formJSON.uuid,
-        administration: formJSON?.administration,
         name: formJSON?.datapoint_name || null,
         json: formJSON ? JSON.stringify(formJSON.answers).replace(/'/g, "''") : null,
         syncedAt: new Date().toISOString(),
       });
       return await conn.tx(db, insertQuery, []);
     },
+    syncForm: async ({ formId, formJSON }) => {
+      const findQuery = query.read('monitoring', { uuid: formJSON.uuid });
+      const { rows } = await conn.tx(db, findQuery, [formJSON.uuid]);
+      if (rows.length) {
+        const monitoringID = rows._array[0].id;
+        const updateQuery = query.update(
+          'monitoring',
+          { id: monitoringID },
+          {
+            json: formJSON ? JSON.stringify(formJSON.answers).replace(/'/g, "''") : null,
+            syncedAt: new Date().toISOString(),
+          },
+        );
+        return await conn.tx(db, updateQuery, [monitoringID]);
+      } else {
+        const insertQuery = query.insert('monitoring', {
+          formId: formId,
+          uuid: formJSON.uuid,
+          name: formJSON?.datapoint_name || null,
+          json: formJSON ? JSON.stringify(formJSON.answers).replace(/'/g, "''") : null,
+          syncedAt: new Date().toISOString(),
+        });
+        return await conn.tx(db, insertQuery, []);
+      }
+    },
     getAllForms: async () => {
       const sqlQuery = 'SELECT formId FROM monitoring';
       const { rows } = await conn.tx(db, sqlQuery);

diff --git a/app/src/database/crud/crud-users.js b/app/src/database/crud/crud-users.js
@@ -45,6 +45,11 @@ const usersQuery = () => {
       const { rows } = await conn.tx(db, query.read('users', { password: passcode }), [passcode]);
       return rows;
     },
+    updateLastSynced: async (id) => {
+      const updateQuery = query.update('users', { id }, { lastSyncedAt: new Date().toISOString() });
+      const { rows } = await conn.tx(db, updateQuery, [id]);
+      return rows;
+    },
   };
 };
 

diff --git a/app/src/database/tables.js b/app/src/database/tables.js
@@ -7,6 +7,7 @@ export const tables = [
       password: 'TEXT',
       active: 'TINYINT',
       token: 'TEXT',
+      lastSyncedAt: 'DATETIME',
     },
   },
   {
@@ -61,7 +62,6 @@ export const tables = [
       formId: 'INTEGER NOT NULL',
       uuid: 'TEXT type UNIQUE',
       name: 'VARCHAR(255)',
-      administration: 'VARCHAR(255)', // TODO: Remove
       syncedAt: 'DATETIME',
       json: 'TEXT',
     },
@@ -78,13 +78,11 @@ export const tables = [
     name: 'jobs',
     fields: {
       id: 'INTEGER PRIMARY KEY NOT NULL',
-      uuid: 'TEXT type UNIQUE', // TODO: Remove if not used
       user: 'INTEGER NOT NULL',
       type: 'VARCHAR(191)',
       status: 'INTEGER NOT NULL',
       attempt: 'INTEGER DEFAULT "0" NOT NULL',
-      active: 'TINYINT',
-      info: 'TEXT',
+      info: 'VARCHAR(255)',
       createdAt: 'DATETIME',
     },
   },

diff --git a/app/src/lib/cascades.js b/app/src/lib/cascades.js
@@ -1,4 +1,3 @@
-import { Asset } from 'expo-asset';
 import * as FileSystem from 'expo-file-system';
 import * as SQLite from 'expo-sqlite';
 import { conn, query } from '../database';
@@ -14,14 +13,21 @@ const createSqliteDir = async () => {
   }
 };
 
-const download = (downloadUrl, fileUrl) => {
+const download = async (downloadUrl, fileUrl, update = false) => {
   const fileSql = fileUrl?.split('/')?.pop(); // get last segment as filename
   const pathSql = `${DIR_NAME}/${fileSql}`;
-  FileSystem.getInfoAsync(FileSystem.documentDirectory + pathSql).then(({ exists }) => {
-    if (!exists) {
-      FileSystem.downloadAsync(downloadUrl, FileSystem.documentDirectory + pathSql);
-    }
-  });
+  console.info('Downloading...', downloadUrl);
+  const { exists } = await FileSystem.getInfoAsync(FileSystem.documentDirectory + pathSql);
+  if (exists && update) {
+    const existing_db = SQLite.openDatabase(fileSql);
+    existing_db.closeAsync();
+    await existing_db.deleteAsync();
+  }
+  if (!exists || update) {
+    await FileSystem.downloadAsync(downloadUrl, FileSystem.documentDirectory + pathSql, {
+      cache: false,
+    });
+  }
 };
 
 const loadDataSource = async (source, id = null) => {

diff --git a/app/src/lib/i18n/ui-text.js b/app/src/lib/i18n/ui-text.js
@@ -100,13 +100,15 @@ const uiText = {
     settingAddNewFormPageTitle: 'Add New Form',
     errorFormsNotLoaded: 'Unable to load forms',
     downloadingData: 'Downloading data',
+    syncDataPointBtn: 'Sync Datapoint',
     syncingText: 'Syncing...',
     syncingFailedText: 'Syncing failed...',
     syncingSuccessText: 'Syncing successful...',
     reSyncingText: 'Re-syncing...',
     doneText: 'Done!',
     about: 'About',
     autoSyncInProgress: 'Auto sync is in progress',
+    connectToInternet: 'Connect to the internet to sync',
   },
   fr: {
     latitude: 'Latitude',

diff --git a/app/src/lib/sync-datapoints.js b/app/src/lib/sync-datapoints.js
@@ -0,0 +1,36 @@
+import { crudMonitoring } from '../database/crud';
+import { DatapointSyncState } from '../store';
+import api from './api';
+
+export const fetchDatapoints = async (pageNumber = 1) => {
+  try {
+    const { data: apiData } = await api.get(`/datapoint-list?page=${pageNumber}`);
+    const { data, total_page: totalPage, current: page } = apiData;
+    DatapointSyncState.update((s) => {
+      s.progress = (page / totalPage) * 100;
+    });
+    if (page < totalPage) {
+      return data.concat(await fetchDatapoints(page + 1));
+    } else {
+      return data;
+    }
+  } catch (error) {
+    return Promise.reject(error);
+  }
+};
+
+export const downloadDatapointsJson = async (formId, url) => {
+  try {
+    const response = await api.get(url);
+    if (response.status === 200) {
+      const jsonData = response.data;
+      const res = await crudMonitoring.syncForm({
+        formId,
+        formJSON: jsonData,
+      });
+      console.info('[SYNCED MONITORING]', res);
+    }
+  } catch (error) {
+    return Promise.reject(error);
+  }
+};
diff --git a/app/src/pages/AuthForm.js b/app/src/pages/AuthForm.js
@@ -63,10 +63,9 @@ const AuthForm = ({ navigation }) => {
       if (status === 'fulfilled') {
         const { data: apiData } = value;
         // download cascades files
-        apiData.cascades.forEach((cascadeFile) => {
+        apiData.cascades.forEach(async (cascadeFile) => {
           const downloadUrl = api.getConfig().baseURL + cascadeFile;
-          console.info('Downloading...', downloadUrl);
-          cascades.download(downloadUrl, cascadeFile);
+          await cascades.download(downloadUrl, cascadeFile);
         });
         // insert all forms to database
         const form = formsUrl?.[index];

diff --git a/app/src/pages/FormPage.js b/app/src/pages/FormPage.js
@@ -9,6 +9,8 @@ import {
 } from 'react-native';
 import { Button, Dialog, Text } from '@rneui/themed';
 import Icon from 'react-native-vector-icons/Ionicons';
+import * as SQLite from 'expo-sqlite';
+
 import { FormContainer } from '../form';
 import { SaveDialogMenu, SaveDropdownMenu } from '../form/support';
 import { BaseLayout } from '../components';
@@ -40,7 +42,17 @@ const FormPage = ({ navigation, route }) => {
   const [currentDataPoint, setCurrentDataPoint] = useState({});
   const [loading, setLoading] = useState(false);
 
+  const closeAllCascades = () => {
+    const { cascades: cascadesFiles } = formJSON || {};
+    cascadesFiles?.forEach((csFile) => {
+      const [dbFile] = csFile?.split('/')?.slice(-1);
+      const connDB = SQLite.openDatabase(dbFile);
+      connDB.closeAsync();
+    });
+  };
+
   const refreshForm = () => {
+    closeAllCascades();
     FormState.update((s) => {
       s.currentValues = {};
       s.visitedQuestionGroup = [];
@@ -182,7 +194,6 @@ const FormPage = ({ navigation, route }) => {
       await crudJobs.addJob({
         user: userId,
         type: SYNC_FORM_SUBMISSION_TASK_NAME,
-        active: 1,
         status: jobStatus.PENDING,
         info: `${currentFormId} | ${datapoitName}`,
       });