diff --git a/app/src/components/SyncService.js b/app/src/components/SyncService.js index 71ee4fbbd..80b02e508 100644 --- a/app/src/components/SyncService.js +++ b/app/src/components/SyncService.js @@ -1,9 +1,14 @@ import { useCallback, useEffect } from 'react'; -import { BuildParamsState, UIState } from '../store'; +import { BuildParamsState, DatapointSyncState, UIState } from '../store'; import { backgroundTask } from '../lib'; -import crudJobs, { jobStatus, MAX_ATTEMPT } from '../database/crud/crud-jobs'; +import crudJobs, { + jobStatus, + MAX_ATTEMPT, + SYNC_DATAPOINT_JOB_NAME, +} from '../database/crud/crud-jobs'; import { SYNC_FORM_SUBMISSION_TASK_NAME, syncStatus } from '../lib/background-task'; import { crudDataPoints } from '../database/crud'; +import { downloadDatapointsJson, fetchDatapoints } from '../lib/sync-datapoints'; /** * This sync only works in the foreground service */ @@ -94,6 +99,60 @@ const SyncService = () => { }; }, [syncInSecond, isOnline, onSync]); + const onSyncDataPoint = useCallback(async () => { + const activeJob = await crudJobs.getActiveJob(SYNC_DATAPOINT_JOB_NAME); + + DatapointSyncState.update((s) => { + s.added = false; + s.inProgress = activeJob ? true : false; + }); + + if (activeJob && activeJob.status === jobStatus.PENDING && activeJob.attempt < MAX_ATTEMPT) { + await crudJobs.updateJob(activeJob.id, { + status: jobStatus.ON_PROGRESS, + }); + + try { + const allData = await fetchDatapoints(); + const urls = allData.map(({ url, form_id: formId }) => ({ url, formId })); + await Promise.all(urls.map(({ formId, url }) => downloadDatapointsJson(formId, url))); + await crudJobs.deleteJob(activeJob.id); + + DatapointSyncState.update((s) => { + s.inProgress = false; + }); + } catch (error) { + DatapointSyncState.update((s) => { + s.added = true; + }); + await crudJobs.updateJob(activeJob.id, { + status: jobStatus.PENDING, + attempt: activeJob.attempt + 1, + info: String(error), + }); + } + } + + if (activeJob && activeJob.status === jobStatus.PENDING && activeJob.attempt === MAX_ATTEMPT) { + await crudJobs.deleteJob(activeJob.id); + } + }, []); + + useEffect(() => { + const unsubsDataSync = DatapointSyncState.subscribe( + (s) => s.added, + (added) => { + if (added) { + onSyncDataPoint(); + } + }, + ); + + return () => { + unsubsDataSync(); + }; + }, [onSyncDataPoint]); + return null; // This is a service component, no rendering is needed }; diff --git a/app/src/database/conn.js b/app/src/database/conn.js index 44adfc70b..0bdd76cf3 100644 --- a/app/src/database/conn.js +++ b/app/src/database/conn.js @@ -76,7 +76,7 @@ const removeDB = async () => { /** * Check user session before deletion */ - const db = openDatabase(); + const db = openDatabase('db.db'); const { rows } = await tx(db, 'SELECT * FROM users where active = ?', [1]); if (rows.length === 0) { /** diff --git a/app/src/database/crud/crud-forms.js b/app/src/database/crud/crud-forms.js index 517097bfc..4b5b89ba2 100644 --- a/app/src/database/crud/crud-forms.js +++ b/app/src/database/crud/crud-forms.js @@ -65,10 +65,13 @@ const formsQuery = () => { }); return await conn.tx(db, insertQuery, []); }, - updateForm: async ({ id: formId, latest = 0 }) => { - // update latest to false - const updateQuery = query.update('forms', { formId }, { latest: latest }); - return await conn.tx(db, updateQuery, [formId]); + updateForm: async ({ userId, formId, version, formJSON, latest = 1 }) => { + const updateQuery = query.update( + 'forms', + { userId, formId }, + { version, latest, json: formJSON ? JSON.stringify(formJSON).replace(/'/g, "''") : null }, + ); + return await conn.tx(db, updateQuery, [userId, formId]); }, getMyForms: async () => { const session = await crudUsers.getActiveUser(); diff --git a/app/src/database/crud/crud-jobs.js b/app/src/database/crud/crud-jobs.js index a24e18187..4b626fea6 100644 --- a/app/src/database/crud/crud-jobs.js +++ b/app/src/database/crud/crud-jobs.js @@ -13,6 +13,8 @@ export const jobStatus = { export const MAX_ATTEMPT = 3; +export const SYNC_DATAPOINT_JOB_NAME = 'sync-form-datapoints'; + const tableName = 'jobs'; const jobsQuery = () => { return { @@ -23,11 +25,12 @@ const jobsQuery = () => { /** * Make sure the app only gets active jobs from current user */ - const where = { active: 1, type, user: session.id }; + const where = { type, user: session.id }; + const params = [type, session.id]; const nocase = false; const order_by = 'createdAt'; const readQuery = query.read(tableName, where, nocase, order_by); - const { rows } = await conn.tx(db, readQuery, [1, type, session.id]); + const { rows } = await conn.tx(db, readQuery, params); if (!rows.length) { return null; } @@ -44,16 +47,15 @@ const jobsQuery = () => { const insertQuery = query.insert(tableName, { ...data, createdAt, - uuid: Crypto.randomUUID(), // TODO: Remove if not needed }); return await conn.tx(db, insertQuery, []); } catch (error) { - return null; + return Promise.reject(error); } }, updateJob: async (id, data) => { try { - const updateQuery = query.update(tableName, { id }, { ...data }); + const updateQuery = query.update(tableName, { id }, data); return await conn.tx(db, updateQuery, [id]); } catch { return null; diff --git a/app/src/database/crud/crud-monitoring.js b/app/src/database/crud/crud-monitoring.js index 9724267b7..dc18191d3 100644 --- a/app/src/database/crud/crud-monitoring.js +++ b/app/src/database/crud/crud-monitoring.js @@ -8,13 +8,37 @@ const monitoringQuery = () => { const insertQuery = query.insert('monitoring', { formId: formId, uuid: formJSON.uuid, - administration: formJSON?.administration, name: formJSON?.datapoint_name || null, json: formJSON ? JSON.stringify(formJSON.answers).replace(/'/g, "''") : null, syncedAt: new Date().toISOString(), }); return await conn.tx(db, insertQuery, []); }, + syncForm: async ({ formId, formJSON }) => { + const findQuery = query.read('monitoring', { uuid: formJSON.uuid }); + const { rows } = await conn.tx(db, findQuery, [formJSON.uuid]); + if (rows.length) { + const monitoringID = rows._array[0].id; + const updateQuery = query.update( + 'monitoring', + { id: monitoringID }, + { + json: formJSON ? JSON.stringify(formJSON.answers).replace(/'/g, "''") : null, + syncedAt: new Date().toISOString(), + }, + ); + return await conn.tx(db, updateQuery, [monitoringID]); + } else { + const insertQuery = query.insert('monitoring', { + formId: formId, + uuid: formJSON.uuid, + name: formJSON?.datapoint_name || null, + json: formJSON ? JSON.stringify(formJSON.answers).replace(/'/g, "''") : null, + syncedAt: new Date().toISOString(), + }); + return await conn.tx(db, insertQuery, []); + } + }, getAllForms: async () => { const sqlQuery = 'SELECT formId FROM monitoring'; const { rows } = await conn.tx(db, sqlQuery); diff --git a/app/src/database/crud/crud-users.js b/app/src/database/crud/crud-users.js index f4bdaaa51..005b45333 100644 --- a/app/src/database/crud/crud-users.js +++ b/app/src/database/crud/crud-users.js @@ -45,6 +45,11 @@ const usersQuery = () => { const { rows } = await conn.tx(db, query.read('users', { password: passcode }), [passcode]); return rows; }, + updateLastSynced: async (id) => { + const updateQuery = query.update('users', { id }, { lastSyncedAt: new Date().toISOString() }); + const { rows } = await conn.tx(db, updateQuery, [id]); + return rows; + }, }; }; diff --git a/app/src/database/tables.js b/app/src/database/tables.js index c03084f4d..d2d33c962 100644 --- a/app/src/database/tables.js +++ b/app/src/database/tables.js @@ -7,6 +7,7 @@ export const tables = [ password: 'TEXT', active: 'TINYINT', token: 'TEXT', + lastSyncedAt: 'DATETIME', }, }, { @@ -61,7 +62,6 @@ export const tables = [ formId: 'INTEGER NOT NULL', uuid: 'TEXT type UNIQUE', name: 'VARCHAR(255)', - administration: 'VARCHAR(255)', // TODO: Remove syncedAt: 'DATETIME', json: 'TEXT', }, @@ -78,13 +78,11 @@ export const tables = [ name: 'jobs', fields: { id: 'INTEGER PRIMARY KEY NOT NULL', - uuid: 'TEXT type UNIQUE', // TODO: Remove if not used user: 'INTEGER NOT NULL', type: 'VARCHAR(191)', status: 'INTEGER NOT NULL', attempt: 'INTEGER DEFAULT "0" NOT NULL', - active: 'TINYINT', - info: 'TEXT', + info: 'VARCHAR(255)', createdAt: 'DATETIME', }, }, diff --git a/app/src/lib/cascades.js b/app/src/lib/cascades.js index f92af2fd0..4020ee990 100644 --- a/app/src/lib/cascades.js +++ b/app/src/lib/cascades.js @@ -1,4 +1,3 @@ -import { Asset } from 'expo-asset'; import * as FileSystem from 'expo-file-system'; import * as SQLite from 'expo-sqlite'; import { conn, query } from '../database'; @@ -14,14 +13,21 @@ const createSqliteDir = async () => { } }; -const download = (downloadUrl, fileUrl) => { +const download = async (downloadUrl, fileUrl, update = false) => { const fileSql = fileUrl?.split('/')?.pop(); // get last segment as filename const pathSql = `${DIR_NAME}/${fileSql}`; - FileSystem.getInfoAsync(FileSystem.documentDirectory + pathSql).then(({ exists }) => { - if (!exists) { - FileSystem.downloadAsync(downloadUrl, FileSystem.documentDirectory + pathSql); - } - }); + console.info('Downloading...', downloadUrl); + const { exists } = await FileSystem.getInfoAsync(FileSystem.documentDirectory + pathSql); + if (exists && update) { + const existing_db = SQLite.openDatabase(fileSql); + existing_db.closeAsync(); + await existing_db.deleteAsync(); + } + if (!exists || update) { + await FileSystem.downloadAsync(downloadUrl, FileSystem.documentDirectory + pathSql, { + cache: false, + }); + } }; const loadDataSource = async (source, id = null) => { diff --git a/app/src/lib/i18n/ui-text.js b/app/src/lib/i18n/ui-text.js index b1949cf9b..648e276ca 100644 --- a/app/src/lib/i18n/ui-text.js +++ b/app/src/lib/i18n/ui-text.js @@ -100,6 +100,7 @@ const uiText = { settingAddNewFormPageTitle: 'Add New Form', errorFormsNotLoaded: 'Unable to load forms', downloadingData: 'Downloading data', + syncDataPointBtn: 'Sync Datapoint', syncingText: 'Syncing...', syncingFailedText: 'Syncing failed...', syncingSuccessText: 'Syncing successful...', @@ -107,6 +108,7 @@ const uiText = { doneText: 'Done!', about: 'About', autoSyncInProgress: 'Auto sync is in progress', + connectToInternet: 'Connect to the internet to sync', }, fr: { latitude: 'Latitude', diff --git a/app/src/lib/sync-datapoints.js b/app/src/lib/sync-datapoints.js new file mode 100644 index 000000000..be72128ee --- /dev/null +++ b/app/src/lib/sync-datapoints.js @@ -0,0 +1,36 @@ +import { crudMonitoring } from '../database/crud'; +import { DatapointSyncState } from '../store'; +import api from './api'; + +export const fetchDatapoints = async (pageNumber = 1) => { + try { + const { data: apiData } = await api.get(`/datapoint-list?page=${pageNumber}`); + const { data, total_page: totalPage, current: page } = apiData; + DatapointSyncState.update((s) => { + s.progress = (page / totalPage) * 100; + }); + if (page < totalPage) { + return data.concat(await fetchDatapoints(page + 1)); + } else { + return data; + } + } catch (error) { + return Promise.reject(error); + } +}; + +export const downloadDatapointsJson = async (formId, url) => { + try { + const response = await api.get(url); + if (response.status === 200) { + const jsonData = response.data; + const res = await crudMonitoring.syncForm({ + formId, + formJSON: jsonData, + }); + console.info('[SYNCED MONITORING]', res); + } + } catch (error) { + return Promise.reject(error); + } +}; diff --git a/app/src/pages/AuthForm.js b/app/src/pages/AuthForm.js index 6c36470d7..35a8acd18 100644 --- a/app/src/pages/AuthForm.js +++ b/app/src/pages/AuthForm.js @@ -63,10 +63,9 @@ const AuthForm = ({ navigation }) => { if (status === 'fulfilled') { const { data: apiData } = value; // download cascades files - apiData.cascades.forEach((cascadeFile) => { + apiData.cascades.forEach(async (cascadeFile) => { const downloadUrl = api.getConfig().baseURL + cascadeFile; - console.info('Downloading...', downloadUrl); - cascades.download(downloadUrl, cascadeFile); + await cascades.download(downloadUrl, cascadeFile); }); // insert all forms to database const form = formsUrl?.[index]; diff --git a/app/src/pages/FormPage.js b/app/src/pages/FormPage.js index be3e15aa0..c3fed78f8 100644 --- a/app/src/pages/FormPage.js +++ b/app/src/pages/FormPage.js @@ -9,6 +9,8 @@ import { } from 'react-native'; import { Button, Dialog, Text } from '@rneui/themed'; import Icon from 'react-native-vector-icons/Ionicons'; +import * as SQLite from 'expo-sqlite'; + import { FormContainer } from '../form'; import { SaveDialogMenu, SaveDropdownMenu } from '../form/support'; import { BaseLayout } from '../components'; @@ -40,7 +42,17 @@ const FormPage = ({ navigation, route }) => { const [currentDataPoint, setCurrentDataPoint] = useState({}); const [loading, setLoading] = useState(false); + const closeAllCascades = () => { + const { cascades: cascadesFiles } = formJSON || {}; + cascadesFiles?.forEach((csFile) => { + const [dbFile] = csFile?.split('/')?.slice(-1); + const connDB = SQLite.openDatabase(dbFile); + connDB.closeAsync(); + }); + }; + const refreshForm = () => { + closeAllCascades(); FormState.update((s) => { s.currentValues = {}; s.visitedQuestionGroup = []; @@ -182,7 +194,6 @@ const FormPage = ({ navigation, route }) => { await crudJobs.addJob({ user: userId, type: SYNC_FORM_SUBMISSION_TASK_NAME, - active: 1, status: jobStatus.PENDING, info: `${currentFormId} | ${datapoitName}`, }); diff --git a/app/src/pages/Home.js b/app/src/pages/Home.js index 493150ac5..41c39ccb2 100644 --- a/app/src/pages/Home.js +++ b/app/src/pages/Home.js @@ -1,13 +1,15 @@ import React, { useState, useEffect, useMemo, useCallback } from 'react'; -import { Button } from '@rneui/themed'; +import { Button, FAB } from '@rneui/themed'; import Icon from 'react-native-vector-icons/Ionicons'; import { Platform, ToastAndroid } from 'react-native'; import { BaseLayout } from '../components'; -import { FormState, UserState, UIState, BuildParamsState } from '../store'; -import { crudForms } from '../database/crud'; -import { i18n } from '../lib'; +import { FormState, UserState, UIState, BuildParamsState, DatapointSyncState } from '../store'; +import { crudForms, crudUsers } from '../database/crud'; +import { api, cascades, i18n } from '../lib'; import * as Notifications from 'expo-notifications'; import * as Location from 'expo-location'; +import * as FileSystem from 'expo-file-system'; +import crudJobs, { SYNC_DATAPOINT_JOB_NAME, jobStatus } from '../database/crud/crud-jobs'; const Home = ({ navigation, route }) => { const params = route?.params || null; @@ -15,11 +17,15 @@ const Home = ({ navigation, route }) => { const [data, setData] = useState([]); const [appLang, setAppLang] = useState('en'); const [loading, setloading] = useState(true); + const [syncLoading, setSyncLoading] = useState(false); const locationIsGranted = UserState.useState((s) => s.locationIsGranted); const gpsAccuracyLevel = BuildParamsState.useState((s) => s.gpsAccuracyLevel); const gpsInterval = BuildParamsState.useState((s) => s.gpsInterval); const isManualSynced = UIState.useState((s) => s.isManualSynced); + const userId = UserState.useState((s) => s.id); + const isOnline = UIState.useState((s) => s.online); + const activeLang = UIState.useState((s) => s.lang); const trans = i18n.text(activeLang); @@ -37,6 +43,59 @@ const Home = ({ navigation, route }) => { const goToUsers = () => { navigation.navigate('Users'); }; + const syncAllForms = async () => { + try { + const endpoints = data.map((d) => api.get(`/form/${d.formId}`)); + const results = await Promise.allSettled(endpoints); + const responses = results.filter(({ status }) => status === 'fulfilled'); + const cascadeFiles = responses.flatMap(({ value: res }) => res.data.cascades); + const downloadFiles = [...new Set(cascadeFiles)]; + + downloadFiles.forEach(async (file) => { + await cascades.download(api.getConfig().baseURL + file, file, true) + }); + + responses.forEach(async ({ value: res }) => { + const { data: apiData } = res; + const { id: formId, version } = apiData; + await crudForms.updateForm({ + userId, + formId, + version, + formJSON: apiData, + latest: 1, + }); + }); + + UIState.update((s) => { + /** + * Refresh homepage to apply latest data + */ + s.isManualSynced = true; + }); + } catch (error) { + return Promise.reject(error); + } + }; + + const handleOnSync = async () => { + setSyncLoading(true); + try { + await syncAllForms(); + await crudUsers.updateLastSynced(userId); + await crudJobs.addJob({ + user: userId, + type: SYNC_DATAPOINT_JOB_NAME, + status: jobStatus.PENDING, + }); + DatapointSyncState.update((s) => { + s.inProgress = true; + s.added = true; + }); + } catch (error) { + ToastAndroid.show(`[ERROR SYNC DATAPOINT]: ${error}`, ToastAndroid.LONG); + } + }; const getUserForms = useCallback(async () => { /** @@ -142,6 +201,21 @@ const Home = ({ navigation, route }) => { }; }, [watchCurrentPosition]); + useEffect(() => { + const unsubsDataSync = DatapointSyncState.subscribe( + (s) => s.inProgress, + (inProgress) => { + if (syncLoading && !inProgress) { + setSyncLoading(false); + } + }, + ); + + return () => { + unsubsDataSync(); + }; + }, [syncLoading]); + return ( { } > + ); }; diff --git a/app/src/store/datapoint-sync.js b/app/src/store/datapoint-sync.js new file mode 100644 index 000000000..e97518dd4 --- /dev/null +++ b/app/src/store/datapoint-sync.js @@ -0,0 +1,10 @@ +import { Store } from 'pullstate'; + +const DatapointSyncState = new Store({ + inProgress: false, + progress: 0, + added: false, + completed: false, +}); + +export default DatapointSyncState; diff --git a/app/src/store/index.js b/app/src/store/index.js index df2fb4abd..1d137c63f 100644 --- a/app/src/store/index.js +++ b/app/src/store/index.js @@ -3,3 +3,4 @@ export { default as BuildParamsState } from './buildParams'; export { default as FormState } from './forms'; export { default as UIState } from './ui'; export { default as UserState } from './users'; +export { default as DatapointSyncState } from './datapoint-sync'; diff --git a/backend/api/v1/v1_data/management/commands/fake_data_seeder.py b/backend/api/v1/v1_data/management/commands/fake_data_seeder.py index db071cfee..4de6d4c99 100644 --- a/backend/api/v1/v1_data/management/commands/fake_data_seeder.py +++ b/backend/api/v1/v1_data/management/commands/fake_data_seeder.py @@ -154,6 +154,7 @@ def seed_data(form, fake_geo, level_names, repeat, test): created_by=SystemUser.objects.order_by('?').first()) data.created = make_aware(created) level_id = administration.id + data.save_to_file data.save() add_fake_answers(data, form.type) else: @@ -165,6 +166,7 @@ def seed_data(form, fake_geo, level_names, repeat, test): administration=Administration.objects.filter( level=level).order_by('?').first(), created_by=SystemUser.objects.order_by('?').first()) + test_data.save_to_file test_data.save() add_fake_answers(test_data, form.type) diff --git a/backend/api/v1/v1_forms/serializers.py b/backend/api/v1/v1_forms/serializers.py index b0984fbbb..6b37ef8f2 100644 --- a/backend/api/v1/v1_forms/serializers.py +++ b/backend/api/v1/v1_forms/serializers.py @@ -261,6 +261,7 @@ def get_cascades(self, instance: Forms): class Meta: model = Forms fields = [ + 'id', 'name', 'version', 'cascades', diff --git a/backend/api/v1/v1_mobile/tests/tests_api.py b/backend/api/v1/v1_mobile/tests/tests_api.py index 24a022d8f..fbe05724a 100644 --- a/backend/api/v1/v1_mobile/tests/tests_api.py +++ b/backend/api/v1/v1_mobile/tests/tests_api.py @@ -126,6 +126,7 @@ def test_get_individual_forms_with_token(self): self.assertEqual( list(response.data), [ + 'id', 'name', 'version', 'cascades',