From 7e27ecbf0d7d923780ff348273cb94696e78fa6a Mon Sep 17 00:00:00 2001 From: "K. Shankari" Date: Tue, 11 Jul 2023 23:32:25 -0700 Subject: [PATCH 01/52] =?UTF-8?q?=F0=9F=97=83=EF=B8=8F=20Check=20and=20cop?= =?UTF-8?q?y=20missing=20local=20storage=20keys=20on=20start=20and=20resum?= =?UTF-8?q?e?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is the core of the workaround for https://github.com/e-mission/e-mission-docs/issues/930 Essentially, we can store key-value data in both web-based local storage, and as a local storage type in the SQLite database. In practice, we have found that both of these are unreliable. On iOS, the webview based local storage is sometimes deleted, and on android, the SQLite reads sometimes return null. Therefore, we plan to treat them as a RAID1 array. We will store identical copies of "local storage" data in both locations, and ensure that they are in sync on every app load and resume. We keep them in sync by geting lists of keys for both storage locations, finding the missing ones in each and copying them over. Testing done: Bunch of keys are present: ``` unifiedlogger.js:49 DEBUG:STORAGE_PLUGIN: app has resumed, checking storage sync storage.js:169 STORAGE_PLUGIN: Called syncAllWebAndNativeValues storage.js:171 STORAGE_PLUGIN: native plugin returned unifiedlogger.js:49 DEBUG:STORAGE_PLUGIN: Comparing web keys CONFIG_PHONE_UI,connection_settings with CONFIG_PHONE_UI,CONFIG_PHONE_UI,connection_settings unifiedlogger.js:49 DEBUG:STORAGE_PLUGIN: Found native keys CONFIG_PHONE_UI,connection_settings missing native keys unifiedlogger.js:49 DEBUG:STORAGE_PLUGIN: Found web keys CONFIG_PHONE_UI,CONFIG_PHONE_UI,connection_settings missing web keys unifiedlogger.js:49 DEBUG:STORAGE_PLUGIN: Syncing all missing keys unifiedlogger.js:49 DEBUG:STORAGE_PLUGIN: For the record, all unique native keys are stats/client_nav_event,stats/client_nav_event,background/battery,CONFIG_PHONE_UI,stats/client_time,config/app_ui_config,CONFIG_PHONE_UI,connection_settings,stats/client_time startprefs.js:72 Not consented in local storage, need to show consent 2startprefs.js:76 Consented in local storage, no need to show consent ``` From the profile screen, use the `Nuke all buffers and cache` option to delete all usercache information. On resume, there are no native keys, but there are a bunch of web keys We copy all of them over. ``` unifiedlogger.js:49 DEBUG:STORAGE_PLUGIN: app has resumed, checking storage sync storage.js:169 STORAGE_PLUGIN: Called syncAllWebAndNativeValues storage.js:171 STORAGE_PLUGIN: native plugin returned unifiedlogger.js:49 DEBUG:STORAGE_PLUGIN: Comparing web keys intro_done,connection_settings,data_collection_consented_protocol,CONFIG_PHONE_UI,prompted-auth with unifiedlogger.js:49 DEBUG:STORAGE_PLUGIN: Found native keys missing native keys intro_done,connection_settings,data_collection_consented_protocol,CONFIG_PHONE_UI,prompted-auth unifiedlogger.js:49 DEBUG:STORAGE_PLUGIN: Found web keys missing web keys unifiedlogger.js:49 DEBUG:STORAGE_PLUGIN: Syncing all missing keys intro_done,connection_settings,data_collection_consented_protocol,CONFIG_PHONE_UI,prompted-auth unifiedlogger.js:49 DEBUG:STORAGE_PLUGIN: For the record, all unique native keys are ``` On the next resume, they are all there ``` DEBUG:STORAGE_PLUGIN: app has resumed, checking storage sync storage.js:169 STORAGE_PLUGIN: Called syncAllWebAndNativeValues storage.js:171 STORAGE_PLUGIN: native plugin returned unifiedlogger.js:49 DEBUG:STORAGE_PLUGIN: Comparing web keys intro_done,connection_settings,CURR_GEOFENCE_LOCATION,data_collection_consented_protocol,CONFIG_PHONE_UI,prompted-auth with intro_done,connection_settings,CURR_GEOFENCE_LOCATION,data_collection_consented_protocol,CONFIG_PHONE_UI,prompted-auth unifiedlogger.js:49 DEBUG:STORAGE_PLUGIN: Found native keys intro_done,connection_settings,CURR_GEOFENCE_LOCATION,data_collection_consented_protocol,CONFIG_PHONE_UI,prompted-auth missing native keys unifiedlogger.js:49 DEBUG:STORAGE_PLUGIN: Found web keys intro_done,connection_settings,CURR_GEOFENCE_LOCATION,data_collection_consented_protocol,CONFIG_PHONE_UI,prompted-auth missing web keys unifiedlogger.js:49 DEBUG:STORAGE_PLUGIN: Syncing all missing keys unifiedlogger.js:49 DEBUG:STORAGE_PLUGIN: For the record, all unique native keys are intro_done,connection_settings,CURR_GEOFENCE_LOCATION,data_collection_consented_protocol,CONFIG_PHONE_UI,prompted-auth,stats/client_time ``` --- www/js/plugin/storage.js | 66 ++++++++++++++++++++++++++++++++++------ 1 file changed, 57 insertions(+), 9 deletions(-) diff --git a/www/js/plugin/storage.js b/www/js/plugin/storage.js index 9d14bb334..afa5d4c3f 100644 --- a/www/js/plugin/storage.js +++ b/www/js/plugin/storage.js @@ -3,7 +3,8 @@ import angular from 'angular'; angular.module('emission.plugin.kvstore', ['emission.plugin.logger', 'LocalStorageModule']) -.factory('KVStore', function($window, Logger, localStorageService, $ionicPopup) { +.factory('KVStore', function($window, Logger, localStorageService, $ionicPopup, + $ionicPlatform) { var logger = Logger; var kvstoreJs = {} /* @@ -46,7 +47,7 @@ angular.module('emission.plugin.kvstore', ['emission.plugin.logger', var getUnifiedValue = function(key) { var ls_stored_val = localStorageService.get(key, undefined); return getNativePlugin().getLocalStorage(key, false).then(function(uc_stored_val) { - logger.log("uc_stored_val = "+JSON.stringify(uc_stored_val)+" ls_stored_val = "+JSON.stringify(ls_stored_val)); + logger.log("for key "+key+" uc_stored_val = "+JSON.stringify(uc_stored_val)+" ls_stored_val = "+JSON.stringify(ls_stored_val)); if (angular.equals(ls_stored_val, uc_stored_val)) { logger.log("local and native values match, already synced"); return uc_stored_val; @@ -54,7 +55,7 @@ angular.module('emission.plugin.kvstore', ['emission.plugin.logger', // the values are different if (ls_stored_val == null) { console.assert(uc_stored_val != null, "uc_stored_val should be non-null"); - logger.log("uc_stored_val = "+JSON.stringify(uc_stored_val)+ + logger.log("for key "+key+"uc_stored_val = "+JSON.stringify(uc_stored_val)+ " ls_stored_val = "+JSON.stringify(ls_stored_val)+ " copying native "+key+" to local..."); localStorageService.set(key, uc_stored_val); @@ -70,7 +71,7 @@ angular.module('emission.plugin.kvstore', ['emission.plugin.logger', ls_stored_val = mungeValue(key, ls_stored_val); $ionicPopup.alert({template: "Local "+key+" found, native " +key+" missing, writing "+key+" to native"}) - logger.log("uc_stored_val = "+JSON.stringify(uc_stored_val)+ + logger.log("for key "+key+"uc_stored_val = "+JSON.stringify(uc_stored_val)+ " ls_stored_val = "+JSON.stringify(ls_stored_val)+ " copying local "+key+" to native..."); return getNativePlugin().putLocalStorage(key, ls_stored_val).then(function() { @@ -83,7 +84,7 @@ angular.module('emission.plugin.kvstore', ['emission.plugin.logger', "uc_stored_val ="+JSON.stringify(uc_stored_val)); $ionicPopup.alert({template: "Local "+key+" found, native " +key+" found, but different, writing "+key+" to local"}) - logger.log("uc_stored_val = "+JSON.stringify(uc_stored_val)+ + logger.log("for key "+key+"uc_stored_val = "+JSON.stringify(uc_stored_val)+ " ls_stored_val = "+JSON.stringify(ls_stored_val)+ " copying native "+key+" to local..."); localStorageService.set(key, uc_stored_val); @@ -139,10 +140,9 @@ angular.module('emission.plugin.kvstore', ['emission.plugin.logger', } /* - * TODO: remove these two functions after we have confirmed that native - * storage is never deleted weirdly, and returned to only native storage. - * In that case, there will be only one clear - of native storage - which - * will be covered using clearAll. + * Unfortunately, there is weird deletion of native + * https://github.com/e-mission/e-mission-docs/issues/930 + * So we cannot remove this if/until we switch to react native */ kvstoreJs.clearOnlyLocal = function() { return localStorageService.clearAll(); @@ -152,5 +152,53 @@ angular.module('emission.plugin.kvstore', ['emission.plugin.logger', return getNativePlugin().clearAll(); } + let findMissing = function(fromKeys, toKeys) { + const foundKeys = []; + const missingKeys = []; + fromKeys.forEach((fk) => { + if (toKeys.includes(fk)) { + foundKeys.push(fk); + } else { + missingKeys.push(fk); + } + }); + return [foundKeys, missingKeys]; + } + + let syncAllWebAndNativeValues = function() { + console.log("STORAGE_PLUGIN: Called syncAllWebAndNativeValues "); + const syncKeys = getNativePlugin().listAllLocalStorageKeys().then((nativeKeys) => { + console.log("STORAGE_PLUGIN: native plugin returned"); + const webKeys = localStorageService.keys(); + // I thought about iterating through the lists and copying over + // only missing values, etc but `getUnifiedValue` already does + // that, and we don't need to copy it + // so let's just find all the missing values and read them + logger.log("STORAGE_PLUGIN: Comparing web keys "+webKeys+" with "+nativeKeys); + let [foundNative, missingNative] = findMissing(webKeys, nativeKeys); + let [foundWeb, missingWeb] = findMissing(nativeKeys, webKeys); + logger.log("STORAGE_PLUGIN: Found native keys "+foundNative+" missing native keys "+missingNative); + logger.log("STORAGE_PLUGIN: Found web keys "+foundWeb+" missing web keys "+missingWeb); + + const allMissing = missingNative.concat(missingWeb); + logger.log("STORAGE_PLUGIN: Syncing all missing keys "+allMissing); + allMissing.forEach(getUnifiedValue); + }); + const listAllKeys = getNativePlugin().listAllUniqueKeys().then((nativeKeys) => { + logger.log("STORAGE_PLUGIN: For the record, all unique native keys are "+nativeKeys); + }); + return Promise.all([syncKeys, listAllKeys]); + } + + $ionicPlatform.ready().then(function() { + Logger.log("STORAGE_PLUGIN: app launched, checking storage sync"); + syncAllWebAndNativeValues(); + }); + + $ionicPlatform.on("resume", function() { + Logger.log("STORAGE_PLUGIN: app has resumed, checking storage sync"); + syncAllWebAndNativeValues(); + }); + return kvstoreJs; }); From dedf98c5510ea47716e02f5589d7d0b10822cecb Mon Sep 17 00:00:00 2001 From: "K. Shankari" Date: Tue, 11 Jul 2023 23:45:42 -0700 Subject: [PATCH 02/52] =?UTF-8?q?=F0=9F=97=83=EF=B8=8F=20Use=20the=20KVsto?= =?UTF-8?q?re=20plugin=20to=20directly=20store=20the=20opcode?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit So it is stored both in the webview local storage and the SQLite database. This means that we no longer need `setOPCode` in the auth plugin, which was simplified in https://github.com/e-mission/cordova-jwt-auth/pull/48 Other plugins were adapter to this simplified version in https://github.com/e-mission/cordova-server-communication/pull/32 and https://github.com/e-mission/cordova-server-sync/pull/55 --- www/js/intro.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/www/js/intro.js b/www/js/intro.js index 9101fbdce..24f18d7fa 100644 --- a/www/js/intro.js +++ b/www/js/intro.js @@ -8,6 +8,7 @@ angular.module('emission.intro', ['emission.splash.startprefs', 'emission.appstatus.permissioncheck', 'emission.i18n.utils', 'emission.config.dynamic', + 'emission.plugin.kvstore', 'ionic-toast', QrCode.module]) @@ -28,7 +29,7 @@ angular.module('emission.intro', ['emission.splash.startprefs', .controller('IntroCtrl', function($scope, $rootScope, $state, $window, $ionicPlatform, $ionicSlideBoxDelegate, - $ionicPopup, $ionicHistory, ionicToast, $timeout, CommHelper, StartPrefs, SurveyLaunch, DynamicConfig, i18nUtils) { + $ionicPopup, $ionicHistory, ionicToast, $timeout, CommHelper, StartPrefs, KVStore, SurveyLaunch, DynamicConfig, i18nUtils) { /* * Move all the state that is currently in the controller body into the init @@ -123,7 +124,9 @@ angular.module('emission.intro', ['emission.splash.startprefs', } $scope.login = function(token) { - window.cordova.plugins.OPCodeAuth.setOPCode(token).then(function(opcode) { + const EXPECTED_METHOD = "prompted-auth"; + const dbStorageObject = {"token": token}; + KVStore.set(EXPECTED_METHOD, dbStorageObject).then(function(opcode) { // ionicToast.show(message, position, stick, time); // $scope.next(); ionicToast.show(opcode, 'middle', false, 2500); From 0c748ed00e77417e23dd005da574745e32ecf2e8 Mon Sep 17 00:00:00 2001 From: "K. Shankari" Date: Tue, 11 Jul 2023 23:49:15 -0700 Subject: [PATCH 03/52] =?UTF-8?q?=F0=9F=97=83=EF=B8=8F=20Store=20the=20dyn?= =?UTF-8?q?amic=20config=20as=20local=20storage=20in=20addition=20to=20RW?= =?UTF-8?q?=20document?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit So it is stored both in the webview local storage and the SQLite database. While reading, we might only have the RW document in the older versions, and we use it to populate the local storage as needed for backwards compatibility and a gradual migration. We might want to remove the RW document in the future, but maybe at that time we will have more reliable SQLite storage anyway. --- www/js/config/dynamic_config.js | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/www/js/config/dynamic_config.js b/www/js/config/dynamic_config.js index 1f858a2b0..e09c0781f 100644 --- a/www/js/config/dynamic_config.js +++ b/www/js/config/dynamic_config.js @@ -2,12 +2,14 @@ import angular from 'angular'; -angular.module('emission.config.dynamic', ['emission.plugin.logger']) +angular.module('emission.config.dynamic', ['emission.plugin.logger', + 'emission.plugin.kvstore']) .factory('DynamicConfig', function($http, $ionicPlatform, - $window, $state, $rootScope, $timeout, Logger) { + $window, $state, $rootScope, $timeout, KVStore, Logger) { // also used in the startprefs class // but without importing this const CONFIG_PHONE_UI="config/app_ui_config"; + const CONFIG_PHONE_UI_KVSTORE ="CONFIG_PHONE_UI"; const LOAD_TIMEOUT = 6000; // 6000 ms = 6 seconds var dc = {}; @@ -62,8 +64,16 @@ angular.module('emission.config.dynamic', ['emission.plugin.logger']) var loadSavedConfig = function() { const nativePlugin = $window.cordova.plugins.BEMUserCache; - return nativePlugin.getDocument(CONFIG_PHONE_UI, false) - .then((savedConfig) => { + const rwDocRead = nativePlugin.getDocument(CONFIG_PHONE_UI, false); + const kvDocRead = KVStore.get(CONFIG_PHONE_UI_KVSTORE); + return Promise.all([rwDocRead, kvDocRead]) + .then(([rwConfig, kvStoreConfig]) => { + const savedConfig = kvStoreConfig? kvStoreConfig : rwConfig; + if (!kvStoreConfig && rwConfig) { + // Backwards compat, can remove at the end of 2023 + Logger.log("DYNAMIC CONFIG: rwConfig found, kvStoreConfig not found, setting to fix backwards compat"); + KVStore.set(CONFIG_PHONE_UI_KVSTORE, rwConfig); + } if (nativePlugin.isEmptyDoc(savedConfig)) { Logger.log("Found empty saved ui config, returning null"); return undefined; @@ -99,9 +109,11 @@ angular.module('emission.config.dynamic', ['emission.plugin.logger']) {joined: {opcode: dc.scannedToken, study_name: newStudyLabel, subgroup: subgroup}}); const storeConfigPromise = $window.cordova.plugins.BEMUserCache.putRWDocument( CONFIG_PHONE_UI, toSaveConfig); + const storeInKVStorePromise = KVStore.set(CONFIG_PHONE_UI_KVSTORE, toSaveConfig); const logSuccess = (storeResults) => Logger.log("UI_CONFIG: Stored dynamic config successfully, result = "+JSON.stringify(storeResults)); // loaded new config, so it is both ready and changed - return storeConfigPromise.then((result) => { + return Promise.all([storeConfigPromise, storeInKVStorePromise]).then( + ([result, kvStoreResult]) => { logSuccess(result); dc.saveAndNotifyConfigChanged(downloadedConfig); dc.saveAndNotifyConfigReady(downloadedConfig); From 2bee22c44f345149c90ae236307e02c6bf933e6f Mon Sep 17 00:00:00 2001 From: "K. Shankari" Date: Wed, 12 Jul 2023 09:10:59 -0700 Subject: [PATCH 04/52] :recycle: Refactor the code to read/set the dynamic config to be in the dynamic config module Before this, in addition to the dynamic config, we read and wrote the dynamic config document directly in other locations as well. ``` www/js//splash/startprefs.js|20| var CONFIGURED_KEY = "config/app_ui_config"; www/js//config/dynamic_config.js|11| const CONFIG_PHONE_UI="config/app_ui_config"; www/js//intro.js|85| const CONFIG_PHONE_UI="config/app_ui_config"; www/js//control/general-settings.js|411| const CONFIG_PHONE_UI="config/app_ui_config"; ``` This meant that the other locations were not reading from the KVstore and not setting the KVStore as well. This meant that: - if the native storage was deleted, then the KVStore value wouldn't be read, and we would assume that the config was empty and we needed to go back to the join page. - if the user disagreed, or tried to log out, only the rwConfig would be deleted, and the KVstore would fill in, so they would stay where they were Bonus fixes: - improved logging in the readConfig function - if the KVStore existed but the rwConfig did not, synced the rwConfig as well for triple redundancy Testing done: - Launch app, see trips in label screen - Go to the profile screen - "Nuke all buffers and cache" - Nuke native data - Go back to the label screen - Refresh - 500 internal server error - loading hangs continuously - Go back to the profile screen - consent, data_collection_consented_protocol, CONFIG_PHONE_UI, intro_done are copied over - note that prompted_auth is *not* - Go back to the label screen and reload - 500 error, loading hangs - Send app to the background and bring it back to the foreground - `syncAllWebAndNativeValues` called - `prompted-auth` and `connection_settings` restored - Refresh - trips load correctly - Logged out - went back to join screen - Tried to log in again, but disagreed - went back to join screen --- www/js/config/dynamic_config.js | 21 +++++++++++++++++++-- www/js/control/general-settings.js | 4 +--- www/js/intro.js | 4 +--- www/js/splash/startprefs.js | 18 +++++++----------- 4 files changed, 28 insertions(+), 19 deletions(-) diff --git a/www/js/config/dynamic_config.js b/www/js/config/dynamic_config.js index e09c0781f..911ae7cf0 100644 --- a/www/js/config/dynamic_config.js +++ b/www/js/config/dynamic_config.js @@ -62,18 +62,28 @@ angular.module('emission.config.dynamic', ['emission.plugin.logger', }); } - var loadSavedConfig = function() { + dc.loadSavedConfig = function() { const nativePlugin = $window.cordova.plugins.BEMUserCache; const rwDocRead = nativePlugin.getDocument(CONFIG_PHONE_UI, false); const kvDocRead = KVStore.get(CONFIG_PHONE_UI_KVSTORE); return Promise.all([rwDocRead, kvDocRead]) .then(([rwConfig, kvStoreConfig]) => { const savedConfig = kvStoreConfig? kvStoreConfig : rwConfig; + Logger.log("DYNAMIC CONFIG: kvStoreConfig key length = "+ Object.keys(kvStoreConfig).length + +" rwConfig key length = "+ Object.keys(rwConfig).length + +" using kvStoreConfig? "+(kvStoreConfig? true: false)); if (!kvStoreConfig && rwConfig) { // Backwards compat, can remove at the end of 2023 Logger.log("DYNAMIC CONFIG: rwConfig found, kvStoreConfig not found, setting to fix backwards compat"); KVStore.set(CONFIG_PHONE_UI_KVSTORE, rwConfig); } + if ((Object.keys(kvStoreConfig).length > 0) + && (Object.keys(rwConfig).length == 0)) { + // Might as well sync the RW config if it doesn't exist and + // have triple-redundancy for this + nativePlugin.putRWDocument(CONFIG_PHONE_UI, kvStoreConfig); + } + Logger.log("DYNAMIC CONFIG: final selected config = "+JSON.stringify(savedConfig)); if (nativePlugin.isEmptyDoc(savedConfig)) { Logger.log("Found empty saved ui config, returning null"); return undefined; @@ -87,6 +97,13 @@ angular.module('emission.config.dynamic', ['emission.plugin.logger', .catch((err) => Logger.displayError(i18next.t('config.unable-read-saved-config'), err)); } + dc.resetConfigAndRefresh = function() { + const resetNativePromise = $window.cordova.plugins.BEMUserCache.putRWDocument(CONFIG_PHONE_UI, {}); + const resetKVStorePromise = KVStore.set(CONFIG_PHONE_UI_KVSTORE, {}); + Promise.all([resetNativePromise, resetKVStorePromise]) + .then($window.location.reload(true)); + } + /** * loadNewConfig download and load a new config from the server if it is a differ * @param {[]} urlComponents specify the label of the config to load @@ -257,7 +274,7 @@ angular.module('emission.config.dynamic', ['emission.plugin.logger', } }; dc.initAtLaunch = function () { - loadSavedConfig().then((existingConfig) => { + dc.loadSavedConfig().then((existingConfig) => { if (!existingConfig) { return Logger.log("UI_CONFIG: No existing config, skipping"); } diff --git a/www/js/control/general-settings.js b/www/js/control/general-settings.js index 2ea96bc1e..c9a9e897f 100644 --- a/www/js/control/general-settings.js +++ b/www/js/control/general-settings.js @@ -408,9 +408,7 @@ angular.module('emission.main.control',['emission.services', if (!res) return; // user cancelled // reset the saved config, then trigger a hard refresh - const CONFIG_PHONE_UI="config/app_ui_config"; - $window.cordova.plugins.BEMUserCache.putRWDocument(CONFIG_PHONE_UI, {}) - .then($window.location.reload(true)); + DynamicConfig.resetConfigAndRefresh(); }); }; diff --git a/www/js/intro.js b/www/js/intro.js index 24f18d7fa..c96b62bef 100644 --- a/www/js/intro.js +++ b/www/js/intro.js @@ -82,9 +82,7 @@ angular.module('emission.intro', ['emission.splash.startprefs', /* If the user does not consent, we boot them back out to the join screen */ $scope.disagree = function() { // reset the saved config, then trigger a hard refresh - const CONFIG_PHONE_UI="config/app_ui_config"; - $window.cordova.plugins.BEMUserCache.putRWDocument(CONFIG_PHONE_UI, {}) - .then($window.location.reload(true)); + DynamicConfig.resetConfigAndRefresh(); }; $scope.agree = function() { diff --git a/www/js/splash/startprefs.js b/www/js/splash/startprefs.js index f688cd189..223c82579 100644 --- a/www/js/splash/startprefs.js +++ b/www/js/splash/startprefs.js @@ -2,10 +2,11 @@ import angular from 'angular'; angular.module('emission.splash.startprefs', ['emission.plugin.logger', 'emission.splash.referral', - 'emission.plugin.kvstore']) + 'emission.plugin.kvstore', + 'emission.config.dynamic']) .factory('StartPrefs', function($window, $state, $interval, $rootScope, $ionicPlatform, - $ionicPopup, KVStore, $http, Logger, ReferralHandler) { + $ionicPopup, KVStore, $http, Logger, ReferralHandler, DynamicConfig) { var logger = Logger; var nTimesCalled = 0; var startprefs = {}; @@ -17,7 +18,6 @@ angular.module('emission.splash.startprefs', ['emission.plugin.logger', var DATA_COLLECTION_CONSENTED_PROTOCOL = 'data_collection_consented_protocol'; var CONSENTED_KEY = "config/consent"; - var CONFIGURED_KEY = "config/app_ui_config"; startprefs.CONSENTED_EVENT = "data_collection_consented"; startprefs.INTRO_DONE_EVENT = "intro_done"; @@ -95,17 +95,13 @@ angular.module('emission.splash.startprefs', ['emission.plugin.logger', } startprefs.readConfig = function() { - const nativePlugin = $window.cordova.plugins.BEMUserCache; - return nativePlugin.getDocument(CONFIGURED_KEY, false).then((read_val) => { - logger.log("in readConfig, read_val = "+JSON.stringify(read_val)); - $rootScope.app_ui_label = read_val; - }); + return DynamicConfig.loadSavedConfig().then((savedConfig) => $rootScope.app_ui_label = savedConfig); } startprefs.hasConfig = function() { - const nativePlugin = $window.cordova.plugins.BEMUserCache; - if ($rootScope.app_ui_label == null || $rootScope.app_ui_label == "" - || nativePlugin.isEmptyDoc($rootScope.app_ui_label)) { + if ($rootScope.app_ui_label == undefined || + $rootScope.app_ui_label == null || + $rootScope.app_ui_label == "") { logger.log("Config not downloaded, need to show join screen"); $rootScope.has_config = false; return false; From f708df10e6d1643376a44428fa5a28b90e20df1e Mon Sep 17 00:00:00 2001 From: "K. Shankari" Date: Wed, 12 Jul 2023 10:08:16 -0700 Subject: [PATCH 05/52] :bug: Account for string responses while parsing nominatim responses --- www/js/diary/addressNamesHelper.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/www/js/diary/addressNamesHelper.ts b/www/js/diary/addressNamesHelper.ts index 0b85307a8..75713dbc3 100644 --- a/www/js/diary/addressNamesHelper.ts +++ b/www/js/diary/addressNamesHelper.ts @@ -153,7 +153,8 @@ export function useAddressNames(tlEntry) { useEffect(() => { if (locData) { - setAddressNames([toAddressName(JSON.parse(locData))]); + const loc = typeof locData === 'string' ? JSON.parse(locData) : locData; + setAddressNames([toAddressName(loc)]); } else if (startLocData && endLocData) { const startLoc = typeof startLocData === 'string' ? JSON.parse(startLocData) : startLocData; const endLoc = typeof endLocData === 'string' ? JSON.parse(endLocData) : endLocData; From cbc08c407ebb33efe65ebf4a5bd67137a438da1f Mon Sep 17 00:00:00 2001 From: "K. Shankari" Date: Wed, 12 Jul 2023 12:00:18 -0700 Subject: [PATCH 06/52] =?UTF-8?q?=F0=9F=93=88=20Record=20various=20flavors?= =?UTF-8?q?=20of=20missing=20keys=20in=20client=20stats?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit So that we can get an automated estimate of how often that happens, instead of having to rely on intermittent reports from participants. --- www/js/plugin/storage.js | 23 ++++++++++++++++++++--- www/js/stats/clientstats.js | 3 ++- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/www/js/plugin/storage.js b/www/js/plugin/storage.js index afa5d4c3f..a14b1db83 100644 --- a/www/js/plugin/storage.js +++ b/www/js/plugin/storage.js @@ -1,10 +1,11 @@ import angular from 'angular'; angular.module('emission.plugin.kvstore', ['emission.plugin.logger', - 'LocalStorageModule']) + 'LocalStorageModule', + 'emission.stats.clientstats']) .factory('KVStore', function($window, Logger, localStorageService, $ionicPopup, - $ionicPlatform) { + $ionicPlatform, ClientStats) { var logger = Logger; var kvstoreJs = {} /* @@ -179,14 +180,30 @@ angular.module('emission.plugin.kvstore', ['emission.plugin.logger', let [foundWeb, missingWeb] = findMissing(nativeKeys, webKeys); logger.log("STORAGE_PLUGIN: Found native keys "+foundNative+" missing native keys "+missingNative); logger.log("STORAGE_PLUGIN: Found web keys "+foundWeb+" missing web keys "+missingWeb); - const allMissing = missingNative.concat(missingWeb); logger.log("STORAGE_PLUGIN: Syncing all missing keys "+allMissing); allMissing.forEach(getUnifiedValue); + if (allMissing.length != 0) { + ClientStats.addReading(ClientStats.getStatKeys().MISSING_KEYS, { + "type": "local_storage_mismatch", + "allMissingLength": allMissing.length, + "missingWebLength": missingWeb.length, + "missingNativeLength": missingNative.length, + "foundWebLength": foundWeb.length, + "foundNativeLength": foundNative.length, + "allMissing": allMissing, + }).then(Logger.log("Logged missing keys to client stats")); + } }); const listAllKeys = getNativePlugin().listAllUniqueKeys().then((nativeKeys) => { logger.log("STORAGE_PLUGIN: For the record, all unique native keys are "+nativeKeys); + if (nativeKeys.length == 0) { + ClientStats.addReading(ClientStats.getStatKeys().MISSING_KEYS, { + "type": "all_native", + }).then(Logger.log("Logged all missing native keys to client stats")); + } }); + return Promise.all([syncKeys, listAllKeys]); } diff --git a/www/js/stats/clientstats.js b/www/js/stats/clientstats.js index e176892f9..7fe4d9cb3 100644 --- a/www/js/stats/clientstats.js +++ b/www/js/stats/clientstats.js @@ -25,7 +25,8 @@ angular.module('emission.stats.clientstats', []) SELECT_LABEL: "select_label", EXPANDED_TRIP: "expanded_trip", NOTIFICATION_OPEN: "notification_open", - REMINDER_PREFS: "reminder_time_prefs" + REMINDER_PREFS: "reminder_time_prefs", + MISSING_KEYS: "missing_keys" }; } From 4dcfbf020655d20f21dd86a86bd2190b8c9ffa6b Mon Sep 17 00:00:00 2001 From: "K. Shankari" Date: Wed, 12 Jul 2023 13:34:22 -0700 Subject: [PATCH 07/52] :bug: Avoid error on startup when the KVStore doesn't have a UI config It returns null, and then `Object.keys` fails. Works for non-null ``` t = {"abc": 1} Object { abc: 1 } Object.keys(t); Array [ "abc" ] ``` Fails for null ``` t = null null Object.keys(t); Uncaught TypeError: can't convert null to object debugger eval code:1 ``` This is the workaround ``` Object.keys(t || {}); Array [] ``` --- www/js/config/dynamic_config.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/www/js/config/dynamic_config.js b/www/js/config/dynamic_config.js index 911ae7cf0..337469527 100644 --- a/www/js/config/dynamic_config.js +++ b/www/js/config/dynamic_config.js @@ -69,16 +69,16 @@ angular.module('emission.config.dynamic', ['emission.plugin.logger', return Promise.all([rwDocRead, kvDocRead]) .then(([rwConfig, kvStoreConfig]) => { const savedConfig = kvStoreConfig? kvStoreConfig : rwConfig; - Logger.log("DYNAMIC CONFIG: kvStoreConfig key length = "+ Object.keys(kvStoreConfig).length - +" rwConfig key length = "+ Object.keys(rwConfig).length + Logger.log("DYNAMIC CONFIG: kvStoreConfig key length = "+ Object.keys(kvStoreConfig || {}).length + +" rwConfig key length = "+ Object.keys(rwConfig || {}).length +" using kvStoreConfig? "+(kvStoreConfig? true: false)); if (!kvStoreConfig && rwConfig) { // Backwards compat, can remove at the end of 2023 Logger.log("DYNAMIC CONFIG: rwConfig found, kvStoreConfig not found, setting to fix backwards compat"); KVStore.set(CONFIG_PHONE_UI_KVSTORE, rwConfig); } - if ((Object.keys(kvStoreConfig).length > 0) - && (Object.keys(rwConfig).length == 0)) { + if ((Object.keys(kvStoreConfig || {}).length > 0) + && (Object.keys(rwConfig || {}).length == 0)) { // Might as well sync the RW config if it doesn't exist and // have triple-redundancy for this nativePlugin.putRWDocument(CONFIG_PHONE_UI, kvStoreConfig); From b7b58ef64ff50f4541a3752ab680f6da9059c066 Mon Sep 17 00:00:00 2001 From: "K. Shankari" Date: Wed, 12 Jul 2023 14:08:45 -0700 Subject: [PATCH 08/52] =?UTF-8?q?=F0=9F=91=B7=20Support=20building=20the?= =?UTF-8?q?=20individual=20platforms=20separately?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Especially while working with native code, this makes the compile-deploy-test cycle much faster since we don't have to wait for an unrelated build to complete --- package.cordovabuild.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/package.cordovabuild.json b/package.cordovabuild.json index d08d08921..9eac316fc 100644 --- a/package.cordovabuild.json +++ b/package.cordovabuild.json @@ -9,7 +9,9 @@ }, "scripts": { "build": "npx webpack --config webpack.prod.js && npx cordova build", - "build-dev": "npx webpack --config webpack.dev.js && npx cordova build" + "build-dev": "npx webpack --config webpack.dev.js && npx cordova build", + "build-dev-android": "npx webpack --config webpack.dev.js && npx cordova build android", + "build-dev-ios": "npx webpack --config webpack.dev.js && npx cordova build ios" }, "devDependencies": { "@babel/core": "^7.21.3", From 259c20f426e7e9a164f5e1ccbd61e351ac8e8b6b Mon Sep 17 00:00:00 2001 From: "K. Shankari" Date: Wed, 12 Jul 2023 14:12:51 -0700 Subject: [PATCH 09/52] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20=20Upgrade=20native?= =?UTF-8?q?=20code=20dependencies=20related=20to=20the=20issue?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This should finally fix: https://github.com/e-mission/e-mission-docs/issues/930 --- package.cordovabuild.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/package.cordovabuild.json b/package.cordovabuild.json index 9eac316fc..120bafb73 100644 --- a/package.cordovabuild.json +++ b/package.cordovabuild.json @@ -122,12 +122,12 @@ "cordova-plugin-customurlscheme": "5.0.2", "cordova-plugin-device": "2.1.0", "cordova-plugin-em-datacollection": "git+https://github.com/e-mission/e-mission-data-collection.git#v1.7.8", - "cordova-plugin-em-opcodeauth": "git+https://github.com/e-mission/cordova-jwt-auth.git#v1.7.1", - "cordova-plugin-em-server-communication": "git+https://github.com/e-mission/cordova-server-communication.git#v1.2.5", - "cordova-plugin-em-serversync": "git+https://github.com/e-mission/cordova-server-sync.git#v1.3.1", + "cordova-plugin-em-opcodeauth": "git+https://github.com/e-mission/cordova-jwt-auth.git#v1.7.2", + "cordova-plugin-em-server-communication": "git+https://github.com/e-mission/cordova-server-communication.git#v1.2.6", + "cordova-plugin-em-serversync": "git+https://github.com/e-mission/cordova-server-sync.git#v1.3.2", "cordova-plugin-em-settings": "git+https://github.com/e-mission/cordova-connection-settings.git#v1.2.3", "cordova-plugin-em-unifiedlogger": "git+https://github.com/e-mission/cordova-unified-logger.git#v1.3.6", - "cordova-plugin-em-usercache": "git+https://github.com/e-mission/cordova-usercache.git#v1.1.5", + "cordova-plugin-em-usercache": "git+https://github.com/e-mission/cordova-usercache.git#v1.1.6", "cordova-plugin-email-composer": "git+https://github.com/katzer/cordova-plugin-email-composer.git#0.10.1", "cordova-plugin-file": "7.0.0", "cordova-plugin-inappbrowser": "5.0.0", From c5b0cc2976993699836ecba74d990c119acc69ec Mon Sep 17 00:00:00 2001 From: Jack Greenlee Date: Fri, 14 Jul 2023 16:30:08 -0400 Subject: [PATCH 10/52] remove unnecessary debug statements These were used during development but are now causing log spew --- www/js/components/LeafletView.jsx | 1 - www/js/diary/addressNamesHelper.ts | 1 - 2 files changed, 2 deletions(-) diff --git a/www/js/components/LeafletView.jsx b/www/js/components/LeafletView.jsx index 02a9eae69..f12774692 100644 --- a/www/js/components/LeafletView.jsx +++ b/www/js/components/LeafletView.jsx @@ -40,7 +40,6 @@ const LeafletView = ({ geojson, opts, ...otherProps }) => { (happens because of FlashList's view recycling on the trip cards: https://shopify.github.io/flash-list/docs/recycling) */ if (geoJsonIdRef.current && geoJsonIdRef.current !== geojson.data.id) { - console.debug('leafletMapRef changed, invalidating map', geoJsonIdRef.current, geojson.data.id); leafletMapRef.current.eachLayer(layer => leafletMapRef.current.removeLayer(layer)); initMap(leafletMapRef.current); } diff --git a/www/js/diary/addressNamesHelper.ts b/www/js/diary/addressNamesHelper.ts index 75713dbc3..154c853ee 100644 --- a/www/js/diary/addressNamesHelper.ts +++ b/www/js/diary/addressNamesHelper.ts @@ -158,7 +158,6 @@ export function useAddressNames(tlEntry) { } else if (startLocData && endLocData) { const startLoc = typeof startLocData === 'string' ? JSON.parse(startLocData) : startLocData; const endLoc = typeof endLocData === 'string' ? JSON.parse(endLocData) : endLocData; - console.debug('useAddressNames: startLocData = ', startLoc, 'endLocData = ', endLoc); setAddressNames([toAddressName(startLoc), toAddressName(endLoc)]); } }, [locData, startLocData, endLocData]); From 903f1a12efdd8d7768c34682c7603663823e6195 Mon Sep 17 00:00:00 2001 From: Jack Greenlee Date: Fri, 14 Jul 2023 21:45:24 -0400 Subject: [PATCH 11/52] make AddNoteButton responsive width There was a fixed with of 150, which causes problems if the button label text was longer. --- www/js/diary/DiaryButton.tsx | 3 ++- www/js/diary/cards/PlaceCard.tsx | 4 ++-- www/js/diary/cards/TripCard.tsx | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/www/js/diary/DiaryButton.tsx b/www/js/diary/DiaryButton.tsx index d197da794..8a50a8c57 100644 --- a/www/js/diary/DiaryButton.tsx +++ b/www/js/diary/DiaryButton.tsx @@ -30,7 +30,8 @@ const buttonStyles = StyleSheet.create({ height: 25, }, label: { - marginHorizontal: 4, + marginLeft: 24, + marginRight: 16, fontSize: 13, fontWeight: '500', flex: 1, diff --git a/www/js/diary/cards/PlaceCard.tsx b/www/js/diary/cards/PlaceCard.tsx index 28e2408ba..fd21e8126 100644 --- a/www/js/diary/cards/PlaceCard.tsx +++ b/www/js/diary/cards/PlaceCard.tsx @@ -41,7 +41,7 @@ const PlaceCard = ({ place }) => { - {/* add note button */} + {/* add note button */} { const s = StyleSheet.create({ notesButton: { paddingHorizontal: 8, - width: 150, + minWidth: 150, margin: 'auto', }, locationText: { diff --git a/www/js/diary/cards/TripCard.tsx b/www/js/diary/cards/TripCard.tsx index 1ded5fef4..52467cc5c 100644 --- a/www/js/diary/cards/TripCard.tsx +++ b/www/js/diary/cards/TripCard.tsx @@ -145,7 +145,7 @@ const s = StyleSheet.create({ notesButton: { paddingHorizontal: 8, paddingVertical: 12, - width: 150, + minWidth: 150, margin: 'auto', }, rightPanel: { From 8b20d86ab556e6496caeb56c79686afb86509b5f Mon Sep 17 00:00:00 2001 From: Jack Greenlee Date: Sat, 15 Jul 2023 00:30:43 -0400 Subject: [PATCH 12/52] add luxon typings Typings for Luxon make it much easier to use; we can benefit from IDE and Typescript features including IntelliSense, autocompletion, and type safety checks. --- package.cordovabuild.json | 1 + package.serve.json | 1 + 2 files changed, 2 insertions(+) diff --git a/package.cordovabuild.json b/package.cordovabuild.json index 120bafb73..72c8304aa 100644 --- a/package.cordovabuild.json +++ b/package.cordovabuild.json @@ -22,6 +22,7 @@ "@babel/preset-react": "^7.18.6", "@babel/preset-typescript": "^7.21.4", "@ionic/cli": "6.20.8", + "@types/luxon": "^3.3.0", "babel-loader": "^9.1.2", "babel-plugin-angularjs-annotate": "^0.10.0", "babel-plugin-optional-require": "^0.3.1", diff --git a/package.serve.json b/package.serve.json index c9bc1c9d8..88dc636a8 100644 --- a/package.serve.json +++ b/package.serve.json @@ -21,6 +21,7 @@ "@babel/preset-react": "^7.18.6", "@babel/preset-typescript": "^7.21.4", "@ionic/cli": "6.20.8", + "@types/luxon": "^3.3.0", "babel-loader": "^9.1.2", "babel-plugin-optional-require": "^0.3.1", "concurrently": "^8.0.1", From 5f36ee8718deb32964709ee681b020f05d6e6ee8 Mon Sep 17 00:00:00 2001 From: Jack Greenlee Date: Sat, 15 Jul 2023 00:38:58 -0400 Subject: [PATCH 13/52] diaryHelper: parse dates with respect to timezone The app interface should be showing the date and time of a trip/place *based on the timezone that the trip was recorded in.* So when we want to parse a trip and get human-readable dates, we can't just use the epoch timestamp. Epoch timestamps only tell us what the date was at a given moment in UTC. We need to consider the timezone of the trip. Thus, all these function need to be modified to accept the `fmt_time` strings (which are ISO 8601 and include timezone +/-). Then we can employ moment.parseZone and format. Result -- dates are now calculated and displayed properly: with respect to the timezone and not abnormal if the date in UTC differed from the actual date. note: the trip object's `display_duration` was removed because it is not even used anywhere in the codebase; `display_time` is consistently used instead. --- www/js/diary/diaryHelper.ts | 59 +++++++++++++++++++++------------- www/js/diary/services.js | 15 ++------- www/js/diary/timelineHelper.ts | 17 +++++----- 3 files changed, 46 insertions(+), 45 deletions(-) diff --git a/www/js/diary/diaryHelper.ts b/www/js/diary/diaryHelper.ts index 75000af0d..0feeb3cb6 100644 --- a/www/js/diary/diaryHelper.ts +++ b/www/js/diary/diaryHelper.ts @@ -3,6 +3,7 @@ import i18next from "i18next"; import moment from "moment"; +import { DateTime } from "luxon"; type MotionType = { name: string, @@ -40,28 +41,29 @@ export function motionTypeOf(motionName: MotionTypeKey | `MotionTypes.${MotionTy } /** - * @param beginTs Unix epoch timestamp in seconds - * @param endTs Unix epoch timestamp in seconds + * @param beginFmtTime An ISO 8601 formatted timestamp (with timezone) + * @param endTs An ISO 8601 formatted timestamp (with timezone) * @returns true if the start and end timestamps fall on different days + * @example isMultiDay("2023-07-13T00:00:00-07:00", "2023-07-14T00:00:00-07:00") => true */ -export function isMultiDay(beginTs: number, endTs: number) { - if (!beginTs || !endTs) return false; - return moment(beginTs * 1000).format('YYYYMMDD') != moment(endTs * 1000).format('YYYYMMDD'); +export function isMultiDay(beginFmtTime: string, endFmtTime: string) { + if (!beginFmtTime || !endFmtTime) return false; + return moment.parseZone(beginFmtTime).format('YYYYMMDD') != moment.parseZone(endFmtTime).format('YYYYMMDD'); } /** - * @param beginTs Unix epoch timestamp in seconds - * @param endTs Unix epoch timestamp in seconds + * @param beginFmtTime An ISO 8601 formatted timestamp (with timezone) + * @param endTs An ISO 8601 formatted timestamp (with timezone) * @returns A formatted range if both params are defined, one formatted date if only one is defined - * @example getFormattedDate(1683115200) => "Wed, May 3, 2023" + * @example getFormattedDate("2023-07-14T00:00:00-07:00") => "Fri, Jul 14, 2023" */ -export function getFormattedDate(beginTs: number, endTs?: number) { - if (!beginTs && !endTs) return; - if (isMultiDay(beginTs, endTs)) { - return `${getFormattedDate(beginTs)} - ${getFormattedDate(endTs)}`; +export function getFormattedDate(beginFmtTime: string, endFmtTime?: string) { + if (!beginFmtTime && !endFmtTime) return; + if (isMultiDay(beginFmtTime, endFmtTime)) { + return `${getFormattedDate(beginFmtTime)} - ${getFormattedDate(endFmtTime)}`; } // only one day given, or both are the same day - const t = moment.unix(beginTs || endTs); + const t = moment.parseZone(beginFmtTime || endFmtTime); // We use ddd LL to get Wed, May 3, 2023 or equivalent // LL only has the date, month and year // LLLL has the day of the week, but also the time @@ -69,20 +71,31 @@ export function getFormattedDate(beginTs: number, endTs?: number) { } /** - * @param beginTs Unix epoch timestamp in seconds - * @param endTs Unix epoch timestamp in seconds + * @param beginFmtTime An ISO 8601 formatted timestamp (with timezone) + * @param endTs An ISO 8601 formatted timestamp (with timezone) * @returns A formatted range if both params are defined, one formatted date if only one is defined - * @example getFormattedDateAbbr(1683115200) => "Wed, May 3" + * @example getFormattedDate("2023-07-14T00:00:00-07:00") => "Fri, Jul 14" */ -export function getFormattedDateAbbr(beginTs: number, endTs?: number) { - if (!beginTs && !endTs) return; - if (isMultiDay(beginTs, endTs)) { - return `${getFormattedDateAbbr(beginTs)} - ${getFormattedDateAbbr(endTs)}`; +export function getFormattedDateAbbr(beginFmtTime: string, endFmtTime?: string) { + if (!beginFmtTime && !endFmtTime) return; + if (isMultiDay(beginFmtTime, endFmtTime)) { + return `${getFormattedDateAbbr(beginFmtTime)} - ${getFormattedDateAbbr(endFmtTime)}`; } // only one day given, or both are the same day - const t = (beginTs || endTs) * 1000; - const opts: Intl.DateTimeFormatOptions = { weekday: 'short', month: 'short', day: 'numeric' }; - return Intl.DateTimeFormat(i18next.resolvedLanguage, opts).format(new Date(t)); + const dt = DateTime.fromISO(beginFmtTime || endFmtTime, { setZone: true }); + return dt.toLocaleString({ weekday: 'short', month: 'short', day: 'numeric' }); } +/** + * @param beginFmtTime An ISO 8601 formatted timestamp (with timezone) + * @param endFmtTime An ISO 8601 formatted timestamp (with timezone) + * @returns A human-readable, approximate time range, e.g. "2 hours" + */ +export function getFormattedTimeRange(beginFmtTime: string, endFmtTime: string) { + if (!beginFmtTime || !endFmtTime) return; + const beginMoment = moment.parseZone(beginFmtTime); + const endMoment = moment.parseZone(endFmtTime); + return endMoment.to(beginMoment, true); +}; + // the rest is TODO, still in services.js diff --git a/www/js/diary/services.js b/www/js/diary/services.js index 342422786..60ba59f72 100644 --- a/www/js/diary/services.js +++ b/www/js/diary/services.js @@ -1,7 +1,7 @@ 'use strict'; import angular from 'angular'; -import { motionTypeOf } from './diaryHelper'; +import { getFormattedTimeRange, motionTypeOf } from './diaryHelper'; angular.module('emission.main.diary.services', ['emission.plugin.logger', 'emission.services']) @@ -50,7 +50,7 @@ angular.module('emission.main.diary.services', ['emission.plugin.logger', dh.getFormattedSectionProperties = (trip, ImperialConfig) => { return trip.sections?.map((s) => ({ fmt_time: dh.getLocalTimeString(s.start_local_dt), - fmt_time_range: dh.getFormattedTimeRange(s.end_ts, s.start_ts), + fmt_time_range: getFormattedTimeRange(s.start_fmt_time, s.end_fmt_time), fmt_distance: ImperialConfig.getFormattedDistance(s.distance), fmt_distance_suffix: ImperialConfig.distanceSuffix, icon: motionTypeOf(s.sensed_mode_str)?.icon, @@ -66,17 +66,6 @@ angular.module('emission.main.diary.services', ['emission.plugin.logger', return moment(mdt).format("LT"); }; - dh.getFormattedTimeRange = function(end_ts_in_secs, start_ts_in_secs) { - if (isNaN(end_ts_in_secs) || isNaN(start_ts_in_secs)) return; - var startMoment = moment(start_ts_in_secs * 1000); - var endMoment = moment(end_ts_in_secs * 1000); - return endMoment.to(startMoment, true); - }; - dh.getFormattedDuration = function(duration_in_secs) { - if (isNaN(duration_in_secs)) return; - return moment.duration(duration_in_secs * 1000).humanize() - }; - /* this function was formerly 'CommonGraph.getDisplayName()', located in 'common/services.js' */ dh.getNominatimLocName = function(loc_geojson) { diff --git a/www/js/diary/timelineHelper.ts b/www/js/diary/timelineHelper.ts index 918f64618..32bcbcbde 100644 --- a/www/js/diary/timelineHelper.ts +++ b/www/js/diary/timelineHelper.ts @@ -1,5 +1,5 @@ import { getAngularService } from "../angular-react-helper"; -import { getFormattedDate, getFormattedDateAbbr, isMultiDay } from "./diaryHelper"; +import { getFormattedDate, getFormattedDateAbbr, getFormattedTimeRange, isMultiDay } from "./diaryHelper"; /** * @description Unpacks composite trips into a Map object of timeline items, by id. @@ -37,20 +37,19 @@ let DiaryHelper; */ export function populateBasicClasses(tlEntry) { DiaryHelper = DiaryHelper || getAngularService('DiaryHelper'); - const beginTs = tlEntry.start_ts || tlEntry.enter_ts; - const endTs = tlEntry.end_ts || tlEntry.exit_ts; + const beginFmt = tlEntry.start_fmt_time || tlEntry.enter_fmt_time; + const endFmt = tlEntry.end_fmt_time || tlEntry.exit_fmt_time; const beginDt = tlEntry.start_local_dt || tlEntry.enter_local_dt; const endDt = tlEntry.end_local_dt || tlEntry.exit_local_dt; - const tlEntryIsMultiDay = isMultiDay(beginTs, endTs); - tlEntry.display_date = getFormattedDate(beginTs, endTs); + const tlEntryIsMultiDay = isMultiDay(beginFmt, endFmt); + tlEntry.display_date = getFormattedDate(beginFmt, endFmt); tlEntry.display_start_time = DiaryHelper.getLocalTimeString(beginDt); tlEntry.display_end_time = DiaryHelper.getLocalTimeString(endDt); if (tlEntryIsMultiDay) { - tlEntry.display_start_date_abbr = getFormattedDateAbbr(beginTs); - tlEntry.display_end_date_abbr = getFormattedDateAbbr(endTs); + tlEntry.display_start_date_abbr = getFormattedDateAbbr(beginFmt); + tlEntry.display_end_date_abbr = getFormattedDateAbbr(endFmt); } - tlEntry.display_duration = DiaryHelper.getFormattedDuration(beginTs, endTs); - tlEntry.display_time = DiaryHelper.getFormattedTimeRange(beginTs, endTs); + tlEntry.display_time = getFormattedTimeRange(beginFmt, endFmt); tlEntry.percentages = DiaryHelper.getPercentages(tlEntry); // Pre-populate start and end names with   so they take up the same amount of vertical space in the UI before they are populated with real data tlEntry.start_display_name = "\xa0"; From 0f95cac1f82e9694b7c17bc692ce091cc30b2f98 Mon Sep 17 00:00:00 2001 From: Jack Greenlee Date: Sun, 16 Jul 2023 23:32:31 -0400 Subject: [PATCH 14/52] update CSP to fix enketo checkboxes' inline SVGs Enketo uses Sass (.scss) to define its styles. The way that the checkbox 'checks' are defined by Enketo is by defining an inline SVG as the background image to the checkbox. See inline SVG usage: https://github.com/enketo/enketo-core/blob/166af8c0d9bfe37ad65c4dbc5a805b458bae0948/src/sass/formhub/_mixins.scss#L16 This does not bundle properly in production builds, causing the checkmark to not appear. This was difficult to troubleshoot because it only happens in production. But on further investigation, I found the cause is the Content Security Policy, which is enforced more strictly in production. Updating this to allow inline evals fixed the issue. --- www/index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/www/index.html b/www/index.html index 4c3452c2c..d5d3266ad 100644 --- a/www/index.html +++ b/www/index.html @@ -3,7 +3,7 @@ - + From e0714b61299f3b5519740061056f57e4be6e533e Mon Sep 17 00:00:00 2001 From: Jack Greenlee Date: Sun, 16 Jul 2023 23:34:59 -0400 Subject: [PATCH 15/52] TimelineScrollList: indicator if no trips loaded Previous to this change, if no trips are received by a request, the loading spinner would show indefinitely. This fixes that behavior, and also implements a placeholder 'banner' which will show if there are no trips in view. --- www/js/diary/LabelTab.tsx | 2 +- www/js/diary/list/TimelineScrollList.tsx | 14 ++++++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/www/js/diary/LabelTab.tsx b/www/js/diary/LabelTab.tsx index 2e78f828e..8076cd461 100644 --- a/www/js/diary/LabelTab.tsx +++ b/www/js/diary/LabelTab.tsx @@ -203,7 +203,7 @@ const LabelTab = () => { }; useEffect(() => { - if (!displayedEntries?.length) return; + if (!displayedEntries) return; invalidateMaps(); setIsLoading(false); }, [displayedEntries]); diff --git a/www/js/diary/list/TimelineScrollList.tsx b/www/js/diary/list/TimelineScrollList.tsx index b47da34d1..570452681 100644 --- a/www/js/diary/list/TimelineScrollList.tsx +++ b/www/js/diary/list/TimelineScrollList.tsx @@ -5,7 +5,7 @@ import TripCard from '../cards/TripCard'; import PlaceCard from '../cards/PlaceCard'; import UntrackedTimeCard from '../cards/UntrackedTimeCard'; import { View } from 'react-native'; -import { ActivityIndicator } from 'react-native-paper'; +import { ActivityIndicator, Banner, IconButton, Text } from 'react-native-paper'; import LoadMoreButton from './LoadMoreButton'; import { useTranslation } from 'react-i18next'; @@ -45,7 +45,17 @@ const TimelineScrollList = ({ listEntries, queriedRange, pipelineRange, loadMore if (isLoading=='replace') { return bigSpinner; } else if (listEntries?.length == 0) { - return "No travel to show"; + return ( + + }> + + {"No travel to show"} + {"Check back after you've taken a few trips"} + + + ); } else { return ( Date: Mon, 17 Jul 2023 18:04:33 -0400 Subject: [PATCH 16/52] disable verify checkmark if trip already labeled trip.verifiability has 3 states: "can-verify", "already-verified", and "cannot-verify". The checkmark button should only be enabled if it is in the 'can-verify' state. --- www/js/survey/multilabel/MultiLabelButtonGroup.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/www/js/survey/multilabel/MultiLabelButtonGroup.tsx b/www/js/survey/multilabel/MultiLabelButtonGroup.tsx index d2dad1466..c44a60422 100644 --- a/www/js/survey/multilabel/MultiLabelButtonGroup.tsx +++ b/www/js/survey/multilabel/MultiLabelButtonGroup.tsx @@ -118,7 +118,7 @@ const MultilabelButtonGroup = ({ trip }) => { From 271319b8a0e65294abbea1ea1e0ab30332548e82 Mon Sep 17 00:00:00 2001 From: Jack Greenlee Date: Mon, 17 Jul 2023 21:52:58 -0400 Subject: [PATCH 17/52] support iOS<14 by transpiling dependencies in prod In order to support iPhone 6, we must support iOS 12. Unfortunately Safari did not implement static properties in classes until version 14, and many of our dependencies use this modern syntax in the JS they expose. To remedy this, we must tell babel to perform extra transpilation for production builds. This will lengthen the build process by about 1 minute, but it should guarantee that the JS exposed by the bundle is safe. Awaiting testing --- webpack.prod.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/webpack.prod.js b/webpack.prod.js index 5b0ea8268..c08fc140c 100644 --- a/webpack.prod.js +++ b/webpack.prod.js @@ -24,6 +24,13 @@ module.exports = merge(common, { plugins: ["angularjs-annotate"], }, }, + { + test: /\.(js|jsx|ts|tsx)$/, + loader: 'babel-loader', + options: { + presets: ['@babel/preset-env', '@babel/preset-react', '@babel/preset-typescript'], + }, + }, ], }, }); From 24644f18c26bb7344cf78c11d1b34f5a91293304 Mon Sep 17 00:00:00 2001 From: Jack Greenlee Date: Wed, 19 Jul 2023 16:03:13 -0400 Subject: [PATCH 18/52] refactor DateSelect+FilterSelect into NavBarButton NavBarButton is new and will contain the styling + presentational components that are common to both DateSelect and FilterSelect. It could be used in other tabs too if we wish to have more buttons embedded in the top navbar. So then, DateSelect and FilterSelect will both be composed of a NavBarButton with different contents inside the button. FilterSelect has reworked to no longer use HTML. Instead of an HTML was not updating. --- www/js/components/NavBarButton.tsx | 56 ++++++++++++++++ www/js/diary/list/DateSelect.tsx | 36 +++++------ www/js/diary/list/FilterSelect.tsx | 100 +++++++++++++---------------- 3 files changed, 114 insertions(+), 78 deletions(-) create mode 100644 www/js/components/NavBarButton.tsx diff --git a/www/js/components/NavBarButton.tsx b/www/js/components/NavBarButton.tsx new file mode 100644 index 000000000..d58f70f93 --- /dev/null +++ b/www/js/components/NavBarButton.tsx @@ -0,0 +1,56 @@ +import React from "react"; +import { View, StyleSheet } from "react-native"; +import color from "color"; +import { Button, IconButton, useTheme } from "react-native-paper"; + +const NavBarButton = ({ children, icon, onPressAction }) => { + + const { colors } = useTheme(); + const buttonColor = color(colors.onBackground).alpha(.07).rgb().string(); + const outlineColor = color(colors.onBackground).alpha(.2).rgb().string(); + + return (<> + + ); +}; + +export const s = StyleSheet.create({ + btn: { + borderRadius: 10, + marginLeft: 5, + }, + label: { + fontSize: 12.5, + fontWeight: '400', + height: '100%', + marginHorizontal: 'auto', + marginVertical: 'auto', + display: 'flex', + }, + icon: { + margin: 'auto', + width: 'auto', + height: 'auto', + }, + textWrapper: { + lineHeight: '100%', + marginHorizontal: 5, + justifyContent: 'space-evenly', + alignItems: 'center', + }, +}); + +export default NavBarButton; diff --git a/www/js/diary/list/DateSelect.tsx b/www/js/diary/list/DateSelect.tsx index 93cb0d860..dad48f029 100644 --- a/www/js/diary/list/DateSelect.tsx +++ b/www/js/diary/list/DateSelect.tsx @@ -5,14 +5,14 @@ */ import React, { useEffect, useState, useMemo, useContext } from "react"; -import { View, Text } from "react-native"; +import { Text, StyleSheet } from "react-native"; import moment from "moment"; -import color from "color"; import { LabelTabContext } from "../LabelTab"; import { DatePickerModal } from "react-native-paper-dates"; -import { Divider, Button, IconButton, useTheme } from "react-native-paper"; +import { Divider, useTheme } from "react-native-paper"; import i18next from "i18next"; import { useTranslation } from "react-i18next"; +import NavBarButton from "../../components/NavBarButton"; const DateSelect = ({ tsRange, loadSpecificWeekFn }) => { @@ -56,25 +56,12 @@ const DateSelect = ({ tsRange, loadSpecificWeekFn }) => { }, [setOpen, loadSpecificWeekFn] ); - - const buttonColor = color(colors.onBackground).alpha(.07).rgb().string(); - const outlineColor = color(colors.onBackground).alpha(.2).rgb().string(); - return (<> - + setOpen(true)}> + {dateRange[0]} + + {dateRange[1]} + { ); }; +export const s = StyleSheet.create({ + divider: { + width: '3ch', + marginHorizontal: 'auto', + } +}); + export default DateSelect; diff --git a/www/js/diary/list/FilterSelect.tsx b/www/js/diary/list/FilterSelect.tsx index 5d962a803..2aa2f1154 100644 --- a/www/js/diary/list/FilterSelect.tsx +++ b/www/js/diary/list/FilterSelect.tsx @@ -4,79 +4,65 @@ when we have fully migrated to React Native. */ -import React, { useEffect, useState } from "react"; -import { angularize } from "../../angular-react-helper"; +import React, { useState, useMemo } from "react"; +import { Modal } from "react-native"; import { useTranslation } from "react-i18next"; -import { array, number } from "prop-types"; +import NavBarButton from "../../components/NavBarButton"; +import { RadioButton, Text, Dialog } from "react-native-paper"; const FilterSelect = ({ filters, setFilters, numListDisplayed, numListTotal }) => { const { t } = useTranslation(); - const [selectedFilter, setSelectedFilter] = useState(); + const [modalVisible, setModalVisible] = useState(false); + const selectedFilter = useMemo(() => filters?.find(f => f.state)?.key || 'show-all', [filters]); + const labelDisplayText = useMemo(() => { + if (!filters) + return '...'; + const selectedFilterObj = filters?.find(f => f.state); + if (!selectedFilterObj) return t('diary.show-all') + ` (${numListTotal||0})`; + return selectedFilterObj.text + ` (${numListDisplayed||0}/${numListTotal||0})`; + }, [filters, numListDisplayed, numListTotal]); - useEffect(() => { - if (!selectedFilter) { - setSelectedFilter(filters?.find(f => f.state)?.key); - } - }, [filters]); - - useEffect(() => { - if (!selectedFilter) return; - if (selectedFilter == 'show-all') { + function chooseFilter(filterKey) { + if (filterKey == 'show-all') { setFilters(filters.map(f => ({ ...f, state: false }))); } else { setFilters(filters.map(f => { - if (f.key === selectedFilter) { + if (f.key === filterKey) { return { ...f, state: true }; } else { return { ...f, state: false }; } })); } - }, [selectedFilter]); - - const selectStyle = s.select; - if (!filters) - selectStyle.pointerEvents = 'none'; - - return ( - - ); -}; - -/* using 'any' here because React Native doesn't have types - for some of these CSS props */ -const s: any = { - select: { - minHeight: 34, - appearance: 'button', - WebkitAppearance: 'button', - borderRadius: 10, - margin: 8, - fontSize: 13, - flexGrow: 0, - color: '#222', - border: '1px solid rgb(20 20 20 / .2)', + /* We must wait to close the modal until this function is done running, + else the click event might leak to the content behind the modal */ + setTimeout(() => setModalVisible(false)); /* setTimeout with no delay defers the call until + the next event loop cycle */ } -} -FilterSelect.propTypes = { - filters: array, - numListDisplayed: number, - numListTotal: number, -} + return (<> + setModalVisible(true)}> + + {labelDisplayText} + + + setModalVisible(false)}> + setModalVisible(false)}> + {/* TODO - add title */} + {/* {t('diary.filter-travel')} */} + + chooseFilter(k)} value={selectedFilter}> + {filters.map(f => ( + + ))} + + + + + + ); +}; export default FilterSelect; From 2f00d33e2cedba651538b40aaba57cedb5838b64 Mon Sep 17 00:00:00 2001 From: Jack Greenlee Date: Wed, 19 Jul 2023 16:03:33 -0400 Subject: [PATCH 19/52] implement new colors for MotionTypes --- www/js/diary/diaryHelper.ts | 41 +++++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/www/js/diary/diaryHelper.ts b/www/js/diary/diaryHelper.ts index 0feeb3cb6..8b6f0a8e7 100644 --- a/www/js/diary/diaryHelper.ts +++ b/www/js/diary/diaryHelper.ts @@ -5,28 +5,39 @@ import i18next from "i18next"; import moment from "moment"; import { DateTime } from "luxon"; +const modeColors = { + red: '#b9003d', // oklch(50% 0.37 15) // car + orange: '#b25200', // oklch(55% 0.37 50) // air, hsr + green: '#007e46', // oklch(52% 0.37 155) // bike + blue: '#0068a5', // oklch(50% 0.37 245) // walk + periwinkle: '#5e45cd', // oklch(50% 0.2 285) // light rail, train, tram, subway + magenta: '#8e35a1', // oklch(50% 0.18 320) // bus + grey: '#484848', // oklch(40% 0 0) // unprocessed / unknown + taupe: '#7d5857', // oklch(50% 0.05 15) // ferry, trolleybus, nonstandard modes +} + type MotionType = { name: string, icon: string, color: string } const MotionTypes: {[k: string]: MotionType} = { - IN_VEHICLE: { name: "IN_VEHICLE", icon: "speedometer", color: "purple" }, - ON_FOOT: { name: "ON_FOOT", icon: "walk", color: "brown" }, - BICYCLING: { name: "BICYCLING", icon: "bike", color: "green" }, - UNKNOWN: { name: "UNKNOWN", icon: "help", color: "orange" }, - WALKING: { name: "WALKING", icon: "walk", color: "brown" }, - CAR: { name: "CAR", icon: "car", color: "red" }, - AIR_OR_HSR: { name: "AIR_OR_HSR", icon: "airplane", color: "red" }, + IN_VEHICLE: { name: "IN_VEHICLE", icon: "speedometer", color: modeColors.red }, + ON_FOOT: { name: "ON_FOOT", icon: "walk", color: modeColors.blue }, + BICYCLING: { name: "BICYCLING", icon: "bike", color: modeColors.green }, + UNKNOWN: { name: "UNKNOWN", icon: "help", color: modeColors.grey }, + WALKING: { name: "WALKING", icon: "walk", color: modeColors.blue }, + CAR: { name: "CAR", icon: "car", color: modeColors.red }, + AIR_OR_HSR: { name: "AIR_OR_HSR", icon: "airplane", color: modeColors.orange }, // based on OSM routes/tags: - BUS: { name: "BUS", icon: "bus-side", color: "red" }, - LIGHT_RAIL: { name: "LIGHT_RAIL", icon: "train-car-passenger", color: "red" }, - TRAIN: { name: "TRAIN", icon: "train-car-passenger", color: "red" }, - TRAM: { name: "TRAM", icon: "fas fa-tram", color: "red" }, - SUBWAY: { name: "SUBWAY", icon: "subway-variant", color: "red" }, - FERRY: { name: "FERRY", icon: "ferry", color: "red" }, - TROLLEYBUS: { name: "TROLLEYBUS", icon: "bus-side", color: "red" }, - UNPROCESSED: { name: "UNPROCESSED", icon: "help", color: "orange" } + BUS: { name: "BUS", icon: "bus-side", color: modeColors.magenta }, + LIGHT_RAIL: { name: "LIGHT_RAIL", icon: "train-car-passenger", color: modeColors.periwinkle }, + TRAIN: { name: "TRAIN", icon: "train-car-passenger", color: modeColors.periwinkle }, + TRAM: { name: "TRAM", icon: "fas fa-tram", color: modeColors.periwinkle }, + SUBWAY: { name: "SUBWAY", icon: "subway-variant", color: modeColors.periwinkle }, + FERRY: { name: "FERRY", icon: "ferry", color: modeColors.taupe }, + TROLLEYBUS: { name: "TROLLEYBUS", icon: "bus-side", color: modeColors.taupe }, + UNPROCESSED: { name: "UNPROCESSED", icon: "help", color: modeColors.grey } } type MotionTypeKey = keyof typeof MotionTypes; From c0632f7c90fd22aa0792003d7c64cbfcc821a38a Mon Sep 17 00:00:00 2001 From: Jack Greenlee Date: Wed, 19 Jul 2023 16:10:25 -0400 Subject: [PATCH 20/52] update descriptions, DateSelect + FilterSelect --- www/js/diary/list/DateSelect.tsx | 10 ++++++---- www/js/diary/list/FilterSelect.tsx | 11 +++++++---- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/www/js/diary/list/DateSelect.tsx b/www/js/diary/list/DateSelect.tsx index dad48f029..b41a900a7 100644 --- a/www/js/diary/list/DateSelect.tsx +++ b/www/js/diary/list/DateSelect.tsx @@ -1,7 +1,9 @@ -/* A component wrapped around an element that allows the user to pick a date, - used in the Label screen. - This is a temporary solution; this component includes HTML and we will need to be rewritten - when we have fully migrated to React Native. +/* This button launches a modal to select a date, which determines which week of + travel should be displayed in the Label screen. + The button itself is a NavBarButton, which shows the currently selected date range, + a calendar icon, and launches the modal when clicked. + The modal is a DatePickerModal from react-native-paper-dates, which shows a calendar + and allows the user to select a date. */ import React, { useEffect, useState, useMemo, useContext } from "react"; diff --git a/www/js/diary/list/FilterSelect.tsx b/www/js/diary/list/FilterSelect.tsx index 2aa2f1154..d1906f462 100644 --- a/www/js/diary/list/FilterSelect.tsx +++ b/www/js/diary/list/FilterSelect.tsx @@ -1,7 +1,10 @@ -/* A component wrapped around a