Release 2024-45

.github/workflows/ci.yaml

@@ -116,6 +116,9 @@ jobs:
       # viz-test-tool
       SLACK_BOT_OAUTH_TOKEN: cat1234
 
+      # Disable zendesk notifications in tests
+      ZENDESK_NOTIFICATIONS_DISABLED: 'true'
+
     strategy:
       fail-fast: false
       matrix:

packages/admin-panel/src/api/mutations/useLogin.js

@@ -2,7 +2,7 @@
  * Tupaia
  *  Copyright (c) 2017 - 2024 Beyond Essential Systems Pty Ltd
  */
-
+import { getBrowserTimeZone } from '@tupaia/utils';
 import { useMutation, useQueryClient } from '@tanstack/react-query';
 import { useLocation, useNavigate } from 'react-router-dom';
 import { post } from '../../VizBuilderApp/api';
@@ -19,6 +19,7 @@ export const useLogin = homeLink => {
         data: {
           emailAddress: email,
           password,
+          timezone: getBrowserTimeZone(),
         },
       });
     },

packages/admin-panel/src/routes/surveys/optionSets.js

@@ -22,7 +22,7 @@ const OPTION_SET_COLUMNS = [
     type: 'export',
     actionConfig: {
       exportEndpoint: 'optionSets',
-      fileName: '{name}',
+      fileName: '{name}.xlsx',
     },
   },
   {

packages/admin-panel/src/routes/surveys/surveys.js

@@ -131,7 +131,7 @@ const SURVEY_FIELDS = {
       options: SERVICE_TYPES,
       setFieldsOnChange: (newValue, currentRecord = null) => {
         const { dhisInstanceCode = 'regional' } = currentRecord
-          ? currentRecord['data_group.config'] ?? {}
+          ? (currentRecord['data_group.config'] ?? {})
           : {};
         const config = newValue === 'dhis' ? { dhisInstanceCode } : {};
         return { 'data_group.config': config };
@@ -217,7 +217,7 @@ const SURVEY_COLUMNS = [
     type: 'export',
     actionConfig: {
       exportEndpoint: 'surveys',
-      fileName: '{name}',
+      fileName: '{name}.xlsx',
     },
   },
   {

packages/admin-panel/src/routes/visualisations/dashboardItems.jsx

@@ -66,7 +66,7 @@ const COLUMNS = [
     type: 'export',
     actionConfig: {
       exportEndpoint: 'dashboardVisualisation',
-      fileName: '{code}',
+      fileName: '{code}.json',
     },
   },
   {

packages/admin-panel/src/routes/visualisations/dataTables.js

@@ -67,7 +67,7 @@ const COLUMNS = [
     type: 'export',
     actionConfig: {
       exportEndpoint: 'dataTable',
-      fileName: '{code}',
+      fileName: '{code}.json',
     },
   },
   {

packages/admin-panel/src/routes/visualisations/mapOverlays.jsx

@@ -142,7 +142,7 @@ const COLUMNS = [
     type: 'export',
     actionConfig: {
       exportEndpoint: 'mapOverlayVisualisation',
-      fileName: '{code}',
+      fileName: '{code}.json',
     },
   },
   {

packages/central-server/.env.example

@@ -13,3 +13,9 @@ DHIS_SYNC_DISABLE=
 KOBO_SYNC_DISABLE=
 FEED_SCRAPER_DISABLE=
 MS1_SYNC_DISABLE=
+
+
+ZENDESK_API_TOKEN=
+ZENDESK_SUBDOMAIN=
+ZENDESK_EMAIL=
+ZENDESK_NOTIFICATIONS_DISABLE=

packages/central-server/package.json

@@ -47,6 +47,7 @@
     "case": "^1.6.1",
     "compare-versions": "^6.1.0",
     "cors": "^2.8.5",
+    "countries-and-timezones": "^3.6.0",
     "countrynames": "^0.1.1",
     "date-fns": "^2.29.2",
     "del": "^2.2.2",
@@ -84,6 +85,7 @@
     "mocha": "^10.2.0",
     "npm-run-all": "^4.1.5",
     "nyc": "^15.1.0",
+    "proxyquire": "^2.1.3",
     "sinon": "^9.0.2",
     "sinon-chai": "^3.3.0",
     "sinon-test": "^3.0.0",

packages/central-server/src/apiV2/authenticate.js

@@ -4,10 +4,11 @@
  */
 
 import winston from 'winston';
-
+import { getCountryForTimezone } from 'countries-and-timezones';
 import { getAuthorizationObject, getUserAndPassFromBasicAuth } from '@tupaia/auth';
 import { respond, reduceToDictionary } from '@tupaia/utils';
 import { allowNoPermissions } from '../permissions';
+import { createSupportTicket } from '../utilities';
 
 const GRANT_TYPES = {
   PASSWORD: 'password',
@@ -98,6 +99,46 @@ const checkApiClientAuthentication = async req => {
   }
 };
 
+const checkUserLocationAccess = async (req, user) => {
+  if (!user) return;
+  const { body, models } = req;
+  const { timezone } = body;
+
+  // The easiest way to get the country code is to use the timezone and get the most likely country using this timezone. This doesn't infringe on the user's privacy as the timezone is a very broad location. It also doesn't require the user to provide their location, which is a barrier to entry for some users.
+  const country = getCountryForTimezone(timezone);
+  if (!country) return;
+  // the ID is the ISO country code.
+  const { id, name } = country;
+
+  const existingEntry = await models.userCountryAccessAttempt.findOne({
+    user_id: user.id,
+    country_code: id,
+  });
+
+  // If there is already an entry for this user and country, return
+  if (existingEntry) return;
+
+  const userEntryCount = await models.userCountryAccessAttempt.count({
+    user_id: user.id,
+  });
+
+  const hasAnyEntries = userEntryCount > 0;
+
+  await models.userCountryAccessAttempt.create({
+    user_id: user.id,
+    country_code: id,
+  });
+
+  // Don't send an email if this is the first time the user has attempted to login
+  if (!hasAnyEntries) return;
+
+  // create a support ticket if the user has attempted to login from a new country
+  await createSupportTicket(
+    'User attempted to login from a new country',
+    `User ${user.first_name} ${user.last_name} (${user.id} - ${user.email}) attempted to access Tupaia from a new country: ${name}`,
+  );
+};
+
 /**
  * Handler for a POST to the /auth endpoint
  * By default, or if URL parameters include grantType=password, will check the email address and
@@ -126,5 +167,7 @@ export async function authenticate(req, res) {
     permissionGroups: permissionGroupsByCountryId,
   });
 
+  await checkUserLocationAccess(req, user);
+
   respond(res, authorizationObject);
 }

packages/central-server/src/database/meditrakSyncQueue/MeditrakSyncQueue.js

@@ -51,16 +51,9 @@ export class MeditrakSyncQueue extends ChangeHandler {
    * @private
    */
   async refreshPermissionsBasedView(database) {
-    try {
-      const start = Date.now();
-      await database.executeSql(
-        `REFRESH MATERIALIZED VIEW CONCURRENTLY permissions_based_meditrak_sync_queue;`,
-      );
-      const end = Date.now();
-      winston.info(`permissions_based_meditrak_sync_queue refresh took: ${end - start}ms`);
-    } catch (error) {
-      winston.error(`permissions_based_meditrak_sync_queue refresh failed: ${error.message}`);
-    }
+    await database.executeSql(
+      `REFRESH MATERIALIZED VIEW CONCURRENTLY permissions_based_meditrak_sync_queue;`,
+    );
   }
 
   /**

packages/central-server/src/tests/apiV2/authenticate/authenticate.test.js

@@ -103,4 +103,46 @@ describe('Authenticate', function () {
     expect(userId).to.equal(userAccount.id);
     expect(apiClientUserId).to.equal(apiClientUserAccount.id);
   });
+
+  it('should add a new entry to the user_country_access_attempts table if one does not already exist', async () => {
+    await app.post('auth?grantType=password', {
+      headers: {
+        authorization: createBasicHeader(apiClientUserAccount.email, apiClientSecret),
+      },
+      body: {
+        emailAddress: userAccount.email,
+        password: userAccountPassword,
+        deviceName: 'test_device',
+        timezone: 'Pacific/Auckland',
+      },
+    });
+    const entries = await models.userCountryAccessAttempt.find({
+      user_id: userAccount.id,
+      country_code: 'NZ',
+    });
+    expect(entries).to.have.length(1);
+  });
+
+  it('should not add a new entry to the user_country_access_attempts table if one does already exist', async () => {
+    await models.userCountryAccessAttempt.create({
+      user_id: userAccount.id,
+      country_code: 'WS',
+    });
+    await app.post('auth?grantType=password', {
+      headers: {
+        authorization: createBasicHeader(apiClientUserAccount.email, apiClientSecret),
+      },
+      body: {
+        emailAddress: userAccount.email,
+        password: userAccountPassword,
+        deviceName: 'test_device',
+        timezone: 'Pacific/Apia',
+      },
+    });
+    const entries = await models.userCountryAccessAttempt.find({
+      user_id: userAccount.id,
+      country_code: 'WS',
+    });
+    expect(entries).to.have.length(1);
+  });
 });

packages/central-server/src/tests/utilities/createSupportTicket.test.js

@@ -0,0 +1,49 @@
+/**
+ * Tupaia
+ * Copyright (c) 2017 - 2024 Beyond Essential Systems Pty Ltd
+ */
+
+import { expect } from 'chai';
+import sinon from 'sinon';
+import proxyquire from 'proxyquire';
+
+const fetchWithTimeoutStub = sinon.stub().resolves();
+const requireEnvStub = sinon.stub().returns('test_value');
+
+// Use proxyquire to replace 'fetchWithTimeout' with the stub - See [here](https://stackoverflow.com/a/52591287) for an explanation about why destructured imports can't be stubbed
+const { createSupportTicket } = proxyquire('../../utilities/createSupportTicket', {
+  '@tupaia/utils': {
+    fetchWithTimeout: fetchWithTimeoutStub,
+    requireEnv: requireEnvStub,
+    getIsProductionEnvironment: () => true,
+  },
+});
+
+describe('Create support ticket', () => {
+  after(() => {
+    // Reset the stub after each test
+    fetchWithTimeoutStub.reset();
+    requireEnvStub.reset();
+  });
+  it("should not create a support ticket if ZENDESK_NOTIFICATIONS_DISABLE is set to 'true'", async () => {
+    process.env.ZENDESK_NOTIFICATIONS_DISABLE = 'true';
+    await createSupportTicket('test_subject', 'test_message');
+    sinon.assert.notCalled(fetchWithTimeoutStub);
+  });
+
+  it('should create a support ticket if ZENDESK_NOTIFICATIONS_DISABLE is not set to true', async () => {
+    process.env.ZENDESK_NOTIFICATIONS_DISABLE = 'false';
+    await createSupportTicket('test_subject', 'test_message');
+    expect(fetchWithTimeoutStub).to.have.been.calledOnce;
+    expect(fetchWithTimeoutStub).to.have.been.calledWith('test_value/tickets', {
+      method: 'POST',
+      headers: {
+        Authorization: `Basic ${Buffer.from('test_value/token:test_value').toString('base64')}`,
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify({
+        ticket: { subject: 'test_subject', comment: { body: 'test_message' } },
+      }),
+    });
+  });
+});

packages/central-server/src/utilities/createSupportTicket.js

@@ -0,0 +1,58 @@
+/**
+ * Tupaia
+ * Copyright (c) 2017 - 2024 Beyond Essential Systems Pty Ltd
+ */
+import { fetchWithTimeout, getIsProductionEnvironment, requireEnv } from '@tupaia/utils';
+import { sendEmail } from '@tupaia/server-utils';
+
+const emailInternally = async (subject, message) => {
+  const sendTo = requireEnv('DEV_EMAIL_ADDRESS');
+  return sendEmail(sendTo, {
+    subject,
+    templateName: 'generic',
+    templateContext: {
+      userName: 'Tupaia Admin',
+      message,
+    },
+  });
+};
+
+export const createSupportTicket = async (subject, message) => {
+  // If ZENDESK_NOTIFICATIONS_DISABLE is set to true, do not create a support ticket
+  if (process.env.ZENDESK_NOTIFICATIONS_DISABLE === 'true') return;
+
+  // If we are not in a production environment, send an email to the dev team instead of creating a support ticket
+  if (!getIsProductionEnvironment()) {
+    return emailInternally(subject, message);
+  }
+
+  try {
+    const zendeskApi = requireEnv('ZENDESK_API_URL');
+    const apiToken = requireEnv('ZENDESK_API_TOKEN');
+    const email = requireEnv('ZENDESK_EMAIL');
+
+    const url = `${zendeskApi}/tickets`;
+
+    const ticketData = {
+      subject,
+      comment: {
+        body: message,
+      },
+    };
+
+    const base64Credentials = Buffer.from(`${email}/token:${apiToken}`).toString('base64');
+
+    const requestConfig = {
+      method: 'POST',
+      headers: {
+        Authorization: `Basic ${base64Credentials}`,
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify({ ticket: ticketData }),
+    };
+
+    await fetchWithTimeout(url, requestConfig);
+  } catch (error) {
+    console.error('Error creating support ticket:', error);
+  }
+};

packages/central-server/src/utilities/index.js

@@ -7,3 +7,4 @@ export { getApiUrl } from './getApiUrl';
 export { getTempDirectory } from './getTempDirectory';
 export { resourceToRecordType } from './resourceToRecordType';
 export { getStandardisedImageName } from './getStandardisedImageName';
+export { createSupportTicket } from './createSupportTicket';

packages/database/src/ModelRegistry.js

@@ -20,6 +20,7 @@ export class ModelRegistry {
       ...baseModelClasses,
       ...extraModelClasses,
     };
+
     this.generateModels(schemata);
     if (useNotifiers) {
       this.initialiseNotifiers();

packages/database/src/changeHandlers/AnalyticsRefresher.js

@@ -23,14 +23,7 @@ export class AnalyticsRefresher extends ChangeHandler {
   }
 
   static refreshAnalytics = async database => {
-    try {
-      const start = Date.now();
-      await database.executeSql(`SELECT mv$refreshMaterializedView('analytics', 'public', true);`);
-      const end = Date.now();
-      winston.info(`Analytics table refresh took: ${end - start}ms`);
-    } catch (error) {
-      winston.error(`Analytics table refresh failed: ${error.message}`);
-    }
+    await database.executeSql(`SELECT mv$refreshMaterializedView('analytics', 'public', true);`);
   };
 
   handleChanges(transactingModels) {