Merge branch 'main' into kalvis/organizations-settings

jsapp/js/components/common/inlineMessage.scss

@@ -50,6 +50,11 @@
   .k-icon {color: colors.$kobo-amber;}
 }
 
+.k-inline-message--type-info {
+  background-color: colors.$kobo-bg-blue;
+  .k-icon {color: colors.$kobo-blue;}
+}
+
 // We need a bit stronger specificity here
 .k-inline-message p.k-inline-message__message {
   margin: 0;

jsapp/js/components/common/inlineMessage.tsx

@@ -5,7 +5,7 @@ import Icon from 'js/components/common/icon';
 import './inlineMessage.scss';
 
 /** Influences the background color and the icon color */
-export type InlineMessageType = 'default' | 'error' | 'success' | 'warning';
+export type InlineMessageType = 'default' | 'error' | 'success' | 'warning' | 'info';
 
 interface InlineMessageProps {
   type: InlineMessageType;

jsapp/js/stores/useSession.tsx

@@ -0,0 +1,49 @@
+import sessionStore from './session';
+import {useEffect, useState} from 'react';
+import {reaction} from 'mobx';
+import type {AccountResponse} from '../dataInterface';
+
+/**
+ * Hook to use the session store in functional components.
+ * This hook provides a way to access teh current logged account, information
+ * regarding the anonymous state of the login and session methods.
+ *
+ * This hook uses MobX reactions to track the current account and update the
+ * state accordingly.
+ * In the future we should update this hook to use react-query and drop the usage of mob-x
+ */
+export const useSession = () => {
+  const [currentLoggedAccount, setCurrentLoggedAccount] =
+    useState<AccountResponse>();
+  const [isAnonymous, setIsAnonymous] = useState<boolean>(true);
+  const [isPending, setIsPending] = useState<boolean>(false);
+
+  useEffect(() => {
+    // We need to setup a reaction for every observable we want to track
+    // Generic reaction to sessionStore won't fire the re-rendering of the hook
+    const currentAccountReactionDisposer = reaction(
+      () => sessionStore.currentAccount,
+      (currentAccount) => {
+        if (sessionStore.isLoggedIn) {
+          setCurrentLoggedAccount(currentAccount as AccountResponse);
+          setIsAnonymous(false);
+          setIsPending(sessionStore.isPending);
+        }
+      },
+      {fireImmediately: true}
+    );
+
+    return () => {
+      currentAccountReactionDisposer();
+    };
+  }, []);
+
+  return {
+    currentLoggedAccount,
+    isAnonymous,
+    isPending,
+    logOut: sessionStore.logOut.bind(sessionStore),
+    logOutAll: sessionStore.logOutAll.bind(sessionStore),
+    refreshAccount: sessionStore.refreshAccount.bind(sessionStore),
+  };
+};

kobo/apps/openrosa/apps/logger/constants.py

@@ -0,0 +1,2 @@
+
+SUBMISSIONS_SUSPENDED_HEARTBEAT_KEY = 'kobo:update_attachment_storage_bytes:heartbeat'

kobo/apps/openrosa/apps/logger/management/commands/update_attachment_storage_bytes.py

@@ -1,17 +1,24 @@
 from __future__ import annotations
 
+import time
+
 from django.conf import settings
 from django.contrib.auth import get_user_model
 from django.core.management.base import BaseCommand
 from django.db.models import OuterRef, Subquery, Sum
+from django_redis import get_redis_connection
 
+from kobo.apps.openrosa.apps.logger.constants import (
+    SUBMISSIONS_SUSPENDED_HEARTBEAT_KEY
+)
 from kobo.apps.openrosa.apps.logger.models.attachment import Attachment
 from kobo.apps.openrosa.apps.logger.models.xform import XForm
 from kobo.apps.openrosa.apps.main.models.user_profile import UserProfile
 from kobo.apps.openrosa.libs.utils.jsonbfield_helper import ReplaceValues
 
 
 class Command(BaseCommand):
+
     help = (
         'Retroactively calculate the total attachment file storage '
         'per xform and user profile'
@@ -22,6 +29,7 @@ def __init__(self, *args, **kwargs):
         self._verbosity = 0
         self._force = False
         self._sync = False
+        self._redis_client = get_redis_connection()
 
     def add_arguments(self, parser):
         parser.add_argument(
@@ -52,10 +60,14 @@ def add_arguments(self, parser):
         )
 
         parser.add_argument(
-            '-l', '--skip-lock-release',
+            '-nl', '--no-lock',
             action='store_true',
             default=False,
-            help='Do not attempts to remove submission lock on user profiles. Default is False',
+            help=(
+                'Do not lock accounts from receiving submissions while updating '
+                'storage counters.\n'
+                'WARNING: This may result in discrepancies. The default value is False.'
+            )
         )
 
     def handle(self, *args, **kwargs):
@@ -65,7 +77,7 @@ def handle(self, *args, **kwargs):
         self._sync = kwargs['sync']
         chunks = kwargs['chunks']
         username = kwargs['username']
-        skip_lock_release = kwargs['skip_lock_release']
+        no_lock = kwargs['no_lock']
 
         if self._force and self._sync:
             self.stderr.write(
@@ -89,7 +101,7 @@ def handle(self, *args, **kwargs):
                 '`force` option has been enabled'
             )
 
-        if not skip_lock_release:
+        if not no_lock:
             self._release_locks()
 
         profile_queryset = self._reset_user_profile_counters()
@@ -112,57 +124,62 @@ def handle(self, *args, **kwargs):
                     )
                 continue
 
-            self._lock_user_profile(user)
+            if not no_lock:
+                self._lock_user_profile(user)
 
-            for xform in user_xforms.iterator(chunk_size=chunks):
+            try:
+                for xform in user_xforms.iterator(chunk_size=chunks):
 
-                # write out xform progress
-                if self._verbosity > 1:
-                    self.stdout.write(
-                        f"Calculating attachments for xform_id #{xform['pk']}"
-                        f" (user {user.username})"
-                    )
-                # aggregate total media file size for all media per xform
-                form_attachments = Attachment.objects.filter(
-                    instance__xform_id=xform['pk'],
-                ).aggregate(total=Sum('media_file_size'))
-
-                if form_attachments['total']:
-                    if (
-                        xform['attachment_storage_bytes']
-                        == form_attachments['total']
-                    ):
-                        if self._verbosity > 2:
-                            self.stdout.write(
-                                '\tSkipping xform update! '
-                                'Attachment storage is already accurate'
+                    self._heartbeat(user)
+
+                    # write out xform progress
+                    if self._verbosity > 1:
+                        self.stdout.write(
+                            f"Calculating attachments for xform_id #{xform['pk']}"
+                            f" (user {user.username})"
+                        )
+                    # aggregate total media file size for all media per xform
+                    form_attachments = Attachment.objects.filter(
+                        instance__xform_id=xform['pk'],
+                    ).aggregate(total=Sum('media_file_size'))
+
+                    if form_attachments['total']:
+                        if (
+                            xform['attachment_storage_bytes']
+                            == form_attachments['total']
+                        ):
+                            if self._verbosity > 2:
+                                self.stdout.write(
+                                    '\tSkipping xform update! '
+                                    'Attachment storage is already accurate'
+                                )
+                        else:
+                            if self._verbosity > 2:
+                                self.stdout.write(
+                                    f'\tUpdating xform attachment storage to '
+                                    f"{form_attachments['total']} bytes"
+                                )
+
+                            XForm.all_objects.filter(
+                                pk=xform['pk']
+                            ).update(
+                                attachment_storage_bytes=form_attachments['total']
                             )
+
                     else:
                         if self._verbosity > 2:
-                            self.stdout.write(
-                                f'\tUpdating xform attachment storage to '
-                                f"{form_attachments['total']} bytes"
+                            self.stdout.write('\tNo attachments found')
+                        if not xform['attachment_storage_bytes'] == 0:
+                            XForm.all_objects.filter(
+                                pk=xform['pk']
+                            ).update(
+                                attachment_storage_bytes=0
                             )
 
-                        XForm.all_objects.filter(
-                            pk=xform['pk']
-                        ).update(
-                            attachment_storage_bytes=form_attachments['total']
-                        )
-
-                else:
-                    if self._verbosity > 2:
-                        self.stdout.write('\tNo attachments found')
-                    if not xform['attachment_storage_bytes'] == 0:
-                        XForm.all_objects.filter(
-                            pk=xform['pk']
-                        ).update(
-                            attachment_storage_bytes=0
-                        )
-
-            # need to call `update_user_profile()` one more time outside the loop
-            # because the last user profile will not be up-to-date otherwise
-            self._update_user_profile(user)
+                self._update_user_profile(user)
+            finally:
+                if not no_lock:
+                    self._release_lock(user)
 
         if self._verbosity >= 1:
             self.stdout.write('Done!')
@@ -187,6 +204,13 @@ def _get_queryset(self, profile_queryset, username):
 
         return users.order_by('pk')
 
+    def _heartbeat(self, user: settings.AUTH_USER_MODEL):
+        self._redis_client.hset(
+            SUBMISSIONS_SUSPENDED_HEARTBEAT_KEY, mapping={
+                user.username: int(time.time())
+            }
+        )
+
     def _lock_user_profile(self, user: settings.AUTH_USER_MODEL):
         # Retrieve or create user's profile.
         (
@@ -204,7 +228,18 @@ def _lock_user_profile(self, user: settings.AUTH_USER_MODEL):
             # new submissions from coming in while the
             # `attachment_storage_bytes` is being calculated.
             user_profile.submissions_suspended = True
-            user_profile.save(update_fields=['submissions_suspended'])
+            user_profile.metadata['attachments_counting_status'] = 'not-completed'
+            user_profile.save(update_fields=['metadata', 'submissions_suspended'])
+
+        self._heartbeat(user)
+
+    def _release_lock(self, user: settings.AUTH_USER_MODEL):
+        # Release any locks on the users' profile from getting submissions
+        if self._verbosity > 1:
+            self.stdout.write(f'Releasing submission lock for {user.username}…')
+
+        UserProfile.objects.filter(user_id=user.pk).update(submissions_suspended=False)
+        self._redis_client.hdel(SUBMISSIONS_SUSPENDED_HEARTBEAT_KEY, user.username)
 
     def _release_locks(self):
         # Release any locks on the users' profile from getting submissions
@@ -244,7 +279,7 @@ def _update_user_profile(self, user: settings.AUTH_USER_MODEL):
                 f'{user.username}’s profile'
             )
 
-        # Update user's profile (and lock the related row)
+        # Update user's profile
         updates = {
             'attachments_counting_status': 'complete',
         }
@@ -265,5 +300,4 @@ def _update_user_profile(self, user: settings.AUTH_USER_MODEL):
                 'metadata',
                 updates=updates,
             ),
-            submissions_suspended=False,
         )

kobo/apps/openrosa/apps/logger/signals.py

@@ -47,10 +47,8 @@ def pre_delete_attachment(instance, **kwargs):
 
     if file_size and attachment.deleted_at is None:
         with transaction.atomic():
-            """
-            Update both counters at the same time (in a transaction) to avoid
-            desynchronization as much as possible
-            """
+            # Update both counters simultaneously within a transaction to minimize
+            # the risk of desynchronization.
             UserProfile.objects.filter(
                 user_id=xform.user_id
             ).update(