Batch migrate olm and megolm sessions

MatrixSDK.xcodeproj/project.pbxproj

@@ -1858,6 +1858,8 @@
 		ED4114EC292E498100728459 /* MXBackgroundCryptoV2.swift in Sources */ = {isa = PBXBuildFile; fileRef = ED4114EA292E498100728459 /* MXBackgroundCryptoV2.swift */; };
 		ED4114EE292E49C000728459 /* MXLegacyBackgroundCrypto.swift in Sources */ = {isa = PBXBuildFile; fileRef = ED4114ED292E49C000728459 /* MXLegacyBackgroundCrypto.swift */; };
 		ED4114EF292E49C000728459 /* MXLegacyBackgroundCrypto.swift in Sources */ = {isa = PBXBuildFile; fileRef = ED4114ED292E49C000728459 /* MXLegacyBackgroundCrypto.swift */; };
+		ED4368B129784CCE002B6272 /* MXRealmCryptoStoreTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = ED4368B029784CCE002B6272 /* MXRealmCryptoStoreTests.swift */; };
+		ED4368B229784CCE002B6272 /* MXRealmCryptoStoreTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = ED4368B029784CCE002B6272 /* MXRealmCryptoStoreTests.swift */; };
 		ED44F01128180BCC00452A5D /* MXSharedHistoryKeyRequest.swift in Sources */ = {isa = PBXBuildFile; fileRef = ED44F01028180BCC00452A5D /* MXSharedHistoryKeyRequest.swift */; };
 		ED44F01228180BCC00452A5D /* MXSharedHistoryKeyRequest.swift in Sources */ = {isa = PBXBuildFile; fileRef = ED44F01028180BCC00452A5D /* MXSharedHistoryKeyRequest.swift */; };
 		ED44F01428180EAB00452A5D /* MXSharedHistoryKeyManager.swift in Sources */ = {isa = PBXBuildFile; fileRef = ED44F01328180EAB00452A5D /* MXSharedHistoryKeyManager.swift */; };
@@ -3070,6 +3072,7 @@
 		ED4114E7292E496C00728459 /* MXBackgroundCrypto.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MXBackgroundCrypto.swift; sourceTree = "<group>"; };
 		ED4114EA292E498100728459 /* MXBackgroundCryptoV2.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MXBackgroundCryptoV2.swift; sourceTree = "<group>"; };
 		ED4114ED292E49C000728459 /* MXLegacyBackgroundCrypto.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MXLegacyBackgroundCrypto.swift; sourceTree = "<group>"; };
+		ED4368B029784CCE002B6272 /* MXRealmCryptoStoreTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MXRealmCryptoStoreTests.swift; sourceTree = "<group>"; };
 		ED44F01028180BCC00452A5D /* MXSharedHistoryKeyRequest.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MXSharedHistoryKeyRequest.swift; sourceTree = "<group>"; };
 		ED44F01328180EAB00452A5D /* MXSharedHistoryKeyManager.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MXSharedHistoryKeyManager.swift; sourceTree = "<group>"; };
 		ED44F01728180F1C00452A5D /* MXSharedHistoryKeyManagerUnitTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MXSharedHistoryKeyManagerUnitTests.swift; sourceTree = "<group>"; };
@@ -5467,6 +5470,14 @@
 			path = Crypto;
 			sourceTree = "<group>";
 		};
+		ED4368AF29784CA3002B6272 /* MXRealmCryptoStore */ = {
+			isa = PBXGroup;
+			children = (
+				ED4368B029784CCE002B6272 /* MXRealmCryptoStoreTests.swift */,
+			);
+			path = MXRealmCryptoStore;
+			sourceTree = "<group>";
+		};
 		ED44F01628180F1300452A5D /* KeySharing */ = {
 			isa = PBXGroup;
 			children = (
@@ -5571,6 +5582,7 @@
 		ED6DAC1328C78D3700ECDCB6 /* Store */ = {
 			isa = PBXGroup;
 			children = (
+				ED4368AF29784CA3002B6272 /* MXRealmCryptoStore */,
 				ED6DAC1428C78D4000ECDCB6 /* MXMemoryCryptoStore.swift */,
 			);
 			path = Store;
@@ -7348,6 +7360,7 @@
 				EDB4209527DF822B0036AF39 /* MXEventsByTypesEnumeratorOnArrayTests.swift in Sources */,
 				EC40385D28A16EDA0067D5B8 /* MXAes256KeyBackupTests.m in Sources */,
 				ED6DAC0728C77E1100ECDCB6 /* MXForwardedRoomKeyEventContentUnitTests.swift in Sources */,
+				ED4368B129784CCE002B6272 /* MXRealmCryptoStoreTests.swift in Sources */,
 				3A9E2B4328EB3960000DB2A7 /* MXMatrixVersionsUnitTests.swift in Sources */,
 				3265CB3B1A151C3800E24B2F /* MXRoomStateTests.m in Sources */,
 				ED8F1D302885AB0300F897E7 /* MXTrustLevelSourceUnitTests.swift in Sources */,
@@ -8006,6 +8019,7 @@
 				EDB4209627DF822B0036AF39 /* MXEventsByTypesEnumeratorOnArrayTests.swift in Sources */,
 				EC40385E28A16EDA0067D5B8 /* MXAes256KeyBackupTests.m in Sources */,
 				ED6DAC0828C77E1100ECDCB6 /* MXForwardedRoomKeyEventContentUnitTests.swift in Sources */,
+				ED4368B229784CCE002B6272 /* MXRealmCryptoStoreTests.swift in Sources */,
 				3A9E2B4428EB3960000DB2A7 /* MXMatrixVersionsUnitTests.swift in Sources */,
 				32B477AA2638186000EA5800 /* MXHTTPAdditionalHeadersUnitTests.m in Sources */,
 				B135066A27EA100100BD3276 /* MXBeaconInfoUnitTests.swift in Sources */,

MatrixSDK/Background/MXBackgroundCryptoStore.m

@@ -209,18 +209,6 @@ - (MXOlmSession*)sessionWithDevice:(NSString*)deviceKey andSessionId:(NSString*)
     return sessions;
 }
 
-- (NSArray<MXOlmSession *> *)sessions
-{
-    NSArray<MXOlmSession*> *bgSessions = [bgCryptoStore sessions] ?: @[];
-    NSArray<MXOlmSession*> *appSessions = [cryptoStore sessions] ?: @[];
-
-    NSMutableArray<MXOlmSession*> *sessions = [NSMutableArray array];
-    [sessions addObjectsFromArray:bgSessions];
-    [sessions addObjectsFromArray:appSessions];
-
-    return sessions;
-}
-
 - (void)storeSession:(MXOlmSession*)session
 {
     [bgCryptoStore storeSession:session];
@@ -356,6 +344,22 @@ - (void)storeDeviceSyncToken:(NSString*)deviceSyncToken
     NSAssert(NO, @"This method should be useless in the context of MXBackgroundCryptoStore");
 }
 
+- (void)enumerateSessionsBy:(NSInteger)batchSize block:(void (^)(NSArray<MXOlmSession *> *, double))block
+{
+    NSAssert(NO, @"This method should be useless in the context of MXBackgroundCryptoStore");
+}
+
+- (void)enumerateInboundGroupSessionsBy:(NSInteger)batchSize block:(void (^)(NSArray<MXOlmInboundGroupSession *> *, NSSet<NSString *> *, double))block
+{
+    NSAssert(NO, @"This method should be useless in the context of MXBackgroundCryptoStore");
+}
+
+- (NSUInteger)sessionsCount
+{
+    NSAssert(NO, @"This method should be useless in the context of MXBackgroundCryptoStore");
+    return 0;
+}
+
 - (NSArray<MXOlmInboundGroupSession*> *)inboundGroupSessions
 {
     NSAssert(NO, @"This method should be useless in the context of MXBackgroundCryptoStore");

MatrixSDK/Crypto/Data/Store/MXCryptoStore.h

@@ -296,11 +296,22 @@
 - (NSArray<MXOlmSession*>*)sessionsWithDevice:(NSString*)deviceKey;
 
 /**
- Retrieve all end-to-end sessions between this device and all other devices
+ Enumerate all end-to-end sessions in batches of `batchSize`
+
+ Each block is internally wrapped in `@autoreleasepool` so that memory footprint remains constant
+ regardless of the number of stored sessions.
+
+ @param batchSize the max number of sessions in a single batch
+ @param block function that will be executed with each batch, incl. list of sessions and current progress of batching
+ */
+- (void)enumerateSessionsBy:(NSInteger)batchSize
+                      block:(void (^)(NSArray <MXOlmSession *> *sessions,
+                                      double progress))block;
 
- @return a array of end-to-end sessions.
+/**
+ The number of stored end-to-end sessions
  */
-- (NSArray<MXOlmSession*>*)sessions;
+- (NSUInteger)sessionsCount;
 
 /**
  Store inbound group sessions.
@@ -336,6 +347,19 @@
  */
 - (NSArray<MXOlmInboundGroupSession*> *)inboundGroupSessions;
 
+/**
+ Enumerate all inbound group sessions in batches of `batchSize`
+
+ Each block is internally wrapped in `@autoreleasepool` so that memory footprint remains constant
+ regardless of the number of stored sessions.
+
+ @param batchSize the max number of sessions in a single batch
+ @param block function that will be executed with each batch, incl. list of sessions and current progress of batching
+ */
+- (void)enumerateInboundGroupSessionsBy:(NSInteger)batchSize
+                                  block:(void (^)(NSArray <MXOlmInboundGroupSession *> *sessions,
+                                                  NSSet <NSString *> *backedUp,
+                                                  double progress))block;
 
 /**
  Store outbound group session.

MatrixSDK/Crypto/Data/Store/MXRealmCryptoStore/MXRealmCryptoStore.m

@@ -862,34 +862,19 @@ - (MXOlmSession*)sessionWithDevice:(NSString*)deviceKey andSessionId:(NSString*)
 {
     MXRealmOlmSession *realmOlmSession = [MXRealmOlmSession objectsInRealm:self.realm
                                                                      where:@"sessionId = %@ AND deviceKey = %@", sessionId, deviceKey].firstObject;
-
-    MXOlmSession *mxOlmSession;
-    if (realmOlmSession.olmSessionData)
-    {
-        OLMSession *olmSession = [NSKeyedUnarchiver unarchiveObjectWithData:realmOlmSession.olmSessionData];
-
-        mxOlmSession = [[MXOlmSession alloc] initWithOlmSession:olmSession deviceKey:realmOlmSession.deviceKey];
-        mxOlmSession.lastReceivedMessageTs = realmOlmSession.lastReceivedMessageTs;
-    }
-
-    return mxOlmSession;
+    return [self olmSessionForRealmSession:realmOlmSession];
 }
 
 - (void)performSessionOperationWithDevice:(NSString*)deviceKey andSessionId:(NSString*)sessionId block:(void (^)(MXOlmSession *olmSession))block
 {
     [self.realm transactionWithName:@"[MXRealmCryptoStore] performSessionOperationWithDevice" block:^{
         MXRealmOlmSession *realmOlmSession = [MXRealmOlmSession objectsInRealm:self.realm
                                                                          where:@"sessionId = %@ AND deviceKey = %@", sessionId, deviceKey].firstObject;
-        if (realmOlmSession.olmSessionData)
+        MXOlmSession *session = [self olmSessionForRealmSession:realmOlmSession];
+        if (session)
         {
-            OLMSession *olmSession = [NSKeyedUnarchiver unarchiveObjectWithData:realmOlmSession.olmSessionData];
-
-            MXOlmSession *mxOlmSession = [[MXOlmSession alloc] initWithOlmSession:olmSession deviceKey:realmOlmSession.deviceKey];
-            mxOlmSession.lastReceivedMessageTs = realmOlmSession.lastReceivedMessageTs;
-
-            block(mxOlmSession);
-
-            realmOlmSession.olmSessionData = [NSKeyedArchiver archivedDataWithRootObject:mxOlmSession.session];
+            block(session);
+            realmOlmSession.olmSessionData = [NSKeyedArchiver archivedDataWithRootObject:session.session];
         }
         else
         {
@@ -915,14 +900,10 @@ - (void)performSessionOperationWithDevice:(NSString*)deviceKey andSessionId:(NSS
             sessionsWithDevice = [NSMutableArray array];
         }
 
-        if (realmOlmSession.olmSessionData)
+        MXOlmSession *session = [self olmSessionForRealmSession:realmOlmSession];
+        if (session)
         {
-            OLMSession *olmSession = [NSKeyedUnarchiver unarchiveObjectWithData:realmOlmSession.olmSessionData];
-
-            MXOlmSession *mxOlmSession = [[MXOlmSession alloc] initWithOlmSession:olmSession deviceKey:realmOlmSession.deviceKey];
-            mxOlmSession.lastReceivedMessageTs = realmOlmSession.lastReceivedMessageTs;
-
-            [sessionsWithDevice addObject:mxOlmSession];
+            [sessionsWithDevice addObject:session];
         }
     }
 
@@ -936,18 +917,64 @@ - (void)performSessionOperationWithDevice:(NSString*)deviceKey andSessionId:(NSS
     RLMResults<MXRealmOlmSession *> *realmOlmSessions = [MXRealmOlmSession allObjectsInRealm:self.realm];
     for (MXRealmOlmSession *realmOlmSession in realmOlmSessions)
     {
-        if (realmOlmSession.olmSessionData)
+        MXOlmSession *session = [self olmSessionForRealmSession:realmOlmSession];
+        if (session)
         {
-            OLMSession *olmSession = [NSKeyedUnarchiver unarchiveObjectWithData:realmOlmSession.olmSessionData];
+            [sessions addObject:session];
+        }
+    }
+
+    return sessions;
+}
+
+- (void)enumerateSessionsBy:(NSInteger)batchSize
+                      block:(void (^)(NSArray<MXOlmSession *> *sessions,
+                                      double progress))block
+{
+    RLMResults<MXRealmOlmSession *> *query = [MXRealmOlmSession allObjectsInRealm:self.realm];
+    for (NSInteger i = 0; i < query.count; i += batchSize)
+    {
+        @autoreleasepool {
+            NSInteger count = MIN(batchSize, query.count - i);
+            NSIndexSet *batchSet = [NSIndexSet indexSetWithIndexesInRange:NSMakeRange(i, count)];
+            MXLogDebug(@"[MXRealmCryptoStore] enumerateSessionsBy: Batch %@", batchSet);
 
-            MXOlmSession *mxOlmSession = [[MXOlmSession alloc] initWithOlmSession:olmSession deviceKey:realmOlmSession.deviceKey];
-            mxOlmSession.lastReceivedMessageTs = realmOlmSession.lastReceivedMessageTs;
+            NSMutableArray *sessions = [NSMutableArray array];
+            for (MXRealmOlmSession *realmOlmSession in [query objectsAtIndexes:batchSet])
+            {
+                MXOlmSession *session = [self olmSessionForRealmSession:realmOlmSession];
+                if (session)
+                {
+                    [sessions addObject:session];
+                }
+            }
 
-            [sessions addObject:mxOlmSession];
+            double progress = (double)(batchSet.lastIndex + 1)/(double)query.count;
+            block(sessions.copy, progress);
         }
     }
+}
+
+- (NSUInteger)sessionsCount
+{
+    RLMResults<MXRealmOlmSession *> *sessions = [MXRealmOlmSession allObjectsInRealm:self.realm];
+    return sessions.count;
+}
+
+- (MXOlmSession *)olmSessionForRealmSession:(MXRealmOlmSession *)realmSession
+{
+    if (!realmSession.olmSessionData)
+    {
+        MXLogFailure(@"[MXRealmCryptoStore] olmSessionForRealmSession: Missing olm session data");
+        return nil;
+    }
 
-    return sessions;
+    OLMSession *olmSession = [NSKeyedUnarchiver unarchiveObjectWithData:realmSession.olmSessionData];
+
+    MXOlmSession *session = [[MXOlmSession alloc] initWithOlmSession:olmSession deviceKey:realmSession.deviceKey];
+    session.lastReceivedMessageTs = realmSession.lastReceivedMessageTs;
+
+    return session;
 }
 
 #pragma mark - MXRealmOlmInboundGroupSession
@@ -1061,6 +1088,36 @@ - (void)performSessionOperationWithGroupSessionWithId:(NSString*)sessionId sende
     return sessions;
 }
 
+- (void)enumerateInboundGroupSessionsBy:(NSInteger)batchSize
+                                  block:(void (^)(NSArray<MXOlmInboundGroupSession *> *sessions,
+                                                  NSSet<NSString *> *backedUp,
+                                                  double progress))block
+{
+    RLMResults<MXRealmOlmInboundGroupSession *> *query = [MXRealmOlmInboundGroupSession allObjectsInRealm:self.realm];
+    for (NSInteger i = 0; i < query.count; i += batchSize)
+    {
+        @autoreleasepool {
+            NSInteger count = MIN(batchSize, query.count - i);
+            NSIndexSet *batchSet = [NSIndexSet indexSetWithIndexesInRange:NSMakeRange(i, count)];
+            MXLogDebug(@"[MXRealmCryptoStore] enumerateInboundGroupSessions: Batch %@", batchSet);
+
+            NSMutableArray *sessions = [NSMutableArray array];
+            NSMutableSet *backedUp = [NSMutableSet set];
+            for (MXRealmOlmInboundGroupSession *realmSession in [query objectsAtIndexes:batchSet])
+            {
+                [sessions addObject:[NSKeyedUnarchiver unarchiveObjectWithData:realmSession.olmInboundGroupSessionData]];
+                if (realmSession.backedUp)
+                {
+                    [backedUp addObject:realmSession.sessionId];
+                }
+            }
+
+            double progress = (double)(batchSet.lastIndex + 1)/(double)query.count;
+            block(sessions.copy, backedUp.copy, progress);
+        }
+    }
+}
+
 - (void)removeInboundGroupSessionWithId:(NSString*)sessionId andSenderKey:(NSString*)senderKey
 {
     RLMRealm *realm = self.realm;

MatrixSDK/Crypto/MXCrypto.h

@@ -403,13 +403,17 @@ MX_ASSUME_MISSING_NULLABILITY_BEGIN
                                         error:(NSError **)error;
 
 /**
- Check if the user has previously enabled crypto.
- If yes, init the crypto module.
+ Initialize the crypto module
+
+ If the user has previously enabled crypto it will be opened, otherwise a new crypto
+ store will be created.
 
+ @param migrationProgress a block called repeatedly with percentage of migration done, if any necessasry
  @param complete a block called in any case when the operation completes.
  */
-+ (void)checkCryptoWithMatrixSession:(MXSession*)mxSession
-                            complete:(void (^)(id<MXCrypto> crypto, NSError *error))complete;
++ (void)initializeCryptoWithMatrixSession:(MXSession*)mxSession
+                        migrationProgress:(void (^)(double progress))migrationProgress
+                                 complete:(void (^)(id<MXCrypto> crypto, NSError *error))complete;
 
 /**
  Stores the exportedOlmDevice related to the credentials into the store.