Skip to content

Commit

Permalink
Batch migrate olm and megolm sessions
Browse files Browse the repository at this point in the history
  • Loading branch information
@Anderas
Anderas committed Jan 18, 2023
1 parent 9a40692 commit f42240e
Show file tree
Hide file tree
Showing 16 changed files with 599 additions and 167 deletions.
14 changes: 14 additions & 0 deletions MatrixSDK.xcodeproj/project.pbxproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1858,6 +1858,8 @@
ED4114EC292E498100728459 /* MXBackgroundCryptoV2.swift in Sources */ = {isa = PBXBuildFile; fileRef = ED4114EA292E498100728459 /* MXBackgroundCryptoV2.swift */; };
ED4114EE292E49C000728459 /* MXLegacyBackgroundCrypto.swift in Sources */ = {isa = PBXBuildFile; fileRef = ED4114ED292E49C000728459 /* MXLegacyBackgroundCrypto.swift */; };
ED4114EF292E49C000728459 /* MXLegacyBackgroundCrypto.swift in Sources */ = {isa = PBXBuildFile; fileRef = ED4114ED292E49C000728459 /* MXLegacyBackgroundCrypto.swift */; };
ED4368B129784CCE002B6272 /* MXRealmCryptoStoreTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = ED4368B029784CCE002B6272 /* MXRealmCryptoStoreTests.swift */; };
ED4368B229784CCE002B6272 /* MXRealmCryptoStoreTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = ED4368B029784CCE002B6272 /* MXRealmCryptoStoreTests.swift */; };
ED44F01128180BCC00452A5D /* MXSharedHistoryKeyRequest.swift in Sources */ = {isa = PBXBuildFile; fileRef = ED44F01028180BCC00452A5D /* MXSharedHistoryKeyRequest.swift */; };
ED44F01228180BCC00452A5D /* MXSharedHistoryKeyRequest.swift in Sources */ = {isa = PBXBuildFile; fileRef = ED44F01028180BCC00452A5D /* MXSharedHistoryKeyRequest.swift */; };
ED44F01428180EAB00452A5D /* MXSharedHistoryKeyManager.swift in Sources */ = {isa = PBXBuildFile; fileRef = ED44F01328180EAB00452A5D /* MXSharedHistoryKeyManager.swift */; };
Expand Down Expand Up @@ -3070,6 +3072,7 @@
ED4114E7292E496C00728459 /* MXBackgroundCrypto.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MXBackgroundCrypto.swift; sourceTree = "<group>"; };
ED4114EA292E498100728459 /* MXBackgroundCryptoV2.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MXBackgroundCryptoV2.swift; sourceTree = "<group>"; };
ED4114ED292E49C000728459 /* MXLegacyBackgroundCrypto.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MXLegacyBackgroundCrypto.swift; sourceTree = "<group>"; };
ED4368B029784CCE002B6272 /* MXRealmCryptoStoreTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MXRealmCryptoStoreTests.swift; sourceTree = "<group>"; };
ED44F01028180BCC00452A5D /* MXSharedHistoryKeyRequest.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MXSharedHistoryKeyRequest.swift; sourceTree = "<group>"; };
ED44F01328180EAB00452A5D /* MXSharedHistoryKeyManager.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MXSharedHistoryKeyManager.swift; sourceTree = "<group>"; };
ED44F01728180F1C00452A5D /* MXSharedHistoryKeyManagerUnitTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MXSharedHistoryKeyManagerUnitTests.swift; sourceTree = "<group>"; };
Expand Down Expand Up @@ -5467,6 +5470,14 @@
path = Crypto;
sourceTree = "<group>";
};
ED4368AF29784CA3002B6272 /* MXRealmCryptoStore */ = {
isa = PBXGroup;
children = (
ED4368B029784CCE002B6272 /* MXRealmCryptoStoreTests.swift */,
);
path = MXRealmCryptoStore;
sourceTree = "<group>";
};
ED44F01628180F1300452A5D /* KeySharing */ = {
isa = PBXGroup;
children = (
Expand Down Expand Up @@ -5571,6 +5582,7 @@
ED6DAC1328C78D3700ECDCB6 /* Store */ = {
isa = PBXGroup;
children = (
ED4368AF29784CA3002B6272 /* MXRealmCryptoStore */,
ED6DAC1428C78D4000ECDCB6 /* MXMemoryCryptoStore.swift */,
);
path = Store;
Expand Down Expand Up @@ -7348,6 +7360,7 @@
EDB4209527DF822B0036AF39 /* MXEventsByTypesEnumeratorOnArrayTests.swift in Sources */,
EC40385D28A16EDA0067D5B8 /* MXAes256KeyBackupTests.m in Sources */,
ED6DAC0728C77E1100ECDCB6 /* MXForwardedRoomKeyEventContentUnitTests.swift in Sources */,
ED4368B129784CCE002B6272 /* MXRealmCryptoStoreTests.swift in Sources */,
3A9E2B4328EB3960000DB2A7 /* MXMatrixVersionsUnitTests.swift in Sources */,
3265CB3B1A151C3800E24B2F /* MXRoomStateTests.m in Sources */,
ED8F1D302885AB0300F897E7 /* MXTrustLevelSourceUnitTests.swift in Sources */,
Expand Down Expand Up @@ -8006,6 +8019,7 @@
EDB4209627DF822B0036AF39 /* MXEventsByTypesEnumeratorOnArrayTests.swift in Sources */,
EC40385E28A16EDA0067D5B8 /* MXAes256KeyBackupTests.m in Sources */,
ED6DAC0828C77E1100ECDCB6 /* MXForwardedRoomKeyEventContentUnitTests.swift in Sources */,
ED4368B229784CCE002B6272 /* MXRealmCryptoStoreTests.swift in Sources */,
3A9E2B4428EB3960000DB2A7 /* MXMatrixVersionsUnitTests.swift in Sources */,
32B477AA2638186000EA5800 /* MXHTTPAdditionalHeadersUnitTests.m in Sources */,
B135066A27EA100100BD3276 /* MXBeaconInfoUnitTests.swift in Sources */,
Expand Down
28 changes: 16 additions & 12 deletions MatrixSDK/Background/MXBackgroundCryptoStore.m
View file
Original file line number Diff line number Diff line change
Expand Up @@ -209,18 +209,6 @@ - (MXOlmSession*)sessionWithDevice:(NSString*)deviceKey andSessionId:(NSString*)
return sessions;
}

- (NSArray<MXOlmSession *> *)sessions
{
NSArray<MXOlmSession*> *bgSessions = [bgCryptoStore sessions] ?: @[];
NSArray<MXOlmSession*> *appSessions = [cryptoStore sessions] ?: @[];

NSMutableArray<MXOlmSession*> *sessions = [NSMutableArray array];
[sessions addObjectsFromArray:bgSessions];
[sessions addObjectsFromArray:appSessions];

return sessions;
}

- (void)storeSession:(MXOlmSession*)session
{
[bgCryptoStore storeSession:session];
Expand Down Expand Up @@ -356,6 +344,22 @@ - (void)storeDeviceSyncToken:(NSString*)deviceSyncToken
NSAssert(NO, @"This method should be useless in the context of MXBackgroundCryptoStore");
}

- (void)enumerateSessionsBy:(NSInteger)batchSize block:(void (^)(NSArray<MXOlmSession *> *, double))block
{
NSAssert(NO, @"This method should be useless in the context of MXBackgroundCryptoStore");
}

- (void)enumerateInboundGroupSessionsBy:(NSInteger)batchSize block:(void (^)(NSArray<MXOlmInboundGroupSession *> *, NSSet<NSString *> *, double))block
{
NSAssert(NO, @"This method should be useless in the context of MXBackgroundCryptoStore");
}

- (NSUInteger)sessionsCount
{
NSAssert(NO, @"This method should be useless in the context of MXBackgroundCryptoStore");
return 0;
}

- (NSArray<MXOlmInboundGroupSession*> *)inboundGroupSessions
{
NSAssert(NO, @"This method should be useless in the context of MXBackgroundCryptoStore");
Expand Down
30 changes: 27 additions & 3 deletions MatrixSDK/Crypto/Data/Store/MXCryptoStore.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -296,11 +296,22 @@
- (NSArray<MXOlmSession*>*)sessionsWithDevice:(NSString*)deviceKey;

/**
Retrieve all end-to-end sessions between this device and all other devices
Enumerate all end-to-end sessions in batches of `batchSize`
Each block is internally wrapped in `@autoreleasepool` so that memory footprint remains constant
regardless of the number of stored sessions.
@param batchSize the max number of sessions in a single batch
@param block function that will be executed with each batch, incl. list of sessions and current progress of batching
*/
- (void)enumerateSessionsBy:(NSInteger)batchSize
block:(void (^)(NSArray <MXOlmSession *> *sessions,
double progress))block;

@return a array of end-to-end sessions.
/**
The number of stored end-to-end sessions
*/
- (NSArray<MXOlmSession*>*)sessions;
- (NSUInteger)sessionsCount;

/**
Store inbound group sessions.
Expand Down Expand Up @@ -336,6 +347,19 @@
*/
- (NSArray<MXOlmInboundGroupSession*> *)inboundGroupSessions;

/**
Enumerate all inbound group sessions in batches of `batchSize`
Each block is internally wrapped in `@autoreleasepool` so that memory footprint remains constant
regardless of the number of stored sessions.
@param batchSize the max number of sessions in a single batch
@param block function that will be executed with each batch, incl. list of sessions and current progress of batching
*/
- (void)enumerateInboundGroupSessionsBy:(NSInteger)batchSize
block:(void (^)(NSArray <MXOlmInboundGroupSession *> *sessions,
NSSet <NSString *> *backedUp,
double progress))block;

/**
Store outbound group session.
Expand Down
123 changes: 90 additions & 33 deletions MatrixSDK/Crypto/Data/Store/MXRealmCryptoStore/MXRealmCryptoStore.m
View file
Original file line number Diff line number Diff line change
Expand Up @@ -862,34 +862,19 @@ - (MXOlmSession*)sessionWithDevice:(NSString*)deviceKey andSessionId:(NSString*)
{
MXRealmOlmSession *realmOlmSession = [MXRealmOlmSession objectsInRealm:self.realm
where:@"sessionId = %@ AND deviceKey = %@", sessionId, deviceKey].firstObject;

MXOlmSession *mxOlmSession;
if (realmOlmSession.olmSessionData)
{
OLMSession *olmSession = [NSKeyedUnarchiver unarchiveObjectWithData:realmOlmSession.olmSessionData];

mxOlmSession = [[MXOlmSession alloc] initWithOlmSession:olmSession deviceKey:realmOlmSession.deviceKey];
mxOlmSession.lastReceivedMessageTs = realmOlmSession.lastReceivedMessageTs;
}

return mxOlmSession;
return [self olmSessionForRealmSession:realmOlmSession];
}

- (void)performSessionOperationWithDevice:(NSString*)deviceKey andSessionId:(NSString*)sessionId block:(void (^)(MXOlmSession *olmSession))block
{
[self.realm transactionWithName:@"[MXRealmCryptoStore] performSessionOperationWithDevice" block:^{
MXRealmOlmSession *realmOlmSession = [MXRealmOlmSession objectsInRealm:self.realm
where:@"sessionId = %@ AND deviceKey = %@", sessionId, deviceKey].firstObject;
if (realmOlmSession.olmSessionData)
MXOlmSession *session = [self olmSessionForRealmSession:realmOlmSession];
if (session)
{
OLMSession *olmSession = [NSKeyedUnarchiver unarchiveObjectWithData:realmOlmSession.olmSessionData];

MXOlmSession *mxOlmSession = [[MXOlmSession alloc] initWithOlmSession:olmSession deviceKey:realmOlmSession.deviceKey];
mxOlmSession.lastReceivedMessageTs = realmOlmSession.lastReceivedMessageTs;

block(mxOlmSession);

realmOlmSession.olmSessionData = [NSKeyedArchiver archivedDataWithRootObject:mxOlmSession.session];
block(session);
realmOlmSession.olmSessionData = [NSKeyedArchiver archivedDataWithRootObject:session.session];
}
else
{
Expand All @@ -915,14 +900,10 @@ - (void)performSessionOperationWithDevice:(NSString*)deviceKey andSessionId:(NSS
sessionsWithDevice = [NSMutableArray array];
}

if (realmOlmSession.olmSessionData)
MXOlmSession *session = [self olmSessionForRealmSession:realmOlmSession];
if (session)
{
OLMSession *olmSession = [NSKeyedUnarchiver unarchiveObjectWithData:realmOlmSession.olmSessionData];

MXOlmSession *mxOlmSession = [[MXOlmSession alloc] initWithOlmSession:olmSession deviceKey:realmOlmSession.deviceKey];
mxOlmSession.lastReceivedMessageTs = realmOlmSession.lastReceivedMessageTs;

[sessionsWithDevice addObject:mxOlmSession];
[sessionsWithDevice addObject:session];
}
}

Expand All @@ -936,18 +917,64 @@ - (void)performSessionOperationWithDevice:(NSString*)deviceKey andSessionId:(NSS
RLMResults<MXRealmOlmSession *> *realmOlmSessions = [MXRealmOlmSession allObjectsInRealm:self.realm];
for (MXRealmOlmSession *realmOlmSession in realmOlmSessions)
{
if (realmOlmSession.olmSessionData)
MXOlmSession *session = [self olmSessionForRealmSession:realmOlmSession];
if (session)
{
OLMSession *olmSession = [NSKeyedUnarchiver unarchiveObjectWithData:realmOlmSession.olmSessionData];
[sessions addObject:session];
}
}

return sessions;
}

- (void)enumerateSessionsBy:(NSInteger)batchSize
block:(void (^)(NSArray<MXOlmSession *> *sessions,
double progress))block
{
RLMResults<MXRealmOlmSession *> *query = [MXRealmOlmSession allObjectsInRealm:self.realm];
for (NSInteger i = 0; i < query.count; i += batchSize)
{
@autoreleasepool {
NSInteger count = MIN(batchSize, query.count - i);
NSIndexSet *batchSet = [NSIndexSet indexSetWithIndexesInRange:NSMakeRange(i, count)];
MXLogDebug(@"[MXRealmCryptoStore] enumerateSessionsBy: Batch %@", batchSet);

MXOlmSession *mxOlmSession = [[MXOlmSession alloc] initWithOlmSession:olmSession deviceKey:realmOlmSession.deviceKey];
mxOlmSession.lastReceivedMessageTs = realmOlmSession.lastReceivedMessageTs;
NSMutableArray *sessions = [NSMutableArray array];
for (MXRealmOlmSession *realmOlmSession in [query objectsAtIndexes:batchSet])
{
MXOlmSession *session = [self olmSessionForRealmSession:realmOlmSession];
if (session)
{
[sessions addObject:session];
}
}

[sessions addObject:mxOlmSession];
double progress = (double)(batchSet.lastIndex + 1)/(double)query.count;
block(sessions.copy, progress);
}
}
}

- (NSUInteger)sessionsCount
{
RLMResults<MXRealmOlmSession *> *sessions = [MXRealmOlmSession allObjectsInRealm:self.realm];
return sessions.count;
}

- (MXOlmSession *)olmSessionForRealmSession:(MXRealmOlmSession *)realmSession
{
if (!realmSession.olmSessionData)
{
MXLogFailure(@"[MXRealmCryptoStore] olmSessionForRealmSession: Missing olm session data");
return nil;
}

return sessions;
OLMSession *olmSession = [NSKeyedUnarchiver unarchiveObjectWithData:realmSession.olmSessionData];

MXOlmSession *session = [[MXOlmSession alloc] initWithOlmSession:olmSession deviceKey:realmSession.deviceKey];
session.lastReceivedMessageTs = realmSession.lastReceivedMessageTs;

return session;
}

#pragma mark - MXRealmOlmInboundGroupSession
Expand Down Expand Up @@ -1061,6 +1088,36 @@ - (void)performSessionOperationWithGroupSessionWithId:(NSString*)sessionId sende
return sessions;
}

- (void)enumerateInboundGroupSessionsBy:(NSInteger)batchSize
block:(void (^)(NSArray<MXOlmInboundGroupSession *> *sessions,
NSSet<NSString *> *backedUp,
double progress))block
{
RLMResults<MXRealmOlmInboundGroupSession *> *query = [MXRealmOlmInboundGroupSession allObjectsInRealm:self.realm];
for (NSInteger i = 0; i < query.count; i += batchSize)
{
@autoreleasepool {
NSInteger count = MIN(batchSize, query.count - i);
NSIndexSet *batchSet = [NSIndexSet indexSetWithIndexesInRange:NSMakeRange(i, count)];
MXLogDebug(@"[MXRealmCryptoStore] enumerateInboundGroupSessions: Batch %@", batchSet);

NSMutableArray *sessions = [NSMutableArray array];
NSMutableSet *backedUp = [NSMutableSet set];
for (MXRealmOlmInboundGroupSession *realmSession in [query objectsAtIndexes:batchSet])
{
[sessions addObject:[NSKeyedUnarchiver unarchiveObjectWithData:realmSession.olmInboundGroupSessionData]];
if (realmSession.backedUp)
{
[backedUp addObject:realmSession.sessionId];
}
}

double progress = (double)(batchSet.lastIndex + 1)/(double)query.count;
block(sessions.copy, backedUp.copy, progress);
}
}
}

- (void)removeInboundGroupSessionWithId:(NSString*)sessionId andSenderKey:(NSString*)senderKey
{
RLMRealm *realm = self.realm;
Expand Down
12 changes: 8 additions & 4 deletions MatrixSDK/Crypto/MXCrypto.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -403,13 +403,17 @@ MX_ASSUME_MISSING_NULLABILITY_BEGIN
error:(NSError **)error;

/**
Check if the user has previously enabled crypto.
If yes, init the crypto module.
Initialize the crypto module
If the user has previously enabled crypto it will be opened, otherwise a new crypto
store will be created.
@param migrationProgress a block called repeatedly with percentage of migration done, if any necessasry
@param complete a block called in any case when the operation completes.
*/
+ (void)checkCryptoWithMatrixSession:(MXSession*)mxSession
complete:(void (^)(id<MXCrypto> crypto, NSError *error))complete;
+ (void)initializeCryptoWithMatrixSession:(MXSession*)mxSession
migrationProgress:(void (^)(double progress))migrationProgress
complete:(void (^)(id<MXCrypto> crypto, NSError *error))complete;

/**
Stores the exportedOlmDevice related to the credentials into the store.
Expand Down
Loading

0 comments on commit f42240e

Please sign in to comment.