merge: improve recreateChain

Resolves #159

packages/backend/src/queue/processors/ImportNotesProcessorService.ts

@@ -69,51 +69,40 @@ export class ImportNotesProcessorService {
 		}
 	}
 
-	// Function was taken from Firefish and edited to remove renoteId and make it run in only one for loop instead of two
+	// Function was taken from Firefish and modified for our needs
 	@bindThis
-	private async recreateChain(arr: any[]) {
+	private async recreateChain(idField: string, replyField: string, arr: any[]): Promise<any[]> {
 		type NotesMap = {
 			[id: string]: any;
 		};
 		const notesTree: any[] = [];
-		const lookup: NotesMap = {};
+		const noteById: NotesMap = {};
+		const notesWaitingForParent: NotesMap = {};
+
 		for await (const note of arr) {
-			lookup[`${note.id}`] = note;
+			noteById[note[idField]] = note;
 			note.childNotes = [];
-			let parent = null;
-
-			if (note.replyId == null) {
-				notesTree.push(note);
-			} else {
-				parent = lookup[`${note.replyId}`];
+
+			const children = notesWaitingForParent[note[idField]];
+			if (children) {
+				note.childNotes.push(...children);
 			}
 
-			if (parent) parent.childNotes.push(note);
-		}
-		return notesTree;
-	}
+			if (note[replyField] == null) {
+				notesTree.push(note);
+				continue;
+			}
 
-	@bindThis
-	private async recreateTwitChain(arr: any[]) {
-		type TweetsMap = {
-			[id: string]: any;
-		};
-		const tweetsTree: any[] = [];
-		const lookup: TweetsMap = {};
-		for await (const tweet of arr) {
-			lookup[`${tweet.id_str}`] = tweet;
-			tweet.replies = [];
-			let parent = null;
-
-			if (!tweet.in_reply_to_status_id_str) {
-				tweetsTree.push(tweet);
+			const parent = noteById[note[replyField]];
+			if (parent) {
+				parent.childNotes.push(note);
 			} else {
-				parent = lookup[`${tweet.in_reply_to_status_id_str}`];
+				notesWaitingForParent[note[replyField]] ||= [];
+				notesWaitingForParent[note[replyField]].push(note);
 			}
-
-			if (parent) parent.replies.push(tweet);
 		}
-		return tweetsTree;
+
+		return notesTree;
 	}
 
 	@bindThis
@@ -178,9 +167,7 @@ export class ImportNotesProcessorService {
 				const tweets = Object.keys(fakeWindow.window.YTD.tweets.part0).reduce((m, key, i, obj) => {
 					return m.concat(fakeWindow.window.YTD.tweets.part0[key].tweet);
 				}, []);
-				// Due to the way twitter outputs the tweets the entire array needs to be reversed for the recreate function.
-				const reversedTweets = tweets.reverse();
-				const processedTweets = await this.recreateTwitChain(reversedTweets);
+				const processedTweets = await this.recreateChain("id_str", "in_reply_to_status_id_str", tweets);
 				this.queueService.createImportTweetsToDbJob(job.data.user, processedTweets, null);
 			} finally {
 				cleanup();
@@ -248,7 +235,7 @@ export class ImportNotesProcessorService {
 
 			const notesJson = fs.readFileSync(path, 'utf-8');
 			const notes = JSON.parse(notesJson);
-			const processedNotes = await this.recreateChain(notes);
+			const processedNotes = await this.recreateChain("id", "replyId", notes);
 			this.queueService.createImportKeyNotesToDbJob(job.data.user, processedNotes, null);
 			cleanup();
 		}
@@ -484,9 +471,9 @@ export class ImportNotesProcessorService {
 		try {
 			const date = new Date(tweet.created_at);
 			const textReplaceURLs = tweet.entities.urls && tweet.entities.urls.length > 0 ? await replaceTwitterUrls(tweet.full_text, tweet.entities.urls) : tweet.full_text;
-			const text = tweet.entities.user_mentions && tweet.entities.user_mentions.length > 0 ? await replaceTwitterMentions(textReplaceURLs, tweet.entities.user_mentions) : textReplaceURLs; 
+			const text = tweet.entities.user_mentions && tweet.entities.user_mentions.length > 0 ? await replaceTwitterMentions(textReplaceURLs, tweet.entities.user_mentions) : textReplaceURLs;
 			const files: MiDriveFile[] = [];
-			
+
 			if (tweet.extended_entities && this.isIterable(tweet.extended_entities.media)) {
 				for await (const file of tweet.extended_entities.media) {
 					if (file.video_info) {
@@ -543,7 +530,7 @@ export class ImportNotesProcessorService {
 				}
 			}
 			const createdNote = await this.noteCreateService.import(user, { createdAt: date, reply: parentNote, text: text, files: files });
-			if (tweet.replies) this.queueService.createImportTweetsToDbJob(user, tweet.replies, createdNote.id);
+			if (tweet.childNotes) this.queueService.createImportTweetsToDbJob(user, tweet.childNotes, createdNote.id);
 		} catch (e) {
 			this.logger.warn(`Error: ${e}`);
 		}