diff --git a/bin/u-wave-core.js b/bin/u-wave-core.js index d23a9b64..59cfefff 100755 --- a/bin/u-wave-core.js +++ b/bin/u-wave-core.js @@ -37,6 +37,10 @@ const envSchema = { type: 'number', default: 6042, }, + MONGODB_URL: { + type: 'string', + format: 'uri', + }, REDIS_URL: { type: 'string', format: 'uri', @@ -108,6 +112,8 @@ const uw = uwave({ redis: config.REDIS_URL, sqlite: config.SQLITE_PATH, secret, + // This property is untyped, it is propagated to the also-untyped MongoDB -> SQL migration + mongo: config.MONGODB_URL, }); uw.on('redisError', (err) => { diff --git a/src/migrations/003-populate-sql.cjs b/src/migrations/003-populate-sql.cjs index 1b84e677..1f8e92bc 100644 --- a/src/migrations/003-populate-sql.cjs +++ b/src/migrations/003-populate-sql.cjs @@ -396,6 +396,42 @@ const userSchema = new mongoose.Schema({ minimize: false, }); +async function* asyncChunks(iter, chunkSize) { + let chunk = []; + for await (const element of iter) { + chunk.push(element); + if (chunk.length >= chunkSize) { + yield chunk; + chunk = []; + } + } + if (chunk.length > 0) { + yield chunk; + } +} + +function zip(a, b) { + const iterA = a[Symbol.iterator](); + const iterB = b[Symbol.iterator](); + const iter = { + next() { + const itemA = iterA.next(); + const itemB = iterB.next(); + + if (itemA.done !== itemB.done) { + throw new Error('zip: iterators have different lengths'); + } + + return { + value: [itemA.value, itemB.value], + done: itemA.done, + }; + }, + }; + iter[Symbol.iterator] = () => iter; + return iter; +} + /** * @param {import('umzug').MigrationParams} params */ @@ -427,7 +463,13 @@ async function up({ context: uw }) { const motd = await uw.redis.get('motd'); /** @type {Map} */ - const idMap = new Map(); + const mediaIDs = new Map(); + /** @type {Map} */ + const userIDs = new Map(); + /** @type {Map} */ + const playlistIDs = new Map(); + /** @type {Map} */ + const playlistItemIDs = new Map(); await db.transaction().execute(async (tx) => { for await (const config of models.Config.find().lean()) { @@ -443,27 +485,29 @@ async function up({ context: uw }) { .execute(); } - for await (const media of models.Media.find().lean()) { - const id = randomUUID(); - await tx.insertInto('media') - .values({ - id, + for await (const medias of asyncChunks(models.Media.find().lean(), 50)) { + const rows = await tx.insertInto('media') + .values(medias.map((media) => ({ + id: randomUUID(), sourceType: media.sourceType, sourceID: media.sourceID, sourceData: jsonb(media.sourceData), artist: media.artist, - title: media.title, + title: media.title ?? '', duration: media.duration, thumbnail: media.thumbnail, - createdAt: media.createdAt.toISOString(), - updatedAt: media.updatedAt.toISOString(), - }) + createdAt: (media.createdAt ?? media.updatedAt ?? new Date()).toISOString(), + updatedAt: (media.updatedAt ?? new Date()).toISOString(), + }))) .onConflict((conflict) => conflict.columns(['sourceType', 'sourceID']).doUpdateSet({ updatedAt: (eb) => eb.ref('excluded.updatedAt'), })) + .returning('id') .execute(); - idMap.set(media._id.toString(), id); + for (const [media, row] of zip(medias, rows)) { + mediaIDs.set(media._id.toString(), row.id); + } } const roles = await models.AclRole.find().lean(); @@ -491,7 +535,7 @@ async function up({ context: uw }) { for await (const user of models.User.find().lean()) { const userID = randomUUID(); - idMap.set(user._id.toString(), userID); + userIDs.set(user._id.toString(), userID); await tx.insertInto('users') .values({ @@ -499,7 +543,7 @@ async function up({ context: uw }) { username: user.username, slug: user.slug, createdAt: user.createdAt.toISOString(), - updatedAt: user.updatedAt.toISOString(), + updatedAt: (user.updatedAt ?? user.createdAt).toISOString(), }) .execute(); @@ -511,35 +555,38 @@ async function up({ context: uw }) { for await (const playlist of models.Playlist.where('author', user._id).lean()) { const playlistID = randomUUID(); - idMap.set(playlist._id.toString(), playlistID); + playlistIDs.set(playlist._id.toString(), playlistID); await tx.insertInto('playlists') .values({ id: playlistID, name: playlist.name, userID, - createdAt: playlist.createdAt.toISOString(), - updatedAt: playlist.updatedAt.toISOString(), + // Old objects use the `.created` property + createdAt: (playlist.createdAt ?? playlist.created).toISOString(), + updatedAt: (playlist.updatedAt ?? playlist.created).toISOString(), }) .execute(); const items = []; for (const itemMongoID of playlist.media) { const itemID = randomUUID(); - idMap.set(itemMongoID.toString(), itemID); + playlistItemIDs.set(itemMongoID.toString(), itemID); const item = await models.PlaylistItem.findById(itemMongoID).lean(); + const mediaID = mediaIDs.get(item.media.toString()); + await tx.insertInto('playlistItems') .values({ id: itemID, playlistID, - mediaID: idMap.get(item.media.toString()), + mediaID, artist: item.artist, title: item.title, start: item.start, - end: item.end, - createdAt: item.createdAt.toISOString(), - updatedAt: item.updatedAt.toISOString(), + end: item.end ?? 0, // Not ideal, but what can we do + createdAt: (item.createdAt ?? item.updatedAt ?? new Date()).toISOString(), + updatedAt: (item.updatedAt ?? new Date()).toISOString(), }) .execute(); @@ -551,10 +598,20 @@ async function up({ context: uw }) { .set({ items: jsonb(items) }) .execute(); } + + if (user.activePlaylist != null) { + const activePlaylistID = playlistIDs.get(user.activePlaylist.toString()); + if (activePlaylistID != null) { + await tx.updateTable('users') + .where('id', '=', userID) + .set({ activePlaylistID }) + .execute(); + } + } } for await (const entry of models.Authentication.find().lean()) { - const userID = idMap.get(entry.user.toString()); + const userID = userIDs.get(entry.user.toString()); if (userID == null) { throw new Error('Migration failure: unknown user ID'); } @@ -576,9 +633,8 @@ async function up({ context: uw }) { for await (const entry of models.HistoryEntry.find().lean()) { const entryID = randomUUID(); - idMap.set(entry._id.toString(), entryID); - const userID = idMap.get(entry.user.toString()); - const mediaID = idMap.get(entry.media.media.toString()); + const userID = userIDs.get(entry.user.toString()); + const mediaID = mediaIDs.get(entry.media.media.toString()); await tx.insertInto('historyEntries') .values({ id: entryID, @@ -597,14 +653,14 @@ async function up({ context: uw }) { for (const id of entry.upvotes) { feedback.set(id.toString(), { historyEntryID: entryID, - userID: idMap.get(id.toString()), + userID: userIDs.get(id.toString()), vote: 1, }); } for (const id of entry.downvotes) { feedback.set(id.toString(), { historyEntryID: entryID, - userID: idMap.get(id.toString()), + userID: userIDs.get(id.toString()), vote: -1, }); } @@ -615,7 +671,7 @@ async function up({ context: uw }) { } else { feedback.set(id.toString(), { historyEntryID: entryID, - userID: idMap.get(id.toString()), + userID: userIDs.get(id.toString()), favorite: 1, }); }