From 2866e9719df1ce006f4c2e38f0f662171b67cae1 Mon Sep 17 00:00:00 2001 From: CraftyDH <23249291+CraftyDH@users.noreply.github.com> Date: Tue, 12 Mar 2024 22:25:34 +1100 Subject: [PATCH] Update crobot to use nocodb backend. --- example.env | 2 +- src/baserow-integration.ts | 298 ---------------------------------- src/baserow-types.ts | 107 ------------ src/commands/flush-members.ts | 4 +- src/door-status.ts | 2 +- src/index.ts | 4 +- src/nocodb-integration.ts | 225 +++++++++++++++++++++++++ src/nocodb-types.ts | 70 ++++++++ 8 files changed, 301 insertions(+), 411 deletions(-) delete mode 100644 src/baserow-integration.ts delete mode 100644 src/baserow-types.ts create mode 100644 src/nocodb-integration.ts create mode 100644 src/nocodb-types.ts diff --git a/example.env b/example.env index 2179a2a..22a1fc8 100644 --- a/example.env +++ b/example.env @@ -4,5 +4,5 @@ CLIENT_ID= MAINTAINERS= CSSA_SERVER= GITHUB_TOKEN= -BASEROW_TOKEN= +NOCODB_TOKEN= WEBSOCKET_SECRET= diff --git a/src/baserow-integration.ts b/src/baserow-integration.ts deleted file mode 100644 index af43130..0000000 --- a/src/baserow-integration.ts +++ /dev/null @@ -1,298 +0,0 @@ -import { - areItemsEqual, - BASEROW_ITEM_FIELDS, - BaserowGetRowsResponse, - BaserowItem, - BaserowWebhook, -} from "./baserow-types"; -import express, { Express, Request } from "express"; -import { GuildMember, Snowflake } from "discord.js"; - -const baserowData: Map = new Map(); -const MEMBER_ROLE_ID = "753524901708693558"; -const LIFE_MEMBER_ROLE_ID = "702889882598506558"; -const CSSA_SERVER_ID = process.env.CSSA_SERVER as Snowflake; -if (!CSSA_SERVER_ID) throw new Error("CSSA_SERVER not set."); - -function transformUsername(username: string): [string, number]; -function transformUsername( - username: string | null, -): [string, number] | undefined { - if (username === null) return undefined; - // New discord names are stored lowercase, and case insensitive - if (!username.includes("#")) { - username = username.toLowerCase(); - return [username, 0]; - } - const [name, discriminator] = username.split("#"); - return [name, Number.parseInt(discriminator, 10)]; -} - -const isLifeMember = (item: BaserowItem): boolean => { - return item[BASEROW_ITEM_FIELDS.FLAGS].some( - (flag) => flag.value === "Life Member", - ); -}; - -export async function refreshBaserowData() { - // Fetch data from baserow - // Store in baserowData - // Fire events for onRowCreate, onRowUpdate, onRowDelete as necessary - - // Force update all discord members - await cssaGuild.fetch(); - await cssaGuild.members.fetch(); - - const fetchedData: BaserowItem[] = []; - let nextUrl: string | null = - "https://baserow.cssa.club/api/database/rows/table/511/"; - while (nextUrl) { - // eslint-disable-next-line no-await-in-loop - const baserowPage: BaserowGetRowsResponse = await fetch(nextUrl, { - headers: new Headers({ - Authorization: `Token ${process.env.BASEROW_TOKEN}`, - }), - }).then((response) => response.json()); - fetchedData.push(...baserowPage.results); - nextUrl = baserowPage.next; - } - const deletedRowIds = new Set(baserowData.keys()); - const eventPromises: Promise[] = []; - for (const row of fetchedData) { - const existingItem = baserowData.get(row.id); - if (existingItem) { - if (!areItemsEqual(existingItem, row)) { - eventPromises.push(onRowUpdate(row, existingItem)); - baserowData.set(row.id, row); - } - deletedRowIds.delete(row.id); - } else { - eventPromises.push(onRowCreate(row)); - baserowData.set(row.id, row); - } - } - for (const id of deletedRowIds) { - const existingItem = baserowData.get(id); - if (existingItem) { - eventPromises.push(onRowDelete(existingItem)); - } else { - console.warn("Deleted item not found in baserowData"); - } - baserowData.delete(id); - } - - // Drop any members not in the fetched data - const memberRole = await cssaGuild.roles.fetch(MEMBER_ROLE_ID); - if (!memberRole) throw new Error("Couldn't get member role."); - const members = memberRole.members; - const memberUsernames: Map> = new Map(); - for (const item of fetchedData) { - const username = item[BASEROW_ITEM_FIELDS.DISCORD_USERNAME]; - if (!username) continue; - const [name, discriminator] = transformUsername(username); - if (!memberUsernames.has(name)) memberUsernames.set(name, new Set()); - memberUsernames.get(name)?.add(discriminator); - } - for (const member of members.values()) { - const validDiscriminators = memberUsernames.get(member.user.username); - if ( - validDiscriminators === undefined || - !validDiscriminators.has(Number.parseInt(member.user.discriminator, 10)) - ) - eventPromises.push(member.roles.remove(memberRole)); - } - - return Promise.all(eventPromises).then(() => { - console.log("Baserow data refreshed."); - }); -} - -export async function attachBaserowWebhookListener(expressApp: Express) { - await refreshBaserowData(); - - expressApp.use(express.json()); - - expressApp.post( - "/membership/update", - async (request: Request, response) => { - if (request.headers["x-cssa-secret"] != process.env.WEBSOCKET_SECRET) { - console.warn("Illegal websocket update."); - response.status(401).send(); - return; - } - console.log("Received baserow webhook."); - const webhookBody: BaserowWebhook = request.body; - - // Collect all async operations for better concurrency - const promises: Promise[] = []; - - switch (webhookBody.event_type) { - case "rows.created": { - for (const item of webhookBody.items) { - promises.push( - onRowCreate(item).then(() => { - baserowData.set(item.id, item); - }), - ); - } - break; - } - case "rows.updated": { - for (const item of webhookBody.items) { - const oldItem = webhookBody.old_items.find( - (oldItem) => oldItem.id === item.id, - ); - if (oldItem) { - promises.push( - onRowUpdate(item, oldItem).then(() => { - baserowData.set(item.id, item); - }), - ); - } else { - console.warn("Updated item not found in webhookBody.old_items"); - } - } - break; - } - case "rows.deleted": { - for (const id of webhookBody.row_ids) { - const item = baserowData.get(id); - if (item) { - promises.push( - onRowDelete(item).then(() => { - baserowData.delete(id); - }), - ); - } else { - console.warn("Deleted item not found in baserowData"); - } - } - break; - } - } - - response.status(204).send(); - - await Promise.all(promises); - }, - ); -} - -class MemberNotFoundError extends Error {} - -async function getUser(rawUsername: string): Promise { - const [username, discriminator] = transformUsername(rawUsername); - for (const member of cssaGuild.members.cache.values()) { - if ( - member.user.username === username && - member.user.discriminator === discriminator.toString(10) - ) - return member; - } - const fetchedMembers = await cssaGuild.members.fetch({ query: username }); - for (const member of fetchedMembers.values()) { - if ( - member.user.username === username && - member.user.discriminator === discriminator.toString(10) - ) - return member; - } - console.log(`Couldn't find member in guild: ${username}#${discriminator}`); - throw new MemberNotFoundError("Couldn't find member in guild: " + username); -} - -export async function performRoleUpdate( - username: string, - roleType: "member" | "lifeMember", - operation: "add" | "remove", -) { - const member = await getUser(username); - const roleId = roleType === "member" ? MEMBER_ROLE_ID : LIFE_MEMBER_ROLE_ID; - const alreadySatisfied = - operation === "add" - ? member.roles.cache.has(roleId) - : !member.roles.cache.has(roleId); - if (alreadySatisfied) { - console.log( - `Role update not required: ${operation} ${roleType} for ${username}`, - ); - return; - } - console.log( - `Performing role update: ${operation} ${roleType} for ${username}`, - ); - await (operation === "add" - ? member.roles.add(roleId) - : member.roles.remove(roleId)); -} - -export async function onRowCreate(row: BaserowItem) { - const username = row[BASEROW_ITEM_FIELDS.DISCORD_USERNAME]; - if (!username) return; - try { - await performRoleUpdate(username, "member", "add"); - if (isLifeMember(row)) { - await performRoleUpdate(username, "lifeMember", "add"); - } - } catch (error) { - if (!(error instanceof MemberNotFoundError)) { - console.warn("Error while processing onRowCreate:", error); - } - } -} - -export async function onRowUpdate(row: BaserowItem, oldRow: BaserowItem) { - if ( - row[BASEROW_ITEM_FIELDS.DISCORD_USERNAME] !== - oldRow[BASEROW_ITEM_FIELDS.DISCORD_USERNAME] - ) { - const oldUsername = oldRow[BASEROW_ITEM_FIELDS.DISCORD_USERNAME]; - if (oldUsername) { - try { - await performRoleUpdate(oldUsername, "member", "remove"); - } catch (error) { - if (!(error instanceof MemberNotFoundError)) { - console.warn("Error while processing onRowUpdate:", error); - } - } - // Never remove life member role, as it is often added manually - } - const newUsername = row[BASEROW_ITEM_FIELDS.DISCORD_USERNAME]; - if (newUsername) { - try { - await performRoleUpdate(newUsername, "member", "add"); - if (isLifeMember(row)) { - await performRoleUpdate(newUsername, "lifeMember", "add"); - } - } catch (error) { - if (!(error instanceof MemberNotFoundError)) { - console.warn("Error while processing onRowUpdate:", error); - } - } - } - } - if (row[BASEROW_ITEM_FIELDS.FLAGS] !== oldRow[BASEROW_ITEM_FIELDS.FLAGS]) { - // Flags changed - // Add life member role if appropriate - const username = row[BASEROW_ITEM_FIELDS.DISCORD_USERNAME]; - if (!username) return; - if (!isLifeMember(row)) return; - try { - await performRoleUpdate(username, "lifeMember", "add"); - } catch (error) { - if (!(error instanceof MemberNotFoundError)) { - console.warn("Error while processing onRowUpdate:", error); - } - } - } -} - -export async function onRowDelete(row: BaserowItem) { - const username = row[BASEROW_ITEM_FIELDS.DISCORD_USERNAME]; - if (!username) return; - try { - await performRoleUpdate(username, "member", "remove"); - } catch (error) { - console.warn("Error while processing onRowDelete:", error); - } -} diff --git a/src/baserow-types.ts b/src/baserow-types.ts deleted file mode 100644 index eadc804..0000000 --- a/src/baserow-types.ts +++ /dev/null @@ -1,107 +0,0 @@ -type Nullable = T | null; - -export const BASEROW_ITEM_FIELDS = { - ID: "field_4740", - FIRST_NAME: "field_4741", - LAST_NAME: "field_4742", - UID: "field_4743", - EMAIL: "field_4744", - FLAGS: "field_4746", - MEMBERSHIP_CARD_NUMBER: "field_4748", - CREATED_AT: "field_4749", - DISCORD_USERNAME: "field_4760", -} as const; - -export type BASEROW_ITEM_FIELDS = - (typeof BASEROW_ITEM_FIELDS)[keyof typeof BASEROW_ITEM_FIELDS]; - -interface BaserowBaseWebhook { - table_id: 511; - database_id: 97; - workspace_id: 96; - event_id: string; -} - -interface BaserowCreatedWebhook extends BaserowBaseWebhook { - event_type: "rows.created"; - items: BaserowItem[]; -} - -interface BaserowUpdatedWebhook extends BaserowBaseWebhook { - event_type: "rows.updated"; - items: BaserowItem[]; - old_items: BaserowItem[]; -} - -interface BaserowDeletedWebhook extends BaserowBaseWebhook { - event_type: "rows.deleted"; - row_ids: number[]; -} - -export type BaserowWebhook = - | BaserowCreatedWebhook - | BaserowUpdatedWebhook - | BaserowDeletedWebhook; - -export interface BaserowGetRowsResponse { - count: number; - next: Nullable; - previous: Nullable; - results: BaserowItem[]; -} - -export interface BaserowItem { - id: number; - order: string; - // Member ID - [BASEROW_ITEM_FIELDS.ID]: string; - // First Name - [BASEROW_ITEM_FIELDS.FIRST_NAME]: Nullable; - // Last Name - [BASEROW_ITEM_FIELDS.LAST_NAME]: Nullable; - // UID - [BASEROW_ITEM_FIELDS.UID]: Nullable; - // Email - [BASEROW_ITEM_FIELDS.EMAIL]: Nullable; - // Flags - [BASEROW_ITEM_FIELDS.FLAGS]: BaserowFlags[]; - // Membership Card Number - [BASEROW_ITEM_FIELDS.MEMBERSHIP_CARD_NUMBER]: Nullable; - // Created At - [BASEROW_ITEM_FIELDS.CREATED_AT]: Date; - // Discord Username - [BASEROW_ITEM_FIELDS.DISCORD_USERNAME]: Nullable; -} - -type BaserowFlags = - | { - id: 2522; - color: "cyan"; - value: "Committee"; - } - | { - id: 2521; - color: "green"; - value: "Life Member"; - } - | { - id: 2523; - color: "purple"; - value: "CRO"; - }; - -export function areItemsEqual(a: BaserowItem, b: BaserowItem): boolean { - if (a.id !== b.id) return false; - for (const field of Object.values(BASEROW_ITEM_FIELDS)) { - if (field === BASEROW_ITEM_FIELDS.FLAGS) { - if (a[field].length !== b[field].length) return false; - for (let index = 0; index < a[field].length; index++) { - if (a[field][index].id !== b[field][index].id) return false; - if (a[field][index].color !== b[field][index].color) return false; - if (a[field][index].value !== b[field][index].value) return false; - } - } - if (a[field] !== b[field]) return false; - } - return true; -} diff --git a/src/commands/flush-members.ts b/src/commands/flush-members.ts index 7a87acd..f216c4c 100644 --- a/src/commands/flush-members.ts +++ b/src/commands/flush-members.ts @@ -3,7 +3,7 @@ import { SlashCommandBuilder, PermissionsBitField, } from "discord.js"; -import { refreshBaserowData } from "../baserow-integration"; +import { refreshDBData } from "../nocodb-integration"; export const data = new SlashCommandBuilder() .setName("flushmembers") @@ -37,7 +37,7 @@ export async function execute( return; } - await refreshBaserowData(); + await refreshDBData(); await interaction.reply({ content: "Members role flushed.", diff --git a/src/door-status.ts b/src/door-status.ts index 12ff2f2..d9fdf11 100644 --- a/src/door-status.ts +++ b/src/door-status.ts @@ -34,7 +34,7 @@ export async function attachDoorServer(app: Express) { await setStatusChannelName("CR is loading..."); // Assume the door is closed until we hear otherwise - let status: boolean = false; + let status = false; // Set a timer to show an error message if the sensor doesn't check in let timer: NodeJS.Timeout | undefined = setTimeout(() => { diff --git a/src/index.ts b/src/index.ts index 58504d1..2925d47 100644 --- a/src/index.ts +++ b/src/index.ts @@ -7,7 +7,7 @@ config({ path: ".env" }); import { initDiscord } from "./discord-client"; import { attachDoorServer } from "./door-status"; import addReactionEvents from "./reacts"; -import { attachBaserowWebhookListener } from "./baserow-integration"; +import { attachNocoDBWebhookListener } from "./nocodb-integration"; import { startServerIcon } from "./server-icon"; import registerCommands from "./command-registry"; @@ -28,7 +28,7 @@ async function main(): Promise { // Initialise the express app and attach the door server const expressApp = express(); await attachDoorServer(expressApp); - await attachBaserowWebhookListener(expressApp); + await attachNocoDBWebhookListener(expressApp); // Start the server icon and add reaction events startServerIcon(); diff --git a/src/nocodb-integration.ts b/src/nocodb-integration.ts new file mode 100644 index 0000000..4d130b2 --- /dev/null +++ b/src/nocodb-integration.ts @@ -0,0 +1,225 @@ +import { NocoDBWebhook, DBGetRowsResponse, DBItem } from "./nocodb-types"; +import express, { Express, Request } from "express"; +import { GuildMember, Snowflake } from "discord.js"; + +const MEMBER_ROLE_ID = "753524901708693558"; +const LIFE_MEMBER_ROLE_ID = "702889882598506558"; +const CSSA_SERVER_ID = process.env.CSSA_SERVER as Snowflake; +const DB_REQUEST_URL = + "https://db.cssa.club/api/v2/tables/mj43d15qwi3hi8p/records?fields=id%2Cdiscord_username%2Clife_member%2Ccommittee%2Ccro&where=where%3D%28discord_username%2Cisnot%2Cnull%29&limit=1000&shuffle=0&offset=0"; + +if (!CSSA_SERVER_ID) throw new Error("CSSA_SERVER not set."); + +function transformUsername(username: string): [string, number]; +function transformUsername( + username: string | null, +): [string, number] | undefined { + if (username === null) return undefined; + // New discord names are stored lowercase, and case insensitive + if (!username.includes("#")) { + username = username.toLowerCase(); + return [username, 0]; + } + const [name, discriminator] = username.split("#"); + return [name, Number.parseInt(discriminator, 10)]; +} + +interface DBItemDiscord { + item: DBItem; + discord: GuildMember; +} + +async function getDatabaseItemDiscord( + item: DBItem, +): Promise { + try { + return { + item, + discord: await getUser(item.discord_username), + }; + } catch { + return undefined; + } +} + +export async function refreshDBData() { + // Fire events for onRowCreate, onRowUpdate, onRowDelete as necessary + + // Force update all discord members + await Promise.all([cssaGuild.fetch(), cssaGuild.members.fetch()]); + + const databaseResp: DBGetRowsResponse = await fetch(DB_REQUEST_URL, { + headers: new Headers({ + "xc-token": process.env.NOCODB_TOKEN || "", + }), + }).then((response) => response.json()); + const fetchedData = databaseResp.list; + + // Associate a discord user for each entry + const itemsDiscordPromises: Promise[] = []; + for (const row of fetchedData) { + itemsDiscordPromises.push(getDatabaseItemDiscord(row)); + } + + const itemsDiscordProm = await Promise.all(itemsDiscordPromises); + const itemsDiscord = itemsDiscordProm.flatMap((f) => (f ? [f] : [])); + const discordUsers: Set = new Set(); + + // Get a set of discord users + for (const row of itemsDiscord) { + if (discordUsers.has(row.discord)) { + console.log( + `Warning: User ${row.item.discord_username} exists twice in db.`, + ); + } else { + discordUsers.add(row.discord); + } + } + + // Sync each found user + const eventPromises: Promise[] = []; + for (const row of itemsDiscord) { + eventPromises.push(onRowUpdate(row)); + } + + // Drop any members not in the fetched data + const memberRole = await cssaGuild.roles.fetch(MEMBER_ROLE_ID); + if (!memberRole) throw new Error("Couldn't get member role."); + + for (const member of memberRole.members.values()) { + if (discordUsers.has(member)) continue; + eventPromises.push(onRowDelete(member)); + } + + return Promise.all(eventPromises).then(() => { + console.log("DB data refreshed."); + }); +} + +export async function attachNocoDBWebhookListener(expressApp: Express) { + await refreshDBData(); + + expressApp.use(express.json()); + + expressApp.post( + "/membership/update", + async (request: Request, response) => { + if (request.headers["x-cssa-secret"] != process.env.WEBSOCKET_SECRET) { + console.warn("Illegal websocket update."); + response.status(401).send(); + return; + } + console.log("Received db webhook."); + const webhookBody: NocoDBWebhook = request.body; + + // Collect all async operations for better concurrency + const promises: Promise[] = []; + + switch (webhookBody.type) { + case "records.after.insert": { + for (const item of webhookBody.data.rows) { + promises.push( + getDatabaseItemDiscord(item).then((index) => { + if (index) onRowUpdate(index); + }), + ); + } + break; + } + case "records.after.update": { + const ids = new Set(); + for (const item of webhookBody.data.rows) { + ids.add(item.discord_username); + promises.push( + getDatabaseItemDiscord(item).then((index) => { + if (index) onRowUpdate(index); + }), + ); + } + for (const item of webhookBody.data.previous_rows) { + if (!ids.has(item.discord_username)) { + promises.push( + getDatabaseItemDiscord(item).then((index) => { + if (index) onRowDelete(index.discord); + }), + ); + } + } + break; + } + case "rows.after.delete": { + for (const item of webhookBody.data.rows) { + promises.push( + getDatabaseItemDiscord(item).then((index) => { + if (index) onRowDelete(index.discord); + }), + ); + } + break; + } + } + response.status(204).send(); + await Promise.all(promises); + }, + ); +} + +class MemberNotFoundError extends Error {} + +async function getUser(rawUsername: string): Promise { + const [username, discriminator] = transformUsername(rawUsername); + for (const member of cssaGuild.members.cache.values()) { + if ( + member.user.username === username && + member.user.discriminator === discriminator.toString(10) + ) + return member; + } + const fetchedMembers = await cssaGuild.members.fetch({ query: username }); + for (const member of fetchedMembers.values()) { + if ( + member.user.username === username && + member.user.discriminator === discriminator.toString(10) + ) + return member; + } + console.log(`Couldn't find member in guild: ${username}#${discriminator}`); + throw new MemberNotFoundError("Couldn't find member in guild: " + username); +} + +export async function performRoleUpdate( + member: GuildMember, + roleId: string, + operation: "add" | "remove", +) { + const alreadySatisfied = + operation === "add" + ? member.roles.cache.has(roleId) + : !member.roles.cache.has(roleId); + + if (alreadySatisfied) { + return; + } + console.log( + `Performing role update: ${operation} ${roleId} for ${member.user.username}`, + ); + await (operation === "add" + ? member.roles.add(roleId) + : member.roles.remove(roleId)); +} + +export async function onRowUpdateWebhook(row: DBItem) { + getUser(row.discord_username); +} + +export async function onRowUpdate(row: DBItemDiscord) { + const discord = row.discord; + await performRoleUpdate(discord, MEMBER_ROLE_ID, "add"); + if (row.item.life_member) { + await performRoleUpdate(discord, LIFE_MEMBER_ROLE_ID, "add"); + } +} + +export async function onRowDelete(row: GuildMember) { + await performRoleUpdate(row, MEMBER_ROLE_ID, "remove"); +} diff --git a/src/nocodb-types.ts b/src/nocodb-types.ts new file mode 100644 index 0000000..0c30455 --- /dev/null +++ b/src/nocodb-types.ts @@ -0,0 +1,70 @@ +interface NocoDBBaseWebhook { + id: string; + data: { + table_id: string; + table_name: string; + view_id: string; + view_name: string; + }; +} + +interface NocoDBCreatedWebhook extends NocoDBBaseWebhook { + type: "records.after.insert"; + data: { + table_id: string; + table_name: string; + view_id: string; + view_name: string; + rows: DBItem[]; + }; +} + +interface NocoDBUpdatedWebhook extends NocoDBBaseWebhook { + type: "records.after.update"; + data: { + table_id: string; + table_name: string; + view_id: string; + view_name: string; + previous_rows: DBItem[]; + rows: DBItem[]; + }; +} + +interface NocoDBDeletedWebhook extends NocoDBBaseWebhook { + type: "rows.after.delete"; + data: { + table_id: string; + table_name: string; + view_id: string; + view_name: string; + previous_rows: DBItem[]; + rows: DBItem[]; + }; +} + +export type NocoDBWebhook = + | NocoDBCreatedWebhook + | NocoDBUpdatedWebhook + | NocoDBDeletedWebhook; + +export interface DBGetRowsResponse { + list: [DBItem]; + PageInfo: [DBPageInfo]; +} + +export interface DBPageInfo { + pageSize: number; + totalRows: number; + isFirstPage: boolean; + isLastPage: boolean; + page: number; +} + +export interface DBItem { + id: number; + discord_username: string; + life_member: boolean; + committee: boolean; + cro: boolean; +}