Rewrite initial bookmark resolution algorithm

.github/workflows/deploy.yml

@@ -13,7 +13,7 @@ on:
         default: true
 
 env:
-  node_version: 22
+  node_version: lts/*
   redis-version: 7
   FORCE_COLOR: 1
   NPM_CONFIG_COLOR: always

eslint.config.js

@@ -3,5 +3,6 @@ import neostandard, { resolveIgnoresFromGitignore } from 'neostandard'
 // Used for editors and canary testing
 
 export default neostandard({
+  ts: true,
   ignores: resolveIgnoresFromGitignore(),
 })

package.json

@@ -13,12 +13,13 @@
   "devDependencies": {
     "@voxpelli/tsconfig": "^15.0.0",
     "auto-changelog": "^2.0.0",
-    "custompatch": "^1.0.28",
+    "custompatch": "^1.1.4",
     "dependency-cruiser": "^16.3.2",
     "gh-release": "^7.0.0",
+    "knip": "^5.37.1",
     "neostandard": "^0.12.0",
     "npm-run-all2": "^7.0.1",
-    "typescript": "~5.6.2"
+    "typescript": "~5.7.2"
   },
   "funding": {
     "type": "individual",
@@ -33,7 +34,7 @@
     "url": "https://github.com/hifiwi-fi/breadcrum.net.git"
   },
   "scripts": {
-    "clean": "rm -rf node_modules && rm -rf ./packages/*/node_modules && rm package-lock.json",
+    "clean": "rm -rf node_modules && rm -rf ./packages/*/node_modules && rm -f package-lock.json",
     "prepublishOnly": "git push --follow-tags && gh-release -y",
     "postinstall": "custompatch",
     "release": "git push --follow-tags && gh-release -y",

packages/web/routes/api/archives/archive-query-create.js → packages/resources/archives/archive-query-create.js

@@ -1,16 +1,27 @@
+/**
+ * @import { PgClient } from '../types/pg-client.js'
+ */
 import SQL from '@nearform/sql'
 
+/**
+ * @typedef {{
+ *  id: string
+ *  url: string
+ *  title: string
+ * }} ArchiveCreate
+ */
+
 /**
  * Creates an archive entry in the database for a given bookmark.
  *
  * @param {Object} params - The parameters for creating an archive.
- * @param {import('@fastify/postgres').PostgresDb} params.client - The database client for executing queries, an instance of a pg connection from `fastify.pg` or `node-pg`.
+ * @param {PgClient} params.client - The database client for executing queries, an instance of a pg connection from `fastify.pg` or `node-pg`.
  * @param {string} params.userId - The ID of the user who owns the bookmark.
  * @param {string} params.bookmarkId - The ID of the bookmark being archived.
  * @param {string} params.bookmarkTitle - The title of the bookmark.
- * @param {string} params.extractionMethod - The method used for archiving the bookmark.
+ * @param {'server'} params.extractionMethod - The method used for archiving the bookmark.
  * @param {string} params.url - The URL of the bookmark.
- * @returns {Promise<Object>} A promise that resolves to the newly created archive object, including its ID, URL, and title.
+ * @returns {Promise<ArchiveCreate>} A promise that resolves to the newly created archive object, including its ID, URL, and title.
  */
 export async function createArchive ({
   client,
@@ -27,6 +38,7 @@ export async function createArchive ({
           `
 
   const archiveResults = await client.query(createArchiveQuery)
+  /** @type {ArchiveCreate} */
   const archive = archiveResults.rows[0]
   return archive
 }

packages/resources/archives/resolve-archive-queue.js

@@ -0,0 +1,40 @@
+/**
+ * @import { Queue, Worker, Processor } from 'bullmq'
+ */
+
+/**
+ * Data relating to resolve document jobs
+ *
+ * @typedef {{
+ *   url: string
+ *   userId: string
+ *   archiveId: string
+ * }} ResolveArchiveData
+ */
+
+export const resolveArchiveQName = 'resolveArchive'
+export const resolveArchiveJobName = 'resolve-archive'
+
+/**
+ * @typedef {Queue<
+ *          ResolveArchiveData,
+ *          null,
+ *          typeof resolveArchiveJobName
+ * >} ResolveArchiveQ
+ */
+
+/**
+ * @typedef {Worker<
+ *          ResolveArchiveData,
+ *          null,
+ *          typeof resolveArchiveJobName
+ * >} ResolveArchiveW
+ */
+
+/**
+ * @typedef {Processor<
+ *          ResolveArchiveData,
+ *          null,
+ *          typeof resolveArchiveJobName
+ * >} ResolveArchiveP
+ */

packages/resources/bookmarks/normalize-url.js

@@ -0,0 +1,10 @@
+/**
+ * Normalizes a URL object by modifying its host property if necessary.
+ *
+ * @param {URL} url - The URL string to be normalized.
+ * @returns {Promise<URL>} An object containing the normalized URL string.
+ */
+export async function normalizeURL (url) {
+  if (url.host === 'm.youtube.com') url.host = 'www.youtube.com'
+  return url
+}

packages/resources/bookmarks/resolve-bookmark-queue.js

@@ -0,0 +1,54 @@
+/**
+ * @import { Queue, Worker, Processor } from 'bullmq'
+ */
+
+/**
+ * User-provided metadata for a bookmark.
+ *
+ * @typedef {{
+ *   title: string | undefined
+ *   tags: string[]
+ *   summary: string | undefined
+ * }} UserProvidedMeta
+ */
+
+/**
+ * Data required to initialize a bookmark.
+ *
+ * @typedef {{
+ *   userId: string
+ *   bookmarkId: string
+ *   url: string
+ *   resolveBookmark: boolean
+ *   resolveArchive: boolean
+ *   resolveEpisode: boolean
+ *   userProvidedMeta: UserProvidedMeta
+ * }} ResolveBookmarkData
+ */
+
+export const resolveBookmarkQName = 'resolveBookmark'
+export const resolveBookmarkJobName = 'resolve-bookmark'
+
+/**
+ * @typedef {Queue<
+ *          ResolveBookmarkData,
+ *          null,
+ *          typeof resolveBookmarkJobName
+ * >} ResolveBookmarkQ
+ */
+
+/**
+ * @typedef {Worker<
+ *          ResolveBookmarkData,
+ *          null,
+ *          typeof resolveBookmarkJobName
+ * >} ResolveBookmarkW
+ */
+
+/**
+ * @typedef {Processor<
+ *          ResolveBookmarkData,
+ *          null,
+ *          typeof resolveBookmarkJobName
+ * >} ResolveBookmarkP
+ */

packages/resources/bullmq/default-job-options.js

@@ -0,0 +1,14 @@
+/**
+ * @import { JobsOptions } from 'bullmq'
+ */
+
+/** @type {JobsOptions} */
+export const defaultJobOptions = {
+  removeOnComplete: {
+    age: 3600, // keep up to 1 hour
+    count: 1000, // keep up to 1000 jobs
+  },
+  removeOnFail: {
+    age: 24 * 3600, // keep up to 24 hours
+  }
+}

packages/web/routes/api/episodes/episode-query-create.js → packages/resources/episodes/episode-query-create.js

@@ -1,10 +1,20 @@
 /**
- * @import { FastifyInstance } from 'fastify'
- * @import { PoolClient } from 'pg'
+ * @import { PgClient } from '../types/pg-client.js'
+ * @import { MediumTypes } from './yt-dlp-api-client.js'
  */
 
 import SQL from '@nearform/sql'
-import { getOrCreateDefaultFeed } from '../feeds/default-feed/default-feed-query.js'
+import { getOrCreateDefaultFeed } from '../feeds/default-feed-query.js'
+
+/**
+ * @typedef {{
+ *   id: string,
+ *   type: string,
+ *   medium: MediumTypes,
+ *   podcast_feed_id: string,
+ *   url: string
+ * }} CreatedEpisode
+ */
 
 /**
  * Creates an episode entry in the database.
@@ -13,19 +23,13 @@ import { getOrCreateDefaultFeed } from '../feeds/default-feed/default-feed-query
  * into the `episodes` table with the provided details and returns the newly created episode object.
  *
  * @param {object} params - The parameters for creating an episode.
- * @param {PoolClient | FastifyInstance['pg']} params.client - The database client for executing queries, an instance of a pg connection from `fastify.pg` or `node-pg`.
+ * @param {PgClient} params.client - The database client for executing queries, an instance of a pg connection from `fastify.pg` or `node-pg`.
  * @param {string} params.userId - The ID of the user who owns the episode.
  * @param {string} params.bookmarkId - The ID of the bookmark associated with the episode.
  * @param {string} params.type - The type of the episode.
- * @param {string} params.medium - The medium of the episode (e.g., audio, video).
+ * @param {MediumTypes} params.medium - The medium of the episode (e.g., audio, video).
  * @param {string} params.url - The URL of the episode.
- * @returns {Promise<{
- *   id: string,
- *   type: string,
- *   medium: string,
- *   podcast_feed_id: string,
- *   url: string
- * }>} A promise that resolves to the newly created episode object, including its ID, type, medium, podcast feed ID, and URL.
+ * @returns {Promise<CreatedEpisode>} A promise that resolves to the newly created episode object, including its ID, type, medium, podcast feed ID, and URL.
  */
 export async function createEpisode ({
   client,

packages/resources/episodes/resolve-episode-queue.js

@@ -0,0 +1,43 @@
+/**
+ * @import { Queue, Worker, Processor } from 'bullmq'
+ * @import { MediumTypes } from './yt-dlp-api-client.js'
+ */
+
+/**
+ * Data required to resolve an episode.
+ *
+ * @typedef {{
+ *   userId: string
+ *   bookmarkTitle: string | undefined
+ *   episodeId: string
+ *   url: string
+ *   medium: MediumTypes
+ * }} ResolveEpisodeData
+ */
+
+export const resolveEpisodeQName = 'resolveEpisode'
+export const resolveEpisodeJobName = 'resolve-episode'
+
+/**
+ * @typedef {Queue<
+ *          ResolveEpisodeData,
+ *          null,
+ *          typeof resolveEpisodeJobName
+ * >} ResolveEpisodeQ
+ */
+
+/**
+ * @typedef {Worker<
+ *          ResolveEpisodeData,
+ *          null,
+ *          typeof resolveEpisodeJobName
+ * >} ResolveEpisodeW
+ */
+
+/**
+ * @typedef {Processor<
+ *          ResolveEpisodeData,
+ *          null,
+ *          typeof resolveEpisodeJobName
+ * >} ResolveEpisodeP
+ */

packages/resources/episodes/resolve-type.js

@@ -1,15 +1,15 @@
 /**
- * @param  {object} metadata
- * @param {string} [metadata.ext]
- * @param {string} [metadata._type]
+ * @param {object} media
+ * @param {string} [media.ext]
+ * @param {string} [media._type]
  * @return {'audio' | 'video' | string | null | undefined }
  */
-export function resolveType (metadata) {
+export function resolveType (media) {
   return (
-    ['mp3', 'm4a'].includes(metadata.ext ?? '')
+    ['mp3', 'm4a'].includes(media.ext ?? '')
       ? 'audio'
-      : ['mp4', 'mov', 'm3u8'].includes(metadata.ext ?? '')
+      : ['mp4', 'mov', 'm3u8'].includes(media.ext ?? '')
           ? 'video'
-          : metadata._type
+          : media._type
   )
 }

packages/web/routes/api/feeds/default-feed/default-feed-query.js → packages/resources/feeds/default-feed-query.js

@@ -1,7 +1,7 @@
 /* eslint-disable camelcase */
+
 /**
- * @import { FastifyInstance } from 'fastify'
- * @import { PoolClient } from 'pg'
+ * @import { PgClient } from '@breadcrum/resources/types/pg-client.js'
  */
 
 import SQL from '@nearform/sql'
@@ -17,7 +17,7 @@ import SQL from '@nearform/sql'
  * within a database transaction to ensure data consistency.
  *
  * @param {Object} params - The function parameters.
- * @param {PoolClient | FastifyInstance['pg']} params.client - The database client for executing queries, an instance of a pg connection from `fastify.pg` or `node-pg`.
+ * @param {PgClient} params.client - The database client for executing queries, an instance of a pg connection from `fastify.pg` or `node-pg`.
  * @param {string} params.userId - The ID of the user for whom to retrieve or create the default podcast feed.
  * @returns {Promise<string>} The ID of the default podcast feed for the given user.
  * @throws {Error} Throws an error if the database transaction fails.

packages/resources/package.json

@@ -11,15 +11,17 @@
   },
   "dependencies": {
     "@nearform/sql": "^1.10.5",
+    "bullmq": "^5.1.12",
+    "ipaddr": "^0.1.0",
     "undici": "^7.0.0"
   },
   "devDependencies": {
     "@types/node": "^22.0.0",
     "@voxpelli/tsconfig": "^15.0.0",
-    "npm-run-all2": "^7.0.1",
     "neostandard": "^0.12.0",
+    "npm-run-all2": "^7.0.1",
     "tap": "^21.0.0",
-    "typescript": "~5.6.2"
+    "typescript": "~5.7.2"
   },
   "tap": {
     "serial": [],

packages/resources/tags/put-tags-query.js

@@ -2,8 +2,7 @@ import SQL from '@nearform/sql'
 
 /**
  * @import { FastifyInstance } from 'fastify'
- * @import { PoolClient } from 'pg'
- * @import { PostgresDb } from '@fastify/postgres'
+ * @import { PgClient } from '@breadcrum/resources/types/pg-client.js'
  */
 
 /**
@@ -13,7 +12,7 @@ import SQL from '@nearform/sql'
  * @exports
  * @param {Object} params - Parameters to shape the query.
  * @param {FastifyInstance} params.fastify - Fastify instance, used for logging and other utilities.
- * @param {PoolClient | PostgresDb?} [params.pg] - PostgreSQL connection or transaction client for executing the query.
+ * @param {PgClient?} [params.pg] - PostgreSQL connection or transaction client for executing the query.
  * @param {string} params.userId - UserID of the owner
  * @param {string} params.bookmarkId - The Bookmark ID to add tags to
  * @param {Array<string>} params.tags- List of tags to associate with the bookmark.
@@ -25,7 +24,6 @@ export async function putTagsQuery ({
   bookmarkId,
   tags,
 }) {
-  // @ts-ignore
   pg = pg ?? fastify.pg
 
   if (!pg) throw new Error('A postgres client is required')
@@ -54,6 +52,6 @@ export async function putTagsQuery ({
 
   await pg.query(applyTags)
 
-  // @ts-ignore
-  fastify.prom.tagAppliedCounter.inc(tagsResults.rows.length)
+  // @ts-expect-error TODO: Fix this ignore
+  fastify?.prom?.tagAppliedCounter?.inc?.(tagsResults.rows.length)
 }