WIP

WIP

WIP

# Conflicts:
#	packages/worker/package.json
#	packages/worker/workers/bookmark-initializer/index.js

Deps

WIP

Refactor PgClient types

WIP

.github/workflows/deploy.yml

@@ -13,7 +13,7 @@ on:
         default: true
 
 env:
-  node_version: 22
+  node_version: lts/*
   redis-version: 7
   FORCE_COLOR: 1
   NPM_CONFIG_COLOR: always

eslint.config.js

@@ -3,5 +3,6 @@ import neostandard, { resolveIgnoresFromGitignore } from 'neostandard'
 // Used for editors and canary testing
 
 export default neostandard({
+  ts: true,
   ignores: resolveIgnoresFromGitignore(),
 })

package.json

@@ -16,9 +16,10 @@
     "custompatch": "^1.0.28",
     "dependency-cruiser": "^16.3.2",
     "gh-release": "^7.0.0",
+    "knip": "^5.37.1",
     "neostandard": "^0.12.0",
     "npm-run-all2": "^7.0.1",
-    "typescript": "~5.6.2"
+    "typescript": "~5.7.2"
   },
   "funding": {
     "type": "individual",
@@ -33,7 +34,7 @@
     "url": "https://github.com/hifiwi-fi/breadcrum.net.git"
   },
   "scripts": {
-    "clean": "rm -rf node_modules && rm -rf ./packages/*/node_modules && rm package-lock.json",
+    "clean": "rm -rf node_modules && rm -rf ./packages/*/node_modules && rm -f package-lock.json",
     "prepublishOnly": "git push --follow-tags && gh-release -y",
     "postinstall": "custompatch",
     "release": "git push --follow-tags && gh-release -y",

packages/resources/archives/resolve-archive-queue.js

@@ -0,0 +1,42 @@
+/**
+ * @import { Queue, Worker, Processor } from 'bullmq'
+ */
+
+/**
+ * Data relating to resolve document jobs
+ *
+ * @typedef {{
+ *   url: string
+ *   userId: string
+ *   archive: boolean
+ *   archiveId: string
+ *   archiveURL: string
+ * }} ResolveArchiveData
+ */
+
+export const resolveArchiveQName = 'resolveArchive'
+export const resolveArchiveJobName = 'resolve-archive'
+
+/**
+ * @typedef {Queue<
+ *          ResolveArchiveData,
+ *          null,
+ *          typeof resolveArchiveJobName
+ * >} ResolveArchiveQ
+ */
+
+/**
+ * @typedef {Worker<
+ *          ResolveArchiveData,
+ *          null,
+ *          typeof resolveArchiveJobName
+ * >} ResolveArchiveW
+ */
+
+/**
+ * @typedef {Processor<
+ *          ResolveArchiveData,
+ *          null,
+ *          typeof resolveArchiveJobName
+ * >} ResolveArchiveP
+ */

packages/resources/bookmarks/normalize-url.js

@@ -0,0 +1,10 @@
+/**
+ * Normalizes a URL object by modifying its host property if necessary.
+ *
+ * @param {URL} url - The URL string to be normalized.
+ * @returns {Promise<URL>} An object containing the normalized URL string.
+ */
+export async function normalizeURL (url) {
+  if (url.host === 'm.youtube.com') url.host = 'www.youtube.com'
+  return url
+}

packages/resources/bookmarks/resolve-bookmark-queue.js

@@ -0,0 +1,54 @@
+/**
+ * @import { Queue, Worker, Processor } from 'bullmq'
+ */
+
+/**
+ * User-provided metadata for a bookmark.
+ *
+ * @typedef {{
+ *   title: string
+ *   tags: string[]
+ *   summary: string
+ * }} UserProvidedMeta
+ */
+
+/**
+ * Data required to initialize a bookmark.
+ *
+ * @typedef {{
+ *   userId: string
+ *   bookmarkId: string
+ *   url: string
+ *   resolveBookmark: boolean
+ *   resolveArchive: boolean
+ *   resolveEpisode: boolean
+ *   userProvidedMeta: UserProvidedMeta
+ * }} ResolveBookmarkData
+ */
+
+export const resolveBookmarkQName = 'resolveBookmark'
+export const resolveBookmarkJobName = 'resolve-bookmark'
+
+/**
+ * @typedef {Queue<
+ *          ResolveBookmarkData,
+ *          null,
+ *          typeof resolveBookmarkJobName
+ * >} ResolveBookmarkQ
+ */
+
+/**
+ * @typedef {Worker<
+ *          ResolveBookmarkData,
+ *          null,
+ *          typeof resolveBookmarkJobName
+ * >} ResolveBookmarkW
+ */
+
+/**
+ * @typedef {Processor<
+ *          ResolveBookmarkData,
+ *          null,
+ *          typeof resolveBookmarkJobName
+ * >} ResolveBookmarkP
+ */

packages/resources/bullmq/default-job-options.js

@@ -0,0 +1,14 @@
+/**
+ * @import { JobsOptions } from 'bullmq'
+ */
+
+/** @type {JobsOptions} */
+export const defaultJobOptions = {
+  removeOnComplete: {
+    age: 3600, // keep up to 1 hour
+    count: 1000, // keep up to 1000 jobs
+  },
+  removeOnFail: {
+    age: 24 * 3600, // keep up to 24 hours
+  }
+}

packages/web/routes/api/episodes/episode-query-create.js → packages/resources/episodes/episode-query-create.js

@@ -1,10 +1,19 @@
 /**
- * @import { FastifyInstance } from 'fastify'
- * @import { PoolClient } from 'pg'
+ * @import { PgClient } from '../types/pg-client.js'
  */
 
 import SQL from '@nearform/sql'
-import { getOrCreateDefaultFeed } from '../feeds/default-feed/default-feed-query.js'
+import { getOrCreateDefaultFeed } from '../feeds/default-feed-query.js'
+
+/**
+ * @typedef {{
+ *   id: string,
+ *   type: string,
+ *   medium: string,
+ *   podcast_feed_id: string,
+ *   url: string
+ * }} CreatedEpisode
+ */
 
 /**
  * Creates an episode entry in the database.
@@ -13,19 +22,13 @@ import { getOrCreateDefaultFeed } from '../feeds/default-feed/default-feed-query
  * into the `episodes` table with the provided details and returns the newly created episode object.
  *
  * @param {object} params - The parameters for creating an episode.
- * @param {PoolClient | FastifyInstance['pg']} params.client - The database client for executing queries, an instance of a pg connection from `fastify.pg` or `node-pg`.
+ * @param {PgClient} params.client - The database client for executing queries, an instance of a pg connection from `fastify.pg` or `node-pg`.
  * @param {string} params.userId - The ID of the user who owns the episode.
  * @param {string} params.bookmarkId - The ID of the bookmark associated with the episode.
  * @param {string} params.type - The type of the episode.
  * @param {string} params.medium - The medium of the episode (e.g., audio, video).
  * @param {string} params.url - The URL of the episode.
- * @returns {Promise<{
- *   id: string,
- *   type: string,
- *   medium: string,
- *   podcast_feed_id: string,
- *   url: string
- * }>} A promise that resolves to the newly created episode object, including its ID, type, medium, podcast feed ID, and URL.
+ * @returns {Promise<CreatedEpisode>} A promise that resolves to the newly created episode object, including its ID, type, medium, podcast feed ID, and URL.
  */
 export async function createEpisode ({
   client,

packages/resources/episodes/resolve-episode-queue.js

@@ -0,0 +1,43 @@
+/**
+ * @import { Queue, Worker, Processor } from 'bullmq'
+ * @import { MediumTypes } from './yt-dlp-api-client.js'
+ */
+
+/**
+ * Data required to resolve an episode.
+ *
+ * @typedef {{
+ *   userId: string
+ *   bookmarkTitle: string
+ *   episodeId: string
+ *   url: string
+ *   medium: MediumTypes
+ * }} ResolveEpisodeData
+ */
+
+export const resolveEpisodeQName = 'resolveEpisode'
+export const resolveEpisodeJobName = 'resolve-episode'
+
+/**
+ * @typedef {Queue<
+ *          ResolveEpisodeData,
+ *          null,
+ *          typeof resolveEpisodeJobName
+ * >} ResolveEpisodeQ
+ */
+
+/**
+ * @typedef {Worker<
+ *          ResolveEpisodeData,
+ *          null,
+ *          typeof resolveEpisodeJobName
+ * >} ResolveEpisodeW
+ */
+
+/**
+ * @typedef {Processor<
+ *          ResolveEpisodeData,
+ *          null,
+ *          typeof resolveEpisodeJobName
+ * >} ResolveEpisodeP
+ */

packages/resources/episodes/resolve-type.js

@@ -1,15 +1,15 @@
 /**
- * @param  {object} metadata
- * @param {string} [metadata.ext]
- * @param {string} [metadata._type]
+ * @param {object} media
+ * @param {string} [media.ext]
+ * @param {string} [media._type]
  * @return {'audio' | 'video' | string | null | undefined }
  */
-export function resolveType (metadata) {
+export function resolveType (media) {
   return (
-    ['mp3', 'm4a'].includes(metadata.ext ?? '')
+    ['mp3', 'm4a'].includes(media.ext ?? '')
       ? 'audio'
-      : ['mp4', 'mov', 'm3u8'].includes(metadata.ext ?? '')
+      : ['mp4', 'mov', 'm3u8'].includes(media.ext ?? '')
           ? 'video'
-          : metadata._type
+          : media._type
   )
 }

packages/web/routes/api/feeds/default-feed/default-feed-query.js → packages/resources/feeds/default-feed-query.js

@@ -1,7 +1,7 @@
 /* eslint-disable camelcase */
+
 /**
- * @import { FastifyInstance } from 'fastify'
- * @import { PoolClient } from 'pg'
+ * @import { PgClient } from '@breadcrum/resources/types/pg-client.js'
  */
 
 import SQL from '@nearform/sql'
@@ -17,7 +17,7 @@ import SQL from '@nearform/sql'
  * within a database transaction to ensure data consistency.
  *
  * @param {Object} params - The function parameters.
- * @param {PoolClient | FastifyInstance['pg']} params.client - The database client for executing queries, an instance of a pg connection from `fastify.pg` or `node-pg`.
+ * @param {PgClient} params.client - The database client for executing queries, an instance of a pg connection from `fastify.pg` or `node-pg`.
  * @param {string} params.userId - The ID of the user for whom to retrieve or create the default podcast feed.
  * @returns {Promise<string>} The ID of the default podcast feed for the given user.
  * @throws {Error} Throws an error if the database transaction fails.

packages/resources/package.json

@@ -11,6 +11,7 @@
   },
   "dependencies": {
     "@nearform/sql": "^1.10.5",
+    "bullmq": "^5.1.12",
     "undici": "^7.0.0"
   },
   "devDependencies": {
@@ -19,7 +20,7 @@
     "npm-run-all2": "^7.0.1",
     "neostandard": "^0.12.0",
     "tap": "^21.0.0",
-    "typescript": "~5.6.2"
+    "typescript": "~5.7.2"
   },
   "tap": {
     "serial": [],

packages/resources/tags/put-tags-query.js

@@ -2,8 +2,7 @@ import SQL from '@nearform/sql'
 
 /**
  * @import { FastifyInstance } from 'fastify'
- * @import { PoolClient } from 'pg'
- * @import { PostgresDb } from '@fastify/postgres'
+ * @import { PgClient } from '@breadcrum/resources/types/pg-client.js'
  */
 
 /**
@@ -13,7 +12,7 @@ import SQL from '@nearform/sql'
  * @exports
  * @param {Object} params - Parameters to shape the query.
  * @param {FastifyInstance} params.fastify - Fastify instance, used for logging and other utilities.
- * @param {PoolClient | PostgresDb?} [params.pg] - PostgreSQL connection or transaction client for executing the query.
+ * @param {PgClient?} [params.pg] - PostgreSQL connection or transaction client for executing the query.
  * @param {string} params.userId - UserID of the owner
  * @param {string} params.bookmarkId - The Bookmark ID to add tags to
  * @param {Array<string>} params.tags- List of tags to associate with the bookmark.
@@ -25,7 +24,6 @@ export async function putTagsQuery ({
   bookmarkId,
   tags,
 }) {
-  // @ts-ignore
   pg = pg ?? fastify.pg
 
   if (!pg) throw new Error('A postgres client is required')

packages/resources/tsconfig.json

@@ -2,9 +2,7 @@
   "extends": "@voxpelli/tsconfig/node20.json",
   "compilerOptions": {
     "noEmit": true,
-    "skipLibCheck": true,
-    "composite": true,
-    "incremental": true,
+    "skipLibCheck": false
   },
   "include": [
     "**/*"

packages/resources/types/pg-client.d.ts

@@ -0,0 +1,4 @@
+import { PoolClient } from 'pg'
+import { PostgresDb } from '@fastify/postgres'
+
+export type PgClient = PoolClient | PostgresDb

packages/resources/urls/ssrf-check.js

@@ -0,0 +1,38 @@
+/**
+ * @param  {Url} url A URL object to check
+ * @return {[type]}     [description]
+ */
+export async function ssrfCheck (url) {
+  try {
+    // Validate protocol
+    if (url.protocol !== 'https:' && url.protocol !== 'http:') {
+      return {
+        ssrf: false
+      }
+    }
+
+    // Block cloud metadata endpoints
+    const blockedHostnames = ['169.254.169.254', 'metadata.google.internal']
+    if (blockedHostnames.includes(url.hostname)) {
+      return false // Block specific metadata endpoints
+    }
+
+    // Resolve hostname to IP addresses
+    const addresses = await dns.lookup(url.hostname, { all: true })
+
+    // Validate each resolved IP
+    for (const { address } of addresses) {
+      if (isBlockedIP(address)) {
+        return false // Blocked IP detected
+      }
+    }
+
+    // Additional DNS rebinding check
+    await dnsRebindingCheck(url.hostname)
+
+    return true // Passed all checks, URL is safe
+  } catch (err) {
+    console.error('Error validating URL:', err.message)
+    return false // Fail closed on error
+  }
+}

packages/web/migrations/019.do.normalized-urls.sql

@@ -0,0 +1,4 @@
+alter table bookmarks
+  add column original_url text;
+
+comment on column bookmarks.original_url is 'The original, unnormalized URL of the bookmark.';

packages/web/migrations/019.undo.normalized-urls.sql

@@ -0,0 +1,2 @@
+alter table bookmarks
+  drop column if exists original_url;