WIP

WIP

WIP

# Conflicts:
#	packages/worker/package.json
#	packages/worker/workers/bookmark-initializer/index.js

Deps

WIP

Refactor PgClient types

WIP

.github/workflows/deploy.yml

@@ -13,7 +13,7 @@ on:
         default: true
 
 env:
-  node_version: 22
+  node_version: lts/*
   redis-version: 7
   FORCE_COLOR: 1
   NPM_CONFIG_COLOR: always

package.json

@@ -16,9 +16,9 @@
     "custompatch": "^1.0.28",
     "dependency-cruiser": "^16.3.2",
     "gh-release": "^7.0.0",
+    "knip": "^5.37.1",
     "neostandard": "^0.12.0",
     "npm-run-all2": "^7.0.1",
-    "typescript": "~5.6.2"
   },
   "funding": {
     "type": "individual",
@@ -33,7 +33,7 @@
     "url": "https://github.com/hifiwi-fi/breadcrum.net.git"
   },
   "scripts": {
-    "clean": "rm -rf node_modules && rm -rf ./packages/*/node_modules && rm package-lock.json",
+    "clean": "rm -rf node_modules && rm -rf ./packages/*/node_modules && rm -f package-lock.json",
     "prepublishOnly": "git push --follow-tags && gh-release -y",
     "postinstall": "custompatch",
     "release": "git push --follow-tags && gh-release -y",

packages/resources/bookmarks/normalize-url.js

@@ -0,0 +1,11 @@
+/**
+ * Normalizes a URL object by modifying its host property if necessary.
+ *
+ * @param {string} url - The URL string to be normalized.
+ * @returns {Promise<URL>} An object containing the normalized URL string.
+ */
+export async function normalizeURL (url) {
+  const urlObj = new URL(url)
+  if (urlObj.host === 'm.youtube.com') urlObj.host = 'www.youtube.com'
+  return urlObj
+}

packages/resources/episodes/episode-query-create.js

@@ -0,0 +1,48 @@
+/**
+ * @import { PgClient } from '@breadcrum/resources/types/pg-client.js'
+ */
+
+import SQL from '@nearform/sql'
+import { getOrCreateDefaultFeed } from '../feeds/default-feed-query.js'
+
+/**
+ * Creates an episode entry in the database.
+ *
+ * This function first retrieves or creates a default feed for the given user. Then, it inserts a new episode
+ * into the `episodes` table with the provided details and returns the newly created episode object.
+ *
+ * @param {object} params - The parameters for creating an episode.
+ * @param {PgClient} params.client - The database client for executing queries, an instance of a pg connection from `fastify.pg` or `node-pg`.
+ * @param {string} params.userId - The ID of the user who owns the episode.
+ * @param {string} params.bookmarkId - The ID of the bookmark associated with the episode.
+ * @param {string} params.type - The type of the episode.
+ * @param {string} params.medium - The medium of the episode (e.g., audio, video).
+ * @param {string} params.url - The URL of the episode.
+ * @returns {Promise<{
+ *   id: string,
+ *   type: string,
+ *   medium: string,
+ *   podcast_feed_id: string,
+ *   url: string
+ * }>} A promise that resolves to the newly created episode object, including its ID, type, medium, podcast feed ID, and URL.
+ */
+export async function createEpisode ({
+  client,
+  userId,
+  bookmarkId,
+  type,
+  medium,
+  url,
+}) {
+  const defaultFeedId = await getOrCreateDefaultFeed({ client, userId })
+
+  const createEpisodeQuery = SQL`
+          INSERT INTO episodes (owner_id, podcast_feed_id, bookmark_id, type, medium, url)
+          VALUES (${userId}, ${defaultFeedId}, ${bookmarkId}, ${type}, ${medium}, ${url})
+          returning id, type, medium, podcast_feed_id, url;
+          `
+
+  const episodeResults = await client.query(createEpisodeQuery)
+  const episode = episodeResults.rows[0]
+  return episode
+}

packages/resources/feeds/default-feed-query.js

@@ -0,0 +1,66 @@
+/* eslint-disable camelcase */
+
+/**
+ * @import { PgClient } from '@breadcrum/resources/types/pg-client.js'
+ */
+
+import SQL from '@nearform/sql'
+
+/**
+ * Retrieves or creates a default podcast feed for a given user.
+ *
+ * This function first attempts to retrieve the user's default podcast feed ID from the database.
+ * If the user does not have a default podcast feed, the function creates a new podcast feed,
+ * assigns it as the user's default, and returns the new feed's ID.
+ *
+ * The process of creating a new feed and updating the user's default feed ID is performed
+ * within a database transaction to ensure data consistency.
+ *
+ * @param {Object} params - The function parameters.
+ * @param {PgClient} params.client - The database client for executing queries, an instance of a pg connection from `fastify.pg` or `node-pg`.
+ * @param {string} params.userId - The ID of the user for whom to retrieve or create the default podcast feed.
+ * @returns {Promise<string>} The ID of the default podcast feed for the given user.
+ * @throws {Error} Throws an error if the database transaction fails.
+ */
+export async function getOrCreateDefaultFeed ({
+  client,
+  userId,
+}) {
+  const getDefaultFeed = SQL`
+    SELECT default_podcast_feed_id
+    FROM users
+    WHERE id = ${userId};
+  `
+
+  const defaultFeedResults = await client.query(getDefaultFeed)
+  let { default_podcast_feed_id } = defaultFeedResults.rows[0]
+
+  if (!default_podcast_feed_id) {
+    try {
+      await client.query(SQL`BEGIN`)
+      const createDefaultFeed = SQL`
+      INSERT INTO podcast_feeds (owner_id)
+      VALUES (${userId})
+      returning id;
+      `
+
+      const defaultFeedResults = await client.query(createDefaultFeed)
+
+      default_podcast_feed_id = defaultFeedResults.rows[0].id
+
+      const applyDefaultFeed = SQL`
+      update users
+      set default_podcast_feed_id = ${default_podcast_feed_id}
+      where id = ${userId};
+      `
+
+      await client.query(applyDefaultFeed)
+      await client.query(SQL`COMMIT`)
+    } catch (err) {
+      await client.query(SQL`ROLLBACK`)
+      throw err
+    }
+  }
+
+  return default_podcast_feed_id
+}

packages/resources/package.json

@@ -19,7 +19,7 @@
     "npm-run-all2": "^7.0.1",
     "neostandard": "^0.12.0",
     "tap": "^21.0.0",
-    "typescript": "~5.6.2"
+    "typescript": "~5.7.2"
   },
   "tap": {
     "serial": [],

packages/resources/tags/put-tags-query.js

@@ -2,8 +2,7 @@ import SQL from '@nearform/sql'
 
 /**
  * @import { FastifyInstance } from 'fastify'
- * @import { PoolClient } from 'pg'
- * @import { PostgresDb } from '@fastify/postgres'
+ * @import { PgClient } from '@breadcrum/resources/types/pg-client.js'
  */
 
 /**
@@ -13,7 +12,7 @@ import SQL from '@nearform/sql'
  * @exports
  * @param {Object} params - Parameters to shape the query.
  * @param {FastifyInstance} params.fastify - Fastify instance, used for logging and other utilities.
- * @param {PoolClient | PostgresDb?} [params.pg] - PostgreSQL connection or transaction client for executing the query.
+ * @param {PgClient?} [params.pg] - PostgreSQL connection or transaction client for executing the query.
  * @param {string} params.userId - UserID of the owner
  * @param {string} params.bookmarkId - The Bookmark ID to add tags to
  * @param {Array<string>} params.tags- List of tags to associate with the bookmark.
@@ -25,7 +24,6 @@ export async function putTagsQuery ({
   bookmarkId,
   tags,
 }) {
-  // @ts-ignore
   pg = pg ?? fastify.pg
 
   if (!pg) throw new Error('A postgres client is required')

packages/resources/tsconfig.json

@@ -2,9 +2,7 @@
   "extends": "@voxpelli/tsconfig/node20.json",
   "compilerOptions": {
     "noEmit": true,
-    "skipLibCheck": true,
-    "composite": true,
-    "incremental": true,
+    "skipLibCheck": true
   },
   "include": [
     "**/*"

packages/resources/types/pg-client.d.ts

@@ -0,0 +1,4 @@
+import { PoolClient } from 'pg'
+import { PostgresDb } from '@fastify/postgres'
+
+export type PgClient = PoolClient | PostgresDb

packages/resources/urls/ssrf-check.js

@@ -0,0 +1,38 @@
+/**
+ * @param  {Url} url A URL object to check
+ * @return {[type]}     [description]
+ */
+export async function ssrfCheck (url) {
+  try {
+    // Validate protocol
+    if (url.protocol !== 'https:' && url.protocol !== 'http:') {
+      return {
+        ssrf: false
+      }
+    }
+
+    // Block cloud metadata endpoints
+    const blockedHostnames = ['169.254.169.254', 'metadata.google.internal']
+    if (blockedHostnames.includes(url.hostname)) {
+      return false // Block specific metadata endpoints
+    }
+
+    // Resolve hostname to IP addresses
+    const addresses = await dns.lookup(url.hostname, { all: true })
+
+    // Validate each resolved IP
+    for (const { address } of addresses) {
+      if (isBlockedIP(address)) {
+        return false // Blocked IP detected
+      }
+    }
+
+    // Additional DNS rebinding check
+    await dnsRebindingCheck(url.hostname)
+
+    return true // Passed all checks, URL is safe
+  } catch (err) {
+    console.error('Error validating URL:', err.message)
+    return false // Fail closed on error
+  }
+}

packages/web/migrations/019.do.normalized-urls.sql

@@ -0,0 +1,4 @@
+alter table bookmarks
+  add column original_url text;
+
+comment on column bookmarks.original_url is 'The original, unnormalized URL of the bookmark.';

packages/web/migrations/019.undo.normalized-urls.sql

@@ -0,0 +1,2 @@
+alter table bookmarks
+  drop column if exists original_url;

packages/web/package.json

@@ -28,8 +28,8 @@
     "@fastify/swagger": "^9.1.0",
     "@fastify/swagger-ui": "^5.0.1",
     "@nearform/sql": "^1.5.0",
-    "ajv": "^8.17.1",
     "abstract-cache-redis": "^2.0.0",
+    "ajv": "^8.17.1",
     "bullmq": "^5.1.12",
     "clean-deep": "^3.4.0",
     "common-tags": "^1.8.2",
@@ -48,8 +48,7 @@
     "pg": "^8.6.0",
     "postgrator-cli": "^9.0.0",
     "resolve-email": "3.0.10",
-    "undici": "^7.0.0",
-    "webassert": "^3.0.2"
+    "undici": "^7.0.0"
   },
   "optionalDependencies": {
     "sodium-native": "^4.0.0"
@@ -78,7 +77,7 @@
     "tap": "^21.0.0",
     "top-bun": "^10.0.0",
     "type-fest": "^4.26.1",
-    "typescript": "~5.6.2",
+    "typescript": "~5.7.2",
     "uland-isomorphic": "^2.0.0",
     "xmlbuilder": "^15.1.1"
   },
@@ -97,7 +96,7 @@
     "test": "run-s test:*",
     "test:eslint": "eslint . --ignore-pattern 'public/*'",
     "test:tap": "tap",
-    "test-skip:tsc":"tsc",
+    "test-skip:tsc": "tsc",
     "watch": "run-s clean && run-p watch:*",
     "watch:server": "fastify start -w --ignore-watch='node_modules .git ./client ./public' -l info -P -p 3000 --options --address localhost app.js",
     "watch:tob-bun": "npm run build:top-bun -- --watch-only --drafts",

packages/web/plugins/bullmq-decorators.d.ts

@@ -6,6 +6,7 @@ declare module 'fastify' {
     queues: {
       resolveEpisodeQ: Queue
       resolveDocumentQ: Queue
+      initializeBookmarkQ: Queue
     }
   }
 }

packages/web/plugins/bullmq.js

@@ -27,9 +27,15 @@ export default fp(async function (fastify, _) {
       defaultJobOptions,
     })
 
+    const initializeBookmarkQ = new Queue('initializeBookmark', {
+      connection: queueRedis,
+      defaultJobOptions,
+    })
+
     const queues = {
       resolveEpisodeQ,
       resolveDocumentQ,
+      initializeBookmarkQ
     }
 
     fastify.decorate('queues', queues)

packages/web/plugins/flags/index.js

@@ -4,7 +4,7 @@ import { defaultFrontendFlags } from './frontend-flags.js'
 import { defaultBackendFlags } from './backend-flags.js'
 
 /**
- * @import { FastifyInstance } from 'fastify'
+ * @import { PgClient } from '@breadcrum/resources/types/pg-client.js'
  */
 
 /**
@@ -16,7 +16,7 @@ export default fp(async function (fastify, _) {
      * Retrieves the feature flags.
      *
      * @param {Object} options - The options for retrieving flags.
-     * @param {FastifyInstance['pg']} [options.pgClient] - The PostgreSQL client instance.
+     * @param {PgClient} [options.pgClient] - The PostgreSQL client instance.
      * @param {boolean} [options.frontend=true] - Whether to retrieve frontend flags.
      * @param {boolean} [options.backend=true] - Whether to retrieve backend flags.
      * @returns {Promise<Object>} The retrieved flag set.

packages/web/plugins/pg.js

@@ -1,12 +1,11 @@
 import fp from 'fastify-plugin'
 
 /**
- * @import { FastifyInstance } from 'fastify'
- * @import { PoolClient } from 'pg'
+ * @import { PgClient } from '@breadcrum/resources/types/pg-client.js'
  */
 
 /**
- * @typedef {PoolClient | FastifyInstance['pg']} PgClientAlias
+ * @typedef {PgClient} PgClientAlias
  */
 
 /**

packages/web/routes/api/archives/archive-query-get.js

@@ -1,15 +1,15 @@
 /**
  * @import { FastifyInstance } from 'fastify'
- * @import { PoolClient } from 'pg'
  * @import { TypeArchiveRead } from './schemas/schema-archive-read.js'
+ * @import { PgClient } from '@breadcrum/resources/types/pg-client.js'
  */
 
 import SQL from '@nearform/sql'
 
 /**
  * @typedef {ArchiveQueryParams & {
  *   fastify: FastifyInstance,
- *   pg?: PoolClient | FastifyInstance['pg']
+ *   pg?: PgClient
  * }} GetArchivesParams
  */
 

packages/web/routes/api/bookmarks/get-bookmarks-query.js

@@ -1,15 +1,15 @@
 /**
  * @import { FastifyInstance } from 'fastify'
- * @import { PoolClient } from 'pg'
  * @import { TypeBookmarkRead } from './schemas/schema-bookmark-read.js'
+ * @import { PgClient } from '@breadcrum/resources/types/pg-client.js'
  */
 
 import SQL from '@nearform/sql'
 
 /**
  * @typedef {GetBookmarksQueryParams & {
  *   fastify: FastifyInstance,
- *   pg?: PoolClient | FastifyInstance['pg']
+ *   pg?: PgClient
  * }} GetBookmarksParams
  */