Skip to content

Commit

Permalink
feature: Schedule RSS feed refreshes every hour
Browse files Browse the repository at this point in the history
  • Loading branch information
MohamedBassem committed Nov 3, 2024
1 parent fa8286a commit d34b538
Show file tree
Hide file tree
Showing 5 changed files with 66 additions and 11 deletions.
30 changes: 30 additions & 0 deletions apps/workers/feedWorker.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import { and, eq, inArray } from "drizzle-orm";
import { DequeuedJob, Runner } from "liteque";
import cron from "node-cron";
import Parser from "rss-parser";
import { buildImpersonatingTRPCClient } from "trpc";

Expand All @@ -10,6 +11,35 @@ import logger from "@hoarder/shared/logger";
import { FeedQueue } from "@hoarder/shared/queues";
import { BookmarkTypes } from "@hoarder/shared/types/bookmarks";

export const FeedRefreshingWorker = cron.schedule(
"0 * * * *",
() => {
logger.info("[feed] Scheduling feed refreshing jobs ...");
db.query.rssFeedsTable
.findMany({
columns: {
id: true,
},
})
.then((feeds) => {
for (const feed of feeds) {
FeedQueue.enqueue(
{
feedId: feed.id,
},
{
idempotencyKey: feed.id,
},
);
}
});
},
{
runOnInit: false,
scheduled: false,
},
);

export class FeedWorker {
static build() {
logger.info("Starting feed worker ...");
Expand Down
4 changes: 3 additions & 1 deletion apps/workers/index.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import "dotenv/config";

import { FeedWorker } from "feedWorker";
import { FeedRefreshingWorker, FeedWorker } from "feedWorker";
import { TidyAssetsWorker } from "tidyAssetsWorker";

import serverConfig from "@hoarder/shared/config";
Expand All @@ -25,6 +25,7 @@ async function main() {
VideoWorker.build(),
FeedWorker.build(),
];
FeedRefreshingWorker.start();

await Promise.any([
Promise.all([
Expand All @@ -41,6 +42,7 @@ async function main() {
"Shutting down crawler, openai, tidyAssets, video, feed and search workers ...",
);

FeedRefreshingWorker.stop();
crawler.stop();
openai.stop();
search.stop();
Expand Down
6 changes: 4 additions & 2 deletions apps/workers/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
"drizzle-orm": "^0.33.0",
"execa": "9.3.1",
"jsdom": "^24.0.0",
"liteque": "^0.1.3",
"liteque": "^0.2.0",
"metascraper": "^5.45.24",
"metascraper-amazon": "^5.45.22",
"metascraper-description": "^5.45.22",
Expand All @@ -27,6 +27,7 @@
"metascraper-title": "^5.45.22",
"metascraper-twitter": "^5.45.6",
"metascraper-url": "^5.45.22",
"node-cron": "^3.0.3",
"pdf2json": "^3.0.5",
"pdfjs-dist": "^4.0.379",
"puppeteer": "^22.0.0",
Expand All @@ -44,7 +45,8 @@
"@hoarder/prettier-config": "workspace:^0.1.0",
"@types/dompurify": "^3.0.5",
"@types/jsdom": "^21.1.6",
"@types/metascraper": "^5.14.3"
"@types/metascraper": "^5.14.3",
"@types/node-cron": "^3.0.11"
},
"scripts": {
"start": "tsx watch index.ts",
Expand Down
2 changes: 1 addition & 1 deletion packages/shared/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
"type": "module",
"dependencies": {
"glob": "^11.0.0",
"liteque": "^0.1.3",
"liteque": "^0.2.0",
"meilisearch": "^0.37.0",
"ollama": "^0.5.9",
"openai": "^4.67.1",
Expand Down
35 changes: 28 additions & 7 deletions pnpm-lock.yaml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

0 comments on commit d34b538

Please sign in to comment.