From f48d0792e3c48df84a6371fac8eb48a9deac62b4 Mon Sep 17 00:00:00 2001 From: Nathan Sarrazin Date: Wed, 8 Jan 2025 10:50:40 +0100 Subject: [PATCH] feat(websearch): skip pages that dont return success (#1639) --- src/lib/server/websearch/scrape/scrape.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib/server/websearch/scrape/scrape.ts b/src/lib/server/websearch/scrape/scrape.ts index c7b6929490a..9f12a439fe4 100644 --- a/src/lib/server/websearch/scrape/scrape.ts +++ b/src/lib/server/websearch/scrape/scrape.ts @@ -35,6 +35,7 @@ export const scrape = (maxCharsPerElem: number) => export async function scrapeUrl(url: string, maxCharsPerElem: number) { return withPage(url, async (page, res) => { if (!res) throw Error("Failed to load page"); + if (!res.ok()) throw Error(`Failed to load page: ${res.status()}`); // Check if it's a non-html content type that we can handle directly // TODO: direct mappings to markdown can be added for markdown, csv and others