feat(websearch): skip pages that dont return success (#1639)

huggingface · Jan 8, 2025 · f48d079 · f48d079
1 parent c135e93
commit f48d079
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/src/lib/server/websearch/scrape/scrape.ts b/src/lib/server/websearch/scrape/scrape.ts
@@ -35,6 +35,7 @@ export const scrape = (maxCharsPerElem: number) =>
 export async function scrapeUrl(url: string, maxCharsPerElem: number) {
 	return withPage(url, async (page, res) => {
 		if (!res) throw Error("Failed to load page");
+		if (!res.ok()) throw Error(`Failed to load page: ${res.status()}`);
 
 		// Check if it's a non-html content type that we can handle directly
 		// TODO: direct mappings to markdown can be added for markdown, csv and others