diff --git a/.env b/.env index 82ba0888f78..76d11b73dc7 100644 --- a/.env +++ b/.env @@ -18,6 +18,7 @@ SERPER_API_KEY=#your serper.dev api key here SERPAPI_KEY=#your serpapi key here SERPSTACK_API_KEY=#your serpstack api key here USE_LOCAL_WEBSEARCH=#set to true to parse google results yourself, overrides other API keys +SEARXNG_QUERY_URL=# where '' will be replaced with query keywords see https://docs.searxng.org/dev/search_api.html eg https://searxng.yourdomain.com/search?q=&engines=duckduckgo,google&format=json WEBSEARCH_ALLOWLIST=`[]` # if it's defined, allow websites from only this list. WEBSEARCH_BLOCKLIST=`[]` # if it's defined, block websites from this list. diff --git a/README.md b/README.md index ed1d46a2a11..bd9e2616e84 100644 --- a/README.md +++ b/README.md @@ -166,7 +166,7 @@ PUBLIC_APP_DISCLAIMER= You can enable the web search through an API by adding `YDC_API_KEY` ([docs.you.com](https://docs.you.com)) or `SERPER_API_KEY` ([serper.dev](https://serper.dev/)) or `SERPAPI_KEY` ([serpapi.com](https://serpapi.com/)) or `SERPSTACK_API_KEY` ([serpstack.com](https://serpstack.com/)) to your `.env.local`. -You can also simply enable the local websearch by setting `USE_LOCAL_WEBSEARCH=true` in your `.env.local`. +You can also simply enable the local google websearch by setting `USE_LOCAL_WEBSEARCH=true` in your `.env.local` or specify a SearXNG instance by adding the query URL to `SEARXNG_QUERY_URL`. ### Custom models diff --git a/src/lib/server/websearch/searchSearxng.ts b/src/lib/server/websearch/searchSearxng.ts new file mode 100644 index 00000000000..a432003cb16 --- /dev/null +++ b/src/lib/server/websearch/searchSearxng.ts @@ -0,0 +1,34 @@ +import { SEARXNG_QUERY_URL } from "$env/static/private"; + +export async function searchSearxng(query: string) { + const abortController = new AbortController(); + setTimeout(() => abortController.abort(), 10000); + + // Insert the query into the URL template + let url = SEARXNG_QUERY_URL.replace("", query); + + // Check if "&format=json" already exists in the URL + if (!url.includes("&format=json")) { + url += "&format=json"; + } + + // Call the URL to return JSON data + const jsonResponse = await fetch(url, { + signal: abortController.signal, + }) + .then((response) => response.json() as Promise<{ results: { url: string }[] }>) + .catch((error) => { + console.error("Failed to fetch or parse JSON", error); + throw new Error("Failed to fetch or parse JSON"); + }); + + // Extract 'url' elements from the JSON response and trim to the top 5 URLs + const urls = jsonResponse.results.slice(0, 5).map((item) => item.url); + + if (!urls.length) { + throw new Error(`Response doesn't contain any "url" elements`); + } + + // Map URLs to the correct object shape + return { organic_results: urls.map((link) => ({ link })) }; +} diff --git a/src/lib/server/websearch/searchWeb.ts b/src/lib/server/websearch/searchWeb.ts index 2dd7991dcef..94021e5c014 100644 --- a/src/lib/server/websearch/searchWeb.ts +++ b/src/lib/server/websearch/searchWeb.ts @@ -5,15 +5,23 @@ import { SERPER_API_KEY, SERPSTACK_API_KEY, USE_LOCAL_WEBSEARCH, + SEARXNG_QUERY_URL, YDC_API_KEY, } from "$env/static/private"; import { getJson } from "serpapi"; import type { GoogleParameters } from "serpapi"; import { searchWebLocal } from "./searchWebLocal"; +import { searchSearxng } from "./searchSearxng"; // get which SERP api is providing web results export function getWebSearchProvider() { - return YDC_API_KEY ? WebSearchProvider.YOU : WebSearchProvider.GOOGLE; + if (YDC_API_KEY) { + return WebSearchProvider.YOU; + } else if (SEARXNG_QUERY_URL) { + return WebSearchProvider.SEARXNG; + } else { + return WebSearchProvider.GOOGLE; + } } // Show result as JSON @@ -21,6 +29,9 @@ export async function searchWeb(query: string) { if (USE_LOCAL_WEBSEARCH) { return await searchWebLocal(query); } + if (SEARXNG_QUERY_URL) { + return await searchSearxng(query); + } if (SERPER_API_KEY) { return await searchWebSerper(query); } diff --git a/src/lib/types/WebSearch.ts b/src/lib/types/WebSearch.ts index ad4ac744144..e4ea514fcda 100644 --- a/src/lib/types/WebSearch.ts +++ b/src/lib/types/WebSearch.ts @@ -42,4 +42,5 @@ interface YouSearchHit { export enum WebSearchProvider { GOOGLE = "Google", YOU = "You.com", + SEARXNG = "SearXNG", } diff --git a/src/routes/+layout.server.ts b/src/routes/+layout.server.ts index fcf4069d50a..ba55c921f26 100644 --- a/src/routes/+layout.server.ts +++ b/src/routes/+layout.server.ts @@ -12,6 +12,7 @@ import { MESSAGES_BEFORE_LOGIN, YDC_API_KEY, USE_LOCAL_WEBSEARCH, + SEARXNG_QUERY_URL, ENABLE_ASSISTANTS, } from "$env/static/private"; import { ObjectId } from "mongodb"; @@ -126,7 +127,8 @@ export const load: LayoutServerLoad = async ({ locals, depends }) => { SERPER_API_KEY || SERPSTACK_API_KEY || YDC_API_KEY || - USE_LOCAL_WEBSEARCH + USE_LOCAL_WEBSEARCH || + SEARXNG_QUERY_URL ), ethicsModalAccepted: !!settings?.ethicsModalAcceptedAt, ethicsModalAcceptedAt: settings?.ethicsModalAcceptedAt ?? null,