From 7c22da3d41000cdc2174d16c2565a934a38962c1 Mon Sep 17 00:00:00 2001 From: Karlostavitch1 <78771208+Karlostavitch1@users.noreply.github.com> Date: Wed, 14 Feb 2024 20:23:58 +1000 Subject: [PATCH] Added support for SearXNG as a websearch source (#805) * Created searchSearxng.ts for querying and parsing searxng instance. Added searxng to the searchWeb.ts to ensure inclusion in the search initialisation script. Added searxng to +layout.server.ts to ensure the search button is enabled if url provided in .env.local assignment Updated .env to include searxng configuration Updated README.md under Web Search Config to mention addition of searxng URL use in .env * Changes requested consolidated env variable to SEARXNG_QUERY_URL (felt considering the content the word QUERY was more relevant) refactored the searchSearxng.ts to generate the query with the new variable updated related references to variable (.env, searchWeb.ts and +layout.server.ts) Updated readme and variable comment on .env to reflect the same * urefectored to display SearXNG in search dialoge when used. * lint * types --------- Co-authored-by: Nathan Sarrazin --- .env | 1 + README.md | 2 +- src/lib/server/websearch/searchSearxng.ts | 34 +++++++++++++++++++++++ src/lib/server/websearch/searchWeb.ts | 13 ++++++++- src/lib/types/WebSearch.ts | 1 + src/routes/+layout.server.ts | 4 ++- 6 files changed, 52 insertions(+), 3 deletions(-) create mode 100644 src/lib/server/websearch/searchSearxng.ts diff --git a/.env b/.env index 82ba0888f78..76d11b73dc7 100644 --- a/.env +++ b/.env @@ -18,6 +18,7 @@ SERPER_API_KEY=#your serper.dev api key here SERPAPI_KEY=#your serpapi key here SERPSTACK_API_KEY=#your serpstack api key here USE_LOCAL_WEBSEARCH=#set to true to parse google results yourself, overrides other API keys +SEARXNG_QUERY_URL=# where '' will be replaced with query keywords see https://docs.searxng.org/dev/search_api.html eg https://searxng.yourdomain.com/search?q=&engines=duckduckgo,google&format=json WEBSEARCH_ALLOWLIST=`[]` # if it's defined, allow websites from only this list. WEBSEARCH_BLOCKLIST=`[]` # if it's defined, block websites from this list. diff --git a/README.md b/README.md index ed1d46a2a11..bd9e2616e84 100644 --- a/README.md +++ b/README.md @@ -166,7 +166,7 @@ PUBLIC_APP_DISCLAIMER= You can enable the web search through an API by adding `YDC_API_KEY` ([docs.you.com](https://docs.you.com)) or `SERPER_API_KEY` ([serper.dev](https://serper.dev/)) or `SERPAPI_KEY` ([serpapi.com](https://serpapi.com/)) or `SERPSTACK_API_KEY` ([serpstack.com](https://serpstack.com/)) to your `.env.local`. -You can also simply enable the local websearch by setting `USE_LOCAL_WEBSEARCH=true` in your `.env.local`. +You can also simply enable the local google websearch by setting `USE_LOCAL_WEBSEARCH=true` in your `.env.local` or specify a SearXNG instance by adding the query URL to `SEARXNG_QUERY_URL`. ### Custom models diff --git a/src/lib/server/websearch/searchSearxng.ts b/src/lib/server/websearch/searchSearxng.ts new file mode 100644 index 00000000000..a432003cb16 --- /dev/null +++ b/src/lib/server/websearch/searchSearxng.ts @@ -0,0 +1,34 @@ +import { SEARXNG_QUERY_URL } from "$env/static/private"; + +export async function searchSearxng(query: string) { + const abortController = new AbortController(); + setTimeout(() => abortController.abort(), 10000); + + // Insert the query into the URL template + let url = SEARXNG_QUERY_URL.replace("", query); + + // Check if "&format=json" already exists in the URL + if (!url.includes("&format=json")) { + url += "&format=json"; + } + + // Call the URL to return JSON data + const jsonResponse = await fetch(url, { + signal: abortController.signal, + }) + .then((response) => response.json() as Promise<{ results: { url: string }[] }>) + .catch((error) => { + console.error("Failed to fetch or parse JSON", error); + throw new Error("Failed to fetch or parse JSON"); + }); + + // Extract 'url' elements from the JSON response and trim to the top 5 URLs + const urls = jsonResponse.results.slice(0, 5).map((item) => item.url); + + if (!urls.length) { + throw new Error(`Response doesn't contain any "url" elements`); + } + + // Map URLs to the correct object shape + return { organic_results: urls.map((link) => ({ link })) }; +} diff --git a/src/lib/server/websearch/searchWeb.ts b/src/lib/server/websearch/searchWeb.ts index 2dd7991dcef..94021e5c014 100644 --- a/src/lib/server/websearch/searchWeb.ts +++ b/src/lib/server/websearch/searchWeb.ts @@ -5,15 +5,23 @@ import { SERPER_API_KEY, SERPSTACK_API_KEY, USE_LOCAL_WEBSEARCH, + SEARXNG_QUERY_URL, YDC_API_KEY, } from "$env/static/private"; import { getJson } from "serpapi"; import type { GoogleParameters } from "serpapi"; import { searchWebLocal } from "./searchWebLocal"; +import { searchSearxng } from "./searchSearxng"; // get which SERP api is providing web results export function getWebSearchProvider() { - return YDC_API_KEY ? WebSearchProvider.YOU : WebSearchProvider.GOOGLE; + if (YDC_API_KEY) { + return WebSearchProvider.YOU; + } else if (SEARXNG_QUERY_URL) { + return WebSearchProvider.SEARXNG; + } else { + return WebSearchProvider.GOOGLE; + } } // Show result as JSON @@ -21,6 +29,9 @@ export async function searchWeb(query: string) { if (USE_LOCAL_WEBSEARCH) { return await searchWebLocal(query); } + if (SEARXNG_QUERY_URL) { + return await searchSearxng(query); + } if (SERPER_API_KEY) { return await searchWebSerper(query); } diff --git a/src/lib/types/WebSearch.ts b/src/lib/types/WebSearch.ts index ad4ac744144..e4ea514fcda 100644 --- a/src/lib/types/WebSearch.ts +++ b/src/lib/types/WebSearch.ts @@ -42,4 +42,5 @@ interface YouSearchHit { export enum WebSearchProvider { GOOGLE = "Google", YOU = "You.com", + SEARXNG = "SearXNG", } diff --git a/src/routes/+layout.server.ts b/src/routes/+layout.server.ts index fcf4069d50a..ba55c921f26 100644 --- a/src/routes/+layout.server.ts +++ b/src/routes/+layout.server.ts @@ -12,6 +12,7 @@ import { MESSAGES_BEFORE_LOGIN, YDC_API_KEY, USE_LOCAL_WEBSEARCH, + SEARXNG_QUERY_URL, ENABLE_ASSISTANTS, } from "$env/static/private"; import { ObjectId } from "mongodb"; @@ -126,7 +127,8 @@ export const load: LayoutServerLoad = async ({ locals, depends }) => { SERPER_API_KEY || SERPSTACK_API_KEY || YDC_API_KEY || - USE_LOCAL_WEBSEARCH + USE_LOCAL_WEBSEARCH || + SEARXNG_QUERY_URL ), ethicsModalAccepted: !!settings?.ethicsModalAcceptedAt, ethicsModalAcceptedAt: settings?.ethicsModalAcceptedAt ?? null,