Add websearch controls for assistants (#812)

* remove query modifiers from generateQuery

* Add backend for assistant RAG

* Add front-end for updating RAG assistant

* enable web parser to return plaintext directly for matching headers

* Update websearch flow for handling assistant rag preferences

* Add our old blocklist to .env.template

* Enable websearch to run on messages depending on assistant requirements

* reorganized imports

* Rename vars

* use projection

* Add environment variable for assistant rag

* fix assistant rag on runwebsearch

* fix styling if rag is disabled

* make sure we always omit credentials when fetching web pages

* Add new checks for SSRF, with a new env var `ENABLE_LOCAL_FETCH`

* Use DNS to check if the links are local or not

* Add a websearch indicator

* Add more tags to parser

* Add indicators

* Display RAG options in settings view

* ui

* fix rag detection

* bit more spacing

* fix button position in assistant form

* wording (mainly)

* reduce number of tags

* Bump max URLs from 3 to 10

* add ul and ol to parseWeb

* change splitting string

* link style

* wording

* add feedback link

* Update src/routes/settings/(nav)/assistants/[assistantId]/+page.svelte

Co-authored-by: Mishig <mishig.davaadorj@coloradocollege.edu>

* Update src/routes/settings/(nav)/assistants/[assistantId]/+page.svelte

Co-authored-by: Mishig <mishig.davaadorj@coloradocollege.edu>

* Update src/routes/assistants/+page.svelte

Co-authored-by: Mishig <mishig.davaadorj@coloradocollege.edu>

* Update src/routes/settings/(nav)/assistants/[assistantId]/+page.svelte

Co-authored-by: Mishig <mishig.davaadorj@coloradocollege.edu>

* Update src/lib/components/chat/ChatWindow.svelte

Co-authored-by: Mishig <mishig.davaadorj@coloradocollege.edu>

* Update src/routes/settings/(nav)/assistants/[assistantId]/+page.svelte

Co-authored-by: Mishig <mishig.davaadorj@coloradocollege.edu>

* Update src/lib/components/AssistantSettings.svelte

Co-authored-by: Mishig <mishig.davaadorj@coloradocollege.edu>

* lint

* throw error if not a string

* simplify rag check

---------

Co-authored-by: Mishig <mishig.davaadorj@coloradocollege.edu>
Co-authored-by: Victor Mustar <victor.mustar@gmail.com>

.env

@@ -135,7 +135,8 @@ EXPOSE_API=true
 # PUBLIC_APP_DISCLAIMER=1
 
 ENABLE_ASSISTANTS=false #set to true to enable assistants feature
-
+ENABLE_ASSISTANTS_RAG=false # /!\ This will let users specify arbitrary URLs that the server will then request. Make sure you have the proper firewall rules in place. 
+ENABLE_LOCAL_FETCH=false #set to true to disable the blocklist for local fetches. Only enable this if you have the proper firewall rules to prevent SSRF attacks and understand the implications.
 ALTERNATIVE_REDIRECT_URLS=`[]` #valide alternative redirect URL for OAuth
 
 WEBHOOK_URL_REPORT_ASSISTANT=#provide webhook url to get notified when an assistant gets reported

.env.template

@@ -277,8 +277,11 @@ PUBLIC_PLAUSIBLE_SCRIPT_URL="/js/script.js"
 # XFF_DEPTH=2
 
 ENABLE_ASSISTANTS=true
+ENABLE_ASSISTANTS_RAG=true
 EXPOSE_API=true
 
 ALTERNATIVE_REDIRECT_URLS=`[
   huggingchat://login/callback
 ]`
+
+WEBSEARCH_BLOCKLIST=`["youtube.com", "twitter.com"]`

package-lock.json

@@ -20,6 +20,7 @@
 				"handlebars": "^4.7.8",
 				"highlight.js": "^11.7.0",
 				"image-size": "^1.0.2",
+				"ip-address": "^9.0.5",
 				"jsdom": "^22.0.0",
 				"json5": "^2.2.3",
 				"marked": "^4.3.0",

package.json

@@ -62,6 +62,7 @@
 		"handlebars": "^4.7.8",
 		"highlight.js": "^11.7.0",
 		"image-size": "^1.0.2",
+		"ip-address": "^9.0.5",
 		"jsdom": "^22.0.0",
 		"json5": "^2.2.3",
 		"marked": "^4.3.0",

src/lib/components/AssistantSettings.svelte

@@ -5,11 +5,13 @@
 
 	import { onMount } from "svelte";
 	import { applyAction, enhance } from "$app/forms";
+	import { page } from "$app/stores";
 	import { base } from "$app/paths";
 	import CarbonPen from "~icons/carbon/pen";
 	import CarbonUpload from "~icons/carbon/upload";
 
 	import { useSettingsStore } from "$lib/stores/settings";
+	import { isHuggingChat } from "$lib/utils/isHuggingChat";
 
 	type ActionData = {
 		error: boolean;
@@ -71,6 +73,14 @@
 	let deleteExistingAvatar = false;
 
 	let loading = false;
+
+	let ragMode: false | "links" | "domains" | "all" = assistant?.rag?.allowAllDomains
+		? "all"
+		: assistant?.rag?.allowedLinks?.length ?? 0 > 0
+		? "links"
+		: (assistant?.rag?.allowedDomains?.length ?? 0) > 0
+		? "domains"
+		: false;
 </script>
 
 <form
@@ -103,6 +113,24 @@
 			}
 		}
 
+		formData.delete("ragMode");
+
+		if (ragMode === false || !$page.data.enableAssistantsRAG) {
+			formData.set("ragAllowAll", "false");
+			formData.set("ragLinkList", "");
+			formData.set("ragDomainList", "");
+		} else if (ragMode === "all") {
+			formData.set("ragAllowAll", "true");
+			formData.set("ragLinkList", "");
+			formData.set("ragDomainList", "");
+		} else if (ragMode === "links") {
+			formData.set("ragAllowAll", "false");
+			formData.set("ragDomainList", "");
+		} else if (ragMode === "domains") {
+			formData.set("ragAllowAll", "false");
+			formData.set("ragLinkList", "");
+		}
+
 		return async ({ result }) => {
 			loading = false;
 			await applyAction(result);
@@ -126,7 +154,7 @@
 	{/if}
 
 	<div class="grid h-full w-full flex-1 grid-cols-2 gap-6 text-sm max-sm:grid-cols-1">
-		<div class="flex flex-col gap-4">
+		<div class="col-span-1 flex flex-col gap-4">
 			<div>
 				<div class="mb-1 block pb-2 text-sm font-semibold">Avatar</div>
 				<input
@@ -255,21 +283,126 @@
 				</div>
 				<p class="text-xs text-red-500">{getError("inputMessage1", form)}</p>
 			</label>
+			{#if $page.data.enableAssistantsRAG}
+				<div class="mb-4 flex flex-col flex-nowrap">
+					<span class="mt-2 text-smd font-semibold"
+						>Internet access <span
+							class="ml-1 rounded bg-gray-100 px-1 py-0.5 text-xxs font-normal text-gray-600"
+							>Experimental</span
+						>
+
+						{#if isHuggingChat}
+							<a
+								href="https://huggingface.co/spaces/huggingchat/chat-ui/discussions/385"
+								target="_blank"
+								class="ml-0.5 rounded bg-gray-100 px-1 py-0.5 text-xxs font-normal text-gray-700 underline decoration-gray-400"
+								>Give feedback</a
+							>
+						{/if}
+					</span>
+
+					<label class="mt-1">
+						<input
+							checked={!ragMode}
+							on:change={() => (ragMode = false)}
+							type="radio"
+							name="ragMode"
+							value={false}
+						/>
+						<span class="my-2 text-sm" class:font-semibold={!ragMode}> Disabled </span>
+						{#if !ragMode}
+							<span class="block text-xs text-gray-500">
+								Assistant won't look for information from Internet and will be faster to answer.
+							</span>
+						{/if}
+					</label>
+
+					<label class="mt-1">
+						<input
+							checked={ragMode === "all"}
+							on:change={() => (ragMode = "all")}
+							type="radio"
+							name="ragMode"
+							value={"all"}
+						/>
+						<span class="my-2 text-sm" class:font-semibold={ragMode === "all"}> Enabled </span>
+						{#if ragMode === "all"}
+							<span class="block text-xs text-gray-500">
+								Assistant will do a web search on each user request to find information.
+							</span>
+						{/if}
+					</label>
+
+					<label class="mt-1">
+						<input
+							checked={ragMode === "domains"}
+							on:change={() => (ragMode = "domains")}
+							type="radio"
+							name="ragMode"
+							value={false}
+						/>
+						<span class="my-2 text-sm" class:font-semibold={ragMode === "domains"}>
+							Domains search
+						</span>
+					</label>
+					{#if ragMode === "domains"}
+						<span class="mb-2 text-xs text-gray-500">
+							Specify domains and URLs that the application can search, separated by commas.
+						</span>
+
+						<input
+							name="ragDomainList"
+							class="w-full rounded-lg border-2 border-gray-200 bg-gray-100 p-2"
+							placeholder="wikipedia.org,bbc.com"
+							value={assistant?.rag?.allowedDomains?.join(",") ?? ""}
+						/>
+						<p class="text-xs text-red-500">{getError("ragDomainList", form)}</p>
+					{/if}
+
+					<label class="mt-1">
+						<input
+							checked={ragMode === "links"}
+							on:change={() => (ragMode = "links")}
+							type="radio"
+							name="ragMode"
+							value={false}
+						/>
+						<span class="my-2 text-sm" class:font-semibold={ragMode === "links"}>
+							Specific Links
+						</span>
+					</label>
+					{#if ragMode === "links"}
+						<span class="mb-2 text-xs text-gray-500">
+							Specify a maximum of 10 direct URLs that the Assistant will access. HTML & Plain Text
+							only, separated by commas
+						</span>
+						<input
+							name="ragLinkList"
+							class="w-full rounded-lg border-2 border-gray-200 bg-gray-100 p-2"
+							placeholder="https://raw.githubusercontent.com/huggingface/chat-ui/main/README.md"
+							value={assistant?.rag?.allowedLinks.join(",") ?? ""}
+						/>
+						<p class="text-xs text-red-500">{getError("ragLinkList", form)}</p>
+					{/if}
+				</div>
+			{/if}
 		</div>
 
-		<label class="flex flex-col">
-			<div class="mb-1 text-sm font-semibold">Instructions (system prompt)</div>
+		<div class="col-span-1 flex h-full flex-col">
+			<span class="mb-1 text-sm font-semibold"> Instructions (system prompt) </span>
 			<textarea
 				name="preprompt"
-				class="min-h-[8lh] flex-1 rounded-lg border-2 border-gray-200 bg-gray-100 p-2 text-sm"
+				class="mb-20 min-h-[8lh] flex-1 rounded-lg border-2 border-gray-200 bg-gray-100 p-2 text-sm"
 				placeholder="You'll act as..."
 				value={assistant?.preprompt ?? ""}
 			/>
 			<p class="text-xs text-red-500">{getError("preprompt", form)}</p>
-		</label>
+		</div>
 	</div>
 
-	<div class="mt-6 flex justify-end gap-2">
+	<div
+		class="ml-auto mt-6 flex w-fit justify-end gap-2 max-sm:fixed max-sm:bottom-6 max-sm:right-6"
+	>
 		<a
 			href={assistant ? `${base}/settings/assistants/${assistant?._id}` : `${base}/settings`}
 			class="flex items-center justify-center rounded-full bg-gray-200 px-5 py-2 font-semibold text-gray-600"

src/lib/components/chat/AssistantIntroduction.svelte

@@ -3,13 +3,26 @@
 	import IconGear from "~icons/bi/gear-fill";
 	import { base } from "$app/paths";
 	import type { Assistant } from "$lib/types/Assistant";
+	import IconInternet from "../icons/IconInternet.svelte";
 
 	export let assistant: Pick<
 		Assistant,
-		"avatar" | "name" | "modelId" | "createdByName" | "exampleInputs" | "_id" | "description"
+		| "avatar"
+		| "name"
+		| "rag"
+		| "modelId"
+		| "createdByName"
+		| "exampleInputs"
+		| "_id"
+		| "description"
 	>;
 
 	const dispatch = createEventDispatcher<{ message: string }>();
+
+	$: hasRag =
+		assistant?.rag?.allowAllDomains ||
+		(assistant?.rag?.allowedDomains?.length ?? 0) > 0 ||
+		(assistant?.rag?.allowedLinks?.length ?? 0) > 0;
 </script>
 
 <div class="flex h-full w-full flex-col content-center items-center justify-center pb-52">
@@ -17,7 +30,7 @@
 		class="relative mt-auto rounded-2xl bg-gray-100 text-gray-600 dark:border-gray-800 dark:bg-gray-800/60 dark:text-gray-300"
 	>
 		<div
-			class="flex min-w-[80dvw] items-center gap-4 p-4 pr-1 sm:min-w-[440px] md:p-8 md:pt-10 xl:gap-8"
+			class="mt-3 flex min-w-[80dvw] items-center gap-4 p-4 pr-1 sm:min-w-[440px] md:p-8 md:pt-10 xl:gap-8"
 		>
 			{#if assistant.avatar}
 				<img
@@ -39,9 +52,21 @@
 				<p class="-mb-1">Assistant</p>
 
 				<p class="text-xl font-bold sm:text-2xl">{assistant.name}</p>
-				<p class="line-clamp-6 text-sm text-gray-500 dark:text-gray-400">
-					{assistant.description}
-				</p>
+				{#if assistant.description}
+					<p class="line-clamp-6 text-sm text-gray-500 dark:text-gray-400">
+						{assistant.description}
+					</p>
+				{/if}
+
+				{#if hasRag}
+					<div
+						class="flex h-5 w-fit items-center gap-1 rounded-full bg-blue-500/10 pl-1 pr-2 text-xs"
+						title="This assistant uses the websearch."
+					>
+						<IconInternet classNames="text-sm text-blue-600" />
+						Has internet access
+					</div>
+				{/if}
 
 				{#if assistant.createdByName}
 					<p class="pt-2 text-sm text-gray-400 dark:text-gray-500">
@@ -55,6 +80,7 @@
 				{/if}
 			</div>
 		</div>
+
 		<div class="absolute right-3 top-3 md:right-4 md:top-4">
 			<a
 				href="{base}/settings/assistants/{assistant._id.toString()}"

src/lib/components/chat/ChatWindow.svelte

@@ -138,7 +138,7 @@
 		bind:this={chatContainer}
 	>
 		<div class="mx-auto flex h-full max-w-3xl flex-col gap-6 px-5 pt-6 sm:gap-8 xl:max-w-4xl">
-			{#if $page.data?.assistant}
+			{#if $page.data?.assistant && !!messages.length}
 				<a
 					class="mx-auto flex items-center gap-1.5 rounded-full border border-gray-100 bg-gray-50 py-1 pl-1 pr-3 text-sm text-gray-800 hover:bg-gray-100 dark:border-gray-800 dark:bg-gray-800 dark:text-gray-200 dark:hover:bg-gray-700"
 					href="{base}/settings/assistants/{$page.data.assistant._id}"

src/lib/components/icons/IconInternet.svelte

@@ -10,6 +10,7 @@
 	role="img"
 	width="1em"
 	height="1em"
+	fill="currentColor"
 	preserveAspectRatio="xMidYMid meet"
 	viewBox="0 0 20 20"
 >

src/lib/server/isURLLocal.spec.ts

@@ -0,0 +1,31 @@
+import { isURLLocal } from "./isURLLocal";
+import { describe, expect, it } from "vitest";
+
+describe("isURLLocal", async () => {
+	it("should return true for localhost", async () => {
+		expect(await isURLLocal(new URL("http://localhost"))).toBe(true);
+	});
+	it("should return true for 127.0.0.1", async () => {
+		expect(await isURLLocal(new URL("http://127.0.0.1"))).toBe(true);
+	});
+	it("should return true for 127.254.254.254", async () => {
+		expect(await isURLLocal(new URL("http://127.254.254.254"))).toBe(true);
+	});
+	it("should return false for huggingface.co", async () => {
+		expect(await isURLLocal(new URL("https://huggingface.co/"))).toBe(false);
+	});
+	it("should return true for 127.0.0.1.nip.io", async () => {
+		expect(await isURLLocal(new URL("http://127.0.0.1.nip.io"))).toBe(true);
+	});
+	it("should fail on ipv6", async () => {
+		await expect(isURLLocal(new URL("http://[::1]"))).rejects.toThrow();
+	});
+	it("should fail on ipv6 --1.sslip.io", async () => {
+		await expect(isURLLocal(new URL("http://--1.sslip.io"))).rejects.toThrow();
+	});
+	it("should fail on invalid domain names", async () => {
+		await expect(
+			isURLLocal(new URL("http://34329487239847329874923948732984.com/"))
+		).rejects.toThrow();
+	});
+});

src/lib/server/isURLLocal.ts

@@ -0,0 +1,26 @@
+import { Address6, Address4 } from "ip-address";
+
+import dns from "node:dns";
+
+export async function isURLLocal(URL: URL): Promise<boolean> {
+	const isLocal = new Promise<boolean>((resolve, reject) => {
+		dns.lookup(URL.hostname, (err, address, family) => {
+			if (err) {
+				reject(err);
+			}
+			if (family === 4) {
+				const addr = new Address4(address);
+				resolve(addr.isInSubnet(new Address4("127.0.0.0/8")));
+			} else if (family === 6) {
+				const addr = new Address6(address);
+				resolve(
+					addr.isLoopback() || addr.isInSubnet(new Address6("::1/128")) || addr.isLinkLocal()
+				);
+			} else {
+				reject(new Error("Unknown IP family"));
+			}
+		});
+	});
+
+	return isLocal;
+}

src/lib/server/websearch/generateQuery.ts

@@ -1,19 +1,6 @@
 import type { Message } from "$lib/types/Message";
 import { format } from "date-fns";
 import { generateFromDefaultEndpoint } from "../generateFromDefaultEndpoint";
-import { WEBSEARCH_ALLOWLIST, WEBSEARCH_BLOCKLIST } from "$env/static/private";
-import { z } from "zod";
-import JSON5 from "json5";
-
-const listSchema = z.array(z.string()).default([]);
-
-const allowList = listSchema.parse(JSON5.parse(WEBSEARCH_ALLOWLIST));
-const blockList = listSchema.parse(JSON5.parse(WEBSEARCH_BLOCKLIST));
-
-const queryModifier = [
-	...allowList.map((item) => "site:" + item),
-	...blockList.map((item) => "-site:" + item),
-].join(" ");
 
 export async function generateQuery(messages: Message[]) {
 	const currentDate = format(new Date(), "MMMM d, yyyy");
@@ -79,5 +66,5 @@ Current Question: Where is it being hosted?`,
 		preprompt: `You are tasked with generating web search queries. Give me an appropriate query to answer my question for google search. Answer with only the query. Today is ${currentDate}`,
 	});
 
-	return (queryModifier + " " + webQuery).trim();
+	return webQuery.trim();
 }

src/lib/server/websearch/parseWeb.ts

@@ -3,30 +3,37 @@ import { JSDOM, VirtualConsole } from "jsdom";
 export async function parseWeb(url: string) {
 	const abortController = new AbortController();
 	setTimeout(() => abortController.abort(), 10000);
-	const htmlString = await fetch(url, { signal: abortController.signal })
-		.then((response) => response.text())
-		.catch();
+	const r = await fetch(url, { signal: abortController.signal, credentials: "omit" }).catch();
 
-	const virtualConsole = new VirtualConsole();
-	virtualConsole.on("error", () => {
-		// No-op to skip console errors.
-	});
+	if (r.headers.get("content-type")?.includes("text/html")) {
+		const virtualConsole = new VirtualConsole();
+		virtualConsole.on("error", () => {
+			// No-op to skip console errors.
+		});
 
-	// put the html string into a DOM
-	const dom = new JSDOM(htmlString ?? "", {
-		virtualConsole,
-	});
+		// put the html string into a DOM
+		const dom = new JSDOM((await r.text()) ?? "", {
+			virtualConsole,
+		});
 
-	const { document } = dom.window;
-	const textElTags = "p";
-	const paragraphs = document.querySelectorAll(textElTags);
-	if (!paragraphs.length) {
-		throw new Error(`webpage doesn't have any "${textElTags}" element`);
-	}
-	const paragraphTexts = Array.from(paragraphs).map((p) => p.textContent);
+		const { document } = dom.window;
+		const paragraphs = document.querySelectorAll("p, table, pre, ul, ol");
+
+		if (!paragraphs.length) {
+			throw new Error(`webpage doesn't have any parseable element`);
+		}
+		const paragraphTexts = Array.from(paragraphs).map((p) => p.textContent);
 
-	// combine text contents from paragraphs and then remove newlines and multiple spaces
-	const text = paragraphTexts.join(" ").replace(/ {2}|\r\n|\n|\r/gm, "");
+		// combine text contents from paragraphs and then remove newlines and multiple spaces
+		const text = paragraphTexts.join(" ").replace(/ {2}|\r\n|\n|\r/gm, "");
 
-	return text;
+		return text;
+	} else if (
+		r.headers.get("content-type")?.includes("text/plain") ||
+		r.headers.get("content-type")?.includes("text/markdown")
+	) {
+		return r.text();
+	} else {
+		throw new Error("Unsupported content type");
+	}
 }

src/lib/server/websearch/runWebSearch.ts

@@ -1,24 +1,35 @@
 import { searchWeb } from "$lib/server/websearch/searchWeb";
-import type { Message } from "$lib/types/Message";
-import type { WebSearch, WebSearchSource } from "$lib/types/WebSearch";
 import { generateQuery } from "$lib/server/websearch/generateQuery";
 import { parseWeb } from "$lib/server/websearch/parseWeb";
 import { chunk } from "$lib/utils/chunk";
 import { findSimilarSentences } from "$lib/server/sentenceSimilarity";
-import type { Conversation } from "$lib/types/Conversation";
-import type { MessageUpdate } from "$lib/types/MessageUpdate";
 import { getWebSearchProvider } from "./searchWeb";
 import { defaultEmbeddingModel, embeddingModels } from "$lib/server/embeddingModels";
+import { WEBSEARCH_ALLOWLIST, WEBSEARCH_BLOCKLIST, ENABLE_LOCAL_FETCH } from "$env/static/private";
+
+import type { Conversation } from "$lib/types/Conversation";
+import type { MessageUpdate } from "$lib/types/MessageUpdate";
+import type { Message } from "$lib/types/Message";
+import type { WebSearch, WebSearchSource } from "$lib/types/WebSearch";
+import type { Assistant } from "$lib/types/Assistant";
+
+import { z } from "zod";
+import JSON5 from "json5";
+import { isURLLocal } from "../isURLLocal";
 
 const MAX_N_PAGES_SCRAPE = 10 as const;
 const MAX_N_PAGES_EMBED = 5 as const;
 
-const DOMAIN_BLOCKLIST = ["youtube.com", "twitter.com"];
+const listSchema = z.array(z.string()).default([]);
+
+const allowList = listSchema.parse(JSON5.parse(WEBSEARCH_ALLOWLIST));
+const blockList = listSchema.parse(JSON5.parse(WEBSEARCH_BLOCKLIST));
 
 export async function runWebSearch(
 	conv: Conversation,
 	messages: Message[],
-	updatePad: (upd: MessageUpdate) => void
+	updatePad: (upd: MessageUpdate) => void,
+	ragSettings?: Assistant["rag"]
 ) {
 	const prompt = messages[messages.length - 1].content;
 	const webSearch: WebSearch = {
@@ -36,26 +47,66 @@ export async function runWebSearch(
 	}
 
 	try {
-		webSearch.searchQuery = await generateQuery(messages);
-		const searchProvider = getWebSearchProvider();
-		appendUpdate(`Searching ${searchProvider}`, [webSearch.searchQuery]);
-		const results = await searchWeb(webSearch.searchQuery);
-		webSearch.results =
-			(results.organic_results &&
-				results.organic_results.map((el: { title?: string; link: string; text?: string }) => {
-					try {
-						const { title, link, text } = el;
-						const { hostname } = new URL(link);
-						return { title, link, hostname, text };
-					} catch (e) {
-						// Ignore Errors
-						return null;
-					}
-				})) ??
-			[];
+		// if the assistant specified direct links, skip the websearch
+		if (ragSettings && ragSettings?.allowedLinks.length > 0) {
+			appendUpdate("Using links specified in Assistant");
+
+			let linksToUse = [...ragSettings.allowedLinks];
+
+			if (ENABLE_LOCAL_FETCH !== "true") {
+				const localLinks = await Promise.all(
+					linksToUse.map(async (link) => {
+						try {
+							const url = new URL(link);
+							return await isURLLocal(url);
+						} catch (e) {
+							return true;
+						}
+					})
+				);
+
+				linksToUse = linksToUse.filter((_, index) => !localLinks[index]);
+			}
+
+			webSearch.results = linksToUse.map((link) => {
+				return { link, hostname: new URL(link).hostname, title: "", text: "" };
+			});
+		} else {
+			webSearch.searchQuery = await generateQuery(messages);
+			const searchProvider = getWebSearchProvider();
+			appendUpdate(`Searching ${searchProvider}`, [webSearch.searchQuery]);
+
+			if (ragSettings && ragSettings?.allowedDomains.length > 0) {
+				appendUpdate("Filtering on specified domains");
+				webSearch.searchQuery +=
+					" " + ragSettings.allowedDomains.map((item) => "site:" + item).join(" ");
+			}
+
+			// handle the global lists
+			webSearch.searchQuery +=
+				allowList.map((item) => "site:" + item).join(" ") +
+				" " +
+				blockList.map((item) => "-site:" + item).join(" ");
+
+			const results = await searchWeb(webSearch.searchQuery);
+			webSearch.results =
+				(results.organic_results &&
+					results.organic_results.map((el: { title?: string; link: string; text?: string }) => {
+						try {
+							const { title, link, text } = el;
+							const { hostname } = new URL(link);
+							return { title, link, hostname, text };
+						} catch (e) {
+							// Ignore Errors
+							return null;
+						}
+					})) ??
+				[];
+		}
+
 		webSearch.results = webSearch.results.filter((value) => value !== null);
 		webSearch.results = webSearch.results
-			.filter(({ link }) => !DOMAIN_BLOCKLIST.some((el) => link.includes(el))) // filter out blocklist links
+			.filter(({ link }) => !blockList.some((el) => link.includes(el))) // filter out blocklist links
 			.slice(0, MAX_N_PAGES_SCRAPE); // limit to first 10 links only
 
 		// fetch the model

src/lib/types/Assistant.ts

@@ -14,5 +14,10 @@ export interface Assistant extends Timestamps {
 	preprompt: string;
 	userCount?: number;
 	featured?: boolean;
+	rag?: {
+		allowAllDomains: boolean;
+		allowedDomains: string[];
+		allowedLinks: string[];
+	};
 	searchTokens: string[];
 }

src/lib/utils/parseStringToList.ts

@@ -0,0 +1,10 @@
+export function parseStringToList(links: unknown): string[] {
+	if (typeof links !== "string") {
+		throw new Error("Expected a string");
+	}
+
+	return links
+		.split(",")
+		.map((link) => link.trim())
+		.filter((link) => link.length > 0);
+}

src/routes/+layout.server.ts

@@ -14,6 +14,7 @@ import {
 	USE_LOCAL_WEBSEARCH,
 	SEARXNG_QUERY_URL,
 	ENABLE_ASSISTANTS,
+	ENABLE_ASSISTANTS_RAG,
 } from "$env/static/private";
 import { ObjectId } from "mongodb";
 import type { ConvSidebar } from "$lib/types/ConvSidebar";
@@ -186,6 +187,7 @@ export const load: LayoutServerLoad = async ({ locals, depends }) => {
 		},
 		assistant,
 		enableAssistants,
+		enableAssistantsRAG: ENABLE_ASSISTANTS_RAG === "true",
 		loginRequired,
 		loginEnabled: requiresUser,
 		guestMode: requiresUser && messagesBeforeLogin > 0,

src/routes/assistants/+page.svelte

@@ -20,6 +20,7 @@
 	import { getHref } from "$lib/utils/getHref";
 	import { debounce } from "$lib/utils/debounce";
 	import { useSettingsStore } from "$lib/stores/settings";
+	import IconInternet from "$lib/components/icons/IconInternet.svelte";
 	import { isDesktop } from "$lib/utils/isDesktop";
 
 	export let data: PageData;
@@ -201,6 +202,11 @@
 
 		<div class="mt-8 grid grid-cols-2 gap-3 sm:gap-5 md:grid-cols-3 lg:grid-cols-4">
 			{#each data.assistants as assistant (assistant._id)}
+				{@const hasRag =
+					assistant?.rag?.allowAllDomains ||
+					!!assistant?.rag?.allowedDomains?.length ||
+					!!assistant?.rag?.allowedLinks?.length}
+
 				<button
 					class="relative flex flex-col items-center justify-center overflow-hidden text-balance rounded-xl border bg-gray-50/50 px-4 py-6 text-center shadow hover:bg-gray-50 hover:shadow-inner max-sm:px-4 sm:h-64 sm:pb-4 xl:pt-8 dark:border-gray-800/70 dark:bg-gray-950/20 dark:hover:bg-gray-950/40"
 					on:click={() => {
@@ -220,6 +226,16 @@
 							<CarbonUserMultiple class="text-xxs" />{formatUserCount(assistant.userCount)}
 						</div>
 					{/if}
+
+					{#if hasRag}
+						<div
+							class="absolute left-3 top-3 grid size-5 place-items-center rounded-full bg-blue-500/10"
+							title="This assistant uses the websearch."
+						>
+							<IconInternet classNames="text-sm text-blue-600" />
+						</div>
+					{/if}
+
 					{#if assistant.avatar}
 						<img
 							src="{base}/settings/assistants/{assistant._id}/avatar.jpg"

src/routes/conversation/[id]/+server.ts

@@ -1,4 +1,4 @@
-import { MESSAGES_BEFORE_LOGIN } from "$env/static/private";
+import { MESSAGES_BEFORE_LOGIN, ENABLE_ASSISTANTS_RAG } from "$env/static/private";
 import { authCondition, requiresUser } from "$lib/server/auth";
 import { collections } from "$lib/server/database";
 import { models } from "$lib/server/models";
@@ -13,6 +13,7 @@ import { abortedGenerations } from "$lib/server/abortedGenerations";
 import { summarize } from "$lib/server/summarize";
 import { uploadFile } from "$lib/server/files/uploadFile";
 import sizeof from "image-size";
+import type { Assistant } from "$lib/types/Assistant";
 import { convertLegacyConversation } from "$lib/utils/tree/convertLegacyConversation";
 import { isMessageId } from "$lib/utils/tree/isMessageId";
 import { buildSubtree } from "$lib/utils/tree/buildSubtree.js";
@@ -334,9 +335,22 @@ export async function POST({ request, locals, params, getClientAddress }) {
 				}
 			);
 
+			// check if assistant has a rag
+			const rag = (
+				await collections.assistants.findOne<Pick<Assistant, "rag">>(
+					{ _id: conv.assistantId },
+					{ projection: { rag: 1 } }
+				)
+			)?.rag;
+
+			const assistantHasRAG =
+				ENABLE_ASSISTANTS_RAG === "true" &&
+				rag &&
+				(rag.allowedLinks.length > 0 || rag.allowedDomains.length > 0 || rag.allowAllDomains);
+
 			// perform websearch if needed
-			if (webSearch && !isContinue && !conv.assistantId) {
-				messageToWriteTo.webSearch = await runWebSearch(conv, messagesForPrompt, update);
+			if (!isContinue && ((webSearch && !conv.assistantId) || assistantHasRAG)) {
+				messageToWriteTo.webSearch = await runWebSearch(conv, messagesForPrompt, update, rag);
 			}
 
 			// inject websearch result & optionally images into the messages

src/routes/settings/(nav)/+layout.svelte

@@ -32,7 +32,7 @@
 </script>
 
 <div
-	class="grid h-full w-full grid-cols-1 grid-rows-[auto,1fr] content-start gap-x-8 overflow-hidden p-4 md:grid-cols-3 md:grid-rows-[auto,1fr] md:p-8"
+	class="grid h-full w-full grid-cols-1 grid-rows-[auto,1fr] content-start gap-x-4 overflow-hidden p-4 md:grid-cols-3 md:grid-rows-[auto,1fr] md:p-8"
 >
 	<div class="col-span-1 mb-4 flex items-center justify-between md:col-span-3">
 		<h2 class="text-xl font-bold">Settings</h2>

src/routes/settings/(nav)/assistants/[assistantId]/+page.svelte

@@ -13,6 +13,7 @@
 	import CarbonLink from "~icons/carbon/link";
 	import CopyToClipBoardBtn from "$lib/components/CopyToClipBoardBtn.svelte";
 	import ReportModal from "./ReportModal.svelte";
+	import IconInternet from "$lib/components/icons/IconInternet.svelte";
 
 	export let data: PageData;
 
@@ -27,6 +28,11 @@
 	$: shareUrl = `${prefix}/assistant/${assistant?._id}`;
 
 	let displayReportModal = false;
+
+	$: hasRag =
+		assistant?.rag?.allowAllDomains ||
+		!!assistant?.rag?.allowedDomains?.length ||
+		!!assistant?.rag?.allowedLinks?.length;
 </script>
 
 {#if displayReportModal}
@@ -51,10 +57,19 @@
 
 		<div class="flex-1">
 			<div class="mb-1.5">
-				<h1 class="mr-2 inline text-xl font-semibold">
+				<h1 class="mr-1 inline text-xl font-semibold">
 					{assistant?.name}
 				</h1>
-				<span class="rounded-full border px-2 py-0.5 text-sm leading-none text-gray-500"
+
+				{#if hasRag}
+					<span
+						class="inline-grid size-5 place-items-center rounded-full bg-blue-500/10"
+						title="This assistant uses the websearch."
+					>
+						<IconInternet classNames="text-sm text-blue-600" />
+					</span>
+				{/if}
+				<span class="ml-1 rounded-full border px-2 py-0.5 text-sm leading-none text-gray-500"
 					>public</span
 				>
 			</div>
@@ -147,11 +162,48 @@
 		</div>
 	</div>
 
-	<h2 class="mt-4 text-lg font-semibold">System Instructions</h2>
+	<!-- two columns for big screen, single column for small screen -->
+	<div class="mb-12 mt-3">
+		<h2 class="mb-2 font-semibold">System Instructions</h2>
+		<textarea
+			disabled
+			class="box-border h-full min-h-[8lh] w-full rounded-lg border-2 border-gray-200 bg-gray-100 p-2 disabled:cursor-not-allowed"
+			>{assistant?.preprompt}</textarea
+		>
 
-	<textarea
-		disabled
-		class="min-h-[8lh] w-full flex-1 rounded-lg border-2 border-gray-200 bg-gray-100 p-2 disabled:cursor-not-allowed 2xl:min-h-[12lh]"
-		>{assistant?.preprompt}</textarea
-	>
+		{#if hasRag}
+			<div class="mt-4">
+				<h2 class=" font-semibold">Internet Access</h2>
+				{#if assistant?.rag?.allowAllDomains}
+					<p class="text-sm text-gray-500">
+						This Assistant uses Web Search to find information on Internet.
+					</p>
+				{:else if !!assistant?.rag?.allowedDomains && assistant?.rag?.allowedDomains.length}
+					<p class="pb-4 text-sm text-gray-500">
+						This Assistant can use Web Search on the following domains:
+					</p>
+					<ul class="mr-2 flex flex-wrap gap-2.5 text-sm text-gray-800">
+						{#each assistant?.rag?.allowedDomains as domain}
+							<li
+								class="break-all rounded-lg border border-gray-200 bg-gray-100 px-2 py-0.5 leading-tight decoration-gray-400"
+							>
+								<a target="_blank" class="underline" href={domain}>{domain}</a>
+							</li>
+						{/each}
+					</ul>
+				{:else if !!assistant?.rag?.allowedLinks && assistant?.rag?.allowedLinks.length}
+					<p class="pb-4 text-sm text-gray-500">This Assistant can browse the following links:</p>
+					<ul class="mr-2 flex flex-wrap gap-2.5 text-sm text-gray-800">
+						{#each assistant?.rag?.allowedLinks as link}
+							<li
+								class="break-all rounded-lg border border-gray-200 bg-gray-100 px-2 py-0.5 leading-tight decoration-gray-400"
+							>
+								<a target="_blank" class="underline" href={link}>{link}</a>
+							</li>
+						{/each}
+					</ul>
+				{/if}
+			</div>
+		{/if}
+	</div>
 </div>

src/routes/settings/(nav)/assistants/[assistantId]/edit/+page.server.ts

@@ -8,6 +8,7 @@ import { z } from "zod";
 import { sha256 } from "$lib/utils/sha256";
 
 import sharp from "sharp";
+import { parseStringToList } from "$lib/utils/parseStringToList";
 import { generateSearchTokens } from "$lib/utils/searchTokens";
 
 const newAsssistantSchema = z.object({
@@ -20,6 +21,9 @@ const newAsssistantSchema = z.object({
 	exampleInput3: z.string().optional(),
 	exampleInput4: z.string().optional(),
 	avatar: z.union([z.instanceof(File), z.literal("null")]).optional(),
+	ragLinkList: z.preprocess(parseStringToList, z.string().url().array().max(10)),
+	ragDomainList: z.preprocess(parseStringToList, z.string().array()),
+	ragAllowAll: z.preprocess((v) => v === "true", z.boolean()),
 });
 
 const uploadAvatar = async (avatar: File, assistantId: ObjectId): Promise<string> => {
@@ -131,6 +135,11 @@ export const actions: Actions = {
 					exampleInputs,
 					avatar: deleteAvatar ? undefined : hash ?? assistant.avatar,
 					updatedAt: new Date(),
+					rag: {
+						allowedLinks: parse.data.ragLinkList,
+						allowedDomains: parse.data.ragDomainList,
+						allowAllDomains: parse.data.ragAllowAll,
+					},
 					searchTokens: generateSearchTokens(parse.data.name),
 				},
 			}

src/routes/settings/(nav)/assistants/new/+page.server.ts

@@ -7,6 +7,7 @@ import { ObjectId } from "mongodb";
 import { z } from "zod";
 import { sha256 } from "$lib/utils/sha256";
 import sharp from "sharp";
+import { parseStringToList } from "$lib/utils/parseStringToList";
 import { usageLimits } from "$lib/server/usageLimits";
 import { generateSearchTokens } from "$lib/utils/searchTokens";
 
@@ -20,6 +21,9 @@ const newAsssistantSchema = z.object({
 	exampleInput3: z.string().optional(),
 	exampleInput4: z.string().optional(),
 	avatar: z.instanceof(File).optional(),
+	ragLinkList: z.preprocess(parseStringToList, z.string().url().array().max(10)),
+	ragDomainList: z.preprocess(parseStringToList, z.string().array()),
+	ragAllowAll: z.preprocess((v) => v === "true", z.boolean()),
 });
 
 const uploadAvatar = async (avatar: File, assistantId: ObjectId): Promise<string> => {
@@ -113,6 +117,11 @@ export const actions: Actions = {
 			updatedAt: new Date(),
 			userCount: 1,
 			featured: false,
+			rag: {
+				allowedLinks: parse.data.ragLinkList,
+				allowedDomains: parse.data.ragDomainList,
+				allowAllDomains: parse.data.ragAllowAll,
+			},
 			searchTokens: generateSearchTokens(parse.data.name),
 		});