docs: write API documentation

selfie-ui/src/app/components/Playground/PlaygroundQuery.tsx

@@ -3,10 +3,25 @@ import { apiBaseUrl } from "@/app/config";
 import useAsyncTask from "@/app/hooks/useAsyncTask";
 import TaskToast from "@/app/components/TaskToast";
 
-
-const fetchDocuments = async (topic: string, limit?: number, minScore?: number, includeSummary?: boolean) => {
-  const params = new URLSearchParams({ topic, ...(limit && { limit: limit.toString() }), ...(minScore && { min_score: minScore.toString() }), ...(includeSummary !== undefined && { include_summary: includeSummary.toString() }) });
-  const url = `${apiBaseUrl}/v1/index_documents/summary?${params.toString()}`;
+const fetchDocuments = async (
+  query: string,
+  limit?: number,
+  minScore?: number,
+  includeSummary?: boolean,
+  relevanceWeight?: number,
+  recencyWeight?: number,
+  importanceWeight?: number
+) => {
+  const params = new URLSearchParams({
+    query,
+    ...(limit && { limit: limit.toString() }),
+    ...(minScore && { min_score: minScore.toString() }),
+    ...(includeSummary !== undefined && { include_summary: includeSummary.toString() }),
+    ...(relevanceWeight && { relevance_weight: relevanceWeight.toString() }),
+    ...(recencyWeight && { recency_weight: recencyWeight.toString() }),
+    ...(importanceWeight && { importance_weight: importanceWeight.toString() }),
+  });
+  const url = `${apiBaseUrl}/v1/documents/search?${params.toString()}`;
 
   try {
     const response = await fetch(url);
@@ -23,10 +38,14 @@ const PlaygroundQuery = () => {
   const [documents, setDocuments] = useState([]);
   const [summary, setSummary] = useState("");
   const [isSummaryLoading, setSummaryLoading] = useState(false);
-  const [score, setScore] = useState(0);
+  const [averageScore, setAverageScore] = useState(0);
+  const [totalResults, setTotalResults] = useState(0);
   const [limit, setLimit] = useState<number | undefined>();
   const [minScore, setMinScore] = useState<number | undefined>();
   const [includeSummary, setIncludeSummary] = useState(true);
+  const [relevanceWeight, setRelevanceWeight] = useState<number | undefined>();
+  const [recencyWeight, setRecencyWeight] = useState<number | undefined>();
+  const [importanceWeight, setImportanceWeight] = useState<number | undefined>();
 
   const handleInputChange = (setter: React.Dispatch<React.SetStateAction<any>>) => (e: React.ChangeEvent<HTMLInputElement>) => {
     const value = e.target.type === "number" ? Number(e.target.value) || undefined : e.target.value;
@@ -41,16 +60,26 @@ const PlaygroundQuery = () => {
     e.preventDefault();
 
     executeTask(async () => {
-      setScore(0);
+      setAverageScore(0);
+      setTotalResults(0);
       setDocuments([]);
       setSummary("");
       setSummaryLoading(true);
-      const results = await fetchDocuments(query, limit, minScore, includeSummary);
-      setScore(results.score);
+      const results = await fetchDocuments(
+        query,
+        limit,
+        minScore,
+        includeSummary,
+        relevanceWeight,
+        recencyWeight,
+        importanceWeight
+      );
+      setAverageScore(results.average_score);
+      setTotalResults(results.total_results);
       setDocuments(results.documents);
       setSummary(results.summary);
       setSummaryLoading(false);
-      console.log("Searching with:", query, limit, minScore, includeSummary);
+      console.log("Searching with:", query, limit, minScore, includeSummary, relevanceWeight, recencyWeight, importanceWeight);
     }, {
       start: "Searching...",
       success: "Search complete",
@@ -62,12 +91,9 @@ const PlaygroundQuery = () => {
     return (
       <div key={i} className="card prose prose-sm bordered mb-4 bg-base-200 w-full max-w-full">
         <div className="card-body">
-          {/*<h2 className="card-title">Document {doc.id}</h2>*/}
-          <h2 className="card-title m-0">Embedding document {i}</h2>
+          <h2 className="card-title m-0">Embedding Document {doc.id}</h2>
           <pre className="m-0">{doc.text}</pre>
           <ul className="m-0">
-            {/*<li>Score: {doc.score}</li>*/}
-            {/* only 2 decimal */}
             <li>Overall score: {doc.score.toFixed(2)}</li>
             <li>Relevance score: {doc.relevance.toFixed(2)}</li>
             <li>Recency score: {doc.recency.toFixed(2)}</li>
@@ -114,7 +140,39 @@ const PlaygroundQuery = () => {
                 onChange={(e) => setMinScore(Number(e.target.value) || undefined)}
                 min="0"
                 max="1"
-                step="0.1"
+                step="0.01"
+              />
+            </div>
+
+            <div className="form-control mb-2">
+              {/*<label className="label">*/}
+              {/*  <span className="label-text">Relevance Weight</span>*/}
+              {/*</label>*/}
+              <input
+                type="number"
+                className="input input-sm input-bordered"
+                value={relevanceWeight === undefined ? "" : relevanceWeight}
+                placeholder="Relevance weight (optional)"
+                onChange={(e) => setRelevanceWeight(e.target.value ? Number(e.target.value) : undefined)}
+                min="0"
+                max="1"
+                step="0.01"
+              />
+            </div>
+
+            <div className="form-control mb-2">
+              {/*<label className="label">*/}
+              {/*  <span className="label-text">Recency Weight</span>*/}
+              {/*</label>*/}
+              <input
+                type="number"
+                className="input input-sm input-bordered"
+                value={recencyWeight === undefined ? "" : recencyWeight}
+                placeholder="Recency weight (optional)"
+                onChange={(e) => setRecencyWeight(e.target.value ? Number(e.target.value) : undefined)}
+                min="0"
+                max="1"
+                step="0.01"
               />
             </div>
 
@@ -145,34 +203,62 @@ const PlaygroundQuery = () => {
             </label>
           </form>
         </div>
-        {!!summary && <div className="lg:w-1/2 mb-4">
-          {/*<Tooltip tip="Search for anything" />*/}
-          <p>{summary}</p>
-          {documents.length ? <p className="mt-4">Result Score: {score.toFixed(2)}</p> : null }
-        </div>}
+        {!!summary && (
+          <div className="lg:w-1/2 mb-4">
+            {/*<Tooltip tip="Search for anything" />*/}
+            <p>{summary}</p>
+            {documents.length ? (
+              <div className="mt-4">
+                <p>Total Results: {totalResults}</p>
+                <p>Average Score: {averageScore.toFixed(2)}</p>
+              </div>
+            ) : null}
+          </div>
+        )}
       </div>
-      {!!score && <div>
-        {documents.map(renderDocument)}
-      </div>}
+      {documents.length > 0 && <div>{documents.map(renderDocument)}</div>}
     </div>
   );
 };
 
-const SearchIcon = () => <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16" fill="currentColor"
-                              className="w-4 h-4 opacity-70">
-  <path fillRule="evenodd"
-        d="M9.965 11.026a5 5 0 1 1 1.06-1.06l2.755 2.754a.75.75 0 1 1-1.06 1.06l-2.755-2.754ZM10.5 7a3.5 3.5 0 1 1-7 0 3.5 3.5 0 0 1 7 0Z"
-        clipRule="evenodd" />
-</svg>;
-
-const LoadingIcon = () => <svg xmlns="http://www.w3.org/2000/svg"
-                               width="16px" height="16px" viewBox="0 0 100 100" preserveAspectRatio="xMidYMid">
-  <circle cx="50" cy="50" fill="none" stroke="currentColor" strokeWidth="10" r="35"
-          strokeDasharray="164.93361431346415 56.97787143782138">
-    <animateTransform attributeName="transform" type="rotate" repeatCount="indefinite" dur="1s"
-                      values="0 50 50;360 50 50" keyTimes="0;1"></animateTransform>
-  </circle>
-</svg>;
+const SearchIcon = () => (
+  <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16" fill="currentColor" className="w-4 h-4 opacity-70">
+    <path
+      fillRule="evenodd"
+      d="M9.965 11.026a5 5 0 1 1 1.06-1.06l2.755 2.754a.75.75 0 1 1-1.06 1.06l-2.755-2.754ZM10.5 7a3.5 3.5 0 1 1-7 0 3.5 3.5 0 0 1 7 0Z"
+      clipRule="evenodd"
+    />
+  </svg>
+);
+
+const LoadingIcon = () => (
+  <svg
+    xmlns="http://www.w3.org/2000/svg"
+    width="16px"
+    height="16px"
+    viewBox="0 0 100 100"
+    preserveAspectRatio="xMidYMid"
+  >
+    <circle
+      cx="50"
+      cy="50"
+      fill="none"
+      stroke="currentColor"
+      strokeWidth="10"
+      r="35"
+      strokeDasharray="164.93361431346415 56.97787143782138"
+    >
+      <animateTransform
+        attributeName="transform"
+        type="rotate"
+        repeatCount="indefinite"
+        dur="1s"
+        values="0 50 50;360 50 50"
+        keyTimes="0;1"
+      ></animateTransform>
+    </circle>
+  </svg>
+);
 
 PlaygroundQuery.displayName = "PlaygroundQuery";
 

selfie-ui/src/app/page.tsx

@@ -88,9 +88,12 @@ const App = () => {
             ))}
             <li></li>
             <li>
-              <a className="link link-hover" href={`${apiBaseUrl}/docs`} target="_blank">
+              <a className="link link-hover" href={`${apiBaseUrl}/redoc`} target="_blank">
                 API Docs
               </a>
+              <a className="link link-hover" href={`${apiBaseUrl}/docs`} target="_blank">
+                API Sandbox
+              </a>
             </li>
             <li>
               <a className="link link-hover" href="https://github.com/vana-com/selfie" target="_blank"

selfie/api/__init__.py

@@ -10,10 +10,8 @@
 
 from selfie.api.completions import router as completions_router
 from selfie.api.connectors import router as connectors_router
-from selfie.api.data_sources import router as data_sources_router
 from selfie.api.document_connections import router as document_connections_router
 from selfie.api.documents import router as documents_router
-from selfie.api.index_documents import router as index_documents_router
 from selfie.api.models import router as models_router
 from selfie.api.connectors import router as connectors_router
 from selfie.config import get_app_config
@@ -22,7 +20,51 @@
 
 config = get_app_config()
 
-app = FastAPI(root_path="/v1")
+
+description = """
+The Selfie API is a RESTful API for interacting with the Selfie platform. It provides endpoints for generating chat and text completions, configuring and managing Selfie, and managing data sources, documents, and data indexing.
+"""
+
+tags_metadata = [
+    {
+        "name": "Completions",
+        "description": """Endpoints for generating chat and text completions. Selfie completion endpoints can be used as drop-in replacements for endpoints in the [OpenAI API](https://platform.openai.com/docs/api-reference).
+
+These endpoints generally include additional functionality not present in the OpenAI API, e.g. you can use a flag to control whether or not to Selfie data is used during text generation.
+
+Please see the [API Usage Guide](https://github.com/vana-com/selfie/?tab=readme-ov-file#api-usage-guide) for more information on how to use these endpoints.
+""",
+    },
+    {
+        "name": "Search",
+        "description": "Endpoints for searching and analyzing documents.",
+    },
+    {
+        "name": "Data Management",
+        "description": """Endpoints for managing data sources, documents, and data indexing.
+
+These endpoints are primarily intended to be used by the Selfie UI."""
+    },
+    {
+        "name": "Configuration",
+        "description": """Endpoints for configuring and managing Selfie.
+
+These endpoints are primarily intended to be used by the Selfie UI."""
+    },
+    # {
+    #     "name": "Deprecated",
+    #     "description": "Endpoints that are deprecated and should not be used.",
+    # }
+]
+
+
+app = FastAPI(
+    title="Selfie",
+    description=description,
+    root_path="/v1",
+    openapi_tags=tags_metadata,
+    version="0.1.0",  # TODO: dynamically fetch version
+)
 
 app.add_middleware(
     CORSMiddleware,
@@ -48,9 +90,7 @@
 app.include_router(completions_router)
 app.include_router(connectors_router)
 app.include_router(document_connections_router)
-app.include_router(data_sources_router)
 app.include_router(documents_router)
-app.include_router(index_documents_router)
 app.include_router(models_router)
 app.include_router(connectors_router)
 
@@ -72,6 +112,6 @@ async def dispatch(self, request: Request, call_next):
 app.add_middleware(CleanURLMiddleware)
 
 
-@app.get("/", response_class=HTMLResponse)
+@app.get("/", response_class=HTMLResponse, include_in_schema=False)
 async def serve_index_html():
     return FileResponse(os.path.join(static_files_dir, "index.html"))

selfie/api/completions.py

@@ -8,18 +8,24 @@
 
 from selfie.text_generation import completion
 
-router = APIRouter()
+router = APIRouter(tags=["Completions"])
 
 
-@router.post("/chat/completions")
+@router.post("/chat/completions",
+             description="""
+             Creates a response for the given conversation in [the style of OpenAI](https://platform.openai.com/docs/api-reference/chat/create).
+             """)
 async def create_chat_completion(
         request: ChatCompletionRequest,
 ) -> LlamaCppChatCompletionResponse | LitellmCompletionResponse:
     return await completion(request)
 
 
 # TODO can StreamingResponse's schema be defined?
-@router.post("/completions")
+@router.post("/completions",
+                description="""
+                Creates a response for the given prompt in [the style of OpenAI](https://platform.openai.com/docs/api-reference/completions/create).
+                """)
 async def create_completion(
         request: CompletionRequest,
 ) -> LlamaCppCompletionResponse | LitellmCompletionResponse: