Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feature/Add new doc store upsert and refresh API #3556

Merged
merged 1 commit into from
Nov 25, 2024
Merged

Feature/Add new doc store upsert and refresh API #3556

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
252 changes: 130 additions & 122 deletions packages/api-documentation/src/yml/swagger.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -305,6 +305,10 @@ paths:
type: string
format: binary
description: Files to be uploaded
base64:
type: boolean
default: false
description: Return contents of the files in base64 format
required:
- files
required: true
Expand Down Expand Up @@ -618,171 +622,109 @@ paths:
'500':
description: Internal server error

/document-store/loader/preview:
/document-store/upsert/{id}:
post:
tags:
- document-store
security:
- bearerAuth: []
summary: Preview document chunks
description: Preview document chunks from loader
operationId: previewChunking
summary: Upsert new document to document store
description: Upsert new document to document store
operationId: upsertDocument
parameters:
- in: path
name: id
required: true
schema:
type: string
format: uuid
description: Document store ID
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/DocumentStoreLoaderForPreview'
required: true
responses:
'200':
description: Successfully preview chunks
content:
application/json:
schema:
type: object
properties:
chunks:
type: array
items:
$ref: '#/components/schemas/Document'
totalChunks:
type: integer
example: 10
previewChunkCount:
type: integer
example: 5
'400':
description: Invalid request body
'500':
description: Internal server error

/document-store/loader/process:
post:
tags:
- document-store
security:
- bearerAuth: []
summary: Process loading & chunking operation
description: Process loading & chunking operation of document from loader
operationId: processChunking
requestBody:
content:
application/json:
$ref: '#/components/schemas/DocumentStoreLoaderForUpsert'
multipart/form-data:
schema:
type: object
required:
- storeId
- id
properties:
storeId:
type: string
description: Document store ID
example: '603a7b51-ae7c-4b0a-8865-e454ed2f6766'
id:
files:
type: array
items:
type: string
format: binary
description: Files to be uploaded
loader:
type: string
description: Document loader ID. If your URL is /document-stores/{storeId}/{id}, then id is the last part of the URL
example: 'c427e569-b81a-469a-b14c-fa73dd5bae49'
required: true
responses:
'200':
description: Successfully process chunking operation
content:
application/json:
schema:
$ref: '#/components/schemas/DocumentStoreFileChunkPagedResponse'

'400':
description: Invalid request body
'500':
description: Internal server error

/document-store/vectorstore/save:
post:
tags:
- document-store
security:
- bearerAuth: []
summary: Save upsert configuration of document store
description: Save upsert configuration of document store
operationId: saveVectorStoreConfig
requestBody:
content:
application/json:
schema:
type: object
required:
- storeId
properties:
storeId:
nullable: true
example: '{"name":"plainText","config":{"text":"why the sky is blue"}}'
description: Loader configurations
splitter:
type: string
description: Document store ID
example: '603a7b51-ae7c-4b0a-8865-e454ed2f6766'
embeddingName:
nullable: true
example: '{"name":"recursiveCharacterTextSplitter","config":{"chunkSize":2000}}'
description: Splitter configurations
embedding:
type: string
description: Name of the embedding
example: 'openAIEmbeddings'
embeddingConfig:
type: object
description: Configuration of the embedding
example: { 'model': 'text-embedding-ada-002', 'credential': '1eba5808-c55b-4817-a285-b0c92846a7ad' }
vectorStoreName:
nullable: true
example: '{"name":"openAIEmbeddings","config":{"modelName":"text-embedding-ada-002"}}'
description: Embedding configurations
vectorStore:
type: string
description: Name of the vector store
example: 'faiss'
vectorStoreConfig:
type: object
description: Configuration of the embedding
example: { 'basePath': './faiss' }
recordManagerName:
nullable: true
example: '{"name":"faiss"}'
description: Vector Store configurations
recordManager:
type: string
description: Name of the record manager
example: 'SQLiteRecordManager'
recordManagerConfig:
type: object
description: Configuration of the embedding
example: { 'databaseFilePath': './recordManager.db' }
nullable: true
example: '{"name":"postgresRecordManager"}'
description: Record Manager configurations
required:
- files
required: true
responses:
'200':
description: Successfully save upsert configuration of document store
description: Successfully execute upsert operation
content:
application/json:
schema:
$ref: '#/components/schemas/DocumentStore'
$ref: '#/components/schemas/VectorUpsertResponse'

'400':
description: Invalid request body
'500':
description: Internal server error

/document-store/vectorstore/insert:
/document-store/refresh/{id}:
post:
tags:
- document-store
security:
- bearerAuth: []
summary: Upsert chunks from document store
description: Upsert chunks from document store using the saved configuration
operationId: insertIntoVectorStore
summary: Re-process and upsert all documents in document store
description: Re-process and upsert all existing documents in document store
operationId: refreshDocument
parameters:
- in: path
name: id
required: true
schema:
type: string
format: uuid
description: Document store ID
requestBody:
content:
application/json:
schema:
type: object
required:
- storeId
properties:
storeId:
type: string
description: Document store ID
example: '603a7b51-ae7c-4b0a-8865-e454ed2f6766'
$ref: '#/components/schemas/DocumentStoreLoaderForRefresh'
required: true
responses:
'200':
description: Successfully save upsert configuration of document store
description: Successfully execute refresh operation
content:
application/json:
schema:
type: array
items:
$ref: '#/components/schemas/VectorUpsertResponse'

'400':
Expand Down Expand Up @@ -2220,6 +2162,72 @@ components:
description:
type: string

DocumentStoreLoaderForUpsert:
type: object
properties:
docId:
type: string
format: uuid
description: Document ID within the store. If provided, existing configuration from the document will be used for the new document
loader:
type: object
properties:
name:
type: string
example: plainText
description: Name of the loader (camelCase)
config:
type: object
description: Configuration for the loader
splitter:
type: object
properties:
name:
type: string
example: recursiveCharacterTextSplitter
description: Name of the text splitter (camelCase)
config:
type: object
description: Configuration for the text splitter
embedding:
type: object
properties:
name:
type: string
example: openAIEmbeddings
description: Name of the embedding generator (camelCase)
config:
type: object
description: Configuration for the embedding generator
vectorStore:
type: object
properties:
name:
type: string
example: faiss
description: Name of the vector store (camelCase)
config:
type: object
description: Configuration for the vector store
recordManager:
type: object
properties:
name:
type: string
example: postgresRecordManager
description: Name of the record manager (camelCase)
config:
type: object
description: Configuration for the record manager

DocumentStoreLoaderForRefresh:
type: object
properties:
items:
type: array
items:
$ref: '#/components/schemas/DocumentStoreLoaderForUpsert'

ChatMessageFeedback:
type: object
properties:
Expand Down
Loading
Loading