Skip to content

Commit

Permalink
[Prostgres Vector Store] Add PGVector Driver option + Fix null charac…
Browse files Browse the repository at this point in the history 
…ter issue w/ TypeORM Driver (#3367)

* Add PGVector Driver option + Fix null character issue w/ TypeORM Driver

* Handle file upload case with PGVector

* Cleanup

* Fix data filtering for chatflow uploaded files

* Add distanceStrategy parameter

* Fix query to improve chatflow uploaded files filtering

* Ensure PGVector release connections

* Await client connected

* Make Postgres credentials optionnal when set on env variables

* Document env variables in nodes directories

* Prevent reuse client

* Fix empty metadataFilter

* Update CONTRIBUTING.md

* Update Postgres.ts

---------

Co-authored-by: Henry Heng <[email protected]>
  • Loading branch information
@JJK801 @HenryHengZJ
JJK801 and HenryHengZJ authored Nov 1, 2024
1 parent 39380a4 commit 15d59a9
Show file tree
Hide file tree
Showing 10 changed files with 535 additions and 204 deletions.
36 changes: 22 additions & 14 deletions packages/components/nodes/recordmanager/PostgresRecordManager/PostgresRecordManager.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,10 @@ import { ICommonObject, INode, INodeData, INodeParams } from '../../../src/Inter
import { getBaseClasses, getCredentialData, getCredentialParam } from '../../../src/utils'
import { ListKeyOptions, RecordManagerInterface, UpdateOptions } from '@langchain/community/indexes/base'
import { DataSource, QueryRunner } from 'typeorm'
import { getHost } from '../../vectorstores/Postgres/utils'
import { getDatabase, getPort, getTableName } from './utils'

const serverCredentialsExists = !!process.env.POSTGRES_RECORDMANAGER_USER && !!process.env.POSTGRES_RECORDMANAGER_PASSWORD

class PostgresRecordManager_RecordManager implements INode {
label: string
Expand Down Expand Up @@ -29,18 +33,22 @@ class PostgresRecordManager_RecordManager implements INode {
{
label: 'Host',
name: 'host',
type: 'string'
type: 'string',
placeholder: getHost(),
optional: !!getHost()
},
{
label: 'Database',
name: 'database',
type: 'string'
type: 'string',
placeholder: getDatabase(),
optional: !!getDatabase()
},
{
label: 'Port',
name: 'port',
type: 'number',
placeholder: '5432',
placeholder: getPort(),
optional: true
},
{
Expand All @@ -54,7 +62,7 @@ class PostgresRecordManager_RecordManager implements INode {
label: 'Table Name',
name: 'tableName',
type: 'string',
placeholder: 'upsertion_records',
placeholder: getTableName(),
additionalParams: true,
optional: true
},
Expand Down Expand Up @@ -110,16 +118,16 @@ class PostgresRecordManager_RecordManager implements INode {
label: 'Connect Credential',
name: 'credential',
type: 'credential',
credentialNames: ['PostgresApi']
credentialNames: ['PostgresApi'],
optional: serverCredentialsExists
}
}

async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
const user = getCredentialParam('user', credentialData, nodeData)
const password = getCredentialParam('password', credentialData, nodeData)
const _tableName = nodeData.inputs?.tableName as string
const tableName = _tableName ? _tableName : 'upsertion_records'
const user = getCredentialParam('user', credentialData, nodeData, process.env.POSTGRES_RECORDMANAGER_USER)
const password = getCredentialParam('password', credentialData, nodeData, process.env.POSTGRES_RECORDMANAGER_PASSWORD)
const tableName = getTableName(nodeData)
const additionalConfig = nodeData.inputs?.additionalConfig as string
const _namespace = nodeData.inputs?.namespace as string
const namespace = _namespace ? _namespace : options.chatflowid
Expand All @@ -139,11 +147,11 @@ class PostgresRecordManager_RecordManager implements INode {
const postgresConnectionOptions = {
...additionalConfiguration,
type: 'postgres',
host: nodeData.inputs?.host as string,
port: nodeData.inputs?.port as number,
host: getHost(nodeData),
port: getPort(nodeData),
username: user,
password: password,
database: nodeData.inputs?.database as string
database: getDatabase(nodeData)
}

const args = {
Expand All @@ -162,7 +170,7 @@ class PostgresRecordManager_RecordManager implements INode {

type PostgresRecordManagerOptions = {
postgresConnectionOptions: any
tableName?: string
tableName: string
}

class PostgresRecordManager implements RecordManagerInterface {
Expand All @@ -180,7 +188,7 @@ class PostgresRecordManager implements RecordManagerInterface {
const { postgresConnectionOptions, tableName } = config
this.namespace = namespace
this.datasource = new DataSource(postgresConnectionOptions)
this.tableName = tableName || 'upsertion_records'
this.tableName = tableName
}

async createSchema(): Promise<void> {
Expand Down
18 changes: 18 additions & 0 deletions packages/components/nodes/recordmanager/PostgresRecordManager/README.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
# Postgres Record Manager

Postgres Record Manager integration for Flowise

## 🌱 Env Variables

| Variable | Description | Type | Default |
| ---------------------------- | ----------------------------------------------------------------------------------------------- | ------------------------------------------------ | ----------------------------------- |
| POSTGRES_RECORDMANAGER_HOST | Default `host` for Postgres Record Manager | String | |
| POSTGRES_RECORDMANAGER_PORT | Default `port` for Postgres Record Manager | Number | 5432 |
| POSTGRES_RECORDMANAGER_USER | Default `user` for Postgres Record Manager | String | |
| POSTGRES_RECORDMANAGER_PASSWORD | Default `password` for Postgres Record Manager | String | |
| POSTGRES_RECORDMANAGER_DATABASE | Default `database` for Postgres Record Manager | String | |
| POSTGRES_RECORDMANAGER_TABLE_NAME | Default `tableName` for Postgres Record Manager | String | upsertion_records |

## License

Source code in this repository is made available under the [Apache License Version 2.0](https://github.com/FlowiseAI/Flowise/blob/master/LICENSE.md).
17 changes: 17 additions & 0 deletions packages/components/nodes/recordmanager/PostgresRecordManager/utils.ts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
import { defaultChain, INodeData } from '../../../src'

export function getHost(nodeData?: INodeData) {
return defaultChain(nodeData?.inputs?.host, process.env.POSTGRES_RECORDMANAGER_HOST)
}

export function getDatabase(nodeData?: INodeData) {
return defaultChain(nodeData?.inputs?.database, process.env.POSTGRES_RECORDMANAGER_DATABASE)
}

export function getPort(nodeData?: INodeData) {
return defaultChain(nodeData?.inputs?.port, process.env.POSTGRES_RECORDMANAGER_PORT, '5432')
}

export function getTableName(nodeData?: INodeData) {
return defaultChain(nodeData?.inputs?.tableName, process.env.POSTGRES_RECORDMANAGER_TABLE_NAME, 'upsertion_records')
}
Loading

0 comments on commit 15d59a9

Please sign in to comment.