From 889fe0c013b62d8b6df8410b4bc0221d883e2a5c Mon Sep 17 00:00:00 2001 From: ryjiang Date: Mon, 23 Dec 2024 19:31:57 +0800 Subject: [PATCH] [2.4] add alterCollectionFieldProperties API (#393) * finish alter collection field properties Signed-off-by: ryjiang * fix test Signed-off-by: ryjiang --------- Signed-off-by: ryjiang --- milvus/grpc/Collection.ts | 46 +++++++++++++++++++++++++++++++++ milvus/grpc/Data.ts | 2 +- milvus/types/Collection.ts | 50 +++++++++++++++++++++--------------- milvus/types/Common.ts | 6 ++--- milvus/utils/Format.ts | 42 +++++++++++++++++++++--------- test/grpc/Collection.spec.ts | 39 +++++++++++++++++++++++++++- test/utils/Format.spec.ts | 3 +++ 7 files changed, 150 insertions(+), 38 deletions(-) diff --git a/milvus/grpc/Collection.ts b/milvus/grpc/Collection.ts index 2db6e3c6..85628f86 100644 --- a/milvus/grpc/Collection.ts +++ b/milvus/grpc/Collection.ts @@ -55,6 +55,7 @@ import { CreateCollectionWithSchemaReq, FieldSchema, DropCollectionPropertiesReq, + AlterCollectionFieldPropertiesReq, isVectorType, } from '../'; @@ -331,6 +332,51 @@ export class Collection extends Database { */ alterCollection = this.alterCollectionProperties; + /** + * Modifies a collection field's properties. + * Note that this operation only modifies the properties of the field, not the field itself. + * + * @param {AlterCollectionFieldPropertiesReq} data - The request parameters. + * @param {string} data.collection_name - The name of the collection to modify. + * @param {string} data.field_name - The name of the field to modify. + * @param {Object} data.properties - The properties to modify. For example, to change field mmap setting and max_length, use { 'mmap.enabled', true, max_length: 128}. + * @param {string} [data.db_name] - The name of the database where the collection is located. + * @param {number} [data.timeout] - An optional duration of time in milliseconds to allow for the RPC. If it is set to undefined, the client keeps waiting until the server responds or error occurs. Default is undefined. + * + * @returns {Promise} The response status of the operation. + * @returns {string} status.error_code - The error code of the operation. + * @returns {string} status.reason - The reason for the error, if any. + * + * @example + * ``` + * const milvusClient = new milvusClient(MILUVS_ADDRESS); + * const resStatus = await milvusClient.alterCollectionField({ + * collection_name: 'my-collection', + * field_name: 'my-field', + * properties: {"mmap.enabled": true} + * }); + * ``` + */ + async alterCollectionFieldProperties( + data: AlterCollectionFieldPropertiesReq + ): Promise { + const req: any = { + collection_name: data.collection_name, + field_name: data.field_name, + properties: parseToKeyValue(data.properties), + }; + if (data.db_name) { + req.db_name = data.db_name; + } + const promise = await promisify( + this.channelPool, + 'AlterCollectionField', + req, + data?.timeout || this.timeout + ); + return promise; + } + /** * Drops collection properties. * Note that this operation only deletes the properties of the collection, not the collection itself. diff --git a/milvus/grpc/Data.ts b/milvus/grpc/Data.ts index 219254f0..f57dfc2c 100644 --- a/milvus/grpc/Data.ts +++ b/milvus/grpc/Data.ts @@ -153,7 +153,7 @@ export class Data extends Collection { name: v.name, type: v.data_type, // milvus return string here elementType: v.element_type, - dim: Number(findKeyValue(v.type_params, 'dim')), + dim: Number(v.dim), data: [], // values container }, ]) diff --git a/milvus/types/Collection.ts b/milvus/types/Collection.ts index baa25d60..9006cd52 100644 --- a/milvus/types/Collection.ts +++ b/milvus/types/Collection.ts @@ -15,25 +15,6 @@ import { ShowCollectionsType, } from '../'; -// returned from milvus -export interface FieldSchema { - type_params: KeyValuePair[]; - index_params: KeyValuePair[]; - fieldID: string | number; - name: string; - is_primary_key?: boolean; - description: string; - data_type: keyof typeof DataType; - autoID: boolean; - state: string; - element_type?: keyof typeof DataType; - default_value?: number | string; - dataType: DataType; - is_partition_key?: boolean; - is_dynamic?: boolean; - is_clustering_key?: boolean; -} - export interface CollectionData { name: string; id: string; @@ -56,8 +37,30 @@ export interface ReplicaInfo { node_ids: string[]; } -export type TypeParam = string | number; -export type TypeParamKey = 'dim' | 'max_length' | 'max_capacity'; +export type TypeParam = string | number | boolean | Record; +export type TypeParamKey = + | 'dim' + | 'max_length' + | 'max_capacity' + | 'mmap.enabled'; + +// returned from milvus +export type FieldSchema = { + type_params: KeyValuePair[]; + index_params: KeyValuePair[]; + fieldID: string | number; + name: string; + is_primary_key: boolean; + description: string; + data_type: keyof typeof DataType; + autoID: boolean; + state: string; + element_type?: keyof typeof DataType; + dataType: DataType; + is_partition_key?: boolean; + is_dynamic?: boolean; + is_clustering_key?: boolean; +} & Partial>; // create collection export interface FieldType { @@ -265,3 +268,8 @@ export interface ListAliasesResponse extends resStatusResponse { aliases: string[]; collection_name: string; } + +export interface AlterCollectionFieldPropertiesReq extends collectionNameReq { + field_name: string; // required, field name + properties: Properties; // required, properties +} diff --git a/milvus/types/Common.ts b/milvus/types/Common.ts index 5d575ecb..43bb435f 100644 --- a/milvus/types/Common.ts +++ b/milvus/types/Common.ts @@ -11,9 +11,9 @@ export interface MsgBase { }; } -export interface KeyValuePair { - key: string; - value: string | number; +export interface KeyValuePair { + key: T; + value: U; } interface NumberArray { diff --git a/milvus/utils/Format.ts b/milvus/utils/Format.ts index 9fa0e0f1..9cff3a88 100644 --- a/milvus/utils/Format.ts +++ b/milvus/utils/Format.ts @@ -42,6 +42,8 @@ import { bf16BytesToF32Array, f16BytesToF32Array, keyValueObj, + TypeParamKey, + TypeParam, } from '../'; /** @@ -189,23 +191,33 @@ export const formatAddress = (address: string) => { */ export const assignTypeParams = ( field: FieldType, - typeParamKeys: string[] = ['dim', 'max_length', 'max_capacity'] + typeParamKeys: TypeParamKey[] = [ + 'dim', + 'max_length', + 'max_capacity', + 'mmap.enabled', + ] ) => { - let newField = cloneObj(field); + const newField = cloneObj(field); + + // Initialize `type_params` if undefined + newField.type_params ??= {} as Record; + typeParamKeys.forEach(key => { - if (newField.hasOwnProperty(key)) { - // if the property exists in the field object, assign it to the type_params object - newField.type_params = newField.type_params || {}; - newField.type_params[key] = String(newField[key as keyof FieldType]); - // delete the property from the field object + if (key in newField) { + const value = newField[key as keyof FieldType]; + // Convert the value to a string, JSON-stringify if it’s an object + newField.type_params![key] = + typeof value === 'object' ? JSON.stringify(value) : String(value ?? ''); delete newField[key as keyof FieldType]; } - - if (newField.type_params && newField.type_params[key]) { - // if the property already exists in the type_params object, convert it to a string - newField.type_params[key] = String(newField.type_params[key]); - } }); + + // delete type_params if it's empty + if (!Object.keys(newField.type_params).length) { + delete newField.type_params; + } + return newField; }; @@ -361,6 +373,12 @@ export const formatDescribedCol = ( // add a dataType property which indicate datatype number newData.schema?.fields?.forEach(f => { f.dataType = DataTypeMap[f.data_type]; + // extract type params(key value pair = {key: 'xxx', value: any}), and assign it to the field object(key) + if (f.type_params && f.type_params.length > 0) { + f.type_params.forEach(keyValuePair => { + f[keyValuePair.key] = keyValuePair.value; + }); + } }); return newData; diff --git a/test/grpc/Collection.spec.ts b/test/grpc/Collection.spec.ts index fbd3ea4f..2fbcf963 100644 --- a/test/grpc/Collection.spec.ts +++ b/test/grpc/Collection.spec.ts @@ -79,7 +79,9 @@ describe(`Collection API`, () => { expect( Boolean( - formatKeyValueData(describe.properties, ['mmap.enabled'])['mmap.enabled'] + formatKeyValueData(describe.properties, ['mmap.enabled'])[ + 'mmap.enabled' + ] ) ).toEqual(true); }); @@ -434,6 +436,41 @@ describe(`Collection API`, () => { ).toEqual(value2); }); + it(`Alter collection field properties should success`, async () => { + const key = 'mmap.enabled'; + const value = true; + const alter = await milvusClient.alterCollectionFieldProperties({ + collection_name: LOAD_COLLECTION_NAME, + field_name: 'json', + properties: { [key]: value }, + db_name: 'Collection', // pass test case + }); + expect(alter.error_code).toEqual(ErrorCode.SUCCESS); + const describe = await milvusClient.describeCollection({ + collection_name: LOAD_COLLECTION_NAME, + }); + // find json field + const jsonField = describe.schema.fields.find( + f => f.name === 'json' + ) as any; + expect(jsonField['mmap.enabled']).toEqual('true'); + const alter2 = await milvusClient.alterCollectionFieldProperties({ + collection_name: LOAD_COLLECTION_NAME, + field_name: 'varChar', + properties: { max_length: 1024 }, + db_name: 'Collection', // pass test case + }); + expect(alter2.error_code).toEqual(ErrorCode.SUCCESS); + const describe2 = await milvusClient.describeCollection({ + collection_name: LOAD_COLLECTION_NAME, + }); + // find varChar field + const varCharField = describe2.schema.fields.find( + f => f.name === 'varChar' + ) as any; + expect(varCharField['max_length']).toEqual('1024'); + }); + it(`Load Collection Sync throw COLLECTION_NAME_IS_REQUIRED`, async () => { try { await milvusClient.loadCollectionSync({} as any); diff --git a/test/utils/Format.spec.ts b/test/utils/Format.spec.ts index 84b77077..a852acd7 100644 --- a/test/utils/Format.spec.ts +++ b/test/utils/Format.spec.ts @@ -204,6 +204,7 @@ describe('utils/format', () => { type_params: { dim: 100, max_length: 50, + 'mmap.enabled': true, }, dim: 200, max_length: 75, @@ -214,6 +215,7 @@ describe('utils/format', () => { type_params: { dim: '200', max_length: '75', + 'mmap.enabled': true, }, }; expect(assignTypeParams(field)).toEqual(expectedOutput); @@ -414,6 +416,7 @@ describe('utils/format', () => { const formatted = formatDescribedCol(response); expect(formatted.schema.fields[0].dataType).toBe(101); + expect(formatted.schema.fields[0].dim).toBe('128'); expect(formatted.schema.fields[1].dataType).toBe(5); });