From 5d49dafc923c4d23be3fa1c6295ed3152c1a5f6e Mon Sep 17 00:00:00 2001 From: Alvaro Viebrantz Date: Fri, 16 Aug 2024 15:14:21 -0400 Subject: [PATCH] fix: remove field name customization --- src/adapt/index.ts | 5 +---- src/adapt/options.ts | 19 +++++++++++++------ src/adapt/proto.ts | 12 ++---------- system-test/managed_writer_client_test.ts | 2 +- test/adapt/proto.ts | 8 ++++---- 5 files changed, 21 insertions(+), 25 deletions(-) diff --git a/src/adapt/index.ts b/src/adapt/index.ts index 56765e59..9afc0bdd 100644 --- a/src/adapt/index.ts +++ b/src/adapt/index.ts @@ -19,7 +19,4 @@ export { export {convertBigQuerySchemaToStorageTableSchema} from './schema'; -export { - withChangeType, - withChangeSequenceNumber, -} from './options'; +export {withChangeType, withChangeSequenceNumber} from './options'; diff --git a/src/adapt/options.ts b/src/adapt/options.ts index 16089aae..d87c5ae2 100644 --- a/src/adapt/options.ts +++ b/src/adapt/options.ts @@ -13,26 +13,33 @@ // limitations under the License. export type AdaptOptions = { - changeSequenceNumberFieldName: string; addChangeSequenceNumber: boolean; - changeTypeFieldName: string; addChangeType: boolean; }; export type AdaptOption = (opts: AdaptOptions) => AdaptOptions; -export function withChangeType(fieldName?: string): AdaptOption { +/** + * Add pseudocolumn `_CHANGE_TYPE` for BigQuery Change Data Capture. + * Used to define the type of change to be professed for each row. + * The pseudocolumn `_CHANGE_TYPE` only accepts the values UPSERT and DELETE. + * See more: https://cloud.google.com/bigquery/docs/change-data-capture#specify_changes_to_existing_records + */ +export function withChangeType(): AdaptOption { return (opts: AdaptOptions) => ({ ...opts, - changeTypeFieldName: fieldName || 'changeType', addChangeType: true, }); } -export function withChangeSequenceNumber(fieldName?: string): AdaptOption { +/** + * Add pseudocolumn `_CHANGE_SEQUENCE_NUMBER` for BigQuery Change Data Capture. + * Used to change behavior of ordering records with same primary key. + * See more: https://cloud.google.com/bigquery/docs/change-data-capture#manage_custom_ordering + */ +export function withChangeSequenceNumber(): AdaptOption { return (opts: AdaptOptions) => ({ ...opts, - changeSequenceNumberFieldName: fieldName || 'changeSequenceNumber', addChangeSequenceNumber: true, }); } diff --git a/src/adapt/proto.ts b/src/adapt/proto.ts index 5ad19c59..96ffc335 100644 --- a/src/adapt/proto.ts +++ b/src/adapt/proto.ts @@ -98,9 +98,7 @@ function convertStorageSchemaToFileDescriptorInternal( ...opts: AdaptOption[] ): FileDescriptorSet { let adaptOpts: AdaptOptions = { - changeSequenceNumberFieldName: 'changeSequenceNumber', addChangeSequenceNumber: false, - changeTypeFieldName: 'changeType', addChangeType: false, }; opts.forEach(f => { @@ -177,14 +175,11 @@ function convertStorageSchemaToFileDescriptorInternal( name: '_CHANGE_SEQUENCE_NUMBER', type: 'STRING', mode: 'REQUIRED', - description: - 'pseudocolumn only accepts values, written in a fixed format written in hexadecimal, separated into sections by a forward slash', }, 991, scope, useProto3 ); - //fdp.jsonName = adaptOpts.changeSequenceNumberFieldName; fields.push(fdp); } if (adaptOpts.addChangeType) { @@ -192,15 +187,12 @@ function convertStorageSchemaToFileDescriptorInternal( { name: '_CHANGE_TYPE', type: 'STRING', - mode: 'REQUIRED', - description: - 'pseudocolumn to indicate the type of change. Only accepts `INSERT`, `UPSERT` and `DELETE`', + mode: 'REQUIRED', }, 992, scope, useProto3 ); - fdp.jsonName = adaptOpts.changeTypeFieldName; fields.push(fdp); } } @@ -330,7 +322,7 @@ function convertTableFieldSchemaToFieldDescriptorProto( type: pType, label: label, options: { - packed: shouldPackType(pType, label, useProto3), + packed: shouldPackType(pType, label, useProto3), }, proto3Optional: isProto3Optional(label, useProto3), }); diff --git a/system-test/managed_writer_client_test.ts b/system-test/managed_writer_client_test.ts index 39d45c73..614ecca3 100644 --- a/system-test/managed_writer_client_test.ts +++ b/system-test/managed_writer_client_test.ts @@ -836,7 +836,7 @@ describe('managedwriter.WriterClient', () => { adapt.convertStorageSchemaToProto2Descriptor( storageSchema, 'root', - adapt.withChangeType('_CHANGE_TYPE') + adapt.withChangeType() ); const row1 = { diff --git a/test/adapt/proto.ts b/test/adapt/proto.ts index d06611c8..2e34b02b 100644 --- a/test/adapt/proto.ts +++ b/test/adapt/proto.ts @@ -95,8 +95,8 @@ describe('Adapt Protos', () => { const protoDescriptor = adapt.convertStorageSchemaToProto2Descriptor( storageSchema, 'Test', - adapt.withChangeType('change'), - adapt.withChangeSequenceNumber('seq') + adapt.withChangeType(), + adapt.withChangeSequenceNumber() ); assert.notEqual(protoDescriptor, null); if (!protoDescriptor) { @@ -106,8 +106,8 @@ describe('Adapt Protos', () => { const raw = { id: 1, username: 'Alice', - change: 'INSERT', - seq: 'FF', + _CHANGE_TYPE: 'INSERT', + _CHANGE_SEQUENCE_NUMBER: 'FF', }; const serialized = TestProto.encode(raw).finish(); const decoded = TestProto.decode(serialized).toJSON();