From b878767f99097f1a60ed422cb260ef1561bd6074 Mon Sep 17 00:00:00 2001 From: Alvaro Viebrantz Date: Fri, 5 Apr 2024 16:05:54 -0400 Subject: [PATCH 1/7] feat: add support for RANGE type --- src/bigquery.ts | 345 ++++++++++++++++++++++++++++++++++------------- test/bigquery.ts | 131 ++++++++++++++++++ 2 files changed, 383 insertions(+), 93 deletions(-) diff --git a/src/bigquery.ts b/src/bigquery.ts index 010a7071..7eb59b2f 100644 --- a/src/bigquery.ts +++ b/src/bigquery.ts @@ -191,6 +191,11 @@ export interface BigQueryDatetimeOptions { fractional?: string | number; } +export interface BigQueryRangeOptions { + start?: BigQueryDate | BigQueryDatetime | BigQueryTimestamp | string; + end?: BigQueryDate | BigQueryDatetime | BigQueryTimestamp | string; +} + export type ProvidedTypeArray = Array; export interface ProvidedTypeStruct { @@ -582,10 +587,10 @@ export class BigQuery extends Service { let value = field.v; if (schemaField.mode === 'REPEATED') { value = (value as TableRowField[]).map(val => { - return convert(schemaField, val.v, options); + return convertSchemaFieldValue(schemaField, val.v, options); }); } else { - value = convert(schemaField, value, options); + value = convertSchemaFieldValue(schemaField, value, options); } // eslint-disable-next-line @typescript-eslint/no-explicit-any const fieldObject: any = {}; @@ -594,97 +599,6 @@ export class BigQuery extends Service { }); } - function convert( - schemaField: TableField, - // eslint-disable-next-line @typescript-eslint/no-explicit-any - value: any, - options: { - wrapIntegers: boolean | IntegerTypeCastOptions; - selectedFields?: string[]; - parseJSON?: boolean; - } - ) { - if (is.null(value)) { - return value; - } - - switch (schemaField.type) { - case 'BOOLEAN': - case 'BOOL': { - value = value.toLowerCase() === 'true'; - break; - } - case 'BYTES': { - value = Buffer.from(value, 'base64'); - break; - } - case 'FLOAT': - case 'FLOAT64': { - value = Number(value); - break; - } - case 'INTEGER': - case 'INT64': { - const {wrapIntegers} = options; - value = wrapIntegers - ? typeof wrapIntegers === 'object' - ? BigQuery.int( - {integerValue: value, schemaFieldName: schemaField.name}, - wrapIntegers - ).valueOf() - : BigQuery.int(value) - : Number(value); - break; - } - case 'NUMERIC': { - value = new Big(value); - break; - } - case 'BIGNUMERIC': { - value = new Big(value); - break; - } - case 'RECORD': { - value = BigQuery.mergeSchemaWithRows_( - schemaField, - value, - options - ).pop(); - break; - } - case 'DATE': { - value = BigQuery.date(value); - break; - } - case 'DATETIME': { - value = BigQuery.datetime(value); - break; - } - case 'TIME': { - value = BigQuery.time(value); - break; - } - case 'TIMESTAMP': { - const pd = new PreciseDate(BigInt(value) * BigInt(1000)); - value = BigQuery.timestamp(pd); - break; - } - case 'GEOGRAPHY': { - value = BigQuery.geography(value); - break; - } - case 'JSON': { - const {parseJSON} = options; - value = parseJSON ? JSON.parse(value) : value; - break; - } - default: - break; - } - - return value; - } - // eslint-disable-next-line @typescript-eslint/no-explicit-any function flattenRows(rows: any[]) { return rows.reduce((acc, row) => { @@ -945,6 +859,42 @@ export class BigQuery extends Service { return BigQuery.timestamp(value); } + /** + * A range represents contiguous range between two dates, datetimes, or timestamps. + * The lower and upper bound for the range are optional. + * The lower bound is inclusive and the upper bound is exclusive. + * + * @method BigQuery.range + * @param {string|BigQueryRangeOptions} value The range literal or start/end with dates/datetimes/timestamp ranges. + * + * @example + * ``` + * const {BigQuery} = require('@google-cloud/bigquery'); + * const timestampRange = BigQuery.range('[2020-10-01 12:00:00+08, 2020-12-31 12:00:00+08)', 'TIMESTAMP'); + * ``` + */ + static range(value: string | BigQueryRangeOptions, elementType: string) { + return new BigQueryRange(value, elementType); + } + + /** + * A range represents contiguous range between two dates, datetimes, or timestamps. + * The lower and upper bound for the range are optional. + * The lower bound is inclusive and the upper bound is exclusive. + * + * @param {string|BigQueryRangeOptions} value The range literal or start/end with dates/datetimes/timestamp ranges. + * + * @example + * ``` + * const {BigQuery} = require('@google-cloud/bigquery'); + * const bigquery = new BigQuery(); + * const timestampRange = bigquery.range('[2020-10-01 12:00:00+08, 2020-12-31 12:00:00+08)', 'TIMESTAMP'); + * ``` + */ + range(value: string, elementType: string) { + return BigQuery.range(value, elementType); + } + /** * A BigQueryInt wraps 'INT64' values. Can be used to maintain precision. * @@ -2390,9 +2340,218 @@ promisifyAll(BigQuery, { 'job', 'time', 'timestamp', + 'range', ], }); +function convertSchemaFieldValue( + schemaField: TableField, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + value: any, + options: { + wrapIntegers: boolean | IntegerTypeCastOptions; + selectedFields?: string[]; + parseJSON?: boolean; + } +) { + if (is.null(value)) { + return value; + } + + switch (schemaField.type) { + case 'BOOLEAN': + case 'BOOL': { + value = value.toLowerCase() === 'true'; + break; + } + case 'BYTES': { + value = Buffer.from(value, 'base64'); + break; + } + case 'FLOAT': + case 'FLOAT64': { + value = Number(value); + break; + } + case 'INTEGER': + case 'INT64': { + const {wrapIntegers} = options; + value = wrapIntegers + ? typeof wrapIntegers === 'object' + ? BigQuery.int( + {integerValue: value, schemaFieldName: schemaField.name}, + wrapIntegers + ).valueOf() + : BigQuery.int(value) + : Number(value); + break; + } + case 'NUMERIC': { + value = new Big(value); + break; + } + case 'BIGNUMERIC': { + value = new Big(value); + break; + } + case 'RECORD': { + value = BigQuery.mergeSchemaWithRows_(schemaField, value, options).pop(); + break; + } + case 'DATE': { + value = BigQuery.date(value); + break; + } + case 'DATETIME': { + value = BigQuery.datetime(value); + break; + } + case 'TIME': { + value = BigQuery.time(value); + break; + } + case 'TIMESTAMP': { + const pd = new PreciseDate(BigInt(value) * BigInt(1000)); + value = BigQuery.timestamp(pd); + break; + } + case 'GEOGRAPHY': { + value = BigQuery.geography(value); + break; + } + case 'JSON': { + const {parseJSON} = options; + value = parseJSON ? JSON.parse(value) : value; + break; + } + case 'RANGE': { + value = BigQueryRange.fromSchemaValue_( + value, + schemaField.rangeElementType!.type! + ); + break; + } + default: + break; + } + + return value; +} + +/** + * Range class for BigQuery. + * A range represents contiguous range between two dates, datetimes, or timestamps. + * The lower and upper bound for the range are optional. + * The lower bound is inclusive and the upper bound is exclusive. + * See https://cloud.google.com/bigquery/docs/reference/standard-sql/lexical#range_literals + */ +export class BigQueryRange { + elementType?: string; + start?: BigQueryTimestamp | BigQueryDate | BigQueryDatetime; + end?: BigQueryTimestamp | BigQueryDate | BigQueryDatetime; + value: string; + constructor(value: string | BigQueryRangeOptions, elementType?: string) { + if (typeof value === 'string') { + if (!elementType) { + throw new Error( + 'invalid RANGE. Element type required when using RANGE literal string.' + ); + } + let cleanedValue = value; + if (cleanedValue.startsWith('[') || cleanedValue.startsWith('(')) { + cleanedValue = cleanedValue.substring(1); + } + if (cleanedValue.endsWith(')') || cleanedValue.endsWith(']')) { + cleanedValue = cleanedValue.substring(0, cleanedValue.length - 1); + } + const parts = cleanedValue.split(','); + if (parts.length !== 2) { + throw new Error( + 'invalid RANGE. See RANGE literal format docs for more information.' + ); + } + + const [start, end] = parts.map((s: string) => s.trim()); + this.start = this.convertElement_(start, elementType); + this.end = this.convertElement_(end, elementType); + this.elementType = elementType; + } else { + const {start, end} = value; + if (start && end) { + if (typeof start !== typeof end) { + throw Error( + 'upper and lower bound on a RANGE should be of the same type.' + ); + } + } + const inferredType = + { + BigQueryDate: 'DATE', + BigQueryDatetime: 'DATETIME', + BigQueryTimestamp: 'TIMESTAMP', + }[(start || end || Object).constructor.name] || elementType; + this.start = this.convertElement_(start, inferredType); + this.end = this.convertElement_(end, inferredType); + this.elementType = inferredType; + } + this.value = `[${this.start ? this.start.value : 'UNBOUNDED'}, ${this.end ? this.end.value : 'UNBOUNDED'})`; + } + + static fromSchemaValue_(value: string, elementType: string): BigQueryRange { + let cleanedValue = value; + if (cleanedValue.startsWith('[') || cleanedValue.startsWith('(')) { + cleanedValue = cleanedValue.substring(1); + } + if (cleanedValue.endsWith(')') || cleanedValue.endsWith(']')) { + cleanedValue = cleanedValue.substring(0, cleanedValue.length - 1); + } + const parts = cleanedValue.split(','); + if (parts.length !== 2) { + throw new Error( + 'invalid RANGE. See RANGE literal format docs for more information.' + ); + } + + const [start, end] = parts.map((s: string) => s.trim()); + const convertRangeSchemaValue = (value: string) => { + if (value === 'UNBOUNDED' || value === 'NULL') { + return null; + } + return convertSchemaFieldValue({type: elementType}, value, { + wrapIntegers: false, + }); + }; + return BigQuery.range( + { + start: convertRangeSchemaValue(start), + end: convertRangeSchemaValue(end), + }, + elementType + ); + } + + convertElement_( + value?: string | BigQueryDate | BigQueryDatetime | BigQueryTimestamp, + elementType?: string + ) { + if (typeof value === 'string') { + if (value === 'UNBOUNDED' || value === 'NULL') { + return undefined; + } + switch (elementType) { + case 'DATE': + return new BigQueryDate(value); + case 'DATETIME': + return new BigQueryDatetime(value); + case 'TIMESTAMP': + return new BigQueryTimestamp(value); + } + return undefined; + } + return value; + } +} + /** * Date class for BigQuery. */ diff --git a/test/bigquery.ts b/test/bigquery.ts index b1022acd..33d8767e 100644 --- a/test/bigquery.ts +++ b/test/bigquery.ts @@ -89,6 +89,7 @@ const fakePfy = Object.assign({}, pfy, { 'job', 'time', 'timestamp', + 'range', ]); }, }); @@ -462,6 +463,14 @@ describe('BigQuery', () => { input, }; }); + + sandbox.stub(BigQuery, 'range').callsFake((input, elementType) => { + return { + type: 'fakeRange', + input, + elementType, + }; + }); }); it('should merge the schema and flatten the rows', () => { @@ -520,6 +529,7 @@ describe('BigQuery', () => { {v: 'datetime-input'}, {v: 'time-input'}, {v: 'geography-input'}, + {v: '[2020-10-01 12:00:00+08, 2020-12-31 12:00:00+08)'}, ], }, expected: { @@ -562,6 +572,20 @@ describe('BigQuery', () => { input: 'geography-input', type: 'fakeGeography', }, + range: { + type: 'fakeRange', + input: { + end: { + input: '2020-12-31 12:00:00+08', + type: 'fakeDatetime', + }, + start: { + input: '2020-10-01 12:00:00+08', + type: 'fakeDatetime', + }, + }, + elementType: 'DATETIME', + }, }, }, ]; @@ -629,6 +653,14 @@ describe('BigQuery', () => { type: 'GEOGRAPHY', }); + schemaObject.fields.push({ + name: 'range', + type: 'RANGE', + rangeElementType: { + type: 'DATETIME', + }, + }); + const rawRows = rows.map(x => x.raw); const mergedRows = BigQuery.mergeSchemaWithRows_(schemaObject, rawRows, { wrapIntegers: false, @@ -910,6 +942,105 @@ describe('BigQuery', () => { }); }); + describe('range', () => { + const INPUT_DATE_RANGE = '[2020-01-01, 2020-12-31)'; + const INPUT_DATETIME_RANGE = '[2020-01-01 12:00:00, 2020-12-31 12:00:00)'; + const INPUT_TIMESTAMP_RANGE = + '[2020-10-01 12:00:00+08, 2020-12-31 12:00:00+08)'; + + it('should have the correct constructor name', () => { + const range = bq.range(INPUT_DATE_RANGE, 'DATE'); + assert.strictEqual(range.constructor.name, 'BigQueryRange'); + }); + + it('should accept a string literal', () => { + const dateRange = bq.range(INPUT_DATE_RANGE, 'DATE'); + assert.strictEqual(dateRange.value, '[2020-01-01, 2020-12-31)'); + assert.strictEqual(dateRange.start.value, '2020-01-01'); + assert.strictEqual(dateRange.end.value, '2020-12-31'); + + const datetimeRange = bq.range(INPUT_DATETIME_RANGE, 'DATETIME'); + assert.strictEqual( + datetimeRange.value, + '[2020-01-01 12:00:00, 2020-12-31 12:00:00)' + ); + assert.strictEqual(datetimeRange.start.value, '2020-01-01 12:00:00'); + assert.strictEqual(datetimeRange.end.value, '2020-12-31 12:00:00'); + + const timestampRange = bq.range(INPUT_TIMESTAMP_RANGE, 'TIMESTAMP'); + assert.strictEqual( + timestampRange.value, + '[2020-10-01T04:00:00.000Z, 2020-12-31T04:00:00.000Z)' + ); + assert.strictEqual( + timestampRange.start.value, + '2020-10-01T04:00:00.000Z' + ); + assert.strictEqual(timestampRange.end.value, '2020-12-31T04:00:00.000Z'); + }); + + it('should accept a BigQueryDate|BigQueryDatetime|BigQueryTimestamp objects', () => { + const dateRange = bq.range({ + start: bq.date('2020-01-01'), + end: bq.date('2020-12-31'), + }); + assert.strictEqual(dateRange.value, INPUT_DATE_RANGE); + assert.strictEqual(dateRange.elementType, 'DATE'); + + const datetimeRange = bq.range({ + start: bq.datetime('2020-01-01 12:00:00'), + end: bq.datetime('2020-12-31 12:00:00'), + }); + assert.strictEqual(datetimeRange.value, INPUT_DATETIME_RANGE); + assert.strictEqual(datetimeRange.elementType, 'DATETIME'); + + const timestampRange = bq.range({ + start: bq.timestamp('2020-10-01 12:00:00+08'), + end: bq.timestamp('2020-12-31 12:00:00+08'), + }); + assert.strictEqual( + timestampRange.value, + '[2020-10-01T04:00:00.000Z, 2020-12-31T04:00:00.000Z)' + ); + assert.strictEqual(timestampRange.elementType, 'TIMESTAMP'); + }); + + it('should accept a start/end as string with element type', () => { + const dateRange = bq.range( + { + start: '2020-01-01', + end: '2020-12-31', + }, + 'DATE' + ); + assert.strictEqual(dateRange.value, INPUT_DATE_RANGE); + assert.strictEqual(dateRange.elementType, 'DATE'); + + const datetimeRange = bq.range( + { + start: '2020-01-01 12:00:00', + end: '2020-12-31 12:00:00', + }, + 'DATETIME' + ); + assert.strictEqual(datetimeRange.value, INPUT_DATETIME_RANGE); + assert.strictEqual(datetimeRange.elementType, 'DATETIME'); + + const timestampRange = bq.range( + { + start: '2020-10-01 12:00:00+08', + end: '2020-12-31 12:00:00+08', + }, + 'TIMESTAMP' + ); + assert.strictEqual( + timestampRange.value, + '[2020-10-01T04:00:00.000Z, 2020-12-31T04:00:00.000Z)' + ); + assert.strictEqual(timestampRange.elementType, 'TIMESTAMP'); + }); + }); + describe('geography', () => { const INPUT_STRING = 'POINT(1 2)'; From 95679bfcf3b6665a1355e89c4c6bf24efa74241f Mon Sep 17 00:00:00 2001 From: Alvaro Viebrantz Date: Fri, 5 Apr 2024 17:07:49 -0400 Subject: [PATCH 2/7] feat: support RANGE insertion and query param --- src/bigquery.ts | 35 +++++++++++++++++++++-- src/table.ts | 9 +++++- system-test/bigquery.ts | 62 ++++++++++++++++++++++++++++++++++++----- test/bigquery.ts | 11 ++++++++ test/table.ts | 9 ++++++ 5 files changed, 116 insertions(+), 10 deletions(-) diff --git a/src/bigquery.ts b/src/bigquery.ts index 7eb59b2f..bf20f12d 100644 --- a/src/bigquery.ts +++ b/src/bigquery.ts @@ -866,6 +866,7 @@ export class BigQuery extends Service { * * @method BigQuery.range * @param {string|BigQueryRangeOptions} value The range literal or start/end with dates/datetimes/timestamp ranges. + * @param {string} elementType The range element type - DATE|DATETIME|TIMESTAMP * * @example * ``` @@ -873,7 +874,10 @@ export class BigQuery extends Service { * const timestampRange = BigQuery.range('[2020-10-01 12:00:00+08, 2020-12-31 12:00:00+08)', 'TIMESTAMP'); * ``` */ - static range(value: string | BigQueryRangeOptions, elementType: string) { + static range( + value: string | BigQueryRangeOptions, + elementType?: string + ): BigQueryRange { return new BigQueryRange(value, elementType); } @@ -883,6 +887,7 @@ export class BigQuery extends Service { * The lower bound is inclusive and the upper bound is exclusive. * * @param {string|BigQueryRangeOptions} value The range literal or start/end with dates/datetimes/timestamp ranges. + * @param {string} elementType The range element type - DATE|DATETIME|TIMESTAMP * * @example * ``` @@ -891,7 +896,7 @@ export class BigQuery extends Service { * const timestampRange = bigquery.range('[2020-10-01 12:00:00+08, 2020-12-31 12:00:00+08)', 'TIMESTAMP'); * ``` */ - range(value: string, elementType: string) { + range(value: string, elementType?: string): BigQueryRange { return BigQuery.range(value, elementType); } @@ -1084,6 +1089,13 @@ export class BigQuery extends Service { typeName = 'INT64'; } else if (value instanceof Geography) { typeName = 'GEOGRAPHY'; + } else if (value instanceof BigQueryRange) { + return { + type: 'RANGE', + rangeElementType: { + type: value.elementType, + }, + }; } else if (Array.isArray(value)) { if (value.length === 0) { throw new Error( @@ -1190,6 +1202,24 @@ export class BigQuery extends Service { }, {} ); + } else if (typeName === 'RANGE') { + let rangeValue: bigquery.IRangeValue; + if (value instanceof BigQueryRange) { + rangeValue = value; + } else { + rangeValue = BigQuery.range( + value, + queryParameter.parameterType?.rangeElementType?.type + ); + } + queryParameter.parameterValue!.rangeValue = { + start: { + value: rangeValue.start && rangeValue.start.value, + }, + end: { + value: rangeValue.end && rangeValue.end.value, + }, + }; } else if (typeName === 'JSON' && is.object(value)) { queryParameter.parameterValue!.value = JSON.stringify(value); } else { @@ -1216,6 +1246,7 @@ export class BigQuery extends Service { type!.indexOf('TIME') > -1 || type!.indexOf('DATE') > -1 || type!.indexOf('GEOGRAPHY') > -1 || + type!.indexOf('RANGE') > -1 || type!.indexOf('BigQueryInt') > -1 ); } diff --git a/src/table.ts b/src/table.ts index 6897bebb..2854c304 100644 --- a/src/table.ts +++ b/src/table.ts @@ -52,7 +52,7 @@ import {GoogleErrorBody} from '@google-cloud/common/build/src/util'; import {Duplex, Writable} from 'stream'; import {JobMetadata} from './job'; import bigquery from './types'; -import {IntegerTypeCastOptions} from './bigquery'; +import {BigQueryRange, IntegerTypeCastOptions} from './bigquery'; import {RowQueue} from './rowQueue'; // eslint-disable-next-line @typescript-eslint/no-var-requires @@ -576,6 +576,13 @@ class Table extends ServiceObject { return value.toFixed(); } + if (value instanceof BigQueryRange) { + return { + start: value.start && value.start.value, + end: value.end && value.end.value, + }; + } + const customTypeConstructorNames = [ 'BigQueryDate', 'BigQueryDatetime', diff --git a/system-test/bigquery.ts b/system-test/bigquery.ts index fa3b21d5..0e6b9253 100644 --- a/system-test/bigquery.ts +++ b/system-test/bigquery.ts @@ -1332,6 +1332,25 @@ describe('BigQuery', () => { ); }); + it('should work with RANGE types', done => { + bigquery.query( + { + query: 'SELECT ? r', + params: [ + bigquery.range( + '[2020-10-01 12:00:00+08, 2020-12-31 12:00:00+08)', + 'TIMESTAMP' + ), + ], + }, + (err, rows) => { + assert.ifError(err); + assert.strictEqual(rows!.length, 1); + done(); + } + ); + }); + it('should work with multiple types', done => { bigquery.query( { @@ -1602,6 +1621,25 @@ describe('BigQuery', () => { ); }); + it('should work with RANGE types', done => { + bigquery.query( + { + query: 'SELECT @r r', + params: { + r: bigquery.range( + '[2020-10-01 12:00:00+08, 2020-12-31 12:00:00+08)', + 'TIMESTAMP' + ), + }, + }, + (err, rows) => { + assert.ifError(err); + assert.strictEqual(rows!.length, 1); + done(); + } + ); + }); + it('should work with multiple types', done => { bigquery.query( { @@ -1659,18 +1697,27 @@ describe('BigQuery', () => { const TIMESTAMP = bigquery.timestamp(new Date()); const NUMERIC = new Big('123.456'); const GEOGRAPHY = bigquery.geography('POINT(1 2)'); + const RANGE = bigquery.range( + '[2020-10-01 12:00:00+08, 2020-12-31 12:00:00+08)', + 'TIMESTAMP' + ); before(() => { table = dataset.table(generateName('table')); return table.create({ schema: [ - 'date:DATE', - 'datetime:DATETIME', - 'time:TIME', - 'timestamp:TIMESTAMP', - 'numeric:NUMERIC', - 'geography:GEOGRAPHY', - ].join(', '), + {name: 'date', type: 'DATE'}, + {name: 'datetime', type: 'DATETIME'}, + {name: 'time', type: 'TIME'}, + {name: 'timestamp', type: 'TIMESTAMP'}, + {name: 'numeric', type: 'NUMERIC'}, + {name: 'geography', type: 'GEOGRAPHY'}, + { + name: 'range', + type: 'RANGE', + rangeElementType: {type: 'TIMESTAMP'}, + }, + ], }); }); @@ -1682,6 +1729,7 @@ describe('BigQuery', () => { timestamp: TIMESTAMP, numeric: NUMERIC, geography: GEOGRAPHY, + range: RANGE, }); }); }); diff --git a/test/bigquery.ts b/test/bigquery.ts index 33d8767e..3e52ea29 100644 --- a/test/bigquery.ts +++ b/test/bigquery.ts @@ -1315,6 +1315,15 @@ describe('BigQuery', () => { BigQuery.getTypeDescriptorFromValue_(bq.geography('POINT (1 1')).type, 'GEOGRAPHY' ); + assert.strictEqual( + BigQuery.getTypeDescriptorFromValue_( + bq.range( + '[2020-10-01 12:00:00+08, 2020-12-31 12:00:00+08)', + 'TIMESTAMP' + ) + ).type, + 'RANGE' + ); }); it('should return correct type for an array', () => { @@ -1755,10 +1764,12 @@ describe('BigQuery', () => { const time = {type: 'TIME'}; const date = {type: 'DATE'}; const geo = {type: 'GEOGRAPHY'}; + const range = {type: 'RANGE'}; assert.strictEqual(BigQuery._isCustomType(time), true); assert.strictEqual(BigQuery._isCustomType(date), true); assert.strictEqual(BigQuery._isCustomType(geo), true); + assert.strictEqual(BigQuery._isCustomType(range), true); }); }); }); diff --git a/test/table.ts b/test/table.ts index 84d71ce5..f1d31530 100644 --- a/test/table.ts +++ b/test/table.ts @@ -373,6 +373,13 @@ describe('BigQuery/Table', () => { this.value = value; } } + + class BigQueryRange { + value: {}; + constructor(value: {}) { + this.value = value; + } + } class BigQueryInt { value: {}; constructor(value: {}) { @@ -385,12 +392,14 @@ describe('BigQuery/Table', () => { const time = new BigQueryTime('time'); const timestamp = new BigQueryTimestamp('timestamp'); const integer = new BigQueryInt('integer'); + const range = new BigQueryRange('range'); assert.strictEqual(Table.encodeValue_(date), 'date'); assert.strictEqual(Table.encodeValue_(datetime), 'datetime'); assert.strictEqual(Table.encodeValue_(time), 'time'); assert.strictEqual(Table.encodeValue_(timestamp), 'timestamp'); assert.strictEqual(Table.encodeValue_(integer), 'integer'); + assert.strictEqual(Table.encodeValue_(range), 'range'); }); it('should properly encode arrays', () => { From c8768f5a4550690633ffe13fd3a96d10b0588d05 Mon Sep 17 00:00:00 2001 From: Alvaro Viebrantz Date: Fri, 5 Apr 2024 17:13:56 -0400 Subject: [PATCH 3/7] fix: table encodeValue_ range --- test/table.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/test/table.ts b/test/table.ts index f1d31530..47d269f8 100644 --- a/test/table.ts +++ b/test/table.ts @@ -346,6 +346,15 @@ describe('BigQuery/Table', () => { const date = new Date(); assert.strictEqual(Table.encodeValue_(date), date.toJSON()); + + const range = BigQuery.range( + '[2020-10-01 12:00:00+08, 2020-12-31 12:00:00+08)', + 'TIMESTAMP' + ); + assert.deepEqual(Table.encodeValue_(range), { + start: '2020-10-01T04:00:00.000Z', + end: '2020-12-31T04:00:00.000Z', + }); }); it('should properly encode custom types', () => { @@ -373,13 +382,6 @@ describe('BigQuery/Table', () => { this.value = value; } } - - class BigQueryRange { - value: {}; - constructor(value: {}) { - this.value = value; - } - } class BigQueryInt { value: {}; constructor(value: {}) { @@ -392,14 +394,12 @@ describe('BigQuery/Table', () => { const time = new BigQueryTime('time'); const timestamp = new BigQueryTimestamp('timestamp'); const integer = new BigQueryInt('integer'); - const range = new BigQueryRange('range'); assert.strictEqual(Table.encodeValue_(date), 'date'); assert.strictEqual(Table.encodeValue_(datetime), 'datetime'); assert.strictEqual(Table.encodeValue_(time), 'time'); assert.strictEqual(Table.encodeValue_(timestamp), 'timestamp'); assert.strictEqual(Table.encodeValue_(integer), 'integer'); - assert.strictEqual(Table.encodeValue_(range), 'range'); }); it('should properly encode arrays', () => { From 96274d3193960fe0e61a5fa4a7d49ce486c30c54 Mon Sep 17 00:00:00 2001 From: Alvaro Viebrantz Date: Mon, 8 Apr 2024 14:19:01 -0400 Subject: [PATCH 4/7] feat: improve BigQueryRange class with value and literalValue --- src/bigquery.ts | 23 +++++++++++++++------- src/table.ts | 8 +------- test/bigquery.ts | 51 +++++++++++++++++++++++++++++++----------------- 3 files changed, 50 insertions(+), 32 deletions(-) diff --git a/src/bigquery.ts b/src/bigquery.ts index bf20f12d..b9db90af 100644 --- a/src/bigquery.ts +++ b/src/bigquery.ts @@ -1203,7 +1203,7 @@ export class BigQuery extends Service { {} ); } else if (typeName === 'RANGE') { - let rangeValue: bigquery.IRangeValue; + let rangeValue: BigQueryRange; if (value instanceof BigQueryRange) { rangeValue = value; } else { @@ -1211,13 +1211,13 @@ export class BigQuery extends Service { value, queryParameter.parameterType?.rangeElementType?.type ); - } + } queryParameter.parameterValue!.rangeValue = { start: { - value: rangeValue.start && rangeValue.start.value, + value: rangeValue.value.start, }, end: { - value: rangeValue.end && rangeValue.end.value, + value: rangeValue.value.end, }, }; } else if (typeName === 'JSON' && is.object(value)) { @@ -2479,8 +2479,7 @@ function convertSchemaFieldValue( export class BigQueryRange { elementType?: string; start?: BigQueryTimestamp | BigQueryDate | BigQueryDatetime; - end?: BigQueryTimestamp | BigQueryDate | BigQueryDatetime; - value: string; + end?: BigQueryTimestamp | BigQueryDate | BigQueryDatetime; constructor(value: string | BigQueryRangeOptions, elementType?: string) { if (typeof value === 'string') { if (!elementType) { @@ -2525,7 +2524,17 @@ export class BigQueryRange { this.end = this.convertElement_(end, inferredType); this.elementType = inferredType; } - this.value = `[${this.start ? this.start.value : 'UNBOUNDED'}, ${this.end ? this.end.value : 'UNBOUNDED'})`; + } + + public get literalValue() { + return `[${this.start ? this.start.value : 'UNBOUNDED'}, ${this.end ? this.end.value : 'UNBOUNDED'})`; + } + + public get value(){ + return { + start: this.start ? this.start.value : 'UNBOUNDED', + end: this.end ? this.end.value : 'UNBOUNDED' + } } static fromSchemaValue_(value: string, elementType: string): BigQueryRange { diff --git a/src/table.ts b/src/table.ts index 2854c304..dd30eca2 100644 --- a/src/table.ts +++ b/src/table.ts @@ -576,19 +576,13 @@ class Table extends ServiceObject { return value.toFixed(); } - if (value instanceof BigQueryRange) { - return { - start: value.start && value.start.value, - end: value.end && value.end.value, - }; - } - const customTypeConstructorNames = [ 'BigQueryDate', 'BigQueryDatetime', 'BigQueryInt', 'BigQueryTime', 'BigQueryTimestamp', + 'BigQueryRange', 'Geography', ]; const constructorName = value.constructor?.name; diff --git a/test/bigquery.ts b/test/bigquery.ts index 3e52ea29..d56bc565 100644 --- a/test/bigquery.ts +++ b/test/bigquery.ts @@ -955,28 +955,31 @@ describe('BigQuery', () => { it('should accept a string literal', () => { const dateRange = bq.range(INPUT_DATE_RANGE, 'DATE'); - assert.strictEqual(dateRange.value, '[2020-01-01, 2020-12-31)'); - assert.strictEqual(dateRange.start.value, '2020-01-01'); - assert.strictEqual(dateRange.end.value, '2020-12-31'); + assert.strictEqual(dateRange.literalValue, '[2020-01-01, 2020-12-31)'); + assert.deepStrictEqual(dateRange.value, { + start: '2020-01-01', + end: '2020-12-31', + }); const datetimeRange = bq.range(INPUT_DATETIME_RANGE, 'DATETIME'); assert.strictEqual( - datetimeRange.value, + datetimeRange.literalValue, '[2020-01-01 12:00:00, 2020-12-31 12:00:00)' ); - assert.strictEqual(datetimeRange.start.value, '2020-01-01 12:00:00'); - assert.strictEqual(datetimeRange.end.value, '2020-12-31 12:00:00'); + assert.deepStrictEqual(datetimeRange.value, { + start: '2020-01-01 12:00:00', + end: '2020-12-31 12:00:00', + }); const timestampRange = bq.range(INPUT_TIMESTAMP_RANGE, 'TIMESTAMP'); assert.strictEqual( - timestampRange.value, + timestampRange.literalValue, '[2020-10-01T04:00:00.000Z, 2020-12-31T04:00:00.000Z)' ); - assert.strictEqual( - timestampRange.start.value, - '2020-10-01T04:00:00.000Z' - ); - assert.strictEqual(timestampRange.end.value, '2020-12-31T04:00:00.000Z'); + assert.deepStrictEqual(timestampRange.value, { + start: '2020-10-01T04:00:00.000Z', + end: '2020-12-31T04:00:00.000Z', + }); }); it('should accept a BigQueryDate|BigQueryDatetime|BigQueryTimestamp objects', () => { @@ -984,25 +987,37 @@ describe('BigQuery', () => { start: bq.date('2020-01-01'), end: bq.date('2020-12-31'), }); - assert.strictEqual(dateRange.value, INPUT_DATE_RANGE); + assert.strictEqual(dateRange.literalValue, INPUT_DATE_RANGE); assert.strictEqual(dateRange.elementType, 'DATE'); + assert.deepStrictEqual(dateRange.value, { + start: '2020-01-01', + end: '2020-12-31', + }); const datetimeRange = bq.range({ start: bq.datetime('2020-01-01 12:00:00'), end: bq.datetime('2020-12-31 12:00:00'), }); - assert.strictEqual(datetimeRange.value, INPUT_DATETIME_RANGE); + assert.strictEqual(datetimeRange.literalValue, INPUT_DATETIME_RANGE); assert.strictEqual(datetimeRange.elementType, 'DATETIME'); + assert.deepStrictEqual(datetimeRange.value, { + start: '2020-01-01 12:00:00', + end: '2020-12-31 12:00:00', + }); const timestampRange = bq.range({ start: bq.timestamp('2020-10-01 12:00:00+08'), end: bq.timestamp('2020-12-31 12:00:00+08'), }); assert.strictEqual( - timestampRange.value, + timestampRange.literalValue, '[2020-10-01T04:00:00.000Z, 2020-12-31T04:00:00.000Z)' ); assert.strictEqual(timestampRange.elementType, 'TIMESTAMP'); + assert.deepStrictEqual(timestampRange.value, { + start: '2020-10-01T04:00:00.000Z', + end: '2020-12-31T04:00:00.000Z', + }); }); it('should accept a start/end as string with element type', () => { @@ -1013,7 +1028,7 @@ describe('BigQuery', () => { }, 'DATE' ); - assert.strictEqual(dateRange.value, INPUT_DATE_RANGE); + assert.strictEqual(dateRange.literalValue, INPUT_DATE_RANGE); assert.strictEqual(dateRange.elementType, 'DATE'); const datetimeRange = bq.range( @@ -1023,7 +1038,7 @@ describe('BigQuery', () => { }, 'DATETIME' ); - assert.strictEqual(datetimeRange.value, INPUT_DATETIME_RANGE); + assert.strictEqual(datetimeRange.literalValue, INPUT_DATETIME_RANGE); assert.strictEqual(datetimeRange.elementType, 'DATETIME'); const timestampRange = bq.range( @@ -1034,7 +1049,7 @@ describe('BigQuery', () => { 'TIMESTAMP' ); assert.strictEqual( - timestampRange.value, + timestampRange.literalValue, '[2020-10-01T04:00:00.000Z, 2020-12-31T04:00:00.000Z)' ); assert.strictEqual(timestampRange.elementType, 'TIMESTAMP'); From a2cc18bd75d1bd264f40a37bcedee6d433b31e62 Mon Sep 17 00:00:00 2001 From: Alvaro Viebrantz Date: Mon, 8 Apr 2024 14:21:45 -0400 Subject: [PATCH 5/7] fix: lint issues --- src/bigquery.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/bigquery.ts b/src/bigquery.ts index b9db90af..e6c228b3 100644 --- a/src/bigquery.ts +++ b/src/bigquery.ts @@ -1211,7 +1211,7 @@ export class BigQuery extends Service { value, queryParameter.parameterType?.rangeElementType?.type ); - } + } queryParameter.parameterValue!.rangeValue = { start: { value: rangeValue.value.start, @@ -2479,7 +2479,7 @@ function convertSchemaFieldValue( export class BigQueryRange { elementType?: string; start?: BigQueryTimestamp | BigQueryDate | BigQueryDatetime; - end?: BigQueryTimestamp | BigQueryDate | BigQueryDatetime; + end?: BigQueryTimestamp | BigQueryDate | BigQueryDatetime; constructor(value: string | BigQueryRangeOptions, elementType?: string) { if (typeof value === 'string') { if (!elementType) { @@ -2530,11 +2530,11 @@ export class BigQueryRange { return `[${this.start ? this.start.value : 'UNBOUNDED'}, ${this.end ? this.end.value : 'UNBOUNDED'})`; } - public get value(){ + public get value() { return { start: this.start ? this.start.value : 'UNBOUNDED', - end: this.end ? this.end.value : 'UNBOUNDED' - } + end: this.end ? this.end.value : 'UNBOUNDED', + }; } static fromSchemaValue_(value: string, elementType: string): BigQueryRange { From 40197b87afb373fe43020753f1d1c29c79edb55e Mon Sep 17 00:00:00 2001 From: Alvaro Viebrantz Date: Wed, 17 Apr 2024 11:34:09 -0400 Subject: [PATCH 6/7] fix: address review comments --- src/bigquery.ts | 44 +++++++++++++++++++++------------------ test/bigquery.ts | 54 ++++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 69 insertions(+), 29 deletions(-) diff --git a/src/bigquery.ts b/src/bigquery.ts index 7bc129ab..9b1ab73c 100644 --- a/src/bigquery.ts +++ b/src/bigquery.ts @@ -865,7 +865,7 @@ export class BigQuery extends Service { * The lower bound is inclusive and the upper bound is exclusive. * * @method BigQuery.range - * @param {string|BigQueryRangeOptions} value The range literal or start/end with dates/datetimes/timestamp ranges. + * @param {string|BigQueryRangeOptions} value The range API string or start/end with dates/datetimes/timestamp ranges. * @param {string} elementType The range element type - DATE|DATETIME|TIMESTAMP * * @example @@ -886,7 +886,7 @@ export class BigQuery extends Service { * The lower and upper bound for the range are optional. * The lower bound is inclusive and the upper bound is exclusive. * - * @param {string|BigQueryRangeOptions} value The range literal or start/end with dates/datetimes/timestamp ranges. + * @param {string|BigQueryRangeOptions} value The range API string or start/end with dates/datetimes/timestamp ranges. * @param {string} elementType The range element type - DATE|DATETIME|TIMESTAMP * * @example @@ -2481,28 +2481,16 @@ export class BigQueryRange { elementType?: string; start?: BigQueryTimestamp | BigQueryDate | BigQueryDatetime; end?: BigQueryTimestamp | BigQueryDate | BigQueryDatetime; + constructor(value: string | BigQueryRangeOptions, elementType?: string) { if (typeof value === 'string') { if (!elementType) { throw new Error( - 'invalid RANGE. Element type required when using RANGE literal string.' - ); - } - let cleanedValue = value; - if (cleanedValue.startsWith('[') || cleanedValue.startsWith('(')) { - cleanedValue = cleanedValue.substring(1); - } - if (cleanedValue.endsWith(')') || cleanedValue.endsWith(']')) { - cleanedValue = cleanedValue.substring(0, cleanedValue.length - 1); - } - const parts = cleanedValue.split(','); - if (parts.length !== 2) { - throw new Error( - 'invalid RANGE. See RANGE literal format docs for more information.' + 'invalid RANGE. Element type required when using RANGE API string.' ); } - const [start, end] = parts.map((s: string) => s.trim()); + const [start, end] = BigQueryRange.fromStringValue_(value); this.start = this.convertElement_(start, elementType); this.end = this.convertElement_(end, elementType); this.elementType = elementType; @@ -2527,10 +2515,21 @@ export class BigQueryRange { } } - public get literalValue() { + /* + * Get Range string representation used by the BigQuery API. + */ + public get apiValue() { return `[${this.start ? this.start.value : 'UNBOUNDED'}, ${this.end ? this.end.value : 'UNBOUNDED'})`; } + /* + * Get Range literal representation accordingly to + * https://cloud.google.com/bigquery/docs/reference/standard-sql/lexical#range_literals + */ + public get literalValue() { + return `RANGE<${this.elementType}> ${this.apiValue}`; + } + public get value() { return { start: this.start ? this.start.value : 'UNBOUNDED', @@ -2538,7 +2537,7 @@ export class BigQueryRange { }; } - static fromSchemaValue_(value: string, elementType: string): BigQueryRange { + private static fromStringValue_(value: string): [start: string, end: string] { let cleanedValue = value; if (cleanedValue.startsWith('[') || cleanedValue.startsWith('(')) { cleanedValue = cleanedValue.substring(1); @@ -2554,6 +2553,11 @@ export class BigQueryRange { } const [start, end] = parts.map((s: string) => s.trim()); + return [start, end]; + } + + static fromSchemaValue_(value: string, elementType: string): BigQueryRange { + const [start, end] = BigQueryRange.fromStringValue_(value); const convertRangeSchemaValue = (value: string) => { if (value === 'UNBOUNDED' || value === 'NULL') { return null; @@ -2571,7 +2575,7 @@ export class BigQueryRange { ); } - convertElement_( + private convertElement_( value?: string | BigQueryDate | BigQueryDatetime | BigQueryTimestamp, elementType?: string ) { diff --git a/test/bigquery.ts b/test/bigquery.ts index 14264451..4fcf93e6 100644 --- a/test/bigquery.ts +++ b/test/bigquery.ts @@ -987,7 +987,11 @@ describe('BigQuery', () => { it('should accept a string literal', () => { const dateRange = bq.range(INPUT_DATE_RANGE, 'DATE'); - assert.strictEqual(dateRange.literalValue, '[2020-01-01, 2020-12-31)'); + assert.strictEqual(dateRange.apiValue, '[2020-01-01, 2020-12-31)'); + assert.strictEqual( + dateRange.literalValue, + 'RANGE [2020-01-01, 2020-12-31)' + ); assert.deepStrictEqual(dateRange.value, { start: '2020-01-01', end: '2020-12-31', @@ -995,9 +999,13 @@ describe('BigQuery', () => { const datetimeRange = bq.range(INPUT_DATETIME_RANGE, 'DATETIME'); assert.strictEqual( - datetimeRange.literalValue, + datetimeRange.apiValue, '[2020-01-01 12:00:00, 2020-12-31 12:00:00)' ); + assert.strictEqual( + datetimeRange.literalValue, + 'RANGE [2020-01-01 12:00:00, 2020-12-31 12:00:00)' + ); assert.deepStrictEqual(datetimeRange.value, { start: '2020-01-01 12:00:00', end: '2020-12-31 12:00:00', @@ -1005,9 +1013,13 @@ describe('BigQuery', () => { const timestampRange = bq.range(INPUT_TIMESTAMP_RANGE, 'TIMESTAMP'); assert.strictEqual( - timestampRange.literalValue, + timestampRange.apiValue, '[2020-10-01T04:00:00.000Z, 2020-12-31T04:00:00.000Z)' ); + assert.strictEqual( + timestampRange.literalValue, + 'RANGE [2020-10-01T04:00:00.000Z, 2020-12-31T04:00:00.000Z)' + ); assert.deepStrictEqual(timestampRange.value, { start: '2020-10-01T04:00:00.000Z', end: '2020-12-31T04:00:00.000Z', @@ -1019,7 +1031,11 @@ describe('BigQuery', () => { start: bq.date('2020-01-01'), end: bq.date('2020-12-31'), }); - assert.strictEqual(dateRange.literalValue, INPUT_DATE_RANGE); + assert.strictEqual(dateRange.apiValue, INPUT_DATE_RANGE); + assert.strictEqual( + dateRange.literalValue, + `RANGE ${INPUT_DATE_RANGE}` + ); assert.strictEqual(dateRange.elementType, 'DATE'); assert.deepStrictEqual(dateRange.value, { start: '2020-01-01', @@ -1030,7 +1046,11 @@ describe('BigQuery', () => { start: bq.datetime('2020-01-01 12:00:00'), end: bq.datetime('2020-12-31 12:00:00'), }); - assert.strictEqual(datetimeRange.literalValue, INPUT_DATETIME_RANGE); + assert.strictEqual(datetimeRange.apiValue, INPUT_DATETIME_RANGE); + assert.strictEqual( + datetimeRange.literalValue, + `RANGE ${INPUT_DATETIME_RANGE}` + ); assert.strictEqual(datetimeRange.elementType, 'DATETIME'); assert.deepStrictEqual(datetimeRange.value, { start: '2020-01-01 12:00:00', @@ -1042,9 +1062,13 @@ describe('BigQuery', () => { end: bq.timestamp('2020-12-31 12:00:00+08'), }); assert.strictEqual( - timestampRange.literalValue, + timestampRange.apiValue, '[2020-10-01T04:00:00.000Z, 2020-12-31T04:00:00.000Z)' ); + assert.strictEqual( + timestampRange.literalValue, + 'RANGE [2020-10-01T04:00:00.000Z, 2020-12-31T04:00:00.000Z)' + ); assert.strictEqual(timestampRange.elementType, 'TIMESTAMP'); assert.deepStrictEqual(timestampRange.value, { start: '2020-10-01T04:00:00.000Z', @@ -1060,7 +1084,11 @@ describe('BigQuery', () => { }, 'DATE' ); - assert.strictEqual(dateRange.literalValue, INPUT_DATE_RANGE); + assert.strictEqual(dateRange.apiValue, INPUT_DATE_RANGE); + assert.strictEqual( + dateRange.literalValue, + `RANGE ${INPUT_DATE_RANGE}` + ); assert.strictEqual(dateRange.elementType, 'DATE'); const datetimeRange = bq.range( @@ -1070,7 +1098,11 @@ describe('BigQuery', () => { }, 'DATETIME' ); - assert.strictEqual(datetimeRange.literalValue, INPUT_DATETIME_RANGE); + assert.strictEqual(datetimeRange.apiValue, INPUT_DATETIME_RANGE); + assert.strictEqual( + datetimeRange.literalValue, + `RANGE ${INPUT_DATETIME_RANGE}` + ); assert.strictEqual(datetimeRange.elementType, 'DATETIME'); const timestampRange = bq.range( @@ -1081,9 +1113,13 @@ describe('BigQuery', () => { 'TIMESTAMP' ); assert.strictEqual( - timestampRange.literalValue, + timestampRange.apiValue, '[2020-10-01T04:00:00.000Z, 2020-12-31T04:00:00.000Z)' ); + assert.strictEqual( + timestampRange.literalValue, + 'RANGE [2020-10-01T04:00:00.000Z, 2020-12-31T04:00:00.000Z)' + ); assert.strictEqual(timestampRange.elementType, 'TIMESTAMP'); }); }); From 2caf0864aa94d3f80f5f352b86b49ff11c25663c Mon Sep 17 00:00:00 2001 From: Alvaro Viebrantz Date: Wed, 17 Apr 2024 11:41:41 -0400 Subject: [PATCH 7/7] test: add test for missing range input --- test/bigquery.ts | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/test/bigquery.ts b/test/bigquery.ts index 4fcf93e6..4d760931 100644 --- a/test/bigquery.ts +++ b/test/bigquery.ts @@ -1122,6 +1122,36 @@ describe('BigQuery', () => { ); assert.strictEqual(timestampRange.elementType, 'TIMESTAMP'); }); + + it('should accept a Range with start and/or end missing', () => { + const dateRange = bq.range( + { + start: '2020-01-01', + }, + 'DATE' + ); + assert.strictEqual( + dateRange.literalValue, + 'RANGE [2020-01-01, UNBOUNDED)' + ); + + const datetimeRange = bq.range( + { + end: '2020-12-31 12:00:00', + }, + 'DATETIME' + ); + assert.strictEqual( + datetimeRange.literalValue, + 'RANGE [UNBOUNDED, 2020-12-31 12:00:00)' + ); + + const timestampRange = bq.range({}, 'TIMESTAMP'); + assert.strictEqual( + timestampRange.literalValue, + 'RANGE [UNBOUNDED, UNBOUNDED)' + ); + }); }); describe('geography', () => {