From 817b2f092f4241f53c4b52c0683f2d9949e2be32 Mon Sep 17 00:00:00 2001 From: Jacob Cable Date: Fri, 17 Jan 2025 13:38:39 +0000 Subject: [PATCH] fix(firestore-bigquery-export): do not cluster invalid types --- firestore-bigquery-export/CHANGELOG.md | 4 ++++ firestore-bigquery-export/README.md | 4 +++- firestore-bigquery-export/extension.yaml | 18 ++++++++++++------ .../functions/package.json | 2 +- 4 files changed, 20 insertions(+), 8 deletions(-) diff --git a/firestore-bigquery-export/CHANGELOG.md b/firestore-bigquery-export/CHANGELOG.md index 9bbd348c7..d9c90d39e 100644 --- a/firestore-bigquery-export/CHANGELOG.md +++ b/firestore-bigquery-export/CHANGELOG.md @@ -1,3 +1,7 @@ +## Version 0.1.57 + +fix - do not add/update clustering if an invalid clustering field is present. + ## Version 0.1.56 feat - improve sync strategy by immediately writing to BQ, and using cloud tasks only as a last resort diff --git a/firestore-bigquery-export/README.md b/firestore-bigquery-export/README.md index b40ce5318..864ad2fc4 100644 --- a/firestore-bigquery-export/README.md +++ b/firestore-bigquery-export/README.md @@ -183,7 +183,9 @@ essential for the script to insert data into an already partitioned table.) * BigQuery SQL Time Partitioning table schema field(column) type: Parameter for BigQuery SQL schema field type for the selected Time Partitioning Firestore Document field option. Cannot be changed if Table is already partitioned. -* BigQuery SQL table clustering: This parameter will allow you to set up Clustering for the BigQuery Table created by the extension. (for example: `data,document_id,timestamp`- no whitespaces). You can select up to 4 comma separated fields. The order of the specified columns determines the sort order of the data. Available schema extensions table fields for clustering: `document_id, document_name, timestamp, event_id, operation, data`. +* BigQuery SQL table clustering: This parameter allows you to set up clustering for the BigQuery table created by the extension. Specify up to 4 comma-separated fields (for example: `data,document_id,timestamp` - no whitespaces). The order of the specified columns determines the sort order of the data. +Note: Cluster columns must be top-level, non-repeated columns of one of the following types: BIGNUMERIC, BOOL, DATE, DATETIME, GEOGRAPHY, INT64, NUMERIC, RANGE, STRING, TIMESTAMP. Clustering will not be added if a field with an invalid type is present in this parameter. +Available schema extensions table fields for clustering include: `document_id, document_name, timestamp, event_id, operation, data`. * Maximum number of synced documents per second: This parameter will set the maximum number of syncronised documents per second with BQ. Please note, any other external updates to a Big Query table will be included within this quota. Ensure that you have a set a low enough number to compensate. Defaults to 10. diff --git a/firestore-bigquery-export/extension.yaml b/firestore-bigquery-export/extension.yaml index 8e7e3e8c8..ee4f1615e 100644 --- a/firestore-bigquery-export/extension.yaml +++ b/firestore-bigquery-export/extension.yaml @@ -310,12 +310,18 @@ params: - param: CLUSTERING label: BigQuery SQL table clustering description: >- - This parameter will allow you to set up Clustering for the BigQuery Table - created by the extension. (for example: `data,document_id,timestamp`- no - whitespaces). You can select up to 4 comma separated fields. The order of - the specified columns determines the sort order of the data. Available - schema extensions table fields for clustering: `document_id, - document_name, timestamp, event_id, operation, data`. + This parameter allows you to set up clustering for the BigQuery table + created by the extension. Specify up to 4 comma-separated fields (for example: + `data,document_id,timestamp` - no whitespaces). The order of the specified + columns determines the sort order of the data. + + Note: Cluster columns must be top-level, non-repeated columns of one of the + following types: BIGNUMERIC, BOOL, DATE, DATETIME, GEOGRAPHY, INT64, NUMERIC, + RANGE, STRING, TIMESTAMP. Clustering will not be added if a field with an invalid + type is present in this parameter. + + Available schema extensions table fields for clustering include: `document_id, document_name, timestamp, event_id, + operation, data`. type: string validationRegex: ^[^,\s]+(?:,[^,\s]+){0,3}$ validationErrorMessage: diff --git a/firestore-bigquery-export/functions/package.json b/firestore-bigquery-export/functions/package.json index 5450224fd..51a9136c8 100644 --- a/firestore-bigquery-export/functions/package.json +++ b/firestore-bigquery-export/functions/package.json @@ -13,7 +13,7 @@ "author": "Jan Wyszynski ", "license": "Apache-2.0", "dependencies": { - "@firebaseextensions/firestore-bigquery-change-tracker": "^1.1.38", + "@firebaseextensions/firestore-bigquery-change-tracker": "^1.1.39", "@google-cloud/bigquery": "^7.6.0", "@types/chai": "^4.1.6", "@types/express-serve-static-core": "4.17.30",