-
Notifications
You must be signed in to change notification settings - Fork 4.3k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Add argument checks and tests for BQ StorageAPI sinks. #27213
Changes from all commits
172eb1b
1def740
15e909d
1d44573
bc144d0
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -3138,16 +3138,44 @@ public WriteResult expand(PCollection<T> input) { | |
: getTriggeringFrequency(); | ||
checkArgument( | ||
triggeringFrequency != null, | ||
"When writing an unbounded PCollection via FILE_LOADS or STORAGE_API_WRITES, " | ||
"When writing an unbounded PCollection via FILE_LOADS or STORAGE_WRITE_API, " | ||
+ "triggering frequency must be specified"); | ||
} else { | ||
checkArgument( | ||
getTriggeringFrequency() == null && getNumFileShards() == 0, | ||
"Triggering frequency or number of file shards can be specified only when writing an" | ||
+ " unbounded PCollection via FILE_LOADS or STORAGE_API_WRITES, but: the collection" | ||
+ " was %s and the method was %s", | ||
getTriggeringFrequency() == null, | ||
"Triggering frequency can be specified only when writing an unbounded PCollection via" | ||
+ " FILE_LOADS or STORAGE_WRITE_API, but the collection was %s and the method was" | ||
+ " %s.", | ||
input.isBounded(), | ||
method); | ||
|
||
if (method == Method.STORAGE_WRITE_API) { | ||
BigQueryOptions bqOptions = input.getPipeline().getOptions().as(BigQueryOptions.class); | ||
if (getStorageApiTriggeringFrequency(bqOptions) != null) { | ||
LOG.warn( | ||
"The setting of storageApiTriggeringFrequency in BigQueryOptions is ignored." | ||
+ " It is only supported when writing an unbounded PCollection via" | ||
+ " STORAGE_WRITE_API, but the collection was {} and the method was {}.", | ||
input.isBounded(), | ||
method); | ||
} | ||
} | ||
|
||
checkArgument( | ||
(getNumFileShards() == 0), | ||
"Number of file shards can be specified only when writing an unbounded PCollection via" | ||
+ " FILE_LOADS, but the collection was %s and the method was %s", | ||
input.isBounded(), | ||
method); | ||
|
||
if (getNumStorageWriteApiStreams() != 0) { | ||
LOG.warn( | ||
"The setting of numStorageWriteApiStreams is ignored. It can be specified only" | ||
+ " when writing an unbounded PCollection via STORAGE_WRITE_API, but the collection" | ||
+ " was {} and the method was {}.", | ||
input.isBounded(), | ||
method); | ||
} | ||
} | ||
|
||
if (method != Method.STORAGE_WRITE_API && method != Method.STORAGE_API_AT_LEAST_ONCE) { | ||
|
@@ -3174,6 +3202,27 @@ public WriteResult expand(PCollection<T> input) { | |
|
||
if (input.isBounded() == IsBounded.BOUNDED) { | ||
checkArgument(!getAutoSharding(), "Auto-sharding is only applicable to unbounded input."); | ||
} else { | ||
if (method == Method.STORAGE_WRITE_API) { | ||
if (getNumStorageWriteApiStreams() > 0 && getAutoSharding()) { | ||
LOG.warn( | ||
"The setting of numStorageWriteApiStreams is ignored. It is only supported when" | ||
+ " autoSharding is not enabled."); | ||
} | ||
} else if (method == Method.FILE_LOADS) { | ||
if (getNumFileShards() > 0 && getAutoSharding()) { | ||
LOG.warn( | ||
"The setting of numFileShards is ignored. It is only supported when autoSharding is" | ||
+ " not enabled."); | ||
} | ||
} else if (method != Method.STREAMING_INSERTS) { | ||
LOG.warn( | ||
"The setting of auto-sharding is ignored. It is only supported when writing an" | ||
+ " unbounded PCollection via FILE_LOADS, STREAMING_INSERTS or" | ||
+ " STORAGE_WRITE_API, but the collection was {} and the method was {}.", | ||
input.isBounded(), | ||
method); | ||
} | ||
Comment on lines
+3218
to
+3225
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think at this point it makes more sense to check |
||
} | ||
|
||
if (getJsonTimePartitioning() != null) { | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -570,4 +570,17 @@ public Table getTableResource(String projectId, String datasetId, String tableId | |
MAX_QUERY_RETRIES, tableId), | ||
lastException); | ||
} | ||
|
||
public void updateTableSchema( | ||
String projectId, String datasetId, String tableId, TableSchema newSchema) { | ||
try { | ||
this.bqClient | ||
.tables() | ||
.patch(projectId, datasetId, tableId, new Table().setSchema(newSchema)) | ||
.execute(); | ||
LOG.info("Successfully updated the schema of table: " + tableId); | ||
} catch (Exception e) { | ||
LOG.debug("Exceptions caught when updating table schema: " + e.getMessage()); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Severity should maybe be raised to info/warning for visibility? |
||
} | ||
} | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
nit: Could be more concise; using
STORAGE_WRITE_API
isn't the problem here