Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: [datastream] Max concurrent backfill tasks #4254

Merged
merged 4 commits into from
May 16, 2023
Merged

feat: [datastream] Max concurrent backfill tasks #4254

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
01160fe
feat: Max concurrent backfill tasks
gcf-owl-bot[bot] May 7, 2023
e85366d
🦉 Updates from OwlBot post-processor
gcf-owl-bot[bot] May 7, 2023
ac3fd4a
Merge branch 'main' into owl-bot-copy-packages-google-cloud-datastream
gcf-merge-on-green[bot] May 10, 2023
42c0ae8
Merge branch 'main' into owl-bot-copy-packages-google-cloud-datastream
ddelgrosso1 May 16, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
99 changes: 49 additions & 50 deletions packages/google-cloud-datastream/README.md
View file
Open in desktop

Large diffs are not rendered by default.

119 changes: 76 additions & 43 deletions packages/google-cloud-datastream/protos/google/cloud/datastream/v1/datastream.proto
View file
Open in desktop

Large diffs are not rendered by default.

118 changes: 72 additions & 46 deletions ...ages/google-cloud-datastream/protos/google/cloud/datastream/v1/datastream_resources.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2022 Google LLC
// Copyright 2023 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -103,14 +103,12 @@ message GcsProfile {
}

// BigQuery warehouse profile.
message BigQueryProfile {
message BigQueryProfile {}

}

// Static IP address connectivity.
message StaticServiceIpConnectivity {

}
// Static IP address connectivity. Used when the source database is configured
// to allow incoming connections from the Datastream public IP addresses
// for the region specified in the connection profile.
message StaticServiceIpConnectivity {}

// Forward SSH Tunnel connectivity.
message ForwardSshTunnelConnectivity {
Expand Down Expand Up @@ -181,10 +179,12 @@ message PrivateConnection {
string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY];

// Output only. The create time of the resource.
google.protobuf.Timestamp create_time = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
google.protobuf.Timestamp create_time = 2
[(google.api.field_behavior) = OUTPUT_ONLY];

// Output only. The update time of the resource.
google.protobuf.Timestamp update_time = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
google.protobuf.Timestamp update_time = 3
[(google.api.field_behavior) = OUTPUT_ONLY];

// Labels.
map<string, string> labels = 4;
Expand All @@ -195,7 +195,8 @@ message PrivateConnection {
// Output only. The state of the Private Connection.
State state = 6 [(google.api.field_behavior) = OUTPUT_ONLY];

// Output only. In case of error, the details of the error in a user-friendly format.
// Output only. In case of error, the details of the error in a user-friendly
// format.
Error error = 7 [(google.api.field_behavior) = OUTPUT_ONLY];

// VPC Peering Config.
Expand Down Expand Up @@ -226,10 +227,12 @@ message Route {
string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY];

// Output only. The create time of the resource.
google.protobuf.Timestamp create_time = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
google.protobuf.Timestamp create_time = 2
[(google.api.field_behavior) = OUTPUT_ONLY];

// Output only. The update time of the resource.
google.protobuf.Timestamp update_time = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
google.protobuf.Timestamp update_time = 3
[(google.api.field_behavior) = OUTPUT_ONLY];

// Labels.
map<string, string> labels = 4;
Expand Down Expand Up @@ -262,8 +265,8 @@ message MysqlSslConfig {
// Output only. Indicates whether the client_certificate field is set.
bool client_certificate_set = 4 [(google.api.field_behavior) = OUTPUT_ONLY];

// Input only. PEM-encoded certificate of the CA that signed the source database
// server's certificate.
// Input only. PEM-encoded certificate of the CA that signed the source
// database server's certificate.
string ca_certificate = 5 [(google.api.field_behavior) = INPUT_ONLY];

// Output only. Indicates whether the ca_certificate field is set.
Expand All @@ -282,10 +285,12 @@ message ConnectionProfile {
string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY];

// Output only. The create time of the resource.
google.protobuf.Timestamp create_time = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
google.protobuf.Timestamp create_time = 2
[(google.api.field_behavior) = OUTPUT_ONLY];

// Output only. The update time of the resource.
google.protobuf.Timestamp update_time = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
google.protobuf.Timestamp update_time = 3
[(google.api.field_behavior) = OUTPUT_ONLY];

// Labels.
map<string, string> labels = 4;
Expand Down Expand Up @@ -383,31 +388,31 @@ message OracleRdbms {
// Oracle data source configuration
message OracleSourceConfig {
// Configuration to drop large object values.
message DropLargeObjects {

}
message DropLargeObjects {}

// Configuration to stream large object values.
message StreamLargeObjects {

}
message StreamLargeObjects {}

// Oracle objects to include in the stream.
OracleRdbms include_objects = 1;

// Oracle objects to exclude from the stream.
OracleRdbms exclude_objects = 2;

// Maximum number of concurrent CDC tasks. The number should be non negative.
// If not set (or set to 0), the system's default value will be used.
// Maximum number of concurrent CDC tasks. The number should be non-negative.
// If not set (or set to 0), the system's default value is used.
int32 max_concurrent_cdc_tasks = 3;

// Maximum number of concurrent backfill tasks. The number should be
// non-negative. If not set (or set to 0), the system's default value is used.
int32 max_concurrent_backfill_tasks = 4;

// The configuration for handle Oracle large objects.
oneof large_objects_handling {
// Drop large object values.
DropLargeObjects drop_large_objects = 100;

// Stream large object values.
// Stream large object values. NOTE: This feature is currently experimental.
StreamLargeObjects stream_large_objects = 102;
}
}
Expand Down Expand Up @@ -473,13 +478,21 @@ message PostgresqlSourceConfig {
// PostgreSQL objects to exclude from the stream.
PostgresqlRdbms exclude_objects = 2;

// Required. The name of the logical replication slot that's configured with the
// pgoutput plugin.
string replication_slot = 3 [(google.api.field_behavior) = REQUIRED];
// Required. Immutable. The name of the logical replication slot that's
// configured with the pgoutput plugin.
string replication_slot = 3 [
(google.api.field_behavior) = REQUIRED,
(google.api.field_behavior) = IMMUTABLE
];

// Required. The name of the publication that includes the set of all tables that are
// defined in the stream's include_objects.
// Required. The name of the publication that includes the set of all tables
// that are defined in the stream's include_objects.
string publication = 4 [(google.api.field_behavior) = REQUIRED];

// Maximum number of concurrent backfill tasks. The number should be non
// negative. If not set (or set to 0), the system's default value will be
// used.
int32 max_concurrent_backfill_tasks = 5;
}

// MySQL Column.
Expand Down Expand Up @@ -544,6 +557,11 @@ message MysqlSourceConfig {
// Maximum number of concurrent CDC tasks. The number should be non negative.
// If not set (or set to 0), the system's default value will be used.
int32 max_concurrent_cdc_tasks = 3;

// Maximum number of concurrent backfill tasks. The number should be non
// negative. If not set (or set to 0), the system's default value will be
// used.
int32 max_concurrent_backfill_tasks = 4;
}

// The configuration of the stream source.
Expand Down Expand Up @@ -571,9 +589,7 @@ message SourceConfig {
}

// AVRO file format configuration.
message AvroFileFormat {

}
message AvroFileFormat {}

// JSON file format configuration.
message JsonFileFormat {
Expand Down Expand Up @@ -617,7 +633,8 @@ message GcsDestinationConfig {
int32 file_rotation_mb = 2;

// The maximum duration for which new events are added before a file is
// closed and a new file is created.
// closed and a new file is created. Values within the range of 15-60 seconds
// are allowed.
google.protobuf.Duration file_rotation_interval = 3;

// File Format that the data should be written in.
Expand All @@ -630,9 +647,11 @@ message GcsDestinationConfig {
}
}

// BigQuery destination configuration
message BigQueryDestinationConfig {
// A single target dataset to which all data will be streamed.
message SingleTargetDataset {
// The dataset ID of the target dataset.
string dataset_id = 1;
}

Expand Down Expand Up @@ -661,6 +680,7 @@ message BigQueryDestinationConfig {
string kms_key_name = 3;
}

// The dataset template to use for dynamic dataset creation.
DatasetTemplate dataset_template = 2;
}

Expand Down Expand Up @@ -759,18 +779,18 @@ message Stream {
}

// Backfill strategy to disable automatic backfill for the Stream's objects.
message BackfillNoneStrategy {

}
message BackfillNoneStrategy {}

// Output only. The stream's name.
string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY];

// Output only. The creation time of the stream.
google.protobuf.Timestamp create_time = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
google.protobuf.Timestamp create_time = 2
[(google.api.field_behavior) = OUTPUT_ONLY];

// Output only. The last update time of the stream.
google.protobuf.Timestamp update_time = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
google.protobuf.Timestamp update_time = 3
[(google.api.field_behavior) = OUTPUT_ONLY];

// Labels.
map<string, string> labels = 4;
Expand All @@ -782,7 +802,8 @@ message Stream {
SourceConfig source_config = 6 [(google.api.field_behavior) = REQUIRED];

// Required. Destination connection profile configuration.
DestinationConfig destination_config = 7 [(google.api.field_behavior) = REQUIRED];
DestinationConfig destination_config = 7
[(google.api.field_behavior) = REQUIRED];

// The state of the stream.
State state = 8;
Expand All @@ -804,7 +825,8 @@ message Stream {
// If provided, it will be used to encrypt the data.
// If left blank, data will be encrypted using an internal Stream-specific
// encryption key provisioned through KMS.
optional string customer_managed_encryption_key = 10 [(google.api.field_behavior) = IMMUTABLE];
optional string customer_managed_encryption_key = 10
[(google.api.field_behavior) = IMMUTABLE];
}

// A specific stream object (e.g a specific DB table).
Expand All @@ -818,10 +840,12 @@ message StreamObject {
string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY];

// Output only. The creation time of the object.
google.protobuf.Timestamp create_time = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
google.protobuf.Timestamp create_time = 2
[(google.api.field_behavior) = OUTPUT_ONLY];

// Output only. The last update time of the object.
google.protobuf.Timestamp update_time = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
google.protobuf.Timestamp update_time = 3
[(google.api.field_behavior) = OUTPUT_ONLY];

// Required. Display name.
string display_name = 5 [(google.api.field_behavior) = REQUIRED];
Expand Down Expand Up @@ -930,10 +954,12 @@ message BackfillJob {
Trigger trigger = 2;

// Output only. Backfill job's start time.
google.protobuf.Timestamp last_start_time = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
google.protobuf.Timestamp last_start_time = 3
[(google.api.field_behavior) = OUTPUT_ONLY];

// Output only. Backfill job's end time.
google.protobuf.Timestamp last_end_time = 4 [(google.api.field_behavior) = OUTPUT_ONLY];
google.protobuf.Timestamp last_end_time = 4
[(google.api.field_behavior) = OUTPUT_ONLY];

// Output only. Errors which caused the backfill job to fail.
repeated Error errors = 5 [(google.api.field_behavior) = OUTPUT_ONLY];
Expand Down
24 changes: 24 additions & 0 deletions packages/google-cloud-datastream/protos/protos.d.ts
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading