fix: Modify client lib retry policy for CreateWriteStream with longer…

… backoff, more error code and longer overall time (#279)

feat: add fields to eventually contain row level errors
Committer: @gnanda
PiperOrigin-RevId: 456324780
Source-Link: googleapis/googleapis@f24b37a
Source-Link: googleapis/googleapis-gen@33f9d81
Copy-Tag: eyJwIjoiLmdpdGh1Yi8uT3dsQm90LnlhbWwiLCJoIjoiMzNmOWQ4MTQwODIxMTcxMTZjNGI2OGE2ZjVhYWMzZjQyYmVjMzVjMiJ9
feat: support regapic LRO
PiperOrigin-RevId: 456946341
Source-Link: googleapis/googleapis@88fd18d
Source-Link: googleapis/googleapis-gen@accfa37
Copy-Tag: eyJwIjoiLmdpdGh1Yi8uT3dsQm90LnlhbWwiLCJoIjoiYWNjZmEzNzFmNjY3NDM5MzEzMzM1YzY0MDQyYjA2M2MxYzUzMTAyZSJ9
See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md
fix: Modify client lib retry policy for CreateWriteStream with longer backoff, more error code and longer overall time
PiperOrigin-RevId: 457061436
Source-Link: googleapis/googleapis@8ff130b
Source-Link: googleapis/googleapis-gen@2eb0fac
Copy-Tag: eyJwIjoiLmdpdGh1Yi8uT3dsQm90LnlhbWwiLCJoIjoiMmViMGZhY2E3MTdkOWNmNDRiODM4YjdkYjVlODYyNDUxYjhhODZlZiJ9
See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md
Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>

protos/google/cloud/bigquery/storage/v1/storage.proto

@@ -248,11 +248,13 @@ message CreateReadSessionRequest {
   // Max initial number of streams. If unset or zero, the server will
   // provide a value of streams so as to produce reasonable throughput. Must be
   // non-negative. The number of streams may be lower than the requested number,
-  // depending on the amount parallelism that is reasonable for the table. Error
-  // will be returned if the max count is greater than the current system
-  // max limit of 1,000.
+  // depending on the amount parallelism that is reasonable for the table.
+  // There is a default system max limit of 1,000.
   //
-  // Streams must be read starting from offset 0.
+  // This must be greater than or equal to preferred_min_stream_count.
+  // Typically, clients should either leave this unset to let the system to
+  // determine an upper bound OR set this a size for the maximum "units of work"
+  // it can gracefully handle.
   int32 max_stream_count = 3;
 }
 
@@ -329,7 +331,7 @@ message ReadRowsResponse {
 
   // The schema for the read. If read_options.selected_fields is set, the
   // schema may be different from the table schema as it will only contain
-  // the selected fields. This schema is equivelant to the one returned by
+  // the selected fields. This schema is equivalent to the one returned by
   // CreateSession. This field is only populated in the first ReadRowsResponse
   // RPC.
   oneof schema {
@@ -488,6 +490,11 @@ message AppendRowsResponse {
   // use it to input new type of message. It will be empty when no schema
   // updates have occurred.
   TableSchema updated_schema = 3;
+
+  // If a request failed due to corrupted rows, no rows in the batch will be
+  // appended. The API will return row level error info, so that the caller can
+  // remove the bad rows and retry the request.
+  repeated RowError row_errors = 4;
 }
 
 // Request message for `GetWriteStreamRequest`.
@@ -622,3 +629,24 @@ message StorageError {
   // Message that describes the error.
   string error_message = 3;
 }
+
+// The message that presents row level error info in a request.
+message RowError {
+  // Error code for `RowError`.
+  enum RowErrorCode {
+    // Default error.
+    ROW_ERROR_CODE_UNSPECIFIED = 0;
+
+    // One or more fields in the row has errors.
+    FIELDS_ERROR = 1;
+  }
+
+  // Index of the malformed row in the request.
+  int64 index = 1;
+
+  // Structured error reason for a row error.
+  RowErrorCode code = 2;
+
+  // Description of the issue encountered when processing the row.
+  string message = 3;
+}

protos/google/cloud/bigquery/storage/v1/stream.proto

@@ -32,6 +32,7 @@ option php_namespace = "Google\\Cloud\\BigQuery\\Storage\\V1";
 
 // Data format for input or output data.
 enum DataFormat {
+  // Data format is unspecified.
   DATA_FORMAT_UNSPECIFIED = 0;
 
   // Avro is a standard open source row based file format.
@@ -91,7 +92,7 @@ message ReadSession {
   // automatically assigned and currently cannot be specified or updated.
   google.protobuf.Timestamp expire_time = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
 
-  // Immutable. Data format of the output data.
+  // Immutable. Data format of the output data. DATA_FORMAT_UNSPECIFIED not supported.
   DataFormat data_format = 3 [(google.api.field_behavior) = IMMUTABLE];
 
   // The schema for the read. If read_options.selected_fields is set, the

protos/google/cloud/bigquery/storage/v1/table.proto

@@ -25,7 +25,9 @@ option java_outer_classname = "TableProto";
 option java_package = "com.google.cloud.bigquery.storage.v1";
 option php_namespace = "Google\\Cloud\\BigQuery\\Storage\\V1";
 
-// Schema of a table.
+// Schema of a table. This schema is a subset of
+// google.cloud.bigquery.v2.TableSchema containing information necessary to
+// generate valid message to write to BigQuery.
 message TableSchema {
   // Describes the fields in a table.
   repeated TableFieldSchema fields = 1;