feat: Add Grounding feature to PredictionService.GenerateContent

packages/google-cloud-aiplatform/README.md

@@ -562,6 +562,7 @@ Samples are in the [`samples/`](https://github.com/googleapis/google-cloud-node/
 | Persistent_resource_service.get_persistent_resource | [source code](https://github.com/googleapis/google-cloud-node/blob/main/packages/google-cloud-aiplatform/samples/generated/v1beta1/persistent_resource_service.get_persistent_resource.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/google-cloud-node&page=editor&open_in_editor=packages/google-cloud-aiplatform/samples/generated/v1beta1/persistent_resource_service.get_persistent_resource.js,packages/google-cloud-aiplatform/samples/README.md) |
 | Persistent_resource_service.list_persistent_resources | [source code](https://github.com/googleapis/google-cloud-node/blob/main/packages/google-cloud-aiplatform/samples/generated/v1beta1/persistent_resource_service.list_persistent_resources.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/google-cloud-node&page=editor&open_in_editor=packages/google-cloud-aiplatform/samples/generated/v1beta1/persistent_resource_service.list_persistent_resources.js,packages/google-cloud-aiplatform/samples/README.md) |
 | Persistent_resource_service.update_persistent_resource | [source code](https://github.com/googleapis/google-cloud-node/blob/main/packages/google-cloud-aiplatform/samples/generated/v1beta1/persistent_resource_service.update_persistent_resource.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/google-cloud-node&page=editor&open_in_editor=packages/google-cloud-aiplatform/samples/generated/v1beta1/persistent_resource_service.update_persistent_resource.js,packages/google-cloud-aiplatform/samples/README.md) |
+| Pipeline_service.batch_cancel_pipeline_jobs | [source code](https://github.com/googleapis/google-cloud-node/blob/main/packages/google-cloud-aiplatform/samples/generated/v1beta1/pipeline_service.batch_cancel_pipeline_jobs.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/google-cloud-node&page=editor&open_in_editor=packages/google-cloud-aiplatform/samples/generated/v1beta1/pipeline_service.batch_cancel_pipeline_jobs.js,packages/google-cloud-aiplatform/samples/README.md) |
 | Pipeline_service.batch_delete_pipeline_jobs | [source code](https://github.com/googleapis/google-cloud-node/blob/main/packages/google-cloud-aiplatform/samples/generated/v1beta1/pipeline_service.batch_delete_pipeline_jobs.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/google-cloud-node&page=editor&open_in_editor=packages/google-cloud-aiplatform/samples/generated/v1beta1/pipeline_service.batch_delete_pipeline_jobs.js,packages/google-cloud-aiplatform/samples/README.md) |
 | Pipeline_service.cancel_pipeline_job | [source code](https://github.com/googleapis/google-cloud-node/blob/main/packages/google-cloud-aiplatform/samples/generated/v1beta1/pipeline_service.cancel_pipeline_job.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/google-cloud-node&page=editor&open_in_editor=packages/google-cloud-aiplatform/samples/generated/v1beta1/pipeline_service.cancel_pipeline_job.js,packages/google-cloud-aiplatform/samples/README.md) |
 | Pipeline_service.cancel_training_pipeline | [source code](https://github.com/googleapis/google-cloud-node/blob/main/packages/google-cloud-aiplatform/samples/generated/v1beta1/pipeline_service.cancel_training_pipeline.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/google-cloud-node&page=editor&open_in_editor=packages/google-cloud-aiplatform/samples/generated/v1beta1/pipeline_service.cancel_training_pipeline.js,packages/google-cloud-aiplatform/samples/README.md) |

packages/google-cloud-aiplatform/protos/google/cloud/aiplatform/v1/custom_job.proto

@@ -349,9 +349,4 @@ message Scheduling {
   // job starts running. If true, overrides
   // `Scheduling.restart_job_on_worker_restart` to false.
   bool disable_retries = 5 [(google.api.field_behavior) = OPTIONAL];
-
-  // Optional. This is the maximum time a user will wait in the QRM queue for
-  // resources. Default is 1 day
-  google.protobuf.Duration max_wait_duration = 6
-      [(google.api.field_behavior) = OPTIONAL];
 }

packages/google-cloud-aiplatform/protos/google/cloud/aiplatform/v1/explanation.proto

@@ -44,6 +44,14 @@ message Explanation {
   // [Attribution.output_index][google.cloud.aiplatform.v1.Attribution.output_index]
   // can be used to identify which output this attribution is explaining.
   //
+  // By default, we provide Shapley values for the predicted class. However,
+  // you can configure the explanation request to generate Shapley values for
+  // any other classes too. For example, if a model predicts a probability of
+  // `0.4` for approving a loan application, the model's decision is to reject
+  // the application since `p(reject) = 0.6 > p(approve) = 0.4`, and the default
+  // Shapley values would be computed for rejection decision and not approval,
+  // even though the latter might be the positive class.
+  //
   // If users set
   // [ExplanationParameters.top_k][google.cloud.aiplatform.v1.ExplanationParameters.top_k],
   // the attributions are sorted by

packages/google-cloud-aiplatform/protos/google/cloud/aiplatform/v1/feature_online_store_admin_service.proto

@@ -319,6 +319,7 @@ message UpdateFeatureOnlineStoreRequest {
   // Updatable fields:
   //
   //   * `big_query_source`
+  //   * `bigtable`
   //   * `labels`
   //   * `sync_config`
   google.protobuf.FieldMask update_mask = 2;
@@ -480,6 +481,7 @@ message UpdateFeatureViewRequest {
   // Updatable fields:
   //
   //   * `labels`
+  //   * `serviceAgentType`
   google.protobuf.FieldMask update_mask = 2;
 }
 

packages/google-cloud-aiplatform/protos/google/cloud/aiplatform/v1/index_endpoint.proto

@@ -291,4 +291,9 @@ message IndexPrivateEndpoints {
   // Output only. The name of the service attachment resource. Populated if
   // private service connect is enabled.
   string service_attachment = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
+
+  // Output only. PscAutomatedEndpoints is populated if private service connect
+  // is enabled if PscAutomatedConfig is set.
+  repeated PscAutomatedEndpoints psc_automated_endpoints = 3
+      [(google.api.field_behavior) = OUTPUT_ONLY];
 }

packages/google-cloud-aiplatform/protos/google/cloud/aiplatform/v1/index_service.proto

@@ -252,6 +252,16 @@ message UpsertDatapointsRequest {
 
   // A list of datapoints to be created/updated.
   repeated IndexDatapoint datapoints = 2;
+
+  // Optional. Update mask is used to specify the fields to be overwritten in
+  // the datapoints by the update. The fields specified in the update_mask are
+  // relative to each IndexDatapoint inside datapoints, not the full request.
+  //
+  // Updatable fields:
+  //
+  //   * Use `all_restricts` to update both restricts and numeric_restricts.
+  google.protobuf.FieldMask update_mask = 3
+      [(google.api.field_behavior) = OPTIONAL];
 }
 
 // Response message for
@@ -307,6 +317,25 @@ message NearestNeighborSearchOperationMetadata {
 
       // The `namespace` field is missing.
       NAMESPACE_MISSING = 7;
+
+      // Generic catch-all error. Only used for validation failure where the
+      // root cause cannot be easily retrieved programmatically.
+      PARSING_ERROR = 8;
+
+      // There are multiple restricts with the same `namespace` value.
+      DUPLICATE_NAMESPACE = 9;
+
+      // Numeric restrict has operator specified in datapoint.
+      OP_IN_DATAPOINT = 10;
+
+      // Numeric restrict has multiple values specified.
+      MULTIPLE_VALUES = 11;
+
+      // Numeric restrict has invalid numeric value specified.
+      INVALID_NUMERIC_VALUE = 12;
+
+      // File is not in UTF_8 format.
+      INVALID_ENCODING = 13;
     }
 
     // The error type of this record.

packages/google-cloud-aiplatform/protos/google/cloud/aiplatform/v1/model.proto

@@ -249,12 +249,12 @@ message Model {
   // deploying this Model. The specification is ingested upon
   // [ModelService.UploadModel][google.cloud.aiplatform.v1.ModelService.UploadModel],
   // and all binaries it contains are copied and stored internally by Vertex AI.
-  // Not present for AutoML Models or Large Models.
+  // Not required for AutoML Models.
   ModelContainerSpec container_spec = 9
       [(google.api.field_behavior) = INPUT_ONLY];
 
   // Immutable. The path to the directory containing the Model artifact and any
-  // of its supporting files. Not present for AutoML Models or Large Models.
+  // of its supporting files. Not required for AutoML Models.
   string artifact_uri = 26 [(google.api.field_behavior) = IMMUTABLE];
 
   // Output only. When this Model is deployed, its prediction resources are
@@ -437,7 +437,8 @@ message Model {
   EncryptionSpec encryption_spec = 24;
 
   // Output only. Source of a model. It can either be automl training pipeline,
-  // custom training pipeline, BigQuery ML, or existing Vertex AI Model.
+  // custom training pipeline, BigQuery ML, or saved and tuned from Genie or
+  // Model Garden.
   ModelSourceInfo model_source_info = 38
       [(google.api.field_behavior) = OUTPUT_ONLY];
 
@@ -766,6 +767,10 @@ message Port {
 // Detail description of the source information of the model.
 message ModelSourceInfo {
   // Source of the model.
+  // Different from `objective` field, this `ModelSourceType` enum
+  // indicates the source from which the model was accessed or obtained,
+  // whereas the `objective` indicates the overall aim or function of this
+  // model.
   enum ModelSourceType {
     // Should not be used.
     MODEL_SOURCE_TYPE_UNSPECIFIED = 0;
@@ -787,6 +792,9 @@ message ModelSourceInfo {
 
     // The Model is uploaded by text embedding finetuning pipeline.
     CUSTOM_TEXT_EMBEDDING = 6;
+
+    // The Model is saved or tuned from Marketplace.
+    MARKETPLACE = 7;
   }
 
   // Type of the model source.

packages/google-cloud-aiplatform/protos/google/cloud/aiplatform/v1/model_deployment_monitoring_job.proto

@@ -263,6 +263,11 @@ message ModelDeploymentMonitoringBigQueryTable {
   // & analysis. Format:
   // `bq://<project_id>.model_deployment_monitoring_<endpoint_id>.<tolower(log_source)>_<tolower(log_type)>`
   string bigquery_table_path = 3;
+
+  // Output only. The schema version of the request/response logging BigQuery
+  // table. Default to v1 if unset.
+  string request_response_logging_schema_version = 4
+      [(google.api.field_behavior) = OUTPUT_ONLY];
 }
 
 // ModelDeploymentMonitoringObjectiveConfig contains the pair of

packages/google-cloud-aiplatform/protos/google/cloud/aiplatform/v1/prediction_service.proto

@@ -653,8 +653,7 @@ message GenerateContentRequest {
   //
   // A `Tool` is a piece of code that enables the system to interact with
   // external systems to perform an action, or set of actions, outside of
-  // knowledge and scope of the model. The only supported tool is currently
-  // `Function`
+  // knowledge and scope of the model.
   repeated Tool tools = 6 [(google.api.field_behavior) = OPTIONAL];
 
   // Optional. Per request settings for blocking unsafe content.

packages/google-cloud-aiplatform/protos/google/cloud/aiplatform/v1/publisher_model.proto

@@ -46,10 +46,10 @@ message PublisherModel {
       string resource_name = 2;
 
       // Use case (CUJ) of the resource.
-      string use_case = 3;
+      string use_case = 3 [deprecated = true];
 
       // Description of the resource.
-      string description = 4;
+      string description = 4 [deprecated = true];
     }
   }
 
@@ -72,8 +72,20 @@ message PublisherModel {
       map<string, ResourceReference> references = 1
           [(google.api.field_behavior) = REQUIRED];
 
-      // Required. The title of the regional resource reference.
+      // Required.
       string title = 2 [(google.api.field_behavior) = REQUIRED];
+
+      // Optional. Title of the resource.
+      optional string resource_title = 3
+          [(google.api.field_behavior) = OPTIONAL];
+
+      // Optional. Use case (CUJ) of the resource.
+      optional string resource_use_case = 4
+          [(google.api.field_behavior) = OPTIONAL];
+
+      // Optional. Description of the resource.
+      optional string resource_description = 5
+          [(google.api.field_behavior) = OPTIONAL];
     }
 
     // Rest API docs.
@@ -86,6 +98,20 @@ message PublisherModel {
       string title = 2 [(google.api.field_behavior) = REQUIRED];
     }
 
+    // Open notebooks.
+    message OpenNotebooks {
+      // Required. Regional resource references to notebooks.
+      repeated RegionalResourceReferences notebooks = 1
+          [(google.api.field_behavior) = REQUIRED];
+    }
+
+    // Open fine tuning pipelines.
+    message OpenFineTuningPipelines {
+      // Required. Regional resource references to fine tuning pipelines.
+      repeated RegionalResourceReferences fine_tuning_pipelines = 1
+          [(google.api.field_behavior) = REQUIRED];
+    }
+
     // Model metadata that is needed for UploadModel or
     // DeployModel/CreateEndpoint requests.
     message Deploy {
@@ -131,13 +157,24 @@ message PublisherModel {
       string public_artifact_uri = 9 [(google.api.field_behavior) = OPTIONAL];
     }
 
+    // Configurations for PublisherModel GKE deployment
+    message DeployGke {
+      // Optional. GKE deployment configuration in yaml format.
+      repeated string gke_yaml_configs = 1
+          [(google.api.field_behavior) = OPTIONAL];
+    }
+
     // Optional. To view Rest API docs.
     ViewRestApi view_rest_api = 1 [(google.api.field_behavior) = OPTIONAL];
 
     // Optional. Open notebook of the PublisherModel.
     RegionalResourceReferences open_notebook = 2
         [(google.api.field_behavior) = OPTIONAL];
 
+    // Optional. Open notebooks of the PublisherModel.
+    optional OpenNotebooks open_notebooks = 12
+        [(google.api.field_behavior) = OPTIONAL];
+
     // Optional. Create application using the PublisherModel.
     RegionalResourceReferences create_application = 3
         [(google.api.field_behavior) = OPTIONAL];
@@ -146,6 +183,10 @@ message PublisherModel {
     RegionalResourceReferences open_fine_tuning_pipeline = 4
         [(google.api.field_behavior) = OPTIONAL];
 
+    // Optional. Open fine-tuning pipelines of the PublisherModel.
+    optional OpenFineTuningPipelines open_fine_tuning_pipelines = 13
+        [(google.api.field_behavior) = OPTIONAL];
+
     // Optional. Open prompt-tuning pipeline of the PublisherModel.
     RegionalResourceReferences open_prompt_tuning_pipeline = 5
         [(google.api.field_behavior) = OPTIONAL];
@@ -157,6 +198,9 @@ message PublisherModel {
     // Optional. Deploy the PublisherModel to Vertex Endpoint.
     Deploy deploy = 7 [(google.api.field_behavior) = OPTIONAL];
 
+    // Optional. Deploy PublisherModel to Google Kubernetes Engine.
+    DeployGke deploy_gke = 14 [(google.api.field_behavior) = OPTIONAL];
+
     // Optional. Open in Generation AI Studio.
     RegionalResourceReferences open_generation_ai_studio = 8
         [(google.api.field_behavior) = OPTIONAL];

packages/google-cloud-aiplatform/protos/google/cloud/aiplatform/v1/service_networking.proto

@@ -37,3 +37,16 @@ message PrivateServiceConnectConfig {
   // attachment.
   repeated string project_allowlist = 2;
 }
+
+// PscAutomatedEndpoints defines the output of the forwarding rule
+// automatically created by each PscAutomationConfig.
+message PscAutomatedEndpoints {
+  // Corresponding project_id in pscAutomationConfigs
+  string project_id = 1;
+
+  // Corresponding network in pscAutomationConfigs.
+  string network = 2;
+
+  // Ip Address created by the automated forwarding rule.
+  string match_address = 3;
+}

packages/google-cloud-aiplatform/protos/google/cloud/aiplatform/v1beta1/content.proto

@@ -291,4 +291,60 @@ message Candidate {
   // Output only. Source attribution of the generated content.
   CitationMetadata citation_metadata = 6
       [(google.api.field_behavior) = OUTPUT_ONLY];
+
+  // Output only. Metadata specifies sources used to ground generated content.
+  GroundingMetadata grounding_metadata = 7
+      [(google.api.field_behavior) = OUTPUT_ONLY];
+}
+
+// Segment of the content.
+message Segment {
+  // Output only. The index of a Part object within its parent Content object.
+  int32 part_index = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
+
+  // Output only. Start index in the given Part, measured in bytes. Offset from
+  // the start of the Part, inclusive, starting at zero.
+  int32 start_index = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
+
+  // Output only. End index in the given Part, measured in bytes. Offset from
+  // the start of the Part, exclusive, starting at zero.
+  int32 end_index = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
+}
+
+// Grounding attribution.
+message GroundingAttribution {
+  // Attribution from the web.
+  message Web {
+    // Output only. URI reference of the attribution.
+    string uri = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
+
+    // Output only. Title of the attribution.
+    string title = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
+  }
+
+  oneof reference {
+    // Optional. Attribution from the web.
+    Web web = 3 [(google.api.field_behavior) = OPTIONAL];
+  }
+
+  // Output only. Segment of the content this attribution belongs to.
+  Segment segment = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
+
+  // Optional. Output only. Confidence score of the attribution. Ranges from 0
+  // to 1. 1 is the most confident.
+  optional float confidence_score = 2 [
+    (google.api.field_behavior) = OPTIONAL,
+    (google.api.field_behavior) = OUTPUT_ONLY
+  ];
+}
+
+// Metadata returned to client when grounding is enabled.
+message GroundingMetadata {
+  // Optional. Web search queries for the following-up web search.
+  repeated string web_search_queries = 1
+      [(google.api.field_behavior) = OPTIONAL];
+
+  // Optional. List of grounding attributions.
+  repeated GroundingAttribution grounding_attributions = 2
+      [(google.api.field_behavior) = OPTIONAL];
 }

packages/google-cloud-aiplatform/protos/google/cloud/aiplatform/v1beta1/custom_job.proto

@@ -363,9 +363,4 @@ message Scheduling {
   // job starts running. If true, overrides
   // `Scheduling.restart_job_on_worker_restart` to false.
   bool disable_retries = 5 [(google.api.field_behavior) = OPTIONAL];
-
-  // Optional. This is the maximum time a user will wait in the QRM queue for
-  // resources. Default is 1 day
-  google.protobuf.Duration max_wait_duration = 6
-      [(google.api.field_behavior) = OPTIONAL];
 }

packages/google-cloud-aiplatform/protos/google/cloud/aiplatform/v1beta1/endpoint_service.proto

@@ -220,6 +220,7 @@ message ListEndpointsRequest {
   //     * A key including a space must be quoted. `labels."a key"`.
   //
   // Some examples:
+  //
   //   * `endpoint=1`
   //   * `displayName="myDisplayName"`
   //   * `labels.myKey="myValue"`

packages/google-cloud-aiplatform/protos/google/cloud/aiplatform/v1beta1/explanation.proto

@@ -44,6 +44,14 @@ message Explanation {
   // [Attribution.output_index][google.cloud.aiplatform.v1beta1.Attribution.output_index]
   // can be used to identify which output this attribution is explaining.
   //
+  // By default, we provide Shapley values for the predicted class. However,
+  // you can configure the explanation request to generate Shapley values for
+  // any other classes too. For example, if a model predicts a probability of
+  // `0.4` for approving a loan application, the model's decision is to reject
+  // the application since `p(reject) = 0.6 > p(approve) = 0.4`, and the default
+  // Shapley values would be computed for rejection decision and not approval,
+  // even though the latter might be the positive class.
+  //
   // If users set
   // [ExplanationParameters.top_k][google.cloud.aiplatform.v1beta1.ExplanationParameters.top_k],
   // the attributions are sorted by

packages/google-cloud-aiplatform/protos/google/cloud/aiplatform/v1beta1/feature_online_store_admin_service.proto

@@ -319,6 +319,7 @@ message UpdateFeatureOnlineStoreRequest {
   // Updatable fields:
   //
   //   * `big_query_source`
+  //   * `bigtable`
   //   * `labels`
   //   * `sync_config`
   google.protobuf.FieldMask update_mask = 2;
@@ -480,6 +481,7 @@ message UpdateFeatureViewRequest {
   // Updatable fields:
   //
   //   * `labels`
+  //   * `serviceAgentType`
   google.protobuf.FieldMask update_mask = 2;
 }