feat: [dialogflow] add baseline model configuration for conversation …

…summarization (#4255) * feat: extended StreamingListCallCompanionEvents timeout to 600 seconds feat: added debug info for StreamingDetectIntent feat: added GenerateStatelessSummary method PiperOrigin-RevId: 530102437 Source-Link: googleapis/googleapis@cea4ed9 Source-Link: googleapis/googleapis-gen@624b26c Copy-Tag: eyJwIjoicGFja2FnZXMvZ29vZ2xlLWNsb3VkLWRpYWxvZ2Zsb3cvLk93bEJvdC55YW1sIiwiaCI6IjYyNGIyNmMwOTlmYjJjNjAxNzQxZDc3NDAxNTUwMmJlZjk5ZTFiZWEifQ== * feat: extended StreamingListCallCompanionEvents timeout to 600 seconds feat: added debug info for StreamingDetectIntent feat: added GenerateStatelessSummary method PiperOrigin-RevId: 530104294 Source-Link: googleapis/googleapis@92bec94 Source-Link: googleapis/googleapis-gen@69bf7b3 Copy-Tag: eyJwIjoicGFja2FnZXMvZ29vZ2xlLWNsb3VkLWRpYWxvZ2Zsb3cvLk93bEJvdC55YW1sIiwiaCI6IjY5YmY3YjNkNTQxZDFhMmY0YjMwZDI5YWRkOGFlYmU0ZTJhZDEzYzUifQ== * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * feat: add baseline model configuration for conversation summarization PiperOrigin-RevId: 530747460 Source-Link: googleapis/googleapis@4048e76 Source-Link: googleapis/googleapis-gen@82d3834 Copy-Tag: eyJwIjoicGFja2FnZXMvZ29vZ2xlLWNsb3VkLWRpYWxvZ2Zsb3cvLk93bEJvdC55YW1sIiwiaCI6IjgyZDM4MzQ4NzNkZGRhOWQzY2VjZGFlYzY1ZTllNmJhNzdlMzc2MDAifQ== * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * feat: add baseline model configuration for conversation summarization PiperOrigin-RevId: 530749453 Source-Link: googleapis/googleapis@620a023 Source-Link: googleapis/googleapis-gen@300d638 Copy-Tag: eyJwIjoicGFja2FnZXMvZ29vZ2xlLWNsb3VkLWRpYWxvZ2Zsb3cvLk93bEJvdC55YW1sIiwiaCI6IjMwMGQ2MzgxOWMyYTI3ZWNkN2YzNWVkZjg4MDUxYjQ0YWIyOTE4MjgifQ== * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --------- Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com> Co-authored-by: sofisl <[email protected]> Co-authored-by: Denis DelGrosso <[email protected]>
googleapis · May 16, 2023 · 88877b9 · 88877b9
1 parent f2c4a78
commit 88877b9
Show file tree

Hide file tree

Showing 65 changed files with 25,066 additions and 18,169 deletions.
diff --git a/packages/google-cloud-dialogflow/README.md b/packages/google-cloud-dialogflow/README.md
diff --git a/packages/google-cloud-dialogflow/protos/google/cloud/dialogflow/v2/agent.proto b/packages/google-cloud-dialogflow/protos/google/cloud/dialogflow/v2/agent.proto
@@ -1,4 +1,4 @@
-// Copyright 2022 Google LLC
+// Copyright 2023 Google LLC
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

diff --git a/packages/google-cloud-dialogflow/protos/google/cloud/dialogflow/v2/answer_record.proto b/packages/google-cloud-dialogflow/protos/google/cloud/dialogflow/v2/answer_record.proto
@@ -1,4 +1,4 @@
-// Copyright 2022 Google LLC
+// Copyright 2023 Google LLC
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -208,6 +208,8 @@ message AnswerFeedback {
 
   // Indicates whether the answer/item was clicked by the human agent
   // or not. Default to false.
+  // For knowledge search, the answer record is considered to be clicked if the
+  // answer was copied or any URI was clicked.
   bool clicked = 3;
 
   // Time when the answer/item was clicked.

diff --git a/packages/google-cloud-dialogflow/protos/google/cloud/dialogflow/v2/audio_config.proto b/packages/google-cloud-dialogflow/protos/google/cloud/dialogflow/v2/audio_config.proto
@@ -1,4 +1,4 @@
-// Copyright 2022 Google LLC
+// Copyright 2023 Google LLC
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -241,6 +241,12 @@ message InputAudioConfig {
   // [Cloud Speech API
   // documentation](https://cloud.google.com/speech-to-text/docs/basics#select-model)
   // for more details.
+  // If you specify a model, the following models typically have the best
+  // performance:
+  //
+  // - phone_call (best for Agent Assist and telephony)
+  // - latest_short (best for Dialogflow non-telephony)
+  // - command_and_search (best for very short utterances and commands)
   string model = 7;
 
   // Which variant of the [Speech
@@ -266,6 +272,9 @@ message InputAudioConfig {
   // If `false` and recognition doesn't return any result, trigger
   // `NO_SPEECH_RECOGNIZED` event to Dialogflow agent.
   bool disable_no_speech_recognized_event = 14;
+
+  // Enable automatic punctuation option at the speech backend.
+  bool enable_automatic_punctuation = 17;
 }
 
 // Gender of the voice as described in

diff --git a/packages/google-cloud-dialogflow/protos/google/cloud/dialogflow/v2/context.proto b/packages/google-cloud-dialogflow/protos/google/cloud/dialogflow/v2/context.proto
@@ -1,4 +1,4 @@
-// Copyright 2022 Google LLC
+// Copyright 2023 Google LLC
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

diff --git a/packages/google-cloud-dialogflow/protos/google/cloud/dialogflow/v2/conversation.proto b/packages/google-cloud-dialogflow/protos/google/cloud/dialogflow/v2/conversation.proto
@@ -1,4 +1,4 @@
-// Copyright 2022 Google LLC
+// Copyright 2023 Google LLC
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -20,6 +20,7 @@ import "google/api/annotations.proto";
 import "google/api/client.proto";
 import "google/api/field_behavior.proto";
 import "google/api/resource.proto";
+import "google/cloud/dialogflow/v2/conversation_profile.proto";
 import "google/cloud/dialogflow/v2/participant.proto";
 import "google/protobuf/timestamp.proto";
 
@@ -139,6 +140,20 @@ service Conversations {
     };
     option (google.api.method_signature) = "conversation";
   }
+
+  // Generates and returns a summary for a conversation that does not have a
+  // resource created for it.
+  rpc GenerateStatelessSummary(GenerateStatelessSummaryRequest)
+      returns (GenerateStatelessSummaryResponse) {
+    option (google.api.http) = {
+      post: "/v2/{stateless_conversation.parent=projects/*}/suggestions:generateStatelessSummary"
+      body: "*"
+      additional_bindings {
+        post: "/v2/{stateless_conversation.parent=projects/*/locations/*}/suggestions:generateStatelessSummary"
+        body: "*"
+      }
+    };
+  }
 }
 
 // Represents a conversation.
@@ -461,3 +476,82 @@ message SuggestConversationSummaryResponse {
   // request if there weren't that many messages in the conversation.
   int32 context_size = 3;
 }
+
+// The request message for
+// [Conversations.GenerateStatelessSummary][google.cloud.dialogflow.v2.Conversations.GenerateStatelessSummary].
+message GenerateStatelessSummaryRequest {
+  // The minimum amount of information required to generate a Summary without
+  // having a Conversation resource created.
+  message MinimalConversation {
+    // Required. The messages that the Summary will be generated from. It is
+    // expected that this message content is already redacted and does not
+    // contain any PII. Required fields: {content, language_code, participant,
+    // participant_role} Optional fields: {send_time} If send_time is not
+    // provided, then the messages must be provided in chronological order.
+    repeated Message messages = 1 [(google.api.field_behavior) = REQUIRED];
+
+    // Required. The parent resource to charge for the Summary's generation.
+    // Format: `projects/<Project ID>/locations/<Location ID>`.
+    string parent = 2 [
+      (google.api.field_behavior) = REQUIRED,
+      (google.api.resource_reference) = {
+        type: "locations.googleapis.com/Location"
+      }
+    ];
+  }
+
+  // Required. The conversation to suggest a summary for.
+  MinimalConversation stateless_conversation = 1
+      [(google.api.field_behavior) = REQUIRED];
+
+  // Required. A ConversationProfile containing information required for Summary
+  // generation.
+  // Required fields: {language_code, security_settings}
+  // Optional fields: {agent_assistant_config}
+  ConversationProfile conversation_profile = 2
+      [(google.api.field_behavior) = REQUIRED];
+
+  // The name of the latest conversation message used as context for
+  // generating a Summary. If empty, the latest message of the conversation will
+  // be used. The format is specific to the user and the names of the messages
+  // provided.
+  string latest_message = 3 [(google.api.resource_reference) = {
+    type: "dialogflow.googleapis.com/Message"
+  }];
+
+  // Max number of messages prior to and including
+  // [latest_message] to use as context when compiling the
+  // suggestion. By default 500 and at most 1000.
+  int32 max_context_size = 4;
+}
+
+// The response message for
+// [Conversations.GenerateStatelessSummary][google.cloud.dialogflow.v2.Conversations.GenerateStatelessSummary].
+message GenerateStatelessSummaryResponse {
+  // Generated summary for a conversation.
+  message Summary {
+    // The summary content that is concatenated into one string.
+    string text = 1;
+
+    // The summary content that is divided into sections. The key is the
+    // section's name and the value is the section's content. There is no
+    // specific format for the key or value.
+    map<string, string> text_sections = 2;
+  }
+
+  // Generated summary.
+  Summary summary = 1;
+
+  // The name of the latest conversation message used as context for
+  // compiling suggestion. The format is specific to the user and the names of
+  // the messages provided.
+  string latest_message = 2 [(google.api.resource_reference) = {
+    type: "dialogflow.googleapis.com/Message"
+  }];
+
+  // Number of messages prior to and including
+  // [last_conversation_message][] used to compile the suggestion. It may be
+  // smaller than the [GenerateStatelessSummaryRequest.context_size][] field in
+  // the request if there weren't that many messages in the conversation.
+  int32 context_size = 3;
+}
diff --git a/...ages/google-cloud-dialogflow/protos/google/cloud/dialogflow/v2/conversation_dataset.proto b/...ages/google-cloud-dialogflow/protos/google/cloud/dialogflow/v2/conversation_dataset.proto
@@ -1,4 +1,4 @@
-// Copyright 2022 Google LLC
+// Copyright 2023 Google LLC
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

diff --git a/packages/google-cloud-dialogflow/protos/google/cloud/dialogflow/v2/conversation_event.proto b/packages/google-cloud-dialogflow/protos/google/cloud/dialogflow/v2/conversation_event.proto
@@ -1,4 +1,4 @@
-// Copyright 2022 Google LLC
+// Copyright 2023 Google LLC
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

diff --git a/packages/google-cloud-dialogflow/protos/google/cloud/dialogflow/v2/conversation_model.proto b/packages/google-cloud-dialogflow/protos/google/cloud/dialogflow/v2/conversation_model.proto
@@ -1,4 +1,4 @@
-// Copyright 2022 Google LLC
+// Copyright 2023 Google LLC
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

diff --git a/...ages/google-cloud-dialogflow/protos/google/cloud/dialogflow/v2/conversation_profile.proto b/...ages/google-cloud-dialogflow/protos/google/cloud/dialogflow/v2/conversation_profile.proto
@@ -1,4 +1,4 @@
-// Copyright 2022 Google LLC
+// Copyright 2023 Google LLC
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -558,6 +558,16 @@ message HumanAgentAssistantConfig {
     string model = 1 [(google.api.resource_reference) = {
       type: "dialogflow.googleapis.com/ConversationModel"
     }];
+
+    // Version of current baseline model. It will be ignored if
+    // [model][google.cloud.dialogflow.v2.HumanAgentAssistantConfig.ConversationModelConfig.model]
+    // is set. Valid versions are:
+    //   Article Suggestion baseline model:
+    //     - 0.9
+    //     - 1.0 (default)
+    //   Summarization baseline model:
+    //     - 1.0
+    string baseline_model_version = 8;
   }
 
   // Config to process conversation.

diff --git a/packages/google-cloud-dialogflow/protos/google/cloud/dialogflow/v2/document.proto b/packages/google-cloud-dialogflow/protos/google/cloud/dialogflow/v2/document.proto
@@ -1,4 +1,4 @@
-// Copyright 2022 Google LLC
+// Copyright 2023 Google LLC
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -582,8 +582,13 @@ message ReloadDocumentRequest {
   ];
 
   // The source for document reloading.
+  //
   // If provided, the service will load the contents from the source
   // and update document in the knowledge base.
+  //
+  // Reloading from a new document source is allowed for smart messaging
+  // documents only. If you want to update the source for other document types,
+  // please delete the existing document and create a new one instead.
   oneof source {
     // Optional. The path of gcs source file for reloading document content. For
     // now, only gcs uri is supported.

diff --git a/packages/google-cloud-dialogflow/protos/google/cloud/dialogflow/v2/entity_type.proto b/packages/google-cloud-dialogflow/protos/google/cloud/dialogflow/v2/entity_type.proto
@@ -1,4 +1,4 @@
-// Copyright 2022 Google LLC
+// Copyright 2023 Google LLC
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

diff --git a/packages/google-cloud-dialogflow/protos/google/cloud/dialogflow/v2/environment.proto b/packages/google-cloud-dialogflow/protos/google/cloud/dialogflow/v2/environment.proto
@@ -1,4 +1,4 @@
-// Copyright 2022 Google LLC
+// Copyright 2023 Google LLC
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

diff --git a/packages/google-cloud-dialogflow/protos/google/cloud/dialogflow/v2/fulfillment.proto b/packages/google-cloud-dialogflow/protos/google/cloud/dialogflow/v2/fulfillment.proto
@@ -1,4 +1,4 @@
-// Copyright 2022 Google LLC
+// Copyright 2023 Google LLC
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

diff --git a/packages/google-cloud-dialogflow/protos/google/cloud/dialogflow/v2/gcs.proto b/packages/google-cloud-dialogflow/protos/google/cloud/dialogflow/v2/gcs.proto
@@ -1,4 +1,4 @@
-// Copyright 2022 Google LLC
+// Copyright 2023 Google LLC
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

diff --git a/...ogle-cloud-dialogflow/protos/google/cloud/dialogflow/v2/human_agent_assistant_event.proto b/...ogle-cloud-dialogflow/protos/google/cloud/dialogflow/v2/human_agent_assistant_event.proto
@@ -1,4 +1,4 @@
-// Copyright 2022 Google LLC
+// Copyright 2023 Google LLC
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

diff --git a/packages/google-cloud-dialogflow/protos/google/cloud/dialogflow/v2/intent.proto b/packages/google-cloud-dialogflow/protos/google/cloud/dialogflow/v2/intent.proto
@@ -1,4 +1,4 @@
-// Copyright 2022 Google LLC
+// Copyright 2023 Google LLC
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

diff --git a/packages/google-cloud-dialogflow/protos/google/cloud/dialogflow/v2/knowledge_base.proto b/packages/google-cloud-dialogflow/protos/google/cloud/dialogflow/v2/knowledge_base.proto
@@ -1,4 +1,4 @@
-// Copyright 2022 Google LLC
+// Copyright 2023 Google LLC
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

diff --git a/packages/google-cloud-dialogflow/protos/google/cloud/dialogflow/v2/participant.proto b/packages/google-cloud-dialogflow/protos/google/cloud/dialogflow/v2/participant.proto
@@ -1,4 +1,4 @@
-// Copyright 2022 Google LLC
+// Copyright 2023 Google LLC
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -607,6 +607,10 @@ message StreamingAnalyzeContentRequest {
   // `Fulfillment`s in Dialogflow virtual agent have been configured to return
   // partial responses.
   bool enable_partial_automated_agent_reply = 12;
+
+  // If true, `StreamingAnalyzeContentResponse.debugging_info` will get
+  // populated.
+  bool enable_debugging_info = 19;
 }
 
 // The top-level message returned from the `StreamingAnalyzeContent` method.
@@ -677,6 +681,10 @@ message StreamingAnalyzeContentResponse {
 
   // Indicates the parameters of DTMF.
   DtmfParameters dtmf_parameters = 10;
+
+  // Debugging info that would get populated when
+  // `StreamingAnalyzeContentRequest.enable_debugging_info` is set to true.
+  CloudConversationDebuggingInfo debugging_info = 11;
 }
 
 // The request message for

diff --git a/packages/google-cloud-dialogflow/protos/google/cloud/dialogflow/v2/session.proto b/packages/google-cloud-dialogflow/protos/google/cloud/dialogflow/v2/session.proto
@@ -1,4 +1,4 @@
-// Copyright 2022 Google LLC
+// Copyright 2023 Google LLC
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -483,6 +483,74 @@ message StreamingDetectIntentRequest {
   // `query_input` was set to a streaming input audio config. The complete audio
   // over all streaming messages must not exceed 1 minute.
   bytes input_audio = 6;
+
+  // if true, `StreamingDetectIntentResponse.debugging_info` will get populated.
+  bool enable_debugging_info = 8;
+}
+
+// Cloud conversation info for easier debugging.
+// It will get populated in `StreamingDetectIntentResponse` or
+// `StreamingAnalyzeContentResponse` when the flag `enable_debugging_info` is
+// set to true in corresponding requests.
+message CloudConversationDebuggingInfo {
+  // Number of input audio data chunks in streaming requests.
+  int32 audio_data_chunks = 1;
+
+  // Time offset of the end of speech utterance relative to the
+  // beginning of the first audio chunk.
+  google.protobuf.Duration result_end_time_offset = 2;
+
+  // Duration of first audio chunk.
+  google.protobuf.Duration first_audio_duration = 3;
+
+  // Whether client used single utterance mode.
+  bool single_utterance = 5;
+
+  // Time offsets of the speech partial results relative to the beginning of
+  // the stream.
+  repeated google.protobuf.Duration speech_partial_results_end_times = 6;
+
+  // Time offsets of the speech final results (is_final=true) relative to the
+  // beginning of the stream.
+  repeated google.protobuf.Duration speech_final_results_end_times = 7;
+
+  // Total number of partial responses.
+  int32 partial_responses = 8;
+
+  // Time offset of Speaker ID stream close time relative to the Speech stream
+  // close time in milliseconds. Only meaningful for conversations involving
+  // passive verification.
+  int32 speaker_id_passive_latency_ms_offset = 9;
+
+  // Whether a barge-in event is triggered in this request.
+  bool bargein_event_triggered = 10;
+
+  // Whether speech uses single utterance mode.
+  bool speech_single_utterance = 11;
+
+  // Time offsets of the DTMF partial results relative to the beginning of
+  // the stream.
+  repeated google.protobuf.Duration dtmf_partial_results_times = 12;
+
+  // Time offsets of the DTMF final results relative to the beginning of
+  // the stream.
+  repeated google.protobuf.Duration dtmf_final_results_times = 13;
+
+  // Time offset of the end-of-single-utterance signal relative to the
+  // beginning of the stream.
+  google.protobuf.Duration single_utterance_end_time_offset = 14;
+
+  // No speech timeout settings observed at runtime.
+  google.protobuf.Duration no_speech_timeout = 15;
+
+  // Whether the streaming terminates with an injected text query.
+  bool is_input_text = 16;
+
+  // Client half close time in terms of input audio duration.
+  google.protobuf.Duration client_half_close_time_offset = 17;
+
+  // Client half close time in terms of API streaming duration.
+  google.protobuf.Duration client_half_close_streaming_time_offset = 18;
 }
 
 // The top-level message returned from the
@@ -527,6 +595,10 @@ message StreamingDetectIntentResponse {
 
   // The config used by the speech synthesizer to generate the output audio.
   OutputAudioConfig output_audio_config = 6;
+
+  // Debugging info that would get populated when
+  // `StreamingDetectIntentRequest.enable_debugging_info` is set to true.
+  CloudConversationDebuggingInfo debugging_info = 8;
 }
 
 // Contains a speech recognition result corresponding to a portion of the audio