[OpenAI.Inference] Adding content filter support (#24652)

* first-pass adding annotation * FilterResult definitions * move Filter definitions to common * cleaning up naming and imports * adding version headers * added descriptions to objects * correct self_harm JSON projection * Correct to "prompt_annotations" * "prompt_annotations" for chat, too * Update specification/cognitiveservices/OpenAI.Inference/models/completions.common.tsp Co-authored-by: Shawn Fang <[email protected]> * Add per-choice content_filter_results to /completions * tsp compile after merge * recompile with 0.46.0 after today's update --------- Co-authored-by: Travis Wilson <[email protected]> Co-authored-by: Shawn Fang <[email protected]>
Azure · Jul 12, 2023 · 812c8a0 · 812c8a0
1 parent f7c77a5
commit 812c8a0
Show file tree

Hide file tree

Showing 5 changed files with 393 additions and 0 deletions.
diff --git a/specification/cognitiveservices/OpenAI.Inference/models/chat.completions.tsp b/specification/cognitiveservices/OpenAI.Inference/models/chat.completions.tsp
@@ -190,6 +190,7 @@ model ChatChoice {
   index: int32;
 
   #suppress "@azure-tools/typespec-azure-core/no-nullable" "The operation already returns nulls"
+  #suppress "@azure-tools/typespec-autorest/union-unsupported" "OpenAPI v2 support deferred"
   @doc("The reason that this chat completions choice completed its generated.")
   @projectedName("json", "finish_reason")
   finishReason: CompletionsFinishReason | null;
@@ -198,6 +199,15 @@ model ChatChoice {
   @projectedName("json", "delta")
   @projectedName("csharp", "InternalStreamingDeltaMessage")
   delta?: ChatMessage;
+
+  @doc("""
+    Information about the content filtering category (hate, sexual, violence, self_harm), if it 
+    has been detected, as well as the severity level (very_low, low, medium, high-scale that 
+    determines the intensity and risk level of harmful content) and if it has been filtered or not.
+    """)
+  @added(ServiceApiVersions.v2023_06_01_Preview)
+  @projectedName("json", "content_filter_results")
+  contentFilterResults?: ContentFilterResults;
 }
 
 @doc("""
@@ -227,6 +237,14 @@ model ChatCompletions {
   @projectedName("json", "choices")
   choices: ChatChoice[];
 
+  @doc("""
+  Content filtering results for zero or more prompts in the request. In a streaming request, 
+  results for different prompts may arrive at different times or in different orders.
+  """)
+  @added(ServiceApiVersions.v2023_06_01_Preview)
+  @projectedName("json", "prompt_annotations")
+  promptFilterResults?: PromptFilterResult[];
+
   @doc("""
     Usage information for tokens processed and generated as part of this completions operation.
     """)

diff --git a/specification/cognitiveservices/OpenAI.Inference/models/completions.common.tsp b/specification/cognitiveservices/OpenAI.Inference/models/completions.common.tsp
@@ -43,3 +43,96 @@ enum CompletionsFinishReason {
   @added(ServiceApiVersions.v2023_07_01_Preview)
   functionCall: "function_call",
 }
+
+@added(ServiceApiVersions.v2023_06_01_Preview)
+@doc("Ratings for the intensity and risk level of harmful content.")
+enum ContentFilterSeverity {
+  @doc("""
+  Content may be related to violence, self-harm, sexual, or hate categories but the terms 
+  are used in general, journalistic, scientific, medical, and similar professional contexts, 
+  which are appropriate for most audiences.
+  """)
+  safe: "safe",
+
+  @doc("""
+  Content that expresses prejudiced, judgmental, or opinionated views, includes offensive 
+  use of language, stereotyping, use cases exploring a fictional world (for example, gaming,
+  literature) and depictions at low intensity.
+  """)
+  low: "low",
+
+  @doc("""
+  Content that uses offensive, insulting, mocking, intimidating, or demeaning language 
+  towards specific identity groups, includes depictions of seeking and executing harmful 
+  instructions, fantasies, glorification, promotion of harm at medium intensity.
+  """)
+  medium: "medium",
+
+  @doc("""
+  Content that displays explicit and severe harmful instructions, actions, 
+  damage, or abuse; includes endorsement, glorification, or promotion of severe 
+  harmful acts, extreme or illegal forms of harm, radicalization, or non-consensual 
+  power exchange or abuse.
+  """)
+  high: "high",
+}
+
+@added(ServiceApiVersions.v2023_06_01_Preview)
+@doc("Information about filtered content severity level and if it has been filtered or not.")
+model ContentFilterResult {
+  @doc("Ratings for the intensity and risk level of filtered content.")
+  @projectedName("json", "severity")
+  severity: ContentFilterSeverity;
+
+  @doc("A value indicating whether or not the content has been filtered.")
+  @projectedName("json", "filtered")
+  filtered: boolean;
+}
+
+@added(ServiceApiVersions.v2023_06_01_Preview)
+@doc("Information about the content filtering category, if it has been detected.")
+model ContentFilterResults {
+  @doc("""
+  Describes language related to anatomical organs and genitals, romantic relationships,
+   acts portrayed in erotic or affectionate terms, physical sexual acts, including 
+   those portrayed as an assault or a forced sexual violent act against one’s will, 
+   prostitution, pornography, and abuse.
+  """)
+  sexual: ContentFilterResult;
+
+  @doc("""
+  Describes language related to physical actions intended to hurt, injure, damage, or 
+  kill someone or something; describes weapons, etc.
+  """)
+  violence: ContentFilterResult;
+
+  @doc("""
+  Describes language attacks or uses that include pejorative or discriminatory language 
+  with reference to a person or identity group on the basis of certain differentiating 
+  attributes of these groups including but not limited to race, ethnicity, nationality,
+  gender identity and expression, sexual orientation, religion, immigration status, ability
+  status, personal appearance, and body size.
+  """)
+  hate: ContentFilterResult;
+
+  @doc("""
+  Describes language related to physical actions intended to purposely hurt, injure,
+  or damage one’s body, or kill oneself.
+  """)
+  @projectedName("json", "self_harm")
+  selfHarm: ContentFilterResult;
+}
+
+@added(ServiceApiVersions.v2023_06_01_Preview)
+@doc("""
+Content filtering results for a single prompt in the request.
+""")
+model PromptFilterResult {
+  @doc("The index of this prompt in the set of prompt results")
+  @projectedName("json", "prompt_index")
+  promptIndex: int32;
+
+  @doc("Content filtering results for this prompt")
+  @projectedName("json", "content_filter_results")
+  contentFilterResults?: ContentFilterResults;
+}
diff --git a/specification/cognitiveservices/OpenAI.Inference/models/completions.create.tsp b/specification/cognitiveservices/OpenAI.Inference/models/completions.create.tsp
@@ -1,9 +1,11 @@
 import "@typespec/rest";
 import "@typespec/http";
+import "@typespec/versioning";
 import "./completions.common.tsp";
 
 using TypeSpec.Rest;
 using TypeSpec.Http;
+using TypeSpec.Versioning;
 
 namespace Azure.OpenAI;
 
@@ -159,6 +161,14 @@ model Completions {
   @projectedName("csharp", "InternalCreatedSecondsAfterUnixEpoch")
   created: int32;
 
+  @doc("""
+  Content filtering results for zero or more prompts in the request. In a streaming request, 
+  results for different prompts may arrive at different times or in different orders.
+  """)
+  @added(ServiceApiVersions.v2023_06_01_Preview)
+  @projectedName("json", "prompt_annotations")
+  promptFilterResults?: PromptFilterResult[];
+
   @doc("""
     The collection of completions choices associated with this completions response.
     Generally, `n` choices are generated per provided prompt with a default value of 1.
@@ -188,13 +198,23 @@ model Choice {
   @projectedName("json", "index")
   index: int32;
 
+  @doc("""
+  Information about the content filtering category (hate, sexual, violence, self_harm), if it 
+  has been detected, as well as the severity level (very_low, low, medium, high-scale that 
+  determines the intensity and risk level of harmful content) and if it has been filtered or not.
+  """)
+  @added(ServiceApiVersions.v2023_06_01_Preview)
+  @projectedName("json", "content_filter_results")
+  contentFilterResults?: ContentFilterResults;
+
   #suppress "@azure-tools/typespec-azure-core/no-nullable" "The operation already returns nulls"
   @doc("The log probabilities model for tokens associated with this completions choice.")
   @projectedName("json", "logprobs")
   @projectedName("csharp", "LogProbabilityModel")
   logprobs: CompletionsLogProbabilityModel | null;
 
   #suppress "@azure-tools/typespec-azure-core/no-nullable" "The operation already returns nulls"
+  #suppress "@azure-tools/typespec-autorest/union-unsupported" "OpenAPI v2 support deferred"
   @doc("Reason for finishing")
   @projectedName("json", "finish_reason")
   finishReason: CompletionsFinishReason | null;

diff --git a/...nitiveservices/data-plane/AzureOpenAI/inference/preview/2023-06-01-preview/generated.json b/...nitiveservices/data-plane/AzureOpenAI/inference/preview/2023-06-01-preview/generated.json
@@ -470,6 +470,11 @@
         "delta": {
           "$ref": "#/definitions/ChatMessage",
           "description": "The delta message content for a streaming response."
+        },
+        "content_filter_results": {
+          "$ref": "#/definitions/ContentFilterResults",
+          "description": "Information about the content filtering category (hate, sexual, violence, self_harm), if it \nhas been detected, as well as the severity level (very_low, low, medium, high-scale that \ndetermines the intensity and risk level of harmful content) and if it has been filtered or not.",
+          "x-ms-client-name": "contentFilterResults"
         }
       },
       "required": [
@@ -498,6 +503,15 @@
           },
           "x-ms-identifiers": []
         },
+        "prompt_annotations": {
+          "type": "array",
+          "description": "Content filtering results for zero or more prompts in the request. In a streaming request, \nresults for different prompts may arrive at different times or in different orders.",
+          "items": {
+            "$ref": "#/definitions/PromptFilterResult"
+          },
+          "x-ms-client-name": "promptFilterResults",
+          "x-ms-identifiers": []
+        },
         "usage": {
           "$ref": "#/definitions/CompletionsUsage",
           "description": "Usage information for tokens processed and generated as part of this completions operation."
@@ -649,6 +663,11 @@
           "format": "int32",
           "description": "The ordered index associated with this completions choice."
         },
+        "content_filter_results": {
+          "$ref": "#/definitions/ContentFilterResults",
+          "description": "Information about the content filtering category (hate, sexual, violence, self_harm), if it \nhas been detected, as well as the severity level (very_low, low, medium, high-scale that \ndetermines the intensity and risk level of harmful content) and if it has been filtered or not.",
+          "x-ms-client-name": "contentFilterResults"
+        },
         "logprobs": {
           "type": "object",
           "description": "The log probabilities model for tokens associated with this completions choice.",
@@ -686,6 +705,15 @@
           "format": "int32",
           "description": "The first timestamp associated with generation activity for this completions response,\nrepresented as seconds since the beginning of the Unix epoch of 00:00 on 1 Jan 1970."
         },
+        "prompt_annotations": {
+          "type": "array",
+          "description": "Content filtering results for zero or more prompts in the request. In a streaming request, \nresults for different prompts may arrive at different times or in different orders.",
+          "items": {
+            "$ref": "#/definitions/PromptFilterResult"
+          },
+          "x-ms-client-name": "promptFilterResults",
+          "x-ms-identifiers": []
+        },
         "choices": {
           "type": "array",
           "description": "The collection of completions choices associated with this completions response.\nGenerally, `n` choices are generated per provided prompt with a default value of 1.\nToken limits and other settings may limit the number of choices generated.",
@@ -905,6 +933,89 @@
         "total_tokens"
       ]
     },
+    "ContentFilterResult": {
+      "type": "object",
+      "description": "Information about filtered content severity level and if it has been filtered or not.",
+      "properties": {
+        "severity": {
+          "$ref": "#/definitions/ContentFilterSeverity",
+          "description": "Ratings for the intensity and risk level of filtered content."
+        },
+        "filtered": {
+          "type": "boolean",
+          "description": "A value indicating whether or not the content has been filtered."
+        }
+      },
+      "required": [
+        "severity",
+        "filtered"
+      ]
+    },
+    "ContentFilterResults": {
+      "type": "object",
+      "description": "Information about the content filtering category, if it has been detected.",
+      "properties": {
+        "sexual": {
+          "$ref": "#/definitions/ContentFilterResult",
+          "description": "Describes language related to anatomical organs and genitals, romantic relationships,\n acts portrayed in erotic or affectionate terms, physical sexual acts, including \n those portrayed as an assault or a forced sexual violent act against one’s will, \n prostitution, pornography, and abuse."
+        },
+        "violence": {
+          "$ref": "#/definitions/ContentFilterResult",
+          "description": "Describes language related to physical actions intended to hurt, injure, damage, or \nkill someone or something; describes weapons, etc."
+        },
+        "hate": {
+          "$ref": "#/definitions/ContentFilterResult",
+          "description": "Describes language attacks or uses that include pejorative or discriminatory language \nwith reference to a person or identity group on the basis of certain differentiating \nattributes of these groups including but not limited to race, ethnicity, nationality,\ngender identity and expression, sexual orientation, religion, immigration status, ability\nstatus, personal appearance, and body size."
+        },
+        "self_harm": {
+          "$ref": "#/definitions/ContentFilterResult",
+          "description": "Describes language related to physical actions intended to purposely hurt, injure,\nor damage one’s body, or kill oneself.",
+          "x-ms-client-name": "selfHarm"
+        }
+      },
+      "required": [
+        "sexual",
+        "violence",
+        "hate",
+        "self_harm"
+      ]
+    },
+    "ContentFilterSeverity": {
+      "type": "string",
+      "description": "Ratings for the intensity and risk level of harmful content.",
+      "enum": [
+        "safe",
+        "low",
+        "medium",
+        "high"
+      ],
+      "x-ms-enum": {
+        "name": "ContentFilterSeverity",
+        "modelAsString": true,
+        "values": [
+          {
+            "name": "safe",
+            "value": "safe",
+            "description": "Content may be related to violence, self-harm, sexual, or hate categories but the terms \nare used in general, journalistic, scientific, medical, and similar professional contexts, \nwhich are appropriate for most audiences."
+          },
+          {
+            "name": "low",
+            "value": "low",
+            "description": "Content that expresses prejudiced, judgmental, or opinionated views, includes offensive \nuse of language, stereotyping, use cases exploring a fictional world (for example, gaming,\nliterature) and depictions at low intensity."
+          },
+          {
+            "name": "medium",
+            "value": "medium",
+            "description": "Content that uses offensive, insulting, mocking, intimidating, or demeaning language \ntowards specific identity groups, includes depictions of seeking and executing harmful \ninstructions, fantasies, glorification, promotion of harm at medium intensity."
+          },
+          {
+            "name": "high",
+            "value": "high",
+            "description": "Content that displays explicit and severe harmful instructions, actions, \ndamage, or abuse; includes endorsement, glorification, or promotion of severe \nharmful acts, extreme or illegal forms of harm, radicalization, or non-consensual \npower exchange or abuse."
+          }
+        ]
+      }
+    },
     "EmbeddingItem": {
       "type": "object",
       "description": "Representation of a single embeddings relatedness comparison.",
@@ -1100,6 +1211,26 @@
           }
         ]
       }
+    },
+    "PromptFilterResult": {
+      "type": "object",
+      "description": "Content filtering results for a single prompt in the request.",
+      "properties": {
+        "prompt_index": {
+          "type": "integer",
+          "format": "int32",
+          "description": "The index of this prompt in the set of prompt results",
+          "x-ms-client-name": "promptIndex"
+        },
+        "content_filter_results": {
+          "$ref": "#/definitions/ContentFilterResults",
+          "description": "Content filtering results for this prompt",
+          "x-ms-client-name": "contentFilterResults"
+        }
+      },
+      "required": [
+        "prompt_index"
+      ]
     }
   },
   "parameters": {