[Language Text] Mitigating bug in auto detected language for TA4H (#2…

…3720)

It adds a swagger transform to specify the type of `detectedLanguage` to be a string and then attempts to deserialize it in the convenience layer and set the iso name correctly.

sdk/cognitivelanguage/ai-language-text/src/transforms.ts

@@ -30,7 +30,6 @@ import {
   PiiResult as GeneratedPiiEntityRecognitionResult,
   SentenceSentiment as GeneratedSentenceSentiment,
   SentimentResponse as GeneratedSentimentAnalysisResult,
-  HealthcareEntitiesDocumentResult,
   HealthcareLROResult,
   HealthcareRelation,
   HealthcareRelationEntity,
@@ -50,6 +49,8 @@ import {
   CustomEntitiesResultDocumentsItem,
   ExtractedSummaryDocumentResultWithDetectedLanguage,
   AbstractiveSummaryDocumentResultWithDetectedLanguage,
+  HealthcareResultDocumentsItem,
+  DetectedLanguage,
 } from "./generated";
 import {
   AnalyzeActionName,
@@ -74,6 +75,7 @@ import {
   TextAnalysisError,
   TextAnalysisErrorResult,
   TextAnalysisSuccessResult,
+  WithDetectedLanguage,
 } from "./models";
 import {
   AssessmentIndex,
@@ -363,6 +365,18 @@ export async function throwError<T>(p: Promise<T>): Promise<T> {
   }
 }
 
+export function deserializeDetectedLanguage(input: string): DetectedLanguage {
+  function helper(str: string): undefined {
+    try {
+      return JSON.parse(str);
+    } catch (e) {
+      return undefined;
+    }
+  }
+  const obj = helper(input);
+  return obj !== undefined ? obj : ({ iso6391Name: input } as any);
+}
+
 function toHealthcareResult(
   docIds: string[],
   results: GeneratedHealthcareResult
@@ -392,20 +406,23 @@ function toHealthcareResult(
       ),
     });
   }
-  return transformDocumentResults<HealthcareEntitiesDocumentResult, HealthcareSuccessResult>(
-    docIds,
-    results,
-    {
-      processSuccess: ({ entities, relations, ...rest }) => {
-        const newEntities = entities.map(makeHealthcareEntity);
-        return {
-          entities: newEntities,
-          entityRelations: relations.map(makeHealthcareRelation(newEntities)),
-          ...rest,
-        };
-      },
-    }
-  );
+  return transformDocumentResults<
+    HealthcareResultDocumentsItem,
+    WithDetectedLanguage<HealthcareSuccessResult>
+  >(docIds, results, {
+    processSuccess: ({ entities, relations, detectedLanguage, ...rest }) => {
+      const newEntities = entities.map(makeHealthcareEntity);
+      return {
+        entities: newEntities,
+        entityRelations: relations.map(makeHealthcareRelation(newEntities)),
+        // FIXME: remove this mitigation when the API fixes the representation on their end
+        ...(detectedLanguage
+          ? { detectedLanguage: deserializeDetectedLanguage(detectedLanguage) }
+          : {}),
+        ...rest,
+      };
+    },
+  });
 }
 
 function toCustomSingleLabelClassificationResult(

sdk/cognitivelanguage/ai-language-text/swagger/README.md

@@ -319,6 +319,20 @@ directive:
     where: $.definitions.JobState
     transform: $.properties.lastUpdatedDateTime["x-ms-client-name"] = "modifiedOn";
 
+  - from: swagger-document
+    where: $.definitions
+    transform: >
+      if (!$.DocumentDetectedLanguageForHealthcare) {
+          $.DocumentDetectedLanguageForHealthcare = { "type": "object", "properties": { "detectedLanguage": { "type": "string" } } };
+      }
+
+  - from: swagger-document
+    where: $.definitions.HealthcareResult.properties.documents.items.allOf
+    transform: >
+      if ($[1]["$ref"] === "#/definitions/DocumentDetectedLanguage") {
+          $[1]["$ref"] = "#/definitions/DocumentDetectedLanguageForHealthcare";
+      }
+
 # Enhance documentation strings for some exported swagger types
 
   - from: swagger-document

sdk/cognitivelanguage/ai-language-text/test/internal/utils.spec.ts

@@ -0,0 +1,22 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+import { assert } from "@azure/test-utils";
+import { deserializeDetectedLanguage } from "../../src/transforms";
+
+describe("Utilities", function () {
+  describe("deserializeDetectedLanguage", function () {
+    it("deserializes string", function () {
+      assert.deepEqual(deserializeDetectedLanguage("en"), { iso6391Name: "en" } as any);
+    });
+
+    it("deserializes object", function () {
+      const res = {
+        name: "English",
+        iso6391Name: "en",
+        confidenceScore: 0.98,
+      };
+      assert.deepEqual(deserializeDetectedLanguage(JSON.stringify(res)), res);
+    });
+  });
+});

sdk/cognitivelanguage/ai-language-text/test/public/expectations.ts

@@ -8696,7 +8696,7 @@ export const expectation71: any = [
         entityRelations: [],
         id: "0",
         warnings: [],
-        detectedLanguage: { "0": "e", "1": "n" },
+        detectedLanguage: { iso6391Name: "en" },
         isLanguageDefaulted: false,
       },
       {