From cf730f8b9f1fa5586bb64ba2f9635cdb84e02e23 Mon Sep 17 00:00:00 2001 From: Deyaaeldeen Almahallawi Date: Fri, 4 Nov 2022 11:13:53 -0700 Subject: [PATCH 1/5] [Language Text] Mitigating bug in auto detected language for TA4H --- .../src/generated/models/index.ts | 13 ++++-- .../src/generated/models/mappers.ts | 37 +++++++++++----- .../ai-language-text/src/transforms.ts | 43 +++++++++++++------ .../ai-language-text/swagger/README.md | 3 +- 4 files changed, 67 insertions(+), 29 deletions(-) diff --git a/sdk/cognitivelanguage/ai-language-text/src/generated/models/index.ts b/sdk/cognitivelanguage/ai-language-text/src/generated/models/index.ts index 6888df54800e..d8adb9cd0371 100644 --- a/sdk/cognitivelanguage/ai-language-text/src/generated/models/index.ts +++ b/sdk/cognitivelanguage/ai-language-text/src/generated/models/index.ts @@ -397,6 +397,11 @@ export interface HealthcareRelationEntity { role: string; } +export interface DocumentDetectedLanguageForHealthcare { + /** If 'language' is set to 'auto' for the document in the request this field will contain a 2 letter ISO 639-1 representation of the language detected for this document. */ + detectedLanguage?: string; +} + export interface PreBuiltResult { /** Errors by document id. */ errors: InputError[]; @@ -944,10 +949,6 @@ export interface CustomLabelClassificationResultDocumentsItem extends ClassificationDocumentResult, DocumentDetectedLanguage {} -export interface HealthcareResultDocumentsItem - extends HealthcareEntitiesDocumentResult, - DocumentDetectedLanguage {} - export interface SentimentResponseDocumentsItem extends SentimentDocumentResult, DocumentDetectedLanguage {} @@ -977,6 +978,10 @@ export interface AbstractiveSummaryDocumentResultWithDetectedLanguage extends AbstractiveSummaryDocumentResult, DocumentDetectedLanguage {} +export interface HealthcareResultDocumentsItem + extends HealthcareEntitiesDocumentResult, + DocumentDetectedLanguageForHealthcare {} + export interface HealthcareResult extends PreBuiltResult { documents: HealthcareResultDocumentsItem[]; } diff --git a/sdk/cognitivelanguage/ai-language-text/src/generated/models/mappers.ts b/sdk/cognitivelanguage/ai-language-text/src/generated/models/mappers.ts index 3404091e4737..1b3bf0929446 100644 --- a/sdk/cognitivelanguage/ai-language-text/src/generated/models/mappers.ts +++ b/sdk/cognitivelanguage/ai-language-text/src/generated/models/mappers.ts @@ -1025,6 +1025,21 @@ export const HealthcareRelationEntity: coreClient.CompositeMapper = { } }; +export const DocumentDetectedLanguageForHealthcare: coreClient.CompositeMapper = { + type: { + name: "Composite", + className: "DocumentDetectedLanguageForHealthcare", + modelProperties: { + detectedLanguage: { + serializedName: "detectedLanguage", + type: { + name: "String" + } + } + } + } +}; + export const PreBuiltResult: coreClient.CompositeMapper = { type: { name: "Composite", @@ -2818,17 +2833,6 @@ export const CustomLabelClassificationResultDocumentsItem: coreClient.CompositeM } }; -export const HealthcareResultDocumentsItem: coreClient.CompositeMapper = { - type: { - name: "Composite", - className: "HealthcareResultDocumentsItem", - modelProperties: { - ...HealthcareEntitiesDocumentResult.type.modelProperties, - ...DocumentDetectedLanguage.type.modelProperties - } - } -}; - export const SentimentResponseDocumentsItem: coreClient.CompositeMapper = { type: { name: "Composite", @@ -2906,6 +2910,17 @@ export const AbstractiveSummaryDocumentResultWithDetectedLanguage: coreClient.Co } }; +export const HealthcareResultDocumentsItem: coreClient.CompositeMapper = { + type: { + name: "Composite", + className: "HealthcareResultDocumentsItem", + modelProperties: { + ...HealthcareEntitiesDocumentResult.type.modelProperties, + ...DocumentDetectedLanguageForHealthcare.type.modelProperties + } + } +}; + export const HealthcareResult: coreClient.CompositeMapper = { type: { name: "Composite", diff --git a/sdk/cognitivelanguage/ai-language-text/src/transforms.ts b/sdk/cognitivelanguage/ai-language-text/src/transforms.ts index 57907420391d..5746cf30d669 100644 --- a/sdk/cognitivelanguage/ai-language-text/src/transforms.ts +++ b/sdk/cognitivelanguage/ai-language-text/src/transforms.ts @@ -50,6 +50,8 @@ import { CustomEntitiesResultDocumentsItem, ExtractedSummaryDocumentResultWithDetectedLanguage, AbstractiveSummaryDocumentResultWithDetectedLanguage, + HealthcareResultDocumentsItem, + DetectedLanguage, } from "./generated"; import { AnalyzeActionName, @@ -74,6 +76,7 @@ import { TextAnalysisError, TextAnalysisErrorResult, TextAnalysisSuccessResult, + WithDetectedLanguage, } from "./models"; import { AssessmentIndex, @@ -392,20 +395,34 @@ function toHealthcareResult( ), }); } - return transformDocumentResults( - docIds, - results, - { - processSuccess: ({ entities, relations, ...rest }) => { - const newEntities = entities.map(makeHealthcareEntity); - return { - entities: newEntities, - entityRelations: relations.map(makeHealthcareRelation(newEntities)), - ...rest, - }; - }, + function deserializeDetectedLanguage(input: string): DetectedLanguage { + function helper(str: string): undefined { + try { + return JSON.parse(str); + } catch (e) { + return undefined; + } } - ); + const obj = helper(input); + return obj !== undefined ? obj : ({ iso6391Name: input } as any); + } + return transformDocumentResults< + HealthcareResultDocumentsItem, + WithDetectedLanguage + >(docIds, results, { + processSuccess: ({ entities, relations, detectedLanguage, ...rest }) => { + const newEntities = entities.map(makeHealthcareEntity); + return { + entities: newEntities, + entityRelations: relations.map(makeHealthcareRelation(newEntities)), + // FIXME: remove this mitigation when the API fixes the representation on their end + ...(detectedLanguage + ? { detectedLanguage: deserializeDetectedLanguage(detectedLanguage) } + : {}), + ...rest, + }; + }, + }); } function toCustomSingleLabelClassificationResult( diff --git a/sdk/cognitivelanguage/ai-language-text/swagger/README.md b/sdk/cognitivelanguage/ai-language-text/swagger/README.md index 36496dc485f7..aefaf852a4cc 100644 --- a/sdk/cognitivelanguage/ai-language-text/swagger/README.md +++ b/sdk/cognitivelanguage/ai-language-text/swagger/README.md @@ -12,7 +12,8 @@ generate-metadata: false license-header: MICROSOFT_MIT_NO_VERSION output-folder: ../ source-code-folder-path: ./src/generated -input-file: https://github.com/Azure/azure-rest-api-specs/blob/ac205086f477776e8d9aa4ff771e98f174afbea2/specification/cognitiveservices/data-plane/Language/preview/2022-10-01-preview/analyzetext.json +input-file: ./swagger.json +#input-file: https://github.com/Azure/azure-rest-api-specs/blob/ac205086f477776e8d9aa4ff771e98f174afbea2/specification/cognitiveservices/data-plane/Language/preview/2022-10-01-preview/analyzetext.json add-credentials: false package-version: 1.1.0-beta.1 v3: true From efa3b679a2a4ee30d581877975196b0fb5d118d0 Mon Sep 17 00:00:00 2001 From: Deyaaeldeen Almahallawi Date: Fri, 4 Nov 2022 11:23:27 -0700 Subject: [PATCH 2/5] update expectation --- sdk/cognitivelanguage/ai-language-text/src/transforms.ts | 1 - .../ai-language-text/test/public/analyzeBatch.spec.ts | 2 +- .../ai-language-text/test/public/expectations.ts | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/sdk/cognitivelanguage/ai-language-text/src/transforms.ts b/sdk/cognitivelanguage/ai-language-text/src/transforms.ts index 5746cf30d669..31d445e3dfdc 100644 --- a/sdk/cognitivelanguage/ai-language-text/src/transforms.ts +++ b/sdk/cognitivelanguage/ai-language-text/src/transforms.ts @@ -30,7 +30,6 @@ import { PiiResult as GeneratedPiiEntityRecognitionResult, SentenceSentiment as GeneratedSentenceSentiment, SentimentResponse as GeneratedSentimentAnalysisResult, - HealthcareEntitiesDocumentResult, HealthcareLROResult, HealthcareRelation, HealthcareRelationEntity, diff --git a/sdk/cognitivelanguage/ai-language-text/test/public/analyzeBatch.spec.ts b/sdk/cognitivelanguage/ai-language-text/test/public/analyzeBatch.spec.ts index 380df6667f91..dfe42122e9b8 100644 --- a/sdk/cognitivelanguage/ai-language-text/test/public/analyzeBatch.spec.ts +++ b/sdk/cognitivelanguage/ai-language-text/test/public/analyzeBatch.spec.ts @@ -862,7 +862,7 @@ matrix([["APIKey", "AAD"]] as const, async (authMethod: AuthMethod) => { await assertActionsResults(await poller.pollUntilDone(), expectation15); }); - it("whole batch input with auto language detection", async function () { + it.only("whole batch input with auto language detection", async function () { const docs = [ "I will go to the park.", "Este es un document escrito en Español.", diff --git a/sdk/cognitivelanguage/ai-language-text/test/public/expectations.ts b/sdk/cognitivelanguage/ai-language-text/test/public/expectations.ts index 41c935b4d9df..aa767c4da445 100644 --- a/sdk/cognitivelanguage/ai-language-text/test/public/expectations.ts +++ b/sdk/cognitivelanguage/ai-language-text/test/public/expectations.ts @@ -8696,7 +8696,7 @@ export const expectation71: any = [ entityRelations: [], id: "0", warnings: [], - detectedLanguage: { "0": "e", "1": "n" }, + detectedLanguage: { iso6391Name: "en" }, isLanguageDefaulted: false, }, { From ff777d50bbe1692b8abeddc76ccb912facecc40a Mon Sep 17 00:00:00 2001 From: Deyaaeldeen Almahallawi Date: Fri, 4 Nov 2022 12:27:47 -0700 Subject: [PATCH 3/5] add swagger transform --- .../src/generated/models/index.ts | 1 - .../ai-language-text/swagger/README.md | 18 ++++++++++++++++-- .../test/public/analyzeBatch.spec.ts | 2 +- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/sdk/cognitivelanguage/ai-language-text/src/generated/models/index.ts b/sdk/cognitivelanguage/ai-language-text/src/generated/models/index.ts index d8adb9cd0371..21056df62ae5 100644 --- a/sdk/cognitivelanguage/ai-language-text/src/generated/models/index.ts +++ b/sdk/cognitivelanguage/ai-language-text/src/generated/models/index.ts @@ -398,7 +398,6 @@ export interface HealthcareRelationEntity { } export interface DocumentDetectedLanguageForHealthcare { - /** If 'language' is set to 'auto' for the document in the request this field will contain a 2 letter ISO 639-1 representation of the language detected for this document. */ detectedLanguage?: string; } diff --git a/sdk/cognitivelanguage/ai-language-text/swagger/README.md b/sdk/cognitivelanguage/ai-language-text/swagger/README.md index aefaf852a4cc..44a50d7d73b1 100644 --- a/sdk/cognitivelanguage/ai-language-text/swagger/README.md +++ b/sdk/cognitivelanguage/ai-language-text/swagger/README.md @@ -12,8 +12,8 @@ generate-metadata: false license-header: MICROSOFT_MIT_NO_VERSION output-folder: ../ source-code-folder-path: ./src/generated -input-file: ./swagger.json -#input-file: https://github.com/Azure/azure-rest-api-specs/blob/ac205086f477776e8d9aa4ff771e98f174afbea2/specification/cognitiveservices/data-plane/Language/preview/2022-10-01-preview/analyzetext.json +# input-file: ./swagger.json +input-file: https://github.com/Azure/azure-rest-api-specs/blob/ac205086f477776e8d9aa4ff771e98f174afbea2/specification/cognitiveservices/data-plane/Language/preview/2022-10-01-preview/analyzetext.json add-credentials: false package-version: 1.1.0-beta.1 v3: true @@ -320,6 +320,20 @@ directive: where: $.definitions.JobState transform: $.properties.lastUpdatedDateTime["x-ms-client-name"] = "modifiedOn"; + - from: swagger-document + where: $.definitions + transform: > + if (!$.DocumentDetectedLanguageForHealthcare) { + $.DocumentDetectedLanguageForHealthcare = { "type": "object", "properties": { "detectedLanguage": { "type": "string" } } }; + } + + - from: swagger-document + where: $.definitions.HealthcareResult.properties.documents.items.allOf + transform: > + if ($[1]["$ref"] === "#/definitions/DocumentDetectedLanguage") { + $[1]["$ref"] = "#/definitions/DocumentDetectedLanguageForHealthcare"; + } + # Enhance documentation strings for some exported swagger types - from: swagger-document diff --git a/sdk/cognitivelanguage/ai-language-text/test/public/analyzeBatch.spec.ts b/sdk/cognitivelanguage/ai-language-text/test/public/analyzeBatch.spec.ts index dfe42122e9b8..380df6667f91 100644 --- a/sdk/cognitivelanguage/ai-language-text/test/public/analyzeBatch.spec.ts +++ b/sdk/cognitivelanguage/ai-language-text/test/public/analyzeBatch.spec.ts @@ -862,7 +862,7 @@ matrix([["APIKey", "AAD"]] as const, async (authMethod: AuthMethod) => { await assertActionsResults(await poller.pollUntilDone(), expectation15); }); - it.only("whole batch input with auto language detection", async function () { + it("whole batch input with auto language detection", async function () { const docs = [ "I will go to the park.", "Este es un document escrito en Español.", From 5a41892974e9ffdce8a88e6dff3ae8530eb2e53a Mon Sep 17 00:00:00 2001 From: Deyaaeldeen Almahallawi Date: Fri, 4 Nov 2022 12:29:36 -0700 Subject: [PATCH 4/5] edit autorest configs --- sdk/cognitivelanguage/ai-language-text/swagger/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/sdk/cognitivelanguage/ai-language-text/swagger/README.md b/sdk/cognitivelanguage/ai-language-text/swagger/README.md index 44a50d7d73b1..d6763c2dc3f1 100644 --- a/sdk/cognitivelanguage/ai-language-text/swagger/README.md +++ b/sdk/cognitivelanguage/ai-language-text/swagger/README.md @@ -12,7 +12,6 @@ generate-metadata: false license-header: MICROSOFT_MIT_NO_VERSION output-folder: ../ source-code-folder-path: ./src/generated -# input-file: ./swagger.json input-file: https://github.com/Azure/azure-rest-api-specs/blob/ac205086f477776e8d9aa4ff771e98f174afbea2/specification/cognitiveservices/data-plane/Language/preview/2022-10-01-preview/analyzetext.json add-credentials: false package-version: 1.1.0-beta.1 From c5fc72b0978987abcf2caef48aad457e45a089cd Mon Sep 17 00:00:00 2001 From: Deyaaeldeen Almahallawi Date: Fri, 4 Nov 2022 14:43:32 -0700 Subject: [PATCH 5/5] add a couple tests --- .../ai-language-text/src/transforms.ts | 23 ++++++++++--------- .../test/internal/utils.spec.ts | 22 ++++++++++++++++++ 2 files changed, 34 insertions(+), 11 deletions(-) create mode 100644 sdk/cognitivelanguage/ai-language-text/test/internal/utils.spec.ts diff --git a/sdk/cognitivelanguage/ai-language-text/src/transforms.ts b/sdk/cognitivelanguage/ai-language-text/src/transforms.ts index 31d445e3dfdc..cd14a73cf637 100644 --- a/sdk/cognitivelanguage/ai-language-text/src/transforms.ts +++ b/sdk/cognitivelanguage/ai-language-text/src/transforms.ts @@ -365,6 +365,18 @@ export async function throwError(p: Promise): Promise { } } +export function deserializeDetectedLanguage(input: string): DetectedLanguage { + function helper(str: string): undefined { + try { + return JSON.parse(str); + } catch (e) { + return undefined; + } + } + const obj = helper(input); + return obj !== undefined ? obj : ({ iso6391Name: input } as any); +} + function toHealthcareResult( docIds: string[], results: GeneratedHealthcareResult @@ -394,17 +406,6 @@ function toHealthcareResult( ), }); } - function deserializeDetectedLanguage(input: string): DetectedLanguage { - function helper(str: string): undefined { - try { - return JSON.parse(str); - } catch (e) { - return undefined; - } - } - const obj = helper(input); - return obj !== undefined ? obj : ({ iso6391Name: input } as any); - } return transformDocumentResults< HealthcareResultDocumentsItem, WithDetectedLanguage diff --git a/sdk/cognitivelanguage/ai-language-text/test/internal/utils.spec.ts b/sdk/cognitivelanguage/ai-language-text/test/internal/utils.spec.ts new file mode 100644 index 000000000000..7ba81a90a211 --- /dev/null +++ b/sdk/cognitivelanguage/ai-language-text/test/internal/utils.spec.ts @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import { assert } from "@azure/test-utils"; +import { deserializeDetectedLanguage } from "../../src/transforms"; + +describe("Utilities", function () { + describe("deserializeDetectedLanguage", function () { + it("deserializes string", function () { + assert.deepEqual(deserializeDetectedLanguage("en"), { iso6391Name: "en" } as any); + }); + + it("deserializes object", function () { + const res = { + name: "English", + iso6391Name: "en", + confidenceScore: 0.98, + }; + assert.deepEqual(deserializeDetectedLanguage(JSON.stringify(res)), res); + }); + }); +});