diff --git a/sdk/textanalytics/ai-text-analytics/CHANGELOG.md b/sdk/textanalytics/ai-text-analytics/CHANGELOG.md index fa34646916fa..1cbd39aef682 100644 --- a/sdk/textanalytics/ai-text-analytics/CHANGELOG.md +++ b/sdk/textanalytics/ai-text-analytics/CHANGELOG.md @@ -1,6 +1,10 @@ # Release History -## 5.2.0-beta.1 (Unreleased) +## 5.1.0-beta.3 (Unreleased) + +- We are now targeting the service's v3.1-preview.3 API as the default instead of v3.1-preview.2. +- We now have added support for performing multiple analyses at once with the introduction of the `beginAnalyze` API. +- We now have added support for the recognition of healthcare entities with the introduction of the `beginAnalyzeHealthcare` API. ## 5.1.0-beta.2 (2020-10-06) @@ -17,7 +21,7 @@ ## 5.0.1 (2020-08-18) -- Handles REST exceptions with code InvalidDocumentBatch in a way that maintains backward compatability. +- Handles REST exceptions with code InvalidDocumentBatch in a way that maintains backward compatibility. ## 5.0.0 (2020-07-27) diff --git a/sdk/textanalytics/ai-text-analytics/README.md b/sdk/textanalytics/ai-text-analytics/README.md index cf182ed3fae2..e7f702aebfbe 100644 --- a/sdk/textanalytics/ai-text-analytics/README.md +++ b/sdk/textanalytics/ai-text-analytics/README.md @@ -2,7 +2,7 @@ [Azure TextAnalytics](https://azure.microsoft.com/services/cognitive-services/text-analytics/) is a cloud-based service that provides advanced natural language processing over raw text, and includes six main functions: -__Note:__ This SDK targets Azure Text Analytics service API version 3.1.0-preview.2. +**Note:** This SDK targets Azure Text Analytics service API version 3.1.0-preview.2. - Language Detection - Sentiment Analysis @@ -10,13 +10,16 @@ __Note:__ This SDK targets Azure Text Analytics service API version 3.1.0-previe - Named Entity Recognition - Recognition of Personally Identifiable Information - Linked Entity Recognition +- Healthcare Analysis +- Batch Analysis Use the client library to: - Detect what language input text is written in. - Determine what customers think of your brand or topic by analyzing raw text for clues about positive or negative sentiment. - Automatically extract key phrases to quickly identify the main points. -- Identify and categorize entities in your text as people, places, organizations, date/time, quantities, percentages, currencies, and more. +- Identify and categorize entities in your text as people, places, organizations, date/time, quantities, percentages, currencies, healthcare specific, and more. +- Perform multiple of the above tasks at once. [Source code](https://github.com/Azure/azure-sdk-for-js/blob/master/sdk/textanalytics/ai-text-analytics/) | [Package (NPM)](https://www.npmjs.com/package/@azure/ai-text-analytics) | @@ -113,7 +116,7 @@ For example, each document can be passed as a string in an array, e.g. const documents = [ "I hated the movie. It was so slow!", "The movie made it into my top ten favorites.", - "What a great movie!", + "What a great movie!" ]; ``` @@ -123,7 +126,7 @@ or, if you wish to pass in a per-item document `id` or `language`/`countryHint`, const textDocumentInputs = [ { id: "1", language: "en", text: "I hated the movie. It was so slow!" }, { id: "2", language: "en", text: "The movie made it into my top ten favorites." }, - { id: "3", language: "en", text: "What a great movie!" }, + { id: "3", language: "en", text: "What a great movie!" } ]; ``` @@ -186,7 +189,7 @@ const client = new TextAnalyticsClient("", new AzureKeyCredential("", new AzureKeyCredential("", new AzureKeyCredential("", new AzureKeyCredential("", new AzureKeyCredential("")); + +const documents = [ + "Prescribed 100mg ibuprofen, taken twice daily.", + "Patient does not suffer from high blood pressure." +]; + +async function main() { + const poller = await client.beginAnalyzeHealthcare(documents); + const results = await poller.pollUntilDone(); + + for await (const result of results) { + console.log(`- Document ${result.id}`); + if (!result.error) { + console.log(" Recognized Entities:"); + for (const entity of result.entities) { + console.log(` - Entity ${entity.text} of type ${entity.category}`); + } + } else console.error(" Error:", result.error); + } +} + +main(); +``` + +### Analyze + +Analyze enables the application of multiple analyses at once. + +```javascript +const { TextAnalyticsClient, AzureKeyCredential } = require("@azure/ai-text-analytics"); + +const client = new TextAnalyticsClient("", new AzureKeyCredential("")); + +const documents = [ + "Microsoft was founded by Bill Gates and Paul Allen.", + "The employee's SSN is 555-55-5555.", + "Easter Island, a Chilean territory, is a remote volcanic island in Polynesia.", + "I use Azure Functions to develop my product." +]; + +async function main() { + const tasks = { + entityRecognitionTasks: [{ modelVersion: "latest" }], + entityRecognitionPiiTasks: [{ modelVersion: "latest" }], + keyPhraseExtractionTasks: [{ modelVersion: "latest" }] + }; + const poller = await client.beginAnalyze(documents, tasks); + const resultPages = await poller.pollUntilDone(); + + for await (const page of resultPages) { + const keyPhrasesResults = page.keyPhrasesExtractionResults[0]; + for (const doc of keyPhrasesResults) { + console.log(`- Document ${doc.id}`); + if (!doc.error) { + console.log(" Key phrases:"); + for (const phrase of doc.keyPhrases) { + console.log(` ${phrase}`); + } + } else { + console.error(" Error:", doc.error); + } + } + + const entitiesResults = page.entitiesRecognitionResults[0]; + for (const doc of entitiesResults) { + console.log(`- Document ${doc.id}`); + if (!doc.error) { + console.log(" Entities:"); + for (const entity of doc.entities) { + console.log(` ${entity}`); + } + } else { + console.error(" Error:", doc.error); + } + } + + const piiEntitiesResults = page.piiEntitiesRecognitionResults[0]; + for (const doc of piiEntitiesResults) { + console.log(`- Document ${doc.id}`); + if (!doc.error) { + console.log(" Pii Entities:"); + for (const entity of doc.entities) { + console.log(` ${entity}`); + } + } else { + console.error(" Error:", doc.error); + } + } + } +} + +main(); +``` + ## Troubleshooting ### Enable logs @@ -421,4 +532,4 @@ If you'd like to contribute to this library, please read the [contributing guide [register_aad_app]: https://docs.microsoft.com/azure/cognitive-services/authentication#assign-a-role-to-a-service-principal [defaultazurecredential]: https://github.com/Azure/azure-sdk-for-js/tree/master/sdk/identity/identity#defaultazurecredential [data_limits]: https://docs.microsoft.com/azure/cognitive-services/text-analytics/overview#data-limits -[analyze_sentiment_opinion_mining_sample]: https://github.com/Azure/azure-sdk-for-js/blob/master/sdk/textanalytics/ai-text-analytics/samples/typescript/src/analyzeSentimentWithOpinionMining.ts \ No newline at end of file +[analyze_sentiment_opinion_mining_sample]: https://github.com/Azure/azure-sdk-for-js/blob/master/sdk/textanalytics/ai-text-analytics/samples/typescript/src/analyzeSentimentWithOpinionMining.ts diff --git a/sdk/textanalytics/ai-text-analytics/package.json b/sdk/textanalytics/ai-text-analytics/package.json index d590e0cc8606..896e5adcc69d 100644 --- a/sdk/textanalytics/ai-text-analytics/package.json +++ b/sdk/textanalytics/ai-text-analytics/package.json @@ -3,7 +3,7 @@ "sdk-type": "client", "author": "Microsoft Corporation", "description": "An isomorphic client library for the Azure Text Analytics service.", - "version": "5.2.0-beta.1", + "version": "5.1.0-beta.3", "keywords": [ "node", "azure", diff --git a/sdk/textanalytics/ai-text-analytics/review/ai-text-analytics.api.md b/sdk/textanalytics/ai-text-analytics/review/ai-text-analytics.api.md index 8e114b07823b..571a83c96173 100644 --- a/sdk/textanalytics/ai-text-analytics/review/ai-text-analytics.api.md +++ b/sdk/textanalytics/ai-text-analytics/review/ai-text-analytics.api.md @@ -14,41 +14,18 @@ import { PollOperationState } from '@azure/core-lro'; import { TokenCredential } from '@azure/core-auth'; // @public -export interface AnalyzeEntitiesResult extends RecognizeCategorizedEntitiesSuccessResult { - // (undocumented) - type: "Entities"; -} - -// @public -export interface AnalyzeErrorResult extends TextAnalyticsErrorResult { - // (undocumented) - type: "Error"; -} - -// @public -export interface AnalyzeJobOptions extends TextAnalyticsOperationOptions { -} - -// @public (undocumented) -export interface AnalyzeKeyPhrasesResult extends ExtractKeyPhrasesSuccessResult { - // (undocumented) - type: "KeyPhrases"; -} - -// @public (undocumented) -export interface AnalyzePiiEntitiesResult extends RecognizePiiEntitiesSuccessResult { - // (undocumented) - type: "PiiEntities"; +export interface AnalyzeJobOptions extends OperationOptions { + includeStatistics?: boolean; } // @public export type AnalyzePollerLike = PollerLike; // @public -export type AnalyzeResult = AnalyzeEntitiesResult | AnalyzePiiEntitiesResult | AnalyzeKeyPhrasesResult | AnalyzeErrorResult; - -// @public -export interface AnalyzeResultsArray extends Array { +export interface AnalyzeResult { + entitiesRecognitionResults?: RecognizeCategorizedEntitiesResultArray[]; + keyPhrasesExtractionResults?: ExtractKeyPhrasesResultArray[]; + piiEntitiesRecognitionResults?: RecognizePiiEntitiesResultArray[]; } // @public @@ -98,9 +75,7 @@ export type BeginAnalyzeHealthcareOperationState = PollOperationState; +export type PagedAsyncIterableAnalyzeResults = PagedAsyncIterableIterator; // @public export type PagedAsyncIterableHealthEntities = PagedAsyncIterableIterator; @@ -326,18 +283,11 @@ export enum PiiEntityDomainType { PROTECTED_HEALTH_INFORMATION = "PHI" } -// @public (undocumented) +// @public export type PiiTask = { - parameters?: PiiTaskParameters; -}; - -// @public (undocumented) -export interface PiiTaskParameters { - // (undocumented) domain?: PiiTaskParametersDomain; - // (undocumented) modelVersion?: string; -} +}; // @public export type PiiTaskParametersDomain = "phi" | "none" | string; diff --git a/sdk/textanalytics/ai-text-analytics/samples/javascript/README.md b/sdk/textanalytics/ai-text-analytics/samples/javascript/README.md index 930b8fe0fc05..f8ac567b50b4 100644 --- a/sdk/textanalytics/ai-text-analytics/samples/javascript/README.md +++ b/sdk/textanalytics/ai-text-analytics/samples/javascript/README.md @@ -24,6 +24,8 @@ These sample programs show how to use the JavaScript client libraries for Azure | [recognizeLinkedEntities.js][recognizelinkedentities] | detects entities that have links to more information on the web | | [authenticationMethods.js][authenticationmethods] | authenticates a service client using both Azure Active Directory and an API key | | [recognizeEntities.js][recognizeentities] | detects entites in a piece of text and prints them along with the entity type | +| [beginAnalyzeHealthcare.js][beginanalyzehealthcare] | detects healthcare entities of a piece of text | +| [beginAnalyze.js][beginanalyze] | applies multiple tasks at once | ## Prerequisites @@ -70,6 +72,8 @@ Take a look at our [API Documentation][apiref] for more information about the AP [recognizepii]: https://github.com/Azure/azure-sdk-for-js/blob/master/sdk/textanalytics/ai-text-analytics/samples/javascript/recognizePii.js [recognizelinkedentities]: https://github.com/Azure/azure-sdk-for-js/tree/master/sdk/textanalytics/ai-text-analytics/samples/javascript/recognizeLinkedEntities.js [recognizeentities]: https://github.com/Azure/azure-sdk-for-js/tree/master/sdk/textanalytics/ai-text-analytics/samples/javascript/recognizeEntities.js +[beginanalyze]: https://github.com/Azure/azure-sdk-for-js/tree/master/sdk/textanalytics/ai-text-analytics/samples/javascript/beginAnalyze.js +[beginanalyzehealthcare]: https://github.com/Azure/azure-sdk-for-js/tree/master/sdk/textanalytics/ai-text-analytics/samples/javascript/beginAnalyzeHealthcare.js [apiref]: https://docs.microsoft.com/javascript/api/@azure/ai-text-analytics [azcogsvc]: https://docs.microsoft.com/azure/cognitive-services/cognitive-services-apis-create-account [freesub]: https://azure.microsoft.com/free/ diff --git a/sdk/textanalytics/ai-text-analytics/samples/javascript/beginAnalyze.js b/sdk/textanalytics/ai-text-analytics/samples/javascript/beginAnalyze.js new file mode 100644 index 000000000000..681b9d25742d --- /dev/null +++ b/sdk/textanalytics/ai-text-analytics/samples/javascript/beginAnalyze.js @@ -0,0 +1,81 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +/** + * extracts key phrases, entities, and pii entities from a piece of text + */ + +const { TextAnalyticsClient, AzureKeyCredential } = require("@azure/ai-text-analytics"); + +// Load the .env file if it exists +const dotenv = require("dotenv"); +dotenv.config(); + +// You will need to set these environment variables or edit the following values +const endpoint = process.env["ENDPOINT"] || ""; +const apiKey = process.env["TEXT_ANALYTICS_API_KEY"] || ""; + +const documents = [ + "Redmond is a city in King County, Washington, United States, located 15 miles east of Seattle.", + "I need to take my cat to the veterinarian.", + "The employee's SSN is 555-55-5555." +]; + +async function main() { + console.log("== Analyze Sample =="); + + const client = new TextAnalyticsClient(endpoint, new AzureKeyCredential(apiKey)); + + const tasks = { + entityRecognitionTasks: [{ modelVersion: "latest" }], + entityRecognitionPiiTasks: [{ modelVersion: "latest" }], + keyPhraseExtractionTasks: [{ modelVersion: "latest" }] + }; + const poller = await client.beginAnalyze(documents, tasks); + const resultPages = await poller.pollUntilDone(); + + for await (const page of resultPages) { + const keyPhrasesResults = page.keyPhrasesExtractionResults[0]; + for (const doc of keyPhrasesResults) { + console.log(`- Document ${doc.id}`); + if (!doc.error) { + console.log(" Key phrases:"); + for (const phrase of doc.keyPhrases) { + console.log(` ${phrase}`); + } + } else { + console.error(" Error:", doc.error); + } + } + + const entitiesResults = page.entitiesRecognitionResults[0]; + for (const doc of entitiesResults) { + console.log(`- Document ${doc.id}`); + if (!doc.error) { + console.log(" Entities:"); + for (const entity of doc.entities) { + console.log(` ${entity}`); + } + } else { + console.error(" Error:", doc.error); + } + } + + const piiEntitiesResults = page.piiEntitiesRecognitionResults[0]; + for (const doc of piiEntitiesResults) { + console.log(`- Document ${doc.id}`); + if (!doc.error) { + console.log(" Pii Entities:"); + for (const entity of doc.entities) { + console.log(` ${entity}`); + } + } else { + console.error(" Error:", doc.error); + } + } + } +} + +main().catch((err) => { + console.error("The sample encountered an error:", err); +}); diff --git a/sdk/textanalytics/ai-text-analytics/samples/javascript/beginAnalyzeHealthcare.js b/sdk/textanalytics/ai-text-analytics/samples/javascript/beginAnalyzeHealthcare.js new file mode 100644 index 000000000000..4cd661ca6ea2 --- /dev/null +++ b/sdk/textanalytics/ai-text-analytics/samples/javascript/beginAnalyzeHealthcare.js @@ -0,0 +1,44 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +/** + * detects healthcare entities in a piece of text and prints them + */ + +const { TextAnalyticsClient, AzureKeyCredential } = require("@azure/ai-text-analytics"); + +// Load the .env file if it exists +const dotenv = require("dotenv"); +dotenv.config(); + +// You will need to set these environment variables or edit the following values +const endpoint = process.env["ENDPOINT"] || ""; +const apiKey = process.env["TEXT_ANALYTICS_API_KEY"] || ""; + +const documents = [ + "Prescribed 100mg ibuprofen, taken twice daily.", + "Patient does not suffer from high blood pressure." +]; + +async function main() { + console.log("== Recognize Healthcare Entities Sample =="); + + const client = new TextAnalyticsClient(endpoint, new AzureKeyCredential(apiKey)); + + const poller = await client.beginAnalyzeHealthcare(documents); + const results = await poller.pollUntilDone(); + + for await (const result of results) { + console.log(`- Document ${result.id}`); + if (!result.error) { + console.log(" Recognized Entities:"); + for (const entity of result.entities) { + console.log(` - Entity ${entity.text} of type ${entity.category}`); + } + } else console.error(" Error:", result.error); + } +} + +main().catch((err) => { + console.error("The sample encountered an error:", err); +}); diff --git a/sdk/textanalytics/ai-text-analytics/samples/typescript/README.md b/sdk/textanalytics/ai-text-analytics/samples/typescript/README.md index b67c66f2dbcd..f3b065b1c14d 100644 --- a/sdk/textanalytics/ai-text-analytics/samples/typescript/README.md +++ b/sdk/textanalytics/ai-text-analytics/samples/typescript/README.md @@ -24,6 +24,8 @@ These sample programs show how to use the TypeScript client libraries for Azure | [recognizeLinkedEntities.ts][recognizelinkedentities] | detects entities that have links to more information on the web | | [authenticationMethods.ts][authenticationmethods] | authenticates a service client using both Azure Active Directory and an API key | | [recognizeEntities.ts][recognizeentities] | detects entites in a piece of text and prints them along with the entity type | +| [beginAnalyzeHealthcare.ts][beginanalyzehealthcare] | detects healthcare entities of a piece of text | +| [beginAnalyze.ts][beginanalyze] | applies multiple tasks at once | ## Prerequisites @@ -79,9 +81,11 @@ Take a look at our [API Documentation][apiref] for more information about the AP [authenticationmethods]: https://github.com/Azure/azure-sdk-for-js/tree/master/sdk/textanalytics/ai-text-analytics/samples/typescript/src/authenticationMethods.ts [detectlanguages]: https://github.com/Azure/azure-sdk-for-js/tree/master/sdk/textanalytics/ai-text-analytics/samples/typescript/src/detectLanguage.ts [extractkeyphrases]: https://github.com/Azure/azure-sdk-for-js/tree/master/sdk/textanalytics/ai-text-analytics/samples/typescript/src/extractKeyPhrases.ts -[recognizepii]:https://github.com/Azure/azure-sdk-for-js/blob/master/sdk/textanalytics/ai-text-analytics/samples/typescript/src/recognizePii.ts +[recognizepii]: https://github.com/Azure/azure-sdk-for-js/blob/master/sdk/textanalytics/ai-text-analytics/samples/typescript/src/recognizePii.ts [recognizelinkedentities]: https://github.com/Azure/azure-sdk-for-js/tree/master/sdk/textanalytics/ai-text-analytics/samples/typescript/src/recognizeLinkedEntities.ts [recognizeentities]: https://github.com/Azure/azure-sdk-for-js/tree/master/sdk/textanalytics/ai-text-analytics/samples/typescript/src/recognizeEntities.ts +[beginanalyze]: https://github.com/Azure/azure-sdk-for-js/tree/master/sdk/textanalytics/ai-text-analytics/samples/typescript/src/beginAnalyze.ts +[beginanalyzehealthcare]: https://github.com/Azure/azure-sdk-for-js/tree/master/sdk/textanalytics/ai-text-analytics/samples/typescript/src/beginAnalyzeHealthcare.ts [apiref]: https://docs.microsoft.com/javascript/api/@azure/ai-text-analytics [azcogsvc]: https://docs.microsoft.com/azure/cognitive-services/cognitive-services-apis-create-account [freesub]: https://azure.microsoft.com/free/ diff --git a/sdk/textanalytics/ai-text-analytics/samples/typescript/src/beginAnalyze.ts b/sdk/textanalytics/ai-text-analytics/samples/typescript/src/beginAnalyze.ts new file mode 100644 index 000000000000..0010984548f5 --- /dev/null +++ b/sdk/textanalytics/ai-text-analytics/samples/typescript/src/beginAnalyze.ts @@ -0,0 +1,81 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +/** + * extracts key phrases, entities, and pii entities from a piece of text + */ + +import { TextAnalyticsClient, AzureKeyCredential } from "@azure/ai-text-analytics"; + +// Load the .env file if it exists +import * as dotenv from "dotenv"; +dotenv.config(); + +// You will need to set these environment variables or edit the following values +const endpoint = process.env["ENDPOINT"] || ""; +const apiKey = process.env["TEXT_ANALYTICS_API_KEY"] || ""; + +const documents = [ + "Redmond is a city in King County, Washington, United States, located 15 miles east of Seattle.", + "I need to take my cat to the veterinarian.", + "The employee's SSN is 555-55-5555." +]; + +export async function main() { + console.log("== Analyze Sample =="); + + const client = new TextAnalyticsClient(endpoint, new AzureKeyCredential(apiKey)); + + const tasks = { + entityRecognitionTasks: [{ modelVersion: "latest" }], + entityRecognitionPiiTasks: [{ modelVersion: "latest" }], + keyPhraseExtractionTasks: [{ modelVersion: "latest" }] + }; + const poller = await client.beginAnalyze(documents, tasks); + const resultPages = await poller.pollUntilDone(); + + for await (const page of resultPages) { + const keyPhrasesResults = page.keyPhrasesExtractionResults![0]; + for (const doc of keyPhrasesResults) { + console.log(`- Document ${doc.id}`); + if (!doc.error) { + console.log(" Key phrases:"); + for (const phrase of doc.keyPhrases) { + console.log(` ${phrase}`); + } + } else { + console.error(" Error:", doc.error); + } + } + + const entitiesResults = page.entitiesRecognitionResults![0]; + for (const doc of entitiesResults) { + console.log(`- Document ${doc.id}`); + if (!doc.error) { + console.log(" Entities:"); + for (const entity of doc.entities) { + console.log(` ${entity}`); + } + } else { + console.error(" Error:", doc.error); + } + } + + const piiEntitiesResults = page.piiEntitiesRecognitionResults![0]; + for (const doc of piiEntitiesResults) { + console.log(`- Document ${doc.id}`); + if (!doc.error) { + console.log(" Pii Entities:"); + for (const entity of doc.entities) { + console.log(` ${entity}`); + } + } else { + console.error(" Error:", doc.error); + } + } + } +} + +main().catch((err) => { + console.error("The sample encountered an error:", err); +}); diff --git a/sdk/textanalytics/ai-text-analytics/samples/typescript/src/beginAnalyzeHealthcare.ts b/sdk/textanalytics/ai-text-analytics/samples/typescript/src/beginAnalyzeHealthcare.ts new file mode 100644 index 000000000000..23c38d32647b --- /dev/null +++ b/sdk/textanalytics/ai-text-analytics/samples/typescript/src/beginAnalyzeHealthcare.ts @@ -0,0 +1,44 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +/** + * detects healthcare entities in a piece of text and prints them + */ + +import { TextAnalyticsClient, AzureKeyCredential } from "@azure/ai-text-analytics"; + +// Load the .env file if it exists +import * as dotenv from "dotenv"; +dotenv.config(); + +// You will need to set these environment variables or edit the following values +const endpoint = process.env["ENDPOINT"] || ""; +const apiKey = process.env["TEXT_ANALYTICS_API_KEY"] || ""; + +const documents = [ + "Prescribed 100mg ibuprofen, taken twice daily.", + "Patient does not suffer from high blood pressure." +]; + +export async function main() { + console.log("== Recognize Healthcare Entities Sample =="); + + const client = new TextAnalyticsClient(endpoint, new AzureKeyCredential(apiKey)); + + const poller = await client.beginAnalyzeHealthcare(documents); + const results = await poller.pollUntilDone(); + + for await (const result of results) { + console.log(`- Document ${result.id}`); + if (!result.error) { + console.log(" Recognized Entities:"); + for (const entity of result.entities) { + console.log(` - Entity ${entity.text} of type ${entity.category}`); + } + } else console.error(" Error:", result.error); + } +} + +main().catch((err) => { + console.error("The sample encountered an error:", err); +}); diff --git a/sdk/textanalytics/ai-text-analytics/src/analyzeResult.ts b/sdk/textanalytics/ai-text-analytics/src/analyzeResult.ts index 06d1cdda8985..0199b54b855a 100644 --- a/sdk/textanalytics/ai-text-analytics/src/analyzeResult.ts +++ b/sdk/textanalytics/ai-text-analytics/src/analyzeResult.ts @@ -2,62 +2,41 @@ // Licensed under the MIT license. import { PagedAsyncIterableIterator } from "@azure/core-paging"; -import { ExtractKeyPhrasesSuccessResult } from "./extractKeyPhrasesResult"; +import { ExtractKeyPhrasesResultArray } from "./extractKeyPhrasesResultArray"; import { TextDocumentBatchStatistics } from "./generated/models"; -import { RecognizeCategorizedEntitiesSuccessResult } from "./recognizeCategorizedEntitiesResult"; -import { RecognizePiiEntitiesSuccessResult } from "./recognizePiiEntitiesResult"; -import { TextAnalyticsErrorResult } from "./textAnalyticsResult"; +import { RecognizeCategorizedEntitiesResultArray } from "./recognizeCategorizedEntitiesResultArray"; +import { RecognizePiiEntitiesResultArray } from "./recognizePiiEntitiesResultArray"; /** - * The results of a successful analyze entities job on a single document. + * The results of a successful analyze job. */ -export interface AnalyzeEntitiesResult extends RecognizeCategorizedEntitiesSuccessResult { - type: "Entities"; -} - -export interface AnalyzePiiEntitiesResult extends RecognizePiiEntitiesSuccessResult { - type: "PiiEntities"; -} - -export interface AnalyzeKeyPhrasesResult extends ExtractKeyPhrasesSuccessResult { - type: "KeyPhrases"; -} - -/** - * An error result from the analyze operation on a single document. - */ -export interface AnalyzeErrorResult extends TextAnalyticsErrorResult { - type: "Error"; +export interface AnalyzeResult { + /** + * Array of the results for each categorized entities recognition task. + */ + entitiesRecognitionResults?: RecognizeCategorizedEntitiesResultArray[]; + /** + * Array of the results for each Pii entities recognition task. + */ + piiEntitiesRecognitionResults?: RecognizePiiEntitiesResultArray[]; + /** + * Array of the results for each key phrases extraction task. + */ + keyPhrasesExtractionResults?: ExtractKeyPhrasesResultArray[]; } /** - * The results of a successful analyze job for a single document. - */ -export type AnalyzeResult = - | AnalyzeEntitiesResult - | AnalyzePiiEntitiesResult - | AnalyzeKeyPhrasesResult - | AnalyzeErrorResult; - -/** - * Array of {@link AnalyzeResult} - */ -export interface AnalyzeResultsArray extends Array {} - -/** - * The results of an analyze job represented as a paginated iterator that can - * either iterate over the results on a document-by-document basis or, by - * byPage(), can iterate over pages of documents. + * The results of an analyze job represented as a paginated iterator that + * iterates over the results of the requested tasks. */ export type PagedAsyncIterableAnalyzeResults = PagedAsyncIterableIterator< AnalyzeResult, - AnalyzeResultsArray + AnalyzeResult >; /** - * The results of an analyze job represented as a paginated iterator that can - * either iterate over the results on a document-by-document basis or, by - * byPage(), can iterate over pages of documents. + * The results of an analyze job represented as a paginated iterator that + * iterates over the results of the requested tasks. */ export interface PaginatedAnalyzeResults extends PagedAsyncIterableAnalyzeResults { /** diff --git a/sdk/textanalytics/ai-text-analytics/src/constants.ts b/sdk/textanalytics/ai-text-analytics/src/constants.ts index 41ad9afbf616..73683a51156b 100644 --- a/sdk/textanalytics/ai-text-analytics/src/constants.ts +++ b/sdk/textanalytics/ai-text-analytics/src/constants.ts @@ -1,4 +1,4 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -export const SDK_VERSION: string = "5.2.0-beta.1"; +export const SDK_VERSION: string = "5.1.0-beta.3"; diff --git a/sdk/textanalytics/ai-text-analytics/src/dom.d.ts b/sdk/textanalytics/ai-text-analytics/src/dom.d.ts new file mode 100644 index 000000000000..88bcf1442b2f --- /dev/null +++ b/sdk/textanalytics/ai-text-analytics/src/dom.d.ts @@ -0,0 +1,4 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +/// diff --git a/sdk/textanalytics/ai-text-analytics/src/generated/generatedClientContext.ts b/sdk/textanalytics/ai-text-analytics/src/generated/generatedClientContext.ts index 5be5f7441879..5dac659849cd 100644 --- a/sdk/textanalytics/ai-text-analytics/src/generated/generatedClientContext.ts +++ b/sdk/textanalytics/ai-text-analytics/src/generated/generatedClientContext.ts @@ -10,7 +10,7 @@ import * as coreHttp from "@azure/core-http"; import { GeneratedClientOptionalParams } from "./models"; const packageName = "@azure/ai-text-analytics"; -const packageVersion = "5.2.0-beta.1"; +const packageVersion = "5.1.0-beta.3"; export class GeneratedClientContext extends coreHttp.ServiceClient { endpoint: string; diff --git a/sdk/textanalytics/ai-text-analytics/src/generated/models/index.ts b/sdk/textanalytics/ai-text-analytics/src/generated/models/index.ts index bb38f45af3f5..a0ff8a446a4a 100644 --- a/sdk/textanalytics/ai-text-analytics/src/generated/models/index.ts +++ b/sdk/textanalytics/ai-text-analytics/src/generated/models/index.ts @@ -175,7 +175,7 @@ export interface TasksState { } export interface TasksStateTasks { - details?: TasksStateTasksDetails; + details?: TaskState; completed: number; failed: number; inProgress: number; @@ -185,18 +185,14 @@ export interface TasksStateTasks { keyPhraseExtractionTasks?: TasksStateTasksKeyPhraseExtractionTasksItem[]; } -export interface TasksStateTasksDetails { - allof?: TaskState; -} - export interface TaskState { lastUpdateDateTime: Date; - name: string; + name?: string; status: State; } export interface Components15Gvwi3SchemasTasksstatePropertiesTasksPropertiesEntityrecognitiontasksItemsAllof1 { - results?: EntitiesResult; + results: EntitiesResult; } export interface EntitiesResult { @@ -303,7 +299,7 @@ export interface DocumentError { } export interface Components15X8E9LSchemasTasksstatePropertiesTasksPropertiesEntityrecognitionpiitasksItemsAllof1 { - results?: PiiResult; + results: PiiResult; } export interface PiiResult { @@ -349,7 +345,7 @@ export interface PiiDocumentEntities { } export interface Components1D9IzucSchemasTasksstatePropertiesTasksPropertiesKeyphraseextractiontasksItemsAllof1 { - results?: KeyPhraseResult; + results: KeyPhraseResult; } export interface KeyPhraseResult { @@ -811,6 +807,8 @@ export type TasksStateTasksEntityRecognitionPiiTasksItem = TaskState & export type TasksStateTasksKeyPhraseExtractionTasksItem = TaskState & Components1D9IzucSchemasTasksstatePropertiesTasksPropertiesKeyphraseextractiontasksItemsAllof1 & {}; +export type TasksStateTasksDetails = TaskState & {}; + export type HealthcareEntity = Entity & { isNegated: boolean; /** @@ -890,13 +888,13 @@ export type ErrorCodeValue = * Defines values for State. */ export type State = - | "notstarted" + | "notStarted" | "running" | "succeeded" | "failed" + | "rejected" | "cancelled" | "cancelling" - | "notStarted" | "partiallyCompleted"; /** * Defines values for DocumentSentimentLabel. diff --git a/sdk/textanalytics/ai-text-analytics/src/generated/models/mappers.ts b/sdk/textanalytics/ai-text-analytics/src/generated/models/mappers.ts index 0faac754fd59..b5f4fb6df7fd 100644 --- a/sdk/textanalytics/ai-text-analytics/src/generated/models/mappers.ts +++ b/sdk/textanalytics/ai-text-analytics/src/generated/models/mappers.ts @@ -457,13 +457,13 @@ export const JobMetadata: coreHttp.CompositeMapper = { type: { name: "Enum", allowedValues: [ - "notstarted", + "notStarted", "running", "succeeded", "failed", + "rejected", "cancelled", "cancelling", - "notStarted", "partiallyCompleted" ] } @@ -497,7 +497,7 @@ export const TasksStateTasks: coreHttp.CompositeMapper = { serializedName: "details", type: { name: "Composite", - className: "TasksStateTasksDetails" + className: "TaskState" } }, completed: { @@ -568,22 +568,6 @@ export const TasksStateTasks: coreHttp.CompositeMapper = { } }; -export const TasksStateTasksDetails: coreHttp.CompositeMapper = { - type: { - name: "Composite", - className: "TasksStateTasksDetails", - modelProperties: { - allof: { - serializedName: "allof", - type: { - name: "Composite", - className: "TaskState" - } - } - } - } -}; - export const TaskState: coreHttp.CompositeMapper = { type: { name: "Composite", @@ -598,7 +582,6 @@ export const TaskState: coreHttp.CompositeMapper = { }, name: { serializedName: "name", - required: true, type: { name: "String" } @@ -609,13 +592,13 @@ export const TaskState: coreHttp.CompositeMapper = { type: { name: "Enum", allowedValues: [ - "notstarted", + "notStarted", "running", "succeeded", "failed", + "rejected", "cancelled", "cancelling", - "notStarted", "partiallyCompleted" ] } @@ -2099,6 +2082,16 @@ export const TasksStateTasksKeyPhraseExtractionTasksItem: coreHttp.CompositeMapp } }; +export const TasksStateTasksDetails: coreHttp.CompositeMapper = { + type: { + name: "Composite", + className: "TasksStateTasksDetails", + modelProperties: { + ...TaskState.type.modelProperties + } + } +}; + export const HealthcareEntity: coreHttp.CompositeMapper = { type: { name: "Composite", diff --git a/sdk/textanalytics/ai-text-analytics/src/index.ts b/sdk/textanalytics/ai-text-analytics/src/index.ts index 88422ddecf5a..3c0ed53b402d 100644 --- a/sdk/textanalytics/ai-text-analytics/src/index.ts +++ b/sdk/textanalytics/ai-text-analytics/src/index.ts @@ -16,10 +16,9 @@ export { RecognizeLinkedEntitiesOptions, PiiEntityDomainType, JobManifestTasks, - EntitiesTaskParameters, EntitiesTask, PiiTask, - PiiTaskParameters, + KeyPhrasesTask, BeginAnalyzeOptions, AnalyzePollerLike, BeginAnalyzeHealthcareOptions, @@ -84,12 +83,7 @@ export { export { PaginatedAnalyzeResults, PagedAsyncIterableAnalyzeResults, - AnalyzeResultsArray, - AnalyzeResult, - AnalyzeEntitiesResult, - AnalyzePiiEntitiesResult, - AnalyzeKeyPhrasesResult, - AnalyzeErrorResult + AnalyzeResult } from "./analyzeResult"; export { TextAnalyticsResult, @@ -119,9 +113,7 @@ export { AspectConfidenceScoreLabel, TokenSentimentValue, TextAnalyticsWarning, - KeyPhrasesTask, PiiTaskParametersDomain, - KeyPhrasesTaskParameters, HealthcareEntity, HealthcareRelation, HealthcareEntityLink diff --git a/sdk/textanalytics/ai-text-analytics/src/lro/analyze/operation.ts b/sdk/textanalytics/ai-text-analytics/src/lro/analyze/operation.ts index 2bcf361aee68..4d30ca2cdbfc 100644 --- a/sdk/textanalytics/ai-text-analytics/src/lro/analyze/operation.ts +++ b/sdk/textanalytics/ai-text-analytics/src/lro/analyze/operation.ts @@ -1,8 +1,6 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -/// - import { AbortSignalLike, OperationOptions, @@ -14,15 +12,11 @@ import { GeneratedClientAnalyzeStatusOptionalParams as AnalyzeJobStatusOptions, JobManifestTasks, State, - TasksStateTasksEntityRecognitionPiiTasksItem, - TasksStateTasksEntityRecognitionTasksItem, - TasksStateTasksKeyPhraseExtractionTasksItem, TextDocumentBatchStatistics, TextDocumentInput } from "../../generated/models"; import { AnalyzeResult, - AnalyzeResultsArray, PagedAsyncIterableAnalyzeResults, PaginatedAnalyzeResults } from "../../analyzeResult"; @@ -42,15 +36,23 @@ import { import { GeneratedClient as Client } from "../../generated"; import { CanonicalCode } from "@opentelemetry/api"; import { createSpan } from "../../tracing"; -import { TextAnalyticsOperationOptions } from "../../textAnalyticsOperationOptions"; -import { makeRecognizeCategorizedEntitiesResult } from "../../recognizeCategorizedEntitiesResult"; -import { makeRecognizePiiEntitiesResult } from "../../recognizePiiEntitiesResult"; -import { makeExtractKeyPhrasesResult } from "../../extractKeyPhrasesResult"; -import { processAndcombineSuccessfulErroneousDocuments } from "../../textAnalyticsResult"; +import { + makeRecognizeCategorizedEntitiesResultArray, + RecognizeCategorizedEntitiesResultArray +} from "../../recognizeCategorizedEntitiesResultArray"; +import { + makeRecognizePiiEntitiesResultArray, + RecognizePiiEntitiesResultArray +} from "../../recognizePiiEntitiesResultArray"; +import { + ExtractKeyPhrasesResultArray, + makeExtractKeyPhrasesResultArray +} from "../../extractKeyPhrasesResultArray"; +import { logger } from "../../logger"; export { State }; interface AnalyzeResultsWithPagination { - result: AnalyzeResultsArray; + result: AnalyzeResult; top?: number; skip?: number; } @@ -70,13 +72,24 @@ interface BeginAnalyzeInternalOptions extends OperationOptions {} /** * Options for configuring analyze jobs. */ -export interface AnalyzeJobOptions extends TextAnalyticsOperationOptions {} +export interface AnalyzeJobOptions extends OperationOptions { + /** + * If set to true, response will contain input and document level statistics. + */ + includeStatistics?: boolean; +} /** * Options for the begin analyze operation. */ export interface BeginAnalyzeOptions { + /** + * Options related to polling from the service. + */ polling?: PollingOptions; + /** + * Options related to the analyze job. + */ analyze?: AnalyzeJobOptions; } @@ -100,7 +113,7 @@ export class BeginAnalyzePollerOperation extends AnalysisPollOperation< private documents: TextDocumentInput[], private tasks: JobManifestTasks, private options: BeginAnalyzeOptions = {}, - private statusOptions: TextAnalyticsStatusOperationOptions + private statusOptions: TextAnalyticsStatusOperationOptions = {} ) { super(state); } @@ -110,12 +123,12 @@ export class BeginAnalyzePollerOperation extends AnalysisPollOperation< * "succeeded" and it returns an iterator for the results and provides a * byPage method to return the results paginated. */ - private listAnalyzeResultsByPage( + private listAnalyzeResults( jobId: string, options: AnalyzeJobStatusOptions = {} ): PagedAsyncIterableAnalyzeResults { - const iter = this._listAnalyzeResults(jobId, options); - const pagedIterator = { + const iter = this._listAnalyzeResultsPaginated(jobId, options); + return { next() { return iter.next(); }, @@ -127,19 +140,6 @@ export class BeginAnalyzePollerOperation extends AnalysisPollOperation< return this._listAnalyzeResultsPaginated(jobId, pageOptions); } }; - return pagedIterator; - } - - /** - * returns an iterator to the results of an analyze job. - */ - private async *_listAnalyzeResults( - jobId: string, - options?: AnalyzeJobStatusOptions - ): AsyncIterableIterator { - for await (const page of this._listAnalyzeResultsPaginated(jobId, options)) { - yield* page; - } } /** @@ -148,7 +148,7 @@ export class BeginAnalyzePollerOperation extends AnalysisPollOperation< private async *_listAnalyzeResultsPaginated( jobId: string, options?: AnalyzeJobStatusOptions - ): AsyncIterableIterator { + ): AsyncIterableIterator { let response = await this._listAnalyzeResultsSinglePage(jobId, options); yield response.result; while (response.skip) { @@ -178,14 +178,32 @@ export class BeginAnalyzePollerOperation extends AnalysisPollOperation< jobId, operationOptionsToRequestOptionsBase(finalOptions) ); - const result: AnalyzeResultsArray = [ - ...makeAnalyzeEntitiesResultArray(this.documents, response.tasks.entityRecognitionTasks), - ...makeAnalyzePiiEntitiesResultArray( - this.documents, - response.tasks.entityRecognitionPiiTasks + const result: AnalyzeResult = { + entitiesRecognitionResults: response.tasks.entityRecognitionTasks?.map( + ({ results }): RecognizeCategorizedEntitiesResultArray => + makeRecognizeCategorizedEntitiesResultArray( + this.documents, + results?.documents, + results?.errors, + results?.modelVersion, + results?.statistics + ) ), - ...makeAnalyzeKeyPhrasesResultArray(this.documents, response.tasks.keyPhraseExtractionTasks) - ]; + piiEntitiesRecognitionResults: response.tasks.entityRecognitionPiiTasks?.map( + ({ results }): RecognizePiiEntitiesResultArray => + makeRecognizePiiEntitiesResultArray(this.documents, results) + ), + keyPhrasesExtractionResults: response.tasks.keyPhraseExtractionTasks?.map( + ({ results }): ExtractKeyPhrasesResultArray => + makeExtractKeyPhrasesResultArray( + this.documents, + results?.documents, + results?.errors, + results?.modelVersion, + results?.statistics + ) + ) + }; return response.nextLink ? { result, ...nextLinkToTopAndSkip(response.nextLink) } : { result }; @@ -235,7 +253,7 @@ export class BeginAnalyzePollerOperation extends AnalysisPollOperation< case "running": break; default: { - throw new Error("Unrecognized state of the analyze job!"); + throw new Error(`Unrecognized state of the analyze job!: ${response.status}`); } } return { done: false }; @@ -298,6 +316,7 @@ export class BeginAnalyzePollerOperation extends AnalysisPollOperation< } state.jobId = getJobID(response.operationLocation); } + const status = await this.getAnalyzeStatus(state.jobId!, { ...this.statusOptions, abortSignal: updatedAbortSignal ? updatedAbortSignal : options.abortSignal @@ -307,7 +326,7 @@ export class BeginAnalyzePollerOperation extends AnalysisPollOperation< if (typeof options.fireProgress === "function") { options.fireProgress(state); } - const pagedIterator = this.listAnalyzeResultsByPage(state.jobId!, this.options.analyze || {}); + const pagedIterator = this.listAnalyzeResults(state.jobId!, this.options.analyze || {}); state.result = Object.assign(pagedIterator, { statistics: status.statistics }); @@ -315,70 +334,11 @@ export class BeginAnalyzePollerOperation extends AnalysisPollOperation< } return this; } -} - -function makeAnalyzeEntitiesResultArray( - documents: TextDocumentInput[], - tasksResults?: TasksStateTasksEntityRecognitionTasksItem[] -): AnalyzeResultsArray { - let result: AnalyzeResultsArray = []; - if (tasksResults) { - for (const task of tasksResults) { - if (task.results) { - const analyzeEntitiesResult: AnalyzeResultsArray = processAndcombineSuccessfulErroneousDocuments( - documents, - task.results, - (doc) => - makeRecognizeCategorizedEntitiesResult( - doc.id, - doc.entities, - doc.warnings, - doc.statistics - ) - ).map((doc) => (doc.error ? { type: "Error", ...doc } : { type: "Entities", ...doc })); - result.push(...analyzeEntitiesResult); - } - } - } - return result; -} - -function makeAnalyzePiiEntitiesResultArray( - documents: TextDocumentInput[], - tasksResults?: TasksStateTasksEntityRecognitionPiiTasksItem[] -): AnalyzeResultsArray { - let result: AnalyzeResultsArray = []; - if (tasksResults) { - for (const task of tasksResults) { - if (task.results) { - const analyzeEntitiesResult: AnalyzeResultsArray = processAndcombineSuccessfulErroneousDocuments( - documents, - task.results, - makeRecognizePiiEntitiesResult - ).map((doc) => (doc.error ? { type: "Error", ...doc } : { type: "PiiEntities", ...doc })); - result.push(...analyzeEntitiesResult); - } - } - } - return result; -} -function makeAnalyzeKeyPhrasesResultArray( - documents: TextDocumentInput[], - tasksResults?: TasksStateTasksKeyPhraseExtractionTasksItem[] -): AnalyzeResultsArray { - let result: AnalyzeResultsArray = []; - if (tasksResults) { - for (const task of tasksResults) { - if (task.results) { - const analyzeKeyPhrasesResult: AnalyzeResultsArray = processAndcombineSuccessfulErroneousDocuments( - documents, - task.results, - (doc) => makeExtractKeyPhrasesResult(doc.id, doc.keyPhrases, doc.warnings, doc.statistics) - ).map((doc) => (doc.error ? { type: "Error", ...doc } : { type: "KeyPhrases", ...doc })); - result.push(...analyzeKeyPhrasesResult); - } - } + async cancel(): Promise { + const state = this.state; + logger.warning(`The service does not yet support cancellation for beginAnalyze.`); + state.isCancelled = true; + return this; } - return result; } diff --git a/sdk/textanalytics/ai-text-analytics/src/lro/analyze/poller.ts b/sdk/textanalytics/ai-text-analytics/src/lro/analyze/poller.ts index 9498894534a4..103fc96bd889 100644 --- a/sdk/textanalytics/ai-text-analytics/src/lro/analyze/poller.ts +++ b/sdk/textanalytics/ai-text-analytics/src/lro/analyze/poller.ts @@ -45,22 +45,20 @@ export class BeginAnalyzePoller extends AnalysisPoller< if (resumeFrom) { state = JSON.parse(resumeFrom).state; } - const { includeStatistics, requestOptions, tracingOptions } = analysisOptions || {}; + const { requestOptions, tracingOptions } = analysisOptions || {}; const operation = new BeginAnalyzePollerOperation( state || {}, client, documents, tasks, { - analyze: analysisOptions, + analyze: { requestOptions, tracingOptions }, polling: { updateIntervalInMs, resumeFrom } }, - // take out modelVersion from the options that will be sent to the status - // API because it is not applicable. - { includeStatistics, requestOptions, tracingOptions } + analysisOptions ); super(operation); diff --git a/sdk/textanalytics/ai-text-analytics/src/lro/health/operation.ts b/sdk/textanalytics/ai-text-analytics/src/lro/health/operation.ts index eacf5ae137df..f30049dd543c 100644 --- a/sdk/textanalytics/ai-text-analytics/src/lro/health/operation.ts +++ b/sdk/textanalytics/ai-text-analytics/src/lro/health/operation.ts @@ -34,7 +34,7 @@ import { TextAnalyticsStatusOperationOptions } from "../poller"; import { GeneratedClient as Client } from "../../generated"; -import { combineSuccessfulErroneousDocuments } from "../../textAnalyticsResult"; +import { combineSuccessfulAndErroneousDocuments } from "../../textAnalyticsResult"; import { CanonicalCode } from "@opentelemetry/api"; import { createSpan } from "../../tracing"; import { TextAnalyticsOperationOptions } from "../../textAnalyticsOperationOptions"; @@ -80,7 +80,13 @@ export interface HealthcareJobOptions extends TextAnalyticsOperationOptions {} * Options for the begin analyze healthcare operation. */ export interface BeginAnalyzeHealthcareOptions { + /** + * Options related to polling from the service. + */ polling?: PollingOptions; + /** + * Options related to the healthcare job. + */ health?: HealthcareJobOptions; } @@ -117,7 +123,7 @@ export class BeginAnalyzeHealthcarePollerOperation extends AnalysisPollOperation options: HealthcareJobStatusOptions = {} ): PagedAsyncIterableHealthEntities { const iter = this._listHealthcareEntities(jobId, options); - const pagedIterator = { + return { next() { return iter.next(); }, @@ -129,7 +135,6 @@ export class BeginAnalyzeHealthcarePollerOperation extends AnalysisPollOperation return this._listHealthcareEntitiesPaginated(jobId, pageOptions); } }; - return pagedIterator; } /** @@ -181,7 +186,7 @@ export class BeginAnalyzeHealthcarePollerOperation extends AnalysisPollOperation operationOptionsToRequestOptionsBase(finalOptions) ); if (response.results) { - const result = combineSuccessfulErroneousDocuments(this.documents, response.results); + const result = combineSuccessfulAndErroneousDocuments(this.documents, response.results); return response.nextLink ? { result, ...nextLinkToTopAndSkip(response.nextLink) } : { result }; diff --git a/sdk/textanalytics/ai-text-analytics/src/lro/poller.ts b/sdk/textanalytics/ai-text-analytics/src/lro/poller.ts index fe2481e6c310..8c80e03ee6a2 100644 --- a/sdk/textanalytics/ai-text-analytics/src/lro/poller.ts +++ b/sdk/textanalytics/ai-text-analytics/src/lro/poller.ts @@ -69,24 +69,21 @@ export abstract class AnalysisPoller extends Poller implements PollOperation { +export abstract class AnalysisPollOperation + implements PollOperation { constructor(public state: TState) {} /** * @summary Meant to reach to the service and update the Poller operation. * @param [options] The optional parameters, which is only an abortSignal from @azure/abort-controller */ - public async update(): Promise> { - throw new Error("Operation not implemented."); - } + public abstract update(): Promise>; /** * @summary Meant to reach to the service and cancel the Poller operation. * @param [options] The optional parameters, which is only an abortSignal from @azure/abort-controller */ - public async cancel(): Promise> { - throw new Error("Operation not implemented."); - } + public abstract cancel(): Promise>; /** * @summary Serializes the Poller operation. diff --git a/sdk/textanalytics/ai-text-analytics/src/recognizePiiEntitiesResultArray.ts b/sdk/textanalytics/ai-text-analytics/src/recognizePiiEntitiesResultArray.ts index 51998da2fb43..d0b837783aa8 100644 --- a/sdk/textanalytics/ai-text-analytics/src/recognizePiiEntitiesResultArray.ts +++ b/sdk/textanalytics/ai-text-analytics/src/recognizePiiEntitiesResultArray.ts @@ -1,11 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -import { - TextDocumentBatchStatistics, - TextDocumentInput, - GeneratedClientEntitiesRecognitionPiiResponse -} from "./generated/models"; +import { TextDocumentBatchStatistics, TextDocumentInput, PiiResult } from "./generated/models"; import { RecognizePiiEntitiesResult, makeRecognizePiiEntitiesResult, @@ -33,7 +29,7 @@ export interface RecognizePiiEntitiesResultArray extends Array( +export function combineSuccessfulAndErroneousDocuments( input: TextDocumentInput[], - response: TextAnalyticsResponse -): (T1 | TextAnalyticsErrorResult)[] { - return processAndcombineSuccessfulErroneousDocuments(input, response, (x) => x); + response: TextAnalyticsResponse +): (TSuccess | TextAnalyticsErrorResult)[] { + return processAndCombineSuccessfulAndErroneousDocuments(input, response, (x) => x); } /** @@ -187,16 +187,16 @@ export function combineSuccessfulErroneousDocuments( input: TextDocumentInput[], - response: TextAnalyticsResponse, - process: (doc: T1) => T2 -): (T2 | TextAnalyticsErrorResult)[] { + response: TextAnalyticsResponse, + process: (doc: TSuccess) => T +): (T | TextAnalyticsErrorResult)[] { const unsortedResult = response.documents - .map((document): T2 | TextAnalyticsErrorResult => process(document)) + .map((document): T | TextAnalyticsErrorResult => process(document)) .concat( response.errors.map((error) => { return makeTextAnalyticsErrorResult(error.id, error.error); @@ -212,10 +212,13 @@ export function processAndcombineSuccessfulErroneousDocuments< * @param input the array of documents sent to the service for processing. * @param response the response received from the service. */ -export function combineSuccessfulErroneousDocumentsWithStatisticsAndModelVersion< - T1 extends TextAnalyticsSuccessResult ->(input: TextDocumentInput[], response: TextAnalyticsResponse): TextAnalyticsResultArray { - const sorted = combineSuccessfulErroneousDocuments(input, response); +export function combineSuccessfulAndErroneousDocumentsWithStatisticsAndModelVersion< + TSuccess extends TextAnalyticsSuccessResult +>( + input: TextDocumentInput[], + response: TextAnalyticsResponse +): TextAnalyticsResultArray { + const sorted = combineSuccessfulAndErroneousDocuments(input, response); return Object.assign(sorted, { statistics: response.statistics, modelVersion: response.modelVersion diff --git a/sdk/textanalytics/ai-text-analytics/src/util.ts b/sdk/textanalytics/ai-text-analytics/src/util.ts index 2f7a5a1ddc8f..9405c7709ccc 100644 --- a/sdk/textanalytics/ai-text-analytics/src/util.ts +++ b/sdk/textanalytics/ai-text-analytics/src/util.ts @@ -2,7 +2,8 @@ // Licensed under the MIT license. import { RestError } from "@azure/core-http"; -import { StringIndexType } from "./generated/models"; +import { URL, URLSearchParams } from "./utils/url"; +import { StringIndexType, StringIndexTypeResponse } from "./generated/models"; import { logger } from "./logger"; export interface IdObject { @@ -70,44 +71,41 @@ export function addStrEncodingParam(options: T): T & { stringIndexType: Strin return { ...options, stringIndexType: jsEncodingUnit }; } -export function addEncodingParamToTask( - task: X & { parameters?: Y & { stringIndexType?: StringIndexType } } -): X & { parameters?: Y & { stringIndexType?: StringIndexType } } { - if (task.parameters) { - task.parameters.stringIndexType = jsEncodingUnit; - } +export function addEncodingParamToTask( + task: X & { stringIndexType?: StringIndexTypeResponse } +): X & { stringIndexType?: StringIndexTypeResponse } { + task.stringIndexType = jsEncodingUnit; return task; } +export function AddParamsToTask(task: X): { parameters?: X } { + return { parameters: task }; +} + export interface PageParam { top: number; skip: number; } -function findParamValue(matches: RegExpMatchArray, param: string): number { - for (let i = 0; i < matches.length; i += 2) { - if (matches[i] === `\$${param}`) { - return parseInt(matches[i + 1]); - } - } - throw new Error(`The parameter \$${param} was not found in nextLink`); -} - export function nextLinkToTopAndSkip(nextLink: string): PageParam { - let regExp = /(?:\?|\&)([^=]+)\=([^\&]+)/g, - match, - matches: string[] = []; - while ((match = regExp.exec(nextLink))) { - matches.push(match[1], match[2]); + const url = new URL(nextLink); + const searchParams = new URLSearchParams(url.searchParams); + let top: number; + if (searchParams.has("$top")) { + top = parseInt(searchParams.get("$top")!); + } else { + throw new Error(`nextLink URL does not have the $top param: ${nextLink}`); } - if (matches) { - return { - skip: findParamValue(matches, "skip"), - top: findParamValue(matches, "top") - }; + let skip: number; + if (searchParams.has("$skip")) { + skip = parseInt(searchParams.get("$skip")!); } else { - throw new Error(`Malformed URL or a URL without parameters found`); + throw new Error(`nextLink URL does not have the $skip param: ${nextLink}`); } + return { + skip: skip, + top: top + }; } export function getJobID(operationLocation: string): string { diff --git a/sdk/textanalytics/ai-text-analytics/src/utils/url.browser.ts b/sdk/textanalytics/ai-text-analytics/src/utils/url.browser.ts new file mode 100644 index 000000000000..085c11b25cf4 --- /dev/null +++ b/sdk/textanalytics/ai-text-analytics/src/utils/url.browser.ts @@ -0,0 +1,7 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +const url = URL; +const urlSearchParams = URLSearchParams; + +export { url as URL, urlSearchParams as URLSearchParams }; diff --git a/sdk/textanalytics/ai-text-analytics/src/utils/url.ts b/sdk/textanalytics/ai-text-analytics/src/utils/url.ts new file mode 100644 index 000000000000..993e69798f9e --- /dev/null +++ b/sdk/textanalytics/ai-text-analytics/src/utils/url.ts @@ -0,0 +1,4 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +export { URL, URLSearchParams } from "url"; diff --git a/sdk/textanalytics/ai-text-analytics/swagger/README.md b/sdk/textanalytics/ai-text-analytics/swagger/README.md index 59c6ea734fab..62530d8e77b4 100644 --- a/sdk/textanalytics/ai-text-analytics/swagger/README.md +++ b/sdk/textanalytics/ai-text-analytics/swagger/README.md @@ -14,7 +14,7 @@ output-folder: ../ source-code-folder-path: ./src/generated input-file: https://raw.githubusercontent.com/Azure/azure-rest-api-specs/master/specification/cognitiveservices/data-plane/TextAnalytics/preview/v3.1-preview.3/TextAnalytics.json add-credentials: false -package-version: 5.2.0-beta.1 +package-version: 5.1.0-beta.3 v3: true use-extension: "@autorest/typescript": "6.0.0-dev.20201027.1" @@ -240,13 +240,11 @@ directive: ```yaml directive: - from: swagger-document - where: $["paths"] + where: $["paths"][*] transform: > - for (var path in $) { - for (var op of Object.values($[path])) { - if (op["x-ms-long-running-operation"]) { - delete op["x-ms-long-running-operation"]; - } + for (var op of Object.values($)) { + if (op["x-ms-long-running-operation"]) { + delete op["x-ms-long-running-operation"]; } } ``` diff --git a/sdk/textanalytics/ai-text-analytics/test/apiKey.spec.ts b/sdk/textanalytics/ai-text-analytics/test/apiKey.spec.ts index 1ff33e58d2fa..72dc977485c5 100644 --- a/sdk/textanalytics/ai-text-analytics/test/apiKey.spec.ts +++ b/sdk/textanalytics/ai-text-analytics/test/apiKey.spec.ts @@ -5,7 +5,7 @@ import { assert, use as chaiUse } from "chai"; import chaiPromises from "chai-as-promised"; chaiUse(chaiPromises); -import { Recorder } from "@azure/test-utils-recorder"; +import { isRecordMode, Recorder } from "@azure/test-utils-recorder"; import { createRecordedClient, testEnv } from "./utils/recordedClient"; import { TextAnalyticsClient, AzureKeyCredential, TextDocumentInput } from "../src/index"; @@ -25,7 +25,7 @@ describe("[API Key] TextAnalyticsClient", function() { const apiKey = new AzureKeyCredential(testEnv.TEXT_ANALYTICS_API_KEY); // eslint-disable-next-line no-invalid-this - this.timeout(10000); + this.timeout(100000); beforeEach(function() { // eslint-disable-next-line no-invalid-this @@ -79,6 +79,9 @@ describe("[API Key] TextAnalyticsClient", function() { }); describe("#health", () => { + if (isRecordMode() || process.env.TEST_MODE === "live") { + this.timeout(1000000); + } it("input strings", async () => { const poller = await client.beginAnalyzeHealthcare([ "Patient does not suffer from high blood pressure.", @@ -88,12 +91,11 @@ describe("[API Key] TextAnalyticsClient", function() { for await (const doc of result) { if (!doc.error) { assert.ok(doc.id); - assert.ok(doc.statistics); assert.ok(doc.entities); assert.ok(doc.relations); } } - }).timeout(1000000); + }); it("input documents", async () => { const poller = await client.beginAnalyzeHealthcare([ @@ -104,7 +106,6 @@ describe("[API Key] TextAnalyticsClient", function() { for await (const doc of result) { if (!doc.error) { assert.ok(doc.id); - assert.ok(doc.statistics); assert.ok(doc.entities); assert.ok(doc.relations); } @@ -125,13 +126,12 @@ describe("[API Key] TextAnalyticsClient", function() { const result3 = (await result.next()).value; if (!result3.error) { assert.ok(result3.id); - assert.ok(result3.statistics); assert.ok(result3.entities); assert.ok(result3.relations); } assert.ok(result1.error); assert.ok(result2.error); - }).timeout(1000000); + }); it("all inputs with errors", async () => { const docs = [ @@ -148,10 +148,10 @@ describe("[API Key] TextAnalyticsClient", function() { assert.ok(result1.error); assert.ok(result2.error); assert.ok(result3.error); - }).timeout(1000000); + }); it("too many documents", async () => { - const docs = Array(1001).fill("random text"); + const docs = Array(11).fill("random text"); try { await client.beginAnalyzeHealthcare(docs); assert.fail("Oops, an exception didn't happen."); @@ -160,15 +160,12 @@ describe("[API Key] TextAnalyticsClient", function() { assert.equal(e.code, "InvalidDocumentBatch"); assert.equal( e.message, - "Batch request contains too many records. Max 1000 records are permitted." + "Batch request contains too many records. Max 10 records are permitted." ); } - }).timeout(1000000); + }); - /** - * The service returns status code 413 which is not included in the swagger. - */ - it.skip("payload too large", async () => { + it("payload too large", async () => { const large_doc = "RECORD #333582770390100 | MH | 85986313 | | 054351 | 2/14/2001 12:00:00 AM | \ CORONARY ARTERY DISEASE | Signed | DIS | Admission Date: 5/22/2001 \ @@ -189,18 +186,15 @@ describe("[API Key] TextAnalyticsClient", function() { assert.fail("Oops, an exception didn't happen."); } catch (e) { assert.equal(e.statusCode, 413); - assert.equal(e.code, "BodyTooLarge"); + assert.equal(e.code, "InvalidDocumentBatch"); assert.equal( e.message, - "Request Payload sent is too large to be processed. Limit request size to: 524288." + "Request Payload sent is too large to be processed. Limit request size to: 524288" ); } }); - /** - * The service does not return warnings. - */ - it.skip("document warnings", async () => { + it("document warnings", async () => { const docs = [{ id: "1", text: "This won't actually create a warning :'(" }]; const poller = await client.beginAnalyzeHealthcare(docs); const result = await poller.pollUntilDone(); @@ -242,7 +236,7 @@ describe("[API Key] TextAnalyticsClient", function() { for await (const doc of result) { assert.equal(parseInt(doc.id), in_order[i++]); } - }).timeout(1000000); + }); it("show stats and model version", async () => { const docs: TextDocumentInput[] = [ @@ -279,7 +273,7 @@ describe("[API Key] TextAnalyticsClient", function() { for await (const doc of result) { assert.isUndefined(doc.error); } - }).timeout(1000000); + }); it("whole batch empty language hint", async () => { const docs = [ @@ -307,7 +301,7 @@ describe("[API Key] TextAnalyticsClient", function() { for await (const doc of result) { assert.isUndefined(doc.error); } - }).timeout(1000000); + }); it("whole batch with multiple languages", async () => { const docs = [ @@ -321,7 +315,7 @@ describe("[API Key] TextAnalyticsClient", function() { for await (const doc of result) { assert.isUndefined(doc.error); } - }).timeout(1000000); + }); it("invalid language hint", async () => { const docs = ["This should fail because we're passing in an invalid language hint"]; @@ -402,8 +396,12 @@ describe("[API Key] TextAnalyticsClient", function() { /** * the service by default returns pages of 20 documents each and this test * makes sure we get all the results and not just the first page. + * + * EDIT: the service decided to process only 10 documents max per request so + * pagination became unneeded. Once the service raises the limit on + * the number of input documents, we should re-enable these tests. */ - it("paged results one loop", async () => { + it.skip("paged results one loop", async () => { const docs = Array(40).fill("random text"); docs.push("Prescribed 100mg ibuprofen, taken twice daily."); const poller = await client.beginAnalyzeHealthcare(docs); @@ -421,9 +419,9 @@ describe("[API Key] TextAnalyticsClient", function() { } } assert.equal(docs.length, count); - }).timeout(1000000); + }); - it("paged results nested loop", async () => { + it.skip("paged results nested loop", async () => { const docs = Array(40).fill("random text"); docs.push("Prescribed 100mg ibuprofen, taken twice daily."); const poller = await client.beginAnalyzeHealthcare(docs); @@ -446,9 +444,9 @@ describe("[API Key] TextAnalyticsClient", function() { } assert.equal(docs.length, docCount); assert.equal(Math.ceil(docs.length / 20), pageCount); - }).timeout(1000000); + }); - it("paged results with custom page size", async () => { + it.skip("paged results with custom page size", async () => { const docs = Array(40).fill("random text"); docs.push("Prescribed 100mg ibuprofen, taken twice daily."); const poller = await client.beginAnalyzeHealthcare(docs); @@ -472,7 +470,7 @@ describe("[API Key] TextAnalyticsClient", function() { } assert.equal(docs.length, docCount); assert.equal(Math.ceil(docs.length / pageSize), pageCount); - }).timeout(1000000); + }); it("cancelled", async () => { const poller = await client.beginAnalyzeHealthcare([ @@ -485,12 +483,183 @@ describe("[API Key] TextAnalyticsClient", function() { }); describe("#analyze", () => { - it.only("input strings", async () => { + if (isRecordMode() || process.env.TEST_MODE === "live") { + this.timeout(1000000); + } + it("single entity recognition task", async () => { + const docs = [ + { id: "1", language: "en", text: "Microsoft was founded by Bill Gates and Paul Allen" }, + { id: "2", language: "es", text: "Microsoft fue fundado por Bill Gates y Paul Allen" } + ]; + + const poller = await client.beginAnalyze(docs, { + entityRecognitionTasks: [{ modelVersion: "latest" }] + }); + const result = await poller.pollUntilDone(); + for await (const page of result) { + const entitiesResult = page.entitiesRecognitionResults; + if (entitiesResult && entitiesResult.length === 1) { + const task = entitiesResult[0]; + for (const result of task) { + if (!result.error) { + assert.ok(result.id); + assert.ok(result.entities); + } else { + assert.fail("did not expect document errors but got one."); + } + } + } else { + assert.fail("expected an array of entities results but did not get one."); + } + } + }); + + it("single key phrases task", async () => { + const docs = [ + { id: "1", language: "en", text: "Microsoft was founded by Bill Gates and Paul Allen" }, + { id: "2", language: "es", text: "Microsoft fue fundado por Bill Gates y Paul Allen" } + ]; + + const poller = await client.beginAnalyze(docs, { + keyPhraseExtractionTasks: [{ modelVersion: "latest" }] + }); + const result = await poller.pollUntilDone(); + for await (const page of result) { + const keyPhrasesResult = page.keyPhrasesExtractionResults; + if (keyPhrasesResult && keyPhrasesResult.length === 1) { + const task = keyPhrasesResult[0]; + assert.equal(task.length, 2); + for (const result of task) { + if (!result.error) { + assert.include(result.keyPhrases, "Paul Allen"); + assert.include(result.keyPhrases, "Bill Gates"); + assert.include(result.keyPhrases, "Microsoft"); + assert.ok(result.id); + } + } + } else { + assert.fail("expected an array of key phrases results but did not get one."); + } + } + }); + + it("single entities recognition task", async () => { const docs = [ - { id: "1", language: "en", text: "Microsoft was founded by Bill Gates and Paul Allen." }, + { + id: "1", + text: "Microsoft was founded by Bill Gates and Paul Allen on April 4, 1975.", + language: "en" + }, { id: "2", - language: "en", + text: "Microsoft fue fundado por Bill Gates y Paul Allen el 4 de abril de 1975.", + language: "es" + }, + { + id: "3", + text: "Microsoft wurde am 4. April 1975 von Bill Gates und Paul Allen gegründet.", + language: "de" + } + ]; + + const poller = await client.beginAnalyze(docs, { + entityRecognitionTasks: [{ modelVersion: "latest" }] + }); + const result = await poller.pollUntilDone(); + for await (const page of result) { + const entitiesResult = page.entitiesRecognitionResults; + if (entitiesResult && entitiesResult.length === 1) { + const task = entitiesResult[0]; + assert.equal(task.length, 3); + for (const doc of task) { + if (!doc.error) { + assert.equal(doc.entities.length, 4); + for (const entity of doc.entities) { + assert.isDefined(entity.text); + assert.isDefined(entity.category); + assert.isDefined(entity.offset); + assert.isDefined(entity.confidenceScore); + } + } + } + } else { + assert.fail("expected an array of entities results but did not get one."); + } + } + }); + + it("single pii entities recognition task", async () => { + const docs = [ + { id: "1", text: "My SSN is 859-98-0987." }, + { + id: "2", + text: + "Your ABA number - 111000025 - is the first 9 digits in the lower left hand corner of your personal check." + }, + { id: "3", text: "Is 998.214.865-68 your Brazilian CPF number?" } + ]; + + const poller = await client.beginAnalyze(docs, { + entityRecognitionPiiTasks: [{ modelVersion: "latest" }] + }); + const result = await poller.pollUntilDone(); + for await (const page of result) { + const entitiesResult = page.piiEntitiesRecognitionResults; + if (entitiesResult && entitiesResult.length === 1) { + const task = entitiesResult[0]; + assert.equal(task.length, 3); + const doc1 = task[0]; + const doc2 = task[1]; + const doc3 = task[2]; + if (!doc1.error) { + assert.equal(doc1.entities[0].text, "859-98-0987"); + assert.equal(doc1.entities[0].category, "U.S. Social Security Number (SSN)"); + } + if (!doc2.error) { + assert.equal(doc2.entities[0].text, "111000025"); + // assert.equal(doc2.entities[0].category, "ABA Routing Number") # Service is currently returning PhoneNumber here + } + if (!doc3.error) { + assert.equal(doc3.entities[0].text, "998.214.865-68"); + assert.equal(doc3.entities[0].category, "Brazil CPF Number"); + } + for (const doc of task) { + if (!doc.error) { + for (const entity of doc.entities) { + assert.isDefined(entity.text); + assert.isDefined(entity.category); + assert.isDefined(entity.offset); + assert.isDefined(entity.confidenceScore); + } + } + } + } else { + assert.fail("expected an array of pii entities results but did not get one."); + } + } + }); + + it("bad request empty string", async () => { + const docs = [""]; + try { + const poller = await client.beginAnalyze(docs, { + entityRecognitionPiiTasks: [{ modelVersion: "latest" }] + }); + await poller.pollUntilDone(); + } catch (e) { + assert.equal(e.statusCode, 400); + } + }); + + /** + * Analyze responds with an InvalidArgument error instead of an InvalidDocument one + */ + it.skip("some documents with errors and multiple tasks", async () => { + const docs = [ + { id: "1", language: "", text: "" }, + { + id: "2", + language: "english", text: "I did not like the hotel we stayed at. It was too expensive." }, { @@ -501,25 +670,465 @@ describe("[API Key] TextAnalyticsClient", function() { ]; const poller = await client.beginAnalyze(docs, { - entityRecognitionTasks: [{ parameters: { modelVersion: "latest" } }] + entityRecognitionTasks: [{ modelVersion: "latest" }], + entityRecognitionPiiTasks: [{ modelVersion: "latest" }], + keyPhraseExtractionTasks: [{ modelVersion: "latest" }] }); const result = await poller.pollUntilDone(); - for await (const doc of result) { - switch (doc.type) { - case "Entities": { - assert.ok(doc.id); - assert.ok(doc.entities); - break; + for await (const page of result) { + const entitiesResult = page.entitiesRecognitionResults; + if (entitiesResult && entitiesResult.length === 1) { + const docs = entitiesResult[0]; + assert.equal(docs.length, 3); + assert.isDefined(docs[0].error); + assert.isDefined(docs[1].error); + assert.isUndefined(docs[2].error); + } else { + assert.fail("expected an array of entities results but did not get one."); + } + + const piiEntitiesResult = page.piiEntitiesRecognitionResults; + if (piiEntitiesResult && piiEntitiesResult.length === 1) { + const docs = piiEntitiesResult[0]; + assert.equal(docs.length, 3); + assert.isDefined(docs[0].error); + assert.isDefined(docs[1].error); + assert.isUndefined(docs[2].error); + } else { + assert.fail("expected an array of pii entities results but did not get one."); + } + + const keyPhrasesResult = page.keyPhrasesExtractionResults; + if (keyPhrasesResult && keyPhrasesResult.length === 1) { + const docs = keyPhrasesResult[0]; + assert.equal(docs.length, 3); + assert.isDefined(docs[0].error); + assert.isDefined(docs[1].error); + assert.isUndefined(docs[2].error); + } else { + assert.fail("expected an array of key phrases results but did not get one."); + } + } + }); + + /** + * Analyze responds with an InvalidArgument error instead of an InvalidDocument one + */ + it.skip("all documents with errors and multiple tasks", async () => { + const docs = [ + { id: "1", language: "", text: "" }, + { + id: "2", + language: "english", + text: "I did not like the hotel we stayed at. It was too expensive." + }, + { + id: "3", + language: "en", + text: "" + } + ]; + + const poller = await client.beginAnalyze(docs, { + entityRecognitionTasks: [{ modelVersion: "latest" }], + entityRecognitionPiiTasks: [{ modelVersion: "latest" }], + keyPhraseExtractionTasks: [{ modelVersion: "latest" }] + }); + const result = await poller.pollUntilDone(); + for await (const page of result) { + const entitiesResult = page.entitiesRecognitionResults; + if (entitiesResult && entitiesResult.length === 1) { + const docs = entitiesResult[0]; + assert.equal(docs.length, 3); + assert.isDefined(docs[0].error); + assert.isDefined(docs[1].error); + assert.isDefined(docs[2].error); + } else { + assert.fail("expected an array of entities results but did not get one."); + } + + const piiEntitiesResult = page.piiEntitiesRecognitionResults; + if (piiEntitiesResult && piiEntitiesResult.length === 1) { + const docs = piiEntitiesResult[0]; + assert.equal(docs.length, 3); + assert.isDefined(docs[0].error); + assert.isDefined(docs[1].error); + assert.isDefined(docs[2].error); + } else { + assert.fail("expected an array of pii entities results but did not get one."); + } + + const keyPhrasesResult = page.keyPhrasesExtractionResults; + if (keyPhrasesResult && keyPhrasesResult.length === 1) { + const docs = keyPhrasesResult[0]; + assert.equal(docs.length, 3); + assert.isDefined(docs[0].error); + assert.isDefined(docs[1].error); + assert.isDefined(docs[2].error); + } else { + assert.fail("expected an array of key phrases results but did not get one."); + } + } + }); + + it("output order is same as the input's one with multiple tasks", async () => { + const docs = [ + { id: "1", text: "one" }, + { id: "2", text: "two" }, + { id: "3", text: "three" }, + { id: "4", text: "four" }, + { id: "5", text: "five" } + ]; + + const poller = await client.beginAnalyze(docs, { + entityRecognitionTasks: [{ modelVersion: "latest" }], + entityRecognitionPiiTasks: [{ modelVersion: "latest" }], + keyPhraseExtractionTasks: [{ modelVersion: "latest" }] + }); + const result = await poller.pollUntilDone(); + for await (const page of result) { + const entitiesResult = page.entitiesRecognitionResults; + if (entitiesResult && entitiesResult.length === 1) { + const docs = entitiesResult[0]; + assert.equal(docs.length, 5); + let i = 1; + for (const doc of docs) { + assert.equal(parseInt(doc.id), i++); } - case "Error": { - assert.fail("Unexpected failure"); + } else { + assert.fail("expected an array of entities results but did not get one."); + } + + const piiEntitiesResult = page.piiEntitiesRecognitionResults; + if (piiEntitiesResult && piiEntitiesResult.length === 1) { + const docs = piiEntitiesResult[0]; + assert.equal(docs.length, 5); + let i = 1; + for (const doc of docs) { + assert.equal(parseInt(doc.id), i++); } - case "KeyPhrases": - case "PiiEntities": { - assert.fail("Unexpected task results"); + } else { + assert.fail("expected an array of pii entities results but did not get one."); + } + + const keyPhrasesResult = page.keyPhrasesExtractionResults; + if (keyPhrasesResult && keyPhrasesResult.length === 1) { + const docs = keyPhrasesResult[0]; + assert.equal(docs.length, 5); + let i = 1; + for (const doc of docs) { + assert.equal(parseInt(doc.id), i++); } + } else { + assert.fail("expected an array of key phrases results but did not get one."); } } - }).timeout(1000000); + }); + + it("out of order input IDs with multiple tasks", async () => { + const docs = [ + { id: "56", text: ":)" }, + { id: "0", text: ":(" }, + { id: "22", text: "w" }, + { id: "19", text: ":P" }, + { id: "1", text: ":D" } + ]; + + const poller = await client.beginAnalyze(docs, { + entityRecognitionTasks: [{ modelVersion: "latest" }], + entityRecognitionPiiTasks: [{ modelVersion: "latest" }], + keyPhraseExtractionTasks: [{ modelVersion: "latest" }] + }); + const result = await poller.pollUntilDone(); + const in_order = ["56", "0", "22", "19", "1"]; + for await (const page of result) { + const entitiesResult = page.entitiesRecognitionResults; + if (entitiesResult && entitiesResult.length === 1) { + const docs = entitiesResult[0]; + assert.equal(docs.length, 5); + let i = 0; + for (const doc of docs) { + assert.equal(doc.id, in_order[i++]); + } + } else { + assert.fail("expected an array of entities results but did not get one."); + } + + const piiEntitiesResult = page.piiEntitiesRecognitionResults; + if (piiEntitiesResult && piiEntitiesResult.length === 1) { + const docs = piiEntitiesResult[0]; + assert.equal(docs.length, 5); + let i = 0; + for (const doc of docs) { + assert.equal(doc.id, in_order[i++]); + } + } else { + assert.fail("expected an array of pii entities results but did not get one."); + } + + const keyPhrasesResult = page.keyPhrasesExtractionResults; + if (keyPhrasesResult && keyPhrasesResult.length === 1) { + const docs = keyPhrasesResult[0]; + assert.equal(docs.length, 5); + let i = 0; + for (const doc of docs) { + assert.equal(doc.id, in_order[i++]); + } + } else { + assert.fail("expected an array of key phrases results but did not get one."); + } + } + }); + + /** + * The service does not returns statistics + */ + it.skip("statistics", async () => { + const docs = [ + { id: "56", text: ":)" }, + { id: "0", text: ":(" }, + { id: "22", text: "" }, + { id: "19", text: ":P" }, + { id: "1", text: ":D" } + ]; + + const poller = await client.beginAnalyze( + docs, + { + entityRecognitionTasks: [{ modelVersion: "latest" }], + entityRecognitionPiiTasks: [{ modelVersion: "latest" }], + keyPhraseExtractionTasks: [{ modelVersion: "latest" }] + }, + { analyze: { includeStatistics: true } } + ); + const result = await poller.pollUntilDone(); + assert.equal(result.statistics?.documentCount, 5); + assert.equal(result.statistics?.transactionCount, 4); + assert.equal(result.statistics?.validDocumentCount, 4); + assert.equal(result.statistics?.erroneousDocumentCount, 1); + }); + + it("whole batch language hint", async () => { + const docs = [ + "This was the best day of my life.", + "I did not like the hotel we stayed at. It was too expensive.", + "The restaurant was not as good as I hoped." + ]; + + const poller = await client.beginAnalyze( + docs, + { + entityRecognitionTasks: [{ modelVersion: "latest" }], + entityRecognitionPiiTasks: [{ modelVersion: "latest" }], + keyPhraseExtractionTasks: [{ modelVersion: "latest" }] + }, + "en" + ); + const result = await poller.pollUntilDone(); + for await (const page of result) { + const entitiesResult = page.entitiesRecognitionResults!; + assert.equal(entitiesResult.length, 1); + for (const docs of entitiesResult) { + assert.equal(docs.length, 3); + for (const doc of docs) { + assert.isUndefined(doc.error); + } + } + } + }); + + it("whole batch with no language hint", async () => { + const docs = [ + "This was the best day of my life.", + "I did not like the hotel we stayed at. It was too expensive.", + "The restaurant was not as good as I hoped." + ]; + + const poller = await client.beginAnalyze( + docs, + { + entityRecognitionTasks: [{ modelVersion: "latest" }], + entityRecognitionPiiTasks: [{ modelVersion: "latest" }], + keyPhraseExtractionTasks: [{ modelVersion: "latest" }] + }, + "" + ); + const result = await poller.pollUntilDone(); + for await (const page of result) { + const entitiesResult = page.entitiesRecognitionResults!; + assert.equal(entitiesResult.length, 1); + for (const docs of entitiesResult) { + assert.equal(docs.length, 3); + for (const doc of docs) { + assert.isUndefined(doc.error); + } + } + } + }); + + it("each doc has a language hint", async () => { + const docs = [ + { id: "1", language: "", text: "I will go to the park." }, + { id: "2", language: "", text: "I did not like the hotel we stayed at." }, + { id: "3", text: "The restaurant had really good food." } + ]; + + const poller = await client.beginAnalyze(docs, { + entityRecognitionTasks: [{ modelVersion: "latest" }], + entityRecognitionPiiTasks: [{ modelVersion: "latest" }], + keyPhraseExtractionTasks: [{ modelVersion: "latest" }] + }); + const result = await poller.pollUntilDone(); + for await (const page of result) { + const entitiesResult = page.entitiesRecognitionResults!; + assert.equal(entitiesResult.length, 1); + for (const docs of entitiesResult) { + assert.equal(docs.length, 3); + for (const doc of docs) { + assert.isUndefined(doc.error); + } + } + } + }); + + it("whole batch input with a language hint", async () => { + const docs = [ + { id: "1", text: "I will go to the park." }, + { id: "2", text: "Este es un document escrito en Español." }, + { id: "3", text: "猫は幸せ" } + ]; + + const poller = await client.beginAnalyze(docs, { + entityRecognitionTasks: [{ modelVersion: "latest" }], + entityRecognitionPiiTasks: [{ modelVersion: "latest" }], + keyPhraseExtractionTasks: [{ modelVersion: "latest" }] + }); + const result = await poller.pollUntilDone(); + for await (const page of result) { + const entitiesResult = page.entitiesRecognitionResults!; + assert.equal(entitiesResult.length, 1); + for (const docs of entitiesResult) { + assert.equal(docs.length, 3); + for (const doc of docs) { + assert.isUndefined(doc.error); + } + } + } + }); + + it("invalid language hint", async () => { + const docs = ["This should fail because we're passing in an invalid language hint"]; + + const poller = await client.beginAnalyze( + docs, + { + entityRecognitionTasks: [{ modelVersion: "latest" }], + entityRecognitionPiiTasks: [{ modelVersion: "latest" }], + keyPhraseExtractionTasks: [{ modelVersion: "latest" }] + }, + "notalanguage" + ); + const result = await poller.pollUntilDone(); + const firstResult = (await result.next()).value; + const entitiesTaskDocs = firstResult?.entitiesRecognitionResults![0]; + for (const doc of entitiesTaskDocs) { + assert.equal(doc.error?.code, "InvalidArgument"); + } + const piiEntitiesTaskDocs = firstResult?.piiEntitiesRecognitionResults![0]; + for (const doc of piiEntitiesTaskDocs) { + assert.equal(doc.error?.code, "InvalidArgument"); + } + const keyPhrasesTaskDocs = firstResult?.keyPhrasesExtractionResults![0]; + for (const doc of keyPhrasesTaskDocs) { + assert.equal(doc.error?.code, "InvalidArgument"); + } + }); + + it.skip("bad model", async () => { + const docs = [ + { + id: "1", + language: "en", + text: "This should fail because we're passing in an invalid language hint" + } + ]; + + const poller = await client.beginAnalyze(docs, { + entityRecognitionTasks: [{ modelVersion: "bad" }], + entityRecognitionPiiTasks: [{ modelVersion: "bad" }], + keyPhraseExtractionTasks: [{ modelVersion: "bad" }] + }); + const result = await poller.pollUntilDone(); + const firstResult = (await result.next()).value; + const entitiesTaskDocs = firstResult?.entitiesRecognitionResults![0]; + for (const doc of entitiesTaskDocs) { + assert.equal(doc.error?.code, "UnknownError"); + } + const piiEntitiesTaskDocs = firstResult?.piiEntitiesRecognitionResults![0]; + for (const doc of piiEntitiesTaskDocs) { + assert.equal(doc.error?.code, "UnknownError"); + } + const keyPhrasesTaskDocs = firstResult?.keyPhrasesExtractionResults![0]; + for (const doc of keyPhrasesTaskDocs) { + assert.equal(doc.error?.code, "UnknownError"); + } + }); + + it("paged results with custom page size", async () => { + const totalDocs = 25; + const docs = Array(totalDocs - 1).fill("random text"); + docs.push("Microsoft was founded by Bill Gates and Paul Allen"); + const poller = await client.beginAnalyze(docs, { + entityRecognitionTasks: [{ modelVersion: "latest" }], + keyPhraseExtractionTasks: [{ modelVersion: "latest" }] + }); + const result = await poller.pollUntilDone(); + let docCount = 0, + pageCount = 0, + pageSize = 10; + for await (const page of result.byPage({ maxPageSize: pageSize })) { + const entitiesTaskDocs = page.entitiesRecognitionResults![0]; + ++pageCount; + for (const doc of entitiesTaskDocs) { + assert.isUndefined(doc.error); + ++docCount; + if (!doc.error) { + if (docCount === totalDocs) { + assert.equal(doc.entities.length, 3); + } else { + assert.equal(doc.entities.length, 0); + } + } + } + } + assert.equal(docs.length, docCount); + assert.equal(Math.ceil(docs.length / pageSize), pageCount); + }); + + it("pii redacted test is not empty", async () => { + const docs = [ + { id: "1", text: "I will go to the park." }, + { id: "2", text: "Este es un document escrito en Español." }, + { id: "3", text: "猫は幸せ" } + ]; + + const poller = await client.beginAnalyze(docs, { + entityRecognitionPiiTasks: [{ modelVersion: "latest" }] + }); + const result = await poller.pollUntilDone(); + for await (const page of result) { + const piiEntitiesResult = page.piiEntitiesRecognitionResults!; + assert.equal(piiEntitiesResult.length, 1); + for (const docs of piiEntitiesResult) { + assert.equal(docs.length, 3); + for (const doc of docs) { + assert.isUndefined(doc.error); + if (!doc.error) { + assert.isNotEmpty(doc.redactedText); + } + } + } + } + }); }); }); diff --git a/sdk/textanalytics/ai-text-analytics/test/collections.spec.ts b/sdk/textanalytics/ai-text-analytics/test/collections.spec.ts index 379237bf98ae..afe4f5d2e9c1 100644 --- a/sdk/textanalytics/ai-text-analytics/test/collections.spec.ts +++ b/sdk/textanalytics/ai-text-analytics/test/collections.spec.ts @@ -369,8 +369,7 @@ describe("RecognizeLinkedEntitiesResultArray", () => { } } ], - modelVersion: "", - _response: {} as any + modelVersion: "" }); const inputOrder = input.map((item) => item.id);