From ff8e1b2eaf005fd3554569ff65946788a8b21eb4 Mon Sep 17 00:00:00 2001 From: Heath Stewart Date: Fri, 14 Jan 2022 15:46:46 -0800 Subject: [PATCH] Enable Conversations live tests (#26346) * Enable Conversations live tests * Fix environment variables * Temporarily disable some tests Disable a few tests until #26379 is resolved so we can get nightly live tests working again. * Fix AnswersFollowupKnowledgeBaseQuestion results Had a second result come back that was 0.30 confidence, which I've seen in the past as well. Tends to vary whenever the model is retrained. Ask for a higher threshold now. --- .../ConversationAnalysisClientLiveTests.cs | 1 + ...Sample3_AnalyzeConversationWithLanguage.cs | 2 + .../tests/QuestionAnsweringClientLiveTests.cs | 2 +- .../AnswersFollowupKnowledgeBaseQuestion.json | 119 +++++++++--------- ...ersFollowupKnowledgeBaseQuestionAsync.json | 119 +++++++++--------- sdk/cognitivelanguage/test-resources.json | 52 ++++---- sdk/cognitivelanguage/tests.yml | 9 +- 7 files changed, 152 insertions(+), 152 deletions(-) diff --git a/sdk/cognitivelanguage/Azure.AI.Language.Conversations/tests/ConversationAnalysisClientLiveTests.cs b/sdk/cognitivelanguage/Azure.AI.Language.Conversations/tests/ConversationAnalysisClientLiveTests.cs index 9a3b02eb79d03..4bfda2ac9461d 100644 --- a/sdk/cognitivelanguage/Azure.AI.Language.Conversations/tests/ConversationAnalysisClientLiveTests.cs +++ b/sdk/cognitivelanguage/Azure.AI.Language.Conversations/tests/ConversationAnalysisClientLiveTests.cs @@ -33,6 +33,7 @@ public async Task AnalyzeConversation() } [RecordedTest] + [Ignore("https://github.com/Azure/azure-sdk-for-net/issues/26379")] public async Task AnalyzeConversationWithLanguage() { AnalyzeConversationOptions options = new AnalyzeConversationOptions() diff --git a/sdk/cognitivelanguage/Azure.AI.Language.Conversations/tests/Samples/Sample3_AnalyzeConversationWithLanguage.cs b/sdk/cognitivelanguage/Azure.AI.Language.Conversations/tests/Samples/Sample3_AnalyzeConversationWithLanguage.cs index d34112975f6bd..93e6aee3e6d41 100644 --- a/sdk/cognitivelanguage/Azure.AI.Language.Conversations/tests/Samples/Sample3_AnalyzeConversationWithLanguage.cs +++ b/sdk/cognitivelanguage/Azure.AI.Language.Conversations/tests/Samples/Sample3_AnalyzeConversationWithLanguage.cs @@ -12,6 +12,7 @@ public partial class ConversationAnalysisClientSamples { [SyncOnly] [RecordedTest] + [Ignore("https://github.com/Azure/azure-sdk-for-net/issues/26379")] public void AnalyzeConversationWithLanguage() { ConversationAnalysisClient client = Client; @@ -49,6 +50,7 @@ public void AnalyzeConversationWithLanguage() [AsyncOnly] [RecordedTest] + [Ignore("https://github.com/Azure/azure-sdk-for-net/issues/26379")] public async Task AnalyzeConversationWithLanguageAsync() { ConversationAnalysisClient client = Client; diff --git a/sdk/cognitivelanguage/Azure.AI.Language.QuestionAnswering/tests/QuestionAnsweringClientLiveTests.cs b/sdk/cognitivelanguage/Azure.AI.Language.QuestionAnswering/tests/QuestionAnsweringClientLiveTests.cs index 00184e3e67dcf..0fd7de34541da 100644 --- a/sdk/cognitivelanguage/Azure.AI.Language.QuestionAnswering/tests/QuestionAnsweringClientLiveTests.cs +++ b/sdk/cognitivelanguage/Azure.AI.Language.QuestionAnswering/tests/QuestionAnsweringClientLiveTests.cs @@ -48,7 +48,7 @@ public async Task AnswersFollowupKnowledgeBaseQuestion() { Size = 3, UserId = "sd53lsY=", - ConfidenceThreshold = 0.2, + ConfidenceThreshold = 0.5, AnswerContext = new(27) { PreviousQuestion = "How long should my Surface battery last?", diff --git a/sdk/cognitivelanguage/Azure.AI.Language.QuestionAnswering/tests/SessionRecords/QuestionAnsweringClientLiveTests/AnswersFollowupKnowledgeBaseQuestion.json b/sdk/cognitivelanguage/Azure.AI.Language.QuestionAnswering/tests/SessionRecords/QuestionAnsweringClientLiveTests/AnswersFollowupKnowledgeBaseQuestion.json index 49fe0f39f8138..330093cf3c7eb 100644 --- a/sdk/cognitivelanguage/Azure.AI.Language.QuestionAnswering/tests/SessionRecords/QuestionAnsweringClientLiveTests/AnswersFollowupKnowledgeBaseQuestion.json +++ b/sdk/cognitivelanguage/Azure.AI.Language.QuestionAnswering/tests/SessionRecords/QuestionAnsweringClientLiveTests/AnswersFollowupKnowledgeBaseQuestion.json @@ -8,19 +8,16 @@ "Content-Length": "328", "Content-Type": "application/json", "Ocp-Apim-Subscription-Key": "Sanitized", - "traceparent": "00-ec7b0797bd60e747982891071f420775-5a85fa315f8e1b42-00", - "User-Agent": [ - "azsdk-net-AI.Language.QuestionAnswering/1.0.0-alpha.20211028.1", - "(.NET 5.0.11; Microsoft Windows 10.0.22000)" - ], - "x-ms-client-request-id": "7247f8716e635a7fbe3a6a4c42039bd6", + "traceparent": "00-460445e1001c07abb71adaa9bdf9c78f-46d808af4f9d3b87-00", + "User-Agent": "azsdk-net-AI.Language.QuestionAnswering/1.1.0-alpha.20220114.1 (.NET 6.0.1; Microsoft Windows 10.0.22000)", + "x-ms-client-request-id": "a2700dbf6638283dc08f4898a0f4c37b", "x-ms-return-client-request-id": "true" }, "RequestBody": { "question": "How long it takes to charge Surface?", "top": 3, "userId": "sd53lsY=", - "confidenceScoreThreshold": 0.2, + "confidenceScoreThreshold": 0.5, "context": { "previousQnaId": 27, "previousUserQuery": "How long should my Surface battery last?" @@ -34,70 +31,68 @@ }, "StatusCode": 200, "ResponseHeaders": { - "apim-request-id": "3ea187c5-69d6-41e6-84c4-7328d42650d2", - "Content-Length": "1594", + "apim-request-id": "b1ab4b2a-d232-4d28-9019-49846131b0dd", + "Content-Length": "1606", "Content-Type": "application/json; charset=utf-8", "csp-billing-usage": "CognitiveServices.TextAnalytics.QuestionAnsweringTextRecords=1", - "Date": "Thu, 28 Oct 2021 21:15:36 GMT", + "Date": "Fri, 14 Jan 2022 23:16:49 GMT", "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", "X-Content-Type-Options": "nosniff", - "x-envoy-upstream-service-time": "2291" + "x-envoy-upstream-service-time": "1339" }, - "ResponseBody": [ - "{\n", - " \u0022answers\u0022: [\n", - " {\n", - " \u0022questions\u0022: [\n", - " \u0022Power and charging\u0022\n", - " ],\n", - " \u0022answer\u0022: \u0022**Power and charging**\\n\\nIt takes two to four hours to charge the Surface Pro 4 battery fully from an empty state. It can take longer if you\u2019re using your Surface for power-intensive activities like gaming or video streaming while you\u2019re charging it.\\n\\nYou can use the USB port on your Surface Pro 4 power supply to charge other devices, like a phone, while your Surface charges. The USB port on the power supply is only for charging, not for data transfer. If you want to use a USB device, plug it into the USB port on your Surface.\u0022,\n", - " \u0022confidenceScore\u0022: 0.6015,\n", - " \u0022id\u0022: 132,\n", - " \u0022source\u0022: \u0022surface-pro-4-user-guide-EN.pdf\u0022,\n", - " \u0022metadata\u0022: {\n", - " \u0022explicitlytaggedheading\u0022: \u0022power and charging\u0022\n", - " },\n", - " \u0022dialog\u0022: {\n", - " \u0022isContextOnly\u0022: false,\n", - " \u0022prompts\u0022: [\n", - " {\n", - " \u0022displayOrder\u0022: 0,\n", - " \u0022qnaId\u0022: 133,\n", - " \u0022displayText\u0022: \u0022Check the battery level\u0022\n", - " },\n", - " {\n", - " \u0022displayOrder\u0022: 1,\n", - " \u0022qnaId\u0022: 134,\n", - " \u0022displayText\u0022: \u0022Lock screen.\u0022\n", - " },\n", - " {\n", - " \u0022displayOrder\u0022: 2,\n", - " \u0022qnaId\u0022: 135,\n", - " \u0022displayText\u0022: \u0022Desktop taskbar.\u0022\n", - " },\n", - " {\n", - " \u0022displayOrder\u0022: 3,\n", - " \u0022qnaId\u0022: 136,\n", - " \u0022displayText\u0022: \u0022Make your battery last\u0022\n", - " }\n", - " ]\n", - " },\n", - " \u0022answerSpan\u0022: {\n", - " \u0022text\u0022: \u0022 two to four hours\u0022,\n", - " \u0022confidenceScore\u0022: 0.3795,\n", - " \u0022offset\u0022: 32,\n", - " \u0022length\u0022: 19\n", - " }\n", - " }\n", - " ]\n", - "}" - ] + "ResponseBody": { + "answers": [ + { + "questions": [ + "Power and charging" + ], + "answer": "**Power and charging**\n\nIt takes two to four hours to charge the Surface Pro 4 battery fully from an empty state. It can take longer if you\u2019re using your Surface for power-intensive activities like gaming or video streaming while you\u2019re charging it.\n\nYou can use the USB port on your Surface Pro 4 power supply to charge other devices, like a phone, while your Surface charges. The USB port on the power supply is only for charging, not for data transfer. If you want to use a USB device, plug it into the USB port on your Surface.", + "confidenceScore": 0.6517000000000001, + "id": 132, + "source": "surface-pro-4-user-guide-EN.pdf", + "metadata": { + "explicitlytaggedheading": "power and charging" + }, + "dialog": { + "isContextOnly": false, + "prompts": [ + { + "displayOrder": 0, + "qnaId": 133, + "displayText": "Check the battery level" + }, + { + "displayOrder": 1, + "qnaId": 134, + "displayText": "Lock screen." + }, + { + "displayOrder": 2, + "qnaId": 135, + "displayText": "Desktop taskbar." + }, + { + "displayOrder": 3, + "qnaId": 136, + "displayText": "Make your battery last" + } + ] + }, + "answerSpan": { + "text": " two to four hours", + "confidenceScore": 0.3795, + "offset": 32, + "length": 19 + } + } + ] + } } ], "Variables": { "QUESTIONANSWERING_ENDPOINT": "https://wuppe.api.cognitive.microsoft.com", "QUESTIONANSWERING_KEY": "Sanitized", "QUESTIONANSWERING_PROJECT": "190a9e13-8ede-4e4b-a8fd-c4d7f2aeab6c", - "RandomSeed": "1573798008" + "RandomSeed": "710190814" } -} \ No newline at end of file +} diff --git a/sdk/cognitivelanguage/Azure.AI.Language.QuestionAnswering/tests/SessionRecords/QuestionAnsweringClientLiveTests/AnswersFollowupKnowledgeBaseQuestionAsync.json b/sdk/cognitivelanguage/Azure.AI.Language.QuestionAnswering/tests/SessionRecords/QuestionAnsweringClientLiveTests/AnswersFollowupKnowledgeBaseQuestionAsync.json index 9278ee6957c4e..0d1d1a5878ba7 100644 --- a/sdk/cognitivelanguage/Azure.AI.Language.QuestionAnswering/tests/SessionRecords/QuestionAnsweringClientLiveTests/AnswersFollowupKnowledgeBaseQuestionAsync.json +++ b/sdk/cognitivelanguage/Azure.AI.Language.QuestionAnswering/tests/SessionRecords/QuestionAnsweringClientLiveTests/AnswersFollowupKnowledgeBaseQuestionAsync.json @@ -8,19 +8,16 @@ "Content-Length": "328", "Content-Type": "application/json", "Ocp-Apim-Subscription-Key": "Sanitized", - "traceparent": "00-dbe7479bb9b7be408e2feb90c2e1954f-06314449ce32b34a-00", - "User-Agent": [ - "azsdk-net-AI.Language.QuestionAnswering/1.0.0-alpha.20211028.1", - "(.NET 5.0.11; Microsoft Windows 10.0.22000)" - ], - "x-ms-client-request-id": "af1d04680c54575fede58c76d8a56576", + "traceparent": "00-a0b00891efe103e60f016dd1c69d369c-e1c4caa5516611e2-00", + "User-Agent": "azsdk-net-AI.Language.QuestionAnswering/1.1.0-alpha.20220114.1 (.NET 6.0.1; Microsoft Windows 10.0.22000)", + "x-ms-client-request-id": "a24bf8be71e022e371dcd7724f3cbe26", "x-ms-return-client-request-id": "true" }, "RequestBody": { "question": "How long it takes to charge Surface?", "top": 3, "userId": "sd53lsY=", - "confidenceScoreThreshold": 0.2, + "confidenceScoreThreshold": 0.5, "context": { "previousQnaId": 27, "previousUserQuery": "How long should my Surface battery last?" @@ -34,70 +31,68 @@ }, "StatusCode": 200, "ResponseHeaders": { - "apim-request-id": "aa17a06d-fd69-47c1-aab0-416162c2d959", - "Content-Length": "1594", + "apim-request-id": "8b6339bf-4428-4970-bb7a-441a60fd3596", + "Content-Length": "1606", "Content-Type": "application/json; charset=utf-8", "csp-billing-usage": "CognitiveServices.TextAnalytics.QuestionAnsweringTextRecords=1", - "Date": "Thu, 28 Oct 2021 21:15:40 GMT", + "Date": "Fri, 14 Jan 2022 23:16:50 GMT", "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", "X-Content-Type-Options": "nosniff", - "x-envoy-upstream-service-time": "900" + "x-envoy-upstream-service-time": "1272" }, - "ResponseBody": [ - "{\n", - " \u0022answers\u0022: [\n", - " {\n", - " \u0022questions\u0022: [\n", - " \u0022Power and charging\u0022\n", - " ],\n", - " \u0022answer\u0022: \u0022**Power and charging**\\n\\nIt takes two to four hours to charge the Surface Pro 4 battery fully from an empty state. It can take longer if you\u2019re using your Surface for power-intensive activities like gaming or video streaming while you\u2019re charging it.\\n\\nYou can use the USB port on your Surface Pro 4 power supply to charge other devices, like a phone, while your Surface charges. The USB port on the power supply is only for charging, not for data transfer. If you want to use a USB device, plug it into the USB port on your Surface.\u0022,\n", - " \u0022confidenceScore\u0022: 0.6015,\n", - " \u0022id\u0022: 132,\n", - " \u0022source\u0022: \u0022surface-pro-4-user-guide-EN.pdf\u0022,\n", - " \u0022metadata\u0022: {\n", - " \u0022explicitlytaggedheading\u0022: \u0022power and charging\u0022\n", - " },\n", - " \u0022dialog\u0022: {\n", - " \u0022isContextOnly\u0022: false,\n", - " \u0022prompts\u0022: [\n", - " {\n", - " \u0022displayOrder\u0022: 0,\n", - " \u0022qnaId\u0022: 133,\n", - " \u0022displayText\u0022: \u0022Check the battery level\u0022\n", - " },\n", - " {\n", - " \u0022displayOrder\u0022: 1,\n", - " \u0022qnaId\u0022: 134,\n", - " \u0022displayText\u0022: \u0022Lock screen.\u0022\n", - " },\n", - " {\n", - " \u0022displayOrder\u0022: 2,\n", - " \u0022qnaId\u0022: 135,\n", - " \u0022displayText\u0022: \u0022Desktop taskbar.\u0022\n", - " },\n", - " {\n", - " \u0022displayOrder\u0022: 3,\n", - " \u0022qnaId\u0022: 136,\n", - " \u0022displayText\u0022: \u0022Make your battery last\u0022\n", - " }\n", - " ]\n", - " },\n", - " \u0022answerSpan\u0022: {\n", - " \u0022text\u0022: \u0022 two to four hours\u0022,\n", - " \u0022confidenceScore\u0022: 0.3795,\n", - " \u0022offset\u0022: 32,\n", - " \u0022length\u0022: 19\n", - " }\n", - " }\n", - " ]\n", - "}" - ] + "ResponseBody": { + "answers": [ + { + "questions": [ + "Power and charging" + ], + "answer": "**Power and charging**\n\nIt takes two to four hours to charge the Surface Pro 4 battery fully from an empty state. It can take longer if you\u2019re using your Surface for power-intensive activities like gaming or video streaming while you\u2019re charging it.\n\nYou can use the USB port on your Surface Pro 4 power supply to charge other devices, like a phone, while your Surface charges. The USB port on the power supply is only for charging, not for data transfer. If you want to use a USB device, plug it into the USB port on your Surface.", + "confidenceScore": 0.6517000000000001, + "id": 132, + "source": "surface-pro-4-user-guide-EN.pdf", + "metadata": { + "explicitlytaggedheading": "power and charging" + }, + "dialog": { + "isContextOnly": false, + "prompts": [ + { + "displayOrder": 0, + "qnaId": 133, + "displayText": "Check the battery level" + }, + { + "displayOrder": 1, + "qnaId": 134, + "displayText": "Lock screen." + }, + { + "displayOrder": 2, + "qnaId": 135, + "displayText": "Desktop taskbar." + }, + { + "displayOrder": 3, + "qnaId": 136, + "displayText": "Make your battery last" + } + ] + }, + "answerSpan": { + "text": " two to four hours", + "confidenceScore": 0.3795, + "offset": 32, + "length": 19 + } + } + ] + } } ], "Variables": { "QUESTIONANSWERING_ENDPOINT": "https://wuppe.api.cognitive.microsoft.com", "QUESTIONANSWERING_KEY": "Sanitized", "QUESTIONANSWERING_PROJECT": "190a9e13-8ede-4e4b-a8fd-c4d7f2aeab6c", - "RandomSeed": "1992895952" + "RandomSeed": "18049392" } -} \ No newline at end of file +} diff --git a/sdk/cognitivelanguage/test-resources.json b/sdk/cognitivelanguage/test-resources.json index 2fbff559d7ea1..693defe568bae 100644 --- a/sdk/cognitivelanguage/test-resources.json +++ b/sdk/cognitivelanguage/test-resources.json @@ -20,29 +20,33 @@ "variables": {}, "resources": [], "outputs": { - "CONVERSATIONS_KEY": { - "type": "string", - "value": "" - }, - "CONVERSATIONS_PROJECT": { - "type": "string", - "value": "" - }, - "CONVERSATIONS_URI": { - "type": "string", - "value": "" - }, - "QUESTIONANSWERING_ENDPOINT": { - "type": "string", - "value": "https://wuppe.api.cognitive.microsoft.com" - }, - "QUESTIONANSWERING_KEY": { - "type": "string", - "value": "" - }, - "QUESTIONANSWERING_PROJECT": { - "type": "string", - "value": "190a9e13-8ede-4e4b-a8fd-c4d7f2aeab6c" - } + // TODO: Uncomment and define when test resources are deployed dynamically. + // Developers should define the following environment variables to run tests locally. + // Live test CIs will use environment variables defined via tests.yml. + // + // "CONVERSATIONS_KEY": { + // "type": "string", + // "value": "" + // }, + // "CONVERSATIONS_PROJECT": { + // "type": "string", + // "value": "antischOne" + // }, + // "CONVERSATIONS_URI": { + // "type": "string", + // "value": "" + // }, + // "QUESTIONANSWERING_ENDPOINT": { + // "type": "string", + // "value": "https://wuppe.api.cognitive.microsoft.com" + // }, + // "QUESTIONANSWERING_KEY": { + // "type": "string", + // "value": "" + // }, + // "QUESTIONANSWERING_PROJECT": { + // "type": "string", + // "value": "190a9e13-8ede-4e4b-a8fd-c4d7f2aeab6c" + // } } } diff --git a/sdk/cognitivelanguage/tests.yml b/sdk/cognitivelanguage/tests.yml index fcb510cc9b435..d56f87c43efe0 100644 --- a/sdk/cognitivelanguage/tests.yml +++ b/sdk/cognitivelanguage/tests.yml @@ -4,7 +4,10 @@ extends: template: ../../eng/pipelines/templates/stages/archetype-sdk-tests.yml parameters: EnvVars: - QNAMAKER_API_KEY: $(qna-key) - QNAMAKER_PROJECT: 190a9e13-8ede-4e4b-a8fd-c4d7f2aeab6c - QNAMAKER_URI: $(qna-uri) + CONVERSATIONS_KEY: $(clu-key) + CONVERSATIONS_PROJECT: antischOne + CONVERSATIONS_URI: $(clu-uri) + QUESTIONANSWERING_KEY: $(qna-key) + QUESTIONANSWERING_PROJECT: 190a9e13-8ede-4e4b-a8fd-c4d7f2aeab6c + QUESTIONANSWERING_ENDPOINT: $(qna-uri) ServiceDirectory: cognitivelanguage