Skip to content

Commit

Permalink
[FormRecognizer] Adding remaining tests for cross-language test parity (
Browse files Browse the repository at this point in the history 
#13006)

* Implemented multipage StartTraining test

* Updated resources json file to support multipage forms

* Added tests for damaged files

* Added tests for valid prefix

* Added tests for bad endpoint

* Added tests for polling from new operation objects

* Added tests for multipage scenario
  • Loading branch information
@kinelski
kinelski authored Jun 26, 2020
1 parent 59625b4 commit eaaff6d
Show file tree
Hide file tree
Showing 53 changed files with 10,066 additions and 55,953 deletions.
Binary file added sdk/formrecognizer/Azure.AI.FormRecognizer/tests/Assets/multi1.pdf
View file
Binary file not shown.
Binary file removed sdk/formrecognizer/Azure.AI.FormRecognizer/tests/Assets/multipage_invoice_noblank.pdf
View file
Binary file not shown.
180 changes: 104 additions & 76 deletions ...nizer/Azure.AI.FormRecognizer/tests/FormRecognizerClient/FormRecognizerClientLiveTests.cs
View file

Large diffs are not rendered by default.

20 changes: 19 additions & 1 deletion ...ecognizer/Azure.AI.FormRecognizer/tests/FormRecognizerClient/FormRecognizerClientTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

using System;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Azure.AI.FormRecognizer.Models;
Expand Down Expand Up @@ -32,7 +33,7 @@ public FormRecognizerClientTests(bool isAsync) : base(isAsync)
/// <returns>The fake <see cref="FormRecognizerClient" />.</returns>
private FormRecognizerClient CreateClient()
{
var fakeEndpoint = new Uri("http://localhost");
var fakeEndpoint = new Uri("http://notreal.azure.com");
var fakeCredential = new AzureKeyCredential("fakeKey");

return new FormRecognizerClient(fakeEndpoint, fakeCredential);
Expand Down Expand Up @@ -98,6 +99,23 @@ public void ConstructorRequiresTheOptions()
Assert.Throws<ArgumentNullException>(() => new FormRecognizerClient(endpoint, keyCredential, null));
}

[Test]
public async Task FormRecognizerClientThrowsWithNonExistingResourceEndpoint()
{
var client = CreateInstrumentedClient();

try
{
var uri = FormRecognizerTestEnvironment.CreateUri(TestFile.Form1);
await client.StartRecognizeContentFromUriAsync(uri);
}
catch (AggregateException ex)
{
var innerExceptions = ex.InnerExceptions.ToList();
Assert.IsTrue(innerExceptions.All(ex => ex is RequestFailedException));
}
}

/// <summary>
/// Verifies functionality of the <see cref="FormRecognizerClient.StartRecognizeContentAsync"/>
/// method.
Expand Down
59 changes: 37 additions & 22 deletions ...ecognizer/Azure.AI.FormRecognizer/tests/FormTrainingClient/FormTrainingClientLiveTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,14 +30,14 @@ public FormTrainingClientLiveTests(bool isAsync) : base(isAsync)
[Test]
public void FormTrainingClientCannotAuthenticateWithFakeApiKey()
{
var client = CreateInstrumentedFormTrainingClient(apiKey: "fakeKey");
var client = CreateFormTrainingClient(apiKey: "fakeKey");
Assert.ThrowsAsync<RequestFailedException>(async () => await client.GetAccountPropertiesAsync());
}

[Test]
public async Task FormTrainingClientCanAuthenticateWithTokenCredential()
{
var client = CreateInstrumentedFormTrainingClient(useTokenCredential: true);
var client = CreateFormTrainingClient(useTokenCredential: true);
var trainingFilesUri = new Uri(TestEnvironment.BlobContainerSasUrl);

TrainingOperation operation = await client.StartTrainingAsync(trainingFilesUri, useTrainingLabels: false);
Expand All @@ -52,12 +52,14 @@ public async Task FormTrainingClientCanAuthenticateWithTokenCredential()
}

[Test]
[TestCase(true)]
[TestCase(false)]
public async Task StartTraining(bool labeled)
[TestCase(true, true)]
[TestCase(true, false)]
[TestCase(false, true)]
[TestCase(false, false)]
public async Task StartTraining(bool singlePage, bool labeled)
{
var client = CreateInstrumentedFormTrainingClient();
var trainingFilesUri = new Uri(TestEnvironment.BlobContainerSasUrl);
var client = CreateFormTrainingClient();
var trainingFilesUri = new Uri(singlePage ? TestEnvironment.BlobContainerSasUrl : TestEnvironment.MultipageBlobContainerSasUrl);

TrainingOperation operation = await client.StartTrainingAsync(trainingFilesUri, labeled);
await operation.WaitForCompletionAsync(PollingInterval);
Expand Down Expand Up @@ -96,24 +98,37 @@ public async Task StartTraining(bool labeled)
}
}

[Test]
public async Task StartTrainingSucceedsWithValidPrefix()
{
var client = CreateFormTrainingClient();
var trainingFilesUri = new Uri(TestEnvironment.BlobContainerSasUrl);

var filter = new TrainingFileFilter { IncludeSubFolders = true, Prefix = "subfolder" };
TrainingOperation operation = await client.StartTrainingAsync(trainingFilesUri, useTrainingLabels: false, filter);

await operation.WaitForCompletionAsync(PollingInterval);

Assert.IsTrue(operation.HasValue);
Assert.AreEqual(CustomFormModelStatus.Ready, operation.Value.Status);
}

[Test]
public async Task StartTrainingFailsWithInvalidPrefix()
{
var client = CreateInstrumentedFormTrainingClient();
var client = CreateFormTrainingClient();
var trainingFilesUri = new Uri(TestEnvironment.BlobContainerSasUrl);
TrainingOperation operation;

var filter = new TrainingFileFilter { IncludeSubFolders = true, Prefix = "invalidPrefix" };

operation = await client.StartTrainingAsync(trainingFilesUri, useTrainingLabels: false, filter);
TrainingOperation operation = await client.StartTrainingAsync(trainingFilesUri, useTrainingLabels: false, filter);

Assert.ThrowsAsync<RequestFailedException>(async () => await operation.WaitForCompletionAsync(PollingInterval));
}

[Test]
public async Task StartTrainingError()
{
var client = CreateInstrumentedFormTrainingClient();
var client = CreateFormTrainingClient();

var containerUrl = new Uri("https://someUrl");

Expand All @@ -129,7 +144,7 @@ public async Task StartTrainingError()
[TestCase(false)]
public async Task TrainingOps(bool labeled)
{
var client = CreateInstrumentedFormTrainingClient();
var client = CreateFormTrainingClient();
var trainingFilesUri = new Uri(TestEnvironment.BlobContainerSasUrl);

TrainingOperation operation = await client.StartTrainingAsync(trainingFilesUri, labeled);
Expand Down Expand Up @@ -195,7 +210,7 @@ public async Task TrainingOps(bool labeled)
[Test]
public void DeleteModelFailsWhenModelDoesNotExist()
{
var client = CreateInstrumentedFormTrainingClient();
var client = CreateFormTrainingClient();
var fakeModelId = "00000000-0000-0000-0000-000000000000";

Assert.ThrowsAsync<RequestFailedException>(async () => await client.DeleteModelAsync(fakeModelId));
Expand All @@ -204,8 +219,8 @@ public void DeleteModelFailsWhenModelDoesNotExist()
[Test]
public async Task CopyModel()
{
var sourceClient = CreateInstrumentedFormTrainingClient();
var targetClient = CreateInstrumentedFormTrainingClient();
var sourceClient = CreateFormTrainingClient();
var targetClient = CreateFormTrainingClient();
var resourceID = TestEnvironment.TargetResourceId;
var region = TestEnvironment.TargetResourceRegion;

Expand All @@ -230,8 +245,8 @@ public async Task CopyModel()
[Ignore("Issue: https://github.com/Azure/azure-sdk-for-net/issues/12319")]
public void CopyModelError()
{
var sourceClient = CreateInstrumentedFormTrainingClient();
var targetClient = CreateInstrumentedFormTrainingClient();
var sourceClient = CreateFormTrainingClient();
var targetClient = CreateFormTrainingClient();
var resourceID = TestEnvironment.TargetResourceId;
var region = TestEnvironment.TargetResourceRegion;

Expand All @@ -243,8 +258,8 @@ public void CopyModelError()
[Test]
public async Task StartCopyModelFailsWithWrongRegion()
{
var sourceClient = CreateInstrumentedFormTrainingClient();
var targetClient = CreateInstrumentedFormTrainingClient();
var sourceClient = CreateFormTrainingClient();
var targetClient = CreateFormTrainingClient();
var resourceID = TestEnvironment.TargetResourceId;
var wrongRegion = TestEnvironment.TargetResourceRegion == "westcentralus" ? "eastus2" : "westcentralus";

Expand All @@ -259,7 +274,7 @@ public async Task StartCopyModelFailsWithWrongRegion()
[Test]
public async Task GetCopyAuthorization()
{
var targetClient = CreateInstrumentedFormTrainingClient();
var targetClient = CreateFormTrainingClient();
var resourceID = TestEnvironment.TargetResourceId;
var region = TestEnvironment.TargetResourceRegion;

Expand All @@ -275,7 +290,7 @@ public async Task GetCopyAuthorization()
[Test]
public async Task SerializeDeserializeCopyAuthorizationAsync()
{
var targetClient = CreateInstrumentedFormTrainingClient();
var targetClient = CreateFormTrainingClient();
var resourceID = TestEnvironment.TargetResourceId;
var region = TestEnvironment.TargetResourceRegion;

Expand Down
20 changes: 19 additions & 1 deletion ...ormrecognizer/Azure.AI.FormRecognizer/tests/FormTrainingClient/FormTrainingClientTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
// Licensed under the MIT License.

using System;
using System.Linq;
using System.Threading.Tasks;
using Azure.AI.FormRecognizer.Training;
using Azure.Core;
using Azure.Core.TestFramework;
Expand Down Expand Up @@ -30,7 +32,7 @@ public FormTrainingClientTests(bool isAsync) : base(isAsync)
/// <returns>The instrumented <see cref="FormTrainingClient" />.</returns>
private FormTrainingClient CreateInstrumentedClient()
{
var fakeEndpoint = new Uri("http://localhost");
var fakeEndpoint = new Uri("http://notreal.azure.com/");
var fakeCredential = new AzureKeyCredential("fakeKey");
var client = new FormTrainingClient(fakeEndpoint, fakeCredential);

Expand Down Expand Up @@ -90,6 +92,22 @@ public void ConstructorRequiresTheOptions()
Assert.Throws<ArgumentNullException>(() => new FormRecognizerClient(endpoint, keyCredential, null));
}

[Test]
public async Task FormTrainingClientThrowsWithNonExistingResourceEndpoint()
{
var client = CreateInstrumentedClient();

try
{
await client.GetAccountPropertiesAsync();
}
catch (AggregateException ex)
{
var innerExceptions = ex.InnerExceptions.ToList();
Assert.IsTrue(innerExceptions.All(ex => ex is RequestFailedException));
}
}

[Test]
public void StartTrainingArgumentValidation()
{
Expand Down
39 changes: 34 additions & 5 deletions ...formrecognizer/Azure.AI.FormRecognizer/tests/Infrastructure/FormRecognizerLiveTestBase.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,14 +18,42 @@ public FormRecognizerLiveTestBase(bool isAsync) : base(isAsync)
Sanitizer = new FormRecognizerRecordedTestSanitizer();
}

/// <summary>
/// Creates a <see cref="FormRecognizerClient" /> with the endpoint and API key provided via environment
/// variables and instruments it to make use of the Azure Core Test Framework functionalities.
/// </summary>
/// <param name="useTokenCredential">Whether or not to use a <see cref="TokenCredential"/> to authenticate. An <see cref="AzureKeyCredential"/> is used by default.</param>
/// <param name="apiKey">The API key to use for authentication. Defaults to <see cref="FormRecognizerTestEnvironment.ApiKey"/>.</param>
/// <param name="skipInstrumenting">Whether or not instrumenting should be skipped. Avoid skipping it as much as possible.</param>
/// <returns>The instrumented <see cref="FormRecognizerClient" />.</returns>
protected FormRecognizerClient CreateFormRecognizerClient(bool useTokenCredential = false, string apiKey = default, bool skipInstrumenting = false)
{
var endpoint = new Uri(TestEnvironment.Endpoint);
var options = Recording.InstrumentClientOptions(new FormRecognizerClientOptions());
FormRecognizerClient client;

if (useTokenCredential)
{
client = new FormRecognizerClient(endpoint, TestEnvironment.Credential, options);
}
else
{
var credential = new AzureKeyCredential(apiKey ?? TestEnvironment.ApiKey);
client = new FormRecognizerClient(endpoint, credential, options);
}

return skipInstrumenting ? client : InstrumentClient(client);
}

/// <summary>
/// Creates a <see cref="FormTrainingClient" /> with the endpoint and API key provided via environment
/// variables and instruments it to make use of the Azure Core Test Framework functionalities.
/// </summary>
/// <param name="useTokenCredential">Whether or not to use a <see cref="TokenCredential"/> to authenticate. An <see cref="AzureKeyCredential"/> is used by default.</param>
/// <param name="apiKey">The API key to use for authentication. Defaults to <see cref="FormRecognizerTestEnvironment.ApiKey"/>.</param>
/// <param name="skipInstrumenting">Whether or not instrumenting should be skipped. Avoid skipping it as much as possible.</param>
/// <returns>The instrumented <see cref="FormTrainingClient" />.</returns>
protected FormTrainingClient CreateInstrumentedFormTrainingClient(bool useTokenCredential = false, string apiKey = default)
protected FormTrainingClient CreateFormTrainingClient(bool useTokenCredential = false, string apiKey = default, bool skipInstrumenting = false)
{
var endpoint = new Uri(TestEnvironment.Endpoint);
var options = Recording.InstrumentClientOptions(new FormRecognizerClientOptions());
Expand All @@ -41,19 +69,20 @@ protected FormTrainingClient CreateInstrumentedFormTrainingClient(bool useTokenC
client = new FormTrainingClient(endpoint, credential, options);
}

return InstrumentClient(client);
return skipInstrumenting ? client : InstrumentClient(client);
}

/// <summary>
/// Trains a model and returns the associated <see cref="DisposableTrainedModel"/> instance, from which
/// the model ID can be obtained. Upon disposal, the model will be deleted.
/// </summary>
/// <param name="useTrainingLabels">If <c>true</c>, use a label file created in the &lt;link-to-label-tool-doc&gt; to provide training-time labels for training a model. If <c>false</c>, the model will be trained from forms only.</param>
/// <param name="useMultipageFiles">Whether or not to use multipage files for training.</param>
/// <returns>A <see cref="DisposableTrainedModel"/> instance from which the trained model ID can be obtained.</returns>
protected async Task<DisposableTrainedModel> CreateDisposableTrainedModelAsync(bool useTrainingLabels)
protected async Task<DisposableTrainedModel> CreateDisposableTrainedModelAsync(bool useTrainingLabels, bool useMultipageFiles = false)
{
var trainingClient = CreateInstrumentedFormTrainingClient();
var trainingFilesUri = new Uri(TestEnvironment.BlobContainerSasUrl);
var trainingClient = CreateFormTrainingClient();
var trainingFilesUri = new Uri(useMultipageFiles ? TestEnvironment.MultipageBlobContainerSasUrl : TestEnvironment.BlobContainerSasUrl);

return await DisposableTrainedModel.TrainModelAsync(trainingClient, trainingFilesUri, useTrainingLabels, PollingInterval);
}
Expand Down
10 changes: 7 additions & 3 deletions ...mrecognizer/Azure.AI.FormRecognizer/tests/Infrastructure/FormRecognizerTestEnvironment.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,13 +23,16 @@ public FormRecognizerTestEnvironment() : base("formrecognizer")
/// <summary>The name of the environment variable from which the Form Recognizer resource's API key will be extracted for the live tests.</summary>
internal const string ApiKeyEnvironmentVariableName = "FORM_RECOGNIZER_API_KEY";

/// <summary>The name of the environment variable for the Blob Container SAS URL use for storing documents used for live tests.</summary>
/// <summary>The name of the environment variable for the Blob Container SAS URL to use for storing documents used for live tests.</summary>
internal const string BlobContainerSasUrlEnvironmentVariableName = "FORM_RECOGNIZER_BLOB_CONTAINER_SAS_URL";

/// <summary>The name of the environment variable for the target resource identifier use for copying custom models live tests.</summary>
/// <summary>The name of the environment variable for the multipage Blob Container SAS URL to use for storing documents used for live tests.</summary>
internal const string MultipageBlobContainerSasUrlEnvironmentVariableName = "FORM_RECOGNIZER_MULTIPAGE_BLOB_CONTAINER_SAS_URL";

/// <summary>The name of the environment variable for the target resource identifier to use for copying custom models live tests.</summary>
internal const string TargetResourceIdEnvironmentVariableName = "FORM_RECOGNIZER_TARGET_RESOURCE_ID";

/// <summary>The name of the environment variable for the target resource region use for copying custom models live tests.</summary>
/// <summary>The name of the environment variable for the target resource region to use for copying custom models live tests.</summary>
internal const string TargetResourceRegionEnvironmentVariableName = "FORM_RECOGNIZER_TARGET_RESOURCE_REGION";

/// <summary>The name of the folder in which test assets are stored.</summary>
Expand All @@ -41,6 +44,7 @@ public FormRecognizerTestEnvironment() : base("formrecognizer")
public string ApiKey => GetRecordedVariable(ApiKeyEnvironmentVariableName);
public string Endpoint => GetRecordedVariable(EndpointEnvironmentVariableName);
public string BlobContainerSasUrl => GetRecordedVariable(BlobContainerSasUrlEnvironmentVariableName);
public string MultipageBlobContainerSasUrl => GetRecordedVariable(MultipageBlobContainerSasUrlEnvironmentVariableName);
public string TargetResourceId => GetRecordedVariable(TargetResourceIdEnvironmentVariableName);
public string TargetResourceRegion => GetRecordedVariable(TargetResourceRegionEnvironmentVariableName);

Expand Down
2 changes: 1 addition & 1 deletion sdk/formrecognizer/Azure.AI.FormRecognizer/tests/Infrastructure/TestFile.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ public static class TestFile
public const string InvoiceLeTiff = "Invoice_1.tiff";

/// <summary>A two-page invoice file.</summary>
public const string InvoiceMultipage = "multipage_invoice_noblank.pdf";
public const string InvoiceMultipage = "multi1.pdf";

/// <summary>A three-page invoice file in which the second page is blank.</summary>
public const string InvoiceMultipageBlank = "multipage_invoice1.pdf";
Expand Down
Loading

0 comments on commit eaaff6d

Please sign in to comment.