Skip to content

Commit

Permalink
Form Recognizer: Implement Recognize for Custom Forms (Azure#11318)
Browse files Browse the repository at this point in the history 
* first steps and test fix

* convert unsupervised analyze result to recognized form

* handle different element reference formats

* pr feedback

* regex updates
  • Loading branch information
@annelo-msft
annelo-msft authored Apr 15, 2020
1 parent b2c31ed commit 401a091
Show file tree
Hide file tree
Showing 8 changed files with 134 additions and 82 deletions.
6 changes: 6 additions & 0 deletions sdk/formrecognizer/Azure.AI.FormRecognizer/src/FieldValue_internal.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,5 +8,11 @@ namespace Azure.AI.FormRecognizer.Models
[CodeGenModel("FieldValue")]
internal partial class FieldValue_internal
{
internal FieldValue_internal(string value)
{
Type = FieldValueType.StringType;
ValueString = value;
Text = value;
}
}
}
74 changes: 40 additions & 34 deletions sdk/formrecognizer/Azure.AI.FormRecognizer/src/FormField.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,35 +4,36 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Text.RegularExpressions;

namespace Azure.AI.FormRecognizer.Models
{
/// <summary>
/// </summary>
public class FormField
{
#pragma warning disable CA1801
internal FormField(KeyValuePair_internal field, ReadResult_internal readResult)
internal FormField(string name, int pageNumber, KeyValuePair_internal field, IReadOnlyList<ReadResult_internal> readResults)
{
#pragma warning restore CA1801
//Confidence = field.Confidence;

//Name = field.Key.Text;
//NameBoundingBox = new BoundingBox(field.Key.BoundingBox);
Confidence = field.Confidence;
Name = name;

//if (field.Key.Elements != null)
//{
// NameTextElements = ConvertTextReferences(readResult, field.Key.Elements);
//}
BoundingBox labelBoundingBox = field.Key.BoundingBox == null ? default : new BoundingBox(field.Key.BoundingBox);
IReadOnlyList<FormContent> labelFormContent = default;
if (field.Key.Elements != null)
{
labelFormContent = ConvertTextReferences(field.Key.Elements, readResults);
}
LabelText = new FieldText(field.Key.Text, pageNumber, labelBoundingBox, labelFormContent);

//Value = field.Value.Text;
//ValueBoundingBox = new BoundingBox(field.Value.BoundingBox);
BoundingBox valueBoundingBox = field.Value.BoundingBox == null ? default : new BoundingBox(field.Value.BoundingBox);
IReadOnlyList<FormContent> valueFormContent = default;
if (field.Value.Elements != null)
{
valueFormContent = ConvertTextReferences(field.Value.Elements, readResults);
}
ValueText = new FieldText(field.Value.Text, pageNumber, valueBoundingBox, valueFormContent);

//if (field.Value.Elements != null)
//{
// ValueTextElements = ConvertTextReferences(readResult, field.Value.Elements);
//}
Value = new FieldValue(new FieldValue_internal(field.Value.Text), readResults);
}

internal FormField(string name, FieldValue_internal fieldValue, IReadOnlyList<ReadResult_internal> readResults)
Expand Down Expand Up @@ -86,35 +87,40 @@ internal static IReadOnlyList<FormContent> ConvertTextReferences(IReadOnlyList<s
return formContent;
}

private static Regex _wordRegex = new Regex(@"/readResults/(?<pageIndex>\d*)/lines/(?<lineIndex>\d*)/words/(?<wordIndex>\d*)$", RegexOptions.Compiled, TimeSpan.FromSeconds(2));
private static Regex _lineRegex = new Regex(@"/readResults/(?<pageIndex>\d*)/lines/(?<lineIndex>\d*)$", RegexOptions.Compiled, TimeSpan.FromSeconds(2));

private static FormContent ResolveTextReference(IReadOnlyList<ReadResult_internal> readResults, string reference)
{
// TODO: Add additional validations here.
// https://github.com/Azure/azure-sdk-for-net/issues/10363

// Example: the following should result in PageIndex = 3, LineIndex = 7, WordIndex = 12
// "#/readResults/3/lines/7/words/12"
string[] segments = reference.Split('/');
// "#/analyzeResult/readResults/3/lines/7/words/12" from DocumentResult
// "#/readResults/3/lines/7/words/12" from PageResult

// Line Reference
if (segments.Length == 5)
// Word Reference
var wordMatch = _wordRegex.Match(reference);
if (wordMatch.Success && wordMatch.Groups.Count == 4)
{
var pageIndex = int.Parse(segments[2], CultureInfo.InvariantCulture);
var lineIndex = int.Parse(segments[4], CultureInfo.InvariantCulture);

return new FormLine(readResults[pageIndex].Lines[lineIndex], pageIndex + 1);
}
else if (segments.Length == 7)
{
var pageIndex = int.Parse(segments[2], CultureInfo.InvariantCulture);
var lineIndex = int.Parse(segments[4], CultureInfo.InvariantCulture);
var wordIndex = int.Parse(segments[6], CultureInfo.InvariantCulture);
int pageIndex = int.Parse(wordMatch.Groups["pageIndex"].Value, CultureInfo.InvariantCulture);
int lineIndex = int.Parse(wordMatch.Groups["lineIndex"].Value, CultureInfo.InvariantCulture);
int wordIndex = int.Parse(wordMatch.Groups["wordIndex"].Value, CultureInfo.InvariantCulture);

return new FormWord(readResults[pageIndex].Lines[lineIndex].Words[wordIndex], pageIndex + 1);
}
else

// Line Reference
var lineMatch = _lineRegex.Match(reference);
if (lineMatch.Success && lineMatch.Groups.Count == 3)
{
throw new InvalidOperationException($"Failed to parse element reference: {reference}");
int pageIndex = int.Parse(lineMatch.Groups["pageIndex"].Value, CultureInfo.InvariantCulture);
int lineIndex = int.Parse(lineMatch.Groups["lineIndex"].Value, CultureInfo.InvariantCulture);

return new FormLine(readResults[pageIndex].Lines[lineIndex], pageIndex + 1);
}

throw new InvalidOperationException($"Failed to parse element reference: {reference}");
}
}
}
8 changes: 3 additions & 5 deletions sdk/formrecognizer/Azure.AI.FormRecognizer/src/FormPageRange.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,10 @@ namespace Azure.AI.FormRecognizer.Models
/// </summary>
public class FormPageRange
{
internal FormPageRange(IReadOnlyList<int> pageRange)
internal FormPageRange(int first, int last)
{
// TODO: validate that PageRange.Length == 2.
// https://github.com/Azure/azure-sdk-for-net/issues/10547
FirstPageNumber = pageRange[0];
LastPageNumber = pageRange[1];
FirstPageNumber = first;
LastPageNumber = last;
}

/// <summary>
Expand Down
53 changes: 27 additions & 26 deletions sdk/formrecognizer/Azure.AI.FormRecognizer/src/FormRecognizerClient.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ public virtual async Task<RecognizeContentOperation> StartRecognizeContentAsync(
[ForwardsClientCalls]
public virtual RecognizeContentOperation StartRecognizeContentFromUri(Uri formFileUri, RecognizeOptions recognizeOptions = default, CancellationToken cancellationToken = default)
{
SourcePath_internal sourcePath = new SourcePath_internal() { Source = formFileUri.ToString() };
SourcePath_internal sourcePath = new SourcePath_internal(formFileUri.ToString());
ResponseWithHeaders<ServiceAnalyzeLayoutAsyncHeaders> response = ServiceClient.RestClient.AnalyzeLayoutAsync(sourcePath, cancellationToken);
//Response response = ServiceClient.RestClient.AnalyzeLayoutAsync(sourcePath, cancellationToken);

Expand All @@ -125,7 +125,7 @@ public virtual RecognizeContentOperation StartRecognizeContentFromUri(Uri formFi
[ForwardsClientCalls]
public virtual async Task<RecognizeContentOperation> StartRecognizeContentFromUriAsync(Uri formFileUri, RecognizeOptions recognizeOptions = default, CancellationToken cancellationToken = default)
{
SourcePath_internal sourcePath = new SourcePath_internal() { Source = formFileUri.ToString() };
SourcePath_internal sourcePath = new SourcePath_internal(formFileUri.ToString());
ResponseWithHeaders<ServiceAnalyzeLayoutAsyncHeaders> response = await ServiceClient.RestClient.AnalyzeLayoutAsyncAsync(sourcePath, cancellationToken).ConfigureAwait(false);
//Response response = await ServiceClient.RestClient.AnalyzeLayoutAsyncAsync(sourcePath, cancellationToken).ConfigureAwait(false);

Expand Down Expand Up @@ -197,7 +197,7 @@ public virtual async Task<RecognizeReceiptsOperation> StartRecognizeReceiptsFrom
{
recognizeOptions ??= new RecognizeOptions();

SourcePath_internal sourcePath = new SourcePath_internal() { Source = receiptFileUri.ToString() };
SourcePath_internal sourcePath = new SourcePath_internal(receiptFileUri.ToString());
ResponseWithHeaders<ServiceAnalyzeReceiptAsyncHeaders> response = await ServiceClient.RestClient.AnalyzeReceiptAsyncAsync(includeTextDetails: recognizeOptions.IncludeTextContent, sourcePath, cancellationToken).ConfigureAwait(false);
return new RecognizeReceiptsOperation(ServiceClient, response.Headers.OperationLocation);
}
Expand All @@ -216,7 +216,7 @@ public virtual RecognizeReceiptsOperation StartRecognizeReceiptsFromUri(Uri rece
{
recognizeOptions ??= new RecognizeOptions();

SourcePath_internal sourcePath = new SourcePath_internal() { Source = receiptFileUri.ToString() };
SourcePath_internal sourcePath = new SourcePath_internal(receiptFileUri.ToString());
ResponseWithHeaders<ServiceAnalyzeReceiptAsyncHeaders> response = ServiceClient.RestClient.AnalyzeReceiptAsync(includeTextDetails: recognizeOptions.IncludeTextContent, sourcePath, cancellationToken);
return new RecognizeReceiptsOperation(ServiceClient, response.Headers.OperationLocation);
}
Expand All @@ -235,14 +235,15 @@ public virtual RecognizeReceiptsOperation StartRecognizeReceiptsFromUri(Uri rece
/// <returns>A <see cref="RecognizeCustomFormsOperation"/> to wait on this long-running operation. Its <see cref="RecognizeCustomFormsOperation"/>.Value upon successful
/// completion will contain extracted pages from the input document.</returns>
[ForwardsClientCalls]
public virtual RecognizeCustomFormsOperation StartRecognizeCustomForms(string modelId, Stream formFileStream, /* ContentType contentType, */ RecognizeOptions recognizeOptions = default, CancellationToken cancellationToken = default)
public virtual RecognizeCustomFormsOperation StartRecognizeCustomForms(string modelId, Stream formFileStream, RecognizeOptions recognizeOptions = default, CancellationToken cancellationToken = default)
{
throw new NotImplementedException();
recognizeOptions ??= new RecognizeOptions();

// TODO: automate content-type detection
// https://github.com/Azure/azure-sdk-for-net/issues/10329

//// TODO: automate content-type detection
//// https://github.com/Azure/azure-sdk-for-net/issues/10329
//ResponseWithHeaders<AnalyzeWithCustomModelHeaders> response = ServiceClient.AnalyzeWithCustomModel(new Guid(modelId), includeTextDetails: includeTextElements, formFileStream, contentType, cancellationToken);
//return new RecognizeFormsOperation(ServiceClient, modelId, response.Headers.OperationLocation);
ResponseWithHeaders<ServiceAnalyzeWithCustomModelHeaders> response = ServiceClient.RestClient.AnalyzeWithCustomModel(new Guid(modelId), ContentType.Pdf, formFileStream, includeTextDetails: recognizeOptions.IncludeTextContent, cancellationToken);
return new RecognizeCustomFormsOperation(ServiceClient, modelId, response.Headers.OperationLocation);
}

/// <summary>
Expand All @@ -257,10 +258,11 @@ public virtual RecognizeCustomFormsOperation StartRecognizeCustomForms(string mo
[ForwardsClientCalls]
public virtual RecognizeCustomFormsOperation StartRecognizeCustomFormsFromUri(string modelId, Uri formFileUri, RecognizeOptions recognizeOptions = default, CancellationToken cancellationToken = default)
{
throw new NotImplementedException();
//SourcePath_internal sourcePath = new SourcePath_internal() { Source = formFileUri.ToString() };
//ResponseWithHeaders<AnalyzeWithCustomModelHeaders> response = ServiceClient.RestClient.AnalyzeWithCustomModel(new Guid(modelId), includeTextDetails: includeTextElements, sourcePath, cancellationToken);
//return new RecognizeCustomFormsOperation(ServiceClient, modelId, response.Headers.OperationLocation);
recognizeOptions ??= new RecognizeOptions();

SourcePath_internal sourcePath = new SourcePath_internal(formFileUri.ToString());
ResponseWithHeaders<ServiceAnalyzeWithCustomModelHeaders> response = ServiceClient.RestClient.AnalyzeWithCustomModel(new Guid(modelId), includeTextDetails: recognizeOptions.IncludeTextContent, sourcePath, cancellationToken);
return new RecognizeCustomFormsOperation(ServiceClient, modelId, response.Headers.OperationLocation);
}

/// <summary>
Expand All @@ -273,15 +275,15 @@ public virtual RecognizeCustomFormsOperation StartRecognizeCustomFormsFromUri(st
/// <returns>A <see cref="RecognizeCustomFormsOperation"/> to wait on this long-running operation. Its <see cref="RecognizeCustomFormsOperation"/>.Value upon successful
/// completion will contain extracted pages from the input document.</returns>
[ForwardsClientCalls]
public virtual async Task<RecognizeCustomFormsOperation> StartRecognizeCustomFormsAsync(string modelId, Stream formFileStream, /* ContentType contentType, */ RecognizeOptions recognizeOptions = default, CancellationToken cancellationToken = default)
public virtual async Task<RecognizeCustomFormsOperation> StartRecognizeCustomFormsAsync(string modelId, Stream formFileStream, RecognizeOptions recognizeOptions = default, CancellationToken cancellationToken = default)
{
await Task.Run(() => { }).ConfigureAwait(false);
throw new NotImplementedException();
recognizeOptions ??= new RecognizeOptions();

//// TODO: automate content-type detection
//// https://github.com/Azure/azure-sdk-for-net/issues/10329
//ResponseWithHeaders<AnalyzeWithCustomModelHeaders> response = await ServiceClient.AnalyzeWithCustomModelAsync(new Guid(modelId), includeTextDetails: includeTextElements, formFileStream, contentType, cancellationToken).ConfigureAwait(false);
//return new RecognizeFormsOperation(ServiceClient, modelId, response.Headers.OperationLocation);
// TODO: automate content-type detection
// https://github.com/Azure/azure-sdk-for-net/issues/10329

ResponseWithHeaders<ServiceAnalyzeWithCustomModelHeaders> response = await ServiceClient.RestClient.AnalyzeWithCustomModelAsync(new Guid(modelId), ContentType.Pdf, formFileStream, includeTextDetails: recognizeOptions.IncludeTextContent, cancellationToken).ConfigureAwait(false);
return new RecognizeCustomFormsOperation(ServiceClient, modelId, response.Headers.OperationLocation);
}

/// <summary>
Expand All @@ -296,12 +298,11 @@ public virtual async Task<RecognizeCustomFormsOperation> StartRecognizeCustomFor
[ForwardsClientCalls]
public virtual async Task<RecognizeCustomFormsOperation> StartRecognizeCustomFormsFromUriAsync(string modelId, Uri formFileUri, RecognizeOptions recognizeOptions = default, CancellationToken cancellationToken = default)
{
await Task.Run(() => { }).ConfigureAwait(false);
throw new NotImplementedException();
recognizeOptions ??= new RecognizeOptions();

//SourcePath_internal sourcePath = new SourcePath_internal() { Source = formFileUri.ToString() };
//ResponseWithHeaders<AnalyzeWithCustomModelHeaders> response = await ServiceClient.RestClient.AnalyzeWithCustomModelAsync(new Guid(modelId), includeTextDetails: includeTextElements, sourcePath, cancellationToken).ConfigureAwait(false);
//return new RecognizeCustomFormsOperation(ServiceClient, modelId, response.Headers.OperationLocation);
SourcePath_internal sourcePath = new SourcePath_internal(formFileUri.ToString());
ResponseWithHeaders<ServiceAnalyzeWithCustomModelHeaders> response = await ServiceClient.RestClient.AnalyzeWithCustomModelAsync(new Guid(modelId), includeTextDetails: recognizeOptions.IncludeTextContent, sourcePath, cancellationToken).ConfigureAwait(false);
return new RecognizeCustomFormsOperation(ServiceClient, modelId, response.Headers.OperationLocation);
}

#endregion
Expand Down
32 changes: 23 additions & 9 deletions sdk/formrecognizer/Azure.AI.FormRecognizer/src/RecognizeCustomFormsOperation.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ private async ValueTask<Response> UpdateStatusAsync(bool async, CancellationToke
if (update.Value.Status == OperationStatus.Succeeded || update.Value.Status == OperationStatus.Failed)
{
_hasCompleted = true;
_value = ConvertToRecognizedForms(update.Value.AnalyzeResult.PageResults, update.Value.AnalyzeResult.ReadResults);
_value = ConvertToRecognizedForms(update.Value.AnalyzeResult);
}

_response = update.GetRawResponse();
Expand All @@ -108,17 +108,31 @@ private async ValueTask<Response> UpdateStatusAsync(bool async, CancellationToke
return GetRawResponse();
}

#pragma warning disable CA1801 // Remove unused parameter
private static IReadOnlyList<RecognizedForm> ConvertToRecognizedForms(IReadOnlyList<PageResult_internal> pageResults, IReadOnlyList<ReadResult_internal> readResults)
#pragma warning restore CA1801 // Remove unused parameter
private static IReadOnlyList<RecognizedForm> ConvertToRecognizedForms(AnalyzeResult_internal analyzeResult)
{
List<RecognizedForm> pages = new List<RecognizedForm>();
for (int i = 0; i < pageResults.Count; i++)
return analyzeResult.DocumentResults?.Count == 0 ?
ConvertUnsupervisedResult(analyzeResult) :
ConvertSupervisedResult(analyzeResult);
}

private static IReadOnlyList<RecognizedForm> ConvertUnsupervisedResult(AnalyzeResult_internal analyzeResult)
{
List<RecognizedForm> forms = new List<RecognizedForm>();
foreach (var pageResult in analyzeResult.PageResults)
{
forms.Add(new RecognizedForm(pageResult, analyzeResult.ReadResults));
}
return forms;
}

private static IReadOnlyList<RecognizedForm> ConvertSupervisedResult(AnalyzeResult_internal analyzeResult)
{
List<RecognizedForm> forms = new List<RecognizedForm>();
foreach (var documentResult in analyzeResult.DocumentResults)
{
// TODO: Implement
//pages.Add(new RecognizedForm(pageResults[i], readResults[i]));
forms.Add(new RecognizedForm(documentResult, analyzeResult.PageResults, analyzeResult.ReadResults));
}
return pages;
return forms;
}
}
}
Loading

0 comments on commit 401a091

Please sign in to comment.