[ai-form-recognizer] Handle Selection Mark Element References (#16710)

* [ai-form-recognizer] Handle ElementReference to selection mark * Update CHANGELOG * Corrected doc string for internal function * FormContent -> FormElement (very old rename) * remove errant it.only
Azure · Aug 4, 2021 · 2e9f122 · 2e9f122
1 parent 74419c7
commit 2e9f122
Show file tree

Hide file tree

Showing 3 changed files with 99 additions and 22 deletions.
diff --git a/sdk/formrecognizer/ai-form-recognizer/CHANGELOG.md b/sdk/formrecognizer/ai-form-recognizer/CHANGELOG.md
@@ -1,6 +1,6 @@
 # Release History
 
-## 3.2.0 (Unreleased)
+## 3.2.0 (2021-08-10)
 
 ### Features Added
 - With the dropping of support for Node.js versions that are no longer in LTS, the dependency on `@types/node` has been updated to version 12. Read our [support policy](https://github.com/Azure/azure-sdk-for-js/blob/main/SUPPORT.md) for more details.
@@ -9,6 +9,8 @@
 
 ### Key Bugs Fixed
 
+- Fixed an issue in which form recognition would sometimes fail due to encountering an element reference pointing to a selection mark, causing an exception to be thrown. These references are now handled correctly.
+
 ## 3.1.0 (2021-05-26)
 
 - This General Availability (GA) release marks the stability of the changes introduced in package versions 3.1.0-beta.1 through 3.1.0-beta.3.

diff --git a/sdk/formrecognizer/ai-form-recognizer/src/transforms.ts b/sdk/formrecognizer/ai-form-recognizer/src/transforms.ts
@@ -51,6 +51,22 @@ function toBoundingBox(original: number[]): Point2D[] {
   ];
 }
 
+/**
+ * Utility type to remove null | undefined from a type.
+ * @internal
+ */
+type NotNull<T> = T extends null | undefined ? never : T;
+
+/**
+ * Extracts the keys of a type whose value types are assignable to a Condition.
+ * @internal
+ */
+type KeysWhere<T, Condition> = NotNull<
+  {
+    [K in keyof T]: T[K] extends Condition ? K : never;
+  }[keyof T]
+>;
+
 /**
  * @internal
  */
@@ -111,26 +127,39 @@ export function toFormPage(original: ReadResultModel): FormPage {
   };
 }
 
-// Note: might need to support other element types in future, e.g., checkbox
-const textPattern = /\/readResults\/(\d+)\/lines\/(\d+)(?:\/words\/(\d+))?/;
+/**
+ * A RegExp that can handle parsing an ElementReference. It handles any keys of FormPage and
+ * Supports optionally specifying "words" at the end. This is kind of hacked together, but
+ * can handle all the valid element references in Form Recognizer so far.
+ */
+const elementReferencePattern = /\/readResults\/(\d+)\/([a-z][a-zA-Z]*)\/(\d+)(?:\/words\/(\d+))?/;
 
 /**
+ * Parse an ElementReference of a known structure.
  * @internal
+ * @param ref - the string representation (JSON Pointer) of the ElementReference
+ * @param readResults - The transformed ReadResults into which the reference points
+ * @returns a reference to the FormElement that the ElementReference refers to
  */
-export function toFormContent(element: string, readResults: FormPage[]): FormElement {
-  const result = textPattern.exec(element);
-  if (!result || !result[0] || !result[1] || !result[2]) {
-    throw new Error(`Unexpected element reference encountered: ${element}`);
+export function elementReferenceToFormElement(ref: string, readResults: FormPage[]): FormElement {
+  const result = elementReferencePattern.exec(ref);
+
+  if (result === null) {
+    throw new Error(`Unexpected element reference encountered: "${ref}"`);
   }
 
   const readIndex = Number.parseInt(result[1]);
-  const lineIndex = Number.parseInt(result[2]);
-  if (result[3]) {
-    const wordIndex = Number.parseInt(result[3]);
-    return readResults[readIndex].lines![lineIndex].words[wordIndex];
-  } else {
-    return readResults[readIndex].lines![lineIndex];
+  const elementKind = result[2] as KeysWhere<FormPage, FormElement[] | undefined>;
+  const elementIndex = Number.parseInt(result[3]);
+  const wordIndex = Number.parseInt(result[4]);
+
+  const baseElement = readResults[readIndex][elementKind]![elementIndex];
+
+  if (!Number.isNaN(wordIndex) && elementKind === "lines") {
+    return (baseElement as FormLine).words[wordIndex];
   }
+
+  return baseElement;
 }
 
 /**
@@ -145,7 +174,9 @@ export function toFieldData(
     pageNumber,
     text: original.text,
     boundingBox: original.boundingBox ? toBoundingBox(original.boundingBox) : undefined,
-    fieldElements: original.elements?.map((element) => toFormContent(element, readResults!))
+    fieldElements: original.elements?.map((element) =>
+      elementReferenceToFormElement(element, readResults!)
+    )
   };
 }
 
@@ -182,7 +213,9 @@ export function toFormTable(
     cells: original.cells.map((cell) => ({
       boundingBox: toBoundingBox(cell.boundingBox),
       columnIndex: cell.columnIndex,
-      fieldElements: cell.elements?.map((element) => toFormContent(element, readResults)),
+      fieldElements: cell.elements?.map((element) =>
+        elementReferenceToFormElement(element, readResults)
+      ),
       rowIndex: cell.rowIndex,
       columnSpan: cell.columnSpan ?? 1,
       rowSpan: cell.rowSpan ?? 1,
@@ -316,7 +349,9 @@ export function toFormFieldFromFieldValueModel(
       pageNumber: original.pageNumber ?? 0,
       text: original.text,
       boundingBox: original.boundingBox ? toBoundingBox(original.boundingBox) : undefined,
-      fieldElements: original.elements?.map((element) => toFormContent(element, readResults))
+      fieldElements: original.elements?.map((element) =>
+        elementReferenceToFormElement(element, readResults)
+      )
     },
     valueType: original.type,
     value

diff --git a/sdk/formrecognizer/ai-form-recognizer/test/internal/transforms.spec.ts b/sdk/formrecognizer/ai-form-recognizer/test/internal/transforms.spec.ts
@@ -6,7 +6,7 @@ import { assert } from "chai";
 import {
   toTextLine,
   toFormPage,
-  toFormContent,
+  elementReferenceToFormElement,
   toFieldData,
   toFormFieldFromKeyValuePairModel,
   toFormFieldFromFieldValueModel,
@@ -25,7 +25,7 @@ import {
   DataTable as DataTableModel,
   DocumentResult as DocumentResultModel
 } from "../../src/generated/models";
-import { Point2D, FormField } from "../../src/models";
+import { Point2D, FormField, FormPage, FormSelectionMark } from "../../src/models";
 
 const supervisedResponseString = `{
   "status": "succeeded",
@@ -2195,25 +2195,65 @@ describe("Transforms", () => {
     lines: [originalLine1, originalLine2]
   };
 
-  it("toExtractedElement() converts word string reference to extracted word", () => {
+  it("elementReferenceToFormContent() converts word string reference to extracted word", () => {
     const stringRef = "#/readResults/0/lines/0/words/0";
     const readResults = [originalReadResult1, originalReadResult2].map(toFormPage);
 
-    const transformed = toFormContent(stringRef, readResults);
+    const transformed = elementReferenceToFormElement(stringRef, readResults);
 
     assert.deepStrictEqual(transformed, readResults[0].lines![0].words[0]);
   });
 
   const formPages = [originalReadResult1, originalReadResult2].map(toFormPage);
 
-  it("toExtractedElement() converts line string reference to extracted line", () => {
+  it("elementReferenceToFormContent() converts line string reference to extracted line", () => {
     const stringRef = "#/readResults/1/lines/1";
 
-    const transformed = toFormContent(stringRef, formPages);
+    const transformed = elementReferenceToFormElement(stringRef, formPages);
 
     assert.deepStrictEqual(transformed, formPages[1].lines![1]);
   });
 
+  it("elementReferenceToFormContent() converts selectionMark reference to FormSelectionMark", () => {
+    const stringRef = "#/readResults/0/selectionMarks/1";
+
+    const input = {
+      selectionMarks: [
+        {
+          kind: "selectionMark",
+          pageNumber: 0,
+          state: "selected",
+          text: "Hello world!",
+          boundingBox: [
+            { x: 0, y: 1 },
+            { x: 2, y: 3 },
+            { x: 4, y: 5 },
+            { x: 6, y: 7 }
+          ],
+          confidence: 0.8
+        },
+        {
+          kind: "selectionMark",
+          pageNumber: 1,
+          state: "unselected",
+          text: "Hello world!",
+          boundingBox: [
+            { x: 0, y: 1 },
+            { x: 2, y: 3 },
+            { x: 4, y: 5 },
+            { x: 6, y: 7 }
+          ],
+          confidence: 0.8
+        }
+      ]
+    } as const;
+
+    assert.deepStrictEqual(
+      elementReferenceToFormElement(stringRef, [(input as unknown) as FormPage]),
+      (input.selectionMarks[1] as unknown) as FormSelectionMark
+    );
+  });
+
   const originalKeyValueElement1 = {
     text: "keyvalue element text",
     boundingBox: [1, 2, 3, 4, 5, 6, 7, 8],