fix: Updates to hOCR Template to follow hOCR Spec (#195)

- Added validation in testing with https://github.com/kba/hocr-spec-python
googleapis · Nov 7, 2023 · 3f52e82 · 3f52e82
1 parent e05cf50
commit 3f52e82
Show file tree

Hide file tree

Showing 10 changed files with 60 additions and 38 deletions.
diff --git a/google/cloud/documentai_toolbox/templates/hocr_document_template.xml.j2 b/google/cloud/documentai_toolbox/templates/hocr_document_template.xml.j2
@@ -6,8 +6,9 @@
 <meta http-equiv="Content-Type" content="text/html;charset=utf-8" />
 <meta name="ocr-system" content="Document AI OCR" />
 <meta name="ocr-langs" content="unknown" />
+<meta name="ocr-scripts" content="unknown" />
 <meta name="ocr-number-of-pages" content="{{ pages|length }}" />
-<meta name="ocr-capabilities" content="ocr_page ocr_carea ocr_par ocr_line ocrx_word" />
+<meta name="ocr-capabilities" content="ocrp_lang ocr_page ocr_carea ocr_par ocr_line ocrx_word" />
 </head>
 <body>
 {% for page in pages -%}
@@ -16,13 +17,13 @@
         {% set bidx = loop.index0 -%}
         <span class='ocr_carea' id='block_{{ page_number }}_{{ bidx }}' title='{{ docai_block.hocr_bounding_box -}}'>{% for paragraph in docai_block.paragraphs -%}
             {% set paridx = loop.index0 -%}
-            <span class='ocr_par' id='par_{{ page_number }}_{{ bidx }}_{{ paridx }}' title='{{ paragraph.hocr_bounding_box -}}'>{% for line in paragraph.lines -%}
+            <p class='ocr_par' id='par_{{ page_number }}_{{ bidx }}_{{ paridx }}' title='{{ paragraph.hocr_bounding_box -}}'>{% for line in paragraph.lines -%}
                 {% set lidx = loop.index0 -%}
                 <span class='ocr_line' id='line_{{ page_number }}_{{ bidx }}_{{ paridx }}_{{ lidx }}' title='{{ line.hocr_bounding_box }}'>{{ line.text }}{% for token in line.tokens -%}
                     {% set tidx = loop.index0 -%}
                     <span class='ocrx_word' id='word_{{ page_number }}_{{ bidx }}_{{ paridx }}_{{ lidx }}_{{ tidx }}' title='{{ token.hocr_bounding_box }}'>{{ token.text }}</span>{% endfor -%}
                 </span>{% endfor -%}
-            </span>{% endfor -%}
+            </p>{% endfor -%}
         </span>{% endfor -%}
     </div>
 {% endfor -%}

diff --git a/samples/snippets/test_convert_document_to_hocr_sample.py b/samples/snippets/test_convert_document_to_hocr_sample.py
@@ -24,7 +24,11 @@ def test_convert_document_to_hocr_sample() -> None:
         document_path=document_path, document_title=document_title
     )
 
-    with open("../../tests/unit/resources/toolbox_invoice_test_0_hocr.xml", "r") as f:
+    with open(
+        "../../tests/unit/resources/toolbox_invoice_test_0_hocr.xml",
+        "r",
+        encoding="utf-8",
+    ) as f:
         expected = f.read()
 
     assert actual == expected
diff --git a/setup.py b/setup.py
@@ -66,6 +66,7 @@
         "immutabledict >= 2.0.0, < 3.0.0dev; python_version<'3.8'",
         "Pillow >= 9.5.0, < 11.0.0",
         "Jinja2 >= 3.1.0, <= 4.0.0",
+        "hocr-spec >= 0.2.0",
     ),
     python_requires=">=3.7",
     classifiers=[

diff --git a/testing/constraints-3.10.txt b/testing/constraints-3.10.txt
@@ -11,3 +11,4 @@ google-cloud-documentai
 google-cloud-storage
 numpy
 pikepdf
+hocr-spec
diff --git a/testing/constraints-3.11.txt b/testing/constraints-3.11.txt
@@ -11,3 +11,4 @@ google-cloud-documentai
 google-cloud-storage
 numpy
 pikepdf
+hocr-spec
diff --git a/testing/constraints-3.7.txt b/testing/constraints-3.7.txt
@@ -14,3 +14,4 @@ google-cloud-documentai==2.20.0
 google-cloud-storage==2.7.0
 numpy==1.19.5
 pikepdf==6.2.9
+hocr-spec==0.2.0
diff --git a/testing/constraints-3.8.txt b/testing/constraints-3.8.txt
@@ -11,3 +11,4 @@ google-cloud-documentai
 google-cloud-storage
 numpy==1.21.6
 pikepdf==8.2.3
+hocr-spec
diff --git a/testing/constraints-3.9.txt b/testing/constraints-3.9.txt
@@ -11,3 +11,4 @@ google-cloud-documentai
 google-cloud-storage
 numpy
 pikepdf
+hocr-spec
diff --git a/tests/unit/resources/toolbox_invoice_test_0_hocr.xml b/tests/unit/resources/toolbox_invoice_test_0_hocr.xml
diff --git a/tests/unit/test_document.py b/tests/unit/test_document.py
@@ -14,6 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from io import BytesIO
 import json
 import os
 import shutil
@@ -32,6 +33,8 @@
 from google.cloud import documentai
 from google.cloud.documentai_toolbox import document, gcs_utilities
 
+from hocr_spec import HocrValidator
+
 
 def get_bytes(file_name):
     result = []
@@ -689,8 +692,15 @@ def test_export_hocr_str():
     )
 
     actual_hocr = wrapped_document.export_hocr_str(title="toolbox_invoice_test-0")
+    assert actual_hocr
+    validator = HocrValidator(profile="standard")
+    report = validator.validate(BytesIO(actual_hocr.encode("utf-8")), parse_strict=True)
+
+    assert report.format("bool")
 
-    with open("tests/unit/resources/toolbox_invoice_test_0_hocr.xml", "r") as f:
+    with open(
+        "tests/unit/resources/toolbox_invoice_test_0_hocr.xml", "r", encoding="utf-8"
+    ) as f:
         expected = f.read()
 
     assert actual_hocr == expected