fix: disable table_as_cells output by default

- now requires env EXTRACT_TABLE_AS_CELLS to be true to output
  table_as_cells in Table elements' metadata

CHANGELOG.md

@@ -1,4 +1,4 @@
-## 0.14.3-dev2
+## 0.14.3-dev3
 
 ### Enhancements
 
@@ -9,11 +9,12 @@
 
 ### Fixes
 
-**Turn off XML resolve entities** Sets `resolve_entities=False` for XML parsing with `lxml`
+* **Turn off XML resolve entities** Sets `resolve_entities=False` for XML parsing with `lxml`
   to avoid text being dynamically injected into the XML document.
 * Add the missing `form_extraction_skip_tables` argument to the `partition_pdf_or_image` call.
-
 * **Chromadb change from Add to Upsert using element_id to make idempotent**
+* **Diable `table_as_cells` output by default** to reduce overhead in partition; now `table_as_cells` is only produced when the env `EXTACT_TABLE_AS_CELLS` is `true`
+
 
 ## 0.14.2
 

unstructured/__version__.py

@@ -1 +1 @@
-__version__ = "0.14.3-dev2"  # pragma: no cover
+__version__ = "0.14.3-dev3"  # pragma: no cover

unstructured/partition/pdf_image/ocr.py

@@ -253,7 +253,7 @@ def supplement_element_with_table_extraction(
     """Supplement the existing layout with table extraction. Any Table elements
     that are extracted will have a metadata fields "text_as_html" where
     the table's text content is rendered into a html string and "table_as_cells"
-    with the raw table cells output from table agent
+    with the raw table cells output from table agent if env_config.EXTRACT_TABLE_AS_CELLS is True
     """
     from unstructured_inference.models.tables import cells_to_html
 
@@ -279,13 +279,15 @@ def supplement_element_with_table_extraction(
         tatr_cells = tables_agent.predict(
             cropped_image, ocr_tokens=table_tokens, result_format="cells"
         )
-        text_as_html = cells_to_html(tatr_cells)
-        simple_table_cells = [
-            SimpleTableCell.from_table_transformer_cell(cell).to_dict() for cell in tatr_cells
-        ]
 
+        text_as_html = cells_to_html(tatr_cells)
         element.text_as_html = text_as_html
-        element.table_as_cells = simple_table_cells
+
+        if env_config.EXTRACT_TABLE_AS_CELLS:
+            simple_table_cells = [
+                SimpleTableCell.from_table_transformer_cell(cell).to_dict() for cell in tatr_cells
+            ]
+            element.table_as_cells = simple_table_cells
 
     return elements
 

unstructured/partition/utils/config.py

@@ -116,6 +116,11 @@ def EXTRACT_IMAGE_BLOCK_CROP_VERTICAL_PAD(self) -> int:
         """
         return self._get_int("EXTRACT_IMAGE_BLOCK_CROP_VERTICAL_PAD", 0)
 
+    @property
+    def EXTRACT_TABLE_AS_CELLS(self) -> bool:
+        """adds `table_as_cells` to a Table element's metadata when it is True"""
+        return self._get_bool("EXTRACT_TABLE_AS_CELLS", False)
+
     @property
     def OCR_LAYOUT_SUBREGION_THRESHOLD(self) -> float:
         """threshold to determine if an OCR region is a sub-region of a given block