Merge branch 'main' into fix/invalid-evaluation-doctype-deduction

CHANGELOG.md

@@ -1,4 +1,14 @@
-## 0.14.3-dev6
+## 0.14.4-dev0
+
+### Enhancements
+
+### Features
+
+### Fixes
+
+* **Fix document type deduction during evaluation** There was a bug that caused the document type for file with more than 2 extensions (or dot symbols) to be inferred incorrectly.
+
+## 0.14.3
 
 ### Enhancements
 
@@ -10,6 +20,7 @@
 
 ### Fixes
 
+* **Fix `partition_pdf()` to keep spaces in the text**. The control character `\t` is now replaced with a space instead of being removed when merging inferred elements with embedded elements.
 * **Turn off XML resolve entities** Sets `resolve_entities=False` for XML parsing with `lxml`
   to avoid text being dynamically injected into the XML document.
 * **Add backward compatibility for the deprecated pdf_infer_table_structure parameter**.
@@ -19,7 +30,6 @@
 * **Diable `table_as_cells` output by default** to reduce overhead in partition; now `table_as_cells` is only produced when the env `EXTACT_TABLE_AS_CELLS` is `true`
 * **Reduce excessive logging** Change per page ocr info level logging into detail level trace logging
 * **Replace try block in `document_to_element_list` for handling HTMLDocument** Use `getattr(element, "type", "")` to get the `type` attribute of an element when it exists. This is more explicit way to handle the special case for HTML documents and prevents other types of attribute error from being silenced by the try block
-* **Fix document type deduction during evaluation** There was a bug that caused the document type for file with more than 2 extensions (or dot symbols) to be inferred incorrectly.
 
 ## 0.14.2
 

test_unstructured/partition/pdf_image/test_pdf_image_utils.py

@@ -347,7 +347,7 @@ def test_annotate_layout_elements_file_not_found_error():
 
 @pytest.mark.parametrize(
     ("text", "expected"),
-    [("c\to\x0cn\ftrol\ncharacter\rs\b", "control characters"), ("\"'\\", "\"'\\")],
+    [("test\tco\x0cn\ftrol\ncharacter\rs\b", "test control characters"), ("\"'\\", "\"'\\")],
 )
 def test_remove_control_characters(text, expected):
     assert pdf_image_utils.remove_control_characters(text) == expected

test_unstructured_ingest/dest/mongodb.sh

@@ -33,6 +33,13 @@ function cleanup() {
 
 trap cleanup EXIT
 
+# NOTE(robinson) - per pymongo docs, pymongo ships with its own version of the bson library,
+# which is incompatible with the bson installed from pypi. bson is installed as part of the
+# astra dependencies.
+# ref: https://pymongo.readthedocs.io/en/stable/installation.html
+pip uninstall -y bson pymongo
+make install-ingest-mongodb
+
 python "$SCRIPT_DIR"/python/test-ingest-mongodb.py \
   --uri "$MONGODB_URI" \
   --database "$MONGODB_DATABASE_NAME" \

test_unstructured_ingest/src/mongodb.sh

@@ -20,7 +20,12 @@ if [ -z "$MONGODB_URI" ] && [ -z "$MONGODB_DATABASE_NAME" ]; then
   exit 8
 fi
 
-# trap cleanup EXIT
+# NOTE(robinson) - per pymongo docs, pymongo ships with its own version of the bson library,
+# which is incompatible with the bson installed from pypi. bson is installed as part of the
+# astra dependencies.
+# ref: https://pymongo.readthedocs.io/en/stable/installation.html
+pip uninstall -y bson pymongo
+make install-ingest-mongodb
 
 PYTHONPATH=. ./unstructured/ingest/main.py \
   mongodb \

test_unstructured_ingest/test-ingest-dest.sh

@@ -34,10 +34,7 @@ all_tests=(
   'sqlite.sh'
   'vectara.sh'
   'weaviate.sh'
-  # NOTE(robinson) - mongo conflicts with astra because it ships with its
-  # own version of bson, and installing bson from pip causes mongo to fail
-  # ref: https://pymongo.readthedocs.io/en/stable/installation.html
-  # 'mongodb.sh'
+  'mongodb.sh'
 )
 
 full_python_matrix_tests=(

test_unstructured_ingest/test-ingest-src.sh

@@ -42,7 +42,7 @@ all_tests=(
   'confluence-diff.sh'
   'confluence-large.sh'
   'airtable-diff.sh'
-  # NOTE(ryan): This test is disabled because it is triggering too many requests to the API
+  # # NOTE(ryan): This test is disabled because it is triggering too many requests to the API
   # 'airtable-large.sh'
   'local-single-file.sh'
   'local-single-file-basic-chunking.sh'
@@ -62,10 +62,7 @@ all_tests=(
   'local-embed-voyageai.sh'
   'sftp.sh'
   'opensearch.sh'
-  # NOTE(robinson) - mongo conflicts with astra because it ships with its
-  # own version of bson, and installing bson from pip causes mongo to fail
-  # ref: https://pymongo.readthedocs.io/en/stable/installation.html
-  # 'mongodb.sh'
+  'mongodb.sh'
 )
 
 full_python_matrix_tests=(

unstructured/__version__.py

@@ -1 +1 @@
-__version__ = "0.14.3-dev6"  # pragma: no cover
+__version__ = "0.14.4-dev0"  # pragma: no cover

unstructured/partition/pdf_image/pdf_image_utils.py

@@ -427,7 +427,7 @@ def remove_control_characters(text: str) -> str:
     """Removes control characters from text."""
 
     # Replace newline character with a space
-    text = text.replace("\n", " ")
+    text = text.replace("\t", " ").replace("\n", " ")
     # Remove other control characters
     out_text = "".join(c for c in text if unicodedata.category(c)[0] != "C")
     return out_text