fix: discard metadata fields if not set in Weaviate (#3578)

* fix weaviate bug in returning embeddings and setting empty meta fields * review comment
deepset-ai · Nov 15, 2022 · ba75d39 · ba75d39
1 parent 6ce2d29
commit ba75d39
Show file tree

Hide file tree

Showing 3 changed files with 16 additions and 15 deletions.
diff --git a/haystack/document_stores/weaviate.py b/haystack/document_stores/weaviate.py
@@ -279,21 +279,21 @@ def _convert_weaviate_result_to_document(
             props.pop("_additional", None)
 
         # We put all additional data of the doc into meta_data and return it in the API
-        meta_data = {k: v for k, v in props.items() if k not in (self.content_field, self.embedding_field)}
-
-        if return_embedding and embedding:
-            embedding = np.asarray(embedding, dtype=np.float32)
+        meta_data = {}
+        for k, v in props.items():
+            if k in (self.content_field, self.embedding_field):
+                continue
+            if v is None:
+                continue
+            meta_data[k] = v
 
         document = Document.from_dict(
-            {
-                "id": id,
-                "content": content,
-                "content_type": content_type,
-                "meta": meta_data,
-                "score": score,
-                "embedding": embedding,
-            }
+            {"id": id, "content": content, "content_type": content_type, "meta": meta_data, "score": score}
         )
+
+        if return_embedding and embedding:
+            document.embedding = np.asarray(embedding, dtype=np.float32)
+
         return document
 
     def _create_document_field_map(self) -> Dict:

diff --git a/test/document_stores/test_base.py b/test/document_stores/test_base.py
@@ -86,8 +86,9 @@ def test_write_with_duplicate_doc_ids(self, ds):
             Document(content="Doc1", id_hash_keys=["content"], meta={"key1": "value1"}),
         ]
         ds.write_documents(duplicate_documents, duplicate_documents="skip")
-        assert len(ds.get_all_documents()) == 1
-        assert ds.get_all_documents()[0] == duplicate_documents[0]
+        results = ds.get_all_documents()
+        assert len(results) == 1
+        assert results[0] == duplicate_documents[0]
         with pytest.raises(Exception):
             ds.write_documents(duplicate_documents, duplicate_documents="fail")
 

diff --git a/test/document_stores/test_weaviate.py b/test/document_stores/test_weaviate.py
@@ -28,7 +28,7 @@ class TestWeaviateDocumentStore(DocumentStoreBaseTestAbstract):
 
     @pytest.fixture
     def ds(self):
-        return WeaviateDocumentStore(index=self.index_name, recreate_index=True)
+        return WeaviateDocumentStore(index=self.index_name, recreate_index=True, return_embedding=True)
 
     @pytest.fixture(scope="class")
     def documents(self):