Merge remote-tracking branch 'origin/master'

zulia-client/build.gradle.kts

@@ -0,0 +1,8 @@
+apply(from = "../maven.gradle")
+
+description = "Zulia Client"
+
+dependencies {
+    compile(project(":zulia-common"))
+}
+

zulia-client/src/main/java/io/zulia/client/ZuliaRESTClient.java

@@ -75,7 +75,7 @@ public void storeAssociated(String uniqueId, String indexName, String fileName,
 			parameters.put(ZuliaConstants.INDEX, indexName);
 			if (meta != null) {
 				ArrayList<Object> list = new ArrayList<>();
-				parameters.put(ZuliaConstants.META, list);
+				parameters.put(ZuliaConstants.META_JSON, list);
 				for (String key : meta.keySet()) {
 					String value = meta.get(key);
 					list.add(key + ":" + value);

zulia-client/src/main/java/io/zulia/client/result/AssociatedResult.java

@@ -1,10 +1,9 @@
 package io.zulia.client.result;
 
-import java.util.HashMap;
-import java.util.Map;
+import io.zulia.util.ZuliaUtil;
+import org.bson.Document;
 
 import static io.zulia.message.ZuliaBase.AssociatedDocument;
-import static io.zulia.message.ZuliaBase.Metadata;
 
 public class AssociatedResult {
 
@@ -14,12 +13,8 @@ public AssociatedResult(AssociatedDocument associatedDocument) {
 		this.associatedDocument = associatedDocument;
 	}
 
-	public Map<String, String> getMeta() {
-		HashMap<String, String> metadata = new HashMap<String, String>();
-		for (Metadata md : associatedDocument.getMetadataList()) {
-			metadata.put(md.getKey(), md.getValue());
-		}
-		return metadata;
+	public Document getMeta() {
+		return ZuliaUtil.byteArrayToMongoDocument(associatedDocument.getMetadata().toByteArray());
 	}
 
 	public String getFilename() {
@@ -39,8 +34,7 @@ public byte[] getDocumentAsBytes() {
 
 	public String getDocumentAsUtf8() {
 		if (hasDocument()) {
-			String contents = associatedDocument.getDocument().toStringUtf8();
-			return contents;
+			return associatedDocument.getDocument().toStringUtf8();
 		}
 		return null;
 	}

zulia-client/src/main/java/io/zulia/client/result/FetchResult.java

@@ -7,12 +7,9 @@
 
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
 
 import static io.zulia.message.ZuliaBase.AssociatedDocument;
-import static io.zulia.message.ZuliaBase.Metadata;
 import static io.zulia.message.ZuliaServiceOuterClass.FetchResponse;
 
 public class FetchResult extends Result {
@@ -53,14 +50,10 @@ public String getIndexName() {
 		return null;
 	}
 
-	public Map<String, String> getMeta() {
+	public Document getMeta() {
 		if (fetchResponse.hasResultDocument()) {
 			ResultDocument rd = fetchResponse.getResultDocument();
-			HashMap<String, String> metadata = new HashMap<>();
-			for (Metadata md : rd.getMetadataList()) {
-				metadata.put(md.getKey(), md.getValue());
-			}
-			return metadata;
+			return ZuliaUtil.byteStringToMongoDocument(rd.getMetadata());
 		}
 		return null;
 	}

zulia-client/src/main/java/io/zulia/doc/AssociatedBuilder.java

@@ -2,7 +2,6 @@
 
 import com.google.protobuf.ByteString;
 import io.zulia.message.ZuliaBase.AssociatedDocument;
-import io.zulia.message.ZuliaBase.Metadata;
 import io.zulia.util.ZuliaUtil;
 import org.bson.Document;
 
@@ -44,17 +43,17 @@ public AssociatedBuilder setDocument(byte[] bytes) {
 	}
 
 	public AssociatedBuilder setDocument(Document document) {
-		adBuilder.setDocument(ByteString.copyFrom(ZuliaUtil.mongoDocumentToByteArray(document)));
+		adBuilder.setDocument(ZuliaUtil.mongoDocumentToByteString(document));
 		return this;
 	}
 
-	public AssociatedBuilder addMetaData(String key, String value) {
-		adBuilder.addMetadata(Metadata.newBuilder().setKey(key).setValue(value));
-		return this;
-	}
-
-	public AssociatedBuilder clearMetaData() {
-		adBuilder.clearMetadata();
+	public AssociatedBuilder setMetadata(Document metadata) {
+		if (metadata != null) {
+			adBuilder.setMetadata(ZuliaUtil.mongoDocumentToByteString(metadata));
+		}
+		else {
+			adBuilder.clearMetadata();
+		}
 		return this;
 	}
 

zulia-client/src/main/java/io/zulia/doc/ResultDocBuilder.java

@@ -1,13 +1,9 @@
 package io.zulia.doc;
 
-import com.google.protobuf.ByteString;
-import io.zulia.message.ZuliaBase.Metadata;
 import io.zulia.message.ZuliaBase.ResultDocument;
 import io.zulia.util.ZuliaUtil;
 import org.bson.Document;
 
-import java.util.HashMap;
-
 public class ResultDocBuilder {
 	private ResultDocument.Builder resultDocumentBuilder;
 
@@ -19,20 +15,18 @@ public ResultDocBuilder() {
 		resultDocumentBuilder = ResultDocument.newBuilder();
 	}
 
-	public ResultDocBuilder addMetaData(String key, String value) {
-		resultDocumentBuilder.addMetadata(Metadata.newBuilder().setKey(key).setValue(value));
-		return this;
-	}
-
-	public ResultDocBuilder addAllMetaData(HashMap<String, String> metadata) {
-		for (String key : metadata.keySet()) {
-			resultDocumentBuilder.addMetadata(Metadata.newBuilder().setKey(key).setValue(metadata.get(key)));
+	public ResultDocBuilder setMetadata(Document metadata) {
+		if (metadata != null) {
+			resultDocumentBuilder.setMetadata(ZuliaUtil.mongoDocumentToByteString(metadata));
+		}
+		else {
+			resultDocumentBuilder.clearMetadata();
 		}
 		return this;
 	}
 
 	public ResultDocBuilder setDocument(Document document) {
-		resultDocumentBuilder.setDocument(ByteString.copyFrom(ZuliaUtil.mongoDocumentToByteArray(document)));
+		resultDocumentBuilder.setDocument(ZuliaUtil.mongoDocumentToByteString(document));
 		return this;
 	}
 

zulia-common/src/main/java/io/zulia/ZuliaConstants.java

@@ -35,7 +35,7 @@ public interface ZuliaConstants {
 	String ROWS = "rows";
 	String ID = "id";
 	String FILE_NAME = "fileName";
-	String META = "meta";
+	String META_JSON = "metaJson";
 	String INDEX = "index";
 	String FACET = "facet";
 	String SORT = "sort";

zulia-common/src/main/java/io/zulia/util/ZuliaUtil.java

@@ -1,5 +1,6 @@
 package io.zulia.util;
 
+import com.google.protobuf.ByteString;
 import org.bson.BsonBinaryReader;
 import org.bson.BsonBinaryWriter;
 import org.bson.Document;
@@ -51,10 +52,24 @@ public static byte[] mongoDocumentToByteArray(Document mongoDocument) {
 		return outputBuffer.toByteArray();
 	}
 
+	public static ByteString mongoDocumentToByteString(Document mongoDocument) {
+		return ByteString.copyFrom(mongoDocumentToByteArray(mongoDocument));
+	}
+
+	public static Document byteStringToMongoDocument(ByteString bytes) {
+		if (bytes != null) {
+			return byteArrayToMongoDocument(bytes.toByteArray());
+		}
+		return new Document();
+	}
+
 	public static Document byteArrayToMongoDocument(byte[] byteArray) {
-		BsonBinaryReader bsonReader = new BsonBinaryReader(ByteBuffer.wrap(byteArray));
+		if (byteArray != null && byteArray.length != 0) {
+			BsonBinaryReader bsonReader = new BsonBinaryReader(ByteBuffer.wrap(byteArray));
 
-		return documentCodec.decode(bsonReader, DecoderContext.builder().build());
+			return documentCodec.decode(bsonReader, DecoderContext.builder().build());
+		}
+		return new Document();
 	}
 
 	public static int computeLevenshteinDistance(String string1, String string2) {

zulia-common/src/main/proto/zulia_base.proto

@@ -19,7 +19,7 @@ message ResultDocument {
     string uniqueId = 1;
     string indexName = 2;
     bytes document = 3;
-    repeated Metadata metadata = 4;
+    bytes metadata = 4;
     int64 timestamp = 5;
 }
 
@@ -29,14 +29,10 @@ message AssociatedDocument {
     string documentUniqueId = 2;
     string indexName = 3;
     bytes document = 4;
-    repeated Metadata metadata = 5;
+    bytes metadata = 5;
     int64 timestamp = 6;
 }
 
-message Metadata {
-    string key = 1;
-    string value = 2;
-}
 
 message ShardCountResponse {
     uint32 shardNumber = 1;

zulia-server/src/main/java/io/zulia/server/filestorage/DocumentStorage.java

@@ -8,7 +8,6 @@
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.util.List;
-import java.util.Map;
 
 public interface DocumentStorage {
 
@@ -20,7 +19,7 @@ public interface DocumentStorage {
 
 	void getAssociatedDocuments(OutputStream outputstream, Document filter) throws IOException;
 
-	void storeAssociatedDocument(String uniqueId, String fileName, InputStream is, long timestamp, Map<String, String> metadataMap) throws Exception;
+	void storeAssociatedDocument(String uniqueId, String fileName, InputStream is, long timestamp, Document metadataMap) throws Exception;
 
 	InputStream getAssociatedDocumentStream(String uniqueId, String filename);
 

zulia-server/src/main/java/io/zulia/server/filestorage/FileDocumentStorage.java

@@ -9,7 +9,6 @@
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.util.List;
-import java.util.Map;
 import java.util.logging.Logger;
 
 public class FileDocumentStorage implements DocumentStorage {
@@ -41,7 +40,7 @@ public void getAssociatedDocuments(OutputStream outputstream, Document filter) t
 	}
 
 	@Override
-	public void storeAssociatedDocument(String uniqueId, String fileName, InputStream is, long timestamp, Map<String, String> metadataMap) throws Exception {
+	public void storeAssociatedDocument(String uniqueId, String fileName, InputStream is, long timestamp, Document metadataMap) throws Exception {
 		throw new RuntimeException("Not implemented");
 	}
 

zulia-server/src/main/java/io/zulia/server/filestorage/MongoDocumentStorage.java

@@ -13,8 +13,8 @@
 import com.mongodb.client.gridfs.model.GridFSUploadOptions;
 import com.mongodb.client.model.IndexOptions;
 import io.zulia.message.ZuliaBase.AssociatedDocument;
-import io.zulia.message.ZuliaBase.Metadata;
 import io.zulia.message.ZuliaQuery.FetchType;
+import io.zulia.util.ZuliaUtil;
 import org.bson.Document;
 
 import java.io.ByteArrayInputStream;
@@ -25,9 +25,7 @@
 import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.Date;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
 import java.util.function.Consumer;
 import java.util.logging.Logger;
 
@@ -100,21 +98,19 @@ public void drop() {
 	}
 
 	@Override
-	public void storeAssociatedDocument(String uniqueId, String fileName, InputStream is, long timestamp, Map<String, String> metadataMap) throws Exception {
+	public void storeAssociatedDocument(String uniqueId, String fileName, InputStream is, long timestamp, Document metadata) throws Exception {
 		GridFSBucket gridFS = createGridFSConnection();
 
 		deleteAssociatedDocument(uniqueId, fileName);
 
-		GridFSUploadOptions gridFSUploadOptions = getGridFSUploadOptions(uniqueId, fileName, timestamp, metadataMap);
+		GridFSUploadOptions gridFSUploadOptions = getGridFSUploadOptions(uniqueId, fileName, timestamp, metadata);
 		gridFS.uploadFromStream(fileName, is, gridFSUploadOptions);
 	}
 
-	private GridFSUploadOptions getGridFSUploadOptions(String uniqueId, String fileName, long timestamp, Map<String, String> metadataMap) {
-		Document metadata = new Document();
-		if (metadataMap != null) {
-			for (String key : metadataMap.keySet()) {
-				metadata.put(key, metadataMap.get(key));
-			}
+	private GridFSUploadOptions getGridFSUploadOptions(String uniqueId, String fileName, long timestamp, Document metadata) {
+
+		if (metadata == null) {
+			metadata = new Document();
 		}
 		metadata.put(TIMESTAMP, timestamp);
 		metadata.put(DOCUMENT_UNIQUE_ID_KEY, uniqueId);
@@ -130,9 +126,12 @@ public void storeAssociatedDocument(AssociatedDocument doc) throws Exception {
 
 		ByteArrayInputStream byteInputStream = new ByteArrayInputStream(bytes);
 
-		Map<String, String> metadata = new HashMap<>();
-		for (Metadata meta : doc.getMetadataList()) {
-			metadata.put(meta.getKey(), meta.getValue());
+		Document metadata;
+		if (doc.getMetadata() != null) {
+			metadata = ZuliaUtil.byteArrayToMongoDocument(doc.getMetadata().toByteArray());
+		}
+		else {
+			metadata = new Document();
 		}
 
 		storeAssociatedDocument(doc.getDocumentUniqueId(), doc.getFilename(), byteInputStream, doc.getTimestamp(), metadata);
@@ -196,9 +195,7 @@ private AssociatedDocument loadGridFSToAssociatedDocument(GridFSBucket gridFS, G
 		aBuilder.setTimestamp(timestamp);
 
 		aBuilder.setDocumentUniqueId((String) metadata.remove(DOCUMENT_UNIQUE_ID_KEY));
-		for (String field : metadata.keySet()) {
-			aBuilder.addMetadata(Metadata.newBuilder().setKey(field).setValue((String) metadata.get(field)));
-		}
+		aBuilder.setMetadata(ZuliaUtil.mongoDocumentToByteString(metadata));
 
 		if (FetchType.FULL.equals(fetchType)) {
 

zulia-server/src/main/java/io/zulia/server/index/ShardDocumentIndexer.java

@@ -2,7 +2,6 @@
 
 import info.debatty.java.lsh.SuperBit;
 import io.zulia.ZuliaConstants;
-import io.zulia.message.ZuliaBase;
 import io.zulia.message.ZuliaIndex;
 import io.zulia.server.config.ServerIndexConfig;
 import io.zulia.server.index.field.BooleanFieldIndexer;
@@ -42,7 +41,7 @@ public ShardDocumentIndexer(ServerIndexConfig indexConfig) {
 
 	}
 
-	public Document getIndexDocument(String uniqueId, long timestamp, org.bson.Document mongoDocument, List<ZuliaBase.Metadata> metadataList) throws Exception {
+	public Document getIndexDocument(String uniqueId, long timestamp, org.bson.Document mongoDocument, org.bson.Document metadata) throws Exception {
 		Document luceneDocument = new Document();
 
 		addStoredFieldsForDocument(mongoDocument, luceneDocument);
@@ -54,13 +53,7 @@ public Document getIndexDocument(String uniqueId, long timestamp, org.bson.Docum
 
 		luceneDocument.add(new StoredField(ZuliaConstants.STORED_DOC_FIELD, new BytesRef(ZuliaUtil.mongoDocumentToByteArray(mongoDocument))));
 
-		org.bson.Document metadataMongoDoc = new org.bson.Document();
-
-		for (ZuliaBase.Metadata metadata : metadataList) {
-			metadataMongoDoc.put(metadata.getKey(), metadata.getValue());
-		}
-
-		luceneDocument.add(new StoredField(ZuliaConstants.STORED_META_FIELD, new BytesRef(ZuliaUtil.mongoDocumentToByteArray(metadataMongoDoc))));
+		luceneDocument.add(new StoredField(ZuliaConstants.STORED_META_FIELD, new BytesRef(ZuliaUtil.mongoDocumentToByteArray(metadata))));
 
 		return luceneDocument;
 

zulia-server/src/main/java/io/zulia/server/index/ShardReader.java

@@ -526,12 +526,7 @@ private void handleStoredDoc(ZuliaQuery.ScoredResult.Builder srBuilder, String u
 
 		if (ZuliaQuery.FetchType.FULL.equals(resultFetchType) || ZuliaQuery.FetchType.META.equals(resultFetchType)) {
 			BytesRef metaRef = d.getBinaryValue(ZuliaConstants.STORED_META_FIELD);
-			org.bson.Document metaMongoDoc = new org.bson.Document();
-			metaMongoDoc.putAll(ZuliaUtil.byteArrayToMongoDocument(metaRef.bytes));
-
-			for (String key : metaMongoDoc.keySet()) {
-				rdBuilder.addMetadata(ZuliaBase.Metadata.newBuilder().setKey(key).setValue(((String) metaMongoDoc.get(key))));
-			}
+			rdBuilder.setMetadata(ByteString.copyFrom(metaRef.bytes));
 		}
 
 		if (ZuliaQuery.FetchType.FULL.equals(resultFetchType)) {
@@ -688,7 +683,7 @@ private ZuliaBase.ResultDocument filterDocument(ZuliaBase.ResultDocument rd, Col
 
 		filterDocument(fieldsToReturn, fieldsToMask, mongoDocument);
 
-		ByteString document = ByteString.copyFrom(ZuliaUtil.mongoDocumentToByteArray(mongoDocument));
+		ByteString document = ZuliaUtil.mongoDocumentToByteString(mongoDocument);
 		resultDocBuilder.setDocument(document);
 
 		return resultDocBuilder.build();