diff --git a/zulia-client/build.gradle.kts b/zulia-client/build.gradle.kts new file mode 100644 index 00000000..964a7556 --- /dev/null +++ b/zulia-client/build.gradle.kts @@ -0,0 +1,8 @@ +apply(from = "../maven.gradle") + +description = "Zulia Client" + +dependencies { + compile(project(":zulia-common")) +} + diff --git a/zulia-client/src/main/java/io/zulia/client/ZuliaRESTClient.java b/zulia-client/src/main/java/io/zulia/client/ZuliaRESTClient.java index 3a205be7..5550bc44 100644 --- a/zulia-client/src/main/java/io/zulia/client/ZuliaRESTClient.java +++ b/zulia-client/src/main/java/io/zulia/client/ZuliaRESTClient.java @@ -75,7 +75,7 @@ public void storeAssociated(String uniqueId, String indexName, String fileName, parameters.put(ZuliaConstants.INDEX, indexName); if (meta != null) { ArrayList list = new ArrayList<>(); - parameters.put(ZuliaConstants.META, list); + parameters.put(ZuliaConstants.META_JSON, list); for (String key : meta.keySet()) { String value = meta.get(key); list.add(key + ":" + value); diff --git a/zulia-client/src/main/java/io/zulia/client/result/AssociatedResult.java b/zulia-client/src/main/java/io/zulia/client/result/AssociatedResult.java index 4a992a17..dd68557a 100644 --- a/zulia-client/src/main/java/io/zulia/client/result/AssociatedResult.java +++ b/zulia-client/src/main/java/io/zulia/client/result/AssociatedResult.java @@ -1,10 +1,9 @@ package io.zulia.client.result; -import java.util.HashMap; -import java.util.Map; +import io.zulia.util.ZuliaUtil; +import org.bson.Document; import static io.zulia.message.ZuliaBase.AssociatedDocument; -import static io.zulia.message.ZuliaBase.Metadata; public class AssociatedResult { @@ -14,12 +13,8 @@ public AssociatedResult(AssociatedDocument associatedDocument) { this.associatedDocument = associatedDocument; } - public Map getMeta() { - HashMap metadata = new HashMap(); - for (Metadata md : associatedDocument.getMetadataList()) { - metadata.put(md.getKey(), md.getValue()); - } - return metadata; + public Document getMeta() { + return ZuliaUtil.byteArrayToMongoDocument(associatedDocument.getMetadata().toByteArray()); } public String getFilename() { @@ -39,8 +34,7 @@ public byte[] getDocumentAsBytes() { public String getDocumentAsUtf8() { if (hasDocument()) { - String contents = associatedDocument.getDocument().toStringUtf8(); - return contents; + return associatedDocument.getDocument().toStringUtf8(); } return null; } diff --git a/zulia-client/src/main/java/io/zulia/client/result/FetchResult.java b/zulia-client/src/main/java/io/zulia/client/result/FetchResult.java index d8d78d7a..7a72b2fa 100644 --- a/zulia-client/src/main/java/io/zulia/client/result/FetchResult.java +++ b/zulia-client/src/main/java/io/zulia/client/result/FetchResult.java @@ -7,12 +7,9 @@ import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.List; -import java.util.Map; import static io.zulia.message.ZuliaBase.AssociatedDocument; -import static io.zulia.message.ZuliaBase.Metadata; import static io.zulia.message.ZuliaServiceOuterClass.FetchResponse; public class FetchResult extends Result { @@ -53,14 +50,10 @@ public String getIndexName() { return null; } - public Map getMeta() { + public Document getMeta() { if (fetchResponse.hasResultDocument()) { ResultDocument rd = fetchResponse.getResultDocument(); - HashMap metadata = new HashMap<>(); - for (Metadata md : rd.getMetadataList()) { - metadata.put(md.getKey(), md.getValue()); - } - return metadata; + return ZuliaUtil.byteStringToMongoDocument(rd.getMetadata()); } return null; } diff --git a/zulia-client/src/main/java/io/zulia/doc/AssociatedBuilder.java b/zulia-client/src/main/java/io/zulia/doc/AssociatedBuilder.java index 607e4ea1..dee6f8c4 100644 --- a/zulia-client/src/main/java/io/zulia/doc/AssociatedBuilder.java +++ b/zulia-client/src/main/java/io/zulia/doc/AssociatedBuilder.java @@ -2,7 +2,6 @@ import com.google.protobuf.ByteString; import io.zulia.message.ZuliaBase.AssociatedDocument; -import io.zulia.message.ZuliaBase.Metadata; import io.zulia.util.ZuliaUtil; import org.bson.Document; @@ -44,17 +43,17 @@ public AssociatedBuilder setDocument(byte[] bytes) { } public AssociatedBuilder setDocument(Document document) { - adBuilder.setDocument(ByteString.copyFrom(ZuliaUtil.mongoDocumentToByteArray(document))); + adBuilder.setDocument(ZuliaUtil.mongoDocumentToByteString(document)); return this; } - public AssociatedBuilder addMetaData(String key, String value) { - adBuilder.addMetadata(Metadata.newBuilder().setKey(key).setValue(value)); - return this; - } - - public AssociatedBuilder clearMetaData() { - adBuilder.clearMetadata(); + public AssociatedBuilder setMetadata(Document metadata) { + if (metadata != null) { + adBuilder.setMetadata(ZuliaUtil.mongoDocumentToByteString(metadata)); + } + else { + adBuilder.clearMetadata(); + } return this; } diff --git a/zulia-client/src/main/java/io/zulia/doc/ResultDocBuilder.java b/zulia-client/src/main/java/io/zulia/doc/ResultDocBuilder.java index 5295d8a3..f37bed49 100644 --- a/zulia-client/src/main/java/io/zulia/doc/ResultDocBuilder.java +++ b/zulia-client/src/main/java/io/zulia/doc/ResultDocBuilder.java @@ -1,13 +1,9 @@ package io.zulia.doc; -import com.google.protobuf.ByteString; -import io.zulia.message.ZuliaBase.Metadata; import io.zulia.message.ZuliaBase.ResultDocument; import io.zulia.util.ZuliaUtil; import org.bson.Document; -import java.util.HashMap; - public class ResultDocBuilder { private ResultDocument.Builder resultDocumentBuilder; @@ -19,20 +15,18 @@ public ResultDocBuilder() { resultDocumentBuilder = ResultDocument.newBuilder(); } - public ResultDocBuilder addMetaData(String key, String value) { - resultDocumentBuilder.addMetadata(Metadata.newBuilder().setKey(key).setValue(value)); - return this; - } - - public ResultDocBuilder addAllMetaData(HashMap metadata) { - for (String key : metadata.keySet()) { - resultDocumentBuilder.addMetadata(Metadata.newBuilder().setKey(key).setValue(metadata.get(key))); + public ResultDocBuilder setMetadata(Document metadata) { + if (metadata != null) { + resultDocumentBuilder.setMetadata(ZuliaUtil.mongoDocumentToByteString(metadata)); + } + else { + resultDocumentBuilder.clearMetadata(); } return this; } public ResultDocBuilder setDocument(Document document) { - resultDocumentBuilder.setDocument(ByteString.copyFrom(ZuliaUtil.mongoDocumentToByteArray(document))); + resultDocumentBuilder.setDocument(ZuliaUtil.mongoDocumentToByteString(document)); return this; } diff --git a/zulia-common/src/main/java/io/zulia/ZuliaConstants.java b/zulia-common/src/main/java/io/zulia/ZuliaConstants.java index 70e93d40..5bd6a93a 100644 --- a/zulia-common/src/main/java/io/zulia/ZuliaConstants.java +++ b/zulia-common/src/main/java/io/zulia/ZuliaConstants.java @@ -35,7 +35,7 @@ public interface ZuliaConstants { String ROWS = "rows"; String ID = "id"; String FILE_NAME = "fileName"; - String META = "meta"; + String META_JSON = "metaJson"; String INDEX = "index"; String FACET = "facet"; String SORT = "sort"; diff --git a/zulia-common/src/main/java/io/zulia/util/ZuliaUtil.java b/zulia-common/src/main/java/io/zulia/util/ZuliaUtil.java index d865ae96..4b9c8dd2 100644 --- a/zulia-common/src/main/java/io/zulia/util/ZuliaUtil.java +++ b/zulia-common/src/main/java/io/zulia/util/ZuliaUtil.java @@ -1,5 +1,6 @@ package io.zulia.util; +import com.google.protobuf.ByteString; import org.bson.BsonBinaryReader; import org.bson.BsonBinaryWriter; import org.bson.Document; @@ -51,10 +52,24 @@ public static byte[] mongoDocumentToByteArray(Document mongoDocument) { return outputBuffer.toByteArray(); } + public static ByteString mongoDocumentToByteString(Document mongoDocument) { + return ByteString.copyFrom(mongoDocumentToByteArray(mongoDocument)); + } + + public static Document byteStringToMongoDocument(ByteString bytes) { + if (bytes != null) { + return byteArrayToMongoDocument(bytes.toByteArray()); + } + return new Document(); + } + public static Document byteArrayToMongoDocument(byte[] byteArray) { - BsonBinaryReader bsonReader = new BsonBinaryReader(ByteBuffer.wrap(byteArray)); + if (byteArray != null && byteArray.length != 0) { + BsonBinaryReader bsonReader = new BsonBinaryReader(ByteBuffer.wrap(byteArray)); - return documentCodec.decode(bsonReader, DecoderContext.builder().build()); + return documentCodec.decode(bsonReader, DecoderContext.builder().build()); + } + return new Document(); } public static int computeLevenshteinDistance(String string1, String string2) { diff --git a/zulia-common/src/main/proto/zulia_base.proto b/zulia-common/src/main/proto/zulia_base.proto index 0d785eee..73cd250a 100644 --- a/zulia-common/src/main/proto/zulia_base.proto +++ b/zulia-common/src/main/proto/zulia_base.proto @@ -19,7 +19,7 @@ message ResultDocument { string uniqueId = 1; string indexName = 2; bytes document = 3; - repeated Metadata metadata = 4; + bytes metadata = 4; int64 timestamp = 5; } @@ -29,14 +29,10 @@ message AssociatedDocument { string documentUniqueId = 2; string indexName = 3; bytes document = 4; - repeated Metadata metadata = 5; + bytes metadata = 5; int64 timestamp = 6; } -message Metadata { - string key = 1; - string value = 2; -} message ShardCountResponse { uint32 shardNumber = 1; diff --git a/zulia-server/src/main/java/io/zulia/server/filestorage/DocumentStorage.java b/zulia-server/src/main/java/io/zulia/server/filestorage/DocumentStorage.java index ec35bb33..3d458e5b 100644 --- a/zulia-server/src/main/java/io/zulia/server/filestorage/DocumentStorage.java +++ b/zulia-server/src/main/java/io/zulia/server/filestorage/DocumentStorage.java @@ -8,7 +8,6 @@ import java.io.InputStream; import java.io.OutputStream; import java.util.List; -import java.util.Map; public interface DocumentStorage { @@ -20,7 +19,7 @@ public interface DocumentStorage { void getAssociatedDocuments(OutputStream outputstream, Document filter) throws IOException; - void storeAssociatedDocument(String uniqueId, String fileName, InputStream is, long timestamp, Map metadataMap) throws Exception; + void storeAssociatedDocument(String uniqueId, String fileName, InputStream is, long timestamp, Document metadataMap) throws Exception; InputStream getAssociatedDocumentStream(String uniqueId, String filename); diff --git a/zulia-server/src/main/java/io/zulia/server/filestorage/FileDocumentStorage.java b/zulia-server/src/main/java/io/zulia/server/filestorage/FileDocumentStorage.java index b1173ca3..0bbb80ee 100644 --- a/zulia-server/src/main/java/io/zulia/server/filestorage/FileDocumentStorage.java +++ b/zulia-server/src/main/java/io/zulia/server/filestorage/FileDocumentStorage.java @@ -9,7 +9,6 @@ import java.io.InputStream; import java.io.OutputStream; import java.util.List; -import java.util.Map; import java.util.logging.Logger; public class FileDocumentStorage implements DocumentStorage { @@ -41,7 +40,7 @@ public void getAssociatedDocuments(OutputStream outputstream, Document filter) t } @Override - public void storeAssociatedDocument(String uniqueId, String fileName, InputStream is, long timestamp, Map metadataMap) throws Exception { + public void storeAssociatedDocument(String uniqueId, String fileName, InputStream is, long timestamp, Document metadataMap) throws Exception { throw new RuntimeException("Not implemented"); } diff --git a/zulia-server/src/main/java/io/zulia/server/filestorage/MongoDocumentStorage.java b/zulia-server/src/main/java/io/zulia/server/filestorage/MongoDocumentStorage.java index 0b82b7fc..da12df3b 100644 --- a/zulia-server/src/main/java/io/zulia/server/filestorage/MongoDocumentStorage.java +++ b/zulia-server/src/main/java/io/zulia/server/filestorage/MongoDocumentStorage.java @@ -13,8 +13,8 @@ import com.mongodb.client.gridfs.model.GridFSUploadOptions; import com.mongodb.client.model.IndexOptions; import io.zulia.message.ZuliaBase.AssociatedDocument; -import io.zulia.message.ZuliaBase.Metadata; import io.zulia.message.ZuliaQuery.FetchType; +import io.zulia.util.ZuliaUtil; import org.bson.Document; import java.io.ByteArrayInputStream; @@ -25,9 +25,7 @@ import java.nio.charset.Charset; import java.util.ArrayList; import java.util.Date; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.function.Consumer; import java.util.logging.Logger; @@ -100,21 +98,19 @@ public void drop() { } @Override - public void storeAssociatedDocument(String uniqueId, String fileName, InputStream is, long timestamp, Map metadataMap) throws Exception { + public void storeAssociatedDocument(String uniqueId, String fileName, InputStream is, long timestamp, Document metadata) throws Exception { GridFSBucket gridFS = createGridFSConnection(); deleteAssociatedDocument(uniqueId, fileName); - GridFSUploadOptions gridFSUploadOptions = getGridFSUploadOptions(uniqueId, fileName, timestamp, metadataMap); + GridFSUploadOptions gridFSUploadOptions = getGridFSUploadOptions(uniqueId, fileName, timestamp, metadata); gridFS.uploadFromStream(fileName, is, gridFSUploadOptions); } - private GridFSUploadOptions getGridFSUploadOptions(String uniqueId, String fileName, long timestamp, Map metadataMap) { - Document metadata = new Document(); - if (metadataMap != null) { - for (String key : metadataMap.keySet()) { - metadata.put(key, metadataMap.get(key)); - } + private GridFSUploadOptions getGridFSUploadOptions(String uniqueId, String fileName, long timestamp, Document metadata) { + + if (metadata == null) { + metadata = new Document(); } metadata.put(TIMESTAMP, timestamp); metadata.put(DOCUMENT_UNIQUE_ID_KEY, uniqueId); @@ -130,9 +126,12 @@ public void storeAssociatedDocument(AssociatedDocument doc) throws Exception { ByteArrayInputStream byteInputStream = new ByteArrayInputStream(bytes); - Map metadata = new HashMap<>(); - for (Metadata meta : doc.getMetadataList()) { - metadata.put(meta.getKey(), meta.getValue()); + Document metadata; + if (doc.getMetadata() != null) { + metadata = ZuliaUtil.byteArrayToMongoDocument(doc.getMetadata().toByteArray()); + } + else { + metadata = new Document(); } storeAssociatedDocument(doc.getDocumentUniqueId(), doc.getFilename(), byteInputStream, doc.getTimestamp(), metadata); @@ -196,9 +195,7 @@ private AssociatedDocument loadGridFSToAssociatedDocument(GridFSBucket gridFS, G aBuilder.setTimestamp(timestamp); aBuilder.setDocumentUniqueId((String) metadata.remove(DOCUMENT_UNIQUE_ID_KEY)); - for (String field : metadata.keySet()) { - aBuilder.addMetadata(Metadata.newBuilder().setKey(field).setValue((String) metadata.get(field))); - } + aBuilder.setMetadata(ZuliaUtil.mongoDocumentToByteString(metadata)); if (FetchType.FULL.equals(fetchType)) { diff --git a/zulia-server/src/main/java/io/zulia/server/index/ShardDocumentIndexer.java b/zulia-server/src/main/java/io/zulia/server/index/ShardDocumentIndexer.java index ebdfb9ed..e6b6afcd 100644 --- a/zulia-server/src/main/java/io/zulia/server/index/ShardDocumentIndexer.java +++ b/zulia-server/src/main/java/io/zulia/server/index/ShardDocumentIndexer.java @@ -2,7 +2,6 @@ import info.debatty.java.lsh.SuperBit; import io.zulia.ZuliaConstants; -import io.zulia.message.ZuliaBase; import io.zulia.message.ZuliaIndex; import io.zulia.server.config.ServerIndexConfig; import io.zulia.server.index.field.BooleanFieldIndexer; @@ -42,7 +41,7 @@ public ShardDocumentIndexer(ServerIndexConfig indexConfig) { } - public Document getIndexDocument(String uniqueId, long timestamp, org.bson.Document mongoDocument, List metadataList) throws Exception { + public Document getIndexDocument(String uniqueId, long timestamp, org.bson.Document mongoDocument, org.bson.Document metadata) throws Exception { Document luceneDocument = new Document(); addStoredFieldsForDocument(mongoDocument, luceneDocument); @@ -54,13 +53,7 @@ public Document getIndexDocument(String uniqueId, long timestamp, org.bson.Docum luceneDocument.add(new StoredField(ZuliaConstants.STORED_DOC_FIELD, new BytesRef(ZuliaUtil.mongoDocumentToByteArray(mongoDocument)))); - org.bson.Document metadataMongoDoc = new org.bson.Document(); - - for (ZuliaBase.Metadata metadata : metadataList) { - metadataMongoDoc.put(metadata.getKey(), metadata.getValue()); - } - - luceneDocument.add(new StoredField(ZuliaConstants.STORED_META_FIELD, new BytesRef(ZuliaUtil.mongoDocumentToByteArray(metadataMongoDoc)))); + luceneDocument.add(new StoredField(ZuliaConstants.STORED_META_FIELD, new BytesRef(ZuliaUtil.mongoDocumentToByteArray(metadata)))); return luceneDocument; diff --git a/zulia-server/src/main/java/io/zulia/server/index/ShardReader.java b/zulia-server/src/main/java/io/zulia/server/index/ShardReader.java index 88d8bbcd..eb05f09f 100644 --- a/zulia-server/src/main/java/io/zulia/server/index/ShardReader.java +++ b/zulia-server/src/main/java/io/zulia/server/index/ShardReader.java @@ -526,12 +526,7 @@ private void handleStoredDoc(ZuliaQuery.ScoredResult.Builder srBuilder, String u if (ZuliaQuery.FetchType.FULL.equals(resultFetchType) || ZuliaQuery.FetchType.META.equals(resultFetchType)) { BytesRef metaRef = d.getBinaryValue(ZuliaConstants.STORED_META_FIELD); - org.bson.Document metaMongoDoc = new org.bson.Document(); - metaMongoDoc.putAll(ZuliaUtil.byteArrayToMongoDocument(metaRef.bytes)); - - for (String key : metaMongoDoc.keySet()) { - rdBuilder.addMetadata(ZuliaBase.Metadata.newBuilder().setKey(key).setValue(((String) metaMongoDoc.get(key)))); - } + rdBuilder.setMetadata(ByteString.copyFrom(metaRef.bytes)); } if (ZuliaQuery.FetchType.FULL.equals(resultFetchType)) { @@ -688,7 +683,7 @@ private ZuliaBase.ResultDocument filterDocument(ZuliaBase.ResultDocument rd, Col filterDocument(fieldsToReturn, fieldsToMask, mongoDocument); - ByteString document = ByteString.copyFrom(ZuliaUtil.mongoDocumentToByteArray(mongoDocument)); + ByteString document = ZuliaUtil.mongoDocumentToByteString(mongoDocument); resultDocBuilder.setDocument(document); return resultDocBuilder.build(); diff --git a/zulia-server/src/main/java/io/zulia/server/index/ShardWriteManager.java b/zulia-server/src/main/java/io/zulia/server/index/ShardWriteManager.java index 4f6b7f24..c88d33e8 100644 --- a/zulia-server/src/main/java/io/zulia/server/index/ShardWriteManager.java +++ b/zulia-server/src/main/java/io/zulia/server/index/ShardWriteManager.java @@ -1,7 +1,6 @@ package io.zulia.server.index; import io.zulia.ZuliaConstants; -import io.zulia.message.ZuliaBase.Metadata; import io.zulia.server.analysis.ZuliaPerFieldAnalyzer; import io.zulia.server.config.ServerIndexConfig; import org.apache.lucene.document.Document; @@ -19,7 +18,6 @@ import java.io.IOException; import java.nio.file.Path; -import java.util.List; import java.util.concurrent.atomic.AtomicLong; import java.util.logging.Logger; @@ -180,8 +178,8 @@ public FacetsConfig getFacetsConfig() { return facetsConfig; } - public void indexDocument(String uniqueId, long timestamp, org.bson.Document mongoDocument, List metadataList) throws Exception { - Document luceneDocument = shardDocumentIndexer.getIndexDocument(uniqueId, timestamp, mongoDocument, metadataList); + public void indexDocument(String uniqueId, long timestamp, org.bson.Document mongoDocument, org.bson.Document metadata) throws Exception { + Document luceneDocument = shardDocumentIndexer.getIndexDocument(uniqueId, timestamp, mongoDocument, metadata); luceneDocument = facetsConfig.build(taxoWriter, luceneDocument); Term updateQuery = new Term(ZuliaConstants.ID_FIELD, uniqueId); indexWriter.updateDocument(updateQuery, luceneDocument); diff --git a/zulia-server/src/main/java/io/zulia/server/index/ZuliaIndex.java b/zulia-server/src/main/java/io/zulia/server/index/ZuliaIndex.java index 1c202c92..aa71ec4d 100644 --- a/zulia-server/src/main/java/io/zulia/server/index/ZuliaIndex.java +++ b/zulia-server/src/main/java/io/zulia/server/index/ZuliaIndex.java @@ -284,9 +284,16 @@ public StoreResponse internalStore(StoreRequest storeRequest) throws Exception { else { document = new Document(); } + Document metadata; + if (resultDocument.getMetadata() != null) { + metadata = ZuliaUtil.byteArrayToMongoDocument(resultDocument.getMetadata().toByteArray()); + } + else { + metadata = new Document(); + } ZuliaShard s = findShardFromUniqueId(uniqueId); - s.index(uniqueId, timestamp, document, resultDocument.getMetadataList()); + s.index(uniqueId, timestamp, document, metadata); } @@ -691,9 +698,7 @@ public GetNumberOfDocsResponse getNumberOfDocs(InternalGetNumberOfDocsRequest re GetNumberOfDocsResponse.Builder responseBuilder = GetNumberOfDocsResponse.newBuilder(); responseBuilder.setNumberOfDocs(0); - for (Future response : responses) - - { + for (Future response : responses) { try { ShardCountResponse scr = response.get(); responseBuilder.addShardCountResponse(scr); @@ -874,14 +879,13 @@ public InternalGetTermsResponse getTerms(final InternalGetTermsRequest request) } - public void storeAssociatedDocument(String uniqueId, String fileName, InputStream is, long clusterTime, HashMap metadataMap) - throws Exception { + public void storeAssociatedDocument(String uniqueId, String fileName, InputStream is, long clusterTime, Document metadata) throws Exception { - documentStorage.storeAssociatedDocument(uniqueId, fileName, is, clusterTime, metadataMap); + documentStorage.storeAssociatedDocument(uniqueId, fileName, is, clusterTime, metadata); } - public InputStream getAssociatedDocumentStream(String uniqueId, String fileName) throws IOException { + public InputStream getAssociatedDocumentStream(String uniqueId, String fileName) { return documentStorage.getAssociatedDocumentStream(uniqueId, fileName); diff --git a/zulia-server/src/main/java/io/zulia/server/index/ZuliaIndexManager.java b/zulia-server/src/main/java/io/zulia/server/index/ZuliaIndexManager.java index 0c582f6f..761d0015 100644 --- a/zulia-server/src/main/java/io/zulia/server/index/ZuliaIndexManager.java +++ b/zulia-server/src/main/java/io/zulia/server/index/ZuliaIndexManager.java @@ -198,11 +198,11 @@ public InputStream getAssociatedDocumentStream(String indexName, String uniqueId return i.getAssociatedDocumentStream(uniqueId, fileName); } - public void storeAssociatedDocument(String indexName, String uniqueId, String fileName, InputStream is, HashMap metadataMap) + public void storeAssociatedDocument(String indexName, String uniqueId, String fileName, InputStream is, Document metadata) throws Exception { ZuliaIndex i = getIndexFromName(indexName); long timestamp = System.currentTimeMillis(); - i.storeAssociatedDocument(uniqueId, fileName, is, timestamp, metadataMap); + i.storeAssociatedDocument(uniqueId, fileName, is, timestamp, metadata); } diff --git a/zulia-server/src/main/java/io/zulia/server/index/ZuliaShard.java b/zulia-server/src/main/java/io/zulia/server/index/ZuliaShard.java index 902cb793..b557509a 100644 --- a/zulia-server/src/main/java/io/zulia/server/index/ZuliaShard.java +++ b/zulia-server/src/main/java/io/zulia/server/index/ZuliaShard.java @@ -1,7 +1,6 @@ package io.zulia.server.index; import io.zulia.message.ZuliaBase; -import io.zulia.message.ZuliaBase.Metadata; import io.zulia.message.ZuliaBase.ShardCountResponse; import io.zulia.message.ZuliaBase.Similarity; import io.zulia.message.ZuliaQuery; @@ -73,7 +72,6 @@ public ShardQueryResponse queryShard(Query query, Map simila } } - public void forceCommit() throws IOException { if (!primary) { throw new IllegalStateException("Cannot force commit from replica: index <" + indexName + "> shard <" + shardNumber + ">"); @@ -95,12 +93,12 @@ public void close() throws IOException { shardWriteManager.close(); } - public void index(String uniqueId, long timestamp, org.bson.Document mongoDocument, List metadataList) throws Exception { + public void index(String uniqueId, long timestamp, org.bson.Document mongoDocument, org.bson.Document metadata) throws Exception { if (!primary) { throw new IllegalStateException("Cannot index document <" + uniqueId + "> from replica: index <" + indexName + "> shard <" + shardNumber + ">"); } - shardWriteManager.indexDocument(uniqueId, timestamp, mongoDocument, metadataList); + shardWriteManager.indexDocument(uniqueId, timestamp, mongoDocument, metadata); if (shardWriteManager.markedChangedCheckIfCommitNeeded()) { forceCommit(); } diff --git a/zulia-server/src/main/java/io/zulia/server/rest/AssociatedResource.java b/zulia-server/src/main/java/io/zulia/server/rest/AssociatedResource.java index 2c30f867..e35cc085 100644 --- a/zulia-server/src/main/java/io/zulia/server/rest/AssociatedResource.java +++ b/zulia-server/src/main/java/io/zulia/server/rest/AssociatedResource.java @@ -16,8 +16,6 @@ import javax.ws.rs.core.Response; import javax.ws.rs.core.StreamingOutput; import java.io.InputStream; -import java.util.HashMap; -import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; @@ -65,27 +63,20 @@ public Response get(@Context Response response, @QueryParam(ZuliaConstants.ID) f @POST @Produces({ MediaType.TEXT_XML }) public Response post(@QueryParam(ZuliaConstants.ID) String uniqueId, @QueryParam(ZuliaConstants.FILE_NAME) String fileName, - @QueryParam(ZuliaConstants.INDEX) String indexName, @QueryParam(ZuliaConstants.META) List meta, InputStream is) { + @QueryParam(ZuliaConstants.INDEX) String indexName, @QueryParam(ZuliaConstants.META_JSON) String metaJson, InputStream is) { if (uniqueId != null && fileName != null && indexName != null) { - HashMap metaMap = new HashMap<>(); - if (meta != null) { - for (String m : meta) { - int colonIndex = m.indexOf(":"); - if (colonIndex != -1) { - String key = m.substring(0, colonIndex); - String value = m.substring(colonIndex + 1).trim(); - metaMap.put(key, value); - } - else { - throw new WebApplicationException("Meta must be in the form key:value"); - } - } + Document metadata; + if (metaJson != null) { + metadata = Document.parse(metaJson); + } + else { + metadata = new Document(); } try { - indexManager.storeAssociatedDocument(indexName, uniqueId, fileName, is, metaMap); + indexManager.storeAssociatedDocument(indexName, uniqueId, fileName, is, metadata); return Response.status(ZuliaConstants.SUCCESS) .entity("Stored associated document with uniqueId <" + uniqueId + "> and fileName <" + fileName + ">").build(); diff --git a/zulia-server/src/test/java/io/zulia/server/test/node/StartStopTest.java b/zulia-server/src/test/java/io/zulia/server/test/node/StartStopTest.java index 4702d8b1..fe5ecff9 100644 --- a/zulia-server/src/test/java/io/zulia/server/test/node/StartStopTest.java +++ b/zulia-server/src/test/java/io/zulia/server/test/node/StartStopTest.java @@ -10,7 +10,9 @@ import io.zulia.fields.FieldConfigBuilder; import io.zulia.message.ZuliaIndex.FacetAs.DateHandling; import io.zulia.message.ZuliaIndex.FieldConfig.FieldType; +import io.zulia.message.ZuliaQuery; import io.zulia.message.ZuliaQuery.FacetCount; +import io.zulia.util.ZuliaUtil; import org.bson.Document; import org.testng.Assert; import org.testng.annotations.Test; @@ -77,6 +79,7 @@ public void index() throws Exception { if (half) { // 1/2 of input mongoDocument.put("country", "US"); + } else { // 1/2 of input mongoDocument.put("country", "France"); @@ -97,7 +100,13 @@ else if (half) { // 2/5 of input } Store s = new Store(uniqueId, FACET_TEST_INDEX); - s.setResultDocument(ResultDocBuilder.newBuilder().setDocument(mongoDocument)); + + ResultDocBuilder resultDocumentBuilder = ResultDocBuilder.newBuilder().setDocument(mongoDocument); + if (half) { + resultDocumentBuilder.setMetadata(new Document("test", "someValue")); + } + + s.setResultDocument(resultDocumentBuilder); zuliaWorkPool.store(s); } @@ -210,6 +219,35 @@ public void confirm() throws Exception { } + { + Query q = new Query(FACET_TEST_INDEX, "country:US", 10).setResultFetchType(ZuliaQuery.FetchType.META); + + QueryResult qr = zuliaWorkPool.query(q); + + for (ZuliaQuery.ScoredResult result : qr.getResults()) { + Document metadata = ZuliaUtil.byteStringToMongoDocument(result.getResultDocument().getMetadata()); + Assert.assertEquals(metadata.getString("test"), "someValue"); + } + + Assert.assertEquals(qr.getTotalHits(), totalRecords / 2, "Total record count filtered on half mismatch"); + + } + + { + Query q = new Query(FACET_TEST_INDEX, "country:US", 10).setResultFetchType(ZuliaQuery.FetchType.FULL); + + QueryResult qr = zuliaWorkPool.query(q); + + for (ZuliaQuery.ScoredResult result : qr.getResults()) { + Document metadata = ZuliaUtil.byteStringToMongoDocument(result.getResultDocument().getMetadata()); + Assert.assertEquals(metadata.getString("test"), "someValue"); + } + + Assert.assertEquals(qr.getTotalHits(), totalRecords / 2, "Total record count filtered on half mismatch"); + + } + + } diff --git a/zulia-server/src/test/java/io/zulia/server/test/util/ResultHelperTest.java b/zulia-server/src/test/java/io/zulia/server/test/util/ResultHelperTest.java index 543027e6..2edc8412 100644 --- a/zulia-server/src/test/java/io/zulia/server/test/util/ResultHelperTest.java +++ b/zulia-server/src/test/java/io/zulia/server/test/util/ResultHelperTest.java @@ -1,6 +1,8 @@ package io.zulia.server.test.util; +import com.google.protobuf.ByteString; import io.zulia.util.ResultHelper; +import io.zulia.util.ZuliaUtil; import org.bson.Document; import org.testng.Assert; import org.testng.annotations.Test; @@ -49,4 +51,27 @@ public void testFieldExtraction() throws Exception { Assert.assertEquals(ResultHelper.getValueFromMongoDocument(testMongoDocument, "myfield"), 40); } + + @Test + public void testSerialization() { + Document testMongoDocument = new Document(); + testMongoDocument.put("field1", "someVal"); + testMongoDocument.put("myfield", 40); + + { + byte[] bytes = ZuliaUtil.mongoDocumentToByteArray(testMongoDocument); + Document testMongoDocumentReborn = ZuliaUtil.byteArrayToMongoDocument(bytes); + + Assert.assertEquals(testMongoDocumentReborn.getString("field1"), "someVal"); + Assert.assertEquals((int) testMongoDocumentReborn.getInteger("myfield"), 40); + } + + { + ByteString byteString = ZuliaUtil.mongoDocumentToByteString(testMongoDocument); + Document testMongoDocumentReborn = ZuliaUtil.byteStringToMongoDocument(byteString); + + Assert.assertEquals(testMongoDocumentReborn.getString("field1"), "someVal"); + Assert.assertEquals((int) testMongoDocumentReborn.getInteger("myfield"), 40); + } + } }