S3 Storage Patch (#66)

* Patch to make the S3 associated document fetch calls recognize the fetchType argument difference between META and FULL
zuliaio · May 10, 2022 · 0955a51 · 0955a51
1 parent 0dec36d
commit 0955a51
Showing 1 changed file with 36 additions and 14 deletions.
diff --git a/zulia-server/src/main/java/io/zulia/server/filestorage/S3DocumentStorage.java b/zulia-server/src/main/java/io/zulia/server/filestorage/S3DocumentStorage.java
@@ -123,26 +123,35 @@ public void storeAssociatedDocument(AssociatedDocument doc) throws Exception {
 	public List<AssociatedDocument> getAssociatedDocuments(String uniqueId, FetchType fetchType) throws Exception {
 		if (FetchType.NONE.equals(fetchType))
 			return Collections.emptyList();
-		FindIterable<Document> found = client.getDatabase(dbName).getCollection(COLLECTION).find(Filters.eq("metadata." + DOCUMENT_UNIQUE_ID_KEY, uniqueId));
 
+		FindIterable<Document> found = client.getDatabase(dbName).getCollection(COLLECTION).find(Filters.eq("metadata." + DOCUMENT_UNIQUE_ID_KEY, uniqueId));
 		List<AssociatedDocument> docs = new ArrayList<>();
-		for (Document doc : found) {
-			docs.add(parseTOC(doc));
+
+		//Have to do it this way because the FindIterable does not implement the streams API.
+		if (FetchType.META.equals(fetchType)) {
+			for (Document doc : found) {
+				docs.add(buildMetadataDocument(doc));
+			}
+		} else if (FetchType.FULL.equals(fetchType)) {
+			for (Document doc : found) {
+				docs.add(buildFullDocument(doc));
+			}
+		} else {
+			return Collections.emptyList();
 		}
+
 		return docs;
 	}
 
 	@Override
 	public AssociatedDocument getAssociatedDocument(String uniqueId, String filename, FetchType fetchType) throws Exception {
-		if (!FetchType.NONE.equals(fetchType)) {
-			String uid = String.join("-", uniqueId, filename);
-			FindIterable<Document> found = client.getDatabase(dbName).getCollection(COLLECTION).find(Filters.eq("metadata." + FILE_UNIQUE_ID_KEY, uid));
-			Document doc = found.first();
-			if (null != doc) {
-				return parseTOC(doc);
-			}
-		}
-		return null;
+		String uid = String.join("-", uniqueId, filename);
+
+		return switch (fetchType) {
+			case NONE, UNRECOGNIZED -> null;
+			case META -> buildMetadataDocument(client.getDatabase(dbName).getCollection(COLLECTION).find(Filters.eq("metadata." + FILE_UNIQUE_ID_KEY, uid)).first());
+			case FULL -> buildFullDocument(client.getDatabase(dbName).getCollection(COLLECTION).find(Filters.eq("metadata." + FILE_UNIQUE_ID_KEY, uid)).first());
+		};
 	}
 
 	@Override
@@ -316,7 +325,7 @@ private Document parseAssociated(AssociatedDocument doc, Long length) {
 		return TOC;
 	}
 
-	private AssociatedDocument parseTOC(Document doc) throws IOException {
+	private AssociatedDocument.Builder parseMongo(Document doc) {
 		AssociatedDocument.Builder aBuilder = AssociatedDocument.newBuilder();
 		aBuilder.setFilename(doc.getString(FILENAME));
 
@@ -329,6 +338,10 @@ private AssociatedDocument parseTOC(Document doc) throws IOException {
 		meta.remove(FILE_UNIQUE_ID_KEY);
 		aBuilder.setMetadata(ZuliaUtil.mongoDocumentToByteString(meta));
 
+		return aBuilder;
+	}
+
+	private void addFileContents(AssociatedDocument.Builder aBuilder, Document doc) throws IOException {
 		Document s3Info = doc.get("s3", Document.class);
 		GetObjectRequest gor = GetObjectRequest.builder()
 				.bucket(s3Info.getString("bucket"))
@@ -339,7 +352,16 @@ private AssociatedDocument parseTOC(Document doc) throws IOException {
 		try (compression) {
 			aBuilder.setDocument(ByteString.readFrom(compression));
 		}
+	}
+
+	private AssociatedDocument buildMetadataDocument(Document doc) {
+		AssociatedDocument.Builder builder = parseMongo(doc);
+		return builder.build();
+	}
 
-		return aBuilder.build();
+	private AssociatedDocument buildFullDocument(Document doc) throws IOException {
+		AssociatedDocument.Builder builder = parseMongo(doc);
+		addFileContents(builder, doc);
+		return builder.build();
 	}
 }