From 1856090a87e86b64f87aff81f2be4bc3879fb4b9 Mon Sep 17 00:00:00 2001 From: Varun Bansal Date: Tue, 4 Apr 2023 15:36:43 +0530 Subject: [PATCH] validate checksum in footer separately for segment metadata files (#6931) Signed-off-by: Varun Bansal --- .../opensearch/common/io/VersionedCodecStreamWrapper.java | 8 +++----- .../index/store/RemoteSegmentStoreDirectory.java | 5 ++++- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/org/opensearch/common/io/VersionedCodecStreamWrapper.java b/server/src/main/java/org/opensearch/common/io/VersionedCodecStreamWrapper.java index 9a1a951b0796e..ff0af3954a3a3 100644 --- a/server/src/main/java/org/opensearch/common/io/VersionedCodecStreamWrapper.java +++ b/server/src/main/java/org/opensearch/common/io/VersionedCodecStreamWrapper.java @@ -11,7 +11,6 @@ import java.io.IOException; import org.apache.lucene.codecs.CodecUtil; -import org.apache.lucene.store.BufferedChecksumIndexInput; import org.apache.lucene.store.ChecksumIndexInput; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; @@ -47,10 +46,9 @@ public VersionedCodecStreamWrapper(IndexIOStreamHandler indexIOStreamHandler, * @return stream content parsed into {@link T} */ public T readStream(IndexInput indexInput) throws IOException { - ChecksumIndexInput checksumIndexInput = new BufferedChecksumIndexInput(indexInput); - int readStreamVersion = checkHeader(checksumIndexInput); - T content = getHandlerForVersion(readStreamVersion).readContent(checksumIndexInput); - checkFooter(checksumIndexInput); + CodecUtil.checksumEntireFile(indexInput); + int readStreamVersion = checkHeader(indexInput); + T content = getHandlerForVersion(readStreamVersion).readContent(indexInput); return content; } diff --git a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java index a0bd32403bd39..095f1c01792e6 100644 --- a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java +++ b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java @@ -17,6 +17,7 @@ import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; import org.opensearch.common.UUIDs; +import org.opensearch.common.lucene.store.ByteArrayIndexInput; import org.opensearch.index.store.remote.metadata.RemoteSegmentMetadata; import org.opensearch.common.io.VersionedCodecStreamWrapper; import org.opensearch.index.store.remote.metadata.RemoteSegmentMetadataHandler; @@ -135,7 +136,9 @@ private Map readLatestMetadataFile() throws IOE private Map readMetadataFile(String metadataFilename) throws IOException { try (IndexInput indexInput = remoteMetadataDirectory.openInput(metadataFilename, IOContext.DEFAULT)) { - RemoteSegmentMetadata metadata = metadataStreamWrapper.readStream(indexInput); + byte[] metadataBytes = new byte[(int) indexInput.length()]; + indexInput.readBytes(metadataBytes, 0, (int) indexInput.length()); + RemoteSegmentMetadata metadata = metadataStreamWrapper.readStream(new ByteArrayIndexInput(metadataFilename, metadataBytes)); return metadata.getMetadata(); } }