From da988b9bb64728bdcbb853c1b2ef809f16fe8073 Mon Sep 17 00:00:00 2001 From: Nikita_Glashchenko Date: Fri, 12 Jul 2019 17:36:53 +0400 Subject: [PATCH 1/3] Remove support for old checkpoint format --- .../index/translog/Checkpoint.java | 52 +++++------------- .../index/translog/TranslogTests.java | 10 ++++ .../org/elasticsearch/index/checkpoint/v1.cpk | Bin 0 -> 48 bytes .../org/elasticsearch/index/checkpoint/v2.cpk | Bin 0 -> 80 bytes 4 files changed, 24 insertions(+), 38 deletions(-) create mode 100644 server/src/test/resources/org/elasticsearch/index/checkpoint/v1.cpk create mode 100644 server/src/test/resources/org/elasticsearch/index/checkpoint/v2.cpk diff --git a/server/src/main/java/org/elasticsearch/index/translog/Checkpoint.java b/server/src/main/java/org/elasticsearch/index/translog/Checkpoint.java index 364203f898cc1..9a3f5b77463be 100644 --- a/server/src/main/java/org/elasticsearch/index/translog/Checkpoint.java +++ b/server/src/main/java/org/elasticsearch/index/translog/Checkpoint.java @@ -47,7 +47,6 @@ final class Checkpoint { final long minTranslogGeneration; final long trimmedAboveSeqNo; - private static final int INITIAL_VERSION = 1; // start with 1, just to recognize there was some magic serialization logic before private static final int VERSION_6_0_0 = 2; // introduction of global checkpoints private static final int CURRENT_VERSION = 3; // introduction of trimmed above seq# @@ -59,10 +58,10 @@ final class Checkpoint { + Integer.BYTES // ops + Long.BYTES // offset + Long.BYTES // generation - + Long.BYTES // minimum sequence number, introduced in 6.0.0 - + Long.BYTES // maximum sequence number, introduced in 6.0.0 - + Long.BYTES // global checkpoint, introduced in 6.0.0 - + Long.BYTES // minimum translog generation in the translog - introduced in 6.0.0 + + Long.BYTES // minimum sequence number + + Long.BYTES // maximum sequence number + + Long.BYTES // global checkpoint + + Long.BYTES // minimum translog generation in the translog + Long.BYTES // maximum reachable (trimmed) sequence number, introduced in 6.4.0 + CodecUtil.footerLength(); @@ -71,17 +70,10 @@ final class Checkpoint { + Integer.BYTES // ops + Long.BYTES // offset + Long.BYTES // generation - + Long.BYTES // minimum sequence number, introduced in 6.0.0 - + Long.BYTES // maximum sequence number, introduced in 6.0.0 - + Long.BYTES // global checkpoint, introduced in 6.0.0 - + Long.BYTES // minimum translog generation in the translog - introduced in 6.0.0 - + CodecUtil.footerLength(); - - // size of 5.0.0 checkpoint - static final int V1_FILE_SIZE = CodecUtil.headerLength(CHECKPOINT_CODEC) - + Integer.BYTES // ops - + Long.BYTES // offset - + Long.BYTES // generation + + Long.BYTES // minimum sequence number + + Long.BYTES // maximum sequence number + + Long.BYTES // global checkpoint + + Long.BYTES // minimum translog generation in the translog + CodecUtil.footerLength(); /** @@ -132,7 +124,7 @@ static Checkpoint emptyTranslogCheckpoint(final long offset, final long generati return new Checkpoint(offset, 0, generation, minSeqNo, maxSeqNo, globalCheckpoint, minTranslogGeneration, trimmedAboveSeqNo); } - static Checkpoint readCheckpointV6_4_0(final DataInput in) throws IOException { + static Checkpoint readCheckpointV3(final DataInput in) throws IOException { final long offset = in.readLong(); final int numOps = in.readInt(); final long generation = in.readLong(); @@ -144,7 +136,7 @@ static Checkpoint readCheckpointV6_4_0(final DataInput in) throws IOException { return new Checkpoint(offset, numOps, generation, minSeqNo, maxSeqNo, globalCheckpoint, minTranslogGeneration, trimmedAboveSeqNo); } - static Checkpoint readCheckpointV6_0_0(final DataInput in) throws IOException { + static Checkpoint readCheckpointV2(final DataInput in) throws IOException { final long offset = in.readLong(); final int numOps = in.readInt(); final long generation = in.readLong(); @@ -156,19 +148,6 @@ static Checkpoint readCheckpointV6_0_0(final DataInput in) throws IOException { return new Checkpoint(offset, numOps, generation, minSeqNo, maxSeqNo, globalCheckpoint, minTranslogGeneration, trimmedAboveSeqNo); } - // reads a checksummed checkpoint introduced in ES 5.0.0 - static Checkpoint readCheckpointV5_0_0(final DataInput in) throws IOException { - final long offset = in.readLong(); - final int numOps = in.readInt(); - final long generation = in.readLong(); - final long minSeqNo = SequenceNumbers.NO_OPS_PERFORMED; - final long maxSeqNo = SequenceNumbers.NO_OPS_PERFORMED; - final long globalCheckpoint = SequenceNumbers.UNASSIGNED_SEQ_NO; - final long minTranslogGeneration = -1; - final long trimmedAboveSeqNo = SequenceNumbers.UNASSIGNED_SEQ_NO; - return new Checkpoint(offset, numOps, generation, minSeqNo, maxSeqNo, globalCheckpoint, minTranslogGeneration, trimmedAboveSeqNo); - } - @Override public String toString() { return "Checkpoint{" + @@ -188,17 +167,14 @@ public static Checkpoint read(Path path) throws IOException { try (IndexInput indexInput = dir.openInput(path.getFileName().toString(), IOContext.DEFAULT)) { // We checksum the entire file before we even go and parse it. If it's corrupted we barf right here. CodecUtil.checksumEntireFile(indexInput); - final int fileVersion = CodecUtil.checkHeader(indexInput, CHECKPOINT_CODEC, INITIAL_VERSION, CURRENT_VERSION); - if (fileVersion == INITIAL_VERSION) { - assert indexInput.length() == V1_FILE_SIZE : indexInput.length(); - return Checkpoint.readCheckpointV5_0_0(indexInput); - } else if (fileVersion == VERSION_6_0_0) { + final int fileVersion = CodecUtil.checkHeader(indexInput, CHECKPOINT_CODEC, VERSION_6_0_0, CURRENT_VERSION); + if (fileVersion == VERSION_6_0_0) { assert indexInput.length() == V2_FILE_SIZE : indexInput.length(); - return Checkpoint.readCheckpointV6_0_0(indexInput); + return Checkpoint.readCheckpointV2(indexInput); } else { assert fileVersion == CURRENT_VERSION : fileVersion; assert indexInput.length() == V3_FILE_SIZE : indexInput.length(); - return Checkpoint.readCheckpointV6_4_0(indexInput); + return Checkpoint.readCheckpointV3(indexInput); } } } diff --git a/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java b/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java index c99fee9dcb8a7..b62120b1ca03b 100644 --- a/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java +++ b/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java @@ -26,6 +26,7 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.TextField; +import org.apache.lucene.index.IndexFormatTooOldException; import org.apache.lucene.index.Term; import org.apache.lucene.mockfile.FilterFileChannel; import org.apache.lucene.mockfile.FilterFileSystemProvider; @@ -2785,6 +2786,15 @@ public void testCheckpointOnDiskFull() throws IOException { assertEquals(read, checkpoint); } + public void testLegacyCheckpointVersion() throws IOException { + expectThrows( + IndexFormatTooOldException.class, + () -> Checkpoint.read(getDataPath("/org/elasticsearch/index/checkpoint/v1.cpk")) + ); + // just check that it doesn't fail + Checkpoint.read(getDataPath("/org/elasticsearch/index/checkpoint/v2.cpk")); + } + /** * Tests that closing views after the translog is fine and we can reopen the translog */ diff --git a/server/src/test/resources/org/elasticsearch/index/checkpoint/v1.cpk b/server/src/test/resources/org/elasticsearch/index/checkpoint/v1.cpk new file mode 100644 index 0000000000000000000000000000000000000000..c3a0a1656cb012412c0bc6fb4a630fc9ed0a9248 GIT binary patch literal 48 zcmcD&o+HkjoL#`cz`*FU^2f6HPHSI>hc=s|H7Z~JzpPktKx6U?pd1*)O#Ipj0J%yM A2><{9 literal 0 HcmV?d00001 diff --git a/server/src/test/resources/org/elasticsearch/index/checkpoint/v2.cpk b/server/src/test/resources/org/elasticsearch/index/checkpoint/v2.cpk new file mode 100644 index 0000000000000000000000000000000000000000..91377cef9d66d905f3fee1e7fc555c957f5359d9 GIT binary patch literal 80 zcmcD&o+HkjoL#`cz`*qOg#Na~urxQOMK2ax-YO2>DSE;+%tQV2CZoI$ Date: Tue, 16 Jul 2019 15:01:36 +0400 Subject: [PATCH 2/3] Fix expected exception in test --- .../java/org/elasticsearch/index/translog/TranslogTests.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java b/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java index 798e93b46c364..50cad733f4413 100644 --- a/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java +++ b/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java @@ -2794,6 +2794,7 @@ public void testCheckpointOnDiskFull() throws IOException { public void testLegacyCheckpointVersion() throws IOException { expectThrows( + TranslogCorruptedException.class, IndexFormatTooOldException.class, () -> Checkpoint.read(getDataPath("/org/elasticsearch/index/checkpoint/v1.cpk")) ); From a4dc57ff08c48f4a69e5108f94389475178ff737 Mon Sep 17 00:00:00 2001 From: Nikita_Glashchenko Date: Fri, 19 Jul 2019 16:28:17 +0400 Subject: [PATCH 3/3] Rename files and improve test --- .../elasticsearch/index/translog/TranslogTests.java | 8 +++++--- .../index/checkpoint/{v1.cpk => v1.ckp.binary} | Bin .../index/checkpoint/{v2.cpk => v2.ckp.binary} | Bin 3 files changed, 5 insertions(+), 3 deletions(-) rename server/src/test/resources/org/elasticsearch/index/checkpoint/{v1.cpk => v1.ckp.binary} (100%) rename server/src/test/resources/org/elasticsearch/index/checkpoint/{v2.cpk => v2.ckp.binary} (100%) diff --git a/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java b/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java index 50cad733f4413..bb3804d3aa49b 100644 --- a/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java +++ b/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java @@ -2796,10 +2796,12 @@ public void testLegacyCheckpointVersion() throws IOException { expectThrows( TranslogCorruptedException.class, IndexFormatTooOldException.class, - () -> Checkpoint.read(getDataPath("/org/elasticsearch/index/checkpoint/v1.cpk")) + () -> Checkpoint.read(getDataPath("/org/elasticsearch/index/checkpoint/v1.ckp.binary")) ); - // just check that it doesn't fail - Checkpoint.read(getDataPath("/org/elasticsearch/index/checkpoint/v2.cpk")); + assertThat(Checkpoint.read(getDataPath("/org/elasticsearch/index/checkpoint/v2.ckp.binary")), + equalTo(new Checkpoint(-1312746831014894010L, 44230819, 4168771208509507653L, 6217263213205155568L, + 8590850694628654668L, 3768575734506660560L, 1476009383806516272L, + SequenceNumbers.UNASSIGNED_SEQ_NO))); } /** diff --git a/server/src/test/resources/org/elasticsearch/index/checkpoint/v1.cpk b/server/src/test/resources/org/elasticsearch/index/checkpoint/v1.ckp.binary similarity index 100% rename from server/src/test/resources/org/elasticsearch/index/checkpoint/v1.cpk rename to server/src/test/resources/org/elasticsearch/index/checkpoint/v1.ckp.binary diff --git a/server/src/test/resources/org/elasticsearch/index/checkpoint/v2.cpk b/server/src/test/resources/org/elasticsearch/index/checkpoint/v2.ckp.binary similarity index 100% rename from server/src/test/resources/org/elasticsearch/index/checkpoint/v2.cpk rename to server/src/test/resources/org/elasticsearch/index/checkpoint/v2.ckp.binary