Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[7.x] Remove support for old translog checkpoint formats #44280

Merged
merged 4 commits into from
Jul 19, 2019
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,6 @@ final class Checkpoint {
final long minTranslogGeneration;
final long trimmedAboveSeqNo;

private static final int INITIAL_VERSION = 1; // start with 1, just to recognize there was some magic serialization logic before
private static final int VERSION_6_0_0 = 2; // introduction of global checkpoints
private static final int CURRENT_VERSION = 3; // introduction of trimmed above seq#

Expand All @@ -63,10 +62,10 @@ final class Checkpoint {
+ Integer.BYTES // ops
+ Long.BYTES // offset
+ Long.BYTES // generation
+ Long.BYTES // minimum sequence number, introduced in 6.0.0
+ Long.BYTES // maximum sequence number, introduced in 6.0.0
+ Long.BYTES // global checkpoint, introduced in 6.0.0
+ Long.BYTES // minimum translog generation in the translog - introduced in 6.0.0
+ Long.BYTES // minimum sequence number
+ Long.BYTES // maximum sequence number
+ Long.BYTES // global checkpoint
+ Long.BYTES // minimum translog generation in the translog
+ Long.BYTES // maximum reachable (trimmed) sequence number, introduced in 6.4.0
+ CodecUtil.footerLength();

Expand All @@ -75,17 +74,10 @@ final class Checkpoint {
+ Integer.BYTES // ops
+ Long.BYTES // offset
+ Long.BYTES // generation
+ Long.BYTES // minimum sequence number, introduced in 6.0.0
+ Long.BYTES // maximum sequence number, introduced in 6.0.0
+ Long.BYTES // global checkpoint, introduced in 6.0.0
+ Long.BYTES // minimum translog generation in the translog - introduced in 6.0.0
+ CodecUtil.footerLength();

// size of 5.0.0 checkpoint
static final int V1_FILE_SIZE = CodecUtil.headerLength(CHECKPOINT_CODEC)
+ Integer.BYTES // ops
+ Long.BYTES // offset
+ Long.BYTES // generation
+ Long.BYTES // minimum sequence number
+ Long.BYTES // maximum sequence number
+ Long.BYTES // global checkpoint
+ Long.BYTES // minimum translog generation in the translog
+ CodecUtil.footerLength();

/**
Expand Down Expand Up @@ -136,7 +128,7 @@ static Checkpoint emptyTranslogCheckpoint(final long offset, final long generati
return new Checkpoint(offset, 0, generation, minSeqNo, maxSeqNo, globalCheckpoint, minTranslogGeneration, trimmedAboveSeqNo);
}

static Checkpoint readCheckpointV6_4_0(final DataInput in) throws IOException {
static Checkpoint readCheckpointV3(final DataInput in) throws IOException {
final long offset = in.readLong();
final int numOps = in.readInt();
final long generation = in.readLong();
Expand All @@ -148,7 +140,7 @@ static Checkpoint readCheckpointV6_4_0(final DataInput in) throws IOException {
return new Checkpoint(offset, numOps, generation, minSeqNo, maxSeqNo, globalCheckpoint, minTranslogGeneration, trimmedAboveSeqNo);
}

static Checkpoint readCheckpointV6_0_0(final DataInput in) throws IOException {
static Checkpoint readCheckpointV2(final DataInput in) throws IOException {
final long offset = in.readLong();
final int numOps = in.readInt();
final long generation = in.readLong();
Expand All @@ -160,19 +152,6 @@ static Checkpoint readCheckpointV6_0_0(final DataInput in) throws IOException {
return new Checkpoint(offset, numOps, generation, minSeqNo, maxSeqNo, globalCheckpoint, minTranslogGeneration, trimmedAboveSeqNo);
}

// reads a checksummed checkpoint introduced in ES 5.0.0
static Checkpoint readCheckpointV5_0_0(final DataInput in) throws IOException {
final long offset = in.readLong();
final int numOps = in.readInt();
final long generation = in.readLong();
final long minSeqNo = SequenceNumbers.NO_OPS_PERFORMED;
final long maxSeqNo = SequenceNumbers.NO_OPS_PERFORMED;
final long globalCheckpoint = SequenceNumbers.UNASSIGNED_SEQ_NO;
final long minTranslogGeneration = -1;
final long trimmedAboveSeqNo = SequenceNumbers.UNASSIGNED_SEQ_NO;
return new Checkpoint(offset, numOps, generation, minSeqNo, maxSeqNo, globalCheckpoint, minTranslogGeneration, trimmedAboveSeqNo);
}

@Override
public String toString() {
return "Checkpoint{" +
Expand All @@ -192,17 +171,14 @@ public static Checkpoint read(Path path) throws IOException {
try (IndexInput indexInput = dir.openInput(path.getFileName().toString(), IOContext.DEFAULT)) {
// We checksum the entire file before we even go and parse it. If it's corrupted we barf right here.
CodecUtil.checksumEntireFile(indexInput);
final int fileVersion = CodecUtil.checkHeader(indexInput, CHECKPOINT_CODEC, INITIAL_VERSION, CURRENT_VERSION);
if (fileVersion == INITIAL_VERSION) {
assert indexInput.length() == V1_FILE_SIZE : indexInput.length();
return Checkpoint.readCheckpointV5_0_0(indexInput);
} else if (fileVersion == VERSION_6_0_0) {
final int fileVersion = CodecUtil.checkHeader(indexInput, CHECKPOINT_CODEC, VERSION_6_0_0, CURRENT_VERSION);
if (fileVersion == VERSION_6_0_0) {
assert indexInput.length() == V2_FILE_SIZE : indexInput.length();
return Checkpoint.readCheckpointV6_0_0(indexInput);
return Checkpoint.readCheckpointV2(indexInput);
} else {
assert fileVersion == CURRENT_VERSION : fileVersion;
assert indexInput.length() == V3_FILE_SIZE : indexInput.length();
return Checkpoint.readCheckpointV6_4_0(indexInput);
return Checkpoint.readCheckpointV3(indexInput);
}
} catch (CorruptIndexException | NoSuchFileException | IndexFormatTooOldException | IndexFormatTooNewException e) {
throw new TranslogCorruptedException(path.toString(), e);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import org.apache.lucene.document.Field;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexFormatTooOldException;
import org.apache.lucene.index.Term;
import org.apache.lucene.mockfile.FilterFileChannel;
import org.apache.lucene.mockfile.FilterFileSystemProvider;
Expand Down Expand Up @@ -2791,6 +2792,18 @@ public void testCheckpointOnDiskFull() throws IOException {
assertEquals(read, checkpoint);
}

public void testLegacyCheckpointVersion() throws IOException {
expectThrows(
TranslogCorruptedException.class,
IndexFormatTooOldException.class,
() -> Checkpoint.read(getDataPath("/org/elasticsearch/index/checkpoint/v1.ckp.binary"))
);
assertThat(Checkpoint.read(getDataPath("/org/elasticsearch/index/checkpoint/v2.ckp.binary")),
equalTo(new Checkpoint(-1312746831014894010L, 44230819, 4168771208509507653L, 6217263213205155568L,
8590850694628654668L, 3768575734506660560L, 1476009383806516272L,
SequenceNumbers.UNASSIGNED_SEQ_NO)));
}

/**
* Tests that closing views after the translog is fine and we can reopen the translog
*/
Expand Down
Binary file not shown.
Binary file not shown.