From 9bd69a782a2ed421dd4e4f663bbcb00daf97ba38 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Fri, 15 Apr 2022 14:08:22 -0500 Subject: [PATCH] Add deprecated API for creating History Ops Snapshot from translog (#2886) (#2917) * Add deprecated API for creating History Ops Snapshot from translog Adds a deprecated API to create a history operations snapshot from the translog. This was superseded by always using the lucene index for peer recovery since the index saves on costly storage but this API enables plugins to selectively use the uncompressed xlog file for speed over disk savings. Note: this API will be either completely removed or refactored in the next release. Signed-off-by: Nicholas Walter Knize Signed-off-by: Sai Kumar Co-authored-by: Nicholas Walter Knize (cherry picked from commit 2069aa372dddccadf8d5087fd76687f1a37cf2ec) --- .../org/opensearch/index/engine/Engine.java | 13 ++++++++++++ .../index/engine/InternalEngine.java | 14 +++++++++++++ .../index/engine/ReadOnlyEngine.java | 14 +++++++++++++ .../opensearch/index/shard/IndexShard.java | 14 ++++++++++++- .../index/engine/InternalEngineTests.java | 20 +++++++++++++++++++ 5 files changed, 74 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/index/engine/Engine.java b/server/src/main/java/org/opensearch/index/engine/Engine.java index bba1d8c069c68..83da0ea6b52f3 100644 --- a/server/src/main/java/org/opensearch/index/engine/Engine.java +++ b/server/src/main/java/org/opensearch/index/engine/Engine.java @@ -742,6 +742,19 @@ public abstract Translog.Snapshot newChangesSnapshot( boolean accurateCount ) throws IOException; + /** + * Reads history operations from the translog file instead of the lucene index + * + * @deprecated reading history operations from the translog file is deprecated and will be removed in the next release + */ + @Deprecated + public abstract Translog.Snapshot newChangesSnapshotFromTranslogFile( + String source, + long fromSeqNo, + long toSeqNo, + boolean requiredFullRange + ) throws IOException; + /** * Counts the number of history operations in the given sequence number range * @param source source of the request diff --git a/server/src/main/java/org/opensearch/index/engine/InternalEngine.java b/server/src/main/java/org/opensearch/index/engine/InternalEngine.java index 84090047d68e8..f6940be979c76 100644 --- a/server/src/main/java/org/opensearch/index/engine/InternalEngine.java +++ b/server/src/main/java/org/opensearch/index/engine/InternalEngine.java @@ -2805,6 +2805,20 @@ public Translog.Snapshot newChangesSnapshot( } } + /** + * Creates a new history snapshot from the translog file instead of the lucene index + * + * @deprecated reading history operations from the translog file is deprecated and will be removed in the next release + * + * Use {@link Engine#newChangesSnapshot(String, long, long, boolean, boolean)} instead + */ + @Deprecated + @Override + public Translog.Snapshot newChangesSnapshotFromTranslogFile(String source, long fromSeqNo, long toSeqNo, boolean requiredFullRange) + throws IOException { + return getTranslog().newSnapshot(fromSeqNo, toSeqNo, requiredFullRange); + } + public int countNumberOfHistoryOperations(String source, long fromSeqNo, long toSeqNo) throws IOException { ensureOpen(); refreshIfNeeded(source, toSeqNo); diff --git a/server/src/main/java/org/opensearch/index/engine/ReadOnlyEngine.java b/server/src/main/java/org/opensearch/index/engine/ReadOnlyEngine.java index 43fe10c217270..a9037de730398 100644 --- a/server/src/main/java/org/opensearch/index/engine/ReadOnlyEngine.java +++ b/server/src/main/java/org/opensearch/index/engine/ReadOnlyEngine.java @@ -335,6 +335,20 @@ public Translog.Snapshot newChangesSnapshot( return newEmptySnapshot(); } + /** + * Creates a new history snapshot from the translog file instead of the lucene index + * + * @deprecated reading history operations from the translog file is deprecated and will be removed in the next release + * + * Use {@link Engine#newChangesSnapshot(String, long, long, boolean, boolean)} instead + */ + @Deprecated + @Override + public Translog.Snapshot newChangesSnapshotFromTranslogFile(String source, long fromSeqNo, long toSeqNo, boolean requiredFullRange) + throws IOException { + return newEmptySnapshot(); + } + @Override public int countNumberOfHistoryOperations(String source, long fromSeqNo, long toSeqNo) throws IOException { try (Translog.Snapshot snapshot = newChangesSnapshot(source, fromSeqNo, toSeqNo, false, true)) { diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index f2630ad05b488..ed851808f5023 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -2233,13 +2233,25 @@ public Closeable acquireHistoryRetentionLock() { /** * Creates a new history snapshot for reading operations since * the provided starting seqno (inclusive) and ending seqno (inclusive) - * The returned snapshot can be retrieved from either Lucene index or translog files. + * The returned snapshot is retrieved from a Lucene index. */ public Translog.Snapshot getHistoryOperations(String reason, long startingSeqNo, long endSeqNo, boolean accurateCount) throws IOException { return getEngine().newChangesSnapshot(reason, startingSeqNo, endSeqNo, true, accurateCount); } + /** + * Creates a new history snapshot from the translog file instead of the lucene index + * + * @deprecated reading history operations from the translog file is deprecated and will be removed in the next release + * + * Use {@link IndexShard#getHistoryOperations(String, long, long, boolean)} instead + */ + @Deprecated + public Translog.Snapshot getHistoryOperationsFromTranslogFile(String reason, long startingSeqNo, long endSeqNo) throws IOException { + return getEngine().newChangesSnapshotFromTranslogFile(reason, startingSeqNo, endSeqNo, true); + } + /** * Checks if we have a completed history of operations since the given starting seqno (inclusive). * This method should be called after acquiring the retention lock; See {@link #acquireHistoryRetentionLock()} diff --git a/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java index 361013149578e..a8a0421ec1ec7 100644 --- a/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java +++ b/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java @@ -6112,10 +6112,30 @@ public void testHistoryBasedOnSource() throws Exception { } } List luceneOps = readAllOperationsBasedOnSource(engine); + // todo remove in next release + List translogOps = readAllOperationsBasedOnTranslog(engine); assertThat(luceneOps.stream().map(o -> o.seqNo()).collect(Collectors.toList()), containsInAnyOrder(expectedSeqNos.toArray())); + assertThat(translogOps.stream().map(o -> o.seqNo()).collect(Collectors.toList()), containsInAnyOrder(expectedSeqNos.toArray())); } } + /** + * Test creating new snapshot from translog file + * + * @deprecated reading history operations from the translog file is deprecated and will be removed in the next release + */ + @Deprecated + private static List readAllOperationsBasedOnTranslog(Engine engine) throws IOException { + final List operations = new ArrayList<>(); + try (Translog.Snapshot snapshot = engine.newChangesSnapshotFromTranslogFile("test", 0, Long.MAX_VALUE, false)) { + Translog.Operation op; + while ((op = snapshot.next()) != null) { + operations.add(op); + } + } + return operations; + } + public void testLuceneHistoryOnPrimary() throws Exception { final List operations = generateSingleDocHistory( false,