From c3b7d04b398ffd0ad417ad273d88b1997f864a2d Mon Sep 17 00:00:00 2001 From: Nicholas Walter Knize Date: Tue, 12 Apr 2022 09:38:12 -0500 Subject: [PATCH 1/2] Add deprecated API for creating History Ops Snapshot from translog Adds a deprecated API to create a history operations snapshot from the translog. This was superseded by always using the lucene index for peer recovery since the index saves on costly storage but this API enables plugins to selectively use the uncompressed xlog file for speed over disk savings. Note: this API will be either completely removed or refactored in the next release. Signed-off-by: Nicholas Walter Knize Signed-off-by: Sai Kumar --- .../org/opensearch/index/engine/Engine.java | 13 ++++++++++++ .../index/engine/InternalEngine.java | 14 +++++++++++++ .../index/engine/ReadOnlyEngine.java | 14 +++++++++++++ .../opensearch/index/shard/IndexShard.java | 14 ++++++++++++- .../index/engine/InternalEngineTests.java | 20 +++++++++++++++++++ 5 files changed, 74 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/index/engine/Engine.java b/server/src/main/java/org/opensearch/index/engine/Engine.java index bba1d8c069c68..83da0ea6b52f3 100644 --- a/server/src/main/java/org/opensearch/index/engine/Engine.java +++ b/server/src/main/java/org/opensearch/index/engine/Engine.java @@ -742,6 +742,19 @@ public abstract Translog.Snapshot newChangesSnapshot( boolean accurateCount ) throws IOException; + /** + * Reads history operations from the translog file instead of the lucene index + * + * @deprecated reading history operations from the translog file is deprecated and will be removed in the next release + */ + @Deprecated + public abstract Translog.Snapshot newChangesSnapshotFromTranslogFile( + String source, + long fromSeqNo, + long toSeqNo, + boolean requiredFullRange + ) throws IOException; + /** * Counts the number of history operations in the given sequence number range * @param source source of the request diff --git a/server/src/main/java/org/opensearch/index/engine/InternalEngine.java b/server/src/main/java/org/opensearch/index/engine/InternalEngine.java index 6bef118e0b61f..8e5308973ae64 100644 --- a/server/src/main/java/org/opensearch/index/engine/InternalEngine.java +++ b/server/src/main/java/org/opensearch/index/engine/InternalEngine.java @@ -2821,6 +2821,20 @@ public Translog.Snapshot newChangesSnapshot( } } + /** + * Creates a new history snapshot from the translog file instead of the lucene index + * + * @deprecated reading history operations from the translog file is deprecated and will be removed in the next release + * + * Use {@link Engine#newChangesSnapshot(String, long, long, boolean, boolean)} instead + */ + @Deprecated + @Override + public Translog.Snapshot newChangesSnapshotFromTranslogFile(String source, long fromSeqNo, long toSeqNo, boolean requiredFullRange) + throws IOException { + return getTranslog().newSnapshot(fromSeqNo, toSeqNo, requiredFullRange); + } + public int countNumberOfHistoryOperations(String source, long fromSeqNo, long toSeqNo) throws IOException { ensureOpen(); refreshIfNeeded(source, toSeqNo); diff --git a/server/src/main/java/org/opensearch/index/engine/ReadOnlyEngine.java b/server/src/main/java/org/opensearch/index/engine/ReadOnlyEngine.java index 43fe10c217270..a9037de730398 100644 --- a/server/src/main/java/org/opensearch/index/engine/ReadOnlyEngine.java +++ b/server/src/main/java/org/opensearch/index/engine/ReadOnlyEngine.java @@ -335,6 +335,20 @@ public Translog.Snapshot newChangesSnapshot( return newEmptySnapshot(); } + /** + * Creates a new history snapshot from the translog file instead of the lucene index + * + * @deprecated reading history operations from the translog file is deprecated and will be removed in the next release + * + * Use {@link Engine#newChangesSnapshot(String, long, long, boolean, boolean)} instead + */ + @Deprecated + @Override + public Translog.Snapshot newChangesSnapshotFromTranslogFile(String source, long fromSeqNo, long toSeqNo, boolean requiredFullRange) + throws IOException { + return newEmptySnapshot(); + } + @Override public int countNumberOfHistoryOperations(String source, long fromSeqNo, long toSeqNo) throws IOException { try (Translog.Snapshot snapshot = newChangesSnapshot(source, fromSeqNo, toSeqNo, false, true)) { diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index f2630ad05b488..ed851808f5023 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -2233,13 +2233,25 @@ public Closeable acquireHistoryRetentionLock() { /** * Creates a new history snapshot for reading operations since * the provided starting seqno (inclusive) and ending seqno (inclusive) - * The returned snapshot can be retrieved from either Lucene index or translog files. + * The returned snapshot is retrieved from a Lucene index. */ public Translog.Snapshot getHistoryOperations(String reason, long startingSeqNo, long endSeqNo, boolean accurateCount) throws IOException { return getEngine().newChangesSnapshot(reason, startingSeqNo, endSeqNo, true, accurateCount); } + /** + * Creates a new history snapshot from the translog file instead of the lucene index + * + * @deprecated reading history operations from the translog file is deprecated and will be removed in the next release + * + * Use {@link IndexShard#getHistoryOperations(String, long, long, boolean)} instead + */ + @Deprecated + public Translog.Snapshot getHistoryOperationsFromTranslogFile(String reason, long startingSeqNo, long endSeqNo) throws IOException { + return getEngine().newChangesSnapshotFromTranslogFile(reason, startingSeqNo, endSeqNo, true); + } + /** * Checks if we have a completed history of operations since the given starting seqno (inclusive). * This method should be called after acquiring the retention lock; See {@link #acquireHistoryRetentionLock()} diff --git a/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java index c33adf3bcb558..b31ec01c1a3c3 100644 --- a/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java +++ b/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java @@ -6318,10 +6318,30 @@ public void testHistoryBasedOnSource() throws Exception { } } List luceneOps = readAllOperationsBasedOnSource(engine); + // todo remove in next release + List translogOps = readAllOperationsBasedOnTranslog(engine); assertThat(luceneOps.stream().map(o -> o.seqNo()).collect(Collectors.toList()), containsInAnyOrder(expectedSeqNos.toArray())); + assertThat(translogOps.stream().map(o -> o.seqNo()).collect(Collectors.toList()), containsInAnyOrder(expectedSeqNos.toArray())); } } + /** + * Test creating new snapshot from translog file + * + * @Deprecated reading history operations from the translog file is deprecated and will be removed in the next release + */ + @Deprecated + private static List readAllOperationsBasedOnTranslog(Engine engine) throws IOException { + final List operations = new ArrayList<>(); + try (Translog.Snapshot snapshot = engine.newChangesSnapshotFromTranslogFile("test", 0, Long.MAX_VALUE, false)) { + Translog.Operation op; + while ((op = snapshot.next()) != null) { + operations.add(op); + } + } + return operations; + } + public void testLuceneHistoryOnPrimary() throws Exception { final List operations = generateSingleDocHistory( false, From 263acf1ae28127a2151eeba2a5cecb775aeb666d Mon Sep 17 00:00:00 2001 From: Nicholas Walter Knize Date: Thu, 14 Apr 2022 15:57:23 -0500 Subject: [PATCH 2/2] fix javadoc casing for deprecated Signed-off-by: Nicholas Walter Knize --- .../java/org/opensearch/index/engine/InternalEngineTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java index b31ec01c1a3c3..ec177f460d606 100644 --- a/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java +++ b/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java @@ -6328,7 +6328,7 @@ public void testHistoryBasedOnSource() throws Exception { /** * Test creating new snapshot from translog file * - * @Deprecated reading history operations from the translog file is deprecated and will be removed in the next release + * @deprecated reading history operations from the translog file is deprecated and will be removed in the next release */ @Deprecated private static List readAllOperationsBasedOnTranslog(Engine engine) throws IOException {