From f0270d0714984c25d617a4128b5497cd82483eac Mon Sep 17 00:00:00 2001 From: Ian Swepston Date: Thu, 10 Oct 2024 12:58:12 -0500 Subject: [PATCH 1/5] Added better handling of commits / file changes so that warming is performed at the correct time(s) --- .../io/zulia/server/index/ShardReader.java | 11 +++++- .../server/index/ShardReaderManager.java | 13 ++++++- .../zulia/server/index/ShardWriteManager.java | 25 +++++++------ .../zulia/server/index/ZuliaIndexManager.java | 1 + .../io/zulia/server/index/ZuliaShard.java | 37 +++++++++++++++++-- .../io/zulia/server/test/node/CacheTest.java | 11 +++++- 6 files changed, 77 insertions(+), 21 deletions(-) diff --git a/zulia-server/src/main/java/io/zulia/server/index/ShardReader.java b/zulia-server/src/main/java/io/zulia/server/index/ShardReader.java index de7316c5..eb4f2516 100644 --- a/zulia-server/src/main/java/io/zulia/server/index/ShardReader.java +++ b/zulia-server/src/main/java/io/zulia/server/index/ShardReader.java @@ -48,9 +48,11 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.StringJoiner; import java.util.function.Consumer; public class ShardReader implements AutoCloseable { @@ -61,12 +63,14 @@ public class ShardReader implements AutoCloseable { private final ServerIndexConfig indexConfig; private final String indexName; private final int shardNumber; + private final long creationTime; private final ZuliaPerFieldAnalyzer zuliaPerFieldAnalyzer; private final Cache queryResultCache; private final Cache pinnedQueryResultCache; public ShardReader(int shardNumber, DirectoryReader indexReader, DirectoryTaxonomyReader taxoReader, ServerIndexConfig indexConfig, ZuliaPerFieldAnalyzer zuliaPerFieldAnalyzer) { + this.creationTime = System.currentTimeMillis(); this.shardNumber = shardNumber; this.indexReader = indexReader; this.taxoReader = taxoReader; @@ -74,8 +78,7 @@ public ShardReader(int shardNumber, DirectoryReader indexReader, DirectoryTaxono this.indexName = indexConfig.getIndexName(); this.zuliaPerFieldAnalyzer = zuliaPerFieldAnalyzer; this.queryResultCache = Caffeine.newBuilder().maximumSize(indexConfig.getIndexSettings().getShardQueryCacheSize()).recordStats().build(); - this.pinnedQueryResultCache = Caffeine.newBuilder().recordStats().build(); - } + this.pinnedQueryResultCache = Caffeine.newBuilder().recordStats().build(); } @Override public void close() throws Exception { @@ -83,6 +86,10 @@ public void close() throws Exception { taxoReader.close(); } + public long getCreationTime() { + return creationTime; + } + public int getTotalFacets() { return taxoReader.getSize(); } diff --git a/zulia-server/src/main/java/io/zulia/server/index/ShardReaderManager.java b/zulia-server/src/main/java/io/zulia/server/index/ShardReaderManager.java index 077ccff4..12265530 100644 --- a/zulia-server/src/main/java/io/zulia/server/index/ShardReaderManager.java +++ b/zulia-server/src/main/java/io/zulia/server/index/ShardReaderManager.java @@ -6,8 +6,11 @@ public class ShardReaderManager extends ReferenceManager { + private long latestShardTime; + public ShardReaderManager(ShardReader initial) { this.current = initial; + this.latestShardTime = initial.getCreationTime(); } @Override @@ -17,7 +20,12 @@ protected void decRef(ShardReader reference) throws IOException { @Override protected ShardReader refreshIfNeeded(ShardReader referenceToRefresh) throws IOException { - return referenceToRefresh.refreshIfNeeded(); + // Evaluate last build time for outside decision making + ShardReader next = referenceToRefresh.refreshIfNeeded(); + if (next != null) { + latestShardTime = next.getCreationTime(); + } + return next; } @Override @@ -30,4 +38,7 @@ protected int getRefCount(ShardReader reference) { return reference.getRefCount(); } + public long getLatestShardTime() { + return latestShardTime; + } } diff --git a/zulia-server/src/main/java/io/zulia/server/index/ShardWriteManager.java b/zulia-server/src/main/java/io/zulia/server/index/ShardWriteManager.java index 0796e015..4ff425c0 100644 --- a/zulia-server/src/main/java/io/zulia/server/index/ShardWriteManager.java +++ b/zulia-server/src/main/java/io/zulia/server/index/ShardWriteManager.java @@ -148,28 +148,25 @@ public boolean needsIdleCommit() { return false; } - public WarmInfo needsSearchWarming() { + public WarmInfo needsSearchWarming(long lastestShardTime) { long currentTime = System.currentTimeMillis(); - long msAfterCommitToWarm = indexConfig.getIndexSettings().getIdleTimeWithoutCommit() * 1000L; - //reassign so it can't change in the middle of the logic Long lastChange = this.lastChange; Long lastCommit = this.lastCommit; Long lastWarm = this.lastWarm; if (lastWarm == null) { - // never warmed so needs warmed - return new WarmInfo(true, lastChange, lastCommit); + // never warmed so needs warmed and the index is idle + boolean idle = lastChange == null || + (Math.abs(currentTime - lastChange) > 1000L && (lastCommit == null || (lastCommit > lastChange))); + return new WarmInfo(idle, lastChange, lastCommit); } if (lastCommit != null && lastChange != null) { // if there has been a change to the index and a commit if (lastChange < lastCommit) { // no changes since last commit - long timeSinceLastCommit = currentTime - lastCommit; - if (timeSinceLastCommit > msAfterCommitToWarm) { - //if the last commit is after the last warming - boolean needsWarm = lastCommit > lastWarm; - return new WarmInfo(needsWarm, lastChange, lastCommit); + if (lastWarm < lastestShardTime) { // Change is committed AND shard reader has been reopened + return new WarmInfo(true, lastChange, lastCommit); } } } @@ -185,8 +182,12 @@ public Long getLastCommit() { return lastCommit; } - public void searchesWarmed() { - lastWarm = System.currentTimeMillis(); + public Long getLastWarm() { + return lastWarm; + } + + public void searchesWarmed(long time) { + lastWarm = time; } public boolean markedChangedCheckIfCommitNeeded() { diff --git a/zulia-server/src/main/java/io/zulia/server/index/ZuliaIndexManager.java b/zulia-server/src/main/java/io/zulia/server/index/ZuliaIndexManager.java index 0413df55..695d8fb4 100644 --- a/zulia-server/src/main/java/io/zulia/server/index/ZuliaIndexManager.java +++ b/zulia-server/src/main/java/io/zulia/server/index/ZuliaIndexManager.java @@ -519,6 +519,7 @@ public UpdateIndexResponse updateIndex(UpdateIndexRequest request) throws Except } } + //TODO(Ian): Check if there was a change before resetting the warming (not compatible w/ List warmingSearches = updateWithAction(updateIndexSettings.getWarmingSearchesOperation(), existingWarmingSearch, warmingSearch, QueryRequest::getSearchLabel); existingSettings.clearWarmingSearches(); diff --git a/zulia-server/src/main/java/io/zulia/server/index/ZuliaShard.java b/zulia-server/src/main/java/io/zulia/server/index/ZuliaShard.java index 18bb5d66..a50df6fc 100644 --- a/zulia-server/src/main/java/io/zulia/server/index/ZuliaShard.java +++ b/zulia-server/src/main/java/io/zulia/server/index/ZuliaShard.java @@ -82,8 +82,7 @@ public void forceCommit() throws IOException { } shardWriteManager.commit(); - shardReaderManager.maybeRefresh(); - + shardReaderManager.maybeRefreshBlocking(); } public void tryIdleCommit() throws IOException { @@ -98,8 +97,18 @@ public void tryWarmSearches(ZuliaIndex zuliaIndex, boolean primary) { EnumSet usesPrimary = EnumSet.of(MasterSlaveSettings.MASTER_ONLY, MasterSlaveSettings.MASTER_IF_AVAILABLE); EnumSet usesReplica = EnumSet.of(MasterSlaveSettings.SLAVE_ONLY, MasterSlaveSettings.MASTER_IF_AVAILABLE); - WarmInfo warmInfo = shardWriteManager.needsSearchWarming(); + long lastestShardTime = shardReaderManager.getLatestShardTime(); + WarmInfo warmInfo = shardWriteManager.needsSearchWarming(lastestShardTime); if (warmInfo.needsWarming()) { + + try { + shardReaderManager.maybeRefreshBlocking(); + } + catch (Exception e) { + LOG.error("Failed to refresh shard reader: ", e); + throw new RuntimeException(e); + } + LOG.info("Started warming searching for index {}", indexName); List warmingSearches = shardWriteManager.getIndexConfig().getWarmingSearches(); @@ -120,6 +129,10 @@ public void tryWarmSearches(ZuliaIndex zuliaIndex, boolean primary) { LOG.info("Index {} commited: canceling warming", indexName); return; } + if (lastestShardTime != shardReaderManager.getLatestShardTime()) { + LOG.info("Index {} reloaded: canceling warming", indexName); + return; + } if (shardNeedsWarmForSearch) { try { @@ -136,7 +149,23 @@ public void tryWarmSearches(ZuliaIndex zuliaIndex, boolean primary) { } - shardWriteManager.searchesWarmed(); + // has the index changed since we made the decision to start warming ? + if (!Objects.equals(warmInfo.lastChanged(), shardWriteManager.getLastChanged())) { + LOG.info("Index {} changed: canceling warming", indexName); + return; + } + if (!Objects.equals(warmInfo.lastCommit(), shardWriteManager.getLastCommit())) { + LOG.info("Index {} commited: canceling warming", indexName); + return; + } + + long warmTime = System.currentTimeMillis(); + if (lastestShardTime != shardReaderManager.getLatestShardTime()) { + LOG.info("Index {} reloaded: canceling warming", indexName); + return; + } + + shardWriteManager.searchesWarmed(warmTime); LOG.info("Finished warming searching for index {}", indexName); } } diff --git a/zulia-server/src/test/java/io/zulia/server/test/node/CacheTest.java b/zulia-server/src/test/java/io/zulia/server/test/node/CacheTest.java index 34dbbbaf..8c0522cc 100644 --- a/zulia-server/src/test/java/io/zulia/server/test/node/CacheTest.java +++ b/zulia-server/src/test/java/io/zulia/server/test/node/CacheTest.java @@ -20,9 +20,12 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestMethodOrder; import org.junit.jupiter.api.extension.RegisterExtension; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; @TestMethodOrder(MethodOrderer.OrderAnnotation.class) public class CacheTest { + private final static Logger LOG = LoggerFactory.getLogger(CacheTest.class); @RegisterExtension static final NodeExtension nodeExtension = new NodeExtension(1); @@ -112,7 +115,8 @@ private void indexRecord(String index, int id, String title, String description, @Order(3) public void searchTest() throws Exception { ZuliaWorkPool zuliaWorkPool = nodeExtension.getClient(); - Thread.sleep(2000); + // Sleep needed because post-update warming takes a long time now since it can (and does) reset after writes based on file io + Thread.sleep(10000); Search search; SearchResult searchResult; @@ -272,7 +276,8 @@ public void searchTest() throws Exception { .addCountFacet(new CountFacet("rating").setTopN(3))); zuliaWorkPool.updateIndex(updateIndex); - Thread.sleep(2000); + // Sleep needed because post-update warming takes a long time now since it can (and does) reset after writes based on file io + Thread.sleep(10000); search = new Search(CACHE_TEST); search.addQuery(new FilterQuery("rating:[2.0 TO 3.5]")); @@ -294,6 +299,8 @@ public void restart() throws Exception { @Test @Order(6) public void confirm() throws Exception { + // Sleep needed because post-update warming takes a long time now since it can (and does) reset after writes based on file io + Thread.sleep(10000); ZuliaWorkPool zuliaWorkPool = nodeExtension.getClient(); Search search; SearchResult searchResult; From b7bb4e8ac95f877d0e9aec4e11e56a9fcfa661c2 Mon Sep 17 00:00:00 2001 From: Ian Swepston Date: Thu, 10 Oct 2024 13:06:53 -0500 Subject: [PATCH 2/5] Beautify --- .../src/main/java/io/zulia/server/index/ShardReader.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/zulia-server/src/main/java/io/zulia/server/index/ShardReader.java b/zulia-server/src/main/java/io/zulia/server/index/ShardReader.java index eb4f2516..9484ef21 100644 --- a/zulia-server/src/main/java/io/zulia/server/index/ShardReader.java +++ b/zulia-server/src/main/java/io/zulia/server/index/ShardReader.java @@ -48,11 +48,9 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.StringJoiner; import java.util.function.Consumer; public class ShardReader implements AutoCloseable { @@ -78,7 +76,8 @@ public ShardReader(int shardNumber, DirectoryReader indexReader, DirectoryTaxono this.indexName = indexConfig.getIndexName(); this.zuliaPerFieldAnalyzer = zuliaPerFieldAnalyzer; this.queryResultCache = Caffeine.newBuilder().maximumSize(indexConfig.getIndexSettings().getShardQueryCacheSize()).recordStats().build(); - this.pinnedQueryResultCache = Caffeine.newBuilder().recordStats().build(); } + this.pinnedQueryResultCache = Caffeine.newBuilder().recordStats().build(); + } @Override public void close() throws Exception { From 75113a4ec0981b4059b8df768abd5b50cb9ec300 Mon Sep 17 00:00:00 2001 From: Ian Swepston Date: Thu, 10 Oct 2024 13:08:29 -0500 Subject: [PATCH 3/5] Beautify --- .../src/main/java/io/zulia/server/index/ShardWriteManager.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/zulia-server/src/main/java/io/zulia/server/index/ShardWriteManager.java b/zulia-server/src/main/java/io/zulia/server/index/ShardWriteManager.java index 4ff425c0..8cad52ac 100644 --- a/zulia-server/src/main/java/io/zulia/server/index/ShardWriteManager.java +++ b/zulia-server/src/main/java/io/zulia/server/index/ShardWriteManager.java @@ -158,8 +158,7 @@ public WarmInfo needsSearchWarming(long lastestShardTime) { if (lastWarm == null) { // never warmed so needs warmed and the index is idle - boolean idle = lastChange == null || - (Math.abs(currentTime - lastChange) > 1000L && (lastCommit == null || (lastCommit > lastChange))); + boolean idle = lastChange == null || (Math.abs(currentTime - lastChange) > 1000L && (lastCommit == null || (lastCommit > lastChange))); return new WarmInfo(idle, lastChange, lastCommit); } From 9a5d864e4d413eaba56262104f6c7a90ae263ff6 Mon Sep 17 00:00:00 2001 From: Ian Swepston Date: Thu, 10 Oct 2024 13:09:07 -0500 Subject: [PATCH 4/5] Beautify --- zulia-server/src/main/java/io/zulia/server/index/ZuliaShard.java | 1 - 1 file changed, 1 deletion(-) diff --git a/zulia-server/src/main/java/io/zulia/server/index/ZuliaShard.java b/zulia-server/src/main/java/io/zulia/server/index/ZuliaShard.java index a50df6fc..61651a99 100644 --- a/zulia-server/src/main/java/io/zulia/server/index/ZuliaShard.java +++ b/zulia-server/src/main/java/io/zulia/server/index/ZuliaShard.java @@ -146,7 +146,6 @@ public void tryWarmSearches(ZuliaIndex zuliaIndex, boolean primary) { } } - } // has the index changed since we made the decision to start warming ? From 140e4d701b191c3f974d04d7a6ea76e0ff686f08 Mon Sep 17 00:00:00 2001 From: Ian Swepston Date: Thu, 10 Oct 2024 13:09:42 -0500 Subject: [PATCH 5/5] Leftover code --- .../src/test/java/io/zulia/server/test/node/CacheTest.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/zulia-server/src/test/java/io/zulia/server/test/node/CacheTest.java b/zulia-server/src/test/java/io/zulia/server/test/node/CacheTest.java index 8c0522cc..11cf03e3 100644 --- a/zulia-server/src/test/java/io/zulia/server/test/node/CacheTest.java +++ b/zulia-server/src/test/java/io/zulia/server/test/node/CacheTest.java @@ -20,12 +20,9 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestMethodOrder; import org.junit.jupiter.api.extension.RegisterExtension; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; @TestMethodOrder(MethodOrderer.OrderAnnotation.class) public class CacheTest { - private final static Logger LOG = LoggerFactory.getLogger(CacheTest.class); @RegisterExtension static final NodeExtension nodeExtension = new NodeExtension(1);