From 71b6b3b724754c6c0af7f38f746b9ce952a645e1 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Tue, 18 Jul 2017 16:39:14 +0200 Subject: [PATCH 1/3] Reduce the overhead of timeouts and low-level search cancellation. Setting a timeout or enforcing low-level search cancellation used to make us wrap the collector and check either the current time or whether the search task was cancelled for every collected document. This can be significant overhead on cheap queries that match many documents. This commit changes the approach to wrap the bulk scorer rather than the collector and exponentially increase the interval between two consecutive checks in order to reduce the overhead of those checks. --- .../internal/CancellableBulkScorer.java | 68 +++++++++++++++++++ .../search/internal/ContextIndexSearcher.java | 53 +++++++++++++++ .../search/query/CancellableCollector.java | 37 ++-------- .../search/query/QueryCollectorContext.java | 25 +------ .../search/query/QueryPhase.java | 59 +++++++++++++--- .../search/SearchCancellationTests.java | 13 +--- .../search/query/QueryPhaseTests.java | 45 ++++++------ .../elasticsearch/test/ESIntegTestCase.java | 6 +- .../test/client/RandomizingClient.java | 7 ++ 9 files changed, 214 insertions(+), 99 deletions(-) create mode 100644 core/src/main/java/org/elasticsearch/search/internal/CancellableBulkScorer.java diff --git a/core/src/main/java/org/elasticsearch/search/internal/CancellableBulkScorer.java b/core/src/main/java/org/elasticsearch/search/internal/CancellableBulkScorer.java new file mode 100644 index 0000000000000..f5eceed52ea32 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/search/internal/CancellableBulkScorer.java @@ -0,0 +1,68 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.internal; + +import org.apache.lucene.search.BulkScorer; +import org.apache.lucene.search.LeafCollector; +import org.apache.lucene.util.Bits; + +import java.io.IOException; +import java.util.Objects; + +/** + * A {@link BulkScorer} wrapper that runs a {@link Runnable} on a regular basis + * so that the query can be interrupted. + */ +final class CancellableBulkScorer extends BulkScorer { + + // we use the BooleanScorer window size as a base interval in order to make sure that we do not + // slow down boolean queries + private static final int INITIAL_INTERVAL = 1 << 11; + + // No point in having intervals that are larger than 1M + private static final int MAX_INTERVAL = 1 << 20; + + private final BulkScorer scorer; + private final Runnable checkCancelled; + + CancellableBulkScorer(BulkScorer scorer, Runnable checkCancelled) { + this.scorer = Objects.requireNonNull(scorer); + this.checkCancelled = Objects.requireNonNull(checkCancelled); + } + + @Override + public int score(LeafCollector collector, Bits acceptDocs, int min, int max) throws IOException { + int interval = INITIAL_INTERVAL; + while (min < max) { + checkCancelled.run(); + final int newMax = (int) Math.min((long) min + interval, max); + min = scorer.score(collector, acceptDocs, min, newMax); + interval = Math.min(interval << 1, MAX_INTERVAL); + } + checkCancelled.run(); + return min; + } + + @Override + public long cost() { + return scorer.cost(); + } + +} diff --git a/core/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java b/core/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java index 9f2df13592b27..17d4c605084cf 100644 --- a/core/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java +++ b/core/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java @@ -20,14 +20,18 @@ package org.elasticsearch.search.internal; import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.Term; import org.apache.lucene.index.TermContext; +import org.apache.lucene.search.BulkScorer; import org.apache.lucene.search.CollectionStatistics; +import org.apache.lucene.search.Collector; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.QueryCache; import org.apache.lucene.search.QueryCachingPolicy; +import org.apache.lucene.search.Scorer; import org.apache.lucene.search.TermStatistics; import org.apache.lucene.search.Weight; import org.elasticsearch.common.lease.Releasable; @@ -40,6 +44,8 @@ import org.elasticsearch.search.profile.query.QueryTimingType; import java.io.IOException; +import java.util.List; +import java.util.Set; /** * Context-aware extension of {@link IndexSearcher}. @@ -58,6 +64,8 @@ public class ContextIndexSearcher extends IndexSearcher implements Releasable { // TODO revisit moving the profiler to inheritance or wrapping model in the future private QueryProfiler profiler; + private Runnable checkCancelled; + public ContextIndexSearcher(Engine.Searcher searcher, QueryCache queryCache, QueryCachingPolicy queryCachingPolicy) { super(searcher.reader()); @@ -76,6 +84,14 @@ public void setProfiler(QueryProfiler profiler) { this.profiler = profiler; } + /** + * Set a {@link Runnable} that will be run on a regular basis while + * collecting documents. + */ + public void setCheckCancelled(Runnable checkCancelled) { + this.checkCancelled = checkCancelled; + } + public void setAggregatedDfs(AggregatedDfs aggregatedDfs) { this.aggregatedDfs = aggregatedDfs; } @@ -133,6 +149,43 @@ public Weight createWeight(Query query, boolean needsScores, float boost) throws } } + @Override + protected void search(List leaves, Weight weight, Collector collector) throws IOException { + final Weight cancellablWeight; + if (checkCancelled != null) { + cancellablWeight = new Weight(weight.getQuery()) { + + @Override + public void extractTerms(Set terms) { + throw new UnsupportedOperationException(); + } + + @Override + public Explanation explain(LeafReaderContext context, int doc) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public Scorer scorer(LeafReaderContext context) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public BulkScorer bulkScorer(LeafReaderContext context) throws IOException { + BulkScorer in = weight.bulkScorer(context); + if (in != null) { + return new CancellableBulkScorer(in, checkCancelled); + } else { + return null; + } + } + }; + } else { + cancellablWeight = weight; + } + super.search(leaves, cancellablWeight, collector); + } + @Override public Explanation explain(Query query, int doc) throws IOException { if (aggregatedDfs != null) { diff --git a/core/src/main/java/org/elasticsearch/search/query/CancellableCollector.java b/core/src/main/java/org/elasticsearch/search/query/CancellableCollector.java index 1c702ac0e1f8d..504a7f3d13da5 100644 --- a/core/src/main/java/org/elasticsearch/search/query/CancellableCollector.java +++ b/core/src/main/java/org/elasticsearch/search/query/CancellableCollector.java @@ -21,58 +21,33 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.Collector; import org.apache.lucene.search.FilterCollector; -import org.apache.lucene.search.FilterLeafCollector; import org.apache.lucene.search.LeafCollector; -import org.elasticsearch.common.inject.Provider; import org.elasticsearch.tasks.TaskCancelledException; import java.io.IOException; +import java.util.function.BooleanSupplier; /** * Collector that checks if the task it is executed under is cancelled. */ public class CancellableCollector extends FilterCollector { - private final Provider cancelled; - private final boolean leafLevel; + private final BooleanSupplier cancelled; /** * Constructor - * @param cancelled supplier of the cancellation flag, the supplier will be called for each segment if lowLevelCancellation is set - * to false and for each collected record if lowLevelCancellation is set to true. In other words this class assumes - * that the supplier is fast, with performance on the order of a volatile read. - * @param lowLevelCancellation true if collector should check for cancellation for each collected record, false if check should be - * performed only once per segment + * @param cancelled supplier of the cancellation flag, the supplier will be called for each segment * @param in wrapped collector */ - public CancellableCollector(Provider cancelled, boolean lowLevelCancellation, Collector in) { + public CancellableCollector(BooleanSupplier cancelled, Collector in) { super(in); this.cancelled = cancelled; - this.leafLevel = lowLevelCancellation; } @Override public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException { - if (cancelled.get()) { + if (cancelled.getAsBoolean()) { throw new TaskCancelledException("cancelled"); } - if (leafLevel) { - return new CancellableLeafCollector(super.getLeafCollector(context)); - } else { - return super.getLeafCollector(context); - } - } - - private class CancellableLeafCollector extends FilterLeafCollector { - private CancellableLeafCollector(LeafCollector in) { - super(in); - } - - @Override - public void collect(int doc) throws IOException { - if (cancelled.get()) { - throw new TaskCancelledException("cancelled"); - } - super.collect(doc); - } + return super.getLeafCollector(context); } } diff --git a/core/src/main/java/org/elasticsearch/search/query/QueryCollectorContext.java b/core/src/main/java/org/elasticsearch/search/query/QueryCollectorContext.java index eaaf07ce305b6..792f79fd16550 100644 --- a/core/src/main/java/org/elasticsearch/search/query/QueryCollectorContext.java +++ b/core/src/main/java/org/elasticsearch/search/query/QueryCollectorContext.java @@ -26,12 +26,9 @@ import org.apache.lucene.search.MultiCollector; import org.apache.lucene.search.Query; import org.apache.lucene.search.Sort; -import org.apache.lucene.search.TimeLimitingCollector; import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.TotalHitCountCollector; import org.apache.lucene.search.Weight; -import org.apache.lucene.util.Counter; -import org.elasticsearch.common.inject.Provider; import org.elasticsearch.common.lucene.MinimumScoreCollector; import org.elasticsearch.common.lucene.search.FilteredCollector; import org.elasticsearch.search.profile.query.InternalProfileCollector; @@ -50,7 +47,6 @@ import static org.elasticsearch.search.profile.query.CollectorResult.REASON_SEARCH_MULTI; import static org.elasticsearch.search.profile.query.CollectorResult.REASON_SEARCH_POST_FILTER; import static org.elasticsearch.search.profile.query.CollectorResult.REASON_SEARCH_TERMINATE_AFTER_COUNT; -import static org.elasticsearch.search.profile.query.CollectorResult.REASON_SEARCH_TIMEOUT; import static org.elasticsearch.search.query.TopDocsCollectorContext.shortcutTotalHitCount; abstract class QueryCollectorContext { @@ -170,31 +166,14 @@ protected InternalProfileCollector createWithProfiler(InternalProfileCollector i }; } - /** - * Creates a time limiting collector limiting the collection to timeOutMillisms. - */ - static QueryCollectorContext createTimeoutCollectorContext(Counter timeEstimate, long timeoutMillis) { - return new QueryCollectorContext(REASON_SEARCH_TIMEOUT) { - @Override - Collector create(Collector in) throws IOException { - return new TimeLimitingCollector(in, timeEstimate, timeoutMillis); - } - - @Override - boolean shouldCollect() { - return false; - } - }; - } - /** * Creates a collector that throws {@link TaskCancelledException} if the search is cancelled */ - static QueryCollectorContext createCancellableCollectorContext(Provider cancelled, boolean lowLevelCancellation) { + static QueryCollectorContext createCancellableCollectorContext(BooleanSupplier cancelled) { return new QueryCollectorContext(REASON_SEARCH_CANCELLED) { @Override Collector create(Collector in) throws IOException { - return new CancellableCollector(cancelled, lowLevelCancellation, in); + return new CancellableCollector(cancelled, in); } @Override diff --git a/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java b/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java index a944f0d1a51c4..57d5b5699b6bb 100644 --- a/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java +++ b/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java @@ -24,6 +24,7 @@ import org.apache.lucene.queries.SearchAfterSortedDocQuery; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.CollectionTerminatedException; import org.apache.lucene.search.Collector; import org.apache.lucene.search.ConstantScoreQuery; import org.apache.lucene.search.EarlyTerminatingSortingCollector; @@ -35,6 +36,8 @@ import org.apache.lucene.search.Sort; import org.apache.lucene.search.TimeLimitingCollector; import org.apache.lucene.search.TopDocs; +import org.apache.lucene.util.Counter; +import org.elasticsearch.action.search.SearchTask; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.settings.Settings; @@ -44,6 +47,7 @@ import org.elasticsearch.search.SearchPhase; import org.elasticsearch.search.SearchService; import org.elasticsearch.search.aggregations.AggregationPhase; +import org.elasticsearch.search.internal.ContextIndexSearcher; import org.elasticsearch.search.internal.ScrollContext; import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.profile.ProfileShardResult; @@ -52,10 +56,11 @@ import org.elasticsearch.search.rescore.RescorePhase; import org.elasticsearch.search.sort.SortAndFormats; import org.elasticsearch.search.suggest.SuggestPhase; +import org.elasticsearch.tasks.TaskCancelledException; import org.elasticsearch.threadpool.ThreadPool; import java.util.LinkedList; -import java.util.concurrent.ExecutorService; +import java.util.function.Consumer; import static org.elasticsearch.search.query.QueryCollectorContext.createCancellableCollectorContext; import static org.elasticsearch.search.query.QueryCollectorContext.createEarlySortingTerminationCollectorContext; @@ -63,9 +68,9 @@ import static org.elasticsearch.search.query.QueryCollectorContext.createFilteredCollectorContext; import static org.elasticsearch.search.query.QueryCollectorContext.createMinScoreCollectorContext; import static org.elasticsearch.search.query.QueryCollectorContext.createMultiCollectorContext; -import static org.elasticsearch.search.query.QueryCollectorContext.createTimeoutCollectorContext; import static org.elasticsearch.search.query.TopDocsCollectorContext.createTopDocsCollectorContext; + /** * Query phase of a search request, used to run the query and get back from each shard information about the matching documents * (document ids and score or sort criteria) so that matches can be reduced on the coordinating node @@ -103,7 +108,8 @@ public void execute(SearchContext searchContext) throws QueryPhaseExecutionExcep aggregationPhase.preProcess(searchContext); Sort indexSort = searchContext.mapperService().getIndexSettings().getIndexSortConfig() .buildIndexSort(searchContext.mapperService()::fullName, searchContext.fieldData()::getForField); - boolean rescore = execute(searchContext, searchContext.searcher(), indexSort); + final ContextIndexSearcher searcher = searchContext.searcher(); + boolean rescore = execute(searchContext, searchContext.searcher(), searcher::setCheckCancelled, indexSort); if (rescore) { // only if we do a regular search rescorePhase.execute(searchContext); @@ -123,7 +129,8 @@ public void execute(SearchContext searchContext) throws QueryPhaseExecutionExcep * wire everything (mapperService, etc.) * @return whether the rescoring phase should be executed */ - static boolean execute(SearchContext searchContext, final IndexSearcher searcher, @Nullable Sort indexSort) throws QueryPhaseExecutionException { + static boolean execute(SearchContext searchContext, final IndexSearcher searcher, + Consumer checkCancellationSetter, @Nullable Sort indexSort) throws QueryPhaseExecutionException { QuerySearchResult queryResult = searchContext.queryResult(); queryResult.searchTimedOut(false); @@ -192,13 +199,49 @@ static boolean execute(SearchContext searchContext, final IndexSearcher searcher boolean timeoutSet = scrollContext == null && searchContext.timeout() != null && searchContext.timeout().equals(SearchService.NO_TIMEOUT) == false; + + final Runnable timeoutRunnable; if (timeoutSet) { - // TODO: change to use our own counter that uses the scheduler in ThreadPool - // throws TimeLimitingCollector.TimeExceededException when timeout has reached - collectors.add(createTimeoutCollectorContext(searchContext.timeEstimateCounter(), searchContext.timeout().millis())); + final Counter counter = searchContext.timeEstimateCounter(); + final long startTime = counter.get(); + final long timeout = searchContext.timeout().millis(); + final long maxTime = startTime + timeout; + timeoutRunnable = () -> { + final long time = counter.get(); + if (time > maxTime) { + queryResult.searchTimedOut(true); + throw new CollectionTerminatedException(); + } + }; + } else { + timeoutRunnable = null; + } + + final Runnable cancellationRunnable; + if (searchContext.lowLevelCancellation()) { + SearchTask task = searchContext.getTask(); + cancellationRunnable = () -> { if (task.isCancelled()) throw new TaskCancelledException("cancelled"); }; + } else { + cancellationRunnable = null; + } + + final Runnable checkCancelled; + if (timeoutRunnable != null && cancellationRunnable != null) { + checkCancelled = () -> { timeoutRunnable.run(); cancellationRunnable.run(); }; + } else if (timeoutRunnable != null) { + checkCancelled = timeoutRunnable; + } else if (cancellationRunnable != null) { + checkCancelled = cancellationRunnable; + } else { + checkCancelled = null; } + + checkCancellationSetter.accept(checkCancelled); + // add cancellable - collectors.add(createCancellableCollectorContext(searchContext.getTask()::isCancelled, searchContext.lowLevelCancellation())); + // this only performs segment-level cancellation, which is cheap and checked regardless of + // searchContext.lowLevelCancellation() + collectors.add(createCancellableCollectorContext(searchContext.getTask()::isCancelled)); final IndexReader reader = searcher.getIndexReader(); final boolean doProfile = searchContext.getProfilers() != null; diff --git a/core/src/test/java/org/elasticsearch/search/SearchCancellationTests.java b/core/src/test/java/org/elasticsearch/search/SearchCancellationTests.java index 2ee81e935e2d5..0b658d95a0f2e 100644 --- a/core/src/test/java/org/elasticsearch/search/SearchCancellationTests.java +++ b/core/src/test/java/org/elasticsearch/search/SearchCancellationTests.java @@ -72,21 +72,10 @@ public static void cleanup() throws IOException { reader = null; } - - public void testLowLevelCancellableCollector() throws IOException { - TotalHitCountCollector collector = new TotalHitCountCollector(); - AtomicBoolean cancelled = new AtomicBoolean(); - CancellableCollector cancellableCollector = new CancellableCollector(cancelled::get, true, collector); - final LeafCollector leafCollector = cancellableCollector.getLeafCollector(reader.leaves().get(0)); - leafCollector.collect(0); - cancelled.set(true); - expectThrows(TaskCancelledException.class, () -> leafCollector.collect(1)); - } - public void testCancellableCollector() throws IOException { TotalHitCountCollector collector = new TotalHitCountCollector(); AtomicBoolean cancelled = new AtomicBoolean(); - CancellableCollector cancellableCollector = new CancellableCollector(cancelled::get, false, collector); + CancellableCollector cancellableCollector = new CancellableCollector(cancelled::get, collector); final LeafCollector leafCollector = cancellableCollector.getLeafCollector(reader.leaves().get(0)); leafCollector.collect(0); cancelled.set(true); diff --git a/core/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java b/core/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java index 3055cda803d6e..d4da2655e8e34 100644 --- a/core/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java +++ b/core/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java @@ -68,7 +68,6 @@ import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.lessThan; -import static org.mockito.Mockito.mock; public class QueryPhaseTests extends IndexShardTestCase { @@ -106,7 +105,7 @@ protected void search(List leaves, Weight weight, Collector c } }; - final boolean rescore = QueryPhase.execute(context, contextSearcher, null); + final boolean rescore = QueryPhase.execute(context, contextSearcher, checkCancelled -> {}, null); assertFalse(rescore); assertEquals(searcher.count(query), context.queryResult().topDocs().totalHits); assertEquals(shouldCollect, collected.get()); @@ -175,12 +174,12 @@ protected void search(List leaves, Weight weight, Collector c } }; - QueryPhase.execute(context, contextSearcher, null); + QueryPhase.execute(context, contextSearcher, checkCancelled -> {}, null); assertEquals(0, context.queryResult().topDocs().totalHits); assertFalse(collected.get()); context.parsedPostFilter(new ParsedQuery(new MatchNoDocsQuery())); - QueryPhase.execute(context, contextSearcher, null); + QueryPhase.execute(context, contextSearcher, checkCancelled -> {}, null); assertEquals(0, context.queryResult().topDocs().totalHits); assertTrue(collected.get()); } @@ -199,12 +198,12 @@ protected void search(List leaves, Weight weight, Collector c } }; - QueryPhase.execute(context, contextSearcher, null); + QueryPhase.execute(context, contextSearcher, checkCancelled -> {}, null); assertEquals(0, context.queryResult().topDocs().totalHits); assertFalse(collected.get()); context.minimumScore(1); - QueryPhase.execute(context, contextSearcher, null); + QueryPhase.execute(context, contextSearcher, checkCancelled -> {}, null); assertEquals(0, context.queryResult().topDocs().totalHits); assertTrue(collected.get()); } @@ -225,7 +224,7 @@ public void testQueryCapturesThreadPoolStats() throws Exception { IndexReader reader = DirectoryReader.open(dir); IndexSearcher contextSearcher = new IndexSearcher(reader); - QueryPhase.execute(context, contextSearcher, null); + QueryPhase.execute(context, contextSearcher, checkCancelled -> {}, null); QuerySearchResult results = context.queryResult(); assertThat(results.serviceTimeEWMA(), greaterThan(0L)); assertThat(results.nodeQueueSize(), greaterThanOrEqualTo(0)); @@ -263,14 +262,14 @@ protected void search(List leaves, Weight weight, Collector c context.setTask(new SearchTask(123L, "", "", "", null)); context.setSize(10); - QueryPhase.execute(context, contextSearcher, null); + QueryPhase.execute(context, contextSearcher, checkCancelled -> {}, null); assertThat(context.queryResult().topDocs().totalHits, equalTo((long) numDocs)); assertTrue(collected.get()); assertNull(context.queryResult().terminatedEarly()); assertThat(context.terminateAfter(), equalTo(0)); assertThat(context.queryResult().getTotalHits(), equalTo((long) numDocs)); - QueryPhase.execute(context, contextSearcher, null); + QueryPhase.execute(context, contextSearcher, checkCancelled -> {}, null); assertThat(context.queryResult().topDocs().totalHits, equalTo((long) numDocs)); assertTrue(collected.get()); assertTrue(context.queryResult().terminatedEarly()); @@ -314,14 +313,14 @@ protected void search(List leaves, Weight weight, Collector c { context.setSize(1); - QueryPhase.execute(context, contextSearcher, null); + QueryPhase.execute(context, contextSearcher, checkCancelled -> {}, null); assertTrue(collected.get()); assertTrue(context.queryResult().terminatedEarly()); assertThat(context.queryResult().topDocs().totalHits, equalTo(1L)); assertThat(context.queryResult().topDocs().scoreDocs.length, equalTo(1)); context.setSize(0); - QueryPhase.execute(context, contextSearcher, null); + QueryPhase.execute(context, contextSearcher, checkCancelled -> {}, null); assertTrue(collected.get()); assertTrue(context.queryResult().terminatedEarly()); assertThat(context.queryResult().topDocs().totalHits, equalTo(1L)); @@ -330,7 +329,7 @@ protected void search(List leaves, Weight weight, Collector c { context.setSize(1); - QueryPhase.execute(context, contextSearcher, null); + QueryPhase.execute(context, contextSearcher, checkCancelled -> {}, null); assertTrue(collected.get()); assertTrue(context.queryResult().terminatedEarly()); assertThat(context.queryResult().topDocs().totalHits, equalTo(1L)); @@ -344,7 +343,7 @@ protected void search(List leaves, Weight weight, Collector c .build(); context.parsedQuery(new ParsedQuery(bq)); collected.set(false); - QueryPhase.execute(context, contextSearcher, null); + QueryPhase.execute(context, contextSearcher, checkCancelled -> {}, null); assertTrue(collected.get()); assertTrue(context.queryResult().terminatedEarly()); assertThat(context.queryResult().topDocs().totalHits, equalTo(1L)); @@ -353,7 +352,7 @@ protected void search(List leaves, Weight weight, Collector c context.setSize(0); context.parsedQuery(new ParsedQuery(bq)); collected.set(false); - QueryPhase.execute(context, contextSearcher, null); + QueryPhase.execute(context, contextSearcher, checkCancelled -> {}, null); assertTrue(collected.get()); assertTrue(context.queryResult().terminatedEarly()); assertThat(context.queryResult().topDocs().totalHits, equalTo(1L)); @@ -364,7 +363,7 @@ protected void search(List leaves, Weight weight, Collector c collected.set(false); TotalHitCountCollector collector = new TotalHitCountCollector(); context.queryCollectors().put(TotalHitCountCollector.class, collector); - QueryPhase.execute(context, contextSearcher, null); + QueryPhase.execute(context, contextSearcher, checkCancelled -> {}, null); assertTrue(collected.get()); assertTrue(context.queryResult().terminatedEarly()); assertThat(context.queryResult().topDocs().totalHits, equalTo(1L)); @@ -375,7 +374,7 @@ protected void search(List leaves, Weight weight, Collector c collected.set(false); TotalHitCountCollector collector = new TotalHitCountCollector(); context.queryCollectors().put(TotalHitCountCollector.class, collector); - QueryPhase.execute(context, contextSearcher, null); + QueryPhase.execute(context, contextSearcher, checkCancelled -> {}, null); assertTrue(collected.get()); assertTrue(context.queryResult().terminatedEarly()); assertThat(context.queryResult().topDocs().totalHits, equalTo(1L)); @@ -421,7 +420,7 @@ protected void search(List leaves, Weight weight, Collector c super.search(leaves, weight, collector); } }; - QueryPhase.execute(context, contextSearcher, sort); + QueryPhase.execute(context, contextSearcher, checkCancelled -> {}, sort); assertTrue(collected.get()); assertTrue(context.queryResult().terminatedEarly()); assertThat(context.queryResult().topDocs().totalHits, equalTo((long) numDocs)); @@ -434,7 +433,7 @@ protected void search(List leaves, Weight weight, Collector c { collected.set(false); context.parsedPostFilter(new ParsedQuery(new MinDocQuery(1))); - QueryPhase.execute(context, contextSearcher, sort); + QueryPhase.execute(context, contextSearcher, checkCancelled -> {}, sort); assertTrue(collected.get()); assertTrue(context.queryResult().terminatedEarly()); assertThat(context.queryResult().topDocs().totalHits, equalTo(numDocs - 1L)); @@ -446,7 +445,7 @@ protected void search(List leaves, Weight weight, Collector c final TotalHitCountCollector totalHitCountCollector = new TotalHitCountCollector(); context.queryCollectors().put(TotalHitCountCollector.class, totalHitCountCollector); collected.set(false); - QueryPhase.execute(context, contextSearcher, sort); + QueryPhase.execute(context, contextSearcher, checkCancelled -> {}, sort); assertTrue(collected.get()); assertTrue(context.queryResult().terminatedEarly()); assertThat(context.queryResult().topDocs().totalHits, equalTo((long) numDocs)); @@ -460,7 +459,7 @@ protected void search(List leaves, Weight weight, Collector c { collected.set(false); context.trackTotalHits(false); - QueryPhase.execute(context, contextSearcher, sort); + QueryPhase.execute(context, contextSearcher, checkCancelled -> {}, sort); assertTrue(collected.get()); assertTrue(context.queryResult().terminatedEarly()); assertThat(context.queryResult().topDocs().totalHits, lessThan((long) numDocs)); @@ -471,7 +470,7 @@ protected void search(List leaves, Weight weight, Collector c final TotalHitCountCollector totalHitCountCollector = new TotalHitCountCollector(); context.queryCollectors().put(TotalHitCountCollector.class, totalHitCountCollector); collected.set(false); - QueryPhase.execute(context, contextSearcher, sort); + QueryPhase.execute(context, contextSearcher, checkCancelled -> {}, sort); assertTrue(collected.get()); assertTrue(context.queryResult().terminatedEarly()); assertThat(context.queryResult().topDocs().totalHits, lessThan((long) numDocs)); @@ -521,7 +520,7 @@ protected void search(List leaves, Weight weight, Collector c } }; - QueryPhase.execute(context, contextSearcher, sort); + QueryPhase.execute(context, contextSearcher, checkCancelled -> {}, sort); assertThat(context.queryResult().topDocs().totalHits, equalTo((long) numDocs)); assertTrue(collected.get()); assertNull(context.queryResult().terminatedEarly()); @@ -530,7 +529,7 @@ protected void search(List leaves, Weight weight, Collector c int sizeMinus1 = context.queryResult().topDocs().scoreDocs.length - 1; FieldDoc lastDoc = (FieldDoc) context.queryResult().topDocs().scoreDocs[sizeMinus1]; - QueryPhase.execute(context, contextSearcher, sort); + QueryPhase.execute(context, contextSearcher, checkCancelled -> {}, sort); assertThat(context.queryResult().topDocs().totalHits, equalTo((long) numDocs)); assertTrue(collected.get()); assertTrue(context.queryResult().terminatedEarly()); diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java index 0eaa3a57df0e3..ac378b1fa843d 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java @@ -97,7 +97,6 @@ import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.json.JsonXContent; @@ -128,6 +127,7 @@ import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.MockSearchService; import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchService; import org.elasticsearch.test.client.RandomizingClient; import org.elasticsearch.test.discovery.TestZenDiscovery; import org.elasticsearch.test.disruption.NetworkDisruption; @@ -1709,7 +1709,9 @@ protected Settings nodeSettings(int nodeOrdinal) { // integration tests that usually create few documents .put(IndicesQueryCache.INDICES_QUERIES_CACHE_ALL_SEGMENTS_SETTING.getKey(), nodeOrdinal % 2 == 0) // wait short time for other active shards before actually deleting, default 30s not needed in tests - .put(IndicesStore.INDICES_STORE_DELETE_SHARD_TIMEOUT.getKey(), new TimeValue(1, TimeUnit.SECONDS)); + .put(IndicesStore.INDICES_STORE_DELETE_SHARD_TIMEOUT.getKey(), new TimeValue(1, TimeUnit.SECONDS)) + // randomly enable low-level search cancellation to make sure it does not alter results + .put(SearchService.LOW_LEVEL_CANCELLATION_SETTING.getKey(), randomBoolean()); if (rarely()) { // Sometimes adjust the minimum search thread pool size, causing // QueueResizingEsThreadPoolExecutor to be used instead of a regular diff --git a/test/framework/src/main/java/org/elasticsearch/test/client/RandomizingClient.java b/test/framework/src/main/java/org/elasticsearch/test/client/RandomizingClient.java index 6759461868a4a..b144898d643d0 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/client/RandomizingClient.java +++ b/test/framework/src/main/java/org/elasticsearch/test/client/RandomizingClient.java @@ -26,10 +26,12 @@ import org.elasticsearch.client.Client; import org.elasticsearch.client.FilterClient; import org.elasticsearch.cluster.routing.Preference; +import org.elasticsearch.common.unit.TimeValue; import java.util.Arrays; import java.util.EnumSet; import java.util.Random; +import java.util.concurrent.TimeUnit; /** A {@link Client} that randomizes request parameters. */ public class RandomizingClient extends FilterClient { @@ -39,6 +41,7 @@ public class RandomizingClient extends FilterClient { private final int batchedReduceSize; private final int maxConcurrentShardRequests; private final int preFilterShardSize; + private final boolean doTimeout; public RandomizingClient(Client client, Random random) { @@ -67,6 +70,7 @@ public RandomizingClient(Client client, Random random) { } else { preFilterShardSize = -1; } + doTimeout = random.nextBoolean(); } @Override @@ -79,6 +83,9 @@ public SearchRequestBuilder prepareSearch(String... indices) { if (preFilterShardSize != -1) { searchRequestBuilder.setPreFilterShardSize(preFilterShardSize); } + if (doTimeout) { + searchRequestBuilder.setTimeout(new TimeValue(1, TimeUnit.DAYS)); + } return searchRequestBuilder; } From 42bbc191665ae7cb607f923e894b31d0f1281f6d Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Wed, 19 Jul 2017 10:38:25 +0200 Subject: [PATCH 2/3] iter --- .../search/internal/ContextIndexSearcher.java | 8 ++++---- .../java/org/elasticsearch/search/query/QueryPhase.java | 9 ++++----- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java b/core/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java index 17d4c605084cf..815486d84a75b 100644 --- a/core/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java +++ b/core/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java @@ -151,9 +151,9 @@ public Weight createWeight(Query query, boolean needsScores, float boost) throws @Override protected void search(List leaves, Weight weight, Collector collector) throws IOException { - final Weight cancellablWeight; + final Weight cancellableWeight; if (checkCancelled != null) { - cancellablWeight = new Weight(weight.getQuery()) { + cancellableWeight = new Weight(weight.getQuery()) { @Override public void extractTerms(Set terms) { @@ -181,9 +181,9 @@ public BulkScorer bulkScorer(LeafReaderContext context) throws IOException { } }; } else { - cancellablWeight = weight; + cancellableWeight = weight; } - super.search(leaves, cancellablWeight, collector); + super.search(leaves, cancellableWeight, collector); } @Override diff --git a/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java b/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java index 57d5b5699b6bb..f914c921a47f5 100644 --- a/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java +++ b/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java @@ -24,7 +24,6 @@ import org.apache.lucene.queries.SearchAfterSortedDocQuery; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.CollectionTerminatedException; import org.apache.lucene.search.Collector; import org.apache.lucene.search.ConstantScoreQuery; import org.apache.lucene.search.EarlyTerminatingSortingCollector; @@ -34,7 +33,6 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.Sort; -import org.apache.lucene.search.TimeLimitingCollector; import org.apache.lucene.search.TopDocs; import org.apache.lucene.util.Counter; import org.elasticsearch.action.search.SearchTask; @@ -209,8 +207,7 @@ static boolean execute(SearchContext searchContext, final IndexSearcher searcher timeoutRunnable = () -> { final long time = counter.get(); if (time > maxTime) { - queryResult.searchTimedOut(true); - throw new CollectionTerminatedException(); + throw new TimeExceededException(); } }; } else { @@ -274,7 +271,7 @@ static boolean execute(SearchContext searchContext, final IndexSearcher searcher if (shouldCollect) { searcher.search(query, queryCollector); } - } catch (TimeLimitingCollector.TimeExceededException e) { + } catch (TimeExceededException e) { assert timeoutSet : "TimeExceededException thrown even though timeout wasn't set"; queryResult.searchTimedOut(true); } finally { @@ -328,4 +325,6 @@ static boolean canEarlyTerminate(Sort indexSort, SearchContext context) { final Sort sort = context.sort() == null ? Sort.RELEVANCE : context.sort().sort; return indexSort != null && EarlyTerminatingSortingCollector.canEarlyTerminate(sort, indexSort); } + + private static class TimeExceededException extends RuntimeException {} } From 9f958258879825a997604915311228d19e796936 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Wed, 19 Jul 2017 14:06:09 +0200 Subject: [PATCH 3/3] iter --- .../indices/IndicesRequestCacheIT.java | 316 +++++++++--------- 1 file changed, 162 insertions(+), 154 deletions(-) diff --git a/core/src/test/java/org/elasticsearch/indices/IndicesRequestCacheIT.java b/core/src/test/java/org/elasticsearch/indices/IndicesRequestCacheIT.java index 534bd23ea5de9..c672d7e5bc509 100644 --- a/core/src/test/java/org/elasticsearch/indices/IndicesRequestCacheIT.java +++ b/core/src/test/java/org/elasticsearch/indices/IndicesRequestCacheIT.java @@ -22,6 +22,7 @@ import org.elasticsearch.action.admin.indices.alias.Alias; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; +import org.elasticsearch.client.Client; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; @@ -47,29 +48,30 @@ public class IndicesRequestCacheIT extends ESIntegTestCase { // One of the primary purposes of the query cache is to cache aggs results public void testCacheAggs() throws Exception { - assertAcked(client().admin().indices().prepareCreate("index") + Client client = client(); + assertAcked(client.admin().indices().prepareCreate("index") .addMapping("type", "f", "type=date") .setSettings(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true).get()); indexRandom(true, - client().prepareIndex("index", "type").setSource("f", "2014-03-10T00:00:00.000Z"), - client().prepareIndex("index", "type").setSource("f", "2014-05-13T00:00:00.000Z")); + client.prepareIndex("index", "type").setSource("f", "2014-03-10T00:00:00.000Z"), + client.prepareIndex("index", "type").setSource("f", "2014-05-13T00:00:00.000Z")); ensureSearchable("index"); // This is not a random example: serialization with time zones writes shared strings // which used to not work well with the query cache because of the handles stream output // see #9500 - final SearchResponse r1 = client().prepareSearch("index").setSize(0).setSearchType(SearchType.QUERY_THEN_FETCH) + final SearchResponse r1 = client.prepareSearch("index").setSize(0).setSearchType(SearchType.QUERY_THEN_FETCH) .addAggregation(dateHistogram("histo").field("f").timeZone(DateTimeZone.forID("+01:00")).minDocCount(0) .dateHistogramInterval(DateHistogramInterval.MONTH)) .get(); assertSearchResponse(r1); // The cached is actually used - assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache() + assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache() .getMemorySizeInBytes(), greaterThan(0L)); for (int i = 0; i < 10; ++i) { - final SearchResponse r2 = client().prepareSearch("index").setSize(0) + final SearchResponse r2 = client.prepareSearch("index").setSize(0) .setSearchType(SearchType.QUERY_THEN_FETCH).addAggregation(dateHistogram("histo").field("f") .timeZone(DateTimeZone.forID("+01:00")).minDocCount(0).dateHistogramInterval(DateHistogramInterval.MONTH)) .get(); @@ -89,411 +91,417 @@ public void testCacheAggs() throws Exception { } public void testQueryRewrite() throws Exception { - assertAcked(client().admin().indices().prepareCreate("index").addMapping("type", "s", "type=date") + Client client = client(); + assertAcked(client.admin().indices().prepareCreate("index").addMapping("type", "s", "type=date") .setSettings(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true, IndexMetaData.SETTING_NUMBER_OF_SHARDS, 5, IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) .get()); - indexRandom(true, client().prepareIndex("index", "type", "1").setRouting("1").setSource("s", "2016-03-19"), - client().prepareIndex("index", "type", "2").setRouting("1").setSource("s", "2016-03-20"), - client().prepareIndex("index", "type", "3").setRouting("1").setSource("s", "2016-03-21"), - client().prepareIndex("index", "type", "4").setRouting("2").setSource("s", "2016-03-22"), - client().prepareIndex("index", "type", "5").setRouting("2").setSource("s", "2016-03-23"), - client().prepareIndex("index", "type", "6").setRouting("2").setSource("s", "2016-03-24"), - client().prepareIndex("index", "type", "7").setRouting("3").setSource("s", "2016-03-25"), - client().prepareIndex("index", "type", "8").setRouting("3").setSource("s", "2016-03-26"), - client().prepareIndex("index", "type", "9").setRouting("3").setSource("s", "2016-03-27")); + indexRandom(true, client.prepareIndex("index", "type", "1").setRouting("1").setSource("s", "2016-03-19"), + client.prepareIndex("index", "type", "2").setRouting("1").setSource("s", "2016-03-20"), + client.prepareIndex("index", "type", "3").setRouting("1").setSource("s", "2016-03-21"), + client.prepareIndex("index", "type", "4").setRouting("2").setSource("s", "2016-03-22"), + client.prepareIndex("index", "type", "5").setRouting("2").setSource("s", "2016-03-23"), + client.prepareIndex("index", "type", "6").setRouting("2").setSource("s", "2016-03-24"), + client.prepareIndex("index", "type", "7").setRouting("3").setSource("s", "2016-03-25"), + client.prepareIndex("index", "type", "8").setRouting("3").setSource("s", "2016-03-26"), + client.prepareIndex("index", "type", "9").setRouting("3").setSource("s", "2016-03-27")); ensureSearchable("index"); - assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), + assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), equalTo(0L)); - assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), + assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), equalTo(0L)); - final SearchResponse r1 = client().prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0) + final SearchResponse r1 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0) .setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-19").lte("2016-03-25")).setPreFilterShardSize(Integer.MAX_VALUE) .get(); assertSearchResponse(r1); assertThat(r1.getHits().getTotalHits(), equalTo(7L)); - assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), + assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), equalTo(0L)); - assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), + assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), equalTo(5L)); - final SearchResponse r2 = client().prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0) + final SearchResponse r2 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0) .setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-20").lte("2016-03-26")) .setPreFilterShardSize(Integer.MAX_VALUE).get(); assertSearchResponse(r2); assertThat(r2.getHits().getTotalHits(), equalTo(7L)); - assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), + assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), equalTo(3L)); - assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), + assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), equalTo(7L)); - final SearchResponse r3 = client().prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0) + final SearchResponse r3 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0) .setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-21").lte("2016-03-27")).setPreFilterShardSize(Integer.MAX_VALUE) .get(); assertSearchResponse(r3); assertThat(r3.getHits().getTotalHits(), equalTo(7L)); - assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), + assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), equalTo(6L)); - assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), + assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), equalTo(9L)); } public void testQueryRewriteMissingValues() throws Exception { - assertAcked(client().admin().indices().prepareCreate("index").addMapping("type", "s", "type=date") + Client client = client(); + assertAcked(client.admin().indices().prepareCreate("index").addMapping("type", "s", "type=date") .setSettings(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true, IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1, IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) .get()); - indexRandom(true, client().prepareIndex("index", "type", "1").setSource("s", "2016-03-19"), - client().prepareIndex("index", "type", "2").setSource("s", "2016-03-20"), - client().prepareIndex("index", "type", "3").setSource("s", "2016-03-21"), - client().prepareIndex("index", "type", "4").setSource("s", "2016-03-22"), - client().prepareIndex("index", "type", "5").setSource("s", "2016-03-23"), - client().prepareIndex("index", "type", "6").setSource("s", "2016-03-24"), - client().prepareIndex("index", "type", "7").setSource("other", "value"), - client().prepareIndex("index", "type", "8").setSource("s", "2016-03-26"), - client().prepareIndex("index", "type", "9").setSource("s", "2016-03-27")); + indexRandom(true, client.prepareIndex("index", "type", "1").setSource("s", "2016-03-19"), + client.prepareIndex("index", "type", "2").setSource("s", "2016-03-20"), + client.prepareIndex("index", "type", "3").setSource("s", "2016-03-21"), + client.prepareIndex("index", "type", "4").setSource("s", "2016-03-22"), + client.prepareIndex("index", "type", "5").setSource("s", "2016-03-23"), + client.prepareIndex("index", "type", "6").setSource("s", "2016-03-24"), + client.prepareIndex("index", "type", "7").setSource("other", "value"), + client.prepareIndex("index", "type", "8").setSource("s", "2016-03-26"), + client.prepareIndex("index", "type", "9").setSource("s", "2016-03-27")); ensureSearchable("index"); - assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), + assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), equalTo(0L)); - assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), + assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), equalTo(0L)); - final SearchResponse r1 = client().prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0) + final SearchResponse r1 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0) .setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-19").lte("2016-03-28")).get(); assertSearchResponse(r1); assertThat(r1.getHits().getTotalHits(), equalTo(8L)); - assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), + assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), equalTo(0L)); - assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), + assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), equalTo(1L)); - final SearchResponse r2 = client().prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0) + final SearchResponse r2 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0) .setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-19").lte("2016-03-28")).get(); assertSearchResponse(r2); assertThat(r2.getHits().getTotalHits(), equalTo(8L)); - assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), + assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), equalTo(1L)); - assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), + assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), equalTo(1L)); - final SearchResponse r3 = client().prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0) + final SearchResponse r3 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0) .setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-19").lte("2016-03-28")).get(); assertSearchResponse(r3); assertThat(r3.getHits().getTotalHits(), equalTo(8L)); - assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), + assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), equalTo(2L)); - assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), + assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), equalTo(1L)); } public void testQueryRewriteDates() throws Exception { - assertAcked(client().admin().indices().prepareCreate("index").addMapping("type", "d", "type=date") + Client client = client(); + assertAcked(client.admin().indices().prepareCreate("index").addMapping("type", "d", "type=date") .setSettings(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true, IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1, IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) .get()); - indexRandom(true, client().prepareIndex("index", "type", "1").setSource("d", "2014-01-01T00:00:00"), - client().prepareIndex("index", "type", "2").setSource("d", "2014-02-01T00:00:00"), - client().prepareIndex("index", "type", "3").setSource("d", "2014-03-01T00:00:00"), - client().prepareIndex("index", "type", "4").setSource("d", "2014-04-01T00:00:00"), - client().prepareIndex("index", "type", "5").setSource("d", "2014-05-01T00:00:00"), - client().prepareIndex("index", "type", "6").setSource("d", "2014-06-01T00:00:00"), - client().prepareIndex("index", "type", "7").setSource("d", "2014-07-01T00:00:00"), - client().prepareIndex("index", "type", "8").setSource("d", "2014-08-01T00:00:00"), - client().prepareIndex("index", "type", "9").setSource("d", "2014-09-01T00:00:00")); + indexRandom(true, client.prepareIndex("index", "type", "1").setSource("d", "2014-01-01T00:00:00"), + client.prepareIndex("index", "type", "2").setSource("d", "2014-02-01T00:00:00"), + client.prepareIndex("index", "type", "3").setSource("d", "2014-03-01T00:00:00"), + client.prepareIndex("index", "type", "4").setSource("d", "2014-04-01T00:00:00"), + client.prepareIndex("index", "type", "5").setSource("d", "2014-05-01T00:00:00"), + client.prepareIndex("index", "type", "6").setSource("d", "2014-06-01T00:00:00"), + client.prepareIndex("index", "type", "7").setSource("d", "2014-07-01T00:00:00"), + client.prepareIndex("index", "type", "8").setSource("d", "2014-08-01T00:00:00"), + client.prepareIndex("index", "type", "9").setSource("d", "2014-09-01T00:00:00")); ensureSearchable("index"); - assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), + assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), equalTo(0L)); - assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), + assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), equalTo(0L)); - final SearchResponse r1 = client().prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0) + final SearchResponse r1 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0) .setQuery(QueryBuilders.rangeQuery("d").gte("2013-01-01T00:00:00").lte("now")) .get(); assertSearchResponse(r1); assertThat(r1.getHits().getTotalHits(), equalTo(9L)); - assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), + assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), equalTo(0L)); - assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), + assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), equalTo(1L)); - final SearchResponse r2 = client().prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0) + final SearchResponse r2 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0) .setQuery(QueryBuilders.rangeQuery("d").gte("2013-01-01T00:00:00").lte("now")) .get(); assertSearchResponse(r2); assertThat(r2.getHits().getTotalHits(), equalTo(9L)); - assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), + assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), equalTo(1L)); - assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), + assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), equalTo(1L)); - final SearchResponse r3 = client().prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0) + final SearchResponse r3 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0) .setQuery(QueryBuilders.rangeQuery("d").gte("2013-01-01T00:00:00").lte("now")) .get(); assertSearchResponse(r3); assertThat(r3.getHits().getTotalHits(), equalTo(9L)); - assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), + assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), equalTo(2L)); - assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), + assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), equalTo(1L)); } public void testQueryRewriteDatesWithNow() throws Exception { - assertAcked(client().admin().indices().prepareCreate("index-1").addMapping("type", "d", "type=date") + Client client = client(); + assertAcked(client.admin().indices().prepareCreate("index-1").addMapping("type", "d", "type=date") .setSettings(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true, IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1, IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) .get()); - assertAcked(client().admin().indices().prepareCreate("index-2").addMapping("type", "d", "type=date") + assertAcked(client.admin().indices().prepareCreate("index-2").addMapping("type", "d", "type=date") .setSettings(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true, IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1, IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) .get()); - assertAcked(client().admin().indices().prepareCreate("index-3").addMapping("type", "d", "type=date") + assertAcked(client.admin().indices().prepareCreate("index-3").addMapping("type", "d", "type=date") .setSettings(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true, IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1, IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) .get()); DateTime now = new DateTime(ISOChronology.getInstanceUTC()); - indexRandom(true, client().prepareIndex("index-1", "type", "1").setSource("d", now), - client().prepareIndex("index-1", "type", "2").setSource("d", now.minusDays(1)), - client().prepareIndex("index-1", "type", "3").setSource("d", now.minusDays(2)), - client().prepareIndex("index-2", "type", "4").setSource("d", now.minusDays(3)), - client().prepareIndex("index-2", "type", "5").setSource("d", now.minusDays(4)), - client().prepareIndex("index-2", "type", "6").setSource("d", now.minusDays(5)), - client().prepareIndex("index-3", "type", "7").setSource("d", now.minusDays(6)), - client().prepareIndex("index-3", "type", "8").setSource("d", now.minusDays(7)), - client().prepareIndex("index-3", "type", "9").setSource("d", now.minusDays(8))); + indexRandom(true, client.prepareIndex("index-1", "type", "1").setSource("d", now), + client.prepareIndex("index-1", "type", "2").setSource("d", now.minusDays(1)), + client.prepareIndex("index-1", "type", "3").setSource("d", now.minusDays(2)), + client.prepareIndex("index-2", "type", "4").setSource("d", now.minusDays(3)), + client.prepareIndex("index-2", "type", "5").setSource("d", now.minusDays(4)), + client.prepareIndex("index-2", "type", "6").setSource("d", now.minusDays(5)), + client.prepareIndex("index-3", "type", "7").setSource("d", now.minusDays(6)), + client.prepareIndex("index-3", "type", "8").setSource("d", now.minusDays(7)), + client.prepareIndex("index-3", "type", "9").setSource("d", now.minusDays(8))); ensureSearchable("index-1", "index-2", "index-3"); assertThat( - client().admin().indices().prepareStats("index-1").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), + client.admin().indices().prepareStats("index-1").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), equalTo(0L)); assertThat( - client().admin().indices().prepareStats("index-1").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), + client.admin().indices().prepareStats("index-1").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), equalTo(0L)); assertThat( - client().admin().indices().prepareStats("index-2").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), + client.admin().indices().prepareStats("index-2").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), equalTo(0L)); assertThat( - client().admin().indices().prepareStats("index-2").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), + client.admin().indices().prepareStats("index-2").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), equalTo(0L)); assertThat( - client().admin().indices().prepareStats("index-3").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), + client.admin().indices().prepareStats("index-3").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), equalTo(0L)); assertThat( - client().admin().indices().prepareStats("index-3").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), + client.admin().indices().prepareStats("index-3").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), equalTo(0L)); - final SearchResponse r1 = client().prepareSearch("index-*").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0) + final SearchResponse r1 = client.prepareSearch("index-*").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0) .setQuery(QueryBuilders.rangeQuery("d").gte("now-7d/d").lte("now")).get(); assertSearchResponse(r1); assertThat(r1.getHits().getTotalHits(), equalTo(8L)); assertThat( - client().admin().indices().prepareStats("index-1").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), + client.admin().indices().prepareStats("index-1").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), equalTo(0L)); assertThat( - client().admin().indices().prepareStats("index-1").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), + client.admin().indices().prepareStats("index-1").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), equalTo(1L)); assertThat( - client().admin().indices().prepareStats("index-2").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), + client.admin().indices().prepareStats("index-2").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), equalTo(0L)); assertThat( - client().admin().indices().prepareStats("index-2").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), + client.admin().indices().prepareStats("index-2").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), equalTo(1L)); // Because the query will INTERSECT with the 3rd index it will not be // rewritten and will still contain `now` so won't be recorded as a // cache miss or cache hit since queries containing now can't be cached assertThat( - client().admin().indices().prepareStats("index-3").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), + client.admin().indices().prepareStats("index-3").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), equalTo(0L)); assertThat( - client().admin().indices().prepareStats("index-3").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), + client.admin().indices().prepareStats("index-3").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), equalTo(0L)); - final SearchResponse r2 = client().prepareSearch("index-*").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0) + final SearchResponse r2 = client.prepareSearch("index-*").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0) .setQuery(QueryBuilders.rangeQuery("d").gte("now-7d/d").lte("now")).get(); assertSearchResponse(r2); assertThat(r2.getHits().getTotalHits(), equalTo(8L)); assertThat( - client().admin().indices().prepareStats("index-1").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), + client.admin().indices().prepareStats("index-1").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), equalTo(1L)); assertThat( - client().admin().indices().prepareStats("index-1").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), + client.admin().indices().prepareStats("index-1").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), equalTo(1L)); assertThat( - client().admin().indices().prepareStats("index-2").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), + client.admin().indices().prepareStats("index-2").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), equalTo(1L)); assertThat( - client().admin().indices().prepareStats("index-2").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), + client.admin().indices().prepareStats("index-2").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), equalTo(1L)); assertThat( - client().admin().indices().prepareStats("index-3").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), + client.admin().indices().prepareStats("index-3").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), equalTo(0L)); assertThat( - client().admin().indices().prepareStats("index-3").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), + client.admin().indices().prepareStats("index-3").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), equalTo(0L)); - final SearchResponse r3 = client().prepareSearch("index-*").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0) + final SearchResponse r3 = client.prepareSearch("index-*").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0) .setQuery(QueryBuilders.rangeQuery("d").gte("now-7d/d").lte("now")).get(); assertSearchResponse(r3); assertThat(r3.getHits().getTotalHits(), equalTo(8L)); assertThat( - client().admin().indices().prepareStats("index-1").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), + client.admin().indices().prepareStats("index-1").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), equalTo(2L)); assertThat( - client().admin().indices().prepareStats("index-1").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), + client.admin().indices().prepareStats("index-1").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), equalTo(1L)); assertThat( - client().admin().indices().prepareStats("index-2").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), + client.admin().indices().prepareStats("index-2").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), equalTo(2L)); assertThat( - client().admin().indices().prepareStats("index-2").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), + client.admin().indices().prepareStats("index-2").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), equalTo(1L)); assertThat( - client().admin().indices().prepareStats("index-3").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), + client.admin().indices().prepareStats("index-3").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), equalTo(0L)); assertThat( - client().admin().indices().prepareStats("index-3").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), + client.admin().indices().prepareStats("index-3").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), equalTo(0L)); } public void testCanCache() throws Exception { - assertAcked(client().admin().indices().prepareCreate("index").addMapping("type", "s", "type=date") + Client client = client(); + assertAcked(client.admin().indices().prepareCreate("index").addMapping("type", "s", "type=date") .setSettings(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true, IndexMetaData.SETTING_NUMBER_OF_SHARDS, 2, IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) .get()); - indexRandom(true, client().prepareIndex("index", "type", "1").setRouting("1").setSource("s", "2016-03-19"), - client().prepareIndex("index", "type", "2").setRouting("1").setSource("s", "2016-03-20"), - client().prepareIndex("index", "type", "3").setRouting("1").setSource("s", "2016-03-21"), - client().prepareIndex("index", "type", "4").setRouting("2").setSource("s", "2016-03-22"), - client().prepareIndex("index", "type", "5").setRouting("2").setSource("s", "2016-03-23"), - client().prepareIndex("index", "type", "6").setRouting("2").setSource("s", "2016-03-24"), - client().prepareIndex("index", "type", "7").setRouting("3").setSource("s", "2016-03-25"), - client().prepareIndex("index", "type", "8").setRouting("3").setSource("s", "2016-03-26"), - client().prepareIndex("index", "type", "9").setRouting("3").setSource("s", "2016-03-27")); + indexRandom(true, client.prepareIndex("index", "type", "1").setRouting("1").setSource("s", "2016-03-19"), + client.prepareIndex("index", "type", "2").setRouting("1").setSource("s", "2016-03-20"), + client.prepareIndex("index", "type", "3").setRouting("1").setSource("s", "2016-03-21"), + client.prepareIndex("index", "type", "4").setRouting("2").setSource("s", "2016-03-22"), + client.prepareIndex("index", "type", "5").setRouting("2").setSource("s", "2016-03-23"), + client.prepareIndex("index", "type", "6").setRouting("2").setSource("s", "2016-03-24"), + client.prepareIndex("index", "type", "7").setRouting("3").setSource("s", "2016-03-25"), + client.prepareIndex("index", "type", "8").setRouting("3").setSource("s", "2016-03-26"), + client.prepareIndex("index", "type", "9").setRouting("3").setSource("s", "2016-03-27")); ensureSearchable("index"); - assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), + assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), equalTo(0L)); - assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), + assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), equalTo(0L)); // If size > 0 we should no cache by default - final SearchResponse r1 = client().prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(1) + final SearchResponse r1 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(1) .setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-19").lte("2016-03-25")).get(); assertSearchResponse(r1); assertThat(r1.getHits().getTotalHits(), equalTo(7L)); - assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), + assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), equalTo(0L)); - assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), + assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), equalTo(0L)); // If search type is DFS_QUERY_THEN_FETCH we should not cache - final SearchResponse r2 = client().prepareSearch("index").setSearchType(SearchType.DFS_QUERY_THEN_FETCH).setSize(0) + final SearchResponse r2 = client.prepareSearch("index").setSearchType(SearchType.DFS_QUERY_THEN_FETCH).setSize(0) .setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-20").lte("2016-03-26")).get(); assertSearchResponse(r2); assertThat(r2.getHits().getTotalHits(), equalTo(7L)); - assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), + assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), equalTo(0L)); - assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), + assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), equalTo(0L)); // If search type is DFS_QUERY_THEN_FETCH we should not cache even if // the cache flag is explicitly set on the request - final SearchResponse r3 = client().prepareSearch("index").setSearchType(SearchType.DFS_QUERY_THEN_FETCH).setSize(0) + final SearchResponse r3 = client.prepareSearch("index").setSearchType(SearchType.DFS_QUERY_THEN_FETCH).setSize(0) .setRequestCache(true).setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-20").lte("2016-03-26")).get(); assertSearchResponse(r3); assertThat(r3.getHits().getTotalHits(), equalTo(7L)); - assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), + assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), equalTo(0L)); - assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), + assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), equalTo(0L)); // If the request has an non-filter aggregation containing now we should not cache - final SearchResponse r5 = client().prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0) + final SearchResponse r5 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0) .setRequestCache(true).setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-20").lte("2016-03-26")) .addAggregation(dateRange("foo").field("s").addRange("now-10y", "now")).get(); assertSearchResponse(r5); assertThat(r5.getHits().getTotalHits(), equalTo(7L)); - assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), + assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), equalTo(0L)); - assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), + assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), equalTo(0L)); // If size > 1 and cache flag is set on the request we should cache - final SearchResponse r6 = client().prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(1) + final SearchResponse r6 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(1) .setRequestCache(true).setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-21").lte("2016-03-27")).get(); assertSearchResponse(r6); assertThat(r6.getHits().getTotalHits(), equalTo(7L)); - assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), + assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), equalTo(0L)); - assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), + assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), equalTo(2L)); // If the request has a filter aggregation containing now we should cache since it gets rewritten - final SearchResponse r4 = client().prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0) + final SearchResponse r4 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0) .setRequestCache(true).setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-20").lte("2016-03-26")) .addAggregation(filter("foo", QueryBuilders.rangeQuery("s").from("now-10y").to("now"))).get(); assertSearchResponse(r4); assertThat(r4.getHits().getTotalHits(), equalTo(7L)); - assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), + assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), equalTo(0L)); - assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), + assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), equalTo(4L)); } public void testCacheWithFilteredAlias() { - assertAcked(client().admin().indices().prepareCreate("index").addMapping("type", "created_at", "type=date") + Client client = client(); + assertAcked(client.admin().indices().prepareCreate("index").addMapping("type", "created_at", "type=date") .setSettings(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true, IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1, IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) .addAlias(new Alias("last_week").filter(QueryBuilders.rangeQuery("created_at").gte("now-7d/d"))) .get()); DateTime now = new DateTime(DateTimeZone.UTC); - client().prepareIndex("index", "type", "1").setRouting("1").setSource("created_at", + client.prepareIndex("index", "type", "1").setRouting("1").setSource("created_at", DateTimeFormat.forPattern("YYYY-MM-dd").print(now)).get(); refresh(); - assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), + assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), equalTo(0L)); - assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), + assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), equalTo(0L)); - SearchResponse r1 = client().prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0) + SearchResponse r1 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0) .setQuery(QueryBuilders.rangeQuery("created_at").gte("now-7d/d")).get(); assertSearchResponse(r1); assertThat(r1.getHits().getTotalHits(), equalTo(1L)); - assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), + assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), equalTo(0L)); - assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), + assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), equalTo(1L)); - r1 = client().prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0) + r1 = client.prepareSearch("index").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0) .setQuery(QueryBuilders.rangeQuery("created_at").gte("now-7d/d")).get(); assertSearchResponse(r1); assertThat(r1.getHits().getTotalHits(), equalTo(1L)); - assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), + assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), equalTo(1L)); - assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), + assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), equalTo(1L)); - r1 = client().prepareSearch("last_week").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0).get(); + r1 = client.prepareSearch("last_week").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0).get(); assertSearchResponse(r1); assertThat(r1.getHits().getTotalHits(), equalTo(1L)); - assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), + assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), equalTo(1L)); - assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), + assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), equalTo(2L)); - r1 = client().prepareSearch("last_week").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0).get(); + r1 = client.prepareSearch("last_week").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0).get(); assertSearchResponse(r1); assertThat(r1.getHits().getTotalHits(), equalTo(1L)); - assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), + assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getHitCount(), equalTo(2L)); - assertThat(client().admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), + assertThat(client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMissCount(), equalTo(2L)); }