Skip to content

Commit

Permalink
Merge branch 'master' into ccr
Browse files Browse the repository at this point in the history
* master:
  Fixed byte buffer leak in Netty4 request handler
  Avoid uid creation in ParsedDocument (#27241)
  Rander sum as zero if count is zero for stats aggregation (#26893) (#27193)
  Add additional explanations around discovery.zen.ping_timeout (#27231)
  Remove unused searcher parameter in SearchService#createContext (#27227)
  Upgrade to Lucene 7.1 (#27225)
  Move IndexShard#getWritingBytes() under InternalEngine (#27209)
  Adjust bwc version for exists query tests
  Introducing took time for _msearch
  • Loading branch information
martijnvg committed Nov 3, 2017
2 parents 57b24a9 + f9e755f commit 0289330
Show file tree
Hide file tree
Showing 68 changed files with 423 additions and 158 deletions.
2 changes: 1 addition & 1 deletion buildSrc/version.properties
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# When updating elasticsearch, please update 'rest' version in core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
elasticsearch = 7.0.0-alpha1
lucene = 7.1.0-snapshot-f33ed4ba12a
lucene = 7.1.0

# optional dependencies
spatial4j = 0.6
Expand Down

This file was deleted.

1 change: 1 addition & 0 deletions core/licenses/lucene-analyzers-common-7.1.0.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
a508bf6b580471ee568dab7d2acfedfa5aadce70

This file was deleted.

1 change: 1 addition & 0 deletions core/licenses/lucene-backward-codecs-7.1.0.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
804a7ce82bba3d085733486bfde4846ecb77ce01

This file was deleted.

1 change: 1 addition & 0 deletions core/licenses/lucene-core-7.1.0.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
dd291b7ebf4845483895724d2562214dc7f40049

This file was deleted.

1 change: 1 addition & 0 deletions core/licenses/lucene-grouping-7.1.0.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
0732d16c16421fca058a2a07ca4081ec7696365b

This file was deleted.

1 change: 1 addition & 0 deletions core/licenses/lucene-highlighter-7.1.0.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
596550daabae765ad685112e0fe7c4f0fdfccb3f

This file was deleted.

1 change: 1 addition & 0 deletions core/licenses/lucene-join-7.1.0.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
5f26dd64c195258a81175772ef7fe105e7d60a26

This file was deleted.

1 change: 1 addition & 0 deletions core/licenses/lucene-memory-7.1.0.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
3ef64c58d0c09ca40d848efa96b585b7476271f2

This file was deleted.

1 change: 1 addition & 0 deletions core/licenses/lucene-misc-7.1.0.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
1496ee5fa62206ee5ddf51042a340d6a9ee3b5de

This file was deleted.

1 change: 1 addition & 0 deletions core/licenses/lucene-queries-7.1.0.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
1554920ab207a3245fa408d022a5c90ad3a1fea3

This file was deleted.

1 change: 1 addition & 0 deletions core/licenses/lucene-queryparser-7.1.0.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
5767c15c5ee97926829fd8a4337e434fa95f3c08

This file was deleted.

1 change: 1 addition & 0 deletions core/licenses/lucene-sandbox-7.1.0.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
691f7b9ac05f3ad2ac7e80733ef70247904bd3ae

This file was deleted.

1 change: 1 addition & 0 deletions core/licenses/lucene-spatial-7.1.0.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
6c64c04d802badb800516a8a574cb993929c3805

This file was deleted.

1 change: 1 addition & 0 deletions core/licenses/lucene-spatial-extras-7.1.0.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
3f1bc1aada8f06b176b782da24b9d7ad9641c41a

This file was deleted.

1 change: 1 addition & 0 deletions core/licenses/lucene-spatial3d-7.1.0.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
8ded650aed23efb775f17be496e3e3870214e23b

This file was deleted.

1 change: 1 addition & 0 deletions core/licenses/lucene-suggest-7.1.0.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
8d0ed1589ebdccf34e888c6efc0134a13a238c85
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,7 @@ protected ShardValidateQueryResponse shardOperation(ShardValidateQueryRequest re
String error = null;
ShardSearchLocalRequest shardSearchLocalRequest = new ShardSearchLocalRequest(request.shardId(), request.types(),
request.nowInMillis(), request.filteringAliases());
SearchContext searchContext = searchService.createSearchContext(shardSearchLocalRequest, SearchService.NO_TIMEOUT, null);
SearchContext searchContext = searchService.createSearchContext(shardSearchLocalRequest, SearchService.NO_TIMEOUT);
try {
ParsedQuery parsedQuery = searchContext.getQueryShardContext().toQuery(request.query());
searchContext.parsedQuery(parsedQuery);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ protected void resolveRequest(ClusterState state, InternalRequest request) {
protected ExplainResponse shardOperation(ExplainRequest request, ShardId shardId) throws IOException {
ShardSearchLocalRequest shardSearchLocalRequest = new ShardSearchLocalRequest(shardId,
new String[]{request.type()}, request.nowInMillis, request.filteringAlias());
SearchContext context = searchService.createSearchContext(shardSearchLocalRequest, SearchService.NO_TIMEOUT, null);
SearchContext context = searchService.createSearchContext(shardSearchLocalRequest, SearchService.NO_TIMEOUT);
Engine.GetResult result = null;
try {
Term uidTerm = context.mapperService().createUidTerm(request.type(), request.id());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,14 @@

import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.ExceptionsHelper;
import org.elasticsearch.Version;
import org.elasticsearch.action.ActionResponse;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Streamable;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.ToXContentObject;
import org.elasticsearch.common.xcontent.XContentBuilder;

Expand Down Expand Up @@ -112,11 +114,14 @@ public Exception getFailure() {

private Item[] items;

private long tookInMillis;

MultiSearchResponse() {
}

public MultiSearchResponse(Item[] items) {
public MultiSearchResponse(Item[] items, long tookInMillis) {
this.items = items;
this.tookInMillis = tookInMillis;
}

@Override
Expand All @@ -131,13 +136,23 @@ public Item[] getResponses() {
return this.items;
}

/**
* How long the msearch took.
*/
public TimeValue getTook() {
return new TimeValue(tookInMillis);
}

@Override
public void readFrom(StreamInput in) throws IOException {
super.readFrom(in);
items = new Item[in.readVInt()];
for (int i = 0; i < items.length; i++) {
items[i] = Item.readItem(in);
}
if (in.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) {
tookInMillis = in.readVLong();
}
}

@Override
Expand All @@ -147,11 +162,15 @@ public void writeTo(StreamOutput out) throws IOException {
for (Item item : items) {
item.writeTo(out);
}
if (out.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) {
out.writeVLong(tookInMillis);
}
}

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject();
builder.field("took", tookInMillis);
builder.startArray(Fields.RESPONSES);
for (Item item : items) {
builder.startObject();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,16 +34,18 @@
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.TransportService;

import java.util.List;
import java.util.Queue;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.LongSupplier;

public class TransportMultiSearchAction extends HandledTransportAction<MultiSearchRequest, MultiSearchResponse> {

private final int availableProcessors;
private final ClusterService clusterService;
private final TransportAction<SearchRequest, SearchResponse> searchAction;
private final LongSupplier relativeTimeProvider;

@Inject
public TransportMultiSearchAction(Settings settings, ThreadPool threadPool, TransportService transportService,
Expand All @@ -53,19 +55,23 @@ public TransportMultiSearchAction(Settings settings, ThreadPool threadPool, Tran
this.clusterService = clusterService;
this.searchAction = searchAction;
this.availableProcessors = EsExecutors.numberOfProcessors(settings);
this.relativeTimeProvider = System::nanoTime;
}

TransportMultiSearchAction(ThreadPool threadPool, ActionFilters actionFilters, TransportService transportService,
ClusterService clusterService, TransportAction<SearchRequest, SearchResponse> searchAction,
IndexNameExpressionResolver resolver, int availableProcessors) {
IndexNameExpressionResolver resolver, int availableProcessors, LongSupplier relativeTimeProvider) {
super(Settings.EMPTY, MultiSearchAction.NAME, threadPool, transportService, actionFilters, resolver, MultiSearchRequest::new);
this.clusterService = clusterService;
this.searchAction = searchAction;
this.availableProcessors = availableProcessors;
this.relativeTimeProvider = relativeTimeProvider;
}

@Override
protected void doExecute(MultiSearchRequest request, ActionListener<MultiSearchResponse> listener) {
final long relativeStartTime = relativeTimeProvider.getAsLong();

ClusterState clusterState = clusterService.state();
clusterState.blocks().globalBlockedRaiseException(ClusterBlockLevel.READ);

Expand All @@ -85,7 +91,7 @@ protected void doExecute(MultiSearchRequest request, ActionListener<MultiSearchR
final AtomicInteger responseCounter = new AtomicInteger(numRequests);
int numConcurrentSearches = Math.min(numRequests, maxConcurrentSearches);
for (int i = 0; i < numConcurrentSearches; i++) {
executeSearch(searchRequestSlots, responses, responseCounter, listener);
executeSearch(searchRequestSlots, responses, responseCounter, listener, relativeStartTime);
}
}

Expand All @@ -111,11 +117,12 @@ static int defaultMaxConcurrentSearches(int availableProcessors, ClusterState st
* @param responseCounter incremented on each response
* @param listener the listener attached to the multi-search request
*/
private void executeSearch(
void executeSearch(
final Queue<SearchRequestSlot> requests,
final AtomicArray<MultiSearchResponse.Item> responses,
final AtomicInteger responseCounter,
final ActionListener<MultiSearchResponse> listener) {
final ActionListener<MultiSearchResponse> listener,
final long relativeStartTime) {
SearchRequestSlot request = requests.poll();
if (request == null) {
/*
Expand Down Expand Up @@ -155,16 +162,25 @@ private void handleResponse(final int responseSlot, final MultiSearchResponse.It
} else {
if (thread == Thread.currentThread()) {
// we are on the same thread, we need to fork to another thread to avoid recursive stack overflow on a single thread
threadPool.generic().execute(() -> executeSearch(requests, responses, responseCounter, listener));
threadPool.generic()
.execute(() -> executeSearch(requests, responses, responseCounter, listener, relativeStartTime));
} else {
// we are on a different thread (we went asynchronous), it's safe to recurse
executeSearch(requests, responses, responseCounter, listener);
executeSearch(requests, responses, responseCounter, listener, relativeStartTime);
}
}
}

private void finish() {
listener.onResponse(new MultiSearchResponse(responses.toArray(new MultiSearchResponse.Item[responses.length()])));
listener.onResponse(new MultiSearchResponse(responses.toArray(new MultiSearchResponse.Item[responses.length()]),
buildTookInMillis()));
}

/**
* Builds how long it took to execute the msearch.
*/
private long buildTookInMillis() {
return TimeUnit.NANOSECONDS.toMillis(relativeTimeProvider.getAsLong() - relativeStartTime);
}
});
}
Expand All @@ -178,7 +194,5 @@ static final class SearchRequestSlot {
this.request = request;
this.responseSlot = responseSlot;
}

}

}
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,9 @@ public MergeStats getMergeStats() {
/** returns the history uuid for the engine */
public abstract String getHistoryUUID();

/** Returns how many bytes we are currently moving from heap to disk */
public abstract long getWritingBytes();

/**
* A throttling class that can be activated, causing the
* {@code acquireThrottle} method to block on a lock when throttling
Expand Down Expand Up @@ -707,7 +710,7 @@ protected void writerSegmentStats(SegmentsStats stats) {
}

/** How much heap is used that would be freed by a refresh. Note that this may throw {@link AlreadyClosedException}. */
public abstract long getIndexBufferRAMBytesUsed();
public abstract long getIndexBufferRAMBytesUsed();

protected Segment[] getSegmentInfo(SegmentInfos lastCommittedSegmentInfos, boolean verbose) {
ensureOpen();
Expand Down
Loading

0 comments on commit 0289330

Please sign in to comment.