Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Snapshot Pagination and Scalability Improvements Backport to 7.x #74676

Merged
merged 4 commits into from
Jun 29, 2021
Merged

Snapshot Pagination and Scalability Improvements Backport to 7.x #74676

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
1462704
Backport start
original-brownbear Jun 29, 2021
307dbc6
Flatten Get Snapshots Response (#74451)
original-brownbear Jun 24, 2021
454991e
fix test
original-brownbear Jun 29, 2021
ebc55b1
urgh 7.x
original-brownbear Jun 29, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 32 additions & 27 deletions client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,22 +33,23 @@
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.repositories.fs.FsRepository;
import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.snapshots.AbstractSnapshotIntegTestCase;
import org.elasticsearch.snapshots.RestoreInfo;
import org.elasticsearch.snapshots.SnapshotInfo;
import org.mockito.internal.util.collections.Sets;

import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;

import static org.elasticsearch.snapshots.SnapshotsService.NO_FEATURE_STATES_VALUE;
import static org.elasticsearch.tasks.TaskResultsService.TASKS_FEATURE_NAME;
import static org.hamcrest.Matchers.contains;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThan;
import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.Matchers.is;

public class SnapshotIT extends ESRestHighLevelClientTestCase {
Expand Down Expand Up @@ -177,50 +178,54 @@ public void testCreateSnapshot() throws Exception {
}

public void testGetSnapshots() throws IOException {
String repository = "test_repository";
String repository1 = "test_repository1";
String repository2 = "test_repository2";
String snapshot1 = "test_snapshot1";
String snapshot2 = "test_snapshot2";

AcknowledgedResponse putRepositoryResponse = createTestRepository(repository, FsRepository.TYPE, "{\"location\": \".\"}");
AcknowledgedResponse putRepositoryResponse =
createTestRepository(repository1, FsRepository.TYPE, "{\"location\": \"loc1\"}");
assertTrue(putRepositoryResponse.isAcknowledged());

CreateSnapshotRequest createSnapshotRequest1 = new CreateSnapshotRequest(repository, snapshot1);
AcknowledgedResponse putRepositoryResponse2 =
createTestRepository(repository2, FsRepository.TYPE, "{\"location\": \"loc2\"}");
assertTrue(putRepositoryResponse2.isAcknowledged());

CreateSnapshotRequest createSnapshotRequest1 = new CreateSnapshotRequest(repository1, snapshot1);
createSnapshotRequest1.waitForCompletion(true);
CreateSnapshotResponse putSnapshotResponse1 = createTestSnapshot(createSnapshotRequest1);
CreateSnapshotRequest createSnapshotRequest2 = new CreateSnapshotRequest(repository, snapshot2);
CreateSnapshotRequest createSnapshotRequest2 = new CreateSnapshotRequest(repository2, snapshot2);
createSnapshotRequest2.waitForCompletion(true);
Map<String, Object> originalMetadata = randomUserMetadata();
Map<String, Object> originalMetadata = AbstractSnapshotIntegTestCase.randomUserMetadata();
createSnapshotRequest2.userMetadata(originalMetadata);
CreateSnapshotResponse putSnapshotResponse2 = createTestSnapshot(createSnapshotRequest2);
// check that the request went ok without parsing JSON here. When using the high level client, check acknowledgement instead.
assertEquals(RestStatus.OK, putSnapshotResponse1.status());
assertEquals(RestStatus.OK, putSnapshotResponse2.status());

GetSnapshotsRequest request;
if (randomBoolean()) {
request = new GetSnapshotsRequest(repository);
} else if (randomBoolean()) {
request = new GetSnapshotsRequest(repository, new String[] {"_all"});
GetSnapshotsRequest request = new GetSnapshotsRequest(
randomFrom(new String[]{"_all"}, new String[]{"*"}, new String[]{repository1, repository2}),
randomFrom(new String[]{"_all"}, new String[]{"*"}, new String[]{snapshot1, snapshot2})
);
request.ignoreUnavailable(true);

} else {
request = new GetSnapshotsRequest(repository, new String[] {snapshot1, snapshot2});
}
GetSnapshotsResponse response = execute(request, highLevelClient().snapshot()::get, highLevelClient().snapshot()::getAsync);

assertEquals(2, response.getSnapshots().size());
assertThat(response.getSnapshots().stream().map((s) -> s.snapshotId().getName()).collect(Collectors.toList()),
contains("test_snapshot1", "test_snapshot2"));
Optional<Map<String, Object>> returnedMetadata = response.getSnapshots().stream()
.filter(s -> s.snapshotId().getName().equals("test_snapshot2"))
.findFirst()
.map(SnapshotInfo::userMetadata);
if (returnedMetadata.isPresent()) {
assertEquals(originalMetadata, returnedMetadata.get());
} else {
assertNull("retrieved metadata is null, expected non-null metadata", originalMetadata);
}
assertThat(response.isFailed(), is(false));
assertEquals(
Sets.newSet(repository1, repository2),
response.getSnapshots().stream().map(SnapshotInfo::repository).collect(Collectors.toSet())
);

assertThat(response.getSnapshots(), hasSize(2));
assertThat(response.getSnapshots().get(0).snapshotId().getName(), equalTo(snapshot1));
assertThat(response.getSnapshots().get(0).repository(), equalTo(repository1));
assertThat(response.getSnapshots().get(1).snapshotId().getName(), equalTo(snapshot2));
assertThat(response.getSnapshots().get(1).userMetadata(), equalTo(originalMetadata));
assertThat(response.getSnapshots().get(1).repository(), equalTo(repository2));
}


public void testSnapshotsStatus() throws IOException {
String testRepository = "test";
String testSnapshot = "snapshot";
Expand Down
13 changes: 8 additions & 5 deletions docs/reference/cat/snapshots.asciidoc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ more repositories. A snapshot is a backup of an index or running {es} cluster.

`GET /_cat/snapshots/<repository>`

`GET /_cat/snapshots`

[[cat-snapshots-api-prereqs]]
==== {api-prereq-title}

Expand All @@ -27,9 +29,10 @@ more repositories. A snapshot is a backup of an index or running {es} cluster.
`<repository>`::
+
--
(Required, string) Snapshot repository used to limit the request.
(Optional, string) Comma-separated list of snapshot repositories used to limit
the request. Accepts wildcard expressions. `_all` returns all repositories.

If the repository fails during the request, {es} returns an error.
If any repository fails during the request, {es} returns an error.
--


Expand Down Expand Up @@ -127,9 +130,9 @@ The API returns the following response:

[source,txt]
--------------------------------------------------
id status start_epoch start_time end_epoch end_time duration indices successful_shards failed_shards total_shards
snap1 FAILED 1445616705 18:11:45 1445616978 18:16:18 4.6m 1 4 1 5
snap2 SUCCESS 1445634298 23:04:58 1445634672 23:11:12 6.2m 2 10 0 10
id repository status start_epoch start_time end_epoch end_time duration indices successful_shards failed_shards total_shards
snap1 repo1 FAILED 1445616705 18:11:45 1445616978 18:16:18 4.6m 1 4 1 5
snap2 repo1 SUCCESS 1445634298 23:04:58 1445634672 23:11:12 6.2m 2 10 0 10
--------------------------------------------------
// TESTRESPONSE[s/FAILED/SUCCESS/ s/14456\d+/\\d+/ s/\d+(\.\d+)?(m|s|ms)/\\d+(\\.\\d+)?(m|s|ms)/]
// TESTRESPONSE[s/\d+:\d+:\d+/\\d+:\\d+:\\d+/]
Expand Down
5 changes: 5 additions & 0 deletions docs/reference/modules/threadpool.asciidoc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,11 @@ There are several thread pools, but the important ones include:
keep-alive of `5m` and a max of `min(5, (`<<node.processors,
`# of allocated processors`>>`) / 2)`.

`snapshot_meta`::
For snapshot repository metadata read operations. Thread pool type is `scaling` with a
keep-alive of `5m` and a max of `min(50, (`<<node.processors,
`# of allocated processors`>>` pass:[ * ]3))`.

`warmer`::
For segment warm-up operations. Thread pool type is `scaling` with a
keep-alive of `5m` and a max of `min(5, (`<<node.processors,
Expand Down
1 change: 1 addition & 0 deletions docs/reference/snapshot-restore/apis/create-snapshot-api.asciidoc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,7 @@ The API returns the following response:
"snapshot": {
"snapshot": "snapshot_2",
"uuid": "vdRctLCxSketdKb54xw67g",
"repository": "my_repository",
"version_id": <version_id>,
"version": <version>,
"indices": [],
Expand Down
180 changes: 177 additions & 3 deletions docs/reference/snapshot-restore/apis/get-snapshot-api.asciidoc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,9 @@ PUT /_snapshot/my_repository

PUT /_snapshot/my_repository/my_snapshot?wait_for_completion=true

PUT /_snapshot/my_repository/snapshot_1?wait_for_completion=true
PUT /_snapshot/my_repository/snapshot_2?wait_for_completion=true
PUT /_snapshot/my_repository/snapshot_3?wait_for_completion=true
----
// TESTSETUP
////
Expand Down Expand Up @@ -57,7 +59,8 @@ Use the get snapshot API to return information about one or more snapshots, incl

`<repository>`::
(Required, string)
Snapshot repository name used to limit the request.
Comma-separated list of snapshot repository names used to limit the request.
Wildcard (`*`) expressions are supported.
+
To get information about all snapshot repositories registered in the
cluster, omit this parameter or use `*` or `_all`.
Expand Down Expand Up @@ -99,6 +102,45 @@ comprising the number of shards in the index, the total size of the index in
bytes, and the maximum number of segments per shard in the index. Defaults to
`false`, meaning that this information is omitted.

`sort`::
(Optional, string)
Allows setting a sort order for the result. Defaults to `start_time`, i.e. sorting by snapshot start time stamp.
+
.Valid values for `sort`
[%collapsible%open]
====
`start_time`::
Sort snapshots by their start time stamp and break ties by snapshot name.

`duration`::
Sort snapshots by their duration and break ties by snapshot name.

`name`::
Sort snapshots by their name.

`index_count`::
Sort snapshots by the number of indices they contain and break ties by snapshot name.
====

`size`::
(Optional, integer)
Maximum number of snapshots to return. Defaults to `0` which means return all that match the request without limit.

`order`::
(Optional, string)
Sort order. Valid values are `asc` for ascending and `desc` for descending order. Defaults to `asc`, meaning ascending order.

`after`::
(Optional, string)
Offset identifier to start pagination from as returned by the `next` field in the response body.

NOTE: The `after` parameter and `next` field allow for iterating through snapshots with some consistency guarantees regarding concurrent
creation or deletion of snapshots. It is guaranteed that any snapshot that exists at the beginning of the iteration and not concurrently
deleted will be seen during the iteration. Snapshots concurrently created may be seen during an iteration.

NOTE: The pagination parameters `size`, `order`, `after` and `sort` are not supported when using `verbose=false` and the sort order for
requests with `verbose=false` is undefined.

[role="child_attributes"]
[[get-snapshot-api-response-body]]
==== {api-response-body-title}
Expand Down Expand Up @@ -236,6 +278,10 @@ The snapshot `state` can be one of the following values:
that were not processed correctly.
====
--
`next`::
(string)
If the request contained a size limit and there might be more results, a `next` field will be added to the response and can be used as the
`after` query parameter to fetch additional results.

[[get-snapshot-api-example]]
==== {api-examples-title}
Expand All @@ -256,6 +302,7 @@ The API returns the following response:
{
"snapshot": "snapshot_2",
"uuid": "vdRctLCxSketdKb54xw67g",
"repository": "my_repository",
"version_id": <version_id>,
"version": <version>,
"indices": [],
Expand All @@ -265,7 +312,7 @@ The API returns the following response:
"state": "SUCCESS",
"start_time": "2020-07-06T21:55:18.129Z",
"start_time_in_millis": 1593093628850,
"end_time": "2020-07-06T21:55:18.876Z",
"end_time": "2020-07-06T21:55:18.129Z",
"end_time_in_millis": 1593094752018,
"duration_in_millis": 0,
"failures": [],
Expand All @@ -283,6 +330,133 @@ The API returns the following response:
// TESTRESPONSE[s/"version": <version>/"version": $body.snapshots.0.version/]
// TESTRESPONSE[s/"start_time": "2020-07-06T21:55:18.129Z"/"start_time": $body.snapshots.0.start_time/]
// TESTRESPONSE[s/"start_time_in_millis": 1593093628850/"start_time_in_millis": $body.snapshots.0.start_time_in_millis/]
// TESTRESPONSE[s/"end_time": "2020-07-06T21:55:18.876Z"/"end_time": $body.snapshots.0.end_time/]
// TESTRESPONSE[s/"end_time": "2020-07-06T21:55:18.129Z"/"end_time": $body.snapshots.0.end_time/]
// TESTRESPONSE[s/"end_time_in_millis": 1593094752018/"end_time_in_millis": $body.snapshots.0.end_time_in_millis/]
// TESTRESPONSE[s/"duration_in_millis": 0/"duration_in_millis": $body.snapshots.0.duration_in_millis/]

The following request returns information for all snapshots with prefix `snapshot` in the `my_repository` repository,
limiting the response size to 2 and sorting by snapshot name.

[source,console]
----
GET /_snapshot/my_repository/snapshot*?size=2&sort=name
----

The API returns the following response:

[source,console-result]
----
{
"snapshots": [
{
"snapshot": "snapshot_1",
"uuid": "dKb54xw67gvdRctLCxSket",
"repository": "my_repository",
"version_id": <version_id>,
"version": <version>,
"indices": [],
"data_streams": [],
"feature_states": [],
"include_global_state": true,
"state": "SUCCESS",
"start_time": "2020-07-06T21:55:18.129Z",
"start_time_in_millis": 1593093628850,
"end_time": "2020-07-06T21:55:18.129Z",
"end_time_in_millis": 1593094752018,
"duration_in_millis": 0,
"failures": [],
"shards": {
"total": 0,
"failed": 0,
"successful": 0
}
},
{
"snapshot": "snapshot_2",
"uuid": "vdRctLCxSketdKb54xw67g",
"repository": "my_repository",
"version_id": <version_id>,
"version": <version>,
"indices": [],
"data_streams": [],
"feature_states": [],
"include_global_state": true,
"state": "SUCCESS",
"start_time": "2020-07-06T21:55:18.130Z",
"start_time_in_millis": 1593093628851,
"end_time": "2020-07-06T21:55:18.130Z",
"end_time_in_millis": 1593094752019,
"duration_in_millis": 1,
"failures": [],
"shards": {
"total": 0,
"failed": 0,
"successful": 0
}
}
],
"next": "c25hcHNob3RfMixteV9yZXBvc2l0b3J5LHNuYXBzaG90XzI="
}
----
// TESTRESPONSE[s/"uuid": "dKb54xw67gvdRctLCxSket"/"uuid": $body.snapshots.0.uuid/]
// TESTRESPONSE[s/"uuid": "vdRctLCxSketdKb54xw67g"/"uuid": $body.snapshots.1.uuid/]
// TESTRESPONSE[s/"version_id": <version_id>/"version_id": $body.snapshots.0.version_id/]
// TESTRESPONSE[s/"version": <version>/"version": $body.snapshots.0.version/]
// TESTRESPONSE[s/"start_time": "2020-07-06T21:55:18.129Z"/"start_time": $body.snapshots.0.start_time/]
// TESTRESPONSE[s/"start_time": "2020-07-06T21:55:18.130Z"/"start_time": $body.snapshots.1.start_time/]
// TESTRESPONSE[s/"start_time_in_millis": 1593093628850/"start_time_in_millis": $body.snapshots.0.start_time_in_millis/]
// TESTRESPONSE[s/"start_time_in_millis": 1593093628851/"start_time_in_millis": $body.snapshots.1.start_time_in_millis/]
// TESTRESPONSE[s/"end_time": "2020-07-06T21:55:18.129Z"/"end_time": $body.snapshots.0.end_time/]
// TESTRESPONSE[s/"end_time": "2020-07-06T21:55:18.130Z"/"end_time": $body.snapshots.1.end_time/]
// TESTRESPONSE[s/"end_time_in_millis": 1593094752018/"end_time_in_millis": $body.snapshots.0.end_time_in_millis/]
// TESTRESPONSE[s/"end_time_in_millis": 1593094752019/"end_time_in_millis": $body.snapshots.1.end_time_in_millis/]
// TESTRESPONSE[s/"duration_in_millis": 0/"duration_in_millis": $body.snapshots.0.duration_in_millis/]
// TESTRESPONSE[s/"duration_in_millis": 1/"duration_in_millis": $body.snapshots.1.duration_in_millis/]

A subsequent request for the remaining snapshots can then be made using the `next` value from the previous response as `after` parameter.

[source,console]
----
GET /_snapshot/my_repository/snapshot*?size=2&sort=name&after=c25hcHNob3RfMixteV9yZXBvc2l0b3J5LHNuYXBzaG90XzI=
----

The API returns the following response:

[source,console-result]
----
{
"snapshots": [
{
"snapshot": "snapshot_3",
"uuid": "dRctdKb54xw67gvLCxSket",
"repository": "my_repository",
"version_id": <version_id>,
"version": <version>,
"indices": [],
"data_streams": [],
"feature_states": [],
"include_global_state": true,
"state": "SUCCESS",
"start_time": "2020-07-06T21:55:18.129Z",
"start_time_in_millis": 1593093628850,
"end_time": "2020-07-06T21:55:18.129Z",
"end_time_in_millis": 1593094752018,
"duration_in_millis": 0,
"failures": [],
"shards": {
"total": 0,
"failed": 0,
"successful": 0
}
}
]
}
----
// TESTRESPONSE[s/"uuid": "dRctdKb54xw67gvLCxSket"/"uuid": $body.snapshots.0.uuid/]
// TESTRESPONSE[s/"version_id": <version_id>/"version_id": $body.snapshots.0.version_id/]
// TESTRESPONSE[s/"version": <version>/"version": $body.snapshots.0.version/]
// TESTRESPONSE[s/"start_time": "2020-07-06T21:55:18.129Z"/"start_time": $body.snapshots.0.start_time/]
// TESTRESPONSE[s/"start_time_in_millis": 1593093628850/"start_time_in_millis": $body.snapshots.0.start_time_in_millis/]
// TESTRESPONSE[s/"end_time": "2020-07-06T21:55:18.129Z"/"end_time": $body.snapshots.0.end_time/]
// TESTRESPONSE[s/"end_time_in_millis": 1593094752018/"end_time_in_millis": $body.snapshots.0.end_time_in_millis/]
// TESTRESPONSE[s/"duration_in_millis": 0/"duration_in_millis": $body.snapshots.0.duration_in_millis/]
Loading