Skip to content

Commit

Permalink
Make DataStreamsSnapshotsIT resilient to failures because of local ti…
Browse files Browse the repository at this point in the history 
…me. (#73522)

Backport #73516 to 7.x branch.

Backing index names contain a date component.

Instead of defining what the expected backing index names are before creating the data streams,
resolve these expected backing index names after the data streams are created.

This way we avoid failures that can occur around midnight (local time), where the expected
names are created before midnight and the data streams are created after midnight.

Closes #73510
  • Loading branch information
@martijnvg
martijnvg authored May 28, 2021
1 parent 48de999 commit 1f48a9e
Showing 1 changed file with 29 additions and 21 deletions.
50 changes: 29 additions & 21 deletions ...ms/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,13 +70,13 @@
@ESIntegTestCase.ClusterScope(transportClientRatio = 0)
public class DataStreamsSnapshotsIT extends AbstractSnapshotIntegTestCase {

private static final String DS_BACKING_INDEX_NAME = DataStream.getDefaultBackingIndexName("ds", 1);
private static final String DS2_BACKING_INDEX_NAME = DataStream.getDefaultBackingIndexName("ds2", 1);
private static final Map<String, Integer> DOCUMENT_SOURCE = Collections.singletonMap("@timestamp", 123);
public static final String REPO = "repo";
public static final String SNAPSHOT = "snap";
private Client client;

private String dsBackingIndexName;
private String ds2BackingIndexName;
private String id;

@Override
Expand All @@ -100,6 +100,14 @@ public void setup() throws Exception {
response = client.execute(CreateDataStreamAction.INSTANCE, request).get();
assertTrue(response.isAcknowledged());

// Resolve backing index names after data streams have been created:
// (these names have a date component, and running around midnight could lead to test failures otherwise)
GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { "*" });
GetDataStreamAction.Response getDataStreamResponse = client.execute(GetDataStreamAction.INSTANCE, getDataStreamRequest).actionGet();
dsBackingIndexName = getDataStreamResponse.getDataStreams().get(0).getDataStream().getIndices().get(0).getName();
// Will be used in some tests, to test renaming while restoring a snapshot:
ds2BackingIndexName = dsBackingIndexName.replace("-ds-", "-ds2-");

IndexResponse indexResponse = client.prepareIndex("ds", "_doc")
.setOpType(DocWriteRequest.OpType.CREATE)
.setSource(DOCUMENT_SOURCE)
Expand Down Expand Up @@ -134,7 +142,7 @@ public void testSnapshotAndRestore() throws Exception {
RestStatus status = createSnapshotResponse.getSnapshotInfo().status();
assertEquals(RestStatus.OK, status);

assertEquals(Collections.singletonList(DS_BACKING_INDEX_NAME), getSnapshot(REPO, SNAPSHOT).indices());
assertEquals(Collections.singletonList(dsBackingIndexName), getSnapshot(REPO, SNAPSHOT).indices());

assertTrue(
client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(new String[] { "ds" }))
Expand All @@ -151,7 +159,7 @@ public void testSnapshotAndRestore() throws Exception {

assertEquals(1, restoreSnapshotResponse.getRestoreInfo().successfulShards());

assertEquals(DOCUMENT_SOURCE, client.prepareGet(DS_BACKING_INDEX_NAME, "_doc", id).get().getSourceAsMap());
assertEquals(DOCUMENT_SOURCE, client.prepareGet(dsBackingIndexName, "_doc", id).get().getSourceAsMap());
SearchHit[] hits = client.prepareSearch("ds").get().getHits().getHits();
assertEquals(1, hits.length);
assertEquals(DOCUMENT_SOURCE, hits[0].getSourceAsMap());
Expand All @@ -162,7 +170,7 @@ public void testSnapshotAndRestore() throws Exception {
).get();
assertEquals(1, ds.getDataStreams().size());
assertEquals(1, ds.getDataStreams().get(0).getDataStream().getIndices().size());
assertEquals(DS_BACKING_INDEX_NAME, ds.getDataStreams().get(0).getDataStream().getIndices().get(0).getName());
assertEquals(dsBackingIndexName, ds.getDataStreams().get(0).getDataStream().getIndices().get(0).getName());

GetAliasesResponse getAliasesResponse = client.admin().indices().getAliases(new GetAliasesRequest("my-alias")).actionGet();
assertThat(getAliasesResponse.getDataStreamAliases().keySet(), containsInAnyOrder("ds", "other-ds"));
Expand All @@ -184,7 +192,7 @@ public void testSnapshotAndRestoreAllDataStreamsInPlace() throws Exception {
RestStatus status = createSnapshotResponse.getSnapshotInfo().status();
assertEquals(RestStatus.OK, status);

assertEquals(Collections.singletonList(DS_BACKING_INDEX_NAME), getSnapshot(REPO, SNAPSHOT).indices());
assertEquals(Collections.singletonList(dsBackingIndexName), getSnapshot(REPO, SNAPSHOT).indices());

// Close all indices:
CloseIndexRequest closeIndexRequest = new CloseIndexRequest("*");
Expand All @@ -199,7 +207,7 @@ public void testSnapshotAndRestoreAllDataStreamsInPlace() throws Exception {
.get();
assertEquals(1, restoreSnapshotResponse.getRestoreInfo().successfulShards());

assertEquals(DOCUMENT_SOURCE, client.prepareGet(DS_BACKING_INDEX_NAME, "_doc", id).get().getSourceAsMap());
assertEquals(DOCUMENT_SOURCE, client.prepareGet(dsBackingIndexName, "_doc", id).get().getSourceAsMap());
SearchHit[] hits = client.prepareSearch("ds").get().getHits().getHits();
assertEquals(1, hits.length);
assertEquals(DOCUMENT_SOURCE, hits[0].getSourceAsMap());
Expand All @@ -211,7 +219,7 @@ public void testSnapshotAndRestoreAllDataStreamsInPlace() throws Exception {
contains(equalTo("ds"), equalTo("other-ds"))
);
java.util.List<Index> backingIndices = ds.getDataStreams().get(0).getDataStream().getIndices();
assertThat(backingIndices.stream().map(Index::getName).collect(Collectors.toList()), contains(DS_BACKING_INDEX_NAME));
assertThat(backingIndices.stream().map(Index::getName).collect(Collectors.toList()), contains(dsBackingIndexName));
backingIndices = ds.getDataStreams().get(1).getDataStream().getIndices();
String expectedBackingIndexName = DataStream.getDefaultBackingIndexName("other-ds", 1);
assertThat(backingIndices.stream().map(Index::getName).collect(Collectors.toList()), contains(expectedBackingIndexName));
Expand All @@ -229,7 +237,7 @@ public void testSnapshotAndRestoreInPlace() throws Exception {
RestStatus status = createSnapshotResponse.getSnapshotInfo().status();
assertEquals(RestStatus.OK, status);

assertEquals(Collections.singletonList(DS_BACKING_INDEX_NAME), getSnapshot(REPO, SNAPSHOT).indices());
assertEquals(Collections.singletonList(dsBackingIndexName), getSnapshot(REPO, SNAPSHOT).indices());

// A rollover after taking snapshot. The new backing index should be a standalone index after restoring
// and not part of the data stream:
Expand All @@ -251,7 +259,7 @@ public void testSnapshotAndRestoreInPlace() throws Exception {
.get();
assertEquals(1, restoreSnapshotResponse.getRestoreInfo().successfulShards());

assertEquals(DOCUMENT_SOURCE, client.prepareGet(DS_BACKING_INDEX_NAME, "_doc", id).get().getSourceAsMap());
assertEquals(DOCUMENT_SOURCE, client.prepareGet(dsBackingIndexName, "_doc", id).get().getSourceAsMap());
SearchHit[] hits = client.prepareSearch("ds").get().getHits().getHits();
assertEquals(1, hits.length);
assertEquals(DOCUMENT_SOURCE, hits[0].getSourceAsMap());
Expand All @@ -264,7 +272,7 @@ public void testSnapshotAndRestoreInPlace() throws Exception {
);
java.util.List<Index> backingIndices = ds.getDataStreams().get(0).getDataStream().getIndices();
assertThat(ds.getDataStreams().get(0).getDataStream().getIndices(), hasSize(1));
assertThat(backingIndices.stream().map(Index::getName).collect(Collectors.toList()), contains(equalTo(DS_BACKING_INDEX_NAME)));
assertThat(backingIndices.stream().map(Index::getName).collect(Collectors.toList()), contains(equalTo(dsBackingIndexName)));

// The backing index created as part of rollover should still exist (but just not part of the data stream)
assertThat(indexExists(DataStream.getDefaultBackingIndexName("ds", 2)), is(true));
Expand All @@ -287,7 +295,7 @@ public void testSnapshotAndRestoreAll() throws Exception {
RestStatus status = createSnapshotResponse.getSnapshotInfo().status();
assertEquals(RestStatus.OK, status);

assertEquals(Collections.singletonList(DS_BACKING_INDEX_NAME), getSnapshot(REPO, SNAPSHOT).indices());
assertEquals(Collections.singletonList(dsBackingIndexName), getSnapshot(REPO, SNAPSHOT).indices());

assertAcked(client.execute(DeleteDataStreamAction.INSTANCE, new DeleteDataStreamAction.Request(new String[] { "*" })).get());
assertAcked(client.admin().indices().prepareDelete("*").setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN));
Expand All @@ -301,7 +309,7 @@ public void testSnapshotAndRestoreAll() throws Exception {

assertEquals(1, restoreSnapshotResponse.getRestoreInfo().successfulShards());

assertEquals(DOCUMENT_SOURCE, client.prepareGet(DS_BACKING_INDEX_NAME, "_doc", id).get().getSourceAsMap());
assertEquals(DOCUMENT_SOURCE, client.prepareGet(dsBackingIndexName, "_doc", id).get().getSourceAsMap());
SearchHit[] hits = client.prepareSearch("ds").get().getHits().getHits();
assertEquals(1, hits.length);
assertEquals(DOCUMENT_SOURCE, hits[0].getSourceAsMap());
Expand All @@ -312,7 +320,7 @@ public void testSnapshotAndRestoreAll() throws Exception {
).get();
assertEquals(1, ds.getDataStreams().size());
assertEquals(1, ds.getDataStreams().get(0).getDataStream().getIndices().size());
assertEquals(DS_BACKING_INDEX_NAME, ds.getDataStreams().get(0).getDataStream().getIndices().get(0).getName());
assertEquals(dsBackingIndexName, ds.getDataStreams().get(0).getDataStream().getIndices().get(0).getName());

GetAliasesResponse getAliasesResponse = client.admin().indices().getAliases(new GetAliasesRequest("my-alias")).actionGet();
assertThat(getAliasesResponse.getDataStreamAliases().keySet(), containsInAnyOrder("ds"));
Expand Down Expand Up @@ -353,9 +361,9 @@ public void testRename() throws Exception {
).get();
assertEquals(1, ds.getDataStreams().size());
assertEquals(1, ds.getDataStreams().get(0).getDataStream().getIndices().size());
assertEquals(DS2_BACKING_INDEX_NAME, ds.getDataStreams().get(0).getDataStream().getIndices().get(0).getName());
assertEquals(ds2BackingIndexName, ds.getDataStreams().get(0).getDataStream().getIndices().get(0).getName());
assertEquals(DOCUMENT_SOURCE, client.prepareSearch("ds2").get().getHits().getHits()[0].getSourceAsMap());
assertEquals(DOCUMENT_SOURCE, client.prepareGet(DS2_BACKING_INDEX_NAME, "_doc", id).get().getSourceAsMap());
assertEquals(DOCUMENT_SOURCE, client.prepareGet(ds2BackingIndexName, "_doc", id).get().getSourceAsMap());

GetAliasesResponse getAliasesResponse = client.admin().indices().getAliases(new GetAliasesRequest("my-alias")).actionGet();
assertThat(getAliasesResponse.getDataStreamAliases().keySet(), containsInAnyOrder("ds", "ds2", "other-ds"));
Expand Down Expand Up @@ -393,15 +401,15 @@ public void testBackingIndexIsNotRenamedWhenRestoringDataStream() {
.prepareRestoreSnapshot(REPO, SNAPSHOT)
.setWaitForCompletion(true)
.setIndices("ds")
.setRenamePattern(DS_BACKING_INDEX_NAME)
.setRenamePattern(dsBackingIndexName)
.setRenameReplacement("new_index_name")
.get();

assertThat(restoreSnapshotResponse.status(), is(RestStatus.OK));

GetDataStreamAction.Request getDSRequest = new GetDataStreamAction.Request(new String[] { "ds" });
GetDataStreamAction.Response response = client.execute(GetDataStreamAction.INSTANCE, getDSRequest).actionGet();
assertThat(response.getDataStreams().get(0).getDataStream().getIndices().get(0).getName(), is(DS_BACKING_INDEX_NAME));
assertThat(response.getDataStreams().get(0).getDataStream().getIndices().get(0).getName(), is(dsBackingIndexName));
}

public void testDataStreamAndBackingIndicesAreRenamedUsingRegex() {
Expand Down Expand Up @@ -444,7 +452,7 @@ public void testDataStreamAndBackingIndicesAreRenamedUsingRegex() {
// data stream "ds" should still exist in the system
GetDataStreamAction.Request getDSRequest = new GetDataStreamAction.Request(new String[] { "ds" });
response = client.execute(GetDataStreamAction.INSTANCE, getDSRequest).actionGet();
assertThat(response.getDataStreams().get(0).getDataStream().getIndices().get(0).getName(), is(DS_BACKING_INDEX_NAME));
assertThat(response.getDataStreams().get(0).getDataStream().getIndices().get(0).getName(), is(dsBackingIndexName));
}

public void testWildcards() throws Exception {
Expand Down Expand Up @@ -476,7 +484,7 @@ public void testWildcards() throws Exception {
).get();
assertEquals(1, ds.getDataStreams().size());
assertEquals(1, ds.getDataStreams().get(0).getDataStream().getIndices().size());
assertEquals(DS2_BACKING_INDEX_NAME, ds.getDataStreams().get(0).getDataStream().getIndices().get(0).getName());
assertEquals(ds2BackingIndexName, ds.getDataStreams().get(0).getDataStream().getIndices().get(0).getName());
assertThat(
"we renamed the restored data stream to one that doesn't match any existing composable template",
ds.getDataStreams().get(0).getIndexTemplate(),
Expand All @@ -489,7 +497,7 @@ public void testDataStreamNotStoredWhenIndexRequested() {
.cluster()
.prepareCreateSnapshot(REPO, "snap2")
.setWaitForCompletion(true)
.setIndices(DS_BACKING_INDEX_NAME)
.setIndices(dsBackingIndexName)
.setIncludeGlobalState(false)
.get();

Expand Down

0 comments on commit 1f48a9e

Please sign in to comment.