From 57d44859bdf26d381688f37ec55afe50b46c98df Mon Sep 17 00:00:00 2001 From: Stephen Crawford <65832608+scrawfor99@users.noreply.github.com> Date: Tue, 3 Jan 2023 17:03:34 -0500 Subject: [PATCH] Fix index exclusion behavior in snapshot restore and clone APIs Standardize snapshot indices parsing so that combinations of included and excluded indices are treated the same regardless of the order they are listed in. Signed-off-by: Stephen Crawford --- CHANGELOG.md | 3 ++- .../opensearch/snapshots/SnapshotUtils.java | 18 +++++++++++++++--- .../snapshots/SnapshotUtilsTests.java | 3 +++ 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 197eededddf00..3701bb9315ec3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,7 +21,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add support for ppc64le architecture ([#5459](https://github.com/opensearch-project/OpenSearch/pull/5459)) - Added @gbbafna as an OpenSearch maintainer ([#5668](https://github.com/opensearch-project/OpenSearch/pull/5668)) - ### Dependencies - Bumps `log4j-core` from 2.18.0 to 2.19.0 - Bumps `reactor-netty-http` from 1.0.18 to 1.0.23 @@ -115,6 +114,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Apply cluster manager throttling settings during bootstrap ([#5524](https://github.com/opensearch-project/OpenSearch/pull/5524)) - Update thresholds map when cluster manager throttling setting is removed ([#5524](https://github.com/opensearch-project/OpenSearch/pull/5524)) - Fix backward compatibility for static cluster manager throttling threshold setting ([#5633](https://github.com/opensearch-project/OpenSearch/pull/5633)) +- Fix index exclusion behavior in snapshot restore and clone APIs ([#5626](https://github.com/opensearch-project/OpenSearch/pull/5626)) + ### Security [Unreleased 3.0]: https://github.com/opensearch-project/OpenSearch/compare/2.4...HEAD diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotUtils.java b/server/src/main/java/org/opensearch/snapshots/SnapshotUtils.java index 3ef3523961df8..5c2efba008652 100644 --- a/server/src/main/java/org/opensearch/snapshots/SnapshotUtils.java +++ b/server/src/main/java/org/opensearch/snapshots/SnapshotUtils.java @@ -49,6 +49,8 @@ import java.util.Set; import java.util.HashMap; import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; /** * Snapshot utilities @@ -69,9 +71,17 @@ public static List filterIndices(List availableIndices, String[] if (IndexNameExpressionResolver.isAllIndices(Arrays.asList(selectedIndices))) { return availableIndices; } + + // Move the exclusions to end of list to ensure they are processed + // after explicitly selected indices are chosen. + final List excludesAtEndSelectedIndices = Stream.concat( + Arrays.stream(selectedIndices).filter(s -> s.isEmpty() || s.charAt(0) != '-'), + Arrays.stream(selectedIndices).filter(s -> !s.isEmpty() && s.charAt(0) == '-') + ).collect(Collectors.toUnmodifiableList()); + Set result = null; - for (int i = 0; i < selectedIndices.length; i++) { - String indexOrPattern = selectedIndices[i]; + for (int i = 0; i < excludesAtEndSelectedIndices.size(); i++) { + String indexOrPattern = excludesAtEndSelectedIndices.get(i); boolean add = true; if (!indexOrPattern.isEmpty()) { if (availableIndices.contains(indexOrPattern)) { @@ -89,7 +99,9 @@ public static List filterIndices(List availableIndices, String[] result = new HashSet<>(); } } else if (indexOrPattern.charAt(0) == '-') { - // if its the first, fill it with all the indices... + // If the first index pattern is an exclusion, then all patterns are exclusions due to the + // reordering logic above. In this case, the request is interpreted as "include all indexes except + // those matching the exclusions" so we add all indices here and then remove the ones that match the exclusion patterns. if (i == 0) { result = new HashSet<>(availableIndices); } diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotUtilsTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotUtilsTests.java index 8dae5026a18bc..4a378acc39b6d 100644 --- a/server/src/test/java/org/opensearch/snapshots/SnapshotUtilsTests.java +++ b/server/src/test/java/org/opensearch/snapshots/SnapshotUtilsTests.java @@ -65,6 +65,9 @@ public void testIndexNameFiltering() { assertIndexNameFiltering(new String[] { "foo", "bar", "baz" }, new String[] { "-ba*" }, new String[] { "foo" }); assertIndexNameFiltering(new String[] { "foo", "bar", "baz" }, new String[] { "+ba*" }, new String[] { "bar", "baz" }); assertIndexNameFiltering(new String[] { "foo", "bar", "baz" }, new String[] { "+bar", "+foo" }, new String[] { "bar", "foo" }); + assertIndexNameFiltering(new String[] { "foo", "bar", "baz" }, new String[] { "-bar", "b*" }, new String[] { "baz" }); + assertIndexNameFiltering(new String[] { "foo", "bar", "baz" }, new String[] { "b*", "-bar" }, new String[] { "baz" }); + assertIndexNameFiltering(new String[] { "foo", "bar", "baz" }, new String[] { "-bar", "-baz" }, new String[] { "foo" }); assertIndexNameFiltering( new String[] { "foo", "bar", "baz" }, new String[] { "zzz", "bar" },