Remove deprecated ._tier allocation filtering settings (#73074)

These settings were deprecated in 7.13+ in #72835 and are now removed by this commit. This commit also ensures that the settings are removed from index metadata when the metadata is loaded. The reason for this is that if we allow the settings to remain (because they are not technically "invalid"), then the index will not be able to be allocated, because the FilterAllocationDecider will be looking for nodes with the _tier attribute.
elastic · May 24, 2021 · 95bccda · 95bccda
1 parent 12c2c1f
commit 95bccda
Show file tree

Hide file tree

Showing 20 changed files with 111 additions and 784 deletions.
diff --git a/docs/reference/index-modules/allocation/data_tier_allocation.asciidoc b/docs/reference/index-modules/allocation/data_tier_allocation.asciidoc
@@ -23,28 +23,6 @@ mounted indices>> exclusively.
 [[data-tier-allocation-filters]]
 ==== Data tier allocation settings
 
-
-`index.routing.allocation.include._tier`::
-
-    Assign the index to a node whose `node.roles` configuration has at
-    least one of to the comma-separated values.
-
-    deprecated::[7.13, Filtering based on `include._tier`, `require._tier` and `exclude._tier` is deprecated, use <<tier-preference-allocation-filter,_tier_preference>> instead]
-
-`index.routing.allocation.require._tier`::
-
-    Assign the index to a node whose `node.roles` configuration has _all_
-    of the comma-separated values.
-
-    deprecated::[7.13, Filtering based on `include._tier`, `require._tier` and `exclude._tier` is deprecated, use <<tier-preference-allocation-filter,_tier_preference>> instead]
-
-`index.routing.allocation.exclude._tier`::
-
-    Assign the index to a node whose `node.roles` configuration has _none_ of the
-    comma-separated values.
-
-    deprecated::[7.13, Filtering based on `include._tier`, `require._tier` and `exclude._tier` is deprecated, use <<tier-preference-allocation-filter,_tier_preference>> instead]
-
 [[tier-preference-allocation-filter]]
 `index.routing.allocation.include._tier_preference`::
 

diff --git a/docs/reference/migration/migrate_8_0/settings.asciidoc b/docs/reference/migration/migrate_8_0/settings.asciidoc
@@ -243,6 +243,30 @@ Discontinue use of the removed setting. Specifying this setting in Elasticsearch
 configuration will result in an error on startup.
 ====
 
+[[tier-filter-setting]]
+.Tier filtering settings removed
+[%collapsible]
+====
+*Details* +
+The cluster and index level settings ending in `._tier` used for filtering the allocation of a shard
+to a particular set of nodes have been removed. Instead, the <<tier-preference-allocation-filter,
+tier preference setting>>, `index.routing.allocation.include._tier_preference` should be used. The
+removed settings are:
+
+Cluster level settings:
+- `cluster.routing.allocation.include._tier`
+- `cluster.routing.allocation.exclude._tier`
+- `cluster.routing.allocation.require._tier`
+Index settings:
+- `index.routing.allocation.include._tier`
+- `index.routing.allocation.exclude._tier`
+- `index.routing.allocation.require._tier`
+
+*Impact* +
+Discontinue use of the removed settings. Specifying any of these cluster settings in Elasticsearch
+configuration will result in an error on startup. Any indices using these settings will have the
+settings archived (and they will have no effect) when the index metadata is loaded.
+
 [[shared-data-path-setting]]
 .Shared data path and per index data path settings deprecated
 [%collapsible]
@@ -256,4 +280,3 @@ per index data path settings.
 
 *Impact* +
 Discontinue use of the deprecated settings.
-====
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java
@@ -74,6 +74,10 @@ public IndexMetadata verifyIndexMetadata(IndexMetadata indexMetadata, Version mi
 
         // First convert any shared_cache searchable snapshot indices to only use _tier_preference: data_frozen
         IndexMetadata newMetadata = convertSharedCacheTierPreference(indexMetadata);
+        // Remove _tier routing settings if available, because though these are technically not
+        // invalid settings, since they are now removed the FilterAllocationDecider treats them as
+        // regular attribute filters, and shards cannot be allocated.
+        newMetadata = removeTierFiltering(newMetadata);
         // Next we have to run this otherwise if we try to create IndexSettings
         // with broken settings it would fail in checkMappingsCompatibility
         newMetadata = archiveBrokenIndexSettings(newMetadata);
@@ -216,4 +220,22 @@ IndexMetadata convertSharedCacheTierPreference(IndexMetadata indexMetadata) {
             return indexMetadata;
         }
     }
+
+    /**
+     * Removes index level ._tier allocation filters, if they exist
+     */
+    IndexMetadata removeTierFiltering(IndexMetadata indexMetadata) {
+        final Settings settings = indexMetadata.getSettings();
+        final Settings.Builder settingsBuilder = Settings.builder().put(settings);
+        // Clear any allocation rules other than preference for tier
+        settingsBuilder.remove("index.routing.allocation.include._tier");
+        settingsBuilder.remove("index.routing.allocation.exclude._tier");
+        settingsBuilder.remove("index.routing.allocation.require._tier");
+        final Settings newSettings = settingsBuilder.build();
+        if (settings.equals(newSettings)) {
+            return indexMetadata;
+        } else {
+            return IndexMetadata.builder(indexMetadata).settings(newSettings).build();
+        }
+    }
 }
diff --git a/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeFilters.java b/server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeFilters.java
@@ -90,10 +90,10 @@ public static DiscoveryNodeFilters trimTier(@Nullable DiscoveryNodeFilters origi
         }
 
         Map<String, String[]> newFilters = original.filters.entrySet().stream()
-            // Remove all entries that start with "_tier", as these will be handled elsewhere
+            // Remove all entries that use "_tier_preference", as these will be handled elsewhere
             .filter(entry -> {
                 String attr = entry.getKey();
-                return attr != null && attr.startsWith("_tier") == false;
+                return attr != null && attr.equals("_tier_preference") == false;
             })
             .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
 

diff --git a/...va/org/elasticsearch/cluster/routing/allocation/decider/FilterAllocationDeciderTests.java b/...va/org/elasticsearch/cluster/routing/allocation/decider/FilterAllocationDeciderTests.java
@@ -129,46 +129,6 @@ public void testFilterInitialRecovery() {
         assertEquals("node passes include/exclude/require filters", decision.getExplanation());
     }
 
-    public void testTierFilterIgnored() {
-        ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
-        FilterAllocationDecider filterAllocationDecider = new FilterAllocationDecider(Settings.EMPTY, clusterSettings);
-        AllocationDeciders allocationDeciders = new AllocationDeciders(
-            Arrays.asList(filterAllocationDecider,
-                new SameShardAllocationDecider(Settings.EMPTY, clusterSettings),
-                new ReplicaAfterPrimaryActiveAllocationDecider()));
-        AllocationService service = new AllocationService(allocationDeciders,
-            new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE,
-            EmptySnapshotsInfoService.INSTANCE);
-        ClusterState state = createInitialClusterState(service, Settings.builder()
-            .put("index.routing.allocation.require._tier", "data_cold")
-            .put("index.routing.allocation.include._tier", "data_cold")
-            .put("index.routing.allocation.include._tier_preference", "data_cold")
-            .put("index.routing.allocation.exclude._tier", "data_cold")
-            .build(),
-            Settings.builder()
-                .put("cluster.routing.allocation.require._tier", "data_cold")
-                .put("cluster.routing.allocation.include._tier", "data_cold")
-                .put("cluster.routing.allocation.exclude._tier", "data_cold")
-                .build());
-        RoutingTable routingTable = state.routingTable();
-        RoutingAllocation allocation = new RoutingAllocation(allocationDeciders, state.getRoutingNodes(), state,
-            null, null, 0);
-        allocation.debugDecision(true);
-        allocation = new RoutingAllocation(allocationDeciders, state.getRoutingNodes(), state,
-            null, null, 0);
-        allocation.debugDecision(true);
-        Decision.Single decision = (Decision.Single) filterAllocationDecider.canAllocate(
-            routingTable.index("idx").shard(0).shards().get(0),
-            state.getRoutingNodes().node("node2"), allocation);
-        assertEquals(decision.toString(), Type.YES, decision.type());
-        assertEquals("node passes include/exclude/require filters", decision.getExplanation());
-        decision = (Decision.Single) filterAllocationDecider.canAllocate(
-            routingTable.index("idx").shard(0).shards().get(0),
-            state.getRoutingNodes().node("node1"), allocation);
-        assertEquals(Type.YES, decision.type());
-        assertEquals("node passes include/exclude/require filters", decision.getExplanation());
-    }
-
     private ClusterState createInitialClusterState(AllocationService service, Settings indexSettings) {
         return createInitialClusterState(service, indexSettings, Settings.EMPTY);
     }

diff --git a/...ternalClusterTest/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageIT.java b/...ternalClusterTest/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageIT.java
@@ -24,7 +24,6 @@
 import org.elasticsearch.xpack.autoscaling.action.GetAutoscalingCapacityAction;
 import org.elasticsearch.xpack.autoscaling.action.PutAutoscalingPolicyAction;
 import org.elasticsearch.xpack.cluster.routing.allocation.DataTierAllocationDecider;
-import org.elasticsearch.xpack.cluster.routing.allocation.DataTierAllocationDeciderTests;
 import org.elasticsearch.xpack.core.DataTier;
 import org.hamcrest.Matchers;
 
@@ -197,7 +196,7 @@ public void testRoles() {
         internalCluster().startMasterOnlyNode();
         ReactiveStorageDeciderService service = new ReactiveStorageDeciderService(
             Settings.EMPTY,
-            new ClusterSettings(Settings.EMPTY, DataTierAllocationDeciderTests.ALL_SETTINGS),
+            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
             null
         );
         assertThat(

diff --git a/...main/java/org/elasticsearch/xpack/autoscaling/storage/ProactiveStorageDeciderService.java b/...main/java/org/elasticsearch/xpack/autoscaling/storage/ProactiveStorageDeciderService.java
@@ -37,7 +37,7 @@ public class ProactiveStorageDeciderService implements AutoscalingDeciderService
 
     public ProactiveStorageDeciderService(Settings settings, ClusterSettings clusterSettings, AllocationDeciders allocationDeciders) {
         this.diskThresholdSettings = new DiskThresholdSettings(settings, clusterSettings);
-        this.dataTierAllocationDecider = new DataTierAllocationDecider(settings, clusterSettings);
+        this.dataTierAllocationDecider = new DataTierAllocationDecider();
         this.allocationDeciders = allocationDeciders;
     }
 

diff --git a/.../main/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderService.java b/.../main/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderService.java
@@ -70,7 +70,7 @@ public class ReactiveStorageDeciderService implements AutoscalingDeciderService
 
     public ReactiveStorageDeciderService(Settings settings, ClusterSettings clusterSettings, AllocationDeciders allocationDeciders) {
         this.diskThresholdSettings = new DiskThresholdSettings(settings, clusterSettings);
-        this.dataTierAllocationDecider = new DataTierAllocationDecider(settings, clusterSettings);
+        this.dataTierAllocationDecider = new DataTierAllocationDecider();
         this.allocationDeciders = allocationDeciders;
     }
 

diff --git a/...java/org/elasticsearch/xpack/autoscaling/storage/ProactiveStorageDeciderServiceTests.java b/...java/org/elasticsearch/xpack/autoscaling/storage/ProactiveStorageDeciderServiceTests.java
@@ -11,9 +11,9 @@
 import org.elasticsearch.cluster.ClusterInfo;
 import org.elasticsearch.cluster.ClusterModule;
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.metadata.DataStreamTestHelper;
 import org.elasticsearch.cluster.DiskUsage;
 import org.elasticsearch.cluster.metadata.DataStream;
+import org.elasticsearch.cluster.metadata.DataStreamTestHelper;
 import org.elasticsearch.cluster.metadata.IndexAbstraction;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
@@ -40,7 +40,6 @@
 import org.elasticsearch.xpack.autoscaling.capacity.AutoscalingCapacity;
 import org.elasticsearch.xpack.autoscaling.capacity.AutoscalingDeciderContext;
 import org.elasticsearch.xpack.autoscaling.capacity.AutoscalingDeciderResult;
-import org.elasticsearch.xpack.cluster.routing.allocation.DataTierAllocationDeciderTests;
 import org.hamcrest.Matchers;
 
 import java.util.ArrayList;
@@ -80,7 +79,7 @@ public void testScale() {
         );
         ClusterState interimState = stateBuilder.build();
         final ClusterState state = startAll(interimState);
-        final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, DataTierAllocationDeciderTests.ALL_SETTINGS);
+        final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
         Collection<AllocationDecider> allocationDecidersList = new ArrayList<>(
             ClusterModule.createAllocationDeciders(Settings.EMPTY, clusterSettings, Collections.emptyList())
         );

diff --git a/...java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderDecisionTests.java b/...java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderDecisionTests.java
@@ -47,7 +47,6 @@
 import org.elasticsearch.xpack.autoscaling.capacity.AutoscalingDeciderContext;
 import org.elasticsearch.xpack.autoscaling.capacity.AutoscalingDeciderResult;
 import org.elasticsearch.xpack.cluster.routing.allocation.DataTierAllocationDecider;
-import org.elasticsearch.xpack.cluster.routing.allocation.DataTierAllocationDeciderTests;
 import org.elasticsearch.xpack.core.DataTier;
 import org.junit.Before;
 
@@ -96,10 +95,7 @@ public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAl
         new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)
     );
 
-    private static final DataTierAllocationDecider DATA_TIER_ALLOCATION_DECIDER = new DataTierAllocationDecider(
-        Settings.EMPTY,
-        new ClusterSettings(Settings.EMPTY, DataTierAllocationDeciderTests.ALL_SETTINGS)
-    );
+    private static final DataTierAllocationDecider DATA_TIER_ALLOCATION_DECIDER = new DataTierAllocationDecider();
 
     private ClusterState state;
     private final int hotNodes = randomIntBetween(1, 8);
@@ -293,8 +289,6 @@ private ClusterState moveToCold(Set<IndexMetadata> candidates) {
 
     private IndexMetadata moveToCold(IndexMetadata imd) {
         Settings.Builder builder = Settings.builder().put(imd.getSettings());
-        overrideSetting(imd, builder, DataTierAllocationDecider.INDEX_ROUTING_REQUIRE_SETTING, DataTier.DATA_COLD);
-        overrideSetting(imd, builder, DataTierAllocationDecider.INDEX_ROUTING_INCLUDE_SETTING, DataTier.DATA_COLD);
         overrideSetting(
             imd,
             builder,
@@ -391,7 +385,7 @@ private void verifyScale(long expectedDifference, String reason, AllocationDecid
     private static void verifyScale(ClusterState state, long expectedDifference, String reason, AllocationDecider... allocationDeciders) {
         ReactiveStorageDeciderService decider = new ReactiveStorageDeciderService(
             Settings.EMPTY,
-            new ClusterSettings(Settings.EMPTY, DataTierAllocationDeciderTests.ALL_SETTINGS),
+            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
             createAllocationDeciders(allocationDeciders)
         );
         TestAutoscalingDeciderContext context = createContext(state, Set.of(DiscoveryNodeRole.DATA_HOT_NODE_ROLE));
@@ -430,16 +424,7 @@ private boolean hasStartedSubjectShard() {
     }
 
     private static AllocationDeciders createAllocationDeciders(AllocationDecider... extraDeciders) {
-        Set<Setting<?>> allSettings = Stream.concat(
-            ClusterSettings.BUILT_IN_CLUSTER_SETTINGS.stream(),
-            Stream.of(
-                DataTierAllocationDecider.CLUSTER_ROUTING_REQUIRE_SETTING,
-                DataTierAllocationDecider.CLUSTER_ROUTING_INCLUDE_SETTING,
-                DataTierAllocationDecider.CLUSTER_ROUTING_EXCLUDE_SETTING
-            )
-        ).collect(Collectors.toSet());
-
-        ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, allSettings);
+        ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
         Collection<AllocationDecider> systemAllocationDeciders = ClusterModule.createAllocationDeciders(
             Settings.builder()
                 .put(
@@ -451,11 +436,9 @@ private static AllocationDeciders createAllocationDeciders(AllocationDecider...
             Collections.emptyList()
         );
         return new AllocationDeciders(
-            Stream.of(
-                Stream.of(extraDeciders),
-                Stream.of(new DataTierAllocationDecider(Settings.EMPTY, clusterSettings)),
-                systemAllocationDeciders.stream()
-            ).flatMap(s -> s).collect(Collectors.toList())
+            Stream.of(Stream.of(extraDeciders), Stream.of(new DataTierAllocationDecider()), systemAllocationDeciders.stream())
+                .flatMap(s -> s)
+                .collect(Collectors.toList())
         );
     }
 
@@ -610,10 +593,7 @@ public Long getShardSize(ShardRouting shardRouting) {
     }
 
     private static ClusterState addRandomIndices(int minShards, int maxShardCopies, ClusterState state) {
-        String[] tierSettingNames = new String[] {
-            DataTierAllocationDecider.INDEX_ROUTING_REQUIRE,
-            DataTierAllocationDecider.INDEX_ROUTING_INCLUDE,
-            DataTierAllocationDecider.INDEX_ROUTING_PREFER };
+        String[] tierSettingNames = new String[] { DataTierAllocationDecider.INDEX_ROUTING_PREFER };
         int shards = randomIntBetween(minShards, 20);
         Metadata.Builder builder = Metadata.builder();
         RoutingTable.Builder routingTableBuilder = RoutingTable.builder();

diff --git a/.../java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderServiceTests.java b/.../java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderServiceTests.java
@@ -50,7 +50,6 @@
 import org.elasticsearch.snapshots.SnapshotShardSizeInfo;
 import org.elasticsearch.xpack.autoscaling.AutoscalingTestCase;
 import org.elasticsearch.xpack.cluster.routing.allocation.DataTierAllocationDecider;
-import org.elasticsearch.xpack.cluster.routing.allocation.DataTierAllocationDeciderTests;
 
 import java.util.Arrays;
 import java.util.Collection;
@@ -421,10 +420,7 @@ public ClusterState addPreference(IndexMetadata indexMetadata, ClusterState clus
 
     public boolean canRemainWithNoNodes(ClusterState clusterState, ShardRouting shardRouting, AllocationDecider... deciders) {
         AllocationDeciders allocationDeciders = new AllocationDeciders(Arrays.asList(deciders));
-        DataTierAllocationDecider dataTierAllocationDecider = new DataTierAllocationDecider(
-            Settings.EMPTY,
-            new ClusterSettings(Settings.EMPTY, DataTierAllocationDeciderTests.ALL_SETTINGS)
-        );
+        DataTierAllocationDecider dataTierAllocationDecider = new DataTierAllocationDecider();
         ReactiveStorageDeciderService.AllocationState allocationState = new ReactiveStorageDeciderService.AllocationState(
             clusterState,
             allocationDeciders,