diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/shared/PartiallyCachedShardAllocationIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/shared/PartiallyCachedShardAllocationIntegTests.java index 0099027f4277e..4598eb86b5613 100644 --- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/shared/PartiallyCachedShardAllocationIntegTests.java +++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/shared/PartiallyCachedShardAllocationIntegTests.java @@ -14,7 +14,9 @@ import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; import org.elasticsearch.action.support.ListenableActionFuture; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.routing.RoutingNode; +import org.elasticsearch.cluster.routing.RoutingNodes; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.ShardRoutingState; import org.elasticsearch.cluster.routing.UnassignedInfo; @@ -50,7 +52,9 @@ import static org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING; import static org.elasticsearch.index.IndexSettings.INDEX_SOFT_DELETES_SETTING; +import static org.elasticsearch.test.NodeRoles.onlyRole; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.xpack.cluster.routing.allocation.DataTierAllocationDecider.INDEX_ROUTING_PREFER; import static org.elasticsearch.xpack.searchablesnapshots.cache.shared.FrozenCacheService.SNAPSHOT_CACHE_SIZE_SETTING; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasItem; @@ -149,6 +153,46 @@ public void testPartialSearchableSnapshotAllocatedToNodesWithCache() throws Exce } } + public void testOnlyPartialSearchableSnapshotAllocatedToDedicatedFrozenNodes() throws Exception { + + final MountSearchableSnapshotRequest req = prepareMountRequest(); + + final List newNodeNames = internalCluster().startNodes( + between(1, 3), + Settings.builder() + .put(SNAPSHOT_CACHE_SIZE_SETTING.getKey(), new ByteSizeValue(randomLongBetween(1, ByteSizeValue.ofMb(10).getBytes()))) + .put(onlyRole(DiscoveryNodeRole.DATA_FROZEN_NODE_ROLE)) + .build() + ); + + createIndex("other-index", Settings.builder().putNull(INDEX_ROUTING_PREFER).build()); + ensureGreen("other-index"); + final RoutingNodes routingNodes = client().admin() + .cluster() + .prepareState() + .clear() + .setRoutingTable(true) + .setNodes(true) + .get() + .getState() + .getRoutingNodes(); + for (RoutingNode routingNode : routingNodes) { + if (newNodeNames.contains(routingNode.node().getName())) { + assertTrue(routingNode + " should be empty in " + routingNodes, routingNode.isEmpty()); + } + } + + final RestoreSnapshotResponse restoreSnapshotResponse = client().execute(MountSearchableSnapshotAction.INSTANCE, req).get(); + assertThat(restoreSnapshotResponse.getRestoreInfo().failedShards(), equalTo(0)); + ensureGreen(req.mountedIndexName()); + + final ClusterState state = client().admin().cluster().prepareState().clear().setNodes(true).setRoutingTable(true).get().getState(); + final Set newNodeIds = newNodeNames.stream().map(n -> state.nodes().resolveNode(n).getId()).collect(Collectors.toSet()); + for (ShardRouting shardRouting : state.routingTable().index(req.mountedIndexName()).shardsWithState(ShardRoutingState.STARTED)) { + assertThat(state.toString(), newNodeIds, hasItem(shardRouting.currentNodeId())); + } + } + public void testPartialSearchableSnapshotDelaysAllocationUntilNodeCacheStatesKnown() throws Exception { assertAcked( diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java index b6290b9b5ec43..685f9360331f8 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java @@ -12,6 +12,8 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.xpack.searchablesnapshots.allocation.decider.DedicatedFrozenNodeAllocationDecider; +import org.elasticsearch.xpack.searchablesnapshots.cache.blob.BlobStoreCacheService; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; @@ -529,7 +531,8 @@ public Collection createAllocationDeciders(Settings settings, return org.elasticsearch.common.collect.List.of( new SearchableSnapshotAllocationDecider(() -> getLicenseState().isAllowed(XPackLicenseState.Feature.SEARCHABLE_SNAPSHOTS)), new SearchableSnapshotEnableAllocationDecider(settings, clusterSettings), - new HasFrozenCacheAllocationDecider(frozenCacheInfoService) + new HasFrozenCacheAllocationDecider(frozenCacheInfoService), + new DedicatedFrozenNodeAllocationDecider() ); } diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/allocation/decider/DedicatedFrozenNodeAllocationDecider.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/allocation/decider/DedicatedFrozenNodeAllocationDecider.java new file mode 100644 index 0000000000000..9f81aef8dcfaa --- /dev/null +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/allocation/decider/DedicatedFrozenNodeAllocationDecider.java @@ -0,0 +1,92 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.searchablesnapshots.allocation.decider; + +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; +import org.elasticsearch.cluster.routing.RoutingNode; +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; +import org.elasticsearch.cluster.routing.allocation.decider.AllocationDecider; +import org.elasticsearch.cluster.routing.allocation.decider.Decision; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsConstants; + +import static org.elasticsearch.cluster.node.DiscoveryNodeRole.DATA_FROZEN_NODE_ROLE; +import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsConstants.SNAPSHOT_PARTIAL_SETTING; + +public class DedicatedFrozenNodeAllocationDecider extends AllocationDecider { + + private static final String NAME = "dedicated_frozen_node"; + + private static final Decision YES_NOT_DEDICATED_FROZEN_NODE = Decision.single( + Decision.Type.YES, + NAME, + "this node's data roles are not exactly [" + DATA_FROZEN_NODE_ROLE.roleName() + "] so it is not a dedicated frozen node" + ); + + private static final Decision YES_IS_PARTIAL_SEARCHABLE_SNAPSHOT = Decision.single( + Decision.Type.YES, + NAME, + "this index is a frozen searchable snapshot so it can be assigned to this dedicated frozen node" + ); + + private static final Decision NO = Decision.single( + Decision.Type.NO, + NAME, + "this node's data roles are exactly [" + + DATA_FROZEN_NODE_ROLE.roleName() + + "] so it may only hold shards from frozen searchable snapshots, but this index is not a frozen searchable snapshot" + ); + + @Override + public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { + return canAllocateToNode(allocation.metadata().getIndexSafe(shardRouting.index()), node.node()); + } + + @Override + public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { + return canAllocateToNode(allocation.metadata().getIndexSafe(shardRouting.index()), node.node()); + } + + @Override + public Decision canAllocate(IndexMetadata indexMetadata, RoutingNode node, RoutingAllocation allocation) { + return canAllocateToNode(indexMetadata, node.node()); + } + + @Override + public Decision shouldAutoExpandToNode(IndexMetadata indexMetadata, DiscoveryNode node, RoutingAllocation allocation) { + return canAllocateToNode(indexMetadata, node); + } + + private Decision canAllocateToNode(IndexMetadata indexMetadata, DiscoveryNode discoveryNode) { + + boolean hasDataFrozenRole = false; + boolean hasOtherDataRole = false; + for (DiscoveryNodeRole role : discoveryNode.getRoles()) { + if (DATA_FROZEN_NODE_ROLE.equals(role)) { + hasDataFrozenRole = true; + } else if (role.canContainData()) { + hasOtherDataRole = true; + break; + } + } + + if (hasDataFrozenRole == false || hasOtherDataRole) { + return YES_NOT_DEDICATED_FROZEN_NODE; + } + + final Settings indexSettings = indexMetadata.getSettings(); + if (SearchableSnapshotsConstants.isSearchableSnapshotStore(indexSettings) && SNAPSHOT_PARTIAL_SETTING.get(indexSettings)) { + return YES_IS_PARTIAL_SEARCHABLE_SNAPSHOT; + } + + return NO; + } +}