Skip to content

Commit

Permalink
Move some constants from SearchableSnapshotsConstants to server (#75308)
Browse files Browse the repository at this point in the history
Elasticsearch's server sometimes has to do things 
differently in order to handle searchable snapshots 
shards. In such cases it relies on index settings 
and hard coded values to know if a shard is a 
searchable snapshot one.

This commit adds a new SearchableSnapshotsSettings 
class in server that provides methods to check if 
an index is a searchable snapshot index. This class 
also contains the names of some index settings 
related to searchable snapshots that are required 
by the server.
  • Loading branch information
tlrx authored Jul 15, 2021
1 parent 5b36c90 commit 9c7a869
Show file tree
Hide file tree
Showing 50 changed files with 201 additions and 205 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@
import java.util.Map;
import java.util.Set;

import static org.elasticsearch.snapshots.SearchableSnapshotsSettings.isPartialSearchableSnapshotIndex;

/**
* This service is responsible for verifying index metadata when an index is introduced
* to the cluster, for example when restarting nodes, importing dangling indices, or restoring
Expand Down Expand Up @@ -202,7 +204,7 @@ IndexMetadata archiveBrokenIndexSettings(IndexMetadata indexMetadata) {
IndexMetadata convertSharedCacheTierPreference(IndexMetadata indexMetadata) {
final Settings settings = indexMetadata.getSettings();
// Only remove these settings for a shared_cache searchable snapshot
if ("snapshot".equals(settings.get("index.store.type", "")) && settings.getAsBoolean("index.store.snapshot.partial", false)) {
if (isPartialSearchableSnapshotIndex(settings)) {
final Settings.Builder settingsBuilder = Settings.builder().put(settings);
// Clear any allocation rules other than preference for tier
settingsBuilder.remove("index.routing.allocation.include._tier");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,7 @@
import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.resolveSettings;
import static org.elasticsearch.index.IndexModule.INDEX_RECOVERY_TYPE_SETTING;
import static org.elasticsearch.index.IndexModule.INDEX_STORE_TYPE_SETTING;
import static org.elasticsearch.snapshots.SearchableSnapshotsSettings.isSearchableSnapshotStore;

/**
* Service responsible for submitting create index requests
Expand Down Expand Up @@ -1080,7 +1081,7 @@ private static List<String> validateIndexCustomPath(Settings settings, @Nullable
*/
static List<String> validateShrinkIndex(ClusterState state, String sourceIndex, String targetIndexName, Settings targetIndexSettings) {
IndexMetadata sourceMetadata = validateResize(state, sourceIndex, targetIndexName, targetIndexSettings);
if ("snapshot".equals(INDEX_STORE_TYPE_SETTING.get(sourceMetadata.getSettings()))) {
if (isSearchableSnapshotStore(sourceMetadata.getSettings())) {
throw new IllegalArgumentException("can't shrink searchable snapshot index [" + sourceIndex + ']');
}
assert INDEX_NUMBER_OF_SHARDS_SETTING.exists(targetIndexSettings);
Expand Down Expand Up @@ -1114,15 +1115,15 @@ static List<String> validateShrinkIndex(ClusterState state, String sourceIndex,

static void validateSplitIndex(ClusterState state, String sourceIndex, String targetIndexName, Settings targetIndexSettings) {
IndexMetadata sourceMetadata = validateResize(state, sourceIndex, targetIndexName, targetIndexSettings);
if ("snapshot".equals(INDEX_STORE_TYPE_SETTING.get(sourceMetadata.getSettings()))) {
if (isSearchableSnapshotStore(sourceMetadata.getSettings())) {
throw new IllegalArgumentException("can't split searchable snapshot index [" + sourceIndex + ']');
}
IndexMetadata.selectSplitShard(0, sourceMetadata, INDEX_NUMBER_OF_SHARDS_SETTING.get(targetIndexSettings));
}

static void validateCloneIndex(ClusterState state, String sourceIndex, String targetIndexName, Settings targetIndexSettings) {
IndexMetadata sourceMetadata = validateResize(state, sourceIndex, targetIndexName, targetIndexSettings);
if ("snapshot".equals(INDEX_STORE_TYPE_SETTING.get(sourceMetadata.getSettings()))) {
if (isSearchableSnapshotStore(sourceMetadata.getSettings())) {
for (Setting<?> nonCloneableSetting : Arrays.asList(INDEX_STORE_TYPE_SETTING, INDEX_RECOVERY_TYPE_SETTING)) {
if (nonCloneableSetting.exists(targetIndexSettings) == false) {
throw new IllegalArgumentException("can't clone searchable snapshot index [" + sourceIndex + "]; setting ["
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,9 @@
import java.util.stream.Stream;

import static java.util.Collections.unmodifiableMap;
import static org.elasticsearch.index.IndexModule.INDEX_STORE_TYPE_SETTING;
import static org.elasticsearch.snapshots.SearchableSnapshotsSettings.SEARCHABLE_SNAPSHOTS_REPOSITORY_NAME_SETTING_KEY;
import static org.elasticsearch.snapshots.SearchableSnapshotsSettings.SEARCHABLE_SNAPSHOTS_REPOSITORY_UUID_SETTING_KEY;
import static org.elasticsearch.snapshots.SearchableSnapshotsSettings.isSearchableSnapshotStore;

/**
* Service responsible for maintaining and providing access to snapshot repositories on nodes.
Expand All @@ -81,9 +83,6 @@ public class RepositoriesService extends AbstractLifecycleComponent implements C
Setting.Property.NodeScope
);

public static final String SEARCHABLE_SNAPSHOTS_REPOSITORY_NAME_SETTING_KEY = "index.store.snapshot.repository_name";
public static final String SEARCHABLE_SNAPSHOTS_REPOSITORY_UUID_SETTING_KEY = "index.store.snapshot.repository_uuid";

private final Map<String, Repository.Factory> typesRegistry;
private final Map<String, Repository.Factory> internalTypesRegistry;

Expand Down Expand Up @@ -738,7 +737,7 @@ private static void ensureNoSearchableSnapshotsIndicesInUse(ClusterState cluster
}

private static boolean indexSettingsMatchRepositoryMetadata(Settings indexSettings, RepositoryMetadata repositoryMetadata) {
if ("snapshot".equals(INDEX_STORE_TYPE_SETTING.get(indexSettings))) {
if (isSearchableSnapshotStore(indexSettings)) {
final String indexRepositoryUuid = indexSettings.get(SEARCHABLE_SNAPSHOTS_REPOSITORY_UUID_SETTING_KEY);
if (Strings.hasLength(indexRepositoryUuid)) {
return Objects.equals(repositoryMetadata.uuid(), indexRepositoryUuid);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@

import static java.util.Collections.emptyMap;
import static java.util.Collections.unmodifiableMap;
import static org.elasticsearch.index.IndexModule.INDEX_STORE_TYPE_SETTING;
import static org.elasticsearch.snapshots.SearchableSnapshotsSettings.isSearchableSnapshotStore;

/**
* This context will execute a file restore of the lucene files. It is primarily designed to be used to
Expand Down Expand Up @@ -175,8 +175,7 @@ public void restore(SnapshotFiles snapshotFiles, Store store, ActionListener<Voi

private void afterRestore(SnapshotFiles snapshotFiles, Store store, StoreFileMetadata restoredSegmentsFile) {
try {
final String indexStoreType = INDEX_STORE_TYPE_SETTING.get(store.indexSettings().getSettings());
if ("snapshot".equals(indexStoreType) == false) {
if (isSearchableSnapshotStore(store.indexSettings().getSettings())) {
Lucene.pruneUnreferencedFiles(restoredSegmentsFile.name(), store.directory());
}
} catch (IOException e) {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/

package org.elasticsearch.snapshots;

import org.elasticsearch.common.settings.Settings;

import static org.elasticsearch.index.IndexModule.INDEX_STORE_TYPE_SETTING;

public final class SearchableSnapshotsSettings {

public static final String SEARCHABLE_SNAPSHOT_STORE_TYPE = "snapshot";
public static final String SEARCHABLE_SNAPSHOT_PARTIAL_SETTING_KEY = "index.store.snapshot.partial";
public static final String SEARCHABLE_SNAPSHOTS_REPOSITORY_NAME_SETTING_KEY = "index.store.snapshot.repository_name";
public static final String SEARCHABLE_SNAPSHOTS_REPOSITORY_UUID_SETTING_KEY = "index.store.snapshot.repository_uuid";

private SearchableSnapshotsSettings() {}

public static boolean isSearchableSnapshotStore(Settings indexSettings) {
return SEARCHABLE_SNAPSHOT_STORE_TYPE.equals(INDEX_STORE_TYPE_SETTING.get(indexSettings));
}

public static boolean isPartialSearchableSnapshotIndex(Settings indexSettings) {
return isSearchableSnapshotStore(indexSettings) && indexSettings.getAsBoolean(SEARCHABLE_SNAPSHOT_PARTIAL_SETTING_KEY, false);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@

import java.util.Objects;

import static org.elasticsearch.snapshots.SearchableSnapshotsSettings.SEARCHABLE_SNAPSHOT_STORE_TYPE;
import static org.hamcrest.Matchers.is;

public class FrozenUtilsTests extends AutoscalingTestCase {
Expand All @@ -46,7 +47,7 @@ public static Settings indexSettings(String tierPreference) {
// pass setting validator.
if (Objects.equals(tierPreference, DataTier.DATA_FROZEN)) {
settings.put(SearchableSnapshotsConstants.SNAPSHOT_PARTIAL_SETTING.getKey(), true)
.put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), SearchableSnapshotsConstants.SNAPSHOT_DIRECTORY_FACTORY_KEY);
.put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), SEARCHABLE_SNAPSHOT_STORE_TYPE);
}
return settings.build();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
package org.elasticsearch.xpack.ccr.action;

import com.carrotsearch.hppc.predicates.ObjectPredicate;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.message.ParameterizedMessage;
Expand All @@ -27,16 +28,17 @@
import org.elasticsearch.cluster.routing.IndexRoutingTable;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.collect.CopyOnWriteHashMap;
import org.elasticsearch.core.Tuple;
import org.elasticsearch.common.component.AbstractLifecycleComponent;
import org.elasticsearch.common.component.Lifecycle;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.core.TimeValue;
import org.elasticsearch.common.util.concurrent.AtomicArray;
import org.elasticsearch.common.util.concurrent.CountDown;
import org.elasticsearch.core.TimeValue;
import org.elasticsearch.core.Tuple;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.license.LicenseUtils;
import org.elasticsearch.snapshots.SearchableSnapshotsSettings;
import org.elasticsearch.transport.NoSuchRemoteClusterException;
import org.elasticsearch.xpack.ccr.Ccr;
import org.elasticsearch.xpack.ccr.CcrLicenseChecker;
Expand All @@ -45,7 +47,6 @@
import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata.AutoFollowPattern;
import org.elasticsearch.xpack.core.ccr.AutoFollowStats;
import org.elasticsearch.xpack.core.ccr.action.PutFollowAction;
import org.elasticsearch.xpack.core.searchablesnapshots.SearchableSnapshotsConstants;

import java.util.ArrayList;
import java.util.Collections;
Expand Down Expand Up @@ -521,7 +522,7 @@ private void checkAutoFollowPattern(String autoFollowPattenName,
}
groupedListener.onResponse(new Tuple<>(indexToFollow, failure));
});
} else if (SearchableSnapshotsConstants.isSearchableSnapshotStore(leaderIndexSettings)) {
} else if (SearchableSnapshotsSettings.isSearchableSnapshotStore(leaderIndexSettings)) {
String message = String.format(Locale.ROOT,
"index to follow [%s] is a searchable snapshot index and cannot be used for cross-cluster replication purpose",
indexToFollow.getName()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
import org.elasticsearch.license.LicenseUtils;
import org.elasticsearch.snapshots.RestoreInfo;
import org.elasticsearch.snapshots.RestoreService;
import org.elasticsearch.snapshots.SearchableSnapshotsSettings;
import org.elasticsearch.tasks.Task;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.TransportService;
Expand All @@ -44,7 +45,6 @@
import org.elasticsearch.xpack.core.ccr.action.FollowParameters;
import org.elasticsearch.xpack.core.ccr.action.PutFollowAction;
import org.elasticsearch.xpack.core.ccr.action.ResumeFollowAction;
import org.elasticsearch.xpack.core.searchablesnapshots.SearchableSnapshotsConstants;

import java.util.ArrayList;
import java.util.Comparator;
Expand Down Expand Up @@ -129,7 +129,7 @@ private void createFollowerIndex(
"] does not have soft deletes enabled"));
return;
}
if (SearchableSnapshotsConstants.isSearchableSnapshotStore(leaderIndexMetadata.getSettings())) {
if (SearchableSnapshotsSettings.isSearchableSnapshotStore(leaderIndexMetadata.getSettings())) {
listener.onFailure(new IllegalArgumentException("leader index [" + request.getLeaderIndex() +
"] is a searchable snapshot index and cannot be used as a leader index for cross-cluster replication purpose"));
return;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
import org.elasticsearch.indices.IndicesService;
import org.elasticsearch.license.LicenseUtils;
import org.elasticsearch.persistent.PersistentTasksService;
import org.elasticsearch.snapshots.SearchableSnapshotsSettings;
import org.elasticsearch.tasks.Task;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.TransportService;
Expand All @@ -52,7 +53,6 @@
import org.elasticsearch.xpack.core.ccr.action.FollowParameters;
import org.elasticsearch.xpack.core.ccr.action.ResumeFollowAction;
import org.elasticsearch.xpack.core.ccr.action.ShardFollowTask;
import org.elasticsearch.xpack.core.searchablesnapshots.SearchableSnapshotsConstants;

import java.io.IOException;
import java.util.Iterator;
Expand Down Expand Up @@ -207,15 +207,15 @@ static void validate(
throw new IllegalArgumentException("leader index [" + leaderIndex.getIndex().getName() +
"] does not have soft deletes enabled");
}
if (SearchableSnapshotsConstants.isSearchableSnapshotStore(leaderIndex.getSettings())) {
if (SearchableSnapshotsSettings.isSearchableSnapshotStore(leaderIndex.getSettings())) {
throw new IllegalArgumentException("leader index [" + leaderIndex.getIndex().getName() +
"] is a searchable snapshot index and cannot be used for cross-cluster replication purpose");
}
if (IndexSettings.INDEX_SOFT_DELETES_SETTING.get(followIndex.getSettings()) == false) {
throw new IllegalArgumentException("follower index [" + request.getFollowerIndex() +
"] does not have soft deletes enabled");
}
if (SearchableSnapshotsConstants.isSearchableSnapshotStore(followIndex.getSettings())) {
if (SearchableSnapshotsSettings.isSearchableSnapshotStore(followIndex.getSettings())) {
throw new IllegalArgumentException("follower index [" + request.getFollowerIndex() +
"] is a searchable snapshot index and cannot be used for cross-cluster replication purpose");
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
package org.elasticsearch.xpack.cluster.routing.allocation;

import com.carrotsearch.hppc.cursors.ObjectCursor;

import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.node.DiscoveryNodeRole;
Expand All @@ -21,6 +22,7 @@
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.IndexModule;
import org.elasticsearch.snapshots.SearchableSnapshotsSettings;
import org.elasticsearch.xpack.core.DataTier;
import org.elasticsearch.xpack.core.searchablesnapshots.SearchableSnapshotsConstants;

Expand Down Expand Up @@ -67,7 +69,7 @@ private static class DataTierValidator implements Setting.Validator<String> {
SearchableSnapshotsConstants.SNAPSHOT_PARTIAL_SETTING);

public static String getDefaultTierPreference(Settings settings) {
if (SearchableSnapshotsConstants.isPartialSearchableSnapshotIndex(settings)) {
if (SearchableSnapshotsSettings.isPartialSearchableSnapshotIndex(settings)) {
return DATA_FROZEN;
} else {
return "";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@
import org.elasticsearch.cluster.node.DiscoveryNodes;
import org.elasticsearch.cluster.routing.allocation.decider.AllocationDecider;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.core.Booleans;
import org.elasticsearch.common.inject.Binder;
import org.elasticsearch.common.inject.multibindings.Multibinder;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
Expand All @@ -34,6 +33,7 @@
import org.elasticsearch.common.settings.SettingsFilter;
import org.elasticsearch.common.util.BigArrays;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.core.Booleans;
import org.elasticsearch.env.Environment;
import org.elasticsearch.env.NodeEnvironment;
import org.elasticsearch.index.IndexSettings;
Expand All @@ -58,6 +58,7 @@
import org.elasticsearch.rest.RestController;
import org.elasticsearch.rest.RestHandler;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.snapshots.SearchableSnapshotsSettings;
import org.elasticsearch.snapshots.sourceonly.SourceOnlySnapshotRepository;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.watcher.ResourceWatcherService;
Expand All @@ -82,12 +83,11 @@
import org.elasticsearch.xpack.core.ssl.SSLConfiguration;
import org.elasticsearch.xpack.core.ssl.SSLConfigurationReloader;
import org.elasticsearch.xpack.core.ssl.SSLService;
import org.elasticsearch.xpack.core.transform.TransformMetadata;
import org.elasticsearch.xpack.core.termsenum.action.TermsEnumAction;
import org.elasticsearch.xpack.core.termsenum.action.TransportTermsEnumAction;
import org.elasticsearch.xpack.core.termsenum.rest.RestTermsEnumAction;
import org.elasticsearch.xpack.core.transform.TransformMetadata;
import org.elasticsearch.xpack.core.watcher.WatcherMetadata;
import org.elasticsearch.xpack.core.searchablesnapshots.SearchableSnapshotsConstants;

import java.nio.file.Files;
import java.nio.file.Path;
Expand Down Expand Up @@ -373,7 +373,7 @@ public Map<String, Repository.Factory> getRepositories(Environment env, NamedXCo
@Override
public Optional<EngineFactory> getEngineFactory(IndexSettings indexSettings) {
if (indexSettings.getValue(SourceOnlySnapshotRepository.SOURCE_ONLY) &&
SearchableSnapshotsConstants.isSearchableSnapshotStore(indexSettings.getSettings()) == false) {
SearchableSnapshotsSettings.isSearchableSnapshotStore(indexSettings.getSettings()) == false) {
return Optional.of(SourceOnlySnapshotRepository.getEngineFactory());
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,18 +9,18 @@
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.client.Client;
import org.elasticsearch.common.xcontent.ParseField;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.ConstructingObjectParser;
import org.elasticsearch.common.xcontent.ParseField;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.snapshots.SearchableSnapshotsSettings;
import org.elasticsearch.xpack.cluster.routing.allocation.DataTierAllocationDecider;
import org.elasticsearch.xpack.core.DataTier;
import org.elasticsearch.xpack.core.ilm.Step.StepKey;
import org.elasticsearch.xpack.core.searchablesnapshots.SearchableSnapshotsConstants;

import java.io.IOException;
import java.util.List;
Expand Down Expand Up @@ -106,7 +106,7 @@ public List<Step> toSteps(Client client, String phase, StepKey nextStepKey) {
Settings indexSettings = clusterState.metadata().index(index).getSettings();

// partially mounted indices will already have data_frozen, and we don't want to change that if they do
if (SearchableSnapshotsConstants.isPartialSearchableSnapshotIndex(indexSettings)) {
if (SearchableSnapshotsSettings.isPartialSearchableSnapshotIndex(indexSettings)) {
String policyName = LifecycleSettings.LIFECYCLE_NAME_SETTING.get(indexSettings);
logger.debug("[{}] action in policy [{}] is configured for index [{}] which is a partially mounted index. " +
"skipping this action", MigrateAction.NAME, policyName, index.getName());
Expand Down
Loading

0 comments on commit 9c7a869

Please sign in to comment.