Skip to content

Commit

Permalink
Remove unused pluggable metadata upgraders (#47297)
Browse files Browse the repository at this point in the history 
Today plugins may provide upgraders for custom metadata and index metadata, but
these upgraders are bypassed during a rolling restart. Fortunately this
extension mechanism is unused by all known plugins. This commit removes these
extension points.
  • Loading branch information
@DaveCTurner
DaveCTurner authored Sep 30, 2019
1 parent 5405f2e commit cb800ab
Show file tree
Hide file tree
Showing 9 changed files with 56 additions and 276 deletions.
18 changes: 2 additions & 16 deletions server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,11 +38,9 @@
import org.elasticsearch.script.ScriptService;

import java.util.AbstractMap;
import java.util.Collection;
import java.util.Collections;
import java.util.Map;
import java.util.Set;
import java.util.function.UnaryOperator;

/**
* This service is responsible for upgrading legacy index metadata to the current version
Expand All @@ -60,22 +58,13 @@ public class MetaDataIndexUpgradeService {
private final NamedXContentRegistry xContentRegistry;
private final MapperRegistry mapperRegistry;
private final IndexScopedSettings indexScopedSettings;
private final UnaryOperator<IndexMetaData> upgraders;

public MetaDataIndexUpgradeService(Settings settings, NamedXContentRegistry xContentRegistry, MapperRegistry mapperRegistry,
IndexScopedSettings indexScopedSettings,
Collection<UnaryOperator<IndexMetaData>> indexMetaDataUpgraders) {
IndexScopedSettings indexScopedSettings) {
this.settings = settings;
this.xContentRegistry = xContentRegistry;
this.mapperRegistry = mapperRegistry;
this.indexScopedSettings = indexScopedSettings;
this.upgraders = indexMetaData -> {
IndexMetaData newIndexMetaData = indexMetaData;
for (UnaryOperator<IndexMetaData> upgrader : indexMetaDataUpgraders) {
newIndexMetaData = upgrader.apply(newIndexMetaData);
}
return newIndexMetaData;
};
}

/**
Expand All @@ -95,14 +84,11 @@ public IndexMetaData upgradeIndexMetaData(IndexMetaData indexMetaData, Version m
return archiveBrokenIndexSettings(indexMetaData);
}
checkSupportedVersion(indexMetaData, minimumIndexCompatibilityVersion);
IndexMetaData newMetaData = indexMetaData;
// we have to run this first otherwise in we try to create IndexSettings
// with broken settings and fail in checkMappingsCompatibility
newMetaData = archiveBrokenIndexSettings(newMetaData);
final IndexMetaData newMetaData = archiveBrokenIndexSettings(indexMetaData);
// only run the check with the upgraded settings!!
checkMappingsCompatibility(newMetaData);
// apply plugin checks
newMetaData = upgraders.apply(newMetaData);
return markAsUpgraded(newMetaData);
}

Expand Down
23 changes: 9 additions & 14 deletions server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
import org.elasticsearch.cluster.coordination.CoordinationState.PersistedState;
import org.elasticsearch.cluster.coordination.InMemoryPersistedState;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
import org.elasticsearch.cluster.metadata.Manifest;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.cluster.metadata.MetaDataIndexUpgradeService;
Expand Down Expand Up @@ -206,7 +207,6 @@ private static boolean isMasterOrDataNode(Settings settings) {
* Elasticsearch 2.0 removed several deprecated features and as well as support for Lucene 3.x. This method calls
* {@link MetaDataIndexUpgradeService} to makes sure that indices are compatible with the current version. The
* MetaDataIndexUpgradeService might also update obsolete settings if needed.
* Allows upgrading global custom meta data via {@link MetaDataUpgrader#customMetaDataUpgraders}
*
* @return input <code>metaData</code> if no upgrade is needed or an upgraded metaData
*/
Expand All @@ -222,11 +222,6 @@ static MetaData upgradeMetaData(MetaData metaData,
changed |= indexMetaData != newMetaData;
upgradedMetaData.put(newMetaData, false);
}
// upgrade global custom meta data
if (applyPluginUpgraders(metaData.getCustoms(), metaDataUpgrader.customMetaDataUpgraders,
upgradedMetaData::removeCustom, upgradedMetaData::putCustom)) {
changed = true;
}
// upgrade current templates
if (applyPluginUpgraders(metaData.getTemplates(), metaDataUpgrader.indexTemplateMetaDataUpgraders,
upgradedMetaData::removeTemplate, (s, indexTemplateMetaData) -> upgradedMetaData.put(indexTemplateMetaData))) {
Expand All @@ -235,21 +230,21 @@ static MetaData upgradeMetaData(MetaData metaData,
return changed ? upgradedMetaData.build() : metaData;
}

private static <Data> boolean applyPluginUpgraders(ImmutableOpenMap<String, Data> existingData,
UnaryOperator<Map<String, Data>> upgrader,
Consumer<String> removeData,
BiConsumer<String, Data> putData) {
private static boolean applyPluginUpgraders(ImmutableOpenMap<String, IndexTemplateMetaData> existingData,
UnaryOperator<Map<String, IndexTemplateMetaData>> upgrader,
Consumer<String> removeData,
BiConsumer<String, IndexTemplateMetaData> putData) {
// collect current data
Map<String, Data> existingMap = new HashMap<>();
for (ObjectObjectCursor<String, Data> customCursor : existingData) {
Map<String, IndexTemplateMetaData> existingMap = new HashMap<>();
for (ObjectObjectCursor<String, IndexTemplateMetaData> customCursor : existingData) {
existingMap.put(customCursor.key, customCursor.value);
}
// upgrade global custom meta data
Map<String, Data> upgradedCustoms = upgrader.apply(existingMap);
Map<String, IndexTemplateMetaData> upgradedCustoms = upgrader.apply(existingMap);
if (upgradedCustoms.equals(existingMap) == false) {
// remove all data first so a plugin can remove custom metadata or templates if needed
existingMap.keySet().forEach(removeData);
for (Map.Entry<String, Data> upgradedCustomEntry : upgradedCustoms.entrySet()) {
for (Map.Entry<String, IndexTemplateMetaData> upgradedCustomEntry : upgradedCustoms.entrySet()) {
putData.accept(upgradedCustomEntry.getKey(), upgradedCustomEntry.getValue());
}
return true;
Expand Down
11 changes: 2 additions & 9 deletions server/src/main/java/org/elasticsearch/node/Node.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,6 @@
import org.elasticsearch.cluster.NodeConnectionsService;
import org.elasticsearch.cluster.action.index.MappingUpdatedAction;
import org.elasticsearch.cluster.metadata.AliasValidator;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.cluster.metadata.MetaDataCreateIndexService;
Expand Down Expand Up @@ -458,19 +457,13 @@ protected Node(
final NetworkModule networkModule = new NetworkModule(settings, pluginsService.filterPlugins(NetworkPlugin.class),
threadPool, bigArrays, pageCacheRecycler, circuitBreakerService, namedWriteableRegistry, xContentRegistry,
networkService, restController);
Collection<UnaryOperator<Map<String, MetaData.Custom>>> customMetaDataUpgraders =
pluginsService.filterPlugins(Plugin.class).stream()
.map(Plugin::getCustomMetaDataUpgrader)
.collect(Collectors.toList());
Collection<UnaryOperator<Map<String, IndexTemplateMetaData>>> indexTemplateMetaDataUpgraders =
pluginsService.filterPlugins(Plugin.class).stream()
.map(Plugin::getIndexTemplateMetaDataUpgrader)
.collect(Collectors.toList());
Collection<UnaryOperator<IndexMetaData>> indexMetaDataUpgraders = pluginsService.filterPlugins(Plugin.class).stream()
.map(Plugin::getIndexMetaDataUpgrader).collect(Collectors.toList());
final MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(customMetaDataUpgraders, indexTemplateMetaDataUpgraders);
final MetaDataUpgrader metaDataUpgrader = new MetaDataUpgrader(indexTemplateMetaDataUpgraders);
final MetaDataIndexUpgradeService metaDataIndexUpgradeService = new MetaDataIndexUpgradeService(settings, xContentRegistry,
indicesModule.getMapperRegistry(), settingsModule.getIndexScopedSettings(), indexMetaDataUpgraders);
indicesModule.getMapperRegistry(), settingsModule.getIndexScopedSettings());
new TemplateUpgradeService(client, clusterService, threadPool, indexTemplateMetaDataUpgraders);
final Transport transport = networkModule.getTransportSupplier().get();
Set<String> taskHeaders = Stream.concat(
Expand Down
13 changes: 1 addition & 12 deletions server/src/main/java/org/elasticsearch/plugins/MetaDataUpgrader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,20 +31,9 @@
* Upgrades {@link MetaData} on startup on behalf of installed {@link Plugin}s
*/
public class MetaDataUpgrader {
public final UnaryOperator<Map<String, MetaData.Custom>> customMetaDataUpgraders;

public final UnaryOperator<Map<String, IndexTemplateMetaData>> indexTemplateMetaDataUpgraders;

public MetaDataUpgrader(Collection<UnaryOperator<Map<String, MetaData.Custom>>> customMetaDataUpgraders,
Collection<UnaryOperator<Map<String, IndexTemplateMetaData>>> indexTemplateMetaDataUpgraders) {
this.customMetaDataUpgraders = customs -> {
Map<String, MetaData.Custom> upgradedCustoms = new HashMap<>(customs);
for (UnaryOperator<Map<String, MetaData.Custom>> customMetaDataUpgrader : customMetaDataUpgraders) {
upgradedCustoms = customMetaDataUpgrader.apply(upgradedCustoms);
}
return upgradedCustoms;
};

public MetaDataUpgrader(Collection<UnaryOperator<Map<String, IndexTemplateMetaData>>> indexTemplateMetaDataUpgraders) {
this.indexTemplateMetaDataUpgraders = templates -> {
Map<String, IndexTemplateMetaData> upgradedTemplates = new HashMap<>(templates);
for (UnaryOperator<Map<String, IndexTemplateMetaData>> upgrader : indexTemplateMetaDataUpgraders) {
Expand Down
33 changes: 0 additions & 33 deletions server/src/main/java/org/elasticsearch/plugins/Plugin.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,7 @@

import org.elasticsearch.bootstrap.BootstrapCheck;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.cluster.node.DiscoveryNodeRole;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.component.LifecycleComponent;
Expand Down Expand Up @@ -143,22 +141,6 @@ public List<SettingUpgrader<?>> getSettingUpgraders() {
return Collections.emptyList();
}

/**
* Provides a function to modify global custom meta data on startup.
* <p>
* Plugins should return the input custom map via {@link UnaryOperator#identity()} if no upgrade is required.
* <p>
* The order of custom meta data upgraders calls is undefined and can change between runs so, it is expected that
* plugins will modify only data owned by them to avoid conflicts.
* <p>
* @return Never {@code null}. The same or upgraded {@code MetaData.Custom} map.
* @throws IllegalStateException if the node should not start because at least one {@code MetaData.Custom}
* is unsupported
*/
public UnaryOperator<Map<String, MetaData.Custom>> getCustomMetaDataUpgrader() {
return UnaryOperator.identity();
}

/**
* Provides a function to modify index template meta data on startup.
* <p>
Expand All @@ -175,21 +157,6 @@ public UnaryOperator<Map<String, IndexTemplateMetaData>> getIndexTemplateMetaDat
return UnaryOperator.identity();
}

/**
* Provides a function to modify index meta data when an index is introduced into the cluster state for the first time.
* <p>
* Plugins should return the input index metadata via {@link UnaryOperator#identity()} if no upgrade is required.
* <p>
* The order of the index upgrader calls for the same index is undefined and can change between runs so, it is expected that
* plugins will modify only indices owned by them to avoid conflicts.
* <p>
* @return Never {@code null}. The same or upgraded {@code IndexMetaData}.
* @throws IllegalStateException if the node should not start because the index is unsupported
*/
public UnaryOperator<IndexMetaData> getIndexMetaDataUpgrader() {
return UnaryOperator.identity();
}

/**
* Provides the list of this plugin's custom thread pools, empty if
* none.
Expand Down
34 changes: 1 addition & 33 deletions ...er/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeServiceTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,44 +120,12 @@ public void testFailUpgrade() {
service.upgradeIndexMetaData(goodMeta, Version.CURRENT.minimumIndexCompatibilityVersion());
}

public void testPluginUpgrade() {
MetaDataIndexUpgradeService service = new MetaDataIndexUpgradeService(Settings.EMPTY, xContentRegistry(),
new MapperRegistry(Collections.emptyMap(), Collections.emptyMap(), MapperPlugin.NOOP_FIELD_FILTER),
IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, Collections.singletonList(
indexMetaData -> IndexMetaData.builder(indexMetaData).settings(
Settings.builder()
.put(indexMetaData.getSettings())
.put("index.refresh_interval", "10s")
).build()));
IndexMetaData src = newIndexMeta("foo", Settings.builder().put("index.refresh_interval", "200s").build());
assertFalse(service.isUpgraded(src));
src = service.upgradeIndexMetaData(src, Version.CURRENT.minimumIndexCompatibilityVersion());
assertTrue(service.isUpgraded(src));
assertEquals("10s", src.getSettings().get("index.refresh_interval"));
assertSame(src, service.upgradeIndexMetaData(src, Version.CURRENT.minimumIndexCompatibilityVersion())); // no double upgrade
}

public void testPluginUpgradeFailure() {
MetaDataIndexUpgradeService service = new MetaDataIndexUpgradeService(Settings.EMPTY, xContentRegistry(),
new MapperRegistry(Collections.emptyMap(), Collections.emptyMap(), MapperPlugin.NOOP_FIELD_FILTER),
IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, Collections.singletonList(
indexMetaData -> {
throw new IllegalStateException("Cannot upgrade index " + indexMetaData.getIndex().getName());
}
));
IndexMetaData src = newIndexMeta("foo", Settings.EMPTY);
String message = expectThrows(IllegalStateException.class, () -> service.upgradeIndexMetaData(src,
Version.CURRENT.minimumIndexCompatibilityVersion())).getMessage();
assertEquals(message, "Cannot upgrade index foo");
}

private MetaDataIndexUpgradeService getMetaDataIndexUpgradeService() {
return new MetaDataIndexUpgradeService(
Settings.EMPTY,
xContentRegistry(),
new MapperRegistry(Collections.emptyMap(), Collections.emptyMap(), MapperPlugin.NOOP_FIELD_FILTER),
IndexScopedSettings.DEFAULT_SCOPED_SETTINGS,
Collections.emptyList());
IndexScopedSettings.DEFAULT_SCOPED_SETTINGS);
}

public static IndexMetaData newIndexMeta(String name, Settings indexSettings) {
Expand Down
Loading

0 comments on commit cb800ab

Please sign in to comment.