From b0421d4035b87c6422da38d8b6c286e7a49215a4 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Mon, 6 Aug 2018 17:09:38 -0400 Subject: [PATCH] Remove PROTO-serialization from IndexMetaData.Custom Switches IndexMetaData.Custom to standard named serialization. In order to achieve this it also removes support for specifying custom index metadata on create index and put template requests. --- .../indices/create/CreateIndexRequest.java | 48 +--- .../create/CreateIndexRequestBuilder.java | 9 - .../create/TransportCreateIndexAction.java | 2 +- .../indices/shrink/TransportResizeAction.java | 1 - .../template/put/PutIndexTemplateRequest.java | 42 +--- .../put/TransportPutIndexTemplateAction.java | 1 - .../cluster/metadata/IndexMetaData.java | 84 +++---- .../metadata/IndexTemplateMetaData.java | 72 +----- .../metadata/MetaDataCreateIndexService.java | 23 -- .../MetaDataIndexTemplateService.java | 3 - .../indices/create/CustomIndexMetadataIT.java | 237 ++++++++++++++++++ .../metadata/IndexCreationTaskTests.java | 23 +- .../metadata/IndexTemplateMetaDataTests.java | 4 +- .../authz/store/NativeRolesStoreTests.java | 2 +- 14 files changed, 301 insertions(+), 250 deletions(-) create mode 100644 server/src/test/java/org/elasticsearch/action/admin/indices/create/CustomIndexMetadataIT.java diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java index 875d17eb54bc8..b5c667426c1b4 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java @@ -29,7 +29,6 @@ import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.AcknowledgedRequest; -import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; @@ -87,8 +86,6 @@ public class CreateIndexRequest extends AcknowledgedRequest private final Set aliases = new HashSet<>(); - private final Map customs = new HashMap<>(); - private ActiveShardCount waitForActiveShards = ActiveShardCount.DEFAULT; public CreateIndexRequest() { @@ -388,18 +385,7 @@ public CreateIndexRequest source(Map source, DeprecationHandler depre } else if (ALIASES.match(name, deprecationHandler)) { aliases((Map) entry.getValue()); } else { - // maybe custom? - IndexMetaData.Custom proto = IndexMetaData.lookupPrototype(name); - if (proto != null) { - try { - customs.put(name, proto.fromMap((Map) entry.getValue())); - } catch (IOException e) { - throw new ElasticsearchParseException("failed to parse custom metadata for [{}]", name); - } - } else { - // found a key which is neither custom defined nor one of the supported ones - throw new ElasticsearchParseException("unknown key [{}] for create index", name); - } + throw new ElasticsearchParseException("unknown key [{}] for create index", name); } } return this; @@ -413,18 +399,6 @@ public Set aliases() { return this.aliases; } - /** - * Adds custom metadata to the index to be created. - */ - public CreateIndexRequest custom(IndexMetaData.Custom custom) { - customs.put(custom.type(), custom); - return this; - } - - public Map customs() { - return this.customs; - } - public ActiveShardCount waitForActiveShards() { return waitForActiveShards; } @@ -474,11 +448,11 @@ public void readFrom(StreamInput in) throws IOException { } mappings.put(type, source); } - int customSize = in.readVInt(); - for (int i = 0; i < customSize; i++) { - String type = in.readString(); - IndexMetaData.Custom customIndexMetaData = IndexMetaData.lookupPrototypeSafe(type).readFrom(in); - customs.put(type, customIndexMetaData); + if (in.getVersion().before(Version.V_7_0_0_alpha1)) { + int customSize = in.readVInt(); + if (customSize > 0) { + throw new IllegalArgumentException("Custom index metadata is not supported in create index request"); + } } int aliasesSize = in.readVInt(); for (int i = 0; i < aliasesSize; i++) { @@ -501,10 +475,8 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(entry.getKey()); out.writeString(entry.getValue()); } - out.writeVInt(customs.size()); - for (Map.Entry entry : customs.entrySet()) { - out.writeString(entry.getKey()); - entry.getValue().writeTo(out); + if (out.getVersion().before(Version.V_7_0_0_alpha1)) { + out.writeVInt(0); // customs } out.writeVInt(aliases.size()); for (Alias alias : aliases) { @@ -542,10 +514,6 @@ public XContentBuilder innerToXContent(XContentBuilder builder, Params params) t alias.toXContent(builder, params); } builder.endObject(); - - for (Map.Entry entry : customs.entrySet()) { - builder.field(entry.getKey(), entry.getValue(), params); - } return builder; } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilder.java index cc8fb2c32c375..d2593e7e94be3 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequestBuilder.java @@ -23,7 +23,6 @@ import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.master.AcknowledgedRequestBuilder; import org.elasticsearch.client.ElasticsearchClient; -import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; @@ -224,14 +223,6 @@ public CreateIndexRequestBuilder setSource(Map source) { return this; } - /** - * Adds custom metadata to the index to be created. - */ - public CreateIndexRequestBuilder addCustom(IndexMetaData.Custom custom) { - request.custom(custom); - return this; - } - /** * Sets the settings and mappings as a single source. */ diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java index 4cf159c439cb5..e4384745d36e8 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java @@ -75,7 +75,7 @@ protected void masterOperation(final CreateIndexRequest request, final ClusterSt final CreateIndexClusterStateUpdateRequest updateRequest = new CreateIndexClusterStateUpdateRequest(request, cause, indexName, request.index()) .ackTimeout(request.timeout()).masterNodeTimeout(request.masterNodeTimeout()) .settings(request.settings()).mappings(request.mappings()) - .aliases(request.aliases()).customs(request.customs()) + .aliases(request.aliases()) .waitForActiveShards(request.waitForActiveShards()); createIndexService.createIndex(updateRequest, ActionListener.wrap(response -> diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java index 7195fd78154c1..5459805416e91 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java @@ -185,7 +185,6 @@ static CreateIndexClusterStateUpdateRequest prepareCreateIndexRequest(final Resi .masterNodeTimeout(targetIndex.masterNodeTimeout()) .settings(targetIndex.settings()) .aliases(targetIndex.aliases()) - .customs(targetIndex.customs()) .waitForActiveShards(targetIndex.waitForActiveShards()) .recoverFrom(metaData.getIndex()) .resizeType(resizeRequest.getResizeType()) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java index d194b9acd1b7f..38f8144195477 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java @@ -27,7 +27,6 @@ import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.MasterNodeRequest; -import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; @@ -88,8 +87,6 @@ public class PutIndexTemplateRequest extends MasterNodeRequest aliases = new HashSet<>(); - private Map customs = new HashMap<>(); - private Integer version; public PutIndexTemplateRequest() { @@ -352,16 +349,6 @@ public PutIndexTemplateRequest source(Map templateSource) { } } else if (name.equals("aliases")) { aliases((Map) entry.getValue()); - } else { - // maybe custom? - IndexMetaData.Custom proto = IndexMetaData.lookupPrototype(name); - if (proto != null) { - try { - customs.put(name, proto.fromMap((Map) entry.getValue())); - } catch (IOException e) { - throw new ElasticsearchParseException("failed to parse custom metadata for [{}]", name); - } - } } } return this; @@ -395,15 +382,6 @@ public PutIndexTemplateRequest source(BytesReference source, XContentType xConte return source(XContentHelper.convertToMap(source, true, xContentType).v2()); } - public PutIndexTemplateRequest custom(IndexMetaData.Custom custom) { - customs.put(custom.type(), custom); - return this; - } - - public Map customs() { - return this.customs; - } - public Set aliases() { return this.aliases; } @@ -499,11 +477,11 @@ public void readFrom(StreamInput in) throws IOException { } mappings.put(type, mappingSource); } - int customSize = in.readVInt(); - for (int i = 0; i < customSize; i++) { - String type = in.readString(); - IndexMetaData.Custom customIndexMetaData = IndexMetaData.lookupPrototypeSafe(type).readFrom(in); - customs.put(type, customIndexMetaData); + if (in.getVersion().before(Version.V_7_0_0_alpha1)) { + int customSize = in.readVInt(); + if (customSize > 0) { + throw new IllegalArgumentException("Custom index metadata is not supported in create template request"); + } } int aliasesSize = in.readVInt(); for (int i = 0; i < aliasesSize; i++) { @@ -530,10 +508,8 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(entry.getKey()); out.writeString(entry.getValue()); } - out.writeVInt(customs.size()); - for (Map.Entry entry : customs.entrySet()) { - out.writeString(entry.getKey()); - entry.getValue().writeTo(out); + if (out.getVersion().before(Version.V_7_0_0_alpha1)) { + out.writeVInt(0); // customs } out.writeVInt(aliases.size()); for (Alias alias : aliases) { @@ -570,10 +546,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } builder.endObject(); - for (Map.Entry entry : customs.entrySet()) { - builder.field(entry.getKey(), entry.getValue(), params); - } - return builder; } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java index 7b46dc602d0ce..a653dbd866649 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java @@ -83,7 +83,6 @@ protected void masterOperation(final PutIndexTemplateRequest request, final Clus .settings(templateSettingsBuilder.build()) .mappings(request.mappings()) .aliases(request.aliases()) - .customs(request.customs()) .create(request.create()) .masterTimeout(request.masterNodeTimeout()) .version(request.version()), diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java index 18b89db72a391..427144269f4da 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java @@ -24,12 +24,16 @@ import com.carrotsearch.hppc.cursors.ObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; +import org.apache.logging.log4j.Logger; import org.elasticsearch.Version; import org.elasticsearch.action.admin.indices.rollover.RolloverInfo; import org.elasticsearch.action.support.ActiveShardCount; +import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.Diff; import org.elasticsearch.cluster.Diffable; import org.elasticsearch.cluster.DiffableUtils; +import org.elasticsearch.cluster.NamedDiffable; +import org.elasticsearch.cluster.NamedDiffableValueSerializer; import org.elasticsearch.cluster.block.ClusterBlock; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.node.DiscoveryNodeFilters; @@ -40,13 +44,16 @@ import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.io.stream.NamedWriteable; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.ToXContentFragment; +import org.elasticsearch.common.xcontent.UnknownNamedObjectException; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentHelper; @@ -80,33 +87,13 @@ public class IndexMetaData implements Diffable, ToXContentFragment { + private static final Logger logger = Loggers.getLogger(IndexMetaData.class); + /** - * This class will be removed in v7.0 + * Custom index metadata components */ - @Deprecated - public interface Custom extends Diffable, ToXContent { - - String type(); - - Custom fromMap(Map map) throws IOException; - - Custom fromXContent(XContentParser parser) throws IOException; - - /** - * Reads the {@link org.elasticsearch.cluster.Diff} from StreamInput - */ - Diff readDiffFrom(StreamInput in) throws IOException; - - /** - * Reads an object of this type from the provided {@linkplain StreamInput}. The receiving instance remains unchanged. - */ - Custom readFrom(StreamInput in) throws IOException; + public interface Custom extends NamedDiffable, ToXContent, NamedWriteable, ClusterState.FeatureAware { - /** - * Merges from this to another, with this being more important, i.e., if something exists in this and another, - * this will prevail. - */ - Custom mergeWith(Custom another); } public static Map customPrototypes = new HashMap<>(); @@ -300,6 +287,7 @@ public Iterator> settings() { public static final String KEY_PRIMARY_TERMS = "primary_terms"; public static final String INDEX_STATE_FILE_PREFIX = "state-"; + private static final NamedDiffableValueSerializer CUSTOM_VALUE_SERIALIZER = new NamedDiffableValueSerializer<>(Custom.class); private final int routingNumShards; private final int routingFactor; private final int routingPartitionSize; @@ -662,7 +650,7 @@ private static class IndexMetaDataDiff implements Diff { primaryTerms = after.primaryTerms; mappings = DiffableUtils.diff(before.mappings, after.mappings, DiffableUtils.getStringKeySerializer()); aliases = DiffableUtils.diff(before.aliases, after.aliases, DiffableUtils.getStringKeySerializer()); - customs = DiffableUtils.diff(before.customs, after.customs, DiffableUtils.getStringKeySerializer()); + customs = DiffableUtils.diff(before.customs, after.customs, DiffableUtils.getStringKeySerializer(), CUSTOM_VALUE_SERIALIZER); inSyncAllocationIds = DiffableUtils.diff(before.inSyncAllocationIds, after.inSyncAllocationIds, DiffableUtils.getVIntKeySerializer(), DiffableUtils.StringSetValueSerializer.getInstance()); rolloverInfos = DiffableUtils.diff(before.rolloverInfos, after.rolloverInfos, DiffableUtils.getStringKeySerializer()); @@ -679,18 +667,7 @@ private static class IndexMetaDataDiff implements Diff { MappingMetaData::readDiffFrom); aliases = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), AliasMetaData::new, AliasMetaData::readDiffFrom); - customs = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), - new DiffableUtils.DiffableValueSerializer() { - @Override - public Custom read(StreamInput in, String key) throws IOException { - return lookupPrototypeSafe(key).readFrom(in); - } - - @Override - public Diff readDiff(StreamInput in, String key) throws IOException { - return lookupPrototypeSafe(key).readDiffFrom(in); - } - }); + customs = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), CUSTOM_VALUE_SERIALIZER); inSyncAllocationIds = DiffableUtils.readImmutableOpenIntMapDiff(in, DiffableUtils.getVIntKeySerializer(), DiffableUtils.StringSetValueSerializer.getInstance()); if (in.getVersion().onOrAfter(Version.V_6_4_0)) { @@ -755,9 +732,8 @@ public static IndexMetaData readFrom(StreamInput in) throws IOException { } int customSize = in.readVInt(); for (int i = 0; i < customSize; i++) { - String type = in.readString(); - Custom customIndexMetaData = lookupPrototypeSafe(type).readFrom(in); - builder.putCustom(type, customIndexMetaData); + Custom customIndexMetaData = in.readNamedWriteable(Custom.class); + builder.putCustom(customIndexMetaData.getWriteableName(), customIndexMetaData); } int inSyncAllocationIdsSize = in.readVInt(); for (int i = 0; i < inSyncAllocationIdsSize; i++) { @@ -790,10 +766,18 @@ public void writeTo(StreamOutput out) throws IOException { for (ObjectCursor cursor : aliases.values()) { cursor.value.writeTo(out); } - out.writeVInt(customs.size()); - for (ObjectObjectCursor cursor : customs) { - out.writeString(cursor.key); - cursor.value.writeTo(out); + // filter out custom metadata not supported by the other node + int numberOfCustoms = 0; + for (final ObjectCursor cursor : customs.values()) { + if (ClusterState.FeatureAware.shouldSerialize(out, cursor.value)) { + numberOfCustoms++; + } + } + out.writeVInt(numberOfCustoms); + for (final ObjectCursor cursor : customs.values()) { + if (ClusterState.FeatureAware.shouldSerialize(out, cursor.value)) { + out.writeNamedWriteable(cursor.value); + } } out.writeVInt(inSyncAllocationIds.size()); for (IntObjectCursor> cursor : inSyncAllocationIds) { @@ -1276,13 +1260,13 @@ public static IndexMetaData fromXContent(XContentParser parser) throws IOExcepti parser.skipChildren(); } else { // check if its a custom index metadata - Custom proto = lookupPrototype(currentFieldName); - if (proto == null) { - //TODO warn + try { + Custom custom = parser.namedObject(Custom.class, currentFieldName, null); + builder.putCustom(custom.getWriteableName(), custom); + } catch (UnknownNamedObjectException ex) { + logger.warn("Skipping unknown custom index metadata object with type {}", currentFieldName); + // Skip children in case the plugin the provided the metadata got uninstalled parser.skipChildren(); - } else { - Custom custom = proto.fromXContent(parser); - builder.putCustom(custom.type(), custom); } } } else if (token == XContentParser.Token.START_ARRAY) { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java index d35a4baa1e680..1ca3ba0ae55dc 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java @@ -87,13 +87,10 @@ public class IndexTemplateMetaData extends AbstractDiffable aliases; - private final ImmutableOpenMap customs; - public IndexTemplateMetaData(String name, int order, Integer version, List patterns, Settings settings, ImmutableOpenMap mappings, - ImmutableOpenMap aliases, - ImmutableOpenMap customs) { + ImmutableOpenMap aliases) { if (patterns == null || patterns.isEmpty()) { throw new IllegalArgumentException("Index patterns must not be null or empty; got " + patterns); } @@ -104,7 +101,6 @@ public IndexTemplateMetaData(String name, int order, Integer version, this.settings = settings; this.mappings = mappings; this.aliases = aliases; - this.customs = customs; } public String name() { @@ -165,19 +161,6 @@ public ImmutableOpenMap getAliases() { return this.aliases; } - public ImmutableOpenMap customs() { - return this.customs; - } - - public ImmutableOpenMap getCustoms() { - return this.customs; - } - - @SuppressWarnings("unchecked") - public T custom(String type) { - return (T) customs.get(type); - } - public static Builder builder(String name) { return new Builder(name); } @@ -227,11 +210,11 @@ public static IndexTemplateMetaData readFrom(StreamInput in) throws IOException AliasMetaData aliasMd = new AliasMetaData(in); builder.putAlias(aliasMd); } - int customSize = in.readVInt(); - for (int i = 0; i < customSize; i++) { - String type = in.readString(); - IndexMetaData.Custom customIndexMetaData = IndexMetaData.lookupPrototypeSafe(type).readFrom(in); - builder.putCustom(type, customIndexMetaData); + if (in.getVersion().before(Version.V_7_0_0_alpha1)) { + int customSize = in.readVInt(); + if (customSize > 0) { + throw new IllegalArgumentException("Custom index metadata is no longer supported in Index Templates"); + } } builder.version(in.readOptionalVInt()); return builder.build(); @@ -260,10 +243,8 @@ public void writeTo(StreamOutput out) throws IOException { for (ObjectCursor cursor : aliases.values()) { cursor.value.writeTo(out); } - out.writeVInt(customs.size()); - for (ObjectObjectCursor cursor : customs) { - out.writeString(cursor.key); - cursor.value.writeTo(out); + if (out.getVersion().before(Version.V_7_0_0_alpha1)) { + out.writeVInt(0); // Custom IndexMetaData } out.writeOptionalVInt(version); } @@ -290,13 +271,10 @@ public static class Builder { private final ImmutableOpenMap.Builder aliases; - private final ImmutableOpenMap.Builder customs; - public Builder(String name) { this.name = name; mappings = ImmutableOpenMap.builder(); aliases = ImmutableOpenMap.builder(); - customs = ImmutableOpenMap.builder(); } public Builder(IndexTemplateMetaData indexTemplateMetaData) { @@ -308,7 +286,6 @@ public Builder(IndexTemplateMetaData indexTemplateMetaData) { mappings = ImmutableOpenMap.builder(indexTemplateMetaData.mappings()); aliases = ImmutableOpenMap.builder(indexTemplateMetaData.aliases()); - customs = ImmutableOpenMap.builder(indexTemplateMetaData.customs()); } public Builder order(int order) { @@ -362,23 +339,8 @@ public Builder putAlias(AliasMetaData.Builder aliasMetaData) { return this; } - public Builder putCustom(String type, IndexMetaData.Custom customIndexMetaData) { - this.customs.put(type, customIndexMetaData); - return this; - } - - public Builder removeCustom(String type) { - this.customs.remove(type); - return this; - } - - public IndexMetaData.Custom getCustom(String type) { - return this.customs.get(type); - } - public IndexTemplateMetaData build() { - return new IndexTemplateMetaData(name, order, version, indexPatterns, settings, mappings.build(), - aliases.build(), customs.build()); + return new IndexTemplateMetaData(name, order, version, indexPatterns, settings, mappings.build(), aliases.build()); } public static void toXContent(IndexTemplateMetaData indexTemplateMetaData, XContentBuilder builder, ToXContent.Params params) @@ -425,12 +387,6 @@ public static void toInnerXContent(IndexTemplateMetaData indexTemplateMetaData, builder.endArray(); } - for (ObjectObjectCursor cursor : indexTemplateMetaData.customs()) { - builder.startObject(cursor.key); - cursor.value.toXContent(builder, params); - builder.endObject(); - } - builder.startObject("aliases"); for (ObjectCursor cursor : indexTemplateMetaData.aliases().values()) { AliasMetaData.Builder.toXContent(cursor.value, builder, params); @@ -468,15 +424,7 @@ public static IndexTemplateMetaData fromXContent(XContentParser parser, String t builder.putAlias(AliasMetaData.Builder.fromXContent(parser)); } } else { - // check if its a custom index metadata - IndexMetaData.Custom proto = IndexMetaData.lookupPrototype(currentFieldName); - if (proto == null) { - //TODO warn - parser.skipChildren(); - } else { - IndexMetaData.Custom custom = proto.fromXContent(parser); - builder.putCustom(custom.type(), custom); - } + throw new IllegalArgumentException("Unsupported template parameter [" + currentFieldName + "]"); } } else if (token == XContentParser.Token.START_ARRAY) { if ("mappings".equals(currentFieldName)) { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java index b19d65090c6b6..e25fb84743952 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java @@ -38,7 +38,6 @@ import org.elasticsearch.cluster.block.ClusterBlock; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.block.ClusterBlocks; -import org.elasticsearch.cluster.metadata.IndexMetaData.Custom; import org.elasticsearch.cluster.metadata.IndexMetaData.State; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.IndexRoutingTable; @@ -287,8 +286,6 @@ public ClusterState execute(ClusterState currentState) throws Exception { List templates = MetaDataIndexTemplateService.findTemplates(currentState.metaData(), request.index()); - Map customs = new HashMap<>(); - // add the request mapping Map> mappings = new HashMap<>(); @@ -300,10 +297,6 @@ public ClusterState execute(ClusterState currentState) throws Exception { mappings.put(entry.getKey(), MapperService.parseMapping(xContentRegistry, entry.getValue())); } - for (Map.Entry entry : request.customs().entrySet()) { - customs.put(entry.getKey(), entry.getValue()); - } - final Index recoverFromIndex = request.recoverFrom(); if (recoverFromIndex == null) { @@ -320,18 +313,6 @@ public ClusterState execute(ClusterState currentState) throws Exception { MapperService.parseMapping(xContentRegistry, mappingString)); } } - // handle custom - for (ObjectObjectCursor cursor : template.customs()) { - String type = cursor.key; - IndexMetaData.Custom custom = cursor.value; - IndexMetaData.Custom existing = customs.get(type); - if (existing == null) { - customs.put(type, custom); - } else { - IndexMetaData.Custom merged = existing.mergeWith(custom); - customs.put(type, merged); - } - } //handle aliases for (ObjectObjectCursor cursor : template.aliases()) { AliasMetaData aliasMetaData = cursor.value; @@ -519,10 +500,6 @@ public ClusterState execute(ClusterState currentState) throws Exception { indexMetaDataBuilder.putAlias(aliasMetaData); } - for (Map.Entry customEntry : customs.entrySet()) { - indexMetaDataBuilder.putCustom(customEntry.getKey(), customEntry.getValue()); - } - indexMetaDataBuilder.state(request.state()); final IndexMetaData indexMetaData; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java index 507eaf412d5fa..215aad9bc0457 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java @@ -179,9 +179,6 @@ public ClusterState execute(ClusterState currentState) throws Exception { .indexRouting(alias.indexRouting()).searchRouting(alias.searchRouting()).build(); templateBuilder.putAlias(aliasMetaData); } - for (Map.Entry entry : request.customs.entrySet()) { - templateBuilder.putCustom(entry.getKey(), entry.getValue()); - } IndexTemplateMetaData template = templateBuilder.build(); MetaData.Builder builder = MetaData.builder(currentState.metaData()).put(template); diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/create/CustomIndexMetadataIT.java b/server/src/test/java/org/elasticsearch/action/admin/indices/create/CustomIndexMetadataIT.java new file mode 100644 index 0000000000000..8731a2126c15f --- /dev/null +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/create/CustomIndexMetadataIT.java @@ -0,0 +1,237 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.indices.create; + +import org.elasticsearch.Version; +import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.AbstractNamedDiffable; +import org.elasticsearch.cluster.ClusterChangedEvent; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterStateListener; +import org.elasticsearch.cluster.ClusterStateUpdateTask; +import org.elasticsearch.cluster.NamedDiff; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.env.Environment; +import org.elasticsearch.env.NodeEnvironment; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.watcher.ResourceWatcherService; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; + +public class CustomIndexMetadataIT extends ESIntegTestCase { + + + public static final class TestIndexMetaData extends AbstractNamedDiffable implements IndexMetaData.Custom { + public static final String TYPE = "testindexmeta"; + + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "painless_execute_request", args -> new TestIndexMetaData((String) args[0])); + + static { + PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), new ParseField("payload")); + } + + + private final String payload; + + public TestIndexMetaData(String payload) { + this.payload = payload; + } + + public TestIndexMetaData(StreamInput in) throws IOException { + this.payload = in.readString(); + } + + @Override + public String getWriteableName() { + return TYPE; + } + + @Override + public Version getMinimalSupportedVersion() { + return Version.V_7_0_0_alpha1; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(payload); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.field("payload", payload); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + TestIndexMetaData that = (TestIndexMetaData) o; + return Objects.equals(payload, that.payload); + } + + @Override + public int hashCode() { + return Objects.hash(payload); + } + + public static TestIndexMetaData fromXContent(XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + + + public static NamedDiff readDiffFrom(StreamInput in) throws IOException { + return readDiffFrom(IndexMetaData.Custom.class, TYPE, in); + } + + public String getPayload() { + return payload; + } + + } + + public static final class CustomMetaDataAdder implements ClusterStateListener { + + private final ClusterService clusterService; + private final AtomicBoolean updateRunning = new AtomicBoolean(); + + public CustomMetaDataAdder(ClusterService clusterService) { + this.clusterService = clusterService; + clusterService.addListener(this); + } + + @Override + public void clusterChanged(ClusterChangedEvent event) { + if (event.localNodeMaster() && event.state().metaData().hasIndex("foobar")) { + TestIndexMetaData testIndexMetaData = event.state().metaData().index("foobar").custom(TestIndexMetaData.TYPE); + if (testIndexMetaData == null && updateRunning.getAndSet(true) == false) { + clusterService.submitStateUpdateTask("add-index-custom-metadata-test", new ClusterStateUpdateTask() { + + @Override + public ClusterState execute(ClusterState currentState) throws Exception { + if (currentState.metaData().hasIndex("foobar")) { + IndexMetaData indexMetaData = currentState.metaData().index("foobar"); + if (indexMetaData.custom(TestIndexMetaData.TYPE) == null) { + ClusterState.Builder builder = ClusterState.builder(currentState); + IndexMetaData.Builder indexMetaDataBuilder = IndexMetaData.builder(indexMetaData); + indexMetaDataBuilder.putCustom(TestIndexMetaData.TYPE, new TestIndexMetaData("payload")); + return builder.metaData(MetaData.builder(currentState.metaData()).put(indexMetaDataBuilder)).build(); + } + } + + return currentState; + } + + @Override + public void onFailure(String source, Exception e) { + updateRunning.set(false); + } + + @Override + public void onNoLongerMaster(String source) { + updateRunning.set(false); + } + + @Override + public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { + updateRunning.set(false); + } + }); + } + } + + } + } + + public static class TestCustomIndexMetadataPlugin extends Plugin { + @Override + public Collection createComponents(Client client, ClusterService clusterService, ThreadPool threadPool, + ResourceWatcherService resourceWatcherService, ScriptService scriptService, + NamedXContentRegistry xContentRegistry, Environment environment, + NodeEnvironment nodeEnvironment, NamedWriteableRegistry namedWriteableRegistry) { + return Collections.singletonList(new CustomMetaDataAdder(clusterService)); + } + + @Override + public List getNamedWriteables() { + return Arrays.asList( + new NamedWriteableRegistry.Entry(IndexMetaData.Custom.class, TestIndexMetaData.TYPE, TestIndexMetaData::new), + new NamedWriteableRegistry.Entry(NamedDiff.class, TestIndexMetaData.TYPE, TestIndexMetaData::readDiffFrom) + ); + } + + @Override + public List getNamedXContent() { + return Collections.singletonList(new NamedXContentRegistry.Entry( + IndexMetaData.Custom.class, new ParseField(TestIndexMetaData.TYPE), TestIndexMetaData::fromXContent)); + } + } + + @Override + protected Collection> nodePlugins() { + ArrayList> plugins = new ArrayList<>(super.nodePlugins()); + plugins.add(TestCustomIndexMetadataPlugin.class); + return plugins; + } + + @Override + protected Collection> transportClientPlugins() { + ArrayList> plugins = new ArrayList<>(super.nodePlugins()); + plugins.add(TestCustomIndexMetadataPlugin.class); + return plugins; + + } + + public void testCreateIndexWithCustomMetaData() throws Exception { + assertAcked(client().admin().indices().prepareCreate("foobar")); + assertBusy(() -> { + ClusterStateResponse response = client().admin().cluster().prepareState().setIndices("foobar").get(); + IndexMetaData indexMetaData = response.getState().getMetaData().index("foobar"); + assertNotNull(indexMetaData); + TestIndexMetaData testIndexMetaData = indexMetaData.custom(TestIndexMetaData.TYPE); + assertNotNull(testIndexMetaData); + assertEquals("payload", testIndexMetaData.getPayload()); + }); + } +} diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexCreationTaskTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexCreationTaskTests.java index 744a29e843c48..aa566458aca6a 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexCreationTaskTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexCreationTaskTests.java @@ -127,14 +127,12 @@ public void testApplyDataFromTemplate() throws Exception { addMatchingTemplate(builder -> builder .putAlias(AliasMetaData.builder("alias1")) .putMapping("mapping1", createMapping()) - .putCustom("custom1", createCustom()) .settings(Settings.builder().put("key1", "value1")) ); final ClusterState result = executeTask(); assertThat(result.metaData().index("test").getAliases(), hasKey("alias1")); - assertThat(result.metaData().index("test").getCustoms(), hasKey("custom1")); assertThat(result.metaData().index("test").getSettings().get("key1"), equalTo("value1")); assertThat(getMappingsFromResponse(), Matchers.hasKey("mapping1")); } @@ -142,22 +140,17 @@ public void testApplyDataFromTemplate() throws Exception { public void testApplyDataFromRequest() throws Exception { setupRequestAlias(new Alias("alias1")); setupRequestMapping("mapping1", createMapping()); - setupRequestCustom("custom1", createCustom()); reqSettings.put("key1", "value1"); final ClusterState result = executeTask(); assertThat(result.metaData().index("test").getAliases(), hasKey("alias1")); - assertThat(result.metaData().index("test").getCustoms(), hasKey("custom1")); + assertThat(result.metaData().index("test").getCustoms().size(), equalTo(0)); assertThat(result.metaData().index("test").getSettings().get("key1"), equalTo("value1")); assertThat(getMappingsFromResponse(), Matchers.hasKey("mapping1")); } public void testRequestDataHavePriorityOverTemplateData() throws Exception { - final IndexMetaData.Custom tplCustom = createCustom(); - final IndexMetaData.Custom reqCustom = createCustom(); - final IndexMetaData.Custom mergedCustom = createCustom(); - when(reqCustom.mergeWith(tplCustom)).thenReturn(mergedCustom); final CompressedXContent tplMapping = createMapping("text"); final CompressedXContent reqMapping = createMapping("keyword"); @@ -165,18 +158,15 @@ public void testRequestDataHavePriorityOverTemplateData() throws Exception { addMatchingTemplate(builder -> builder .putAlias(AliasMetaData.builder("alias1").searchRouting("fromTpl").build()) .putMapping("mapping1", tplMapping) - .putCustom("custom1", tplCustom) .settings(Settings.builder().put("key1", "tplValue")) ); setupRequestAlias(new Alias("alias1").searchRouting("fromReq")); setupRequestMapping("mapping1", reqMapping); - setupRequestCustom("custom1", reqCustom); reqSettings.put("key1", "reqValue"); final ClusterState result = executeTask(); - assertThat(result.metaData().index("test").getCustoms().get("custom1"), equalTo(mergedCustom)); assertThat(result.metaData().index("test").getAliases().get("alias1").getSearchRouting(), equalTo("fromReq")); assertThat(result.metaData().index("test").getSettings().get("key1"), equalTo("reqValue")); assertThat(getMappingsFromResponse().get("mapping1").toString(), equalTo("{type={properties={field={type=keyword}}}}")); @@ -272,7 +262,6 @@ public void testShrinkIndexIgnoresTemplates() throws Exception { addMatchingTemplate(builder -> builder .putAlias(AliasMetaData.builder("alias1").searchRouting("fromTpl").build()) .putMapping("mapping1", createMapping()) - .putCustom("custom1", createCustom()) .settings(Settings.builder().put("key1", "tplValue")) ); @@ -296,7 +285,6 @@ public void testWriteIndex() throws Exception { Boolean writeIndex = randomBoolean() ? null : randomBoolean(); setupRequestAlias(new Alias("alias1").writeIndex(writeIndex)); setupRequestMapping("mapping1", createMapping()); - setupRequestCustom("custom1", createCustom()); reqSettings.put("key1", "value1"); final ClusterState result = executeTask(); @@ -310,7 +298,6 @@ public void testWriteIndexValidationException() throws Exception { .numberOfShards(1).numberOfReplicas(0).build(); idxBuilder.put("test2", existingWriteIndex); setupRequestMapping("mapping1", createMapping()); - setupRequestCustom("custom1", createCustom()); reqSettings.put("key1", "value1"); setupRequestAlias(new Alias("alias1").writeIndex(true)); @@ -342,10 +329,6 @@ private IndexMetaData.Builder createIndexMetaDataBuilder(String name, String uui .numberOfReplicas(numReplicas); } - private IndexMetaData.Custom createCustom() { - return mock(IndexMetaData.Custom.class); - } - private interface MetaDataBuilderConfigurator { void configure(IndexTemplateMetaData.Builder builder) throws IOException; } @@ -372,10 +355,6 @@ private void setupRequestMapping(String mappingKey, CompressedXContent mapping) when(request.mappings()).thenReturn(Collections.singletonMap(mappingKey, mapping.string())); } - private void setupRequestCustom(String customKey, IndexMetaData.Custom custom) throws IOException { - when(request.customs()).thenReturn(Collections.singletonMap(customKey, custom)); - } - private CompressedXContent createMapping() throws IOException { return createMapping("text"); } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaDataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaDataTests.java index 6d489f5feb314..dbebf7dbe71fe 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaDataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaDataTests.java @@ -128,13 +128,13 @@ public void testIndexTemplateMetaDataXContentRoundTrip() throws Exception { public void testValidateInvalidIndexPatterns() throws Exception { final IllegalArgumentException emptyPatternError = expectThrows(IllegalArgumentException.class, () -> { new IndexTemplateMetaData(randomRealisticUnicodeOfLengthBetween(5, 10), randomInt(), randomInt(), - Collections.emptyList(), Settings.EMPTY, ImmutableOpenMap.of(), ImmutableOpenMap.of(), ImmutableOpenMap.of()); + Collections.emptyList(), Settings.EMPTY, ImmutableOpenMap.of(), ImmutableOpenMap.of()); }); assertThat(emptyPatternError.getMessage(), equalTo("Index patterns must not be null or empty; got []")); final IllegalArgumentException nullPatternError = expectThrows(IllegalArgumentException.class, () -> { new IndexTemplateMetaData(randomRealisticUnicodeOfLengthBetween(5, 10), randomInt(), randomInt(), - null, Settings.EMPTY, ImmutableOpenMap.of(), ImmutableOpenMap.of(), ImmutableOpenMap.of()); + null, Settings.EMPTY, ImmutableOpenMap.of(), ImmutableOpenMap.of()); }); assertThat(nullPatternError.getMessage(), equalTo("Index patterns must not be null or empty; got null")); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java index a2c70db3b63e8..4000969187548 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java @@ -259,7 +259,7 @@ private ClusterState getClusterStateWithSecurityIndex() { .put(IndexMetaData.builder(securityIndexName).settings(settings)) .put(new IndexTemplateMetaData(SecurityIndexManager.SECURITY_TEMPLATE_NAME, 0, 0, Collections.singletonList(securityIndexName), Settings.EMPTY, ImmutableOpenMap.of(), - ImmutableOpenMap.of(), ImmutableOpenMap.of())) + ImmutableOpenMap.of())) .build(); if (withAlias) {