Skip to content

Commit

Permalink
Restore a noop _all metadata field for 6x indices (#37808)
Browse files Browse the repository at this point in the history
This commit restores a noop version of the AllFieldMapper that is instanciated only
for indices created in 6x. We need this metadata field mapper to be present in this version
in order to allow the upgrade of indices that explicitly disable _all (enabled: false).
The mapping of these indices contains a reference to the _all field that we cannot remove
in 7 so we'll need to keep this metadata mapper in 7x. Since indices created in 6x will not
be compatible with 8, we'll remove this noop mapper in the next major version.

Closes #37429
  • Loading branch information
jimczi authored Jan 30, 2019
1 parent f51bc00 commit 5dcc805
Show file tree
Hide file tree
Showing 14 changed files with 354 additions and 28 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -67,3 +67,9 @@
field3: value
- match: { hits.total: 1 }
- match: { hits.hits.0._id: q3 }

---
"Index with _all is available":
- do:
indices.get:
index: all-index
Original file line number Diff line number Diff line change
Expand Up @@ -203,3 +203,21 @@
tasks.get:
wait_for_completion: true
task_id: $task

---
"Create an index with _all explicitly disabled":
- skip:
features: warnings
- do:
warnings:
- "[_all] is deprecated in 6.0+ and will be removed in 7.0. As a replacement, you can use [copy_to] on mapping fields to create your own catch all field."
indices.create:
index: all-index
body:
mappings:
type:
_all:
enabled: false
properties:
field:
type: text
Original file line number Diff line number Diff line change
Expand Up @@ -125,3 +125,17 @@
wait_for_completion: true
task_id: $task_id
- match: { task.headers.X-Opaque-Id: "Reindexing Again" }

---
"Index with _all is available":
- do:
indices.get:
index: all-index

- do:
indices.get_mapping:
index: all-index

- is_true: all-index.mappings._all
- match: { all-index.mappings._all.enabled: false}

Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
package org.elasticsearch.action.admin.indices.mapping.get;

import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.Version;
import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetaData;
import org.elasticsearch.action.support.ActionFilters;
import org.elasticsearch.action.support.single.shard.TransportSingleShardAction;
Expand Down Expand Up @@ -91,7 +92,8 @@ protected ShardsIterator shards(ClusterState state, InternalRequest request) {
protected GetFieldMappingsResponse shardOperation(final GetFieldMappingsIndexRequest request, ShardId shardId) {
assert shardId != null;
IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex());
Predicate<String> metadataFieldPredicate = indicesService::isMetaDataField;
Version indexCreatedVersion = indexService.mapperService().getIndexSettings().getIndexVersionCreated();
Predicate<String> metadataFieldPredicate = (f) -> indicesService.isMetaDataField(indexCreatedVersion, f);
Predicate<String> fieldPredicate = metadataFieldPredicate.or(indicesService.getFieldFilter().apply(shardId.getIndexName()));

DocumentMapper mapper = indexService.mapperService().documentMapper();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,8 @@ protected FieldCapabilitiesIndexResponse shardOperation(final FieldCapabilitiesI
for (String field : fieldNames) {
MappedFieldType ft = mapperService.fullName(field);
if (ft != null) {
if (indicesService.isMetaDataField(field) || fieldPredicate.test(ft.name())) {
if (indicesService.isMetaDataField(mapperService.getIndexSettings().getIndexVersionCreated(), field)
|| fieldPredicate.test(ft.name())) {
FieldCapabilities fieldCap = new FieldCapabilities(field, ft.typeName(), ft.isSearchable(), ft.isAggregatable());
responseMap.put(field, fieldCap);
} else {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,175 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.elasticsearch.index.mapper;

import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.search.MatchNoDocsQuery;
import org.apache.lucene.search.Query;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.support.XContentMapValues;
import org.elasticsearch.index.query.QueryShardContext;

import java.io.IOException;
import java.util.Iterator;
import java.util.List;
import java.util.Map;

/**
* Noop mapper that ensures that mappings created in 6x that explicitly disable the _all field
* can be restored in this version.
*
* TODO: Remove in 8
*/
public class AllFieldMapper extends MetadataFieldMapper {
public static final String NAME = "_all";
public static final String CONTENT_TYPE = "_all";

public static class Defaults {
public static final MappedFieldType FIELD_TYPE = new AllFieldType();

static {
FIELD_TYPE.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
FIELD_TYPE.setTokenized(true);
FIELD_TYPE.setName(NAME);
FIELD_TYPE.freeze();
}
}

public static class Builder extends MetadataFieldMapper.Builder<Builder, AllFieldMapper> {
private boolean disableExplicit = false;

public Builder(MappedFieldType existing) {
super(NAME, existing == null ? Defaults.FIELD_TYPE : existing, Defaults.FIELD_TYPE);
builder = this;
}

private Builder setDisableExplicit() {
this.disableExplicit = true;
return this;
}

@Override
public AllFieldMapper build(BuilderContext context) {
return new AllFieldMapper(fieldType, context.indexSettings(), disableExplicit);
}
}

public static class TypeParser implements MetadataFieldMapper.TypeParser {
@Override
public MetadataFieldMapper.Builder<?,?> parse(String name, Map<String, Object> node,
ParserContext parserContext) throws MapperParsingException {
Builder builder = new Builder(parserContext.mapperService().fullName(NAME));
for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext();) {
Map.Entry<String, Object> entry = iterator.next();
String fieldName = entry.getKey();
if (fieldName.equals("enabled")) {
boolean enabled = XContentMapValues.nodeBooleanValue(entry.getValue(), "enabled");
if (enabled) {
throw new IllegalArgumentException("[_all] is disabled in this version.");
}
builder.setDisableExplicit();
iterator.remove();
}
}
return builder;
}

@Override
public MetadataFieldMapper getDefault(MappedFieldType fieldType, ParserContext context) {
final Settings indexSettings = context.mapperService().getIndexSettings().getSettings();
return new AllFieldMapper(indexSettings, Defaults.FIELD_TYPE, false);
}
}

static final class AllFieldType extends StringFieldType {
AllFieldType() {
}

protected AllFieldType(AllFieldType ref) {
super(ref);
}

@Override
public MappedFieldType clone() {
return new AllFieldType(this);
}

@Override
public String typeName() {
return CONTENT_TYPE;
}

@Override
public Query existsQuery(QueryShardContext context) {
return new MatchNoDocsQuery();
}
}

private final boolean disableExplicit;

private AllFieldMapper(Settings indexSettings, MappedFieldType existing, boolean disableExplicit) {
this(existing.clone(), indexSettings, disableExplicit);
}

private AllFieldMapper(MappedFieldType fieldType, Settings indexSettings, boolean disableExplicit) {
super(NAME, fieldType, Defaults.FIELD_TYPE, indexSettings);
this.disableExplicit = disableExplicit;
}

@Override
public void preParse(ParseContext context) throws IOException {
}

@Override
public void postParse(ParseContext context) throws IOException {
super.parse(context);
}

@Override
public void parse(ParseContext context) throws IOException {
// we parse in post parse
}

@Override
protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
// noop mapper
return;
}

@Override
protected String contentType() {
return CONTENT_TYPE;
}

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
boolean includeDefaults = params.paramAsBoolean("include_defaults", false);
if (includeDefaults || disableExplicit) {
builder.startObject(CONTENT_TYPE);
if (disableExplicit) {
builder.field("enabled", false);
}
builder.endObject();
}
return builder;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import org.apache.lucene.search.Weight;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.ElasticsearchGenerationException;
import org.elasticsearch.Version;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.compress.CompressedXContent;
import org.elasticsearch.common.settings.Settings;
Expand Down Expand Up @@ -73,7 +74,9 @@ public Builder(RootObjectMapper.Builder builder, MapperService mapperService) {

final String type = rootObjectMapper.name();
final DocumentMapper existingMapper = mapperService.documentMapper(type);
final Map<String, TypeParser> metadataMapperParsers = mapperService.mapperRegistry.getMetadataMapperParsers();
final Version indexCreatedVersion = mapperService.getIndexSettings().getIndexVersionCreated();
final Map<String, TypeParser> metadataMapperParsers =
mapperService.mapperRegistry.getMetadataMapperParsers(indexCreatedVersion);
for (Map.Entry<String, MetadataFieldMapper.TypeParser> entry : metadataMapperParsers.entrySet()) {
final String name = entry.getKey();
final MetadataFieldMapper existingMetadataMapper = existingMapper == null
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -65,8 +65,8 @@ public DocumentMapperParser(IndexSettings indexSettings, MapperService mapperSer
this.similarityService = similarityService;
this.queryShardContextSupplier = queryShardContextSupplier;
this.typeParsers = mapperRegistry.getMapperParsers();
this.rootTypeParsers = mapperRegistry.getMetadataMapperParsers();
indexVersionCreated = indexSettings.getIndexVersionCreated();
this.indexVersionCreated = indexSettings.getIndexVersionCreated();
this.rootTypeParsers = mapperRegistry.getMetadataMapperParsers(indexVersionCreated);
}

public Mapper.TypeParser.ParserContext parserContext(String type) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import org.apache.lucene.util.RamUsageEstimator;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.ResourceAlreadyExistsException;
import org.elasticsearch.Version;
import org.elasticsearch.action.admin.indices.stats.CommonStats;
import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags;
import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags.Flag;
Expand Down Expand Up @@ -1382,8 +1383,8 @@ public Function<String, Predicate<String>> getFieldFilter() {
/**
* Returns true if the provided field is a registered metadata field (including ones registered via plugins), false otherwise.
*/
public boolean isMetaDataField(String field) {
return mapperRegistry.isMetaDataField(field);
public boolean isMetaDataField(Version indexCreatedVersion, String field) {
return mapperRegistry.isMetaDataField(indexCreatedVersion, field);
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@

package org.elasticsearch.indices.mapper;

import org.elasticsearch.Version;
import org.elasticsearch.index.mapper.AllFieldMapper;
import org.elasticsearch.index.mapper.Mapper;
import org.elasticsearch.index.mapper.MetadataFieldMapper;
import org.elasticsearch.plugins.MapperPlugin;
Expand All @@ -36,13 +38,19 @@ public final class MapperRegistry {

private final Map<String, Mapper.TypeParser> mapperParsers;
private final Map<String, MetadataFieldMapper.TypeParser> metadataMapperParsers;
private final Map<String, MetadataFieldMapper.TypeParser> metadataMapperParsers6x;
private final Function<String, Predicate<String>> fieldFilter;


public MapperRegistry(Map<String, Mapper.TypeParser> mapperParsers,
Map<String, MetadataFieldMapper.TypeParser> metadataMapperParsers, Function<String, Predicate<String>> fieldFilter) {
this.mapperParsers = Collections.unmodifiableMap(new LinkedHashMap<>(mapperParsers));
this.metadataMapperParsers = Collections.unmodifiableMap(new LinkedHashMap<>(metadataMapperParsers));
// add the _all field mapper for indices created in 6x
Map<String, MetadataFieldMapper.TypeParser> metadata6x = new LinkedHashMap<>();
metadata6x.put(AllFieldMapper.NAME, new AllFieldMapper.TypeParser());
metadata6x.putAll(metadataMapperParsers);
this.metadataMapperParsers6x = Collections.unmodifiableMap(metadata6x);
this.fieldFilter = fieldFilter;
}

Expand All @@ -58,15 +66,15 @@ public Map<String, Mapper.TypeParser> getMapperParsers() {
* Return a map of the meta mappers that have been registered. The
* returned map uses the name of the field as a key.
*/
public Map<String, MetadataFieldMapper.TypeParser> getMetadataMapperParsers() {
return metadataMapperParsers;
public Map<String, MetadataFieldMapper.TypeParser> getMetadataMapperParsers(Version indexCreatedVersion) {
return indexCreatedVersion.onOrAfter(Version.V_7_0_0) ? metadataMapperParsers : metadataMapperParsers6x;
}

/**
* Returns true if the provide field is a registered metadata field, false otherwise
* Returns true if the provided field is a registered metadata field, false otherwise
*/
public boolean isMetaDataField(String field) {
return getMetadataMapperParsers().containsKey(field);
public boolean isMetaDataField(Version indexCreatedVersion, String field) {
return getMetadataMapperParsers(indexCreatedVersion).containsKey(field);
}

/**
Expand Down
Loading

0 comments on commit 5dcc805

Please sign in to comment.