diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java index 17442da04138b..21b466e6919b2 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java @@ -358,11 +358,12 @@ private static EsField createField(String fieldName, Map= 0) { String parentName = fieldName.substring(0, dot); fieldName = fieldName.substring(dot + 1); - EsField parent = flattedMapping.get(parentName); + parent = flattedMapping.get(parentName); if (parent == null) { Map map = globalCaps.get(parentName); Function fieldFunction; @@ -387,7 +388,22 @@ private static EsField createField(String fieldName, Map properties = Collections.emptyMap(); } } else { - properties = Collections.emptyMap(); + properties = fromEs(content); } boolean docValues = boolSetting(content.get("doc_values"), esDataType.defaultDocValues); final EsField field; @@ -91,7 +91,8 @@ private static void walkMapping(String name, Object value, Map break; case UNSUPPORTED: String type = content.get("type").toString(); - field = new UnsupportedEsField(name, type); + field = new UnsupportedEsField(name, type, null, properties); + propagateUnsupportedType(name, type, properties); break; default: field = new EsField(name, esDataType, properties, docValues); @@ -113,4 +114,21 @@ private static boolean boolSetting(Object value, boolean defaultValue) { private static int intSetting(Object value, int defaultValue) { return value == null ? defaultValue : Integer.parseInt(value.toString()); } + + private static void propagateUnsupportedType(String inherited, String originalType, Map properties) { + if (properties != null && properties.isEmpty() == false) { + for (Entry entry : properties.entrySet()) { + EsField field = entry.getValue(); + UnsupportedEsField u; + if (field instanceof UnsupportedEsField) { + u = (UnsupportedEsField) field; + u = new UnsupportedEsField(u.getName(), originalType, inherited, u.getProperties()); + } else { + u = new UnsupportedEsField(field.getName(), originalType, inherited, field.getProperties()); + } + entry.setValue(u); + propagateUnsupportedType(inherited, originalType, u.getProperties()); + } + } + } } diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/UnsupportedEsField.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/UnsupportedEsField.java index b41f9458864c2..4987ba97f2ba1 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/UnsupportedEsField.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/UnsupportedEsField.java @@ -5,24 +5,40 @@ */ package org.elasticsearch.xpack.ql.type; -import java.util.Collections; +import java.util.Map; import java.util.Objects; +import java.util.TreeMap; /** - * SQL-related information about an index field that cannot be supported by SQL + * SQL-related information about an index field that cannot be supported by SQL. + * All the subfields (properties) of an unsupported type should also be unsupported. */ public class UnsupportedEsField extends EsField { - private String originalType; - + private final String originalType; + private final String inherited; // for fields belonging to parents (or grandparents) that have an unsupported type + public UnsupportedEsField(String name, String originalType) { - super(name, DataType.UNSUPPORTED, Collections.emptyMap(), false); + this(name, originalType, null, new TreeMap<>()); + } + + public UnsupportedEsField(String name, String originalType, String inherited, Map properties) { + super(name, DataType.UNSUPPORTED, properties, false); this.originalType = originalType; + this.inherited = inherited; } public String getOriginalType() { return originalType; } + + public String getInherited() { + return inherited; + } + + public boolean hasInherited() { + return inherited != null; + } @Override public boolean equals(Object o) { @@ -36,11 +52,12 @@ public boolean equals(Object o) { return false; } UnsupportedEsField that = (UnsupportedEsField) o; - return Objects.equals(originalType, that.originalType); + return Objects.equals(originalType, that.originalType) + && Objects.equals(inherited, that.inherited); } @Override public int hashCode() { - return Objects.hash(super.hashCode(), originalType); + return Objects.hash(super.hashCode(), originalType, inherited); } } diff --git a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/TypesTests.java b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/TypesTests.java index 9fc3fdb0b7594..8be30f389973d 100644 --- a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/TypesTests.java +++ b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/TypesTests.java @@ -188,6 +188,10 @@ public void testUnsupportedTypes() { Map mapping = loadMapping("mapping-unsupported.json"); EsField dt = mapping.get("range"); assertThat(dt.getDataType().typeName(), is("unsupported")); + dt = mapping.get("time_frame"); + assertThat(dt.getDataType().typeName(), is("unsupported")); + dt = mapping.get("flat"); + assertThat(dt.getDataType().typeName(), is("unsupported")); } public static Map loadMapping(String name) { diff --git a/x-pack/plugin/ql/src/test/resources/mapping-multi-field-with-nested.json b/x-pack/plugin/ql/src/test/resources/mapping-multi-field-with-nested.json index e46d64a45e88f..d66dd7efd5fca 100644 --- a/x-pack/plugin/ql/src/test/resources/mapping-multi-field-with-nested.json +++ b/x-pack/plugin/ql/src/test/resources/mapping-multi-field-with-nested.json @@ -7,6 +7,26 @@ "unsupported" : { "type" : "ip_range" }, "date" : { "type" : "date"}, "shape": { "type" : "geo_shape" }, + "x" : { + "type" : "text", + "fields" : { + "y" : { + "type" : "foobar", + "fields" : { + "z" : { + "properties" : { + "v" : { + "type" : "keyword" + }, + "w" : { + "type" : "foo" + } + } + } + } + } + } + }, "some" : { "properties" : { "dotted" : { diff --git a/x-pack/plugin/ql/src/test/resources/mapping-unsupported.json b/x-pack/plugin/ql/src/test/resources/mapping-unsupported.json index 832dc9c0d745d..d3afa7fa52405 100644 --- a/x-pack/plugin/ql/src/test/resources/mapping-unsupported.json +++ b/x-pack/plugin/ql/src/test/resources/mapping-unsupported.json @@ -6,6 +6,9 @@ "time_frame" : { "type" : "date_range", "format" : "yyyy-MM-dd" + }, + "flat" : { + "type" : "flattened" } } } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java index 5119378d6462b..c47f31bcb901f 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java @@ -217,8 +217,14 @@ private static Attribute handleSpecialFields(UnresolvedAttribute u, Attribute na // unsupported types else if (DataTypes.isUnsupported(fa.dataType())) { UnsupportedEsField unsupportedField = (UnsupportedEsField) fa.field(); - named = u.withUnresolvedMessage( - "Cannot use field [" + fa.name() + "] type [" + unsupportedField.getOriginalType() + "] as is unsupported"); + if (unsupportedField.hasInherited()) { + named = u.withUnresolvedMessage( + "Cannot use field [" + fa.name() + "] with unsupported type [" + unsupportedField.getOriginalType() + "] " + + "in hierarchy (field [" + unsupportedField.getInherited() + "])"); + } else { + named = u.withUnresolvedMessage( + "Cannot use field [" + fa.name() + "] with unsupported type [" + unsupportedField.getOriginalType() + "]"); + } } // compound fields else if (allowCompound == false && fa.dataType().isPrimitive() == false) { diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java index b6bedfe02b86e..7cfcc2bdc7642 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java @@ -482,19 +482,33 @@ public void testGroupByScalarFunctionWithAggOnTarget() { } public void testUnsupportedType() { - assertEquals("1:8: Cannot use field [unsupported] type [ip_range] as is unsupported", + assertEquals("1:8: Cannot use field [unsupported] with unsupported type [ip_range]", error("SELECT unsupported FROM test")); } public void testUnsupportedStarExpansion() { - assertEquals("1:8: Cannot use field [unsupported] type [ip_range] as is unsupported", + assertEquals("1:8: Cannot use field [unsupported] with unsupported type [ip_range]", error("SELECT unsupported.* FROM test")); } public void testUnsupportedTypeInFilter() { - assertEquals("1:26: Cannot use field [unsupported] type [ip_range] as is unsupported", + assertEquals("1:26: Cannot use field [unsupported] with unsupported type [ip_range]", error("SELECT * FROM test WHERE unsupported > 1")); } + + public void testValidRootFieldWithUnsupportedChildren() { + accept("SELECT x FROM test"); + } + + public void testUnsupportedTypeInHierarchy() { + assertEquals("1:8: Cannot use field [x.y.z.w] with unsupported type [foobar] in hierarchy (field [y])", + error("SELECT x.y.z.w FROM test")); + assertEquals("1:8: Cannot use field [x.y.z.v] with unsupported type [foobar] in hierarchy (field [y])", + error("SELECT x.y.z.v FROM test")); + assertEquals("1:8: Cannot use field [x.y.z] with unsupported type [foobar] in hierarchy (field [y])", + error("SELECT x.y.z.* FROM test")); + assertEquals("1:8: Cannot use field [x.y] with unsupported type [foobar]", error("SELECT x.y FROM test")); + } public void testTermEqualitOnInexact() { assertEquals("1:26: [text = 'value'] cannot operate on first argument field of data type [text]: " + @@ -509,12 +523,12 @@ public void testTermEqualityOnAmbiguous() { } public void testUnsupportedTypeInFunction() { - assertEquals("1:12: Cannot use field [unsupported] type [ip_range] as is unsupported", + assertEquals("1:12: Cannot use field [unsupported] with unsupported type [ip_range]", error("SELECT ABS(unsupported) FROM test")); } public void testUnsupportedTypeInOrder() { - assertEquals("1:29: Cannot use field [unsupported] type [ip_range] as is unsupported", + assertEquals("1:29: Cannot use field [unsupported] with unsupported type [ip_range]", error("SELECT * FROM test ORDER BY unsupported")); } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java index f58925c16d828..181d0791f93b8 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java @@ -128,6 +128,94 @@ public void testMetaFieldsAreIgnored() throws Exception { assertEquals(DataType.INTEGER, esIndex.mapping().get("_meta_field").getDataType()); assertEquals(DataType.KEYWORD, esIndex.mapping().get("text").getDataType()); } + + public void testFlattenedHiddenSubfield() throws Exception { + Map> fieldCaps = new HashMap<>(); + addFieldCaps(fieldCaps, "some_field", "flattened", false, false); + addFieldCaps(fieldCaps, "some_field._keyed", "flattened", false, false); + addFieldCaps(fieldCaps, "another_field", "object", true, false); + addFieldCaps(fieldCaps, "another_field._keyed", "keyword", true, false); + addFieldCaps(fieldCaps, "nested_field", "object", false, false); + addFieldCaps(fieldCaps, "nested_field.sub_field", "flattened", true, true); + addFieldCaps(fieldCaps, "nested_field.sub_field._keyed", "flattened", true, true); + addFieldCaps(fieldCaps, "text", "keyword", true, true); + + String wildcard = "*"; + IndexResolution resolution = IndexResolver.mergedMappings(wildcard, new String[] { "index" }, fieldCaps); + assertTrue(resolution.isValid()); + + EsIndex esIndex = resolution.get(); + assertEquals(wildcard, esIndex.name()); + assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("some_field").getDataType()); + assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("some_field").getProperties().get("_keyed").getDataType()); + assertEquals(DataType.OBJECT, esIndex.mapping().get("nested_field").getDataType()); + assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("nested_field").getProperties().get("sub_field").getDataType()); + assertEquals(DataType.UNSUPPORTED, + esIndex.mapping().get("nested_field").getProperties().get("sub_field").getProperties().get("_keyed").getDataType()); + assertEquals(DataType.KEYWORD, esIndex.mapping().get("text").getDataType()); + assertEquals(DataType.OBJECT, esIndex.mapping().get("another_field").getDataType()); + assertEquals(DataType.KEYWORD, esIndex.mapping().get("another_field").getProperties().get("_keyed").getDataType()); + } + + public void testPropagateUnsupportedTypeToSubFields() throws Exception { + // generate a field type having the name of the format "foobar43" + String esFieldType = randomAlphaOfLengthBetween(5, 10) + randomIntBetween(-100, 100); + Map> fieldCaps = new HashMap<>(); + addFieldCaps(fieldCaps, "a", "text", false, false); + addFieldCaps(fieldCaps, "a.b", esFieldType, false, false); + addFieldCaps(fieldCaps, "a.b.c", "object", true, false); + addFieldCaps(fieldCaps, "a.b.c.d", "keyword", true, false); + addFieldCaps(fieldCaps, "a.b.c.e", "foo", true, true); + + String wildcard = "*"; + IndexResolution resolution = IndexResolver.mergedMappings(wildcard, new String[] { "index" }, fieldCaps); + assertTrue(resolution.isValid()); + + EsIndex esIndex = resolution.get(); + assertEquals(wildcard, esIndex.name()); + assertEquals(DataType.TEXT, esIndex.mapping().get("a").getDataType()); + assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("a").getProperties().get("b").getDataType()); + assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("a").getProperties().get("b").getProperties().get("c").getDataType()); + assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("a").getProperties().get("b").getProperties().get("c") + .getProperties().get("d").getDataType()); + assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("a").getProperties().get("b").getProperties().get("c") + .getProperties().get("e").getDataType()); + } + + public void testRandomMappingFieldTypeMappedAsUnsupported() throws Exception { + // generate a field type having the name of the format "foobar43" + String esFieldType = randomAlphaOfLengthBetween(5, 10) + randomIntBetween(-100, 100); + + Map> fieldCaps = new HashMap<>(); + addFieldCaps(fieldCaps, "some_field", esFieldType, false, false); + addFieldCaps(fieldCaps, "another_field", "object", true, false); + addFieldCaps(fieldCaps, "another_field._foo", esFieldType, true, false); + addFieldCaps(fieldCaps, "nested_field", "object", false, false); + addFieldCaps(fieldCaps, "nested_field.sub_field1", esFieldType, true, true); + addFieldCaps(fieldCaps, "nested_field.sub_field1.bar", esFieldType, true, true); + addFieldCaps(fieldCaps, "nested_field.sub_field2", esFieldType, true, true); + // even if this is of a supported type, because it belongs to an UNSUPPORTED type parent, it should also be UNSUPPORTED + addFieldCaps(fieldCaps, "nested_field.sub_field2.bar", "keyword", true, true); + addFieldCaps(fieldCaps, "text", "keyword", true, true); + + String wildcard = "*"; + IndexResolution resolution = IndexResolver.mergedMappings(wildcard, new String[] { "index" }, fieldCaps); + assertTrue(resolution.isValid()); + + EsIndex esIndex = resolution.get(); + assertEquals(wildcard, esIndex.name()); + assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("some_field").getDataType()); + assertEquals(DataType.OBJECT, esIndex.mapping().get("nested_field").getDataType()); + assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("nested_field").getProperties().get("sub_field1").getDataType()); + assertEquals(DataType.UNSUPPORTED, + esIndex.mapping().get("nested_field").getProperties().get("sub_field1").getProperties().get("bar").getDataType()); + assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("nested_field").getProperties().get("sub_field2").getDataType()); + assertEquals(DataType.UNSUPPORTED, + esIndex.mapping().get("nested_field").getProperties().get("sub_field2").getProperties().get("bar").getDataType()); + assertEquals(DataType.KEYWORD, esIndex.mapping().get("text").getDataType()); + assertEquals(DataType.OBJECT, esIndex.mapping().get("another_field").getDataType()); + assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("another_field").getProperties().get("_foo").getDataType()); + } public void testMergeIncompatibleCapabilitiesOfObjectFields() throws Exception { Map> fieldCaps = new HashMap<>(); @@ -330,7 +418,7 @@ private static void assertEqualsMaps(Map left, Map right) { private void addFieldCaps(Map> fieldCaps, String name, String type, boolean isSearchable, boolean isAggregatable) { Map cap = new HashMap<>(); - cap.put(name, new FieldCapabilities(name, type, isSearchable, isAggregatable, Collections.emptyMap())); + cap.put(type, new FieldCapabilities(name, type, isSearchable, isAggregatable, Collections.emptyMap())); fieldCaps.put(name, cap); } } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java index cfadd61828e23..5bdeff1f20e42 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysColumnsTests.java @@ -463,7 +463,7 @@ public void testSysColumnsNoArg() throws Exception { public void testSysColumnsWithCatalogWildcard() throws Exception { executeCommand("SYS COLUMNS CATALOG 'cluster' TABLE LIKE 'test' LIKE '%'", emptyList(), r -> { - assertEquals(14, r.size()); + assertEquals(15, r.size()); assertEquals(CLUSTER_NAME, r.column(0)); assertEquals("test", r.column(2)); assertEquals("bool", r.column(3)); @@ -476,7 +476,7 @@ public void testSysColumnsWithCatalogWildcard() throws Exception { public void testSysColumnsWithMissingCatalog() throws Exception { executeCommand("SYS COLUMNS TABLE LIKE 'test' LIKE '%'", emptyList(), r -> { - assertEquals(14, r.size()); + assertEquals(15, r.size()); assertEquals(CLUSTER_NAME, r.column(0)); assertEquals("test", r.column(2)); assertEquals("bool", r.column(3)); @@ -489,7 +489,7 @@ public void testSysColumnsWithMissingCatalog() throws Exception { public void testSysColumnsWithNullCatalog() throws Exception { executeCommand("SYS COLUMNS CATALOG ? TABLE LIKE 'test' LIKE '%'", singletonList(new SqlTypedParamValue("keyword", null)), r -> { - assertEquals(14, r.size()); + assertEquals(15, r.size()); assertEquals(CLUSTER_NAME, r.column(0)); assertEquals("test", r.column(2)); assertEquals("bool", r.column(3));