nodes = new ArrayList<>();
- while (token != XContentParser.Token.END_ARRAY) {
- nodes.add(parseCoordinates(parser));
- token = parser.nextToken();
+ protected static Coordinate readFromStream(StreamInput in) throws IOException {
+ return new Coordinate(in.readDouble(), in.readDouble());
+ }
+
+ @Override
+ public void writeTo(StreamOutput out) throws IOException {
+ out.writeVInt(coordinates.size());
+ for (Coordinate point : coordinates) {
+ writeCoordinateTo(point, out);
}
+ }
+
+ protected static void writeCoordinateTo(Coordinate coordinate, StreamOutput out) throws IOException {
+ out.writeDouble(coordinate.x);
+ out.writeDouble(coordinate.y);
+ }
- return new CoordinateNode(nodes);
+ @SuppressWarnings("unchecked")
+ private E thisRef() {
+ return (E)this;
}
/**
- * Create a new {@link ShapeBuilder} from {@link XContent}
- * @param parser parser to read the GeoShape from
- * @return {@link ShapeBuilder} read from the parser or null
- * if the parsers current token has been null
- * @throws IOException if the input could not be read
+ * Add a new coordinate to the collection
+ * @param longitude longitude of the coordinate
+ * @param latitude latitude of the coordinate
+ * @return this
*/
- public static ShapeBuilder parse(XContentParser parser) throws IOException {
- return GeoShapeType.parse(parser, null);
+ public E coordinate(double longitude, double latitude) {
+ return this.coordinate(new Coordinate(longitude, latitude));
}
/**
- * Create a new {@link ShapeBuilder} from {@link XContent}
- * @param parser parser to read the GeoShape from
- * @param geoDocMapper document field mapper reference required for spatial parameters relevant
- * to the shape construction process (e.g., orientation)
- * todo: refactor to place build specific parameters in the SpatialContext
- * @return {@link ShapeBuilder} read from the parser or null
- * if the parsers current token has been null
- * @throws IOException if the input could not be read
+ * Add a new coordinate to the collection
+ * @param coordinate coordinate of the point
+ * @return this
*/
- public static ShapeBuilder parse(XContentParser parser, GeoShapeFieldMapper geoDocMapper) throws IOException {
- return GeoShapeType.parse(parser, geoDocMapper);
+ public E coordinate(Coordinate coordinate) {
+ this.coordinates.add(coordinate);
+ return thisRef();
}
- protected static XContentBuilder toXContent(XContentBuilder builder, Coordinate coordinate) throws IOException {
- return builder.startArray().value(coordinate.x).value(coordinate.y).endArray();
+ /**
+ * Add a array of coordinates to the collection
+ *
+ * @param coordinates array of {@link Coordinate}s to add
+ * @return this
+ */
+ public E coordinates(Coordinate...coordinates) {
+ return this.coordinates(Arrays.asList(coordinates));
}
- protected static void writeCoordinateTo(Coordinate coordinate, StreamOutput out) throws IOException {
- out.writeDouble(coordinate.x);
- out.writeDouble(coordinate.y);
+ /**
+ * Add a collection of coordinates to the collection
+ *
+ * @param coordinates array of {@link Coordinate}s to add
+ * @return this
+ */
+ public E coordinates(Collection coordinates) {
+ this.coordinates.addAll(coordinates);
+ return thisRef();
}
- protected static Coordinate readFromStream(StreamInput in) throws IOException {
- return new Coordinate(in.readDouble(), in.readDouble());
+ /**
+ * Copy all coordinate to a new Array
+ *
+ * @param closed if set to true the first point of the array is repeated as last element
+ * @return Array of coordinates
+ */
+ protected Coordinate[] coordinates(boolean closed) {
+ Coordinate[] result = coordinates.toArray(new Coordinate[coordinates.size() + (closed?1:0)]);
+ if(closed) {
+ result[result.length-1] = result[0];
+ }
+ return result;
+ }
+
+ protected JtsGeometry jtsGeometry(Geometry geom) {
+ //dateline180Check is false because ElasticSearch does it's own dateline wrapping
+ JtsGeometry jtsGeometry = new JtsGeometry(geom, SPATIAL_CONTEXT, false, MULTI_POLYGON_MAY_OVERLAP);
+ if (AUTO_VALIDATE_JTS_GEOMETRY)
+ jtsGeometry.validate();
+ if (AUTO_INDEX_JTS_GEOMETRY)
+ jtsGeometry.index();
+ return jtsGeometry;
}
+ /**
+ * Create a new Shape from this builder. Since calling this method could change the
+ * defined shape. (by inserting new coordinates or change the position of points)
+ * the builder looses its validity. So this method should only be called once on a builder
+ * @return new {@link Shape} defined by the builder
+ */
+ public abstract T build();
+
protected static Coordinate shift(Coordinate coordinate, double dateline) {
if (dateline == 0) {
return coordinate;
@@ -255,58 +274,6 @@ protected static int intersections(double dateline, Edge[] edges) {
return numIntersections;
}
- /**
- * Node used to represent a tree of coordinates.
- *
- * Can either be a leaf node consisting of a Coordinate, or a parent with
- * children
- */
- protected static class CoordinateNode implements ToXContentObject {
-
- protected final Coordinate coordinate;
- protected final List children;
-
- /**
- * Creates a new leaf CoordinateNode
- *
- * @param coordinate
- * Coordinate for the Node
- */
- protected CoordinateNode(Coordinate coordinate) {
- this.coordinate = coordinate;
- this.children = null;
- }
-
- /**
- * Creates a new parent CoordinateNode
- *
- * @param children
- * Children of the Node
- */
- protected CoordinateNode(List children) {
- this.children = children;
- this.coordinate = null;
- }
-
- protected boolean isEmpty() {
- return (coordinate == null && (children == null || children.isEmpty()));
- }
-
- @Override
- public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
- if (children == null) {
- builder.startArray().value(coordinate.x).value(coordinate.y).endArray();
- } else {
- builder.startArray();
- for (CoordinateNode child : children) {
- child.toXContent(builder, params);
- }
- builder.endArray();
- }
- return builder;
- }
- }
-
/**
* This helper class implements a linked list for {@link Coordinate}. It contains
* fields for a dateline intersection and component id
@@ -415,293 +382,50 @@ public static Orientation fromString(String orientation) {
}
}
- public static final String FIELD_TYPE = "type";
- public static final String FIELD_COORDINATES = "coordinates";
- public static final String FIELD_GEOMETRIES = "geometries";
- public static final String FIELD_ORIENTATION = "orientation";
-
protected static final boolean debugEnabled() {
return LOGGER.isDebugEnabled() || DEBUG;
}
+ protected static XContentBuilder toXContent(XContentBuilder builder, Coordinate coordinate) throws IOException {
+ return builder.startArray().value(coordinate.x).value(coordinate.y).endArray();
+ }
+
/**
- * Enumeration that lists all {@link GeoShapeType}s that can be handled
+ * builds an array of coordinates to a {@link XContentBuilder}
+ *
+ * @param builder builder to use
+ * @param closed repeat the first point at the end of the array if it's not already defines as last element of the array
+ * @return the builder
*/
- public enum GeoShapeType {
- POINT("point"),
- MULTIPOINT("multipoint"),
- LINESTRING("linestring"),
- MULTILINESTRING("multilinestring"),
- POLYGON("polygon"),
- MULTIPOLYGON("multipolygon"),
- GEOMETRYCOLLECTION("geometrycollection"),
- ENVELOPE("envelope"),
- CIRCLE("circle");
-
- private final String shapename;
-
- GeoShapeType(String shapename) {
- this.shapename = shapename;
- }
-
- protected String shapeName() {
- return shapename;
- }
-
- public static GeoShapeType forName(String geoshapename) {
- String typename = geoshapename.toLowerCase(Locale.ROOT);
- for (GeoShapeType type : values()) {
- if(type.shapename.equals(typename)) {
- return type;
- }
- }
- throw new IllegalArgumentException("unknown geo_shape ["+geoshapename+"]");
- }
-
- public static ShapeBuilder parse(XContentParser parser) throws IOException {
- return parse(parser, null);
- }
-
- /**
- * Parse the geometry specified by the source document and return a ShapeBuilder instance used to
- * build the actual geometry
- * @param parser - parse utility object including source document
- * @param shapeMapper - field mapper needed for index specific parameters
- * @return ShapeBuilder - a builder instance used to create the geometry
- */
- public static ShapeBuilder parse(XContentParser parser, GeoShapeFieldMapper shapeMapper) throws IOException {
- if (parser.currentToken() == XContentParser.Token.VALUE_NULL) {
- return null;
- } else if (parser.currentToken() != XContentParser.Token.START_OBJECT) {
- throw new ElasticsearchParseException("shape must be an object consisting of type and coordinates");
- }
-
- GeoShapeType shapeType = null;
- Distance radius = null;
- CoordinateNode node = null;
- GeometryCollectionBuilder geometryCollections = null;
-
- Orientation requestedOrientation = (shapeMapper == null) ? Orientation.RIGHT : shapeMapper.fieldType().orientation();
- boolean coerce = (shapeMapper == null) ? GeoShapeFieldMapper.Defaults.COERCE.value() : shapeMapper.coerce().value();
-
- XContentParser.Token token;
- while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
- if (token == XContentParser.Token.FIELD_NAME) {
- String fieldName = parser.currentName();
-
- if (FIELD_TYPE.equals(fieldName)) {
- parser.nextToken();
- shapeType = GeoShapeType.forName(parser.text());
- } else if (FIELD_COORDINATES.equals(fieldName)) {
- parser.nextToken();
- node = parseCoordinates(parser);
- } else if (FIELD_GEOMETRIES.equals(fieldName)) {
- parser.nextToken();
- geometryCollections = parseGeometries(parser, shapeMapper);
- } else if (CircleBuilder.FIELD_RADIUS.equals(fieldName)) {
- parser.nextToken();
- radius = Distance.parseDistance(parser.text());
- } else if (FIELD_ORIENTATION.equals(fieldName)) {
- parser.nextToken();
- requestedOrientation = Orientation.fromString(parser.text());
- } else {
- parser.nextToken();
- parser.skipChildren();
- }
- }
- }
-
- if (shapeType == null) {
- throw new ElasticsearchParseException("shape type not included");
- } else if (node == null && GeoShapeType.GEOMETRYCOLLECTION != shapeType) {
- throw new ElasticsearchParseException("coordinates not included");
- } else if (geometryCollections == null && GeoShapeType.GEOMETRYCOLLECTION == shapeType) {
- throw new ElasticsearchParseException("geometries not included");
- } else if (radius != null && GeoShapeType.CIRCLE != shapeType) {
- throw new ElasticsearchParseException("field [{}] is supported for [{}] only", CircleBuilder.FIELD_RADIUS,
- CircleBuilder.TYPE);
- }
-
- switch (shapeType) {
- case POINT: return parsePoint(node);
- case MULTIPOINT: return parseMultiPoint(node);
- case LINESTRING: return parseLineString(node);
- case MULTILINESTRING: return parseMultiLine(node);
- case POLYGON: return parsePolygon(node, requestedOrientation, coerce);
- case MULTIPOLYGON: return parseMultiPolygon(node, requestedOrientation, coerce);
- case CIRCLE: return parseCircle(node, radius);
- case ENVELOPE: return parseEnvelope(node);
- case GEOMETRYCOLLECTION: return geometryCollections;
- default:
- throw new ElasticsearchParseException("shape type [{}] not included", shapeType);
- }
- }
-
- protected static void validatePointNode(CoordinateNode node) {
- if (node.isEmpty()) {
- throw new ElasticsearchParseException(
- "invalid number of points (0) provided when expecting a single coordinate ([lat, lng])");
- } else if (node.coordinate == null) {
- if (node.children.isEmpty() == false) {
- throw new ElasticsearchParseException("multipoint data provided when single point data expected.");
- }
- }
- }
-
- protected static PointBuilder parsePoint(CoordinateNode node) {
- validatePointNode(node);
- return ShapeBuilders.newPoint(node.coordinate);
- }
-
- protected static CircleBuilder parseCircle(CoordinateNode coordinates, Distance radius) {
- return ShapeBuilders.newCircleBuilder().center(coordinates.coordinate).radius(radius);
- }
-
- protected static EnvelopeBuilder parseEnvelope(CoordinateNode coordinates) {
- // validate the coordinate array for envelope type
- if (coordinates.children.size() != 2) {
- throw new ElasticsearchParseException(
- "invalid number of points [{}] provided for geo_shape [{}] when expecting an array of 2 coordinates",
- coordinates.children.size(), GeoShapeType.ENVELOPE.shapename);
- }
- // verify coordinate bounds, correct if necessary
- Coordinate uL = coordinates.children.get(0).coordinate;
- Coordinate lR = coordinates.children.get(1).coordinate;
- if (((lR.x < uL.x) || (uL.y < lR.y))) {
- Coordinate uLtmp = uL;
- uL = new Coordinate(Math.min(uL.x, lR.x), Math.max(uL.y, lR.y));
- lR = new Coordinate(Math.max(uLtmp.x, lR.x), Math.min(uLtmp.y, lR.y));
+ protected XContentBuilder coordinatesToXcontent(XContentBuilder builder, boolean closed) throws IOException {
+ builder.startArray();
+ for(Coordinate coord : coordinates) {
+ toXContent(builder, coord);
+ }
+ if(closed) {
+ Coordinate start = coordinates.get(0);
+ Coordinate end = coordinates.get(coordinates.size()-1);
+ if(start.x != end.x || start.y != end.y) {
+ toXContent(builder, coordinates.get(0));
}
- return ShapeBuilders.newEnvelope(uL, lR);
- }
-
- protected static void validateMultiPointNode(CoordinateNode coordinates) {
- if (coordinates.children == null || coordinates.children.isEmpty()) {
- if (coordinates.coordinate != null) {
- throw new ElasticsearchParseException("single coordinate found when expecting an array of " +
- "coordinates. change type to point or change data to an array of >0 coordinates");
- }
- throw new ElasticsearchParseException("no data provided for multipoint object when expecting " +
- ">0 points (e.g., [[lat, lng]] or [[lat, lng], ...])");
- } else {
- for (CoordinateNode point : coordinates.children) {
- validatePointNode(point);
- }
- }
- }
-
- protected static MultiPointBuilder parseMultiPoint(CoordinateNode coordinates) {
- validateMultiPointNode(coordinates);
- CoordinatesBuilder points = new CoordinatesBuilder();
- for (CoordinateNode node : coordinates.children) {
- points.coordinate(node.coordinate);
- }
- return new MultiPointBuilder(points.build());
- }
-
- protected static LineStringBuilder parseLineString(CoordinateNode coordinates) {
- /**
- * Per GeoJSON spec (http://geojson.org/geojson-spec.html#linestring)
- * "coordinates" member must be an array of two or more positions
- * LineStringBuilder should throw a graceful exception if < 2 coordinates/points are provided
- */
- if (coordinates.children.size() < 2) {
- throw new ElasticsearchParseException("invalid number of points in LineString (found [{}] - must be >= 2)",
- coordinates.children.size());
- }
-
- CoordinatesBuilder line = new CoordinatesBuilder();
- for (CoordinateNode node : coordinates.children) {
- line.coordinate(node.coordinate);
- }
- return ShapeBuilders.newLineString(line);
- }
-
- protected static MultiLineStringBuilder parseMultiLine(CoordinateNode coordinates) {
- MultiLineStringBuilder multiline = ShapeBuilders.newMultiLinestring();
- for (CoordinateNode node : coordinates.children) {
- multiline.linestring(parseLineString(node));
- }
- return multiline;
- }
-
- protected static LineStringBuilder parseLinearRing(CoordinateNode coordinates, boolean coerce) {
- /**
- * Per GeoJSON spec (http://geojson.org/geojson-spec.html#linestring)
- * A LinearRing is closed LineString with 4 or more positions. The first and last positions
- * are equivalent (they represent equivalent points). Though a LinearRing is not explicitly
- * represented as a GeoJSON geometry type, it is referred to in the Polygon geometry type definition.
- */
- if (coordinates.children == null) {
- String error = "Invalid LinearRing found.";
- error += (coordinates.coordinate == null) ?
- " No coordinate array provided" : " Found a single coordinate when expecting a coordinate array";
- throw new ElasticsearchParseException(error);
- }
-
- int numValidPts = coerce ? 3 : 4;
- if (coordinates.children.size() < numValidPts) {
- throw new ElasticsearchParseException("invalid number of points in LinearRing (found [{}] - must be >= [{}])",
- coordinates.children.size(), numValidPts);
- }
-
- if (!coordinates.children.get(0).coordinate.equals(
- coordinates.children.get(coordinates.children.size() - 1).coordinate)) {
- if (coerce) {
- coordinates.children.add(coordinates.children.get(0));
- } else {
- throw new ElasticsearchParseException("invalid LinearRing found (coordinates are not closed)");
- }
- }
- return parseLineString(coordinates);
- }
-
- protected static PolygonBuilder parsePolygon(CoordinateNode coordinates, final Orientation orientation, final boolean coerce) {
- if (coordinates.children == null || coordinates.children.isEmpty()) {
- throw new ElasticsearchParseException(
- "invalid LinearRing provided for type polygon. Linear ring must be an array of coordinates");
- }
-
- LineStringBuilder shell = parseLinearRing(coordinates.children.get(0), coerce);
- PolygonBuilder polygon = new PolygonBuilder(shell, orientation);
- for (int i = 1; i < coordinates.children.size(); i++) {
- polygon.hole(parseLinearRing(coordinates.children.get(i), coerce));
- }
- return polygon;
}
+ builder.endArray();
+ return builder;
+ }
- protected static MultiPolygonBuilder parseMultiPolygon(CoordinateNode coordinates, final Orientation orientation,
- final boolean coerce) {
- MultiPolygonBuilder polygons = ShapeBuilders.newMultiPolygon(orientation);
- for (CoordinateNode node : coordinates.children) {
- polygons.polygon(parsePolygon(node, orientation, coerce));
- }
- return polygons;
- }
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (!(o instanceof ShapeBuilder)) return false;
- /**
- * Parse the geometries array of a GeometryCollection
- *
- * @param parser Parser that will be read from
- * @return Geometry[] geometries of the GeometryCollection
- * @throws IOException Thrown if an error occurs while reading from the XContentParser
- */
- protected static GeometryCollectionBuilder parseGeometries(XContentParser parser, GeoShapeFieldMapper mapper) throws
- IOException {
- if (parser.currentToken() != XContentParser.Token.START_ARRAY) {
- throw new ElasticsearchParseException("geometries must be an array of geojson objects");
- }
+ ShapeBuilder that = (ShapeBuilder) o;
- XContentParser.Token token = parser.nextToken();
- GeometryCollectionBuilder geometryCollection = ShapeBuilders.newGeometryCollection();
- while (token != XContentParser.Token.END_ARRAY) {
- ShapeBuilder shapeBuilder = GeoShapeType.parse(parser);
- geometryCollection.shape(shapeBuilder);
- token = parser.nextToken();
- }
+ return Objects.equals(coordinates, that.coordinates);
+ }
- return geometryCollection;
- }
+ @Override
+ public int hashCode() {
+ return Objects.hash(coordinates);
}
@Override
diff --git a/core/src/main/java/org/elasticsearch/common/geo/builders/ShapeBuilders.java b/core/src/main/java/org/elasticsearch/common/geo/builders/ShapeBuilders.java
deleted file mode 100644
index e0afa4c20d50c..0000000000000
--- a/core/src/main/java/org/elasticsearch/common/geo/builders/ShapeBuilders.java
+++ /dev/null
@@ -1,153 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.common.geo.builders;
-
-import java.util.List;
-
-import com.vividsolutions.jts.geom.Coordinate;
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry.Entry;
-
-/**
- * A collection of static methods for creating ShapeBuilders.
- */
-public class ShapeBuilders {
-
- /**
- * Create a new point
- *
- * @param longitude longitude of the point
- * @param latitude latitude of the point
- * @return a new {@link PointBuilder}
- */
- public static PointBuilder newPoint(double longitude, double latitude) {
- return ShapeBuilders.newPoint(new Coordinate(longitude, latitude));
- }
-
- /**
- * Create a new {@link PointBuilder} from a {@link Coordinate}
- * @param coordinate coordinate defining the position of the point
- * @return a new {@link PointBuilder}
- */
- public static PointBuilder newPoint(Coordinate coordinate) {
- return new PointBuilder().coordinate(coordinate);
- }
-
- /**
- * Create a new set of points
- * @return new {@link MultiPointBuilder}
- */
- public static MultiPointBuilder newMultiPoint(List points) {
- return new MultiPointBuilder(points);
- }
-
- /**
- * Create a new lineString
- * @return a new {@link LineStringBuilder}
- */
- public static LineStringBuilder newLineString(List list) {
- return new LineStringBuilder(list);
- }
-
- /**
- * Create a new lineString
- * @return a new {@link LineStringBuilder}
- */
- public static LineStringBuilder newLineString(CoordinatesBuilder coordinates) {
- return new LineStringBuilder(coordinates);
- }
-
- /**
- * Create a new Collection of lineStrings
- * @return a new {@link MultiLineStringBuilder}
- */
- public static MultiLineStringBuilder newMultiLinestring() {
- return new MultiLineStringBuilder();
- }
-
- /**
- * Create a new PolygonBuilder
- * @return a new {@link PolygonBuilder}
- */
- public static PolygonBuilder newPolygon(List shell) {
- return new PolygonBuilder(new CoordinatesBuilder().coordinates(shell));
- }
-
- /**
- * Create a new PolygonBuilder
- * @return a new {@link PolygonBuilder}
- */
- public static PolygonBuilder newPolygon(CoordinatesBuilder shell) {
- return new PolygonBuilder(shell);
- }
-
- /**
- * Create a new Collection of polygons
- * @return a new {@link MultiPolygonBuilder}
- */
- public static MultiPolygonBuilder newMultiPolygon() {
- return new MultiPolygonBuilder();
- }
-
- /**
- * Create a new Collection of polygons
- * @return a new {@link MultiPolygonBuilder}
- */
- public static MultiPolygonBuilder newMultiPolygon(ShapeBuilder.Orientation orientation) {
- return new MultiPolygonBuilder(orientation);
- }
-
- /**
- * Create a new GeometryCollection
- * @return a new {@link GeometryCollectionBuilder}
- */
- public static GeometryCollectionBuilder newGeometryCollection() {
- return new GeometryCollectionBuilder();
- }
-
- /**
- * create a new Circle
- *
- * @return a new {@link CircleBuilder}
- */
- public static CircleBuilder newCircleBuilder() {
- return new CircleBuilder();
- }
-
- /**
- * create a new rectangle
- *
- * @return a new {@link EnvelopeBuilder}
- */
- public static EnvelopeBuilder newEnvelope(Coordinate topLeft, Coordinate bottomRight) {
- return new EnvelopeBuilder(topLeft, bottomRight);
- }
-
- public static void register(List namedWriteables) {
- namedWriteables.add(new Entry(ShapeBuilder.class, PointBuilder.TYPE.shapeName(), PointBuilder::new));
- namedWriteables.add(new Entry(ShapeBuilder.class, CircleBuilder.TYPE.shapeName(), CircleBuilder::new));
- namedWriteables.add(new Entry(ShapeBuilder.class, EnvelopeBuilder.TYPE.shapeName(), EnvelopeBuilder::new));
- namedWriteables.add(new Entry(ShapeBuilder.class, MultiPointBuilder.TYPE.shapeName(), MultiPointBuilder::new));
- namedWriteables.add(new Entry(ShapeBuilder.class, LineStringBuilder.TYPE.shapeName(), LineStringBuilder::new));
- namedWriteables.add(new Entry(ShapeBuilder.class, MultiLineStringBuilder.TYPE.shapeName(), MultiLineStringBuilder::new));
- namedWriteables.add(new Entry(ShapeBuilder.class, PolygonBuilder.TYPE.shapeName(), PolygonBuilder::new));
- namedWriteables.add(new Entry(ShapeBuilder.class, MultiPolygonBuilder.TYPE.shapeName(), MultiPolygonBuilder::new));
- namedWriteables.add(new Entry(ShapeBuilder.class, GeometryCollectionBuilder.TYPE.shapeName(), GeometryCollectionBuilder::new));
- }
-}
diff --git a/core/src/main/java/org/elasticsearch/common/geo/parsers/CoordinateNode.java b/core/src/main/java/org/elasticsearch/common/geo/parsers/CoordinateNode.java
new file mode 100644
index 0000000000000..d766d75d5ec1a
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/common/geo/parsers/CoordinateNode.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.common.geo.parsers;
+
+import com.vividsolutions.jts.geom.Coordinate;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * Node used to represent a tree of coordinates.
+ *
+ * Can either be a leaf node consisting of a Coordinate, or a parent with
+ * children
+ */
+public class CoordinateNode implements ToXContentObject {
+ public final Coordinate coordinate;
+ public final List children;
+
+ /**
+ * Creates a new leaf CoordinateNode
+ *
+ * @param coordinate
+ * Coordinate for the Node
+ */
+ protected CoordinateNode(Coordinate coordinate) {
+ this.coordinate = coordinate;
+ this.children = null;
+ }
+
+ /**
+ * Creates a new parent CoordinateNode
+ *
+ * @param children
+ * Children of the Node
+ */
+ protected CoordinateNode(List children) {
+ this.children = children;
+ this.coordinate = null;
+ }
+
+ public boolean isEmpty() {
+ return (coordinate == null && (children == null || children.isEmpty()));
+ }
+
+ public boolean isMultiPoint() {
+ return children != null && children.size() > 1;
+ }
+
+ @Override
+ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+ if (children == null) {
+ builder.startArray().value(coordinate.x).value(coordinate.y).endArray();
+ } else {
+ builder.startArray();
+ for (CoordinateNode child : children) {
+ child.toXContent(builder, params);
+ }
+ builder.endArray();
+ }
+ return builder;
+ }
+}
diff --git a/core/src/main/java/org/elasticsearch/common/geo/parsers/GeoJsonParser.java b/core/src/main/java/org/elasticsearch/common/geo/parsers/GeoJsonParser.java
new file mode 100644
index 0000000000000..90145448be326
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/common/geo/parsers/GeoJsonParser.java
@@ -0,0 +1,194 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.common.geo.parsers;
+
+import com.vividsolutions.jts.geom.Coordinate;
+import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.common.Explicit;
+import org.elasticsearch.common.geo.GeoShapeType;
+import org.elasticsearch.common.geo.builders.CircleBuilder;
+import org.elasticsearch.common.geo.builders.GeometryCollectionBuilder;
+import org.elasticsearch.common.geo.builders.ShapeBuilder;
+import org.elasticsearch.common.unit.DistanceUnit;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.mapper.GeoShapeFieldMapper;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Parses shape geometry represented in geojson
+ *
+ * complies with geojson specification: https://tools.ietf.org/html/rfc7946
+ */
+abstract class GeoJsonParser {
+ protected static ShapeBuilder parse(XContentParser parser, GeoShapeFieldMapper shapeMapper)
+ throws IOException {
+ GeoShapeType shapeType = null;
+ DistanceUnit.Distance radius = null;
+ CoordinateNode coordinateNode = null;
+ GeometryCollectionBuilder geometryCollections = null;
+
+ ShapeBuilder.Orientation requestedOrientation =
+ (shapeMapper == null) ? ShapeBuilder.Orientation.RIGHT : shapeMapper.fieldType().orientation();
+ Explicit coerce = (shapeMapper == null) ? GeoShapeFieldMapper.Defaults.COERCE : shapeMapper.coerce();
+
+ String malformedException = null;
+
+ XContentParser.Token token;
+ while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+ if (token == XContentParser.Token.FIELD_NAME) {
+ String fieldName = parser.currentName();
+
+ if (ShapeParser.FIELD_TYPE.match(fieldName)) {
+ parser.nextToken();
+ final GeoShapeType type = GeoShapeType.forName(parser.text());
+ if (shapeType != null && shapeType.equals(type) == false) {
+ malformedException = ShapeParser.FIELD_TYPE + " already parsed as ["
+ + shapeType + "] cannot redefine as [" + type + "]";
+ } else {
+ shapeType = type;
+ }
+ } else if (ShapeParser.FIELD_COORDINATES.match(fieldName)) {
+ parser.nextToken();
+ coordinateNode = parseCoordinates(parser);
+ } else if (ShapeParser.FIELD_GEOMETRIES.match(fieldName)) {
+ if (shapeType == null) {
+ shapeType = GeoShapeType.GEOMETRYCOLLECTION;
+ } else if (shapeType.equals(GeoShapeType.GEOMETRYCOLLECTION) == false) {
+ malformedException = "cannot have [" + ShapeParser.FIELD_GEOMETRIES + "] with type set to ["
+ + shapeType + "]";
+ }
+ parser.nextToken();
+ geometryCollections = parseGeometries(parser, shapeMapper);
+ } else if (CircleBuilder.FIELD_RADIUS.match(fieldName)) {
+ if (shapeType == null) {
+ shapeType = GeoShapeType.CIRCLE;
+ } else if (shapeType != null && shapeType.equals(GeoShapeType.CIRCLE) == false) {
+ malformedException = "cannot have [" + CircleBuilder.FIELD_RADIUS + "] with type set to ["
+ + shapeType + "]";
+ }
+ parser.nextToken();
+ radius = DistanceUnit.Distance.parseDistance(parser.text());
+ } else if (ShapeParser.FIELD_ORIENTATION.match(fieldName)) {
+ if (shapeType != null
+ && (shapeType.equals(GeoShapeType.POLYGON) || shapeType.equals(GeoShapeType.MULTIPOLYGON)) == false) {
+ malformedException = "cannot have [" + ShapeParser.FIELD_ORIENTATION + "] with type set to [" + shapeType + "]";
+ }
+ parser.nextToken();
+ requestedOrientation = ShapeBuilder.Orientation.fromString(parser.text());
+ } else {
+ parser.nextToken();
+ parser.skipChildren();
+ }
+ }
+ }
+
+ if (malformedException != null) {
+ throw new ElasticsearchParseException(malformedException);
+ } else if (shapeType == null) {
+ throw new ElasticsearchParseException("shape type not included");
+ } else if (coordinateNode == null && GeoShapeType.GEOMETRYCOLLECTION != shapeType) {
+ throw new ElasticsearchParseException("coordinates not included");
+ } else if (geometryCollections == null && GeoShapeType.GEOMETRYCOLLECTION == shapeType) {
+ throw new ElasticsearchParseException("geometries not included");
+ } else if (radius != null && GeoShapeType.CIRCLE != shapeType) {
+ throw new ElasticsearchParseException("field [{}] is supported for [{}] only", CircleBuilder.FIELD_RADIUS,
+ CircleBuilder.TYPE);
+ }
+
+ if (shapeType == null) {
+ throw new ElasticsearchParseException("shape type [{}] not included", shapeType);
+ }
+
+ if (shapeType.equals(GeoShapeType.GEOMETRYCOLLECTION)) {
+ return geometryCollections;
+ }
+
+ return shapeType.getBuilder(coordinateNode, radius, requestedOrientation, coerce.value());
+ }
+
+ /**
+ * Recursive method which parses the arrays of coordinates used to define
+ * Shapes
+ *
+ * @param parser
+ * Parser that will be read from
+ * @return CoordinateNode representing the start of the coordinate tree
+ * @throws IOException
+ * Thrown if an error occurs while reading from the
+ * XContentParser
+ */
+ private static CoordinateNode parseCoordinates(XContentParser parser) throws IOException {
+ XContentParser.Token token = parser.nextToken();
+ // Base cases
+ if (token != XContentParser.Token.START_ARRAY &&
+ token != XContentParser.Token.END_ARRAY &&
+ token != XContentParser.Token.VALUE_NULL) {
+ return new CoordinateNode(parseCoordinate(parser));
+ } else if (token == XContentParser.Token.VALUE_NULL) {
+ throw new IllegalArgumentException("coordinates cannot contain NULL values)");
+ }
+
+ List nodes = new ArrayList<>();
+ while (token != XContentParser.Token.END_ARRAY) {
+ nodes.add(parseCoordinates(parser));
+ token = parser.nextToken();
+ }
+
+ return new CoordinateNode(nodes);
+ }
+
+ private static Coordinate parseCoordinate(XContentParser parser) throws IOException {
+ double lon = parser.doubleValue();
+ parser.nextToken();
+ double lat = parser.doubleValue();
+ XContentParser.Token token = parser.nextToken();
+ while (token == XContentParser.Token.VALUE_NUMBER) {
+ token = parser.nextToken();
+ }
+ // todo support z/alt
+ return new Coordinate(lon, lat);
+ }
+
+ /**
+ * Parse the geometries array of a GeometryCollection
+ *
+ * @param parser Parser that will be read from
+ * @return Geometry[] geometries of the GeometryCollection
+ * @throws IOException Thrown if an error occurs while reading from the XContentParser
+ */
+ static GeometryCollectionBuilder parseGeometries(XContentParser parser, GeoShapeFieldMapper mapper) throws
+ IOException {
+ if (parser.currentToken() != XContentParser.Token.START_ARRAY) {
+ throw new ElasticsearchParseException("geometries must be an array of geojson objects");
+ }
+
+ XContentParser.Token token = parser.nextToken();
+ GeometryCollectionBuilder geometryCollection = new GeometryCollectionBuilder();
+ while (token != XContentParser.Token.END_ARRAY) {
+ ShapeBuilder shapeBuilder = ShapeParser.parse(parser);
+ geometryCollection.shape(shapeBuilder);
+ token = parser.nextToken();
+ }
+
+ return geometryCollection;
+ }
+}
diff --git a/core/src/main/java/org/elasticsearch/common/geo/parsers/ShapeParser.java b/core/src/main/java/org/elasticsearch/common/geo/parsers/ShapeParser.java
new file mode 100644
index 0000000000000..39540f902fedf
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/common/geo/parsers/ShapeParser.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.common.geo.parsers;
+
+import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.geo.builders.ShapeBuilder;
+import org.elasticsearch.common.xcontent.XContent;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.mapper.GeoShapeFieldMapper;
+
+import java.io.IOException;
+
+/**
+ * first point of entry for a shape parser
+ */
+public interface ShapeParser {
+ ParseField FIELD_TYPE = new ParseField("type");
+ ParseField FIELD_COORDINATES = new ParseField("coordinates");
+ ParseField FIELD_GEOMETRIES = new ParseField("geometries");
+ ParseField FIELD_ORIENTATION = new ParseField("orientation");
+
+ /**
+ * Create a new {@link ShapeBuilder} from {@link XContent}
+ * @param parser parser to read the GeoShape from
+ * @param shapeMapper document field mapper reference required for spatial parameters relevant
+ * to the shape construction process (e.g., orientation)
+ * todo: refactor to place build specific parameters in the SpatialContext
+ * @return {@link ShapeBuilder} read from the parser or null
+ * if the parsers current token has been null
+ * @throws IOException if the input could not be read
+ */
+ static ShapeBuilder parse(XContentParser parser, GeoShapeFieldMapper shapeMapper) throws IOException {
+ if (parser.currentToken() == XContentParser.Token.VALUE_NULL) {
+ return null;
+ } if (parser.currentToken() == XContentParser.Token.START_OBJECT) {
+ return GeoJsonParser.parse(parser, shapeMapper);
+ }
+ throw new ElasticsearchParseException("shape must be an object consisting of type and coordinates");
+ }
+
+ /**
+ * Create a new {@link ShapeBuilder} from {@link XContent}
+ * @param parser parser to read the GeoShape from
+ * @return {@link ShapeBuilder} read from the parser or null
+ * if the parsers current token has been null
+ * @throws IOException if the input could not be read
+ */
+ static ShapeBuilder parse(XContentParser parser) throws IOException {
+ return parse(parser, null);
+ }
+}
diff --git a/core/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java b/core/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java
index 61f32c67c20cb..38eaef1d14df9 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java
@@ -264,20 +264,16 @@ public synchronized void addSettingsUpdateConsumer(Setting setting, Consu
}
/**
- * Validates that all settings in the builder are registered and valid
+ * Validates that all given settings are registered and valid
+ * @param settings the settings to validate
+ * @param validateDependencies if true settings dependencies are validated as well.
+ * @see Setting#getSettingsDependencies(String)
*/
- public final void validate(Settings.Builder settingsBuilder) {
- validate(settingsBuilder.build());
- }
-
- /**
- * * Validates that all given settings are registered and valid
- */
- public final void validate(Settings settings) {
+ public final void validate(Settings settings, boolean validateDependencies) {
List exceptions = new ArrayList<>();
for (String key : settings.keySet()) { // settings iterate in deterministic fashion
try {
- validate(key, settings);
+ validate(key, settings, validateDependencies);
} catch (RuntimeException ex) {
exceptions.add(ex);
}
@@ -285,12 +281,11 @@ public final void validate(Settings settings) {
ExceptionsHelper.rethrowAndSuppress(exceptions);
}
-
/**
* Validates that the setting is valid
*/
- public final void validate(String key, Settings settings) {
- Setting setting = get(key);
+ void validate(String key, Settings settings, boolean validateDependencies) {
+ Setting setting = getRaw(key);
if (setting == null) {
LevensteinDistance ld = new LevensteinDistance();
List> scoredKeys = new ArrayList<>();
@@ -315,6 +310,20 @@ public final void validate(String key, Settings settings) {
"settings";
}
throw new IllegalArgumentException(msg);
+ } else {
+ Set settingsDependencies = setting.getSettingsDependencies(key);
+ if (setting.hasComplexMatcher()) {
+ setting = setting.getConcreteSetting(key);
+ }
+ if (validateDependencies && settingsDependencies.isEmpty() == false) {
+ Set settingKeys = settings.keySet();
+ for (String requiredSetting : settingsDependencies) {
+ if (settingKeys.contains(requiredSetting) == false) {
+ throw new IllegalArgumentException("Missing required setting ["
+ + requiredSetting + "] for setting [" + setting.getKey() + "]");
+ }
+ }
+ }
}
setting.get(settings);
}
@@ -375,7 +384,18 @@ default Runnable updater(Settings current, Settings previous) {
/**
* Returns the {@link Setting} for the given key or null if the setting can not be found.
*/
- public Setting get(String key) {
+ public final Setting get(String key) {
+ Setting raw = getRaw(key);
+ if (raw == null) {
+ return null;
+ } if (raw.hasComplexMatcher()) {
+ return raw.getConcreteSetting(key);
+ } else {
+ return raw;
+ }
+ }
+
+ private Setting getRaw(String key) {
Setting setting = keySettings.get(key);
if (setting != null) {
return setting;
@@ -383,7 +403,8 @@ public Setting get(String key) {
for (Map.Entry> entry : complexMatchers.entrySet()) {
if (entry.getValue().match(key)) {
assert assertMatcher(key, 1);
- return entry.getValue().getConcreteSetting(key);
+ assert entry.getValue().hasComplexMatcher();
+ return entry.getValue();
}
}
return null;
@@ -513,7 +534,7 @@ private boolean updateSettings(Settings toApply, Settings.Builder target, Settin
} else if (get(key) == null) {
throw new IllegalArgumentException(type + " setting [" + key + "], not recognized");
} else if (isNull == false && canUpdate.test(key)) {
- validate(key, toApply);
+ validate(key, toApply, false); // we might not have a full picture here do to a dependency validation
settingsBuilder.copy(key, toApply);
updates.copy(key, toApply);
changed = true;
@@ -654,7 +675,7 @@ public String setValue(String value) {
* representation. Otherwise false
*/
// TODO this should be replaced by Setting.Property.HIDDEN or something like this.
- protected boolean isPrivateSetting(String key) {
+ public boolean isPrivateSetting(String key) {
return false;
}
}
diff --git a/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java
index 962e61b5c3c68..d40488eaa34f8 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java
@@ -191,7 +191,7 @@ protected void validateSettingKey(Setting setting) {
}
@Override
- protected boolean isPrivateSetting(String key) {
+ public boolean isPrivateSetting(String key) {
switch (key) {
case IndexMetaData.SETTING_CREATION_DATE:
case IndexMetaData.SETTING_INDEX_UUID:
diff --git a/core/src/main/java/org/elasticsearch/common/settings/Setting.java b/core/src/main/java/org/elasticsearch/common/settings/Setting.java
index 9b99e67c8c4da..abc589aedafc3 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/Setting.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/Setting.java
@@ -42,6 +42,7 @@
import java.util.Collections;
import java.util.EnumSet;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.IdentityHashMap;
import java.util.Iterator;
import java.util.List;
@@ -126,7 +127,7 @@ public enum Property {
private static final EnumSet EMPTY_PROPERTIES = EnumSet.noneOf(Property.class);
private Setting(Key key, @Nullable Setting fallbackSetting, Function defaultValue, Function parser,
- Validator validator, Property... properties) {
+ Validator validator, Property... properties) {
assert this instanceof SecureSetting || this.isGroupSetting() || parser.apply(defaultValue.apply(Settings.EMPTY)) != null
: "parser returned null";
this.key = key;
@@ -457,6 +458,14 @@ public Setting getConcreteSetting(String key) {
return this;
}
+ /**
+ * Returns a set of settings that are required at validation time. Unless all of the dependencies are present in the settings
+ * object validation of setting must fail.
+ */
+ public Set getSettingsDependencies(String key) {
+ return Collections.emptySet();
+ }
+
/**
* Build a new updater with a noop validator.
*/
@@ -519,11 +528,13 @@ public String toString() {
public static class AffixSetting extends Setting {
private final AffixKey key;
private final Function> delegateFactory;
+ private final Set dependencies;
- public AffixSetting(AffixKey key, Setting delegate, Function> delegateFactory) {
+ public AffixSetting(AffixKey key, Setting delegate, Function> delegateFactory, AffixSetting... dependencies) {
super(key, delegate.defaultValue, delegate.parser, delegate.properties.toArray(new Property[0]));
this.key = key;
this.delegateFactory = delegateFactory;
+ this.dependencies = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(dependencies)));
}
boolean isGroupSetting() {
@@ -534,6 +545,15 @@ private Stream matchStream(Settings settings) {
return settings.keySet().stream().filter((key) -> match(key)).map(settingKey -> key.getConcreteString(settingKey));
}
+ public Set getSettingsDependencies(String settingsKey) {
+ if (dependencies.isEmpty()) {
+ return Collections.emptySet();
+ } else {
+ String namespace = key.getNamespace(settingsKey);
+ return dependencies.stream().map(s -> s.key.toConcreteKey(namespace).key).collect(Collectors.toSet());
+ }
+ }
+
AbstractScopedSettings.SettingUpdater, T>> newAffixUpdater(
BiConsumer consumer, Logger logger, BiConsumer validator) {
return new AbstractScopedSettings.SettingUpdater, T>>() {
@@ -659,6 +679,13 @@ public Stream> getAllConcreteSettings(Settings settings) {
return matchStream(settings).distinct().map(this::getConcreteSetting);
}
+ /**
+ * Returns distinct namespaces for the given settings
+ */
+ public Set getNamespaces(Settings settings) {
+ return settings.keySet().stream().filter(this::match).map(key::getNamespace).collect(Collectors.toSet());
+ }
+
/**
* Returns a map of all namespaces to it's values give the provided settings
*/
@@ -1184,13 +1211,15 @@ public static AffixSetting prefixKeySetting(String prefix, Function AffixSetting affixKeySetting(String prefix, String suffix, Function> delegateFactory) {
- return affixKeySetting(new AffixKey(prefix, suffix), delegateFactory);
+ public static AffixSetting affixKeySetting(String prefix, String suffix, Function> delegateFactory,
+ AffixSetting... dependencies) {
+ return affixKeySetting(new AffixKey(prefix, suffix), delegateFactory, dependencies);
}
- private static AffixSetting affixKeySetting(AffixKey key, Function> delegateFactory) {
+ private static AffixSetting affixKeySetting(AffixKey key, Function> delegateFactory,
+ AffixSetting... dependencies) {
Setting delegate = delegateFactory.apply("_na_");
- return new AffixSetting<>(key, delegate, delegateFactory);
+ return new AffixSetting<>(key, delegate, delegateFactory, dependencies);
};
diff --git a/core/src/main/java/org/elasticsearch/common/settings/SettingsModule.java b/core/src/main/java/org/elasticsearch/common/settings/SettingsModule.java
index 45b511e1cc168..0304b20e992e5 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/SettingsModule.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/SettingsModule.java
@@ -132,7 +132,7 @@ public SettingsModule(Settings settings, List> additionalSettings, Li
}
}
// by now we are fully configured, lets check node level settings for unregistered index settings
- clusterSettings.validate(settings);
+ clusterSettings.validate(settings, true);
this.settingsFilter = new SettingsFilter(settings, settingsFilterPattern);
}
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/UnicastZenPing.java b/core/src/main/java/org/elasticsearch/discovery/zen/UnicastZenPing.java
index d0a9a212bdd14..06269706e0d5f 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/UnicastZenPing.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/UnicastZenPing.java
@@ -575,7 +575,8 @@ public void handleResponse(UnicastPingResponse response) {
@Override
public void handleException(TransportException exp) {
- if (exp instanceof ConnectTransportException || exp.getCause() instanceof ConnectTransportException) {
+ if (exp instanceof ConnectTransportException || exp.getCause() instanceof ConnectTransportException ||
+ exp.getCause() instanceof AlreadyClosedException) {
// ok, not connected...
logger.trace((Supplier) () -> new ParameterizedMessage("failed to connect to {}", node), exp);
} else if (closed == false) {
@@ -608,6 +609,9 @@ class UnicastPingRequestHandler implements TransportRequestHandler> IFD getForField(MappedFieldType fieldType
final String fieldName = fieldType.name();
IndexFieldData.Builder builder = fieldType.fielddataBuilder(fullyQualifiedIndexName);
- IndexFieldDataCache cache;
- synchronized (this) {
- cache = fieldDataCaches.get(fieldName);
- if (cache == null) {
- String cacheType = indexSettings.getValue(INDEX_FIELDDATA_CACHE_KEY);
- if (FIELDDATA_CACHE_VALUE_NODE.equals(cacheType)) {
- cache = indicesFieldDataCache.buildIndexFieldDataCache(listener, index(), fieldName);
- } else if ("none".equals(cacheType)){
- cache = new IndexFieldDataCache.None();
- } else {
- throw new IllegalArgumentException("cache type not supported [" + cacheType + "] for field [" + fieldName + "]");
+ IndexFieldDataCache cache = fieldDataCaches.get(fieldName);
+ if (cache == null) {
+ //for perf reason, only synchronize when cache is null
+ synchronized (this) {
+ cache = fieldDataCaches.get(fieldName);
+ //double checked locking to make sure it is thread safe
+ //especially when other threads calling clear() or clearField()
+ if (cache == null) {
+ String cacheType = indexSettings.getValue(INDEX_FIELDDATA_CACHE_KEY);
+ if (FIELDDATA_CACHE_VALUE_NODE.equals(cacheType)) {
+ cache = indicesFieldDataCache.buildIndexFieldDataCache(listener, index(), fieldName);
+ } else if ("none".equals(cacheType)){
+ cache = new IndexFieldDataCache.None();
+ } else {
+ throw new IllegalArgumentException("cache type not supported [" + cacheType + "] for field [" + fieldName + "]");
+ }
+ fieldDataCaches.put(fieldName, cache);
}
- fieldDataCaches.put(fieldName, cache);
}
}
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/DocValuesIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/DocValuesIndexFieldData.java
index 4a066328bc2a9..698b289d758be 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/DocValuesIndexFieldData.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/DocValuesIndexFieldData.java
@@ -46,13 +46,11 @@ public abstract class DocValuesIndexFieldData {
protected final Index index;
protected final String fieldName;
- protected final Logger logger;
public DocValuesIndexFieldData(Index index, String fieldName) {
super();
this.index = index;
this.fieldName = fieldName;
- this.logger = Loggers.getLogger(getClass());
}
public final String getFieldName() {
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVOrdinalsIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVOrdinalsIndexFieldData.java
index 0834d2479f072..4b3643dda059a 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVOrdinalsIndexFieldData.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVOrdinalsIndexFieldData.java
@@ -19,6 +19,7 @@
package org.elasticsearch.index.fielddata.plain;
+import org.apache.logging.log4j.Logger;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.OrdinalMap;
@@ -28,6 +29,7 @@
import org.apache.lucene.search.SortedSetSortField;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData;
import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
@@ -48,6 +50,7 @@ public class SortedSetDVOrdinalsIndexFieldData extends DocValuesIndexFieldData i
private final IndexFieldDataCache cache;
private final CircuitBreakerService breakerService;
private final Function> scriptFunction;
+ private static final Logger logger = Loggers.getLogger(SortedSetDVOrdinalsIndexFieldData.class);
public SortedSetDVOrdinalsIndexFieldData(IndexSettings indexSettings, IndexFieldDataCache cache, String fieldName,
CircuitBreakerService breakerService, Function> scriptFunction) {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java
index c605b8d093644..495bdf49cb9fc 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java
@@ -21,7 +21,6 @@
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.Term;
-import org.apache.lucene.search.DocValuesFieldExistsQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.spatial.prefix.PrefixTreeStrategy;
@@ -37,6 +36,7 @@
import org.elasticsearch.common.geo.SpatialStrategy;
import org.elasticsearch.common.geo.builders.ShapeBuilder;
import org.elasticsearch.common.geo.builders.ShapeBuilder.Orientation;
+import org.elasticsearch.common.geo.parsers.ShapeParser;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.DistanceUnit;
import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -54,6 +54,8 @@
import java.util.Map;
import java.util.Objects;
+import static org.elasticsearch.index.mapper.GeoPointFieldMapper.Names.IGNORE_MALFORMED;
+
/**
* FieldMapper for indexing {@link org.locationtech.spatial4j.shape.Shape}s.
*
@@ -96,6 +98,7 @@ public static class Defaults {
public static final Orientation ORIENTATION = Orientation.RIGHT;
public static final double LEGACY_DISTANCE_ERROR_PCT = 0.025d;
public static final Explicit COERCE = new Explicit<>(false, false);
+ public static final Explicit IGNORE_MALFORMED = new Explicit<>(false, false);
public static final MappedFieldType FIELD_TYPE = new GeoShapeFieldType();
@@ -115,6 +118,7 @@ public static class Defaults {
public static class Builder extends FieldMapper.Builder {
private Boolean coerce;
+ private Boolean ignoreMalformed;
public Builder(String name) {
super(name, Defaults.FIELD_TYPE, Defaults.FIELD_TYPE);
@@ -145,6 +149,21 @@ protected Explicit coerce(BuilderContext context) {
return Defaults.COERCE;
}
+ public Builder ignoreMalformed(boolean ignoreMalformed) {
+ this.ignoreMalformed = ignoreMalformed;
+ return builder;
+ }
+
+ protected Explicit ignoreMalformed(BuilderContext context) {
+ if (ignoreMalformed != null) {
+ return new Explicit<>(ignoreMalformed, true);
+ }
+ if (context.indexSettings() != null) {
+ return new Explicit<>(IGNORE_MALFORMED_SETTING.get(context.indexSettings()), false);
+ }
+ return Defaults.IGNORE_MALFORMED;
+ }
+
@Override
public GeoShapeFieldMapper build(BuilderContext context) {
GeoShapeFieldType geoShapeFieldType = (GeoShapeFieldType)fieldType;
@@ -154,8 +173,8 @@ public GeoShapeFieldMapper build(BuilderContext context) {
}
setupFieldType(context);
- return new GeoShapeFieldMapper(name, fieldType, coerce(context), context.indexSettings(), multiFieldsBuilder.build(this,
- context), copyTo);
+ return new GeoShapeFieldMapper(name, fieldType, ignoreMalformed(context), coerce(context), context.indexSettings(),
+ multiFieldsBuilder.build(this, context), copyTo);
}
}
@@ -186,6 +205,9 @@ public Mapper.Builder parse(String name, Map node, ParserContext
} else if (Names.STRATEGY.equals(fieldName)) {
builder.fieldType().setStrategyName(fieldNode.toString());
iterator.remove();
+ } else if (IGNORE_MALFORMED.equals(fieldName)) {
+ builder.ignoreMalformed(TypeParsers.nodeBooleanValue(fieldName, "ignore_malformed", fieldNode, parserContext));
+ iterator.remove();
} else if (Names.COERCE.equals(fieldName)) {
builder.coerce(TypeParsers.nodeBooleanValue(fieldName, Names.COERCE, fieldNode, parserContext));
iterator.remove();
@@ -428,11 +450,13 @@ public Query termQuery(Object value, QueryShardContext context) {
}
protected Explicit coerce;
+ protected Explicit ignoreMalformed;
- public GeoShapeFieldMapper(String simpleName, MappedFieldType fieldType, Explicit coerce, Settings indexSettings,
- MultiFields multiFields, CopyTo copyTo) {
+ public GeoShapeFieldMapper(String simpleName, MappedFieldType fieldType, Explicit ignoreMalformed,
+ Explicit coerce, Settings indexSettings, MultiFields multiFields, CopyTo copyTo) {
super(simpleName, fieldType, Defaults.FIELD_TYPE, indexSettings, multiFields, copyTo);
this.coerce = coerce;
+ this.ignoreMalformed = ignoreMalformed;
}
@Override
@@ -445,7 +469,7 @@ public Mapper parse(ParseContext context) throws IOException {
try {
Shape shape = context.parseExternalValue(Shape.class);
if (shape == null) {
- ShapeBuilder shapeBuilder = ShapeBuilder.parse(context.parser(), this);
+ ShapeBuilder shapeBuilder = ShapeParser.parse(context.parser(), this);
if (shapeBuilder == null) {
return null;
}
@@ -453,7 +477,7 @@ public Mapper parse(ParseContext context) throws IOException {
}
if (fieldType().pointsOnly() && !(shape instanceof Point)) {
throw new MapperParsingException("[{" + fieldType().name() + "}] is configured for points only but a " +
- ((shape instanceof JtsGeometry) ? ((JtsGeometry)shape).getGeom().getGeometryType() : shape.getClass()) + " was found");
+ ((shape instanceof JtsGeometry) ? ((JtsGeometry) shape).getGeom().getGeometryType() : shape.getClass()) + " was found");
}
List fields = new ArrayList<>(Arrays.asList(fieldType().defaultStrategy().createIndexableFields(shape)));
createFieldNamesField(context, fields);
@@ -461,7 +485,9 @@ public Mapper parse(ParseContext context) throws IOException {
context.doc().add(field);
}
} catch (Exception e) {
- throw new MapperParsingException("failed to parse [" + fieldType().name() + "]", e);
+ if (ignoreMalformed.value() == false) {
+ throw new MapperParsingException("failed to parse [" + fieldType().name() + "]", e);
+ }
}
return null;
}
@@ -478,6 +504,9 @@ protected void doMerge(Mapper mergeWith, boolean updateAllTypes) {
if (gsfm.coerce.explicit()) {
this.coerce = gsfm.coerce;
}
+ if (gsfm.ignoreMalformed.explicit()) {
+ this.ignoreMalformed = gsfm.ignoreMalformed;
+ }
}
@Override
@@ -506,7 +535,10 @@ protected void doXContentBody(XContentBuilder builder, boolean includeDefaults,
builder.field(Names.STRATEGY_POINTS_ONLY, fieldType().pointsOnly());
}
if (includeDefaults || coerce.explicit()) {
- builder.field("coerce", coerce.value());
+ builder.field(Names.COERCE, coerce.value());
+ }
+ if (includeDefaults || ignoreMalformed.explicit()) {
+ builder.field(IGNORE_MALFORMED, ignoreMalformed.value());
}
}
@@ -514,6 +546,10 @@ public Explicit coerce() {
return coerce;
}
+ public Explicit ignoreMalformed() {
+ return ignoreMalformed;
+ }
+
@Override
protected String contentType() {
return CONTENT_TYPE;
diff --git a/core/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java
index 0aef7a3528ea1..0424cf6f14bb8 100644
--- a/core/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java
@@ -39,6 +39,7 @@
import org.elasticsearch.common.geo.ShapesAvailability;
import org.elasticsearch.common.geo.SpatialStrategy;
import org.elasticsearch.common.geo.builders.ShapeBuilder;
+import org.elasticsearch.common.geo.parsers.ShapeParser;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
@@ -410,7 +411,7 @@ public void onResponse(GetResponse response) {
if (pathElements[currentPathSlot].equals(parser.currentName())) {
parser.nextToken();
if (++currentPathSlot == pathElements.length) {
- listener.onResponse(ShapeBuilder.parse(parser));
+ listener.onResponse(ShapeParser.parse(parser));
}
} else {
parser.nextToken();
@@ -517,7 +518,7 @@ public static GeoShapeQueryBuilder fromXContent(XContentParser parser) throws IO
currentFieldName = parser.currentName();
token = parser.nextToken();
if (SHAPE_FIELD.match(currentFieldName)) {
- shape = ShapeBuilder.parse(parser);
+ shape = ShapeParser.parse(parser);
} else if (STRATEGY_FIELD.match(currentFieldName)) {
String strategyName = parser.text();
strategy = SpatialStrategy.fromString(strategyName);
diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesService.java b/core/src/main/java/org/elasticsearch/indices/IndicesService.java
index bfe69097b7bac..0caf9ba35089f 100644
--- a/core/src/main/java/org/elasticsearch/indices/IndicesService.java
+++ b/core/src/main/java/org/elasticsearch/indices/IndicesService.java
@@ -1111,6 +1111,12 @@ public void close() {
* Can the shard request be cached at all?
*/
public boolean canCache(ShardSearchRequest request, SearchContext context) {
+ // Queries that create a scroll context cannot use the cache.
+ // They modify the search context during their execution so using the cache
+ // may invalidate the scroll for the next query.
+ if (request.scroll() != null) {
+ return false;
+ }
// We cannot cache with DFS because results depend not only on the content of the index but also
// on the overridden statistics. So if you ran two queries on the same index with different stats
@@ -1119,6 +1125,7 @@ public boolean canCache(ShardSearchRequest request, SearchContext context) {
if (SearchType.QUERY_THEN_FETCH != context.searchType()) {
return false;
}
+
IndexSettings settings = context.indexShard().indexSettings();
// if not explicitly set in the request, use the index setting, if not, use the request
if (request.requestCache() == null) {
diff --git a/core/src/main/java/org/elasticsearch/search/SearchModule.java b/core/src/main/java/org/elasticsearch/search/SearchModule.java
index 6a56624730cf7..dd7cf962d6cd7 100644
--- a/core/src/main/java/org/elasticsearch/search/SearchModule.java
+++ b/core/src/main/java/org/elasticsearch/search/SearchModule.java
@@ -21,8 +21,8 @@
import org.apache.lucene.search.BooleanQuery;
import org.elasticsearch.common.NamedRegistry;
+import org.elasticsearch.common.geo.GeoShapeType;
import org.elasticsearch.common.geo.ShapesAvailability;
-import org.elasticsearch.common.geo.builders.ShapeBuilders;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry.Entry;
import org.elasticsearch.common.io.stream.Writeable;
@@ -250,6 +250,7 @@
import org.elasticsearch.search.suggest.term.TermSuggestionBuilder;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.function.Consumer;
@@ -531,7 +532,7 @@ private void registerPipelineAggregation(PipelineAggregationSpec spec) {
private void registerShapes() {
if (ShapesAvailability.JTS_AVAILABLE && ShapesAvailability.SPATIAL4J_AVAILABLE) {
- ShapeBuilders.register(namedWriteables);
+ namedWriteables.addAll(GeoShapeType.getShapeWriteables());
}
}
diff --git a/core/src/main/resources/org/elasticsearch/bootstrap/security.policy b/core/src/main/resources/org/elasticsearch/bootstrap/security.policy
index 7268e0f72380b..603e95c3102d0 100644
--- a/core/src/main/resources/org/elasticsearch/bootstrap/security.policy
+++ b/core/src/main/resources/org/elasticsearch/bootstrap/security.policy
@@ -24,14 +24,14 @@
//// SecurityManager impl:
//// Must have all permissions to properly perform access checks
-grant codeBase "${codebase.securesm-1.1.jar}" {
+grant codeBase "${codebase.securesm}" {
permission java.security.AllPermission;
};
//// Very special jar permissions:
//// These are dangerous permissions that we don't want to grant to everything.
-grant codeBase "${codebase.lucene-core-7.1.0.jar}" {
+grant codeBase "${codebase.lucene-core}" {
// needed to allow MMapDirectory's "unmap hack" (die unmap hack, die)
// java 8 package
permission java.lang.RuntimePermission "accessClassInPackage.sun.misc";
@@ -42,7 +42,7 @@ grant codeBase "${codebase.lucene-core-7.1.0.jar}" {
permission java.lang.RuntimePermission "accessDeclaredMembers";
};
-grant codeBase "${codebase.lucene-misc-7.1.0.jar}" {
+grant codeBase "${codebase.lucene-misc}" {
// needed to allow shard shrinking to use hard-links if possible via lucenes HardlinkCopyDirectoryWrapper
permission java.nio.file.LinkPermission "hard";
};
diff --git a/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy b/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
index 453621b138e0a..539587c409d42 100644
--- a/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
+++ b/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
@@ -21,7 +21,7 @@
//// These are mock objects and test management that we allow test framework libs
//// to provide on our behalf. But tests themselves cannot do this stuff!
-grant codeBase "${codebase.securemock-1.2.jar}" {
+grant codeBase "${codebase.securemock}" {
// needed to access ReflectionFactory (see below)
permission java.lang.RuntimePermission "accessClassInPackage.sun.reflect";
// needed for reflection in ibm jdk
@@ -33,7 +33,7 @@ grant codeBase "${codebase.securemock-1.2.jar}" {
permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
};
-grant codeBase "${codebase.lucene-test-framework-7.1.0.jar}" {
+grant codeBase "${codebase.lucene-test-framework}" {
// needed by RamUsageTester
permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
// needed for testing hardlinks in StoreRecoveryTests since we install MockFS
@@ -42,7 +42,7 @@ grant codeBase "${codebase.lucene-test-framework-7.1.0.jar}" {
permission java.lang.RuntimePermission "accessDeclaredMembers";
};
-grant codeBase "${codebase.randomizedtesting-runner-2.5.2.jar}" {
+grant codeBase "${codebase.randomizedtesting-runner}" {
// optionally needed for access to private test methods (e.g. beforeClass)
permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
// needed to fail tests on uncaught exceptions from other threads
@@ -53,12 +53,12 @@ grant codeBase "${codebase.randomizedtesting-runner-2.5.2.jar}" {
permission java.lang.RuntimePermission "accessDeclaredMembers";
};
-grant codeBase "${codebase.junit-4.12.jar}" {
+grant codeBase "${codebase.junit}" {
// needed for TestClass creation
permission java.lang.RuntimePermission "accessDeclaredMembers";
};
-grant codeBase "${codebase.mocksocket-1.2.jar}" {
+grant codeBase "${codebase.mocksocket}" {
// mocksocket makes and accepts socket connections
permission java.net.SocketPermission "*", "accept,connect";
};
@@ -70,12 +70,12 @@ grant codeBase "${codebase.elasticsearch-rest-client}" {
permission java.net.NetPermission "getProxySelector";
};
-grant codeBase "${codebase.httpcore-nio-4.4.5.jar}" {
+grant codeBase "${codebase.httpcore-nio}" {
// httpcore makes socket connections for rest tests
permission java.net.SocketPermission "*", "connect";
};
-grant codeBase "${codebase.httpasyncclient-4.1.2.jar}" {
+grant codeBase "${codebase.httpasyncclient}" {
// httpasyncclient makes socket connections for rest tests
permission java.net.SocketPermission "*", "connect";
// rest client uses system properties which gets the default proxy
diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/IndexGraveyardTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/IndexGraveyardTests.java
index 9178d112acade..ef801dad28eb4 100644
--- a/core/src/test/java/org/elasticsearch/cluster/metadata/IndexGraveyardTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/metadata/IndexGraveyardTests.java
@@ -19,6 +19,7 @@
package org.elasticsearch.cluster.metadata;
+import org.elasticsearch.common.Strings;
import org.elasticsearch.common.UUIDs;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.common.settings.Settings;
@@ -36,6 +37,7 @@
import java.util.List;
import java.util.stream.Collectors;
+import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.lessThan;
import static org.hamcrest.Matchers.not;
@@ -66,6 +68,11 @@ public void testXContent() throws IOException {
builder.startObject();
graveyard.toXContent(builder, ToXContent.EMPTY_PARAMS);
builder.endObject();
+ if (graveyard.getTombstones().size() > 0) {
+ // check that date properly printed
+ assertThat(Strings.toString(graveyard, false, true),
+ containsString(XContentBuilder.DEFAULT_DATE_PRINTER.print(graveyard.getTombstones().get(0).getDeleteDateInMillis())));
+ }
XContentParser parser = createParser(JsonXContent.jsonXContent, builder.bytes());
parser.nextToken(); // the beginning of the parser
assertThat(IndexGraveyard.fromXContent(parser), equalTo(graveyard));
diff --git a/core/src/test/java/org/elasticsearch/common/geo/BaseGeoParsingTestCase.java b/core/src/test/java/org/elasticsearch/common/geo/BaseGeoParsingTestCase.java
new file mode 100644
index 0000000000000..4c2f58cf6c336
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/common/geo/BaseGeoParsingTestCase.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.common.geo;
+
+import com.vividsolutions.jts.geom.GeometryFactory;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+
+import static org.elasticsearch.common.geo.builders.ShapeBuilder.SPATIAL_CONTEXT;
+
+/**
+ * Created by nknize on 9/22/17.
+ */
+abstract class BaseGeoParsingTestCase extends ESTestCase {
+ protected static final GeometryFactory GEOMETRY_FACTORY = SPATIAL_CONTEXT.getGeometryFactory();
+
+ public abstract void testParsePoint() throws IOException;
+ public abstract void testParseMultiPoint() throws IOException;
+ public abstract void testParseLineString() throws IOException;
+ public abstract void testParseMultiLineString() throws IOException;
+ public abstract void testParsePolygon() throws IOException;
+ public abstract void testParseMultiPolygon() throws IOException;
+}
diff --git a/core/src/test/java/org/elasticsearch/common/geo/GeoJSONShapeParserTests.java b/core/src/test/java/org/elasticsearch/common/geo/GeoJsonShapeParserTests.java
similarity index 97%
rename from core/src/test/java/org/elasticsearch/common/geo/GeoJSONShapeParserTests.java
rename to core/src/test/java/org/elasticsearch/common/geo/GeoJsonShapeParserTests.java
index 5ac50b91bd3f5..32f384d96b118 100644
--- a/core/src/test/java/org/elasticsearch/common/geo/GeoJSONShapeParserTests.java
+++ b/core/src/test/java/org/elasticsearch/common/geo/GeoJsonShapeParserTests.java
@@ -21,7 +21,6 @@
import com.vividsolutions.jts.geom.Coordinate;
import com.vividsolutions.jts.geom.Geometry;
-import com.vividsolutions.jts.geom.GeometryFactory;
import com.vividsolutions.jts.geom.LineString;
import com.vividsolutions.jts.geom.LinearRing;
import com.vividsolutions.jts.geom.MultiLineString;
@@ -29,12 +28,11 @@
import com.vividsolutions.jts.geom.Polygon;
import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.common.geo.builders.ShapeBuilder;
+import org.elasticsearch.common.geo.parsers.ShapeParser;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.json.JsonXContent;
-import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.test.hamcrest.ElasticsearchGeoAssertions;
import org.locationtech.spatial4j.exception.InvalidShapeException;
import org.locationtech.spatial4j.shape.Circle;
@@ -55,11 +53,10 @@
/**
* Tests for {@code GeoJSONShapeParser}
*/
-public class GeoJSONShapeParserTests extends ESTestCase {
+public class GeoJsonShapeParserTests extends BaseGeoParsingTestCase {
- private static final GeometryFactory GEOMETRY_FACTORY = SPATIAL_CONTEXT.getGeometryFactory();
-
- public void testParseSimplePoint() throws IOException {
+ @Override
+ public void testParsePoint() throws IOException {
XContentBuilder pointGeoJson = XContentFactory.jsonBuilder()
.startObject()
.field("type", "Point")
@@ -70,6 +67,7 @@ public void testParseSimplePoint() throws IOException {
assertGeometryEquals(new JtsPoint(expected, SPATIAL_CONTEXT), pointGeoJson);
}
+ @Override
public void testParseLineString() throws IOException {
XContentBuilder lineGeoJson = XContentFactory.jsonBuilder()
.startObject()
@@ -89,6 +87,7 @@ public void testParseLineString() throws IOException {
assertGeometryEquals(jtsGeom(expected), lineGeoJson);
}
+ @Override
public void testParseMultiLineString() throws IOException {
XContentBuilder multilinesGeoJson = XContentFactory.jsonBuilder()
.startObject()
@@ -205,7 +204,8 @@ public void testParseEnvelope() throws IOException {
ElasticsearchGeoAssertions.assertValidException(parser, ElasticsearchParseException.class);
}
- public void testParsePolygonNoHoles() throws IOException {
+ @Override
+ public void testParsePolygon() throws IOException {
XContentBuilder polygonGeoJson = XContentFactory.jsonBuilder()
.startObject()
.field("type", "Polygon")
@@ -344,7 +344,7 @@ public void testParseOGCPolygonWithoutHoles() throws IOException {
XContentParser parser = createParser(JsonXContent.jsonXContent, polygonGeoJson);
parser.nextToken();
- Shape shape = ShapeBuilder.parse(parser).build();
+ Shape shape = ShapeParser.parse(parser).build();
ElasticsearchGeoAssertions.assertPolygon(shape);
@@ -364,7 +364,7 @@ public void testParseOGCPolygonWithoutHoles() throws IOException {
parser = createParser(JsonXContent.jsonXContent, polygonGeoJson);
parser.nextToken();
- shape = ShapeBuilder.parse(parser).build();
+ shape = ShapeParser.parse(parser).build();
ElasticsearchGeoAssertions.assertMultiPolygon(shape);
@@ -384,7 +384,7 @@ public void testParseOGCPolygonWithoutHoles() throws IOException {
parser = createParser(JsonXContent.jsonXContent, polygonGeoJson);
parser.nextToken();
- shape = ShapeBuilder.parse(parser).build();
+ shape = ShapeParser.parse(parser).build();
ElasticsearchGeoAssertions.assertPolygon(shape);
@@ -404,7 +404,7 @@ public void testParseOGCPolygonWithoutHoles() throws IOException {
parser = createParser(JsonXContent.jsonXContent, polygonGeoJson);
parser.nextToken();
- shape = ShapeBuilder.parse(parser).build();
+ shape = ShapeParser.parse(parser).build();
ElasticsearchGeoAssertions.assertMultiPolygon(shape);
}
@@ -432,7 +432,7 @@ public void testParseOGCPolygonWithHoles() throws IOException {
XContentParser parser = createParser(JsonXContent.jsonXContent, polygonGeoJson);
parser.nextToken();
- Shape shape = ShapeBuilder.parse(parser).build();
+ Shape shape = ShapeParser.parse(parser).build();
ElasticsearchGeoAssertions.assertPolygon(shape);
@@ -458,7 +458,7 @@ public void testParseOGCPolygonWithHoles() throws IOException {
parser = createParser(JsonXContent.jsonXContent, polygonGeoJson);
parser.nextToken();
- shape = ShapeBuilder.parse(parser).build();
+ shape = ShapeParser.parse(parser).build();
ElasticsearchGeoAssertions.assertMultiPolygon(shape);
@@ -484,7 +484,7 @@ public void testParseOGCPolygonWithHoles() throws IOException {
parser = createParser(JsonXContent.jsonXContent, polygonGeoJson);
parser.nextToken();
- shape = ShapeBuilder.parse(parser).build();
+ shape = ShapeParser.parse(parser).build();
ElasticsearchGeoAssertions.assertPolygon(shape);
@@ -510,7 +510,7 @@ public void testParseOGCPolygonWithHoles() throws IOException {
parser = createParser(JsonXContent.jsonXContent, polygonGeoJson);
parser.nextToken();
- shape = ShapeBuilder.parse(parser).build();
+ shape = ShapeParser.parse(parser).build();
ElasticsearchGeoAssertions.assertMultiPolygon(shape);
}
@@ -671,6 +671,7 @@ public void testParseSelfCrossingPolygon() throws IOException {
ElasticsearchGeoAssertions.assertValidException(parser, InvalidShapeException.class);
}
+ @Override
public void testParseMultiPoint() throws IOException {
XContentBuilder multiPointGeoJson = XContentFactory.jsonBuilder()
.startObject()
@@ -687,6 +688,7 @@ public void testParseMultiPoint() throws IOException {
assertGeometryEquals(expected, multiPointGeoJson);
}
+ @Override
public void testParseMultiPolygon() throws IOException {
// test #1: two polygons; one without hole, one with hole
XContentBuilder multiPolygonGeoJson = XContentFactory.jsonBuilder()
@@ -882,7 +884,7 @@ public void testParseOrientationOption() throws IOException {
XContentParser parser = createParser(polygonGeoJson);
parser.nextToken();
- Shape shape = ShapeBuilder.parse(parser).build();
+ Shape shape = ShapeParser.parse(parser).build();
ElasticsearchGeoAssertions.assertPolygon(shape);
@@ -911,7 +913,7 @@ public void testParseOrientationOption() throws IOException {
parser = createParser(polygonGeoJson);
parser.nextToken();
- shape = ShapeBuilder.parse(parser).build();
+ shape = ShapeParser.parse(parser).build();
ElasticsearchGeoAssertions.assertPolygon(shape);
@@ -940,7 +942,7 @@ public void testParseOrientationOption() throws IOException {
parser = createParser(polygonGeoJson);
parser.nextToken();
- shape = ShapeBuilder.parse(parser).build();
+ shape = ShapeParser.parse(parser).build();
ElasticsearchGeoAssertions.assertPolygon(shape);
@@ -969,7 +971,7 @@ public void testParseOrientationOption() throws IOException {
parser = createParser(polygonGeoJson);
parser.nextToken();
- shape = ShapeBuilder.parse(parser).build();
+ shape = ShapeParser.parse(parser).build();
ElasticsearchGeoAssertions.assertMultiPolygon(shape);
@@ -998,7 +1000,7 @@ public void testParseOrientationOption() throws IOException {
parser = createParser(polygonGeoJson);
parser.nextToken();
- shape = ShapeBuilder.parse(parser).build();
+ shape = ShapeParser.parse(parser).build();
ElasticsearchGeoAssertions.assertMultiPolygon(shape);
@@ -1027,7 +1029,7 @@ public void testParseOrientationOption() throws IOException {
parser = createParser(polygonGeoJson);
parser.nextToken();
- shape = ShapeBuilder.parse(parser).build();
+ shape = ShapeParser.parse(parser).build();
ElasticsearchGeoAssertions.assertMultiPolygon(shape);
}
@@ -1035,7 +1037,7 @@ public void testParseOrientationOption() throws IOException {
private void assertGeometryEquals(Shape expected, XContentBuilder geoJson) throws IOException {
XContentParser parser = createParser(geoJson);
parser.nextToken();
- ElasticsearchGeoAssertions.assertEquals(expected, ShapeBuilder.parse(parser).build());
+ ElasticsearchGeoAssertions.assertEquals(expected, ShapeParser.parse(parser).build());
}
private ShapeCollection shapeCollection(Shape... shapes) {
diff --git a/core/src/test/java/org/elasticsearch/common/geo/ShapeBuilderTests.java b/core/src/test/java/org/elasticsearch/common/geo/ShapeBuilderTests.java
index dc01a39cb8145..d1f7d5601a6cc 100644
--- a/core/src/test/java/org/elasticsearch/common/geo/ShapeBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/common/geo/ShapeBuilderTests.java
@@ -24,10 +24,13 @@
import com.vividsolutions.jts.geom.Polygon;
import org.elasticsearch.common.geo.builders.CoordinatesBuilder;
+import org.elasticsearch.common.geo.builders.CircleBuilder;
+import org.elasticsearch.common.geo.builders.EnvelopeBuilder;
import org.elasticsearch.common.geo.builders.LineStringBuilder;
+import org.elasticsearch.common.geo.builders.MultiLineStringBuilder;
+import org.elasticsearch.common.geo.builders.PointBuilder;
import org.elasticsearch.common.geo.builders.PolygonBuilder;
import org.elasticsearch.common.geo.builders.ShapeBuilder;
-import org.elasticsearch.common.geo.builders.ShapeBuilders;
import org.elasticsearch.test.ESTestCase;
import org.locationtech.spatial4j.exception.InvalidShapeException;
import org.locationtech.spatial4j.shape.Circle;
@@ -46,13 +49,13 @@
public class ShapeBuilderTests extends ESTestCase {
public void testNewPoint() {
- Point point = ShapeBuilders.newPoint(-100, 45).build();
+ Point point = new PointBuilder().coordinate(-100, 45).build();
assertEquals(-100D, point.getX(), 0.0d);
assertEquals(45D, point.getY(), 0.0d);
}
public void testNewRectangle() {
- Rectangle rectangle = ShapeBuilders.newEnvelope(new Coordinate(-45, 30), new Coordinate(45, -30)).build();
+ Rectangle rectangle = new EnvelopeBuilder(new Coordinate(-45, 30), new Coordinate(45, -30)).build();
assertEquals(-45D, rectangle.getMinX(), 0.0d);
assertEquals(-30D, rectangle.getMinY(), 0.0d);
assertEquals(45D, rectangle.getMaxX(), 0.0d);
@@ -60,7 +63,7 @@ public void testNewRectangle() {
}
public void testNewPolygon() {
- Polygon polygon = ShapeBuilders.newPolygon(new CoordinatesBuilder()
+ Polygon polygon = new PolygonBuilder(new CoordinatesBuilder()
.coordinate(-45, 30)
.coordinate(45, 30)
.coordinate(45, -30)
@@ -75,7 +78,7 @@ public void testNewPolygon() {
}
public void testNewPolygon_coordinate() {
- Polygon polygon = ShapeBuilders.newPolygon(new CoordinatesBuilder()
+ Polygon polygon = new PolygonBuilder(new CoordinatesBuilder()
.coordinate(new Coordinate(-45, 30))
.coordinate(new Coordinate(45, 30))
.coordinate(new Coordinate(45, -30))
@@ -90,7 +93,7 @@ public void testNewPolygon_coordinate() {
}
public void testNewPolygon_coordinates() {
- Polygon polygon = ShapeBuilders.newPolygon(new CoordinatesBuilder()
+ Polygon polygon = new PolygonBuilder(new CoordinatesBuilder()
.coordinates(new Coordinate(-45, 30), new Coordinate(45, 30), new Coordinate(45, -30), new Coordinate(-45, -30), new Coordinate(-45, 30))
).toPolygon();
@@ -103,7 +106,7 @@ public void testNewPolygon_coordinates() {
public void testLineStringBuilder() {
// Building a simple LineString
- ShapeBuilders.newLineString(new CoordinatesBuilder()
+ new LineStringBuilder(new CoordinatesBuilder()
.coordinate(-130.0, 55.0)
.coordinate(-130.0, -40.0)
.coordinate(-15.0, -40.0)
@@ -114,7 +117,7 @@ public void testLineStringBuilder() {
.coordinate(-110.0, 55.0)).build();
// Building a linestring that needs to be wrapped
- ShapeBuilders.newLineString(new CoordinatesBuilder()
+ new LineStringBuilder(new CoordinatesBuilder()
.coordinate(100.0, 50.0)
.coordinate(110.0, -40.0)
.coordinate(240.0, -40.0)
@@ -127,7 +130,7 @@ public void testLineStringBuilder() {
.build();
// Building a lineString on the dateline
- ShapeBuilders.newLineString(new CoordinatesBuilder()
+ new LineStringBuilder(new CoordinatesBuilder()
.coordinate(-180.0, 80.0)
.coordinate(-180.0, 40.0)
.coordinate(-180.0, -40.0)
@@ -136,7 +139,7 @@ public void testLineStringBuilder() {
.build();
// Building a lineString on the dateline
- ShapeBuilders.newLineString(new CoordinatesBuilder()
+ new LineStringBuilder(new CoordinatesBuilder()
.coordinate(180.0, 80.0)
.coordinate(180.0, 40.0)
.coordinate(180.0, -40.0)
@@ -146,7 +149,7 @@ public void testLineStringBuilder() {
}
public void testMultiLineString() {
- ShapeBuilders.newMultiLinestring()
+ new MultiLineStringBuilder()
.linestring(new LineStringBuilder(new CoordinatesBuilder()
.coordinate(-100.0, 50.0)
.coordinate(50.0, 50.0)
@@ -164,7 +167,7 @@ public void testMultiLineString() {
.build();
// LineString that needs to be wrapped
- ShapeBuilders.newMultiLinestring()
+ new MultiLineStringBuilder()
.linestring(new LineStringBuilder(new CoordinatesBuilder()
.coordinate(150.0, 60.0)
.coordinate(200.0, 60.0)
@@ -183,7 +186,7 @@ public void testMultiLineString() {
}
public void testPolygonSelfIntersection() {
- PolygonBuilder newPolygon = ShapeBuilders.newPolygon(new CoordinatesBuilder()
+ PolygonBuilder newPolygon = new PolygonBuilder(new CoordinatesBuilder()
.coordinate(-40.0, 50.0)
.coordinate(40.0, 50.0)
.coordinate(-40.0, -50.0)
@@ -194,31 +197,31 @@ public void testPolygonSelfIntersection() {
public void testGeoCircle() {
double earthCircumference = 40075016.69;
- Circle circle = ShapeBuilders.newCircleBuilder().center(0, 0).radius("100m").build();
+ Circle circle = new CircleBuilder().center(0, 0).radius("100m").build();
assertEquals((360 * 100) / earthCircumference, circle.getRadius(), 0.00000001);
assertEquals(new PointImpl(0, 0, ShapeBuilder.SPATIAL_CONTEXT), circle.getCenter());
- circle = ShapeBuilders.newCircleBuilder().center(+180, 0).radius("100m").build();
+ circle = new CircleBuilder().center(+180, 0).radius("100m").build();
assertEquals((360 * 100) / earthCircumference, circle.getRadius(), 0.00000001);
assertEquals(new PointImpl(180, 0, ShapeBuilder.SPATIAL_CONTEXT), circle.getCenter());
- circle = ShapeBuilders.newCircleBuilder().center(-180, 0).radius("100m").build();
+ circle = new CircleBuilder().center(-180, 0).radius("100m").build();
assertEquals((360 * 100) / earthCircumference, circle.getRadius(), 0.00000001);
assertEquals(new PointImpl(-180, 0, ShapeBuilder.SPATIAL_CONTEXT), circle.getCenter());
- circle = ShapeBuilders.newCircleBuilder().center(0, 90).radius("100m").build();
+ circle = new CircleBuilder().center(0, 90).radius("100m").build();
assertEquals((360 * 100) / earthCircumference, circle.getRadius(), 0.00000001);
assertEquals(new PointImpl(0, 90, ShapeBuilder.SPATIAL_CONTEXT), circle.getCenter());
- circle = ShapeBuilders.newCircleBuilder().center(0, -90).radius("100m").build();
+ circle = new CircleBuilder().center(0, -90).radius("100m").build();
assertEquals((360 * 100) / earthCircumference, circle.getRadius(), 0.00000001);
assertEquals(new PointImpl(0, -90, ShapeBuilder.SPATIAL_CONTEXT), circle.getCenter());
double randomLat = (randomDouble() * 180) - 90;
double randomLon = (randomDouble() * 360) - 180;
double randomRadius = randomIntBetween(1, (int) earthCircumference / 4);
- circle = ShapeBuilders.newCircleBuilder().center(randomLon, randomLat).radius(randomRadius + "m").build();
+ circle = new CircleBuilder().center(randomLon, randomLat).radius(randomRadius + "m").build();
assertEquals((360 * randomRadius) / earthCircumference, circle.getRadius(), 0.00000001);
assertEquals(new PointImpl(randomLon, randomLat, ShapeBuilder.SPATIAL_CONTEXT), circle.getCenter());
}
public void testPolygonWrapping() {
- Shape shape = ShapeBuilders.newPolygon(new CoordinatesBuilder()
+ Shape shape = new PolygonBuilder(new CoordinatesBuilder()
.coordinate(-150.0, 65.0)
.coordinate(-250.0, 65.0)
.coordinate(-250.0, -65.0)
@@ -231,7 +234,7 @@ public void testPolygonWrapping() {
}
public void testLineStringWrapping() {
- Shape shape = ShapeBuilders.newLineString(new CoordinatesBuilder()
+ Shape shape = new LineStringBuilder(new CoordinatesBuilder()
.coordinate(-150.0, 65.0)
.coordinate(-250.0, 65.0)
.coordinate(-250.0, -65.0)
@@ -248,7 +251,7 @@ public void testDatelineOGC() {
// expected results: 3 polygons, 1 with a hole
// a giant c shape
- PolygonBuilder builder = ShapeBuilders.newPolygon(new CoordinatesBuilder()
+ PolygonBuilder builder = new PolygonBuilder(new CoordinatesBuilder()
.coordinate(174,0)
.coordinate(-176,0)
.coordinate(-176,3)
@@ -292,7 +295,7 @@ public void testDateline() {
// expected results: 3 polygons, 1 with a hole
// a giant c shape
- PolygonBuilder builder = ShapeBuilders.newPolygon(new CoordinatesBuilder()
+ PolygonBuilder builder = new PolygonBuilder(new CoordinatesBuilder()
.coordinate(-186,0)
.coordinate(-176,0)
.coordinate(-176,3)
@@ -331,7 +334,7 @@ public void testDateline() {
}
public void testComplexShapeWithHole() {
- PolygonBuilder builder = ShapeBuilders.newPolygon(new CoordinatesBuilder()
+ PolygonBuilder builder = new PolygonBuilder(new CoordinatesBuilder()
.coordinate(-85.0018514,37.1311314)
.coordinate(-85.0016645,37.1315293)
.coordinate(-85.0016246,37.1317069)
@@ -407,7 +410,7 @@ public void testComplexShapeWithHole() {
}
public void testShapeWithHoleAtEdgeEndPoints() {
- PolygonBuilder builder = ShapeBuilders.newPolygon(new CoordinatesBuilder()
+ PolygonBuilder builder = new PolygonBuilder(new CoordinatesBuilder()
.coordinate(-4, 2)
.coordinate(4, 2)
.coordinate(6, 0)
@@ -430,7 +433,7 @@ public void testShapeWithHoleAtEdgeEndPoints() {
}
public void testShapeWithPointOnDateline() {
- PolygonBuilder builder = ShapeBuilders.newPolygon(new CoordinatesBuilder()
+ PolygonBuilder builder = new PolygonBuilder(new CoordinatesBuilder()
.coordinate(180, 0)
.coordinate(176, 4)
.coordinate(176, -4)
@@ -443,7 +446,7 @@ public void testShapeWithPointOnDateline() {
public void testShapeWithEdgeAlongDateline() {
// test case 1: test the positive side of the dateline
- PolygonBuilder builder = ShapeBuilders.newPolygon(new CoordinatesBuilder()
+ PolygonBuilder builder = new PolygonBuilder(new CoordinatesBuilder()
.coordinate(180, 0)
.coordinate(176, 4)
.coordinate(180, -4)
@@ -454,7 +457,7 @@ public void testShapeWithEdgeAlongDateline() {
assertPolygon(shape);
// test case 2: test the negative side of the dateline
- builder = ShapeBuilders.newPolygon(new CoordinatesBuilder()
+ builder = new PolygonBuilder(new CoordinatesBuilder()
.coordinate(-176, 4)
.coordinate(-180, 0)
.coordinate(-180, -4)
@@ -467,7 +470,7 @@ public void testShapeWithEdgeAlongDateline() {
public void testShapeWithBoundaryHoles() {
// test case 1: test the positive side of the dateline
- PolygonBuilder builder = ShapeBuilders.newPolygon(new CoordinatesBuilder()
+ PolygonBuilder builder = new PolygonBuilder(new CoordinatesBuilder()
.coordinate(-177, 10)
.coordinate(176, 15)
.coordinate(172, 0)
@@ -486,7 +489,7 @@ public void testShapeWithBoundaryHoles() {
assertMultiPolygon(shape);
// test case 2: test the negative side of the dateline
- builder = ShapeBuilders.newPolygon(
+ builder = new PolygonBuilder(
new CoordinatesBuilder()
.coordinate(-176, 15)
.coordinate(179, 10)
@@ -510,7 +513,7 @@ public void testShapeWithBoundaryHoles() {
public void testShapeWithTangentialHole() {
// test a shape with one tangential (shared) vertex (should pass)
- PolygonBuilder builder = ShapeBuilders.newPolygon(new CoordinatesBuilder()
+ PolygonBuilder builder = new PolygonBuilder(new CoordinatesBuilder()
.coordinate(179, 10)
.coordinate(168, 15)
.coordinate(164, 0)
@@ -531,7 +534,7 @@ public void testShapeWithTangentialHole() {
public void testShapeWithInvalidTangentialHole() {
// test a shape with one invalid tangential (shared) vertex (should throw exception)
- PolygonBuilder builder = ShapeBuilders.newPolygon(new CoordinatesBuilder()
+ PolygonBuilder builder = new PolygonBuilder(new CoordinatesBuilder()
.coordinate(179, 10)
.coordinate(168, 15)
.coordinate(164, 0)
@@ -552,7 +555,7 @@ public void testShapeWithInvalidTangentialHole() {
public void testBoundaryShapeWithTangentialHole() {
// test a shape with one tangential (shared) vertex for each hole (should pass)
- PolygonBuilder builder = ShapeBuilders.newPolygon(new CoordinatesBuilder()
+ PolygonBuilder builder = new PolygonBuilder(new CoordinatesBuilder()
.coordinate(-177, 10)
.coordinate(176, 15)
.coordinate(172, 0)
@@ -579,7 +582,7 @@ public void testBoundaryShapeWithTangentialHole() {
public void testBoundaryShapeWithInvalidTangentialHole() {
// test shape with two tangential (shared) vertices (should throw exception)
- PolygonBuilder builder = ShapeBuilders.newPolygon(new CoordinatesBuilder()
+ PolygonBuilder builder = new PolygonBuilder(new CoordinatesBuilder()
.coordinate(-177, 10)
.coordinate(176, 15)
.coordinate(172, 0)
@@ -602,7 +605,7 @@ public void testBoundaryShapeWithInvalidTangentialHole() {
* Test an enveloping polygon around the max mercator bounds
*/
public void testBoundaryShape() {
- PolygonBuilder builder = ShapeBuilders.newPolygon(new CoordinatesBuilder()
+ PolygonBuilder builder = new PolygonBuilder(new CoordinatesBuilder()
.coordinate(-180, 90)
.coordinate(180, 90)
.coordinate(180, -90)
@@ -616,7 +619,7 @@ public void testBoundaryShape() {
public void testShapeWithAlternateOrientation() {
// cw: should produce a multi polygon spanning hemispheres
- PolygonBuilder builder = ShapeBuilders.newPolygon(new CoordinatesBuilder()
+ PolygonBuilder builder = new PolygonBuilder(new CoordinatesBuilder()
.coordinate(180, 0)
.coordinate(176, 4)
.coordinate(-176, 4)
@@ -627,7 +630,7 @@ public void testShapeWithAlternateOrientation() {
assertPolygon(shape);
// cw: geo core will convert to ccw across the dateline
- builder = ShapeBuilders.newPolygon(new CoordinatesBuilder()
+ builder = new PolygonBuilder(new CoordinatesBuilder()
.coordinate(180, 0)
.coordinate(-176, 4)
.coordinate(176, 4)
@@ -640,7 +643,7 @@ public void testShapeWithAlternateOrientation() {
}
public void testInvalidShapeWithConsecutiveDuplicatePoints() {
- PolygonBuilder builder = ShapeBuilders.newPolygon(new CoordinatesBuilder()
+ PolygonBuilder builder = new PolygonBuilder(new CoordinatesBuilder()
.coordinate(180, 0)
.coordinate(176, 4)
.coordinate(176, 4)
diff --git a/core/src/test/java/org/elasticsearch/common/geo/builders/AbstractShapeBuilderTestCase.java b/core/src/test/java/org/elasticsearch/common/geo/builders/AbstractShapeBuilderTestCase.java
index c3e4781b3d27c..5ac55832959d7 100644
--- a/core/src/test/java/org/elasticsearch/common/geo/builders/AbstractShapeBuilderTestCase.java
+++ b/core/src/test/java/org/elasticsearch/common/geo/builders/AbstractShapeBuilderTestCase.java
@@ -19,13 +19,14 @@
package org.elasticsearch.common.geo.builders;
+import org.elasticsearch.common.geo.GeoShapeType;
+import org.elasticsearch.common.geo.parsers.ShapeParser;
import org.elasticsearch.common.io.stream.NamedWriteable;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.common.io.stream.Writeable.Reader;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.test.ESTestCase;
@@ -33,8 +34,6 @@
import org.junit.BeforeClass;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode;
@@ -49,9 +48,7 @@ public abstract class AbstractShapeBuilderTestCase exte
@BeforeClass
public static void init() {
if (namedWriteableRegistry == null) {
- List shapes = new ArrayList<>();
- ShapeBuilders.register(shapes);
- namedWriteableRegistry = new NamedWriteableRegistry(shapes);
+ namedWriteableRegistry = new NamedWriteableRegistry(GeoShapeType.getShapeWriteables());
}
}
@@ -82,9 +79,9 @@ public void testFromXContent() throws IOException {
}
XContentBuilder builder = testShape.toXContent(contentBuilder, ToXContent.EMPTY_PARAMS);
XContentBuilder shuffled = shuffleXContent(builder);
- XContentParser shapeParser = createParser(shuffled);
- shapeParser.nextToken();
- ShapeBuilder parsedShape = ShapeBuilder.parse(shapeParser);
+ XContentParser shapeContentParser = createParser(shuffled);
+ shapeContentParser.nextToken();
+ ShapeBuilder parsedShape = ShapeParser.parse(shapeContentParser);
assertNotSame(testShape, parsedShape);
assertEquals(testShape, parsedShape);
assertEquals(testShape.hashCode(), parsedShape.hashCode());
diff --git a/core/src/test/java/org/elasticsearch/common/geo/builders/LineStringBuilderTests.java b/core/src/test/java/org/elasticsearch/common/geo/builders/LineStringBuilderTests.java
index be7772b0da60a..3b5f2662316ca 100644
--- a/core/src/test/java/org/elasticsearch/common/geo/builders/LineStringBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/common/geo/builders/LineStringBuilderTests.java
@@ -49,7 +49,7 @@ protected LineStringBuilder createMutation(LineStringBuilder original) throws IO
}
static LineStringBuilder mutate(LineStringBuilder original) throws IOException {
- LineStringBuilder mutation = (LineStringBuilder) copyShape(original);
+ LineStringBuilder mutation = copyShape(original);
Coordinate[] coordinates = original.coordinates(false);
Coordinate coordinate = randomFrom(coordinates);
if (randomBoolean()) {
@@ -65,7 +65,7 @@ static LineStringBuilder mutate(LineStringBuilder original) throws IOException {
coordinate.y = randomDoubleBetween(-90.0, 90.0, true);
}
}
- return mutation.coordinates(coordinates);
+ return LineStringBuilder.class.cast(mutation.coordinates(coordinates));
}
static LineStringBuilder createRandomShape() {
diff --git a/core/src/test/java/org/elasticsearch/common/geo/builders/MultiLineStringBuilderTests.java b/core/src/test/java/org/elasticsearch/common/geo/builders/MultiLineStringBuilderTests.java
index 9fbaed815be1d..b650939594077 100644
--- a/core/src/test/java/org/elasticsearch/common/geo/builders/MultiLineStringBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/common/geo/builders/MultiLineStringBuilderTests.java
@@ -68,6 +68,6 @@ static MultiLineStringBuilder mutate(MultiLineStringBuilder original) throws IOE
}
static MultiLineStringBuilder createRandomShape() {
- return new MultiLineStringBuilder();
+ return MultiLineStringBuilder.class.cast(RandomShapeGenerator.createShape(random(), ShapeType.MULTILINESTRING));
}
}
diff --git a/core/src/test/java/org/elasticsearch/common/geo/builders/MultiPointBuilderTests.java b/core/src/test/java/org/elasticsearch/common/geo/builders/MultiPointBuilderTests.java
index 14c1acc8c60ee..c0a799e1c306e 100644
--- a/core/src/test/java/org/elasticsearch/common/geo/builders/MultiPointBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/common/geo/builders/MultiPointBuilderTests.java
@@ -70,7 +70,7 @@ static MultiPointBuilder mutate(MultiPointBuilder original) throws IOException {
} else {
coordinates = new Coordinate[]{new Coordinate(1.0, 1.0)};
}
- return mutation.coordinates(coordinates);
+ return MultiPointBuilder.class.cast(mutation.coordinates(coordinates));
}
static MultiPointBuilder createRandomShape() {
diff --git a/core/src/test/java/org/elasticsearch/common/settings/ScopedSettingsTests.java b/core/src/test/java/org/elasticsearch/common/settings/ScopedSettingsTests.java
index bd4ac25a8747b..2015a6b42d16f 100644
--- a/core/src/test/java/org/elasticsearch/common/settings/ScopedSettingsTests.java
+++ b/core/src/test/java/org/elasticsearch/common/settings/ScopedSettingsTests.java
@@ -122,7 +122,7 @@ public void testResetSettingWithIPValidator() {
Settings.Builder builder = Settings.builder();
Settings updates = Settings.builder().putNull("index.routing.allocation.require._ip")
.put("index.some.dyn.setting", 1).build();
- settings.validate(updates);
+ settings.validate(updates, false);
settings.updateDynamicSettings(updates,
Settings.builder().put(currentSettings), builder, "node");
currentSettings = builder.build();
@@ -160,6 +160,26 @@ public void testAddConsumer() {
assertEquals(0, consumer2.get());
}
+ public void testDependentSettings() {
+ Setting.AffixSetting stringSetting = Setting.affixKeySetting("foo.", "name",
+ (k) -> Setting.simpleString(k, Property.Dynamic, Property.NodeScope));
+ Setting.AffixSetting intSetting = Setting.affixKeySetting("foo.", "bar",
+ (k) -> Setting.intSetting(k, 1, Property.Dynamic, Property.NodeScope), stringSetting);
+
+ AbstractScopedSettings service = new ClusterSettings(Settings.EMPTY,new HashSet<>(Arrays.asList(intSetting, stringSetting)));
+
+ IllegalArgumentException iae = expectThrows(IllegalArgumentException.class,
+ () -> service.validate(Settings.builder().put("foo.test.bar", 7).build(), true));
+ assertEquals("Missing required setting [foo.test.name] for setting [foo.test.bar]", iae.getMessage());
+
+ service.validate(Settings.builder()
+ .put("foo.test.name", "test")
+ .put("foo.test.bar", 7)
+ .build(), true);
+
+ service.validate(Settings.builder().put("foo.test.bar", 7).build(), false);
+ }
+
public void testAddConsumerAffix() {
Setting.AffixSetting intSetting = Setting.affixKeySetting("foo.", "bar",
(k) -> Setting.intSetting(k, 1, Property.Dynamic, Property.NodeScope));
@@ -585,7 +605,7 @@ public void testValidateWithSuggestion() {
Settings.EMPTY,
IndexScopedSettings.BUILT_IN_INDEX_SETTINGS);
IllegalArgumentException iae = expectThrows(IllegalArgumentException.class,
- () -> settings.validate(Settings.builder().put("index.numbe_of_replica", "1").build()));
+ () -> settings.validate(Settings.builder().put("index.numbe_of_replica", "1").build(), false));
assertEquals(iae.getMessage(), "unknown setting [index.numbe_of_replica] did you mean [index.number_of_replicas]?");
}
@@ -595,26 +615,23 @@ public void testValidate() {
IndexScopedSettings.BUILT_IN_INDEX_SETTINGS);
String unknownMsgSuffix = " please check that any required plugins are installed, or check the breaking changes documentation for" +
" removed settings";
- settings.validate(Settings.builder().put("index.store.type", "boom"));
- settings.validate(Settings.builder().put("index.store.type", "boom").build());
- IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () ->
- settings.validate(Settings.builder().put("index.store.type", "boom").put("i.am.not.a.setting", true)));
- assertEquals("unknown setting [i.am.not.a.setting]" + unknownMsgSuffix, e.getMessage());
+ settings.validate(Settings.builder().put("index.store.type", "boom").build(), false);
- e = expectThrows(IllegalArgumentException.class, () ->
- settings.validate(Settings.builder().put("index.store.type", "boom").put("i.am.not.a.setting", true).build()));
+ IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () ->
+ settings.validate(Settings.builder().put("index.store.type", "boom").put("i.am.not.a.setting", true).build(), false));
assertEquals("unknown setting [i.am.not.a.setting]" + unknownMsgSuffix, e.getMessage());
e = expectThrows(IllegalArgumentException.class, () ->
- settings.validate(Settings.builder().put("index.store.type", "boom").put("index.number_of_replicas", true).build()));
+ settings.validate(Settings.builder().put("index.store.type", "boom").put("index.number_of_replicas", true).build(), false));
assertEquals("Failed to parse value [true] for setting [index.number_of_replicas]", e.getMessage());
e = expectThrows(IllegalArgumentException.class, () ->
- settings.validate("index.number_of_replicas", Settings.builder().put("index.number_of_replicas", "true").build()));
+ settings.validate("index.number_of_replicas", Settings.builder().put("index.number_of_replicas", "true").build(), false));
assertEquals("Failed to parse value [true] for setting [index.number_of_replicas]", e.getMessage());
e = expectThrows(IllegalArgumentException.class, () ->
- settings.validate("index.similarity.classic.type", Settings.builder().put("index.similarity.classic.type", "mine").build()));
+ settings.validate("index.similarity.classic.type", Settings.builder().put("index.similarity.classic.type", "mine").build(),
+ false));
assertEquals("illegal value for [index.similarity.classic] cannot redefine built-in similarity", e.getMessage());
}
@@ -624,12 +641,12 @@ public void testValidateSecureSettings() {
Settings settings = Settings.builder().setSecureSettings(secureSettings).build();
final ClusterSettings clusterSettings = new ClusterSettings(settings, Collections.emptySet());
- IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> clusterSettings.validate(settings));
+ IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> clusterSettings.validate(settings, false));
assertThat(e.getMessage(), startsWith("unknown secure setting [some.secure.setting]"));
ClusterSettings clusterSettings2 = new ClusterSettings(settings,
Collections.singleton(SecureSetting.secureString("some.secure.setting", null)));
- clusterSettings2.validate(settings);
+ clusterSettings2.validate(settings, false);
}
public void testDiffSecureSettings() {
@@ -722,7 +739,7 @@ public void testLoggingUpdates() {
IllegalArgumentException ex =
expectThrows(
IllegalArgumentException.class,
- () -> settings.validate(Settings.builder().put("logger._root", "boom").build()));
+ () -> settings.validate(Settings.builder().put("logger._root", "boom").build(), false));
assertEquals("Unknown level constant [BOOM].", ex.getMessage());
assertEquals(level, ESLoggerFactory.getRootLogger().getLevel());
settings.applySettings(Settings.builder().put("logger._root", "TRACE").build());
diff --git a/core/src/test/java/org/elasticsearch/common/settings/SettingTests.java b/core/src/test/java/org/elasticsearch/common/settings/SettingTests.java
index 65d51e126c9f6..4a4beb2e0e3ef 100644
--- a/core/src/test/java/org/elasticsearch/common/settings/SettingTests.java
+++ b/core/src/test/java/org/elasticsearch/common/settings/SettingTests.java
@@ -30,6 +30,7 @@
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Function;
import java.util.stream.Collectors;
@@ -42,6 +43,7 @@
import static org.hamcrest.Matchers.is;
public class SettingTests extends ESTestCase {
+
public void testGet() {
Setting booleanSetting = Setting.boolSetting("foo.bar", false, Property.Dynamic, Property.NodeScope);
assertFalse(booleanSetting.get(Settings.EMPTY));
@@ -577,6 +579,22 @@ public void testAffixKeySetting() {
assertFalse(listAffixSetting.match("foo"));
}
+ public void testAffixSettingNamespaces() {
+ Setting.AffixSetting setting =
+ Setting.affixKeySetting("foo.", "enable", (key) -> Setting.boolSetting(key, false, Property.NodeScope));
+ Settings build = Settings.builder()
+ .put("foo.bar.enable", "true")
+ .put("foo.baz.enable", "true")
+ .put("foo.boom.enable", "true")
+ .put("something.else", "true")
+ .build();
+ Set namespaces = setting.getNamespaces(build);
+ assertEquals(3, namespaces.size());
+ assertTrue(namespaces.contains("bar"));
+ assertTrue(namespaces.contains("baz"));
+ assertTrue(namespaces.contains("boom"));
+ }
+
public void testAffixAsMap() {
Setting.AffixSetting setting = Setting.prefixKeySetting("foo.bar.", key ->
Setting.simpleString(key, Property.NodeScope));
diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/UnicastZenPingTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/UnicastZenPingTests.java
index 3c7a49a176635..853294de186d1 100644
--- a/core/src/test/java/org/elasticsearch/discovery/zen/UnicastZenPingTests.java
+++ b/core/src/test/java/org/elasticsearch/discovery/zen/UnicastZenPingTests.java
@@ -258,6 +258,16 @@ protected Version getVersion() {
assertPingCount(handleD, handleA, 0);
assertPingCount(handleD, handleB, 0);
assertPingCount(handleD, handleC, 3);
+
+ zenPingC.close();
+ handleD.counters.clear();
+ logger.info("ping from UZP_D after closing UZP_C");
+ pingResponses = zenPingD.pingAndWait().toList();
+ // check that node does not respond to pings anymore after the ping service has been closed
+ assertThat(pingResponses.size(), equalTo(0));
+ assertPingCount(handleD, handleA, 0);
+ assertPingCount(handleD, handleB, 0);
+ assertPingCount(handleD, handleC, 3);
}
public void testUnknownHostNotCached() throws ExecutionException, InterruptedException {
diff --git a/core/src/test/java/org/elasticsearch/index/IndexSettingsTests.java b/core/src/test/java/org/elasticsearch/index/IndexSettingsTests.java
index 6be786aff88b5..79c306f43f151 100644
--- a/core/src/test/java/org/elasticsearch/index/IndexSettingsTests.java
+++ b/core/src/test/java/org/elasticsearch/index/IndexSettingsTests.java
@@ -498,7 +498,7 @@ public void testSingleTypeSetting() {
assertTrue(index.isSingleType());
expectThrows(IllegalArgumentException.class, () -> {
index.getScopedSettings()
- .validate(Settings.builder().put(IndexSettings.INDEX_MAPPING_SINGLE_TYPE_SETTING_KEY, randomBoolean()).build());
+ .validate(Settings.builder().put(IndexSettings.INDEX_MAPPING_SINGLE_TYPE_SETTING_KEY, randomBoolean()).build(), false);
});
}
{
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java b/core/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java
index 33e3bc201835d..67c8435520aef 100755
--- a/core/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java
@@ -24,13 +24,13 @@
import org.apache.lucene.search.DocValuesFieldExistsQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
+import org.elasticsearch.common.geo.builders.PointBuilder;
+import org.locationtech.spatial4j.shape.Point;
import org.elasticsearch.common.collect.Iterators;
import org.elasticsearch.common.geo.GeoPoint;
-import org.elasticsearch.common.geo.builders.ShapeBuilders;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.query.QueryShardContext;
-import org.locationtech.spatial4j.shape.Point;
import java.io.IOException;
import java.nio.charset.Charset;
@@ -181,7 +181,7 @@ public Mapper parse(ParseContext context) throws IOException {
pointMapper.parse(context.createExternalValueContext(point));
// Let's add a Dummy Shape
- Point shape = ShapeBuilders.newPoint(-100, 45).build();
+ Point shape = new PointBuilder(-100, 45).build();
shapeMapper.parse(context.createExternalValueContext(shape));
context = context.createExternalValueContext(generatedValue);
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/ExternalValuesMapperIntegrationIT.java b/core/src/test/java/org/elasticsearch/index/mapper/ExternalValuesMapperIntegrationIT.java
index f0109e10abbb4..6dacbc9f64ab9 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/ExternalValuesMapperIntegrationIT.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/ExternalValuesMapperIntegrationIT.java
@@ -21,7 +21,7 @@
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.common.geo.ShapeRelation;
-import org.elasticsearch.common.geo.builders.ShapeBuilders;
+import org.elasticsearch.common.geo.builders.PointBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.plugins.Plugin;
@@ -118,7 +118,7 @@ public void testExternalValues() throws Exception {
assertThat(response.getHits().getTotalHits(), equalTo((long) 1));
response = client().prepareSearch("test-idx")
- .setPostFilter(QueryBuilders.geoShapeQuery("field.shape", ShapeBuilders.newPoint(-100, 45)).relation(ShapeRelation.WITHIN))
+ .setPostFilter(QueryBuilders.geoShapeQuery("field.shape", new PointBuilder(-100, 45)).relation(ShapeRelation.WITHIN))
.execute().actionGet();
assertThat(response.getHits().getTotalHits(), equalTo((long) 1));
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java
index 5972a8ecee8c9..e43cfbe1fd1c1 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java
@@ -22,6 +22,7 @@
import org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy;
import org.apache.lucene.spatial.prefix.tree.GeohashPrefixTree;
import org.apache.lucene.spatial.prefix.tree.QuadPrefixTree;
+import org.elasticsearch.common.Explicit;
import org.elasticsearch.common.compress.CompressedXContent;
import org.elasticsearch.common.geo.GeoUtils;
import org.elasticsearch.common.geo.builders.ShapeBuilder;
@@ -103,7 +104,7 @@ public void testOrientationParsing() throws IOException {
}
/**
- * Test that orientation parameter correctly parses
+ * Test that coerce parameter correctly parses
*/
public void testCoerceParsing() throws IOException {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type1")
@@ -136,6 +137,41 @@ public void testCoerceParsing() throws IOException {
assertThat(coerce, equalTo(false));
}
+ /**
+ * Test that ignore_malformed parameter correctly parses
+ */
+ public void testIgnoreMalformedParsing() throws IOException {
+ String mapping = XContentFactory.jsonBuilder().startObject().startObject("type1")
+ .startObject("properties").startObject("location")
+ .field("type", "geo_shape")
+ .field("ignore_malformed", "true")
+ .endObject().endObject()
+ .endObject().endObject().string();
+
+ DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
+ FieldMapper fieldMapper = defaultMapper.mappers().getMapper("location");
+ assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class));
+
+ Explicit ignoreMalformed = ((GeoShapeFieldMapper)fieldMapper).ignoreMalformed();
+ assertThat(ignoreMalformed.value(), equalTo(true));
+
+ // explicit false ignore_malformed test
+ mapping = XContentFactory.jsonBuilder().startObject().startObject("type1")
+ .startObject("properties").startObject("location")
+ .field("type", "geo_shape")
+ .field("ignore_malformed", "false")
+ .endObject().endObject()
+ .endObject().endObject().string();
+
+ defaultMapper = createIndex("test2").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
+ fieldMapper = defaultMapper.mappers().getMapper("location");
+ assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class));
+
+ ignoreMalformed = ((GeoShapeFieldMapper)fieldMapper).ignoreMalformed();
+ assertThat(ignoreMalformed.explicit(), equalTo(true));
+ assertThat(ignoreMalformed.value(), equalTo(false));
+ }
+
public void testGeohashConfiguration() throws IOException {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type1")
.startObject("properties").startObject("location")
diff --git a/core/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderTests.java
index f1df9ff82a1ac..077f0f48d004a 100644
--- a/core/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderTests.java
@@ -32,7 +32,6 @@
import org.elasticsearch.common.geo.SpatialStrategy;
import org.elasticsearch.common.geo.builders.EnvelopeBuilder;
import org.elasticsearch.common.geo.builders.ShapeBuilder;
-import org.elasticsearch.common.geo.builders.ShapeBuilders;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
@@ -200,7 +199,7 @@ public void testInvalidRelation() throws IOException {
// see #3878
public void testThatXContentSerializationInsideOfArrayWorks() throws Exception {
- EnvelopeBuilder envelopeBuilder = ShapeBuilders.newEnvelope(new Coordinate(0, 0), new Coordinate(10, 10));
+ EnvelopeBuilder envelopeBuilder = new EnvelopeBuilder(new Coordinate(0, 0), new Coordinate(10, 10));
GeoShapeQueryBuilder geoQuery = QueryBuilders.geoShapeQuery("searchGeometry", envelopeBuilder);
JsonXContent.contentBuilder().startArray().value(geoQuery).endArray();
}
diff --git a/core/src/test/java/org/elasticsearch/indices/settings/UpdateSettingsIT.java b/core/src/test/java/org/elasticsearch/indices/settings/UpdateSettingsIT.java
index e8c1604c79915..7ff0725449e8d 100644
--- a/core/src/test/java/org/elasticsearch/indices/settings/UpdateSettingsIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/settings/UpdateSettingsIT.java
@@ -85,6 +85,17 @@ protected Collection> nodePlugins() {
public static class DummySettingPlugin extends Plugin {
public static final Setting DUMMY_SETTING = Setting.simpleString("index.dummy",
Setting.Property.IndexScope, Setting.Property.Dynamic);
+
+ public static final Setting.AffixSetting DUMMY_ACCOUNT_USER = Setting.affixKeySetting("index.acc.", "user",
+ k -> Setting.simpleString(k, Setting.Property.IndexScope, Setting.Property.Dynamic));
+ public static final Setting DUMMY_ACCOUNT_PW = Setting.affixKeySetting("index.acc.", "pw",
+ k -> Setting.simpleString(k, Setting.Property.IndexScope, Setting.Property.Dynamic), DUMMY_ACCOUNT_USER);
+
+ public static final Setting.AffixSetting DUMMY_ACCOUNT_USER_CLUSTER = Setting.affixKeySetting("cluster.acc.", "user",
+ k -> Setting.simpleString(k, Setting.Property.NodeScope, Setting.Property.Dynamic));
+ public static final Setting DUMMY_ACCOUNT_PW_CLUSTER = Setting.affixKeySetting("cluster.acc.", "pw",
+ k -> Setting.simpleString(k, Setting.Property.NodeScope, Setting.Property.Dynamic), DUMMY_ACCOUNT_USER_CLUSTER);
+
@Override
public void onIndexModule(IndexModule indexModule) {
indexModule.addSettingsUpdateConsumer(DUMMY_SETTING, (s) -> {}, (s) -> {
@@ -95,7 +106,8 @@ public void onIndexModule(IndexModule indexModule) {
@Override
public List> getSettings() {
- return Collections.singletonList(DUMMY_SETTING);
+ return Arrays.asList(DUMMY_SETTING, DUMMY_ACCOUNT_PW, DUMMY_ACCOUNT_USER,
+ DUMMY_ACCOUNT_PW_CLUSTER, DUMMY_ACCOUNT_USER_CLUSTER);
}
}
@@ -112,6 +124,124 @@ public List> getSettings() {
}
}
+ public void testUpdateDependentClusterSettings() {
+ IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () ->
+ client().admin().cluster().prepareUpdateSettings().setPersistentSettings(Settings.builder()
+ .put("cluster.acc.test.pw", "asdf")).get());
+ assertEquals("Missing required setting [cluster.acc.test.user] for setting [cluster.acc.test.pw]", iae.getMessage());
+
+ iae = expectThrows(IllegalArgumentException.class, () ->
+ client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder()
+ .put("cluster.acc.test.pw", "asdf")).get());
+ assertEquals("Missing required setting [cluster.acc.test.user] for setting [cluster.acc.test.pw]", iae.getMessage());
+
+ iae = expectThrows(IllegalArgumentException.class, () ->
+ client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder()
+ .put("cluster.acc.test.pw", "asdf")).setPersistentSettings(Settings.builder()
+ .put("cluster.acc.test.user", "asdf")).get());
+ assertEquals("Missing required setting [cluster.acc.test.user] for setting [cluster.acc.test.pw]", iae.getMessage());
+
+ if (randomBoolean()) {
+ client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder()
+ .put("cluster.acc.test.pw", "asdf")
+ .put("cluster.acc.test.user", "asdf")).get();
+ iae = expectThrows(IllegalArgumentException.class, () ->
+ client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder()
+ .putNull("cluster.acc.test.user")).get());
+ assertEquals("Missing required setting [cluster.acc.test.user] for setting [cluster.acc.test.pw]", iae.getMessage());
+ client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder()
+ .putNull("cluster.acc.test.pw")
+ .putNull("cluster.acc.test.user")).get();
+ } else {
+ client().admin().cluster().prepareUpdateSettings().setPersistentSettings(Settings.builder()
+ .put("cluster.acc.test.pw", "asdf")
+ .put("cluster.acc.test.user", "asdf")).get();
+
+ iae = expectThrows(IllegalArgumentException.class, () ->
+ client().admin().cluster().prepareUpdateSettings().setPersistentSettings(Settings.builder()
+ .putNull("cluster.acc.test.user")).get());
+ assertEquals("Missing required setting [cluster.acc.test.user] for setting [cluster.acc.test.pw]", iae.getMessage());
+
+ client().admin().cluster().prepareUpdateSettings().setPersistentSettings(Settings.builder()
+ .putNull("cluster.acc.test.pw")
+ .putNull("cluster.acc.test.user")).get();
+ }
+
+ }
+
+ public void testUpdateDependentIndexSettings() {
+ IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () ->
+ prepareCreate("test", Settings.builder().put("index.acc.test.pw", "asdf")).get());
+ assertEquals("Missing required setting [index.acc.test.user] for setting [index.acc.test.pw]", iae.getMessage());
+
+ createIndex("test");
+ for (int i = 0; i < 2; i++) {
+ if (i == 1) {
+ // now do it on a closed index
+ client().admin().indices().prepareClose("test").get();
+ }
+
+ iae = expectThrows(IllegalArgumentException.class, () ->
+ client()
+ .admin()
+ .indices()
+ .prepareUpdateSettings("test")
+ .setSettings(
+ Settings.builder()
+ .put("index.acc.test.pw", "asdf"))
+ .execute()
+ .actionGet());
+ assertEquals("Missing required setting [index.acc.test.user] for setting [index.acc.test.pw]", iae.getMessage());
+
+ // user has no dependency
+ client()
+ .admin()
+ .indices()
+ .prepareUpdateSettings("test")
+ .setSettings(
+ Settings.builder()
+ .put("index.acc.test.user", "asdf"))
+ .execute()
+ .actionGet();
+
+ // now we are consistent
+ client()
+ .admin()
+ .indices()
+ .prepareUpdateSettings("test")
+ .setSettings(
+ Settings.builder()
+ .put("index.acc.test.pw", "test"))
+ .execute()
+ .actionGet();
+
+ // now try to remove it and make sure it fails
+ iae = expectThrows(IllegalArgumentException.class, () ->
+ client()
+ .admin()
+ .indices()
+ .prepareUpdateSettings("test")
+ .setSettings(
+ Settings.builder()
+ .putNull("index.acc.test.user"))
+ .execute()
+ .actionGet());
+ assertEquals("Missing required setting [index.acc.test.user] for setting [index.acc.test.pw]", iae.getMessage());
+
+ // now we are consistent
+ client()
+ .admin()
+ .indices()
+ .prepareUpdateSettings("test")
+ .setSettings(
+ Settings.builder()
+ .putNull("index.acc.test.pw")
+ .putNull("index.acc.test.user"))
+ .execute()
+ .actionGet();
+ }
+ }
+
public void testResetDefault() {
createIndex("test");
diff --git a/core/src/test/java/org/elasticsearch/search/SearchRequestTests.java b/core/src/test/java/org/elasticsearch/search/SearchRequestTests.java
index d37b8b4b13392..eb643885e830a 100644
--- a/core/src/test/java/org/elasticsearch/search/SearchRequestTests.java
+++ b/core/src/test/java/org/elasticsearch/search/SearchRequestTests.java
@@ -91,8 +91,10 @@ public void testValidate() throws IOException {
assertNull(validationErrors);
}
{
- // disabeling `track_total_hits` isn't valid in scroll context
+ // disabling `track_total_hits` isn't valid in scroll context
SearchRequest searchRequest = createSearchRequest().source(new SearchSourceBuilder());
+ // make sure we don't set the request cache for a scroll query
+ searchRequest.requestCache(false);
searchRequest.scroll(new TimeValue(1000));
searchRequest.source().trackTotalHits(false);
ActionRequestValidationException validationErrors = searchRequest.validate();
@@ -103,6 +105,8 @@ public void testValidate() throws IOException {
{
// scroll and `from` isn't valid
SearchRequest searchRequest = createSearchRequest().source(new SearchSourceBuilder());
+ // make sure we don't set the request cache for a scroll query
+ searchRequest.requestCache(false);
searchRequest.scroll(new TimeValue(1000));
searchRequest.source().from(10);
ActionRequestValidationException validationErrors = searchRequest.validate();
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalExtendedStatsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalExtendedStatsTests.java
index 9bc07a251d8a5..143ad4553c7dd 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalExtendedStatsTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/InternalExtendedStatsTests.java
@@ -82,7 +82,7 @@ protected void assertReduced(InternalExtendedStats reduced, List
- "price_field": 2 <2>
- }
- },
- "status_field": {
- "type": "keyword"
- },
- "price_field": {
- "type": "long"
- },
- "field": {
- "type": "text"
- }
- }
- }
- }
-}
---------------------------------------------------
-// CONSOLE
-
-<1> A boost of zero hints to the percolator that if there are other clauses in a conjunction query then these should be
- preferred over this one.
-
-<2> Any boost higher than 1 overrides the default behaviour when it comes to selecting the best clause. The clause
- that has the field with the highest boost will be selected from a conjunction query for extraction.
-
-The steps the percolator field mapper takes when selecting a clause from a conjunction query:
-
-* If there are clauses that have boosted fields then the clause with highest boost field is selected.
-* If there are range based clauses and term based clauses then term based clauses are picked over range based clauses
-* From all term based clauses the clause with longest term is picked.
-* In the case when there are only range based clauses then the range clause with smallest range is picked over clauses with wider ranges.
-
[float]
==== Reindexing your percolator queries
diff --git a/docs/reference/migration/migrate_7_0/search.asciidoc b/docs/reference/migration/migrate_7_0/search.asciidoc
index a2e5d1ccf8563..12847354cf895 100644
--- a/docs/reference/migration/migrate_7_0/search.asciidoc
+++ b/docs/reference/migration/migrate_7_0/search.asciidoc
@@ -33,3 +33,10 @@ The Search API returns `400 - Bad request` while it would previously return
* the number of slices is too large
* keep alive for scroll is too large
* number of filters in the adjacency matrix aggregation is too large
+
+
+==== Scroll queries cannot use the request_cache anymore
+
+Setting `request_cache:true` on a query that creates a scroll ('scroll=1m`)
+has been deprecated in 6 and will now return a `400 - Bad request`.
+Scroll queries are not meant to be cached.
diff --git a/docs/reference/setup/install.asciidoc b/docs/reference/setup/install.asciidoc
index babdccc2d95fe..49709ba1d6dc1 100644
--- a/docs/reference/setup/install.asciidoc
+++ b/docs/reference/setup/install.asciidoc
@@ -41,7 +41,7 @@ Images are available for running Elasticsearch as Docker containers. They may be
downloaded from the Elastic Docker Registry. The default image ships with
{xpack-ref}/index.html[X-Pack] pre-installed.
+
-<>
+{ref}/docker.html[Install {es} with Docker]
[float]
[[config-mgmt-tools]]
@@ -66,4 +66,9 @@ include::install/rpm.asciidoc[]
include::install/windows.asciidoc[]
-include::install/docker.asciidoc[]
+ifdef::include-xpack[]
+:edit_url!:
+include::{xes-repo-dir}/setup/docker.asciidoc[]
+
+:edit_url:
+endif::include-xpack[]
diff --git a/docs/reference/setup/install/docker.asciidoc b/docs/reference/setup/install/docker.asciidoc
deleted file mode 100644
index 1bcdefc5bc2b5..0000000000000
--- a/docs/reference/setup/install/docker.asciidoc
+++ /dev/null
@@ -1,336 +0,0 @@
-[[docker]]
-=== Install Elasticsearch with Docker
-
-Elasticsearch is also available as Docker images.
-The images use https://hub.docker.com/_/centos/[centos:7] as the base image and
-are available with {xpack-ref}/xpack-introduction.html[X-Pack].
-
-A list of all published Docker images and tags can be found in https://www.docker.elastic.co[www.docker.elastic.co]. The source code can be found
-on https://github.com/elastic/elasticsearch-docker/tree/{branch}[GitHub].
-
-==== Image types
-
-The images are available in three different configurations or "flavors". The
-`basic` flavor, which is the default, ships with X-Pack Basic features
-pre-installed and automatically activated with a free licence. The `platinum`
-flavor features all X-Pack functionally under a 30-day trial licence. The `oss`
-flavor does not include X-Pack, and contains only open-source Elasticsearch.
-
-NOTE: {xpack-ref}/xpack-security.html[X-Pack Security] is enabled in the `platinum`
-image. To access your cluster, it's necessary to set an initial password for the
-`elastic` user. The initial password can be set at start up time via the
-`ELASTIC_PASSWORD` environment variable:
-
-["source","txt",subs="attributes"]
---------------------------------------------
-docker run -e ELASTIC_PASSWORD=MagicWord {docker-repo}-platinum:{version}
---------------------------------------------
-
-NOTE: The `platinum` image includes a trial license for 30 days. After that, you
-can obtain one of the https://www.elastic.co/subscriptions[available
-subscriptions] or revert to a Basic licence. The Basic license is free and
-includes a selection of X-Pack features.
-
-Obtaining Elasticsearch for Docker is as simple as issuing a +docker pull+ command against the Elastic Docker registry.
-
-ifeval::["{release-state}"=="unreleased"]
-
-WARNING: Version {version} of Elasticsearch has not yet been released, so no
-Docker image is currently available for this version.
-
-endif::[]
-
-ifeval::["{release-state}"!="unreleased"]
-
-Docker images can be retrieved with the following commands:
-
-["source","sh",subs="attributes"]
---------------------------------------------
-docker pull {docker-repo}:{version}
-docker pull {docker-repo}-platinum:{version}
-docker pull {docker-repo}-oss:{version}
---------------------------------------------
-
-endif::[]
-
-[[docker-cli-run]]
-==== Running Elasticsearch from the command line
-
-[[docker-cli-run-dev-mode]]
-===== Development mode
-
-ifeval::["{release-state}"=="unreleased"]
-
-WARNING: Version {version} of the Elasticsearch Docker image has not yet been released.
-
-endif::[]
-
-ifeval::["{release-state}"!="unreleased"]
-
-Elasticsearch can be quickly started for development or testing use with the following command:
-
-["source","sh",subs="attributes"]
---------------------------------------------
-docker run -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" {docker-image}
---------------------------------------------
-
-endif::[]
-
-[[docker-cli-run-prod-mode]]
-===== Production mode
-
-[[docker-prod-prerequisites]]
-[IMPORTANT]
-=========================
-
-The `vm.max_map_count` kernel setting needs to be set to at least `262144` for production use.
-Depending on your platform:
-
-* Linux
-+
-The `vm.max_map_count` setting should be set permanently in /etc/sysctl.conf:
-+
-[source,sh]
---------------------------------------------
-$ grep vm.max_map_count /etc/sysctl.conf
-vm.max_map_count=262144
-----------------------------------
-+
-To apply the setting on a live system type: `sysctl -w vm.max_map_count=262144`
-+
-* macOS with https://docs.docker.com/engine/installation/mac/#/docker-for-mac[Docker for Mac]
-+
-The `vm.max_map_count` setting must be set within the xhyve virtual machine:
-+
-["source","sh"]
---------------------------------------------
-$ screen ~/Library/Containers/com.docker.docker/Data/com.docker.driver.amd64-linux/tty
---------------------------------------------
-+
-Log in with 'root' and no password.
-Then configure the `sysctl` setting as you would for Linux:
-+
-["source","sh"]
---------------------------------------------
-sysctl -w vm.max_map_count=262144
---------------------------------------------
-+
-* Windows and macOS with https://www.docker.com/products/docker-toolbox[Docker Toolbox]
-+
-The `vm.max_map_count` setting must be set via docker-machine:
-+
-["source","txt"]
---------------------------------------------
-docker-machine ssh
-sudo sysctl -w vm.max_map_count=262144
---------------------------------------------
-=========================
-
-The following example brings up a cluster comprising two Elasticsearch nodes.
-To bring up the cluster, use the <> and just type:
-
-ifeval::["{release-state}"=="unreleased"]
-
-WARNING: Version {version} of Elasticsearch has not yet been released, so a
-`docker-compose.yml` is not available for this version.
-
-endif::[]
-
-ifeval::["{release-state}"!="unreleased"]
-
-["source","sh"]
---------------------------------------------
-docker-compose up
---------------------------------------------
-
-endif::[]
-
-[NOTE]
-`docker-compose` is not pre-installed with Docker on Linux.
-Instructions for installing it can be found on the
-https://docs.docker.com/compose/install/#install-using-pip[Docker Compose webpage].
-
-The node `elasticsearch` listens on `localhost:9200` while `elasticsearch2`
-talks to `elasticsearch` over a Docker network.
-
-This example also uses https://docs.docker.com/engine/tutorials/dockervolumes[Docker named volumes], called `esdata1` and `esdata2` which will be created if not already present.
-
-[[docker-prod-cluster-composefile]]
-`docker-compose.yml`:
-ifeval::["{release-state}"=="unreleased"]
-
-WARNING: Version {version} of Elasticsearch has not yet been released, so a
-`docker-compose.yml` is not available for this version.
-
-endif::[]
-
-ifeval::["{release-state}"!="unreleased"]
-["source","yaml",subs="attributes"]
---------------------------------------------
-version: 2.2
-services:
- elasticsearch:
- image: {docker-image}
- container_name: elasticsearch
- environment:
- - cluster.name=docker-cluster
- - bootstrap.memory_lock=true
- - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
- ulimits:
- memlock:
- soft: -1
- hard: -1
- volumes:
- - esdata1:/usr/share/elasticsearch/data
- ports:
- - 9200:9200
- networks:
- - esnet
- elasticsearch2:
- image: {docker-image}
- container_name: elasticsearch2
- environment:
- - cluster.name=docker-cluster
- - bootstrap.memory_lock=true
- - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
- - "discovery.zen.ping.unicast.hosts=elasticsearch"
- ulimits:
- memlock:
- soft: -1
- hard: -1
- volumes:
- - esdata2:/usr/share/elasticsearch/data
- networks:
- - esnet
-
-volumes:
- esdata1:
- driver: local
- esdata2:
- driver: local
-
-networks:
- esnet:
---------------------------------------------
-endif::[]
-
-To stop the cluster, type `docker-compose down`. Data volumes will persist, so it's possible to start the cluster again with the same data using `docker-compose up`.
-To destroy the cluster **and the data volumes**, just type `docker-compose down -v`.
-
-===== Inspect status of cluster:
-
-["source","txt"]
---------------------------------------------
-curl http://127.0.0.1:9200/_cat/health
-1472225929 15:38:49 docker-cluster green 2 2 4 2 0 0 0 0 - 100.0%
---------------------------------------------
-// NOTCONSOLE
-
-Log messages go to the console and are handled by the configured Docker logging driver. By default you can access logs with `docker logs`.
-
-[[docker-configuration-methods]]
-==== Configuring Elasticsearch with Docker
-
-Elasticsearch loads its configuration from files under `/usr/share/elasticsearch/config/`. These configuration files are documented in <> and <>.
-
-The image offers several methods for configuring Elasticsearch settings with the conventional approach being to provide customized files, i.e. `elasticsearch.yml`, but it's also possible to use environment variables to set options:
-
-===== A. Present the parameters via Docker environment variables
-For example, to define the cluster name with `docker run` you can pass `-e "cluster.name=mynewclustername"`. Double quotes are required.
-
-===== B. Bind-mounted configuration
-Create your custom config file and mount this over the image's corresponding file.
-For example, bind-mounting a `custom_elasticsearch.yml` with `docker run` can be accomplished with the parameter:
-
-["source","sh"]
---------------------------------------------
--v full_path_to/custom_elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
---------------------------------------------
-IMPORTANT: The container **runs Elasticsearch as user `elasticsearch` using uid:gid `1000:1000`**. Bind mounted host directories and files, such as `custom_elasticsearch.yml` above, **need to be accessible by this user**. For the https://www.elastic.co/guide/en/elasticsearch/reference/current/important-settings.html#path-settings[data and log dirs], such as `/usr/share/elasticsearch/data`, write access is required as well. Also see note 1 below.
-
-===== C. Customized image
-In some environments, it may make more sense to prepare a custom image containing your configuration. A `Dockerfile` to achieve this may be as simple as:
-
-["source","sh",subs="attributes"]
---------------------------------------------
-FROM docker.elastic.co/elasticsearch/elasticsearch:{version}
-COPY --chown=elasticsearch:elasticsearch elasticsearch.yml /usr/share/elasticsearch/config/
---------------------------------------------
-
-You could then build and try the image with something like:
-
-["source","sh"]
---------------------------------------------
-docker build --tag=elasticsearch-custom .
-docker run -ti -v /usr/share/elasticsearch/data elasticsearch-custom
---------------------------------------------
-
-===== D. Override the image's default https://docs.docker.com/engine/reference/run/#cmd-default-command-or-options[CMD]
-
-Options can be passed as command-line options to the Elasticsearch process by
-overriding the default command for the image. For example:
-
-["source","sh"]
---------------------------------------------
-docker run bin/elasticsearch -Ecluster.name=mynewclustername
---------------------------------------------
-
-==== Notes for production use and defaults
-
-We have collected a number of best practices for production use.
-Any Docker parameters mentioned below assume the use of `docker run`.
-
-. By default, Elasticsearch runs inside the container as user `elasticsearch` using uid:gid `1000:1000`.
-+
-CAUTION: One exception is https://docs.openshift.com/container-platform/3.6/creating_images/guidelines.html#openshift-specific-guidelines[Openshift] which runs containers using an arbitrarily assigned user ID. Openshift will present persistent volumes with the gid set to `0` which will work without any adjustments.
-+
-If you are bind-mounting a local directory or file, ensure it is readable by this user, while the <> additionally require write access. A good strategy is to grant group access to gid `1000` or `0` for the local directory. As an example, to prepare a local directory for storing data through a bind-mount:
-+
- mkdir esdatadir
- chmod g+rwx esdatadir
- chgrp 1000 esdatadir
-+
-As a last resort, you can also force the container to mutate the ownership of any bind-mounts used for the <> through the environment variable `TAKE_FILE_OWNERSHIP`; in this case they will be owned by uid:gid `1000:0` providing read/write access to the elasticsearch process as required.
-+
-. It is important to ensure increased ulimits for <> and <> are available for the Elasticsearch containers. Verify the https://github.com/moby/moby/tree/ea4d1243953e6b652082305a9c3cda8656edab26/contrib/init[init system] for the Docker daemon is already setting those to acceptable values and, if needed, adjust them in the Daemon, or override them per container, for example using `docker run`:
-+
- --ulimit nofile=65536:65536
-+
-NOTE: One way of checking the Docker daemon defaults for the aforementioned ulimits is by running:
-+
- docker run --rm centos:7 /bin/bash -c 'ulimit -Hn && ulimit -Sn && ulimit -Hu && ulimit -Su'
-+
-. Swapping needs to be disabled for performance and node stability. This can be
-achieved through any of the methods mentioned in the
-<>. If you opt for the
-`bootstrap.memory_lock: true` approach, apart from defining it through any of
-the <>, you will
-additionally need the `memlock: true` ulimit, either defined in the
-https://docs.docker.com/engine/reference/commandline/dockerd/#default-ulimits[Docker
-Daemon] or specifically set for the container. This is demonstrated above in the
-<>. If using `docker run`:
-+
- -e "bootstrap.memory_lock=true" --ulimit memlock=-1:-1
-+
-. The image https://docs.docker.com/engine/reference/builder/#/expose[exposes] TCP ports 9200 and 9300. For clusters it is recommended to randomize the published ports with `--publish-all`, unless you are pinning one container per host.
-+
-. Use the `ES_JAVA_OPTS` environment variable to set heap size, e.g. to use 16GB
-use `-e ES_JAVA_OPTS="-Xms16g -Xmx16g"` with `docker run`.
-+
-. Pin your deployments to a specific version of the Elasticsearch Docker image, e.g. +docker.elastic.co/elasticsearch/elasticsearch:{version}+.
-+
-. Always use a volume bound on `/usr/share/elasticsearch/data`, as shown in the <>, for the following reasons:
-+
-.. The data of your elasticsearch node won't be lost if the container is killed
-.. Elasticsearch is I/O sensitive and the Docker storage driver is not ideal for fast I/O
-.. It allows the use of advanced https://docs.docker.com/engine/extend/plugins/#volume-plugins[Docker volume plugins]
-+
-. If you are using the devicemapper storage driver, make sure you are not using
-the default `loop-lvm` mode. Configure docker-engine to use
-https://docs.docker.com/engine/userguide/storagedriver/device-mapper-driver/#configure-docker-with-devicemapper[direct-lvm]
-instead.
-+
-. Consider centralizing your logs by using a different https://docs.docker.com/engine/admin/logging/overview/[logging driver]. Also note that the default json-file logging driver is not ideally suited for production use.
-
-
-include::next-steps.asciidoc[]
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JsonProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JsonProcessor.java
index d4c7efe0d5092..8ca31787b5aaa 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JsonProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JsonProcessor.java
@@ -19,14 +19,24 @@
package org.elasticsearch.ingest.common;
+import com.fasterxml.jackson.core.JsonParseException;
+import org.apache.lucene.util.BytesRef;
import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentParserUtils;
+import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.common.xcontent.json.JsonXContentParser;
import org.elasticsearch.ingest.AbstractProcessor;
import org.elasticsearch.ingest.ConfigurationUtils;
import org.elasticsearch.ingest.IngestDocument;
import org.elasticsearch.ingest.Processor;
+import java.io.IOException;
import java.util.Map;
import static org.elasticsearch.ingest.ConfigurationUtils.newConfigurationException;
@@ -64,17 +74,36 @@ boolean isAddToRoot() {
@Override
public void execute(IngestDocument document) throws Exception {
- String stringValue = document.getFieldValue(field, String.class);
- try {
- Map mapValue = XContentHelper.convertToMap(JsonXContent.jsonXContent, stringValue, false);
- if (addToRoot) {
- for (Map.Entry entry : mapValue.entrySet()) {
+ Object fieldValue = document.getFieldValue(field, Object.class);
+ BytesReference bytesRef = (fieldValue == null) ? new BytesArray("null") : new BytesArray(fieldValue.toString());
+ try (XContentParser parser = JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY, bytesRef)) {
+ XContentParser.Token token = parser.nextToken();
+ Object value = null;
+ if (token == XContentParser.Token.VALUE_NULL) {
+ value = null;
+ } else if (token == XContentParser.Token.VALUE_STRING) {
+ value = parser.text();
+ } else if (token == XContentParser.Token.VALUE_NUMBER) {
+ value = parser.numberValue();
+ } else if (token == XContentParser.Token.VALUE_BOOLEAN) {
+ value = parser.booleanValue();
+ } else if (token == XContentParser.Token.START_OBJECT) {
+ value = parser.map();
+ } else if (token == XContentParser.Token.START_ARRAY) {
+ value = parser.list();
+ } else if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) {
+ throw new IllegalArgumentException("cannot read binary value");
+ }
+ if (addToRoot && (value instanceof Map)) {
+ for (Map.Entry entry : ((Map) value).entrySet()) {
document.setFieldValue(entry.getKey(), entry.getValue());
}
+ } else if (addToRoot) {
+ throw new IllegalArgumentException("cannot add non-map fields to root of document");
} else {
- document.setFieldValue(targetField, mapValue);
+ document.setFieldValue(targetField, value);
}
- } catch (ElasticsearchParseException e) {
+ } catch (IOException e) {
throw new IllegalArgumentException(e);
}
}
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorTests.java
index 66ffb87e8da14..ef17935962d0e 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorTests.java
@@ -21,15 +21,19 @@
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.ingest.IngestDocument;
import org.elasticsearch.ingest.RandomDocumentPicks;
import org.elasticsearch.test.ESTestCase;
+import java.util.Arrays;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
+import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
public class JsonProcessorTests extends ESTestCase {
@@ -44,7 +48,7 @@ public void testExecute() throws Exception {
Map randomJsonMap = RandomDocumentPicks.randomSource(random());
XContentBuilder builder = JsonXContent.contentBuilder().map(randomJsonMap);
- String randomJson = XContentHelper.convertToJson(builder.bytes(), false);
+ String randomJson = XContentHelper.convertToJson(builder.bytes(), false, XContentType.JSON);
document.put(randomField, randomJson);
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
@@ -53,16 +57,84 @@ public void testExecute() throws Exception {
assertIngestDocument(ingestDocument.getFieldValue(randomTargetField, Object.class), jsonified);
}
- public void testInvalidJson() {
+ public void testInvalidValue() {
JsonProcessor jsonProcessor = new JsonProcessor("tag", "field", "target_field", false);
Map document = new HashMap<>();
- document.put("field", "invalid json");
+ document.put("field", "blah blah");
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
Exception exception = expectThrows(IllegalArgumentException.class, () -> jsonProcessor.execute(ingestDocument));
- assertThat(exception.getCause().getCause().getMessage(), equalTo("Unrecognized token"
- + " 'invalid': was expecting ('true', 'false' or 'null')\n"
- + " at [Source: invalid json; line: 1, column: 8]"));
+ assertThat(exception.getCause().getMessage(), containsString("Unrecognized token 'blah': " +
+ "was expecting ('true', 'false' or 'null')"));
+ }
+
+ public void testByteArray() {
+ JsonProcessor jsonProcessor = new JsonProcessor("tag", "field", "target_field", false);
+ Map document = new HashMap<>();
+ document.put("field", new byte[] { 0, 1 });
+ IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
+
+ Exception exception = expectThrows(IllegalArgumentException.class, () -> jsonProcessor.execute(ingestDocument));
+ assertThat(exception.getCause().getMessage(), containsString("Unrecognized token 'B': was expecting ('true', 'false' or 'null')"));
+ }
+
+ public void testNull() throws Exception {
+ JsonProcessor jsonProcessor = new JsonProcessor("tag", "field", "target_field", false);
+ Map document = new HashMap<>();
+ document.put("field", null);
+ IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
+ jsonProcessor.execute(ingestDocument);
+ assertNull(ingestDocument.getFieldValue("target_field", Object.class));
+ }
+
+ public void testBoolean() throws Exception {
+ JsonProcessor jsonProcessor = new JsonProcessor("tag", "field", "target_field", false);
+ Map document = new HashMap<>();
+ boolean value = true;
+ document.put("field", value);
+ IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
+ jsonProcessor.execute(ingestDocument);
+ assertThat(ingestDocument.getFieldValue("target_field", Object.class), equalTo(value));
+ }
+
+ public void testInteger() throws Exception {
+ JsonProcessor jsonProcessor = new JsonProcessor("tag", "field", "target_field", false);
+ Map document = new HashMap<>();
+ int value = 3;
+ document.put("field", value);
+ IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
+ jsonProcessor.execute(ingestDocument);
+ assertThat(ingestDocument.getFieldValue("target_field", Object.class), equalTo(value));
+ }
+
+ public void testDouble() throws Exception {
+ JsonProcessor jsonProcessor = new JsonProcessor("tag", "field", "target_field", false);
+ Map document = new HashMap<>();
+ double value = 3.0;
+ document.put("field", value);
+ IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
+ jsonProcessor.execute(ingestDocument);
+ assertThat(ingestDocument.getFieldValue("target_field", Object.class), equalTo(value));
+ }
+
+ public void testString() throws Exception {
+ JsonProcessor jsonProcessor = new JsonProcessor("tag", "field", "target_field", false);
+ Map document = new HashMap<>();
+ String value = "hello world";
+ document.put("field", "\"" + value + "\"");
+ IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
+ jsonProcessor.execute(ingestDocument);
+ assertThat(ingestDocument.getFieldValue("target_field", Object.class), equalTo(value));
+ }
+
+ public void testArray() throws Exception {
+ JsonProcessor jsonProcessor = new JsonProcessor("tag", "field", "target_field", false);
+ Map document = new HashMap<>();
+ List value = Arrays.asList(true, true, false);
+ document.put("field", value.toString());
+ IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
+ jsonProcessor.execute(ingestDocument);
+ assertThat(ingestDocument.getFieldValue("target_field", Object.class), equalTo(value));
}
public void testFieldMissing() {
@@ -96,4 +168,13 @@ public void testAddToRoot() throws Exception {
assertIngestDocument(ingestDocument, expectedIngestDocument);
}
+
+ public void testAddBoolToRoot() {
+ JsonProcessor jsonProcessor = new JsonProcessor("tag", "field", "target_field", true);
+ Map document = new HashMap<>();
+ document.put("field", true);
+ IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
+ Exception exception = expectThrows(IllegalArgumentException.class, () -> jsonProcessor.execute(ingestDocument));
+ assertThat(exception.getMessage(), containsString("cannot add non-map fields to root of document"));
+ }
}
diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/140_json.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/140_json.yml
index 3d9f6a97c081a..81761ba509e10 100644
--- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/140_json.yml
+++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/140_json.yml
@@ -15,7 +15,32 @@ teardown:
"processors": [
{
"json" : {
- "field" : "foo"
+ "field" : "foo_object"
+ }
+ },
+ {
+ "json" : {
+ "field" : "foo_array"
+ }
+ },
+ {
+ "json" : {
+ "field" : "foo_null"
+ }
+ },
+ {
+ "json" : {
+ "field" : "foo_string"
+ }
+ },
+ {
+ "json" : {
+ "field" : "foo_number"
+ }
+ },
+ {
+ "json" : {
+ "field" : "foo_boolean"
}
}
]
@@ -29,7 +54,12 @@ teardown:
id: 1
pipeline: "1"
body: {
- foo: "{\"hello\": \"world\"}"
+ foo_object: "{\"hello\": \"world\"}",
+ foo_array: "[1, 2, 3]",
+ foo_null: null,
+ foo_string: "\"bla bla\"",
+ foo_number: 3,
+ foo_boolean: "true"
}
- do:
@@ -37,4 +67,9 @@ teardown:
index: test
type: test
id: 1
- - match: { _source.foo.hello: "world" }
+ - match: { _source.foo_object.hello: "world" }
+ - match: { _source.foo_array.0: 1 }
+ - match: { _source.foo_string: "bla bla" }
+ - match: { _source.foo_number: 3 }
+ - is_true: _source.foo_boolean
+ - is_false: _source.foo_null
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java
index 910c716db6934..be18c62d1f2e2 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java
@@ -183,6 +183,10 @@ QueryStore getQueryStore() {
return queryStore;
}
+ Query getCandidateMatchesQuery() {
+ return candidateMatchesQuery;
+ }
+
// Comparing identity here to avoid being cached
// Note that in theory if the same instance gets used multiple times it could still get cached,
// however since we create a new query instance each time we this query this shouldn't happen and thus
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java
index db1b444dcd28e..f4e295d4863df 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java
@@ -639,7 +639,7 @@ protected Analyzer getWrappedAnalyzer(String fieldName) {
String name = this.name != null ? this.name : field;
PercolatorFieldMapper.FieldType pft = (PercolatorFieldMapper.FieldType) fieldType;
PercolateQuery.QueryStore queryStore = createStore(pft.queryBuilderField, percolateShardContext, mapUnmappedFieldsAsString);
- return pft.percolateQuery(name, queryStore, documents, docSearcher);
+ return pft.percolateQuery(name, queryStore, documents, docSearcher, context.indexVersionCreated());
}
public String getField() {
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java
index 06c1423eb238d..35bce1f4f4f3f 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java
@@ -20,6 +20,7 @@
import org.apache.lucene.document.BinaryRange;
import org.apache.lucene.document.Field;
+import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.IndexOptions;
@@ -30,10 +31,12 @@
import org.apache.lucene.index.Term;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.search.BooleanClause.Occur;
+import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.CoveringQuery;
import org.apache.lucene.search.DocValuesFieldExistsQuery;
import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.LongValuesSource;
import org.apache.lucene.search.MatchNoDocsQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermInSetQuery;
@@ -44,6 +47,7 @@
import org.elasticsearch.action.support.PlainActionFuture;
import org.elasticsearch.common.ParsingException;
import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.hash.MurmurHash3;
import org.elasticsearch.common.io.stream.OutputStreamStreamOutput;
import org.elasticsearch.common.logging.DeprecationLogger;
@@ -62,6 +66,7 @@
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.mapper.Mapper;
import org.elasticsearch.index.mapper.MapperParsingException;
+import org.elasticsearch.index.mapper.NumberFieldMapper;
import org.elasticsearch.index.mapper.ParseContext;
import org.elasticsearch.index.mapper.RangeFieldMapper;
import org.elasticsearch.index.mapper.RangeFieldMapper.RangeType;
@@ -87,9 +92,6 @@
import java.util.Map;
import java.util.function.Supplier;
-import static org.elasticsearch.common.xcontent.support.XContentMapValues.isObject;
-import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeFloatValue;
-import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeStringValue;
import static org.elasticsearch.index.query.AbstractQueryBuilder.parseInnerQueryBuilder;
public class PercolatorFieldMapper extends FieldMapper {
@@ -113,11 +115,11 @@ public class PercolatorFieldMapper extends FieldMapper {
static final String EXTRACTION_RESULT_FIELD_NAME = "extraction_result";
static final String QUERY_BUILDER_FIELD_NAME = "query_builder_field";
static final String RANGE_FIELD_NAME = "range_field";
+ static final String MINIMUM_SHOULD_MATCH_FIELD_NAME = "minimum_should_match_field";
static class Builder extends FieldMapper.Builder {
private final Supplier queryShardContext;
- private final Map boostFields = new HashMap<>();
Builder(String fieldName, Supplier queryShardContext) {
super(fieldName, FIELD_TYPE, FIELD_TYPE);
@@ -138,15 +140,13 @@ public PercolatorFieldMapper build(BuilderContext context) {
// have to introduce a new field type...
RangeFieldMapper rangeFieldMapper = createExtractedRangeFieldBuilder(RANGE_FIELD_NAME, RangeType.IP, context);
fieldType.rangeField = rangeFieldMapper.fieldType();
+ NumberFieldMapper minimumShouldMatchFieldMapper = createMinimumShouldMatchField(context);
+ fieldType.minimumShouldMatchField = minimumShouldMatchFieldMapper.fieldType();
context.path().remove();
setupFieldType(context);
return new PercolatorFieldMapper(name(), fieldType, defaultFieldType, context.indexSettings(),
multiFieldsBuilder.build(this, context), copyTo, queryShardContext, extractedTermsField,
- extractionResultField, queryBuilderField, rangeFieldMapper, Collections.unmodifiableMap(boostFields));
- }
-
- void addBoostField(String field, float boost) {
- this.boostFields.put(field, boost);
+ extractionResultField, queryBuilderField, rangeFieldMapper, minimumShouldMatchFieldMapper);
}
static KeywordFieldMapper createExtractQueryFieldBuilder(String name, BuilderContext context) {
@@ -173,30 +173,23 @@ static RangeFieldMapper createExtractedRangeFieldBuilder(String name, RangeType
return builder.build(context);
}
+ static NumberFieldMapper createMinimumShouldMatchField(BuilderContext context) {
+ NumberFieldMapper.Builder builder =
+ new NumberFieldMapper.Builder(MINIMUM_SHOULD_MATCH_FIELD_NAME, NumberFieldMapper.NumberType.INTEGER);
+ builder.index(false);
+ builder.store(false);
+ builder.docValues(true);
+ builder.fieldType().setDocValuesType(DocValuesType.NUMERIC);
+ return builder.build(context);
+ }
+
}
static class TypeParser implements FieldMapper.TypeParser {
@Override
public Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException {
- Builder builder = new Builder(name, parserContext.queryShardContextSupplier());
- for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext();) {
- Map.Entry entry = iterator.next();
- String propName = entry.getKey();
- Object propNode = entry.getValue();
- if (propName.equals("boost_fields")) {
- if (isObject(propNode)) {
- for (Map.Entry innerEntry : ((Map) propNode).entrySet()) {
- String fieldName = nodeStringValue(innerEntry.getKey(), null);
- builder.addBoostField(fieldName, nodeFloatValue(innerEntry.getValue()));
- }
- } else {
- throw new IllegalArgumentException("boost_fields [" + propNode + "] is not an object");
- }
- iterator.remove();
- }
- }
- return builder;
+ return new Builder(name, parserContext.queryShardContextSupplier());
}
}
@@ -205,6 +198,7 @@ static class FieldType extends MappedFieldType {
MappedFieldType queryTermsField;
MappedFieldType extractionResultField;
MappedFieldType queryBuilderField;
+ MappedFieldType minimumShouldMatchField;
RangeFieldMapper.RangeFieldType rangeField;
@@ -220,6 +214,7 @@ static class FieldType extends MappedFieldType {
extractionResultField = ref.extractionResultField;
queryBuilderField = ref.queryBuilderField;
rangeField = ref.rangeField;
+ minimumShouldMatchField = ref.minimumShouldMatchField;
}
@Override
@@ -247,23 +242,38 @@ public Query termQuery(Object value, QueryShardContext context) {
}
Query percolateQuery(String name, PercolateQuery.QueryStore queryStore, List documents,
- IndexSearcher searcher) throws IOException {
+ IndexSearcher searcher, Version indexVersion) throws IOException {
IndexReader indexReader = searcher.getIndexReader();
- Query candidateMatchesQuery = createCandidateQuery(indexReader);
+ Tuple, Boolean> t = createCandidateQueryClauses(indexReader);
+ BooleanQuery.Builder candidateQuery = new BooleanQuery.Builder();
+ if (t.v2() && indexVersion.onOrAfter(Version.V_6_1_0)) {
+ LongValuesSource valuesSource = LongValuesSource.fromIntField(minimumShouldMatchField.name());
+ candidateQuery.add(new CoveringQuery(t.v1(), valuesSource), BooleanClause.Occur.SHOULD);
+ } else {
+ for (Query query : t.v1()) {
+ candidateQuery.add(query, BooleanClause.Occur.SHOULD);
+ }
+ }
+ // include extractionResultField:failed, because docs with this term have no extractedTermsField
+ // and otherwise we would fail to return these docs. Docs that failed query term extraction
+ // always need to be verified by MemoryIndex:
+ candidateQuery.add(new TermQuery(new Term(extractionResultField.name(), EXTRACTION_FAILED)), BooleanClause.Occur.SHOULD);
+
Query verifiedMatchesQuery;
- // We can only skip the MemoryIndex verification when percolating a single document.
- // When the document being percolated contains a nested object field then the MemoryIndex contains multiple
- // documents. In this case the term query that indicates whether memory index verification can be skipped
- // can incorrectly indicate that non nested queries would match, while their nested variants would not.
- if (indexReader.maxDoc() == 1) {
+ // We can only skip the MemoryIndex verification when percolating a single non nested document. We cannot
+ // skip MemoryIndex verification when percolating multiple documents, because when terms and
+ // ranges are extracted from IndexReader backed by a RamDirectory holding multiple documents we do
+ // not know to which document the terms belong too and for certain queries we incorrectly emit candidate
+ // matches as actual match.
+ if (t.v2() && indexReader.maxDoc() == 1) {
verifiedMatchesQuery = new TermQuery(new Term(extractionResultField.name(), EXTRACTION_COMPLETE));
} else {
- verifiedMatchesQuery = new MatchNoDocsQuery("multiple/nested docs, so no verified matches");
+ verifiedMatchesQuery = new MatchNoDocsQuery("multiple or nested docs or CoveringQuery could not be used");
}
- return new PercolateQuery(name, queryStore, documents, candidateMatchesQuery, searcher, verifiedMatchesQuery);
+ return new PercolateQuery(name, queryStore, documents, candidateQuery.build(), searcher, verifiedMatchesQuery);
}
- Query createCandidateQuery(IndexReader indexReader) throws IOException {
+ Tuple, Boolean> createCandidateQueryClauses(IndexReader indexReader) throws IOException {
List extractedTerms = new ArrayList<>();
Map> encodedPointValuesByField = new HashMap<>();
@@ -290,14 +300,17 @@ Query createCandidateQuery(IndexReader indexReader) throws IOException {
}
}
- BooleanQuery.Builder builder = new BooleanQuery.Builder();
- if (extractedTerms.size() != 0) {
- builder.add(new TermInSetQuery(queryTermsField.name(), extractedTerms), Occur.SHOULD);
+ final boolean canUseMinimumShouldMatchField;
+ final List queries = new ArrayList<>();
+ if (extractedTerms.size() + encodedPointValuesByField.size() <= BooleanQuery.getMaxClauseCount()) {
+ canUseMinimumShouldMatchField = true;
+ for (BytesRef extractedTerm : extractedTerms) {
+ queries.add(new TermQuery(new Term(queryTermsField.name(), extractedTerm)));
+ }
+ } else {
+ canUseMinimumShouldMatchField = false;
+ queries.add(new TermInSetQuery(queryTermsField.name(), extractedTerms));
}
- // include extractionResultField:failed, because docs with this term have no extractedTermsField
- // and otherwise we would fail to return these docs. Docs that failed query term extraction
- // always need to be verified by MemoryIndex:
- builder.add(new TermQuery(new Term(extractionResultField.name(), EXTRACTION_FAILED)), Occur.SHOULD);
for (Map.Entry> entry : encodedPointValuesByField.entrySet()) {
String rangeFieldName = entry.getKey();
@@ -305,9 +318,9 @@ Query createCandidateQuery(IndexReader indexReader) throws IOException {
byte[] min = encodedPointValues.get(0);
byte[] max = encodedPointValues.get(1);
Query query = BinaryRange.newIntersectsQuery(rangeField.name(), encodeRange(rangeFieldName, min, max));
- builder.add(query, Occur.SHOULD);
+ queries.add(query);
}
- return builder.build();
+ return new Tuple<>(queries, canUseMinimumShouldMatchField);
}
}
@@ -317,24 +330,24 @@ Query createCandidateQuery(IndexReader indexReader) throws IOException {
private KeywordFieldMapper queryTermsField;
private KeywordFieldMapper extractionResultField;
private BinaryFieldMapper queryBuilderField;
+ private NumberFieldMapper minimumShouldMatchFieldMapper;
private RangeFieldMapper rangeFieldMapper;
- private Map boostFields;
PercolatorFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType,
- Settings indexSettings, MultiFields multiFields, CopyTo copyTo,
- Supplier queryShardContext,
- KeywordFieldMapper queryTermsField, KeywordFieldMapper extractionResultField,
- BinaryFieldMapper queryBuilderField, RangeFieldMapper rangeFieldMapper,
- Map boostFields) {
+ Settings indexSettings, MultiFields multiFields, CopyTo copyTo,
+ Supplier queryShardContext,
+ KeywordFieldMapper queryTermsField, KeywordFieldMapper extractionResultField,
+ BinaryFieldMapper queryBuilderField, RangeFieldMapper rangeFieldMapper,
+ NumberFieldMapper minimumShouldMatchFieldMapper) {
super(simpleName, fieldType, defaultFieldType, indexSettings, multiFields, copyTo);
this.queryShardContext = queryShardContext;
this.queryTermsField = queryTermsField;
this.extractionResultField = extractionResultField;
this.queryBuilderField = queryBuilderField;
+ this.minimumShouldMatchFieldMapper = minimumShouldMatchFieldMapper;
this.mapUnmappedFieldAsText = getMapUnmappedFieldAsText(indexSettings);
this.rangeFieldMapper = rangeFieldMapper;
- this.boostFields = boostFields;
}
private static boolean getMapUnmappedFieldAsText(Settings indexSettings) {
@@ -361,6 +374,7 @@ public FieldMapper updateFieldType(Map fullNameToFieldT
KeywordFieldMapper extractionResultUpdated = (KeywordFieldMapper) extractionResultField.updateFieldType(fullNameToFieldType);
BinaryFieldMapper queryBuilderUpdated = (BinaryFieldMapper) queryBuilderField.updateFieldType(fullNameToFieldType);
RangeFieldMapper rangeFieldMapperUpdated = (RangeFieldMapper) rangeFieldMapper.updateFieldType(fullNameToFieldType);
+ NumberFieldMapper msmFieldMapperUpdated = (NumberFieldMapper) minimumShouldMatchFieldMapper.updateFieldType(fullNameToFieldType);
if (updated == this && queryTermsUpdated == queryTermsField && extractionResultUpdated == extractionResultField
&& queryBuilderUpdated == queryBuilderField && rangeFieldMapperUpdated == rangeFieldMapper) {
@@ -373,6 +387,7 @@ public FieldMapper updateFieldType(Map fullNameToFieldT
updated.extractionResultField = extractionResultUpdated;
updated.queryBuilderField = queryBuilderUpdated;
updated.rangeFieldMapper = rangeFieldMapperUpdated;
+ updated.minimumShouldMatchFieldMapper = msmFieldMapperUpdated;
return updated;
}
@@ -429,7 +444,8 @@ void processQuery(Query query, ParseContext context) {
FieldType pft = (FieldType) this.fieldType();
QueryAnalyzer.Result result;
try {
- result = QueryAnalyzer.analyze(query, boostFields);
+ Version indexVersion = context.mapperService().getIndexSettings().getIndexVersionCreated();
+ result = QueryAnalyzer.analyze(query, indexVersion);
} catch (QueryAnalyzer.UnsupportedQueryException e) {
doc.add(new Field(pft.extractionResultField.name(), EXTRACTION_FAILED, extractionResultField.fieldType()));
return;
@@ -457,6 +473,9 @@ void processQuery(Query query, ParseContext context) {
for (IndexableField field : fields) {
context.doc().add(field);
}
+ if (context.mapperService().getIndexSettings().getIndexVersionCreated().onOrAfter(Version.V_6_1_0)) {
+ doc.add(new NumericDocValuesField(minimumShouldMatchFieldMapper.name(), result.minimumShouldMatch));
+ }
}
static Query parseQuery(QueryShardContext context, boolean mapUnmappedFieldsAsString, XContentParser parser) throws IOException {
@@ -491,7 +510,9 @@ private static QueryBuilder parseQueryBuilder(XContentParser parser, XContentLoc
@Override
public Iterator iterator() {
- return Arrays.asList(queryTermsField, extractionResultField, queryBuilderField, rangeFieldMapper).iterator();
+ return Arrays.asList(
+ queryTermsField, extractionResultField, queryBuilderField, minimumShouldMatchFieldMapper, rangeFieldMapper
+ ).iterator();
}
@Override
@@ -504,28 +525,6 @@ protected String contentType() {
return CONTENT_TYPE;
}
- @Override
- protected void doMerge(Mapper mergeWith, boolean updateAllTypes) {
- super.doMerge(mergeWith, updateAllTypes);
- PercolatorFieldMapper percolatorMergeWith = (PercolatorFieldMapper) mergeWith;
-
- // Updating the boost_fields can be allowed, because it doesn't break previously indexed percolator queries
- // However the updated boost_fields to completely take effect, percolator queries prior to the mapping update need to be reindexed
- boostFields = percolatorMergeWith.boostFields;
- }
-
- @Override
- protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException {
- super.doXContentBody(builder, includeDefaults, params);
- if (boostFields.isEmpty() == false) {
- builder.startObject("boost_fields");
- for (Map.Entry entry : boostFields.entrySet()) {
- builder.field(entry.getKey(), entry.getValue());
- }
- builder.endObject();
- }
- }
-
boolean isMapUnmappedFieldAsText() {
return mapUnmappedFieldAsText;
}
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/QueryAnalyzer.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/QueryAnalyzer.java
index 8c2a6d7a4553b..940f9ebab5a9a 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/QueryAnalyzer.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/QueryAnalyzer.java
@@ -45,6 +45,7 @@
import org.apache.lucene.search.spans.SpanTermQuery;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.NumericUtils;
+import org.elasticsearch.Version;
import org.elasticsearch.common.logging.LoggerMessageFormat;
import org.elasticsearch.common.lucene.search.function.FunctionScoreQuery;
import org.elasticsearch.index.search.ESToParentBlockJoinQuery;
@@ -59,16 +60,15 @@
import java.util.Objects;
import java.util.Set;
import java.util.function.BiFunction;
-import java.util.function.Predicate;
import static java.util.stream.Collectors.toSet;
final class QueryAnalyzer {
- private static final Map, BiFunction, Result>> queryProcessors;
+ private static final Map, BiFunction> queryProcessors;
static {
- Map, BiFunction, Result>> map = new HashMap<>();
+ Map, BiFunction> map = new HashMap<>();
map.put(MatchNoDocsQuery.class, matchNoDocsQuery());
map.put(ConstantScoreQuery.class, constantScoreQuery());
map.put(BoostQuery.class, boostQuery());
@@ -119,161 +119,196 @@ private QueryAnalyzer() {
* Sometimes the query analyzer can't always extract terms or ranges from a sub query, if that happens then
* query analysis is stopped and an UnsupportedQueryException is thrown. So that the caller can mark
* this query in such a way that the PercolatorQuery always verifies if this query with the MemoryIndex.
+ *
+ * @param query The query to analyze.
+ * @param indexVersion The create version of the index containing the percolator queries.
*/
- static Result analyze(Query query, Map boosts) {
+ static Result analyze(Query query, Version indexVersion) {
Class queryClass = query.getClass();
if (queryClass.isAnonymousClass()) {
// Sometimes queries have anonymous classes in that case we need the direct super class.
// (for example blended term query)
queryClass = queryClass.getSuperclass();
}
- BiFunction, Result> queryProcessor = queryProcessors.get(queryClass);
+ BiFunction queryProcessor = queryProcessors.get(queryClass);
if (queryProcessor != null) {
- return queryProcessor.apply(query, boosts);
+ return queryProcessor.apply(query, indexVersion);
} else {
throw new UnsupportedQueryException(query);
}
}
- private static BiFunction, Result> matchNoDocsQuery() {
- return (query, boosts) -> new Result(true, Collections.emptySet());
+ private static BiFunction matchNoDocsQuery() {
+ return (query, version) -> new Result(true, Collections.emptySet(), 1);
}
- private static BiFunction, Result> constantScoreQuery() {
- return (query, boosts)-> {
+ private static BiFunction constantScoreQuery() {
+ return (query, boosts) -> {
Query wrappedQuery = ((ConstantScoreQuery) query).getQuery();
return analyze(wrappedQuery, boosts);
};
}
- private static BiFunction, Result> boostQuery() {
- return (query, boosts) -> {
+ private static BiFunction boostQuery() {
+ return (query, version) -> {
Query wrappedQuery = ((BoostQuery) query).getQuery();
- return analyze(wrappedQuery, boosts);
+ return analyze(wrappedQuery, version);
};
}
- private static BiFunction, Result> termQuery() {
- return (query, boosts) -> {
+ private static BiFunction termQuery() {
+ return (query, version) -> {
TermQuery termQuery = (TermQuery) query;
- return new Result(true, Collections.singleton(new QueryExtraction(termQuery.getTerm())));
+ return new Result(true, Collections.singleton(new QueryExtraction(termQuery.getTerm())), 1);
};
}
- private static BiFunction, Result> termInSetQuery() {
- return (query, boosts) -> {
+ private static BiFunction termInSetQuery() {
+ return (query, version) -> {
TermInSetQuery termInSetQuery = (TermInSetQuery) query;
Set terms = new HashSet<>();
PrefixCodedTerms.TermIterator iterator = termInSetQuery.getTermData().iterator();
for (BytesRef term = iterator.next(); term != null; term = iterator.next()) {
terms.add(new QueryExtraction(new Term(iterator.field(), term)));
}
- return new Result(true, terms);
+ return new Result(true, terms, 1);
};
}
- private static BiFunction, Result> synonymQuery() {
- return (query, boosts) -> {
+ private static BiFunction synonymQuery() {
+ return (query, version) -> {
Set terms = ((SynonymQuery) query).getTerms().stream().map(QueryExtraction::new).collect(toSet());
- return new Result(true, terms);
+ return new Result(true, terms, 1);
};
}
- private static BiFunction, Result> commonTermsQuery() {
- return (query, boosts) -> {
+ private static BiFunction commonTermsQuery() {
+ return (query, version) -> {
Set terms = ((CommonTermsQuery) query).getTerms().stream().map(QueryExtraction::new).collect(toSet());
- return new Result(false, terms);
+ return new Result(false, terms, 1);
};
}
- private static BiFunction, Result> blendedTermQuery() {
- return (query, boosts) -> {
+ private static BiFunction blendedTermQuery() {
+ return (query, version) -> {
Set terms = ((BlendedTermQuery) query).getTerms().stream().map(QueryExtraction::new).collect(toSet());
- return new Result(true, terms);
+ return new Result(true, terms, 1);
};
}
- private static BiFunction, Result> phraseQuery() {
- return (query, boosts) -> {
+ private static BiFunction phraseQuery() {
+ return (query, version) -> {
Term[] terms = ((PhraseQuery) query).getTerms();
if (terms.length == 0) {
- return new Result(true, Collections.emptySet());
+ return new Result(true, Collections.emptySet(), 1);
}
- // the longest term is likely to be the rarest,
- // so from a performance perspective it makes sense to extract that
- Term longestTerm = terms[0];
- for (Term term : terms) {
- if (longestTerm.bytes().length < term.bytes().length) {
- longestTerm = term;
+ if (version.onOrAfter(Version.V_6_1_0)) {
+ Set extractions = Arrays.stream(terms).map(QueryExtraction::new).collect(toSet());
+ return new Result(false, extractions, extractions.size());
+ } else {
+ // the longest term is likely to be the rarest,
+ // so from a performance perspective it makes sense to extract that
+ Term longestTerm = terms[0];
+ for (Term term : terms) {
+ if (longestTerm.bytes().length < term.bytes().length) {
+ longestTerm = term;
+ }
}
+ return new Result(false, Collections.singleton(new QueryExtraction(longestTerm)), 1);
}
- return new Result(false, Collections.singleton(new QueryExtraction(longestTerm)));
};
}
- private static BiFunction, Result> multiPhraseQuery() {
- return (query, boosts) -> {
+ private static BiFunction multiPhraseQuery() {
+ return (query, version) -> {
Term[][] terms = ((MultiPhraseQuery) query).getTermArrays();
if (terms.length == 0) {
- return new Result(true, Collections.emptySet());
+ return new Result(true, Collections.emptySet(), 1);
}
- Set bestTermArr = null;
- for (Term[] termArr : terms) {
- Set queryExtractions = Arrays.stream(termArr).map(QueryExtraction::new).collect(toSet());
- bestTermArr = selectBestExtraction(boosts, bestTermArr, queryExtractions);
+ if (version.onOrAfter(Version.V_6_1_0)) {
+ Set extractions = new HashSet<>();
+ for (Term[] termArr : terms) {
+ extractions.addAll(Arrays.stream(termArr).map(QueryExtraction::new).collect(toSet()));
+ }
+ return new Result(false, extractions, terms.length);
+ } else {
+ Set bestTermArr = null;
+ for (Term[] termArr : terms) {
+ Set queryExtractions = Arrays.stream(termArr).map(QueryExtraction::new).collect(toSet());
+ bestTermArr = selectBestExtraction(bestTermArr, queryExtractions);
+ }
+ return new Result(false, bestTermArr, 1);
}
- return new Result(false, bestTermArr);
};
}
- private static BiFunction, Result> spanTermQuery() {
- return (query, boosts) -> {
+ private static BiFunction spanTermQuery() {
+ return (query, version) -> {
Term term = ((SpanTermQuery) query).getTerm();
- return new Result(true, Collections.singleton(new QueryExtraction(term)));
+ return new Result(true, Collections.singleton(new QueryExtraction(term)), 1);
};
}
- private static BiFunction, Result> spanNearQuery() {
- return (query, boosts) -> {
- Set bestClauses = null;
+ private static BiFunction spanNearQuery() {
+ return (query, version) -> {
SpanNearQuery spanNearQuery = (SpanNearQuery) query;
- for (SpanQuery clause : spanNearQuery.getClauses()) {
- Result temp = analyze(clause, boosts);
- bestClauses = selectBestExtraction(boosts, temp.extractions, bestClauses);
+ if (version.onOrAfter(Version.V_6_1_0)) {
+ Set results = Arrays.stream(spanNearQuery.getClauses()).map(clause -> analyze(clause, version)).collect(toSet());
+ int msm = 0;
+ Set extractions = new HashSet<>();
+ Set seenRangeFields = new HashSet<>();
+ for (Result result : results) {
+ QueryExtraction[] t = result.extractions.toArray(new QueryExtraction[1]);
+ if (result.extractions.size() == 1 && t[0].range != null) {
+ if (seenRangeFields.add(t[0].range.fieldName)) {
+ msm += 1;
+ }
+ } else {
+ msm += result.minimumShouldMatch;
+ }
+ extractions.addAll(result.extractions);
+ }
+ return new Result(false, extractions, msm);
+ } else {
+ Set bestClauses = null;
+ for (SpanQuery clause : spanNearQuery.getClauses()) {
+ Result temp = analyze(clause, version);
+ bestClauses = selectBestExtraction(temp.extractions, bestClauses);
+ }
+ return new Result(false, bestClauses, 1);
}
- return new Result(false, bestClauses);
};
}
- private static BiFunction, Result> spanOrQuery() {
- return (query, boosts) -> {
+ private static BiFunction spanOrQuery() {
+ return (query, version) -> {
Set terms = new HashSet<>();
SpanOrQuery spanOrQuery = (SpanOrQuery) query;
for (SpanQuery clause : spanOrQuery.getClauses()) {
- terms.addAll(analyze(clause, boosts).extractions);
+ terms.addAll(analyze(clause, version).extractions);
}
- return new Result(false, terms);
+ return new Result(false, terms, 1);
};
}
- private static BiFunction, Result> spanNotQuery() {
- return (query, boosts) -> {
- Result result = analyze(((SpanNotQuery) query).getInclude(), boosts);
- return new Result(false, result.extractions);
+ private static BiFunction spanNotQuery() {
+ return (query, version) -> {
+ Result result = analyze(((SpanNotQuery) query).getInclude(), version);
+ return new Result(false, result.extractions, result.minimumShouldMatch);
};
}
- private static BiFunction, Result> spanFirstQuery() {
- return (query, boosts) -> {
- Result result = analyze(((SpanFirstQuery) query).getMatch(), boosts);
- return new Result(false, result.extractions);
+ private static BiFunction spanFirstQuery() {
+ return (query, version) -> {
+ Result result = analyze(((SpanFirstQuery) query).getMatch(), version);
+ return new Result(false, result.extractions, result.minimumShouldMatch);
};
}
- private static BiFunction, Result> booleanQuery() {
- return (query, boosts) -> {
+ private static BiFunction booleanQuery() {
+ return (query, version) -> {
BooleanQuery bq = (BooleanQuery) query;
List clauses = bq.clauses();
int minimumShouldMatch = bq.getMinimumNumberShouldMatch();
@@ -292,34 +327,89 @@ private static BiFunction, Result> booleanQuery() {
}
}
if (numRequiredClauses > 0) {
- Set bestClause = null;
- UnsupportedQueryException uqe = null;
- for (BooleanClause clause : clauses) {
- if (clause.isRequired() == false) {
- // skip must_not clauses, we don't need to remember the things that do *not* match...
- // skip should clauses, this bq has must clauses, so we don't need to remember should clauses,
- // since they are completely optional.
- continue;
+ if (version.onOrAfter(Version.V_6_1_0)) {
+ UnsupportedQueryException uqe = null;
+ List results = new ArrayList<>(numRequiredClauses);
+ for (BooleanClause clause : clauses) {
+ if (clause.isRequired()) {
+ // skip must_not clauses, we don't need to remember the things that do *not* match...
+ // skip should clauses, this bq has must clauses, so we don't need to remember should clauses,
+ // since they are completely optional.
+
+ try {
+ results.add(analyze(clause.getQuery(), version));
+ } catch (UnsupportedQueryException e) {
+ uqe = e;
+ }
+ }
}
- Result temp;
- try {
- temp = analyze(clause.getQuery(), boosts);
- } catch (UnsupportedQueryException e) {
- uqe = e;
- continue;
+ if (results.isEmpty()) {
+ if (uqe != null) {
+ // we're unable to select the best clause and an exception occurred, so we bail
+ throw uqe;
+ } else {
+ // We didn't find a clause and no exception occurred, so this bq only contained MatchNoDocsQueries,
+ return new Result(true, Collections.emptySet(), 1);
+ }
+ } else {
+ int msm = 0;
+ boolean requiredShouldClauses = minimumShouldMatch > 0 && numOptionalClauses > 0;
+ boolean verified = uqe == null && numProhibitedClauses == 0 && requiredShouldClauses == false;
+ Set extractions = new HashSet<>();
+ Set seenRangeFields = new HashSet<>();
+ for (Result result : results) {
+ QueryExtraction[] t = result.extractions.toArray(new QueryExtraction[1]);
+ if (result.extractions.size() == 1 && t[0].range != null) {
+ // In case of range queries each extraction does not simply increment the minimum_should_match
+ // for that percolator query like for a term based extraction, so that can lead to more false
+ // positives for percolator queries with range queries than term based queries.
+ // The is because the way number fields are extracted from the document to be percolated.
+ // Per field a single range is extracted and if a percolator query has two or more range queries
+ // on the same field than the the minimum should match can be higher than clauses in the CoveringQuery.
+ // Therefore right now the minimum should match is incremented once per number field when processing
+ // the percolator query at index time.
+ if (seenRangeFields.add(t[0].range.fieldName)) {
+ msm += 1;
+ }
+ } else {
+ msm += result.minimumShouldMatch;
+ }
+ verified &= result.verified;
+ extractions.addAll(result.extractions);
+ }
+ return new Result(verified, extractions, msm);
}
- bestClause = selectBestExtraction(boosts, temp.extractions, bestClause);
- }
- if (bestClause != null) {
- return new Result(false, bestClause);
} else {
- if (uqe != null) {
- // we're unable to select the best clause and an exception occurred, so we bail
- throw uqe;
+ Set bestClause = null;
+ UnsupportedQueryException uqe = null;
+ for (BooleanClause clause : clauses) {
+ if (clause.isRequired() == false) {
+ // skip must_not clauses, we don't need to remember the things that do *not* match...
+ // skip should clauses, this bq has must clauses, so we don't need to remember should clauses,
+ // since they are completely optional.
+ continue;
+ }
+
+ Result temp;
+ try {
+ temp = analyze(clause.getQuery(), version);
+ } catch (UnsupportedQueryException e) {
+ uqe = e;
+ continue;
+ }
+ bestClause = selectBestExtraction(temp.extractions, bestClause);
+ }
+ if (bestClause != null) {
+ return new Result(false, bestClause, 1);
} else {
- // We didn't find a clause and no exception occurred, so this bq only contained MatchNoDocsQueries,
- return new Result(true, Collections.emptySet());
+ if (uqe != null) {
+ // we're unable to select the best clause and an exception occurred, so we bail
+ throw uqe;
+ } else {
+ // We didn't find a clause and no exception occurred, so this bq only contained MatchNoDocsQueries,
+ return new Result(true, Collections.emptySet(), 1);
+ }
}
}
} else {
@@ -329,33 +419,33 @@ private static BiFunction, Result> booleanQuery() {
disjunctions.add(clause.getQuery());
}
}
- return handleDisjunction(disjunctions, minimumShouldMatch, numProhibitedClauses > 0, boosts);
+ return handleDisjunction(disjunctions, minimumShouldMatch, numProhibitedClauses > 0, version);
}
};
}
- private static BiFunction, Result> disjunctionMaxQuery() {
- return (query, boosts) -> {
+ private static BiFunction disjunctionMaxQuery() {
+ return (query, version) -> {
List disjuncts = ((DisjunctionMaxQuery) query).getDisjuncts();
- return handleDisjunction(disjuncts, 1, false, boosts);
+ return handleDisjunction(disjuncts, 1, false, version);
};
}
- private static BiFunction, Result> functionScoreQuery() {
- return (query, boosts) -> {
+ private static BiFunction functionScoreQuery() {
+ return (query, version) -> {
FunctionScoreQuery functionScoreQuery = (FunctionScoreQuery) query;
- Result result = analyze(functionScoreQuery.getSubQuery(), boosts);
+ Result result = analyze(functionScoreQuery.getSubQuery(), version);
// If min_score is specified we can't guarantee upfront that this percolator query matches,
// so in that case we set verified to false.
// (if it matches with the percolator document matches with the extracted terms.
// Min score filters out docs, which is different than the functions, which just influences the score.)
boolean verified = functionScoreQuery.getMinScore() == null;
- return new Result(verified, result.extractions);
+ return new Result(verified, result.extractions, result.minimumShouldMatch);
};
}
- private static BiFunction, Result> pointRangeQuery() {
- return (query, boosts) -> {
+ private static BiFunction pointRangeQuery() {
+ return (query, version) -> {
PointRangeQuery pointRangeQuery = (PointRangeQuery) query;
if (pointRangeQuery.getNumDims() != 1) {
throw new UnsupportedQueryException(query);
@@ -367,14 +457,13 @@ private static BiFunction, Result> pointRangeQuery() {
// Need to check whether upper is not smaller than lower, otherwise NumericUtils.subtract(...) fails IAE
// If upper is really smaller than lower then we deal with like MatchNoDocsQuery. (verified and no extractions)
if (new BytesRef(lowerPoint).compareTo(new BytesRef(upperPoint)) > 0) {
- return new Result(true, Collections.emptySet());
+ return new Result(true, Collections.emptySet(), 1);
}
byte[] interval = new byte[16];
NumericUtils.subtract(16, 0, prepad(upperPoint), prepad(lowerPoint), interval);
return new Result(false, Collections.singleton(new QueryExtraction(
- new Range(pointRangeQuery.getField(), lowerPoint, upperPoint, interval))
- ));
+ new Range(pointRangeQuery.getField(), lowerPoint, upperPoint, interval))), 1);
};
}
@@ -385,82 +474,83 @@ private static byte[] prepad(byte[] original) {
return result;
}
- private static BiFunction, Result> indexOrDocValuesQuery() {
- return (query, boosts) -> {
+ private static BiFunction