fix extrusion querying for polygons with coincident points

src/style/style_layer/fill_extrusion_style_layer.js

@@ -67,7 +67,7 @@ function dot(a, b) {
     return a.x * b.x + a.y * b.y;
 }
 
-function getIntersectionDistance(projectedQueryGeometry: Array<Point>, projectedFace: Array<Point>) {
+export function getIntersectionDistance(projectedQueryGeometry: Array<Point>, projectedFace: Array<Point>) {
 
     if (projectedQueryGeometry.length === 1) {
         // For point queries calculate the z at which the point intersects the face
@@ -77,9 +77,20 @@ function getIntersectionDistance(projectedQueryGeometry: Array<Point>, projected
         // triangle of the face, using only the xy plane. It doesn't matter if the
         // point is outside the first triangle because all the triangles in the face
         // are in the same plane.
-        const a = projectedFace[0];
-        const b = projectedFace[1];
-        const c = projectedFace[3];
+        //
+        // Check whether points are coincident and use other points if they are.
+        let i = 0;
+        const a = projectedFace[i++];
+        let b, c;
+        while (!b || a.equals(b)) {
+            b = projectedFace[i++];
+            if (!b) return Infinity;
+        }
+        while (!c || a.equals(c) || b.equals(c)) {
+            c = projectedFace[i++];
+            if (!c) return Infinity;
+        }
+
         const p = projectedQueryGeometry[0];
 
         const ab = b.sub(a);

test/integration/query-tests/regressions/mapbox-gl-js#8999/expected.json

@@ -0,0 +1,33 @@
+[
+  {
+    "geometry": {
+      "type": "Polygon",
+      "coordinates": [
+        [
+          [
+            -19.9951171875,
+            -40.01078714046552
+          ],
+          [
+            -19.9951171875,
+            40.01078714046551
+          ],
+          [
+            39.990234375,
+            40.01078714046551
+          ],
+          [
+            -19.9951171875,
+            -40.01078714046552
+          ]
+        ]
+      ]
+    },
+    "type": "Feature",
+    "properties": {
+      "layer": "extrusion_three_points"
+    },
+    "source": "extrusion_three_points",
+    "state": {}
+  }
+]

test/integration/query-tests/regressions/mapbox-gl-js#8999/style.json

@@ -0,0 +1,65 @@
+{
+  "version": 8,
+  "metadata": {
+    "test": {
+      "debug": true,
+      "width": 200,
+      "height": 200,
+      "queryGeometry": [
+          100,
+          40
+      ]
+    }
+  },
+  "pitch": 0,
+  "center": [
+      0,
+      0
+  ],
+  "zoom": 0,
+  "sources": {
+    "extrusion_three_points": {
+      "type": "geojson",
+        "data": {
+            "type": "Feature",
+            "properties": {
+                "layer": "extrusion_three_points"
+            },
+            "geometry": {
+                "type": "Polygon",
+                "coordinates": [
+                    [
+                        [
+                            -20,
+                            -40
+                        ],
+                        [
+                            -20,
+                            40
+                        ],
+                        [
+                            40,
+                            40
+                        ],
+                        [
+                            -20,
+                            -40
+                        ]
+                    ]
+                ]
+            }
+        }
+	}
+  },
+  "layers": [
+    {
+      "id": "extrusion_three_points",
+      "type": "fill-extrusion",
+      "source": "extrusion_three_points",
+      "paint": {
+        "fill-extrusion-color": "#ecc",
+        "fill-extrusion-height": 0
+      }
+    }
+  ]
+}

test/unit/style/style_layer/fill_extrusion_style_layer.js

@@ -0,0 +1,45 @@
+import {test} from '../../../util/test';
+import {getIntersectionDistance} from '../../../../src/style/style_layer/fill_extrusion_style_layer';
+import Point from '@mapbox/point-geometry';
+
+test('getIntersectionDistance', (t) => {
+    const queryPoint = [new Point(100, 100)];
+    const z = 3;
+    const a = new Point(100, -90);
+    const b = new Point(110, 110);
+    const c = new Point(-110, 110);
+    a.z = z;
+    b.z = z;
+    c.z = z;
+
+    t.test('one point', (t) => {
+        const projectedFace = [a, a];
+        t.equal(getIntersectionDistance(queryPoint, projectedFace), Infinity);
+        t.end();
+    });
+
+    t.test('two points', (t) => {
+        const projectedFace = [a, b, a];
+        t.equal(getIntersectionDistance(queryPoint, projectedFace), Infinity);
+        t.end();
+    });
+
+    t.test('two points coincident', (t) => {
+        const projectedFace = [a, a, a, b, b, b, b, a];
+        t.equal(getIntersectionDistance(queryPoint, projectedFace), Infinity);
+        t.end();
+    });
+
+    t.test('three points', (t) => {
+        const projectedFace = [a, b, c, a];
+        t.equal(getIntersectionDistance(queryPoint, projectedFace), z);
+        t.end();
+    });
+
+    t.test('three points coincident points', (t) => {
+        const projectedFace = [a, a, b, b, b, c, c, a];
+        t.equal(getIntersectionDistance(queryPoint, projectedFace), z);
+        t.end();
+    });
+    t.end();
+});