improve 3d bbox projection logic

app/packages/looker/package.json

@@ -26,6 +26,7 @@
         "lodash": "^4.17.21",
         "lru-cache": "^6.0.0",
         "mime": "^2.5.2",
+        "monotone-convex-hull-2d": "^1.0.1",
         "uuid": "^8.3.2"
     },
     "devDependencies": {

app/packages/looker/src/overlays/detection.ts

@@ -21,6 +21,7 @@ export interface DetectionLabel extends RegularLabel {
   dimensions?: [number, number, number];
   location?: [number, number, number];
   rotation?: [number, number, number];
+  convexHull?: Coordinates[];
 }
 
 export default class DetectionOverlay<
@@ -229,48 +230,32 @@ export default class DetectionOverlay<
     state: Readonly<State>,
     color: string
   ) {
-    const [tlx, tly, w, h] = this.label.bounding_box;
-    const [boxCenterX, boxCenterY] = t(state, tlx + w / 2, tly + h / 2);
-
-    const hasRotationAroundZAxis =
-      this.label.rotation && this.label.rotation[2] !== 0;
-
-    if (hasRotationAroundZAxis) {
-      // translate to center of box before rotating
-      ctx.translate(boxCenterX, boxCenterY);
-      // modifies current transformation matrix so that all subsequent drawings are rotated
-      ctx.rotate(-this.label.rotation[2]);
-      // translate back to undo the translation into the center of the box
-      ctx.translate(-boxCenterX, -boxCenterY);
-    }
+    const convexHull = this.label.convexHull;
 
     const previousAlpha = ctx.globalAlpha;
-    ctx.beginPath();
     // use double stoke width to make the box more visible
     ctx.lineWidth = state.strokeWidth * 2;
     ctx.fillStyle = color;
     ctx.strokeStyle = color;
-    ctx.moveTo(...t(state, tlx, tly));
-    ctx.lineTo(...t(state, tlx + w, tly));
-    ctx.lineTo(...t(state, tlx + w, tly + h));
-    ctx.lineTo(...t(state, tlx, tly + h));
+
+    ctx.beginPath();
+
+    // draw a polyline that defines the convex hull of the projected corners and fill it
+    ctx.moveTo(...t(state, convexHull[0][0], convexHull[0][1]));
+    for (let i = 1; i < convexHull.length; i++) {
+      ctx.lineTo(...t(state, convexHull[i][0], convexHull[i][1]));
+    }
+
     ctx.closePath();
     ctx.stroke();
 
     // fill with some transparency
-    ctx.globalAlpha = state.options.alpha * 0.5;
-    ctx.fillRect(...t(state, tlx, tly), w, h);
+    ctx.globalAlpha = state.options.alpha * 0.3;
+
     ctx.fill();
 
     // restore previous alpha
     ctx.globalAlpha = previousAlpha;
-
-    if (hasRotationAroundZAxis) {
-      // undo rotation to reset current transformation matrix
-      ctx.translate(boxCenterX, boxCenterY);
-      ctx.rotate(this.label.rotation[2]);
-      ctx.translate(-boxCenterX, -boxCenterY);
-    }
   }
 
   private strokeRect(

app/packages/looker/src/state.ts

@@ -65,8 +65,9 @@ export type OrthogrpahicProjectionMetadata = {
   filepath: string;
   height: number;
   width: number;
-  min_bound: [number, number];
-  max_bound: [number, number];
+  min_bound: [number, number, number];
+  max_bound: [number, number, number];
+  normal: [number, number, number];
 };
 
 export type GenericLabel = {

app/packages/looker/src/worker/label-3d-projection-utils.test.ts

@@ -0,0 +1,22 @@
+import { describe, expect, it } from "vitest";
+import { Vec3, projectTo2D } from "./label-3d-projection-utils";
+
+describe("projectTo2D", () => {
+  it("should project a point to the xz plane", () => {
+    const point: Vec3 = [1, 2, 3];
+    const projectedPoint = projectTo2D(point, "xz");
+    expect(projectedPoint).toEqual([1, 3]);
+  });
+
+  it("should project a point to the xy plane", () => {
+    const point: Vec3 = [1, 2, 3];
+    const projectedPoint = projectTo2D(point, "xy");
+    expect(projectedPoint).toEqual([1, 2]);
+  });
+
+  it("should project a point to the yz plane", () => {
+    const point: Vec3 = [1, 2, 3];
+    const projectedPoint = projectTo2D(point, "yz");
+    expect(projectedPoint).toEqual([2, 3]);
+  });
+});

app/packages/looker/src/worker/label-3d-projection-utils.ts

@@ -0,0 +1,83 @@
+import { Euler, Vector3 } from "three";
+
+export type Vec3 = [number, number, number];
+export type Vec2 = [number, number];
+
+export interface BoundingBox3D {
+  dimensions: Vec3;
+  location: Vec3;
+  rotation: Vec3; // rotation angles in radians
+}
+
+export interface BoundingBox2D {
+  tlx: number; // top-left corner of the bounding box, x
+  tly: number; // top-left corner of the bounding box, y
+  width: number; // width of the bounding box
+  height: number; // height of the bounding box
+}
+
+export const rotatePoint = (point: Vec3, rotation: Vec3): Vec3 => {
+  const threePoint = new Vector3(...point);
+  const threeRotation = new Euler(...rotation);
+
+  return threePoint.applyEuler(threeRotation).toArray() as Vec3;
+};
+
+export const projectTo2D = (point: Vec3, plane: "xz" | "xy" | "yz"): Vec2 => {
+  switch (plane) {
+    case "xz":
+      return [point[0], point[2]];
+    case "xy":
+      return [point[0], point[1]];
+    case "yz":
+      return [point[1], point[2]];
+  }
+};
+
+export const getProjectedCorners = (
+  box: BoundingBox3D,
+  plane: "xz" | "xy" | "yz"
+) => {
+  const { dimensions, location, rotation } = box;
+  const [dx, dy, dz] = dimensions;
+  const halfDimensions = [dx / 2, dy / 2, dz / 2] as Vec3;
+
+  // Generate the 8 corners of the 3D bounding box
+  const corners: Vec3[] = [
+    // left bottom back
+    [-halfDimensions[0], -halfDimensions[1], -halfDimensions[2]],
+    // left bottom front
+    [-halfDimensions[0], -halfDimensions[1], halfDimensions[2]],
+    // left top back
+    [-halfDimensions[0], halfDimensions[1], -halfDimensions[2]],
+    // left top front
+    [-halfDimensions[0], halfDimensions[1], halfDimensions[2]],
+    // right bottom back
+    [halfDimensions[0], -halfDimensions[1], -halfDimensions[2]],
+    // right bottom front
+    [halfDimensions[0], -halfDimensions[1], halfDimensions[2]],
+    // right top back
+    [halfDimensions[0], halfDimensions[1], -halfDimensions[2]],
+    // right top front
+    [halfDimensions[0], halfDimensions[1], halfDimensions[2]],
+  ];
+
+  // rotate first, and translate
+  const transformedCorners = corners.map((corner) => {
+    const newRotation = rotation;
+
+    const rotated = rotatePoint(corner, newRotation);
+    return [
+      rotated[0] + location[0],
+      rotated[1] + location[1],
+      rotated[2] + location[2],
+    ] as Vec3;
+  });
+
+  // project the 3D points to 2D based on the specified plane
+  const projectedCorners: Vec2[] = transformedCorners.map((corner) =>
+    projectTo2D(corner, plane)
+  );
+
+  return { projectedCorners };
+};