improve 3d bbox projection logic

app/packages/looker/package.json

@@ -26,6 +26,7 @@
         "lodash": "^4.17.21",
         "lru-cache": "^6.0.0",
         "mime": "^2.5.2",
+        "monotone-convex-hull-2d": "^1.0.1",
         "uuid": "^8.3.2"
     },
     "devDependencies": {

app/packages/looker/src/overlays/detection.ts

@@ -21,6 +21,7 @@ export interface DetectionLabel extends RegularLabel {
   dimensions?: [number, number, number];
   location?: [number, number, number];
   rotation?: [number, number, number];
+  convexHull?: Coordinates[];
 }
 
 export default class DetectionOverlay<
@@ -229,48 +230,32 @@ export default class DetectionOverlay<
     state: Readonly<State>,
     color: string
   ) {
-    const [tlx, tly, w, h] = this.label.bounding_box;
-    const [boxCenterX, boxCenterY] = t(state, tlx + w / 2, tly + h / 2);
-
-    const hasRotationAroundZAxis =
-      this.label.rotation && this.label.rotation[2] !== 0;
-
-    if (hasRotationAroundZAxis) {
-      // translate to center of box before rotating
-      ctx.translate(boxCenterX, boxCenterY);
-      // modifies current transformation matrix so that all subsequent drawings are rotated
-      ctx.rotate(-this.label.rotation[2]);
-      // translate back to undo the translation into the center of the box
-      ctx.translate(-boxCenterX, -boxCenterY);
-    }
+    const convexHull = this.label.convexHull;
 
     const previousAlpha = ctx.globalAlpha;
-    ctx.beginPath();
     // use double stoke width to make the box more visible
     ctx.lineWidth = state.strokeWidth * 2;
     ctx.fillStyle = color;
     ctx.strokeStyle = color;
-    ctx.moveTo(...t(state, tlx, tly));
-    ctx.lineTo(...t(state, tlx + w, tly));
-    ctx.lineTo(...t(state, tlx + w, tly + h));
-    ctx.lineTo(...t(state, tlx, tly + h));
+
+    ctx.beginPath();
+
+    // draw a polyline that defines the convex hull of the projected corners
+    ctx.moveTo(...t(state, convexHull[0][0], convexHull[0][1]));
+    for (let i = 1; i < convexHull.length; i++) {
+      ctx.lineTo(...t(state, convexHull[i][0], convexHull[i][1]));
+    }
+
     ctx.closePath();
     ctx.stroke();
 
     // fill with some transparency
-    ctx.globalAlpha = state.options.alpha * 0.5;
-    ctx.fillRect(...t(state, tlx, tly), w, h);
+    ctx.globalAlpha = state.options.alpha * 0.3;
+
     ctx.fill();
 
     // restore previous alpha
     ctx.globalAlpha = previousAlpha;
-
-    if (hasRotationAroundZAxis) {
-      // undo rotation to reset current transformation matrix
-      ctx.translate(boxCenterX, boxCenterY);
-      ctx.rotate(this.label.rotation[2]);
-      ctx.translate(-boxCenterX, -boxCenterY);
-    }
   }
 
   private strokeRect(

app/packages/looker/src/state.ts

@@ -65,8 +65,9 @@ export type OrthogrpahicProjectionMetadata = {
   filepath: string;
   height: number;
   width: number;
-  min_bound: [number, number];
-  max_bound: [number, number];
+  min_bound: [number, number, number];
+  max_bound: [number, number, number];
+  normal: [number, number, number];
 };
 
 export type GenericLabel = {

app/packages/looker/src/worker/label-3d-projection-utils.test.ts

@@ -0,0 +1,76 @@
+import { describe, expect, it } from "vitest";
+import { Vec3, projectTo2D, rotatePoint } from "./label-3d-projection-utils";
+
+describe("rotatePoint", () => {
+  it("should correctly rotate a point around the origin - y axis", () => {
+    const point: Vec3 = [1, 0, 0];
+    const rotation: Vec3 = [0, Math.PI / 2, 0];
+    const rotatedPoint = rotatePoint(point, rotation);
+    expect(rotatedPoint[0]).toBeCloseTo(0);
+    expect(rotatedPoint[1]).toBeCloseTo(0);
+    expect(rotatedPoint[2]).toBeCloseTo(-1);
+  });
+
+  it("should correctly rotate a point around the origin - x axis", () => {
+    const point: Vec3 = [1, 0, 0];
+    const rotation: Vec3 = [Math.PI / 2, 0, 0];
+    const rotatedPoint = rotatePoint(point, rotation);
+    expect(rotatedPoint[0]).toBeCloseTo(1);
+    expect(rotatedPoint[1]).toBeCloseTo(0);
+    expect(rotatedPoint[2]).toBeCloseTo(0);
+  });
+
+  it("should correctly rotate a point around the origin - z axis", () => {
+    const point: Vec3 = [1, 0, 0];
+    const rotation: Vec3 = [0, 0, Math.PI / 2];
+    const rotatedPoint = rotatePoint(point, rotation);
+    expect(rotatedPoint[0]).toBeCloseTo(0);
+    expect(rotatedPoint[1]).toBeCloseTo(1);
+    expect(rotatedPoint[2]).toBeCloseTo(0);
+  });
+
+  it("should correctly rotate a point around the origin - z axis (2)", () => {
+    const point: Vec3 = [1, 0, 0];
+    const rotation: Vec3 = [0, 0, Math.PI];
+    const rotatedPoint = rotatePoint(point, rotation);
+    expect(rotatedPoint[0]).toBeCloseTo(-1);
+    expect(rotatedPoint[1]).toBeCloseTo(0);
+    expect(rotatedPoint[2]).toBeCloseTo(0);
+  });
+
+  it("should correctly rotate a point around the origin - xyz axis", () => {
+    const point: Vec3 = [1, 0, 0];
+    const rotation: Vec3 = [0, Math.PI / 4, Math.PI / 2];
+    const rotatedPoint = rotatePoint(point, rotation);
+    expect(rotatedPoint[0]).toBeCloseTo(0);
+    expect(rotatedPoint[1]).toBeCloseTo(Math.sqrt(2) / 2);
+    expect(rotatedPoint[2]).toBeCloseTo(-Math.sqrt(2) / 2);
+  });
+
+  it("should handle no rotation", () => {
+    const point: Vec3 = [1, 2, 3];
+    const rotation: Vec3 = [0, 0, 0];
+    const rotatedPoint = rotatePoint(point, rotation);
+    expect(rotatedPoint).toEqual([1, 2, 3]);
+  });
+});
+
+describe("projectTo2D", () => {
+  it("should project a point to the xz plane", () => {
+    const point: Vec3 = [1, 2, 3];
+    const projectedPoint = projectTo2D(point, "xz");
+    expect(projectedPoint).toEqual([1, 3]);
+  });
+
+  it("should project a point to the xy plane", () => {
+    const point: Vec3 = [1, 2, 3];
+    const projectedPoint = projectTo2D(point, "xy");
+    expect(projectedPoint).toEqual([1, 2]);
+  });
+
+  it("should project a point to the yz plane", () => {
+    const point: Vec3 = [1, 2, 3];
+    const projectedPoint = projectTo2D(point, "yz");
+    expect(projectedPoint).toEqual([2, 3]);
+  });
+});

app/packages/looker/src/worker/label-3d-projection-utils.ts

@@ -0,0 +1,103 @@
+export type Vec3 = [number, number, number];
+export type Vec2 = [number, number];
+
+export interface BoundingBox3D {
+  dimensions: Vec3;
+  location: Vec3;
+  rotation: Vec3; // rotation angles in radians
+}
+
+export interface BoundingBox2D {
+  tlx: number; // top-left corner of the bounding box, x
+  tly: number; // top-left corner of the bounding box, y
+  width: number; // width of the bounding box
+  height: number; // height of the bounding box
+}
+
+export const rotatePoint = (point: Vec3, rotation: Vec3): Vec3 => {
+  // https://en.wikipedia.org/wiki/Rotation_matrix
+
+  const [x, y, z] = point;
+  const [rx, ry, rz] = rotation;
+
+  const cosX = Math.cos(rx);
+  const sinX = Math.sin(rx);
+  const cosY = Math.cos(ry);
+  const sinY = Math.sin(ry);
+  const cosZ = Math.cos(rz);
+  const sinZ = Math.sin(rz);
+
+  // rotation around X axis
+  const y1 = cosX * y - sinX * z;
+  const z1 = sinX * y + cosX * z;
+
+  // rotation around Y axis
+  const x2 = cosY * x + sinY * z1;
+  const z2 = cosY * z1 - sinY * x;
+
+  // rotation around Z axis
+  const xRotated = cosZ * x2 - sinZ * y1;
+  const yRotated = sinZ * x2 + cosZ * y1;
+  const zRotated = z2;
+
+  return [xRotated, yRotated, zRotated];
+};
+
+export const projectTo2D = (point: Vec3, plane: "xz" | "xy" | "yz"): Vec2 => {
+  switch (plane) {
+    case "xz":
+      return [point[0], point[2]];
+    case "xy":
+      return [point[0], point[1]];
+    case "yz":
+      return [point[1], point[2]];
+  }
+};
+
+export const getProjectedCorners = (
+  box: BoundingBox3D,
+  plane: "xz" | "xy" | "yz"
+) => {
+  const { dimensions, location, rotation } = box;
+  const [dx, dy, dz] = dimensions;
+  const halfDimensions = [dx / 2, dy / 2, dz / 2] as Vec3;
+
+  // Generate the 8 corners of the 3D bounding box
+  const corners: Vec3[] = [
+    // left bottom back
+    [-halfDimensions[0], -halfDimensions[1], -halfDimensions[2]],
+    // left bottom front
+    [-halfDimensions[0], -halfDimensions[1], halfDimensions[2]],
+    // left top back
+    [-halfDimensions[0], halfDimensions[1], -halfDimensions[2]],
+    // left top front
+    [-halfDimensions[0], halfDimensions[1], halfDimensions[2]],
+    // right bottom back
+    [halfDimensions[0], -halfDimensions[1], -halfDimensions[2]],
+    // right bottom front
+    [halfDimensions[0], -halfDimensions[1], halfDimensions[2]],
+    // right top back
+    [halfDimensions[0], halfDimensions[1], -halfDimensions[2]],
+    // right top front
+    [halfDimensions[0], halfDimensions[1], halfDimensions[2]],
+  ];
+
+  // rotate first, and translate
+  const transformedCorners = corners.map((corner) => {
+    const newRotation = rotation;
+
+    const rotated = rotatePoint(corner, newRotation);
+    return [
+      rotated[0] + location[0],
+      rotated[1] + location[1],
+      rotated[2] + location[2],
+    ] as Vec3;
+  });
+
+  // project the 3D points to 2D based on the specified plane
+  const projectedCorners: Vec2[] = transformedCorners.map((corner) =>
+    projectTo2D(corner, plane)
+  );
+
+  return { projectedCorners };
+};