Skip to content

Commit

Permalink
Improved 3D transform ingestion & affine transform support (#2102)
Browse files Browse the repository at this point in the history 
* Transform is now Transform3D and has an affine transform with various options

* todo notes on sparse enums in transform3d

* from instead of into glam impls

* add more utilities, port rust examples

* better ui display for transform3d

* transform3d now also captures direction parent/child child/parent

* Fix images with same render order not layering transparently

* nicer draw order sample

* every draw order given out is individual again to avoid z fighting

* affine3 python interface wip

* unified affine transform logging from python

* hide zero translations

* fix up log_rigid3 and deprecate it

* example for log_affine3

* Rename RotationAxisAngle

* py-lint fixes

* re-enable data_table_sizes_basics test

* more docs and doc tests for transform3d

* implement pinhole via transform3d.py

* spelling

* better affine transform error messages on python, fix tensor logging again

* linting, small improvements

* update all uses of log_rigid3 in python examples

* fix old python incompatibility issues

* split up transform enum into three different components
rust only so far

* wire up python sdk again to new components

* fix pinhole camera not being categorized as spatial

* enable custom pinhole ui

* Rename to TranslationAndMat3

* fixup some tests in re_log_types

* mono component documentation

* fixup test_clean_for_polars_modify

* change python api to log_transform3d

* explicit quaternion type for python

* Translation3D is now its own enum to allow explicit non-logged translation

* fix quaternion logging in examples

* slightly better error messages on wrong type in python transform api

* Rust fmt

* rust test fix

* Avoid using attr.dataclass

* comment and cosmetic fixes, fixes for ros demo

* translation/rotation/scale are now options

* rename creation methods in rust api fro Transform3D

* fixup SpaceInfoConnection to be more versatile

* Rigid3D helper class on python

* a bit nicer rust api, less word duplication

* Quaternion is no longer a dataclass  in order to enforce spelling out the ordering

* fix documentation of datalayout for matrix on python transform api

* test/lint fixes

* doc fix

* fixup ros demo

---------

Co-authored-by: Jeremy Leibs <[email protected]>
  • Loading branch information
@Wumpf @jleibs @emilk
2 people authored and emilk committed Jun 15, 2023
1 parent 631c268 commit 1d30c6e
Show file tree
Hide file tree
Showing 63 changed files with 2,191 additions and 921 deletions.
151 changes: 53 additions & 98 deletions crates/re_arrow_store/src/arrow_util.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -233,109 +233,64 @@ fn test_clean_for_polars_nomodify() {
assert_eq!(cell.as_arrow_ref(), &*cleaned);
}

#[test]
fn test_clean_for_polars_modify() {
use re_log_types::{DataCell, Pinhole, Transform};
// transforms are a nice pathological type with both Unions and FixedSizeLists
let transforms = vec![Transform::Pinhole(Pinhole {
image_from_cam: [[1.0, 2.0, 3.0], [4.0, 5.0, 6.0], [7.0, 8.0, 9.0]].into(),
resolution: None,
})];
#[cfg(test)]
mod tests {
use arrow2::datatypes::{DataType, Field, UnionMode};
use arrow2_convert::{ArrowDeserialize, ArrowField, ArrowSerialize};
use re_log_types::{component_types::Vec3D, Component, DataCell};

let cell: DataCell = transforms.try_into().unwrap();
assert_eq!(
*cell.datatype(),
DataType::Union(
vec![
Field::new("Unknown", DataType::Boolean, false),
Field::new(
"Rigid3",
DataType::Struct(vec![
Field::new(
"rotation",
DataType::FixedSizeList(
Box::new(Field::new("item", DataType::Float32, false)),
4
),
false
),
Field::new(
"translation",
DataType::FixedSizeList(
Box::new(Field::new("item", DataType::Float32, false)),
3
),
false
)
]),
false
),
Field::new(
"Pinhole",
DataType::Struct(vec![
Field::new(
"image_from_cam",
DataType::FixedSizeList(
Box::new(Field::new("item", DataType::Float32, false)),
9
),
false,
),
Field::new(
"resolution",
DataType::FixedSizeList(
Box::new(Field::new("item", DataType::Float32, false)),
2
),
true,
),
]),
false
)
],
None,
UnionMode::Dense
),
);
use crate::ArrayExt;

let cleaned = cell.as_arrow_ref().clean_for_polars();
#[derive(Clone, Copy, Debug, PartialEq, ArrowField, ArrowSerialize, ArrowDeserialize)]
#[arrow_field(type = "dense")]
enum TestComponentWithUnionAndFixedSizeList {
Bool(bool),
Vec3D(Vec3D),
}

assert_eq!(
*cleaned.data_type(),
DataType::Struct(vec![
Field::new("Unknown", DataType::Boolean, false),
Field::new(
"Rigid3",
DataType::Struct(vec![
Field::new(
"rotation",
DataType::List(Box::new(Field::new("item", DataType::Float32, false)),),
false
),
impl Component for TestComponentWithUnionAndFixedSizeList {
fn name() -> re_log_types::ComponentName {
"test_component_with_union_and_fixed_size_list".into()
}
}

#[test]
fn test_clean_for_polars_modify() {
// Pick a type with both Unions and FixedSizeLists
let elements = vec![TestComponentWithUnionAndFixedSizeList::Bool(false)];

let cell: DataCell = elements.try_into().unwrap();
assert_eq!(
*cell.datatype(),
DataType::Union(
vec![
Field::new("Bool", DataType::Boolean, false),
Field::new(
"translation",
DataType::List(Box::new(Field::new("item", DataType::Float32, false)),),
"Vec3D",
DataType::FixedSizeList(
Box::new(Field::new("item", DataType::Float32, false)),
3
),
false
)
]),
false
),
Field::new(
"Pinhole",
DataType::Struct(vec![
Field::new(
"image_from_cam",
DataType::List(Box::new(Field::new("item", DataType::Float32, false))),
false,
),
Field::new(
"resolution",
DataType::List(Box::new(Field::new("item", DataType::Float32, false))),
true,
),
]),
false
],
None,
UnionMode::Dense
)
],),
);
);

let cleaned = cell.as_arrow_ref().clean_for_polars();

assert_eq!(
*cleaned.data_type(),
DataType::Struct(vec![
Field::new("Bool", DataType::Boolean, false),
Field::new(
"Vec3D",
DataType::List(Box::new(Field::new("item", DataType::Float32, false))),
false
)
],)
);
}
}
3 changes: 2 additions & 1 deletion crates/re_data_ui/src/component_ui_registry.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ pub fn create_component_ui_registry() -> ComponentUiRegistry {
add::<re_log_types::component_types::Mesh3D>(&mut registry);
// add::<re_log_types::component_types::Point2D>(&mut registry);
// add::<re_log_types::component_types::Point3D>(&mut registry);
add::<re_log_types::component_types::Pinhole>(&mut registry);
// add::<re_log_types::component_types::Quaternion>(&mut registry);
// add::<re_log_types::component_types::Radius>(&mut registry);
add::<re_log_types::component_types::Rect2D>(&mut registry);
Expand All @@ -59,7 +60,7 @@ pub fn create_component_ui_registry() -> ComponentUiRegistry {
// add::<re_log_types::component_types::Size3D>(&mut registry);
add::<re_log_types::component_types::Tensor>(&mut registry);
add::<re_log_types::component_types::TextEntry>(&mut registry);
add::<re_log_types::component_types::Transform>(&mut registry);
add::<re_log_types::component_types::Transform3D>(&mut registry);
add::<re_log_types::component_types::Vec2D>(&mut registry);
add::<re_log_types::component_types::Vec3D>(&mut registry);
add::<re_log_types::ViewCoordinates>(&mut registry);
Expand Down
108 changes: 2 additions & 106 deletions crates/re_data_ui/src/data.rs
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,8 @@
use egui::Vec2;

use re_format::format_f32;
use re_log_types::{
component_types::ColorRGBA,
component_types::{LineStrip2D, LineStrip3D, Mat3x3, Rect2D, Vec2D, Vec3D, Vec4D},
Pinhole, Rigid3, Transform, ViewCoordinates,
use re_log_types::component_types::{
ColorRGBA, LineStrip2D, LineStrip3D, Mat3x3, Rect2D, Vec2D, Vec3D, Vec4D, ViewCoordinates,
};
use re_viewer_context::{UiVerbosity, ViewerContext};

Expand Down Expand Up @@ -53,24 +51,6 @@ impl DataUi for ColorRGBA {
}
}

impl DataUi for Transform {
fn data_ui(
&self,
ctx: &mut ViewerContext<'_>,
ui: &mut egui::Ui,
verbosity: UiVerbosity,
query: &re_arrow_store::LatestAtQuery,
) {
match self {
Transform::Unknown => {
ui.label("Unknown transform");
}
Transform::Rigid3(rigid3) => rigid3.data_ui(ctx, ui, verbosity, query),
Transform::Pinhole(pinhole) => pinhole.data_ui(ctx, ui, verbosity, query),
}
}
}

impl DataUi for ViewCoordinates {
fn data_ui(
&self,
Expand All @@ -90,90 +70,6 @@ impl DataUi for ViewCoordinates {
}
}

impl DataUi for Rigid3 {
#[allow(clippy::only_used_in_recursion)]
fn data_ui(
&self,
ctx: &mut ViewerContext<'_>,
ui: &mut egui::Ui,
verbosity: UiVerbosity,
query: &re_arrow_store::LatestAtQuery,
) {
match verbosity {
UiVerbosity::Small => {
ui.label("Rigid 3D transform").on_hover_ui(|ui| {
self.data_ui(ctx, ui, UiVerbosity::All, query);
});
}

UiVerbosity::All | UiVerbosity::Reduced => {
let pose = self.parent_from_child(); // TODO(emilk): which one to show?
let rotation = pose.rotation();
let translation = pose.translation();

ui.vertical(|ui| {
ui.label("Rigid 3D transform:");
ui.indent("rigid3", |ui| {
egui::Grid::new("rigid3").num_columns(2).show(ui, |ui| {
ui.label("rotation");
ui.monospace(format!("{rotation:?}"));
ui.end_row();

ui.label("translation");
ui.monospace(format!("{translation:?}"));
ui.end_row();
});
});
});
}
}
}
}

impl DataUi for Pinhole {
fn data_ui(
&self,
ctx: &mut ViewerContext<'_>,
ui: &mut egui::Ui,
verbosity: UiVerbosity,
query: &re_arrow_store::LatestAtQuery,
) {
match verbosity {
UiVerbosity::Small => {
ui.label("Pinhole transform").on_hover_ui(|ui| {
self.data_ui(ctx, ui, UiVerbosity::All, query);
});
}

UiVerbosity::All | UiVerbosity::Reduced => {
let Pinhole {
image_from_cam: image_from_view,
resolution,
} = self;

ui.vertical(|ui| {
ui.label("Pinhole transform:");
ui.indent("pinole", |ui| {
ui.horizontal(|ui| {
ui.label("resolution:");
if let Some(re_log_types::component_types::Vec2D([x, y])) = resolution {
ui.monospace(format!("{x}x{y}"));
} else {
ui.weak("(none)");
}
});

ui.label("image from view:");
ui.indent("image_from_view", |ui| {
image_from_view.data_ui(ctx, ui, verbosity, query);
});
});
});
}
}
}
}

impl DataUi for Mat3x3 {
fn data_ui(
&self,
Expand Down
2 changes: 2 additions & 0 deletions crates/re_data_ui/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ mod instance_path;
mod item;
pub mod item_ui;
mod log_msg;
mod pinhole;
mod transform3d;

pub use crate::image::{
show_zoomed_image_region, show_zoomed_image_region_area_outline,
Expand Down
48 changes: 48 additions & 0 deletions crates/re_data_ui/src/pinhole.rs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
use re_log_types::component_types::Pinhole;
use re_viewer_context::{UiVerbosity, ViewerContext};

use crate::DataUi;

impl DataUi for Pinhole {
fn data_ui(
&self,
ctx: &mut ViewerContext<'_>,
ui: &mut egui::Ui,
verbosity: UiVerbosity,
query: &re_arrow_store::LatestAtQuery,
) {
match verbosity {
UiVerbosity::Small => {
ui.label("Pinhole transform").on_hover_ui(|ui| {
self.data_ui(ctx, ui, UiVerbosity::All, query);
});
}

UiVerbosity::All | UiVerbosity::Reduced => {
let Pinhole {
image_from_cam: image_from_view,
resolution,
} = self;

ui.vertical(|ui| {
ui.label("Pinhole transform:");
ui.indent("pinole", |ui| {
ui.horizontal(|ui| {
ui.label("resolution:");
if let Some(re_log_types::component_types::Vec2D([x, y])) = resolution {
ui.monospace(format!("{x}x{y}"));
} else {
ui.weak("(none)");
}
});

ui.label("image from view:");
ui.indent("image_from_view", |ui| {
image_from_view.data_ui(ctx, ui, verbosity, query);
});
});
});
}
}
}
}
Loading

0 comments on commit 1d30c6e

Please sign in to comment.