From c360db4149de1d7ca0cf1ea5ce46763fc410d356 Mon Sep 17 00:00:00 2001 From: mosure Date: Wed, 24 Jan 2024 17:00:25 -0600 Subject: [PATCH 01/10] feat: conditional cov3d --- src/gaussian/f32.rs | 61 ++++++++++++++ src/gaussian/packed.rs | 28 ++++++- src/material/spherindrical_harmonics.rs | 0 src/material/spherindrical_harmonics.wgsl | 0 src/render/gaussian.wgsl | 21 +++++ src/render/gaussian_4d.wgsl | 98 +++++++++++++++++++++++ 6 files changed, 207 insertions(+), 1 deletion(-) create mode 100644 src/material/spherindrical_harmonics.rs create mode 100644 src/material/spherindrical_harmonics.wgsl create mode 100644 src/render/gaussian_4d.wgsl diff --git a/src/gaussian/f32.rs b/src/gaussian/f32.rs index 34b62aea..faa2c985 100644 --- a/src/gaussian/f32.rs +++ b/src/gaussian/f32.rs @@ -53,6 +53,41 @@ impl From<[f32; 4]> for PositionVisibility { } } + +#[derive( + Clone, + Debug, + Default, + Copy, + PartialEq, + Reflect, + ShaderType, + Pod, + Zeroable, + Serialize, + Deserialize, +)] +#[repr(C)] +pub struct PositionOpacity { + pub position: Position, + pub opacity: f32, +} + +impl From<[f32; 4]> for PositionOpacity { + fn from(position_visibility: [f32; 4]) -> Self { + Self { + position: [ + position_visibility[0], + position_visibility[1], + position_visibility[2], + ], + opacity: position_visibility[3], + } + } +} + + + #[derive( Clone, Debug, @@ -77,6 +112,32 @@ impl From<[f32; 4]> for Rotation { } } + +#[derive( + Clone, + Debug, + Default, + Copy, + PartialEq, + Reflect, + ShaderType, + Pod, + Zeroable, + Serialize, + Deserialize, +)] +#[repr(C)] +pub struct Scale4d { + pub scale: [f32; 4], +} + +impl From<[f32; 4]> for Scale4d { + fn from(scale: [f32; 4]) -> Self { + Self { scale } + } +} + + #[derive( Clone, Debug, diff --git a/src/gaussian/packed.rs b/src/gaussian/packed.rs index c978ea2e..04b2c662 100644 --- a/src/gaussian/packed.rs +++ b/src/gaussian/packed.rs @@ -12,11 +12,16 @@ use serde::{ use crate::{ gaussian::f32::{ + PositionOpacity, PositionVisibility, Rotation, + Scale4d, ScaleOpacity, }, - material::spherical_harmonics::SphericalHarmonicCoefficients, + material::spherical_harmonics::{ + SphericalHarmonicCoefficients, + SpherindricalHarmonicCoefficients, + }, }; #[derive( @@ -38,3 +43,24 @@ pub struct Gaussian { pub scale_opacity: ScaleOpacity, pub spherical_harmonic: SphericalHarmonicCoefficients, } + + +#[derive( + Clone, + Debug, + Default, + Copy, + PartialEq, + Reflect, + Pod, + Zeroable, + Serialize, + Deserialize, +)] +#[repr(C)] +pub struct Gaussian4d { + pub isomorphic_rotations: [Rotation; 2], + pub position_opacity: PositionOpacity, + pub scale: Scale4d, + pub spherindrical_harmonic: SpherindricalHarmonicCoefficients, +} diff --git a/src/material/spherindrical_harmonics.rs b/src/material/spherindrical_harmonics.rs new file mode 100644 index 00000000..e69de29b diff --git a/src/material/spherindrical_harmonics.wgsl b/src/material/spherindrical_harmonics.wgsl new file mode 100644 index 00000000..e69de29b diff --git a/src/render/gaussian.wgsl b/src/render/gaussian.wgsl index 5db51cd8..864e35cc 100644 --- a/src/render/gaussian.wgsl +++ b/src/render/gaussian.wgsl @@ -12,6 +12,7 @@ #import bevy_gaussian_splatting::depth::{ depth_to_rgb, } +#import bevy_gaussian_splatting::gaussian_4d::compute_cov3d_conditional #import bevy_gaussian_splatting::transform::{ world_to_clip, in_frustum, @@ -181,7 +182,27 @@ fn compute_cov2d( let rotation = get_rotation(index); let scale = get_scale(index); +#ifdef GAUSSIAN_4d + let rotation_r = get_rotation_r(index); + + let decomposed = compute_cov3d_conditional( + position, + scale, + rotation, + rotation_r, + ); + + // TODO: propagate to discard_quad + if decomposed.mask { + return vec3(0.0, 0.0, 0.0); + } + + let cov3d = decomposed.cov3d; + let position = position + decomposed.delta_mean; + let opacity = decomposed.opacity; +#else let cov3d = compute_cov3d(scale, rotation); +#endif #endif let Vrk = mat3x3( diff --git a/src/render/gaussian_4d.wgsl b/src/render/gaussian_4d.wgsl new file mode 100644 index 00000000..247cfa9b --- /dev/null +++ b/src/render/gaussian_4d.wgsl @@ -0,0 +1,98 @@ +#define_import_path bevy_gaussian_splatting::gaussian_4d + +#import bevy_gaussian_splatting::bindings::{ + view, + globals, + gaussian_uniforms, +} + + +struct DecomposedGaussian4d { + cov3d: array, + delta_mean: vec3, + opacity: f32, + mask: bool, +} + + +fn compute_cov3d_conditional( + position: vec3, + scale: vec4, + rotation: vec4, + rotation_r: vec4, +) -> DecomposedGaussian4d { + let dt = globals.delta_time; + + let S = mat4x4( + gaussian_uniforms.global_scale * scale.x, 0.0, 0.0, 0.0, + 0.0, gaussian_uniforms.global_scale * scale.y, 0.0, 0.0, + 0.0, 0.0, gaussian_uniforms.global_scale * scale.z, 0.0, + 0.0, 0.0, 0.0, gaussian_uniforms.global_scale * scale.w, // TODO: separate spatial and time scale uniforms + ); + + let a = rotation.x; + let b = rotation.y; + let c = rotation.z; + let d = rotation.w; + + let p = rotation_r.x; + let q = rotation_r.y; + let r = rotation_r.z; + let s = rotation_r.w; + + let M_l = mat4x4( + a, -b, -c, -d, + b, a, -d, c, + c, d, a, -b, + d, -c, b, a, + ); + + let M_r = mat4x4( + p, q, r, s, + -q, p, -s, r, + -r, s, p, -q, + -s, -r, q, p, + ); + + let R = M_r * M_l; + let M = S * R; + let Sigma = transpose(M) * M; + + let cov_t = Sigma[3][3]; + let marginal_t = exp(-0.5 * dt * dt / cov_t); + + let mask = marginal_t > 0.05; + if (!mask) { + return DecomposedGaussian4d( + 0.0, 0.0, 0.0, + 0.0, 0.0, 0.0, + vec3(0.0, 0.0, 0.0), + 0.0, + false, + ); + } + + let opacity *= marginal_t; + + let cov11 = mat3x3( + Sigma[0][0], Sigma[0][1], Sigma[0][2], + Sigma[1][0], Sigma[1][1], Sigma[1][2], + Sigma[2][0], Sigma[2][1], Sigma[2][2], + ); + let cov12 = vec3(Sigma[0][3], Sigma[1][3], Sigma[2][3]); + let cov3d_condition = cov11 - outerProduct(cov12, cov12) / cov_t; + + let delta_mean = cov12 / cov_t * dt; + + return DecomposedGaussian4d( + cov3d_condition[0][0], + cov3d_condition[0][1], + cov3d_condition[0][2], + cov3d_condition[1][1], + cov3d_condition[1][2], + cov3d_condition[2][2], + delta_mean, + opacity, + mask, + ); +} From 32fd1d5466c0740da55715b70d66c08150f64979 Mon Sep 17 00:00:00 2001 From: mosure Date: Wed, 24 Jan 2024 17:09:44 -0600 Subject: [PATCH 02/10] docs: gaussian cloud creation tools --- CONTRIBUTING.md | 2 +- README.md | 45 ++++-------------- docs/CREDITS.md | 32 +++++++++++++ tools/README.md => docs/TOOLING.md | 0 .../bevy_gaussian_splatting_demo.webp | Bin docs/{ => assets}/bike.png | Bin docs/{ => assets}/cactus.gif | Bin docs/{ => assets}/go.gif | Bin docs/{ => assets}/notferris.png | Bin docs/{ => assets}/notferris2.webp | Bin 10 files changed, 43 insertions(+), 36 deletions(-) create mode 100644 docs/CREDITS.md rename tools/README.md => docs/TOOLING.md (100%) rename docs/{ => assets}/bevy_gaussian_splatting_demo.webp (100%) rename docs/{ => assets}/bike.png (100%) rename docs/{ => assets}/cactus.gif (100%) rename docs/{ => assets}/go.gif (100%) rename docs/{ => assets}/notferris.png (100%) rename docs/{ => assets}/notferris2.webp (100%) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 014df1f5..4b50464e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,6 +1,6 @@ # contributing to bevy_gaussian_splatting -![alt text](docs/notferris2.webp) +![alt text](docs/assets/notferris2.webp) thank you for your interest in contributing to `bevy_gaussian_splatting`! contributions of all forms are welcome and appreciated. this includes code contributions, bug reports, documentation, or any other form of support. diff --git a/README.md b/README.md index 0b0c580c..89ec4d8e 100644 --- a/README.md +++ b/README.md @@ -10,8 +10,8 @@ bevy gaussian splatting render pipeline plugin. view the [live demo](https://mosure.github.io/bevy_gaussian_splatting/index.html?arg1=cactus.gcloud) -![Alt text](docs/bevy_gaussian_splatting_demo.webp) -![Alt text](docs/go.gif) +![alt text](docs/assets/bevy_gaussian_splatting_demo.webp) +![alt text](docs/assets/go.gif) ## capabilities @@ -69,6 +69,13 @@ fn setup_gaussian_cloud( - aabb vs. obb gaussian comparison via `cargo run --bin compare_aabb_obb` +### creating gaussian clouds + +- [X] 3d gaussian clouds: [gaussian-splatting](https://github.com/graphdeco-inria/gaussian-splatting) +- [X] 4d gaussian clouds: [4d-gaussian-splatting](https://fudan-zvg.github.io/4d-gaussian-splatting/) +- [ ] edge-device training pipeline: [burn_gaussian_splatting](https://github.com/mosure/burn_gaussian_splatting) + + ## compatible bevy versions | `bevy_gaussian_splatting` | `bevy` | @@ -76,38 +83,6 @@ fn setup_gaussian_cloud( | `0.4 - 1.0` | `0.12` | | `0.1 - 0.3` | `0.11` | + ## projects using this plugin - [kitt2](https://github.com/cs50victor/kitt2) - -# credits - -- [4d gaussians](https://github.com/hustvl/4DGaussians) -- [4d-gaussian-splatting](https://fudan-zvg.github.io/4d-gaussian-splatting/) -- [bevy](https://github.com/bevyengine/bevy) -- [bevy-hanabi](https://github.com/djeedai/bevy_hanabi) -- [d3ga](https://zielon.github.io/d3ga/) -- [deformable-3d-gaussians](https://github.com/ingra14m/Deformable-3D-Gaussians) -- [diff-gaussian-rasterization](https://github.com/graphdeco-inria/diff-gaussian-rasterization) -- [dreamgaussian](https://github.com/dreamgaussian/dreamgaussian) -- [dynamic-3d-gaussians](https://github.com/JonathonLuiten/Dynamic3DGaussians) -- [ewa splatting](https://www.cs.umd.edu/~zwicker/publications/EWASplatting-TVCG02.pdf) -- [gaussian-grouping](https://github.com/lkeab/gaussian-grouping) -- [gaussian-splatting](https://github.com/graphdeco-inria/gaussian-splatting) -- [gaussian-splatting-viewer](https://github.com/limacv/GaussianSplattingViewer/tree/main) -- [gaussian-splatting-web](https://github.com/cvlab-epfl/gaussian-splatting-web) -- [gir](https://3dgir.github.io/) -- [making gaussian splats smaller](https://aras-p.info/blog/2023/09/13/Making-Gaussian-Splats-smaller/) -- [masked-spacetime-hashing](https://github.com/masked-spacetime-hashing/msth) -- [onesweep](https://arxiv.org/ftp/arxiv/papers/2206/2206.01784.pdf) -- [pasture](https://github.com/Mortano/pasture) -- [phys-gaussian](https://xpandora.github.io/PhysGaussian/) -- [point-visualizer](https://github.com/mosure/point-visualizer) -- [rusty-automata](https://github.com/mosure/rusty-automata) -- [scaffold-gs](https://city-super.github.io/scaffold-gs/) -- [shader-one-sweep](https://github.com/b0nes164/ShaderOneSweep) -- [spacetime-gaussians](https://github.com/oppo-us-research/SpacetimeGaussians) -- [splat](https://github.com/antimatter15/splat) -- [splatter](https://github.com/Lichtso/splatter) -- [sturdy-dollop](https://github.com/mosure/sturdy-dollop) -- [sugar](https://github.com/Anttwo/SuGaR) -- [taichi_3d_gaussian_splatting](https://github.com/wanmeihuali/taichi_3d_gaussian_splatting) diff --git a/docs/CREDITS.md b/docs/CREDITS.md new file mode 100644 index 00000000..f4feaf3f --- /dev/null +++ b/docs/CREDITS.md @@ -0,0 +1,32 @@ +# bevy_gaussian_splatting credits + +- [4d gaussians](https://github.com/hustvl/4DGaussians) +- [4d-gaussian-splatting](https://fudan-zvg.github.io/4d-gaussian-splatting/) +- [bevy](https://github.com/bevyengine/bevy) +- [bevy-hanabi](https://github.com/djeedai/bevy_hanabi) +- [d3ga](https://zielon.github.io/d3ga/) +- [deformable-3d-gaussians](https://github.com/ingra14m/Deformable-3D-Gaussians) +- [diff-gaussian-rasterization](https://github.com/graphdeco-inria/diff-gaussian-rasterization) +- [dreamgaussian](https://github.com/dreamgaussian/dreamgaussian) +- [dynamic-3d-gaussians](https://github.com/JonathonLuiten/Dynamic3DGaussians) +- [ewa splatting](https://www.cs.umd.edu/~zwicker/publications/EWASplatting-TVCG02.pdf) +- [gaussian-grouping](https://github.com/lkeab/gaussian-grouping) +- [gaussian-splatting](https://github.com/graphdeco-inria/gaussian-splatting) +- [gaussian-splatting-viewer](https://github.com/limacv/GaussianSplattingViewer/tree/main) +- [gaussian-splatting-web](https://github.com/cvlab-epfl/gaussian-splatting-web) +- [gir](https://3dgir.github.io/) +- [making gaussian splats smaller](https://aras-p.info/blog/2023/09/13/Making-Gaussian-Splats-smaller/) +- [masked-spacetime-hashing](https://github.com/masked-spacetime-hashing/msth) +- [onesweep](https://arxiv.org/ftp/arxiv/papers/2206/2206.01784.pdf) +- [pasture](https://github.com/Mortano/pasture) +- [phys-gaussian](https://xpandora.github.io/PhysGaussian/) +- [point-visualizer](https://github.com/mosure/point-visualizer) +- [rusty-automata](https://github.com/mosure/rusty-automata) +- [scaffold-gs](https://city-super.github.io/scaffold-gs/) +- [shader-one-sweep](https://github.com/b0nes164/ShaderOneSweep) +- [spacetime-gaussians](https://github.com/oppo-us-research/SpacetimeGaussians) +- [splat](https://github.com/antimatter15/splat) +- [splatter](https://github.com/Lichtso/splatter) +- [sturdy-dollop](https://github.com/mosure/sturdy-dollop) +- [sugar](https://github.com/Anttwo/SuGaR) +- [taichi_3d_gaussian_splatting](https://github.com/wanmeihuali/taichi_3d_gaussian_splatting) diff --git a/tools/README.md b/docs/TOOLING.md similarity index 100% rename from tools/README.md rename to docs/TOOLING.md diff --git a/docs/bevy_gaussian_splatting_demo.webp b/docs/assets/bevy_gaussian_splatting_demo.webp similarity index 100% rename from docs/bevy_gaussian_splatting_demo.webp rename to docs/assets/bevy_gaussian_splatting_demo.webp diff --git a/docs/bike.png b/docs/assets/bike.png similarity index 100% rename from docs/bike.png rename to docs/assets/bike.png diff --git a/docs/cactus.gif b/docs/assets/cactus.gif similarity index 100% rename from docs/cactus.gif rename to docs/assets/cactus.gif diff --git a/docs/go.gif b/docs/assets/go.gif similarity index 100% rename from docs/go.gif rename to docs/assets/go.gif diff --git a/docs/notferris.png b/docs/assets/notferris.png similarity index 100% rename from docs/notferris.png rename to docs/assets/notferris.png diff --git a/docs/notferris2.webp b/docs/assets/notferris2.webp similarity index 100% rename from docs/notferris2.webp rename to docs/assets/notferris2.webp From 49d281499ca2236b65a83dd1a0bd50623f1e0a20 Mon Sep 17 00:00:00 2001 From: mosure Date: Wed, 24 Jan 2024 17:11:42 -0600 Subject: [PATCH 03/10] docs: add a refactor todo --- src/render/gaussian.wgsl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/render/gaussian.wgsl b/src/render/gaussian.wgsl index 864e35cc..b2f015db 100644 --- a/src/render/gaussian.wgsl +++ b/src/render/gaussian.wgsl @@ -200,6 +200,7 @@ fn compute_cov2d( let cov3d = decomposed.cov3d; let position = position + decomposed.delta_mean; let opacity = decomposed.opacity; + // TODO: refactor compute_cov2d to accept cov3d input (allows opacity/updated position to be used in parent) #else let cov3d = compute_cov3d(scale, rotation); #endif From 1b59f70708538197eda3f4623f23dce22783e50a Mon Sep 17 00:00:00 2001 From: mosure Date: Thu, 25 Jan 2024 15:10:55 -0600 Subject: [PATCH 04/10] feat: spherindrical harmonics scaffold --- src/material/mod.rs | 1 + src/material/spherindrical_harmonics.rs | 45 +++++++++++++++++++ src/material/spherindrical_harmonics.wgsl | 54 +++++++++++++++++++++++ 3 files changed, 100 insertions(+) diff --git a/src/material/mod.rs b/src/material/mod.rs index 368720f5..19de651e 100644 --- a/src/material/mod.rs +++ b/src/material/mod.rs @@ -2,6 +2,7 @@ use bevy::prelude::*; pub mod depth; pub mod spherical_harmonics; +pub mod spherindrical_harmonics; #[cfg(feature = "material_noise")] pub mod noise; diff --git a/src/material/spherindrical_harmonics.rs b/src/material/spherindrical_harmonics.rs index e69de29b..205f97f0 100644 --- a/src/material/spherindrical_harmonics.rs +++ b/src/material/spherindrical_harmonics.rs @@ -0,0 +1,45 @@ +use std::marker::Copy; + +use bevy::{ + prelude::*, + asset::load_internal_asset, + render::render_resource::ShaderType, +}; +use bytemuck::{ + Pod, + Zeroable, +}; +use serde::{ + Deserialize, + Serialize, + Serializer, + ser::SerializeTuple, +}; + +#[cfg(feature = "f16")] +use half::f16; + +use crate::material::spherical_harmonics::{ + +}; + + +#[cfg(feature = "f16")] +#[derive( + Clone, + Copy, + Debug, + PartialEq, + Reflect, + ShaderType, + Pod, + Zeroable, + Serialize, + Deserialize, +)] +#[repr(C)] +pub struct SpherindricalHarmonicCoefficients { + #[reflect(ignore)] + #[serde(serialize_with = "coefficients_serializer", deserialize_with = "coefficients_deserializer")] + pub coefficients: [u32; HALF_SH_COEFF_COUNT], +} diff --git a/src/material/spherindrical_harmonics.wgsl b/src/material/spherindrical_harmonics.wgsl index e69de29b..d8789340 100644 --- a/src/material/spherindrical_harmonics.wgsl +++ b/src/material/spherindrical_harmonics.wgsl @@ -0,0 +1,54 @@ +#define_import_path bevy_gaussian_splatting::spherindrical_harmonics +#import bevy_gaussian_splatting::bindings::globals +#import bevy_gaussian_splatting::spherical_harmonics::{ + shc, + spherical_harmonics_lookup, +} + + +fn spherindrical_harmonics_lookup( + ray_direction: vec3, + sh: array, +) -> vec3 { + let rds = ray_direction * ray_direction; + let dir_t = globals.time; + + var color = vec3(0.5); + + color += shc[ 0] * vec3(sh[0], sh[1], sh[2]); + +#if SH_DEG > 0 + color += shc[ 1] * vec3(sh[ 3], sh[ 4], sh[ 5]) * ray_direction.y; + color += shc[ 2] * vec3(sh[ 6], sh[ 7], sh[ 8]) * ray_direction.z; + color += shc[ 3] * vec3(sh[ 9], sh[10], sh[11]) * ray_direction.x; +#endif + +#if SH_DEG > 1 + color += shc[ 4] * vec3(sh[12], sh[13], sh[14]) * ray_direction.x * ray_direction.y; + color += shc[ 5] * vec3(sh[15], sh[16], sh[17]) * ray_direction.y * ray_direction.z; + color += shc[ 6] * vec3(sh[18], sh[19], sh[20]) * (2.0 * rds.z - rds.x - rds.y); + color += shc[ 7] * vec3(sh[21], sh[22], sh[23]) * ray_direction.x * ray_direction.z; + color += shc[ 8] * vec3(sh[24], sh[25], sh[26]) * (rds.x - rds.y); +#endif + +#if SH_DEG > 2 + color += shc[ 9] * vec3(sh[27], sh[28], sh[29]) * ray_direction.y * (3.0 * rds.x - rds.y); + color += shc[10] * vec3(sh[30], sh[31], sh[32]) * ray_direction.x * ray_direction.y * ray_direction.z; + color += shc[11] * vec3(sh[33], sh[34], sh[35]) * ray_direction.y * (4.0 * rds.z - rds.x - rds.y); + color += shc[12] * vec3(sh[36], sh[37], sh[38]) * ray_direction.z * (2.0 * rds.z - 3.0 * rds.x - 3.0 * rds.y); + color += shc[13] * vec3(sh[39], sh[40], sh[41]) * ray_direction.x * (4.0 * rds.z - rds.x - rds.y); + color += shc[14] * vec3(sh[42], sh[43], sh[44]) * ray_direction.z * (rds.x - rds.y); + color += shc[15] * vec3(sh[45], sh[46], sh[47]) * ray_direction.x * (rds.x - 3.0 * rds.y); +#endif + +// TODO: add SH_DEG and SH_DEG_T shader defines +#if SH_DEG_T > 0 + +#endif + +#if SH_DEG_T > 1 + +#endif + + return color; +} From 5dcac36e4b629bcb1e3d920ecfe901595b8cd270 Mon Sep 17 00:00:00 2001 From: mosure Date: Thu, 25 Jan 2024 15:30:57 -0600 Subject: [PATCH 05/10] feat: 4d shader defines --- src/material/spherical_harmonics.rs | 4 ++-- src/material/spherindrical_harmonics.rs | 5 ++++- src/material/spherindrical_harmonics.wgsl | 10 +++++----- src/render/gaussian.wgsl | 2 +- src/render/mod.rs | 9 +++++++++ 5 files changed, 21 insertions(+), 9 deletions(-) diff --git a/src/material/spherical_harmonics.rs b/src/material/spherical_harmonics.rs index 2db78d43..84c28cb3 100644 --- a/src/material/spherical_harmonics.rs +++ b/src/material/spherical_harmonics.rs @@ -47,10 +47,10 @@ const fn num_sh_coefficients(degree: usize) -> usize { #[cfg(feature = "web")] -const SH_DEGREE: usize = 0; +pub const SH_DEGREE: usize = 0; #[cfg(not(feature = "web"))] -const SH_DEGREE: usize = 3; +pub const SH_DEGREE: usize = 3; pub const SH_CHANNELS: usize = 3; pub const SH_COEFF_COUNT_PER_CHANNEL: usize = num_sh_coefficients(SH_DEGREE); diff --git a/src/material/spherindrical_harmonics.rs b/src/material/spherindrical_harmonics.rs index 205f97f0..06d99398 100644 --- a/src/material/spherindrical_harmonics.rs +++ b/src/material/spherindrical_harmonics.rs @@ -20,10 +20,13 @@ use serde::{ use half::f16; use crate::material::spherical_harmonics::{ - + SH_DEGREE, }; +const SH_4D_DEGREE_TIME: usize = 0; + + #[cfg(feature = "f16")] #[derive( Clone, diff --git a/src/material/spherindrical_harmonics.wgsl b/src/material/spherindrical_harmonics.wgsl index d8789340..bbcd191e 100644 --- a/src/material/spherindrical_harmonics.wgsl +++ b/src/material/spherindrical_harmonics.wgsl @@ -17,13 +17,13 @@ fn spherindrical_harmonics_lookup( color += shc[ 0] * vec3(sh[0], sh[1], sh[2]); -#if SH_DEG > 0 +#if SH_DEGREE > 0 color += shc[ 1] * vec3(sh[ 3], sh[ 4], sh[ 5]) * ray_direction.y; color += shc[ 2] * vec3(sh[ 6], sh[ 7], sh[ 8]) * ray_direction.z; color += shc[ 3] * vec3(sh[ 9], sh[10], sh[11]) * ray_direction.x; #endif -#if SH_DEG > 1 +#if SH_DEGREE > 1 color += shc[ 4] * vec3(sh[12], sh[13], sh[14]) * ray_direction.x * ray_direction.y; color += shc[ 5] * vec3(sh[15], sh[16], sh[17]) * ray_direction.y * ray_direction.z; color += shc[ 6] * vec3(sh[18], sh[19], sh[20]) * (2.0 * rds.z - rds.x - rds.y); @@ -31,7 +31,7 @@ fn spherindrical_harmonics_lookup( color += shc[ 8] * vec3(sh[24], sh[25], sh[26]) * (rds.x - rds.y); #endif -#if SH_DEG > 2 +#if SH_DEGREE > 2 color += shc[ 9] * vec3(sh[27], sh[28], sh[29]) * ray_direction.y * (3.0 * rds.x - rds.y); color += shc[10] * vec3(sh[30], sh[31], sh[32]) * ray_direction.x * ray_direction.y * ray_direction.z; color += shc[11] * vec3(sh[33], sh[34], sh[35]) * ray_direction.y * (4.0 * rds.z - rds.x - rds.y); @@ -42,11 +42,11 @@ fn spherindrical_harmonics_lookup( #endif // TODO: add SH_DEG and SH_DEG_T shader defines -#if SH_DEG_T > 0 +#if SH_DEGREE_TIME > 0 #endif -#if SH_DEG_T > 1 +#if SH_DEGREE_TIME > 1 #endif diff --git a/src/render/gaussian.wgsl b/src/render/gaussian.wgsl index b2f015db..6010e427 100644 --- a/src/render/gaussian.wgsl +++ b/src/render/gaussian.wgsl @@ -182,7 +182,7 @@ fn compute_cov2d( let rotation = get_rotation(index); let scale = get_scale(index); -#ifdef GAUSSIAN_4d +#ifdef GAUSSIAN_4D let rotation_r = get_rotation_r(index); let decomposed = compute_cov3d_conditional( diff --git a/src/render/mod.rs b/src/render/mod.rs index 0e0136a2..023f4adf 100644 --- a/src/render/mod.rs +++ b/src/render/mod.rs @@ -66,6 +66,7 @@ use crate::{ material::spherical_harmonics::{ HALF_SH_COEFF_COUNT, SH_COEFF_COUNT, + SH_DEGREE, SH_VEC4_PLANES, }, morph::MorphPlugin, @@ -304,6 +305,7 @@ fn queue_gaussians( let key = GaussianCloudPipelineKey { aabb: settings.aabb, + gaussian_4d: false, // TODO: determine GaussianCloud(3d) or SpaceTimeGaussianCloud(4d) - better queue system abstraction visualize_bounding_box: settings.visualize_bounding_box, visualize_depth: settings.visualize_depth, draw_mode: settings.draw_mode, @@ -492,6 +494,8 @@ pub fn shader_defs( let defines = ShaderDefines::default(); let mut shader_defs = vec![ ShaderDefVal::UInt("SH_COEFF_COUNT".into(), SH_COEFF_COUNT as u32), + ShaderDefVal::UInt("SH_DEGREE".into(), SH_DEGREE as u32), + ShaderDefVal::UInt("SH_DEGREE_TIME".into(), SH_DEGREE as u32), ShaderDefVal::UInt("HALF_SH_COEFF_COUNT".into(), HALF_SH_COEFF_COUNT as u32), ShaderDefVal::UInt("SH_VEC4_PLANES".into(), SH_VEC4_PLANES as u32), ShaderDefVal::UInt("RADIX_BASE".into(), defines.radix_base), @@ -525,6 +529,10 @@ pub fn shader_defs( shader_defs.push("VISUALIZE_DEPTH".into()); } + if key.gaussian_4d { + shader_defs.push("GAUSSIAN_4D".into()); + } + #[cfg(feature = "packed")] shader_defs.push("PACKED".into()); @@ -573,6 +581,7 @@ pub fn shader_defs( #[derive(PartialEq, Eq, Hash, Clone, Copy, Default)] pub struct GaussianCloudPipelineKey { pub aabb: bool, + pub gaussian_4d: bool, pub visualize_bounding_box: bool, pub visualize_depth: bool, pub draw_mode: GaussianCloudDrawMode, From a23c334178dbda8f60d4ce65958d789e4a75bfb9 Mon Sep 17 00:00:00 2001 From: mosure Date: Thu, 25 Jan 2024 19:45:53 -0600 Subject: [PATCH 06/10] feat: spherindrical constants --- src/material/spherindrical_harmonics.rs | 199 +++++++++++++++++++++++- src/render/mod.rs | 2 +- 2 files changed, 198 insertions(+), 3 deletions(-) diff --git a/src/material/spherindrical_harmonics.rs b/src/material/spherindrical_harmonics.rs index 06d99398..e16b8b74 100644 --- a/src/material/spherindrical_harmonics.rs +++ b/src/material/spherindrical_harmonics.rs @@ -20,14 +20,78 @@ use serde::{ use half::f16; use crate::material::spherical_harmonics::{ + SH_CHANNELS, SH_DEGREE, }; -const SH_4D_DEGREE_TIME: usize = 0; +const fn gcd(a: usize, b: usize) -> usize { + if b == 0 { + a + } else { + gcd(b, a % b) + } +} + + +pub const SH_4D_DEGREE_TIME: usize = 2; + +pub const SH_4D_COEFF_COUNT_PER_CHANNEL: usize = (SH_DEGREE + 1).pow(2) * (SH_4D_DEGREE_TIME + 1); +pub const SH_4D_COEFF_COUNT: usize = (SH_4D_COEFF_COUNT_PER_CHANNEL * SH_CHANNELS + 3) & !3; + +pub const HALF_SH_4D_COEFF_COUNT: usize = (SH_4D_COEFF_COUNT / 2 + 3) & !3; + +// TODO: calculate POD_PLANE_COUNT for f16 and f32 based on a switch for HALF_SH_4D_COEFF_COUNT vs. SH_4D_COEFF_COUNT +pub const MAX_POD_U32_ARRAY_SIZE: usize = 32; +pub const POD_ARRAY_SIZE: usize = gcd(HALF_SH_4D_COEFF_COUNT, MAX_POD_U32_ARRAY_SIZE); +pub const POD_PLANE_COUNT: usize = HALF_SH_4D_COEFF_COUNT / POD_ARRAY_SIZE; + +pub const WASTE: usize = POD_PLANE_COUNT * POD_ARRAY_SIZE - HALF_SH_4D_COEFF_COUNT; +static_assertions::const_assert_eq!(WASTE, 0); #[cfg(feature = "f16")] +pub const SH_4D_VEC4_PLANES: usize = HALF_SH_4D_COEFF_COUNT / 4; +#[cfg(feature = "f32")] +pub const SH_4D_VEC4_PLANES: usize = SH_4D_COEFF_COUNT / 4; + + +const SPHERINDRICAL_HARMONICS_SHADER_HANDLE: Handle = Handle::weak_from_u128(512346253); + +pub struct SpherindricalHarmonicCoefficientsPlugin; +impl Plugin for SpherindricalHarmonicCoefficientsPlugin { + fn build(&self, app: &mut App) { + load_internal_asset!( + app, + SPHERINDRICAL_HARMONICS_SHADER_HANDLE, + "spherindrical_harmonics.wgsl", + Shader::from_wgsl + ); + } +} + + +#[cfg(feature = "f16")] +#[derive( + Clone, + Copy, + Debug, + PartialEq, + Reflect, + ShaderType, + Pod, + Zeroable, + Serialize, + Deserialize, +)] +#[repr(C)] +pub struct SpherindricalHarmonicCoefficients { + #[reflect(ignore)] + #[serde(serialize_with = "coefficients_serializer", deserialize_with = "coefficients_deserializer")] + pub coefficients: [[u32; POD_ARRAY_SIZE]; POD_PLANE_COUNT], +} + +#[cfg(feature = "f32")] #[derive( Clone, Copy, @@ -44,5 +108,136 @@ const SH_4D_DEGREE_TIME: usize = 0; pub struct SpherindricalHarmonicCoefficients { #[reflect(ignore)] #[serde(serialize_with = "coefficients_serializer", deserialize_with = "coefficients_deserializer")] - pub coefficients: [u32; HALF_SH_COEFF_COUNT], + pub coefficients: [u32; SH_4D_COEFF_COUNT], +} + + +#[cfg(feature = "f16")] +impl Default for SpherindricalHarmonicCoefficients { + fn default() -> Self { + Self { + coefficients: [[0; POD_ARRAY_SIZE]; POD_PLANE_COUNT], + } + } +} + +#[cfg(feature = "f32")] +impl Default for SpherindricalHarmonicCoefficients { + fn default() -> Self { + Self { + coefficients: [0.0; SH_4D_COEFF_COUNT], + } + } +} + + +impl SpherindricalHarmonicCoefficients { + #[cfg(feature = "f16")] + pub fn set(&mut self, index: usize, value: f32) { + let quantized = f16::from_f32(value).to_bits(); + self.coefficients[index / 2] = match index % 2 { + 0 => (self.coefficients[index / 2] & 0xffff0000) | (quantized as u32), + 1 => (self.coefficients[index / 2] & 0x0000ffff) | ((quantized as u32) << 16), + _ => unreachable!(), + }; + } + + #[cfg(feature = "f32")] + pub fn set(&mut self, index: usize, value: f32) { + self.coefficients[index] = value; + } +} + + + +#[cfg(feature = "f16")] +fn coefficients_serializer(n: &[u32; HALF_SH_4D_COEFF_COUNT], s: S) -> Result +where + S: Serializer, +{ + let mut tup = s.serialize_tuple(SH_4D_COEFF_COUNT)?; + for &x in n.iter() { + tup.serialize_element(&x)?; + } + + tup.end() +} + +#[cfg(feature = "f16")] +fn coefficients_deserializer<'de, D>(d: D) -> Result<[u32; HALF_SH_4D_COEFF_COUNT], D::Error> +where + D: serde::Deserializer<'de>, +{ + struct CoefficientsVisitor; + + impl<'de> serde::de::Visitor<'de> for CoefficientsVisitor { + type Value = [u32; HALF_SH_4D_COEFF_COUNT]; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("an array of floats") + } + + fn visit_seq(self, mut seq: A) -> Result<[u32; HALF_SH_4D_COEFF_COUNT], A::Error> + where + A: serde::de::SeqAccess<'de>, + { + let mut coefficients = [0; HALF_SH_4D_COEFF_COUNT]; + + for (i, coefficient) in coefficients.iter_mut().enumerate().take(SH_4D_COEFF_COUNT) { + *coefficient = seq + .next_element()? + .ok_or_else(|| serde::de::Error::invalid_length(i, &self))?; + } + Ok(coefficients) + } + } + + d.deserialize_tuple(SH_4D_COEFF_COUNT, CoefficientsVisitor) +} + + +#[cfg(feature = "f32")] +fn coefficients_serializer(n: &[f32; SH_4D_COEFF_COUNT], s: S) -> Result +where + S: Serializer, +{ + let mut tup = s.serialize_tuple(SH_4D_COEFF_COUNT)?; + for &x in n.iter() { + tup.serialize_element(&x)?; + } + + tup.end() } + +#[cfg(feature = "f32")] +fn coefficients_deserializer<'de, D>(d: D) -> Result<[f32; SH_4D_COEFF_COUNT], D::Error> +where + D: serde::Deserializer<'de>, +{ + struct CoefficientsVisitor; + + impl<'de> serde::de::Visitor<'de> for CoefficientsVisitor { + type Value = [f32; SH_4D_COEFF_COUNT]; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("an array of floats") + } + + fn visit_seq(self, mut seq: A) -> Result<[f32; SH_4D_COEFF_COUNT], A::Error> + where + A: serde::de::SeqAccess<'de>, + { + let mut coefficients = [0.0; SH_4D_COEFF_COUNT]; + + for (i, coefficient) in coefficients.iter_mut().enumerate().take(SH_4D_COEFF_COUNT) { + *coefficient = seq + .next_element()? + .ok_or_else(|| serde::de::Error::invalid_length(i, &self))?; + } + Ok(coefficients) + } + } + + d.deserialize_tuple(SH_4D_COEFF_COUNT, CoefficientsVisitor) +} + diff --git a/src/render/mod.rs b/src/render/mod.rs index 023f4adf..b122bfa8 100644 --- a/src/render/mod.rs +++ b/src/render/mod.rs @@ -183,7 +183,7 @@ pub struct GpuGaussianSplattingBundle { pub settings: GaussianCloudSettings, pub settings_uniform: GaussianCloudUniform, pub sorted_entries: Handle, - pub cloud_handle: Handle, + pub cloud_handle: Handle, // TODO: handle 4d gaussian cloud } #[derive(Debug, Clone)] From e349302ae30083c94cb24763c718463f0793f998 Mon Sep 17 00:00:00 2001 From: mosure Date: Fri, 26 Jan 2024 14:11:02 -0600 Subject: [PATCH 07/10] fix: spherindrical build errors and duplicate code --- src/gaussian/packed.rs | 6 +-- src/lib.rs | 1 + src/material/spherical_harmonics.rs | 10 +++-- src/material/spherindrical_harmonics.rs | 51 +++++++++++++------------ src/math/mod.rs | 13 +++++++ 5 files changed, 49 insertions(+), 32 deletions(-) create mode 100644 src/math/mod.rs diff --git a/src/gaussian/packed.rs b/src/gaussian/packed.rs index 04b2c662..8c150bbe 100644 --- a/src/gaussian/packed.rs +++ b/src/gaussian/packed.rs @@ -18,9 +18,9 @@ use crate::{ Scale4d, ScaleOpacity, }, - material::spherical_harmonics::{ - SphericalHarmonicCoefficients, - SpherindricalHarmonicCoefficients, + material::{ + spherical_harmonics::SphericalHarmonicCoefficients, + spherindrical_harmonics::SpherindricalHarmonicCoefficients, }, }; diff --git a/src/lib.rs b/src/lib.rs index 7d363d1d..c51fcb1c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -16,6 +16,7 @@ use render::RenderPipelinePlugin; pub mod gaussian; pub mod io; pub mod material; +pub mod math; pub mod morph; pub mod query; pub mod render; diff --git a/src/material/spherical_harmonics.rs b/src/material/spherical_harmonics.rs index 84c28cb3..b0e90a03 100644 --- a/src/material/spherical_harmonics.rs +++ b/src/material/spherical_harmonics.rs @@ -19,6 +19,8 @@ use serde::{ #[cfg(feature = "f16")] use half::f16; +use crate::math::pad_4; + const SPHERICAL_HARMONICS_SHADER_HANDLE: Handle = Handle::weak_from_u128(834667312); @@ -54,10 +56,10 @@ pub const SH_DEGREE: usize = 3; pub const SH_CHANNELS: usize = 3; pub const SH_COEFF_COUNT_PER_CHANNEL: usize = num_sh_coefficients(SH_DEGREE); -pub const SH_COEFF_COUNT: usize = (SH_COEFF_COUNT_PER_CHANNEL * SH_CHANNELS + 3) & !3; +pub const SH_COEFF_COUNT: usize = pad_4(SH_COEFF_COUNT_PER_CHANNEL * SH_CHANNELS); pub const HALF_SH_COEFF_COUNT: usize = SH_COEFF_COUNT / 2; -pub const PADDED_HALF_SH_COEFF_COUNT: usize = (HALF_SH_COEFF_COUNT + 3) & !3; +pub const PADDED_HALF_SH_COEFF_COUNT: usize = pad_4(HALF_SH_COEFF_COUNT); #[cfg(feature = "f16")] pub const SH_VEC4_PLANES: usize = PADDED_HALF_SH_COEFF_COUNT / 4; @@ -149,7 +151,7 @@ fn coefficients_serializer(n: &[u32; HALF_SH_COEFF_COUNT], s: S) -> Result usize { - if b == 0 { - a - } else { - gcd(b, a % b) - } -} - - pub const SH_4D_DEGREE_TIME: usize = 2; pub const SH_4D_COEFF_COUNT_PER_CHANNEL: usize = (SH_DEGREE + 1).pow(2) * (SH_4D_DEGREE_TIME + 1); -pub const SH_4D_COEFF_COUNT: usize = (SH_4D_COEFF_COUNT_PER_CHANNEL * SH_CHANNELS + 3) & !3; +pub const SH_4D_COEFF_COUNT: usize = pad_4(SH_4D_COEFF_COUNT_PER_CHANNEL * SH_CHANNELS); -pub const HALF_SH_4D_COEFF_COUNT: usize = (SH_4D_COEFF_COUNT / 2 + 3) & !3; +pub const HALF_SH_4D_COEFF_COUNT: usize = pad_4(SH_4D_COEFF_COUNT / 2); // TODO: calculate POD_PLANE_COUNT for f16 and f32 based on a switch for HALF_SH_4D_COEFF_COUNT vs. SH_4D_COEFF_COUNT pub const MAX_POD_U32_ARRAY_SIZE: usize = 32; @@ -125,7 +122,7 @@ impl Default for SpherindricalHarmonicCoefficients { impl Default for SpherindricalHarmonicCoefficients { fn default() -> Self { Self { - coefficients: [0.0; SH_4D_COEFF_COUNT], + coefficients: [[0.0; POD_ARRAY_SIZE]; POD_PLANE_COUNT], } } } @@ -135,9 +132,13 @@ impl SpherindricalHarmonicCoefficients { #[cfg(feature = "f16")] pub fn set(&mut self, index: usize, value: f32) { let quantized = f16::from_f32(value).to_bits(); - self.coefficients[index / 2] = match index % 2 { - 0 => (self.coefficients[index / 2] & 0xffff0000) | (quantized as u32), - 1 => (self.coefficients[index / 2] & 0x0000ffff) | ((quantized as u32) << 16), + let pair_index = index / 2; + let pod_index = pair_index / POD_ARRAY_SIZE; + let pod_offset = pair_index % POD_ARRAY_SIZE; + + self.coefficients[pod_index][pod_offset] = match index % 2 { + 0 => (self.coefficients[pod_index][pod_offset] & 0xffff0000) | (quantized as u32), + 1 => (self.coefficients[pod_index][pod_offset] & 0x0000ffff) | ((quantized as u32) << 16), _ => unreachable!(), }; } @@ -151,11 +152,11 @@ impl SpherindricalHarmonicCoefficients { #[cfg(feature = "f16")] -fn coefficients_serializer(n: &[u32; HALF_SH_4D_COEFF_COUNT], s: S) -> Result +fn coefficients_serializer(n: &[[u32; POD_ARRAY_SIZE]; POD_PLANE_COUNT], s: S) -> Result where S: Serializer, { - let mut tup = s.serialize_tuple(SH_4D_COEFF_COUNT)?; + let mut tup = s.serialize_tuple(HALF_SH_4D_COEFF_COUNT)?; for &x in n.iter() { tup.serialize_element(&x)?; } @@ -164,24 +165,24 @@ where } #[cfg(feature = "f16")] -fn coefficients_deserializer<'de, D>(d: D) -> Result<[u32; HALF_SH_4D_COEFF_COUNT], D::Error> +fn coefficients_deserializer<'de, D>(d: D) -> Result<[[u32; POD_ARRAY_SIZE]; POD_PLANE_COUNT], D::Error> where D: serde::Deserializer<'de>, { struct CoefficientsVisitor; impl<'de> serde::de::Visitor<'de> for CoefficientsVisitor { - type Value = [u32; HALF_SH_4D_COEFF_COUNT]; + type Value = [[u32; POD_ARRAY_SIZE]; POD_PLANE_COUNT]; fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { formatter.write_str("an array of floats") } - fn visit_seq(self, mut seq: A) -> Result<[u32; HALF_SH_4D_COEFF_COUNT], A::Error> + fn visit_seq(self, mut seq: A) -> Result<[[u32; POD_ARRAY_SIZE]; POD_PLANE_COUNT], A::Error> where A: serde::de::SeqAccess<'de>, { - let mut coefficients = [0; HALF_SH_4D_COEFF_COUNT]; + let mut coefficients = [[0; POD_ARRAY_SIZE]; POD_PLANE_COUNT]; for (i, coefficient) in coefficients.iter_mut().enumerate().take(SH_4D_COEFF_COUNT) { *coefficient = seq @@ -192,7 +193,7 @@ where } } - d.deserialize_tuple(SH_4D_COEFF_COUNT, CoefficientsVisitor) + d.deserialize_tuple(HALF_SH_4D_COEFF_COUNT, CoefficientsVisitor) } diff --git a/src/math/mod.rs b/src/math/mod.rs new file mode 100644 index 00000000..98b885e9 --- /dev/null +++ b/src/math/mod.rs @@ -0,0 +1,13 @@ + +pub const fn gcd(a: usize, b: usize) -> usize { + if b == 0 { + a + } else { + gcd(b, a % b) + } +} + + +pub const fn pad_4(x: usize) -> usize { + (x + 3) & !3 +} From d5ff9a0cb4f005ac895e6b6c6bb193fd93d898fb Mon Sep 17 00:00:00 2001 From: mosure Date: Tue, 30 Jan 2024 16:03:05 -0600 Subject: [PATCH 08/10] feat: gaussian cloud abstraction --- src/gaussian/cloud.rs | 49 +++++++++++++++++------ src/gaussian/cloud_4d.rs | 0 src/gaussian/f16.rs | 80 ++++++++++++++++++++++++++++++++++++- src/gaussian/mod.rs | 1 + src/render/gaussian_4d.wgsl | 6 +-- src/render/mod.rs | 8 ++++ 6 files changed, 127 insertions(+), 17 deletions(-) create mode 100644 src/gaussian/cloud_4d.rs diff --git a/src/gaussian/cloud.rs b/src/gaussian/cloud.rs index ebbadcef..443160a3 100644 --- a/src/gaussian/cloud.rs +++ b/src/gaussian/cloud.rs @@ -44,7 +44,6 @@ use crate::gaussian::f16::{ }; -#[cfg(feature = "f16")] #[derive( Asset, Clone, @@ -58,6 +57,28 @@ use crate::gaussian::f16::{ )] #[uuid = "ac2f08eb-bc32-aabb-ff21-51571ea332d5"] pub struct GaussianCloud { + pub cloud_3d: Option, + pub cloud_4d: Option, +} + +pub trait GaussianCloudTrait { + fn is_empty(&self) -> bool; + fn len(&self) -> usize; + fn len_sqrt_ceil(&self) -> usize; + fn square_len(&self) -> usize; +} + + +#[cfg(feature = "f16")] +#[derive( + Debug, + Default, + PartialEq, + Reflect, + Serialize, + Deserialize, +)] +pub struct GaussianCloud3d { pub position_visibility: Vec, pub spherical_harmonic: Vec, @@ -82,7 +103,7 @@ pub struct GaussianCloud { Deserialize, )] #[uuid = "ac2f08eb-bc32-aabb-ff21-51571ea332d5"] -pub struct GaussianCloud { +pub struct GaussianCloud3d { pub position_visibility: Vec, pub spherical_harmonic: Vec, @@ -96,23 +117,25 @@ pub struct GaussianCloud { pub scale_opacity: Vec, } -impl GaussianCloud { - pub fn is_empty(&self) -> bool { +impl GaussianCloudTrait for GaussianCloud3d { + fn is_empty(&self) -> bool { self.position_visibility.is_empty() } - pub fn len(&self) -> usize { + fn len(&self) -> usize { self.position_visibility.len() } - pub fn len_sqrt_ceil(&self) -> usize { + fn len_sqrt_ceil(&self) -> usize { (self.len() as f32).sqrt().ceil() as usize } - pub fn square_len(&self) -> usize { + fn square_len(&self) -> usize { self.len_sqrt_ceil().pow(2) } +} +impl GaussianCloud3d { pub fn position(&self, index: usize) -> &[f32; 3] { &self.position_visibility[index].position } @@ -272,7 +295,7 @@ impl GaussianCloud { } -impl GaussianCloud { +impl GaussianCloud3d { #[cfg(feature = "f16")] pub fn subset(&self, indicies: &[usize]) -> Self { let mut position_visibility = Vec::with_capacity(indicies.len()); @@ -341,7 +364,7 @@ impl GaussianCloud { } -impl GaussianCloud { +impl GaussianCloud3d { #[cfg(feature = "f16")] pub fn from_gaussians(gaussians: Vec) -> Self { let mut position_visibility = Vec::with_capacity(gaussians.len()); @@ -365,7 +388,7 @@ impl GaussianCloud { } #[allow(unused_mut)] - let mut cloud = GaussianCloud { + let mut cloud = GaussianCloud3d { position_visibility, spherical_harmonic, @@ -470,13 +493,13 @@ impl GaussianCloud { gaussians.push(gaussians[0]); - GaussianCloud::from_gaussians(gaussians) + GaussianCloud3d::from_gaussians(gaussians) } } -impl FromIterator for GaussianCloud { +impl FromIterator for GaussianCloud3d { fn from_iter>(iter: I) -> Self { let gaussians = iter.into_iter().collect::>(); - GaussianCloud::from_gaussians(gaussians) + GaussianCloud3d::from_gaussians(gaussians) } } diff --git a/src/gaussian/cloud_4d.rs b/src/gaussian/cloud_4d.rs new file mode 100644 index 00000000..e69de29b diff --git a/src/gaussian/f16.rs b/src/gaussian/f16.rs index 0f0a3dfa..1abf7679 100644 --- a/src/gaussian/f16.rs +++ b/src/gaussian/f16.rs @@ -21,7 +21,10 @@ use crate::gaussian::{ Rotation, ScaleOpacity, }, - packed::Gaussian, + packed::{ + Gaussian, + Gaussian4d, + }, }; @@ -197,6 +200,81 @@ impl From<[u32; 4]> for Covariance3dOpacityPacked128 { } +#[derive( + Clone, + Debug, + Default, + Copy, + PartialEq, + Reflect, + ShaderType, + Pod, + Zeroable, + Serialize, + Deserialize, +)] +#[repr(C)] +pub struct IsotropicRotations { + pub rotation: [u32; 2], + pub rotation_r: [u32; 2], +} + +impl IsotropicRotations { + pub fn from_gaussian(gaussian: &Gaussian4d) -> Self { + let rotation = gaussian.isomorphic_rotations[0]; + let rotation = gaussian.isomorphic_rotations[1]; + + Self { + rotation: [ + pack_f32s_to_u32(rotation.rotation[0], rotation.rotation[1]), + pack_f32s_to_u32(rotation.rotation[2], rotation.rotation[3]), + ], + rotation_r: [ + pack_f32s_to_u32(rotation.rotation[0], rotation.rotation[1]), + pack_f32s_to_u32(rotation.rotation[2], rotation.rotation[3]), + ], + } + } + + pub fn rotations(&self) -> [Rotation; 2] { + let (u0, l0) = unpack_u32_to_f32s(self.rotation[0]); + let (u1, l1) = unpack_u32_to_f32s(self.rotation[1]); + + let (u0_r, l0_r) = unpack_u32_to_f32s(self.rotation_r[0]); + let (u1_r, l1_r) = unpack_u32_to_f32s(self.rotation_r[1]); + + [ + Rotation { + rotation: [ + u0, + l0, + u1, + l1, + ], + }, + Rotation { + rotation: [ + u0_r, + l0_r, + u1_r, + l1_r, + ], + }, + ] + } +} + +impl From<[u32; 4]> for IsotropicRotations { + fn from(rotations: [u32; 4]) -> Self { + Self { + rotation: [rotations[0], rotations[1]], + rotation_r: [rotations[2], rotations[3]], + } + } +} + + + pub fn pack_f32s_to_u32(upper: f32, lower: f32) -> u32 { pack_f16s_to_u32( f16::from_f32(upper), diff --git a/src/gaussian/mod.rs b/src/gaussian/mod.rs index 1271e21f..78037294 100644 --- a/src/gaussian/mod.rs +++ b/src/gaussian/mod.rs @@ -1,6 +1,7 @@ use static_assertions::assert_cfg; pub mod cloud; +pub mod cloud_4d; pub mod covariance; pub mod f32; pub mod packed; diff --git a/src/render/gaussian_4d.wgsl b/src/render/gaussian_4d.wgsl index 247cfa9b..a4f135a6 100644 --- a/src/render/gaussian_4d.wgsl +++ b/src/render/gaussian_4d.wgsl @@ -10,7 +10,7 @@ struct DecomposedGaussian4d { cov3d: array, delta_mean: vec3, - opacity: f32, + opacity_modifier: f32, mask: bool, } @@ -72,7 +72,7 @@ fn compute_cov3d_conditional( ); } - let opacity *= marginal_t; + let opacity_modifier = marginal_t; let cov11 = mat3x3( Sigma[0][0], Sigma[0][1], Sigma[0][2], @@ -92,7 +92,7 @@ fn compute_cov3d_conditional( cov3d_condition[1][2], cov3d_condition[2][2], delta_mean, - opacity, + opacity_modifier, mask, ); } diff --git a/src/render/mod.rs b/src/render/mod.rs index b122bfa8..ce566289 100644 --- a/src/render/mod.rs +++ b/src/render/mod.rs @@ -88,6 +88,7 @@ mod texture; const BINDINGS_SHADER_HANDLE: Handle = Handle::weak_from_u128(675257236); const GAUSSIAN_SHADER_HANDLE: Handle = Handle::weak_from_u128(68294581); +const GAUSSIAN_4D_SHADER_HANDLE: Handle = Handle::weak_from_u128(513623421); const PACKED_SHADER_HANDLE: Handle = Handle::weak_from_u128(123623514); const PLANAR_SHADER_HANDLE: Handle = Handle::weak_from_u128(72345231); const TEXTURE_SHADER_HANDLE: Handle = Handle::weak_from_u128(26345735); @@ -113,6 +114,13 @@ impl Plugin for RenderPipelinePlugin { Shader::from_wgsl ); + load_internal_asset!( + app, + GAUSSIAN_4D_SHADER_HANDLE, + "gaussian_4d.wgsl", + Shader::from_wgsl + ); + load_internal_asset!( app, PACKED_SHADER_HANDLE, From e1af290d38de272f46559927a289bca844e52418 Mon Sep 17 00:00:00 2001 From: mosure Date: Fri, 9 Feb 2024 15:44:53 -0600 Subject: [PATCH 09/10] docs: analytics --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 89ec4d8e..b9e31a4d 100644 --- a/README.md +++ b/README.md @@ -2,10 +2,8 @@ [![test](https://github.com/mosure/bevy_gaussian_splatting/workflows/test/badge.svg)](https://github.com/Mosure/bevy_gaussian_splatting/actions?query=workflow%3Atest) [![GitHub License](https://img.shields.io/github/license/mosure/bevy_gaussian_splatting)](https://raw.githubusercontent.com/mosure/bevy_gaussian_splatting/main/LICENSE) -[![GitHub Last Commit](https://img.shields.io/github/last-commit/mosure/bevy_gaussian_splatting)](https://github.com/mosure/bevy_gaussian_splatting) [![GitHub Releases](https://img.shields.io/github/v/release/mosure/bevy_gaussian_splatting?include_prereleases&sort=semver)](https://github.com/mosure/bevy_gaussian_splatting/releases) [![GitHub Issues](https://img.shields.io/github/issues/mosure/bevy_gaussian_splatting)](https://github.com/mosure/bevy_gaussian_splatting/issues) -[![Average time to resolve an issue](https://isitmaintained.com/badge/resolution/mosure/bevy_gaussian_splatting.svg)](http://isitmaintained.com/project/mosure/bevy_gaussian_splatting) [![crates.io](https://img.shields.io/crates/v/bevy_gaussian_splatting.svg)](https://crates.io/crates/bevy_gaussian_splatting) bevy gaussian splatting render pipeline plugin. view the [live demo](https://mosure.github.io/bevy_gaussian_splatting/index.html?arg1=cactus.gcloud) @@ -84,5 +82,9 @@ fn setup_gaussian_cloud( | `0.1 - 0.3` | `0.11` | +## analytics +![Alt](https://repobeats.axiom.co/api/embed/4f273f05f00ec57e90be34727e85952039e1a712.svg "analytics") + + ## projects using this plugin - [kitt2](https://github.com/cs50victor/kitt2) From f035ec9976fabb3d844c679416d4cb89155204fc Mon Sep 17 00:00:00 2001 From: mosure Date: Sun, 11 Feb 2024 19:40:39 -0600 Subject: [PATCH 10/10] checkin - massive rename effort --- benches/io.rs | 4 +- examples/headless.rs | 8 +- src/gaussian/cloud.rs | 285 +++++++++++-------------- src/gaussian/cloud_4d.rs | 403 +++++++++++++++++++++++++++++++++++ src/gaussian/rand.rs | 6 +- src/io/gcloud/bincode2.rs | 2 +- src/io/gcloud/flexbuffers.rs | 6 +- src/io/loader.rs | 8 +- src/io/writer.rs | 4 +- src/lib.rs | 10 +- src/material/noise.rs | 6 +- src/query/select.rs | 14 +- src/query/sparse.rs | 8 +- src/render/mod.rs | 32 +-- src/render/packed.rs | 6 +- src/render/planar.rs | 6 +- src/render/texture.rs | 6 +- src/sort/mod.rs | 6 +- src/sort/radix.rs | 14 +- src/sort/rayon.rs | 6 +- src/sort/std.rs | 6 +- tests/gaussian.rs | 2 +- tests/gpu/gaussian.rs | 8 +- tests/gpu/radix.rs | 8 +- tests/io.rs | 2 +- tools/compare_aabb_obb.rs | 4 +- tools/ply_to_gcloud.rs | 2 +- viewer/viewer.rs | 12 +- 28 files changed, 630 insertions(+), 254 deletions(-) diff --git a/benches/io.rs b/benches/io.rs index 4e060d93..4c9d97bc 100644 --- a/benches/io.rs +++ b/benches/io.rs @@ -8,7 +8,7 @@ use criterion::{ use bevy_gaussian_splatting::{ Gaussian, - GaussianCloud, + Cloud, io::codec::GaussianCloudCodec, random_gaussians, }; @@ -33,7 +33,7 @@ fn gaussian_cloud_decode_benchmark(c: &mut Criterion) { let gaussians = random_gaussians(*count); let bytes = gaussians.encode(); - b.iter(|| GaussianCloud::decode(bytes.as_slice())); + b.iter(|| Cloud::decode(bytes.as_slice())); }, ); } diff --git a/examples/headless.rs b/examples/headless.rs index 0943ccfb..49c4d79e 100644 --- a/examples/headless.rs +++ b/examples/headless.rs @@ -16,7 +16,7 @@ use bevy_panorbit_camera::{ }; use bevy_gaussian_splatting::{ - GaussianCloud, + Cloud, GaussianSplattingBundle, GaussianSplattingPlugin, random_gaussians, @@ -369,12 +369,12 @@ fn setup_gaussian_cloud( mut commands: Commands, asset_server: Res, gaussian_splatting_viewer: Res, - mut gaussian_assets: ResMut>, + mut gaussian_assets: ResMut>, mut scene_controller: ResMut, mut images: ResMut>, render_device: Res, ) { - let cloud: Handle; + let cloud: Handle; if gaussian_splatting_viewer.gaussian_count > 0 { println!("generating {} gaussians", gaussian_splatting_viewer.gaussian_count); @@ -383,7 +383,7 @@ fn setup_gaussian_cloud( println!("loading {}", gaussian_splatting_viewer.input_file); cloud = asset_server.load(&gaussian_splatting_viewer.input_file); } else { - cloud = gaussian_assets.add(GaussianCloud::test_model()); + cloud = gaussian_assets.add(Cloud::test_model()); } let render_target = frame_capture::scene::setup_render_target( diff --git a/src/gaussian/cloud.rs b/src/gaussian/cloud.rs index 443160a3..306240b7 100644 --- a/src/gaussian/cloud.rs +++ b/src/gaussian/cloud.rs @@ -19,6 +19,7 @@ use rayon::prelude::*; #[allow(unused_imports)] use crate::{ gaussian::{ + cloud_4d::GaussianCloud4d, f32::{ Covariance3dOpacity, Position, @@ -56,21 +57,30 @@ use crate::gaussian::f16::{ Deserialize, )] #[uuid = "ac2f08eb-bc32-aabb-ff21-51571ea332d5"] -pub struct GaussianCloud { +pub struct Cloud { pub cloud_3d: Option, pub cloud_4d: Option, } -pub trait GaussianCloudTrait { +pub trait GaussianCloud { fn is_empty(&self) -> bool; fn len(&self) -> usize; fn len_sqrt_ceil(&self) -> usize; fn square_len(&self) -> usize; + + fn packed(&self, index: usize) -> PackedType; + fn iter(&self) -> dyn Iterator; + + // TODO: become opinionated about packed vs. planar representation + fn to_packed(&self) -> Vec; + + fn test_model() -> Self; } #[cfg(feature = "f16")] #[derive( + Clone, Debug, Default, PartialEq, @@ -117,7 +127,7 @@ pub struct GaussianCloud3d { pub scale_opacity: Vec, } -impl GaussianCloudTrait for GaussianCloud3d { +impl GaussianCloud for GaussianCloud3d { fn is_empty(&self) -> bool { self.position_visibility.is_empty() } @@ -133,76 +143,13 @@ impl GaussianCloudTrait for GaussianCloud3d { fn square_len(&self) -> usize { self.len_sqrt_ceil().pow(2) } -} - -impl GaussianCloud3d { - pub fn position(&self, index: usize) -> &[f32; 3] { - &self.position_visibility[index].position - } - - pub fn position_mut(&mut self, index: usize) -> &mut [f32; 3] { - &mut self.position_visibility[index].position - } - - pub fn position_iter(&self) -> impl Iterator + '_ { - self.position_visibility.iter() - .map(|position_visibility| &position_visibility.position) - } - - #[cfg(feature = "sort_rayon")] - pub fn position_par_iter(&self) -> impl IndexedParallelIterator { - self.position_visibility.par_iter() - .map(|position_visibility| &position_visibility.position) - } - - - pub fn visibility(&self, index: usize) -> f32 { - self.position_visibility[index].visibility - } - - pub fn visibility_mut(&mut self, index: usize) -> &mut f32 { - &mut self.position_visibility[index].visibility - } - - - // pub fn rotation(&self, index: usize) -> &[f32; 4] { - // #[cfg(feature = "f16")] - // return &self.rotation_scale_opacity_packed128[index].rotation; - - // #[cfg(feature = "f32")] - // return &self.rotation[index].rotation; - // } - // pub fn rotation_mut(&mut self, index: usize) -> &mut [f32; 4] { - // #[cfg(feature = "f16")] - // return &mut self.rotation_scale_opacity_packed128[index].rotation; - - // #[cfg(feature = "f32")] - // return &mut self.rotation[index].rotation; - // } - - - // pub fn scale(&self, index: usize) -> &[f32; 3] { - // #[cfg(feature = "f16")] - // return &self.rotation_scale_opacity_packed128[index].scale; - - // #[cfg(feature = "f32")] - // return &self.scale_opacity[index].scale; - // } - - // pub fn scale_mut(&mut self, index: usize) -> &mut [f32; 3] { - // #[cfg(feature = "f16")] - // return &mut self.rotation_scale_opacity_packed128[index].scale; - - // #[cfg(feature = "f32")] - // return &mut self.scale_opacity[index].scale; - // } #[cfg(all( not(feature = "precompute_covariance_3d"), feature = "f16", ))] - pub fn gaussian(&self, index: usize) -> Gaussian { + fn packed(&self, index: usize) -> Gaussian { let rso = self.rotation_scale_opacity_packed128[index]; let rotation = rso.rotation(); @@ -217,7 +164,7 @@ impl GaussianCloud3d { } #[cfg(feature = "f32")] - pub fn gaussian(&self, index: usize) -> Gaussian { + fn packed(&self, index: usize) -> Gaussian { Gaussian { position_visibility: self.position_visibility[index], spherical_harmonic: self.spherical_harmonic[index], @@ -230,7 +177,7 @@ impl GaussianCloud3d { not(feature = "precompute_covariance_3d"), feature = "f16", ))] - pub fn gaussian_iter(&self) -> impl Iterator + '_ { + fn iter(&self) -> dyn Iterator { self.position_visibility.iter() .zip(self.spherical_harmonic.iter()) .zip(self.rotation_scale_opacity_packed128.iter()) @@ -246,7 +193,7 @@ impl GaussianCloud3d { } #[cfg(feature = "f32")] - pub fn gaussian_iter(&self) -> impl Iterator + '_ { + fn iter(&self) -> dyn Iterator { self.position_visibility.iter() .zip(self.spherical_harmonic.iter()) .zip(self.rotation.iter()) @@ -262,6 +209,116 @@ impl GaussianCloud3d { }) } + fn to_packed(&self) -> Vec { + let mut gaussians = Vec::with_capacity(self.len()); + + for index in 0..self.len() { + gaussians.push(self.packed(index)); + } + + gaussians + } + + fn test_model() -> Self { + let mut rng = rand::thread_rng(); + + let origin = Gaussian { + rotation: [ + 1.0, + 0.0, + 0.0, + 0.0, + ].into(), + position_visibility: [ + 0.0, + 0.0, + 0.0, + 1.0, + ].into(), + scale_opacity: [ + 0.5, + 0.5, + 0.5, + 0.5, + ].into(), + spherical_harmonic: SphericalHarmonicCoefficients { + coefficients: { + #[cfg(feature = "f16")] + { + let mut coefficients = [0_u32; HALF_SH_COEFF_COUNT]; + + for coefficient in coefficients.iter_mut() { + let upper = rng.gen_range(-1.0..1.0); + let lower = rng.gen_range(-1.0..1.0); + + *coefficient = pack_f32s_to_u32(upper, lower); + } + + coefficients + } + + #[cfg(feature = "f32")] + { + let mut coefficients = [0.0; SH_COEFF_COUNT]; + + for coefficient in coefficients.iter_mut() { + *coefficient = rng.gen_range(-1.0..1.0); + } + + coefficients + } + }, + }, + }; + let mut gaussians: Vec = Vec::new(); + + for &x in [-0.5, 0.5].iter() { + for &y in [-0.5, 0.5].iter() { + for &z in [-0.5, 0.5].iter() { + let mut g = origin; + g.position_visibility = [x, y, z, 1.0].into(); + gaussians.push(g); + + gaussians.last_mut().unwrap().spherical_harmonic.coefficients.shuffle(&mut rng); + } + } + } + + gaussians.push(gaussians[0]); + + GaussianCloud3d::from_packed(gaussians) + } +} + +impl GaussianCloud3d { + pub fn position(&self, index: usize) -> &[f32; 3] { + &self.position_visibility[index].position + } + + pub fn position_mut(&mut self, index: usize) -> &mut [f32; 3] { + &mut self.position_visibility[index].position + } + + pub fn position_iter(&self) -> impl Iterator + '_ { + self.position_visibility.iter() + .map(|position_visibility| &position_visibility.position) + } + + #[cfg(feature = "sort_rayon")] + pub fn position_par_iter(&self) -> impl IndexedParallelIterator { + self.position_visibility.par_iter() + .map(|position_visibility| &position_visibility.position) + } + + + pub fn visibility(&self, index: usize) -> f32 { + self.position_visibility[index].visibility + } + + pub fn visibility_mut(&mut self, index: usize) -> &mut f32 { + &mut self.position_visibility[index].visibility + } + pub fn spherical_harmonic(&self, index: usize) -> &SphericalHarmonicCoefficients { &self.spherical_harmonic[index] @@ -351,22 +408,8 @@ impl GaussianCloud3d { } } - #[cfg(feature = "f32")] - pub fn to_packed(&self) -> Vec { - let mut gaussians = Vec::with_capacity(self.len()); - - for index in 0..self.len() { - gaussians.push(self.gaussian(index)); - } - - gaussians - } -} - - -impl GaussianCloud3d { #[cfg(feature = "f16")] - pub fn from_gaussians(gaussians: Vec) -> Self { + fn from_packed(gaussians: Vec) -> Self { let mut position_visibility = Vec::with_capacity(gaussians.len()); let mut spherical_harmonic = Vec::with_capacity(gaussians.len()); @@ -404,7 +447,7 @@ impl GaussianCloud3d { } #[cfg(feature = "f32")] - pub fn from_gaussians(gaussians: Vec) -> Self { + fn from_packed(gaussians: Vec) -> Self { let mut position_visibility = Vec::with_capacity(gaussians.len()); let mut spherical_harmonic = Vec::with_capacity(gaussians.len()); let mut rotation = Vec::with_capacity(gaussians.len()); @@ -425,76 +468,6 @@ impl GaussianCloud3d { scale_opacity, } } - - pub fn test_model() -> Self { - let mut rng = rand::thread_rng(); - - let origin = Gaussian { - rotation: [ - 1.0, - 0.0, - 0.0, - 0.0, - ].into(), - position_visibility: [ - 0.0, - 0.0, - 0.0, - 1.0, - ].into(), - scale_opacity: [ - 0.5, - 0.5, - 0.5, - 0.5, - ].into(), - spherical_harmonic: SphericalHarmonicCoefficients { - coefficients: { - #[cfg(feature = "f16")] - { - let mut coefficients = [0_u32; HALF_SH_COEFF_COUNT]; - - for coefficient in coefficients.iter_mut() { - let upper = rng.gen_range(-1.0..1.0); - let lower = rng.gen_range(-1.0..1.0); - - *coefficient = pack_f32s_to_u32(upper, lower); - } - - coefficients - } - - #[cfg(feature = "f32")] - { - let mut coefficients = [0.0; SH_COEFF_COUNT]; - - for coefficient in coefficients.iter_mut() { - *coefficient = rng.gen_range(-1.0..1.0); - } - - coefficients - } - }, - }, - }; - let mut gaussians: Vec = Vec::new(); - - for &x in [-0.5, 0.5].iter() { - for &y in [-0.5, 0.5].iter() { - for &z in [-0.5, 0.5].iter() { - let mut g = origin; - g.position_visibility = [x, y, z, 1.0].into(); - gaussians.push(g); - - gaussians.last_mut().unwrap().spherical_harmonic.coefficients.shuffle(&mut rng); - } - } - } - - gaussians.push(gaussians[0]); - - GaussianCloud3d::from_gaussians(gaussians) - } } impl FromIterator for GaussianCloud3d { diff --git a/src/gaussian/cloud_4d.rs b/src/gaussian/cloud_4d.rs index e69de29b..9feecfd1 100644 --- a/src/gaussian/cloud_4d.rs +++ b/src/gaussian/cloud_4d.rs @@ -0,0 +1,403 @@ +use rand::{ + seq::SliceRandom, + Rng, +}; +use std::iter::FromIterator; + +use bevy::prelude::*; +use serde::{ + Deserialize, + Serialize, +}; + +#[cfg(feature = "sort_rayon")] +use rayon::prelude::*; + +#[allow(unused_imports)] +use crate::{ + gaussian::{ + cloud::{ + Cloud, + GaussianCloud, + }, + f32::{ + Covariance3dOpacity, + Position, + PositionVisibility, + Rotation, + ScaleOpacity, + }, + packed::Gaussian4d, + }, + material::spherical_harmonics::{ + HALF_SH_COEFF_COUNT, + SH_COEFF_COUNT, + SphericalHarmonicCoefficients, + }, +}; + +#[allow(unused_imports)] +#[cfg(feature = "f16")] +use crate::gaussian::f16::{ + Covariance3dOpacityPacked128, + RotationScaleOpacityPacked128, + pack_f32s_to_u32, +}; + + + +#[cfg(feature = "f16")] +#[derive( + Debug, + Default, + PartialEq, + Reflect, + Serialize, + Deserialize, +)] +pub struct GaussianCloud4d { + pub position_visibility: Vec, + + pub spherical_harmonic: Vec, + + #[cfg(not(feature = "precompute_covariance_3d"))] + pub rotation_scale_opacity_packed128: Vec, + + #[cfg(feature = "precompute_covariance_3d")] + pub covariance_3d_opacity_packed128: Vec, +} + +#[cfg(feature = "f32")] +#[derive( + Asset, + Clone, + Debug, + Default, + PartialEq, + Reflect, + TypeUuid, + Serialize, + Deserialize, +)] +#[uuid = "ac2f08eb-bc32-aabb-ff21-51571ea332d5"] +pub struct GaussianCloud4d { + pub position_visibility: Vec, + + pub spherical_harmonic: Vec, + + #[cfg(feature = "precompute_covariance_3d")] + pub covariance_3d: Vec, + + #[cfg(not(feature = "precompute_covariance_3d"))] + pub rotation: Vec, + #[cfg(not(feature = "precompute_covariance_3d"))] + pub scale_opacity: Vec, +} + +impl GaussianCloud for GaussianCloud4d { + fn is_empty(&self) -> bool { + self.position_visibility.is_empty() + } + + fn len(&self) -> usize { + self.position_visibility.len() + } + + fn len_sqrt_ceil(&self) -> usize { + (self.len() as f32).sqrt().ceil() as usize + } + + fn square_len(&self) -> usize { + self.len_sqrt_ceil().pow(2) + } + + + #[cfg(all( + not(feature = "precompute_covariance_3d"), + feature = "f16", + ))] + fn packed(&self, index: usize) -> Gaussian4d { + let rso = self.rotation_scale_opacity_packed128[index]; + + let rotation = rso.rotation(); + let scale_opacity = rso.scale_opacity(); + + Gaussian4d { + isomorphic_rotations: todo!(), + position_opacity: todo!(), + scale: todo!(), + spherindrical_harmonic: todo!(), + } + } + + #[cfg(feature = "f32")] + fn packed(&self, index: usize) -> Gaussian4d { + Gaussian4d { + position_visibility: self.position_visibility[index], + spherical_harmonic: self.spherical_harmonic[index], + rotation: self.rotation[index], + scale_opacity: self.scale_opacity[index], + } + } + + #[cfg(all( + not(feature = "precompute_covariance_3d"), + feature = "f16", + ))] + fn iter(&self) -> std::vec::IntoIter { + self.position_visibility.iter() + .zip(self.spherical_harmonic.iter()) + .zip(self.rotation_scale_opacity_packed128.iter()) + .map(|((position_visibility, spherical_harmonic), rotation_scale_opacity)| { + Gaussian4d { + isomorphic_rotations: todo!(), + position_opacity: todo!(), + scale: todo!(), + spherindrical_harmonic: todo!(), + } + }) + } + + #[cfg(feature = "f32")] + fn iter(&self) -> dyn Iterator { + self.position_visibility.iter() + .zip(self.spherical_harmonic.iter()) + .zip(self.rotation.iter()) + .zip(self.scale_opacity.iter()) + .map(|(((position_visibility, spherical_harmonic), rotation), scale_opacity)| { + Gaussian { + position_visibility: *position_visibility, + spherical_harmonic: *spherical_harmonic, + + rotation: *rotation, + scale_opacity: *scale_opacity, + } + }) + } + + fn to_packed(&self) -> Vec { + let mut gaussians = Vec::with_capacity(self.len()); + + for index in 0..self.len() { + gaussians.push(self.packed(index)); + } + + gaussians + } + + fn test_model() -> Self { + let mut rng = rand::thread_rng(); + + let origin = Gaussian4d { + isomorphic_rotations: todo!(), + position_opacity: todo!(), + scale: todo!(), + spherindrical_harmonic: todo!(), + }; + let mut gaussians: Vec = Vec::new(); + + for &x in [-0.5, 0.5].iter() { + for &y in [-0.5, 0.5].iter() { + for &z in [-0.5, 0.5].iter() { + let mut g = origin; + g.position_opacity = [x, y, z, 0.5].into(); + gaussians.push(g); + + gaussians.last_mut().unwrap().spherindrical_harmonic.coefficients.shuffle(&mut rng); + } + } + } + + gaussians.push(gaussians[0]); + + GaussianCloud4d::from_packed(gaussians) + } +} + +impl GaussianCloud4d { + pub fn position(&self, index: usize) -> &[f32; 3] { + &self.position_visibility[index].position + } + + pub fn position_mut(&mut self, index: usize) -> &mut [f32; 3] { + &mut self.position_visibility[index].position + } + + pub fn position_iter(&self) -> impl Iterator + '_ { + self.position_visibility.iter() + .map(|position_visibility| &position_visibility.position) + } + + #[cfg(feature = "sort_rayon")] + pub fn position_par_iter(&self) -> impl IndexedParallelIterator { + self.position_visibility.par_iter() + .map(|position_visibility| &position_visibility.position) + } + + + pub fn visibility(&self, index: usize) -> f32 { + self.position_visibility[index].visibility + } + + pub fn visibility_mut(&mut self, index: usize) -> &mut f32 { + &mut self.position_visibility[index].visibility + } + + + pub fn spherical_harmonic(&self, index: usize) -> &SphericalHarmonicCoefficients { + &self.spherical_harmonic[index] + } + + pub fn spherical_harmonic_mut(&mut self, index: usize) -> &mut SphericalHarmonicCoefficients { + &mut self.spherical_harmonic[index] + } + + pub fn resize_to_square(&mut self) { + #[cfg(all(feature = "buffer_texture", feature = "f16"))] + { + self.position_visibility.resize(self.square_len(), PositionVisibility::default()); + self.spherical_harmonic.resize(self.square_len(), SphericalHarmonicCoefficients::default()); + + #[cfg(feature = "precompute_covariance_3d")] + self.covariance_3d_opacity_packed128.resize(self.square_len(), Covariance3dOpacityPacked128::default()); + #[cfg(not(feature = "precompute_covariance_3d"))] + self.rotation_scale_opacity_packed128.resize(self.square_len(), RotationScaleOpacityPacked128::default()); + } + + #[cfg(all(feature = "buffer_texture", feature = "f32"))] + { + self.position_visibility.resize(self.square_len(), PositionVisibility::default()); + self.spherical_harmonic.resize(self.square_len(), SphericalHarmonicCoefficients::default()); + self.rotation.resize(self.square_len(), Rotation::default()); + self.scale_opacity.resize(self.square_len(), ScaleOpacity::default()); + self.covariance_3d.resize(self.square_len(), Covariance3dOpacity::default()); + } + } +} + + +impl GaussianCloud4d { + #[cfg(feature = "f16")] + pub fn subset(&self, indicies: &[usize]) -> Self { + let mut position_visibility = Vec::with_capacity(indicies.len()); + let mut spherical_harmonic = Vec::with_capacity(indicies.len()); + + #[cfg(feature = "precompute_covariance_3d")] + let mut covariance_3d_opacity_packed128 = Vec::with_capacity(indicies.len()); + + #[cfg(not(feature = "precompute_covariance_3d"))] + let mut rotation_scale_opacity_packed128 = Vec::with_capacity(indicies.len()); + + for &index in indicies.iter() { + position_visibility.push(self.position_visibility[index]); + spherical_harmonic.push(self.spherical_harmonic[index]); + + #[cfg(feature = "precompute_covariance_3d")] + covariance_3d_opacity_packed128.push(self.covariance_3d_opacity_packed128[index]); + + #[cfg(not(feature = "precompute_covariance_3d"))] + rotation_scale_opacity_packed128.push(self.rotation_scale_opacity_packed128[index]); + } + + Self { + position_visibility, + spherical_harmonic, + + #[cfg(feature = "precompute_covariance_3d")] + covariance_3d_opacity_packed128, + #[cfg(not(feature = "precompute_covariance_3d"))] + rotation_scale_opacity_packed128, + } + } + + #[cfg(feature = "f32")] + pub fn subset(&self, indicies: &[usize]) -> Self { + let mut position_visibility = Vec::with_capacity(indicies.len()); + let mut spherical_harmonic = Vec::with_capacity(indicies.len()); + let mut rotation = Vec::with_capacity(indicies.len()); + let mut scale_opacity = Vec::with_capacity(indicies.len()); + + for &index in indicies.iter() { + position_visibility.push(self.position_visibility[index]); + spherical_harmonic.push(self.spherical_harmonic[index]); + rotation.push(self.rotation[index]); + scale_opacity.push(self.scale_opacity[index]); + } + + Self { + position_visibility, + spherical_harmonic, + rotation, + scale_opacity, + } + } + + + #[cfg(feature = "f16")] + fn from_packed(gaussians: Vec) -> Self { + let mut position_visibility = Vec::with_capacity(gaussians.len()); + let mut spherical_harmonic = Vec::with_capacity(gaussians.len()); + + #[cfg(feature = "precompute_covariance_3d")] + let mut covariance_3d_opacity_packed128 = Vec::with_capacity(gaussians.len()); + + #[cfg(not(feature = "precompute_covariance_3d"))] + let mut rotation_scale_opacity_packed128 = Vec::with_capacity(gaussians.len()); + + for gaussian in gaussians { + position_visibility.push(gaussian.position_visibility); + spherical_harmonic.push(gaussian.spherical_harmonic); + + #[cfg(feature = "precompute_covariance_3d")] + covariance_3d_opacity_packed128.push(Covariance3dOpacityPacked128::from_gaussian(&gaussian)); + + #[cfg(not(feature = "precompute_covariance_3d"))] + rotation_scale_opacity_packed128.push(RotationScaleOpacityPacked128::from_gaussian(&gaussian)); + } + + #[allow(unused_mut)] + let mut cloud = GaussianCloud4d { + position_visibility, + spherical_harmonic, + + #[cfg(feature = "precompute_covariance_3d")] + covariance_3d_opacity_packed128, + #[cfg(not(feature = "precompute_covariance_3d"))] + rotation_scale_opacity_packed128, + }; + + cloud.resize_to_square(); + + cloud + } + + #[cfg(feature = "f32")] + fn from_packed(gaussians: Vec) -> Self { + let mut position_visibility = Vec::with_capacity(gaussians.len()); + let mut spherical_harmonic = Vec::with_capacity(gaussians.len()); + let mut rotation = Vec::with_capacity(gaussians.len()); + let mut scale_opacity = Vec::with_capacity(gaussians.len()); + + for gaussian in gaussians { + position_visibility.push(gaussian.position_visibility); + spherical_harmonic.push(gaussian.spherical_harmonic); + + rotation.push(gaussian.rotation); + scale_opacity.push(gaussian.scale_opacity); + } + + Self { + position_visibility, + spherical_harmonic, + rotation, + scale_opacity, + } + } +} + +impl FromIterator for GaussianCloud4d { + fn from_iter>(iter: I) -> Self { + let gaussians = iter.into_iter().collect::>(); + GaussianCloud4d::from_packed(gaussians) + } +} diff --git a/src/gaussian/rand.rs b/src/gaussian/rand.rs index 4d4cab4c..e351e2e6 100644 --- a/src/gaussian/rand.rs +++ b/src/gaussian/rand.rs @@ -9,7 +9,7 @@ use crate::gaussian::f16::pack_f32s_to_u32; #[allow(unused_imports)] use crate::{ gaussian::{ - cloud::GaussianCloud, + cloud::Cloud, packed::Gaussian, }, material::spherical_harmonics::{ @@ -69,7 +69,7 @@ impl Distribution for rand::distributions::Standard { } } -pub fn random_gaussians(n: usize) -> GaussianCloud { +pub fn random_gaussians(n: usize) -> Cloud { let mut rng = rand::thread_rng(); let mut gaussians: Vec = Vec::with_capacity(n); @@ -77,6 +77,6 @@ pub fn random_gaussians(n: usize) -> GaussianCloud { gaussians.push(rng.gen()); } - GaussianCloud::from_gaussians(gaussians) + Cloud::from_gaussians(gaussians) } diff --git a/src/io/gcloud/bincode2.rs b/src/io/gcloud/bincode2.rs index 7e9b0874..0e6943cd 100644 --- a/src/io/gcloud/bincode2.rs +++ b/src/io/gcloud/bincode2.rs @@ -9,7 +9,7 @@ use flate2::{ }; use crate::{ - gaussian::GaussianCloud, + gaussian::Cloud, io::codec::GaussianCloudCodec, }; diff --git a/src/io/gcloud/flexbuffers.rs b/src/io/gcloud/flexbuffers.rs index a2490d80..a1bc8aae 100644 --- a/src/io/gcloud/flexbuffers.rs +++ b/src/io/gcloud/flexbuffers.rs @@ -8,12 +8,12 @@ use serde::{ }; use crate::{ - GaussianCloud, + Cloud, io::codec::GaussianCloudCodec, }; -impl GaussianCloudCodec for GaussianCloud { +impl GaussianCloudCodec for Cloud { fn encode(&self) -> Vec { let mut serializer = FlexbufferSerializer::new(); self.serialize(&mut serializer).expect("failed to serialize cloud"); @@ -23,7 +23,7 @@ impl GaussianCloudCodec for GaussianCloud { fn decode(data: &[u8]) -> Self { let reader = Reader::get_root(data).expect("failed to read flexbuffer"); - let cloud = GaussianCloud::deserialize(reader).expect("deserialization failed"); + let cloud = Cloud::deserialize(reader).expect("deserialization failed"); cloud } diff --git a/src/io/loader.rs b/src/io/loader.rs index 21e6c086..e0dcfce6 100644 --- a/src/io/loader.rs +++ b/src/io/loader.rs @@ -16,7 +16,7 @@ use bevy::{ }; use crate::{ - GaussianCloud, + Cloud, io::codec::GaussianCloudCodec, }; @@ -25,7 +25,7 @@ use crate::{ pub struct GaussianCloudLoader; impl AssetLoader for GaussianCloudLoader { - type Asset = GaussianCloud; + type Asset = Cloud; type Settings = (); type Error = std::io::Error; @@ -49,7 +49,7 @@ impl AssetLoader for GaussianCloudLoader { let gaussians = crate::io::ply::parse_ply(&mut f)?; - Ok(GaussianCloud::from_gaussians(gaussians)) + Ok(Cloud::from_gaussians(gaussians)) } #[cfg(not(feature = "io_ply"))] @@ -58,7 +58,7 @@ impl AssetLoader for GaussianCloudLoader { } }, Some(ext) if ext == "gcloud" => { - let cloud = GaussianCloud::decode(bytes.as_slice()); + let cloud = Cloud::decode(bytes.as_slice()); Ok(cloud) }, diff --git a/src/io/writer.rs b/src/io/writer.rs index e12bd533..7ee3e757 100644 --- a/src/io/writer.rs +++ b/src/io/writer.rs @@ -1,13 +1,13 @@ use std::io::Write; use crate::{ - GaussianCloud, + Cloud, io::codec::GaussianCloudCodec, }; pub fn write_gaussian_cloud_to_file( - cloud: &GaussianCloud, + cloud: &Cloud, path: &str, ) { let gcloud_file = std::fs::File::create(path).expect("failed to create file"); diff --git a/src/lib.rs b/src/lib.rs index c51fcb1c..f2e98d16 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,7 +2,7 @@ use bevy::prelude::*; pub use gaussian::{ packed::Gaussian, - cloud::GaussianCloud, + cloud::Cloud, rand::random_gaussians, settings::GaussianCloudSettings, }; @@ -30,7 +30,7 @@ pub mod noise; #[derive(Bundle, Default, Reflect)] pub struct GaussianSplattingBundle { pub settings: GaussianCloudSettings, - pub cloud: Handle, + pub cloud: Handle, pub visibility: Visibility, } @@ -45,9 +45,9 @@ impl Plugin for GaussianSplattingPlugin { fn build(&self, app: &mut App) { // TODO: allow hot reloading of GaussianCloud handle through inspector UI app.register_type::(); - app.register_type::(); - app.init_asset::(); - app.register_asset_reflect::(); + app.register_type::(); + app.init_asset::(); + app.register_asset_reflect::(); app.init_asset_loader::(); diff --git a/src/material/noise.rs b/src/material/noise.rs index 720a8b94..7a429349 100644 --- a/src/material/noise.rs +++ b/src/material/noise.rs @@ -7,7 +7,7 @@ use noise::{ use crate::{ Gaussian, - GaussianCloud, + Cloud, }; @@ -38,10 +38,10 @@ impl Plugin for NoiseMaterialPlugin { fn apply_noise_cpu( - mut gaussian_clouds_res: ResMut>, + mut gaussian_clouds_res: ResMut>, mut selections: Query<( Entity, - &Handle, + &Handle, &NoiseMaterial, Changed, )>, diff --git a/src/query/select.rs b/src/query/select.rs index 015c88dd..7d4fdbb3 100644 --- a/src/query/select.rs +++ b/src/query/select.rs @@ -5,7 +5,7 @@ use bevy::{ asset::LoadState, }; -use crate::{GaussianCloud, io::writer::write_gaussian_cloud_to_file}; +use crate::{Cloud, io::writer::write_gaussian_cloud_to_file}; #[derive(Component, Debug, Default, Reflect)] @@ -56,10 +56,10 @@ impl Plugin for SelectPlugin { fn apply_selection( asset_server: Res, - mut gaussian_clouds_res: ResMut>, + mut gaussian_clouds_res: ResMut>, mut selections: Query<( Entity, - &Handle, + &Handle, &mut Select, )>, ) { @@ -103,10 +103,10 @@ pub struct InvertSelectionEvent; fn invert_selection( mut events: EventReader, - mut gaussian_clouds_res: ResMut>, + mut gaussian_clouds_res: ResMut>, mut selections: Query<( Entity, - &Handle, + &Handle, &mut Select, )>, ) { @@ -152,10 +152,10 @@ pub struct SaveSelectionEvent; pub fn save_selection( mut events: EventReader, - mut gaussian_clouds_res: ResMut>, + mut gaussian_clouds_res: ResMut>, mut selections: Query<( Entity, - &Handle, + &Handle, &Select, )>, ) { diff --git a/src/query/sparse.rs b/src/query/sparse.rs index c0cd215a..c8c7e896 100644 --- a/src/query/sparse.rs +++ b/src/query/sparse.rs @@ -11,7 +11,7 @@ use typenum::consts::U3; use crate::{ Gaussian, - GaussianCloud, + Cloud, query::select::Select, }; @@ -44,7 +44,7 @@ impl Default for SparseSelect { impl SparseSelect { pub fn select( &self, - cloud: &GaussianCloud, + cloud: &Cloud, ) -> Select { let tree = KdTree::build_by_ordered_float(cloud.gaussian_iter().collect()); @@ -86,10 +86,10 @@ impl KdPoint for Gaussian { fn select_sparse_handler( mut commands: Commands, asset_server: Res, - gaussian_clouds_res: Res>, + gaussian_clouds_res: Res>, mut selections: Query<( Entity, - &Handle, + &Handle, &mut SparseSelect, )>, ) { diff --git a/src/render/mod.rs b/src/render/mod.rs index ce566289..26af217b 100644 --- a/src/render/mod.rs +++ b/src/render/mod.rs @@ -57,7 +57,7 @@ use bevy::{ use crate::{ gaussian::{ - cloud::GaussianCloud, + cloud::Cloud, settings::{ GaussianCloudDrawMode, GaussianCloudSettings, @@ -149,7 +149,7 @@ impl Plugin for RenderPipelinePlugin { Shader::from_wgsl ); - app.add_plugins(RenderAssetPlugin::::default()); + app.add_plugins(RenderAssetPlugin::::default()); app.add_plugins(UniformComponentPlugin::::default()); app.add_plugins(( @@ -191,7 +191,7 @@ pub struct GpuGaussianSplattingBundle { pub settings: GaussianCloudSettings, pub settings_uniform: GaussianCloudUniform, pub sorted_entries: Handle, - pub cloud_handle: Handle, // TODO: handle 4d gaussian cloud + pub cloud_handle: Handle, // TODO: handle 4d gaussian cloud } #[derive(Debug, Clone)] @@ -206,10 +206,10 @@ pub struct GpuGaussianCloud { pub draw_indirect_buffer: Buffer, #[cfg(feature = "debug_gpu")] - pub debug_gpu: GaussianCloud, + pub debug_gpu: Cloud, } -impl RenderAsset for GaussianCloud { - type ExtractedAsset = GaussianCloud; +impl RenderAsset for Cloud { + type ExtractedAsset = Cloud; type PreparedAsset = GpuGaussianCloud; type Param = SRes; @@ -234,7 +234,7 @@ impl RenderAsset for GaussianCloud { usage: BufferUsages::INDIRECT | BufferUsages::COPY_DST | BufferUsages::STORAGE | BufferUsages::COPY_SRC, }); - // TODO: (extract GaussianCloud, TextureBuffers) when feature buffer_texture is enabled + // TODO: (extract Cloud, TextureBuffers) when feature buffer_texture is enabled Ok(GpuGaussianCloud { count, @@ -254,7 +254,7 @@ impl RenderAsset for GaussianCloud { #[cfg(feature = "buffer_storage")] type GpuGaussianBundleQuery = ( Entity, - &'static Handle, + &'static Handle, &'static Handle, &'static GaussianCloudSettings, (), @@ -263,7 +263,7 @@ type GpuGaussianBundleQuery = ( #[cfg(feature = "buffer_texture")] type GpuGaussianBundleQuery = ( Entity, - &'static Handle, + &'static Handle, &'static Handle, &'static GaussianCloudSettings, &'static texture::GpuTextureBuffers, @@ -276,7 +276,7 @@ fn queue_gaussians( custom_pipeline: Res, mut pipelines: ResMut>, pipeline_cache: Res, - gaussian_clouds: Res>, + gaussian_clouds: Res>, sorted_entries: Res>, mut views: Query<( &ExtractedView, @@ -682,13 +682,13 @@ pub fn extract_gaussians( mut commands: Commands, mut prev_commands_len: Local, asset_server: Res, - gaussian_cloud_res: Res>, + gaussian_cloud_res: Res>, gaussians_query: Extract< Query<( Entity, // &ComputedVisibility, &Visibility, - &Handle, + &Handle, &Handle, &GaussianCloudSettings, )>, @@ -758,7 +758,7 @@ fn queue_gaussian_bind_group( render_device: Res, gaussian_uniforms: Res>, asset_server: Res, - gaussian_cloud_res: Res>, + gaussian_cloud_res: Res>, sorted_entries_res: Res>, gaussian_clouds: Query, @@ -972,10 +972,10 @@ impl RenderCommand

for SetGaussianUniformBindGr pub struct DrawGaussianInstanced; impl RenderCommand

for DrawGaussianInstanced { - type Param = SRes>; + type Param = SRes>; type ViewWorldQuery = (); type ItemWorldQuery = ( - Read>, + Read>, Read, ); @@ -987,7 +987,7 @@ impl RenderCommand

for DrawGaussianInstanced { handle, bind_groups, ): ( - &'w Handle, + &'w Handle, &'w GaussianCloudBindGroup, ), gaussian_clouds: SystemParamItem<'w, '_, Self::Param>, diff --git a/src/render/packed.rs b/src/render/packed.rs index e8b3e7aa..c6d2ec96 100644 --- a/src/render/packed.rs +++ b/src/render/packed.rs @@ -20,12 +20,12 @@ use bevy::render::{ use crate::{ gaussian::{ - cloud::GaussianCloud, + cloud::Cloud, packed::Gaussian, }, render::{ GaussianCloudPipeline, - GpuGaussianCloud, + GpuCloud, }, }; @@ -38,7 +38,7 @@ pub struct PackedBuffers { pub fn prepare_cloud( render_device: &RenderDevice, - cloud: &GaussianCloud, + cloud: &Cloud, ) -> PackedBuffers { let gaussians = render_device.create_buffer_with_data(&BufferInitDescriptor { label: Some("packed_gaussian_cloud_buffer"), diff --git a/src/render/planar.rs b/src/render/planar.rs index b41d47c0..6558a14a 100644 --- a/src/render/planar.rs +++ b/src/render/planar.rs @@ -7,7 +7,7 @@ use bevy::render::{ #[allow(unused_imports)] use crate::{ gaussian::{ - cloud::GaussianCloud, + cloud::Cloud, f32::{ PositionVisibility, Rotation, @@ -58,7 +58,7 @@ pub struct PlanarBuffers { #[cfg(feature = "f16")] pub fn prepare_cloud( render_device: &RenderDevice, - cloud: &GaussianCloud, + cloud: &Cloud, ) -> PlanarBuffers { let position_visibility = render_device.create_buffer_with_data(&BufferInitDescriptor { label: Some("planar_position_visibility_buffer"), @@ -101,7 +101,7 @@ pub fn prepare_cloud( #[cfg(feature = "f32")] pub fn prepare_cloud( render_device: &RenderDevice, - cloud: &GaussianCloud, + cloud: &Cloud, ) -> PlanarBuffers { let position_visibility = render_device.create_buffer_with_data(&BufferInitDescriptor { label: Some("planar_f32_position_visibility_buffer"), diff --git a/src/render/texture.rs b/src/render/texture.rs index 2bbff4ce..d0293406 100644 --- a/src/render/texture.rs +++ b/src/render/texture.rs @@ -35,7 +35,7 @@ use static_assertions::assert_cfg; #[allow(unused_imports)] use crate::{ gaussian::{ - cloud::GaussianCloud, + cloud::Cloud, f32::{ PositionVisibility, Rotation, @@ -217,11 +217,11 @@ pub fn queue_gpu_texture_buffers( fn queue_textures( mut commands: Commands, asset_server: Res, - gaussian_cloud_res: Res>, + gaussian_cloud_res: Res>, mut images: ResMut>, clouds: Query<( Entity, - &Handle, + &Handle, Without, )>, ) { diff --git a/src/sort/mod.rs b/src/sort/mod.rs index e44baf1f..9a0a1cbc 100644 --- a/src/sort/mod.rs +++ b/src/sort/mod.rs @@ -23,7 +23,7 @@ use bytemuck::{ use static_assertions::assert_cfg; use crate::{ - GaussianCloud, + Cloud, GaussianCloudSettings, }; @@ -140,11 +140,11 @@ fn update_textures_on_change( fn auto_insert_sorted_entries( mut commands: Commands, asset_server: Res, - gaussian_clouds_res: Res>, + gaussian_clouds_res: Res>, mut sorted_entries_res: ResMut>, gaussian_clouds: Query<( Entity, - &Handle, + &Handle, &GaussianCloudSettings, Without>, )>, diff --git a/src/sort/radix.rs b/src/sort/radix.rs index 85003014..bf1229f1 100644 --- a/src/sort/radix.rs +++ b/src/sort/radix.rs @@ -50,7 +50,7 @@ use bevy::{ use static_assertions::assert_cfg; use crate::{ - gaussian::cloud::GaussianCloud, + gaussian::cloud::Cloud, GaussianCloudSettings, render::{ GaussianCloudBindGroup, @@ -140,7 +140,7 @@ impl Plugin for RadixSortPlugin { #[derive(Resource, Default)] pub struct RadixSortBuffers { pub asset_map: HashMap< - AssetId, + AssetId, GpuRadixBuffers, >, } @@ -201,7 +201,7 @@ impl GpuRadixBuffers { fn update_sort_buffers( - gpu_gaussian_clouds: Res>, + gpu_gaussian_clouds: Res>, mut sort_buffers: ResMut, render_device: Res, ) { @@ -360,11 +360,11 @@ pub fn queue_radix_bind_group( radix_pipeline: Res, render_device: Res, asset_server: Res, - gaussian_cloud_res: Res>, + gaussian_cloud_res: Res>, sorted_entries_res: Res>, gaussian_clouds: Query<( Entity, - &Handle, + &Handle, &Handle, &GaussianCloudSettings, )>, @@ -489,7 +489,7 @@ pub fn queue_radix_bind_group( pub struct RadixSortNode { gaussian_clouds: QueryState<( - &'static Handle, + &'static Handle, &'static GaussianCloudBindGroup, &'static RadixBindGroup, )>, @@ -562,7 +562,7 @@ impl Node for RadixSortNode { cloud_bind_group, radix_bind_group, ) in self.gaussian_clouds.iter_manual(world) { - let cloud = world.get_resource::>().unwrap().get(cloud_handle).unwrap(); + let cloud = world.get_resource::>().unwrap().get(cloud_handle).unwrap(); assert!(sort_buffers.asset_map.contains_key(&cloud_handle.id())); let sorting_assets = &sort_buffers.asset_map[&cloud_handle.id()]; diff --git a/src/sort/rayon.rs b/src/sort/rayon.rs index a2890e68..a43a0758 100644 --- a/src/sort/rayon.rs +++ b/src/sort/rayon.rs @@ -7,7 +7,7 @@ use bevy::{ use rayon::prelude::*; use crate::{ - GaussianCloud, + Cloud, GaussianCloudSettings, sort::{ SortedEntries, @@ -28,10 +28,10 @@ impl Plugin for RayonSortPlugin { #[allow(clippy::too_many_arguments)] pub fn rayon_sort( asset_server: Res, - gaussian_clouds_res: Res>, + gaussian_clouds_res: Res>, mut sorted_entries_res: ResMut>, gaussian_clouds: Query<( - &Handle, + &Handle, &Handle, &GaussianCloudSettings, )>, diff --git a/src/sort/std.rs b/src/sort/std.rs index 14816e83..592efd87 100644 --- a/src/sort/std.rs +++ b/src/sort/std.rs @@ -5,7 +5,7 @@ use bevy::{ }; use crate::{ - GaussianCloud, + Cloud, GaussianCloudSettings, sort::{ SortedEntries, @@ -27,10 +27,10 @@ impl Plugin for StdSortPlugin { #[allow(clippy::too_many_arguments)] pub fn std_sort( asset_server: Res, - gaussian_clouds_res: Res>, + gaussian_clouds_res: Res>, mut sorted_entries_res: ResMut>, gaussian_clouds: Query<( - &Handle, + &Handle, &Handle, &GaussianCloudSettings, )>, diff --git a/tests/gaussian.rs b/tests/gaussian.rs index 5cb2a9fd..c7a9b59e 100644 --- a/tests/gaussian.rs +++ b/tests/gaussian.rs @@ -1,5 +1,5 @@ use bevy_gaussian_splatting::{ - GaussianCloud, + Cloud, io::codec::GaussianCloudCodec, random_gaussians, }; diff --git a/tests/gpu/gaussian.rs b/tests/gpu/gaussian.rs index d8563aa8..d509e3ac 100644 --- a/tests/gpu/gaussian.rs +++ b/tests/gpu/gaussian.rs @@ -13,7 +13,7 @@ use bevy::{ }; use bevy_gaussian_splatting::{ - GaussianCloud, + Cloud, GaussianSplattingBundle, random_gaussians, }; @@ -41,7 +41,7 @@ fn main() { fn setup( mut commands: Commands, - mut gaussian_assets: ResMut>, + mut gaussian_assets: ResMut>, ) { let cloud = gaussian_assets.add(random_gaussians(10000)); @@ -106,11 +106,11 @@ fn save_captures(captures: Arc>>) { } fn capture_ready( - // gaussian_cloud_assets: Res>, + // gaussian_cloud_assets: Res>, // asset_server: Res, // gaussian_clouds: Query< // Entity, - // &Handle, + // &Handle, // >, main_window: Query>, mut screenshot_manager: ResMut, diff --git a/tests/gpu/radix.rs b/tests/gpu/radix.rs index 4740d763..5ad3f730 100644 --- a/tests/gpu/radix.rs +++ b/tests/gpu/radix.rs @@ -35,7 +35,7 @@ use bevy::{ }; use bevy_gaussian_splatting::{ - GaussianCloud, + Cloud, GaussianSplattingBundle, random_gaussians, sort::SortedEntries, @@ -82,7 +82,7 @@ fn main() { fn setup( mut commands: Commands, - mut gaussian_assets: ResMut>, + mut gaussian_assets: ResMut>, ) { let cloud = gaussian_assets.add(random_gaussians(10000)); @@ -110,7 +110,7 @@ fn setup( pub struct RadixTestNode { gaussian_clouds: QueryState<( - &'static Handle, + &'static Handle, &'static Handle, )>, state: TestStateArc, @@ -170,7 +170,7 @@ impl Node for RadixTestNode { cloud_handle, sorted_entries_handle, ) in self.gaussian_clouds.iter_manual(world) { - let gaussian_cloud_res = world.get_resource::>().unwrap(); + let gaussian_cloud_res = world.get_resource::>().unwrap(); let sorted_entries_res = world.get_resource::>().unwrap(); let mut state = self.state.lock().unwrap(); diff --git a/tests/io.rs b/tests/io.rs index d6469b62..8d842c00 100644 --- a/tests/io.rs +++ b/tests/io.rs @@ -1,5 +1,5 @@ use bevy_gaussian_splatting::{ - GaussianCloud, + Cloud, io::codec::GaussianCloudCodec, random_gaussians, }; diff --git a/tools/compare_aabb_obb.rs b/tools/compare_aabb_obb.rs index 5777208f..4a4c8ffd 100644 --- a/tools/compare_aabb_obb.rs +++ b/tools/compare_aabb_obb.rs @@ -16,7 +16,7 @@ use bevy_panorbit_camera::{ use bevy_gaussian_splatting::{ Gaussian, - GaussianCloud, + Cloud, GaussianCloudSettings, GaussianSplattingBundle, GaussianSplattingPlugin, @@ -30,7 +30,7 @@ use bevy_gaussian_splatting::{ pub fn setup_aabb_obb_compare( mut commands: Commands, - mut gaussian_assets: ResMut>, + mut gaussian_assets: ResMut>, ) { let mut blue_sh = SphericalHarmonicCoefficients::default(); blue_sh.set(2, 5.0); diff --git a/tools/ply_to_gcloud.rs b/tools/ply_to_gcloud.rs index b3a52f61..6eaa5279 100644 --- a/tools/ply_to_gcloud.rs +++ b/tools/ply_to_gcloud.rs @@ -4,7 +4,7 @@ use byte_unit::{ }; use bevy_gaussian_splatting::{ - GaussianCloud, + Cloud, io::{ ply::parse_ply, writer::write_gaussian_cloud_to_file, diff --git a/viewer/viewer.rs b/viewer/viewer.rs index fa3bc1f1..bcdb3bfb 100644 --- a/viewer/viewer.rs +++ b/viewer/viewer.rs @@ -27,7 +27,7 @@ use bevy_panorbit_camera::{ }; use bevy_gaussian_splatting::{ - GaussianCloud, + Cloud, GaussianSplattingBundle, GaussianSplattingPlugin, random_gaussians, @@ -61,9 +61,9 @@ fn setup_gaussian_cloud( mut commands: Commands, asset_server: Res, gaussian_splatting_viewer: Res, - mut gaussian_assets: ResMut>, + mut gaussian_assets: ResMut>, ) { - let cloud: Handle; + let cloud: Handle; if gaussian_splatting_viewer.gaussian_count > 0 { log(&format!("generating {} gaussians", gaussian_splatting_viewer.gaussian_count)); cloud = gaussian_assets.add(random_gaussians(gaussian_splatting_viewer.gaussian_count)); @@ -106,7 +106,7 @@ fn setup_particle_behavior( mut particle_behavior_assets: ResMut>, gaussian_cloud: Query<( Entity, - &Handle, + &Handle, Without>, )>, ) { @@ -132,7 +132,7 @@ fn setup_noise_material( asset_server: Res, gaussian_clouds: Query<( Entity, - &Handle, + &Handle, Without, )>, ) { @@ -159,7 +159,7 @@ fn setup_sparse_select( mut commands: Commands, gaussian_cloud: Query<( Entity, - &Handle, + &Handle, Without, )>, ) {