Merge pull request #396 from ShaddyDC/fix-large-buffers

Allow handling large external buffers

examples/export/main.rs

@@ -3,6 +3,7 @@ use gltf_json as json;
 use std::{fs, mem};
 
 use json::validation::Checked::Valid;
+use json::validation::USize64;
 use std::borrow::Cow;
 use std::io::Write;
 
@@ -37,7 +38,7 @@ fn bounding_coords(points: &[Vertex]) -> ([f32; 3], [f32; 3]) {
     (min, max)
 }
 
-fn align_to_multiple_of_four(n: &mut u32) {
+fn align_to_multiple_of_four(n: &mut usize) {
     *n = (*n + 3) & !3;
 }
 
@@ -71,9 +72,9 @@ fn export(output: Output) {
 
     let (min, max) = bounding_coords(&triangle_vertices);
 
-    let buffer_length = (triangle_vertices.len() * mem::size_of::<Vertex>()) as u32;
+    let buffer_length = triangle_vertices.len() * mem::size_of::<Vertex>();
     let buffer = json::Buffer {
-        byte_length: buffer_length,
+        byte_length: USize64::from(buffer_length),
         extensions: Default::default(),
         extras: Default::default(),
         name: None,
@@ -87,16 +88,16 @@ fn export(output: Output) {
         buffer: json::Index::new(0),
         byte_length: buffer.byte_length,
         byte_offset: None,
-        byte_stride: Some(mem::size_of::<Vertex>() as u32),
+        byte_stride: Some(json::buffer::Stride(mem::size_of::<Vertex>())),
         extensions: Default::default(),
         extras: Default::default(),
         name: None,
         target: Some(Valid(json::buffer::Target::ArrayBuffer)),
     };
     let positions = json::Accessor {
         buffer_view: Some(json::Index::new(0)),
-        byte_offset: Some(0),
-        count: triangle_vertices.len() as u32,
+        byte_offset: Some(USize64(0)),
+        count: USize64::from(triangle_vertices.len()),
         component_type: Valid(json::accessor::GenericComponentType(
             json::accessor::ComponentType::F32,
         )),
@@ -111,8 +112,8 @@ fn export(output: Output) {
     };
     let colors = json::Accessor {
         buffer_view: Some(json::Index::new(0)),
-        byte_offset: Some((3 * mem::size_of::<f32>()) as u32),
-        count: triangle_vertices.len() as u32,
+        byte_offset: Some(USize64::from(3 * mem::size_of::<f32>())),
+        count: USize64::from(triangle_vertices.len()),
         component_type: Valid(json::accessor::GenericComponentType(
             json::accessor::ComponentType::F32,
         )),
@@ -192,13 +193,16 @@ fn export(output: Output) {
         }
         Output::Binary => {
             let json_string = json::serialize::to_string(&root).expect("Serialization error");
-            let mut json_offset = json_string.len() as u32;
+            let mut json_offset = json_string.len();
             align_to_multiple_of_four(&mut json_offset);
             let glb = gltf::binary::Glb {
                 header: gltf::binary::Header {
                     magic: *b"glTF",
                     version: 2,
-                    length: json_offset + buffer_length,
+                    // N.B., the size of binary glTF file is limited to range of `u32`.
+                    length: (json_offset + buffer_length)
+                        .try_into()
+                        .expect("file size exceeds binary glTF limit"),
                 },
                 bin: Some(Cow::Owned(to_padded_byte_vector(triangle_vertices))),
                 json: Cow::Owned(json_string.into_bytes()),

gltf-json/src/accessor.rs

@@ -1,4 +1,4 @@
-use crate::validation::{Checked, Error, Validate};
+use crate::validation::{Checked, Error, USize64, Validate};
 use crate::{buffer, extensions, Extras, Index, Path, Root};
 use gltf_derive::Validate;
 use serde::{de, ser};
@@ -94,7 +94,7 @@ pub mod sparse {
 
         /// The offset relative to the start of the parent `BufferView` in bytes.
         #[serde(default, rename = "byteOffset")]
-        pub byte_offset: u32,
+        pub byte_offset: USize64,
 
         /// The data type of each index.
         #[serde(rename = "componentType")]
@@ -115,7 +115,7 @@ pub mod sparse {
     #[derive(Clone, Debug, Deserialize, Serialize, Validate)]
     pub struct Sparse {
         /// The number of attributes encoded in this sparse accessor.
-        pub count: u32,
+        pub count: USize64,
 
         /// Index array of size `count` that points to those accessor attributes
         /// that deviate from their initialization value.
@@ -154,7 +154,7 @@ pub mod sparse {
 
         /// The offset relative to the start of the parent buffer view in bytes.
         #[serde(default, rename = "byteOffset")]
-        pub byte_offset: u32,
+        pub byte_offset: USize64,
 
         /// Extension specific data.
         #[serde(default, skip_serializing_if = "Option::is_none")]
@@ -183,11 +183,11 @@ pub struct Accessor {
     /// This field can be omitted in sparse accessors.
     #[serde(default, rename = "byteOffset")]
     #[serde(skip_serializing_if = "Option::is_none")]
-    pub byte_offset: Option<u32>,
+    pub byte_offset: Option<USize64>,
 
     /// The number of components within the buffer view - not to be confused
     /// with the number of bytes in the buffer view.
-    pub count: u32,
+    pub count: USize64,
 
     /// The data type of components in the attribute.
     #[serde(rename = "componentType")]

gltf-json/src/buffer.rs

@@ -1,5 +1,5 @@
-use crate::validation::Checked;
-use crate::{extensions, Extras, Index};
+use crate::validation::{Checked, Error, USize64, Validate};
+use crate::{extensions, Extras, Index, Path, Root};
 use gltf_derive::Validate;
 use serde::{de, ser};
 use serde_derive::{Deserialize, Serialize};
@@ -12,10 +12,10 @@ pub const ARRAY_BUFFER: u32 = 34_962;
 pub const ELEMENT_ARRAY_BUFFER: u32 = 34_963;
 
 /// The minimum byte stride.
-pub const MIN_BYTE_STRIDE: u32 = 4;
+pub const MIN_BYTE_STRIDE: usize = 4;
 
 /// The maximum byte stride.
-pub const MAX_BYTE_STRIDE: u32 = 252;
+pub const MAX_BYTE_STRIDE: usize = 252;
 
 /// All valid GPU buffer targets.
 pub const VALID_TARGETS: &[u32] = &[ARRAY_BUFFER, ELEMENT_ARRAY_BUFFER];
@@ -42,12 +42,28 @@ impl ser::Serialize for Target {
     }
 }
 
+/// Distance between individual items in a buffer view, measured in bytes.
+#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, Hash, PartialEq, Serialize)]
+pub struct Stride(pub usize);
+
+impl Validate for Stride {
+    fn validate<P, R>(&self, _root: &Root, path: P, report: &mut R)
+    where
+        P: Fn() -> Path,
+        R: FnMut(&dyn Fn() -> Path, Error),
+    {
+        if self.0 < MIN_BYTE_STRIDE || self.0 > MAX_BYTE_STRIDE {
+            report(&path, Error::Invalid);
+        }
+    }
+}
+
 /// A buffer points to binary data representing geometry, animations, or skins.
 #[derive(Clone, Debug, Deserialize, Serialize, Validate)]
 pub struct Buffer {
     /// The length of the buffer in bytes.
     #[serde(default, rename = "byteLength")]
-    pub byte_length: u32,
+    pub byte_length: USize64,
 
     /// Optional user-defined name for this object.
     #[cfg(feature = "names")]
@@ -81,22 +97,22 @@ pub struct View {
 
     /// The length of the `BufferView` in bytes.
     #[serde(rename = "byteLength")]
-    pub byte_length: u32,
+    pub byte_length: USize64,
 
     /// Offset into the parent buffer in bytes.
     #[serde(
         default,
         rename = "byteOffset",
         skip_serializing_if = "Option::is_none"
     )]
-    pub byte_offset: Option<u32>,
+    pub byte_offset: Option<USize64>,
 
     /// The stride in bytes between vertex attributes or other interleavable data.
     ///
     /// When zero, data is assumed to be tightly packed.
     #[serde(rename = "byteStride")]
     #[serde(skip_serializing_if = "Option::is_none")]
-    pub byte_stride: Option<u32>,
+    pub byte_stride: Option<Stride>,
 
     /// Optional user-defined name for this object.
     #[cfg(feature = "names")]

gltf-json/src/validation.rs

@@ -27,6 +27,9 @@ pub enum Error {
 
     /// Some required data has been omitted.
     Missing,
+
+    /// A memory size or offset exceeds the system limits.
+    Oversize,
 }
 
 /// Specifies a type that has been pre-validated during deserialization or otherwise.
@@ -103,6 +106,44 @@ impl<T> Validate for Checked<T> {
     }
 }
 
+/// Validates the suitability of 64-bit byte offsets/sizes on 32-bit systems.
+#[derive(
+    Clone,
+    Copy,
+    Debug,
+    Default,
+    Eq,
+    Hash,
+    PartialEq,
+    serde_derive::Deserialize,
+    serde_derive::Serialize,
+)]
+pub struct USize64(pub u64);
+
+impl From<u64> for USize64 {
+    fn from(value: u64) -> Self {
+        Self(value)
+    }
+}
+
+impl From<usize> for USize64 {
+    fn from(value: usize) -> Self {
+        Self(value as u64)
+    }
+}
+
+impl Validate for USize64 {
+    fn validate<P, R>(&self, _root: &Root, path: P, report: &mut R)
+    where
+        P: Fn() -> Path,
+        R: FnMut(&dyn Fn() -> Path, Error),
+    {
+        if usize::try_from(self.0).is_err() {
+            report(&path, Error::Oversize);
+        }
+    }
+}
+
 impl<K: ToString + Validate, V: Validate> Validate for BTreeMap<K, V> {
     fn validate<P, R>(&self, root: &Root, path: P, report: &mut R)
     where
@@ -174,6 +215,7 @@ impl std::fmt::Display for Error {
                 Error::IndexOutOfBounds => "Index out of bounds",
                 Error::Invalid => "Invalid value",
                 Error::Missing => "Missing data",
+                Error::Oversize => "Size exceeds system limits",
             }
         )
     }

src/accessor/mod.rs

@@ -126,13 +126,13 @@ impl<'a> Accessor<'a> {
     pub fn offset(&self) -> usize {
         // TODO: Change this function to return Option<usize> in the next
         // version and return None for sparse accessors.
-        self.json.byte_offset.unwrap_or(0) as usize
+        self.json.byte_offset.unwrap_or_default().0 as usize
     }
 
     /// Returns the number of components within the buffer view - not to be confused
     /// with the number of bytes in the buffer view.
     pub fn count(&self) -> usize {
-        self.json.count as usize
+        self.json.count.0 as usize
     }
 
     /// Returns the data type of components in the attribute.

src/accessor/sparse.rs

@@ -37,8 +37,8 @@ impl<'a> Indices<'a> {
     }
 
     /// The offset relative to the start of the parent buffer view in bytes.
-    pub fn offset(&self) -> u32 {
-        self.json.byte_offset
+    pub fn offset(&self) -> usize {
+        self.json.byte_offset.0 as usize
     }
 
     /// The data type of each index.
@@ -73,8 +73,8 @@ impl<'a> Sparse<'a> {
     }
 
     /// Returns the number of attributes encoded in this sparse accessor.
-    pub fn count(&self) -> u32 {
-        self.json.count
+    pub fn count(&self) -> usize {
+        self.json.count.0 as usize
     }
 
     /// Returns an index array of size `count` that points to those accessor
@@ -120,8 +120,8 @@ impl<'a> Values<'a> {
     }
 
     /// The offset relative to the start of the parent buffer view in bytes.
-    pub fn offset(&self) -> u32 {
-        self.json.byte_offset
+    pub fn offset(&self) -> usize {
+        self.json.byte_offset.0 as usize
     }
 
     /// Optional application specific data.

src/accessor/util.rs

@@ -303,14 +303,14 @@ impl<'a, 's, T: Item> Iter<'s, T> {
 
                 let indices = sparse.indices();
                 let values = sparse.values();
-                let sparse_count = sparse.count() as usize;
+                let sparse_count = sparse.count();
 
                 let index_iter = {
                     let view = indices.view();
                     let index_size = indices.index_type().size();
                     let stride = view.stride().unwrap_or(index_size);
 
-                    let start = indices.offset() as usize;
+                    let start = indices.offset();
                     let end = start + stride * (sparse_count - 1) + index_size;
                     let subslice = buffer_view_slice(view, &get_buffer_data)
                         .and_then(|slice| slice.get(start..end))?;
@@ -332,7 +332,7 @@ impl<'a, 's, T: Item> Iter<'s, T> {
                     let view = values.view();
                     let stride = view.stride().unwrap_or(mem::size_of::<T>());
 
-                    let start = values.offset() as usize;
+                    let start = values.offset();
                     let end = start + stride * (sparse_count - 1) + mem::size_of::<T>();
                     let subslice = buffer_view_slice(view, &get_buffer_data)
                         .and_then(|slice| slice.get(start..end))?;

src/animation/mod.rs

@@ -4,6 +4,7 @@ use crate::{accessor, scene, Document};
 use crate::Buffer;
 
 pub use json::animation::{Interpolation, Property};
+#[cfg(feature = "extensions")]
 use serde_json::{Map, Value};
 
 /// Iterators.

src/buffer.rs

@@ -4,6 +4,7 @@ use std::ops;
 use crate::Document;
 
 pub use json::buffer::Target;
+#[cfg(feature = "extensions")]
 use serde_json::{Map, Value};
 
 /// A buffer points to binary data representing geometry, animations, or skins.
@@ -92,7 +93,7 @@ impl<'a> Buffer<'a> {
 
     /// The length of the buffer in bytes.
     pub fn length(&self) -> usize {
-        self.json.byte_length as usize
+        self.json.byte_length.0 as usize
     }
 
     /// Optional user-defined name for this object.
@@ -151,12 +152,12 @@ impl<'a> View<'a> {
 
     /// Returns the length of the buffer view in bytes.
     pub fn length(&self) -> usize {
-        self.json.byte_length as usize
+        self.json.byte_length.0 as usize
     }
 
     /// Returns the offset into the parent buffer in bytes.
     pub fn offset(&self) -> usize {
-        self.json.byte_offset.unwrap_or(0) as usize
+        self.json.byte_offset.unwrap_or_default().0 as usize
     }
 
     /// Returns the stride in bytes between vertex attributes or other interleavable
@@ -165,10 +166,10 @@ impl<'a> View<'a> {
         self.json.byte_stride.and_then(|x| {
             // Treat byte_stride == 0 same as not specifying stride.
             // This is technically a validation error, but best way we can handle it here
-            if x == 0 {
+            if x.0 == 0 {
                 None
             } else {
-                Some(x as usize)
+                Some(x.0)
             }
         })
     }