Allow handling large external buffers

examples/export/main.rs

@@ -3,6 +3,7 @@ use gltf_json as json;
 use std::{fs, mem};
 
 use json::validation::Checked::Valid;
+use json::validation::USize64;
 use std::borrow::Cow;
 use std::io::Write;
 
@@ -37,7 +38,7 @@ fn bounding_coords(points: &[Vertex]) -> ([f32; 3], [f32; 3]) {
  (min, max)
 }
 
-fn align_to_multiple_of_four(n: &mut u32) {
+fn align_to_multiple_of_four(n: &mut usize) {
  *n = (*n + 3) & !3;
 }
 
@@ -71,9 +72,9 @@ fn export(output: Output) {
 
  let (min, max) = bounding_coords(&triangle_vertices);
 
- let buffer_length = (triangle_vertices.len() * mem::size_of::<Vertex>()) as u32;
+ let buffer_length = triangle_vertices.len() * mem::size_of::<Vertex>();
  let buffer = json::Buffer {
- byte_length: buffer_length,
+ byte_length: USize64::from(buffer_length),
  extensions: Default::default(),
  extras: Default::default(),
  name: None,
@@ -87,16 +88,16 @@ fn export(output: Output) {
  buffer: json::Index::new(0),
  byte_length: buffer.byte_length,
  byte_offset: None,
- byte_stride: Some(mem::size_of::<Vertex>() as u32),
+ byte_stride: Some(json::buffer::Stride(mem::size_of::<Vertex>())),
  extensions: Default::default(),
  extras: Default::default(),
  name: None,
  target: Some(Valid(json::buffer::Target::ArrayBuffer)),
  };
  let positions = json::Accessor {
  buffer_view: Some(json::Index::new(0)),
- byte_offset: Some(0),
- count: triangle_vertices.len() as u32,
+ byte_offset: Some(USize64(0)),
+ count: USize64::from(triangle_vertices.len()),
  component_type: Valid(json::accessor::GenericComponentType(
  json::accessor::ComponentType::F32,
  )),
@@ -111,8 +112,8 @@ fn export(output: Output) {
  };
  let colors = json::Accessor {
  buffer_view: Some(json::Index::new(0)),
- byte_offset: Some((3 * mem::size_of::<f32>()) as u32),
- count: triangle_vertices.len() as u32,
+ byte_offset: Some(USize64::from(3 * mem::size_of::<f32>())),
+ count: USize64::from(triangle_vertices.len()),
  component_type: Valid(json::accessor::GenericComponentType(
  json::accessor::ComponentType::F32,
  )),
@@ -192,13 +193,16 @@ fn export(output: Output) {
  }
  Output::Binary => {
  let json_string = json::serialize::to_string(&root).expect("Serialization error");
- let mut json_offset = json_string.len() as u32;
+ let mut json_offset = json_string.len();
  align_to_multiple_of_four(&mut json_offset);
  let glb = gltf::binary::Glb {
  header: gltf::binary::Header {
  magic: *b"glTF",
  version: 2,
- length: json_offset + buffer_length,
+ // N.B., the size of binary glTF file is limited to range of `u32`.
+ length: (json_offset + buffer_length)
+ .try_into()
+ .expect("file size exceeds binary glTF limit"),
  },
  bin: Some(Cow::Owned(to_padded_byte_vector(triangle_vertices))),
  json: Cow::Owned(json_string.into_bytes()),

gltf-json/src/accessor.rs

@@ -1,4 +1,4 @@
-use crate::validation::{Checked, Error, Validate};
+use crate::validation::{Checked, Error, USize64, Validate};
 use crate::{buffer, extensions, Extras, Index, Path, Root};
 use gltf_derive::Validate;
 use serde::{de, ser};
@@ -94,7 +94,7 @@ pub mod sparse {
 
  /// The offset relative to the start of the parent `BufferView` in bytes.
  #[serde(default, rename = "byteOffset")]
- pub byte_offset: u32,
+ pub byte_offset: USize64,
 
  /// The data type of each index.
  #[serde(rename = "componentType")]
@@ -115,7 +115,7 @@ pub mod sparse {
  #[derive(Clone, Debug, Deserialize, Serialize, Validate)]
  pub struct Sparse {
  /// The number of attributes encoded in this sparse accessor.
- pub count: u32,
+ pub count: USize64,
 
  /// Index array of size `count` that points to those accessor attributes
  /// that deviate from their initialization value.
@@ -154,7 +154,7 @@ pub mod sparse {
 
  /// The offset relative to the start of the parent buffer view in bytes.
  #[serde(default, rename = "byteOffset")]
- pub byte_offset: u32,
+ pub byte_offset: USize64,
 
  /// Extension specific data.
  #[serde(default, skip_serializing_if = "Option::is_none")]
@@ -183,11 +183,11 @@ pub struct Accessor {
  /// This field can be omitted in sparse accessors.
  #[serde(default, rename = "byteOffset")]
  #[serde(skip_serializing_if = "Option::is_none")]
- pub byte_offset: Option<u32>,
+ pub byte_offset: Option<USize64>,
 
  /// The number of components within the buffer view - not to be confused
  /// with the number of bytes in the buffer view.
- pub count: u32,
+ pub count: USize64,
 
  /// The data type of components in the attribute.
  #[serde(rename = "componentType")]

gltf-json/src/buffer.rs

@@ -1,5 +1,5 @@
-use crate::validation::Checked;
-use crate::{extensions, Extras, Index};
+use crate::validation::{Checked, Error, USize64, Validate};
+use crate::{extensions, Extras, Index, Path, Root};
 use gltf_derive::Validate;
 use serde::{de, ser};
 use serde_derive::{Deserialize, Serialize};
@@ -12,10 +12,10 @@ pub const ARRAY_BUFFER: u32 = 34_962;
 pub const ELEMENT_ARRAY_BUFFER: u32 = 34_963;
 
 /// The minimum byte stride.
-pub const MIN_BYTE_STRIDE: u32 = 4;
+pub const MIN_BYTE_STRIDE: usize = 4;
 
 /// The maximum byte stride.
-pub const MAX_BYTE_STRIDE: u32 = 252;
+pub const MAX_BYTE_STRIDE: usize = 252;
 
 /// All valid GPU buffer targets.
 pub const VALID_TARGETS: &[u32] = &[ARRAY_BUFFER, ELEMENT_ARRAY_BUFFER];
@@ -42,12 +42,28 @@ impl ser::Serialize for Target {
  }
 }
 
+/// Distance between individual items in a buffer view, measured in bytes.
+#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, Hash, PartialEq, Serialize)]
+pub struct Stride(pub usize);
+
+impl Validate for Stride {
+ fn validate<P, R>(&self, _root: &Root, path: P, report: &mut R)
+ where
+ P: Fn() -> Path,
+ R: FnMut(&dyn Fn() -> Path, Error),
+ {
+ if self.0 < MIN_BYTE_STRIDE || self.0 > MAX_BYTE_STRIDE {
+ report(&path, Error::Invalid);
+ }
+ }
+}
+
 /// A buffer points to binary data representing geometry, animations, or skins.
 #[derive(Clone, Debug, Deserialize, Serialize, Validate)]
 pub struct Buffer {
  /// The length of the buffer in bytes.
  #[serde(default, rename = "byteLength")]
- pub byte_length: u32,
+ pub byte_length: USize64,
 
  /// Optional user-defined name for this object.
  #[cfg(feature = "names")]
@@ -81,22 +97,22 @@ pub struct View {
 
  /// The length of the `BufferView` in bytes.
  #[serde(rename = "byteLength")]
- pub byte_length: u32,
+ pub byte_length: USize64,
 
  /// Offset into the parent buffer in bytes.
  #[serde(
  default,
  rename = "byteOffset",
  skip_serializing_if = "Option::is_none"
  )]
- pub byte_offset: Option<u32>,
+ pub byte_offset: Option<USize64>,
 
  /// The stride in bytes between vertex attributes or other interleavable data.
  ///
  /// When zero, data is assumed to be tightly packed.
  #[serde(rename = "byteStride")]
  #[serde(skip_serializing_if = "Option::is_none")]
- pub byte_stride: Option<u32>,
+ pub byte_stride: Option<Stride>,
 
  /// Optional user-defined name for this object.
  #[cfg(feature = "names")]

gltf-json/src/validation.rs

@@ -27,6 +27,9 @@ pub enum Error {
 
  /// Some required data has been omitted.
  Missing,
+
+ /// A memory size or offset exceeds the system limits.
+ Oversize,
 }
 
 /// Specifies a type that has been pre-validated during deserialization or otherwise.
@@ -103,6 +106,44 @@ impl<T> Validate for Checked<T> {
  }
 }
 
+/// Validates the suitability of 64-bit byte offsets/sizes on 32-bit systems.
+#[derive(
+ Clone,
+ Copy,
+ Debug,
+ Default,
+ Eq,
+ Hash,
+ PartialEq,
+ serde_derive::Deserialize,
+ serde_derive::Serialize,
+)]
+pub struct USize64(pub u64);
+
+impl From<u64> for USize64 {
+ fn from(value: u64) -> Self {
+ Self(value)
+ }
+}
+
+impl From<usize> for USize64 {
+ fn from(value: usize) -> Self {
+ Self(value as u64)
+ }
+}
+
+impl Validate for USize64 {
+ fn validate<P, R>(&self, _root: &Root, path: P, report: &mut R)
+ where
+ P: Fn() -> Path,
+ R: FnMut(&dyn Fn() -> Path, Error),
+ {
+ if usize::try_from(self.0).is_err() {
+ report(&path, Error::Oversize);
+ }
+ }
+}
+
 impl<K: ToString + Validate, V: Validate> Validate for BTreeMap<K, V> {
  fn validate<P, R>(&self, root: &Root, path: P, report: &mut R)
  where
@@ -174,6 +215,7 @@ impl std::fmt::Display for Error {
  Error::IndexOutOfBounds => "Index out of bounds",
  Error::Invalid => "Invalid value",
  Error::Missing => "Missing data",
+ Error::Oversize => "Size exceeds system limits",
  }
  )
  }

src/accessor/mod.rs

@@ -126,13 +126,13 @@ impl<'a> Accessor<'a> {
  pub fn offset(&self) -> usize {
  // TODO: Change this function to return Option<usize> in the next
  // version and return None for sparse accessors.
- self.json.byte_offset.unwrap_or(0) as usize
+ self.json.byte_offset.unwrap_or_default().0 as usize
  }
 
  /// Returns the number of components within the buffer view - not to be confused
  /// with the number of bytes in the buffer view.
  pub fn count(&self) -> usize {
- self.json.count as usize
+ self.json.count.0 as usize
  }
 
  /// Returns the data type of components in the attribute.

src/accessor/sparse.rs

@@ -37,8 +37,8 @@ impl<'a> Indices<'a> {
  }
 
  /// The offset relative to the start of the parent buffer view in bytes.
- pub fn offset(&self) -> u32 {
- self.json.byte_offset
+ pub fn offset(&self) -> usize {
+ self.json.byte_offset.0 as usize
  }
 
  /// The data type of each index.
@@ -73,8 +73,8 @@ impl<'a> Sparse<'a> {
  }
 
  /// Returns the number of attributes encoded in this sparse accessor.
- pub fn count(&self) -> u32 {
- self.json.count
+ pub fn count(&self) -> usize {
+ self.json.count.0 as usize
  }
 
  /// Returns an index array of size `count` that points to those accessor
@@ -120,8 +120,8 @@ impl<'a> Values<'a> {
  }
 
  /// The offset relative to the start of the parent buffer view in bytes.
- pub fn offset(&self) -> u32 {
- self.json.byte_offset
+ pub fn offset(&self) -> usize {
+ self.json.byte_offset.0 as usize
  }
 
  /// Optional application specific data.

src/accessor/util.rs

@@ -303,14 +303,14 @@ impl<'a, 's, T: Item> Iter<'s, T> {
 
  let indices = sparse.indices();
  let values = sparse.values();
- let sparse_count = sparse.count() as usize;
+ let sparse_count = sparse.count();
 
  let index_iter = {
  let view = indices.view();
  let index_size = indices.index_type().size();
  let stride = view.stride().unwrap_or(index_size);
 
- let start = indices.offset() as usize;
+ let start = indices.offset();
  let end = start + stride * (sparse_count - 1) + index_size;
  let subslice = buffer_view_slice(view, &get_buffer_data)
  .and_then(|slice| slice.get(start..end))?;
@@ -332,7 +332,7 @@ impl<'a, 's, T: Item> Iter<'s, T> {
  let view = values.view();
  let stride = view.stride().unwrap_or(mem::size_of::<T>());
 
- let start = values.offset() as usize;
+ let start = values.offset();
  let end = start + stride * (sparse_count - 1) + mem::size_of::<T>();
  let subslice = buffer_view_slice(view, &get_buffer_data)
  .and_then(|slice| slice.get(start..end))?;

src/animation/mod.rs

@@ -4,6 +4,7 @@ use crate::{accessor, scene, Document};
 use crate::Buffer;
 
 pub use json::animation::{Interpolation, Property};
+#[cfg(feature = "extensions")]
 use serde_json::{Map, Value};
 
 /// Iterators.

src/buffer.rs

@@ -4,6 +4,7 @@ use std::ops;
 use crate::Document;
 
 pub use json::buffer::Target;
+#[cfg(feature = "extensions")]
 use serde_json::{Map, Value};
 
 /// A buffer points to binary data representing geometry, animations, or skins.
@@ -92,7 +93,7 @@ impl<'a> Buffer<'a> {
 
  /// The length of the buffer in bytes.
  pub fn length(&self) -> usize {
- self.json.byte_length as usize
+ self.json.byte_length.0 as usize
  }
 
  /// Optional user-defined name for this object.
@@ -151,12 +152,12 @@ impl<'a> View<'a> {
 
  /// Returns the length of the buffer view in bytes.
  pub fn length(&self) -> usize {
- self.json.byte_length as usize
+ self.json.byte_length.0 as usize
  }
 
  /// Returns the offset into the parent buffer in bytes.
  pub fn offset(&self) -> usize {
- self.json.byte_offset.unwrap_or(0) as usize
+ self.json.byte_offset.unwrap_or_default().0 as usize
  }
 
  /// Returns the stride in bytes between vertex attributes or other interleavable
@@ -165,10 +166,10 @@ impl<'a> View<'a> {
  self.json.byte_stride.and_then(|x| {
  // Treat byte_stride == 0 same as not specifying stride.
  // This is technically a validation error, but best way we can handle it here
- if x == 0 {
+ if x.0 == 0 {
  None
  } else {
- Some(x as usize)
+ Some(x.0)
  }
  })
  }