From d4b1d57f3cf0f325520785f48ee5b9d6d7e45841 Mon Sep 17 00:00:00 2001 From: Jim Blandy Date: Wed, 23 Nov 2022 14:29:20 -0800 Subject: [PATCH] Move `ResourceMetadata` into its own module, and give it a proper interface (#3213) --- CHANGELOG.md | 1 + wgpu-core/src/track/buffer.rs | 45 +++--- wgpu-core/src/track/metadata.rs | 268 +++++++++++++++++++++++++++++++ wgpu-core/src/track/mod.rs | 176 +------------------- wgpu-core/src/track/stateless.rs | 53 +++--- wgpu-core/src/track/texture.rs | 66 ++++---- 6 files changed, 342 insertions(+), 267 deletions(-) create mode 100644 wgpu-core/src/track/metadata.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index a9d6fb5e37..74efa7bd9a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -112,6 +112,7 @@ Bottom level categories: - Update the `minimum supported rust version` to 1.64 - Use cargo 1.64 workspace inheritance feature. By @jinleili in [#3107](https://github.com/gfx-rs/wgpu/pull/3107) +- Move `ResourceMetadata` into its own module. By @jimblandy in [#3213](https://github.com/gfx-rs/wgpu/pull/3213) #### Vulkan diff --git a/wgpu-core/src/track/buffer.rs b/wgpu-core/src/track/buffer.rs index c021a349ac..65f02c467d 100644 --- a/wgpu-core/src/track/buffer.rs +++ b/wgpu-core/src/track/buffer.rs @@ -13,8 +13,8 @@ use crate::{ id::{BufferId, TypedId, Valid}, resource::Buffer, track::{ - invalid_resource_state, iterate_bitvec_indices, skip_barrier, ResourceMetadata, - ResourceMetadataProvider, ResourceUses, UsageConflict, + invalid_resource_state, skip_barrier, ResourceMetadata, ResourceMetadataProvider, + ResourceUses, UsageConflict, }, LifeGuard, RefCount, }; @@ -124,7 +124,7 @@ impl BufferUsageScope { /// Returns a list of all buffers tracked. pub fn used(&self) -> impl Iterator> + '_ { - self.metadata.used() + self.metadata.owned_ids() } /// Merge the list of buffer states in the given bind group into this usage scope. @@ -180,7 +180,7 @@ impl BufferUsageScope { self.set_size(incoming_size); } - for index in iterate_bitvec_indices(&scope.metadata.owned) { + for index in scope.metadata.owned_indices() { self.tracker_assert_in_bounds(index); scope.tracker_assert_in_bounds(index); @@ -293,7 +293,7 @@ impl BufferTracker { /// Returns a list of all buffers tracked. pub fn used(&self) -> impl Iterator> + '_ { - self.metadata.used() + self.metadata.owned_ids() } /// Drains all currently pending transitions. @@ -316,7 +316,7 @@ impl BufferTracker { self.tracker_assert_in_bounds(index); unsafe { - let currently_owned = self.metadata.owned.get(index).unwrap_unchecked(); + let currently_owned = self.metadata.contains_unchecked(index); if currently_owned { panic!("Tried to insert buffer already tracked"); @@ -394,7 +394,7 @@ impl BufferTracker { self.set_size(incoming_size); } - for index in iterate_bitvec_indices(&tracker.metadata.owned) { + for index in tracker.metadata.owned_indices() { self.tracker_assert_in_bounds(index); tracker.tracker_assert_in_bounds(index); unsafe { @@ -434,7 +434,7 @@ impl BufferTracker { self.set_size(incoming_size); } - for index in iterate_bitvec_indices(&scope.metadata.owned) { + for index in scope.metadata.owned_indices() { self.tracker_assert_in_bounds(index); scope.tracker_assert_in_bounds(index); unsafe { @@ -492,7 +492,7 @@ impl BufferTracker { scope.tracker_assert_in_bounds(index); - if unsafe { !scope.metadata.owned.get(index).unwrap_unchecked() } { + if unsafe { !scope.metadata.contains_unchecked(index) } { continue; } unsafe { @@ -514,7 +514,7 @@ impl BufferTracker { ) }; - unsafe { scope.metadata.reset(index) }; + unsafe { scope.metadata.remove(index) }; } } @@ -529,22 +529,19 @@ impl BufferTracker { let (index32, epoch, _) = id.0.unzip(); let index = index32 as usize; - if index > self.metadata.owned.len() { + if index > self.metadata.size() { return false; } self.tracker_assert_in_bounds(index); unsafe { - if self.metadata.owned.get(index).unwrap_unchecked() { - let existing_epoch = self.metadata.epochs.get_unchecked(index); - let existing_ref_count = self.metadata.ref_counts.get_unchecked(index); - - if *existing_epoch == epoch - && existing_ref_count.as_ref().unwrap_unchecked().load() == 1 - { - self.metadata.reset(index); + if self.metadata.contains_unchecked(index) { + let existing_epoch = self.metadata.get_epoch_unchecked(index); + let existing_ref_count = self.metadata.get_ref_count_unchecked(index); + if existing_epoch == epoch && existing_ref_count.load() == 1 { + self.metadata.remove(index); return true; } } @@ -600,7 +597,7 @@ unsafe fn insert_or_merge( state_provider: BufferStateProvider<'_>, metadata_provider: ResourceMetadataProvider<'_, A>, ) -> Result<(), UsageConflict> { - let currently_owned = unsafe { resource_metadata.owned.get(index).unwrap_unchecked() }; + let currently_owned = unsafe { resource_metadata.contains_unchecked(index) }; if !currently_owned { unsafe { @@ -659,7 +656,7 @@ unsafe fn insert_or_barrier_update( metadata_provider: ResourceMetadataProvider<'_, A>, barriers: &mut Vec>, ) { - let currently_owned = unsafe { resource_metadata.owned.get(index).unwrap_unchecked() }; + let currently_owned = unsafe { resource_metadata.contains_unchecked(index) }; if !currently_owned { unsafe { @@ -720,11 +717,7 @@ unsafe fn insert( *current_states.get_unchecked_mut(index) = new_end_state; let (epoch, ref_count) = metadata_provider.get_own(life_guard, index); - - resource_metadata.owned.set(index, true); - - *resource_metadata.epochs.get_unchecked_mut(index) = epoch; - *resource_metadata.ref_counts.get_unchecked_mut(index) = Some(ref_count); + resource_metadata.insert(index, epoch, ref_count); } } diff --git a/wgpu-core/src/track/metadata.rs b/wgpu-core/src/track/metadata.rs new file mode 100644 index 0000000000..728ff5ca0e --- /dev/null +++ b/wgpu-core/src/track/metadata.rs @@ -0,0 +1,268 @@ +//! The `ResourceMetadata` type. + +use crate::{ + hub, + id::{self, TypedId}, + Epoch, LifeGuard, RefCount, +}; +use bit_vec::BitVec; +use std::{borrow::Cow, marker::PhantomData, mem}; + +/// A set of resources, holding a [`RefCount`] and epoch for each member. +/// +/// Testing for membership is fast, and iterating over members is +/// reasonably fast in practice. Storage consumption is proportional +/// to the largest id index of any member, not to the number of +/// members, but a bit vector tracks occupancy, so iteration touches +/// only occupied elements. +#[derive(Debug)] +pub(super) struct ResourceMetadata { + /// If the resource with index `i` is a member, `owned[i]` is `true`. + owned: BitVec, + + /// A vector parallel to `owned`, holding clones of members' `RefCount`s. + ref_counts: Vec>, + + /// A vector parallel to `owned`, holding the epoch of each members' id. + epochs: Vec, + + /// This tells Rust that this type should be covariant with `A`. + _phantom: PhantomData, +} + +impl ResourceMetadata { + pub(super) fn new() -> Self { + Self { + owned: BitVec::default(), + ref_counts: Vec::new(), + epochs: Vec::new(), + + _phantom: PhantomData, + } + } + + /// Returns the number of indices we can accommodate. + pub(super) fn size(&self) -> usize { + self.owned.len() + } + + pub(super) fn set_size(&mut self, size: usize) { + self.ref_counts.resize(size, None); + self.epochs.resize(size, u32::MAX); + + resize_bitvec(&mut self.owned, size); + } + + /// Ensures a given index is in bounds for all arrays and does + /// sanity checks of the presence of a refcount. + /// + /// In release mode this function is completely empty and is removed. + #[cfg_attr(not(feature = "strict_asserts"), allow(unused_variables))] + pub(super) fn tracker_assert_in_bounds(&self, index: usize) { + strict_assert!(index < self.owned.len()); + strict_assert!(index < self.ref_counts.len()); + strict_assert!(index < self.epochs.len()); + + strict_assert!(if self.contains(index) { + self.ref_counts[index].is_some() + } else { + true + }); + } + + /// Returns true if the tracker owns no resources. + /// + /// This is a O(n) operation. + pub(super) fn is_empty(&self) -> bool { + !self.owned.any() + } + + /// Returns true if the set contains the resource with the given index. + pub(super) fn contains(&self, index: usize) -> bool { + self.owned[index] + } + + /// Returns true if the set contains the resource with the given index. + /// + /// # Safety + /// + /// The given `index` must be in bounds for this `ResourceMetadata`'s + /// existing tables. See `tracker_assert_in_bounds`. + #[inline(always)] + pub(super) unsafe fn contains_unchecked(&self, index: usize) -> bool { + unsafe { self.owned.get(index).unwrap_unchecked() } + } + + /// Insert a resource into the set. + /// + /// Add the resource with the given index, epoch, and reference count to the + /// set. + /// + /// # Safety + /// + /// The given `index` must be in bounds for this `ResourceMetadata`'s + /// existing tables. See `tracker_assert_in_bounds`. + #[inline(always)] + pub(super) unsafe fn insert(&mut self, index: usize, epoch: Epoch, ref_count: RefCount) { + self.owned.set(index, true); + unsafe { + *self.epochs.get_unchecked_mut(index) = epoch; + *self.ref_counts.get_unchecked_mut(index) = Some(ref_count); + } + } + + /// Get the [`RefCount`] of the resource with the given index. + /// + /// # Safety + /// + /// The given `index` must be in bounds for this `ResourceMetadata`'s + /// existing tables. See `tracker_assert_in_bounds`. + #[inline(always)] + pub(super) unsafe fn get_ref_count_unchecked(&self, index: usize) -> &RefCount { + unsafe { + self.ref_counts + .get_unchecked(index) + .as_ref() + .unwrap_unchecked() + } + } + + /// Get the [`Epoch`] of the id of the resource with the given index. + /// + /// # Safety + /// + /// The given `index` must be in bounds for this `ResourceMetadata`'s + /// existing tables. See `tracker_assert_in_bounds`. + #[inline(always)] + pub(super) unsafe fn get_epoch_unchecked(&self, index: usize) -> Epoch { + unsafe { *self.epochs.get_unchecked(index) } + } + + /// Returns an iterator over the ids for all resources owned by `self`. + pub(super) fn owned_ids(&self) -> impl Iterator> + '_ { + if !self.owned.is_empty() { + self.tracker_assert_in_bounds(self.owned.len() - 1) + }; + iterate_bitvec_indices(&self.owned).map(move |index| { + let epoch = unsafe { *self.epochs.get_unchecked(index) }; + id::Valid(Id::zip(index as u32, epoch, A::VARIANT)) + }) + } + + /// Returns an iterator over the indices of all resources owned by `self`. + pub(super) fn owned_indices(&self) -> impl Iterator + '_ { + if !self.owned.is_empty() { + self.tracker_assert_in_bounds(self.owned.len() - 1) + }; + iterate_bitvec_indices(&self.owned) + } + + /// Remove the resource with the given index from the set. + pub(super) unsafe fn remove(&mut self, index: usize) { + unsafe { + *self.ref_counts.get_unchecked_mut(index) = None; + *self.epochs.get_unchecked_mut(index) = u32::MAX; + } + self.owned.set(index, false); + } +} + +/// A source of resource metadata. +/// +/// This is used to abstract over the various places +/// trackers can get new resource metadata from. +pub(super) enum ResourceMetadataProvider<'a, A: hub::HalApi> { + /// Comes directly from explicit values. + Direct { + epoch: Epoch, + ref_count: Cow<'a, RefCount>, + }, + /// Comes from another metadata tracker. + Indirect { metadata: &'a ResourceMetadata }, + /// The epoch is given directly, but the life count comes from the resource itself. + Resource { epoch: Epoch }, +} +impl ResourceMetadataProvider<'_, A> { + /// Get the epoch and an owned refcount from this. + /// + /// # Safety + /// + /// - The index must be in bounds of the metadata tracker if this uses an indirect source. + /// - life_guard must be Some if this uses a Resource source. + #[inline(always)] + pub(super) unsafe fn get_own( + self, + life_guard: Option<&LifeGuard>, + index: usize, + ) -> (Epoch, RefCount) { + match self { + ResourceMetadataProvider::Direct { epoch, ref_count } => { + (epoch, ref_count.into_owned()) + } + ResourceMetadataProvider::Indirect { metadata } => { + metadata.tracker_assert_in_bounds(index); + (unsafe { *metadata.epochs.get_unchecked(index) }, { + let ref_count = unsafe { metadata.ref_counts.get_unchecked(index) }; + unsafe { ref_count.clone().unwrap_unchecked() } + }) + } + ResourceMetadataProvider::Resource { epoch } => { + strict_assert!(life_guard.is_some()); + (epoch, unsafe { life_guard.unwrap_unchecked() }.add_ref()) + } + } + } + /// Get the epoch from this. + /// + /// # Safety + /// + /// - The index must be in bounds of the metadata tracker if this uses an indirect source. + #[inline(always)] + pub(super) unsafe fn get_epoch(self, index: usize) -> Epoch { + match self { + ResourceMetadataProvider::Direct { epoch, .. } + | ResourceMetadataProvider::Resource { epoch, .. } => epoch, + ResourceMetadataProvider::Indirect { metadata } => { + metadata.tracker_assert_in_bounds(index); + unsafe { *metadata.epochs.get_unchecked(index) } + } + } + } +} + +/// Resizes the given bitvec to the given size. I'm not sure why this is hard to do but it is. +fn resize_bitvec(vec: &mut BitVec, size: usize) { + let owned_size_to_grow = size.checked_sub(vec.len()); + if let Some(delta) = owned_size_to_grow { + if delta != 0 { + vec.grow(delta, false); + } + } else { + vec.truncate(size); + } +} + +/// Produces an iterator that yields the indexes of all bits that are set in the bitvec. +/// +/// Will skip entire usize's worth of bits if they are all false. +fn iterate_bitvec_indices(ownership: &BitVec) -> impl Iterator + '_ { + const BITS_PER_BLOCK: usize = mem::size_of::() * 8; + + let size = ownership.len(); + + ownership + .blocks() + .enumerate() + .filter(|&(_, word)| word != 0) + .flat_map(move |(word_index, mut word)| { + let bit_start = word_index * BITS_PER_BLOCK; + let bit_end = (bit_start + BITS_PER_BLOCK).min(size); + + (bit_start..bit_end).filter(move |_| { + let active = word & 0b1 != 0; + word >>= 1; + + active + }) + }) +} diff --git a/wgpu-core/src/track/mod.rs b/wgpu-core/src/track/mod.rs index fe44bd9067..9d85e1ab7b 100644 --- a/wgpu-core/src/track/mod.rs +++ b/wgpu-core/src/track/mod.rs @@ -94,6 +94,7 @@ Device <- CommandBuffer = insert(device.start, device.end, buffer.start, buffer. */ mod buffer; +mod metadata; mod range; mod stateless; mod texture; @@ -101,14 +102,14 @@ mod texture; use crate::{ binding_model, command, conv, hub, id::{self, TypedId}, - pipeline, resource, Epoch, LifeGuard, RefCount, + pipeline, resource, }; -use bit_vec::BitVec; -use std::{borrow::Cow, fmt, marker::PhantomData, mem, num::NonZeroU32, ops}; +use std::{fmt, num::NonZeroU32, ops}; use thiserror::Error; pub(crate) use buffer::{BufferBindGroupState, BufferTracker, BufferUsageScope}; +use metadata::{ResourceMetadata, ResourceMetadataProvider}; pub(crate) use stateless::{StatelessBindGroupSate, StatelessTracker}; pub(crate) use texture::{ TextureBindGroupState, TextureSelector, TextureTracker, TextureUsageScope, @@ -205,43 +206,6 @@ fn skip_barrier(old_state: T, new_state: T) -> bool { old_state == new_state && old_state.all_ordered() } -/// Resizes the given bitvec to the given size. I'm not sure why this is hard to do but it is. -fn resize_bitvec(vec: &mut BitVec, size: usize) { - let owned_size_to_grow = size.checked_sub(vec.len()); - if let Some(delta) = owned_size_to_grow { - if delta != 0 { - vec.grow(delta, false); - } - } else { - vec.truncate(size); - } -} - -/// Produces an iterator that yields the indexes of all bits that are set in the bitvec. -/// -/// Will skip entire usize's worth of bits if they are all false. -fn iterate_bitvec_indices(ownership: &BitVec) -> impl Iterator + '_ { - const BITS_PER_BLOCK: usize = mem::size_of::() * 8; - - let size = ownership.len(); - - ownership - .blocks() - .enumerate() - .filter(|&(_, word)| word != 0) - .flat_map(move |(word_index, mut word)| { - let bit_start = word_index * BITS_PER_BLOCK; - let bit_end = (bit_start + BITS_PER_BLOCK).min(size); - - (bit_start..bit_end).filter(move |_| { - let active = word & 0b1 != 0; - word >>= 1; - - active - }) - }) -} - #[derive(Clone, Debug, Error, Eq, PartialEq)] pub enum UsageConflict { #[error("Attempted to use invalid buffer")] @@ -342,138 +306,6 @@ impl fmt::Display for InvalidUse { } } -/// SOA container for storing metadata of a resource. -/// -/// This contins the ownership bitvec, the refcount of -/// the resource, and the epoch of the object's full ID. -#[derive(Debug)] -pub(crate) struct ResourceMetadata { - owned: BitVec, - ref_counts: Vec>, - epochs: Vec, - - _phantom: PhantomData, -} -impl ResourceMetadata { - pub fn new() -> Self { - Self { - owned: BitVec::default(), - ref_counts: Vec::new(), - epochs: Vec::new(), - - _phantom: PhantomData, - } - } - - pub fn set_size(&mut self, size: usize) { - self.ref_counts.resize(size, None); - self.epochs.resize(size, u32::MAX); - - resize_bitvec(&mut self.owned, size); - } - - /// Ensures a given index is in bounds for all arrays and does - /// sanity checks of the presence of a refcount. - /// - /// In release mode this function is completely empty and is removed. - #[cfg_attr(not(feature = "strict_asserts"), allow(unused_variables))] - fn tracker_assert_in_bounds(&self, index: usize) { - strict_assert!(index < self.owned.len()); - strict_assert!(index < self.ref_counts.len()); - strict_assert!(index < self.epochs.len()); - - strict_assert!(if self.owned.get(index).unwrap() { - self.ref_counts[index].is_some() - } else { - true - }); - } - - /// Returns true if the tracker owns no resources. - /// - /// This is a O(n) operation. - fn is_empty(&self) -> bool { - !self.owned.any() - } - - /// Returns ids for all resources we own. - fn used(&self) -> impl Iterator> + '_ { - if !self.owned.is_empty() { - self.tracker_assert_in_bounds(self.owned.len() - 1) - }; - iterate_bitvec_indices(&self.owned).map(move |index| { - let epoch = unsafe { *self.epochs.get_unchecked(index) }; - id::Valid(Id::zip(index as u32, epoch, A::VARIANT)) - }) - } - - /// Resets the metadata for a given index to sane "invalid" values. - unsafe fn reset(&mut self, index: usize) { - unsafe { *self.ref_counts.get_unchecked_mut(index) = None }; - unsafe { *self.epochs.get_unchecked_mut(index) = u32::MAX }; - self.owned.set(index, false); - } -} - -/// A source of resource metadata. -/// -/// This is used to abstract over the various places -/// trackers can get new resource metadata from. -enum ResourceMetadataProvider<'a, A: hub::HalApi> { - /// Comes directly from explicit values. - Direct { - epoch: Epoch, - ref_count: Cow<'a, RefCount>, - }, - /// Comes from another metadata tracker. - Indirect { metadata: &'a ResourceMetadata }, - /// The epoch is given directly, but the life count comes from the resource itself. - Resource { epoch: Epoch }, -} -impl ResourceMetadataProvider<'_, A> { - /// Get the epoch and an owned refcount from this. - /// - /// # Safety - /// - /// - The index must be in bounds of the metadata tracker if this uses an indirect source. - /// - life_guard must be Some if this uses a Resource source. - #[inline(always)] - unsafe fn get_own(self, life_guard: Option<&LifeGuard>, index: usize) -> (Epoch, RefCount) { - match self { - ResourceMetadataProvider::Direct { epoch, ref_count } => { - (epoch, ref_count.into_owned()) - } - ResourceMetadataProvider::Indirect { metadata } => { - metadata.tracker_assert_in_bounds(index); - (unsafe { *metadata.epochs.get_unchecked(index) }, { - let ref_count = unsafe { metadata.ref_counts.get_unchecked(index) }; - unsafe { ref_count.clone().unwrap_unchecked() } - }) - } - ResourceMetadataProvider::Resource { epoch } => { - strict_assert!(life_guard.is_some()); - (epoch, unsafe { life_guard.unwrap_unchecked() }.add_ref()) - } - } - } - /// Get the epoch from this. - /// - /// # Safety - /// - /// - The index must be in bounds of the metadata tracker if this uses an indirect source. - #[inline(always)] - unsafe fn get_epoch(self, index: usize) -> Epoch { - match self { - ResourceMetadataProvider::Direct { epoch, .. } - | ResourceMetadataProvider::Resource { epoch, .. } => epoch, - ResourceMetadataProvider::Indirect { metadata } => { - metadata.tracker_assert_in_bounds(index); - unsafe { *metadata.epochs.get_unchecked(index) } - } - } - } -} - /// All the usages that a bind group contains. The uses are not deduplicated in any way /// and may include conflicting uses. This is fully compliant by the WebGPU spec. /// diff --git a/wgpu-core/src/track/stateless.rs b/wgpu-core/src/track/stateless.rs index 13117cb53e..1d0fd5997a 100644 --- a/wgpu-core/src/track/stateless.rs +++ b/wgpu-core/src/track/stateless.rs @@ -9,7 +9,7 @@ use std::marker::PhantomData; use crate::{ hub, id::{TypedId, Valid}, - track::{iterate_bitvec_indices, ResourceMetadata}, + track::ResourceMetadata, RefCount, }; @@ -84,14 +84,14 @@ impl StatelessTracker { /// Extend the vectors to let the given index be valid. fn allow_index(&mut self, index: usize) { - if index >= self.metadata.owned.len() { + if index >= self.metadata.size() { self.set_size(index + 1); } } /// Returns a list of all resources tracked. pub fn used(&self) -> impl Iterator> + '_ { - self.metadata.used() + self.metadata.owned_ids() } /// Inserts a single resource into the resource tracker. @@ -109,9 +109,7 @@ impl StatelessTracker { self.tracker_assert_in_bounds(index); unsafe { - *self.metadata.epochs.get_unchecked_mut(index) = epoch; - *self.metadata.ref_counts.get_unchecked_mut(index) = Some(ref_count); - self.metadata.owned.set(index, true); + self.metadata.insert(index, epoch, ref_count); } } @@ -130,9 +128,8 @@ impl StatelessTracker { self.tracker_assert_in_bounds(index); unsafe { - *self.metadata.epochs.get_unchecked_mut(index) = epoch; - *self.metadata.ref_counts.get_unchecked_mut(index) = Some(item.life_guard().add_ref()); - self.metadata.owned.set(index, true); + self.metadata + .insert(index, epoch, item.life_guard().add_ref()); } Some(item) @@ -143,30 +140,21 @@ impl StatelessTracker { /// If the ID is higher than the length of internal vectors, /// the vectors will be extended. A call to set_size is not needed. pub fn add_from_tracker(&mut self, other: &Self) { - let incoming_size = other.metadata.owned.len(); - if incoming_size > self.metadata.owned.len() { + let incoming_size = other.metadata.size(); + if incoming_size > self.metadata.size() { self.set_size(incoming_size); } - for index in iterate_bitvec_indices(&other.metadata.owned) { + for index in other.metadata.owned_indices() { self.tracker_assert_in_bounds(index); other.tracker_assert_in_bounds(index); unsafe { - let previously_owned = self.metadata.owned.get(index).unwrap_unchecked(); + let previously_owned = self.metadata.contains_unchecked(index); if !previously_owned { - self.metadata.owned.set(index, true); - - let other_ref_count = other - .metadata - .ref_counts - .get_unchecked(index) - .clone() - .unwrap_unchecked(); - *self.metadata.ref_counts.get_unchecked_mut(index) = Some(other_ref_count); - - let epoch = *other.metadata.epochs.get_unchecked(index); - *self.metadata.epochs.get_unchecked_mut(index) = epoch; + let epoch = other.metadata.get_epoch_unchecked(index); + let other_ref_count = other.metadata.get_ref_count_unchecked(index); + self.metadata.insert(index, epoch, other_ref_count.clone()); } } } @@ -183,22 +171,19 @@ impl StatelessTracker { let (index32, epoch, _) = id.0.unzip(); let index = index32 as usize; - if index > self.metadata.owned.len() { + if index > self.metadata.size() { return false; } self.tracker_assert_in_bounds(index); unsafe { - if self.metadata.owned.get(index).unwrap_unchecked() { - let existing_epoch = self.metadata.epochs.get_unchecked(index); - let existing_ref_count = self.metadata.ref_counts.get_unchecked(index); - - if *existing_epoch == epoch - && existing_ref_count.as_ref().unwrap_unchecked().load() == 1 - { - self.metadata.reset(index); + if self.metadata.contains_unchecked(index) { + let existing_epoch = self.metadata.get_epoch_unchecked(index); + let existing_ref_count = self.metadata.get_ref_count_unchecked(index); + if existing_epoch == epoch && existing_ref_count.load() == 1 { + self.metadata.remove(index); return true; } } diff --git a/wgpu-core/src/track/texture.rs b/wgpu-core/src/track/texture.rs index 0e0271688a..5b22fb527f 100644 --- a/wgpu-core/src/track/texture.rs +++ b/wgpu-core/src/track/texture.rs @@ -25,8 +25,8 @@ use crate::{ id::{TextureId, TypedId, Valid}, resource::Texture, track::{ - invalid_resource_state, iterate_bitvec_indices, skip_barrier, ResourceMetadata, - ResourceMetadataProvider, ResourceUses, UsageConflict, + invalid_resource_state, skip_barrier, ResourceMetadata, ResourceMetadataProvider, + ResourceUses, UsageConflict, }, LifeGuard, RefCount, }; @@ -238,7 +238,7 @@ impl TextureUsageScope { strict_assert!(index < self.set.simple.len()); - strict_assert!(if self.metadata.owned.get(index).unwrap() + strict_assert!(if self.metadata.contains(index) && self.set.simple[index] == TextureUses::COMPLEX { self.set.complex.contains_key(&(index as u32)) @@ -258,7 +258,7 @@ impl TextureUsageScope { /// Returns a list of all textures tracked. pub fn used(&self) -> impl Iterator> + '_ { - self.metadata.used() + self.metadata.owned_ids() } /// Returns true if the tracker owns no resources. @@ -285,7 +285,7 @@ impl TextureUsageScope { self.set_size(incoming_size); } - for index in iterate_bitvec_indices(&scope.metadata.owned) { + for index in scope.metadata.owned_indices() { let index32 = index as u32; self.tracker_assert_in_bounds(index); @@ -411,14 +411,14 @@ impl TextureTracker { strict_assert!(index < self.start_set.simple.len()); strict_assert!(index < self.end_set.simple.len()); - strict_assert!(if self.metadata.owned.get(index).unwrap() + strict_assert!(if self.metadata.contains(index) && self.start_set.simple[index] == TextureUses::COMPLEX { self.start_set.complex.contains_key(&(index as u32)) } else { true }); - strict_assert!(if self.metadata.owned.get(index).unwrap() + strict_assert!(if self.metadata.contains(index) && self.end_set.simple[index] == TextureUses::COMPLEX { self.end_set.complex.contains_key(&(index as u32)) @@ -447,7 +447,7 @@ impl TextureTracker { /// Returns a list of all textures tracked. pub fn used(&self) -> impl Iterator> + '_ { - self.metadata.used() + self.metadata.owned_ids() } /// Drains all currently pending transitions. @@ -469,8 +469,7 @@ impl TextureTracker { self.tracker_assert_in_bounds(index); - let ref_count = unsafe { self.metadata.ref_counts.get_unchecked(index) }; - unsafe { ref_count.as_ref().unwrap_unchecked() } + unsafe { self.metadata.get_ref_count_unchecked(index) } } /// Inserts a single texture and a state into the resource tracker. @@ -488,7 +487,7 @@ impl TextureTracker { self.tracker_assert_in_bounds(index); unsafe { - let currently_owned = self.metadata.owned.get(index).unwrap_unchecked(); + let currently_owned = self.metadata.contains_unchecked(index); if currently_owned { panic!("Tried to insert texture already tracked"); @@ -573,7 +572,7 @@ impl TextureTracker { self.set_size(incoming_size); } - for index in iterate_bitvec_indices(&tracker.metadata.owned) { + for index in tracker.metadata.owned_indices() { let index32 = index as u32; self.tracker_assert_in_bounds(index); @@ -619,7 +618,7 @@ impl TextureTracker { self.set_size(incoming_size); } - for index in iterate_bitvec_indices(&scope.metadata.owned) { + for index in scope.metadata.owned_indices() { let index32 = index as u32; self.tracker_assert_in_bounds(index); @@ -677,7 +676,7 @@ impl TextureTracker { let index = index32 as usize; scope.tracker_assert_in_bounds(index); - if unsafe { !scope.metadata.owned.get(index).unwrap_unchecked() } { + if unsafe { !scope.metadata.contains_unchecked(index) } { continue; } let texture_data = unsafe { texture_data_from_texture(storage, index32) }; @@ -698,7 +697,7 @@ impl TextureTracker { ) }; - unsafe { scope.metadata.reset(index) }; + unsafe { scope.metadata.remove(index) }; } } @@ -712,21 +711,21 @@ impl TextureTracker { let (index32, epoch, _) = id.0.unzip(); let index = index32 as usize; - if index > self.metadata.owned.len() { + if index > self.metadata.size() { return false; } self.tracker_assert_in_bounds(index); unsafe { - if self.metadata.owned.get(index).unwrap_unchecked() { - let existing_epoch = *self.metadata.epochs.get_unchecked_mut(index); + if self.metadata.contains_unchecked(index) { + let existing_epoch = self.metadata.get_epoch_unchecked(index); assert_eq!(existing_epoch, epoch); self.start_set.complex.remove(&index32); self.end_set.complex.remove(&index32); - self.metadata.reset(index); + self.metadata.remove(index); return true; } @@ -746,24 +745,22 @@ impl TextureTracker { let (index32, epoch, _) = id.0.unzip(); let index = index32 as usize; - if index > self.metadata.owned.len() { + if index > self.metadata.size() { return false; } self.tracker_assert_in_bounds(index); unsafe { - if self.metadata.owned.get(index).unwrap_unchecked() { - let existing_epoch = self.metadata.epochs.get_unchecked(index); - let existing_ref_count = self.metadata.ref_counts.get_unchecked(index); + if self.metadata.contains_unchecked(index) { + let existing_epoch = self.metadata.get_epoch_unchecked(index); + let existing_ref_count = self.metadata.get_ref_count_unchecked(index); - if *existing_epoch == epoch - && existing_ref_count.as_ref().unwrap_unchecked().load() == 1 - { + if existing_epoch == epoch && existing_ref_count.load() == 1 { self.start_set.complex.remove(&index32); self.end_set.complex.remove(&index32); - self.metadata.reset(index); + self.metadata.remove(index); return true; } @@ -906,7 +903,7 @@ unsafe fn insert_or_merge( state_provider: TextureStateProvider<'_>, metadata_provider: ResourceMetadataProvider<'_, A>, ) -> Result<(), UsageConflict> { - let currently_owned = unsafe { resource_metadata.owned.get(index).unwrap_unchecked() }; + let currently_owned = unsafe { resource_metadata.contains_unchecked(index) }; if !currently_owned { unsafe { @@ -967,7 +964,7 @@ unsafe fn insert_or_barrier_update( metadata_provider: ResourceMetadataProvider<'_, A>, barriers: &mut Vec>, ) { - let currently_owned = unsafe { resource_metadata.owned.get(index).unwrap_unchecked() }; + let currently_owned = unsafe { resource_metadata.contains_unchecked(index) }; if !currently_owned { unsafe { @@ -1092,12 +1089,11 @@ unsafe fn insert( } } - let (epoch, ref_count) = - unsafe { metadata_provider.get_own(texture_data.map(|(life_guard, _)| life_guard), index) }; - - resource_metadata.owned.set(index, true); - unsafe { *resource_metadata.epochs.get_unchecked_mut(index) = epoch }; - unsafe { *resource_metadata.ref_counts.get_unchecked_mut(index) = Some(ref_count) }; + unsafe { + let (epoch, ref_count) = + metadata_provider.get_own(texture_data.map(|(life_guard, _)| life_guard), index); + resource_metadata.insert(index, epoch, ref_count); + } } #[inline(always)]