From 3eb599a156302856cfe2929aea46afce2a266e08 Mon Sep 17 00:00:00 2001 From: Connor Fitzgerald Date: Thu, 28 Apr 2022 09:28:09 -0400 Subject: [PATCH 01/35] Texture state full is dead code --- wgpu-core/src/track/range.rs | 1 + wgpu-core/src/track/texture.rs | 23 +++-------------------- wgpu-hal/src/lib.rs | 5 +++-- 3 files changed, 7 insertions(+), 22 deletions(-) diff --git a/wgpu-core/src/track/range.rs b/wgpu-core/src/track/range.rs index c2238439db..db43155a99 100644 --- a/wgpu-core/src/track/range.rs +++ b/wgpu-core/src/track/range.rs @@ -15,6 +15,7 @@ pub struct RangedStates { } impl RangedStates { + #[cfg_attr(not(test), allow(dead_code))] pub fn empty() -> Self { Self { ranges: SmallVec::new(), diff --git a/wgpu-core/src/track/texture.rs b/wgpu-core/src/track/texture.rs index c519ee07a5..7338dab753 100644 --- a/wgpu-core/src/track/texture.rs +++ b/wgpu-core/src/track/texture.rs @@ -19,8 +19,6 @@ pub struct TextureSelector { #[derive(Clone, Debug, Default, PartialEq)] pub(crate) struct TextureState { mips: ArrayVec, - /// True if we have the information about all the subresources here - full: bool, } impl PendingTransition { @@ -44,7 +42,6 @@ impl TextureState { }) .take(mip_level_count as usize) .collect(), - full: true, } } } @@ -61,9 +58,7 @@ impl ResourceState for TextureState { // initial state to whatever we need, which we can always make // to be the same as the query result for the known subresources. let num_levels = self.mips.len(); - if self.full { - assert!(num_levels >= selector.levels.end as usize); - } + assert!(num_levels >= selector.levels.end as usize); let mip_start = num_levels.min(selector.levels.start as usize); let mip_end = num_levels.min(selector.levels.end as usize); for mip in self.mips[mip_start..mip_end].iter() { @@ -86,13 +81,7 @@ impl ResourceState for TextureState { usage: Self::Usage, mut output: Option<&mut Vec>>, ) -> Result<(), PendingTransition> { - if self.full { - assert!(self.mips.len() >= selector.levels.end as usize); - } else { - while self.mips.len() < selector.levels.end as usize { - self.mips.push(PlaneStates::empty()); - } - } + assert!(self.mips.len() >= selector.levels.end as usize); for (mip_id, mip) in self.mips[selector.levels.start as usize..selector.levels.end as usize] .iter_mut() .enumerate() @@ -143,13 +132,7 @@ impl ResourceState for TextureState { mut output: Option<&mut Vec>>, ) -> Result<(), PendingTransition> { let mut temp = Vec::new(); - if self.full { - assert!(self.mips.len() >= other.mips.len()); - } else { - while self.mips.len() < other.mips.len() { - self.mips.push(PlaneStates::empty()); - } - } + assert!(self.mips.len() >= other.mips.len()); for (mip_id, (mip_self, mip_other)) in self.mips.iter_mut().zip(&other.mips).enumerate() { let level = mip_id as u32; diff --git a/wgpu-hal/src/lib.rs b/wgpu-hal/src/lib.rs index 891381e50a..f5a278722a 100644 --- a/wgpu-hal/src/lib.rs +++ b/wgpu-hal/src/lib.rs @@ -626,7 +626,7 @@ bitflags!( bitflags::bitflags! { /// Similar to `wgt::BufferUsages` but for internal use. - pub struct BufferUses: u32 { + pub struct BufferUses: u16 { const MAP_READ = 1 << 0; const MAP_WRITE = 1 << 1; const COPY_SRC = 1 << 2; @@ -653,7 +653,7 @@ bitflags::bitflags! { bitflags::bitflags! { /// Similar to `wgt::TextureUsages` but for internal use. - pub struct TextureUses: u32 { + pub struct TextureUses: u16 { const COPY_SRC = 1 << 0; const COPY_DST = 1 << 1; const RESOURCE = 1 << 2; @@ -662,6 +662,7 @@ bitflags::bitflags! { const DEPTH_STENCIL_WRITE = 1 << 5; const STORAGE_READ = 1 << 6; const STORAGE_WRITE = 1 << 7; + const COMPLEX = 1 << 8; /// The combination of usages that can be used together (read-only). const INCLUSIVE = Self::COPY_SRC.bits | Self::RESOURCE.bits | Self::DEPTH_STENCIL_READ.bits; /// The combination of exclusive usages (write-only and read-write). From aaff782f180968b9e58c35563cb5488d952455ea Mon Sep 17 00:00:00 2001 From: Connor Fitzgerald Date: Sun, 1 May 2022 22:28:24 -0400 Subject: [PATCH 02/35] temp --- Cargo.lock | 1 + wgpu-core/Cargo.toml | 1 + wgpu-core/src/command/bundle.rs | 2 +- wgpu-core/src/command/clear.rs | 6 +- wgpu-core/src/command/memory_init.rs | 4 +- wgpu-core/src/command/mod.rs | 4 +- wgpu-core/src/conv.rs | 8 +- wgpu-core/src/device/mod.rs | 10 +- wgpu-core/src/device/queue.rs | 4 +- wgpu-core/src/hub.rs | 8 + wgpu-core/src/present.rs | 2 +- wgpu-core/src/track/mod.rs | 36 +-- wgpu-core/src/track/range.rs | 1 - wgpu-core/src/track/texture.rs | 372 ++++++++++++++++++++++++++- 14 files changed, 414 insertions(+), 45 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 962fcb1de5..19bb034004 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2090,6 +2090,7 @@ name = "wgpu-core" version = "0.12.0" dependencies = [ "arrayvec", + "bit-vec", "bitflags", "cfg_aliases", "codespan-reporting", diff --git a/wgpu-core/Cargo.toml b/wgpu-core/Cargo.toml index cafa3d5436..ad4bd46321 100644 --- a/wgpu-core/Cargo.toml +++ b/wgpu-core/Cargo.toml @@ -26,6 +26,7 @@ vulkan-portability = ["hal/vulkan"] [dependencies] arrayvec = "0.7" bitflags = "1.0" +bit-vec = "0.6" codespan-reporting = "0.11" copyless = "0.1" fxhash = "0.2" diff --git a/wgpu-core/src/command/bundle.rs b/wgpu-core/src/command/bundle.rs index 5bd7eafb1a..c04e56b37d 100644 --- a/wgpu-core/src/command/bundle.rs +++ b/wgpu-core/src/command/bundle.rs @@ -117,7 +117,7 @@ impl RenderBundleEncoder { }, sample_count: { let sc = desc.sample_count; - if sc == 0 || sc > 32 || !conv::is_power_of_two(sc) { + if sc == 0 || sc > 32 || !conv::is_power_of_two_u32(sc) { return Err(CreateRenderBundleError::InvalidSampleCount(sc)); } sc diff --git a/wgpu-core/src/command/clear.rs b/wgpu-core/src/command/clear.rs index 0bd909c76e..8bfc227acb 100644 --- a/wgpu-core/src/command/clear.rs +++ b/wgpu-core/src/command/clear.rs @@ -10,7 +10,7 @@ use crate::{ id::{BufferId, CommandEncoderId, DeviceId, TextureId, Valid}, init_tracker::{MemoryInitKind, TextureInitRange}, resource::{Texture, TextureClearMode}, - track::{ResourceTracker, TextureSelector, TextureState}, + track::{ResourceTracker, TextureSelector, OldTextureState}, }; use hal::{auxil::align_to, CommandEncoder as _}; @@ -236,7 +236,7 @@ pub(crate) fn clear_texture( dst_texture: &Texture, range: TextureInitRange, encoder: &mut A::CommandEncoder, - texture_tracker: &mut ResourceTracker, + texture_tracker: &mut ResourceTracker, device: &Device, ) -> Result<(), ClearError> { clear_texture_no_device( @@ -255,7 +255,7 @@ pub(crate) fn clear_texture_no_device( dst_texture: &Texture, range: TextureInitRange, encoder: &mut A::CommandEncoder, - texture_tracker: &mut ResourceTracker, + texture_tracker: &mut ResourceTracker, alignments: &hal::Alignments, zero_buffer: &A::Buffer, ) -> Result<(), ClearError> { diff --git a/wgpu-core/src/command/memory_init.rs b/wgpu-core/src/command/memory_init.rs index ea0b303609..7bd4389660 100644 --- a/wgpu-core/src/command/memory_init.rs +++ b/wgpu-core/src/command/memory_init.rs @@ -8,7 +8,7 @@ use crate::{ id::{self, TextureId}, init_tracker::*, resource::{Buffer, Texture}, - track::{ResourceTracker, TextureState, TrackerSet}, + track::{ResourceTracker, OldTextureState, TrackerSet}, FastHashMap, }; @@ -127,7 +127,7 @@ pub(crate) fn fixup_discarded_surfaces< inits: InitIter, encoder: &mut A::CommandEncoder, texture_guard: &Storage, TextureId>, - texture_tracker: &mut ResourceTracker, + texture_tracker: &mut ResourceTracker, device: &Device, ) { for init in inits { diff --git a/wgpu-core/src/command/mod.rs b/wgpu-core/src/command/mod.rs index a4e7967307..9a8a7ea340 100644 --- a/wgpu-core/src/command/mod.rs +++ b/wgpu-core/src/command/mod.rs @@ -23,7 +23,7 @@ use crate::{ hub::{Global, GlobalIdentityHandlerFactory, HalApi, Storage, Token}, id, resource::{Buffer, Texture}, - track::{BufferState, ResourceTracker, TextureState, TrackerSet}, + track::{BufferState, ResourceTracker, OldTextureState, TrackerSet}, Label, Stored, }; @@ -142,7 +142,7 @@ impl CommandBuffer { raw: &mut A::CommandEncoder, base: &mut TrackerSet, head_buffers: &ResourceTracker, - head_textures: &ResourceTracker, + head_textures: &ResourceTracker, buffer_guard: &Storage, id::BufferId>, texture_guard: &Storage, id::TextureId>, ) { diff --git a/wgpu-core/src/conv.rs b/wgpu-core/src/conv.rs index b6632b5903..253cd24d3f 100644 --- a/wgpu-core/src/conv.rs +++ b/wgpu-core/src/conv.rs @@ -1,6 +1,10 @@ use crate::resource; -pub fn is_power_of_two(val: u32) -> bool { +pub fn is_power_of_two_u16(val: u16) -> bool { + val != 0 && (val & (val - 1)) == 0 +} + +pub fn is_power_of_two_u32(val: u32) -> bool { val != 0 && (val & (val - 1)) == 0 } @@ -146,7 +150,7 @@ pub fn check_texture_dimension_size( return Err(Tde::LimitExceeded { dim, given, limit }); } } - if sample_size == 0 || sample_size > sample_limit || !is_power_of_two(sample_size) { + if sample_size == 0 || sample_size > sample_limit || !is_power_of_two_u32(sample_size) { return Err(Tde::InvalidSampleCount(sample_size)); } diff --git a/wgpu-core/src/device/mod.rs b/wgpu-core/src/device/mod.rs index 760de1195c..98307f0d93 100644 --- a/wgpu-core/src/device/mod.rs +++ b/wgpu-core/src/device/mod.rs @@ -8,7 +8,7 @@ use crate::{ TextureInitTracker, TextureInitTrackerAction, }, instance, pipeline, present, resource, - track::{BufferState, TextureSelector, TextureState, TrackerSet, UsageConflict}, + track::{BufferState, TextureSelector, OldTextureState, TrackerSet, UsageConflict}, validation::{self, check_buffer_usage, check_texture_usage}, FastHashMap, Label, LabelHelpers as _, LifeGuard, MultiRefCount, RefCount, Stored, SubmissionIndex, DOWNLEVEL_ERROR_MESSAGE, @@ -1058,7 +1058,7 @@ impl Device { let anisotropy_clamp = if let Some(clamp) = desc.anisotropy_clamp { let clamp = clamp.get(); - let valid_clamp = clamp <= hal::MAX_ANISOTROPY && conv::is_power_of_two(clamp as u32); + let valid_clamp = clamp <= hal::MAX_ANISOTROPY && conv::is_power_of_two_u32(clamp as u32); if !valid_clamp { return Err(resource::CreateSamplerError::InvalidClamp(clamp)); } @@ -2551,7 +2551,7 @@ impl Device { let samples = { let sc = desc.multisample.count; - if sc == 0 || sc > 32 || !conv::is_power_of_two(sc) { + if sc == 0 || sc > 32 || !conv::is_power_of_two_u32(sc) { return Err(pipeline::CreateRenderPipelineError::InvalidSampleCount(sc)); } sc @@ -3457,7 +3457,7 @@ impl Global { .trackers .lock() .textures - .init(id, ref_count, TextureState::new(num_levels, num_layers)) + .init(id, ref_count, OldTextureState::new(num_levels, num_layers)) .unwrap(); return (id.0, None); }; @@ -3535,7 +3535,7 @@ impl Global { .trackers .lock() .textures - .init(id, ref_count, TextureState::new(num_levels, num_layers)) + .init(id, ref_count, OldTextureState::new(num_levels, num_layers)) .unwrap(); return (id.0, None); }; diff --git a/wgpu-core/src/device/queue.rs b/wgpu-core/src/device/queue.rs index 572f49e5dc..001d102ac4 100644 --- a/wgpu-core/src/device/queue.rs +++ b/wgpu-core/src/device/queue.rs @@ -683,7 +683,7 @@ impl Global { let ref_count = cmdbuf.trackers.textures.get_ref_count(id); //TODO: better error handling here? // register it in the temporary tracker. - let mut ts = track::TextureState::default(); + let mut ts = track::OldTextureState::default(); let _ = ts.change( id, texture.full_range.clone(), @@ -846,7 +846,7 @@ impl Global { let ref_count = texture.life_guard.add_ref(); //TODO: better error handling here? // register it in the temporary tracker. - let mut ts = track::TextureState::default(); + let mut ts = track::OldTextureState::default(); let _ = ts.change( id::Valid(id), texture.full_range.clone(), diff --git a/wgpu-core/src/hub.rs b/wgpu-core/src/hub.rs index 57f6a2dfe9..273c70b3bf 100644 --- a/wgpu-core/src/hub.rs +++ b/wgpu-core/src/hub.rs @@ -191,6 +191,14 @@ impl Storage { result } + pub(crate) unsafe fn get_unchecked(&self, id: u32) -> Result<&T, InvalidId> { + match self.map[id as usize] { + Element::Occupied(ref v, _) => Ok(v), + Element::Vacant => panic!("{}[{}] does not exist", self.kind, id), + Element::Error(_, _) => Err(InvalidId), + } + } + pub(crate) fn label_for_invalid_id(&self, id: I) -> &str { let (index, _, _) = id.unzip(); match self.map[index as usize] { diff --git a/wgpu-core/src/present.rs b/wgpu-core/src/present.rs index 833e65d6be..dd50c08d2f 100644 --- a/wgpu-core/src/present.rs +++ b/wgpu-core/src/present.rs @@ -190,7 +190,7 @@ impl Global { use track::ResourceState as _; // register it in the device tracker as uninitialized let mut trackers = device.trackers.lock(); - let mut ts = track::TextureState::default(); + let mut ts = track::OldTextureState::default(); let _ = ts.change( id, track::TextureSelector { diff --git a/wgpu-core/src/track/mod.rs b/wgpu-core/src/track/mod.rs index bcf5eb4a4b..268c31a76a 100644 --- a/wgpu-core/src/track/mod.rs +++ b/wgpu-core/src/track/mod.rs @@ -14,7 +14,7 @@ use std::{ use thiserror::Error; pub(crate) use buffer::BufferState; -pub(crate) use texture::{TextureSelector, TextureState}; +pub(crate) use texture::{OldTextureState, TextureSelector}; /// A single unit of state tracking. It keeps an initial /// usage as well as the last/current one, similar to `Range`. @@ -109,7 +109,7 @@ struct Resource { /// based on the contents. #[derive(Debug, PartialEq)] pub(crate) struct PendingTransition { - pub id: Valid, + pub id: u32, pub selector: S::Selector, pub usage: ops::Range, } @@ -131,14 +131,15 @@ impl PendingTransition { impl From> for UsageConflict { fn from(e: PendingTransition) -> Self { - Self::Buffer { - id: e.id.0, - combined_use: e.usage.end, - } + // Self::Buffer { + // id: e.id.0, + // combined_use: e.usage.end, + // } + todo!() } } -impl PendingTransition { +impl PendingTransition { /// Produce the hal barrier corresponding to the transition. pub fn into_hal<'a, A: hal::Api>( self, @@ -164,14 +165,15 @@ impl PendingTransition { } } -impl From> for UsageConflict { - fn from(e: PendingTransition) -> Self { - Self::Texture { - id: e.id.0, - mip_levels: e.selector.levels.start..e.selector.levels.end, - array_layers: e.selector.layers.start..e.selector.layers.end, - combined_use: e.usage.end, - } +impl From> for UsageConflict { + fn from(e: PendingTransition) -> Self { + // Self::Texture { + // id: e.id.0, + // mip_levels: e.selector.levels.start..e.selector.levels.end, + // array_layers: e.selector.layers.start..e.selector.layers.end, + // combined_use: e.usage.end, + // } + todo!() } } @@ -582,7 +584,7 @@ pub enum UsageConflict { #[derive(Debug)] pub(crate) struct TrackerSet { pub buffers: ResourceTracker, - pub textures: ResourceTracker, + pub textures: ResourceTracker, pub views: ResourceTracker>, pub bind_groups: ResourceTracker>, pub samplers: ResourceTracker>, @@ -650,7 +652,7 @@ impl TrackerSet { #[derive(Debug)] pub(crate) struct StatefulTrackerSubset { pub buffers: ResourceTracker, - pub textures: ResourceTracker, + pub textures: ResourceTracker, } impl StatefulTrackerSubset { diff --git a/wgpu-core/src/track/range.rs b/wgpu-core/src/track/range.rs index db43155a99..9f3b316eb9 100644 --- a/wgpu-core/src/track/range.rs +++ b/wgpu-core/src/track/range.rs @@ -65,7 +65,6 @@ impl RangedStates { } /// Merge the neighboring ranges together, where possible. - #[allow(clippy::suspicious_operation_groupings)] pub fn coalesce(&mut self) { let mut num_removed = 0; let mut iter = self.ranges.iter_mut(); diff --git a/wgpu-core/src/track/texture.rs b/wgpu-core/src/track/texture.rs index 7338dab753..1471af54ef 100644 --- a/wgpu-core/src/track/texture.rs +++ b/wgpu-core/src/track/texture.rs @@ -1,10 +1,16 @@ use super::{range::RangedStates, PendingTransition, ResourceState, Unit}; -use crate::id::{TextureId, Valid}; +use crate::{ + conv, hub, + id::{TextureId, TypedId, Valid}, + resource::Texture, +}; +use bit_vec::BitVec; use hal::TextureUses; use arrayvec::ArrayVec; +use naga::FastHashMap; -use std::{iter, ops::Range}; +use std::{iter, ops::Range, vec::Drain}; type PlaneStates = RangedStates>; @@ -17,11 +23,11 @@ pub struct TextureSelector { } #[derive(Clone, Debug, Default, PartialEq)] -pub(crate) struct TextureState { +pub(crate) struct OldTextureState { mips: ArrayVec, } -impl PendingTransition { +impl PendingTransition { fn collapse(self) -> Result { if self.usage.start.is_empty() || self.usage.start == self.usage.end @@ -34,7 +40,7 @@ impl PendingTransition { } } -impl TextureState { +impl OldTextureState { pub fn new(mip_level_count: u32, array_layer_count: u32) -> Self { Self { mips: iter::repeat_with(|| { @@ -46,7 +52,7 @@ impl TextureState { } } -impl ResourceState for TextureState { +impl ResourceState for OldTextureState { type Id = TextureId; type Selector = TextureSelector; type Usage = TextureUses; @@ -213,6 +219,354 @@ impl ResourceState for TextureState { } } +fn invalid_texture_state(state: TextureUses) -> bool { + // Is power of two also means "is one bit set". We check for this as if + // we're in any exclusive state, we must only be in a single state. + state.intersects(hal::TextureUses::EXCLUSIVE) && !conv::is_power_of_two_u16(state.bits()) +} + +#[derive(Clone, Debug, Default, PartialEq)] +pub(crate) struct ComplexTextureState { + mips: ArrayVec, { hal::MAX_MIP_LEVELS as usize }>, +} + +impl ComplexTextureState { + pub fn new(mip_level_count: u32, array_layer_count: u32) -> Self { + Self { + mips: iter::repeat_with(|| { + RangedStates::from_range(0..array_layer_count, TextureUses::UNINITIALIZED) + }) + .take(mip_level_count as usize) + .collect(), + } + } +} + +fn resize_bitvec(vec: &mut BitVec, size: usize) { + let owned_size_to_grow = size.checked_sub(vec.len()); + if let Some(delta) = owned_size_to_grow { + if delta != 0 { + vec.grow(delta, false); + } + } else { + vec.truncate(size); + } +} + +pub struct TextureStateSet { + simple: Vec, + complex: FastHashMap, +} +impl TextureStateSet { + pub fn new() -> Self { + Self { + simple: Vec::new(), + complex: FastHashMap::default(), + } + } +} + +pub struct TextureUsageScope { + set: TextureStateSet, + owned: BitVec, +} + +impl TextureUsageScope { + pub fn new() -> Self { + Self { + set: TextureStateSet::new(), + owned: BitVec::default(), + } + } + + pub fn set_max_index(&mut self, size: usize) { + self.set.simple.resize(size, TextureUses::UNINITIALIZED); + + resize_bitvec(&mut self.owned, size); + } + + /// # Safety + /// + /// `id` must be a valid ID and have an ID value less than the last call to set_max_index. + unsafe fn extend( + &mut self, + storage: &hub::Storage, TextureId>, + id: Valid, + selector: Option, + new_state: TextureUses, + ) -> Result<(), ()> { + let (index, _, _) = id.0.unzip(); + let index = index as usize; + + let currently_active = self.owned.get(index).unwrap_unchecked(); + if currently_active { + let current_state = *self.set.simple.get_unchecked(index); + match (current_state == hal::TextureUses::COMPLEX, selector) { + // Both our usages are simple + (false, None) => { + let merged_state = current_state | new_state; + + if invalid_texture_state(merged_state) { + // Conflicting states + return Err(()); + } + + *self.set.simple.get_unchecked_mut(index) = merged_state; + + return Ok(()); + } + // The old usage is complex. + (true, selector) => return self.extend_complex(storage, id, selector, new_state), + + // The old usage is simple, so demote it to a complex one. + (false, Some(selector)) => { + *self.set.simple.get_unchecked_mut(index) = hal::TextureUses::COMPLEX; + + // Demote our simple state to a complex one. + self.extend_complex(storage, id, None, current_state)?; + + // Extend that complex state with our new complex state. + return self.extend_complex(storage, id, Some(selector), new_state); + } + } + } + + // We're the first to use this resource, let's add it. + self.owned.set(index, true); + + if let Some(selector) = selector { + *self.set.simple.get_unchecked_mut(index) = hal::TextureUses::COMPLEX; + self.extend_complex(storage, id, Some(selector), new_state)?; + } else { + *self.set.simple.get_unchecked_mut(index) = new_state; + } + + Ok(()) + } + + #[cold] + fn extend_complex( + &mut self, + storage: &hub::Storage, TextureId>, + id: Valid, + selector: Option, + new_state: TextureUses, + ) -> Result<(), ()> { + let texture = &storage[id]; + + let (id, _, _) = id.0.unzip(); + + // Create the complex entry for this texture. + let complex = self.set.complex.entry(id).or_insert_with(|| { + ComplexTextureState::new( + texture.desc.mip_level_count, + texture.desc.array_layer_count(), + ) + }); + + let mips; + let layers; + match selector { + Some(selector) => { + mips = selector.levels.clone(); + layers = selector.layers.clone(); + } + None => { + mips = 0..texture.desc.mip_level_count; + layers = 0..texture.desc.array_layer_count(); + } + } + + // Go through all relevant mip levels + for mip in mips { + let mip_state = &mut complex.mips[mip as usize]; + + // Set our state. + for (_, current_state) in mip_state.isolate(&layers, new_state) { + let merged = *current_state | new_state; + if invalid_texture_state(merged) { + return Err(()); + } + *current_state = merged; + } + } + + Ok(()) + } +} + +pub(crate) struct TextureTracker { + start_set: TextureStateSet, + end_set: TextureStateSet, + /// Temporary storage for collecting transitions. + temp: Vec>, + owned: BitVec, +} +impl TextureTracker { + pub fn new() -> Self { + Self { + start_set: TextureStateSet::new(), + end_set: TextureStateSet::new(), + temp: Vec::new(), + owned: BitVec::default(), + } + } + + fn set_max_index(&mut self, size: usize) { + self.start_set + .simple + .resize(size, TextureUses::UNINITIALIZED); + self.end_set.simple.resize(size, TextureUses::UNINITIALIZED); + + resize_bitvec(&mut self.owned, size); + } + + pub fn transition_to_other( + &mut self, + storage: &hub::Storage, TextureId>, + incoming_set: &TextureStateSet, + incoming_ownership: &BitVec, + ) -> Drain> { + let incoming_size = incoming_set.simple.len(); + if incoming_size > self.start_set.simple.len() { + self.set_max_index(incoming_size); + } + + for (word_index, mut word) in incoming_ownership.blocks().enumerate() { + if word == 0 { + continue; + } + + let bit_start = word_index * 64; + let bit_end = ((word_index + 1) * 64).min(incoming_size); + + for index in bit_start..bit_end { + if word & 0b1 == 0 { + continue; + } + word >>= 1; + + unsafe { self.transition_index_to_other(storage, incoming_set, index) }; + } + } + + self.temp.drain(..) + } + + unsafe fn transition_index_to_other( + &mut self, + storage: &hub::Storage, TextureId>, + incoming_set: &TextureStateSet, + index: usize, + ) { + let old_tracked = self.owned.get(index).unwrap_unchecked(); + let old_state = *self.end_set.simple.get_unchecked(index); + let new_state = *incoming_set.simple.get_unchecked(index); + + match ( + old_tracked, + old_state == TextureUses::COMPLEX, + new_state == TextureUses::COMPLEX, + ) { + (false, _, false) => { + *self.start_set.simple.get_unchecked_mut(index) = new_state; + *self.end_set.simple.get_unchecked_mut(index) = new_state; + + self.owned.set(index, true); + } + (false, _, true) => { + *self.start_set.simple.get_unchecked_mut(index) = TextureUses::COMPLEX; + *self.end_set.simple.get_unchecked_mut(index) = TextureUses::COMPLEX; + + let complex_state = &incoming_set.complex[&(index as u32)]; + self.start_set + .complex + .insert(index as u32, complex_state.clone()); + self.end_set + .complex + .insert(index as u32, complex_state.clone()); + + self.owned.set(index, true); + } + (true, false, false) => { + // If the state didn't change and all the usages are ordered, the hardware + // will guarentee the order of accesses, so we do not need to issue a barrier at all + if old_state == new_state && old_state.contains(TextureUses::ORDERED) { + return; + } + + self.temp.push(PendingTransition { + id: index as u32, + selector: storage.get_unchecked(index as u32).unwrap().full_range, + usage: old_state..new_state, + }); + } + (true, true, true) => { + self.transition_index_to_other_complex(storage, incoming_set, index); + } + (true, true, false) => {} + (true, false, true) => {} + } + } + + #[cold] + unsafe fn transition_index_to_other_complex( + &mut self, + storage: &hub::Storage, TextureId>, + incoming_set: &TextureStateSet, + index: usize, + ) { + let old_complex = self.end_set.complex.get(&(index as u32)).unwrap_unchecked(); + let new_complex = self.end_set.complex.get(&(index as u32)).unwrap_unchecked(); + + let mut temp = Vec::new(); + debug_assert!(old_complex.mips.len() >= new_complex.mips.len()); + + for (mip_id, (mip_self, mip_other)) in old_complex + .mips + .iter_mut() + .zip(&new_complex.mips) + .enumerate() + { + let level = mip_id as u32; + temp.extend(mip_self.merge(mip_other, 0)); + + for (layers, states) in temp.drain(..) { + match states { + Range { + start: Some(start), + end: Some(end), + } => { + // If the state didn't change and all the usages are ordered, the hardware + // will guarentee the order of accesses, so we do not need to issue a barrier at all + if start == end && TextureUses::ORDERED.contains(start) { + return; + } + // TODO: Can't satisfy clippy here unless we modify + // `TextureSelector` to use `std::ops::RangeBounds`. + #[allow(clippy::range_plus_one)] + let pending = PendingTransition { + id: index as u32, + selector: TextureSelector { + levels: level..level + 1, + layers: layers.clone(), + }, + usage: start..end, + }; + + self.temp.push(pending); + + for (_, state) in mip_self.isolate(&layers, end) { + *state = end; + } + } + _ => unreachable!(), + }; + } + } + } +} + #[cfg(test)] mod test { //TODO: change() tests @@ -221,7 +575,7 @@ mod test { #[test] fn query() { - let mut ts = TextureState::default(); + let mut ts = OldTextureState::default(); ts.mips.push(PlaneStates::empty()); ts.mips.push(PlaneStates::from_slice(&[ (1..3, Unit::new(TextureUses::RESOURCE)), @@ -266,12 +620,12 @@ mod test { #[test] fn merge() { let id = Id::dummy(); - let mut ts1 = TextureState::default(); + let mut ts1 = OldTextureState::default(); ts1.mips.push(PlaneStates::from_slice(&[( 1..3, Unit::new(TextureUses::RESOURCE), )])); - let mut ts2 = TextureState::default(); + let mut ts2 = OldTextureState::default(); assert_eq!( ts1.merge(id, &ts2, None), Ok(()), From 5d5298d77d468ec491a85ca6055eff84ee1b7bf7 Mon Sep 17 00:00:00 2001 From: Connor Fitzgerald Date: Mon, 2 May 2022 03:24:09 -0400 Subject: [PATCH 03/35] temp2 - next is lifetime only --- wgpu-core/src/command/clear.rs | 12 +- wgpu-core/src/command/memory_init.rs | 2 +- wgpu-core/src/command/mod.rs | 2 +- wgpu-core/src/command/render.rs | 4 +- wgpu-core/src/command/transfer.rs | 2 +- wgpu-core/src/device/mod.rs | 19 +- wgpu-core/src/init_tracker/texture.rs | 2 +- wgpu-core/src/present.rs | 4 +- wgpu-core/src/track/buffer.rs | 272 ++++++++---- wgpu-core/src/track/mod.rs | 595 +++++--------------------- wgpu-core/src/track/range.rs | 8 + wgpu-core/src/track/texture.rs | 423 ++++++++---------- 12 files changed, 495 insertions(+), 850 deletions(-) diff --git a/wgpu-core/src/command/clear.rs b/wgpu-core/src/command/clear.rs index 8bfc227acb..64261fabc9 100644 --- a/wgpu-core/src/command/clear.rs +++ b/wgpu-core/src/command/clear.rs @@ -10,7 +10,7 @@ use crate::{ id::{BufferId, CommandEncoderId, DeviceId, TextureId, Valid}, init_tracker::{MemoryInitKind, TextureInitRange}, resource::{Texture, TextureClearMode}, - track::{ResourceTracker, TextureSelector, OldTextureState}, + track::{OldTextureState, ResourceTracker, TextureSelector}, }; use hal::{auxil::align_to, CommandEncoder as _}; @@ -191,13 +191,13 @@ impl Global { // Check if subresource level range is valid let subresource_level_end = match subresource_range.mip_level_count { Some(count) => subresource_range.base_mip_level + count.get(), - None => dst_texture.full_range.levels.end, + None => dst_texture.full_range.mips.end, }; - if dst_texture.full_range.levels.start > subresource_range.base_mip_level - || dst_texture.full_range.levels.end < subresource_level_end + if dst_texture.full_range.mips.start > subresource_range.base_mip_level + || dst_texture.full_range.mips.end < subresource_level_end { return Err(ClearError::InvalidTextureLevelRange { - texture_level_range: dst_texture.full_range.levels.clone(), + texture_level_range: dst_texture.full_range.mips.clone(), subresource_base_mip_level: subresource_range.base_mip_level, subresource_mip_level_count: subresource_range.mip_level_count, }); @@ -277,7 +277,7 @@ pub(crate) fn clear_texture_no_device( }; let selector = TextureSelector { - levels: range.mip_range.clone(), + mips: range.mip_range.clone(), layers: range.layer_range.clone(), }; diff --git a/wgpu-core/src/command/memory_init.rs b/wgpu-core/src/command/memory_init.rs index 7bd4389660..134e86f083 100644 --- a/wgpu-core/src/command/memory_init.rs +++ b/wgpu-core/src/command/memory_init.rs @@ -8,7 +8,7 @@ use crate::{ id::{self, TextureId}, init_tracker::*, resource::{Buffer, Texture}, - track::{ResourceTracker, OldTextureState, TrackerSet}, + track::{OldTextureState, ResourceTracker, TrackerSet}, FastHashMap, }; diff --git a/wgpu-core/src/command/mod.rs b/wgpu-core/src/command/mod.rs index 9a8a7ea340..57d06d4d61 100644 --- a/wgpu-core/src/command/mod.rs +++ b/wgpu-core/src/command/mod.rs @@ -23,7 +23,7 @@ use crate::{ hub::{Global, GlobalIdentityHandlerFactory, HalApi, Storage, Token}, id, resource::{Buffer, Texture}, - track::{BufferState, ResourceTracker, OldTextureState, TrackerSet}, + track::{BufferState, ResourceTracker, TrackerSet}, Label, Stored, }; diff --git a/wgpu-core/src/command/render.rs b/wgpu-core/src/command/render.rs index 2cd83a178f..07ae6c8a5d 100644 --- a/wgpu-core/src/command/render.rs +++ b/wgpu-core/src/command/render.rs @@ -605,7 +605,7 @@ impl<'a, A: HalApi> RenderPassInfo<'a, A> { // but recording the discard right away be alright since the texture can't be used during the pass anyways texture_memory_actions.discard(TextureSurfaceDiscard { texture: view.parent_id.value.0, - mip_level: view.selector.levels.start, + mip_level: view.selector.mips.start, layer: view.selector.layers.start, }); } @@ -779,7 +779,7 @@ impl<'a, A: HalApi> RenderPassInfo<'a, A> { // Both are discarded using the regular path. discarded_surfaces.push(TextureSurfaceDiscard { texture: view.parent_id.value.0, - mip_level: view.selector.levels.start, + mip_level: view.selector.mips.start, layer: view.selector.layers.start, }); } diff --git a/wgpu-core/src/command/transfer.rs b/wgpu-core/src/command/transfer.rs index 1033feecee..8a1be65e28 100644 --- a/wgpu-core/src/command/transfer.rs +++ b/wgpu-core/src/command/transfer.rs @@ -191,7 +191,7 @@ pub(crate) fn extract_texture_selector( aspect: copy_aspect, }; let selector = TextureSelector { - levels: copy_texture.mip_level..copy_texture.mip_level + 1, + mips: copy_texture.mip_level..copy_texture.mip_level + 1, layers, }; diff --git a/wgpu-core/src/device/mod.rs b/wgpu-core/src/device/mod.rs index 98307f0d93..d7af1f5b5a 100644 --- a/wgpu-core/src/device/mod.rs +++ b/wgpu-core/src/device/mod.rs @@ -8,7 +8,7 @@ use crate::{ TextureInitTracker, TextureInitTrackerAction, }, instance, pipeline, present, resource, - track::{BufferState, TextureSelector, OldTextureState, TrackerSet, UsageConflict}, + track::{BufferState, OldTextureState, TextureSelector, TrackerSet, UsageConflict}, validation::{self, check_buffer_usage, check_texture_usage}, FastHashMap, Label, LabelHelpers as _, LifeGuard, MultiRefCount, RefCount, Stored, SubmissionIndex, DOWNLEVEL_ERROR_MESSAGE, @@ -655,7 +655,7 @@ impl Device { desc.array_layer_count(), ), full_range: TextureSelector { - levels: 0..desc.mip_level_count, + mips: 0..desc.mip_level_count, layers: 0..desc.array_layer_count(), }, life_guard: LifeGuard::new(desc.label.borrow_or_default()), @@ -876,7 +876,7 @@ impl Device { _ => texture.desc.array_layer_count(), }, }; - let level_end = texture.full_range.levels.end; + let level_end = texture.full_range.mips.end; let layer_end = texture.full_range.layers.end; if required_level_count > level_end { return Err(resource::CreateTextureViewError::TooManyMipLevels { @@ -927,7 +927,7 @@ impl Device { .array_layer_count .map_or(layer_end, |_| required_layer_count); let selector = TextureSelector { - levels: desc.range.base_mip_level..end_level, + mips: desc.range.base_mip_level..end_level, layers: desc.range.base_array_layer..end_layer, }; @@ -976,7 +976,7 @@ impl Device { } _ => hal::TextureUses::all(), }; - let mask_mip_level = if selector.levels.end - selector.levels.start != 1 { + let mask_mip_level = if selector.mips.end - selector.mips.start != 1 { hal::TextureUses::RESOURCE } else { hal::TextureUses::all() @@ -1058,7 +1058,8 @@ impl Device { let anisotropy_clamp = if let Some(clamp) = desc.anisotropy_clamp { let clamp = clamp.get(); - let valid_clamp = clamp <= hal::MAX_ANISOTROPY && conv::is_power_of_two_u32(clamp as u32); + let valid_clamp = + clamp <= hal::MAX_ANISOTROPY && conv::is_power_of_two_u32(clamp as u32); if !valid_clamp { return Err(resource::CreateSamplerError::InvalidClamp(clamp)); } @@ -2018,7 +2019,7 @@ impl Device { }); } - let mip_level_count = view.selector.levels.end - view.selector.levels.start; + let mip_level_count = view.selector.mips.end - view.selector.mips.start; if mip_level_count != 1 { return Err(Error::InvalidStorageTextureMipLevelCount { binding, @@ -3446,7 +3447,7 @@ impl Global { Ok(texture) => texture, Err(error) => break error, }; - let num_levels = texture.full_range.levels.end; + let num_levels = texture.full_range.mips.end; let num_layers = texture.full_range.layers.end; let ref_count = texture.life_guard.add_ref(); @@ -3524,7 +3525,7 @@ impl Global { texture.initialization_status = TextureInitTracker::new(desc.mip_level_count, 0); - let num_levels = texture.full_range.levels.end; + let num_levels = texture.full_range.mips.end; let num_layers = texture.full_range.layers.end; let ref_count = texture.life_guard.add_ref(); diff --git a/wgpu-core/src/init_tracker/texture.rs b/wgpu-core/src/init_tracker/texture.rs index a2913ea3d4..87bfcdc887 100644 --- a/wgpu-core/src/init_tracker/texture.rs +++ b/wgpu-core/src/init_tracker/texture.rs @@ -26,7 +26,7 @@ pub(crate) fn has_copy_partial_init_tracker_coverage( impl From for TextureInitRange { fn from(selector: TextureSelector) -> Self { TextureInitRange { - mip_range: selector.levels, + mip_range: selector.mips, layer_range: selector.layers, } } diff --git a/wgpu-core/src/present.rs b/wgpu-core/src/present.rs index dd50c08d2f..9ae07df347 100644 --- a/wgpu-core/src/present.rs +++ b/wgpu-core/src/present.rs @@ -174,7 +174,7 @@ impl Global { initialization_status: TextureInitTracker::new(1, 1), full_range: track::TextureSelector { layers: 0..1, - levels: 0..1, + mips: 0..1, }, life_guard: LifeGuard::new(""), clear_mode: resource::TextureClearMode::RenderPass { @@ -195,7 +195,7 @@ impl Global { id, track::TextureSelector { layers: 0..1, - levels: 0..1, + mips: 0..1, }, hal::TextureUses::UNINITIALIZED, None, diff --git a/wgpu-core/src/track/buffer.rs b/wgpu-core/src/track/buffer.rs index 01e3a79798..01f41d7121 100644 --- a/wgpu-core/src/track/buffer.rs +++ b/wgpu-core/src/track/buffer.rs @@ -1,122 +1,208 @@ -use super::{PendingTransition, ResourceState, Unit}; -use crate::id::{BufferId, Valid}; +use std::vec::Drain; + +use super::PendingTransition; +use crate::{ + hub, + id::{BufferId, TypedId, Valid}, + resource::Buffer, + track::{invalid_resource_state, resize_bitvec, skip_barrier, ResourceUses, UsageConflict}, +}; +use bit_vec::BitVec; use hal::BufferUses; -pub(crate) type BufferState = Unit; +impl ResourceUses for BufferUses { + const EXCLUSIVE: Self = Self::EXCLUSIVE; -impl PendingTransition { - fn collapse(self) -> Result { - if self.usage.start.is_empty() - || self.usage.start == self.usage.end - || !BufferUses::EXCLUSIVE.intersects(self.usage.start | self.usage.end) - { - Ok(self.usage.start | self.usage.end) - } else { - Err(self) - } + type Id = BufferId; + type Selector = (); + + fn bits(self) -> u16 { + self.bits() + } + + fn all_ordered(self) -> bool { + self.contains(Self::ORDERED) + } + + fn any_exclusive(self) -> bool { + self.intersects(Self::EXCLUSIVE) } } -impl Default for BufferState { - fn default() -> Self { +pub struct BufferBindGroupState { + buffers: Vec<(Valid, BufferUses)>, +} + +pub(crate) struct BufferUsageScope { + state: Vec, + owned: BitVec, +} + +impl BufferUsageScope { + pub fn new() -> Self { Self { - first: None, - last: BufferUses::empty(), + state: Vec::new(), + owned: BitVec::default(), } } -} -impl BufferState { - pub fn with_usage(usage: BufferUses) -> Self { - Unit::new(usage) + pub fn set_max_index(&mut self, size: usize) { + self.state.resize(size, BufferUses::empty()); + + resize_bitvec(&mut self.owned, size); } -} -impl ResourceState for BufferState { - type Id = BufferId; - type Selector = (); - type Usage = BufferUses; + pub unsafe fn extend_from_bind_group( + &mut self, + storage: &hub::Storage, BufferId>, + bind_group: &BufferBindGroupState, + ) -> Result<(), UsageConflict> { + for &(id, state) in &bind_group.buffers { + self.extend(storage, id, state)?; + } - fn query(&self, _selector: Self::Selector) -> Option { - Some(self.last) + Ok(()) } - fn change( + pub unsafe fn extend( &mut self, - id: Valid, - _selector: Self::Selector, - usage: Self::Usage, - output: Option<&mut Vec>>, - ) -> Result<(), PendingTransition> { - let old = self.last; - if old != usage || !BufferUses::ORDERED.contains(usage) { - let pending = PendingTransition { - id, - selector: (), - usage: old..usage, - }; - *self = match output { - None => { - assert_eq!( - self.first, None, - "extending a state that is already a transition" - ); - Unit::new(pending.collapse()?) - } - Some(transitions) => { - transitions.push(pending); - Unit { - first: self.first.or(Some(old)), - last: usage, - } - } - }; + storage: &hub::Storage, BufferId>, + id: Valid, + new_state: BufferUses, + ) -> Result<(), UsageConflict> { + let (index, _, _) = id.0.unzip(); + let index = index as usize; + + let currently_active = self.owned.get(index).unwrap_unchecked(); + if currently_active { + let current_state = *self.state.get_unchecked(index); + + let merged_state = current_state | new_state; + + if invalid_resource_state(merged_state) { + return Err(UsageConflict::from_buffer(id, current_state, new_state)); + } + + *self.state.get_unchecked_mut(index) = merged_state; } + + // We're the first to use this resource, let's add it. + self.owned.set(index, true); + + *self.state.get_unchecked_mut(index) = new_state; + Ok(()) } +} - fn merge( +pub(crate) struct BufferTracker { + start: Vec, + end: Vec, + temp: Vec>, + owned: BitVec, +} +impl BufferTracker { + pub fn new() -> Self { + Self { + start: Vec::new(), + end: Vec::new(), + temp: Vec::new(), + owned: BitVec::default(), + } + } + + fn set_max_index(&mut self, size: usize) { + self.start.resize(size, BufferUses::empty()); + self.end.resize(size, BufferUses::empty()); + + resize_bitvec(&mut self.owned, size); + } + + pub fn change_states( &mut self, - id: Valid, - other: &Self, - output: Option<&mut Vec>>, - ) -> Result<(), PendingTransition> { - let old = self.last; - let new = other.port(); - if old == new && BufferUses::ORDERED.contains(new) { - if output.is_some() && self.first.is_none() { - *self = Unit { - first: Some(old), - last: other.last, - }; + storage: &hub::Storage, BufferId>, + incoming_set: &Vec, + incoming_ownership: &BitVec, + ) -> Drain> { + let incoming_size = incoming_set.len(); + if incoming_size > self.start.len() { + self.set_max_index(incoming_size); + } + + for (word_index, mut word) in incoming_ownership.blocks().enumerate() { + if word == 0 { + continue; } - } else { - let pending = PendingTransition { - id, - selector: (), - usage: old..new, - }; - *self = match output { - None => { - assert_eq!( - self.first, None, - "extending a state that is already a transition" - ); - Unit::new(pending.collapse()?) - } - Some(transitions) => { - transitions.push(pending); - Unit { - first: self.first.or(Some(old)), - last: other.last, - } + + let bit_start = word_index * 64; + let bit_end = ((word_index + 1) * 64).min(incoming_size); + + for index in bit_start..bit_end { + if word & 0b1 == 0 { + continue; } - }; + word >>= 1; + + unsafe { self.transition(storage, incoming_set, index) }; + } } - Ok(()) + + self.temp.drain(..) } - fn optimize(&mut self) {} + pub unsafe fn change_states_bind_group( + &mut self, + storage: &hub::Storage, BufferId>, + incoming_set: &Vec, + incoming_ownership: &mut BitVec, + bind_group_state: &BufferBindGroupState, + ) -> Drain> { + let incoming_size = incoming_set.len(); + if incoming_size > self.start.len() { + self.set_max_index(incoming_size); + } + + for &(index, _) in bind_group_state.buffers.iter() { + let index = index.0.unzip().0 as usize; + if !incoming_ownership.get(index).unwrap_unchecked() { + continue; + } + self.transition(storage, incoming_set, index); + incoming_ownership.set(index, false); + } + + self.temp.drain(..) + } + + unsafe fn transition( + &mut self, + storage: &hub::Storage, BufferId>, + incoming_set: &Vec, + index: usize, + ) { + let old_tracked = self.owned.get(index).unwrap_unchecked(); + let old_state = *self.end.get_unchecked(index); + let new_state = *incoming_set.get_unchecked(index); + + if old_tracked { + if skip_barrier(old_state, new_state) { + return; + } + + self.temp.push(PendingTransition { + id: index as u32, + selector: (), + usage: old_state..new_state, + }); + + *self.end.get_unchecked_mut(index) = new_state; + } else { + *self.start.get_unchecked_mut(index) = new_state; + *self.end.get_unchecked_mut(index) = new_state; + + self.owned.set(index, true); + } + } } #[cfg(test)] diff --git a/wgpu-core/src/track/mod.rs b/wgpu-core/src/track/mod.rs index 268c31a76a..7b7f0b5c15 100644 --- a/wgpu-core/src/track/mod.rs +++ b/wgpu-core/src/track/mod.rs @@ -3,118 +3,33 @@ mod range; mod texture; use crate::{ - hub, + conv, hub, id::{self, TypedId, Valid}, resource, Epoch, FastHashMap, Index, RefCount, }; +use bit_vec::BitVec; use std::{ collections::hash_map::Entry, fmt, marker::PhantomData, num::NonZeroU32, ops, vec::Drain, }; use thiserror::Error; pub(crate) use buffer::BufferState; -pub(crate) use texture::{OldTextureState, TextureSelector}; - -/// A single unit of state tracking. It keeps an initial -/// usage as well as the last/current one, similar to `Range`. -#[derive(Clone, Copy, Debug, PartialEq)] -pub struct Unit { - first: Option, - last: U, -} - -impl Unit { - /// Create a new unit from a given usage. - fn new(usage: U) -> Self { - Self { - first: None, - last: usage, - } - } - - /// Return a usage to link to. - fn port(&self) -> U { - self.first.unwrap_or(self.last) - } -} - -/// The main trait that abstracts away the tracking logic of -/// a particular resource type, like a buffer or a texture. -pub(crate) trait ResourceState: Clone + Default { - /// Corresponding `HUB` identifier. - type Id: Copy + fmt::Debug + TypedId; - /// A type specifying the sub-resources. - type Selector: fmt::Debug; - /// Usage type for a `Unit` of a sub-resource. - type Usage: fmt::Debug; - - /// Check if all the selected sub-resources have the same - /// usage, and return it. - /// - /// Returns `None` if no sub-resources - /// are intersecting with the selector, or their usage - /// isn't consistent. - fn query(&self, selector: Self::Selector) -> Option; - - /// Change the last usage of the selected sub-resources. - /// - /// If `output` is specified, it's filled with the - /// `PendingTransition` objects corresponding to smaller - /// sub-resource transitions. The old usage is replaced by - /// the new one. - /// - /// If `output` is `None`, the old usage is extended with - /// the new usage. The error is returned if it's not possible, - /// specifying the conflicting transition. Extension can only - /// be done for read-only usages. - fn change( - &mut self, - id: Valid, - selector: Self::Selector, - usage: Self::Usage, - output: Option<&mut Vec>>, - ) -> Result<(), PendingTransition>; - - /// Merge the state of this resource tracked by a different instance - /// with the current one. - /// - /// Same rules for `output` apply as with `change()`: last usage state - /// is either replaced (when `output` is provided) with a - /// `PendingTransition` pushed to this vector, or extended with the - /// other read-only usage, unless there is a usage conflict, and - /// the error is generated (returning the conflict). - fn merge( - &mut self, - id: Valid, - other: &Self, - output: Option<&mut Vec>>, - ) -> Result<(), PendingTransition>; - - /// Try to optimize the internal representation. - fn optimize(&mut self); -} - -/// Structure wrapping the abstract tracking state with the relevant resource -/// data, such as the reference count and the epoch. -#[derive(Clone)] -struct Resource { - ref_count: RefCount, - state: S, - epoch: Epoch, -} +pub(crate) use texture::{ + TextureBindGroupState, TextureSelector, TextureTracker, TextureUsageScope, +}; /// A structure containing all the information about a particular resource /// transition. User code should be able to generate a pipeline barrier /// based on the contents. #[derive(Debug, PartialEq)] -pub(crate) struct PendingTransition { +pub(crate) struct PendingTransition { pub id: u32, pub selector: S::Selector, - pub usage: ops::Range, + pub usage: ops::Range, } -impl PendingTransition { +impl PendingTransition { /// Produce the hal barrier corresponding to the transition. pub fn into_hal<'a, A: hal::Api>( self, @@ -129,17 +44,7 @@ impl PendingTransition { } } -impl From> for UsageConflict { - fn from(e: PendingTransition) -> Self { - // Self::Buffer { - // id: e.id.0, - // combined_use: e.usage.end, - // } - todo!() - } -} - -impl PendingTransition { +impl PendingTransition { /// Produce the hal barrier corresponding to the transition. pub fn into_hal<'a, A: hal::Api>( self, @@ -151,10 +56,8 @@ impl PendingTransition { texture, range: wgt::ImageSubresourceRange { aspect: wgt::TextureAspect::All, - base_mip_level: self.selector.levels.start, - mip_level_count: NonZeroU32::new( - self.selector.levels.end - self.selector.levels.start, - ), + base_mip_level: self.selector.mips.start, + mip_level_count: NonZeroU32::new(self.selector.mips.end - self.selector.mips.start), base_array_layer: self.selector.layers.start, array_layer_count: NonZeroU32::new( self.selector.layers.end - self.selector.layers.start, @@ -165,414 +68,134 @@ impl PendingTransition { } } -impl From> for UsageConflict { - fn from(e: PendingTransition) -> Self { - // Self::Texture { - // id: e.id.0, - // mip_levels: e.selector.levels.start..e.selector.levels.end, - // array_layers: e.selector.layers.start..e.selector.layers.end, - // combined_use: e.usage.end, - // } - todo!() - } -} +trait ResourceUses: + fmt::Debug + ops::BitAnd + ops::BitOr + PartialEq + Sized +{ + const EXCLUSIVE: Self; -#[derive(Clone, Debug, Error)] -pub enum UseExtendError { - #[error("resource is invalid")] - InvalidResource, - #[error("total usage {0:?} is not valid")] - Conflict(U), -} + type Id: Copy + fmt::Debug + TypedId; + type Selector: fmt::Debug; -/// A tracker for all resources of a given type. -pub(crate) struct ResourceTracker { - /// An association of known resource indices with their tracked states. - map: FastHashMap>, - /// Temporary storage for collecting transitions. - temp: Vec>, - /// The backend variant for all the tracked resources. - backend: wgt::Backend, + fn bits(self) -> u16; + fn all_ordered(self) -> bool; + fn any_exclusive(self) -> bool; } -impl fmt::Debug for ResourceTracker { - fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - self.map - .iter() - .map(|(&index, res)| ((index, res.epoch), &res.state)) - .collect::>() - .fmt(formatter) - } +fn invalid_resource_state(state: T) -> bool { + // Is power of two also means "is one bit set". We check for this as if + // we're in any exclusive state, we must only be in a single state. + state.any_exclusive() && !conv::is_power_of_two_u16(state.bits()) } -#[allow( - // Explicit lifetimes are easier to reason about here. - clippy::needless_lifetimes, -)] -impl ResourceTracker { - /// Create a new empty tracker. - pub fn new(backend: wgt::Backend) -> Self { - Self { - map: FastHashMap::default(), - temp: Vec::new(), - backend, - } - } +fn skip_barrier(old_state: T, new_state: T) -> bool { + // If the state didn't change and all the usages are ordered, the hardware + // will guarentee the order of accesses, so we do not need to issue a barrier at all + old_state == new_state && old_state.all_ordered() +} - /// Remove an id from the tracked map. - pub(crate) fn remove(&mut self, id: Valid) -> bool { - let (index, epoch, backend) = id.0.unzip(); - debug_assert_eq!(backend, self.backend); - match self.map.remove(&index) { - Some(resource) => { - assert_eq!(resource.epoch, epoch); - true - } - None => false, +fn resize_bitvec(vec: &mut BitVec, size: usize) { + let owned_size_to_grow = size.checked_sub(vec.len()); + if let Some(delta) = owned_size_to_grow { + if delta != 0 { + vec.grow(delta, false); } + } else { + vec.truncate(size); } +} - /// Remove the resource `id`, only if `self` is holding the last reference to it. - /// - /// Return `true` if we did remove the resource; the underlying hal resource - /// is ready to be freed. - /// - /// This is generally only meaningful to apply to members of - /// [`Device::trackers`], which holds all resources allocated with that - /// [`Device`]. Other trackers should never be the final reference. - /// - /// [`Device`]: crate::device::Device - /// [`Device::trackers`]: crate::device::Device::trackers - pub(crate) fn remove_abandoned(&mut self, id: Valid) -> bool { - let (index, epoch, backend) = id.0.unzip(); - debug_assert_eq!(backend, self.backend); - match self.map.entry(index) { - // This code explicitly ignores requests for IDs that are no longer valid, - // i.e. corresponding to removed entries, or entries that got re-filled - // with new elements (having different epochs). - // This is needed because of the asynchronous nature of the device internals. - // As such, by the time a resource is added to the suspected list, it may - // already be fully removed from all the trackers (and be a stale ID). - // see https://github.com/gfx-rs/wgpu/issues/1996 - Entry::Occupied(e) => { - // see https://github.com/gfx-rs/wgpu/issues/1996 - if e.get().epoch == epoch && e.get().ref_count.load() == 1 { - e.remove(); - true - } else { - false - } - } - _ => false, +#[derive(Clone, Debug, Error)] +pub enum UsageConflict { + #[error("Attempted to use buffer {id:?} with {invalid_use}.")] + Buffer { + id: id::BufferId, + invalid_use: InvalidUse, + }, + #[error("Attempted to use a texture {id:?} mips {mip_levels:?} layers {array_layers:?} with {invalid_use}.")] + Texture { + id: id::TextureId, + mip_levels: ops::Range, + array_layers: ops::Range, + invalid_use: InvalidUse, + }, +} +impl UsageConflict { + fn from_buffer( + id: Valid, + current_state: hal::BufferUses, + new_state: hal::BufferUses, + ) -> Self { + Self::Buffer { + id: id.0, + invalid_use: InvalidUse { + current_state, + new_state, + }, } } - /// Try to optimize the internal representation. - pub(crate) fn optimize(&mut self) { - for resource in self.map.values_mut() { - resource.state.optimize(); - } - } + fn from_texture( + storage: &hub::Storage, id::TextureId>, + id: Valid, + selector: Option, + current_state: hal::TextureUses, + new_state: hal::TextureUses, + ) -> Self { + let texture = &storage[id]; - /// Return an iterator over used resources keys. - pub fn used<'a>(&'a self) -> impl 'a + Iterator> { - let backend = self.backend; - self.map - .iter() - .map(move |(&index, resource)| Valid(S::Id::zip(index, resource.epoch, backend))) - } + let mips; + let layers; - pub fn get_ref_count(&self, id: Valid) -> &RefCount { - let (index, _, _) = id.0.unzip(); - &self.map[&index].ref_count - } - - /// Return true if there is nothing here. - pub fn is_empty(&self) -> bool { - self.map.is_empty() - } - - /// Clear the tracked contents. - pub fn clear(&mut self) { - self.map.clear(); - } - - /// Begin tracking a new resource `id` in state `state`. - /// - /// Hold `ref_count` in the tracker. - /// - /// Returns false if the resource is already registered. - pub(crate) fn init( - &mut self, - id: Valid, - ref_count: RefCount, - state: S, - ) -> Result<(), &S> { - let (index, epoch, backend) = id.0.unzip(); - debug_assert_eq!(backend, self.backend); - match self.map.entry(index) { - Entry::Vacant(e) => { - e.insert(Resource { - ref_count, - state, - epoch, - }); - Ok(()) + match selector { + Some(selector) => { + mips = selector.mips; + layers = selector.layers; } - Entry::Occupied(e) => Err(&e.into_mut().state), - } - } - - /// Query the usage of a resource selector. - /// - /// Returns `Some(Usage)` only if this usage is consistent - /// across the given selector. - #[allow(unused)] // TODO: figure out if this needs to be removed - pub fn query(&self, id: Valid, selector: S::Selector) -> Option { - let (index, epoch, backend) = id.0.unzip(); - debug_assert_eq!(backend, self.backend); - let res = self.map.get(&index)?; - assert_eq!(res.epoch, epoch); - res.state.query(selector) - } - - /// Make sure that a resource is tracked, and return a mutable reference to it. - /// - /// If the resource isn't tracked, start it in the default state, and take a - /// clone of `ref_count`. - /// - /// The `self_backend` and `map` arguments should be the `backend` and `map` - /// fields of a `ResourceTracker`. Ideally this function would just take - /// `&mut self` and access those from there, but that would upset the borrow - /// checker in some callers, who want to borrow `ResourceTracker::temp` - /// alongside our return value. The approach taken here has the caller - /// borrow both `map` and `temp`, so the borrow checker can see that they - /// don't alias. - fn get_or_insert<'a>( - self_backend: wgt::Backend, - map: &'a mut FastHashMap>, - id: Valid, - ref_count: &RefCount, - ) -> &'a mut Resource { - let (index, epoch, backend) = id.0.unzip(); - debug_assert_eq!(self_backend, backend); - match map.entry(index) { - Entry::Vacant(e) => e.insert(Resource { - ref_count: ref_count.clone(), - state: S::default(), - epoch, - }), - Entry::Occupied(e) => { - assert_eq!(e.get().epoch, epoch); - e.into_mut() + None => { + mips = texture.full_range.mips; + layers = texture.full_range.layers; } } - } - - /// Return a mutable reference to `id`'s state. - fn get<'a>( - self_backend: wgt::Backend, - map: &'a mut FastHashMap>, - id: Valid, - ) -> &'a mut Resource { - let (index, epoch, backend) = id.0.unzip(); - debug_assert_eq!(self_backend, backend); - let e = map.get_mut(&index).unwrap(); - assert_eq!(e.epoch, epoch); - e - } - - /// Extend the usage of `id`, tracking it if necessary. - /// - /// Returns conflicting transition as an error. - pub(crate) fn change_extend( - &mut self, - id: Valid, - ref_count: &RefCount, - selector: S::Selector, - usage: S::Usage, - ) -> Result<(), PendingTransition> { - Self::get_or_insert(self.backend, &mut self.map, id, ref_count) - .state - .change(id, selector, usage, None) - } - - /// Replace the usage of a specified resource. - pub(crate) fn change_replace( - &mut self, - id: Valid, - ref_count: &RefCount, - selector: S::Selector, - usage: S::Usage, - ) -> Drain> { - let res = Self::get_or_insert(self.backend, &mut self.map, id, ref_count); - res.state - .change(id, selector, usage, Some(&mut self.temp)) - .ok(); //TODO: unwrap? - self.temp.drain(..) - } - - /// Replace the usage of a specified already tracked resource. - /// (panics if the resource is not yet tracked) - pub(crate) fn change_replace_tracked( - &mut self, - id: Valid, - selector: S::Selector, - usage: S::Usage, - ) -> Drain> { - let res = Self::get(self.backend, &mut self.map, id); - res.state - .change(id, selector, usage, Some(&mut self.temp)) - .ok(); - self.temp.drain(..) - } - - /// Merge another tracker into `self` by extending the current states - /// without any transitions. - pub(crate) fn merge_extend(&mut self, other: &Self) -> Result<(), PendingTransition> { - debug_assert_eq!(self.backend, other.backend); - for (&index, new) in other.map.iter() { - match self.map.entry(index) { - Entry::Vacant(e) => { - e.insert(new.clone()); - } - Entry::Occupied(e) => { - assert_eq!( - e.get().epoch, - new.epoch, - "ID {:?} wasn't properly removed", - S::Id::zip(index, e.get().epoch, self.backend) - ); - let id = Valid(S::Id::zip(index, new.epoch, self.backend)); - e.into_mut().state.merge(id, &new.state, None)?; - } - } - } - Ok(()) - } - /// Merge another tracker, adding it's transitions to `self`. - /// Transitions the current usage to the new one. - pub(crate) fn merge_replace<'a>(&'a mut self, other: &'a Self) -> Drain> { - for (&index, new) in other.map.iter() { - match self.map.entry(index) { - Entry::Vacant(e) => { - e.insert(new.clone()); - } - Entry::Occupied(e) => { - assert_eq!( - e.get().epoch, - new.epoch, - "ID {:?} wasn't properly removed", - S::Id::zip(index, e.get().epoch, self.backend) - ); - let id = Valid(S::Id::zip(index, new.epoch, self.backend)); - e.into_mut() - .state - .merge(id, &new.state, Some(&mut self.temp)) - .ok(); //TODO: unwrap? - } - } + Self::Texture { + id: id.0, + mip_levels: mips, + array_layers: layers, + invalid_use: InvalidUse { + current_state, + new_state, + }, } - self.temp.drain(..) - } - - /// Use a given resource provided by an `Id` with the specified usage. - /// Combines storage access by 'Id' with the transition that extends - /// the last read-only usage, if possible. - /// - /// Returns the old usage as an error if there is a conflict. - pub(crate) fn use_extend<'a, T: 'a + hub::Resource>( - &mut self, - storage: &'a hub::Storage, - id: S::Id, - selector: S::Selector, - usage: S::Usage, - ) -> Result<&'a T, UseExtendError> { - let item = storage - .get(id) - .map_err(|_| UseExtendError::InvalidResource)?; - self.change_extend( - Valid(id), - item.life_guard().ref_count.as_ref().unwrap(), - selector, - usage, - ) - .map(|()| item) - .map_err(|pending| UseExtendError::Conflict(pending.usage.end)) } +} - /// Use a given resource provided by an `Id` with the specified usage. - /// Combines storage access by 'Id' with the transition that replaces - /// the last usage with a new one, returning an iterator over these - /// transitions. - pub(crate) fn use_replace<'a, T: 'a + hub::Resource>( - &mut self, - storage: &'a hub::Storage, - id: S::Id, - selector: S::Selector, - usage: S::Usage, - ) -> Result<(&'a T, Drain>), S::Id> { - let item = storage.get(id).map_err(|_| id)?; - let drain = self.change_replace( - Valid(id), - item.life_guard().ref_count.as_ref().unwrap(), - selector, - usage, - ); - Ok((item, drain)) - } +#[derive(Clone, Debug)] +struct InvalidUse { + current_state: T, + new_state: T, } -impl ResourceState for PhantomData { - type Id = I; - type Selector = (); - type Usage = (); +impl fmt::Display for InvalidUse { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let current = self.current_state; + let new = self.new_state; - fn query(&self, _selector: Self::Selector) -> Option { - Some(()) - } + let current_exclusive = current & T::EXCLUSIVE; + let new_exclusive = new & T::EXCLUSIVE; - fn change( - &mut self, - _id: Valid, - _selector: Self::Selector, - _usage: Self::Usage, - _output: Option<&mut Vec>>, - ) -> Result<(), PendingTransition> { - Ok(()) - } + let exclusive = current_exclusive | new_exclusive; - fn merge( - &mut self, - _id: Valid, - _other: &Self, - _output: Option<&mut Vec>>, - ) -> Result<(), PendingTransition> { - Ok(()) + write!( + f, + "conflicting usages. Current usage {current:?} and new usage {new:?}. \ + {exclusive:?} is an exclusive usage and cannot be used with any other\ + usages within the usage scope (renderpass or compute dispatch)" + ) } - - fn optimize(&mut self) {} } -pub const DUMMY_SELECTOR: () = (); - -#[derive(Clone, Debug, Error)] -pub enum UsageConflict { - #[error( - "Attempted to use buffer {id:?} as a combination of {combined_use:?} within a usage scope." - )] - Buffer { - id: id::BufferId, - combined_use: hal::BufferUses, - }, - #[error("Attempted to use texture {id:?} mips {mip_levels:?} layers {array_layers:?} as a combination of {combined_use:?} within a usage scope.")] - Texture { - id: id::TextureId, - mip_levels: ops::Range, - array_layers: ops::Range, - combined_use: hal::TextureUses, - }, +pub(crate) struct BindGroupStates { + pub textures: TextureBindGroupState, } /// A set of trackers for all relevant resources. diff --git a/wgpu-core/src/track/range.rs b/wgpu-core/src/track/range.rs index 9f3b316eb9..fb58313b99 100644 --- a/wgpu-core/src/track/range.rs +++ b/wgpu-core/src/track/range.rs @@ -36,6 +36,14 @@ impl RangedStates { } } + pub fn iter(&self) -> impl Iterator, T)> { + self.ranges.iter() + } + + pub fn into_iter(self) -> impl Iterator, T)> { + self.ranges.into_iter() + } + /// Clear all the ranges. pub fn clear(&mut self) { self.ranges.clear(); diff --git a/wgpu-core/src/track/texture.rs b/wgpu-core/src/track/texture.rs index 1471af54ef..1332f8ee83 100644 --- a/wgpu-core/src/track/texture.rs +++ b/wgpu-core/src/track/texture.rs @@ -1,8 +1,9 @@ -use super::{range::RangedStates, PendingTransition, ResourceState, Unit}; +use super::{range::RangedStates, PendingTransition}; use crate::{ - conv, hub, + hub, id::{TextureId, TypedId, Valid}, resource::Texture, + track::{invalid_resource_state, resize_bitvec, skip_barrier, ResourceUses, UsageConflict}, }; use bit_vec::BitVec; use hal::TextureUses; @@ -12,226 +13,38 @@ use naga::FastHashMap; use std::{iter, ops::Range, vec::Drain}; -type PlaneStates = RangedStates>; - #[derive(Clone, Debug, PartialEq, Eq)] pub struct TextureSelector { - //TODO: rename to `mip_levels` and `array_layers` for consistency - //pub aspects: hal::FormatAspects, - pub levels: Range, + pub mips: Range, pub layers: Range, } -#[derive(Clone, Debug, Default, PartialEq)] -pub(crate) struct OldTextureState { - mips: ArrayVec, -} - -impl PendingTransition { - fn collapse(self) -> Result { - if self.usage.start.is_empty() - || self.usage.start == self.usage.end - || !TextureUses::EXCLUSIVE.intersects(self.usage.start | self.usage.end) - { - Ok(self.usage.start | self.usage.end) - } else { - Err(self) - } - } -} - -impl OldTextureState { - pub fn new(mip_level_count: u32, array_layer_count: u32) -> Self { - Self { - mips: iter::repeat_with(|| { - PlaneStates::from_range(0..array_layer_count, Unit::new(TextureUses::UNINITIALIZED)) - }) - .take(mip_level_count as usize) - .collect(), - } - } -} +impl ResourceUses for TextureUses { + const EXCLUSIVE: Self = Self::EXCLUSIVE; -impl ResourceState for OldTextureState { type Id = TextureId; type Selector = TextureSelector; - type Usage = TextureUses; - - fn query(&self, selector: Self::Selector) -> Option { - let mut result = None; - // Note: we only consider the subresources tracked by `self`. - // If some are not known to `self`, it means the can assume the - // initial state to whatever we need, which we can always make - // to be the same as the query result for the known subresources. - let num_levels = self.mips.len(); - assert!(num_levels >= selector.levels.end as usize); - let mip_start = num_levels.min(selector.levels.start as usize); - let mip_end = num_levels.min(selector.levels.end as usize); - for mip in self.mips[mip_start..mip_end].iter() { - match mip.query(&selector.layers, |unit| unit.last) { - None => {} - Some(Ok(usage)) if result == Some(usage) => {} - Some(Ok(usage)) if result.is_none() => { - result = Some(usage); - } - Some(Ok(_)) | Some(Err(())) => return None, - } - } - result - } - fn change( - &mut self, - id: Valid, - selector: Self::Selector, - usage: Self::Usage, - mut output: Option<&mut Vec>>, - ) -> Result<(), PendingTransition> { - assert!(self.mips.len() >= selector.levels.end as usize); - for (mip_id, mip) in self.mips[selector.levels.start as usize..selector.levels.end as usize] - .iter_mut() - .enumerate() - { - let level = selector.levels.start + mip_id as u32; - let layers = mip.isolate(&selector.layers, Unit::new(usage)); - for &mut (ref range, ref mut unit) in layers { - if unit.last == usage && TextureUses::ORDERED.contains(usage) { - continue; - } - // TODO: Can't satisfy clippy here unless we modify - // `TextureSelector` to use `std::ops::RangeBounds`. - #[allow(clippy::range_plus_one)] - let pending = PendingTransition { - id, - selector: TextureSelector { - levels: level..level + 1, - layers: range.clone(), - }, - usage: unit.last..usage, - }; - - *unit = match output { - None => { - assert_eq!( - unit.first, None, - "extending a state that is already a transition" - ); - Unit::new(pending.collapse()?) - } - Some(ref mut out) => { - out.push(pending); - Unit { - first: unit.first.or(Some(unit.last)), - last: usage, - } - } - }; - } - } - Ok(()) + fn bits(self) -> u16 { + self.bits() } - fn merge( - &mut self, - id: Valid, - other: &Self, - mut output: Option<&mut Vec>>, - ) -> Result<(), PendingTransition> { - let mut temp = Vec::new(); - assert!(self.mips.len() >= other.mips.len()); - - for (mip_id, (mip_self, mip_other)) in self.mips.iter_mut().zip(&other.mips).enumerate() { - let level = mip_id as u32; - temp.extend(mip_self.merge(mip_other, 0)); - mip_self.clear(); - - for (layers, states) in temp.drain(..) { - let unit = match states { - Range { - start: None, - end: None, - } => unreachable!(), - Range { - start: Some(start), - end: None, - } => start, - Range { - start: None, - end: Some(end), - } => end, - Range { - start: Some(start), - end: Some(end), - } => { - let to_usage = end.port(); - if start.last == to_usage && TextureUses::ORDERED.contains(to_usage) { - Unit { - first: match output { - None => start.first, - Some(_) => start.first.or(Some(start.last)), - }, - last: end.last, - } - } else { - // TODO: Can't satisfy clippy here unless we modify - // `TextureSelector` to use `std::ops::RangeBounds`. - #[allow(clippy::range_plus_one)] - let pending = PendingTransition { - id, - selector: TextureSelector { - levels: level..level + 1, - layers: layers.clone(), - }, - usage: start.last..to_usage, - }; - - match output { - None => { - assert_eq!( - start.first, None, - "extending a state that is already a transition" - ); - Unit::new(pending.collapse()?) - } - Some(ref mut out) => { - out.push(pending); - Unit { - // this has to leave a valid `first` state - first: start.first.or(Some(start.last)), - last: end.last, - } - } - } - } - } - }; - mip_self.append(layers, unit); - } - } - - Ok(()) + fn all_ordered(self) -> bool { + self.contains(Self::ORDERED) } - fn optimize(&mut self) { - for mip in self.mips.iter_mut() { - mip.coalesce(); - } + fn any_exclusive(self) -> bool { + self.intersects(Self::EXCLUSIVE) } } -fn invalid_texture_state(state: TextureUses) -> bool { - // Is power of two also means "is one bit set". We check for this as if - // we're in any exclusive state, we must only be in a single state. - state.intersects(hal::TextureUses::EXCLUSIVE) && !conv::is_power_of_two_u16(state.bits()) -} - #[derive(Clone, Debug, Default, PartialEq)] -pub(crate) struct ComplexTextureState { +struct ComplexTextureState { mips: ArrayVec, { hal::MAX_MIP_LEVELS as usize }>, } impl ComplexTextureState { - pub fn new(mip_level_count: u32, array_layer_count: u32) -> Self { + fn new(mip_level_count: u32, array_layer_count: u32) -> Self { Self { mips: iter::repeat_with(|| { RangedStates::from_range(0..array_layer_count, TextureUses::UNINITIALIZED) @@ -242,15 +55,9 @@ impl ComplexTextureState { } } -fn resize_bitvec(vec: &mut BitVec, size: usize) { - let owned_size_to_grow = size.checked_sub(vec.len()); - if let Some(delta) = owned_size_to_grow { - if delta != 0 { - vec.grow(delta, false); - } - } else { - vec.truncate(size); - } +// TODO: This representation could be optimized in a couple ways, but keep it simple for now. +pub struct TextureBindGroupState { + textures: Vec<(Valid, Option, TextureUses)>, } pub struct TextureStateSet { @@ -285,16 +92,28 @@ impl TextureUsageScope { resize_bitvec(&mut self.owned, size); } + pub unsafe fn extend_from_bind_group( + &mut self, + storage: &hub::Storage, TextureId>, + bind_group: &TextureBindGroupState, + ) -> Result<(), UsageConflict> { + for (id, selector, state) in &bind_group.textures { + self.extend(storage, *id, selector.clone(), *state)?; + } + + Ok(()) + } + /// # Safety /// /// `id` must be a valid ID and have an ID value less than the last call to set_max_index. - unsafe fn extend( + pub unsafe fn extend( &mut self, storage: &hub::Storage, TextureId>, id: Valid, selector: Option, new_state: TextureUses, - ) -> Result<(), ()> { + ) -> Result<(), UsageConflict> { let (index, _, _) = id.0.unzip(); let index = index as usize; @@ -306,9 +125,14 @@ impl TextureUsageScope { (false, None) => { let merged_state = current_state | new_state; - if invalid_texture_state(merged_state) { - // Conflicting states - return Err(()); + if invalid_resource_state(merged_state) { + return Err(UsageConflict::from_texture( + storage, + id, + selector, + current_state, + new_state, + )); } *self.set.simple.get_unchecked_mut(index) = merged_state; @@ -351,13 +175,13 @@ impl TextureUsageScope { id: Valid, selector: Option, new_state: TextureUses, - ) -> Result<(), ()> { + ) -> Result<(), UsageConflict> { let texture = &storage[id]; - let (id, _, _) = id.0.unzip(); + let (index, _, _) = id.0.unzip(); // Create the complex entry for this texture. - let complex = self.set.complex.entry(id).or_insert_with(|| { + let complex = self.set.complex.entry(index).or_insert_with(|| { ComplexTextureState::new( texture.desc.mip_level_count, texture.desc.array_layer_count(), @@ -368,12 +192,12 @@ impl TextureUsageScope { let layers; match selector { Some(selector) => { - mips = selector.levels.clone(); + mips = selector.mips.clone(); layers = selector.layers.clone(); } None => { - mips = 0..texture.desc.mip_level_count; - layers = 0..texture.desc.array_layer_count(); + mips = texture.full_range.mips; + layers = texture.full_range.layers; } } @@ -384,8 +208,14 @@ impl TextureUsageScope { // Set our state. for (_, current_state) in mip_state.isolate(&layers, new_state) { let merged = *current_state | new_state; - if invalid_texture_state(merged) { - return Err(()); + if invalid_resource_state(merged) { + return Err(UsageConflict::from_texture( + storage, + id, + selector, + *current_state, + new_state, + )); } *current_state = merged; } @@ -399,7 +229,7 @@ pub(crate) struct TextureTracker { start_set: TextureStateSet, end_set: TextureStateSet, /// Temporary storage for collecting transitions. - temp: Vec>, + temp: Vec>, owned: BitVec, } impl TextureTracker { @@ -421,12 +251,12 @@ impl TextureTracker { resize_bitvec(&mut self.owned, size); } - pub fn transition_to_other( + pub fn change_states( &mut self, storage: &hub::Storage, TextureId>, incoming_set: &TextureStateSet, incoming_ownership: &BitVec, - ) -> Drain> { + ) -> Drain> { let incoming_size = incoming_set.simple.len(); if incoming_size > self.start_set.simple.len() { self.set_max_index(incoming_size); @@ -446,14 +276,38 @@ impl TextureTracker { } word >>= 1; - unsafe { self.transition_index_to_other(storage, incoming_set, index) }; + unsafe { self.transition(storage, incoming_set, index) }; } } self.temp.drain(..) } - unsafe fn transition_index_to_other( + pub unsafe fn change_states_bind_group( + &mut self, + storage: &hub::Storage, TextureId>, + incoming_set: &TextureStateSet, + incoming_ownership: &mut BitVec, + bind_group_state: &TextureBindGroupState, + ) -> Drain> { + let incoming_size = incoming_set.simple.len(); + if incoming_size > self.start_set.simple.len() { + self.set_max_index(incoming_size); + } + + for &(index, _, _) in bind_group_state.textures.iter() { + let index = index.0.unzip().0 as usize; + if !incoming_ownership.get(index).unwrap_unchecked() { + continue; + } + self.transition(storage, incoming_set, index); + incoming_ownership.set(index, false); + } + + self.temp.drain(..) + } + + unsafe fn transition( &mut self, storage: &hub::Storage, TextureId>, incoming_set: &TextureStateSet, @@ -478,7 +332,7 @@ impl TextureTracker { *self.start_set.simple.get_unchecked_mut(index) = TextureUses::COMPLEX; *self.end_set.simple.get_unchecked_mut(index) = TextureUses::COMPLEX; - let complex_state = &incoming_set.complex[&(index as u32)]; + let complex_state = incoming_set.complex.get(&(index as u32)).unwrap_unchecked(); self.start_set .complex .insert(index as u32, complex_state.clone()); @@ -489,9 +343,7 @@ impl TextureTracker { self.owned.set(index, true); } (true, false, false) => { - // If the state didn't change and all the usages are ordered, the hardware - // will guarentee the order of accesses, so we do not need to issue a barrier at all - if old_state == new_state && old_state.contains(TextureUses::ORDERED) { + if skip_barrier(old_state, new_state) { return; } @@ -500,17 +352,23 @@ impl TextureTracker { selector: storage.get_unchecked(index as u32).unwrap().full_range, usage: old_state..new_state, }); + + *self.end_set.simple.get_unchecked_mut(index) = new_state; } (true, true, true) => { - self.transition_index_to_other_complex(storage, incoming_set, index); + self.transition_complex_to_complex(storage, incoming_set, index); + } + (true, true, false) => { + self.transition_complex_to_simple(storage, incoming_set, index, new_state); + } + (true, false, true) => { + self.transition_simple_to_complex(storage, incoming_set, index, old_state); } - (true, true, false) => {} - (true, false, true) => {} } } #[cold] - unsafe fn transition_index_to_other_complex( + unsafe fn transition_complex_to_complex( &mut self, storage: &hub::Storage, TextureId>, incoming_set: &TextureStateSet, @@ -537,9 +395,7 @@ impl TextureTracker { start: Some(start), end: Some(end), } => { - // If the state didn't change and all the usages are ordered, the hardware - // will guarentee the order of accesses, so we do not need to issue a barrier at all - if start == end && TextureUses::ORDERED.contains(start) { + if skip_barrier(start, end) { return; } // TODO: Can't satisfy clippy here unless we modify @@ -548,7 +404,7 @@ impl TextureTracker { let pending = PendingTransition { id: index as u32, selector: TextureSelector { - levels: level..level + 1, + mips: level..level + 1, layers: layers.clone(), }, usage: start..end, @@ -565,6 +421,77 @@ impl TextureTracker { } } } + + #[cold] + unsafe fn transition_complex_to_simple( + &mut self, + storage: &hub::Storage, TextureId>, + incoming_set: &TextureStateSet, + index: usize, + new_state: TextureUses, + ) { + let old_complex = self + .end_set + .complex + .remove(&(index as u32)) + .unwrap_unchecked(); + + for (mip_index, mips) in old_complex.mips.into_iter().enumerate() { + let mip_index = mip_index as u32; + for (layer, old_state) in mips.into_iter() { + if skip_barrier(old_state, new_state) { + continue; + } + + #[allow(clippy::range_plus_one)] + self.temp.push(PendingTransition { + id: index as u32, + selector: TextureSelector { + mips: mip_index..mip_index + 1, + layers: layer, + }, + usage: old_state..new_state, + }) + } + } + + *self.end_set.simple.get_unchecked_mut(index) = new_state; + } + + #[cold] + unsafe fn transition_simple_to_complex( + &mut self, + storage: &hub::Storage, TextureId>, + incoming_set: &TextureStateSet, + index: usize, + old_state: TextureUses, + ) { + let new_complex = incoming_set.complex.get(&(index as u32)).unwrap_unchecked(); + + for (mip_index, mips) in new_complex.mips.iter().enumerate() { + let mip_index = mip_index as u32; + for (layer, new_state) in mips.into_iter() { + if skip_barrier(old_state, new_state) { + continue; + } + + #[allow(clippy::range_plus_one)] + self.temp.push(PendingTransition { + id: index as u32, + selector: TextureSelector { + mips: mip_index..mip_index + 1, + layers: layer, + }, + usage: old_state..new_state, + }) + } + } + + self.end_set + .complex + .insert(index as u32, new_complex.clone()); + *self.end_set.simple.get_unchecked_mut(index) = TextureUses::COMPLEX; + } } #[cfg(test)] @@ -585,7 +512,7 @@ mod test { assert_eq!( ts.query(TextureSelector { - levels: 1..2, + mips: 1..2, layers: 2..5, }), // level 1 matches @@ -593,7 +520,7 @@ mod test { ); assert_eq!( ts.query(TextureSelector { - levels: 0..2, + mips: 0..2, layers: 2..5, }), // level 0 is empty, level 1 matches @@ -601,7 +528,7 @@ mod test { ); assert_eq!( ts.query(TextureSelector { - levels: 1..2, + mips: 1..2, layers: 1..5, }), // level 1 matches with gaps @@ -609,7 +536,7 @@ mod test { ); assert_eq!( ts.query(TextureSelector { - levels: 1..2, + mips: 1..2, layers: 4..6, }), // level 1 doesn't match @@ -656,7 +583,7 @@ mod test { Err(PendingTransition { id, selector: TextureSelector { - levels: 0..1, + mips: 0..1, layers: 1..2, }, usage: TextureUses::RESOURCE | TextureUses::COPY_SRC..TextureUses::COPY_DST, @@ -682,7 +609,7 @@ mod test { PendingTransition { id, selector: TextureSelector { - levels: 0..1, + mips: 0..1, layers: 1..2, }, usage: TextureUses::RESOURCE | TextureUses::COPY_SRC..TextureUses::COPY_DST, @@ -690,7 +617,7 @@ mod test { PendingTransition { id, selector: TextureSelector { - levels: 0..1, + mips: 0..1, layers: 2..3, }, // the transition links the end of the base rage (..SAMPLED) @@ -742,7 +669,7 @@ mod test { &[PendingTransition { id, selector: TextureSelector { - levels: 0..1, + mips: 0..1, layers: 2..3, }, usage: TextureUses::COPY_SRC..TextureUses::COPY_DST, From 4fc1ea402e025d73b52612c4a2af5e15c777cc1c Mon Sep 17 00:00:00 2001 From: Connor Fitzgerald Date: Tue, 3 May 2022 00:47:10 -0400 Subject: [PATCH 04/35] temp3 - all trackers in place --- wgpu-core/src/binding_model.rs | 13 +- wgpu-core/src/command/bundle.rs | 16 +- wgpu-core/src/command/clear.rs | 6 +- wgpu-core/src/command/draw.rs | 8 +- wgpu-core/src/command/memory_init.rs | 4 +- wgpu-core/src/command/mod.rs | 19 +- wgpu-core/src/command/render.rs | 184 +++++++++--------- wgpu-core/src/hub.rs | 8 +- wgpu-core/src/track/buffer.rs | 190 +++++------------- wgpu-core/src/track/mod.rs | 193 +++++++++--------- wgpu-core/src/track/stateless.rs | 50 +++++ wgpu-core/src/track/texture.rs | 280 ++++++--------------------- 12 files changed, 370 insertions(+), 601 deletions(-) create mode 100644 wgpu-core/src/track/stateless.rs diff --git a/wgpu-core/src/binding_model.rs b/wgpu-core/src/binding_model.rs index 030574842a..e91df3308d 100644 --- a/wgpu-core/src/binding_model.rs +++ b/wgpu-core/src/binding_model.rs @@ -4,7 +4,7 @@ use crate::{ hub::Resource, id::{BindGroupLayoutId, BufferId, DeviceId, SamplerId, TextureViewId, Valid}, init_tracker::{BufferInitTrackerAction, TextureInitTrackerAction}, - track::{TrackerSet, UsageConflict, DUMMY_SELECTOR}, + track::{UsageConflict, BindGroupStates}, validation::{MissingBufferUsageError, MissingTextureUsageError}, FastHashMap, Label, LifeGuard, MultiRefCount, Stored, }; @@ -17,7 +17,7 @@ use serde::Deserialize; use serde::Serialize; use std::{ - borrow::{Borrow, Cow}, + borrow::{Cow}, ops::Range, }; @@ -709,13 +709,12 @@ pub(crate) fn buffer_binding_type_alignment( } } -#[derive(Debug)] pub struct BindGroup { pub(crate) raw: A::BindGroup, pub(crate) device_id: Stored, pub(crate) layout_id: Valid, pub(crate) life_guard: LifeGuard, - pub(crate) used: TrackerSet, + pub(crate) used: BindGroupStates, pub(crate) used_buffer_ranges: Vec, pub(crate) used_texture_ranges: Vec, pub(crate) dynamic_binding_info: Vec, @@ -766,12 +765,6 @@ impl BindGroup { } } -impl Borrow<()> for BindGroup { - fn borrow(&self) -> &() { - &DUMMY_SELECTOR - } -} - impl Resource for BindGroup { const TYPE: &'static str = "BindGroup"; diff --git a/wgpu-core/src/command/bundle.rs b/wgpu-core/src/command/bundle.rs index c04e56b37d..181799f102 100644 --- a/wgpu-core/src/command/bundle.rs +++ b/wgpu-core/src/command/bundle.rs @@ -49,7 +49,7 @@ use crate::{ id, init_tracker::{BufferInitTrackerAction, MemoryInitKind, TextureInitTrackerAction}, pipeline::PipelineFlags, - track::{TrackerSet, UsageConflict}, + track::{TrackerSet, UsageConflict, UsageScope, RenderBundleScope}, validation::check_buffer_usage, Label, LabelHelpers, LifeGuard, Stored, }; @@ -599,23 +599,23 @@ pub type RenderBundleDescriptor<'a> = wgt::RenderBundleDescriptor>; // The plan is to back it by an actual Vulkan secondary buffer, D3D12 Bundle, // or Metal indirect command buffer. #[derive(Debug)] -pub struct RenderBundle { +pub struct RenderBundle { // Normalized command stream. It can be executed verbatim, // without re-binding anything on the pipeline change. base: BasePass, pub(super) is_ds_read_only: bool, pub(crate) device_id: Stored, - pub(crate) used: TrackerSet, + pub(crate) used: RenderBundleScope, pub(super) buffer_memory_init_actions: Vec, pub(super) texture_memory_init_actions: Vec, pub(super) context: RenderPassContext, pub(crate) life_guard: LifeGuard, } -unsafe impl Send for RenderBundle {} -unsafe impl Sync for RenderBundle {} +unsafe impl Send for RenderBundle {} +unsafe impl Sync for RenderBundle {} -impl RenderBundle { +impl RenderBundle { /// Actually encode the contents into a native command buffer. /// /// This is partially duplicating the logic of `command_encoder_run_render_pass`. @@ -625,7 +625,7 @@ impl RenderBundle { /// Note that the function isn't expected to fail, generally. /// All the validation has already been done by this point. /// The only failure condition is if some of the used buffers are destroyed. - pub(super) unsafe fn execute( + pub(super) unsafe fn execute( &self, raw: &mut A::CommandEncoder, pipeline_layout_guard: &Storage< @@ -814,7 +814,7 @@ impl RenderBundle { } } -impl Resource for RenderBundle { +impl Resource for RenderBundle { const TYPE: &'static str = "RenderBundle"; fn life_guard(&self) -> &LifeGuard { diff --git a/wgpu-core/src/command/clear.rs b/wgpu-core/src/command/clear.rs index 64261fabc9..61f51b07bf 100644 --- a/wgpu-core/src/command/clear.rs +++ b/wgpu-core/src/command/clear.rs @@ -10,7 +10,7 @@ use crate::{ id::{BufferId, CommandEncoderId, DeviceId, TextureId, Valid}, init_tracker::{MemoryInitKind, TextureInitRange}, resource::{Texture, TextureClearMode}, - track::{OldTextureState, ResourceTracker, TextureSelector}, + track::{OldTextureState, ResourceTracker, TextureSelector, TextureTracker}, }; use hal::{auxil::align_to, CommandEncoder as _}; @@ -236,7 +236,7 @@ pub(crate) fn clear_texture( dst_texture: &Texture, range: TextureInitRange, encoder: &mut A::CommandEncoder, - texture_tracker: &mut ResourceTracker, + texture_tracker: &mut TextureTracker, device: &Device, ) -> Result<(), ClearError> { clear_texture_no_device( @@ -255,7 +255,7 @@ pub(crate) fn clear_texture_no_device( dst_texture: &Texture, range: TextureInitRange, encoder: &mut A::CommandEncoder, - texture_tracker: &mut ResourceTracker, + texture_tracker: &mut TextureTracker, alignments: &hal::Alignments, zero_buffer: &A::Buffer, ) -> Result<(), ClearError> { diff --git a/wgpu-core/src/command/draw.rs b/wgpu-core/src/command/draw.rs index e8e53e3ade..60267b7723 100644 --- a/wgpu-core/src/command/draw.rs +++ b/wgpu-core/src/command/draw.rs @@ -5,7 +5,7 @@ use crate::{ binding_model::{LateMinBufferBindingSizeMismatch, PushConstantUploadError}, error::ErrorFormatter, id, - track::UseExtendError, + track::{UseExtendError, UsageConflict}, validation::{MissingBufferUsageError, MissingTextureUsageError}, }; use wgt::{BufferAddress, BufferSize, Color}; @@ -13,7 +13,7 @@ use wgt::{BufferAddress, BufferSize, Color}; use std::num::NonZeroU32; use thiserror::Error; -pub type BufferError = UseExtendError; +pub type BufferError = UsageConflict; /// Error validating a draw call. #[derive(Clone, Debug, Error, PartialEq)] @@ -79,8 +79,8 @@ pub enum RenderCommandError { IncompatiblePipelineTargets(#[from] crate::device::RenderPassCompatibilityError), #[error("pipeline writes to depth/stencil, while the pass has read-only depth/stencil")] IncompatiblePipelineRods, - #[error("buffer {0:?} is in error {1:?}")] - Buffer(id::BufferId, BufferError), + #[error(transparent)] + Buffer(#[from] BufferError), #[error("buffer {0:?} is destroyed")] DestroyedBuffer(id::BufferId), #[error(transparent)] diff --git a/wgpu-core/src/command/memory_init.rs b/wgpu-core/src/command/memory_init.rs index 134e86f083..674e987c07 100644 --- a/wgpu-core/src/command/memory_init.rs +++ b/wgpu-core/src/command/memory_init.rs @@ -8,7 +8,7 @@ use crate::{ id::{self, TextureId}, init_tracker::*, resource::{Buffer, Texture}, - track::{OldTextureState, ResourceTracker, TrackerSet}, + track::{OldTextureState, ResourceTracker, TrackerSet, TextureTracker}, FastHashMap, }; @@ -127,7 +127,7 @@ pub(crate) fn fixup_discarded_surfaces< inits: InitIter, encoder: &mut A::CommandEncoder, texture_guard: &Storage, TextureId>, - texture_tracker: &mut ResourceTracker, + texture_tracker: &mut TextureTracker, device: &Device, ) { for init in inits { diff --git a/wgpu-core/src/command/mod.rs b/wgpu-core/src/command/mod.rs index 57d06d4d61..b590c897e0 100644 --- a/wgpu-core/src/command/mod.rs +++ b/wgpu-core/src/command/mod.rs @@ -19,6 +19,7 @@ use self::memory_init::CommandBufferTextureMemoryActions; use crate::error::{ErrorFormatter, PrettyError}; use crate::init_tracker::BufferInitTrackerAction; +use crate::track::{Tracker, UsageScope}; use crate::{ hub::{Global, GlobalIdentityHandlerFactory, HalApi, Storage, Token}, id, @@ -96,7 +97,7 @@ pub struct CommandBuffer { encoder: CommandEncoder, status: CommandEncoderStatus, pub(crate) device_id: Stored, - pub(crate) trackers: TrackerSet, + pub(crate) trackers: Tracker, buffer_memory_init_actions: Vec, texture_memory_actions: CommandBufferTextureMemoryActions, limits: wgt::Limits, @@ -140,19 +141,21 @@ impl CommandBuffer { pub(crate) fn insert_barriers( raw: &mut A::CommandEncoder, - base: &mut TrackerSet, - head_buffers: &ResourceTracker, - head_textures: &ResourceTracker, + base: &mut Tracker, + head: &UsageScope, buffer_guard: &Storage, id::BufferId>, texture_guard: &Storage, id::TextureId>, ) { profiling::scope!("insert_barriers"); debug_assert_eq!(A::VARIANT, base.backend()); - let buffer_barriers = base.buffers.merge_replace(head_buffers).map(|pending| { - let buf = &buffer_guard[pending.id]; - pending.into_hal(buf) - }); + let buffer_barriers = base + .buffers + .change_states_scope(&*buffer_guard, &head.buffers) + .map(|pending| { + let buf = unsafe { &buffer_guard.get_unchecked(pending.id) }; + pending.into_hal(buf) + }); let texture_barriers = base.textures.merge_replace(head_textures).map(|pending| { let tex = &texture_guard[pending.id]; pending.into_hal(tex) diff --git a/wgpu-core/src/command/render.rs b/wgpu-core/src/command/render.rs index 07ae6c8a5d..1996cc565a 100644 --- a/wgpu-core/src/command/render.rs +++ b/wgpu-core/src/command/render.rs @@ -1,6 +1,7 @@ use crate::{ binding_model::BindError, command::{ + self, bind::Binder, end_pipeline_statistics_query, memory_init::{fixup_discarded_surfaces, SurfacesInDiscardState}, @@ -17,8 +18,8 @@ use crate::{ id, init_tracker::{MemoryInitKind, TextureInitRange, TextureInitTrackerAction}, pipeline::PipelineFlags, - resource::{Texture, TextureView}, - track::{StatefulTrackerSubset, TextureSelector, UsageConflict}, + resource::{self, Texture, TextureView}, + track::{TextureSelector, UsageConflict, UsageScope}, validation::{ check_buffer_usage, check_texture_usage, MissingBufferUsageError, MissingTextureUsageError, }, @@ -38,7 +39,6 @@ use serde::Deserialize; #[cfg(any(feature = "serial-pass", feature = "trace"))] use serde::Serialize; -use crate::track::UseExtendError; use std::{borrow::Cow, fmt, iter, marker::PhantomData, mem, num::NonZeroU32, ops::Range, str}; use super::{memory_init::TextureSurfaceDiscard, CommandBufferTextureMemoryActions}; @@ -563,7 +563,7 @@ type AttachmentDataVec = ArrayVec; struct RenderPassInfo<'a, A: hal::Api> { context: RenderPassContext, - trackers: StatefulTrackerSubset, + usage_scope: UsageScope, render_attachments: AttachmentDataVec>, // All render attachments, including depth/stencil is_ds_read_only: bool, extent: wgt::Extent3d, @@ -934,7 +934,7 @@ impl<'a, A: HalApi> RenderPassInfo<'a, A> { Ok(Self { context, - trackers: StatefulTrackerSubset::new(A::VARIANT), + usage_scope: UsageScope::new(), render_attachments, is_ds_read_only, extent, @@ -949,7 +949,7 @@ impl<'a, A: HalApi> RenderPassInfo<'a, A> { mut self, raw: &mut A::CommandEncoder, texture_guard: &Storage, id::TextureId>, - ) -> Result<(StatefulTrackerSubset, SurfacesInDiscardState), RenderPassErrorInner> { + ) -> Result<(UsageScope, SurfacesInDiscardState), RenderPassErrorInner> { profiling::scope!("finish", "RenderPassInfo"); unsafe { raw.end_render_pass(); @@ -963,7 +963,7 @@ impl<'a, A: HalApi> RenderPassInfo<'a, A> { check_texture_usage(texture.desc.usage, TextureUsages::RENDER_ATTACHMENT)?; // the tracker set of the pass is always in "extend" mode - self.trackers + self.usage_scope .textures .change_extend( ra.texture_id.value, @@ -1012,7 +1012,7 @@ impl<'a, A: HalApi> RenderPassInfo<'a, A> { } } - Ok((self.trackers, self.pending_discard_init_fixups)) + Ok((self.usage_scope, self.pending_discard_init_fixups)) } } @@ -1047,7 +1047,7 @@ impl Global { let mut token = Token::root(); let (device_guard, mut token) = hub.devices.read(&mut token); - let (trackers, query_reset_state, pending_discard_init_fixups) = { + let (scope, query_reset_state, pending_discard_init_fixups) = { let (mut cmb_guard, mut token) = hub.command_buffers.write(&mut token); let cmd_buf = CommandBuffer::get_encoder_mut(&mut *cmb_guard, encoder_id) @@ -1136,20 +1136,24 @@ impl Global { ); dynamic_offset_count += num_dynamic_offsets as usize; - let bind_group = cmd_buf - .trackers - .bind_groups - .use_extend(&*bind_group_guard, bind_group_id, (), ()) - .map_err(|_| RenderCommandError::InvalidBindGroup(bind_group_id)) - .map_pass_err(scope)?; + let bind_group = unsafe { + cmd_buf + .trackers + .bind_groups + .extend(&*bind_group_guard, bind_group_id) + }; bind_group .validate_dynamic_bindings(&temp_offsets, &cmd_buf.limits) .map_pass_err(scope)?; // merge the resource tracker in - info.trackers - .merge_extend(&bind_group.used) - .map_pass_err(scope)?; + unsafe { + info.usage_scope.extend_from_bind_group( + &*buffer_guard, + &*texture_guard, + &bind_group.used, + ) + } //Note: stateless trackers are not merged: the lifetime reference // is held to the bind group itself. @@ -1198,12 +1202,12 @@ impl Global { let scope = PassErrorScope::SetPipelineRender(pipeline_id); state.pipeline = Some(pipeline_id); - let pipeline = cmd_buf - .trackers - .render_pipes - .use_extend(&*pipeline_guard, pipeline_id, (), ()) - .map_err(|_| RenderCommandError::InvalidPipeline(pipeline_id)) - .map_pass_err(scope)?; + let pipeline = unsafe { + cmd_buf + .trackers + .render_pipelines + .extend(&*pipeline_guard, pipeline_id) + }; info.context .check_compatible(&pipeline.pass_context) @@ -1309,12 +1313,12 @@ impl Global { size, } => { let scope = PassErrorScope::SetIndexBuffer(buffer_id); - let buffer = info - .trackers - .buffers - .use_extend(&*buffer_guard, buffer_id, (), hal::BufferUses::INDEX) - .map_err(|e| RenderCommandError::Buffer(buffer_id, e)) - .map_pass_err(scope)?; + let buffer: &resource::Buffer = unsafe { + info.usage_scope + .buffers + .extend(&*buffer_guard, buffer_id, hal::BufferUses::INDEX) + .map_pass_err(scope)? + }; check_buffer_usage(buffer.usage, BufferUsages::INDEX) .map_pass_err(scope)?; let buf_raw = buffer @@ -1356,12 +1360,12 @@ impl Global { size, } => { let scope = PassErrorScope::SetVertexBuffer(buffer_id); - let buffer = info - .trackers - .buffers - .use_extend(&*buffer_guard, buffer_id, (), hal::BufferUses::VERTEX) - .map_err(|e| RenderCommandError::Buffer(buffer_id, e)) - .map_pass_err(scope)?; + let buffer: &resource::Buffer = unsafe { + info.usage_scope + .buffers + .extend(&*buffer_guard, buffer_id, hal::BufferUses::VERTEX) + .map_pass_err(scope)? + }; check_buffer_usage(buffer.usage, BufferUsages::VERTEX) .map_pass_err(scope)?; let buf_raw = buffer @@ -1614,12 +1618,12 @@ impl Global { .require_downlevel_flags(wgt::DownlevelFlags::INDIRECT_EXECUTION) .map_pass_err(scope)?; - let indirect_buffer = info - .trackers - .buffers - .use_extend(&*buffer_guard, buffer_id, (), hal::BufferUses::INDIRECT) - .map_err(|e| RenderCommandError::Buffer(buffer_id, e)) - .map_pass_err(scope)?; + let indirect_buffer: &resource::Buffer = unsafe { + info.usage_scope + .buffers + .extend(&*buffer_guard, buffer_id, hal::BufferUses::INDIRECT) + .map_pass_err(scope)? + }; check_buffer_usage(indirect_buffer.usage, BufferUsages::INDIRECT) .map_pass_err(scope)?; let indirect_raw = indirect_buffer @@ -1685,12 +1689,12 @@ impl Global { .require_downlevel_flags(wgt::DownlevelFlags::INDIRECT_EXECUTION) .map_pass_err(scope)?; - let indirect_buffer = info - .trackers - .buffers - .use_extend(&*buffer_guard, buffer_id, (), hal::BufferUses::INDIRECT) - .map_err(|e| RenderCommandError::Buffer(buffer_id, e)) - .map_pass_err(scope)?; + let indirect_buffer: &resource::Buffer = unsafe { + info.usage_scope + .buffers + .extend(&*buffer_guard, buffer_id, hal::BufferUses::INDIRECT) + .map_pass_err(scope)? + }; check_buffer_usage(indirect_buffer.usage, BufferUsages::INDIRECT) .map_pass_err(scope)?; let indirect_raw = indirect_buffer @@ -1699,17 +1703,12 @@ impl Global { .ok_or(RenderCommandError::DestroyedBuffer(buffer_id)) .map_pass_err(scope)?; - let count_buffer = info - .trackers - .buffers - .use_extend( - &*buffer_guard, - count_buffer_id, - (), - hal::BufferUses::INDIRECT, - ) - .map_err(|e| RenderCommandError::Buffer(count_buffer_id, e)) - .map_pass_err(scope)?; + let count_buffer: &resource::Buffer = unsafe { + info.usage_scope + .buffers + .extend(&*buffer_guard, count_buffer_id, hal::BufferUses::INDIRECT) + .map_pass_err(scope)? + }; check_buffer_usage(count_buffer.usage, BufferUsages::INDIRECT) .map_pass_err(scope)?; let count_raw = count_buffer @@ -1811,17 +1810,14 @@ impl Global { } => { let scope = PassErrorScope::WriteTimestamp; - let query_set = cmd_buf - .trackers - .query_sets - .use_extend(&*query_set_guard, query_set_id, (), ()) - .map_err(|e| match e { - UseExtendError::InvalidResource => { - RenderCommandError::InvalidQuerySet(query_set_id) - } - _ => unreachable!(), - }) - .map_pass_err(scope)?; + let query_set: &resource::QuerySet = unsafe { + cmd_buf + .trackers + .query_sets + .extend(&*query_set_guard, query_set_id) + .map_err(|_| RenderCommandError::InvalidQuerySet(query_set_id)) + .map_pass_err(scope)? + }; query_set .validate_and_write_timestamp( @@ -1838,17 +1834,14 @@ impl Global { } => { let scope = PassErrorScope::BeginPipelineStatisticsQuery; - let query_set = cmd_buf - .trackers - .query_sets - .use_extend(&*query_set_guard, query_set_id, (), ()) - .map_err(|e| match e { - UseExtendError::InvalidResource => { - RenderCommandError::InvalidQuerySet(query_set_id) - } - _ => unreachable!(), - }) - .map_pass_err(scope)?; + let query_set: &resource::QuerySet = unsafe { + cmd_buf + .trackers + .query_sets + .extend(&*query_set_guard, query_set_id) + .map_err(|_| RenderCommandError::InvalidQuerySet(query_set_id)) + .map_pass_err(scope)? + }; query_set .validate_and_begin_pipeline_statistics_query( @@ -1868,12 +1861,14 @@ impl Global { } RenderCommand::ExecuteBundle(bundle_id) => { let scope = PassErrorScope::ExecuteBundle; - let bundle = cmd_buf - .trackers - .bundles - .use_extend(&*bundle_guard, bundle_id, (), ()) - .map_err(|_| RenderCommandError::InvalidRenderBundle(bundle_id)) - .map_pass_err(scope)?; + let bundle: &command::RenderBundle = unsafe { + cmd_buf + .trackers + .bundles + .extend(&*bundle_guard, bundle_id) + .map_err(|_| RenderCommandError::InvalidRenderBundle(bundle_id)) + .map_pass_err(scope)? + }; info.context .check_compatible(&bundle.context) @@ -1924,9 +1919,15 @@ impl Global { }) .map_pass_err(scope)?; - info.trackers - .merge_extend(&bundle.used) - .map_pass_err(scope)?; + unsafe { + info.usage_scope + .extend_from_render_bundle( + &*buffer_guard, + &*texture_guard, + &bundle.used, + ) + .map_pass_err(scope)? + }; // Start tracking the bind groups specifically, as they are the only // compound resources, to make it easier to update submission indices // later at submission time. @@ -1977,8 +1978,7 @@ impl Global { super::CommandBuffer::insert_barriers( transit, &mut cmd_buf.trackers, - &trackers.buffers, - &trackers.textures, + &scope, &*buffer_guard, &*texture_guard, ); diff --git a/wgpu-core/src/hub.rs b/wgpu-core/src/hub.rs index 273c70b3bf..da5e20a248 100644 --- a/wgpu-core/src/hub.rs +++ b/wgpu-core/src/hub.rs @@ -191,11 +191,11 @@ impl Storage { result } - pub(crate) unsafe fn get_unchecked(&self, id: u32) -> Result<&T, InvalidId> { + pub(crate) unsafe fn get_unchecked(&self, id: u32) -> &T { match self.map[id as usize] { - Element::Occupied(ref v, _) => Ok(v), + Element::Occupied(ref v, e) => v, Element::Vacant => panic!("{}[{}] does not exist", self.kind, id), - Element::Error(_, _) => Err(InvalidId), + Element::Error(_, _) => panic!(""), } } @@ -627,7 +627,7 @@ pub struct Hub { pub bind_group_layouts: Registry, id::BindGroupLayoutId, F>, pub bind_groups: Registry, id::BindGroupId, F>, pub command_buffers: Registry, id::CommandBufferId, F>, - pub render_bundles: Registry, + pub render_bundles: Registry, id::RenderBundleId, F>, pub render_pipelines: Registry, id::RenderPipelineId, F>, pub compute_pipelines: Registry, id::ComputePipelineId, F>, pub query_sets: Registry, id::QuerySetId, F>, diff --git a/wgpu-core/src/track/buffer.rs b/wgpu-core/src/track/buffer.rs index 01f41d7121..8db353c376 100644 --- a/wgpu-core/src/track/buffer.rs +++ b/wgpu-core/src/track/buffer.rs @@ -5,7 +5,10 @@ use crate::{ hub, id::{BufferId, TypedId, Valid}, resource::Buffer, - track::{invalid_resource_state, resize_bitvec, skip_barrier, ResourceUses, UsageConflict}, + track::{ + invalid_resource_state, iterate_bitvec, resize_bitvec, skip_barrier, ResourceUses, + UsageConflict, + }, }; use bit_vec::BitVec; use hal::BufferUses; @@ -58,19 +61,48 @@ impl BufferUsageScope { bind_group: &BufferBindGroupState, ) -> Result<(), UsageConflict> { for &(id, state) in &bind_group.buffers { - self.extend(storage, id, state)?; + self.extend(storage, id.0, state)?; } Ok(()) } - pub unsafe fn extend( + pub unsafe fn extend_from_scope( &mut self, storage: &hub::Storage, BufferId>, - id: Valid, + scope: &Self, + ) -> Result<(), UsageConflict> { + let incoming_size = scope.state.len(); + if incoming_size > self.state.len() { + self.set_max_index(incoming_size); + } + + iterate_bitvec(&scope.owned, |index| { + unsafe { self.extend_inner(storage, index as u32, *scope.state.get_unchecked(index)) }; + }); + + Ok(()) + } + + pub unsafe fn extend<'a, A: hal::Api>( + &mut self, + storage: &'a hub::Storage, BufferId>, + id: BufferId, + new_state: BufferUses, + ) -> Result<&'a Buffer, UsageConflict> { + self.extend_inner(storage, id.unzip().0, new_state)?; + + Ok(storage + .get(id) + .map_err(|_| UsageConflict::BufferInvalid { id })?) + } + + unsafe fn extend_inner<'a, A: hal::Api>( + &mut self, + storage: &'a hub::Storage, BufferId>, + index: u32, new_state: BufferUses, ) -> Result<(), UsageConflict> { - let (index, _, _) = id.0.unzip(); let index = index as usize; let currently_active = self.owned.get(index).unwrap_unchecked(); @@ -80,7 +112,7 @@ impl BufferUsageScope { let merged_state = current_state | new_state; if invalid_resource_state(merged_state) { - return Err(UsageConflict::from_buffer(id, current_state, new_state)); + return Err(UsageConflict::from_buffer(index, current_state, new_state)); } *self.state.get_unchecked_mut(index) = merged_state; @@ -118,7 +150,15 @@ impl BufferTracker { resize_bitvec(&mut self.owned, size); } - pub fn change_states( + pub fn change_states_scope( + &mut self, + storage: &hub::Storage, BufferId>, + scope: &BufferUsageScope, + ) -> Drain> { + self.change_states_inner(storage, &scope.state, &scope.owned) + } + + fn change_states_inner( &mut self, storage: &hub::Storage, BufferId>, incoming_set: &Vec, @@ -129,23 +169,9 @@ impl BufferTracker { self.set_max_index(incoming_size); } - for (word_index, mut word) in incoming_ownership.blocks().enumerate() { - if word == 0 { - continue; - } - - let bit_start = word_index * 64; - let bit_end = ((word_index + 1) * 64).min(incoming_size); - - for index in bit_start..bit_end { - if word & 0b1 == 0 { - continue; - } - word >>= 1; - - unsafe { self.transition(storage, incoming_set, index) }; - } - } + iterate_bitvec(incoming_ownership, |index| { + unsafe { self.transition(storage, incoming_set, index) }; + }); self.temp.drain(..) } @@ -204,119 +230,3 @@ impl BufferTracker { } } } - -#[cfg(test)] -mod test { - use super::*; - use crate::id::Id; - - #[test] - fn change_extend() { - let mut bs = Unit { - first: None, - last: BufferUses::INDEX, - }; - let id = Id::dummy(); - assert_eq!( - bs.change(id, (), BufferUses::STORAGE_WRITE, None), - Err(PendingTransition { - id, - selector: (), - usage: BufferUses::INDEX..BufferUses::STORAGE_WRITE, - }), - ); - bs.change(id, (), BufferUses::VERTEX, None).unwrap(); - bs.change(id, (), BufferUses::INDEX, None).unwrap(); - assert_eq!(bs, Unit::new(BufferUses::VERTEX | BufferUses::INDEX)); - } - - #[test] - fn change_replace() { - let mut bs = Unit { - first: None, - last: BufferUses::STORAGE_WRITE, - }; - let id = Id::dummy(); - let mut list = Vec::new(); - bs.change(id, (), BufferUses::VERTEX, Some(&mut list)) - .unwrap(); - assert_eq!( - &list, - &[PendingTransition { - id, - selector: (), - usage: BufferUses::STORAGE_WRITE..BufferUses::VERTEX, - }], - ); - assert_eq!( - bs, - Unit { - first: Some(BufferUses::STORAGE_WRITE), - last: BufferUses::VERTEX, - } - ); - - list.clear(); - bs.change(id, (), BufferUses::STORAGE_WRITE, Some(&mut list)) - .unwrap(); - assert_eq!( - &list, - &[PendingTransition { - id, - selector: (), - usage: BufferUses::VERTEX..BufferUses::STORAGE_WRITE, - }], - ); - assert_eq!( - bs, - Unit { - first: Some(BufferUses::STORAGE_WRITE), - last: BufferUses::STORAGE_WRITE, - } - ); - } - - #[test] - fn merge_replace() { - let mut bs = Unit { - first: None, - last: BufferUses::empty(), - }; - let other_smooth = Unit { - first: Some(BufferUses::empty()), - last: BufferUses::COPY_DST, - }; - let id = Id::dummy(); - let mut list = Vec::new(); - bs.merge(id, &other_smooth, Some(&mut list)).unwrap(); - assert!(list.is_empty()); - assert_eq!( - bs, - Unit { - first: Some(BufferUses::empty()), - last: BufferUses::COPY_DST, - } - ); - - let other_rough = Unit { - first: Some(BufferUses::empty()), - last: BufferUses::UNIFORM, - }; - bs.merge(id, &other_rough, Some(&mut list)).unwrap(); - assert_eq!( - &list, - &[PendingTransition { - id, - selector: (), - usage: BufferUses::COPY_DST..BufferUses::empty(), - }], - ); - assert_eq!( - bs, - Unit { - first: Some(BufferUses::empty()), - last: BufferUses::UNIFORM, - } - ); - } -} diff --git a/wgpu-core/src/track/mod.rs b/wgpu-core/src/track/mod.rs index 7b7f0b5c15..6a1e57a808 100644 --- a/wgpu-core/src/track/mod.rs +++ b/wgpu-core/src/track/mod.rs @@ -1,20 +1,20 @@ mod buffer; mod range; +mod stateless; mod texture; use crate::{ - conv, hub, - id::{self, TypedId, Valid}, - resource, Epoch, FastHashMap, Index, RefCount, + binding_model, command, conv, hub, + id::{self, TypedId}, + pipeline, resource, }; use bit_vec::BitVec; -use std::{ - collections::hash_map::Entry, fmt, marker::PhantomData, num::NonZeroU32, ops, vec::Drain, -}; +use std::{fmt, num::NonZeroU32, ops}; use thiserror::Error; -pub(crate) use buffer::BufferState; +pub(crate) use buffer::{BufferBindGroupState, BufferTracker, BufferUsageScope}; +pub(crate) use stateless::{StatelessBindGroupSate, StatelessTracker}; pub(crate) use texture::{ TextureBindGroupState, TextureSelector, TextureTracker, TextureUsageScope, }; @@ -104,8 +104,33 @@ fn resize_bitvec(vec: &mut BitVec, size: usize) { } } +fn iterate_bitvec(ownership: &BitVec, mut func: impl FnMut(usize)) { + let size = ownership.len(); + for (word_index, mut word) in ownership.blocks().enumerate() { + if word == 0 { + continue; + } + + let bit_start = word_index * 64; + let bit_end = (bit_start + 64).min(size); + + for index in bit_start..bit_end { + if word & 0b1 == 0 { + continue; + } + word >>= 1; + + func(index); + } + } +} + #[derive(Clone, Debug, Error)] pub enum UsageConflict { + #[error("Attempted to use buffer {id} which is invalid.")] + BufferInvalid { id: u32 }, + #[error("Attempted to use texture {id} which is invalid.")] + TextureInvalid { id: u32 }, #[error("Attempted to use buffer {id:?} with {invalid_use}.")] Buffer { id: id::BufferId, @@ -120,13 +145,9 @@ pub enum UsageConflict { }, } impl UsageConflict { - fn from_buffer( - id: Valid, - current_state: hal::BufferUses, - new_state: hal::BufferUses, - ) -> Self { + fn from_buffer(id: u32, current_state: hal::BufferUses, new_state: hal::BufferUses) -> Self { Self::Buffer { - id: id.0, + id, invalid_use: InvalidUse { current_state, new_state, @@ -136,12 +157,12 @@ impl UsageConflict { fn from_texture( storage: &hub::Storage, id::TextureId>, - id: Valid, + id: u32, selector: Option, current_state: hal::TextureUses, new_state: hal::TextureUses, ) -> Self { - let texture = &storage[id]; + let texture = unsafe { storage.get_unchecked(id) }; let mips; let layers; @@ -194,109 +215,73 @@ impl fmt::Display for InvalidUse { } } -pub(crate) struct BindGroupStates { +pub(crate) struct BindGroupStates { + pub buffers: BufferBindGroupState, pub textures: TextureBindGroupState, + pub views: StatelessBindGroupSate, id::TextureViewId>, + pub samplers: StatelessBindGroupSate, id::SamplerId>, } -/// A set of trackers for all relevant resources. -/// -/// `Device` uses this to track all resources allocated from that device. -/// Resources like `BindGroup`, `CommandBuffer`, and so on that may own a -/// variety of other resources also use a value of this type to keep track of -/// everything they're depending on. -#[derive(Debug)] -pub(crate) struct TrackerSet { - pub buffers: ResourceTracker, - pub textures: ResourceTracker, - pub views: ResourceTracker>, - pub bind_groups: ResourceTracker>, - pub samplers: ResourceTracker>, - pub compute_pipes: ResourceTracker>, - pub render_pipes: ResourceTracker>, - pub bundles: ResourceTracker>, - pub query_sets: ResourceTracker>, +pub(crate) struct RenderBundleScope { + pub buffers: BufferUsageScope, + pub textures: TextureUsageScope, + pub views: StatelessTracker, id::TextureViewId>, + pub samplers: StatelessTracker, id::SamplerId>, + pub bind_groups: StatelessTracker, id::BindGroupId>, + pub render_pipelines: StatelessTracker, id::RenderPipelineId>, + pub query_sets: StatelessTracker, id::QuerySetId>, } -impl TrackerSet { - /// Create an empty set. - pub fn new(backend: wgt::Backend) -> Self { +pub(crate) struct UsageScope { + pub buffers: BufferUsageScope, + pub textures: TextureUsageScope, +} + +impl UsageScope { + pub fn new() -> Self { Self { - buffers: ResourceTracker::new(backend), - textures: ResourceTracker::new(backend), - views: ResourceTracker::new(backend), - bind_groups: ResourceTracker::new(backend), - samplers: ResourceTracker::new(backend), - compute_pipes: ResourceTracker::new(backend), - render_pipes: ResourceTracker::new(backend), - bundles: ResourceTracker::new(backend), - query_sets: ResourceTracker::new(backend), + buffers: BufferUsageScope::new(), + textures: TextureUsageScope::new(), } } - /// Clear all the trackers. - pub fn _clear(&mut self) { - self.buffers.clear(); - self.textures.clear(); - self.views.clear(); - self.bind_groups.clear(); - self.samplers.clear(); - self.compute_pipes.clear(); - self.render_pipes.clear(); - self.bundles.clear(); - self.query_sets.clear(); - } + pub unsafe fn extend_from_bind_group( + &mut self, + buffers: &hub::Storage, id::BufferId>, + textures: &hub::Storage, id::TextureId>, + bind_group: &BindGroupStates, + ) -> Result<(), UsageConflict> { + self.buffers + .extend_from_bind_group(buffers, &bind_group.buffers)?; + self.textures + .extend_from_bind_group(textures, &bind_group.textures)?; - /// Try to optimize the tracking representation. - pub fn optimize(&mut self) { - self.buffers.optimize(); - self.textures.optimize(); - self.views.optimize(); - self.bind_groups.optimize(); - self.samplers.optimize(); - self.compute_pipes.optimize(); - self.render_pipes.optimize(); - self.bundles.optimize(); - self.query_sets.optimize(); - } - - /// Merge only the stateful trackers of another instance by extending - /// the usage. Returns a conflict if any. - pub fn merge_extend_stateful(&mut self, other: &Self) -> Result<(), UsageConflict> { - self.buffers.merge_extend(&other.buffers)?; - self.textures.merge_extend(&other.textures)?; Ok(()) } - pub fn backend(&self) -> wgt::Backend { - self.buffers.backend - } -} - -#[derive(Debug)] -pub(crate) struct StatefulTrackerSubset { - pub buffers: ResourceTracker, - pub textures: ResourceTracker, -} - -impl StatefulTrackerSubset { - /// Create an empty set. - pub fn new(backend: wgt::Backend) -> Self { - Self { - buffers: ResourceTracker::new(backend), - textures: ResourceTracker::new(backend), - } - } - - /// Clear all the trackers. - pub fn clear(&mut self) { - self.buffers.clear(); - self.textures.clear(); - } + pub unsafe fn extend_from_render_bundle( + &mut self, + buffers: &hub::Storage, id::BufferId>, + textures: &hub::Storage, id::TextureId>, + render_bundle: &RenderBundleScope, + ) -> Result<(), UsageConflict> { + self.buffers + .extend_from_scope(buffers, &render_bundle.buffers)?; + self.textures + .extend_from_scope(textures, &render_bundle.textures)?; - /// Merge all the trackers of another tracker the usage. - pub fn merge_extend(&mut self, other: &TrackerSet) -> Result<(), UsageConflict> { - self.buffers.merge_extend(&other.buffers)?; - self.textures.merge_extend(&other.textures)?; Ok(()) } } + +pub(crate) struct Tracker { + pub buffers: BufferTracker, + pub textures: TextureTracker, + pub views: StatelessTracker, id::TextureViewId>, + pub samplers: StatelessTracker, id::SamplerId>, + pub bind_groups: StatelessTracker, id::BindGroupId>, + pub compute_pipelines: StatelessTracker, id::ComputePipelineId>, + pub render_pipelines: StatelessTracker, id::RenderPipelineId>, + pub bundles: StatelessTracker, id::RenderBundleId>, + pub query_sets: StatelessTracker, id::QuerySetId>, +} diff --git a/wgpu-core/src/track/stateless.rs b/wgpu-core/src/track/stateless.rs new file mode 100644 index 0000000000..468f427409 --- /dev/null +++ b/wgpu-core/src/track/stateless.rs @@ -0,0 +1,50 @@ +use std::marker::PhantomData; + +use bit_vec::BitVec; + +use crate::{ + hub, + id::{TypedId, Valid}, + track::resize_bitvec, +}; + +pub struct StatelessBindGroupSate { + resource: Vec>, + + _phantom: PhantomData, +} + +pub struct StatelessTracker { + owned: BitVec, + + _phantom: PhantomData<(T, Id)>, +} + +impl StatelessTracker { + pub fn new() -> Self { + Self { + owned: BitVec::default(), + _phantom: PhantomData, + } + } + + pub fn set_max_index(&mut self, size: usize) { + resize_bitvec(&mut self.owned, size); + } + + pub unsafe fn extend<'a>(&mut self, storage: &'a hub::Storage, id: Id) -> Option<&'a T> { + self.owned.set(id.unzip().0 as usize, true); + + storage.get(id).ok() + } + + pub unsafe fn extend_from_bind_group( + &mut self, + storage: &hub::Storage, + bind_group: &StatelessBindGroupSate, + ) { + for &id in &bind_group.resource { + self.owned.set(id.0.unzip().0 as usize, true); + } + } +} diff --git a/wgpu-core/src/track/texture.rs b/wgpu-core/src/track/texture.rs index 1332f8ee83..ebf4ad0cca 100644 --- a/wgpu-core/src/track/texture.rs +++ b/wgpu-core/src/track/texture.rs @@ -3,7 +3,10 @@ use crate::{ hub, id::{TextureId, TypedId, Valid}, resource::Texture, - track::{invalid_resource_state, resize_bitvec, skip_barrier, ResourceUses, UsageConflict}, + track::{ + invalid_resource_state, iterate_bitvec, resize_bitvec, skip_barrier, ResourceUses, + UsageConflict, + }, }; use bit_vec::BitVec; use hal::TextureUses; @@ -92,6 +95,23 @@ impl TextureUsageScope { resize_bitvec(&mut self.owned, size); } + pub fn extend_from_scope( + &mut self, + storage: &hub::Storage, TextureId>, + scope: &Self, + ) -> Drain> { + let incoming_size = scope.set.simple.len(); + if incoming_size > self.set.simple.len() { + self.set_max_index(incoming_size); + } + + iterate_bitvec(&scope.set.simple, |index| { + todo!() + }); + + self.temp.drain(..) + } + pub unsafe fn extend_from_bind_group( &mut self, storage: &hub::Storage, TextureId>, @@ -107,14 +127,30 @@ impl TextureUsageScope { /// # Safety /// /// `id` must be a valid ID and have an ID value less than the last call to set_max_index. - pub unsafe fn extend( + pub unsafe fn extend<'a, A: hal::Api>( + &mut self, + storage: &'a hub::Storage, TextureId>, + id: TextureId, + selector: Option, + new_state: TextureUses, + ) -> Result<&'a Texture, UsageConflict> { + let (index, _, _) = id.unzip(); + self.extend_inner(storage, index, selector, new_state)?; + Ok(storage + .get(id) + .map_err(|_| UsageConflict::TextureInvalid { id: index })?) + } + + /// # Safety + /// + /// `id` must be a valid ID and have an ID value less than the last call to set_max_index. + unsafe fn extend_inner( &mut self, storage: &hub::Storage, TextureId>, - id: Valid, + index: u32, selector: Option, new_state: TextureUses, ) -> Result<(), UsageConflict> { - let (index, _, _) = id.0.unzip(); let index = index as usize; let currently_active = self.owned.get(index).unwrap_unchecked(); @@ -128,7 +164,7 @@ impl TextureUsageScope { if invalid_resource_state(merged_state) { return Err(UsageConflict::from_texture( storage, - id, + index, selector, current_state, new_state, @@ -140,17 +176,19 @@ impl TextureUsageScope { return Ok(()); } // The old usage is complex. - (true, selector) => return self.extend_complex(storage, id, selector, new_state), + (true, selector) => { + return self.extend_complex(storage, index, selector, new_state) + } // The old usage is simple, so demote it to a complex one. (false, Some(selector)) => { *self.set.simple.get_unchecked_mut(index) = hal::TextureUses::COMPLEX; // Demote our simple state to a complex one. - self.extend_complex(storage, id, None, current_state)?; + self.extend_complex(storage, index, None, current_state)?; // Extend that complex state with our new complex state. - return self.extend_complex(storage, id, Some(selector), new_state); + return self.extend_complex(storage, index, Some(selector), new_state); } } } @@ -160,7 +198,7 @@ impl TextureUsageScope { if let Some(selector) = selector { *self.set.simple.get_unchecked_mut(index) = hal::TextureUses::COMPLEX; - self.extend_complex(storage, id, Some(selector), new_state)?; + self.extend_complex(storage, index, Some(selector), new_state)?; } else { *self.set.simple.get_unchecked_mut(index) = new_state; } @@ -169,16 +207,14 @@ impl TextureUsageScope { } #[cold] - fn extend_complex( + unsafe fn extend_complex( &mut self, storage: &hub::Storage, TextureId>, - id: Valid, + index: u32, selector: Option, new_state: TextureUses, ) -> Result<(), UsageConflict> { - let texture = &storage[id]; - - let (index, _, _) = id.0.unzip(); + let texture = storage.get_unchecked(index); // Create the complex entry for this texture. let complex = self.set.complex.entry(index).or_insert_with(|| { @@ -211,7 +247,7 @@ impl TextureUsageScope { if invalid_resource_state(merged) { return Err(UsageConflict::from_texture( storage, - id, + index, selector, *current_state, new_state, @@ -262,23 +298,9 @@ impl TextureTracker { self.set_max_index(incoming_size); } - for (word_index, mut word) in incoming_ownership.blocks().enumerate() { - if word == 0 { - continue; - } - - let bit_start = word_index * 64; - let bit_end = ((word_index + 1) * 64).min(incoming_size); - - for index in bit_start..bit_end { - if word & 0b1 == 0 { - continue; - } - word >>= 1; - - unsafe { self.transition(storage, incoming_set, index) }; - } - } + iterate_bitvec(incoming_ownership, |index| { + unsafe { self.transition(storage, incoming_set, index) }; + }); self.temp.drain(..) } @@ -493,197 +515,3 @@ impl TextureTracker { *self.end_set.simple.get_unchecked_mut(index) = TextureUses::COMPLEX; } } - -#[cfg(test)] -mod test { - //TODO: change() tests - use super::*; - use crate::id::Id; - - #[test] - fn query() { - let mut ts = OldTextureState::default(); - ts.mips.push(PlaneStates::empty()); - ts.mips.push(PlaneStates::from_slice(&[ - (1..3, Unit::new(TextureUses::RESOURCE)), - (3..5, Unit::new(TextureUses::RESOURCE)), - (5..6, Unit::new(TextureUses::STORAGE_READ)), - ])); - - assert_eq!( - ts.query(TextureSelector { - mips: 1..2, - layers: 2..5, - }), - // level 1 matches - Some(TextureUses::RESOURCE), - ); - assert_eq!( - ts.query(TextureSelector { - mips: 0..2, - layers: 2..5, - }), - // level 0 is empty, level 1 matches - Some(TextureUses::RESOURCE), - ); - assert_eq!( - ts.query(TextureSelector { - mips: 1..2, - layers: 1..5, - }), - // level 1 matches with gaps - Some(TextureUses::RESOURCE), - ); - assert_eq!( - ts.query(TextureSelector { - mips: 1..2, - layers: 4..6, - }), - // level 1 doesn't match - None, - ); - } - - #[test] - fn merge() { - let id = Id::dummy(); - let mut ts1 = OldTextureState::default(); - ts1.mips.push(PlaneStates::from_slice(&[( - 1..3, - Unit::new(TextureUses::RESOURCE), - )])); - let mut ts2 = OldTextureState::default(); - assert_eq!( - ts1.merge(id, &ts2, None), - Ok(()), - "failed to merge with an empty" - ); - - ts2.mips.push(PlaneStates::from_slice(&[( - 1..2, - Unit::new(TextureUses::COPY_SRC), - )])); - assert_eq!( - ts1.merge(Id::dummy(), &ts2, None), - Ok(()), - "failed to extend a compatible state" - ); - assert_eq!( - ts1.mips[0].query(&(1..2), |&v| v), - Some(Ok(Unit { - first: None, - last: TextureUses::RESOURCE | TextureUses::COPY_SRC, - })), - "wrong extension result" - ); - - ts2.mips[0] = PlaneStates::from_slice(&[(1..2, Unit::new(TextureUses::COPY_DST))]); - assert_eq!( - ts1.clone().merge(Id::dummy(), &ts2, None), - Err(PendingTransition { - id, - selector: TextureSelector { - mips: 0..1, - layers: 1..2, - }, - usage: TextureUses::RESOURCE | TextureUses::COPY_SRC..TextureUses::COPY_DST, - }), - "wrong error on extending with incompatible state" - ); - - let mut list = Vec::new(); - ts2.mips[0] = PlaneStates::from_slice(&[ - (1..2, Unit::new(TextureUses::COPY_DST)), - ( - 2..3, - Unit { - first: Some(TextureUses::COPY_SRC), - last: TextureUses::COLOR_TARGET, - }, - ), - ]); - ts1.merge(Id::dummy(), &ts2, Some(&mut list)).unwrap(); - assert_eq!( - &list, - &[ - PendingTransition { - id, - selector: TextureSelector { - mips: 0..1, - layers: 1..2, - }, - usage: TextureUses::RESOURCE | TextureUses::COPY_SRC..TextureUses::COPY_DST, - }, - PendingTransition { - id, - selector: TextureSelector { - mips: 0..1, - layers: 2..3, - }, - // the transition links the end of the base rage (..SAMPLED) - // with the start of the next range (COPY_SRC..) - usage: TextureUses::RESOURCE..TextureUses::COPY_SRC, - }, - ], - "replacing produced wrong transitions" - ); - assert_eq!( - ts1.mips[0].query(&(1..2), |&v| v), - Some(Ok(Unit { - first: Some(TextureUses::RESOURCE | TextureUses::COPY_SRC), - last: TextureUses::COPY_DST, - })), - "wrong final layer 1 state" - ); - assert_eq!( - ts1.mips[0].query(&(2..3), |&v| v), - Some(Ok(Unit { - first: Some(TextureUses::RESOURCE), - last: TextureUses::COLOR_TARGET, - })), - "wrong final layer 2 state" - ); - - list.clear(); - ts2.mips[0] = PlaneStates::from_slice(&[( - 2..3, - Unit { - first: Some(TextureUses::COLOR_TARGET), - last: TextureUses::COPY_SRC, - }, - )]); - ts1.merge(Id::dummy(), &ts2, Some(&mut list)).unwrap(); - assert_eq!(&list, &[], "unexpected replacing transition"); - - list.clear(); - ts2.mips[0] = PlaneStates::from_slice(&[( - 2..3, - Unit { - first: Some(TextureUses::COPY_DST), - last: TextureUses::COPY_DST, - }, - )]); - ts1.merge(Id::dummy(), &ts2, Some(&mut list)).unwrap(); - assert_eq!( - &list, - &[PendingTransition { - id, - selector: TextureSelector { - mips: 0..1, - layers: 2..3, - }, - usage: TextureUses::COPY_SRC..TextureUses::COPY_DST, - },], - "invalid replacing transition" - ); - assert_eq!( - ts1.mips[0].query(&(2..3), |&v| v), - Some(Ok(Unit { - // the initial state here is never expected to change - first: Some(TextureUses::RESOURCE), - last: TextureUses::COPY_DST, - })), - "wrong final layer 2 state" - ); - } -} From 97e565359ef5edc7bba64f238823a5cc00a75811 Mon Sep 17 00:00:00 2001 From: Connor Fitzgerald Date: Tue, 3 May 2022 01:37:27 -0400 Subject: [PATCH 05/35] temp4 - continue muddling along --- wgpu-core/src/command/clear.rs | 2 +- wgpu-core/src/command/draw.rs | 2 +- wgpu-core/src/command/mod.rs | 16 ++++++++------ wgpu-core/src/command/query.rs | 31 ++++++++++++-------------- wgpu-core/src/command/render.rs | 23 +++++++++++-------- wgpu-core/src/resource.rs | 25 +-------------------- wgpu-core/src/track/mod.rs | 38 ++++++++++++++++++++++++++++++++ wgpu-core/src/track/stateless.rs | 11 +++++++++ wgpu-core/src/track/texture.rs | 10 ++++++++- 9 files changed, 98 insertions(+), 60 deletions(-) diff --git a/wgpu-core/src/command/clear.rs b/wgpu-core/src/command/clear.rs index 61f51b07bf..8beb873a6d 100644 --- a/wgpu-core/src/command/clear.rs +++ b/wgpu-core/src/command/clear.rs @@ -10,7 +10,7 @@ use crate::{ id::{BufferId, CommandEncoderId, DeviceId, TextureId, Valid}, init_tracker::{MemoryInitKind, TextureInitRange}, resource::{Texture, TextureClearMode}, - track::{OldTextureState, ResourceTracker, TextureSelector, TextureTracker}, + track::{TextureSelector, TextureTracker}, }; use hal::{auxil::align_to, CommandEncoder as _}; diff --git a/wgpu-core/src/command/draw.rs b/wgpu-core/src/command/draw.rs index 60267b7723..653b4df459 100644 --- a/wgpu-core/src/command/draw.rs +++ b/wgpu-core/src/command/draw.rs @@ -5,7 +5,7 @@ use crate::{ binding_model::{LateMinBufferBindingSizeMismatch, PushConstantUploadError}, error::ErrorFormatter, id, - track::{UseExtendError, UsageConflict}, + track::{UsageConflict}, validation::{MissingBufferUsageError, MissingTextureUsageError}, }; use wgt::{BufferAddress, BufferSize, Color}; diff --git a/wgpu-core/src/command/mod.rs b/wgpu-core/src/command/mod.rs index b590c897e0..9915b7dbac 100644 --- a/wgpu-core/src/command/mod.rs +++ b/wgpu-core/src/command/mod.rs @@ -24,7 +24,6 @@ use crate::{ hub::{Global, GlobalIdentityHandlerFactory, HalApi, Storage, Token}, id, resource::{Buffer, Texture}, - track::{BufferState, ResourceTracker, TrackerSet}, Label, Stored, }; @@ -85,7 +84,7 @@ impl CommandEncoder { pub struct BakedCommands { pub(crate) encoder: A::CommandEncoder, pub(crate) list: Vec, - pub(crate) trackers: TrackerSet, + pub(crate) trackers: Tracker, buffer_memory_init_actions: Vec, texture_memory_actions: CommandBufferTextureMemoryActions, } @@ -125,7 +124,7 @@ impl CommandBuffer { }, status: CommandEncoderStatus::Recording, device_id, - trackers: TrackerSet::new(A::VARIANT), + trackers: Tracker::new(), buffer_memory_init_actions: Default::default(), texture_memory_actions: Default::default(), limits, @@ -156,10 +155,13 @@ impl CommandBuffer { let buf = unsafe { &buffer_guard.get_unchecked(pending.id) }; pending.into_hal(buf) }); - let texture_barriers = base.textures.merge_replace(head_textures).map(|pending| { - let tex = &texture_guard[pending.id]; - pending.into_hal(tex) - }); + let texture_barriers = base + .textures + .change_states_scope(&*texture_guard, &head.textures) + .map(|pending| { + let tex = unsafe { texture_guard.get_unchecked(pending.id) }; + pending.into_hal(tex) + }); unsafe { raw.transition_buffers(buffer_barriers); diff --git a/wgpu-core/src/command/query.rs b/wgpu-core/src/command/query.rs index 42a0dd905b..b67a4092ae 100644 --- a/wgpu-core/src/command/query.rs +++ b/wgpu-core/src/command/query.rs @@ -8,7 +8,6 @@ use crate::{ id::{self, Id, TypedId}, init_tracker::MemoryInitKind, resource::QuerySet, - track::UseExtendError, Epoch, FastHashMap, Index, }; use std::{iter, marker::PhantomData}; @@ -297,14 +296,13 @@ impl Global { }); } - let query_set = cmd_buf - .trackers - .query_sets - .use_extend(&*query_set_guard, query_set_id, (), ()) - .map_err(|e| match e { - UseExtendError::InvalidResource => QueryError::InvalidQuerySet(query_set_id), - _ => unreachable!(), - })?; + let query_set = unsafe { + cmd_buf + .trackers + .query_sets + .extend(&*query_set_guard, query_set_id) + .map_err(|_| QueryError::InvalidQuerySet(query_set_id))? + }; query_set.validate_and_write_timestamp(raw_encoder, query_set_id, query_index, None)?; @@ -345,14 +343,13 @@ impl Global { return Err(QueryError::Resolve(ResolveError::BufferOffsetAlignment)); } - let query_set = cmd_buf - .trackers - .query_sets - .use_extend(&*query_set_guard, query_set_id, (), ()) - .map_err(|e| match e { - UseExtendError::InvalidResource => QueryError::InvalidQuerySet(query_set_id), - _ => unreachable!(), - })?; + let query_set = unsafe { + cmd_buf + .trackers + .query_sets + .extend(&*query_set_guard, query_set_id) + .map_err(|_| QueryError::InvalidQuerySet(query_set_id))? + }; let (dst_buffer, dst_pending) = cmd_buf .trackers diff --git a/wgpu-core/src/command/render.rs b/wgpu-core/src/command/render.rs index 1996cc565a..b2a157b826 100644 --- a/wgpu-core/src/command/render.rs +++ b/wgpu-core/src/command/render.rs @@ -1072,10 +1072,12 @@ impl Global { let (bundle_guard, mut token) = hub.render_bundles.read(&mut token); let (pipeline_layout_guard, mut token) = hub.pipeline_layouts.read(&mut token); let (bind_group_guard, mut token) = hub.bind_groups.read(&mut token); + let (render_pipe_guard, mut token) = hub.render_pipelines.read(&mut token); let (pipeline_guard, mut token) = hub.render_pipelines.read(&mut token); let (query_set_guard, mut token) = hub.query_sets.read(&mut token); let (buffer_guard, mut token) = hub.buffers.read(&mut token); let (texture_guard, mut token) = hub.textures.read(&mut token); + let (sampler_guard, mut token) = hub.samplers.read(&mut token); let (view_guard, _) = hub.texture_views.read(&mut token); log::trace!( @@ -1926,16 +1928,19 @@ impl Global { &*texture_guard, &bundle.used, ) - .map_pass_err(scope)? + .map_pass_err(scope)?; + cmd_buf + .trackers + .extend_from_render_bundle( + &*view_guard, + &*sampler_guard, + &*bind_group_guard, + &*render_pipe_guard, + &*query_set_guard, + &bundle.used, + ) + .map_pass_err(scope)?; }; - // Start tracking the bind groups specifically, as they are the only - // compound resources, to make it easier to update submission indices - // later at submission time. - cmd_buf - .trackers - .bind_groups - .merge_extend(&bundle.used.bind_groups) - .unwrap(); state.reset_bundle(); } } diff --git a/wgpu-core/src/resource.rs b/wgpu-core/src/resource.rs index 3f12aba0d5..0ef751cb5b 100644 --- a/wgpu-core/src/resource.rs +++ b/wgpu-core/src/resource.rs @@ -3,7 +3,7 @@ use crate::{ hub::{Global, GlobalIdentityHandlerFactory, HalApi, Resource, Token}, id::{DeviceId, SurfaceId, TextureId, Valid}, init_tracker::{BufferInitTracker, TextureInitTracker}, - track::{TextureSelector, DUMMY_SELECTOR}, + track::{TextureSelector}, validation::MissingBufferUsageError, Label, LifeGuard, RefCount, Stored, }; @@ -149,12 +149,6 @@ impl Resource for Buffer { } } -impl Borrow<()> for Buffer { - fn borrow(&self) -> &() { - &DUMMY_SELECTOR - } -} - pub type TextureDescriptor<'a> = wgt::TextureDescriptor>; #[derive(Debug)] @@ -428,12 +422,6 @@ impl Resource for TextureView { } } -impl Borrow<()> for TextureView { - fn borrow(&self) -> &() { - &DUMMY_SELECTOR - } -} - /// Describes a [`Sampler`] #[derive(Clone, Debug, PartialEq)] #[cfg_attr(feature = "trace", derive(serde::Serialize))] @@ -510,11 +498,6 @@ impl Resource for Sampler { } } -impl Borrow<()> for Sampler { - fn borrow(&self) -> &() { - &DUMMY_SELECTOR - } -} #[derive(Clone, Debug, Error)] pub enum CreateQuerySetError { #[error(transparent)] @@ -545,12 +528,6 @@ impl Resource for QuerySet { } } -impl Borrow<()> for QuerySet { - fn borrow(&self) -> &() { - &DUMMY_SELECTOR - } -} - #[derive(Clone, Debug, Error)] pub enum DestroyError { #[error("resource is invalid")] diff --git a/wgpu-core/src/track/mod.rs b/wgpu-core/src/track/mod.rs index 6a1e57a808..be41143ef3 100644 --- a/wgpu-core/src/track/mod.rs +++ b/wgpu-core/src/track/mod.rs @@ -285,3 +285,41 @@ pub(crate) struct Tracker { pub bundles: StatelessTracker, id::RenderBundleId>, pub query_sets: StatelessTracker, id::QuerySetId>, } + +impl Tracker { + pub fn new() -> Self { + Self { + buffers: BufferTracker::new(), + textures: TextureTracker::new(), + views: StatelessTracker::new(), + samplers: StatelessTracker::new(), + bind_groups: StatelessTracker::new(), + compute_pipelines: StatelessTracker::new(), + render_pipelines: StatelessTracker::new(), + bundles: StatelessTracker::new(), + query_sets: StatelessTracker::new(), + } + } + + pub unsafe fn extend_from_render_bundle( + &mut self, + views: hub::Storage, id::TextureViewId>, + samplers: hub::Storage, id::SamplerId>, + bind_groups: hub::Storage, id::BindGroupId>, + render_pipelines: hub::Storage, id::RenderPipelineId>, + query_sets: hub::Storage, id::QuerySetId>, + render_bundle: &RenderBundleScope, + ) -> Result<(), UsageConflict> { + self.views.extend_from_tracker(&views, &render_bundle.views); + self.samplers + .extend_from_tracker(&samplers, &render_bundle.samplers); + self.bind_groups + .extend_from_tracker(&bind_groups, &render_bundle.bind_groups); + self.render_pipelines + .extend_from_tracker(&render_pipelines, &render_bundle.render_pipelines); + self.query_sets + .extend_from_tracker(&query_sets, &render_bundle.query_sets); + + Ok(()) + } +} diff --git a/wgpu-core/src/track/stateless.rs b/wgpu-core/src/track/stateless.rs index 468f427409..1edbda884c 100644 --- a/wgpu-core/src/track/stateless.rs +++ b/wgpu-core/src/track/stateless.rs @@ -47,4 +47,15 @@ impl StatelessTracker { self.owned.set(id.0.unzip().0 as usize, true); } } + + pub fn extend_from_tracker(&mut self, storage: &hub::Storage, other: &Self) { + let incoming_size = other.owned.len(); + if incoming_size > self.owned.len() { + self.set_max_index(incoming_size); + } + + for (left, &right) in unsafe { self.owned.storage_mut().iter_mut() }.zip(other.owned.storage()) { + *left |= right; + } + } } diff --git a/wgpu-core/src/track/texture.rs b/wgpu-core/src/track/texture.rs index ebf4ad0cca..affa312a90 100644 --- a/wgpu-core/src/track/texture.rs +++ b/wgpu-core/src/track/texture.rs @@ -287,7 +287,15 @@ impl TextureTracker { resize_bitvec(&mut self.owned, size); } - pub fn change_states( + pub fn change_states_scope( + &mut self, + storage: &hub::Storage, TextureId>, + scope: &TextureUsageScope, + ) -> Drain> { + self.change_states_inner(storage, &scope.set, &scope.owned) + } + + fn change_states_inner( &mut self, storage: &hub::Storage, TextureId>, incoming_set: &TextureStateSet, From 26c3c7bdea71fd4ec51cdf9de34029b2c8fe0d5a Mon Sep 17 00:00:00 2001 From: Connor Fitzgerald Date: Tue, 3 May 2022 02:21:59 -0400 Subject: [PATCH 06/35] temp5 - doodle doodle --- wgpu-core/src/command/bundle.rs | 78 ++++++++++++++++------------ wgpu-core/src/command/compute.rs | 2 +- wgpu-core/src/command/memory_init.rs | 2 +- wgpu-core/src/device/life.rs | 1 - wgpu-core/src/device/mod.rs | 2 +- wgpu-core/src/hub.rs | 6 +-- wgpu-core/src/id.rs | 2 +- wgpu-core/src/track/mod.rs | 14 +++++ 8 files changed, 65 insertions(+), 42 deletions(-) diff --git a/wgpu-core/src/command/bundle.rs b/wgpu-core/src/command/bundle.rs index 181799f102..8138d526d8 100644 --- a/wgpu-core/src/command/bundle.rs +++ b/wgpu-core/src/command/bundle.rs @@ -45,11 +45,12 @@ use crate::{ SHADER_STAGE_COUNT, }, error::{ErrorFormatter, PrettyError}, - hub::{GlobalIdentityHandlerFactory, HalApi, Hub, Resource, Storage, Token}, + hub::{GlobalIdentityHandlerFactory, Hub, Resource, Storage, Token}, id, init_tracker::{BufferInitTrackerAction, MemoryInitKind, TextureInitTrackerAction}, - pipeline::PipelineFlags, - track::{TrackerSet, UsageConflict, UsageScope, RenderBundleScope}, + pipeline::{PipelineFlags, self}, + resource, + track::{RenderBundleScope, UsageConflict}, validation::check_buffer_usage, Label, LabelHelpers, LifeGuard, Stored, }; @@ -173,14 +174,14 @@ impl RenderBundleEncoder { device: &Device, hub: &Hub, token: &mut Token>, - ) -> Result { + ) -> Result, RenderBundleError> { let (pipeline_layout_guard, mut token) = hub.pipeline_layouts.read(token); let (bind_group_guard, mut token) = hub.bind_groups.read(&mut token); let (pipeline_guard, mut token) = hub.render_pipelines.read(&mut token); let (buffer_guard, _) = hub.buffers.read(&mut token); let mut state = State { - trackers: TrackerSet::new(self.parent_id.backend()), + trackers: RenderBundleScope::new(), index: IndexState::new(), vertex: (0..hal::MAX_VERTEX_BUFFERS) .map(|_| VertexState::new()) @@ -267,11 +268,13 @@ impl RenderBundleEncoder { state.pipeline = Some(pipeline_id); - let pipeline = state - .trackers - .render_pipes - .use_extend(&*pipeline_guard, pipeline_id, (), ()) - .unwrap(); + let pipeline: &pipeline::RenderPipeline = unsafe { + state + .trackers + .render_pipelines + .extend(&*pipeline_guard, pipeline_id) + .unwrap() + }; self.context .check_compatible(&pipeline.pass_context) @@ -306,11 +309,13 @@ impl RenderBundleEncoder { size, } => { let scope = PassErrorScope::SetIndexBuffer(buffer_id); - let buffer = state - .trackers - .buffers - .use_extend(&*buffer_guard, buffer_id, (), hal::BufferUses::INDEX) - .unwrap(); + let buffer: &resource::Buffer = unsafe { + state + .trackers + .buffers + .extend(&*buffer_guard, buffer_id, hal::BufferUses::INDEX) + .map_pass_err(scope)? + }; check_buffer_usage(buffer.usage, wgt::BufferUsages::INDEX) .map_pass_err(scope)?; @@ -333,11 +338,13 @@ impl RenderBundleEncoder { size, } => { let scope = PassErrorScope::SetVertexBuffer(buffer_id); - let buffer = state - .trackers - .buffers - .use_extend(&*buffer_guard, buffer_id, (), hal::BufferUses::VERTEX) - .unwrap(); + let buffer: &resource::Buffer = unsafe { + state + .trackers + .buffers + .extend(&*buffer_guard, buffer_id, hal::BufferUses::VERTEX) + .map_pass_err(scope)? + }; check_buffer_usage(buffer.usage, wgt::BufferUsages::VERTEX) .map_pass_err(scope)?; @@ -458,11 +465,13 @@ impl RenderBundleEncoder { .require_downlevel_flags(wgt::DownlevelFlags::INDIRECT_EXECUTION) .map_pass_err(scope)?; - let buffer = state - .trackers - .buffers - .use_extend(&*buffer_guard, buffer_id, (), hal::BufferUses::INDIRECT) - .unwrap(); + let buffer: &resource::Buffer = unsafe { + state + .trackers + .buffers + .extend(&*buffer_guard, buffer_id, hal::BufferUses::INDIRECT) + .map_pass_err(scope)? + }; check_buffer_usage(buffer.usage, wgt::BufferUsages::INDIRECT) .map_pass_err(scope)?; @@ -491,12 +500,13 @@ impl RenderBundleEncoder { .require_downlevel_flags(wgt::DownlevelFlags::INDIRECT_EXECUTION) .map_pass_err(scope)?; - let buffer = state - .trackers - .buffers - .use_extend(&*buffer_guard, buffer_id, (), hal::BufferUses::INDIRECT) - .map_err(|err| RenderCommandError::Buffer(buffer_id, err)) - .map_pass_err(scope)?; + let buffer: &resource::Buffer = unsafe { + state + .trackers + .buffers + .extend(&*buffer_guard, buffer_id, hal::BufferUses::INDIRECT) + .map_pass_err(scope)? + }; check_buffer_usage(buffer.usage, wgt::BufferUsages::INDIRECT) .map_pass_err(scope)?; @@ -992,8 +1002,8 @@ struct VertexLimitState { } #[derive(Debug)] -struct State { - trackers: TrackerSet, +struct State { + trackers: RenderBundleScope, index: IndexState, vertex: ArrayVec, bind: ArrayVec, @@ -1004,7 +1014,7 @@ struct State { pipeline: Option, } -impl State { +impl State { fn vertex_limits(&self) -> VertexLimitState { let mut vert_state = VertexLimitState { vertex_limit: u32::MAX, diff --git a/wgpu-core/src/command/compute.rs b/wgpu-core/src/command/compute.rs index 9881e797d1..e5f672d77b 100644 --- a/wgpu-core/src/command/compute.rs +++ b/wgpu-core/src/command/compute.rs @@ -15,7 +15,7 @@ use crate::{ id, init_tracker::MemoryInitKind, resource::{Buffer, Texture}, - track::{StatefulTrackerSubset, TrackerSet, UsageConflict, UseExtendError}, + track::UsageConflict, validation::{check_buffer_usage, MissingBufferUsageError}, Label, }; diff --git a/wgpu-core/src/command/memory_init.rs b/wgpu-core/src/command/memory_init.rs index 674e987c07..acd1e387e4 100644 --- a/wgpu-core/src/command/memory_init.rs +++ b/wgpu-core/src/command/memory_init.rs @@ -8,7 +8,7 @@ use crate::{ id::{self, TextureId}, init_tracker::*, resource::{Buffer, Texture}, - track::{OldTextureState, ResourceTracker, TrackerSet, TextureTracker}, + track::{TextureTracker}, FastHashMap, }; diff --git a/wgpu-core/src/device/life.rs b/wgpu-core/src/device/life.rs index 9d0c8d96e3..cc9c6b5045 100644 --- a/wgpu-core/src/device/life.rs +++ b/wgpu-core/src/device/life.rs @@ -7,7 +7,6 @@ use crate::{ }, hub::{GlobalIdentityHandlerFactory, HalApi, Hub, Token}, id, resource, - track::TrackerSet, RefCount, Stored, SubmissionIndex, }; use smallvec::SmallVec; diff --git a/wgpu-core/src/device/mod.rs b/wgpu-core/src/device/mod.rs index d7af1f5b5a..f5f10aadd0 100644 --- a/wgpu-core/src/device/mod.rs +++ b/wgpu-core/src/device/mod.rs @@ -8,7 +8,7 @@ use crate::{ TextureInitTracker, TextureInitTrackerAction, }, instance, pipeline, present, resource, - track::{BufferState, OldTextureState, TextureSelector, TrackerSet, UsageConflict}, + track::{TextureSelector, UsageConflict}, validation::{self, check_buffer_usage, check_texture_usage}, FastHashMap, Label, LabelHelpers as _, LifeGuard, MultiRefCount, RefCount, Stored, SubmissionIndex, DOWNLEVEL_ERROR_MESSAGE, diff --git a/wgpu-core/src/hub.rs b/wgpu-core/src/hub.rs index da5e20a248..c4bf14cda7 100644 --- a/wgpu-core/src/hub.rs +++ b/wgpu-core/src/hub.rs @@ -310,7 +310,7 @@ impl Access> for Surface {} impl Access> for Adapter {} impl Access> for Root {} impl Access> for Device {} -impl Access> for RenderBundle {} +impl Access> for RenderBundle {} impl Access> for Root {} impl Access> for Device {} impl Access> for PipelineLayout {} @@ -321,8 +321,8 @@ impl Access> for PipelineLayout {} impl Access> for CommandBuffer {} impl Access> for Root {} impl Access> for Device {} -impl Access for Device {} -impl Access for CommandBuffer {} +impl Access> for Device {} +impl Access> for CommandBuffer {} impl Access> for Device {} impl Access> for BindGroup {} impl Access> for Device {} diff --git a/wgpu-core/src/id.rs b/wgpu-core/src/id.rs index cbd7669e02..b36ef9637d 100644 --- a/wgpu-core/src/id.rs +++ b/wgpu-core/src/id.rs @@ -184,7 +184,7 @@ pub type CommandBufferId = Id>; pub type RenderPassEncoderId = *mut crate::command::RenderPass; pub type ComputePassEncoderId = *mut crate::command::ComputePass; pub type RenderBundleEncoderId = *mut crate::command::RenderBundleEncoder; -pub type RenderBundleId = Id; +pub type RenderBundleId = Id>; pub type QuerySetId = Id>; #[test] diff --git a/wgpu-core/src/track/mod.rs b/wgpu-core/src/track/mod.rs index be41143ef3..4b75fd5855 100644 --- a/wgpu-core/src/track/mod.rs +++ b/wgpu-core/src/track/mod.rs @@ -232,6 +232,20 @@ pub(crate) struct RenderBundleScope { pub query_sets: StatelessTracker, id::QuerySetId>, } +impl RenderBundleScope { + pub fn new() -> Self { + Self { + buffers: BufferTracker::new(), + textures: TextureTracker::new(), + views: StatelessTracker::new(), + samplers: StatelessTracker::new(), + bind_groups: StatelessTracker::new(), + render_pipelines: StatelessTracker::new(), + query_sets: StatelessTracker::new(), + } + } +} + pub(crate) struct UsageScope { pub buffers: BufferUsageScope, pub textures: TextureUsageScope, From c0deee91204298ab01684d4ed35cd76a4e3abfb5 Mon Sep 17 00:00:00 2001 From: Connor Fitzgerald Date: Tue, 3 May 2022 20:18:11 -0400 Subject: [PATCH 07/35] temp6 - continuous progress is continuous --- wgpu-core/src/binding_model.rs | 11 +- wgpu-core/src/command/bundle.rs | 39 +++--- wgpu-core/src/command/clear.rs | 33 +++-- wgpu-core/src/command/compute.rs | 128 ++++++++++--------- wgpu-core/src/command/draw.rs | 11 +- wgpu-core/src/command/memory_init.rs | 27 ++-- wgpu-core/src/command/mod.rs | 43 ++++--- wgpu-core/src/command/query.rs | 18 +-- wgpu-core/src/command/render.rs | 87 +++++++------ wgpu-core/src/command/transfer.rs | 184 ++++++++++++++------------- wgpu-core/src/device/life.rs | 3 +- wgpu-core/src/device/mod.rs | 55 ++++---- wgpu-core/src/device/queue.rs | 34 +++-- wgpu-core/src/resource.rs | 2 +- wgpu-core/src/track/buffer.rs | 84 +++++++++--- wgpu-core/src/track/mod.rs | 69 +++++++--- wgpu-core/src/track/stateless.rs | 29 +++-- wgpu-core/src/track/texture.rs | 96 +++++++++----- 18 files changed, 572 insertions(+), 381 deletions(-) diff --git a/wgpu-core/src/binding_model.rs b/wgpu-core/src/binding_model.rs index e91df3308d..2c8e695bc3 100644 --- a/wgpu-core/src/binding_model.rs +++ b/wgpu-core/src/binding_model.rs @@ -2,9 +2,9 @@ use crate::{ device::{DeviceError, MissingDownlevelFlags, MissingFeatures, SHADER_STAGE_COUNT}, error::{ErrorFormatter, PrettyError}, hub::Resource, - id::{BindGroupLayoutId, BufferId, DeviceId, SamplerId, TextureViewId, Valid}, + id::{BindGroupLayoutId, BufferId, DeviceId, SamplerId, TextureId, TextureViewId, Valid}, init_tracker::{BufferInitTrackerAction, TextureInitTrackerAction}, - track::{UsageConflict, BindGroupStates}, + track::{BindGroupStates, UsageConflict}, validation::{MissingBufferUsageError, MissingTextureUsageError}, FastHashMap, Label, LifeGuard, MultiRefCount, Stored, }; @@ -16,10 +16,7 @@ use serde::Deserialize; #[cfg(feature = "trace")] use serde::Serialize; -use std::{ - borrow::{Cow}, - ops::Range, -}; +use std::{borrow::Cow, ops::Range}; use thiserror::Error; @@ -63,6 +60,8 @@ pub enum CreateBindGroupError { InvalidBuffer(BufferId), #[error("texture view {0:?} is invalid")] InvalidTextureView(TextureViewId), + #[error("texture {0:?} is invalid")] + InvalidTexture(TextureId), #[error("sampler {0:?} is invalid")] InvalidSampler(SamplerId), #[error( diff --git a/wgpu-core/src/command/bundle.rs b/wgpu-core/src/command/bundle.rs index 8138d526d8..73b73b61b6 100644 --- a/wgpu-core/src/command/bundle.rs +++ b/wgpu-core/src/command/bundle.rs @@ -48,9 +48,9 @@ use crate::{ hub::{GlobalIdentityHandlerFactory, Hub, Resource, Storage, Token}, id, init_tracker::{BufferInitTrackerAction, MemoryInitKind, TextureInitTrackerAction}, - pipeline::{PipelineFlags, self}, + pipeline::{self, PipelineFlags}, resource, - track::{RenderBundleScope, UsageConflict}, + track::RenderBundleScope, validation::check_buffer_usage, Label, LabelHelpers, LifeGuard, Stored, }; @@ -178,7 +178,8 @@ impl RenderBundleEncoder { let (pipeline_layout_guard, mut token) = hub.pipeline_layouts.read(token); let (bind_group_guard, mut token) = hub.bind_groups.read(&mut token); let (pipeline_guard, mut token) = hub.render_pipelines.read(&mut token); - let (buffer_guard, _) = hub.buffers.read(&mut token); + let (buffer_guard, mut token) = hub.buffers.read(&mut token); + let (texture_guard, mut token) = hub.textures.read(&mut token); let mut state = State { trackers: RenderBundleScope::new(), @@ -222,12 +223,14 @@ impl RenderBundleEncoder { let offsets = &base.dynamic_offsets[..num_dynamic_offsets as usize]; base.dynamic_offsets = &base.dynamic_offsets[num_dynamic_offsets as usize..]; - let bind_group = state - .trackers - .bind_groups - .use_extend(&*bind_group_guard, bind_group_id, (), ()) - .map_err(|_| RenderCommandError::InvalidBindGroup(bind_group_id)) - .map_pass_err(scope)?; + let bind_group = unsafe { + state + .trackers + .bind_groups + .extend(&*bind_group_guard, bind_group_id) + .ok_or_else(|| RenderCommandError::InvalidBindGroup(bind_group_id)) + .map_pass_err(scope)? + }; if bind_group.dynamic_binding_info.len() != offsets.len() { return Err(RenderCommandError::InvalidDynamicOffsetCount { actual: offsets.len(), @@ -256,10 +259,16 @@ impl RenderBundleEncoder { texture_memory_init_actions.extend_from_slice(&bind_group.used_texture_ranges); state.set_bind_group(index, bind_group_id, bind_group.layout_id, offsets); - state - .trackers - .merge_extend_stateful(&bind_group.used) - .map_pass_err(scope)?; + unsafe { + state + .trackers + .extend_from_bind_group( + &*buffer_guard, + &*texture_guard, + &bind_group.used, + ) + .map_pass_err(scope)? + }; //Note: stateless trackers are not merged: the lifetime reference // is held to the bind group itself. } @@ -608,7 +617,6 @@ pub type RenderBundleDescriptor<'a> = wgt::RenderBundleDescriptor>; //Note: here, `RenderBundle` is just wrapping a raw stream of render commands. // The plan is to back it by an actual Vulkan secondary buffer, D3D12 Bundle, // or Metal indirect command buffer. -#[derive(Debug)] pub struct RenderBundle { // Normalized command stream. It can be executed verbatim, // without re-binding anything on the pipeline change. @@ -1001,7 +1009,6 @@ struct VertexLimitState { instance_limit_slot: u32, } -#[derive(Debug)] struct State { trackers: RenderBundleScope, index: IndexState, @@ -1173,8 +1180,6 @@ pub(super) enum RenderBundleErrorInner { #[error(transparent)] RenderCommand(RenderCommandError), #[error(transparent)] - ResourceUsageConflict(#[from] UsageConflict), - #[error(transparent)] Draw(#[from] DrawError), #[error(transparent)] MissingDownlevelFlags(#[from] MissingDownlevelFlags), diff --git a/wgpu-core/src/command/clear.rs b/wgpu-core/src/command/clear.rs index 8beb873a6d..1aefa82d3a 100644 --- a/wgpu-core/src/command/clear.rs +++ b/wgpu-core/src/command/clear.rs @@ -6,7 +6,7 @@ use crate::{ command::CommandBuffer, device::Device, get_lowest_common_denom, - hub::{Global, GlobalIdentityHandlerFactory, HalApi, Resource, Token}, + hub::{self, Global, GlobalIdentityHandlerFactory, HalApi, Token}, id::{BufferId, CommandEncoderId, DeviceId, TextureId, Valid}, init_tracker::{MemoryInitKind, TextureInitRange}, resource::{Texture, TextureClearMode}, @@ -87,11 +87,13 @@ impl Global { list.push(TraceCommand::ClearBuffer { dst, offset, size }); } - let (dst_buffer, dst_pending) = cmd_buf - .trackers - .buffers - .use_replace(&*buffer_guard, dst, (), hal::BufferUses::COPY_DST) - .map_err(ClearError::InvalidBuffer)?; + let (dst_buffer, dst_pending) = unsafe { + cmd_buf + .trackers + .buffers + .change_state(&*buffer_guard, dst, hal::BufferUses::COPY_DST) + .ok_or_else(|| ClearError::InvalidBuffer(dst))? + }; let dst_raw = dst_buffer .raw .as_ref() @@ -139,7 +141,7 @@ impl Global { let dst_barrier = dst_pending.map(|pending| pending.into_hal(dst_buffer)); let cmd_buf_raw = cmd_buf.encoder.open(); unsafe { - cmd_buf_raw.transition_buffers(dst_barrier); + cmd_buf_raw.transition_buffers(dst_barrier.into_iter()); cmd_buf_raw.clear_buffer(dst_raw, offset..end); } Ok(()) @@ -218,6 +220,7 @@ impl Global { } clear_texture( + &*texture_guard, Valid(dst), dst_texture, TextureInitRange { @@ -232,6 +235,7 @@ impl Global { } pub(crate) fn clear_texture( + storage: &hub::Storage, TextureId>, dst_texture_id: Valid, dst_texture: &Texture, range: TextureInitRange, @@ -240,6 +244,7 @@ pub(crate) fn clear_texture( device: &Device, ) -> Result<(), ClearError> { clear_texture_no_device( + storage, dst_texture_id, dst_texture, range, @@ -251,6 +256,7 @@ pub(crate) fn clear_texture( } pub(crate) fn clear_texture_no_device( + storage: &hub::Storage, TextureId>, dst_texture_id: Valid, dst_texture: &Texture, range: TextureInitRange, @@ -287,14 +293,13 @@ pub(crate) fn clear_texture_no_device( // On the other hand, when coming via command_encoder_clear_texture, the life_guard is still there since in order to call it a texture object is needed. // // We could in theory distinguish these two scenarios in the internal clear_texture api in order to remove this check and call the cheaper change_replace_tracked whenever possible. - let dst_barrier = if let Some(ref_count) = dst_texture.life_guard().ref_count.as_ref() { - texture_tracker.change_replace(dst_texture_id, ref_count, selector, clear_usage) - } else { - texture_tracker.change_replace_tracked(dst_texture_id, selector, clear_usage) - } - .map(|pending| pending.into_hal(dst_texture)); + let dst_barrier = texture_tracker + .change_state(storage, dst_texture_id.0, selector, clear_usage) + .unwrap() + .1 + .map(|pending| pending.into_hal(dst_texture)); unsafe { - encoder.transition_textures(dst_barrier); + encoder.transition_textures(dst_barrier.into_iter()); } // Record actual clearing diff --git a/wgpu-core/src/command/compute.rs b/wgpu-core/src/command/compute.rs index e5f672d77b..d89884ded1 100644 --- a/wgpu-core/src/command/compute.rs +++ b/wgpu-core/src/command/compute.rs @@ -14,8 +14,9 @@ use crate::{ hub::{Global, GlobalIdentityHandlerFactory, HalApi, Storage, Token}, id, init_tracker::MemoryInitKind, - resource::{Buffer, Texture}, - track::UsageConflict, + pipeline, + resource::{self, Buffer, Texture}, + track::{Tracker, UsageConflict, UsageScope}, validation::{check_buffer_usage, MissingBufferUsageError}, Label, }; @@ -228,11 +229,10 @@ where } } -#[derive(Debug)] struct State { binder: Binder, pipeline: Option, - trackers: StatefulTrackerSubset, + scope: UsageScope, debug_scope_depth: u32, } @@ -256,29 +256,37 @@ impl State { fn flush_states( &mut self, raw_encoder: &mut A::CommandEncoder, - base_trackers: &mut TrackerSet, + base_trackers: &mut Tracker, bind_group_guard: &Storage, id::BindGroupId>, buffer_guard: &Storage, id::BufferId>, texture_guard: &Storage, id::TextureId>, ) -> Result<(), UsageConflict> { for id in self.binder.list_active() { - self.trackers.merge_extend(&bind_group_guard[id].used)?; - //Note: stateless trackers are not merged: the lifetime reference + unsafe { + self.scope.extend_from_bind_group( + buffer_guard, + texture_guard, + &bind_group_guard[id].used, + )? + }; + // Note: stateless trackers are not merged: the lifetime reference // is held to the bind group itself. } - log::trace!("Encoding dispatch barriers"); + for id in self.binder.list_active() { + unsafe { + base_trackers.extend_from_bind_group( + buffer_guard, + texture_guard, + &mut self.scope, + &bind_group_guard[id].used, + ) + } + } - CommandBuffer::insert_barriers( - raw_encoder, - base_trackers, - &self.trackers.buffers, - &self.trackers.textures, - buffer_guard, - texture_guard, - ); + log::trace!("Encoding dispatch barriers"); - self.trackers.clear(); + CommandBuffer::drain_barriers(raw_encoder, base_trackers, buffer_guard, texture_guard); Ok(()) } } @@ -309,8 +317,9 @@ impl Global { let (device_guard, mut token) = hub.devices.read(&mut token); let (mut cmd_buf_guard, mut token) = hub.command_buffers.write(&mut token); - let cmd_buf = CommandBuffer::get_encoder_mut(&mut *cmd_buf_guard, encoder_id) - .map_pass_err(init_scope)?; + let cmd_buf: &mut CommandBuffer = + CommandBuffer::get_encoder_mut(&mut *cmd_buf_guard, encoder_id) + .map_pass_err(init_scope)?; // will be reset to true if recording is done without errors cmd_buf.status = CommandEncoderStatus::Error; let raw = cmd_buf.encoder.open(); @@ -335,7 +344,7 @@ impl Global { let mut state = State { binder: Binder::new(), pipeline: None, - trackers: StatefulTrackerSubset::new(A::VARIANT), + scope: UsageScope::new(), debug_scope_depth: 0, }; let mut temp_offsets = Vec::new(); @@ -376,12 +385,14 @@ impl Global { ); dynamic_offset_count += num_dynamic_offsets as usize; - let bind_group = cmd_buf - .trackers - .bind_groups - .use_extend(&*bind_group_guard, bind_group_id, (), ()) - .map_err(|_| ComputePassErrorInner::InvalidBindGroup(bind_group_id)) - .map_pass_err(scope)?; + let bind_group: &BindGroup = unsafe { + cmd_buf + .trackers + .bind_groups + .extend(&*bind_group_guard, bind_group_id) + .ok_or_else(|| ComputePassErrorInner::InvalidBindGroup(bind_group_id)) + .map_pass_err(scope)? + }; bind_group .validate_dynamic_bindings(&temp_offsets, &cmd_buf.limits) .map_pass_err(scope)?; @@ -431,12 +442,14 @@ impl Global { state.pipeline = Some(pipeline_id); - let pipeline = cmd_buf - .trackers - .compute_pipes - .use_extend(&*pipeline_guard, pipeline_id, (), ()) - .map_err(|_| ComputePassErrorInner::InvalidPipeline(pipeline_id)) - .map_pass_err(scope)?; + let pipeline: &pipeline::ComputePipeline = unsafe { + cmd_buf + .trackers + .compute_pipelines + .extend(&*pipeline_guard, pipeline_id) + .ok_or_else(|| ComputePassErrorInner::InvalidPipeline(pipeline_id)) + .map_pass_err(scope)? + }; unsafe { raw.set_compute_pipeline(&pipeline.raw); @@ -584,12 +597,13 @@ impl Global { .require_downlevel_flags(wgt::DownlevelFlags::INDIRECT_EXECUTION) .map_pass_err(scope)?; - let indirect_buffer = state - .trackers - .buffers - .use_extend(&*buffer_guard, buffer_id, (), hal::BufferUses::INDIRECT) - .map_err(|_| ComputePassErrorInner::InvalidIndirectBuffer(buffer_id)) - .map_pass_err(scope)?; + let indirect_buffer: &Buffer = unsafe { + state + .scope + .buffers + .extend(&*buffer_guard, buffer_id, hal::BufferUses::INDIRECT) + .map_pass_err(scope)? + }; check_buffer_usage(indirect_buffer.usage, wgt::BufferUsages::INDIRECT) .map_pass_err(scope)?; @@ -667,17 +681,14 @@ impl Global { } => { let scope = PassErrorScope::WriteTimestamp; - let query_set = cmd_buf - .trackers - .query_sets - .use_extend(&*query_set_guard, query_set_id, (), ()) - .map_err(|e| match e { - UseExtendError::InvalidResource => { - ComputePassErrorInner::InvalidQuerySet(query_set_id) - } - _ => unreachable!(), - }) - .map_pass_err(scope)?; + let query_set: &resource::QuerySet = unsafe { + cmd_buf + .trackers + .query_sets + .extend(&*query_set_guard, query_set_id) + .ok_or_else(|| ComputePassErrorInner::InvalidQuerySet(query_set_id)) + .map_pass_err(scope)? + }; query_set .validate_and_write_timestamp(raw, query_set_id, query_index, None) @@ -689,17 +700,14 @@ impl Global { } => { let scope = PassErrorScope::BeginPipelineStatisticsQuery; - let query_set = cmd_buf - .trackers - .query_sets - .use_extend(&*query_set_guard, query_set_id, (), ()) - .map_err(|e| match e { - UseExtendError::InvalidResource => { - ComputePassErrorInner::InvalidQuerySet(query_set_id) - } - _ => unreachable!(), - }) - .map_pass_err(scope)?; + let query_set: &resource::QuerySet = unsafe { + cmd_buf + .trackers + .query_sets + .extend(&*query_set_guard, query_set_id) + .ok_or_else(|| ComputePassErrorInner::InvalidQuerySet(query_set_id)) + .map_pass_err(scope)? + }; query_set .validate_and_begin_pipeline_statistics_query( diff --git a/wgpu-core/src/command/draw.rs b/wgpu-core/src/command/draw.rs index 653b4df459..5e96fcddec 100644 --- a/wgpu-core/src/command/draw.rs +++ b/wgpu-core/src/command/draw.rs @@ -5,7 +5,7 @@ use crate::{ binding_model::{LateMinBufferBindingSizeMismatch, PushConstantUploadError}, error::ErrorFormatter, id, - track::{UsageConflict}, + track::UsageConflict, validation::{MissingBufferUsageError, MissingTextureUsageError}, }; use wgt::{BufferAddress, BufferSize, Color}; @@ -13,8 +13,6 @@ use wgt::{BufferAddress, BufferSize, Color}; use std::num::NonZeroU32; use thiserror::Error; -pub type BufferError = UsageConflict; - /// Error validating a draw call. #[derive(Clone, Debug, Error, PartialEq)] pub enum DrawError { @@ -80,7 +78,7 @@ pub enum RenderCommandError { #[error("pipeline writes to depth/stencil, while the pass has read-only depth/stencil")] IncompatiblePipelineRods, #[error(transparent)] - Buffer(#[from] BufferError), + UsageConflict(#[from] UsageConflict), #[error("buffer {0:?} is destroyed")] DestroyedBuffer(id::BufferId), #[error(transparent)] @@ -106,8 +104,9 @@ impl crate::error::PrettyError for RenderCommandError { Self::InvalidPipeline(id) => { fmt.render_pipeline_label(&id); } - Self::Buffer(id, ..) | Self::DestroyedBuffer(id) => { - fmt.buffer_label(&id); + Self::UsageConflict(_, ..) | Self::DestroyedBuffer(_) => { + // fmt.buffer_label(&id); + todo!() } _ => {} }; diff --git a/wgpu-core/src/command/memory_init.rs b/wgpu-core/src/command/memory_init.rs index acd1e387e4..8d66c271a2 100644 --- a/wgpu-core/src/command/memory_init.rs +++ b/wgpu-core/src/command/memory_init.rs @@ -8,7 +8,7 @@ use crate::{ id::{self, TextureId}, init_tracker::*, resource::{Buffer, Texture}, - track::{TextureTracker}, + track::{TextureTracker, Tracker}, FastHashMap, }; @@ -132,6 +132,7 @@ pub(crate) fn fixup_discarded_surfaces< ) { for init in inits { clear_texture( + texture_guard, id::Valid(init.texture), texture_guard.get(init.texture).unwrap(), TextureInitRange { @@ -150,7 +151,7 @@ impl BakedCommands { // inserts all buffer initializations that are going to be needed for executing the commands and updates resource init states accordingly pub(crate) fn initialize_buffer_memory( &mut self, - device_tracker: &mut TrackerSet, + device_tracker: &mut Tracker, buffer_guard: &mut Storage, id::BufferId>, ) -> Result<(), DestroyedBufferError> { // Gather init ranges for each buffer so we can collapse them. @@ -202,11 +203,13 @@ impl BakedCommands { // Don't do use_replace since the buffer may already no longer have a ref_count. // However, we *know* that it is currently in use, so the tracker must already know about it. - let transition = device_tracker.buffers.change_replace_tracked( - id::Valid(buffer_id), - (), - hal::BufferUses::COPY_DST, - ); + let transition = unsafe { + device_tracker + .buffers + .change_state(buffer_guard, buffer_id, hal::BufferUses::COPY_DST) + .unwrap() + .1 + }; let buffer = buffer_guard .get_mut(buffer_id) @@ -214,8 +217,11 @@ impl BakedCommands { let raw_buf = buffer.raw.as_ref().ok_or(DestroyedBufferError(buffer_id))?; unsafe { - self.encoder - .transition_buffers(transition.map(|pending| pending.into_hal(buffer))); + self.encoder.transition_buffers( + transition + .map(|pending| pending.into_hal(buffer)) + .into_iter(), + ); } for range in ranges.iter() { @@ -234,7 +240,7 @@ impl BakedCommands { // any textures that are left discarded by this command buffer will be marked as uninitialized pub(crate) fn initialize_texture_memory( &mut self, - device_tracker: &mut TrackerSet, + device_tracker: &mut Tracker, texture_guard: &mut Storage, TextureId>, device: &Device, ) -> Result<(), DestroyedTextureError> { @@ -274,6 +280,7 @@ impl BakedCommands { // TODO: Could we attempt some range collapsing here? for range in ranges.drain(..) { clear_texture( + texture_guard, id::Valid(texture_use.id), &*texture, range, diff --git a/wgpu-core/src/command/mod.rs b/wgpu-core/src/command/mod.rs index 9915b7dbac..886d1d917a 100644 --- a/wgpu-core/src/command/mod.rs +++ b/wgpu-core/src/command/mod.rs @@ -146,22 +146,31 @@ impl CommandBuffer { texture_guard: &Storage, id::TextureId>, ) { profiling::scope!("insert_barriers"); - debug_assert_eq!(A::VARIANT, base.backend()); - - let buffer_barriers = base - .buffers - .change_states_scope(&*buffer_guard, &head.buffers) - .map(|pending| { - let buf = unsafe { &buffer_guard.get_unchecked(pending.id) }; - pending.into_hal(buf) - }); - let texture_barriers = base - .textures - .change_states_scope(&*texture_guard, &head.textures) - .map(|pending| { - let tex = unsafe { texture_guard.get_unchecked(pending.id) }; - pending.into_hal(tex) - }); + + base.buffers + .change_states_scope(&*buffer_guard, &head.buffers); + base.textures + .change_states_scope(&*texture_guard, &head.textures); + + Self::drain_barriers(raw, base, buffer_guard, texture_guard); + } + + pub(crate) fn drain_barriers( + raw: &mut A::CommandEncoder, + base: &mut Tracker, + buffer_guard: &Storage, id::BufferId>, + texture_guard: &Storage, id::TextureId>, + ) { + profiling::scope!("drain_barriers"); + + let buffer_barriers = base.buffers.drain().map(|pending| { + let buf = unsafe { &buffer_guard.get_unchecked(pending.id) }; + pending.into_hal(buf) + }); + let texture_barriers = base.textures.drain().map(|pending| { + let tex = unsafe { texture_guard.get_unchecked(pending.id) }; + pending.into_hal(tex) + }); unsafe { raw.transition_buffers(buffer_barriers); @@ -302,7 +311,7 @@ impl Global { cmd_buf.status = CommandEncoderStatus::Finished; //Note: if we want to stop tracking the swapchain texture view, // this is the place to do it. - log::trace!("Command buffer {:?} {:#?}", encoder_id, cmd_buf.trackers); + log::trace!("Command buffer {:?}", encoder_id); None } CommandEncoderStatus::Finished => Some(CommandEncoderError::NotRecording), diff --git a/wgpu-core/src/command/query.rs b/wgpu-core/src/command/query.rs index b67a4092ae..e5b4601f35 100644 --- a/wgpu-core/src/command/query.rs +++ b/wgpu-core/src/command/query.rs @@ -301,7 +301,7 @@ impl Global { .trackers .query_sets .extend(&*query_set_guard, query_set_id) - .map_err(|_| QueryError::InvalidQuerySet(query_set_id))? + .ok_or_else(|| QueryError::InvalidQuerySet(query_set_id))? }; query_set.validate_and_write_timestamp(raw_encoder, query_set_id, query_index, None)?; @@ -348,14 +348,16 @@ impl Global { .trackers .query_sets .extend(&*query_set_guard, query_set_id) - .map_err(|_| QueryError::InvalidQuerySet(query_set_id))? + .ok_or_else(|| QueryError::InvalidQuerySet(query_set_id))? }; - let (dst_buffer, dst_pending) = cmd_buf - .trackers - .buffers - .use_replace(&*buffer_guard, destination, (), hal::BufferUses::COPY_DST) - .map_err(QueryError::InvalidBuffer)?; + let (dst_buffer, dst_pending) = unsafe { + cmd_buf + .trackers + .buffers + .change_state(&*buffer_guard, destination, hal::BufferUses::COPY_DST) + .ok_or_else(|| QueryError::InvalidBuffer(destination))? + }; let dst_barrier = dst_pending.map(|pending| pending.into_hal(dst_buffer)); if !dst_buffer.usage.contains(wgt::BufferUsages::COPY_DST) { @@ -404,7 +406,7 @@ impl Global { )); unsafe { - raw_encoder.transition_buffers(dst_barrier); + raw_encoder.transition_buffers(dst_barrier.into_iter()); raw_encoder.copy_query_results( &query_set.raw, start_query..end_query, diff --git a/wgpu-core/src/command/render.rs b/wgpu-core/src/command/render.rs index b2a157b826..b31c234fc0 100644 --- a/wgpu-core/src/command/render.rs +++ b/wgpu-core/src/command/render.rs @@ -696,11 +696,13 @@ impl<'a, A: HalApi> RenderPassInfo<'a, A> { let mut depth_stencil = None; if let Some(at) = depth_stencil_attachment { - let view = cmd_buf - .trackers - .views - .use_extend(&*view_guard, at.view, (), ()) - .map_err(|_| RenderPassErrorInner::InvalidAttachment(at.view))?; + let view = unsafe { + cmd_buf + .trackers + .views + .extend(&*view_guard, at.view) + .ok_or_else(|| RenderPassErrorInner::InvalidAttachment(at.view))? + }; check_multiview(view)?; add_view(view, "depth")?; @@ -805,11 +807,13 @@ impl<'a, A: HalApi> RenderPassInfo<'a, A> { } for at in color_attachments { - let color_view = cmd_buf - .trackers - .views - .use_extend(&*view_guard, at.view, (), ()) - .map_err(|_| RenderPassErrorInner::InvalidAttachment(at.view))?; + let color_view = unsafe { + cmd_buf + .trackers + .views + .extend(&*view_guard, at.view) + .ok_or_else(|| RenderPassErrorInner::InvalidAttachment(at.view))? + }; check_multiview(color_view)?; add_view(color_view, "color")?; @@ -835,11 +839,13 @@ impl<'a, A: HalApi> RenderPassInfo<'a, A> { let mut hal_resolve_target = None; if let Some(resolve_target) = at.resolve_target { - let resolve_view = cmd_buf - .trackers - .views - .use_extend(&*view_guard, resolve_target, (), ()) - .map_err(|_| RenderPassErrorInner::InvalidAttachment(resolve_target))?; + let resolve_view = unsafe { + cmd_buf + .trackers + .views + .extend(&*view_guard, resolve_target) + .ok_or_else(|| RenderPassErrorInner::InvalidAttachment(resolve_target))? + }; check_multiview(resolve_view)?; if color_view.extent != resolve_view.extent { @@ -963,15 +969,17 @@ impl<'a, A: HalApi> RenderPassInfo<'a, A> { check_texture_usage(texture.desc.usage, TextureUsages::RENDER_ATTACHMENT)?; // the tracker set of the pass is always in "extend" mode - self.usage_scope - .textures - .change_extend( - ra.texture_id.value, - &ra.texture_id.ref_count, - ra.selector.clone(), - ra.usage, - ) - .map_err(UsageConflict::from)?; + unsafe { + self.usage_scope + .textures + .extend( + &*texture_guard, + ra.texture_id.value.0, + Some(ra.selector.clone()), + ra.usage, + ) + .map_err(UsageConflict::from)? + }; } // If either only stencil or depth was discarded, we put in a special clear pass to keep the init status of the aspects in sync. @@ -1073,12 +1081,11 @@ impl Global { let (pipeline_layout_guard, mut token) = hub.pipeline_layouts.read(&mut token); let (bind_group_guard, mut token) = hub.bind_groups.read(&mut token); let (render_pipe_guard, mut token) = hub.render_pipelines.read(&mut token); - let (pipeline_guard, mut token) = hub.render_pipelines.read(&mut token); let (query_set_guard, mut token) = hub.query_sets.read(&mut token); let (buffer_guard, mut token) = hub.buffers.read(&mut token); let (texture_guard, mut token) = hub.textures.read(&mut token); - let (sampler_guard, mut token) = hub.samplers.read(&mut token); - let (view_guard, _) = hub.texture_views.read(&mut token); + let (view_guard, mut token) = hub.texture_views.read(&mut token); + let (sampler_guard, _) = hub.samplers.read(&mut token); log::trace!( "Encoding render pass begin in command buffer {:?}", @@ -1143,6 +1150,8 @@ impl Global { .trackers .bind_groups .extend(&*bind_group_guard, bind_group_id) + .ok_or_else(|| RenderCommandError::InvalidBindGroup(bind_group_id)) + .map_pass_err(scope)? }; bind_group .validate_dynamic_bindings(&temp_offsets, &cmd_buf.limits) @@ -1150,11 +1159,13 @@ impl Global { // merge the resource tracker in unsafe { - info.usage_scope.extend_from_bind_group( - &*buffer_guard, - &*texture_guard, - &bind_group.used, - ) + info.usage_scope + .extend_from_bind_group( + &*buffer_guard, + &*texture_guard, + &bind_group.used, + ) + .map_pass_err(scope)?; } //Note: stateless trackers are not merged: the lifetime reference // is held to the bind group itself. @@ -1208,7 +1219,9 @@ impl Global { cmd_buf .trackers .render_pipelines - .extend(&*pipeline_guard, pipeline_id) + .extend(&*render_pipe_guard, pipeline_id) + .ok_or_else(|| RenderCommandError::InvalidPipeline(pipeline_id)) + .map_pass_err(scope)? }; info.context @@ -1817,7 +1830,7 @@ impl Global { .trackers .query_sets .extend(&*query_set_guard, query_set_id) - .map_err(|_| RenderCommandError::InvalidQuerySet(query_set_id)) + .ok_or_else(|| RenderCommandError::InvalidQuerySet(query_set_id)) .map_pass_err(scope)? }; @@ -1841,7 +1854,7 @@ impl Global { .trackers .query_sets .extend(&*query_set_guard, query_set_id) - .map_err(|_| RenderCommandError::InvalidQuerySet(query_set_id)) + .ok_or_else(|| RenderCommandError::InvalidQuerySet(query_set_id)) .map_pass_err(scope)? }; @@ -1868,7 +1881,7 @@ impl Global { .trackers .bundles .extend(&*bundle_guard, bundle_id) - .map_err(|_| RenderCommandError::InvalidRenderBundle(bundle_id)) + .ok_or_else(|| RenderCommandError::InvalidRenderBundle(bundle_id)) .map_pass_err(scope)? }; @@ -1907,7 +1920,7 @@ impl Global { raw, &*pipeline_layout_guard, &*bind_group_guard, - &*pipeline_guard, + &*render_pipe_guard, &*buffer_guard, ) } diff --git a/wgpu-core/src/command/transfer.rs b/wgpu-core/src/command/transfer.rs index 8a1be65e28..f594a3110c 100644 --- a/wgpu-core/src/command/transfer.rs +++ b/wgpu-core/src/command/transfer.rs @@ -15,6 +15,7 @@ use crate::{ track::TextureSelector, }; +use arrayvec::ArrayVec; use hal::CommandEncoder as _; use thiserror::Error; use wgt::{BufferAddress, BufferUsages, Extent3d, TextureUsages}; @@ -409,6 +410,7 @@ fn handle_texture_init( let cmd_buf_raw = cmd_buf.encoder.open(); for init in immediate_inits { clear_texture( + texture_guard, Valid(init.texture), texture, TextureInitRange { @@ -517,11 +519,13 @@ impl Global { }); } - let (src_buffer, src_pending) = cmd_buf - .trackers - .buffers - .use_replace(&*buffer_guard, source, (), hal::BufferUses::COPY_SRC) - .map_err(TransferError::InvalidBuffer)?; + let (src_buffer, src_pending) = unsafe { + cmd_buf + .trackers + .buffers + .change_state(&*buffer_guard, source, hal::BufferUses::COPY_SRC) + .ok_or_else(|| TransferError::InvalidBuffer(source))? + }; let src_raw = src_buffer .raw .as_ref() @@ -530,15 +534,15 @@ impl Global { return Err(TransferError::MissingCopySrcUsageFlag.into()); } // expecting only a single barrier - let src_barrier = src_pending - .map(|pending| pending.into_hal(src_buffer)) - .next(); - - let (dst_buffer, dst_pending) = cmd_buf - .trackers - .buffers - .use_replace(&*buffer_guard, destination, (), hal::BufferUses::COPY_DST) - .map_err(TransferError::InvalidBuffer)?; + let src_barrier = src_pending.map(|pending| pending.into_hal(src_buffer)); + + let (dst_buffer, dst_pending) = unsafe { + cmd_buf + .trackers + .buffers + .change_state(&*buffer_guard, destination, hal::BufferUses::COPY_DST) + .ok_or_else(|| TransferError::InvalidBuffer(destination))? + }; let dst_raw = dst_buffer .raw .as_ref() @@ -546,9 +550,7 @@ impl Global { if !dst_buffer.usage.contains(BufferUsages::COPY_DST) { return Err(TransferError::MissingCopyDstUsageFlag(Some(destination), None).into()); } - let dst_barrier = dst_pending - .map(|pending| pending.into_hal(dst_buffer)) - .next(); + let dst_barrier = dst_pending.map(|pending| pending.into_hal(dst_buffer)); if size % wgt::COPY_BUFFER_ALIGNMENT != 0 { return Err(TransferError::UnalignedCopySize(size).into()); @@ -655,11 +657,13 @@ impl Global { // Handle texture init *before* dealing with barrier transitions so we have an easier time inserting "immediate-inits" that may be required by prior discards in rare cases. handle_dst_texture_init(cmd_buf, device, destination, copy_size, &texture_guard)?; - let (src_buffer, src_pending) = cmd_buf - .trackers - .buffers - .use_replace(&*buffer_guard, source.buffer, (), hal::BufferUses::COPY_SRC) - .map_err(TransferError::InvalidBuffer)?; + let (src_buffer, src_pending) = unsafe { + cmd_buf + .trackers + .buffers + .change_state(&*buffer_guard, source.buffer, hal::BufferUses::COPY_SRC) + .ok_or_else(|| TransferError::InvalidBuffer(source.buffer))? + }; let src_raw = src_buffer .raw .as_ref() @@ -667,18 +671,20 @@ impl Global { if !src_buffer.usage.contains(BufferUsages::COPY_SRC) { return Err(TransferError::MissingCopySrcUsageFlag.into()); } - let src_barriers = src_pending.map(|pending| pending.into_hal(src_buffer)); - - let (dst_texture, dst_pending) = cmd_buf - .trackers - .textures - .use_replace( - &*texture_guard, - destination.texture, - dst_range, - hal::TextureUses::COPY_DST, - ) - .unwrap(); + let src_barrier = src_pending.map(|pending| pending.into_hal(src_buffer)); + + let (dst_texture, dst_pending) = unsafe { + cmd_buf + .trackers + .textures + .change_state( + &*texture_guard, + destination.texture, + dst_range, + hal::TextureUses::COPY_DST, + ) + .ok_or_else(|| TransferError::InvalidTexture(destination.texture))? + }; let dst_raw = dst_texture .inner .as_raw() @@ -688,7 +694,7 @@ impl Global { TransferError::MissingCopyDstUsageFlag(None, Some(destination.texture)).into(), ); } - let dst_barriers = dst_pending.map(|pending| pending.into_hal(dst_texture)); + let dst_barrier = dst_pending.map(|pending| pending.into_hal(dst_texture)); let format_desc = dst_texture.desc.format.describe(); let (hal_copy_size, array_layer_count) = validate_texture_copy_range( @@ -735,8 +741,8 @@ impl Global { let cmd_buf_raw = cmd_buf.encoder.open(); unsafe { - cmd_buf_raw.transition_textures(dst_barriers); - cmd_buf_raw.transition_buffers(src_barriers); + cmd_buf_raw.transition_textures(dst_barrier.into_iter()); + cmd_buf_raw.transition_buffers(src_barrier.into_iter()); cmd_buf_raw.copy_buffer_to_texture(src_raw, dst_raw, regions); } Ok(()) @@ -782,16 +788,18 @@ impl Global { // Handle texture init *before* dealing with barrier transitions so we have an easier time inserting "immediate-inits" that may be required by prior discards in rare cases. handle_src_texture_init(cmd_buf, device, source, copy_size, &texture_guard)?; - let (src_texture, src_pending) = cmd_buf - .trackers - .textures - .use_replace( - &*texture_guard, - source.texture, - src_range, - hal::TextureUses::COPY_SRC, - ) - .unwrap(); + let (src_texture, src_pending) = unsafe { + cmd_buf + .trackers + .textures + .change_state( + &*texture_guard, + source.texture, + src_range, + hal::TextureUses::COPY_SRC, + ) + .ok_or_else(|| TransferError::InvalidTexture(source.texture))? + }; let src_raw = src_texture .inner .as_raw() @@ -799,18 +807,19 @@ impl Global { if !src_texture.desc.usage.contains(TextureUsages::COPY_SRC) { return Err(TransferError::MissingCopySrcUsageFlag.into()); } - let src_barriers = src_pending.map(|pending| pending.into_hal(src_texture)); - - let (dst_buffer, dst_pending) = cmd_buf - .trackers - .buffers - .use_replace( - &*buffer_guard, - destination.buffer, - (), - hal::BufferUses::COPY_DST, - ) - .map_err(TransferError::InvalidBuffer)?; + let src_barrier = src_pending.map(|pending| pending.into_hal(src_texture)); + + let (dst_buffer, dst_pending) = unsafe { + cmd_buf + .trackers + .buffers + .change_state( + &*buffer_guard, + destination.buffer, + hal::BufferUses::COPY_DST, + ) + .ok_or_else(|| TransferError::InvalidBuffer(destination.buffer))? + }; let dst_raw = dst_buffer .raw .as_ref() @@ -820,7 +829,7 @@ impl Global { TransferError::MissingCopyDstUsageFlag(Some(destination.buffer), None).into(), ); } - let dst_barriers = dst_pending.map(|pending| pending.into_hal(dst_buffer)); + let dst_barrier = dst_pending.map(|pending| pending.into_hal(dst_buffer)); let format_desc = src_texture.desc.format.describe(); let (hal_copy_size, array_layer_count) = @@ -875,8 +884,8 @@ impl Global { }); let cmd_buf_raw = cmd_buf.encoder.open(); unsafe { - cmd_buf_raw.transition_buffers(dst_barriers); - cmd_buf_raw.transition_textures(src_barriers); + cmd_buf_raw.transition_buffers(dst_barrier.into_iter()); + cmd_buf_raw.transition_textures(src_barrier.into_iter()); cmd_buf_raw.copy_texture_to_buffer( src_raw, hal::TextureUses::COPY_SRC, @@ -933,16 +942,18 @@ impl Global { handle_src_texture_init(cmd_buf, device, source, copy_size, &texture_guard)?; handle_dst_texture_init(cmd_buf, device, destination, copy_size, &texture_guard)?; - let (src_texture, src_pending) = cmd_buf - .trackers - .textures - .use_replace( - &*texture_guard, - source.texture, - src_range, - hal::TextureUses::COPY_SRC, - ) - .unwrap(); + let (src_texture, src_pending) = unsafe { + cmd_buf + .trackers + .textures + .change_state( + &*texture_guard, + source.texture, + src_range, + hal::TextureUses::COPY_SRC, + ) + .ok_or_else(|| TransferError::InvalidTexture(source.texture))? + }; let src_raw = src_texture .inner .as_raw() @@ -953,20 +964,23 @@ impl Global { //TODO: try to avoid this the collection. It's needed because both // `src_pending` and `dst_pending` try to hold `trackers.textures` mutably. - let mut barriers = src_pending + let mut barriers: ArrayVec<_, 2> = src_pending .map(|pending| pending.into_hal(src_texture)) - .collect::>(); - - let (dst_texture, dst_pending) = cmd_buf - .trackers - .textures - .use_replace( - &*texture_guard, - destination.texture, - dst_range, - hal::TextureUses::COPY_DST, - ) - .unwrap(); + .into_iter() + .collect(); + + let (dst_texture, dst_pending) = unsafe { + cmd_buf + .trackers + .textures + .change_state( + &*texture_guard, + destination.texture, + dst_range, + hal::TextureUses::COPY_DST, + ) + .ok_or(TransferError::InvalidTexture(destination.texture))? + }; let dst_raw = dst_texture .inner .as_raw() diff --git a/wgpu-core/src/device/life.rs b/wgpu-core/src/device/life.rs index cc9c6b5045..9d69ae52d6 100644 --- a/wgpu-core/src/device/life.rs +++ b/wgpu-core/src/device/life.rs @@ -7,6 +7,7 @@ use crate::{ }, hub::{GlobalIdentityHandlerFactory, HalApi, Hub, Token}, id, resource, + track::Tracker, RefCount, Stored, SubmissionIndex, }; use smallvec::SmallVec; @@ -490,7 +491,7 @@ impl LifetimeTracker { pub(super) fn triage_suspected( &mut self, hub: &Hub, - trackers: &Mutex, + trackers: &Mutex>, #[cfg(feature = "trace")] trace: Option<&Mutex>, token: &mut Token>, ) { diff --git a/wgpu-core/src/device/mod.rs b/wgpu-core/src/device/mod.rs index f5f10aadd0..5fd07af03b 100644 --- a/wgpu-core/src/device/mod.rs +++ b/wgpu-core/src/device/mod.rs @@ -1,14 +1,16 @@ use crate::{ binding_model, command, conv, device::life::WaitIdleError, - hub::{Global, GlobalIdentityHandlerFactory, HalApi, Hub, Input, InvalidId, Storage, Token}, + hub::{ + self, Global, GlobalIdentityHandlerFactory, HalApi, Hub, Input, InvalidId, Storage, Token, + }, id, init_tracker::{ BufferInitTracker, BufferInitTrackerAction, MemoryInitKind, TextureInitRange, TextureInitTracker, TextureInitTrackerAction, }, instance, pipeline, present, resource, - track::{TextureSelector, UsageConflict}, + track::{BindGroupStates, TextureSelector, Tracker, UsageConflict}, validation::{self, check_buffer_usage, check_texture_usage}, FastHashMap, Label, LabelHelpers as _, LifeGuard, MultiRefCount, RefCount, Stored, SubmissionIndex, DOWNLEVEL_ERROR_MESSAGE, @@ -281,7 +283,7 @@ pub struct Device { /// All live resources allocated with this [`Device`]. /// /// Has to be locked temporarily only (locked last) - pub(crate) trackers: Mutex, + pub(crate) trackers: Mutex>, // Life tracker should be locked right after the device and before anything else. life_tracker: Mutex>, /// Temporary storage for resource management functions. Cleared at the end @@ -394,7 +396,7 @@ impl Device { command_allocator: Mutex::new(com_alloc), active_submission_index: 0, fence, - trackers: Mutex::new(TrackerSet::new(A::VARIANT)), + trackers: Mutex::new(Tracker::new()), life_tracker: Mutex::new(life::LifetimeTracker::new()), temp_suspected: life::SuspectedResources::default(), #[cfg(feature = "trace")] @@ -495,7 +497,7 @@ impl Device { fn untrack<'this, 'token: 'this, G: GlobalIdentityHandlerFactory>( &'this mut self, hub: &Hub, - trackers: &TrackerSet, + trackers: &Tracker, token: &mut Token<'token, Self>, ) { self.temp_suspected.clear(); @@ -1495,7 +1497,7 @@ impl Device { used_buffer_ranges: &mut Vec, dynamic_binding_info: &mut Vec, late_buffer_binding_sizes: &mut FastHashMap, - used: &mut TrackerSet, + used: &mut BindGroupStates, storage: &'a Storage, id::BufferId>, limits: &wgt::Limits, ) -> Result, binding_model::CreateBindGroupError> { @@ -1544,8 +1546,8 @@ impl Device { let buffer = used .buffers - .use_extend(storage, bb.buffer_id, (), internal_use) - .map_err(|_| Error::InvalidBuffer(bb.buffer_id))?; + .extend(storage, bb.buffer_id, internal_use) + .ok_or_else(|| Error::InvalidBuffer(bb.buffer_id))?; check_buffer_usage(buffer.usage, pub_usage)?; let raw_buffer = buffer .raw @@ -1614,26 +1616,25 @@ impl Device { fn create_texture_binding( view: &resource::TextureView, - texture_guard: &parking_lot::lock_api::RwLockReadGuard< - parking_lot::RawRwLock, - Storage, id::Id>>, - >, + texture_guard: &hub::Storage, id::TextureId>, internal_use: hal::TextureUses, pub_usage: wgt::TextureUsages, - used: &mut TrackerSet, + used: &mut BindGroupStates, used_texture_ranges: &mut Vec, ) -> Result<(), binding_model::CreateBindGroupError> { // Careful here: the texture may no longer have its own ref count, // if it was deleted by the user. let texture = &texture_guard[view.parent_id.value]; used.textures - .change_extend( - view.parent_id.value, - &view.parent_id.ref_count, - view.selector.clone(), + .extend( + texture_guard, + view.parent_id.value.0, + Some(view.selector.clone()), internal_use, ) - .map_err(UsageConflict::from)?; + .ok_or_else(|| { + binding_model::CreateBindGroupError::InvalidTexture(view.parent_id.value) + })?; check_texture_usage(texture.desc.usage, pub_usage)?; used_texture_ranges.push(TextureInitTrackerAction { @@ -1675,7 +1676,7 @@ impl Device { // it needs to be in BGL iteration order, not BG entry order. let mut late_buffer_binding_sizes = FastHashMap::default(); // fill out the descriptors - let mut used = TrackerSet::new(A::VARIANT); + let mut used = BindGroupStates::new(); let (buffer_guard, mut token) = hub.buffers.read(token); let (texture_guard, mut token) = hub.textures.read(&mut token); //skip token @@ -1739,8 +1740,8 @@ impl Device { wgt::BindingType::Sampler(ty) => { let sampler = used .samplers - .use_extend(&*sampler_guard, id, (), ()) - .map_err(|_| Error::InvalidSampler(id))?; + .extend(&*sampler_guard, id) + .ok_or_else(|| Error::InvalidSampler(id))?; // Allowed sampler values for filtering and comparison let (allowed_filtering, allowed_comparison) = match ty { @@ -1788,8 +1789,8 @@ impl Device { for &id in bindings_array.iter() { let sampler = used .samplers - .use_extend(&*sampler_guard, id, (), ()) - .map_err(|_| Error::InvalidSampler(id))?; + .extend(&*sampler_guard, id) + .ok_or_else(|| Error::InvalidSampler(id))?; hal_samplers.push(&sampler.raw); } @@ -1798,8 +1799,8 @@ impl Device { Br::TextureView(id) => { let view = used .views - .use_extend(&*texture_view_guard, id, (), ()) - .map_err(|_| Error::InvalidTextureView(id))?; + .extend(&*texture_view_guard, id) + .ok_or_else(|| Error::InvalidTextureView(id))?; let (pub_usage, internal_use) = Self::texture_use_parameters( binding, decl, @@ -1829,8 +1830,8 @@ impl Device { for &id in bindings_array.iter() { let view = used .views - .use_extend(&*texture_view_guard, id, (), ()) - .map_err(|_| Error::InvalidTextureView(id))?; + .extend(&*texture_view_guard, id) + .ok_or_else(|| Error::InvalidTextureView(id))?; let (pub_usage, internal_use) = Self::texture_use_parameters(binding, decl, view, "SampledTextureArray, ReadonlyStorageTextureArray or WriteonlyStorageTextureArray")?; diff --git a/wgpu-core/src/device/queue.rs b/wgpu-core/src/device/queue.rs index 001d102ac4..089d8a7bd6 100644 --- a/wgpu-core/src/device/queue.rs +++ b/wgpu-core/src/device/queue.rs @@ -284,10 +284,12 @@ impl Global { .map_err(DeviceError::from)?; let mut trackers = device.trackers.lock(); - let (dst, transition) = trackers - .buffers - .use_replace(&*buffer_guard, buffer_id, (), hal::BufferUses::COPY_DST) - .map_err(TransferError::InvalidBuffer)?; + let (dst, transition) = unsafe { + trackers + .buffers + .change_state(&*buffer_guard, buffer_id, hal::BufferUses::COPY_DST) + .ok_or_else(|| TransferError::InvalidBuffer(buffer_id))? + }; let dst_raw = dst .raw .as_ref() @@ -452,6 +454,7 @@ impl Global { .collect::>>() { crate::command::clear_texture_no_device( + &*texture_guard, id::Valid(destination.texture), &*dst, TextureInitRange { @@ -471,15 +474,17 @@ impl Global { } } - let (dst, transition) = trackers - .textures - .use_replace( - &*texture_guard, - destination.texture, - selector, - hal::TextureUses::COPY_DST, - ) - .unwrap(); + let (dst, transition) = unsafe { + trackers + .textures + .change_state( + &*texture_guard, + destination.texture, + selector, + hal::TextureUses::COPY_DST, + ) + .ok_or_else(|| TransferError::InvalidTexture(destination.texture))? + }; let (hal_copy_size, array_layer_count) = validate_texture_copy_range(destination, &dst.desc, CopySide::Destination, size)?; @@ -561,7 +566,8 @@ impl Global { .ok_or(TransferError::InvalidTexture(destination.texture))?; unsafe { - encoder.transition_textures(transition.map(|pending| pending.into_hal(dst))); + encoder + .transition_textures(transition.map(|pending| pending.into_hal(dst)).into_iter()); encoder.transition_buffers(iter::once(barrier)); encoder.copy_buffer_to_texture(&stage.buffer, dst_raw, regions); } diff --git a/wgpu-core/src/resource.rs b/wgpu-core/src/resource.rs index 0ef751cb5b..e132c40e72 100644 --- a/wgpu-core/src/resource.rs +++ b/wgpu-core/src/resource.rs @@ -3,7 +3,7 @@ use crate::{ hub::{Global, GlobalIdentityHandlerFactory, HalApi, Resource, Token}, id::{DeviceId, SurfaceId, TextureId, Valid}, init_tracker::{BufferInitTracker, TextureInitTracker}, - track::{TextureSelector}, + track::TextureSelector, validation::MissingBufferUsageError, Label, LifeGuard, RefCount, Stored, }; diff --git a/wgpu-core/src/track/buffer.rs b/wgpu-core/src/track/buffer.rs index 8db353c376..0524b958d3 100644 --- a/wgpu-core/src/track/buffer.rs +++ b/wgpu-core/src/track/buffer.rs @@ -3,7 +3,7 @@ use std::vec::Drain; use super::PendingTransition; use crate::{ hub, - id::{BufferId, TypedId, Valid}, + id::{BufferId, TypedId}, resource::Buffer, track::{ invalid_resource_state, iterate_bitvec, resize_bitvec, skip_barrier, ResourceUses, @@ -33,9 +33,28 @@ impl ResourceUses for BufferUses { } pub struct BufferBindGroupState { - buffers: Vec<(Valid, BufferUses)>, + buffers: Vec<(BufferId, BufferUses)>, +} +impl BufferBindGroupState { + pub fn new() -> Self { + Self { + buffers: Vec::new(), + } + } + + pub fn extend<'a, A: hal::Api>( + &mut self, + storage: &'a hub::Storage, BufferId>, + id: BufferId, + state: BufferUses, + ) -> Option<&'a Buffer> { + self.buffers.push((id, state)); + + storage.get(id).ok() + } } +#[derive(Debug)] pub(crate) struct BufferUsageScope { state: Vec, owned: BitVec, @@ -61,7 +80,7 @@ impl BufferUsageScope { bind_group: &BufferBindGroupState, ) -> Result<(), UsageConflict> { for &(id, state) in &bind_group.buffers { - self.extend(storage, id.0, state)?; + self.extend(storage, id, state)?; } Ok(()) @@ -94,7 +113,7 @@ impl BufferUsageScope { Ok(storage .get(id) - .map_err(|_| UsageConflict::BufferInvalid { id })?) + .map_err(|_| UsageConflict::BufferInvalid { id: id.unzip().0 })?) } unsafe fn extend_inner<'a, A: hal::Api>( @@ -112,7 +131,11 @@ impl BufferUsageScope { let merged_state = current_state | new_state; if invalid_resource_state(merged_state) { - return Err(UsageConflict::from_buffer(index, current_state, new_state)); + return Err(UsageConflict::from_buffer( + index as u32, + current_state, + new_state, + )); } *self.state.get_unchecked_mut(index) = merged_state; @@ -150,11 +173,27 @@ impl BufferTracker { resize_bitvec(&mut self.owned, size); } + pub fn drain(&mut self) -> Drain> { + self.temp.drain(..) + } + + pub unsafe fn change_state<'a, A: hal::Api>( + &mut self, + storage: &'a hub::Storage, BufferId>, + id: BufferId, + state: BufferUses, + ) -> Option<(&'a Buffer, Option>)> { + self.transition_inner(storage, id.unzip().0 as usize, state); + + let value = storage.get(id).ok()?; + Some((value, self.temp.pop())) + } + pub fn change_states_scope( &mut self, storage: &hub::Storage, BufferId>, scope: &BufferUsageScope, - ) -> Drain> { + ) { self.change_states_inner(storage, &scope.state, &scope.owned) } @@ -163,7 +202,7 @@ impl BufferTracker { storage: &hub::Storage, BufferId>, incoming_set: &Vec, incoming_ownership: &BitVec, - ) -> Drain> { + ) { let incoming_size = incoming_set.len(); if incoming_size > self.start.len() { self.set_max_index(incoming_size); @@ -172,32 +211,27 @@ impl BufferTracker { iterate_bitvec(incoming_ownership, |index| { unsafe { self.transition(storage, incoming_set, index) }; }); - - self.temp.drain(..) } pub unsafe fn change_states_bind_group( &mut self, storage: &hub::Storage, BufferId>, - incoming_set: &Vec, - incoming_ownership: &mut BitVec, + scope: &mut BufferUsageScope, bind_group_state: &BufferBindGroupState, - ) -> Drain> { - let incoming_size = incoming_set.len(); + ) { + let incoming_size = scope.state.len(); if incoming_size > self.start.len() { self.set_max_index(incoming_size); } for &(index, _) in bind_group_state.buffers.iter() { - let index = index.0.unzip().0 as usize; - if !incoming_ownership.get(index).unwrap_unchecked() { + let index = index.unzip().0 as usize; + if !scope.owned.get(index).unwrap_unchecked() { continue; } - self.transition(storage, incoming_set, index); - incoming_ownership.set(index, false); + self.transition(storage, &scope.state, index); + scope.owned.set(index, false); } - - self.temp.drain(..) } unsafe fn transition( @@ -205,10 +239,20 @@ impl BufferTracker { storage: &hub::Storage, BufferId>, incoming_set: &Vec, index: usize, + ) { + let new_state = *incoming_set.get_unchecked(index); + + self.transition_inner(storage, index, new_state); + } + + unsafe fn transition_inner( + &mut self, + storage: &hub::Storage, BufferId>, + index: usize, + new_state: BufferUses, ) { let old_tracked = self.owned.get(index).unwrap_unchecked(); let old_state = *self.end.get_unchecked(index); - let new_state = *incoming_set.get_unchecked(index); if old_tracked { if skip_barrier(old_state, new_state) { diff --git a/wgpu-core/src/track/mod.rs b/wgpu-core/src/track/mod.rs index 4b75fd5855..cc4e9a4412 100644 --- a/wgpu-core/src/track/mod.rs +++ b/wgpu-core/src/track/mod.rs @@ -133,12 +133,12 @@ pub enum UsageConflict { TextureInvalid { id: u32 }, #[error("Attempted to use buffer {id:?} with {invalid_use}.")] Buffer { - id: id::BufferId, + id: u32, invalid_use: InvalidUse, }, #[error("Attempted to use a texture {id:?} mips {mip_levels:?} layers {array_layers:?} with {invalid_use}.")] Texture { - id: id::TextureId, + id: u32, mip_levels: ops::Range, array_layers: ops::Range, invalid_use: InvalidUse, @@ -179,7 +179,7 @@ impl UsageConflict { } Self::Texture { - id: id.0, + id, mip_levels: mips, array_layers: layers, invalid_use: InvalidUse { @@ -222,6 +222,17 @@ pub(crate) struct BindGroupStates { pub samplers: StatelessBindGroupSate, id::SamplerId>, } +impl BindGroupStates { + pub fn new() -> Self { + Self { + buffers: BufferBindGroupState::new(), + textures: TextureBindGroupState::new(), + views: StatelessBindGroupSate::new(), + samplers: StatelessBindGroupSate::new(), + } + } +} + pub(crate) struct RenderBundleScope { pub buffers: BufferUsageScope, pub textures: TextureUsageScope, @@ -235,8 +246,8 @@ pub(crate) struct RenderBundleScope { impl RenderBundleScope { pub fn new() -> Self { Self { - buffers: BufferTracker::new(), - textures: TextureTracker::new(), + buffers: BufferUsageScope::new(), + textures: TextureUsageScope::new(), views: StatelessTracker::new(), samplers: StatelessTracker::new(), bind_groups: StatelessTracker::new(), @@ -244,8 +255,23 @@ impl RenderBundleScope { query_sets: StatelessTracker::new(), } } + + pub unsafe fn extend_from_bind_group( + &mut self, + buffers: &hub::Storage, id::BufferId>, + textures: &hub::Storage, id::TextureId>, + bind_group: &BindGroupStates, + ) -> Result<(), UsageConflict> { + self.buffers + .extend_from_bind_group(buffers, &bind_group.buffers)?; + self.textures + .extend_from_bind_group(textures, &bind_group.textures)?; + + Ok(()) + } } +#[derive(Debug)] pub(crate) struct UsageScope { pub buffers: BufferUsageScope, pub textures: TextureUsageScope, @@ -315,24 +341,37 @@ impl Tracker { } } + pub unsafe fn extend_from_bind_group( + &mut self, + buffers: &hub::Storage, id::BufferId>, + textures: &hub::Storage, id::TextureId>, + scope: &mut UsageScope, + bind_group: &BindGroupStates, + ) { + self.buffers + .change_states_bind_group(buffers, &mut scope.buffers, &bind_group.buffers); + self.textures + .change_states_bind_group(textures, &mut scope.textures, &bind_group.textures); + } + pub unsafe fn extend_from_render_bundle( &mut self, - views: hub::Storage, id::TextureViewId>, - samplers: hub::Storage, id::SamplerId>, - bind_groups: hub::Storage, id::BindGroupId>, - render_pipelines: hub::Storage, id::RenderPipelineId>, - query_sets: hub::Storage, id::QuerySetId>, + views: &hub::Storage, id::TextureViewId>, + samplers: &hub::Storage, id::SamplerId>, + bind_groups: &hub::Storage, id::BindGroupId>, + render_pipelines: &hub::Storage, id::RenderPipelineId>, + query_sets: &hub::Storage, id::QuerySetId>, render_bundle: &RenderBundleScope, ) -> Result<(), UsageConflict> { - self.views.extend_from_tracker(&views, &render_bundle.views); + self.views.extend_from_tracker(views, &render_bundle.views); self.samplers - .extend_from_tracker(&samplers, &render_bundle.samplers); + .extend_from_tracker(samplers, &render_bundle.samplers); self.bind_groups - .extend_from_tracker(&bind_groups, &render_bundle.bind_groups); + .extend_from_tracker(bind_groups, &render_bundle.bind_groups); self.render_pipelines - .extend_from_tracker(&render_pipelines, &render_bundle.render_pipelines); + .extend_from_tracker(render_pipelines, &render_bundle.render_pipelines); self.query_sets - .extend_from_tracker(&query_sets, &render_bundle.query_sets); + .extend_from_tracker(query_sets, &render_bundle.query_sets); Ok(()) } diff --git a/wgpu-core/src/track/stateless.rs b/wgpu-core/src/track/stateless.rs index 1edbda884c..7abc60f588 100644 --- a/wgpu-core/src/track/stateless.rs +++ b/wgpu-core/src/track/stateless.rs @@ -2,18 +2,29 @@ use std::marker::PhantomData; use bit_vec::BitVec; -use crate::{ - hub, - id::{TypedId, Valid}, - track::resize_bitvec, -}; +use crate::{hub, id::TypedId, track::resize_bitvec}; pub struct StatelessBindGroupSate { - resource: Vec>, + resource: Vec, _phantom: PhantomData, } +impl StatelessBindGroupSate { + pub fn new() -> Self { + Self { + resource: Vec::new(), + _phantom: PhantomData, + } + } + + pub fn extend<'a>(&mut self, storage: &'a hub::Storage, id: Id) -> Option<&'a T> { + self.resource.push(id); + + storage.get(id).ok() + } +} + pub struct StatelessTracker { owned: BitVec, @@ -44,7 +55,7 @@ impl StatelessTracker { bind_group: &StatelessBindGroupSate, ) { for &id in &bind_group.resource { - self.owned.set(id.0.unzip().0 as usize, true); + self.owned.set(id.unzip().0 as usize, true); } } @@ -54,7 +65,9 @@ impl StatelessTracker { self.set_max_index(incoming_size); } - for (left, &right) in unsafe { self.owned.storage_mut().iter_mut() }.zip(other.owned.storage()) { + for (left, &right) in + unsafe { self.owned.storage_mut().iter_mut() }.zip(other.owned.storage()) + { *left |= right; } } diff --git a/wgpu-core/src/track/texture.rs b/wgpu-core/src/track/texture.rs index affa312a90..d620448d07 100644 --- a/wgpu-core/src/track/texture.rs +++ b/wgpu-core/src/track/texture.rs @@ -1,7 +1,7 @@ use super::{range::RangedStates, PendingTransition}; use crate::{ hub, - id::{TextureId, TypedId, Valid}, + id::{TextureId, TypedId}, resource::Texture, track::{ invalid_resource_state, iterate_bitvec, resize_bitvec, skip_barrier, ResourceUses, @@ -60,9 +60,29 @@ impl ComplexTextureState { // TODO: This representation could be optimized in a couple ways, but keep it simple for now. pub struct TextureBindGroupState { - textures: Vec<(Valid, Option, TextureUses)>, + textures: Vec<(TextureId, Option, TextureUses)>, } +impl TextureBindGroupState { + pub fn new() -> Self { + Self { + textures: Vec::new(), + } + } + + pub fn extend<'a, A: hal::Api>( + &mut self, + storage: &'a hub::Storage, TextureId>, + id: TextureId, + selector: Option, + state: TextureUses, + ) -> Option<&'a Texture> { + self.textures.push((id, selector, state)); + storage.get(id).ok() + } +} + +#[derive(Debug)] pub struct TextureStateSet { simple: Vec, complex: FastHashMap, @@ -76,6 +96,7 @@ impl TextureStateSet { } } +#[derive(Debug)] pub struct TextureUsageScope { set: TextureStateSet, owned: BitVec, @@ -99,17 +120,15 @@ impl TextureUsageScope { &mut self, storage: &hub::Storage, TextureId>, scope: &Self, - ) -> Drain> { + ) -> Result<(), UsageConflict> { let incoming_size = scope.set.simple.len(); if incoming_size > self.set.simple.len() { self.set_max_index(incoming_size); } - iterate_bitvec(&scope.set.simple, |index| { - todo!() - }); + iterate_bitvec(&scope.owned, |index| todo!()); - self.temp.drain(..) + Ok(()) } pub unsafe fn extend_from_bind_group( @@ -147,11 +166,11 @@ impl TextureUsageScope { unsafe fn extend_inner( &mut self, storage: &hub::Storage, TextureId>, - index: u32, + id: u32, selector: Option, new_state: TextureUses, ) -> Result<(), UsageConflict> { - let index = index as usize; + let index = id as usize; let currently_active = self.owned.get(index).unwrap_unchecked(); if currently_active { @@ -164,7 +183,7 @@ impl TextureUsageScope { if invalid_resource_state(merged_state) { return Err(UsageConflict::from_texture( storage, - index, + id, selector, current_state, new_state, @@ -176,19 +195,17 @@ impl TextureUsageScope { return Ok(()); } // The old usage is complex. - (true, selector) => { - return self.extend_complex(storage, index, selector, new_state) - } + (true, selector) => return self.extend_complex(storage, id, selector, new_state), // The old usage is simple, so demote it to a complex one. (false, Some(selector)) => { *self.set.simple.get_unchecked_mut(index) = hal::TextureUses::COMPLEX; // Demote our simple state to a complex one. - self.extend_complex(storage, index, None, current_state)?; + self.extend_complex(storage, id, None, current_state)?; // Extend that complex state with our new complex state. - return self.extend_complex(storage, index, Some(selector), new_state); + return self.extend_complex(storage, id, Some(selector), new_state); } } } @@ -198,7 +215,7 @@ impl TextureUsageScope { if let Some(selector) = selector { *self.set.simple.get_unchecked_mut(index) = hal::TextureUses::COMPLEX; - self.extend_complex(storage, index, Some(selector), new_state)?; + self.extend_complex(storage, id, Some(selector), new_state)?; } else { *self.set.simple.get_unchecked_mut(index) = new_state; } @@ -210,14 +227,14 @@ impl TextureUsageScope { unsafe fn extend_complex( &mut self, storage: &hub::Storage, TextureId>, - index: u32, + id: u32, selector: Option, new_state: TextureUses, ) -> Result<(), UsageConflict> { - let texture = storage.get_unchecked(index); + let texture = storage.get_unchecked(id); // Create the complex entry for this texture. - let complex = self.set.complex.entry(index).or_insert_with(|| { + let complex = self.set.complex.entry(id).or_insert_with(|| { ComplexTextureState::new( texture.desc.mip_level_count, texture.desc.array_layer_count(), @@ -247,7 +264,7 @@ impl TextureUsageScope { if invalid_resource_state(merged) { return Err(UsageConflict::from_texture( storage, - index, + id, selector, *current_state, new_state, @@ -287,11 +304,25 @@ impl TextureTracker { resize_bitvec(&mut self.owned, size); } + pub fn drain(&mut self) -> Drain> { + self.temp.drain(..) + } + + pub fn change_state<'a, A: hal::Api>( + &mut self, + storage: &'a hub::Storage, TextureId>, + id: TextureId, + selector: TextureSelector, + new_usage: TextureUses, + ) -> Option<(&'a Texture, Option>)> { + todo!() + } + pub fn change_states_scope( &mut self, storage: &hub::Storage, TextureId>, scope: &TextureUsageScope, - ) -> Drain> { + ) { self.change_states_inner(storage, &scope.set, &scope.owned) } @@ -300,7 +331,7 @@ impl TextureTracker { storage: &hub::Storage, TextureId>, incoming_set: &TextureStateSet, incoming_ownership: &BitVec, - ) -> Drain> { + ) { let incoming_size = incoming_set.simple.len(); if incoming_size > self.start_set.simple.len() { self.set_max_index(incoming_size); @@ -309,32 +340,27 @@ impl TextureTracker { iterate_bitvec(incoming_ownership, |index| { unsafe { self.transition(storage, incoming_set, index) }; }); - - self.temp.drain(..) } pub unsafe fn change_states_bind_group( &mut self, storage: &hub::Storage, TextureId>, - incoming_set: &TextureStateSet, - incoming_ownership: &mut BitVec, + scope: &mut TextureUsageScope, bind_group_state: &TextureBindGroupState, - ) -> Drain> { - let incoming_size = incoming_set.simple.len(); + ) { + let incoming_size = scope.set.simple.len(); if incoming_size > self.start_set.simple.len() { self.set_max_index(incoming_size); } for &(index, _, _) in bind_group_state.textures.iter() { - let index = index.0.unzip().0 as usize; - if !incoming_ownership.get(index).unwrap_unchecked() { + let index = index.unzip().0 as usize; + if !scope.owned.get(index).unwrap_unchecked() { continue; } - self.transition(storage, incoming_set, index); - incoming_ownership.set(index, false); + self.transition(storage, &scope.set, index); + scope.owned.set(index, false); } - - self.temp.drain(..) } unsafe fn transition( @@ -379,7 +405,7 @@ impl TextureTracker { self.temp.push(PendingTransition { id: index as u32, - selector: storage.get_unchecked(index as u32).unwrap().full_range, + selector: storage.get_unchecked(index as u32).full_range, usage: old_state..new_state, }); From 523599178cffa2a6eb52833d611eef5a2605190f Mon Sep 17 00:00:00 2001 From: Connor Fitzgerald Date: Tue, 3 May 2022 23:46:59 -0400 Subject: [PATCH 08/35] temp7 - pro-gress --- wgpu-core/src/device/mod.rs | 114 +++++++++++-------------------- wgpu-core/src/present.rs | 13 +--- wgpu-core/src/track/buffer.rs | 8 +++ wgpu-core/src/track/stateless.rs | 6 ++ wgpu-core/src/track/texture.rs | 9 +++ 5 files changed, 65 insertions(+), 85 deletions(-) diff --git a/wgpu-core/src/device/mod.rs b/wgpu-core/src/device/mod.rs index 5fd07af03b..ef705874d7 100644 --- a/wgpu-core/src/device/mod.rs +++ b/wgpu-core/src/device/mod.rs @@ -1633,7 +1633,7 @@ impl Device { internal_use, ) .ok_or_else(|| { - binding_model::CreateBindGroupError::InvalidTexture(view.parent_id.value) + binding_model::CreateBindGroupError::InvalidTexture(view.parent_id.value.0) })?; check_texture_usage(texture.desc.usage, pub_usage)?; @@ -3182,12 +3182,7 @@ impl Global { let id = fid.assign(buffer, &mut token); log::info!("Created buffer {:?} with {:?}", id, desc); - device - .trackers - .lock() - .buffers - .init(id, ref_count, BufferState::with_usage(buffer_use)) - .unwrap(); + unsafe { device.trackers.lock().buffers.init(id.0, buffer_use) }; return (id.0, None); }; @@ -3455,12 +3450,13 @@ impl Global { let id = fid.assign(texture, &mut token); log::info!("Created texture {:?} with {:?}", id, desc); - device - .trackers - .lock() - .textures - .init(id, ref_count, OldTextureState::new(num_levels, num_layers)) - .unwrap(); + unsafe { + device + .trackers + .lock() + .textures + .init(id.0, hal::TextureUses::UNINITIALIZED) + }; return (id.0, None); }; @@ -3533,12 +3529,13 @@ impl Global { let id = fid.assign(texture, &mut token); log::info!("Created texture {:?} with {:?}", id, desc); - device - .trackers - .lock() - .textures - .init(id, ref_count, OldTextureState::new(num_levels, num_layers)) - .unwrap(); + unsafe { + device + .trackers + .lock() + .textures + .init(id.0, hal::TextureUses::UNINITIALIZED) + }; return (id.0, None); }; @@ -3696,12 +3693,7 @@ impl Global { let ref_count = view.life_guard.add_ref(); let id = fid.assign(view, &mut token); - device - .trackers - .lock() - .views - .init(id, ref_count, PhantomData) - .unwrap(); + unsafe { device.trackers.lock().views.init(id.0) }; return (id.0, None); }; @@ -3794,12 +3786,9 @@ impl Global { let ref_count = sampler.life_guard.add_ref(); let id = fid.assign(sampler, &mut token); - device - .trackers - .lock() - .samplers - .init(id, ref_count, PhantomData) - .unwrap(); + unsafe { + device.trackers.lock().samplers.init(id.0); + } return (id.0, None); }; @@ -4057,18 +4046,9 @@ impl Global { let ref_count = bind_group.life_guard.add_ref(); let id = fid.assign(bind_group, &mut token); - log::debug!( - "Bind group {:?} {:#?}", - id, - hub.bind_groups.read(&mut token).0[id].used - ); + log::debug!("Bind group {:?}", id,); - device - .trackers - .lock() - .bind_groups - .init(id, ref_count, PhantomData) - .unwrap(); + unsafe { device.trackers.lock().bind_groups.init(id.0) }; return (id.0, None); }; @@ -4371,16 +4351,11 @@ impl Global { Err(e) => break e, }; - log::debug!("Render bundle {:#?}", render_bundle.used); + log::debug!("Render bundle"); let ref_count = render_bundle.life_guard.add_ref(); let id = fid.assign(render_bundle, &mut token); - device - .trackers - .lock() - .bundles - .init(id, ref_count, PhantomData) - .unwrap(); + unsafe { device.trackers.lock().bundles.init(id.0) }; return (id.0, None); }; @@ -4455,12 +4430,9 @@ impl Global { let ref_count = query_set.life_guard.add_ref(); let id = fid.assign(query_set, &mut token); - device - .trackers - .lock() - .query_sets - .init(id, ref_count, PhantomData) - .unwrap(); + unsafe { + device.trackers.lock().query_sets.init(id.0); + } return (id.0, None); }; @@ -4551,12 +4523,9 @@ impl Global { let id = fid.assign(pipeline, &mut token); log::info!("Created render pipeline {:?} with {:?}", id, desc); - device - .trackers - .lock() - .render_pipes - .init(id, ref_count, PhantomData) - .unwrap(); + unsafe { + device.trackers.lock().render_pipelines.init(id.0); + } return (id.0, None); }; @@ -4692,12 +4661,7 @@ impl Global { let id = fid.assign(pipeline, &mut token); log::info!("Created compute pipeline {:?} with {:?}", id, desc); - device - .trackers - .lock() - .compute_pipes - .init(id, ref_count, PhantomData) - .unwrap(); + unsafe { device.trackers.lock().compute_pipelines.init(id.0) }; return (id.0, None); }; @@ -5155,16 +5119,20 @@ impl Global { }; log::debug!("Buffer {:?} map state -> Waiting", buffer_id); + let device = &device_guard[buffer.device_id.value]; + + unsafe { + let trackers = device.trackers.lock(); + trackers + .buffers + .change_state(&*buffer_guard, buffer_id, internal_use); + trackers.buffers.drain(); + }; + (buffer.device_id.value, buffer.life_guard.add_ref()) }; let device = &device_guard[device_id]; - device.trackers.lock().buffers.change_replace( - id::Valid(buffer_id), - &ref_count, - (), - internal_use, - ); device .lock_life(&mut token) diff --git a/wgpu-core/src/present.rs b/wgpu-core/src/present.rs index 9ae07df347..fb21c67c3d 100644 --- a/wgpu-core/src/present.rs +++ b/wgpu-core/src/present.rs @@ -187,20 +187,9 @@ impl Global { let id = fid.assign(texture, &mut token); { - use track::ResourceState as _; // register it in the device tracker as uninitialized let mut trackers = device.trackers.lock(); - let mut ts = track::OldTextureState::default(); - let _ = ts.change( - id, - track::TextureSelector { - layers: 0..1, - mips: 0..1, - }, - hal::TextureUses::UNINITIALIZED, - None, - ); - let _ = trackers.textures.init(id, ref_count.clone(), ts); + let _ = unsafe { trackers.textures.init(id.0, hal::TextureUses::UNINITIALIZED) }; } if present.acquired_texture.is_some() { diff --git a/wgpu-core/src/track/buffer.rs b/wgpu-core/src/track/buffer.rs index 0524b958d3..a4ae57061f 100644 --- a/wgpu-core/src/track/buffer.rs +++ b/wgpu-core/src/track/buffer.rs @@ -177,6 +177,14 @@ impl BufferTracker { self.temp.drain(..) } + pub unsafe fn init(&mut self, id: BufferId, state: BufferUses) { + let index = id.unzip().0 as usize; + *self.start.get_unchecked_mut(index) = state; + *self.end.get_unchecked_mut(index) = state; + + self.owned.set(index, true); + } + pub unsafe fn change_state<'a, A: hal::Api>( &mut self, storage: &'a hub::Storage, BufferId>, diff --git a/wgpu-core/src/track/stateless.rs b/wgpu-core/src/track/stateless.rs index 7abc60f588..77a6a31d24 100644 --- a/wgpu-core/src/track/stateless.rs +++ b/wgpu-core/src/track/stateless.rs @@ -43,6 +43,12 @@ impl StatelessTracker { resize_bitvec(&mut self.owned, size); } + pub unsafe fn init(&mut self, id: Id) { + let index = id.unzip().0 as usize; + + self.owned.set(index, true); + } + pub unsafe fn extend<'a>(&mut self, storage: &'a hub::Storage, id: Id) -> Option<&'a T> { self.owned.set(id.unzip().0 as usize, true); diff --git a/wgpu-core/src/track/texture.rs b/wgpu-core/src/track/texture.rs index d620448d07..518c7a1639 100644 --- a/wgpu-core/src/track/texture.rs +++ b/wgpu-core/src/track/texture.rs @@ -308,6 +308,15 @@ impl TextureTracker { self.temp.drain(..) } + pub unsafe fn init(&mut self, id: TextureId, usage: TextureUses) { + let index = id.unzip().0 as usize; + + *self.start_set.simple.get_unchecked_mut(index) = usage; + *self.end_set.simple.get_unchecked_mut(index) = usage; + + self.owned.set(index, true); + } + pub fn change_state<'a, A: hal::Api>( &mut self, storage: &'a hub::Storage, TextureId>, From 483002a567ba04d9fff8e708ef1d458ca69cb323 Mon Sep 17 00:00:00 2001 From: Connor Fitzgerald Date: Thu, 5 May 2022 00:36:16 -0400 Subject: [PATCH 09/35] temp8 - Add refcounts to trackers --- wgpu-core/src/device/life.rs | 2 +- wgpu-core/src/device/mod.rs | 33 ++++++------ wgpu-core/src/present.rs | 6 ++- wgpu-core/src/track/buffer.rs | 88 ++++++++++++++++++++++---------- wgpu-core/src/track/stateless.rs | 67 ++++++++++++++++++------ wgpu-core/src/track/texture.rs | 60 +++++++++++++++++----- 6 files changed, 181 insertions(+), 75 deletions(-) diff --git a/wgpu-core/src/device/life.rs b/wgpu-core/src/device/life.rs index 9d69ae52d6..c1b9211fbd 100644 --- a/wgpu-core/src/device/life.rs +++ b/wgpu-core/src/device/life.rs @@ -829,7 +829,7 @@ impl LifetimeTracker { &mut self, hub: &Hub, raw: &A::Device, - trackers: &Mutex, + trackers: &Mutex>, token: &mut Token>, ) -> Vec { if self.ready_to_map.is_empty() { diff --git a/wgpu-core/src/device/mod.rs b/wgpu-core/src/device/mod.rs index ef705874d7..59feb758e2 100644 --- a/wgpu-core/src/device/mod.rs +++ b/wgpu-core/src/device/mod.rs @@ -1624,11 +1624,12 @@ impl Device { ) -> Result<(), binding_model::CreateBindGroupError> { // Careful here: the texture may no longer have its own ref count, // if it was deleted by the user. - let texture = &texture_guard[view.parent_id.value]; - used.textures - .extend( + let texture = used + .textures + .extend_with_refcount( texture_guard, view.parent_id.value.0, + view.parent_id.ref_count.clone(), Some(view.selector.clone()), internal_use, ) @@ -3451,11 +3452,11 @@ impl Global { log::info!("Created texture {:?} with {:?}", id, desc); unsafe { - device - .trackers - .lock() - .textures - .init(id.0, hal::TextureUses::UNINITIALIZED) + device.trackers.lock().textures.init( + id.0, + ref_count, + hal::TextureUses::UNINITIALIZED, + ) }; return (id.0, None); }; @@ -3534,7 +3535,7 @@ impl Global { .trackers .lock() .textures - .init(id.0, hal::TextureUses::UNINITIALIZED) + .init(id.0, ref_count, hal::TextureUses::UNINITIALIZED) }; return (id.0, None); }; @@ -3693,7 +3694,7 @@ impl Global { let ref_count = view.life_guard.add_ref(); let id = fid.assign(view, &mut token); - unsafe { device.trackers.lock().views.init(id.0) }; + unsafe { device.trackers.lock().views.init(id.0, ref_count) }; return (id.0, None); }; @@ -3787,7 +3788,7 @@ impl Global { let id = fid.assign(sampler, &mut token); unsafe { - device.trackers.lock().samplers.init(id.0); + device.trackers.lock().samplers.init(id.0, ref_count); } return (id.0, None); }; @@ -4048,7 +4049,7 @@ impl Global { let id = fid.assign(bind_group, &mut token); log::debug!("Bind group {:?}", id,); - unsafe { device.trackers.lock().bind_groups.init(id.0) }; + unsafe { device.trackers.lock().bind_groups.init(id.0, ref_count) }; return (id.0, None); }; @@ -4355,7 +4356,7 @@ impl Global { let ref_count = render_bundle.life_guard.add_ref(); let id = fid.assign(render_bundle, &mut token); - unsafe { device.trackers.lock().bundles.init(id.0) }; + unsafe { device.trackers.lock().bundles.init(id.0, ref_count) }; return (id.0, None); }; @@ -4431,7 +4432,7 @@ impl Global { let id = fid.assign(query_set, &mut token); unsafe { - device.trackers.lock().query_sets.init(id.0); + device.trackers.lock().query_sets.init(id.0, ref_count); } return (id.0, None); @@ -4524,7 +4525,7 @@ impl Global { log::info!("Created render pipeline {:?} with {:?}", id, desc); unsafe { - device.trackers.lock().render_pipelines.init(id.0); + device.trackers.lock().render_pipelines.init(id.0, ref_count); } return (id.0, None); }; @@ -4661,7 +4662,7 @@ impl Global { let id = fid.assign(pipeline, &mut token); log::info!("Created compute pipeline {:?} with {:?}", id, desc); - unsafe { device.trackers.lock().compute_pipelines.init(id.0) }; + unsafe { device.trackers.lock().compute_pipelines.init(id.0, ref_count) }; return (id.0, None); }; diff --git a/wgpu-core/src/present.rs b/wgpu-core/src/present.rs index fb21c67c3d..b47bd7bbc1 100644 --- a/wgpu-core/src/present.rs +++ b/wgpu-core/src/present.rs @@ -189,7 +189,11 @@ impl Global { { // register it in the device tracker as uninitialized let mut trackers = device.trackers.lock(); - let _ = unsafe { trackers.textures.init(id.0, hal::TextureUses::UNINITIALIZED) }; + let _ = unsafe { + trackers + .textures + .init(id.0, ref_count, hal::TextureUses::UNINITIALIZED) + }; } if present.acquired_texture.is_some() { diff --git a/wgpu-core/src/track/buffer.rs b/wgpu-core/src/track/buffer.rs index a4ae57061f..01c1a9fc71 100644 --- a/wgpu-core/src/track/buffer.rs +++ b/wgpu-core/src/track/buffer.rs @@ -9,6 +9,7 @@ use crate::{ invalid_resource_state, iterate_bitvec, resize_bitvec, skip_barrier, ResourceUses, UsageConflict, }, + RefCount, }; use bit_vec::BitVec; use hal::BufferUses; @@ -33,7 +34,7 @@ impl ResourceUses for BufferUses { } pub struct BufferBindGroupState { - buffers: Vec<(BufferId, BufferUses)>, + buffers: Vec<(BufferId, RefCount, BufferUses)>, } impl BufferBindGroupState { pub fn new() -> Self { @@ -48,9 +49,11 @@ impl BufferBindGroupState { id: BufferId, state: BufferUses, ) -> Option<&'a Buffer> { - self.buffers.push((id, state)); + let buffer = storage.get(id).ok()?; - storage.get(id).ok() + self.buffers.push((id, buffer.life_guard.add_ref(), state)); + + Some(buffer) } } @@ -58,6 +61,7 @@ impl BufferBindGroupState { pub(crate) struct BufferUsageScope { state: Vec, owned: BitVec, + ref_count: Vec>, } impl BufferUsageScope { @@ -65,11 +69,13 @@ impl BufferUsageScope { Self { state: Vec::new(), owned: BitVec::default(), + ref_count: Vec::new(), } } pub fn set_max_index(&mut self, size: usize) { self.state.resize(size, BufferUses::empty()); + self.ref_count.resize(size, None); resize_bitvec(&mut self.owned, size); } @@ -79,8 +85,8 @@ impl BufferUsageScope { storage: &hub::Storage, BufferId>, bind_group: &BufferBindGroupState, ) -> Result<(), UsageConflict> { - for &(id, state) in &bind_group.buffers { - self.extend(storage, id, state)?; + for (id, ref_count, state) in &bind_group.buffers { + self.extend_inner(storage, id.unzip().0, ref_count, *state)?; } Ok(()) @@ -97,7 +103,18 @@ impl BufferUsageScope { } iterate_bitvec(&scope.owned, |index| { - unsafe { self.extend_inner(storage, index as u32, *scope.state.get_unchecked(index)) }; + unsafe { + self.extend_inner( + storage, + index as u32, + scope + .ref_count + .get_unchecked(index) + .as_ref() + .unwrap_unchecked(), + *scope.state.get_unchecked(index), + ) + }; }); Ok(()) @@ -109,17 +126,25 @@ impl BufferUsageScope { id: BufferId, new_state: BufferUses, ) -> Result<&'a Buffer, UsageConflict> { - self.extend_inner(storage, id.unzip().0, new_state)?; - - Ok(storage + let buffer = storage .get(id) - .map_err(|_| UsageConflict::BufferInvalid { id: id.unzip().0 })?) + .map_err(|_| UsageConflict::BufferInvalid { id: id.unzip().0 })?; + + self.extend_inner( + storage, + id.unzip().0, + buffer.life_guard.ref_count.as_ref().unwrap(), + new_state, + )?; + + Ok(buffer) } unsafe fn extend_inner<'a, A: hal::Api>( &mut self, storage: &'a hub::Storage, BufferId>, index: u32, + ref_count: &RefCount, new_state: BufferUses, ) -> Result<(), UsageConflict> { let index = index as usize; @@ -143,6 +168,7 @@ impl BufferUsageScope { // We're the first to use this resource, let's add it. self.owned.set(index, true); + *self.ref_count.get_unchecked_mut(index) = Some(ref_count.clone()); *self.state.get_unchecked_mut(index) = new_state; @@ -155,6 +181,7 @@ pub(crate) struct BufferTracker { end: Vec, temp: Vec>, owned: BitVec, + ref_count: Vec>, } impl BufferTracker { pub fn new() -> Self { @@ -163,12 +190,14 @@ impl BufferTracker { end: Vec::new(), temp: Vec::new(), owned: BitVec::default(), + ref_count: Vec::new(), } } fn set_max_index(&mut self, size: usize) { self.start.resize(size, BufferUses::empty()); self.end.resize(size, BufferUses::empty()); + self.ref_count.resize(size, None); resize_bitvec(&mut self.owned, size); } @@ -191,9 +220,15 @@ impl BufferTracker { id: BufferId, state: BufferUses, ) -> Option<(&'a Buffer, Option>)> { - self.transition_inner(storage, id.unzip().0 as usize, state); - let value = storage.get(id).ok()?; + + self.transition_inner( + storage, + id.unzip().0 as usize, + value.life_guard.ref_count.as_ref().unwrap(), + state, + ); + Some((value, self.temp.pop())) } @@ -202,22 +237,19 @@ impl BufferTracker { storage: &hub::Storage, BufferId>, scope: &BufferUsageScope, ) { - self.change_states_inner(storage, &scope.state, &scope.owned) - } - - fn change_states_inner( - &mut self, - storage: &hub::Storage, BufferId>, - incoming_set: &Vec, - incoming_ownership: &BitVec, - ) { - let incoming_size = incoming_set.len(); + let incoming_size = scope.state.len(); if incoming_size > self.start.len() { self.set_max_index(incoming_size); } - iterate_bitvec(incoming_ownership, |index| { - unsafe { self.transition(storage, incoming_set, index) }; + iterate_bitvec(&scope.owned, |index| unsafe { + let ref_count = scope + .ref_count + .get_unchecked(index) + .as_ref() + .unwrap_unchecked(); + + self.transition(storage, &scope.state, ref_count, index); }); } @@ -232,12 +264,12 @@ impl BufferTracker { self.set_max_index(incoming_size); } - for &(index, _) in bind_group_state.buffers.iter() { + for (index, ref_count, _) in bind_group_state.buffers.iter() { let index = index.unzip().0 as usize; if !scope.owned.get(index).unwrap_unchecked() { continue; } - self.transition(storage, &scope.state, index); + self.transition(storage, &scope.state, ref_count, index); scope.owned.set(index, false); } } @@ -246,17 +278,19 @@ impl BufferTracker { &mut self, storage: &hub::Storage, BufferId>, incoming_set: &Vec, + ref_count: &RefCount, index: usize, ) { let new_state = *incoming_set.get_unchecked(index); - self.transition_inner(storage, index, new_state); + self.transition_inner(storage, index, ref_count, new_state); } unsafe fn transition_inner( &mut self, storage: &hub::Storage, BufferId>, index: usize, + ref_count: &RefCount, new_state: BufferUses, ) { let old_tracked = self.owned.get(index).unwrap_unchecked(); diff --git a/wgpu-core/src/track/stateless.rs b/wgpu-core/src/track/stateless.rs index 77a6a31d24..887052e238 100644 --- a/wgpu-core/src/track/stateless.rs +++ b/wgpu-core/src/track/stateless.rs @@ -2,15 +2,20 @@ use std::marker::PhantomData; use bit_vec::BitVec; -use crate::{hub, id::TypedId, track::resize_bitvec}; +use crate::{ + hub, + id::TypedId, + track::{iterate_bitvec, resize_bitvec}, + RefCount, +}; pub struct StatelessBindGroupSate { - resource: Vec, + resource: Vec<(Id, RefCount)>, _phantom: PhantomData, } -impl StatelessBindGroupSate { +impl StatelessBindGroupSate { pub fn new() -> Self { Self { resource: Vec::new(), @@ -18,41 +23,55 @@ impl StatelessBindGroupSate { } } - pub fn extend<'a>(&mut self, storage: &'a hub::Storage, id: Id) -> Option<&'a T> { - self.resource.push(id); + pub fn extend<'a>( + &mut self, + storage: &'a hub::Storage, + id: Id, + ) -> Option<&'a T> { + let resource = storage.get(id).ok()?; + + self.resource.push((id, resource.life_guard().add_ref())); - storage.get(id).ok() + Some(resource) } } pub struct StatelessTracker { owned: BitVec, + ref_counts: Vec>, _phantom: PhantomData<(T, Id)>, } -impl StatelessTracker { +impl StatelessTracker { pub fn new() -> Self { Self { owned: BitVec::default(), + ref_counts: Vec::new(), _phantom: PhantomData, } } pub fn set_max_index(&mut self, size: usize) { + self.ref_counts.resize(size, None); resize_bitvec(&mut self.owned, size); } - pub unsafe fn init(&mut self, id: Id) { + pub unsafe fn init(&mut self, id: Id, ref_count: RefCount) { let index = id.unzip().0 as usize; self.owned.set(index, true); + *self.ref_counts.get_unchecked_mut(index) = Some(ref_count); } pub unsafe fn extend<'a>(&mut self, storage: &'a hub::Storage, id: Id) -> Option<&'a T> { - self.owned.set(id.unzip().0 as usize, true); + let item = storage.get(id).ok()?; + let index = id.unzip().0 as usize; + + self.owned.set(index, true); + *self.ref_counts.get_unchecked_mut(index) = Some(item.life_guard().add_ref()); - storage.get(id).ok() + Some(item) } pub unsafe fn extend_from_bind_group( @@ -60,8 +79,13 @@ impl StatelessTracker { storage: &hub::Storage, bind_group: &StatelessBindGroupSate, ) { - for &id in &bind_group.resource { - self.owned.set(id.unzip().0 as usize, true); + for (id, ref_count) in &bind_group.resource { + let index = id.unzip().0 as usize; + let previously_owned = self.owned.get(index).unwrap_unchecked(); + if !previously_owned { + self.owned.set(id.unzip().0 as usize, true); + *self.ref_counts.get_unchecked_mut(index) = Some(ref_count.clone()); + } } } @@ -71,10 +95,19 @@ impl StatelessTracker { self.set_max_index(incoming_size); } - for (left, &right) in - unsafe { self.owned.storage_mut().iter_mut() }.zip(other.owned.storage()) - { - *left |= right; - } + iterate_bitvec(&other.owned, |index| unsafe { + let previously_owned = self.owned.get(index).unwrap_unchecked(); + + if !previously_owned { + self.owned.set(index, true); + + let other_ref_count = other + .ref_counts + .get_unchecked(index) + .unwrap_unchecked() + .clone(); + *self.ref_counts.get_unchecked_mut(index) = Some(other_ref_count); + } + }) } } diff --git a/wgpu-core/src/track/texture.rs b/wgpu-core/src/track/texture.rs index 518c7a1639..37791246bf 100644 --- a/wgpu-core/src/track/texture.rs +++ b/wgpu-core/src/track/texture.rs @@ -7,6 +7,7 @@ use crate::{ invalid_resource_state, iterate_bitvec, resize_bitvec, skip_barrier, ResourceUses, UsageConflict, }, + RefCount, }; use bit_vec::BitVec; use hal::TextureUses; @@ -60,7 +61,7 @@ impl ComplexTextureState { // TODO: This representation could be optimized in a couple ways, but keep it simple for now. pub struct TextureBindGroupState { - textures: Vec<(TextureId, Option, TextureUses)>, + textures: Vec<(TextureId, Option, RefCount, TextureUses)>, } impl TextureBindGroupState { pub fn new() -> Self { @@ -69,14 +70,15 @@ impl TextureBindGroupState { } } - pub fn extend<'a, A: hal::Api>( + pub fn extend_with_refcount<'a, A: hal::Api>( &mut self, storage: &'a hub::Storage, TextureId>, id: TextureId, + ref_count: RefCount, selector: Option, state: TextureUses, ) -> Option<&'a Texture> { - self.textures.push((id, selector, state)); + self.textures.push((id, selector, ref_count, state)); storage.get(id).ok() } @@ -100,6 +102,7 @@ impl TextureStateSet { pub struct TextureUsageScope { set: TextureStateSet, owned: BitVec, + ref_counts: Vec>, } impl TextureUsageScope { @@ -107,11 +110,13 @@ impl TextureUsageScope { Self { set: TextureStateSet::new(), owned: BitVec::default(), + ref_counts: Vec::new(), } } pub fn set_max_index(&mut self, size: usize) { self.set.simple.resize(size, TextureUses::UNINITIALIZED); + self.ref_counts.resize(size, None); resize_bitvec(&mut self.owned, size); } @@ -136,8 +141,8 @@ impl TextureUsageScope { storage: &hub::Storage, TextureId>, bind_group: &TextureBindGroupState, ) -> Result<(), UsageConflict> { - for (id, selector, state) in &bind_group.textures { - self.extend(storage, *id, selector.clone(), *state)?; + for (id, selector, ref_count, state) in &bind_group.textures { + self.extend_inner(storage, id.unzip().0, selector.clone(), ref_count, *state)?; } Ok(()) @@ -154,10 +159,19 @@ impl TextureUsageScope { new_state: TextureUses, ) -> Result<&'a Texture, UsageConflict> { let (index, _, _) = id.unzip(); - self.extend_inner(storage, index, selector, new_state)?; - Ok(storage + let tex = storage .get(id) - .map_err(|_| UsageConflict::TextureInvalid { id: index })?) + .map_err(|_| UsageConflict::TextureInvalid { id: index })?; + + self.extend_inner( + storage, + index, + selector, + tex.life_guard.ref_count.as_ref().unwrap(), + new_state, + )?; + + Ok(tex) } /// # Safety @@ -168,6 +182,7 @@ impl TextureUsageScope { storage: &hub::Storage, TextureId>, id: u32, selector: Option, + ref_count: &RefCount, new_state: TextureUses, ) -> Result<(), UsageConflict> { let index = id as usize; @@ -211,6 +226,7 @@ impl TextureUsageScope { } // We're the first to use this resource, let's add it. + *self.ref_counts.get_unchecked_mut(index) = Some(ref_count.clone()); self.owned.set(index, true); if let Some(selector) = selector { @@ -284,6 +300,7 @@ pub(crate) struct TextureTracker { /// Temporary storage for collecting transitions. temp: Vec>, owned: BitVec, + ref_counts: Vec>, } impl TextureTracker { pub fn new() -> Self { @@ -292,6 +309,7 @@ impl TextureTracker { end_set: TextureStateSet::new(), temp: Vec::new(), owned: BitVec::default(), + ref_counts: Vec::new(), } } @@ -300,6 +318,7 @@ impl TextureTracker { .simple .resize(size, TextureUses::UNINITIALIZED); self.end_set.simple.resize(size, TextureUses::UNINITIALIZED); + self.ref_counts.resize(size, None); resize_bitvec(&mut self.owned, size); } @@ -308,13 +327,14 @@ impl TextureTracker { self.temp.drain(..) } - pub unsafe fn init(&mut self, id: TextureId, usage: TextureUses) { + pub unsafe fn init(&mut self, id: TextureId, ref_count: RefCount, usage: TextureUses) { let index = id.unzip().0 as usize; *self.start_set.simple.get_unchecked_mut(index) = usage; *self.end_set.simple.get_unchecked_mut(index) = usage; self.owned.set(index, true); + *self.ref_counts.get_unchecked_mut(index) = Some(ref_count); } pub fn change_state<'a, A: hal::Api>( @@ -332,7 +352,7 @@ impl TextureTracker { storage: &hub::Storage, TextureId>, scope: &TextureUsageScope, ) { - self.change_states_inner(storage, &scope.set, &scope.owned) + self.change_states_inner(storage, &scope.set, &scope.owned, &scope.ref_counts) } fn change_states_inner( @@ -340,6 +360,7 @@ impl TextureTracker { storage: &hub::Storage, TextureId>, incoming_set: &TextureStateSet, incoming_ownership: &BitVec, + incoming_ref_counts: &Vec>, ) { let incoming_size = incoming_set.simple.len(); if incoming_size > self.start_set.simple.len() { @@ -347,7 +368,7 @@ impl TextureTracker { } iterate_bitvec(incoming_ownership, |index| { - unsafe { self.transition(storage, incoming_set, index) }; + unsafe { self.transition(storage, incoming_set, incoming_ref_counts, index) }; }); } @@ -362,12 +383,12 @@ impl TextureTracker { self.set_max_index(incoming_size); } - for &(index, _, _) in bind_group_state.textures.iter() { + for &(index, _, _, _) in bind_group_state.textures.iter() { let index = index.unzip().0 as usize; if !scope.owned.get(index).unwrap_unchecked() { continue; } - self.transition(storage, &scope.set, index); + self.transition(storage, &scope.set, &scope.ref_counts, index); scope.owned.set(index, false); } } @@ -376,6 +397,7 @@ impl TextureTracker { &mut self, storage: &hub::Storage, TextureId>, incoming_set: &TextureStateSet, + incoming_ref_counts: &Vec>, index: usize, ) { let old_tracked = self.owned.get(index).unwrap_unchecked(); @@ -392,6 +414,12 @@ impl TextureTracker { *self.end_set.simple.get_unchecked_mut(index) = new_state; self.owned.set(index, true); + + let ref_count = incoming_ref_counts + .get_unchecked(index) + .unwrap_unchecked() + .clone(); + *self.ref_counts.get_unchecked_mut(index) = Some(ref_count); } (false, _, true) => { *self.start_set.simple.get_unchecked_mut(index) = TextureUses::COMPLEX; @@ -406,6 +434,12 @@ impl TextureTracker { .insert(index as u32, complex_state.clone()); self.owned.set(index, true); + + let ref_count = incoming_ref_counts + .get_unchecked(index) + .unwrap_unchecked() + .clone(); + *self.ref_counts.get_unchecked_mut(index) = Some(ref_count); } (true, false, false) => { if skip_barrier(old_state, new_state) { From f94a4514f5aa0e295fb113215321db35360a0818 Mon Sep 17 00:00:00 2001 From: Connor Fitzgerald Date: Sat, 7 May 2022 01:59:22 -0400 Subject: [PATCH 10/35] temp9 - Soundness, generics, refcounts, and epochs --- wgpu-core/src/binding_model.rs | 8 +- wgpu-core/src/command/bundle.rs | 18 +- wgpu-core/src/command/clear.rs | 8 +- wgpu-core/src/command/compute.rs | 8 +- wgpu-core/src/command/memory_init.rs | 8 +- wgpu-core/src/command/mod.rs | 10 +- wgpu-core/src/command/render.rs | 6 +- wgpu-core/src/command/transfer.rs | 10 +- wgpu-core/src/device/life.rs | 3 +- wgpu-core/src/device/mod.rs | 58 +++--- wgpu-core/src/device/queue.rs | 2 +- wgpu-core/src/hub.rs | 118 ++++++----- wgpu-core/src/track/buffer.rs | 245 ++++++++++++++++------- wgpu-core/src/track/mod.rs | 123 ++++++------ wgpu-core/src/track/stateless.rs | 114 +++++++---- wgpu-core/src/track/texture.rs | 282 +++++++++++++++++++-------- 16 files changed, 657 insertions(+), 364 deletions(-) diff --git a/wgpu-core/src/binding_model.rs b/wgpu-core/src/binding_model.rs index 2c8e695bc3..dae4771fcb 100644 --- a/wgpu-core/src/binding_model.rs +++ b/wgpu-core/src/binding_model.rs @@ -1,7 +1,7 @@ use crate::{ device::{DeviceError, MissingDownlevelFlags, MissingFeatures, SHADER_STAGE_COUNT}, error::{ErrorFormatter, PrettyError}, - hub::Resource, + hub::{Resource, HalApi}, id::{BindGroupLayoutId, BufferId, DeviceId, SamplerId, TextureId, TextureViewId, Valid}, init_tracker::{BufferInitTrackerAction, TextureInitTrackerAction}, track::{BindGroupStates, UsageConflict}, @@ -708,7 +708,7 @@ pub(crate) fn buffer_binding_type_alignment( } } -pub struct BindGroup { +pub struct BindGroup { pub(crate) raw: A::BindGroup, pub(crate) device_id: Stored, pub(crate) layout_id: Valid, @@ -722,7 +722,7 @@ pub struct BindGroup { pub(crate) late_buffer_binding_sizes: Vec, } -impl BindGroup { +impl BindGroup { pub(crate) fn validate_dynamic_bindings( &self, offsets: &[wgt::DynamicOffset], @@ -764,7 +764,7 @@ impl BindGroup { } } -impl Resource for BindGroup { +impl Resource for BindGroup { const TYPE: &'static str = "BindGroup"; fn life_guard(&self) -> &LifeGuard { diff --git a/wgpu-core/src/command/bundle.rs b/wgpu-core/src/command/bundle.rs index 73b73b61b6..533fb232b9 100644 --- a/wgpu-core/src/command/bundle.rs +++ b/wgpu-core/src/command/bundle.rs @@ -45,7 +45,7 @@ use crate::{ SHADER_STAGE_COUNT, }, error::{ErrorFormatter, PrettyError}, - hub::{GlobalIdentityHandlerFactory, Hub, Resource, Storage, Token}, + hub::{GlobalIdentityHandlerFactory, Hub, Resource, Storage, Token, HalApi}, id, init_tracker::{BufferInitTrackerAction, MemoryInitKind, TextureInitTrackerAction}, pipeline::{self, PipelineFlags}, @@ -168,7 +168,7 @@ impl RenderBundleEncoder { self.parent_id } - pub(crate) fn finish( + pub(crate) fn finish( self, desc: &RenderBundleDescriptor, device: &Device, @@ -617,7 +617,7 @@ pub type RenderBundleDescriptor<'a> = wgt::RenderBundleDescriptor>; //Note: here, `RenderBundle` is just wrapping a raw stream of render commands. // The plan is to back it by an actual Vulkan secondary buffer, D3D12 Bundle, // or Metal indirect command buffer. -pub struct RenderBundle { +pub struct RenderBundle { // Normalized command stream. It can be executed verbatim, // without re-binding anything on the pipeline change. base: BasePass, @@ -630,10 +630,10 @@ pub struct RenderBundle { pub(crate) life_guard: LifeGuard, } -unsafe impl Send for RenderBundle {} -unsafe impl Sync for RenderBundle {} +unsafe impl Send for RenderBundle {} +unsafe impl Sync for RenderBundle {} -impl RenderBundle { +impl RenderBundle { /// Actually encode the contents into a native command buffer. /// /// This is partially duplicating the logic of `command_encoder_run_render_pass`. @@ -832,7 +832,7 @@ impl RenderBundle { } } -impl Resource for RenderBundle { +impl Resource for RenderBundle { const TYPE: &'static str = "RenderBundle"; fn life_guard(&self) -> &LifeGuard { @@ -1009,7 +1009,7 @@ struct VertexLimitState { instance_limit_slot: u32, } -struct State { +struct State { trackers: RenderBundleScope, index: IndexState, vertex: ArrayVec, @@ -1021,7 +1021,7 @@ struct State { pipeline: Option, } -impl State { +impl State { fn vertex_limits(&self) -> VertexLimitState { let mut vert_state = VertexLimitState { vertex_limit: u32::MAX, diff --git a/wgpu-core/src/command/clear.rs b/wgpu-core/src/command/clear.rs index 1aefa82d3a..a7fa1051f9 100644 --- a/wgpu-core/src/command/clear.rs +++ b/wgpu-core/src/command/clear.rs @@ -234,13 +234,13 @@ impl Global { } } -pub(crate) fn clear_texture( +pub(crate) fn clear_texture( storage: &hub::Storage, TextureId>, dst_texture_id: Valid, dst_texture: &Texture, range: TextureInitRange, encoder: &mut A::CommandEncoder, - texture_tracker: &mut TextureTracker, + texture_tracker: &mut TextureTracker, device: &Device, ) -> Result<(), ClearError> { clear_texture_no_device( @@ -255,13 +255,13 @@ pub(crate) fn clear_texture( ) } -pub(crate) fn clear_texture_no_device( +pub(crate) fn clear_texture_no_device( storage: &hub::Storage, TextureId>, dst_texture_id: Valid, dst_texture: &Texture, range: TextureInitRange, encoder: &mut A::CommandEncoder, - texture_tracker: &mut TextureTracker, + texture_tracker: &mut TextureTracker, alignments: &hal::Alignments, zero_buffer: &A::Buffer, ) -> Result<(), ClearError> { diff --git a/wgpu-core/src/command/compute.rs b/wgpu-core/src/command/compute.rs index d89884ded1..95ed7b9c65 100644 --- a/wgpu-core/src/command/compute.rs +++ b/wgpu-core/src/command/compute.rs @@ -229,14 +229,14 @@ where } } -struct State { +struct State { binder: Binder, pipeline: Option, - scope: UsageScope, + scope: UsageScope, debug_scope_depth: u32, } -impl State { +impl State { fn is_ready(&self) -> Result<(), DispatchError> { let bind_mask = self.binder.invalid_mask(); if bind_mask != 0 { @@ -253,7 +253,7 @@ impl State { Ok(()) } - fn flush_states( + fn flush_states( &mut self, raw_encoder: &mut A::CommandEncoder, base_trackers: &mut Tracker, diff --git a/wgpu-core/src/command/memory_init.rs b/wgpu-core/src/command/memory_init.rs index 8d66c271a2..d9717cef04 100644 --- a/wgpu-core/src/command/memory_init.rs +++ b/wgpu-core/src/command/memory_init.rs @@ -4,7 +4,7 @@ use hal::CommandEncoder; use crate::{ device::Device, - hub::Storage, + hub::{Storage, HalApi}, id::{self, TextureId}, init_tracker::*, resource::{Buffer, Texture}, @@ -121,13 +121,13 @@ impl CommandBufferTextureMemoryActions { // Utility function that takes discarded surfaces from (several calls to) register_init_action and initializes them on the spot. // Takes care of barriers as well! pub(crate) fn fixup_discarded_surfaces< - A: hal::Api, + A: HalApi, InitIter: Iterator, >( inits: InitIter, encoder: &mut A::CommandEncoder, texture_guard: &Storage, TextureId>, - texture_tracker: &mut TextureTracker, + texture_tracker: &mut TextureTracker, device: &Device, ) { for init in inits { @@ -147,7 +147,7 @@ pub(crate) fn fixup_discarded_surfaces< } } -impl BakedCommands { +impl BakedCommands { // inserts all buffer initializations that are going to be needed for executing the commands and updates resource init states accordingly pub(crate) fn initialize_buffer_memory( &mut self, diff --git a/wgpu-core/src/command/mod.rs b/wgpu-core/src/command/mod.rs index 886d1d917a..796bb1d450 100644 --- a/wgpu-core/src/command/mod.rs +++ b/wgpu-core/src/command/mod.rs @@ -81,7 +81,7 @@ impl CommandEncoder { } } -pub struct BakedCommands { +pub struct BakedCommands { pub(crate) encoder: A::CommandEncoder, pub(crate) list: Vec, pub(crate) trackers: Tracker, @@ -92,7 +92,7 @@ pub struct BakedCommands { pub(crate) struct DestroyedBufferError(pub id::BufferId); pub(crate) struct DestroyedTextureError(pub id::TextureId); -pub struct CommandBuffer { +pub struct CommandBuffer { encoder: CommandEncoder, status: CommandEncoderStatus, pub(crate) device_id: Stored, @@ -141,7 +141,7 @@ impl CommandBuffer { pub(crate) fn insert_barriers( raw: &mut A::CommandEncoder, base: &mut Tracker, - head: &UsageScope, + head: &UsageScope, buffer_guard: &Storage, id::BufferId>, texture_guard: &Storage, id::TextureId>, ) { @@ -179,7 +179,7 @@ impl CommandBuffer { } } -impl CommandBuffer { +impl CommandBuffer { fn get_encoder_mut( storage: &mut Storage, id: id::CommandEncoderId, @@ -212,7 +212,7 @@ impl CommandBuffer { } } -impl crate::hub::Resource for CommandBuffer { +impl crate::hub::Resource for CommandBuffer { const TYPE: &'static str = "CommandBuffer"; fn life_guard(&self) -> &crate::LifeGuard { diff --git a/wgpu-core/src/command/render.rs b/wgpu-core/src/command/render.rs index b31c234fc0..72293103bb 100644 --- a/wgpu-core/src/command/render.rs +++ b/wgpu-core/src/command/render.rs @@ -561,9 +561,9 @@ impl TextureView { const MAX_TOTAL_ATTACHMENTS: usize = hal::MAX_COLOR_TARGETS + hal::MAX_COLOR_TARGETS + 1; type AttachmentDataVec = ArrayVec; -struct RenderPassInfo<'a, A: hal::Api> { +struct RenderPassInfo<'a, A: HalApi> { context: RenderPassContext, - usage_scope: UsageScope, + usage_scope: UsageScope, render_attachments: AttachmentDataVec>, // All render attachments, including depth/stencil is_ds_read_only: bool, extent: wgt::Extent3d, @@ -955,7 +955,7 @@ impl<'a, A: HalApi> RenderPassInfo<'a, A> { mut self, raw: &mut A::CommandEncoder, texture_guard: &Storage, id::TextureId>, - ) -> Result<(UsageScope, SurfacesInDiscardState), RenderPassErrorInner> { + ) -> Result<(UsageScope, SurfacesInDiscardState), RenderPassErrorInner> { profiling::scope!("finish", "RenderPassInfo"); unsafe { raw.end_render_pass(); diff --git a/wgpu-core/src/command/transfer.rs b/wgpu-core/src/command/transfer.rs index f594a3110c..e901ff3fd1 100644 --- a/wgpu-core/src/command/transfer.rs +++ b/wgpu-core/src/command/transfer.rs @@ -6,7 +6,7 @@ use crate::{ device::{Device, MissingDownlevelFlags}, error::{ErrorFormatter, PrettyError}, hub::{Global, GlobalIdentityHandlerFactory, HalApi, Storage, Token}, - id::{BufferId, CommandEncoderId, Id, TextureId, Valid}, + id::{BufferId, CommandEncoderId, TextureId, Valid}, init_tracker::{ has_copy_partial_init_tracker_coverage, MemoryInitKind, TextureInitRange, TextureInitTrackerAction, @@ -381,13 +381,13 @@ pub(crate) fn validate_texture_copy_range( Ok((copy_extent, array_layer_count)) } -fn handle_texture_init( +fn handle_texture_init( init_kind: MemoryInitKind, cmd_buf: &mut CommandBuffer, device: &Device, copy_texture: &ImageCopyTexture, copy_size: &Extent3d, - texture_guard: &Storage, Id>>, + texture_guard: &Storage, TextureId>, texture: &Texture, ) { let init_action = TextureInitTrackerAction { @@ -427,7 +427,7 @@ fn handle_texture_init( } // Ensures the source texture of a transfer is in the right initialization state and records the state for after the transfer operation. -fn handle_src_texture_init( +fn handle_src_texture_init( cmd_buf: &mut CommandBuffer, device: &Device, source: &ImageCopyTexture, @@ -451,7 +451,7 @@ fn handle_src_texture_init( } // Ensures the destination texture of a transfer is in the right initialization state and records the state for after the transfer operation. -fn handle_dst_texture_init( +fn handle_dst_texture_init( cmd_buf: &mut CommandBuffer, device: &Device, destination: &ImageCopyTexture, diff --git a/wgpu-core/src/device/life.rs b/wgpu-core/src/device/life.rs index c1b9211fbd..f559164af3 100644 --- a/wgpu-core/src/device/life.rs +++ b/wgpu-core/src/device/life.rs @@ -273,7 +273,8 @@ pub(super) struct LifetimeTracker { /// Textures can be used in the upcoming submission by `write_texture`. pub future_suspected_textures: Vec>, - /// Resources that are suspected for destruction. + /// Resources whose user handle has died (i.e. drop/destroy has been called) + /// and will likely be ready for destruction soon. pub suspected_resources: SuspectedResources, /// Resources used by queue submissions still in flight. One entry per diff --git a/wgpu-core/src/device/mod.rs b/wgpu-core/src/device/mod.rs index 59feb758e2..61c2586633 100644 --- a/wgpu-core/src/device/mod.rs +++ b/wgpu-core/src/device/mod.rs @@ -2,7 +2,7 @@ use crate::{ binding_model, command, conv, device::life::WaitIdleError, hub::{ - self, Global, GlobalIdentityHandlerFactory, HalApi, Hub, Input, InvalidId, Storage, Token, + Global, GlobalIdentityHandlerFactory, HalApi, Hub, Input, InvalidId, Storage, Token, }, id, init_tracker::{ @@ -10,7 +10,7 @@ use crate::{ TextureInitTracker, TextureInitTrackerAction, }, instance, pipeline, present, resource, - track::{BindGroupStates, TextureSelector, Tracker, UsageConflict}, + track::{BindGroupStates, TextureSelector, Tracker}, validation::{self, check_buffer_usage, check_texture_usage}, FastHashMap, Label, LabelHelpers as _, LifeGuard, MultiRefCount, RefCount, Stored, SubmissionIndex, DOWNLEVEL_ERROR_MESSAGE, @@ -24,7 +24,7 @@ use smallvec::SmallVec; use thiserror::Error; use wgt::{BufferAddress, TextureFormat, TextureViewDimension}; -use std::{borrow::Cow, iter, marker::PhantomData, mem, num::NonZeroU32, ops::Range, ptr}; +use std::{borrow::Cow, iter, mem, num::NonZeroU32, ops::Range, ptr}; mod life; pub mod queue; @@ -258,7 +258,7 @@ impl CommandAllocator { /// 1. `life_tracker` is locked after `hub.devices`, enforced by the type system /// 1. `self.trackers` is locked last (unenforced) /// 1. `self.trace` is locked last (unenforced) -pub struct Device { +pub struct Device { pub(crate) raw: A::Device, pub(crate) adapter_id: Stored, pub(crate) queue: A::Queue, @@ -308,7 +308,7 @@ pub enum CreateDeviceError { FailedToCreateZeroBuffer(#[from] DeviceError), } -impl Device { +impl Device { pub(crate) fn require_features(&self, feature: wgt::Features) -> Result<(), MissingFeatures> { if self.features.contains(feature) { Ok(()) @@ -538,12 +538,12 @@ impl Device { self.temp_suspected.samplers.push(id); } } - for id in trackers.compute_pipes.used() { + for id in trackers.compute_pipelines.used() { if compute_pipe_guard[id].life_guard.ref_count.is_none() { self.temp_suspected.compute_pipelines.push(id); } } - for id in trackers.render_pipes.used() { + for id in trackers.render_pipelines.used() { if render_pipe_guard[id].life_guard.ref_count.is_none() { self.temp_suspected.render_pipelines.push(id); } @@ -1616,7 +1616,7 @@ impl Device { fn create_texture_binding( view: &resource::TextureView, - texture_guard: &hub::Storage, id::TextureId>, + texture_guard: &Storage, id::TextureId>, internal_use: hal::TextureUses, pub_usage: wgt::TextureUsages, used: &mut BindGroupStates, @@ -2882,7 +2882,7 @@ impl Device { } } -impl Device { +impl Device { pub(crate) fn destroy_buffer(&self, buffer: resource::Buffer) { if let Some(raw) = buffer.raw { unsafe { @@ -2928,7 +2928,7 @@ impl Device { } } -impl crate::hub::Resource for Device { +impl crate::hub::Resource for Device { const TYPE: &'static str = "Device"; fn life_guard(&self) -> &LifeGuard { @@ -2984,7 +2984,7 @@ pub struct ImplicitPipelineIds<'a, G: GlobalIdentityHandlerFactory> { } impl ImplicitPipelineIds<'_, G> { - fn prepare(self, hub: &Hub) -> ImplicitPipelineContext { + fn prepare(self, hub: &Hub) -> ImplicitPipelineContext { ImplicitPipelineContext { root_id: hub.pipeline_layouts.prepare(self.root_id).into_id(), group_ids: self @@ -3183,7 +3183,7 @@ impl Global { let id = fid.assign(buffer, &mut token); log::info!("Created buffer {:?} with {:?}", id, desc); - unsafe { device.trackers.lock().buffers.init(id.0, buffer_use) }; + unsafe { device.trackers.lock().buffers.init(id, ref_count, buffer_use) }; return (id.0, None); }; @@ -3531,11 +3531,11 @@ impl Global { log::info!("Created texture {:?} with {:?}", id, desc); unsafe { - device - .trackers - .lock() - .textures - .init(id.0, ref_count, hal::TextureUses::UNINITIALIZED) + device.trackers.lock().textures.init( + id.0, + ref_count, + hal::TextureUses::UNINITIALIZED, + ) }; return (id.0, None); }; @@ -3694,7 +3694,7 @@ impl Global { let ref_count = view.life_guard.add_ref(); let id = fid.assign(view, &mut token); - unsafe { device.trackers.lock().views.init(id.0, ref_count) }; + unsafe { device.trackers.lock().views.init(id, ref_count) }; return (id.0, None); }; @@ -3788,7 +3788,7 @@ impl Global { let id = fid.assign(sampler, &mut token); unsafe { - device.trackers.lock().samplers.init(id.0, ref_count); + device.trackers.lock().samplers.init(id, ref_count); } return (id.0, None); }; @@ -4049,7 +4049,7 @@ impl Global { let id = fid.assign(bind_group, &mut token); log::debug!("Bind group {:?}", id,); - unsafe { device.trackers.lock().bind_groups.init(id.0, ref_count) }; + unsafe { device.trackers.lock().bind_groups.init(id, ref_count) }; return (id.0, None); }; @@ -4356,7 +4356,7 @@ impl Global { let ref_count = render_bundle.life_guard.add_ref(); let id = fid.assign(render_bundle, &mut token); - unsafe { device.trackers.lock().bundles.init(id.0, ref_count) }; + unsafe { device.trackers.lock().bundles.init(id, ref_count) }; return (id.0, None); }; @@ -4432,7 +4432,7 @@ impl Global { let id = fid.assign(query_set, &mut token); unsafe { - device.trackers.lock().query_sets.init(id.0, ref_count); + device.trackers.lock().query_sets.init(id, ref_count); } return (id.0, None); @@ -4525,7 +4525,11 @@ impl Global { log::info!("Created render pipeline {:?} with {:?}", id, desc); unsafe { - device.trackers.lock().render_pipelines.init(id.0, ref_count); + device + .trackers + .lock() + .render_pipelines + .init(id, ref_count); } return (id.0, None); }; @@ -4662,7 +4666,13 @@ impl Global { let id = fid.assign(pipeline, &mut token); log::info!("Created compute pipeline {:?} with {:?}", id, desc); - unsafe { device.trackers.lock().compute_pipelines.init(id.0, ref_count) }; + unsafe { + device + .trackers + .lock() + .compute_pipelines + .init(id, ref_count) + }; return (id.0, None); }; diff --git a/wgpu-core/src/device/queue.rs b/wgpu-core/src/device/queue.rs index 089d8a7bd6..8f31543400 100644 --- a/wgpu-core/src/device/queue.rs +++ b/wgpu-core/src/device/queue.rs @@ -192,7 +192,7 @@ impl PendingWrites { } } -impl super::Device { +impl super::Device { fn prepare_stage(&mut self, size: wgt::BufferAddress) -> Result, DeviceError> { profiling::scope!("prepare_stage"); let stage_desc = hal::BufferDescriptor { diff --git a/wgpu-core/src/hub.rs b/wgpu-core/src/hub.rs index c4bf14cda7..93dcb9e72f 100644 --- a/wgpu-core/src/hub.rs +++ b/wgpu-core/src/hub.rs @@ -303,56 +303,56 @@ pub enum Root {} impl Access for Root {} impl Access for Root {} impl Access for Instance {} -impl Access> for Root {} -impl Access> for Surface {} -impl Access> for Root {} -impl Access> for Surface {} -impl Access> for Adapter {} -impl Access> for Root {} -impl Access> for Device {} -impl Access> for RenderBundle {} -impl Access> for Root {} -impl Access> for Device {} -impl Access> for PipelineLayout {} -impl Access> for Root {} -impl Access> for Device {} -impl Access> for BindGroupLayout {} -impl Access> for PipelineLayout {} -impl Access> for CommandBuffer {} -impl Access> for Root {} -impl Access> for Device {} -impl Access> for Device {} -impl Access> for CommandBuffer {} -impl Access> for Device {} -impl Access> for BindGroup {} -impl Access> for Device {} -impl Access> for BindGroup {} -impl Access> for ComputePipeline {} -impl Access> for Root {} -impl Access> for Device {} -impl Access> for CommandBuffer {} -impl Access> for RenderPipeline {} -impl Access> for ComputePipeline {} -impl Access> for Sampler {} -impl Access> for Device {} -impl Access> for BindGroupLayout {} -impl Access> for Root {} -impl Access> for Device {} -impl Access> for BindGroupLayout {} -impl Access> for BindGroup {} -impl Access> for CommandBuffer {} -impl Access> for ComputePipeline {} -impl Access> for RenderPipeline {} -impl Access> for QuerySet {} -impl Access> for Root {} -impl Access> for Device {} -impl Access> for Buffer {} -impl Access> for Root {} -impl Access> for Device {} -impl Access> for Texture {} -impl Access> for Root {} -impl Access> for Device {} -impl Access> for TextureView {} +impl Access> for Root {} +impl Access> for Surface {} +impl Access> for Root {} +impl Access> for Surface {} +impl Access> for Adapter {} +impl Access> for Root {} +impl Access> for Device {} +impl Access> for RenderBundle {} +impl Access> for Root {} +impl Access> for Device {} +impl Access> for PipelineLayout {} +impl Access> for Root {} +impl Access> for Device {} +impl Access> for BindGroupLayout {} +impl Access> for PipelineLayout {} +impl Access> for CommandBuffer {} +impl Access> for Root {} +impl Access> for Device {} +impl Access> for Device {} +impl Access> for CommandBuffer {} +impl Access> for Device {} +impl Access> for BindGroup {} +impl Access> for Device {} +impl Access> for BindGroup {} +impl Access> for ComputePipeline {} +impl Access> for Root {} +impl Access> for Device {} +impl Access> for CommandBuffer {} +impl Access> for RenderPipeline {} +impl Access> for ComputePipeline {} +impl Access> for Sampler {} +impl Access> for Device {} +impl Access> for BindGroupLayout {} +impl Access> for Root {} +impl Access> for Device {} +impl Access> for BindGroupLayout {} +impl Access> for BindGroup {} +impl Access> for CommandBuffer {} +impl Access> for ComputePipeline {} +impl Access> for RenderPipeline {} +impl Access> for QuerySet {} +impl Access> for Root {} +impl Access> for Device {} +impl Access> for Buffer {} +impl Access> for Root {} +impl Access> for Device {} +impl Access> for Texture {} +impl Access> for Root {} +impl Access> for Device {} +impl Access> for TextureView {} #[cfg(debug_assertions)] thread_local! { @@ -619,7 +619,7 @@ impl HubReport { } } -pub struct Hub { +pub struct Hub { pub adapters: Registry, id::AdapterId, F>, pub devices: Registry, id::DeviceId, F>, pub pipeline_layouts: Registry, id::PipelineLayoutId, F>, @@ -1001,6 +1001,22 @@ pub trait HalApi: hal::Api { fn get_surface_mut(surface: &mut Surface) -> &mut HalSurface; } +impl HalApi for hal::api::Empty { + const VARIANT: Backend = Backend::Empty; + fn create_instance_from_hal(name: &str, hal_instance: Self::Instance) -> Instance { + unimplemented!("called empty api") + } + fn hub(global: &Global) -> &Hub { + unimplemented!("called empty api") + } + fn get_surface(surface: &Surface) -> &HalSurface { + unimplemented!("called empty api") + } + fn get_surface_mut(surface: &mut Surface) -> &mut HalSurface { + unimplemented!("called empty api") + } +} + #[cfg(vulkan)] impl HalApi for hal::api::Vulkan { const VARIANT: Backend = Backend::Vulkan; diff --git a/wgpu-core/src/track/buffer.rs b/wgpu-core/src/track/buffer.rs index 01c1a9fc71..6035068218 100644 --- a/wgpu-core/src/track/buffer.rs +++ b/wgpu-core/src/track/buffer.rs @@ -1,15 +1,15 @@ -use std::vec::Drain; +use std::{vec::Drain, marker::PhantomData}; use super::PendingTransition; use crate::{ hub, - id::{BufferId, TypedId}, + id::{BufferId, TypedId, Valid}, resource::Buffer, track::{ - invalid_resource_state, iterate_bitvec, resize_bitvec, skip_barrier, ResourceUses, + invalid_resource_state, iterate_bitvec_indices, resize_bitvec, skip_barrier, ResourceUses, UsageConflict, }, - RefCount, + Epoch, RefCount, }; use bit_vec::BitVec; use hal::BufferUses; @@ -33,17 +33,21 @@ impl ResourceUses for BufferUses { } } -pub struct BufferBindGroupState { - buffers: Vec<(BufferId, RefCount, BufferUses)>, +pub struct BufferBindGroupState { + buffers: Vec<(Valid, RefCount, BufferUses)>, + + _phantom: PhantomData } -impl BufferBindGroupState { +impl BufferBindGroupState { pub fn new() -> Self { Self { buffers: Vec::new(), + + _phantom: PhantomData, } } - pub fn extend<'a, A: hal::Api>( + pub fn extend<'a>( &mut self, storage: &'a hub::Storage, BufferId>, id: BufferId, @@ -51,48 +55,76 @@ impl BufferBindGroupState { ) -> Option<&'a Buffer> { let buffer = storage.get(id).ok()?; - self.buffers.push((id, buffer.life_guard.add_ref(), state)); + self.buffers + .push((Valid(id), buffer.life_guard.add_ref(), state)); Some(buffer) } } #[derive(Debug)] -pub(crate) struct BufferUsageScope { +pub(crate) struct BufferUsageScope { state: Vec, + + ref_counts: Vec>, + epochs: Vec, + owned: BitVec, - ref_count: Vec>, + + _phantom: PhantomData, } -impl BufferUsageScope { +impl BufferUsageScope { pub fn new() -> Self { Self { state: Vec::new(), + + ref_counts: Vec::new(), + epochs: Vec::new(), + owned: BitVec::default(), - ref_count: Vec::new(), + + _phantom: PhantomData, } } + fn debug_assert_in_bounds(&self, index: usize) { + debug_assert!(index < self.state.len()); + debug_assert!(index < self.ref_counts.len()); + debug_assert!(index < self.epochs.len()); + debug_assert!(index < self.owned.len()); + + debug_assert!(if self.owned.get(index).unwrap() { + self.ref_counts[index].is_some() + } else { + true + }); + } + pub fn set_max_index(&mut self, size: usize) { self.state.resize(size, BufferUses::empty()); - self.ref_count.resize(size, None); + self.ref_counts.resize(size, None); + self.epochs.resize(size, u32::MAX); resize_bitvec(&mut self.owned, size); } - pub unsafe fn extend_from_bind_group( + pub unsafe fn extend_from_bind_group( &mut self, storage: &hub::Storage, BufferId>, - bind_group: &BufferBindGroupState, + bind_group: &BufferBindGroupState, ) -> Result<(), UsageConflict> { for (id, ref_count, state) in &bind_group.buffers { - self.extend_inner(storage, id.unzip().0, ref_count, *state)?; + let (index32, epoch, _) = id.0.unzip(); + let index = index32 as usize; + + self.extend_inner(storage, *id, index, epoch, ref_count, *state)?; } Ok(()) } - pub unsafe fn extend_from_scope( + pub fn extend_from_scope( &mut self, storage: &hub::Storage, BufferId>, scope: &Self, @@ -102,25 +134,34 @@ impl BufferUsageScope { self.set_max_index(incoming_size); } - iterate_bitvec(&scope.owned, |index| { + for index in iterate_bitvec_indices(&scope.owned) { + self.debug_assert_in_bounds(index); + scope.debug_assert_in_bounds(index); + unsafe { + let ref_count = scope + .ref_counts + .get_unchecked(index) + .as_ref() + .unwrap_unchecked(); + let epoch = *scope.epochs.get_unchecked(index); + let new_state = *scope.state.get_unchecked(index); + self.extend_inner( storage, - index as u32, - scope - .ref_count - .get_unchecked(index) - .as_ref() - .unwrap_unchecked(), - *scope.state.get_unchecked(index), + Valid(BufferId::zip(index as u32, epoch, A::VARIANT)), + index, + epoch, + ref_count, + new_state, ) }; - }); + } Ok(()) } - pub unsafe fn extend<'a, A: hal::Api>( + pub unsafe fn extend<'a>( &mut self, storage: &'a hub::Storage, BufferId>, id: BufferId, @@ -128,11 +169,16 @@ impl BufferUsageScope { ) -> Result<&'a Buffer, UsageConflict> { let buffer = storage .get(id) - .map_err(|_| UsageConflict::BufferInvalid { id: id.unzip().0 })?; + .map_err(|_| UsageConflict::BufferInvalid { id })?; + + let (index32, epoch, _) = id.unzip(); + let index = index32 as usize; self.extend_inner( storage, - id.unzip().0, + Valid(id), + index, + epoch, buffer.life_guard.ref_count.as_ref().unwrap(), new_state, )?; @@ -140,14 +186,16 @@ impl BufferUsageScope { Ok(buffer) } - unsafe fn extend_inner<'a, A: hal::Api>( + unsafe fn extend_inner<'a>( &mut self, storage: &'a hub::Storage, BufferId>, - index: u32, + id: Valid, + index: usize, + epoch: u32, ref_count: &RefCount, new_state: BufferUses, ) -> Result<(), UsageConflict> { - let index = index as usize; + self.debug_assert_in_bounds(index); let currently_active = self.owned.get(index).unwrap_unchecked(); if currently_active { @@ -156,19 +204,16 @@ impl BufferUsageScope { let merged_state = current_state | new_state; if invalid_resource_state(merged_state) { - return Err(UsageConflict::from_buffer( - index as u32, - current_state, - new_state, - )); + return Err(UsageConflict::from_buffer(id.0, current_state, new_state)); } *self.state.get_unchecked_mut(index) = merged_state; } // We're the first to use this resource, let's add it. + *self.epochs.get_unchecked_mut(index) = epoch; + *self.ref_counts.get_unchecked_mut(index) = Some(ref_count.clone()); self.owned.set(index, true); - *self.ref_count.get_unchecked_mut(index) = Some(ref_count.clone()); *self.state.get_unchecked_mut(index) = new_state; @@ -176,45 +221,86 @@ impl BufferUsageScope { } } -pub(crate) struct BufferTracker { +pub(crate) struct BufferTracker { start: Vec, end: Vec, - temp: Vec>, + + epochs: Vec, + ref_counts: Vec>, owned: BitVec, - ref_count: Vec>, + + temp: Vec>, + + _phantom: PhantomData, } -impl BufferTracker { +impl BufferTracker { pub fn new() -> Self { Self { start: Vec::new(), end: Vec::new(), - temp: Vec::new(), + + epochs: Vec::new(), + ref_counts: Vec::new(), owned: BitVec::default(), - ref_count: Vec::new(), + + temp: Vec::new(), + + _phantom: PhantomData, } } + fn debug_assert_in_bounds(&self, index: usize) { + debug_assert!(index < self.start.len()); + debug_assert!(index < self.end.len()); + debug_assert!(index < self.ref_counts.len()); + debug_assert!(index < self.epochs.len()); + debug_assert!(index < self.owned.len()); + + debug_assert!(if self.owned.get(index).unwrap() { + self.ref_counts[index].is_some() + } else { + true + }); + } + fn set_max_index(&mut self, size: usize) { self.start.resize(size, BufferUses::empty()); self.end.resize(size, BufferUses::empty()); - self.ref_count.resize(size, None); + + self.epochs.resize(size, u32::MAX); + self.ref_counts.resize(size, None); resize_bitvec(&mut self.owned, size); } + pub fn used(&self) -> impl Iterator> + '_ { + self.debug_assert_in_bounds(self.owned.len() - 1); + iterate_bitvec_indices(&self.owned).map(move |index| { + let epoch = unsafe { *self.epochs.get_unchecked(index) }; + Valid(BufferId::zip(index as u32, epoch, A::VARIANT)) + }) + } + pub fn drain(&mut self) -> Drain> { self.temp.drain(..) } - pub unsafe fn init(&mut self, id: BufferId, state: BufferUses) { - let index = id.unzip().0 as usize; + pub unsafe fn init(&mut self, id: Valid, ref_count: RefCount, state: BufferUses) { + let (index32, epoch, _) = id.0.unzip(); + let index = index32 as usize; + + self.debug_assert_in_bounds(index); + *self.start.get_unchecked_mut(index) = state; *self.end.get_unchecked_mut(index) = state; + *self.ref_counts.get_unchecked_mut(index) = Some(ref_count); + *self.epochs.get_unchecked_mut(index) = epoch; + self.owned.set(index, true); } - pub unsafe fn change_state<'a, A: hal::Api>( + pub unsafe fn change_state<'a>( &mut self, storage: &'a hub::Storage, BufferId>, id: BufferId, @@ -222,9 +308,13 @@ impl BufferTracker { ) -> Option<(&'a Buffer, Option>)> { let value = storage.get(id).ok()?; + let (index32, epoch, _) = id.unzip(); + let index = index32 as usize; + self.transition_inner( storage, - id.unzip().0 as usize, + index, + epoch, value.life_guard.ref_count.as_ref().unwrap(), state, ); @@ -232,67 +322,81 @@ impl BufferTracker { Some((value, self.temp.pop())) } - pub fn change_states_scope( + pub fn change_states_scope( &mut self, storage: &hub::Storage, BufferId>, - scope: &BufferUsageScope, + scope: &BufferUsageScope, ) { let incoming_size = scope.state.len(); if incoming_size > self.start.len() { self.set_max_index(incoming_size); } - iterate_bitvec(&scope.owned, |index| unsafe { - let ref_count = scope - .ref_count - .get_unchecked(index) - .as_ref() - .unwrap_unchecked(); + for index in iterate_bitvec_indices(&scope.owned) { + scope.debug_assert_in_bounds(index); + unsafe { + let ref_count = scope + .ref_counts + .get_unchecked(index) + .as_ref() + .unwrap_unchecked(); - self.transition(storage, &scope.state, ref_count, index); - }); + let epoch = *scope.epochs.get_unchecked(index); + + self.transition(storage, &scope.state, ref_count, index, epoch); + } + } } - pub unsafe fn change_states_bind_group( + pub unsafe fn change_states_bind_group( &mut self, storage: &hub::Storage, BufferId>, - scope: &mut BufferUsageScope, - bind_group_state: &BufferBindGroupState, + scope: &mut BufferUsageScope, + bind_group_state: &BufferBindGroupState, ) { let incoming_size = scope.state.len(); if incoming_size > self.start.len() { self.set_max_index(incoming_size); } - for (index, ref_count, _) in bind_group_state.buffers.iter() { - let index = index.unzip().0 as usize; + for (id, ref_count, _) in bind_group_state.buffers.iter() { + let (index32, epoch, _) = id.0.unzip(); + let index = index32 as usize; + + scope.debug_assert_in_bounds(index); + if !scope.owned.get(index).unwrap_unchecked() { continue; } - self.transition(storage, &scope.state, ref_count, index); + self.transition(storage, &scope.state, ref_count, index, epoch); + scope.owned.set(index, false); } } - unsafe fn transition( + unsafe fn transition( &mut self, storage: &hub::Storage, BufferId>, incoming_set: &Vec, ref_count: &RefCount, index: usize, + epoch: u32, ) { let new_state = *incoming_set.get_unchecked(index); - self.transition_inner(storage, index, ref_count, new_state); + self.transition_inner(storage, index, epoch, ref_count, new_state); } - unsafe fn transition_inner( + unsafe fn transition_inner( &mut self, storage: &hub::Storage, BufferId>, index: usize, + epoch: u32, ref_count: &RefCount, new_state: BufferUses, ) { + self.debug_assert_in_bounds(index); + let old_tracked = self.owned.get(index).unwrap_unchecked(); let old_state = *self.end.get_unchecked(index); @@ -312,6 +416,9 @@ impl BufferTracker { *self.start.get_unchecked_mut(index) = new_state; *self.end.get_unchecked_mut(index) = new_state; + *self.ref_counts.get_unchecked_mut(index) = Some(ref_count.clone()); + *self.epochs.get_unchecked(index) = epoch; + self.owned.set(index, true); } } diff --git a/wgpu-core/src/track/mod.rs b/wgpu-core/src/track/mod.rs index cc4e9a4412..2401d04b46 100644 --- a/wgpu-core/src/track/mod.rs +++ b/wgpu-core/src/track/mod.rs @@ -10,7 +10,7 @@ use crate::{ }; use bit_vec::BitVec; -use std::{fmt, num::NonZeroU32, ops}; +use std::{fmt, mem, num::NonZeroU32, ops}; use thiserror::Error; pub(crate) use buffer::{BufferBindGroupState, BufferTracker, BufferUsageScope}; @@ -104,48 +104,53 @@ fn resize_bitvec(vec: &mut BitVec, size: usize) { } } -fn iterate_bitvec(ownership: &BitVec, mut func: impl FnMut(usize)) { - let size = ownership.len(); - for (word_index, mut word) in ownership.blocks().enumerate() { - if word == 0 { - continue; - } - - let bit_start = word_index * 64; - let bit_end = (bit_start + 64).min(size); +fn iterate_bitvec_indices(ownership: &BitVec) -> impl Iterator + '_ { + const BITS_PER_BLOCK: usize = mem::size_of::() * 8; - for index in bit_start..bit_end { - if word & 0b1 == 0 { - continue; - } - word >>= 1; + let size = ownership.len(); - func(index); - } - } + ownership + .blocks() + .enumerate() + .filter(|(_, word)| *word != 0) + .flat_map(move |(word_index, mut word)| { + let bit_start = word_index * BITS_PER_BLOCK; + let bit_end = (bit_start + BITS_PER_BLOCK).min(size); + + (bit_start..bit_end).filter(move |index| { + let active = word & 0b1 != 0; + word >>= 1; + + active + }) + }) } #[derive(Clone, Debug, Error)] pub enum UsageConflict { - #[error("Attempted to use buffer {id} which is invalid.")] - BufferInvalid { id: u32 }, - #[error("Attempted to use texture {id} which is invalid.")] - TextureInvalid { id: u32 }, + #[error("Attempted to use buffer {id:?} which is invalid.")] + BufferInvalid { id: id::BufferId }, + #[error("Attempted to use texture {id:?} which is invalid.")] + TextureInvalid { id: id::TextureId }, #[error("Attempted to use buffer {id:?} with {invalid_use}.")] Buffer { - id: u32, + id: id::BufferId, invalid_use: InvalidUse, }, #[error("Attempted to use a texture {id:?} mips {mip_levels:?} layers {array_layers:?} with {invalid_use}.")] Texture { - id: u32, + id: id::TextureId, mip_levels: ops::Range, array_layers: ops::Range, invalid_use: InvalidUse, }, } impl UsageConflict { - fn from_buffer(id: u32, current_state: hal::BufferUses, new_state: hal::BufferUses) -> Self { + fn from_buffer( + id: id::BufferId, + current_state: hal::BufferUses, + new_state: hal::BufferUses, + ) -> Self { Self::Buffer { id, invalid_use: InvalidUse { @@ -157,12 +162,12 @@ impl UsageConflict { fn from_texture( storage: &hub::Storage, id::TextureId>, - id: u32, + id: id::Valid, selector: Option, current_state: hal::TextureUses, new_state: hal::TextureUses, ) -> Self { - let texture = unsafe { storage.get_unchecked(id) }; + let texture = &storage[id]; let mips; let layers; @@ -179,7 +184,7 @@ impl UsageConflict { } Self::Texture { - id, + id: id.0, mip_levels: mips, array_layers: layers, invalid_use: InvalidUse { @@ -215,14 +220,14 @@ impl fmt::Display for InvalidUse { } } -pub(crate) struct BindGroupStates { - pub buffers: BufferBindGroupState, - pub textures: TextureBindGroupState, +pub(crate) struct BindGroupStates { + pub buffers: BufferBindGroupState, + pub textures: TextureBindGroupState, pub views: StatelessBindGroupSate, id::TextureViewId>, pub samplers: StatelessBindGroupSate, id::SamplerId>, } -impl BindGroupStates { +impl BindGroupStates { pub fn new() -> Self { Self { buffers: BufferBindGroupState::new(), @@ -233,17 +238,17 @@ impl BindGroupStates { } } -pub(crate) struct RenderBundleScope { - pub buffers: BufferUsageScope, - pub textures: TextureUsageScope, - pub views: StatelessTracker, id::TextureViewId>, - pub samplers: StatelessTracker, id::SamplerId>, - pub bind_groups: StatelessTracker, id::BindGroupId>, - pub render_pipelines: StatelessTracker, id::RenderPipelineId>, - pub query_sets: StatelessTracker, id::QuerySetId>, +pub(crate) struct RenderBundleScope { + pub buffers: BufferUsageScope, + pub textures: TextureUsageScope, + pub views: StatelessTracker, id::TextureViewId>, + pub samplers: StatelessTracker, id::SamplerId>, + pub bind_groups: StatelessTracker, id::BindGroupId>, + pub render_pipelines: StatelessTracker, id::RenderPipelineId>, + pub query_sets: StatelessTracker, id::QuerySetId>, } -impl RenderBundleScope { +impl RenderBundleScope { pub fn new() -> Self { Self { buffers: BufferUsageScope::new(), @@ -272,12 +277,12 @@ impl RenderBundleScope { } #[derive(Debug)] -pub(crate) struct UsageScope { - pub buffers: BufferUsageScope, - pub textures: TextureUsageScope, +pub(crate) struct UsageScope { + pub buffers: BufferUsageScope, + pub textures: TextureUsageScope, } -impl UsageScope { +impl UsageScope { pub fn new() -> Self { Self { buffers: BufferUsageScope::new(), @@ -285,7 +290,7 @@ impl UsageScope { } } - pub unsafe fn extend_from_bind_group( + pub unsafe fn extend_from_bind_group( &mut self, buffers: &hub::Storage, id::BufferId>, textures: &hub::Storage, id::TextureId>, @@ -299,7 +304,7 @@ impl UsageScope { Ok(()) } - pub unsafe fn extend_from_render_bundle( + pub unsafe fn extend_from_render_bundle( &mut self, buffers: &hub::Storage, id::BufferId>, textures: &hub::Storage, id::TextureId>, @@ -314,19 +319,19 @@ impl UsageScope { } } -pub(crate) struct Tracker { - pub buffers: BufferTracker, - pub textures: TextureTracker, - pub views: StatelessTracker, id::TextureViewId>, - pub samplers: StatelessTracker, id::SamplerId>, - pub bind_groups: StatelessTracker, id::BindGroupId>, - pub compute_pipelines: StatelessTracker, id::ComputePipelineId>, - pub render_pipelines: StatelessTracker, id::RenderPipelineId>, - pub bundles: StatelessTracker, id::RenderBundleId>, - pub query_sets: StatelessTracker, id::QuerySetId>, +pub(crate) struct Tracker { + pub buffers: BufferTracker, + pub textures: TextureTracker, + pub views: StatelessTracker, id::TextureViewId>, + pub samplers: StatelessTracker, id::SamplerId>, + pub bind_groups: StatelessTracker, id::BindGroupId>, + pub compute_pipelines: StatelessTracker, id::ComputePipelineId>, + pub render_pipelines: StatelessTracker, id::RenderPipelineId>, + pub bundles: StatelessTracker, id::RenderBundleId>, + pub query_sets: StatelessTracker, id::QuerySetId>, } -impl Tracker { +impl Tracker { pub fn new() -> Self { Self { buffers: BufferTracker::new(), @@ -345,7 +350,7 @@ impl Tracker { &mut self, buffers: &hub::Storage, id::BufferId>, textures: &hub::Storage, id::TextureId>, - scope: &mut UsageScope, + scope: &mut UsageScope, bind_group: &BindGroupStates, ) { self.buffers diff --git a/wgpu-core/src/track/stateless.rs b/wgpu-core/src/track/stateless.rs index 887052e238..d18da4ae78 100644 --- a/wgpu-core/src/track/stateless.rs +++ b/wgpu-core/src/track/stateless.rs @@ -4,13 +4,13 @@ use bit_vec::BitVec; use crate::{ hub, - id::TypedId, - track::{iterate_bitvec, resize_bitvec}, - RefCount, + id::{TypedId, Valid}, + track::{iterate_bitvec_indices, resize_bitvec}, + Epoch, RefCount, }; pub struct StatelessBindGroupSate { - resource: Vec<(Id, RefCount)>, + resources: Vec<(Valid, RefCount)>, _phantom: PhantomData, } @@ -18,58 +18,92 @@ pub struct StatelessBindGroupSate { impl StatelessBindGroupSate { pub fn new() -> Self { Self { - resource: Vec::new(), + resources: Vec::new(), + _phantom: PhantomData, } } - pub fn extend<'a>( - &mut self, - storage: &'a hub::Storage, - id: Id, - ) -> Option<&'a T> { + pub fn extend<'a>(&mut self, storage: &'a hub::Storage, id: Id) -> Option<&'a T> { let resource = storage.get(id).ok()?; - self.resource.push((id, resource.life_guard().add_ref())); + self.resources + .push((Valid(id), resource.life_guard().add_ref())); Some(resource) } } -pub struct StatelessTracker { - owned: BitVec, +pub struct StatelessTracker { ref_counts: Vec>, + epochs: Vec, + + owned: BitVec, - _phantom: PhantomData<(T, Id)>, + _phantom: PhantomData<(A, T, Id)>, } -impl StatelessTracker { +impl StatelessTracker { pub fn new() -> Self { Self { - owned: BitVec::default(), ref_counts: Vec::new(), + epochs: Vec::new(), + + owned: BitVec::default(), + _phantom: PhantomData, } } + fn debug_assert_in_bounds(&self, index: usize) { + debug_assert!(index < self.ref_counts.len()); + debug_assert!(index < self.epochs.len()); + debug_assert!(index < self.owned.len()); + + debug_assert!(if self.owned.get(index).unwrap() { + self.ref_counts[index].is_some() + } else { + true + }); + } + pub fn set_max_index(&mut self, size: usize) { + self.epochs.resize(size, u32::MAX); self.ref_counts.resize(size, None); + resize_bitvec(&mut self.owned, size); } - pub unsafe fn init(&mut self, id: Id, ref_count: RefCount) { - let index = id.unzip().0 as usize; + pub fn used(&self) -> impl Iterator> + '_ { + self.debug_assert_in_bounds(self.owned.len() - 1); + iterate_bitvec_indices(&self.owned).map(move |index| { + let epoch = unsafe { *self.epochs.get_unchecked(index) }; + Valid(Id::zip(index as u32, epoch, A::VARIANT)) + }) + } - self.owned.set(index, true); + pub unsafe fn init(&mut self, id: Valid, ref_count: RefCount) { + let (index32, epoch, _) = id.0.unzip(); + let index = index32 as usize; + + self.debug_assert_in_bounds(index); + + *self.epochs.get_unchecked_mut(index) = epoch; *self.ref_counts.get_unchecked_mut(index) = Some(ref_count); + self.owned.set(index, true); } pub unsafe fn extend<'a>(&mut self, storage: &'a hub::Storage, id: Id) -> Option<&'a T> { let item = storage.get(id).ok()?; - let index = id.unzip().0 as usize; - self.owned.set(index, true); + let (index32, epoch, _) = id.unzip(); + let index = index32 as usize; + + self.debug_assert_in_bounds(index); + + *self.epochs.get_unchecked_mut(index) = epoch; *self.ref_counts.get_unchecked_mut(index) = Some(item.life_guard().add_ref()); + self.owned.set(index, true); Some(item) } @@ -79,12 +113,16 @@ impl StatelessTracker { storage: &hub::Storage, bind_group: &StatelessBindGroupSate, ) { - for (id, ref_count) in &bind_group.resource { - let index = id.unzip().0 as usize; + for (id, ref_count) in &bind_group.resources { + let (index32, epoch, _) = id.0.unzip(); + let index = index32 as usize; + self.debug_assert_in_bounds(index); + let previously_owned = self.owned.get(index).unwrap_unchecked(); if !previously_owned { - self.owned.set(id.unzip().0 as usize, true); + *self.epochs.get_unchecked_mut(index) = epoch; *self.ref_counts.get_unchecked_mut(index) = Some(ref_count.clone()); + self.owned.set(index, true); } } } @@ -95,19 +133,25 @@ impl StatelessTracker { self.set_max_index(incoming_size); } - iterate_bitvec(&other.owned, |index| unsafe { - let previously_owned = self.owned.get(index).unwrap_unchecked(); + for index in iterate_bitvec_indices(&other.owned) { + self.debug_assert_in_bounds(index); + unsafe { + let previously_owned = self.owned.get(index).unwrap_unchecked(); - if !previously_owned { - self.owned.set(index, true); + if !previously_owned { + self.owned.set(index, true); - let other_ref_count = other - .ref_counts - .get_unchecked(index) - .unwrap_unchecked() - .clone(); - *self.ref_counts.get_unchecked_mut(index) = Some(other_ref_count); + let other_ref_count = other + .ref_counts + .get_unchecked(index) + .unwrap_unchecked() + .clone(); + *self.ref_counts.get_unchecked_mut(index) = Some(other_ref_count); + + let epoch = *other.epochs.get_unchecked(index); + *self.epochs.get_unchecked_mut(index) = epoch; + } } - }) + } } } diff --git a/wgpu-core/src/track/texture.rs b/wgpu-core/src/track/texture.rs index 37791246bf..adff0e2e30 100644 --- a/wgpu-core/src/track/texture.rs +++ b/wgpu-core/src/track/texture.rs @@ -1,13 +1,13 @@ use super::{range::RangedStates, PendingTransition}; use crate::{ hub, - id::{TextureId, TypedId}, + id::{TextureId, TypedId, Valid}, resource::Texture, track::{ - invalid_resource_state, iterate_bitvec, resize_bitvec, skip_barrier, ResourceUses, + invalid_resource_state, iterate_bitvec_indices, resize_bitvec, skip_barrier, ResourceUses, UsageConflict, }, - RefCount, + Epoch, RefCount, }; use bit_vec::BitVec; use hal::TextureUses; @@ -15,7 +15,7 @@ use hal::TextureUses; use arrayvec::ArrayVec; use naga::FastHashMap; -use std::{iter, ops::Range, vec::Drain}; +use std::{iter, marker::PhantomData, ops::Range, vec::Drain}; #[derive(Clone, Debug, PartialEq, Eq)] pub struct TextureSelector { @@ -60,17 +60,26 @@ impl ComplexTextureState { } // TODO: This representation could be optimized in a couple ways, but keep it simple for now. -pub struct TextureBindGroupState { - textures: Vec<(TextureId, Option, RefCount, TextureUses)>, +pub struct TextureBindGroupState { + textures: Vec<( + Valid, + Option, + RefCount, + TextureUses, + )>, + + _phantom: PhantomData, } -impl TextureBindGroupState { +impl TextureBindGroupState { pub fn new() -> Self { Self { textures: Vec::new(), + + _phantom: PhantomData, } } - pub fn extend_with_refcount<'a, A: hal::Api>( + pub fn extend_with_refcount<'a>( &mut self, storage: &'a hub::Storage, TextureId>, id: TextureId, @@ -78,9 +87,11 @@ impl TextureBindGroupState { selector: Option, state: TextureUses, ) -> Option<&'a Texture> { - self.textures.push((id, selector, ref_count, state)); + let value = storage.get(id).ok()?; + + self.textures.push((Valid(id), selector, ref_count, state)); - storage.get(id).ok() + Some(value) } } @@ -99,29 +110,61 @@ impl TextureStateSet { } #[derive(Debug)] -pub struct TextureUsageScope { +pub struct TextureUsageScope { set: TextureStateSet, - owned: BitVec, + ref_counts: Vec>, + epochs: Vec, + + owned: BitVec, + + _phantom: PhantomData, } -impl TextureUsageScope { +impl TextureUsageScope { pub fn new() -> Self { Self { set: TextureStateSet::new(), - owned: BitVec::default(), + + epochs: Vec::new(), ref_counts: Vec::new(), + + owned: BitVec::default(), + + _phantom: PhantomData, } } + fn debug_assert_in_bounds(&self, index: usize) { + debug_assert!(index < self.set.simple.len()); + debug_assert!(index < self.owned.len()); + debug_assert!(index < self.ref_counts.len()); + debug_assert!(index < self.epochs.len()); + + debug_assert!(if self.owned.get(index).unwrap() { + self.ref_counts[index].is_some() + } else { + true + }); + + debug_assert!(if self.owned.get(index).unwrap() + && self.set.simple[index] == TextureUses::COMPLEX + { + self.set.complex.contains_key(&(index as u32)) + } else { + true + }); + } + pub fn set_max_index(&mut self, size: usize) { self.set.simple.resize(size, TextureUses::UNINITIALIZED); self.ref_counts.resize(size, None); + self.epochs.resize(size, u32::MAX); resize_bitvec(&mut self.owned, size); } - pub fn extend_from_scope( + pub fn extend_from_scope( &mut self, storage: &hub::Storage, TextureId>, scope: &Self, @@ -131,18 +174,20 @@ impl TextureUsageScope { self.set_max_index(incoming_size); } - iterate_bitvec(&scope.owned, |index| todo!()); + for index in iterate_bitvec_indices(&scope.owned) { + todo!() + } Ok(()) } - pub unsafe fn extend_from_bind_group( + pub unsafe fn extend_from_bind_group( &mut self, storage: &hub::Storage, TextureId>, - bind_group: &TextureBindGroupState, + bind_group: &TextureBindGroupState, ) -> Result<(), UsageConflict> { for (id, selector, ref_count, state) in &bind_group.textures { - self.extend_inner(storage, id.unzip().0, selector.clone(), ref_count, *state)?; + self.extend_inner(storage, *id, selector.clone(), ref_count, *state)?; } Ok(()) @@ -151,21 +196,20 @@ impl TextureUsageScope { /// # Safety /// /// `id` must be a valid ID and have an ID value less than the last call to set_max_index. - pub unsafe fn extend<'a, A: hal::Api>( + pub unsafe fn extend<'a>( &mut self, storage: &'a hub::Storage, TextureId>, id: TextureId, selector: Option, new_state: TextureUses, ) -> Result<&'a Texture, UsageConflict> { - let (index, _, _) = id.unzip(); let tex = storage .get(id) - .map_err(|_| UsageConflict::TextureInvalid { id: index })?; + .map_err(|_| UsageConflict::TextureInvalid { id })?; self.extend_inner( storage, - index, + Valid(id), selector, tex.life_guard.ref_count.as_ref().unwrap(), new_state, @@ -177,15 +221,18 @@ impl TextureUsageScope { /// # Safety /// /// `id` must be a valid ID and have an ID value less than the last call to set_max_index. - unsafe fn extend_inner( + unsafe fn extend_inner( &mut self, storage: &hub::Storage, TextureId>, - id: u32, + id: Valid, selector: Option, ref_count: &RefCount, new_state: TextureUses, ) -> Result<(), UsageConflict> { - let index = id as usize; + let (index32, epoch, _) = id.0.unzip(); + let index = index32 as usize; + + self.debug_assert_in_bounds(index); let currently_active = self.owned.get(index).unwrap_unchecked(); if currently_active { @@ -210,28 +257,31 @@ impl TextureUsageScope { return Ok(()); } // The old usage is complex. - (true, selector) => return self.extend_complex(storage, id, selector, new_state), + (true, selector) => { + return self.extend_complex(storage, id, index32, selector, new_state) + } // The old usage is simple, so demote it to a complex one. (false, Some(selector)) => { *self.set.simple.get_unchecked_mut(index) = hal::TextureUses::COMPLEX; // Demote our simple state to a complex one. - self.extend_complex(storage, id, None, current_state)?; + self.extend_complex(storage, id, index32, None, current_state)?; // Extend that complex state with our new complex state. - return self.extend_complex(storage, id, Some(selector), new_state); + return self.extend_complex(storage, id, index32, Some(selector), new_state); } } } // We're the first to use this resource, let's add it. *self.ref_counts.get_unchecked_mut(index) = Some(ref_count.clone()); + *self.epochs.get_unchecked_mut(index) = epoch; self.owned.set(index, true); if let Some(selector) = selector { *self.set.simple.get_unchecked_mut(index) = hal::TextureUses::COMPLEX; - self.extend_complex(storage, id, Some(selector), new_state)?; + self.extend_complex(storage, id, index32, Some(selector), new_state)?; } else { *self.set.simple.get_unchecked_mut(index) = new_state; } @@ -240,17 +290,18 @@ impl TextureUsageScope { } #[cold] - unsafe fn extend_complex( + unsafe fn extend_complex( &mut self, storage: &hub::Storage, TextureId>, - id: u32, + id: Valid, + index32: u32, selector: Option, new_state: TextureUses, ) -> Result<(), UsageConflict> { - let texture = storage.get_unchecked(id); + let texture = storage.get_unchecked(index32); // Create the complex entry for this texture. - let complex = self.set.complex.entry(id).or_insert_with(|| { + let complex = self.set.complex.entry(index32).or_insert_with(|| { ComplexTextureState::new( texture.desc.mip_level_count, texture.desc.array_layer_count(), @@ -294,50 +345,100 @@ impl TextureUsageScope { } } -pub(crate) struct TextureTracker { +pub(crate) struct TextureTracker { start_set: TextureStateSet, end_set: TextureStateSet, + + epochs: Vec, + ref_counts: Vec>, + owned: BitVec, + /// Temporary storage for collecting transitions. temp: Vec>, - owned: BitVec, - ref_counts: Vec>, + + _phantom: PhantomData, } -impl TextureTracker { +impl TextureTracker { pub fn new() -> Self { Self { start_set: TextureStateSet::new(), end_set: TextureStateSet::new(), - temp: Vec::new(), - owned: BitVec::default(), + + epochs: Vec::new(), ref_counts: Vec::new(), + owned: BitVec::default(), + + temp: Vec::new(), + + _phantom: PhantomData, } } + fn debug_assert_in_bounds(&self, index: usize) { + debug_assert!(index < self.start_set.simple.len()); + debug_assert!(index < self.end_set.simple.len()); + debug_assert!(index < self.owned.len()); + debug_assert!(index < self.ref_counts.len()); + debug_assert!(index < self.epochs.len()); + + debug_assert!(if self.owned.get(index).unwrap() { + self.ref_counts[index].is_some() + } else { + true + }); + + debug_assert!(if self.owned.get(index).unwrap() + && self.start_set.simple[index] == TextureUses::COMPLEX + { + self.start_set.complex.contains_key(&(index as u32)) + } else { + true + }); + debug_assert!(if self.owned.get(index).unwrap() + && self.end_set.simple[index] == TextureUses::COMPLEX + { + self.end_set.complex.contains_key(&(index as u32)) + } else { + true + }); + } + fn set_max_index(&mut self, size: usize) { self.start_set .simple .resize(size, TextureUses::UNINITIALIZED); self.end_set.simple.resize(size, TextureUses::UNINITIALIZED); self.ref_counts.resize(size, None); + self.epochs.resize(size, u32::MAX); resize_bitvec(&mut self.owned, size); } + pub fn used(&self) -> impl Iterator> + '_ { + self.debug_assert_in_bounds(self.owned.len() - 1); + iterate_bitvec_indices(&self.owned).map(move |index| { + let epoch = unsafe { *self.epochs.get_unchecked(index) }; + Valid(TextureId::zip(index as u32, epoch, A::VARIANT)) + }) + } + pub fn drain(&mut self) -> Drain> { self.temp.drain(..) } pub unsafe fn init(&mut self, id: TextureId, ref_count: RefCount, usage: TextureUses) { - let index = id.unzip().0 as usize; + let (index32, epoch, _) = id.unzip(); + let index = index32 as usize; *self.start_set.simple.get_unchecked_mut(index) = usage; *self.end_set.simple.get_unchecked_mut(index) = usage; - self.owned.set(index, true); + *self.epochs.get_unchecked_mut(index) = epoch; *self.ref_counts.get_unchecked_mut(index) = Some(ref_count); + self.owned.set(index, true); } - pub fn change_state<'a, A: hal::Api>( + pub fn change_state<'a>( &mut self, storage: &'a hub::Storage, TextureId>, id: TextureId, @@ -347,62 +448,67 @@ impl TextureTracker { todo!() } - pub fn change_states_scope( + pub fn change_states_scope( &mut self, storage: &hub::Storage, TextureId>, - scope: &TextureUsageScope, + scope: &TextureUsageScope, ) { - self.change_states_inner(storage, &scope.set, &scope.owned, &scope.ref_counts) - } - - fn change_states_inner( - &mut self, - storage: &hub::Storage, TextureId>, - incoming_set: &TextureStateSet, - incoming_ownership: &BitVec, - incoming_ref_counts: &Vec>, - ) { - let incoming_size = incoming_set.simple.len(); + let incoming_size = scope.set.simple.len(); if incoming_size > self.start_set.simple.len() { self.set_max_index(incoming_size); } - iterate_bitvec(incoming_ownership, |index| { - unsafe { self.transition(storage, incoming_set, incoming_ref_counts, index) }; - }); + scope.debug_assert_in_bounds(scope.owned.len() - 1); + for index in iterate_bitvec_indices(&scope.owned) { + unsafe { + let epoch = *scope.epochs.get_unchecked(index); + self.transition( + storage, + scope, + Valid(TextureId::zip(index as u32, epoch, A::VARIANT)), + index, + ) + }; + } } - pub unsafe fn change_states_bind_group( + pub unsafe fn change_states_bind_group( &mut self, storage: &hub::Storage, TextureId>, - scope: &mut TextureUsageScope, - bind_group_state: &TextureBindGroupState, + scope: &mut TextureUsageScope, + bind_group_state: &TextureBindGroupState, ) { let incoming_size = scope.set.simple.len(); if incoming_size > self.start_set.simple.len() { self.set_max_index(incoming_size); } - for &(index, _, _, _) in bind_group_state.textures.iter() { - let index = index.unzip().0 as usize; + for &(id, _, _, _) in bind_group_state.textures.iter() { + let (index32, epoch, _) = id.0.unzip(); + let index = index32 as usize; + scope.debug_assert_in_bounds(index); + if !scope.owned.get(index).unwrap_unchecked() { continue; } - self.transition(storage, &scope.set, &scope.ref_counts, index); + self.transition(storage, &scope, id, index); scope.owned.set(index, false); } } - unsafe fn transition( + unsafe fn transition( &mut self, storage: &hub::Storage, TextureId>, - incoming_set: &TextureStateSet, - incoming_ref_counts: &Vec>, + scope: &TextureUsageScope, + id: Valid, index: usize, ) { + // Note: both callees of this function call scope.debug_assert_in_bounds. + self.debug_assert_in_bounds(index); + let old_tracked = self.owned.get(index).unwrap_unchecked(); let old_state = *self.end_set.simple.get_unchecked(index); - let new_state = *incoming_set.simple.get_unchecked(index); + let new_state = *scope.set.simple.get_unchecked(index); match ( old_tracked, @@ -415,17 +521,21 @@ impl TextureTracker { self.owned.set(index, true); - let ref_count = incoming_ref_counts + let ref_count = scope + .ref_counts .get_unchecked(index) .unwrap_unchecked() .clone(); *self.ref_counts.get_unchecked_mut(index) = Some(ref_count); + + let epoch = *scope.epochs.get_unchecked(index); + *self.epochs.get_unchecked_mut(index) = epoch; } (false, _, true) => { *self.start_set.simple.get_unchecked_mut(index) = TextureUses::COMPLEX; *self.end_set.simple.get_unchecked_mut(index) = TextureUses::COMPLEX; - let complex_state = incoming_set.complex.get(&(index as u32)).unwrap_unchecked(); + let complex_state = scope.set.complex.get(&(index as u32)).unwrap_unchecked(); self.start_set .complex .insert(index as u32, complex_state.clone()); @@ -435,11 +545,15 @@ impl TextureTracker { self.owned.set(index, true); - let ref_count = incoming_ref_counts + let ref_count = scope + .ref_counts .get_unchecked(index) .unwrap_unchecked() .clone(); *self.ref_counts.get_unchecked_mut(index) = Some(ref_count); + + let epoch = *scope.epochs.get_unchecked(index); + *self.epochs.get_unchecked_mut(index) = epoch; } (true, false, false) => { if skip_barrier(old_state, new_state) { @@ -455,26 +569,25 @@ impl TextureTracker { *self.end_set.simple.get_unchecked_mut(index) = new_state; } (true, true, true) => { - self.transition_complex_to_complex(storage, incoming_set, index); + self.transition_complex_to_complex(storage, scope, index); } (true, true, false) => { - self.transition_complex_to_simple(storage, incoming_set, index, new_state); + self.transition_complex_to_simple(storage, index, new_state); } (true, false, true) => { - self.transition_simple_to_complex(storage, incoming_set, index, old_state); + self.transition_simple_to_complex(storage, scope, index, old_state); } } } - #[cold] - unsafe fn transition_complex_to_complex( + unsafe fn transition_complex_to_complex( &mut self, storage: &hub::Storage, TextureId>, - incoming_set: &TextureStateSet, + scope: &TextureUsageScope, index: usize, ) { let old_complex = self.end_set.complex.get(&(index as u32)).unwrap_unchecked(); - let new_complex = self.end_set.complex.get(&(index as u32)).unwrap_unchecked(); + let new_complex = scope.set.complex.get(&(index as u32)).unwrap_unchecked(); let mut temp = Vec::new(); debug_assert!(old_complex.mips.len() >= new_complex.mips.len()); @@ -521,11 +634,9 @@ impl TextureTracker { } } - #[cold] - unsafe fn transition_complex_to_simple( + unsafe fn transition_complex_to_simple( &mut self, storage: &hub::Storage, TextureId>, - incoming_set: &TextureStateSet, index: usize, new_state: TextureUses, ) { @@ -557,15 +668,14 @@ impl TextureTracker { *self.end_set.simple.get_unchecked_mut(index) = new_state; } - #[cold] - unsafe fn transition_simple_to_complex( + unsafe fn transition_simple_to_complex( &mut self, storage: &hub::Storage, TextureId>, - incoming_set: &TextureStateSet, + scope: &TextureUsageScope, index: usize, old_state: TextureUses, ) { - let new_complex = incoming_set.complex.get(&(index as u32)).unwrap_unchecked(); + let new_complex = scope.set.complex.get(&(index as u32)).unwrap_unchecked(); for (mip_index, mips) in new_complex.mips.iter().enumerate() { let mip_index = mip_index as u32; From a65b60e7252d957054fb4e1e776f5bed1932ca04 Mon Sep 17 00:00:00 2001 From: Connor Fitzgerald Date: Sat, 7 May 2022 04:25:07 -0400 Subject: [PATCH 11/35] temp10 - Compiling?!1?!1?!1 --- wgpu-core/src/binding_model.rs | 2 +- wgpu-core/src/command/bundle.rs | 10 +- wgpu-core/src/command/clear.rs | 43 ++---- wgpu-core/src/command/compute.rs | 8 +- wgpu-core/src/command/memory_init.rs | 10 +- wgpu-core/src/command/mod.rs | 23 +++- wgpu-core/src/command/render.rs | 26 +--- wgpu-core/src/command/transfer.rs | 13 +- wgpu-core/src/device/life.rs | 24 ++-- wgpu-core/src/device/mod.rs | 49 +++---- wgpu-core/src/device/queue.rs | 114 ++++++++-------- wgpu-core/src/hub.rs | 10 +- wgpu-core/src/id.rs | 2 +- wgpu-core/src/present.rs | 8 +- wgpu-core/src/track/buffer.rs | 104 ++++++++++---- wgpu-core/src/track/mod.rs | 43 +++--- wgpu-core/src/track/range.rs | 4 +- wgpu-core/src/track/stateless.rs | 50 +++++-- wgpu-core/src/track/texture.rs | 194 +++++++++++++++++++-------- 19 files changed, 427 insertions(+), 310 deletions(-) diff --git a/wgpu-core/src/binding_model.rs b/wgpu-core/src/binding_model.rs index dae4771fcb..ef97af6d91 100644 --- a/wgpu-core/src/binding_model.rs +++ b/wgpu-core/src/binding_model.rs @@ -1,7 +1,7 @@ use crate::{ device::{DeviceError, MissingDownlevelFlags, MissingFeatures, SHADER_STAGE_COUNT}, error::{ErrorFormatter, PrettyError}, - hub::{Resource, HalApi}, + hub::{HalApi, Resource}, id::{BindGroupLayoutId, BufferId, DeviceId, SamplerId, TextureId, TextureViewId, Valid}, init_tracker::{BufferInitTrackerAction, TextureInitTrackerAction}, track::{BindGroupStates, UsageConflict}, diff --git a/wgpu-core/src/command/bundle.rs b/wgpu-core/src/command/bundle.rs index 533fb232b9..540eb97c00 100644 --- a/wgpu-core/src/command/bundle.rs +++ b/wgpu-core/src/command/bundle.rs @@ -45,7 +45,7 @@ use crate::{ SHADER_STAGE_COUNT, }, error::{ErrorFormatter, PrettyError}, - hub::{GlobalIdentityHandlerFactory, Hub, Resource, Storage, Token, HalApi}, + hub::{GlobalIdentityHandlerFactory, HalApi, Hub, Resource, Storage, Token}, id, init_tracker::{BufferInitTrackerAction, MemoryInitKind, TextureInitTrackerAction}, pipeline::{self, PipelineFlags}, @@ -179,7 +179,7 @@ impl RenderBundleEncoder { let (bind_group_guard, mut token) = hub.bind_groups.read(&mut token); let (pipeline_guard, mut token) = hub.render_pipelines.read(&mut token); let (buffer_guard, mut token) = hub.buffers.read(&mut token); - let (texture_guard, mut token) = hub.textures.read(&mut token); + let (texture_guard, _) = hub.textures.read(&mut token); let mut state = State { trackers: RenderBundleScope::new(), @@ -262,11 +262,7 @@ impl RenderBundleEncoder { unsafe { state .trackers - .extend_from_bind_group( - &*buffer_guard, - &*texture_guard, - &bind_group.used, - ) + .extend_from_bind_group(&*texture_guard, &bind_group.used) .map_pass_err(scope)? }; //Note: stateless trackers are not merged: the lifetime reference diff --git a/wgpu-core/src/command/clear.rs b/wgpu-core/src/command/clear.rs index a7fa1051f9..8042e4106b 100644 --- a/wgpu-core/src/command/clear.rs +++ b/wgpu-core/src/command/clear.rs @@ -4,7 +4,6 @@ use std::{num::NonZeroU32, ops::Range}; use crate::device::trace::Command as TraceCommand; use crate::{ command::CommandBuffer, - device::Device, get_lowest_common_denom, hub::{self, Global, GlobalIdentityHandlerFactory, HalApi, Token}, id::{BufferId, CommandEncoderId, DeviceId, TextureId, Valid}, @@ -219,17 +218,19 @@ impl Global { }); } + let device = &device_guard[cmd_buf.device_id.value]; + clear_texture( &*texture_guard, Valid(dst), - dst_texture, TextureInitRange { mip_range: subresource_range.base_mip_level..subresource_level_end, layer_range: subresource_range.base_array_layer..subresource_layer_end, }, cmd_buf.encoder.open(), &mut cmd_buf.trackers.textures, - &device_guard[cmd_buf.device_id.value], + &device.alignments, + &device.zero_buffer, ) } } @@ -237,34 +238,14 @@ impl Global { pub(crate) fn clear_texture( storage: &hub::Storage, TextureId>, dst_texture_id: Valid, - dst_texture: &Texture, - range: TextureInitRange, - encoder: &mut A::CommandEncoder, - texture_tracker: &mut TextureTracker, - device: &Device, -) -> Result<(), ClearError> { - clear_texture_no_device( - storage, - dst_texture_id, - dst_texture, - range, - encoder, - texture_tracker, - &device.alignments, - &device.zero_buffer, - ) -} - -pub(crate) fn clear_texture_no_device( - storage: &hub::Storage, TextureId>, - dst_texture_id: Valid, - dst_texture: &Texture, range: TextureInitRange, encoder: &mut A::CommandEncoder, texture_tracker: &mut TextureTracker, alignments: &hal::Alignments, zero_buffer: &A::Buffer, ) -> Result<(), ClearError> { + let dst_texture = &storage[dst_texture_id]; + let dst_raw = dst_texture .inner .as_raw() @@ -293,11 +274,13 @@ pub(crate) fn clear_texture_no_device( // On the other hand, when coming via command_encoder_clear_texture, the life_guard is still there since in order to call it a texture object is needed. // // We could in theory distinguish these two scenarios in the internal clear_texture api in order to remove this check and call the cheaper change_replace_tracked whenever possible. - let dst_barrier = texture_tracker - .change_state(storage, dst_texture_id.0, selector, clear_usage) - .unwrap() - .1 - .map(|pending| pending.into_hal(dst_texture)); + let dst_barrier = unsafe { + texture_tracker + .change_state(storage, dst_texture_id.0, selector, clear_usage) + .unwrap() + .1 + .map(|pending| pending.into_hal(dst_texture)) + }; unsafe { encoder.transition_textures(dst_barrier.into_iter()); } diff --git a/wgpu-core/src/command/compute.rs b/wgpu-core/src/command/compute.rs index 95ed7b9c65..302a28edd0 100644 --- a/wgpu-core/src/command/compute.rs +++ b/wgpu-core/src/command/compute.rs @@ -263,11 +263,8 @@ impl State { ) -> Result<(), UsageConflict> { for id in self.binder.list_active() { unsafe { - self.scope.extend_from_bind_group( - buffer_guard, - texture_guard, - &bind_group_guard[id].used, - )? + self.scope + .extend_from_bind_group(texture_guard, &bind_group_guard[id].used)? }; // Note: stateless trackers are not merged: the lifetime reference // is held to the bind group itself. @@ -276,7 +273,6 @@ impl State { for id in self.binder.list_active() { unsafe { base_trackers.extend_from_bind_group( - buffer_guard, texture_guard, &mut self.scope, &bind_group_guard[id].used, diff --git a/wgpu-core/src/command/memory_init.rs b/wgpu-core/src/command/memory_init.rs index d9717cef04..b9587f3527 100644 --- a/wgpu-core/src/command/memory_init.rs +++ b/wgpu-core/src/command/memory_init.rs @@ -4,7 +4,7 @@ use hal::CommandEncoder; use crate::{ device::Device, - hub::{Storage, HalApi}, + hub::{HalApi, Storage}, id::{self, TextureId}, init_tracker::*, resource::{Buffer, Texture}, @@ -134,14 +134,14 @@ pub(crate) fn fixup_discarded_surfaces< clear_texture( texture_guard, id::Valid(init.texture), - texture_guard.get(init.texture).unwrap(), TextureInitRange { mip_range: init.mip_level..(init.mip_level + 1), layer_range: init.layer..(init.layer + 1), }, encoder, texture_tracker, - device, + &device.alignments, + &device.zero_buffer, ) .unwrap(); } @@ -282,11 +282,11 @@ impl BakedCommands { clear_texture( texture_guard, id::Valid(texture_use.id), - &*texture, range, &mut self.encoder, &mut device_tracker.textures, - device, + &device.alignments, + &device.zero_buffer, ) .unwrap(); } diff --git a/wgpu-core/src/command/mod.rs b/wgpu-core/src/command/mod.rs index 796bb1d450..78896bc8cd 100644 --- a/wgpu-core/src/command/mod.rs +++ b/wgpu-core/src/command/mod.rs @@ -10,7 +10,7 @@ mod transfer; use std::slice; -pub(crate) use self::clear::clear_texture_no_device; +pub(crate) use self::clear::clear_texture; pub use self::{ bundle::*, clear::ClearError, compute::*, draw::*, query::*, render::*, transfer::*, }; @@ -138,7 +138,23 @@ impl CommandBuffer { } } - pub(crate) fn insert_barriers( + pub(crate) fn insert_barriers_from_tracker( + raw: &mut A::CommandEncoder, + base: &mut Tracker, + head: &Tracker, + buffer_guard: &Storage, id::BufferId>, + texture_guard: &Storage, id::TextureId>, + ) { + profiling::scope!("insert_barriers"); + + base.buffers.change_states_tracker(&head.buffers); + base.textures + .change_states_tracker(&*texture_guard, &head.textures); + + Self::drain_barriers(raw, base, buffer_guard, texture_guard); + } + + pub(crate) fn insert_barriers_from_scope( raw: &mut A::CommandEncoder, base: &mut Tracker, head: &UsageScope, @@ -147,8 +163,7 @@ impl CommandBuffer { ) { profiling::scope!("insert_barriers"); - base.buffers - .change_states_scope(&*buffer_guard, &head.buffers); + base.buffers.change_states_scope(&head.buffers); base.textures .change_states_scope(&*texture_guard, &head.textures); diff --git a/wgpu-core/src/command/render.rs b/wgpu-core/src/command/render.rs index 72293103bb..b8f79d5dc1 100644 --- a/wgpu-core/src/command/render.rs +++ b/wgpu-core/src/command/render.rs @@ -1084,8 +1084,7 @@ impl Global { let (query_set_guard, mut token) = hub.query_sets.read(&mut token); let (buffer_guard, mut token) = hub.buffers.read(&mut token); let (texture_guard, mut token) = hub.textures.read(&mut token); - let (view_guard, mut token) = hub.texture_views.read(&mut token); - let (sampler_guard, _) = hub.samplers.read(&mut token); + let (view_guard, _) = hub.texture_views.read(&mut token); log::trace!( "Encoding render pass begin in command buffer {:?}", @@ -1160,11 +1159,7 @@ impl Global { // merge the resource tracker in unsafe { info.usage_scope - .extend_from_bind_group( - &*buffer_guard, - &*texture_guard, - &bind_group.used, - ) + .extend_from_bind_group(&*texture_guard, &bind_group.used) .map_pass_err(scope)?; } //Note: stateless trackers are not merged: the lifetime reference @@ -1936,22 +1931,11 @@ impl Global { unsafe { info.usage_scope - .extend_from_render_bundle( - &*buffer_guard, - &*texture_guard, - &bundle.used, - ) + .extend_from_render_bundle(&*texture_guard, &bundle.used) .map_pass_err(scope)?; cmd_buf .trackers - .extend_from_render_bundle( - &*view_guard, - &*sampler_guard, - &*bind_group_guard, - &*render_pipe_guard, - &*query_set_guard, - &bundle.used, - ) + .extend_from_render_bundle(&bundle.used) .map_pass_err(scope)?; }; state.reset_bundle(); @@ -1993,7 +1977,7 @@ impl Global { .map_err(RenderCommandError::InvalidQuerySet) .map_pass_err(PassErrorScope::QueryReset)?; - super::CommandBuffer::insert_barriers( + super::CommandBuffer::insert_barriers_from_scope( transit, &mut cmd_buf.trackers, &scope, diff --git a/wgpu-core/src/command/transfer.rs b/wgpu-core/src/command/transfer.rs index e901ff3fd1..26f0489fb8 100644 --- a/wgpu-core/src/command/transfer.rs +++ b/wgpu-core/src/command/transfer.rs @@ -1,7 +1,7 @@ #[cfg(feature = "trace")] use crate::device::trace::Command as TraceCommand; use crate::{ - command::{CommandBuffer, CommandEncoderError}, + command::{clear_texture, CommandBuffer, CommandEncoderError}, conv, device::{Device, MissingDownlevelFlags}, error::{ErrorFormatter, PrettyError}, @@ -22,8 +22,6 @@ use wgt::{BufferAddress, BufferUsages, Extent3d, TextureUsages}; use std::iter; -use super::clear::clear_texture; - pub type ImageCopyBuffer = wgt::ImageCopyBuffer; pub type ImageCopyTexture = wgt::ImageCopyTexture; @@ -388,7 +386,6 @@ fn handle_texture_init( copy_texture: &ImageCopyTexture, copy_size: &Extent3d, texture_guard: &Storage, TextureId>, - texture: &Texture, ) { let init_action = TextureInitTrackerAction { id: copy_texture.texture, @@ -412,14 +409,14 @@ fn handle_texture_init( clear_texture( texture_guard, Valid(init.texture), - texture, TextureInitRange { mip_range: init.mip_level..(init.mip_level + 1), layer_range: init.layer..(init.layer + 1), }, cmd_buf_raw, &mut cmd_buf.trackers.textures, - device, + &device.alignments, + &device.zero_buffer, ) .unwrap(); } @@ -434,7 +431,7 @@ fn handle_src_texture_init( copy_size: &Extent3d, texture_guard: &Storage, TextureId>, ) -> Result<(), TransferError> { - let texture = texture_guard + let _ = texture_guard .get(source.texture) .map_err(|_| TransferError::InvalidTexture(source.texture))?; @@ -445,7 +442,6 @@ fn handle_src_texture_init( source, copy_size, texture_guard, - texture, ); Ok(()) } @@ -481,7 +477,6 @@ fn handle_dst_texture_init( destination, copy_size, texture_guard, - texture, ); Ok(()) } diff --git a/wgpu-core/src/device/life.rs b/wgpu-core/src/device/life.rs index f559164af3..2dd4648a41 100644 --- a/wgpu-core/src/device/life.rs +++ b/wgpu-core/src/device/life.rs @@ -7,7 +7,7 @@ use crate::{ }, hub::{GlobalIdentityHandlerFactory, HalApi, Hub, Token}, id, resource, - track::Tracker, + track::{BindGroupStates, RenderBundleScope, Tracker}, RefCount, Stored, SubmissionIndex, }; use smallvec::SmallVec; @@ -68,17 +68,23 @@ impl SuspectedResources { self.query_sets.extend_from_slice(&other.query_sets); } - pub(super) fn add_trackers(&mut self, trackers: &TrackerSet) { + pub(super) fn add_render_bundle_scope(&mut self, trackers: &RenderBundleScope) { self.buffers.extend(trackers.buffers.used()); self.textures.extend(trackers.textures.used()); self.texture_views.extend(trackers.views.used()); self.samplers.extend(trackers.samplers.used()); self.bind_groups.extend(trackers.bind_groups.used()); - self.compute_pipelines.extend(trackers.compute_pipes.used()); - self.render_pipelines.extend(trackers.render_pipes.used()); - self.render_bundles.extend(trackers.bundles.used()); + self.render_pipelines + .extend(trackers.render_pipelines.used()); self.query_sets.extend(trackers.query_sets.used()); } + + pub(super) fn add_bind_group_states(&mut self, trackers: &BindGroupStates) { + self.buffers.extend(trackers.buffers.used()); + self.textures.extend(trackers.textures.used()); + self.texture_views.extend(trackers.views.used()); + self.samplers.extend(trackers.samplers.used()); + } } /// Raw backend resources that should be freed shortly. @@ -511,7 +517,7 @@ impl LifetimeTracker { } if let Some(res) = hub.render_bundles.unregister_locked(id.0, &mut *guard) { - self.suspected_resources.add_trackers(&res.used); + self.suspected_resources.add_render_bundle_scope(&res.used); } } } @@ -530,7 +536,7 @@ impl LifetimeTracker { } if let Some(res) = hub.bind_groups.unregister_locked(id.0, &mut *guard) { - self.suspected_resources.add_trackers(&res.used); + self.suspected_resources.add_bind_group_states(&res.used); self.suspected_resources .bind_group_layouts @@ -671,7 +677,7 @@ impl LifetimeTracker { let mut trackers = trackers.lock(); for id in self.suspected_resources.compute_pipelines.drain(..) { - if trackers.compute_pipes.remove_abandoned(id) { + if trackers.compute_pipelines.remove_abandoned(id) { log::debug!("Compute pipeline {:?} will be destroyed", id); #[cfg(feature = "trace")] if let Some(t) = trace { @@ -696,7 +702,7 @@ impl LifetimeTracker { let mut trackers = trackers.lock(); for id in self.suspected_resources.render_pipelines.drain(..) { - if trackers.render_pipes.remove_abandoned(id) { + if trackers.render_pipelines.remove_abandoned(id) { log::debug!("Render pipeline {:?} will be destroyed", id); #[cfg(feature = "trace")] if let Some(t) = trace { diff --git a/wgpu-core/src/device/mod.rs b/wgpu-core/src/device/mod.rs index 61c2586633..6438b42b2c 100644 --- a/wgpu-core/src/device/mod.rs +++ b/wgpu-core/src/device/mod.rs @@ -1,9 +1,7 @@ use crate::{ binding_model, command, conv, device::life::WaitIdleError, - hub::{ - Global, GlobalIdentityHandlerFactory, HalApi, Hub, Input, InvalidId, Storage, Token, - }, + hub::{Global, GlobalIdentityHandlerFactory, HalApi, Hub, Input, InvalidId, Storage, Token}, id, init_tracker::{ BufferInitTracker, BufferInitTrackerAction, MemoryInitKind, TextureInitRange, @@ -3183,7 +3181,13 @@ impl Global { let id = fid.assign(buffer, &mut token); log::info!("Created buffer {:?} with {:?}", id, desc); - unsafe { device.trackers.lock().buffers.init(id, ref_count, buffer_use) }; + unsafe { + device + .trackers + .lock() + .buffers + .init(id, ref_count, buffer_use) + }; return (id.0, None); }; @@ -3444,8 +3448,6 @@ impl Global { Ok(texture) => texture, Err(error) => break error, }; - let num_levels = texture.full_range.mips.end; - let num_layers = texture.full_range.layers.end; let ref_count = texture.life_guard.add_ref(); let id = fid.assign(texture, &mut token); @@ -3523,20 +3525,17 @@ impl Global { texture.initialization_status = TextureInitTracker::new(desc.mip_level_count, 0); - let num_levels = texture.full_range.mips.end; - let num_layers = texture.full_range.layers.end; let ref_count = texture.life_guard.add_ref(); let id = fid.assign(texture, &mut token); log::info!("Created texture {:?} with {:?}", id, desc); - unsafe { - device.trackers.lock().textures.init( - id.0, - ref_count, - hal::TextureUses::UNINITIALIZED, - ) - }; + device + .trackers + .lock() + .textures + .init(id.0, ref_count, hal::TextureUses::UNINITIALIZED); + return (id.0, None); }; @@ -4525,11 +4524,7 @@ impl Global { log::info!("Created render pipeline {:?} with {:?}", id, desc); unsafe { - device - .trackers - .lock() - .render_pipelines - .init(id, ref_count); + device.trackers.lock().render_pipelines.init(id, ref_count); } return (id.0, None); }; @@ -4666,13 +4661,7 @@ impl Global { let id = fid.assign(pipeline, &mut token); log::info!("Created compute pipeline {:?} with {:?}", id, desc); - unsafe { - device - .trackers - .lock() - .compute_pipelines - .init(id, ref_count) - }; + unsafe { device.trackers.lock().compute_pipelines.init(id, ref_count) }; return (id.0, None); }; @@ -5132,15 +5121,17 @@ impl Global { let device = &device_guard[buffer.device_id.value]; + let ret = (buffer.device_id.value, buffer.life_guard.add_ref()); + unsafe { - let trackers = device.trackers.lock(); + let mut trackers = device.trackers.lock(); trackers .buffers .change_state(&*buffer_guard, buffer_id, internal_use); trackers.buffers.drain(); }; - (buffer.device_id.value, buffer.life_guard.add_ref()) + ret }; let device = &device_guard[device_id]; diff --git a/wgpu-core/src/device/queue.rs b/wgpu-core/src/device/queue.rs index 8f31543400..9f84aaee0d 100644 --- a/wgpu-core/src/device/queue.rs +++ b/wgpu-core/src/device/queue.rs @@ -453,10 +453,9 @@ impl Global { .drain(init_layer_range) .collect::>>() { - crate::command::clear_texture_no_device( + crate::command::clear_texture( &*texture_guard, id::Valid(destination.texture), - &*dst, TextureInitRange { mip_range: destination.mip_level..(destination.mip_level + 1), layer_range, @@ -600,7 +599,7 @@ impl Global { device.active_submission_index += 1; let submit_index = device.active_submission_index; let mut active_executions = Vec::new(); - let mut used_surface_textures = track::ResourceTracker::new(A::VARIANT); + let mut used_surface_textures = track::TextureUsageScope::new(); { let (mut command_buffer_guard, mut token) = hub.command_buffers.write(&mut token); @@ -648,7 +647,7 @@ impl Global { } // optimize the tracked states - cmdbuf.trackers.optimize(); + // cmdbuf.trackers.optimize(); // update submission IDs for id in cmdbuf.trackers.buffers.used() { @@ -675,33 +674,35 @@ impl Global { } for id in cmdbuf.trackers.textures.used() { let texture = &mut texture_guard[id]; - match texture.inner { + let should_extend = match texture.inner { TextureInner::Native { raw: None } => { return Err(QueueSubmitError::DestroyedTexture(id.0)); } - TextureInner::Native { raw: Some(_) } => {} + TextureInner::Native { raw: Some(_) } => false, TextureInner::Surface { ref mut has_work, .. } => { - use track::ResourceState as _; - *has_work = true; - let ref_count = cmdbuf.trackers.textures.get_ref_count(id); - //TODO: better error handling here? - // register it in the temporary tracker. - let mut ts = track::OldTextureState::default(); - let _ = ts.change( - id, - texture.full_range.clone(), - hal::TextureUses::empty(), //present - None, - ); - let _ = used_surface_textures.init(id, ref_count.clone(), ts); + true } - } + }; if !texture.life_guard.use_at(submit_index) { device.temp_suspected.textures.push(id); } + if should_extend { + let ref_count = cmdbuf.trackers.textures.get_ref_count(id); + unsafe { + used_surface_textures + .extend_refcount( + &*texture_guard, + id, + None, + ref_count, + hal::TextureUses::empty(), + ) + .unwrap(); + }; + } } for id in cmdbuf.trackers.views.used() { if !texture_view_guard[id].life_guard.use_at(submit_index) { @@ -723,13 +724,13 @@ impl Global { sampler_guard[sub_id].life_guard.use_at(submit_index); } } - assert!(cmdbuf.trackers.samplers.is_empty()); - for id in cmdbuf.trackers.compute_pipes.used() { + // assert!(cmdbuf.trackers.samplers.is_empty()); + for id in cmdbuf.trackers.compute_pipelines.used() { if !compute_pipe_guard[id].life_guard.use_at(submit_index) { device.temp_suspected.compute_pipelines.push(id); } } - for id in cmdbuf.trackers.render_pipes.used() { + for id in cmdbuf.trackers.render_pipelines.used() { if !render_pipe_guard[id].life_guard.use_at(submit_index) { device.temp_suspected.render_pipelines.push(id); } @@ -747,12 +748,12 @@ impl Global { // We need to update the submission indices for the contained // state-less (!) resources as well, excluding the bind groups. // They don't get deleted too early if the bundle goes out of scope. - for sub_id in bundle.used.compute_pipes.used() { - compute_pipe_guard[sub_id].life_guard.use_at(submit_index); - } - for sub_id in bundle.used.render_pipes.used() { + for sub_id in bundle.used.render_pipelines.used() { render_pipe_guard[sub_id].life_guard.use_at(submit_index); } + for sub_id in bundle.used.query_sets.used() { + query_set_guard[sub_id].life_guard.use_at(submit_index); + } } let mut baked = cmdbuf.into_baked(); @@ -772,11 +773,10 @@ impl Global { .map_err(|err| QueueSubmitError::DestroyedTexture(err.0))?; //Note: stateless trackers are not merged: // device already knows these resources exist. - CommandBuffer::insert_barriers( + CommandBuffer::insert_barriers_from_tracker( &mut baked.encoder, &mut *trackers, - &baked.trackers.buffers, - &baked.trackers.textures, + &baked.trackers, &*buffer_guard, &*texture_guard, ); @@ -794,19 +794,19 @@ impl Global { .begin_encoding(Some("(wgpu internal) Present")) .map_err(DeviceError::from)? }; - let texture_barriers = trackers + trackers .textures - .merge_replace(&used_surface_textures) - .map(|pending| { - let tex = &texture_guard[pending.id]; - pending.into_hal(tex) - }); + .change_states_scope(&*texture_guard, &used_surface_textures); + let texture_barriers = trackers.textures.drain().map(|pending| { + let tex = unsafe { texture_guard.get_unchecked(pending.id) }; + pending.into_hal(tex) + }); let present = unsafe { baked.encoder.transition_textures(texture_barriers); baked.encoder.end_encoding().unwrap() }; baked.list.push(present); - used_surface_textures.clear(); + used_surface_textures = track::TextureUsageScope::new(); } // done @@ -816,7 +816,7 @@ impl Global { }); } - log::trace!("Device after submission {}: {:#?}", submit_index, trackers); + log::trace!("Device after submission {}", submit_index); } let super::Device { @@ -846,39 +846,39 @@ impl Global { TextureInner::Surface { ref mut has_work, .. } => { - use track::ResourceState as _; - *has_work = true; let ref_count = texture.life_guard.add_ref(); - //TODO: better error handling here? - // register it in the temporary tracker. - let mut ts = track::OldTextureState::default(); - let _ = ts.change( - id::Valid(id), - texture.full_range.clone(), - hal::TextureUses::empty(), //present - None, - ); - let _ = used_surface_textures.init(id::Valid(id), ref_count, ts); + unsafe { + used_surface_textures + .extend_refcount( + &*texture_guard, + id::Valid(id), + None, + &ref_count, + hal::TextureUses::empty(), + ) + .unwrap() + }; } } } if !used_surface_textures.is_empty() { let mut trackers = device.trackers.lock(); - let texture_barriers = trackers + + trackers .textures - .merge_replace(&used_surface_textures) - .map(|pending| { - let tex = &texture_guard[pending.id]; - pending.into_hal(tex) - }); + .change_states_scope(&*texture_guard, &used_surface_textures); + let texture_barriers = trackers.textures.drain().map(|pending| { + let tex = unsafe { texture_guard.get_unchecked(pending.id) }; + pending.into_hal(tex) + }); + unsafe { pending_writes .command_encoder .transition_textures(texture_barriers); }; - used_surface_textures.clear(); } } diff --git a/wgpu-core/src/hub.rs b/wgpu-core/src/hub.rs index 93dcb9e72f..1f6ab1f3cc 100644 --- a/wgpu-core/src/hub.rs +++ b/wgpu-core/src/hub.rs @@ -193,7 +193,7 @@ impl Storage { pub(crate) unsafe fn get_unchecked(&self, id: u32) -> &T { match self.map[id as usize] { - Element::Occupied(ref v, e) => v, + Element::Occupied(ref v, _) => v, Element::Vacant => panic!("{}[{}] does not exist", self.kind, id), Element::Error(_, _) => panic!(""), } @@ -1003,16 +1003,16 @@ pub trait HalApi: hal::Api { impl HalApi for hal::api::Empty { const VARIANT: Backend = Backend::Empty; - fn create_instance_from_hal(name: &str, hal_instance: Self::Instance) -> Instance { + fn create_instance_from_hal(_: &str, _: Self::Instance) -> Instance { unimplemented!("called empty api") } - fn hub(global: &Global) -> &Hub { + fn hub(_: &Global) -> &Hub { unimplemented!("called empty api") } - fn get_surface(surface: &Surface) -> &HalSurface { + fn get_surface(_: &Surface) -> &HalSurface { unimplemented!("called empty api") } - fn get_surface_mut(surface: &mut Surface) -> &mut HalSurface { + fn get_surface_mut(_: &mut Surface) -> &mut HalSurface { unimplemented!("called empty api") } } diff --git a/wgpu-core/src/id.rs b/wgpu-core/src/id.rs index b36ef9637d..5c91b412b1 100644 --- a/wgpu-core/src/id.rs +++ b/wgpu-core/src/id.rs @@ -135,7 +135,7 @@ pub(crate) struct Valid(pub I); /// Most `wgpu-core` clients should not use this trait. Unusual clients that /// need to construct `Id` values directly, or access their components, like the /// WGPU recording player, may use this trait to do so. -pub trait TypedId { +pub trait TypedId: Copy { fn zip(index: Index, epoch: Epoch, backend: Backend) -> Self; fn unzip(self) -> (Index, Epoch, Backend); } diff --git a/wgpu-core/src/present.rs b/wgpu-core/src/present.rs index b47bd7bbc1..b163081ede 100644 --- a/wgpu-core/src/present.rs +++ b/wgpu-core/src/present.rs @@ -190,9 +190,11 @@ impl Global { // register it in the device tracker as uninitialized let mut trackers = device.trackers.lock(); let _ = unsafe { - trackers - .textures - .init(id.0, ref_count, hal::TextureUses::UNINITIALIZED) + trackers.textures.init( + id.0, + ref_count.clone(), + hal::TextureUses::UNINITIALIZED, + ) }; } diff --git a/wgpu-core/src/track/buffer.rs b/wgpu-core/src/track/buffer.rs index 6035068218..023539bf12 100644 --- a/wgpu-core/src/track/buffer.rs +++ b/wgpu-core/src/track/buffer.rs @@ -1,4 +1,4 @@ -use std::{vec::Drain, marker::PhantomData}; +use std::{marker::PhantomData, vec::Drain}; use super::PendingTransition; use crate::{ @@ -21,7 +21,7 @@ impl ResourceUses for BufferUses { type Selector = (); fn bits(self) -> u16 { - self.bits() + Self::bits(&self) } fn all_ordered(self) -> bool { @@ -33,10 +33,10 @@ impl ResourceUses for BufferUses { } } -pub struct BufferBindGroupState { +pub(crate) struct BufferBindGroupState { buffers: Vec<(Valid, RefCount, BufferUses)>, - _phantom: PhantomData + _phantom: PhantomData, } impl BufferBindGroupState { pub fn new() -> Self { @@ -47,6 +47,10 @@ impl BufferBindGroupState { } } + pub fn used(&self) -> impl Iterator> + '_ { + self.buffers.iter().map(|&(id, _, _)| id) + } + pub fn extend<'a>( &mut self, storage: &'a hub::Storage, BufferId>, @@ -109,26 +113,29 @@ impl BufferUsageScope { resize_bitvec(&mut self.owned, size); } + pub fn used(&self) -> impl Iterator> + '_ { + self.debug_assert_in_bounds(self.owned.len() - 1); + iterate_bitvec_indices(&self.owned).map(move |index| { + let epoch = unsafe { *self.epochs.get_unchecked(index) }; + Valid(BufferId::zip(index as u32, epoch, A::VARIANT)) + }) + } + pub unsafe fn extend_from_bind_group( &mut self, - storage: &hub::Storage, BufferId>, bind_group: &BufferBindGroupState, ) -> Result<(), UsageConflict> { for (id, ref_count, state) in &bind_group.buffers { let (index32, epoch, _) = id.0.unzip(); let index = index32 as usize; - self.extend_inner(storage, *id, index, epoch, ref_count, *state)?; + self.extend_inner(*id, index, epoch, ref_count, *state)?; } Ok(()) } - pub fn extend_from_scope( - &mut self, - storage: &hub::Storage, BufferId>, - scope: &Self, - ) -> Result<(), UsageConflict> { + pub fn extend_from_scope(&mut self, scope: &Self) -> Result<(), UsageConflict> { let incoming_size = scope.state.len(); if incoming_size > self.state.len() { self.set_max_index(incoming_size); @@ -148,13 +155,12 @@ impl BufferUsageScope { let new_state = *scope.state.get_unchecked(index); self.extend_inner( - storage, Valid(BufferId::zip(index as u32, epoch, A::VARIANT)), index, epoch, ref_count, new_state, - ) + )?; }; } @@ -175,7 +181,6 @@ impl BufferUsageScope { let index = index32 as usize; self.extend_inner( - storage, Valid(id), index, epoch, @@ -188,7 +193,6 @@ impl BufferUsageScope { unsafe fn extend_inner<'a>( &mut self, - storage: &'a hub::Storage, BufferId>, id: Valid, index: usize, epoch: u32, @@ -312,7 +316,6 @@ impl BufferTracker { let index = index32 as usize; self.transition_inner( - storage, index, epoch, value.life_guard.ref_count.as_ref().unwrap(), @@ -322,11 +325,29 @@ impl BufferTracker { Some((value, self.temp.pop())) } - pub fn change_states_scope( - &mut self, - storage: &hub::Storage, BufferId>, - scope: &BufferUsageScope, - ) { + pub fn change_states_tracker(&mut self, tracker: &Self) { + let incoming_size = tracker.start.len(); + if incoming_size > self.start.len() { + self.set_max_index(incoming_size); + } + + for index in iterate_bitvec_indices(&tracker.owned) { + tracker.debug_assert_in_bounds(index); + unsafe { + let ref_count = tracker + .ref_counts + .get_unchecked(index) + .as_ref() + .unwrap_unchecked(); + + let epoch = *tracker.epochs.get_unchecked(index); + + self.transition(&tracker.start, ref_count, index, epoch); + } + } + } + + pub fn change_states_scope(&mut self, scope: &BufferUsageScope) { let incoming_size = scope.state.len(); if incoming_size > self.start.len() { self.set_max_index(incoming_size); @@ -343,14 +364,13 @@ impl BufferTracker { let epoch = *scope.epochs.get_unchecked(index); - self.transition(storage, &scope.state, ref_count, index, epoch); + self.transition(&scope.state, ref_count, index, epoch); } } } pub unsafe fn change_states_bind_group( &mut self, - storage: &hub::Storage, BufferId>, scope: &mut BufferUsageScope, bind_group_state: &BufferBindGroupState, ) { @@ -368,7 +388,7 @@ impl BufferTracker { if !scope.owned.get(index).unwrap_unchecked() { continue; } - self.transition(storage, &scope.state, ref_count, index, epoch); + self.transition(&scope.state, ref_count, index, epoch); scope.owned.set(index, false); } @@ -376,7 +396,6 @@ impl BufferTracker { unsafe fn transition( &mut self, - storage: &hub::Storage, BufferId>, incoming_set: &Vec, ref_count: &RefCount, index: usize, @@ -384,12 +403,11 @@ impl BufferTracker { ) { let new_state = *incoming_set.get_unchecked(index); - self.transition_inner(storage, index, epoch, ref_count, new_state); + self.transition_inner(index, epoch, ref_count, new_state); } unsafe fn transition_inner( &mut self, - storage: &hub::Storage, BufferId>, index: usize, epoch: u32, ref_count: &RefCount, @@ -417,9 +435,39 @@ impl BufferTracker { *self.end.get_unchecked_mut(index) = new_state; *self.ref_counts.get_unchecked_mut(index) = Some(ref_count.clone()); - *self.epochs.get_unchecked(index) = epoch; + *self.epochs.get_unchecked_mut(index) = epoch; self.owned.set(index, true); } } + + pub fn remove_abandoned(&mut self, id: Valid) -> bool { + let (index32, epoch, _) = id.0.unzip(); + let index = index32 as usize; + + if index > self.owned.len() { + return false; + } + + self.debug_assert_in_bounds(index); + + unsafe { + if self.owned.get(index).unwrap_unchecked() { + let existing_epoch = self.epochs.get_unchecked_mut(index); + let existing_ref_count = self.ref_counts.get_unchecked_mut(index); + + if *existing_epoch == epoch + && existing_ref_count.as_mut().unwrap_unchecked().load() == 1 + { + self.owned.set(index, false); + *existing_epoch = u32::MAX; + *existing_ref_count = None; + + return true; + } + } + } + + false + } } diff --git a/wgpu-core/src/track/mod.rs b/wgpu-core/src/track/mod.rs index 2401d04b46..115bb75541 100644 --- a/wgpu-core/src/track/mod.rs +++ b/wgpu-core/src/track/mod.rs @@ -68,8 +68,8 @@ impl PendingTransition { } } -trait ResourceUses: - fmt::Debug + ops::BitAnd + ops::BitOr + PartialEq + Sized +pub trait ResourceUses: + fmt::Debug + ops::BitAnd + ops::BitOr + PartialEq + Sized + Copy { const EXCLUSIVE: Self; @@ -117,7 +117,7 @@ fn iterate_bitvec_indices(ownership: &BitVec) -> impl Iterator>= 1; @@ -178,8 +178,8 @@ impl UsageConflict { layers = selector.layers; } None => { - mips = texture.full_range.mips; - layers = texture.full_range.layers; + mips = texture.full_range.mips.clone(); + layers = texture.full_range.layers.clone(); } } @@ -196,7 +196,7 @@ impl UsageConflict { } #[derive(Clone, Debug)] -struct InvalidUse { +pub struct InvalidUse { current_state: T, new_state: T, } @@ -263,12 +263,10 @@ impl RenderBundleScope { pub unsafe fn extend_from_bind_group( &mut self, - buffers: &hub::Storage, id::BufferId>, textures: &hub::Storage, id::TextureId>, bind_group: &BindGroupStates, ) -> Result<(), UsageConflict> { - self.buffers - .extend_from_bind_group(buffers, &bind_group.buffers)?; + self.buffers.extend_from_bind_group(&bind_group.buffers)?; self.textures .extend_from_bind_group(textures, &bind_group.textures)?; @@ -292,12 +290,10 @@ impl UsageScope { pub unsafe fn extend_from_bind_group( &mut self, - buffers: &hub::Storage, id::BufferId>, textures: &hub::Storage, id::TextureId>, bind_group: &BindGroupStates, ) -> Result<(), UsageConflict> { - self.buffers - .extend_from_bind_group(buffers, &bind_group.buffers)?; + self.buffers.extend_from_bind_group(&bind_group.buffers)?; self.textures .extend_from_bind_group(textures, &bind_group.textures)?; @@ -306,12 +302,10 @@ impl UsageScope { pub unsafe fn extend_from_render_bundle( &mut self, - buffers: &hub::Storage, id::BufferId>, textures: &hub::Storage, id::TextureId>, render_bundle: &RenderBundleScope, ) -> Result<(), UsageConflict> { - self.buffers - .extend_from_scope(buffers, &render_bundle.buffers)?; + self.buffers.extend_from_scope(&render_bundle.buffers)?; self.textures .extend_from_scope(textures, &render_bundle.textures)?; @@ -348,35 +342,28 @@ impl Tracker { pub unsafe fn extend_from_bind_group( &mut self, - buffers: &hub::Storage, id::BufferId>, textures: &hub::Storage, id::TextureId>, scope: &mut UsageScope, bind_group: &BindGroupStates, ) { self.buffers - .change_states_bind_group(buffers, &mut scope.buffers, &bind_group.buffers); + .change_states_bind_group(&mut scope.buffers, &bind_group.buffers); self.textures .change_states_bind_group(textures, &mut scope.textures, &bind_group.textures); } pub unsafe fn extend_from_render_bundle( &mut self, - views: &hub::Storage, id::TextureViewId>, - samplers: &hub::Storage, id::SamplerId>, - bind_groups: &hub::Storage, id::BindGroupId>, - render_pipelines: &hub::Storage, id::RenderPipelineId>, - query_sets: &hub::Storage, id::QuerySetId>, render_bundle: &RenderBundleScope, ) -> Result<(), UsageConflict> { - self.views.extend_from_tracker(views, &render_bundle.views); - self.samplers - .extend_from_tracker(samplers, &render_bundle.samplers); + self.views.extend_from_tracker(&render_bundle.views); + self.samplers.extend_from_tracker(&render_bundle.samplers); self.bind_groups - .extend_from_tracker(bind_groups, &render_bundle.bind_groups); + .extend_from_tracker(&render_bundle.bind_groups); self.render_pipelines - .extend_from_tracker(render_pipelines, &render_bundle.render_pipelines); + .extend_from_tracker(&render_bundle.render_pipelines); self.query_sets - .extend_from_tracker(query_sets, &render_bundle.query_sets); + .extend_from_tracker(&render_bundle.query_sets); Ok(()) } diff --git a/wgpu-core/src/track/range.rs b/wgpu-core/src/track/range.rs index fb58313b99..cd1dd86e43 100644 --- a/wgpu-core/src/track/range.rs +++ b/wgpu-core/src/track/range.rs @@ -8,7 +8,7 @@ use std::{cmp::Ordering, fmt::Debug, iter, ops::Range, slice::Iter}; /// optimized for a case where keys of the same values /// are often grouped together linearly. #[derive(Clone, Debug, PartialEq)] -pub struct RangedStates { +pub(crate) struct RangedStates { /// List of ranges, each associated with a singe value. /// Ranges of keys have to be non-intersecting and ordered. ranges: SmallVec<[(Range, T); 1]>, @@ -200,7 +200,7 @@ impl RangedStates { /// A custom iterator that goes through two `RangedStates` and process a merge. #[derive(Debug)] -pub struct Merge<'a, I, T> { +pub(crate) struct Merge<'a, I, T> { base: I, sa: iter::Peekable, T)>>, sb: iter::Peekable, T)>>, diff --git a/wgpu-core/src/track/stateless.rs b/wgpu-core/src/track/stateless.rs index d18da4ae78..062793a001 100644 --- a/wgpu-core/src/track/stateless.rs +++ b/wgpu-core/src/track/stateless.rs @@ -9,7 +9,7 @@ use crate::{ Epoch, RefCount, }; -pub struct StatelessBindGroupSate { +pub(crate) struct StatelessBindGroupSate { resources: Vec<(Valid, RefCount)>, _phantom: PhantomData, @@ -24,6 +24,10 @@ impl StatelessBindGroupSate { } } + pub fn used(&self) -> impl Iterator> + '_ { + self.resources.iter().map(|&(id, _)| id) + } + pub fn extend<'a>(&mut self, storage: &'a hub::Storage, id: Id) -> Option<&'a T> { let resource = storage.get(id).ok()?; @@ -34,7 +38,7 @@ impl StatelessBindGroupSate { } } -pub struct StatelessTracker { +pub(crate) struct StatelessTracker { ref_counts: Vec>, epochs: Vec, @@ -108,11 +112,7 @@ impl StatelessTracker { Some(item) } - pub unsafe fn extend_from_bind_group( - &mut self, - storage: &hub::Storage, - bind_group: &StatelessBindGroupSate, - ) { + pub unsafe fn extend_from_bind_group(&mut self, bind_group: &StatelessBindGroupSate) { for (id, ref_count) in &bind_group.resources { let (index32, epoch, _) = id.0.unzip(); let index = index32 as usize; @@ -127,7 +127,7 @@ impl StatelessTracker { } } - pub fn extend_from_tracker(&mut self, storage: &hub::Storage, other: &Self) { + pub fn extend_from_tracker(&mut self, other: &Self) { let incoming_size = other.owned.len(); if incoming_size > self.owned.len() { self.set_max_index(incoming_size); @@ -144,8 +144,8 @@ impl StatelessTracker { let other_ref_count = other .ref_counts .get_unchecked(index) - .unwrap_unchecked() - .clone(); + .clone() + .unwrap_unchecked(); *self.ref_counts.get_unchecked_mut(index) = Some(other_ref_count); let epoch = *other.epochs.get_unchecked(index); @@ -154,4 +154,34 @@ impl StatelessTracker { } } } + + pub fn remove_abandoned(&mut self, id: Valid) -> bool { + let (index32, epoch, _) = id.0.unzip(); + let index = index32 as usize; + + if index > self.owned.len() { + return false; + } + + self.debug_assert_in_bounds(index); + + unsafe { + if self.owned.get(index).unwrap_unchecked() { + let existing_epoch = self.epochs.get_unchecked_mut(index); + let existing_ref_count = self.ref_counts.get_unchecked_mut(index); + + if *existing_epoch == epoch + && existing_ref_count.as_mut().unwrap_unchecked().load() == 1 + { + self.owned.set(index, false); + *existing_epoch = u32::MAX; + *existing_ref_count = None; + + return true; + } + } + } + + false + } } diff --git a/wgpu-core/src/track/texture.rs b/wgpu-core/src/track/texture.rs index adff0e2e30..0df9df297e 100644 --- a/wgpu-core/src/track/texture.rs +++ b/wgpu-core/src/track/texture.rs @@ -30,7 +30,7 @@ impl ResourceUses for TextureUses { type Selector = TextureSelector; fn bits(self) -> u16 { - self.bits() + Self::bits(&self) } fn all_ordered(self) -> bool { @@ -60,7 +60,7 @@ impl ComplexTextureState { } // TODO: This representation could be optimized in a couple ways, but keep it simple for now. -pub struct TextureBindGroupState { +pub(crate) struct TextureBindGroupState { textures: Vec<( Valid, Option, @@ -79,6 +79,10 @@ impl TextureBindGroupState { } } + pub fn used(&self) -> impl Iterator> + '_ { + self.textures.iter().map(|&(id, _, _, _)| id) + } + pub fn extend_with_refcount<'a>( &mut self, storage: &'a hub::Storage, TextureId>, @@ -96,7 +100,7 @@ impl TextureBindGroupState { } #[derive(Debug)] -pub struct TextureStateSet { +pub(crate) struct TextureStateSet { simple: Vec, complex: FastHashMap, } @@ -110,7 +114,7 @@ impl TextureStateSet { } #[derive(Debug)] -pub struct TextureUsageScope { +pub(crate) struct TextureUsageScope { set: TextureStateSet, ref_counts: Vec>, @@ -164,6 +168,18 @@ impl TextureUsageScope { resize_bitvec(&mut self.owned, size); } + pub fn used(&self) -> impl Iterator> + '_ { + self.debug_assert_in_bounds(self.owned.len() - 1); + iterate_bitvec_indices(&self.owned).map(move |index| { + let epoch = unsafe { *self.epochs.get_unchecked(index) }; + Valid(TextureId::zip(index as u32, epoch, A::VARIANT)) + }) + } + + pub(crate) fn is_empty(&self) -> bool { + !self.owned.any() + } + pub fn extend_from_scope( &mut self, storage: &hub::Storage, TextureId>, @@ -175,6 +191,7 @@ impl TextureUsageScope { } for index in iterate_bitvec_indices(&scope.owned) { + let _ = (storage, index); todo!() } @@ -187,7 +204,7 @@ impl TextureUsageScope { bind_group: &TextureBindGroupState, ) -> Result<(), UsageConflict> { for (id, selector, ref_count, state) in &bind_group.textures { - self.extend_inner(storage, *id, selector.clone(), ref_count, *state)?; + self.extend_refcount(storage, *id, selector.clone(), ref_count, *state)?; } Ok(()) @@ -207,7 +224,7 @@ impl TextureUsageScope { .get(id) .map_err(|_| UsageConflict::TextureInvalid { id })?; - self.extend_inner( + self.extend_refcount( storage, Valid(id), selector, @@ -221,7 +238,7 @@ impl TextureUsageScope { /// # Safety /// /// `id` must be a valid ID and have an ID value less than the last call to set_max_index. - unsafe fn extend_inner( + pub unsafe fn extend_refcount( &mut self, storage: &hub::Storage, TextureId>, id: Valid, @@ -246,7 +263,7 @@ impl TextureUsageScope { return Err(UsageConflict::from_texture( storage, id, - selector, + None, current_state, new_state, )); @@ -311,13 +328,13 @@ impl TextureUsageScope { let mips; let layers; match selector { - Some(selector) => { + Some(ref selector) => { mips = selector.mips.clone(); layers = selector.layers.clone(); } None => { - mips = texture.full_range.mips; - layers = texture.full_range.layers; + mips = texture.full_range.mips.clone(); + layers = texture.full_range.layers.clone(); } } @@ -426,10 +443,19 @@ impl TextureTracker { self.temp.drain(..) } + pub fn get_ref_count(&self, id: Valid) -> &RefCount { + let (index32, _, _) = id.0.unzip(); + let index = index32 as usize; + + self.ref_counts[index].as_ref().unwrap() + } + pub unsafe fn init(&mut self, id: TextureId, ref_count: RefCount, usage: TextureUses) { let (index32, epoch, _) = id.unzip(); let index = index32 as usize; + self.debug_assert_in_bounds(index); + *self.start_set.simple.get_unchecked_mut(index) = usage; *self.end_set.simple.get_unchecked_mut(index) = usage; @@ -438,40 +464,59 @@ impl TextureTracker { self.owned.set(index, true); } - pub fn change_state<'a>( + pub unsafe fn change_state<'a>( &mut self, storage: &'a hub::Storage, TextureId>, id: TextureId, selector: TextureSelector, new_usage: TextureUses, ) -> Option<(&'a Texture, Option>)> { + let _ = (storage, id, selector, new_usage); todo!() } - pub fn change_states_scope( + pub fn change_states_tracker( &mut self, storage: &hub::Storage, TextureId>, - scope: &TextureUsageScope, + tracker: &Self, ) { - let incoming_size = scope.set.simple.len(); + let incoming_size = tracker.start_set.simple.len(); if incoming_size > self.start_set.simple.len() { self.set_max_index(incoming_size); } - scope.debug_assert_in_bounds(scope.owned.len() - 1); - for index in iterate_bitvec_indices(&scope.owned) { + for index in iterate_bitvec_indices(&tracker.owned) { + tracker.debug_assert_in_bounds(index); unsafe { - let epoch = *scope.epochs.get_unchecked(index); self.transition( storage, - scope, - Valid(TextureId::zip(index as u32, epoch, A::VARIANT)), + &tracker.start_set, + &tracker.ref_counts, + &tracker.epochs, index, ) }; } } + pub fn change_states_scope( + &mut self, + storage: &hub::Storage, TextureId>, + scope: &TextureUsageScope, + ) { + let incoming_size = scope.set.simple.len(); + if incoming_size > self.start_set.simple.len() { + self.set_max_index(incoming_size); + } + + for index in iterate_bitvec_indices(&scope.owned) { + scope.debug_assert_in_bounds(index); + unsafe { + self.transition(storage, &scope.set, &scope.ref_counts, &scope.epochs, index) + }; + } + } + pub unsafe fn change_states_bind_group( &mut self, storage: &hub::Storage, TextureId>, @@ -484,14 +529,17 @@ impl TextureTracker { } for &(id, _, _, _) in bind_group_state.textures.iter() { - let (index32, epoch, _) = id.0.unzip(); + let (index32, _, _) = id.0.unzip(); let index = index32 as usize; scope.debug_assert_in_bounds(index); if !scope.owned.get(index).unwrap_unchecked() { continue; } - self.transition(storage, &scope, id, index); + self.transition(storage, &scope.set, &scope.ref_counts, &scope.epochs, index); + + *scope.ref_counts.get_unchecked_mut(index) = None; + *scope.epochs.get_unchecked_mut(index) = u32::MAX; scope.owned.set(index, false); } } @@ -499,8 +547,9 @@ impl TextureTracker { unsafe fn transition( &mut self, storage: &hub::Storage, TextureId>, - scope: &TextureUsageScope, - id: Valid, + incoming_set: &TextureStateSet, + incoming_ref_counts: &[Option], + incoming_epochs: &[Epoch], index: usize, ) { // Note: both callees of this function call scope.debug_assert_in_bounds. @@ -508,7 +557,7 @@ impl TextureTracker { let old_tracked = self.owned.get(index).unwrap_unchecked(); let old_state = *self.end_set.simple.get_unchecked(index); - let new_state = *scope.set.simple.get_unchecked(index); + let new_state = *incoming_set.simple.get_unchecked(index); match ( old_tracked, @@ -521,21 +570,20 @@ impl TextureTracker { self.owned.set(index, true); - let ref_count = scope - .ref_counts + let ref_count = incoming_ref_counts .get_unchecked(index) - .unwrap_unchecked() - .clone(); + .clone() + .unwrap_unchecked(); *self.ref_counts.get_unchecked_mut(index) = Some(ref_count); - let epoch = *scope.epochs.get_unchecked(index); + let epoch = *incoming_epochs.get_unchecked(index); *self.epochs.get_unchecked_mut(index) = epoch; } (false, _, true) => { *self.start_set.simple.get_unchecked_mut(index) = TextureUses::COMPLEX; *self.end_set.simple.get_unchecked_mut(index) = TextureUses::COMPLEX; - let complex_state = scope.set.complex.get(&(index as u32)).unwrap_unchecked(); + let complex_state = incoming_set.complex.get(&(index as u32)).unwrap_unchecked(); self.start_set .complex .insert(index as u32, complex_state.clone()); @@ -545,14 +593,13 @@ impl TextureTracker { self.owned.set(index, true); - let ref_count = scope - .ref_counts + let ref_count = incoming_ref_counts .get_unchecked(index) - .unwrap_unchecked() - .clone(); + .clone() + .unwrap_unchecked(); *self.ref_counts.get_unchecked_mut(index) = Some(ref_count); - let epoch = *scope.epochs.get_unchecked(index); + let epoch = *incoming_epochs.get_unchecked(index); *self.epochs.get_unchecked_mut(index) = epoch; } (true, false, false) => { @@ -562,32 +609,35 @@ impl TextureTracker { self.temp.push(PendingTransition { id: index as u32, - selector: storage.get_unchecked(index as u32).full_range, + selector: storage.get_unchecked(index as u32).full_range.clone(), usage: old_state..new_state, }); *self.end_set.simple.get_unchecked_mut(index) = new_state; } (true, true, true) => { - self.transition_complex_to_complex(storage, scope, index); + self.transition_complex_to_complex(incoming_set, index); } (true, true, false) => { - self.transition_complex_to_simple(storage, index, new_state); + self.transition_complex_to_simple(index, new_state); } (true, false, true) => { - self.transition_simple_to_complex(storage, scope, index, old_state); + self.transition_simple_to_complex(incoming_set, index, old_state); } } } unsafe fn transition_complex_to_complex( &mut self, - storage: &hub::Storage, TextureId>, - scope: &TextureUsageScope, + incoming_set: &TextureStateSet, index: usize, ) { - let old_complex = self.end_set.complex.get(&(index as u32)).unwrap_unchecked(); - let new_complex = scope.set.complex.get(&(index as u32)).unwrap_unchecked(); + let old_complex = self + .end_set + .complex + .get_mut(&(index as u32)) + .unwrap_unchecked(); + let new_complex = incoming_set.complex.get(&(index as u32)).unwrap_unchecked(); let mut temp = Vec::new(); debug_assert!(old_complex.mips.len() >= new_complex.mips.len()); @@ -634,12 +684,7 @@ impl TextureTracker { } } - unsafe fn transition_complex_to_simple( - &mut self, - storage: &hub::Storage, TextureId>, - index: usize, - new_state: TextureUses, - ) { + unsafe fn transition_complex_to_simple(&mut self, index: usize, new_state: TextureUses) { let old_complex = self .end_set .complex @@ -670,16 +715,15 @@ impl TextureTracker { unsafe fn transition_simple_to_complex( &mut self, - storage: &hub::Storage, TextureId>, - scope: &TextureUsageScope, + incoming_set: &TextureStateSet, index: usize, old_state: TextureUses, ) { - let new_complex = scope.set.complex.get(&(index as u32)).unwrap_unchecked(); + let new_complex = incoming_set.complex.get(&(index as u32)).unwrap_unchecked(); for (mip_index, mips) in new_complex.mips.iter().enumerate() { let mip_index = mip_index as u32; - for (layer, new_state) in mips.into_iter() { + for &(ref layer, new_state) in mips.iter() { if skip_barrier(old_state, new_state) { continue; } @@ -689,7 +733,7 @@ impl TextureTracker { id: index as u32, selector: TextureSelector { mips: mip_index..mip_index + 1, - layers: layer, + layers: layer.clone(), }, usage: old_state..new_state, }) @@ -701,4 +745,44 @@ impl TextureTracker { .insert(index as u32, new_complex.clone()); *self.end_set.simple.get_unchecked_mut(index) = TextureUses::COMPLEX; } + + pub fn remove(&mut self, id: Valid) -> bool { + self.remove_inner(id, true) + } + + pub fn remove_abandoned(&mut self, id: Valid) -> bool { + self.remove_inner(id, false) + } + + fn remove_inner(&mut self, id: Valid, force: bool) -> bool { + let (index32, epoch, _) = id.0.unzip(); + let index = index32 as usize; + + if index > self.owned.len() { + return false; + } + + self.debug_assert_in_bounds(index); + + unsafe { + if self.owned.get(index).unwrap_unchecked() { + let existing_epoch = self.epochs.get_unchecked_mut(index); + let existing_ref_count = self.ref_counts.get_unchecked_mut(index); + + if *existing_epoch == epoch + && existing_ref_count.as_mut().unwrap_unchecked().load() == 1 + { + self.owned.set(index, false); + *existing_epoch = u32::MAX; + *existing_ref_count = None; + + return true; + } else if force { + assert_eq!(*existing_epoch, epoch); + } + } + } + + false + } } From dfb10f71a326c10c8cbfbf070636ca8ea0950ceb Mon Sep 17 00:00:00 2001 From: Connor Fitzgerald Date: Sat, 7 May 2022 04:31:20 -0400 Subject: [PATCH 12/35] temp11 - Add bind group state optimization --- wgpu-core/src/device/mod.rs | 2 ++ wgpu-core/src/track/buffer.rs | 5 +++++ wgpu-core/src/track/mod.rs | 7 +++++++ wgpu-core/src/track/stateless.rs | 5 +++++ wgpu-core/src/track/texture.rs | 5 +++++ 5 files changed, 24 insertions(+) diff --git a/wgpu-core/src/device/mod.rs b/wgpu-core/src/device/mod.rs index 6438b42b2c..b03d9cfd3e 100644 --- a/wgpu-core/src/device/mod.rs +++ b/wgpu-core/src/device/mod.rs @@ -1859,6 +1859,8 @@ impl Device { }); } + used.optimize(); + hal_entries.sort_by_key(|entry| entry.binding); for (a, b) in hal_entries.iter().zip(hal_entries.iter().skip(1)) { if a.binding == b.binding { diff --git a/wgpu-core/src/track/buffer.rs b/wgpu-core/src/track/buffer.rs index 023539bf12..f6cfd3bc4d 100644 --- a/wgpu-core/src/track/buffer.rs +++ b/wgpu-core/src/track/buffer.rs @@ -47,6 +47,11 @@ impl BufferBindGroupState { } } + pub(crate) fn optimize(&mut self) { + self.buffers + .sort_unstable_by_key(|&(id, _, _)| id.0.unzip().0); + } + pub fn used(&self) -> impl Iterator> + '_ { self.buffers.iter().map(|&(id, _, _)| id) } diff --git a/wgpu-core/src/track/mod.rs b/wgpu-core/src/track/mod.rs index 115bb75541..65501d7406 100644 --- a/wgpu-core/src/track/mod.rs +++ b/wgpu-core/src/track/mod.rs @@ -236,6 +236,13 @@ impl BindGroupStates { samplers: StatelessBindGroupSate::new(), } } + + pub fn optimize(&mut self) { + self.buffers.optimize(); + self.textures.optimize(); + self.views.optimize(); + self.samplers.optimize(); + } } pub(crate) struct RenderBundleScope { diff --git a/wgpu-core/src/track/stateless.rs b/wgpu-core/src/track/stateless.rs index 062793a001..bba2316e9e 100644 --- a/wgpu-core/src/track/stateless.rs +++ b/wgpu-core/src/track/stateless.rs @@ -24,6 +24,11 @@ impl StatelessBindGroupSate { } } + pub(crate) fn optimize(&mut self) { + self.resources + .sort_unstable_by_key(|&(id, _)| id.0.unzip().0); + } + pub fn used(&self) -> impl Iterator> + '_ { self.resources.iter().map(|&(id, _)| id) } diff --git a/wgpu-core/src/track/texture.rs b/wgpu-core/src/track/texture.rs index 0df9df297e..1a5007869d 100644 --- a/wgpu-core/src/track/texture.rs +++ b/wgpu-core/src/track/texture.rs @@ -79,6 +79,11 @@ impl TextureBindGroupState { } } + pub(crate) fn optimize(&mut self) { + self.textures + .sort_unstable_by_key(|&(id, _, _, _)| id.0.unzip().0); + } + pub fn used(&self) -> impl Iterator> + '_ { self.textures.iter().map(|&(id, _, _, _)| id) } From 22d16b315538cd8e3c8f396699349c6e172fbc9d Mon Sep 17 00:00:00 2001 From: Connor Fitzgerald Date: Sun, 8 May 2022 02:38:23 -0400 Subject: [PATCH 13/35] temp12 - Safety and adding size setting --- wgpu-core/src/command/bundle.rs | 13 +++-- wgpu-core/src/command/compute.rs | 14 +++++- wgpu-core/src/command/render.rs | 34 +++++++++---- wgpu-core/src/device/life.rs | 2 - wgpu-core/src/device/queue.rs | 1 + wgpu-core/src/hub.rs | 4 ++ wgpu-core/src/track/buffer.rs | 54 ++++++++++++-------- wgpu-core/src/track/mod.rs | 84 +++++++++++++++++++++++++++----- wgpu-core/src/track/stateless.rs | 5 +- wgpu-core/src/track/texture.rs | 47 ++++++++++++------ 10 files changed, 194 insertions(+), 64 deletions(-) diff --git a/wgpu-core/src/command/bundle.rs b/wgpu-core/src/command/bundle.rs index 540eb97c00..50da9c3e29 100644 --- a/wgpu-core/src/command/bundle.rs +++ b/wgpu-core/src/command/bundle.rs @@ -34,7 +34,7 @@ invalidations or index format changes. #![allow(clippy::reversed_empty_ranges)] use crate::{ - binding_model::buffer_binding_type_alignment, + binding_model::{buffer_binding_type_alignment, self}, command::{ BasePass, BindGroupStateChange, DrawError, MapPassErr, PassErrorScope, RenderCommand, RenderCommandError, StateChange, @@ -178,11 +178,18 @@ impl RenderBundleEncoder { let (pipeline_layout_guard, mut token) = hub.pipeline_layouts.read(token); let (bind_group_guard, mut token) = hub.bind_groups.read(&mut token); let (pipeline_guard, mut token) = hub.render_pipelines.read(&mut token); + let (query_set_guard, mut token) = hub.query_sets.read(&mut token); let (buffer_guard, mut token) = hub.buffers.read(&mut token); let (texture_guard, _) = hub.textures.read(&mut token); let mut state = State { - trackers: RenderBundleScope::new(), + trackers: RenderBundleScope::new( + &*buffer_guard, + &*texture_guard, + &*bind_group_guard, + &*pipeline_guard, + &*query_set_guard, + ), index: IndexState::new(), vertex: (0..hal::MAX_VERTEX_BUFFERS) .map(|_| VertexState::new()) @@ -223,7 +230,7 @@ impl RenderBundleEncoder { let offsets = &base.dynamic_offsets[..num_dynamic_offsets as usize]; base.dynamic_offsets = &base.dynamic_offsets[num_dynamic_offsets as usize..]; - let bind_group = unsafe { + let bind_group: &binding_model::BindGroup = unsafe { state .trackers .bind_groups diff --git a/wgpu-core/src/command/compute.rs b/wgpu-core/src/command/compute.rs index 302a28edd0..81bcaf00f5 100644 --- a/wgpu-core/src/command/compute.rs +++ b/wgpu-core/src/command/compute.rs @@ -340,7 +340,7 @@ impl Global { let mut state = State { binder: Binder::new(), pipeline: None, - scope: UsageScope::new(), + scope: UsageScope::new(&*buffer_guard, &*texture_guard), debug_scope_depth: 0, }; let mut temp_offsets = Vec::new(); @@ -348,6 +348,18 @@ impl Global { let mut string_offset = 0; let mut active_query = None; + cmd_buf.trackers.set_size( + Some(&*buffer_guard), + Some(&*texture_guard), + None, + None, + Some(&*bind_group_guard), + Some(&*pipeline_guard), + None, + None, + Some(&*query_set_guard), + ); + let hal_desc = hal::ComputePassDescriptor { label: base.label }; unsafe { raw.begin_compute_pass(&hal_desc); diff --git a/wgpu-core/src/command/render.rs b/wgpu-core/src/command/render.rs index b8f79d5dc1..07e83f70cc 100644 --- a/wgpu-core/src/command/render.rs +++ b/wgpu-core/src/command/render.rs @@ -18,7 +18,7 @@ use crate::{ id, init_tracker::{MemoryInitKind, TextureInitRange, TextureInitTrackerAction}, pipeline::PipelineFlags, - resource::{self, Texture, TextureView}, + resource::{self, Buffer, Texture, TextureView}, track::{TextureSelector, UsageConflict, UsageScope}, validation::{ check_buffer_usage, check_texture_usage, MissingBufferUsageError, MissingTextureUsageError, @@ -618,6 +618,7 @@ impl<'a, A: HalApi> RenderPassInfo<'a, A> { depth_stencil_attachment: Option<&RenderPassDepthStencilAttachment>, cmd_buf: &mut CommandBuffer, view_guard: &'a Storage, id::TextureViewId>, + buffer_guard: &'a Storage, id::BufferId>, texture_guard: &'a Storage, id::TextureId>, ) -> Result { profiling::scope!("start", "RenderPassInfo"); @@ -940,7 +941,7 @@ impl<'a, A: HalApi> RenderPassInfo<'a, A> { Ok(Self { context, - usage_scope: UsageScope::new(), + usage_scope: UsageScope::new(buffer_guard, texture_guard), render_attachments, is_ds_read_only, extent, @@ -1080,7 +1081,7 @@ impl Global { let (bundle_guard, mut token) = hub.render_bundles.read(&mut token); let (pipeline_layout_guard, mut token) = hub.pipeline_layouts.read(&mut token); let (bind_group_guard, mut token) = hub.bind_groups.read(&mut token); - let (render_pipe_guard, mut token) = hub.render_pipelines.read(&mut token); + let (render_pipeline_guard, mut token) = hub.render_pipelines.read(&mut token); let (query_set_guard, mut token) = hub.query_sets.read(&mut token); let (buffer_guard, mut token) = hub.buffers.read(&mut token); let (texture_guard, mut token) = hub.textures.read(&mut token); @@ -1098,10 +1099,23 @@ impl Global { depth_stencil_attachment, cmd_buf, &*view_guard, + &*buffer_guard, &*texture_guard, ) .map_pass_err(init_scope)?; + cmd_buf.trackers.set_size( + Some(&*buffer_guard), + Some(&*texture_guard), + Some(&*view_guard), + None, + Some(&*bind_group_guard), + None, + Some(&*render_pipeline_guard), + Some(&*bundle_guard), + Some(&*query_set_guard), + ); + let raw = &mut cmd_buf.encoder.raw; let mut state = State { @@ -1214,7 +1228,7 @@ impl Global { cmd_buf .trackers .render_pipelines - .extend(&*render_pipe_guard, pipeline_id) + .extend(&*render_pipeline_guard, pipeline_id) .ok_or_else(|| RenderCommandError::InvalidPipeline(pipeline_id)) .map_pass_err(scope)? }; @@ -1323,7 +1337,7 @@ impl Global { size, } => { let scope = PassErrorScope::SetIndexBuffer(buffer_id); - let buffer: &resource::Buffer = unsafe { + let buffer: &Buffer = unsafe { info.usage_scope .buffers .extend(&*buffer_guard, buffer_id, hal::BufferUses::INDEX) @@ -1370,7 +1384,7 @@ impl Global { size, } => { let scope = PassErrorScope::SetVertexBuffer(buffer_id); - let buffer: &resource::Buffer = unsafe { + let buffer: &Buffer = unsafe { info.usage_scope .buffers .extend(&*buffer_guard, buffer_id, hal::BufferUses::VERTEX) @@ -1628,7 +1642,7 @@ impl Global { .require_downlevel_flags(wgt::DownlevelFlags::INDIRECT_EXECUTION) .map_pass_err(scope)?; - let indirect_buffer: &resource::Buffer = unsafe { + let indirect_buffer: &Buffer = unsafe { info.usage_scope .buffers .extend(&*buffer_guard, buffer_id, hal::BufferUses::INDIRECT) @@ -1699,7 +1713,7 @@ impl Global { .require_downlevel_flags(wgt::DownlevelFlags::INDIRECT_EXECUTION) .map_pass_err(scope)?; - let indirect_buffer: &resource::Buffer = unsafe { + let indirect_buffer: &Buffer = unsafe { info.usage_scope .buffers .extend(&*buffer_guard, buffer_id, hal::BufferUses::INDIRECT) @@ -1713,7 +1727,7 @@ impl Global { .ok_or(RenderCommandError::DestroyedBuffer(buffer_id)) .map_pass_err(scope)?; - let count_buffer: &resource::Buffer = unsafe { + let count_buffer: &Buffer = unsafe { info.usage_scope .buffers .extend(&*buffer_guard, count_buffer_id, hal::BufferUses::INDIRECT) @@ -1915,7 +1929,7 @@ impl Global { raw, &*pipeline_layout_guard, &*bind_group_guard, - &*render_pipe_guard, + &*render_pipeline_guard, &*buffer_guard, ) } diff --git a/wgpu-core/src/device/life.rs b/wgpu-core/src/device/life.rs index 2dd4648a41..8c6705276f 100644 --- a/wgpu-core/src/device/life.rs +++ b/wgpu-core/src/device/life.rs @@ -71,8 +71,6 @@ impl SuspectedResources { pub(super) fn add_render_bundle_scope(&mut self, trackers: &RenderBundleScope) { self.buffers.extend(trackers.buffers.used()); self.textures.extend(trackers.textures.used()); - self.texture_views.extend(trackers.views.used()); - self.samplers.extend(trackers.samplers.used()); self.bind_groups.extend(trackers.bind_groups.used()); self.render_pipelines .extend(trackers.render_pipelines.used()); diff --git a/wgpu-core/src/device/queue.rs b/wgpu-core/src/device/queue.rs index 9f84aaee0d..ddfa27b5a9 100644 --- a/wgpu-core/src/device/queue.rs +++ b/wgpu-core/src/device/queue.rs @@ -627,6 +627,7 @@ impl Global { // finish all the command buffers first for &cmb_id in command_buffer_ids { + #[allow(unused_mut)] let mut cmdbuf = match hub .command_buffers .unregister_locked(cmb_id, &mut *command_buffer_guard) diff --git a/wgpu-core/src/hub.rs b/wgpu-core/src/hub.rs index 1f6ab1f3cc..76fb5c6aa4 100644 --- a/wgpu-core/src/hub.rs +++ b/wgpu-core/src/hub.rs @@ -271,6 +271,10 @@ impl Storage { }) } + pub(crate) fn len(&self) -> usize { + self.map.len() + } + fn generate_report(&self) -> StorageReport { let mut report = StorageReport { element_size: mem::size_of::(), diff --git a/wgpu-core/src/track/buffer.rs b/wgpu-core/src/track/buffer.rs index f6cfd3bc4d..44a7a91dc2 100644 --- a/wgpu-core/src/track/buffer.rs +++ b/wgpu-core/src/track/buffer.rs @@ -110,7 +110,7 @@ impl BufferUsageScope { }); } - pub fn set_max_index(&mut self, size: usize) { + pub fn set_size(&mut self, size: usize) { self.state.resize(size, BufferUses::empty()); self.ref_counts.resize(size, None); self.epochs.resize(size, u32::MAX); @@ -143,7 +143,7 @@ impl BufferUsageScope { pub fn extend_from_scope(&mut self, scope: &Self) -> Result<(), UsageConflict> { let incoming_size = scope.state.len(); if incoming_size > self.state.len() { - self.set_max_index(incoming_size); + self.set_size(incoming_size); } for index in iterate_bitvec_indices(&scope.owned) { @@ -272,7 +272,7 @@ impl BufferTracker { }); } - fn set_max_index(&mut self, size: usize) { + pub fn set_size(&mut self, size: usize) { self.start.resize(size, BufferUses::empty()); self.end.resize(size, BufferUses::empty()); @@ -282,6 +282,12 @@ impl BufferTracker { resize_bitvec(&mut self.owned, size); } + fn allow_index(&mut self, index: usize) { + if index >= self.start.len() { + self.set_size(index + 1); + } + } + pub fn used(&self) -> impl Iterator> + '_ { self.debug_assert_in_bounds(self.owned.len() - 1); iterate_bitvec_indices(&self.owned).map(move |index| { @@ -294,22 +300,26 @@ impl BufferTracker { self.temp.drain(..) } - pub unsafe fn init(&mut self, id: Valid, ref_count: RefCount, state: BufferUses) { + pub fn init(&mut self, id: Valid, ref_count: RefCount, state: BufferUses) { let (index32, epoch, _) = id.0.unzip(); let index = index32 as usize; + self.allow_index(index); + self.debug_assert_in_bounds(index); - *self.start.get_unchecked_mut(index) = state; - *self.end.get_unchecked_mut(index) = state; + unsafe { + *self.start.get_unchecked_mut(index) = state; + *self.end.get_unchecked_mut(index) = state; - *self.ref_counts.get_unchecked_mut(index) = Some(ref_count); - *self.epochs.get_unchecked_mut(index) = epoch; + *self.ref_counts.get_unchecked_mut(index) = Some(ref_count); + *self.epochs.get_unchecked_mut(index) = epoch; - self.owned.set(index, true); + self.owned.set(index, true); + } } - pub unsafe fn change_state<'a>( + pub fn change_state<'a>( &mut self, storage: &'a hub::Storage, BufferId>, id: BufferId, @@ -320,12 +330,16 @@ impl BufferTracker { let (index32, epoch, _) = id.unzip(); let index = index32 as usize; - self.transition_inner( - index, - epoch, - value.life_guard.ref_count.as_ref().unwrap(), - state, - ); + self.allow_index(index); + + unsafe { + self.transition_inner( + index, + epoch, + value.life_guard.ref_count.as_ref().unwrap(), + state, + ) + }; Some((value, self.temp.pop())) } @@ -333,7 +347,7 @@ impl BufferTracker { pub fn change_states_tracker(&mut self, tracker: &Self) { let incoming_size = tracker.start.len(); if incoming_size > self.start.len() { - self.set_max_index(incoming_size); + self.set_size(incoming_size); } for index in iterate_bitvec_indices(&tracker.owned) { @@ -355,7 +369,7 @@ impl BufferTracker { pub fn change_states_scope(&mut self, scope: &BufferUsageScope) { let incoming_size = scope.state.len(); if incoming_size > self.start.len() { - self.set_max_index(incoming_size); + self.set_size(incoming_size); } for index in iterate_bitvec_indices(&scope.owned) { @@ -381,7 +395,7 @@ impl BufferTracker { ) { let incoming_size = scope.state.len(); if incoming_size > self.start.len() { - self.set_max_index(incoming_size); + self.set_size(incoming_size); } for (id, ref_count, _) in bind_group_state.buffers.iter() { @@ -395,6 +409,8 @@ impl BufferTracker { } self.transition(&scope.state, ref_count, index, epoch); + *scope.ref_counts.get_unchecked_mut(index) = None; + *scope.epochs.get_unchecked_mut(index) = u32::MAX; scope.owned.set(index, false); } } diff --git a/wgpu-core/src/track/mod.rs b/wgpu-core/src/track/mod.rs index 65501d7406..f4e90c34e0 100644 --- a/wgpu-core/src/track/mod.rs +++ b/wgpu-core/src/track/mod.rs @@ -248,24 +248,35 @@ impl BindGroupStates { pub(crate) struct RenderBundleScope { pub buffers: BufferUsageScope, pub textures: TextureUsageScope, - pub views: StatelessTracker, id::TextureViewId>, - pub samplers: StatelessTracker, id::SamplerId>, + // Don't need to track views and samplers, they are never used directly, only by bind groups. pub bind_groups: StatelessTracker, id::BindGroupId>, pub render_pipelines: StatelessTracker, id::RenderPipelineId>, pub query_sets: StatelessTracker, id::QuerySetId>, } impl RenderBundleScope { - pub fn new() -> Self { - Self { + pub fn new( + buffers: &hub::Storage, id::BufferId>, + textures: &hub::Storage, id::TextureId>, + bind_groups: &hub::Storage, id::BindGroupId>, + render_pipelines: &hub::Storage, id::RenderPipelineId>, + query_sets: &hub::Storage, id::QuerySetId>, + ) -> Self { + let mut value = Self { buffers: BufferUsageScope::new(), textures: TextureUsageScope::new(), - views: StatelessTracker::new(), - samplers: StatelessTracker::new(), bind_groups: StatelessTracker::new(), render_pipelines: StatelessTracker::new(), query_sets: StatelessTracker::new(), - } + }; + + value.buffers.set_size(buffers.len()); + value.textures.set_size(textures.len()); + value.bind_groups.set_size(bind_groups.len()); + value.render_pipelines.set_size(render_pipelines.len()); + value.query_sets.set_size(query_sets.len()); + + value } pub unsafe fn extend_from_bind_group( @@ -288,11 +299,19 @@ pub(crate) struct UsageScope { } impl UsageScope { - pub fn new() -> Self { - Self { + pub fn new( + buffers: &hub::Storage, id::BufferId>, + textures: &hub::Storage, id::TextureId>, + ) -> Self { + let mut value = Self { buffers: BufferUsageScope::new(), textures: TextureUsageScope::new(), - } + }; + + value.buffers.set_size(buffers.len()); + value.textures.set_size(textures.len()); + + value } pub unsafe fn extend_from_bind_group( @@ -347,6 +366,49 @@ impl Tracker { } } + pub fn set_size( + &mut self, + buffers: Option<&hub::Storage, id::BufferId>>, + textures: Option<&hub::Storage, id::TextureId>>, + views: Option<&hub::Storage, id::TextureViewId>>, + samplers: Option<&hub::Storage, id::SamplerId>>, + bind_groups: Option<&hub::Storage, id::BindGroupId>>, + compute_pipelines: Option< + &hub::Storage, id::ComputePipelineId>, + >, + render_pipelines: Option<&hub::Storage, id::RenderPipelineId>>, + bundles: Option<&hub::Storage, id::RenderBundleId>>, + query_sets: Option<&hub::Storage, id::QuerySetId>>, + ) { + if let Some(buffers) = buffers { + self.buffers.set_size(buffers.len()); + }; + if let Some(textures) = textures { + self.textures.set_size(textures.len()); + }; + if let Some(views) = views { + self.views.set_size(views.len()); + }; + if let Some(samplers) = samplers { + self.samplers.set_size(samplers.len()); + }; + if let Some(bind_groups) = bind_groups { + self.bind_groups.set_size(bind_groups.len()); + }; + if let Some(compute_pipelines) = compute_pipelines { + self.compute_pipelines.set_size(compute_pipelines.len()); + } + if let Some(render_pipelines) = render_pipelines { + self.render_pipelines.set_size(render_pipelines.len()); + }; + if let Some(bundles) = bundles { + self.bundles.set_size(bundles.len()); + }; + if let Some(query_sets) = query_sets { + self.query_sets.set_size(query_sets.len()); + }; + } + pub unsafe fn extend_from_bind_group( &mut self, textures: &hub::Storage, id::TextureId>, @@ -363,8 +425,6 @@ impl Tracker { &mut self, render_bundle: &RenderBundleScope, ) -> Result<(), UsageConflict> { - self.views.extend_from_tracker(&render_bundle.views); - self.samplers.extend_from_tracker(&render_bundle.samplers); self.bind_groups .extend_from_tracker(&render_bundle.bind_groups); self.render_pipelines diff --git a/wgpu-core/src/track/stateless.rs b/wgpu-core/src/track/stateless.rs index bba2316e9e..dfd96f7a2b 100644 --- a/wgpu-core/src/track/stateless.rs +++ b/wgpu-core/src/track/stateless.rs @@ -76,7 +76,7 @@ impl StatelessTracker { }); } - pub fn set_max_index(&mut self, size: usize) { + pub fn set_size(&mut self, size: usize) { self.epochs.resize(size, u32::MAX); self.ref_counts.resize(size, None); @@ -102,6 +102,7 @@ impl StatelessTracker { self.owned.set(index, true); } + /// Requires set_size to be called pub unsafe fn extend<'a>(&mut self, storage: &'a hub::Storage, id: Id) -> Option<&'a T> { let item = storage.get(id).ok()?; @@ -135,7 +136,7 @@ impl StatelessTracker { pub fn extend_from_tracker(&mut self, other: &Self) { let incoming_size = other.owned.len(); if incoming_size > self.owned.len() { - self.set_max_index(incoming_size); + self.set_size(incoming_size); } for index in iterate_bitvec_indices(&other.owned) { diff --git a/wgpu-core/src/track/texture.rs b/wgpu-core/src/track/texture.rs index 1a5007869d..960ed14e08 100644 --- a/wgpu-core/src/track/texture.rs +++ b/wgpu-core/src/track/texture.rs @@ -165,7 +165,7 @@ impl TextureUsageScope { }); } - pub fn set_max_index(&mut self, size: usize) { + pub fn set_size(&mut self, size: usize) { self.set.simple.resize(size, TextureUses::UNINITIALIZED); self.ref_counts.resize(size, None); self.epochs.resize(size, u32::MAX); @@ -192,7 +192,7 @@ impl TextureUsageScope { ) -> Result<(), UsageConflict> { let incoming_size = scope.set.simple.len(); if incoming_size > self.set.simple.len() { - self.set_max_index(incoming_size); + self.set_size(incoming_size); } for index in iterate_bitvec_indices(&scope.owned) { @@ -217,7 +217,7 @@ impl TextureUsageScope { /// # Safety /// - /// `id` must be a valid ID and have an ID value less than the last call to set_max_index. + /// `id` must be a valid ID and have an ID value less than the last call to set_size. pub unsafe fn extend<'a>( &mut self, storage: &'a hub::Storage, TextureId>, @@ -242,7 +242,7 @@ impl TextureUsageScope { /// # Safety /// - /// `id` must be a valid ID and have an ID value less than the last call to set_max_index. + /// `id` must be a valid ID and have an ID value less than the last call to set_size. pub unsafe fn extend_refcount( &mut self, storage: &hub::Storage, TextureId>, @@ -425,7 +425,7 @@ impl TextureTracker { }); } - fn set_max_index(&mut self, size: usize) { + pub fn set_size(&mut self, size: usize) { self.start_set .simple .resize(size, TextureUses::UNINITIALIZED); @@ -436,6 +436,12 @@ impl TextureTracker { resize_bitvec(&mut self.owned, size); } + fn allow_index(&mut self, index: usize) { + if index >= self.start_set.simple.len() { + self.set_size(index + 1); + } + } + pub fn used(&self) -> impl Iterator> + '_ { self.debug_assert_in_bounds(self.owned.len() - 1); iterate_bitvec_indices(&self.owned).map(move |index| { @@ -455,27 +461,38 @@ impl TextureTracker { self.ref_counts[index].as_ref().unwrap() } - pub unsafe fn init(&mut self, id: TextureId, ref_count: RefCount, usage: TextureUses) { + pub fn init(&mut self, id: TextureId, ref_count: RefCount, usage: TextureUses) { let (index32, epoch, _) = id.unzip(); let index = index32 as usize; + self.allow_index(index); + self.debug_assert_in_bounds(index); - *self.start_set.simple.get_unchecked_mut(index) = usage; - *self.end_set.simple.get_unchecked_mut(index) = usage; + unsafe { + *self.start_set.simple.get_unchecked_mut(index) = usage; + *self.end_set.simple.get_unchecked_mut(index) = usage; - *self.epochs.get_unchecked_mut(index) = epoch; - *self.ref_counts.get_unchecked_mut(index) = Some(ref_count); - self.owned.set(index, true); + *self.epochs.get_unchecked_mut(index) = epoch; + *self.ref_counts.get_unchecked_mut(index) = Some(ref_count); + self.owned.set(index, true); + } } - pub unsafe fn change_state<'a>( + pub fn change_state<'a>( &mut self, storage: &'a hub::Storage, TextureId>, id: TextureId, selector: TextureSelector, new_usage: TextureUses, ) -> Option<(&'a Texture, Option>)> { + let (index32, epoch, _) = id.unzip(); + let index = index32 as usize; + + self.allow_index(index); + + self.debug_assert_in_bounds(index); + let _ = (storage, id, selector, new_usage); todo!() } @@ -487,7 +504,7 @@ impl TextureTracker { ) { let incoming_size = tracker.start_set.simple.len(); if incoming_size > self.start_set.simple.len() { - self.set_max_index(incoming_size); + self.set_size(incoming_size); } for index in iterate_bitvec_indices(&tracker.owned) { @@ -511,7 +528,7 @@ impl TextureTracker { ) { let incoming_size = scope.set.simple.len(); if incoming_size > self.start_set.simple.len() { - self.set_max_index(incoming_size); + self.set_size(incoming_size); } for index in iterate_bitvec_indices(&scope.owned) { @@ -530,7 +547,7 @@ impl TextureTracker { ) { let incoming_size = scope.set.simple.len(); if incoming_size > self.start_set.simple.len() { - self.set_max_index(incoming_size); + self.set_size(incoming_size); } for &(id, _, _, _) in bind_group_state.textures.iter() { From 643c26498b47976ca9a642a8f56af37a8a123751 Mon Sep 17 00:00:00 2001 From: Connor Fitzgerald Date: Sun, 8 May 2022 04:36:05 -0400 Subject: [PATCH 14/35] temp13 - unsafe --- wgpu-core/src/command/clear.rs | 24 ++--- wgpu-core/src/command/memory_init.rs | 12 +-- wgpu-core/src/command/query.rs | 12 +-- wgpu-core/src/command/transfer.rs | 144 ++++++++++++--------------- wgpu-core/src/device/mod.rs | 38 ++++--- wgpu-core/src/device/queue.rs | 30 +++--- wgpu-core/src/present.rs | 12 +-- wgpu-core/src/track/buffer.rs | 2 + wgpu-core/src/track/stateless.rs | 15 --- wgpu-core/src/track/texture.rs | 9 +- 10 files changed, 127 insertions(+), 171 deletions(-) diff --git a/wgpu-core/src/command/clear.rs b/wgpu-core/src/command/clear.rs index 8042e4106b..a0a7450f07 100644 --- a/wgpu-core/src/command/clear.rs +++ b/wgpu-core/src/command/clear.rs @@ -86,13 +86,11 @@ impl Global { list.push(TraceCommand::ClearBuffer { dst, offset, size }); } - let (dst_buffer, dst_pending) = unsafe { - cmd_buf - .trackers - .buffers - .change_state(&*buffer_guard, dst, hal::BufferUses::COPY_DST) - .ok_or_else(|| ClearError::InvalidBuffer(dst))? - }; + let (dst_buffer, dst_pending) = cmd_buf + .trackers + .buffers + .change_state(&*buffer_guard, dst, hal::BufferUses::COPY_DST) + .ok_or_else(|| ClearError::InvalidBuffer(dst))?; let dst_raw = dst_buffer .raw .as_ref() @@ -274,13 +272,11 @@ pub(crate) fn clear_texture( // On the other hand, when coming via command_encoder_clear_texture, the life_guard is still there since in order to call it a texture object is needed. // // We could in theory distinguish these two scenarios in the internal clear_texture api in order to remove this check and call the cheaper change_replace_tracked whenever possible. - let dst_barrier = unsafe { - texture_tracker - .change_state(storage, dst_texture_id.0, selector, clear_usage) - .unwrap() - .1 - .map(|pending| pending.into_hal(dst_texture)) - }; + let dst_barrier = texture_tracker + .change_state(storage, dst_texture_id.0, selector, clear_usage) + .unwrap() + .1 + .map(|pending| pending.into_hal(dst_texture)); unsafe { encoder.transition_textures(dst_barrier.into_iter()); } diff --git a/wgpu-core/src/command/memory_init.rs b/wgpu-core/src/command/memory_init.rs index b9587f3527..b35a6a5e9e 100644 --- a/wgpu-core/src/command/memory_init.rs +++ b/wgpu-core/src/command/memory_init.rs @@ -203,13 +203,11 @@ impl BakedCommands { // Don't do use_replace since the buffer may already no longer have a ref_count. // However, we *know* that it is currently in use, so the tracker must already know about it. - let transition = unsafe { - device_tracker - .buffers - .change_state(buffer_guard, buffer_id, hal::BufferUses::COPY_DST) - .unwrap() - .1 - }; + let transition = device_tracker + .buffers + .change_state(buffer_guard, buffer_id, hal::BufferUses::COPY_DST) + .unwrap() + .1; let buffer = buffer_guard .get_mut(buffer_id) diff --git a/wgpu-core/src/command/query.rs b/wgpu-core/src/command/query.rs index e5b4601f35..60f1d64504 100644 --- a/wgpu-core/src/command/query.rs +++ b/wgpu-core/src/command/query.rs @@ -351,13 +351,11 @@ impl Global { .ok_or_else(|| QueryError::InvalidQuerySet(query_set_id))? }; - let (dst_buffer, dst_pending) = unsafe { - cmd_buf - .trackers - .buffers - .change_state(&*buffer_guard, destination, hal::BufferUses::COPY_DST) - .ok_or_else(|| QueryError::InvalidBuffer(destination))? - }; + let (dst_buffer, dst_pending) = cmd_buf + .trackers + .buffers + .change_state(&*buffer_guard, destination, hal::BufferUses::COPY_DST) + .ok_or_else(|| QueryError::InvalidBuffer(destination))?; let dst_barrier = dst_pending.map(|pending| pending.into_hal(dst_buffer)); if !dst_buffer.usage.contains(wgt::BufferUsages::COPY_DST) { diff --git a/wgpu-core/src/command/transfer.rs b/wgpu-core/src/command/transfer.rs index 26f0489fb8..78a6943230 100644 --- a/wgpu-core/src/command/transfer.rs +++ b/wgpu-core/src/command/transfer.rs @@ -514,13 +514,11 @@ impl Global { }); } - let (src_buffer, src_pending) = unsafe { - cmd_buf - .trackers - .buffers - .change_state(&*buffer_guard, source, hal::BufferUses::COPY_SRC) - .ok_or_else(|| TransferError::InvalidBuffer(source))? - }; + let (src_buffer, src_pending) = cmd_buf + .trackers + .buffers + .change_state(&*buffer_guard, source, hal::BufferUses::COPY_SRC) + .ok_or_else(|| TransferError::InvalidBuffer(source))?; let src_raw = src_buffer .raw .as_ref() @@ -531,13 +529,11 @@ impl Global { // expecting only a single barrier let src_barrier = src_pending.map(|pending| pending.into_hal(src_buffer)); - let (dst_buffer, dst_pending) = unsafe { - cmd_buf - .trackers - .buffers - .change_state(&*buffer_guard, destination, hal::BufferUses::COPY_DST) - .ok_or_else(|| TransferError::InvalidBuffer(destination))? - }; + let (dst_buffer, dst_pending) = cmd_buf + .trackers + .buffers + .change_state(&*buffer_guard, destination, hal::BufferUses::COPY_DST) + .ok_or_else(|| TransferError::InvalidBuffer(destination))?; let dst_raw = dst_buffer .raw .as_ref() @@ -652,13 +648,11 @@ impl Global { // Handle texture init *before* dealing with barrier transitions so we have an easier time inserting "immediate-inits" that may be required by prior discards in rare cases. handle_dst_texture_init(cmd_buf, device, destination, copy_size, &texture_guard)?; - let (src_buffer, src_pending) = unsafe { - cmd_buf - .trackers - .buffers - .change_state(&*buffer_guard, source.buffer, hal::BufferUses::COPY_SRC) - .ok_or_else(|| TransferError::InvalidBuffer(source.buffer))? - }; + let (src_buffer, src_pending) = cmd_buf + .trackers + .buffers + .change_state(&*buffer_guard, source.buffer, hal::BufferUses::COPY_SRC) + .ok_or_else(|| TransferError::InvalidBuffer(source.buffer))?; let src_raw = src_buffer .raw .as_ref() @@ -668,18 +662,16 @@ impl Global { } let src_barrier = src_pending.map(|pending| pending.into_hal(src_buffer)); - let (dst_texture, dst_pending) = unsafe { - cmd_buf - .trackers - .textures - .change_state( - &*texture_guard, - destination.texture, - dst_range, - hal::TextureUses::COPY_DST, - ) - .ok_or_else(|| TransferError::InvalidTexture(destination.texture))? - }; + let (dst_texture, dst_pending) = cmd_buf + .trackers + .textures + .change_state( + &*texture_guard, + destination.texture, + dst_range, + hal::TextureUses::COPY_DST, + ) + .ok_or_else(|| TransferError::InvalidTexture(destination.texture))?; let dst_raw = dst_texture .inner .as_raw() @@ -783,18 +775,16 @@ impl Global { // Handle texture init *before* dealing with barrier transitions so we have an easier time inserting "immediate-inits" that may be required by prior discards in rare cases. handle_src_texture_init(cmd_buf, device, source, copy_size, &texture_guard)?; - let (src_texture, src_pending) = unsafe { - cmd_buf - .trackers - .textures - .change_state( - &*texture_guard, - source.texture, - src_range, - hal::TextureUses::COPY_SRC, - ) - .ok_or_else(|| TransferError::InvalidTexture(source.texture))? - }; + let (src_texture, src_pending) = cmd_buf + .trackers + .textures + .change_state( + &*texture_guard, + source.texture, + src_range, + hal::TextureUses::COPY_SRC, + ) + .ok_or_else(|| TransferError::InvalidTexture(source.texture))?; let src_raw = src_texture .inner .as_raw() @@ -804,17 +794,15 @@ impl Global { } let src_barrier = src_pending.map(|pending| pending.into_hal(src_texture)); - let (dst_buffer, dst_pending) = unsafe { - cmd_buf - .trackers - .buffers - .change_state( - &*buffer_guard, - destination.buffer, - hal::BufferUses::COPY_DST, - ) - .ok_or_else(|| TransferError::InvalidBuffer(destination.buffer))? - }; + let (dst_buffer, dst_pending) = cmd_buf + .trackers + .buffers + .change_state( + &*buffer_guard, + destination.buffer, + hal::BufferUses::COPY_DST, + ) + .ok_or_else(|| TransferError::InvalidBuffer(destination.buffer))?; let dst_raw = dst_buffer .raw .as_ref() @@ -937,18 +925,16 @@ impl Global { handle_src_texture_init(cmd_buf, device, source, copy_size, &texture_guard)?; handle_dst_texture_init(cmd_buf, device, destination, copy_size, &texture_guard)?; - let (src_texture, src_pending) = unsafe { - cmd_buf - .trackers - .textures - .change_state( - &*texture_guard, - source.texture, - src_range, - hal::TextureUses::COPY_SRC, - ) - .ok_or_else(|| TransferError::InvalidTexture(source.texture))? - }; + let (src_texture, src_pending) = cmd_buf + .trackers + .textures + .change_state( + &*texture_guard, + source.texture, + src_range, + hal::TextureUses::COPY_SRC, + ) + .ok_or_else(|| TransferError::InvalidTexture(source.texture))?; let src_raw = src_texture .inner .as_raw() @@ -964,18 +950,16 @@ impl Global { .into_iter() .collect(); - let (dst_texture, dst_pending) = unsafe { - cmd_buf - .trackers - .textures - .change_state( - &*texture_guard, - destination.texture, - dst_range, - hal::TextureUses::COPY_DST, - ) - .ok_or(TransferError::InvalidTexture(destination.texture))? - }; + let (dst_texture, dst_pending) = cmd_buf + .trackers + .textures + .change_state( + &*texture_guard, + destination.texture, + dst_range, + hal::TextureUses::COPY_DST, + ) + .ok_or(TransferError::InvalidTexture(destination.texture))?; let dst_raw = dst_texture .inner .as_raw() diff --git a/wgpu-core/src/device/mod.rs b/wgpu-core/src/device/mod.rs index b03d9cfd3e..4cc111ed15 100644 --- a/wgpu-core/src/device/mod.rs +++ b/wgpu-core/src/device/mod.rs @@ -3183,13 +3183,12 @@ impl Global { let id = fid.assign(buffer, &mut token); log::info!("Created buffer {:?} with {:?}", id, desc); - unsafe { - device - .trackers - .lock() - .buffers - .init(id, ref_count, buffer_use) - }; + device + .trackers + .lock() + .buffers + .init(id, ref_count, buffer_use); + return (id.0, None); }; @@ -3455,13 +3454,12 @@ impl Global { let id = fid.assign(texture, &mut token); log::info!("Created texture {:?} with {:?}", id, desc); - unsafe { - device.trackers.lock().textures.init( - id.0, - ref_count, - hal::TextureUses::UNINITIALIZED, - ) - }; + device + .trackers + .lock() + .textures + .init(id.0, ref_count, hal::TextureUses::UNINITIALIZED); + return (id.0, None); }; @@ -5125,13 +5123,11 @@ impl Global { let ret = (buffer.device_id.value, buffer.life_guard.add_ref()); - unsafe { - let mut trackers = device.trackers.lock(); - trackers - .buffers - .change_state(&*buffer_guard, buffer_id, internal_use); - trackers.buffers.drain(); - }; + let mut trackers = device.trackers.lock(); + trackers + .buffers + .change_state(&*buffer_guard, buffer_id, internal_use); + trackers.buffers.drain(); ret }; diff --git a/wgpu-core/src/device/queue.rs b/wgpu-core/src/device/queue.rs index ddfa27b5a9..d1861a7d81 100644 --- a/wgpu-core/src/device/queue.rs +++ b/wgpu-core/src/device/queue.rs @@ -284,12 +284,10 @@ impl Global { .map_err(DeviceError::from)?; let mut trackers = device.trackers.lock(); - let (dst, transition) = unsafe { - trackers - .buffers - .change_state(&*buffer_guard, buffer_id, hal::BufferUses::COPY_DST) - .ok_or_else(|| TransferError::InvalidBuffer(buffer_id))? - }; + let (dst, transition) = trackers + .buffers + .change_state(&*buffer_guard, buffer_id, hal::BufferUses::COPY_DST) + .ok_or_else(|| TransferError::InvalidBuffer(buffer_id))?; let dst_raw = dst .raw .as_ref() @@ -473,17 +471,15 @@ impl Global { } } - let (dst, transition) = unsafe { - trackers - .textures - .change_state( - &*texture_guard, - destination.texture, - selector, - hal::TextureUses::COPY_DST, - ) - .ok_or_else(|| TransferError::InvalidTexture(destination.texture))? - }; + let (dst, transition) = trackers + .textures + .change_state( + &*texture_guard, + destination.texture, + selector, + hal::TextureUses::COPY_DST, + ) + .ok_or_else(|| TransferError::InvalidTexture(destination.texture))?; let (hal_copy_size, array_layer_count) = validate_texture_copy_range(destination, &dst.desc, CopySide::Destination, size)?; diff --git a/wgpu-core/src/present.rs b/wgpu-core/src/present.rs index b163081ede..d1717bf9b0 100644 --- a/wgpu-core/src/present.rs +++ b/wgpu-core/src/present.rs @@ -189,13 +189,11 @@ impl Global { { // register it in the device tracker as uninitialized let mut trackers = device.trackers.lock(); - let _ = unsafe { - trackers.textures.init( - id.0, - ref_count.clone(), - hal::TextureUses::UNINITIALIZED, - ) - }; + let _ = trackers.textures.init( + id.0, + ref_count.clone(), + hal::TextureUses::UNINITIALIZED, + ); } if present.acquired_texture.is_some() { diff --git a/wgpu-core/src/track/buffer.rs b/wgpu-core/src/track/buffer.rs index 44a7a91dc2..910be34142 100644 --- a/wgpu-core/src/track/buffer.rs +++ b/wgpu-core/src/track/buffer.rs @@ -332,6 +332,8 @@ impl BufferTracker { self.allow_index(index); + self.debug_assert_in_bounds(index); + unsafe { self.transition_inner( index, diff --git a/wgpu-core/src/track/stateless.rs b/wgpu-core/src/track/stateless.rs index dfd96f7a2b..70b5e353b8 100644 --- a/wgpu-core/src/track/stateless.rs +++ b/wgpu-core/src/track/stateless.rs @@ -118,21 +118,6 @@ impl StatelessTracker { Some(item) } - pub unsafe fn extend_from_bind_group(&mut self, bind_group: &StatelessBindGroupSate) { - for (id, ref_count) in &bind_group.resources { - let (index32, epoch, _) = id.0.unzip(); - let index = index32 as usize; - self.debug_assert_in_bounds(index); - - let previously_owned = self.owned.get(index).unwrap_unchecked(); - if !previously_owned { - *self.epochs.get_unchecked_mut(index) = epoch; - *self.ref_counts.get_unchecked_mut(index) = Some(ref_count.clone()); - self.owned.set(index, true); - } - } - } - pub fn extend_from_tracker(&mut self, other: &Self) { let incoming_size = other.owned.len(); if incoming_size > self.owned.len() { diff --git a/wgpu-core/src/track/texture.rs b/wgpu-core/src/track/texture.rs index 960ed14e08..f224a9f6a0 100644 --- a/wgpu-core/src/track/texture.rs +++ b/wgpu-core/src/track/texture.rs @@ -486,15 +486,18 @@ impl TextureTracker { selector: TextureSelector, new_usage: TextureUses, ) -> Option<(&'a Texture, Option>)> { - let (index32, epoch, _) = id.unzip(); + let item = storage.get(id).ok()?; + + let (index32, _epoch, _) = id.unzip(); let index = index32 as usize; self.allow_index(index); self.debug_assert_in_bounds(index); - + let _ = (storage, id, selector, new_usage); - todo!() + + Some((item, self.temp.pop())) } pub fn change_states_tracker( From 27a4e401a4ae15da2ad71615dd43c3efaa441890 Mon Sep 17 00:00:00 2001 From: Connor Fitzgerald Date: Mon, 9 May 2022 15:06:30 -0400 Subject: [PATCH 15/35] temp14 - Abstract all the state transitions --- wgpu-core/src/command/bundle.rs | 2 +- wgpu-core/src/track/mod.rs | 90 ++- wgpu-core/src/track/range.rs | 4 + wgpu-core/src/track/texture.rs | 1004 +++++++++++++++++++------------ 4 files changed, 678 insertions(+), 422 deletions(-) diff --git a/wgpu-core/src/command/bundle.rs b/wgpu-core/src/command/bundle.rs index 50da9c3e29..286f756f93 100644 --- a/wgpu-core/src/command/bundle.rs +++ b/wgpu-core/src/command/bundle.rs @@ -34,7 +34,7 @@ invalidations or index format changes. #![allow(clippy::reversed_empty_ranges)] use crate::{ - binding_model::{buffer_binding_type_alignment, self}, + binding_model::{self, buffer_binding_type_alignment}, command::{ BasePass, BindGroupStateChange, DrawError, MapPassErr, PassErrorScope, RenderCommand, RenderCommandError, StateChange, diff --git a/wgpu-core/src/track/mod.rs b/wgpu-core/src/track/mod.rs index f4e90c34e0..3d87b99227 100644 --- a/wgpu-core/src/track/mod.rs +++ b/wgpu-core/src/track/mod.rs @@ -6,11 +6,11 @@ mod texture; use crate::{ binding_model, command, conv, hub, id::{self, TypedId}, - pipeline, resource, + pipeline, resource, Epoch, RefCount, }; use bit_vec::BitVec; -use std::{fmt, mem, num::NonZeroU32, ops}; +use std::{fmt, marker::PhantomData, mem, num::NonZeroU32, ops}; use thiserror::Error; pub(crate) use buffer::{BufferBindGroupState, BufferTracker, BufferUsageScope}; @@ -160,33 +160,16 @@ impl UsageConflict { } } - fn from_texture( - storage: &hub::Storage, id::TextureId>, - id: id::Valid, - selector: Option, + fn from_texture( + id: id::TextureId, + selector: TextureSelector, current_state: hal::TextureUses, new_state: hal::TextureUses, ) -> Self { - let texture = &storage[id]; - - let mips; - let layers; - - match selector { - Some(selector) => { - mips = selector.mips; - layers = selector.layers; - } - None => { - mips = texture.full_range.mips.clone(); - layers = texture.full_range.layers.clone(); - } - } - Self::Texture { - id: id.0, - mip_levels: mips, - array_layers: layers, + id: id, + mip_levels: selector.mips, + array_layers: selector.layers, invalid_use: InvalidUse { current_state, new_state, @@ -220,6 +203,63 @@ impl fmt::Display for InvalidUse { } } +#[derive(Debug)] +pub(crate) struct ResourceMetadata { + owned: BitVec, + ref_counts: Vec>, + epochs: Vec, + + _phantom: PhantomData, +} +impl ResourceMetadata { + pub fn new() -> Self { + Self { + owned: BitVec::default(), + ref_counts: Vec::new(), + epochs: Vec::new(), + + _phantom: PhantomData, + } + } + + pub fn set_size(&mut self, size: usize) { + self.ref_counts.resize(size, None); + self.epochs.resize(size, u32::MAX); + + resize_bitvec(&mut self.owned, size); + } + + fn debug_assert_in_bounds(&self, index: usize) { + debug_assert!(index < self.owned.len()); + debug_assert!(index < self.ref_counts.len()); + debug_assert!(index < self.epochs.len()); + + debug_assert!(if self.owned.get(index).unwrap() { + self.ref_counts[index].is_some() + } else { + true + }); + } + + fn is_empty(&self) -> bool { + !self.owned.any() + } + + fn used(&self) -> impl Iterator> + '_ { + self.debug_assert_in_bounds(self.owned.len() - 1); + iterate_bitvec_indices(&self.owned).map(move |index| { + let epoch = unsafe { *self.epochs.get_unchecked(index) }; + id::Valid(Id::zip(index as u32, epoch, A::VARIANT)) + }) + } + + unsafe fn reset(&mut self, index: usize) { + *self.ref_counts.get_unchecked_mut(index) = None; + *self.epochs.get_unchecked_mut(index) = u32::MAX; + self.owned.set(index, false); + } +} + pub(crate) struct BindGroupStates { pub buffers: BufferBindGroupState, pub textures: TextureBindGroupState, diff --git a/wgpu-core/src/track/range.rs b/wgpu-core/src/track/range.rs index cd1dd86e43..a64821ec79 100644 --- a/wgpu-core/src/track/range.rs +++ b/wgpu-core/src/track/range.rs @@ -40,6 +40,10 @@ impl RangedStates { self.ranges.iter() } + pub fn iter_mut(&mut self) -> impl Iterator, T)> { + self.ranges.iter_mut() + } + pub fn into_iter(self) -> impl Iterator, T)> { self.ranges.into_iter() } diff --git a/wgpu-core/src/track/texture.rs b/wgpu-core/src/track/texture.rs index f224a9f6a0..2d6f16885d 100644 --- a/wgpu-core/src/track/texture.rs +++ b/wgpu-core/src/track/texture.rs @@ -4,18 +4,17 @@ use crate::{ id::{TextureId, TypedId, Valid}, resource::Texture, track::{ - invalid_resource_state, iterate_bitvec_indices, resize_bitvec, skip_barrier, ResourceUses, - UsageConflict, + invalid_resource_state, iterate_bitvec_indices, skip_barrier, + ResourceMetadata, ResourceUses, UsageConflict, }, Epoch, RefCount, }; -use bit_vec::BitVec; use hal::TextureUses; use arrayvec::ArrayVec; use naga::FastHashMap; -use std::{iter, marker::PhantomData, ops::Range, vec::Drain}; +use std::{borrow::Cow, iter, marker::PhantomData, ops::Range, vec::Drain}; #[derive(Clone, Debug, PartialEq, Eq)] pub struct TextureSelector { @@ -57,6 +56,42 @@ impl ComplexTextureState { .collect(), } } + + fn from_selector_state_iter( + full_range: TextureSelector, + state_iter: impl Iterator, + ) -> Self { + let mut complex = + ComplexTextureState::new(full_range.mips.len() as u32, full_range.layers.len() as u32); + for (selector, desired_state) in state_iter { + let mips = selector.mips.start as usize..selector.mips.end as usize; + for mip in &mut complex.mips[mips] { + for (_, state) in mip.isolate(&selector.layers, TextureUses::UNINITIALIZED) { + *state = desired_state; + } + } + } + complex + } + + fn into_selector_state_iter( + &self, + ) -> impl Iterator + '_ { + self.mips.iter().enumerate().flat_map(|(mip, inner)| { + let mip = mip as u32; + { + inner.iter().map(move |(layers, inner)| { + ( + TextureSelector { + mips: mip..mip + 1, + layers: layers.clone(), + }, + *inner, + ) + }) + } + }) + } } // TODO: This representation could be optimized in a couple ways, but keep it simple for now. @@ -122,12 +157,7 @@ impl TextureStateSet { pub(crate) struct TextureUsageScope { set: TextureStateSet, - ref_counts: Vec>, - epochs: Vec, - - owned: BitVec, - - _phantom: PhantomData, + metadata: ResourceMetadata, } impl TextureUsageScope { @@ -135,28 +165,16 @@ impl TextureUsageScope { Self { set: TextureStateSet::new(), - epochs: Vec::new(), - ref_counts: Vec::new(), - - owned: BitVec::default(), - - _phantom: PhantomData, + metadata: ResourceMetadata::new(), } } fn debug_assert_in_bounds(&self, index: usize) { - debug_assert!(index < self.set.simple.len()); - debug_assert!(index < self.owned.len()); - debug_assert!(index < self.ref_counts.len()); - debug_assert!(index < self.epochs.len()); + self.metadata.debug_assert_in_bounds(index); - debug_assert!(if self.owned.get(index).unwrap() { - self.ref_counts[index].is_some() - } else { - true - }); + debug_assert!(index < self.set.simple.len()); - debug_assert!(if self.owned.get(index).unwrap() + debug_assert!(if self.metadata.owned.get(index).unwrap() && self.set.simple[index] == TextureUses::COMPLEX { self.set.complex.contains_key(&(index as u32)) @@ -167,22 +185,15 @@ impl TextureUsageScope { pub fn set_size(&mut self, size: usize) { self.set.simple.resize(size, TextureUses::UNINITIALIZED); - self.ref_counts.resize(size, None); - self.epochs.resize(size, u32::MAX); - - resize_bitvec(&mut self.owned, size); + self.metadata.set_size(size); } pub fn used(&self) -> impl Iterator> + '_ { - self.debug_assert_in_bounds(self.owned.len() - 1); - iterate_bitvec_indices(&self.owned).map(move |index| { - let epoch = unsafe { *self.epochs.get_unchecked(index) }; - Valid(TextureId::zip(index as u32, epoch, A::VARIANT)) - }) + self.metadata.used() } pub(crate) fn is_empty(&self) -> bool { - !self.owned.any() + self.metadata.is_empty() } pub fn extend_from_scope( @@ -195,9 +206,24 @@ impl TextureUsageScope { self.set_size(incoming_size); } - for index in iterate_bitvec_indices(&scope.owned) { - let _ = (storage, index); - todo!() + for index in iterate_bitvec_indices(&scope.metadata.owned) { + let index32 = index as u32; + + unsafe { + state_combine( + Some(storage), + None, + &mut self.set, + &mut self.metadata, + index32, + index, + LayeredStateProvider::TextureSet { set: &scope.set }, + ResourceMetadataProvider::Indirect { + metadata: &scope.metadata, + }, + None, + )? + }; } Ok(()) @@ -254,114 +280,20 @@ impl TextureUsageScope { let (index32, epoch, _) = id.0.unzip(); let index = index32 as usize; - self.debug_assert_in_bounds(index); - - let currently_active = self.owned.get(index).unwrap_unchecked(); - if currently_active { - let current_state = *self.set.simple.get_unchecked(index); - match (current_state == hal::TextureUses::COMPLEX, selector) { - // Both our usages are simple - (false, None) => { - let merged_state = current_state | new_state; - - if invalid_resource_state(merged_state) { - return Err(UsageConflict::from_texture( - storage, - id, - None, - current_state, - new_state, - )); - } - - *self.set.simple.get_unchecked_mut(index) = merged_state; - - return Ok(()); - } - // The old usage is complex. - (true, selector) => { - return self.extend_complex(storage, id, index32, selector, new_state) - } - - // The old usage is simple, so demote it to a complex one. - (false, Some(selector)) => { - *self.set.simple.get_unchecked_mut(index) = hal::TextureUses::COMPLEX; - - // Demote our simple state to a complex one. - self.extend_complex(storage, id, index32, None, current_state)?; - - // Extend that complex state with our new complex state. - return self.extend_complex(storage, id, index32, Some(selector), new_state); - } - } - } - - // We're the first to use this resource, let's add it. - *self.ref_counts.get_unchecked_mut(index) = Some(ref_count.clone()); - *self.epochs.get_unchecked_mut(index) = epoch; - self.owned.set(index, true); - - if let Some(selector) = selector { - *self.set.simple.get_unchecked_mut(index) = hal::TextureUses::COMPLEX; - self.extend_complex(storage, id, index32, Some(selector), new_state)?; - } else { - *self.set.simple.get_unchecked_mut(index) = new_state; - } - - Ok(()) - } - - #[cold] - unsafe fn extend_complex( - &mut self, - storage: &hub::Storage, TextureId>, - id: Valid, - index32: u32, - selector: Option, - new_state: TextureUses, - ) -> Result<(), UsageConflict> { - let texture = storage.get_unchecked(index32); - - // Create the complex entry for this texture. - let complex = self.set.complex.entry(index32).or_insert_with(|| { - ComplexTextureState::new( - texture.desc.mip_level_count, - texture.desc.array_layer_count(), - ) - }); - - let mips; - let layers; - match selector { - Some(ref selector) => { - mips = selector.mips.clone(); - layers = selector.layers.clone(); - } - None => { - mips = texture.full_range.mips.clone(); - layers = texture.full_range.layers.clone(); - } - } - - // Go through all relevant mip levels - for mip in mips { - let mip_state = &mut complex.mips[mip as usize]; - - // Set our state. - for (_, current_state) in mip_state.isolate(&layers, new_state) { - let merged = *current_state | new_state; - if invalid_resource_state(merged) { - return Err(UsageConflict::from_texture( - storage, - id, - selector, - *current_state, - new_state, - )); - } - *current_state = merged; - } - } + state_combine( + Some(storage), + None, + &mut self.set, + &mut self.metadata, + index32, + index, + LayeredStateProvider::from_option(selector, new_state), + ResourceMetadataProvider::Direct { + epoch: epoch, + ref_count: Cow::Borrowed(ref_count), + }, + None, + )?; Ok(()) } @@ -371,9 +303,7 @@ pub(crate) struct TextureTracker { start_set: TextureStateSet, end_set: TextureStateSet, - epochs: Vec, - ref_counts: Vec>, - owned: BitVec, + metadata: ResourceMetadata, /// Temporary storage for collecting transitions. temp: Vec>, @@ -386,9 +316,7 @@ impl TextureTracker { start_set: TextureStateSet::new(), end_set: TextureStateSet::new(), - epochs: Vec::new(), - ref_counts: Vec::new(), - owned: BitVec::default(), + metadata: ResourceMetadata::new(), temp: Vec::new(), @@ -397,26 +325,19 @@ impl TextureTracker { } fn debug_assert_in_bounds(&self, index: usize) { + self.metadata.debug_assert_in_bounds(index); + debug_assert!(index < self.start_set.simple.len()); debug_assert!(index < self.end_set.simple.len()); - debug_assert!(index < self.owned.len()); - debug_assert!(index < self.ref_counts.len()); - debug_assert!(index < self.epochs.len()); - - debug_assert!(if self.owned.get(index).unwrap() { - self.ref_counts[index].is_some() - } else { - true - }); - debug_assert!(if self.owned.get(index).unwrap() + debug_assert!(if self.metadata.owned.get(index).unwrap() && self.start_set.simple[index] == TextureUses::COMPLEX { self.start_set.complex.contains_key(&(index as u32)) } else { true }); - debug_assert!(if self.owned.get(index).unwrap() + debug_assert!(if self.metadata.owned.get(index).unwrap() && self.end_set.simple[index] == TextureUses::COMPLEX { self.end_set.complex.contains_key(&(index as u32)) @@ -430,10 +351,8 @@ impl TextureTracker { .simple .resize(size, TextureUses::UNINITIALIZED); self.end_set.simple.resize(size, TextureUses::UNINITIALIZED); - self.ref_counts.resize(size, None); - self.epochs.resize(size, u32::MAX); - resize_bitvec(&mut self.owned, size); + self.metadata.set_size(size); } fn allow_index(&mut self, index: usize) { @@ -443,11 +362,7 @@ impl TextureTracker { } pub fn used(&self) -> impl Iterator> + '_ { - self.debug_assert_in_bounds(self.owned.len() - 1); - iterate_bitvec_indices(&self.owned).map(move |index| { - let epoch = unsafe { *self.epochs.get_unchecked(index) }; - Valid(TextureId::zip(index as u32, epoch, A::VARIANT)) - }) + self.metadata.used() } pub fn drain(&mut self) -> Drain> { @@ -458,7 +373,7 @@ impl TextureTracker { let (index32, _, _) = id.0.unzip(); let index = index32 as usize; - self.ref_counts[index].as_ref().unwrap() + self.metadata.ref_counts[index].as_ref().unwrap() } pub fn init(&mut self, id: TextureId, ref_count: RefCount, usage: TextureUses) { @@ -470,12 +385,21 @@ impl TextureTracker { self.debug_assert_in_bounds(index); unsafe { - *self.start_set.simple.get_unchecked_mut(index) = usage; - *self.end_set.simple.get_unchecked_mut(index) = usage; - - *self.epochs.get_unchecked_mut(index) = epoch; - *self.ref_counts.get_unchecked_mut(index) = Some(ref_count); - self.owned.set(index, true); + state_combine( + None, + Some(&mut self.start_set), + &mut self.end_set, + &mut self.metadata, + index32, + index, + LayeredStateProvider::KnownSingle { state: usage }, + ResourceMetadataProvider::Direct { + epoch: epoch, + ref_count: Cow::Owned(ref_count), + }, + None, + ) + .unwrap(); } } @@ -484,20 +408,36 @@ impl TextureTracker { storage: &'a hub::Storage, TextureId>, id: TextureId, selector: TextureSelector, - new_usage: TextureUses, + new_state: TextureUses, ) -> Option<(&'a Texture, Option>)> { - let item = storage.get(id).ok()?; + let texture = storage.get(id).ok()?; - let (index32, _epoch, _) = id.unzip(); + let (index32, epoch, _) = id.unzip(); let index = index32 as usize; self.allow_index(index); self.debug_assert_in_bounds(index); - let _ = (storage, id, selector, new_usage); + unsafe { + state_combine( + Some(storage), + Some(&mut self.start_set), + &mut self.end_set, + &mut self.metadata, + index32, + index, + LayeredStateProvider::Selector { + selector, + state: new_state, + }, + ResourceMetadataProvider::Resource { epoch }, + Some(&mut self.temp), + ) + .unwrap(); + } - Some((item, self.temp.pop())) + Some((texture, self.temp.pop())) } pub fn change_states_tracker( @@ -510,17 +450,29 @@ impl TextureTracker { self.set_size(incoming_size); } - for index in iterate_bitvec_indices(&tracker.owned) { + for index in iterate_bitvec_indices(&tracker.metadata.owned) { + let index32 = index as u32; + + self.debug_assert_in_bounds(index); tracker.debug_assert_in_bounds(index); unsafe { - self.transition( - storage, - &tracker.start_set, - &tracker.ref_counts, - &tracker.epochs, + state_combine( + Some(storage), + Some(&mut self.start_set), + &mut self.end_set, + &mut self.metadata, + index32, index, + LayeredStateProvider::TextureSet { + set: &tracker.end_set, + }, + ResourceMetadataProvider::Indirect { + metadata: &tracker.metadata, + }, + Some(&mut self.temp), ) - }; + .unwrap(); + } } } @@ -534,11 +486,27 @@ impl TextureTracker { self.set_size(incoming_size); } - for index in iterate_bitvec_indices(&scope.owned) { + for index in iterate_bitvec_indices(&scope.metadata.owned) { + let index32 = index as u32; + + self.debug_assert_in_bounds(index); scope.debug_assert_in_bounds(index); unsafe { - self.transition(storage, &scope.set, &scope.ref_counts, &scope.epochs, index) - }; + state_combine( + Some(storage), + Some(&mut self.start_set), + &mut self.end_set, + &mut self.metadata, + index32, + index, + LayeredStateProvider::TextureSet { set: &scope.set }, + ResourceMetadataProvider::Indirect { + metadata: &scope.metadata, + }, + Some(&mut self.temp), + ) + .unwrap(); + } } } @@ -558,256 +526,500 @@ impl TextureTracker { let index = index32 as usize; scope.debug_assert_in_bounds(index); - if !scope.owned.get(index).unwrap_unchecked() { + if !scope.metadata.owned.get(index).unwrap_unchecked() { continue; } - self.transition(storage, &scope.set, &scope.ref_counts, &scope.epochs, index); + unsafe { + state_combine( + Some(storage), + Some(&mut self.start_set), + &mut self.end_set, + &mut self.metadata, + index32, + index, + LayeredStateProvider::TextureSet { set: &scope.set }, + ResourceMetadataProvider::Indirect { + metadata: &scope.metadata, + }, + Some(&mut self.temp), + ) + .unwrap(); + } - *scope.ref_counts.get_unchecked_mut(index) = None; - *scope.epochs.get_unchecked_mut(index) = u32::MAX; - scope.owned.set(index, false); + scope.metadata.reset(index); } } - unsafe fn transition( - &mut self, - storage: &hub::Storage, TextureId>, - incoming_set: &TextureStateSet, - incoming_ref_counts: &[Option], - incoming_epochs: &[Epoch], - index: usize, - ) { - // Note: both callees of this function call scope.debug_assert_in_bounds. - self.debug_assert_in_bounds(index); - - let old_tracked = self.owned.get(index).unwrap_unchecked(); - let old_state = *self.end_set.simple.get_unchecked(index); - let new_state = *incoming_set.simple.get_unchecked(index); + pub fn remove(&mut self, id: Valid) -> bool { + self.remove_inner(id, true) + } - match ( - old_tracked, - old_state == TextureUses::COMPLEX, - new_state == TextureUses::COMPLEX, - ) { - (false, _, false) => { - *self.start_set.simple.get_unchecked_mut(index) = new_state; - *self.end_set.simple.get_unchecked_mut(index) = new_state; + pub fn remove_abandoned(&mut self, id: Valid) -> bool { + self.remove_inner(id, false) + } - self.owned.set(index, true); + fn remove_inner(&mut self, id: Valid, force: bool) -> bool { + let (index32, epoch, _) = id.0.unzip(); + let index = index32 as usize; - let ref_count = incoming_ref_counts - .get_unchecked(index) - .clone() - .unwrap_unchecked(); - *self.ref_counts.get_unchecked_mut(index) = Some(ref_count); + if index > self.metadata.owned.len() { + return false; + } - let epoch = *incoming_epochs.get_unchecked(index); - *self.epochs.get_unchecked_mut(index) = epoch; - } - (false, _, true) => { - *self.start_set.simple.get_unchecked_mut(index) = TextureUses::COMPLEX; - *self.end_set.simple.get_unchecked_mut(index) = TextureUses::COMPLEX; + self.debug_assert_in_bounds(index); - let complex_state = incoming_set.complex.get(&(index as u32)).unwrap_unchecked(); - self.start_set - .complex - .insert(index as u32, complex_state.clone()); - self.end_set - .complex - .insert(index as u32, complex_state.clone()); + unsafe { + if self.metadata.owned.get(index).unwrap_unchecked() { + let existing_epoch = self.metadata.epochs.get_unchecked_mut(index); + let existing_ref_count = self.metadata.ref_counts.get_unchecked_mut(index); - self.owned.set(index, true); + if *existing_epoch == epoch + && existing_ref_count.as_mut().unwrap_unchecked().load() == 1 + { + self.start_set.complex.remove(&index32); + self.end_set.complex.remove(&index32); - let ref_count = incoming_ref_counts - .get_unchecked(index) - .clone() - .unwrap_unchecked(); - *self.ref_counts.get_unchecked_mut(index) = Some(ref_count); + self.metadata.reset(index); - let epoch = *incoming_epochs.get_unchecked(index); - *self.epochs.get_unchecked_mut(index) = epoch; - } - (true, false, false) => { - if skip_barrier(old_state, new_state) { - return; + return true; + } else if force { + assert_eq!(*existing_epoch, epoch); } + } + } - self.temp.push(PendingTransition { - id: index as u32, - selector: storage.get_unchecked(index as u32).full_range.clone(), - usage: old_state..new_state, - }); + false + } +} - *self.end_set.simple.get_unchecked_mut(index) = new_state; - } - (true, true, true) => { - self.transition_complex_to_complex(incoming_set, index); - } - (true, true, false) => { - self.transition_complex_to_simple(index, new_state); - } - (true, false, true) => { - self.transition_simple_to_complex(incoming_set, index, old_state); - } +enum Either { + Left(L), + Right(R), +} + +impl Iterator for Either +where + L: Iterator, + R: Iterator, +{ + type Item = D; + + fn next(&mut self) -> Option { + match self { + Either::Left(inner) => inner.next(), + Either::Right(inner) => inner.next(), } } +} - unsafe fn transition_complex_to_complex( - &mut self, - incoming_set: &TextureStateSet, - index: usize, - ) { - let old_complex = self - .end_set - .complex - .get_mut(&(index as u32)) - .unwrap_unchecked(); - let new_complex = incoming_set.complex.get(&(index as u32)).unwrap_unchecked(); - - let mut temp = Vec::new(); - debug_assert!(old_complex.mips.len() >= new_complex.mips.len()); - - for (mip_id, (mip_self, mip_other)) in old_complex - .mips - .iter_mut() - .zip(&new_complex.mips) - .enumerate() - { - let level = mip_id as u32; - temp.extend(mip_self.merge(mip_other, 0)); - - for (layers, states) in temp.drain(..) { - match states { - Range { - start: Some(start), - end: Some(end), - } => { - if skip_barrier(start, end) { - return; - } - // TODO: Can't satisfy clippy here unless we modify - // `TextureSelector` to use `std::ops::RangeBounds`. - #[allow(clippy::range_plus_one)] - let pending = PendingTransition { - id: index as u32, - selector: TextureSelector { - mips: level..level + 1, - layers: layers.clone(), - }, - usage: start..end, - }; - - self.temp.push(pending); - - for (_, state) in mip_self.isolate(&layers, end) { - *state = end; - } - } - _ => unreachable!(), - }; - } +enum SingleOrManyStates { + Single(S), + Many(M), +} + +enum LayeredStateProvider<'a> { + KnownSingle { + state: TextureUses, + }, + Selector { + selector: TextureSelector, + state: TextureUses, + }, + TextureSet { + set: &'a TextureStateSet, + }, +} +impl<'a> LayeredStateProvider<'a> { + fn from_option(selector: Option, state: TextureUses) -> Self { + match selector { + Some(selector) => Self::Selector { + selector: selector, + state, + }, + None => Self::KnownSingle { state }, } } - unsafe fn transition_complex_to_simple(&mut self, index: usize, new_state: TextureUses) { - let old_complex = self - .end_set - .complex - .remove(&(index as u32)) - .unwrap_unchecked(); + unsafe fn get_layers( + self, + texture: Option<&Texture>, + index32: u32, + index: usize, + ) -> SingleOrManyStates + 'a> + { + match self { + LayeredStateProvider::KnownSingle { state } => SingleOrManyStates::Single(state), + LayeredStateProvider::Selector { selector, state } => { + if texture.unwrap().full_range == selector { + SingleOrManyStates::Single(state) + } else { + SingleOrManyStates::Many(Either::Left(iter::once((selector, state)))) + } + } + LayeredStateProvider::TextureSet { set } => { + let new_state = *set.simple.get_unchecked(index); + + if new_state == TextureUses::COMPLEX { + let new_complex = set.complex.get(&index32).unwrap_unchecked(); - for (mip_index, mips) in old_complex.mips.into_iter().enumerate() { - let mip_index = mip_index as u32; - for (layer, old_state) in mips.into_iter() { - if skip_barrier(old_state, new_state) { - continue; + SingleOrManyStates::Many(Either::Right(new_complex.into_selector_state_iter())) + } else { + SingleOrManyStates::Single(new_state) } + } + } + } +} - #[allow(clippy::range_plus_one)] - self.temp.push(PendingTransition { - id: index as u32, - selector: TextureSelector { - mips: mip_index..mip_index + 1, - layers: layer, - }, - usage: old_state..new_state, - }) +enum ResourceMetadataProvider<'a, A: hub::HalApi> { + Direct { + epoch: Epoch, + ref_count: Cow<'a, RefCount>, + }, + Indirect { + metadata: &'a ResourceMetadata, + }, + Resource { + epoch: Epoch, + }, +} +impl ResourceMetadataProvider<'_, A> { + unsafe fn get_own(self, texture: Option<&Texture>, index: usize) -> (Epoch, RefCount) { + match self { + ResourceMetadataProvider::Direct { epoch, ref_count } => { + (epoch, ref_count.into_owned()) + } + ResourceMetadataProvider::Indirect { metadata } => ( + *metadata.epochs.get_unchecked(index), + metadata + .ref_counts + .get_unchecked(index) + .clone() + .unwrap_unchecked(), + ), + ResourceMetadataProvider::Resource { epoch } => { + (epoch, texture.unwrap().life_guard.add_ref()) + } + } + } + unsafe fn get_epoch(self, index: usize) -> Epoch { + match self { + ResourceMetadataProvider::Direct { epoch, .. } + | ResourceMetadataProvider::Resource { epoch, .. } => epoch, + ResourceMetadataProvider::Indirect { metadata } => { + *metadata.epochs.get_unchecked(index) } } + } +} - *self.end_set.simple.get_unchecked_mut(index) = new_state; +unsafe fn state_combine( + storage: Option<&hub::Storage, TextureId>>, + start_state: Option<&mut TextureStateSet>, + end_state: &mut TextureStateSet, + resource_metadata: &mut ResourceMetadata, + index32: u32, + index: usize, + state_provider: LayeredStateProvider<'_>, + metadata_provider: ResourceMetadataProvider<'_, A>, + barriers: Option<&mut Vec>>, +) -> Result<(), UsageConflict> { + let texture = storage.map(|s| s.get_unchecked(index32)); + + let currently_owned = resource_metadata.owned.get(index).unwrap_unchecked(); + + if currently_owned { + let old_simple = end_state.simple.get_unchecked_mut(index); + + let old_state = if *old_simple == TextureUses::COMPLEX { + SingleOrManyStates::Many(end_state.complex.remove(&index32).unwrap_unchecked()) + } else { + SingleOrManyStates::Single(*old_simple) + }; + + let new_state = set_state( + texture, + start_state, + end_state, + index32, + index, + state_provider, + barriers.is_none(), + ); + + merge_or_transition_state(texture.unwrap(), old_state, new_state, index32, barriers) + .map_err(|partial| { + let epoch = metadata_provider.get_epoch(index); + UsageConflict::from_texture( + TextureId::zip(index32, epoch, A::VARIANT), + partial.selector, + partial.current_state, + partial.new_state, + ) + })?; + } else { + set_state( + texture, + start_state, + end_state, + index32, + index, + state_provider, + false, + ); + + let (epoch, ref_count) = metadata_provider.get_own(texture, index); + + resource_metadata.owned.set(index, true); + *resource_metadata.epochs.get_unchecked_mut(index) = epoch; + *resource_metadata.ref_counts.get_unchecked_mut(index) = Some(ref_count); } - unsafe fn transition_simple_to_complex( - &mut self, - incoming_set: &TextureStateSet, - index: usize, - old_state: TextureUses, - ) { - let new_complex = incoming_set.complex.get(&(index as u32)).unwrap_unchecked(); + Ok(()) +} - for (mip_index, mips) in new_complex.mips.iter().enumerate() { - let mip_index = mip_index as u32; - for &(ref layer, new_state) in mips.iter() { - if skip_barrier(old_state, new_state) { - continue; - } +unsafe fn set_state<'a, A: hub::HalApi>( + texture: Option<&Texture>, + start_state: Option<&mut TextureStateSet>, + end_state: &'a mut TextureStateSet, + index32: u32, + index: usize, + state_provider: LayeredStateProvider, + merging_and_existing: bool, +) -> SingleOrManyStates<&'a mut TextureUses, &'a mut ComplexTextureState> { + match state_provider.get_layers(texture, index32, index) { + SingleOrManyStates::Single(state) => { + let reference = end_state.simple.get_unchecked_mut(index); + + if merging_and_existing && *reference == TextureUses::COMPLEX { + let full_range = texture.unwrap().full_range.clone(); + + let complex = ComplexTextureState::from_selector_state_iter( + full_range.clone(), + iter::once((full_range, state)), + ); + + *reference = state; + let new_state = end_state + .complex + .entry(index32) + .insert_entry(complex) + .into_mut(); - #[allow(clippy::range_plus_one)] - self.temp.push(PendingTransition { - id: index as u32, - selector: TextureSelector { - mips: mip_index..mip_index + 1, - layers: layer.clone(), - }, - usage: old_state..new_state, - }) + return SingleOrManyStates::Many(new_state); } + + if let Some(start_state) = start_state { + *start_state.simple.get_unchecked_mut(index) = state; + } + *reference = state; + SingleOrManyStates::Single(reference) } + SingleOrManyStates::Many(state_iter) => { + let full_range = texture.unwrap().full_range.clone(); - self.end_set - .complex - .insert(index as u32, new_complex.clone()); - *self.end_set.simple.get_unchecked_mut(index) = TextureUses::COMPLEX; - } + let complex = ComplexTextureState::from_selector_state_iter(full_range, state_iter); - pub fn remove(&mut self, id: Valid) -> bool { - self.remove_inner(id, true) + if let Some(start_state) = start_state { + *start_state.simple.get_unchecked_mut(index) = TextureUses::COMPLEX; + start_state.complex.insert(index32, complex.clone()); + } + *end_state.simple.get_unchecked_mut(index) = TextureUses::COMPLEX; + let new_state = end_state + .complex + .entry(index32) + .insert_entry(complex) + .into_mut(); + + SingleOrManyStates::Many(new_state) + } } +} - pub fn remove_abandoned(&mut self, id: Valid) -> bool { - self.remove_inner(id, false) - } +struct PartialUsageConflict { + selector: TextureSelector, + current_state: TextureUses, + new_state: TextureUses, +} - fn remove_inner(&mut self, id: Valid, force: bool) -> bool { - let (index32, epoch, _) = id.0.unzip(); - let index = index32 as usize; +fn merge_or_transition_state( + texture: &Texture, + old_state: SingleOrManyStates, + new_state: SingleOrManyStates<&mut TextureUses, &mut ComplexTextureState>, + index32: u32, + barriers: Option<&mut Vec>>, +) -> Result<(), PartialUsageConflict> { + match (old_state, new_state) { + (SingleOrManyStates::Single(old_simple), SingleOrManyStates::Single(new_simple)) => { + match barriers { + Some(barriers) => { + if skip_barrier(old_simple, *new_simple) { + return Ok(()); + } - if index > self.owned.len() { - return false; - } + #[allow(clippy::range_plus_one)] + barriers.push(PendingTransition { + id: index32, + selector: texture.full_range.clone(), + usage: old_simple..*new_simple, + }) + } + None => { + let merged_state = old_simple | *new_simple; - self.debug_assert_in_bounds(index); + if invalid_resource_state(merged_state) { + return Err(PartialUsageConflict { + selector: texture.full_range.clone(), + current_state: old_simple, + new_state: *new_simple, + }); + } - unsafe { - if self.owned.get(index).unwrap_unchecked() { - let existing_epoch = self.epochs.get_unchecked_mut(index); - let existing_ref_count = self.ref_counts.get_unchecked_mut(index); + *new_simple = merged_state; + } + } - if *existing_epoch == epoch - && existing_ref_count.as_mut().unwrap_unchecked().load() == 1 - { - self.owned.set(index, false); - *existing_epoch = u32::MAX; - *existing_ref_count = None; + Ok(()) + } + (SingleOrManyStates::Single(old_simple), SingleOrManyStates::Many(new_many)) => { + for (mip_index, mips) in new_many.mips.iter_mut().enumerate() { + let mip_index = mip_index as u32; + for (layer, new_state) in mips.iter_mut() { + match barriers { + Some(&mut ref mut barriers) => { + if skip_barrier(old_simple, *new_state) { + return Ok(()); + } + + #[allow(clippy::range_plus_one)] + barriers.push(PendingTransition { + id: index32, + selector: TextureSelector { + mips: mip_index..mip_index + 1, + layers: layer.clone(), + }, + usage: old_simple..*new_state, + }) + } + None => { + let merged_state = old_simple | *new_state; + + if invalid_resource_state(merged_state) { + return Err(PartialUsageConflict { + selector: TextureSelector { + mips: mip_index..mip_index + 1, + layers: layer.clone(), + }, + current_state: old_simple, + new_state: *new_state, + }); + } + + *new_state = merged_state; + } + } + } - return true; - } else if force { - assert_eq!(*existing_epoch, epoch); + mips.coalesce(); + } + + Ok(()) + } + (SingleOrManyStates::Many(old_many), SingleOrManyStates::Single(new_single)) => { + for (mip_index, mips) in old_many.mips.iter().enumerate() { + let mip_index = mip_index as u32; + for &(ref layer, old_state) in mips.iter() { + match barriers { + Some(&mut ref mut barriers) => { + if skip_barrier(old_state, *new_single) { + return Ok(()); + } + + #[allow(clippy::range_plus_one)] + barriers.push(PendingTransition { + id: index32, + selector: TextureSelector { + mips: mip_index..mip_index + 1, + layers: layer.clone(), + }, + usage: old_state..*new_single, + }) + } + None => { + unreachable!(); + } + } } } + + Ok(()) } + (SingleOrManyStates::Many(mut old_complex), SingleOrManyStates::Many(new_complex)) => { + let mut temp = Vec::new(); + debug_assert!(old_complex.mips.len() >= new_complex.mips.len()); + + for (mip_id, (mip_old, mip_new)) in old_complex + .mips + .iter_mut() + .zip(&mut new_complex.mips) + .enumerate() + { + let level = mip_id as u32; + temp.extend(mip_old.merge(mip_new, 0)); + + for (layers, states) in temp.drain(..) { + match states { + Range { + start: Some(start), + end: Some(end), + } => { + match barriers { + Some(&mut ref mut barriers) => { + if skip_barrier(start, end) { + return Ok(()); + } + // TODO: Can't satisfy clippy here unless we modify + // `TextureSelector` to use `std::ops::RangeBounds`. + #[allow(clippy::range_plus_one)] + let pending = PendingTransition { + id: index32, + selector: TextureSelector { + mips: level..level + 1, + layers: layers.clone(), + }, + usage: start..end, + }; + + barriers.push(pending); + } + None => { + let merged_state = start | end; + + if invalid_resource_state(merged_state) { + return Err(PartialUsageConflict { + selector: TextureSelector { + mips: level..level + 1, + layers: layers.clone(), + }, + current_state: start, + new_state: end, + }); + } + + for (_, state) in mip_new.isolate(&layers, end) { + *state = merged_state; + } + } + } + } + _ => unreachable!(), + }; + } - false + mip_new.coalesce(); + } + + Ok(()) + } } } From db41d9b51ecce66d8c47b8505559f9a7ddb0616b Mon Sep 17 00:00:00 2001 From: Connor Fitzgerald Date: Mon, 9 May 2022 15:23:14 -0400 Subject: [PATCH 16/35] temp15 - It verks! kinda... --- wgpu-core/src/device/queue.rs | 4 ++++ wgpu-core/src/track/buffer.rs | 8 +++++-- wgpu-core/src/track/mod.rs | 4 +++- wgpu-core/src/track/stateless.rs | 35 +++++++++++++++++++++++-------- wgpu-core/src/track/texture.rs | 36 ++++++++++++++++---------------- 5 files changed, 57 insertions(+), 30 deletions(-) diff --git a/wgpu-core/src/device/queue.rs b/wgpu-core/src/device/queue.rs index d1861a7d81..d5699edd6d 100644 --- a/wgpu-core/src/device/queue.rs +++ b/wgpu-core/src/device/queue.rs @@ -617,6 +617,8 @@ impl Global { //Note: locking the trackers has to be done after the storages let mut trackers = device.trackers.lock(); + used_surface_textures.set_size(texture_guard.len()); + //TODO: if multiple command buffers are submitted, we can re-use the last // native command buffer of the previous chain instead of always creating // a temporary one, since the chains are not finished. @@ -833,6 +835,8 @@ impl Global { let (_, mut token) = hub.buffers.read(&mut token); // skip token let (mut texture_guard, _) = hub.textures.write(&mut token); + used_surface_textures.set_size(texture_guard.len()); + for &id in pending_writes.dst_textures.iter() { let texture = texture_guard.get_mut(id).unwrap(); match texture.inner { diff --git a/wgpu-core/src/track/buffer.rs b/wgpu-core/src/track/buffer.rs index 910be34142..97dce8e984 100644 --- a/wgpu-core/src/track/buffer.rs +++ b/wgpu-core/src/track/buffer.rs @@ -119,7 +119,9 @@ impl BufferUsageScope { } pub fn used(&self) -> impl Iterator> + '_ { - self.debug_assert_in_bounds(self.owned.len() - 1); + if !self.owned.is_empty() { + self.debug_assert_in_bounds(self.owned.len() - 1) + }; iterate_bitvec_indices(&self.owned).map(move |index| { let epoch = unsafe { *self.epochs.get_unchecked(index) }; Valid(BufferId::zip(index as u32, epoch, A::VARIANT)) @@ -289,7 +291,9 @@ impl BufferTracker { } pub fn used(&self) -> impl Iterator> + '_ { - self.debug_assert_in_bounds(self.owned.len() - 1); + if !self.owned.is_empty() { + self.debug_assert_in_bounds(self.owned.len() - 1) + }; iterate_bitvec_indices(&self.owned).map(move |index| { let epoch = unsafe { *self.epochs.get_unchecked(index) }; Valid(BufferId::zip(index as u32, epoch, A::VARIANT)) diff --git a/wgpu-core/src/track/mod.rs b/wgpu-core/src/track/mod.rs index 3d87b99227..ddebdd0393 100644 --- a/wgpu-core/src/track/mod.rs +++ b/wgpu-core/src/track/mod.rs @@ -246,7 +246,9 @@ impl ResourceMetadata { } fn used(&self) -> impl Iterator> + '_ { - self.debug_assert_in_bounds(self.owned.len() - 1); + if !self.owned.is_empty() { + self.debug_assert_in_bounds(self.owned.len() - 1) + }; iterate_bitvec_indices(&self.owned).map(move |index| { let epoch = unsafe { *self.epochs.get_unchecked(index) }; id::Valid(Id::zip(index as u32, epoch, A::VARIANT)) diff --git a/wgpu-core/src/track/stateless.rs b/wgpu-core/src/track/stateless.rs index 70b5e353b8..96015b27c2 100644 --- a/wgpu-core/src/track/stateless.rs +++ b/wgpu-core/src/track/stateless.rs @@ -84,36 +84,52 @@ impl StatelessTracker { } pub fn used(&self) -> impl Iterator> + '_ { - self.debug_assert_in_bounds(self.owned.len() - 1); + if !self.owned.is_empty() { + self.debug_assert_in_bounds(self.owned.len() - 1) + }; iterate_bitvec_indices(&self.owned).map(move |index| { let epoch = unsafe { *self.epochs.get_unchecked(index) }; Valid(Id::zip(index as u32, epoch, A::VARIANT)) }) } - pub unsafe fn init(&mut self, id: Valid, ref_count: RefCount) { + fn allow_index(&mut self, index: usize) { + if index >= self.owned.len() { + self.set_size(index + 1); + } + } + + pub fn init(&mut self, id: Valid, ref_count: RefCount) { let (index32, epoch, _) = id.0.unzip(); let index = index32 as usize; + self.allow_index(index); + self.debug_assert_in_bounds(index); - *self.epochs.get_unchecked_mut(index) = epoch; - *self.ref_counts.get_unchecked_mut(index) = Some(ref_count); - self.owned.set(index, true); + unsafe { + *self.epochs.get_unchecked_mut(index) = epoch; + *self.ref_counts.get_unchecked_mut(index) = Some(ref_count); + self.owned.set(index, true); + } } /// Requires set_size to be called - pub unsafe fn extend<'a>(&mut self, storage: &'a hub::Storage, id: Id) -> Option<&'a T> { + pub fn extend<'a>(&mut self, storage: &'a hub::Storage, id: Id) -> Option<&'a T> { let item = storage.get(id).ok()?; let (index32, epoch, _) = id.unzip(); let index = index32 as usize; + self.allow_index(index); + self.debug_assert_in_bounds(index); - *self.epochs.get_unchecked_mut(index) = epoch; - *self.ref_counts.get_unchecked_mut(index) = Some(item.life_guard().add_ref()); - self.owned.set(index, true); + unsafe { + *self.epochs.get_unchecked_mut(index) = epoch; + *self.ref_counts.get_unchecked_mut(index) = Some(item.life_guard().add_ref()); + self.owned.set(index, true); + } Some(item) } @@ -126,6 +142,7 @@ impl StatelessTracker { for index in iterate_bitvec_indices(&other.owned) { self.debug_assert_in_bounds(index); + other.debug_assert_in_bounds(index); unsafe { let previously_owned = self.owned.get(index).unwrap_unchecked(); diff --git a/wgpu-core/src/track/texture.rs b/wgpu-core/src/track/texture.rs index 2d6f16885d..47597e6279 100644 --- a/wgpu-core/src/track/texture.rs +++ b/wgpu-core/src/track/texture.rs @@ -4,8 +4,8 @@ use crate::{ id::{TextureId, TypedId, Valid}, resource::Texture, track::{ - invalid_resource_state, iterate_bitvec_indices, skip_barrier, - ResourceMetadata, ResourceUses, UsageConflict, + invalid_resource_state, iterate_bitvec_indices, skip_barrier, ResourceMetadata, + ResourceUses, UsageConflict, }, Epoch, RefCount, }; @@ -280,6 +280,8 @@ impl TextureUsageScope { let (index32, epoch, _) = id.0.unzip(); let index = index32 as usize; + self.debug_assert_in_bounds(index); + state_combine( Some(storage), None, @@ -529,22 +531,20 @@ impl TextureTracker { if !scope.metadata.owned.get(index).unwrap_unchecked() { continue; } - unsafe { - state_combine( - Some(storage), - Some(&mut self.start_set), - &mut self.end_set, - &mut self.metadata, - index32, - index, - LayeredStateProvider::TextureSet { set: &scope.set }, - ResourceMetadataProvider::Indirect { - metadata: &scope.metadata, - }, - Some(&mut self.temp), - ) - .unwrap(); - } + state_combine( + Some(storage), + Some(&mut self.start_set), + &mut self.end_set, + &mut self.metadata, + index32, + index, + LayeredStateProvider::TextureSet { set: &scope.set }, + ResourceMetadataProvider::Indirect { + metadata: &scope.metadata, + }, + Some(&mut self.temp), + ) + .unwrap(); scope.metadata.reset(index); } From 79736f5464b1f13b84d437134da4467c9124cf5c Mon Sep 17 00:00:00 2001 From: Connor Fitzgerald Date: Thu, 12 May 2022 02:21:40 -0400 Subject: [PATCH 17/35] temp16 - it more verks! --- wgpu-core/src/command/render.rs | 5 +++-- wgpu-core/src/present.rs | 1 + wgpu-core/src/track/buffer.rs | 2 +- wgpu-core/src/track/texture.rs | 37 ++++++++++++++++++++++++++------- 4 files changed, 34 insertions(+), 11 deletions(-) diff --git a/wgpu-core/src/command/render.rs b/wgpu-core/src/command/render.rs index 07e83f70cc..4182795c30 100644 --- a/wgpu-core/src/command/render.rs +++ b/wgpu-core/src/command/render.rs @@ -973,10 +973,11 @@ impl<'a, A: HalApi> RenderPassInfo<'a, A> { unsafe { self.usage_scope .textures - .extend( + .extend_refcount( &*texture_guard, - ra.texture_id.value.0, + ra.texture_id.value, Some(ra.selector.clone()), + &ra.texture_id.ref_count, ra.usage, ) .map_err(UsageConflict::from)? diff --git a/wgpu-core/src/present.rs b/wgpu-core/src/present.rs index d1717bf9b0..6e62294d4e 100644 --- a/wgpu-core/src/present.rs +++ b/wgpu-core/src/present.rs @@ -266,6 +266,7 @@ impl Global { // The texture ID got added to the device tracker by `submit()`, // and now we are moving it away. + log::debug!("Removing swapchain texture {:?} from the device tracker", texture_id.value); device.trackers.lock().textures.remove(texture_id.value); let (texture, _) = hub.textures.unregister(texture_id.value.0, &mut token); diff --git a/wgpu-core/src/track/buffer.rs b/wgpu-core/src/track/buffer.rs index 97dce8e984..95527e1cbc 100644 --- a/wgpu-core/src/track/buffer.rs +++ b/wgpu-core/src/track/buffer.rs @@ -25,7 +25,7 @@ impl ResourceUses for BufferUses { } fn all_ordered(self) -> bool { - self.contains(Self::ORDERED) + Self::ORDERED.contains(self) } fn any_exclusive(self) -> bool { diff --git a/wgpu-core/src/track/texture.rs b/wgpu-core/src/track/texture.rs index 47597e6279..5a7560055b 100644 --- a/wgpu-core/src/track/texture.rs +++ b/wgpu-core/src/track/texture.rs @@ -33,7 +33,7 @@ impl ResourceUses for TextureUses { } fn all_ordered(self) -> bool { - self.contains(Self::ORDERED) + Self::ORDERED.contains(self) } fn any_exclusive(self) -> bool { @@ -551,14 +551,33 @@ impl TextureTracker { } pub fn remove(&mut self, id: Valid) -> bool { - self.remove_inner(id, true) - } + let (index32, epoch, _) = id.0.unzip(); + let index = index32 as usize; - pub fn remove_abandoned(&mut self, id: Valid) -> bool { - self.remove_inner(id, false) + if index > self.metadata.owned.len() { + return false; + } + + self.debug_assert_in_bounds(index); + + unsafe { + if self.metadata.owned.get(index).unwrap_unchecked() { + let existing_epoch = *self.metadata.epochs.get_unchecked_mut(index); + assert_eq!(existing_epoch, epoch); + + self.start_set.complex.remove(&index32); + self.end_set.complex.remove(&index32); + + self.metadata.reset(index); + + return true; + } + } + + false } - fn remove_inner(&mut self, id: Valid, force: bool) -> bool { + pub fn remove_abandoned(&mut self, id: Valid) -> bool { let (index32, epoch, _) = id.0.unzip(); let index = index32 as usize; @@ -582,8 +601,6 @@ impl TextureTracker { self.metadata.reset(index); return true; - } else if force { - assert_eq!(*existing_epoch, epoch); } } } @@ -640,6 +657,7 @@ impl<'a> LayeredStateProvider<'a> { } } + #[inline(always)] unsafe fn get_layers( self, texture: Option<&Texture>, @@ -713,6 +731,7 @@ impl ResourceMetadataProvider<'_, A> { } } +#[inline(always)] unsafe fn state_combine( storage: Option<&hub::Storage, TextureId>>, start_state: Option<&mut TextureStateSet>, @@ -778,6 +797,7 @@ unsafe fn state_combine( Ok(()) } +#[inline(always)] unsafe fn set_state<'a, A: hub::HalApi>( texture: Option<&Texture>, start_state: Option<&mut TextureStateSet>, @@ -842,6 +862,7 @@ struct PartialUsageConflict { new_state: TextureUses, } +#[inline(always)] fn merge_or_transition_state( texture: &Texture, old_state: SingleOrManyStates, From 9015fbbae83dda8562f983b8eaca6d7f3f26107c Mon Sep 17 00:00:00 2001 From: Connor Fitzgerald Date: Thu, 12 May 2022 11:30:43 -0400 Subject: [PATCH 18/35] temp17 - debugging and unit tests --- wgpu-core/src/id.rs | 4 +- wgpu-core/src/track/buffer.rs | 4 + wgpu-core/src/track/mod.rs | 7 +- wgpu-core/src/track/range.rs | 443 ++++++++++++++++----------------- wgpu-core/src/track/texture.rs | 357 +++++++++++++++++++------- wgpu-hal/src/lib.rs | 2 +- 6 files changed, 494 insertions(+), 323 deletions(-) diff --git a/wgpu-core/src/id.rs b/wgpu-core/src/id.rs index 5c91b412b1..e7a19b229c 100644 --- a/wgpu-core/src/id.rs +++ b/wgpu-core/src/id.rs @@ -65,8 +65,8 @@ impl From for Id { impl Id { #[cfg(test)] - pub(crate) fn dummy() -> Valid { - Valid(Id(NonZeroId::new(1).unwrap(), PhantomData)) + pub(crate) fn dummy(index: u32) -> Valid { + Valid(Id::zip(index, 1, Backend::Empty)) } pub fn backend(self) -> Backend { diff --git a/wgpu-core/src/track/buffer.rs b/wgpu-core/src/track/buffer.rs index 95527e1cbc..87c353834a 100644 --- a/wgpu-core/src/track/buffer.rs +++ b/wgpu-core/src/track/buffer.rs @@ -31,6 +31,10 @@ impl ResourceUses for BufferUses { fn any_exclusive(self) -> bool { self.intersects(Self::EXCLUSIVE) } + + fn uninit(self) -> bool { + false + } } pub(crate) struct BufferBindGroupState { diff --git a/wgpu-core/src/track/mod.rs b/wgpu-core/src/track/mod.rs index ddebdd0393..d8d37a4b41 100644 --- a/wgpu-core/src/track/mod.rs +++ b/wgpu-core/src/track/mod.rs @@ -79,6 +79,7 @@ pub trait ResourceUses: fn bits(self) -> u16; fn all_ordered(self) -> bool; fn any_exclusive(self) -> bool; + fn uninit(self) -> bool; } fn invalid_resource_state(state: T) -> bool { @@ -90,7 +91,7 @@ fn invalid_resource_state(state: T) -> bool { fn skip_barrier(old_state: T, new_state: T) -> bool { // If the state didn't change and all the usages are ordered, the hardware // will guarentee the order of accesses, so we do not need to issue a barrier at all - old_state == new_state && old_state.all_ordered() + old_state == new_state && (old_state.all_ordered() || old_state.uninit()) } fn resize_bitvec(vec: &mut BitVec, size: usize) { @@ -126,7 +127,7 @@ fn iterate_bitvec_indices(ownership: &BitVec) -> impl Iterator { current_state: T, new_state: T, diff --git a/wgpu-core/src/track/range.rs b/wgpu-core/src/track/range.rs index a64821ec79..8551a1dc11 100644 --- a/wgpu-core/src/track/range.rs +++ b/wgpu-core/src/track/range.rs @@ -2,7 +2,13 @@ //TODO: consider getting rid of it. use smallvec::SmallVec; -use std::{cmp::Ordering, fmt::Debug, iter, ops::Range, slice::Iter}; +use std::{ + cmp::Ordering, + fmt::Debug, + iter, + ops::Range, + slice::{Iter, IterMut}, +}; /// Structure that keeps track of a I -> T mapping, /// optimized for a case where keys of the same values @@ -193,256 +199,235 @@ impl RangedStates { /// /// Each range in the returned iterator is a subset of a range in either /// `self` or `other`, and the value returned as a `Range` from `self` to `other`. - pub fn merge<'a>(&'a self, other: &'a Self, base: I) -> Merge<'a, I, T> { + pub fn merge<'a>(&'a self, other: &'a mut Self, base: I) -> Merge<'a, I, T> { Merge { base, - sa: self.ranges.iter().peekable(), - sb: other.ranges.iter().peekable(), + max: self.ranges.last().unwrap().0.end, + index_a: 0, + slice_a: &self.ranges, + index_b: 0, + slice_b: &mut other.ranges, } } } +pub(crate) struct MergeItem<'a, T>(pub T, pub &'a mut T); + /// A custom iterator that goes through two `RangedStates` and process a merge. #[derive(Debug)] pub(crate) struct Merge<'a, I, T> { base: I, - sa: iter::Peekable, T)>>, - sb: iter::Peekable, T)>>, + max: I, + index_a: usize, + slice_a: &'a [(Range, T)], + index_b: usize, + slice_b: &'a mut [(Range, T)], } -impl<'a, I: Copy + Debug + Ord, T: Copy + Debug> Iterator for Merge<'a, I, T> { - type Item = (Range, Range>); +impl<'a, I: Ord + Copy, T: Copy + Debug> Iterator for Merge<'a, I, T> { + type Item = (Range, MergeItem<'a, T>); fn next(&mut self) -> Option { - match (self.sa.peek(), self.sb.peek()) { - // we have both streams - (Some(&&(ref ra, va)), Some(&&(ref rb, vb))) => { - let (range, usage) = if ra.start < self.base { - // in the middle of the left stream - let (end, end_value) = if self.base == rb.start { - // right stream is starting - debug_assert!(self.base < ra.end); - (rb.end, Some(vb)) - } else { - // right hasn't started yet - debug_assert!(self.base < rb.start); - (rb.start, None) - }; - (self.base..ra.end.min(end), Some(va)..end_value) - } else if rb.start < self.base { - // in the middle of the right stream - let (end, start_value) = if self.base == ra.start { - // left stream is starting - debug_assert!(self.base < rb.end); - (ra.end, Some(va)) - } else { - // left hasn't started yet - debug_assert!(self.base < ra.start); - (ra.start, None) - }; - (self.base..rb.end.min(end), start_value..Some(vb)) - } else { - // no active streams - match ra.start.cmp(&rb.start) { - // both are starting - Ordering::Equal => (ra.start..ra.end.min(rb.end), Some(va)..Some(vb)), - // only left is starting - Ordering::Less => (ra.start..rb.start.min(ra.end), Some(va)..None), - // only right is starting - Ordering::Greater => (rb.start..ra.start.min(rb.end), None..Some(vb)), - } - }; - self.base = range.end; - if ra.end == range.end { - let _ = self.sa.next(); - } - if rb.end == range.end { - let _ = self.sb.next(); - } - Some((range, usage)) - } - // only right stream - (None, Some(&&(ref rb, vb))) => { - let range = self.base.max(rb.start)..rb.end; - self.base = rb.end; - let _ = self.sb.next(); - Some((range, None..Some(vb))) - } - // only left stream - (Some(&&(ref ra, va)), None) => { - let range = self.base.max(ra.start)..ra.end; - self.base = ra.end; - let _ = self.sa.next(); - Some((range, Some(va)..None)) - } - // done - (None, None) => None, + if self.base >= self.max { + return None; } - } -} -#[cfg(test)] -mod test { - //TODO: randomized/fuzzy testing - use super::RangedStates; - use std::{fmt::Debug, ops::Range}; - - fn easy_merge( - ra: &[(Range, T)], - rb: &[(Range, T)], - ) -> Vec<(Range, Range>)> { - RangedStates::from_slice(ra) - .merge(&RangedStates::from_slice(rb), 0) - .collect() - } + let a_end = self.slice_a[self.index_a].0.end; + let b_end = self.slice_b[self.index_b].0.end; - #[test] - fn sane_good() { - let rs = RangedStates::from_slice(&[(1..4, 9u8), (4..5, 9)]); - rs.check_sanity(); - } + let a_item = self.slice_a[self.index_a].1; + let b_item = unsafe { &mut (*self.slice_b.as_mut_ptr().add(self.index_b)).1 }; - #[test] - #[should_panic] - fn sane_empty() { - let rs = RangedStates::from_slice(&[(1..4, 9u8), (5..5, 9)]); - rs.check_sanity(); - } + match a_end.cmp(&b_end) { + Ordering::Less => { + self.index_a += 1; - #[test] - #[should_panic] - fn sane_intersect() { - let rs = RangedStates::from_slice(&[(1..4, 9u8), (3..5, 9)]); - rs.check_sanity(); - } + let range = self.base..a_end; + self.base = a_end; - #[test] - fn coalesce() { - let mut rs = RangedStates::from_slice(&[(1..4, 9u8), (4..5, 9), (5..7, 1), (8..9, 1)]); - rs.coalesce(); - rs.check_sanity(); - assert_eq!(rs.ranges.as_slice(), &[(1..5, 9), (5..7, 1), (8..9, 1),]); - } - - #[test] - fn query() { - let rs = RangedStates::from_slice(&[(1..4, 1u8), (5..7, 2)]); - assert_eq!(rs.query(&(0..1), |v| *v), None); - assert_eq!(rs.query(&(1..3), |v| *v), Some(Ok(1))); - assert_eq!(rs.query(&(1..6), |v| *v), Some(Err(()))); - } - - #[test] - fn isolate() { - let rs = RangedStates::from_slice(&[(1..4, 9u8), (4..5, 9), (5..7, 1), (8..9, 1)]); - assert_eq!(&rs.sanely_isolated(4..5, 0), &[(4..5, 9u8),]); - assert_eq!( - &rs.sanely_isolated(0..6, 0), - &[(0..1, 0), (1..4, 9u8), (4..5, 9), (5..6, 1),] - ); - assert_eq!(&rs.sanely_isolated(8..10, 1), &[(8..9, 1), (9..10, 1),]); - assert_eq!( - &rs.sanely_isolated(6..9, 0), - &[(6..7, 1), (7..8, 0), (8..9, 1),] - ); - } - - #[test] - fn merge_same() { - assert_eq!( - &easy_merge(&[(1..4, 0u8),], &[(1..4, 2u8),],), - &[(1..4, Some(0)..Some(2)),] - ); - } - - #[test] - fn merge_empty() { - assert_eq!( - &easy_merge(&[(1..2, 0u8),], &[],), - &[(1..2, Some(0)..None),] - ); - assert_eq!( - &easy_merge(&[], &[(3..4, 1u8),],), - &[(3..4, None..Some(1)),] - ); - } + Some((range, MergeItem(a_item, b_item))) + } + Ordering::Equal => { + self.index_a += 1; + self.index_b += 1; - #[test] - fn merge_separate() { - assert_eq!( - &easy_merge(&[(1..2, 0u8), (5..6, 1u8),], &[(2..4, 2u8),],), - &[ - (1..2, Some(0)..None), - (2..4, None..Some(2)), - (5..6, Some(1)..None), - ] - ); - } + let range = self.base..a_end; + self.base = a_end; - #[test] - fn merge_subset() { - assert_eq!( - &easy_merge(&[(1..6, 0u8),], &[(2..4, 2u8),],), - &[ - (1..2, Some(0)..None), - (2..4, Some(0)..Some(2)), - (4..6, Some(0)..None), - ] - ); - assert_eq!( - &easy_merge(&[(2..4, 0u8),], &[(1..4, 2u8),],), - &[(1..2, None..Some(2)), (2..4, Some(0)..Some(2)),] - ); - } + Some((range, MergeItem(a_item, b_item))) + } + Ordering::Greater => { + self.index_b += 1; - #[test] - fn merge_all() { - assert_eq!( - &easy_merge(&[(1..4, 0u8), (5..8, 1u8),], &[(2..6, 2u8), (7..9, 3u8),],), - &[ - (1..2, Some(0)..None), - (2..4, Some(0)..Some(2)), - (4..5, None..Some(2)), - (5..6, Some(1)..Some(2)), - (6..7, Some(1)..None), - (7..8, Some(1)..Some(3)), - (8..9, None..Some(3)), - ] - ); - } + let range = self.base..b_end; + self.base = b_end; - #[test] - fn merge_complex() { - assert_eq!( - &easy_merge( - &[ - (0..8, 0u8), - (8..9, 1), - (9..16, 2), - (16..17, 3), - (17..118, 4), - (118..119, 5), - (119..124, 6), - (124..125, 7), - (125..512, 8), - ], - &[(15..16, 10u8), (51..52, 11), (126..127, 12),], - ), - &[ - (0..8, Some(0)..None), - (8..9, Some(1)..None), - (9..15, Some(2)..None), - (15..16, Some(2)..Some(10)), - (16..17, Some(3)..None), - (17..51, Some(4)..None), - (51..52, Some(4)..Some(11)), - (52..118, Some(4)..None), - (118..119, Some(5)..None), - (119..124, Some(6)..None), - (124..125, Some(7)..None), - (125..126, Some(8)..None), - (126..127, Some(8)..Some(12)), - (127..512, Some(8)..None), - ] - ); + Some((range, MergeItem(a_item, b_item))) + } + } } } + +// #[cfg(test)] +// mod test { +// //TODO: randomized/fuzzy testing +// use super::RangedStates; +// use std::{fmt::Debug, ops::Range}; + +// fn easy_merge( +// ra: &[(Range, T)], +// rb: &[(Range, T)], +// ) -> Vec<(Range, Range>)> { +// todo!() +// } + +// #[test] +// fn sane_good() { +// let rs = RangedStates::from_slice(&[(1..4, 9u8), (4..5, 9)]); +// rs.check_sanity(); +// } + +// #[test] +// #[should_panic] +// fn sane_empty() { +// let rs = RangedStates::from_slice(&[(1..4, 9u8), (5..5, 9)]); +// rs.check_sanity(); +// } + +// #[test] +// #[should_panic] +// fn sane_intersect() { +// let rs = RangedStates::from_slice(&[(1..4, 9u8), (3..5, 9)]); +// rs.check_sanity(); +// } + +// #[test] +// fn coalesce() { +// let mut rs = RangedStates::from_slice(&[(1..4, 9u8), (4..5, 9), (5..7, 1), (8..9, 1)]); +// rs.coalesce(); +// rs.check_sanity(); +// assert_eq!(rs.ranges.as_slice(), &[(1..5, 9), (5..7, 1), (8..9, 1),]); +// } + +// #[test] +// fn query() { +// let rs = RangedStates::from_slice(&[(1..4, 1u8), (5..7, 2)]); +// assert_eq!(rs.query(&(0..1), |v| *v), None); +// assert_eq!(rs.query(&(1..3), |v| *v), Some(Ok(1))); +// assert_eq!(rs.query(&(1..6), |v| *v), Some(Err(()))); +// } + +// #[test] +// fn isolate() { +// let rs = RangedStates::from_slice(&[(1..4, 9u8), (4..5, 9), (5..7, 1), (8..9, 1)]); +// assert_eq!(&rs.sanely_isolated(4..5, 0), &[(4..5, 9u8),]); +// assert_eq!( +// &rs.sanely_isolated(0..6, 0), +// &[(0..1, 0), (1..4, 9u8), (4..5, 9), (5..6, 1),] +// ); +// assert_eq!(&rs.sanely_isolated(8..10, 1), &[(8..9, 1), (9..10, 1),]); +// assert_eq!( +// &rs.sanely_isolated(6..9, 0), +// &[(6..7, 1), (7..8, 0), (8..9, 1),] +// ); +// } + +// #[test] +// fn merge_same() { +// assert_eq!( +// &easy_merge(&[(1..4, 0u8),], &[(1..4, 2u8),],), +// &[(1..4, Some(0)..Some(2)),] +// ); +// } + +// #[test] +// fn merge_empty() { +// assert_eq!( +// &easy_merge(&[(1..2, 0u8),], &[],), +// &[(1..2, Some(0)..None),] +// ); +// assert_eq!( +// &easy_merge(&[], &[(3..4, 1u8),],), +// &[(3..4, None..Some(1)),] +// ); +// } + +// #[test] +// fn merge_separate() { +// assert_eq!( +// &easy_merge(&[(1..2, 0u8), (5..6, 1u8),], &[(2..4, 2u8),],), +// &[ +// (1..2, Some(0)..None), +// (2..4, None..Some(2)), +// (5..6, Some(1)..None), +// ] +// ); +// } + +// #[test] +// fn merge_subset() { +// assert_eq!( +// &easy_merge(&[(1..6, 0u8),], &[(2..4, 2u8),],), +// &[ +// (1..2, Some(0)..None), +// (2..4, Some(0)..Some(2)), +// (4..6, Some(0)..None), +// ] +// ); +// assert_eq!( +// &easy_merge(&[(2..4, 0u8),], &[(1..4, 2u8),],), +// &[(1..2, None..Some(2)), (2..4, Some(0)..Some(2)),] +// ); +// } + +// #[test] +// fn merge_all() { +// assert_eq!( +// &easy_merge(&[(1..4, 0u8), (5..8, 1u8),], &[(2..6, 2u8), (7..9, 3u8),],), +// &[ +// (1..2, Some(0)..None), +// (2..4, Some(0)..Some(2)), +// (4..5, None..Some(2)), +// (5..6, Some(1)..Some(2)), +// (6..7, Some(1)..None), +// (7..8, Some(1)..Some(3)), +// (8..9, None..Some(3)), +// ] +// ); +// } + +// #[test] +// fn merge_complex() { +// assert_eq!( +// &easy_merge( +// &[ +// (0..8, 0u8), +// (8..9, 1), +// (9..16, 2), +// (16..17, 3), +// (17..118, 4), +// (118..119, 5), +// (119..124, 6), +// (124..125, 7), +// (125..512, 8), +// ], +// &[(15..16, 10u8), (51..52, 11), (126..127, 12),], +// ), +// &[ +// (0..8, Some(0)..None), +// (8..9, Some(1)..None), +// (9..15, Some(2)..None), +// (15..16, Some(2)..Some(10)), +// (16..17, Some(3)..None), +// (17..51, Some(4)..None), +// (51..52, Some(4)..Some(11)), +// (52..118, Some(4)..None), +// (118..119, Some(5)..None), +// (119..124, Some(6)..None), +// (124..125, Some(7)..None), +// (125..126, Some(8)..None), +// (126..127, Some(8)..Some(12)), +// (127..512, Some(8)..None), +// ] +// ); +// } +// } diff --git a/wgpu-core/src/track/texture.rs b/wgpu-core/src/track/texture.rs index 5a7560055b..45efada285 100644 --- a/wgpu-core/src/track/texture.rs +++ b/wgpu-core/src/track/texture.rs @@ -4,10 +4,10 @@ use crate::{ id::{TextureId, TypedId, Valid}, resource::Texture, track::{ - invalid_resource_state, iterate_bitvec_indices, skip_barrier, ResourceMetadata, - ResourceUses, UsageConflict, + invalid_resource_state, iterate_bitvec_indices, range::MergeItem, skip_barrier, + ResourceMetadata, ResourceUses, UsageConflict, }, - Epoch, RefCount, + Epoch, LifeGuard, RefCount, }; use hal::TextureUses; @@ -39,6 +39,10 @@ impl ResourceUses for TextureUses { fn any_exclusive(self) -> bool { self.intersects(Self::EXCLUSIVE) } + + fn uninit(self) -> bool { + self == Self::UNINITIALIZED + } } #[derive(Clone, Debug, Default, PartialEq)] @@ -145,12 +149,16 @@ pub(crate) struct TextureStateSet { complex: FastHashMap, } impl TextureStateSet { - pub fn new() -> Self { + fn new() -> Self { Self { simple: Vec::new(), complex: FastHashMap::default(), } } + + fn set_size(&mut self, size: usize) { + self.simple.resize(size, TextureUses::UNINITIALIZED); + } } #[derive(Debug)] @@ -184,7 +192,7 @@ impl TextureUsageScope { } pub fn set_size(&mut self, size: usize) { - self.set.simple.resize(size, TextureUses::UNINITIALIZED); + self.set.set_size(size); self.metadata.set_size(size); } @@ -211,7 +219,7 @@ impl TextureUsageScope { unsafe { state_combine( - Some(storage), + Some(texture_data_from_texture(storage, index32)), None, &mut self.set, &mut self.metadata, @@ -241,31 +249,6 @@ impl TextureUsageScope { Ok(()) } - /// # Safety - /// - /// `id` must be a valid ID and have an ID value less than the last call to set_size. - pub unsafe fn extend<'a>( - &mut self, - storage: &'a hub::Storage, TextureId>, - id: TextureId, - selector: Option, - new_state: TextureUses, - ) -> Result<&'a Texture, UsageConflict> { - let tex = storage - .get(id) - .map_err(|_| UsageConflict::TextureInvalid { id })?; - - self.extend_refcount( - storage, - Valid(id), - selector, - tex.life_guard.ref_count.as_ref().unwrap(), - new_state, - )?; - - Ok(tex) - } - /// # Safety /// /// `id` must be a valid ID and have an ID value less than the last call to set_size. @@ -283,7 +266,7 @@ impl TextureUsageScope { self.debug_assert_in_bounds(index); state_combine( - Some(storage), + Some(texture_data_from_texture(storage, index32)), None, &mut self.set, &mut self.metadata, @@ -349,10 +332,8 @@ impl TextureTracker { } pub fn set_size(&mut self, size: usize) { - self.start_set - .simple - .resize(size, TextureUses::UNINITIALIZED); - self.end_set.simple.resize(size, TextureUses::UNINITIALIZED); + self.start_set.set_size(size); + self.end_set.set_size(size); self.metadata.set_size(size); } @@ -423,7 +404,7 @@ impl TextureTracker { unsafe { state_combine( - Some(storage), + Some(texture_data_from_texture(storage, index32)), Some(&mut self.start_set), &mut self.end_set, &mut self.metadata, @@ -459,7 +440,7 @@ impl TextureTracker { tracker.debug_assert_in_bounds(index); unsafe { state_combine( - Some(storage), + Some(texture_data_from_texture(storage, index32)), Some(&mut self.start_set), &mut self.end_set, &mut self.metadata, @@ -495,7 +476,7 @@ impl TextureTracker { scope.debug_assert_in_bounds(index); unsafe { state_combine( - Some(storage), + Some(texture_data_from_texture(storage, index32)), Some(&mut self.start_set), &mut self.end_set, &mut self.metadata, @@ -532,7 +513,7 @@ impl TextureTracker { continue; } state_combine( - Some(storage), + Some(texture_data_from_texture(storage, index32)), Some(&mut self.start_set), &mut self.end_set, &mut self.metadata, @@ -658,9 +639,9 @@ impl<'a> LayeredStateProvider<'a> { } #[inline(always)] - unsafe fn get_layers( + unsafe fn get_layers( self, - texture: Option<&Texture>, + texture_data: Option<(&LifeGuard, &TextureSelector)>, index32: u32, index: usize, ) -> SingleOrManyStates + 'a> @@ -668,7 +649,7 @@ impl<'a> LayeredStateProvider<'a> { match self { LayeredStateProvider::KnownSingle { state } => SingleOrManyStates::Single(state), LayeredStateProvider::Selector { selector, state } => { - if texture.unwrap().full_range == selector { + if *texture_data.unwrap().1 == selector { SingleOrManyStates::Single(state) } else { SingleOrManyStates::Many(Either::Left(iter::once((selector, state)))) @@ -702,7 +683,11 @@ enum ResourceMetadataProvider<'a, A: hub::HalApi> { }, } impl ResourceMetadataProvider<'_, A> { - unsafe fn get_own(self, texture: Option<&Texture>, index: usize) -> (Epoch, RefCount) { + unsafe fn get_own( + self, + texture_data: Option<(&LifeGuard, &TextureSelector)>, + index: usize, + ) -> (Epoch, RefCount) { match self { ResourceMetadataProvider::Direct { epoch, ref_count } => { (epoch, ref_count.into_owned()) @@ -716,7 +701,7 @@ impl ResourceMetadataProvider<'_, A> { .unwrap_unchecked(), ), ResourceMetadataProvider::Resource { epoch } => { - (epoch, texture.unwrap().life_guard.add_ref()) + (epoch, texture_data.unwrap().0.add_ref()) } } } @@ -731,9 +716,18 @@ impl ResourceMetadataProvider<'_, A> { } } +#[inline(always)] +unsafe fn texture_data_from_texture( + storage: &hub::Storage, TextureId>, + index32: u32, +) -> (&LifeGuard, &TextureSelector) { + let texture = storage.get_unchecked(index32); + (&texture.life_guard, &texture.full_range) +} + #[inline(always)] unsafe fn state_combine( - storage: Option<&hub::Storage, TextureId>>, + texture_data: Option<(&LifeGuard, &TextureSelector)>, start_state: Option<&mut TextureStateSet>, end_state: &mut TextureStateSet, resource_metadata: &mut ResourceMetadata, @@ -743,8 +737,6 @@ unsafe fn state_combine( metadata_provider: ResourceMetadataProvider<'_, A>, barriers: Option<&mut Vec>>, ) -> Result<(), UsageConflict> { - let texture = storage.map(|s| s.get_unchecked(index32)); - let currently_owned = resource_metadata.owned.get(index).unwrap_unchecked(); if currently_owned { @@ -757,7 +749,7 @@ unsafe fn state_combine( }; let new_state = set_state( - texture, + texture_data, start_state, end_state, index32, @@ -766,19 +758,25 @@ unsafe fn state_combine( barriers.is_none(), ); - merge_or_transition_state(texture.unwrap(), old_state, new_state, index32, barriers) - .map_err(|partial| { - let epoch = metadata_provider.get_epoch(index); - UsageConflict::from_texture( - TextureId::zip(index32, epoch, A::VARIANT), - partial.selector, - partial.current_state, - partial.new_state, - ) - })?; + merge_or_transition_state( + texture_data.unwrap(), + old_state, + new_state, + index32, + barriers, + ) + .map_err(|partial| { + let epoch = metadata_provider.get_epoch(index); + UsageConflict::from_texture( + TextureId::zip(index32, epoch, A::VARIANT), + partial.selector, + partial.current_state, + partial.new_state, + ) + })?; } else { set_state( - texture, + texture_data, start_state, end_state, index32, @@ -787,7 +785,7 @@ unsafe fn state_combine( false, ); - let (epoch, ref_count) = metadata_provider.get_own(texture, index); + let (epoch, ref_count) = metadata_provider.get_own(texture_data, index); resource_metadata.owned.set(index, true); *resource_metadata.epochs.get_unchecked_mut(index) = epoch; @@ -798,8 +796,8 @@ unsafe fn state_combine( } #[inline(always)] -unsafe fn set_state<'a, A: hub::HalApi>( - texture: Option<&Texture>, +unsafe fn set_state<'a>( + texture_data: Option<(&LifeGuard, &TextureSelector)>, start_state: Option<&mut TextureStateSet>, end_state: &'a mut TextureStateSet, index32: u32, @@ -807,12 +805,12 @@ unsafe fn set_state<'a, A: hub::HalApi>( state_provider: LayeredStateProvider, merging_and_existing: bool, ) -> SingleOrManyStates<&'a mut TextureUses, &'a mut ComplexTextureState> { - match state_provider.get_layers(texture, index32, index) { + match state_provider.get_layers(texture_data, index32, index) { SingleOrManyStates::Single(state) => { let reference = end_state.simple.get_unchecked_mut(index); if merging_and_existing && *reference == TextureUses::COMPLEX { - let full_range = texture.unwrap().full_range.clone(); + let full_range = texture_data.unwrap().1.clone(); let complex = ComplexTextureState::from_selector_state_iter( full_range.clone(), @@ -836,7 +834,7 @@ unsafe fn set_state<'a, A: hub::HalApi>( SingleOrManyStates::Single(reference) } SingleOrManyStates::Many(state_iter) => { - let full_range = texture.unwrap().full_range.clone(); + let full_range = texture_data.unwrap().1.clone(); let complex = ComplexTextureState::from_selector_state_iter(full_range, state_iter); @@ -863,8 +861,8 @@ struct PartialUsageConflict { } #[inline(always)] -fn merge_or_transition_state( - texture: &Texture, +fn merge_or_transition_state( + texture_data: (&LifeGuard, &TextureSelector), old_state: SingleOrManyStates, new_state: SingleOrManyStates<&mut TextureUses, &mut ComplexTextureState>, index32: u32, @@ -881,7 +879,7 @@ fn merge_or_transition_state( #[allow(clippy::range_plus_one)] barriers.push(PendingTransition { id: index32, - selector: texture.full_range.clone(), + selector: texture_data.1.clone(), usage: old_simple..*new_simple, }) } @@ -890,7 +888,7 @@ fn merge_or_transition_state( if invalid_resource_state(merged_state) { return Err(PartialUsageConflict { - selector: texture.full_range.clone(), + selector: texture_data.1.clone(), current_state: old_simple, new_state: *new_simple, }); @@ -906,10 +904,13 @@ fn merge_or_transition_state( for (mip_index, mips) in new_many.mips.iter_mut().enumerate() { let mip_index = mip_index as u32; for (layer, new_state) in mips.iter_mut() { + if *new_state == TextureUses::UNINITIALIZED { + *new_state = old_simple; + } match barriers { Some(&mut ref mut barriers) => { if skip_barrier(old_simple, *new_state) { - return Ok(()); + continue; } #[allow(clippy::range_plus_one)] @@ -953,7 +954,7 @@ fn merge_or_transition_state( match barriers { Some(&mut ref mut barriers) => { if skip_barrier(old_state, *new_single) { - return Ok(()); + continue; } #[allow(clippy::range_plus_one)] @@ -976,7 +977,6 @@ fn merge_or_transition_state( Ok(()) } (SingleOrManyStates::Many(mut old_complex), SingleOrManyStates::Many(new_complex)) => { - let mut temp = Vec::new(); debug_assert!(old_complex.mips.len() >= new_complex.mips.len()); for (mip_id, (mip_old, mip_new)) in old_complex @@ -986,18 +986,17 @@ fn merge_or_transition_state( .enumerate() { let level = mip_id as u32; - temp.extend(mip_old.merge(mip_new, 0)); - for (layers, states) in temp.drain(..) { + for (layers, states) in mip_old.merge(mip_new, 0) { match states { - Range { - start: Some(start), - end: Some(end), - } => { + MergeItem(start, end) => { + if *end == TextureUses::UNINITIALIZED { + *end = start; + } match barriers { Some(&mut ref mut barriers) => { - if skip_barrier(start, end) { - return Ok(()); + if skip_barrier(start, *end) { + continue; } // TODO: Can't satisfy clippy here unless we modify // `TextureSelector` to use `std::ops::RangeBounds`. @@ -1008,13 +1007,13 @@ fn merge_or_transition_state( mips: level..level + 1, layers: layers.clone(), }, - usage: start..end, + usage: start..*end, }; barriers.push(pending); } None => { - let merged_state = start | end; + let merged_state = start | *end; if invalid_resource_state(merged_state) { return Err(PartialUsageConflict { @@ -1023,17 +1022,14 @@ fn merge_or_transition_state( layers: layers.clone(), }, current_state: start, - new_state: end, + new_state: *end, }); } - for (_, state) in mip_new.isolate(&layers, end) { - *state = merged_state; - } + *end = merged_state; } } } - _ => unreachable!(), }; } @@ -1044,3 +1040,188 @@ fn merge_or_transition_state( } } } + +#[cfg(test)] +mod test { + use super::*; + + struct TrackingTest { + life_guard: LifeGuard, + selector: TextureSelector, + set: TextureStateSet, + metadata: ResourceMetadata, + } + + fn setup(mips: Range, layers: Range, states: &[TextureUses]) -> TrackingTest { + let life_guard = LifeGuard::new("test"); + let selector = TextureSelector { mips, layers }; + + let mut set = TextureStateSet::new(); + let mut metadata = ResourceMetadata::::new(); + + set.set_size(states.len()); + metadata.set_size(states.len()); + + for &state in states { + set.simple[0] = state; + + metadata.owned.set(0, true); + metadata.ref_counts[0] = None; // Okay because we only read this through metadata provider + metadata.epochs[0] = 1; + } + + TrackingTest { + life_guard, + selector, + set, + metadata, + } + } + + #[test] + fn simple_transition() { + let mut test = setup(0..1, 0..1, &[TextureUses::UNINITIALIZED]); + + let mut barriers = Vec::new(); + unsafe { + let _ = state_combine( + Some((&test.life_guard, &test.selector)), + None, + &mut test.set, + &mut test.metadata, + 0, + 0, + LayeredStateProvider::KnownSingle { + state: TextureUses::RESOURCE, + }, + ResourceMetadataProvider::Direct { + epoch: 1, + ref_count: Cow::Borrowed(test.life_guard.ref_count.as_ref().unwrap()), + }, + Some(&mut barriers), + ); + }; + assert_eq!(test.set.simple, &[TextureUses::RESOURCE]); + assert_eq!( + barriers, + &[PendingTransition { + id: 0, + selector: test.selector, + usage: TextureUses::UNINITIALIZED..TextureUses::RESOURCE + }] + ); + } + + #[test] + fn simple_merger() { + let mut test = setup(0..1, 0..1, &[TextureUses::COPY_SRC]); + + unsafe { + let res = state_combine( + Some((&test.life_guard, &test.selector)), + None, + &mut test.set, + &mut test.metadata, + 0, + 0, + LayeredStateProvider::KnownSingle { + state: TextureUses::RESOURCE, + }, + ResourceMetadataProvider::Direct { + epoch: 1, + ref_count: Cow::Borrowed(test.life_guard.ref_count.as_ref().unwrap()), + }, + None, + ); + assert_eq!(res, Ok(())); + }; + assert_eq!( + test.set.simple, + &[TextureUses::COPY_SRC | TextureUses::RESOURCE] + ); + } + + #[test] + fn simple_to_complex_transition() { + let mut test = setup(0..1, 0..2, &[TextureUses::RESOURCE]); + + let transition_selector = TextureSelector { + mips: 0..1, + layers: 1..2, + }; + + let mut barriers = Vec::new(); + unsafe { + let _ = state_combine( + Some((&test.life_guard, &test.selector)), + None, + &mut test.set, + &mut test.metadata, + 0, + 0, + LayeredStateProvider::Selector { + selector: transition_selector.clone(), + state: TextureUses::STORAGE_WRITE, + }, + ResourceMetadataProvider::Direct { + epoch: 1, + ref_count: Cow::Borrowed(test.life_guard.ref_count.as_ref().unwrap()), + }, + Some(&mut barriers), + ); + }; + assert_eq!(test.set.simple, &[TextureUses::COMPLEX]); + assert_eq!( + barriers, + &[PendingTransition { + id: 0, + selector: transition_selector, + usage: TextureUses::RESOURCE..TextureUses::STORAGE_WRITE + }] + ); + } + + #[test] + fn simple_to_complex_merger() { + let mut test = setup(0..1, 0..2, &[TextureUses::RESOURCE]); + + let transition_selector = TextureSelector { + mips: 0..1, + layers: 1..2, + }; + + unsafe { + let res = state_combine( + Some((&test.life_guard, &test.selector)), + None, + &mut test.set, + &mut test.metadata, + 0, + 0, + LayeredStateProvider::Selector { + selector: transition_selector.clone(), + state: TextureUses::DEPTH_STENCIL_READ, + }, + ResourceMetadataProvider::Direct { + epoch: 1, + ref_count: Cow::Borrowed(test.life_guard.ref_count.as_ref().unwrap()), + }, + None, + ); + assert_eq!(res, Ok(())); + }; + assert_eq!(test.set.simple, &[TextureUses::COMPLEX]); + let complex = test.set.complex.get(&0).unwrap(); + let mip0: Vec<_> = complex.mips[0].iter().cloned().collect(); + assert_eq!( + mip0, + &[ + (0..1, TextureUses::RESOURCE), + ( + 1..2, + TextureUses::RESOURCE | TextureUses::DEPTH_STENCIL_READ + ) + ] + ); + } +} diff --git a/wgpu-hal/src/lib.rs b/wgpu-hal/src/lib.rs index f5a278722a..7918b297c7 100644 --- a/wgpu-hal/src/lib.rs +++ b/wgpu-hal/src/lib.rs @@ -673,7 +673,7 @@ bitflags::bitflags! { /// still need to be pipeline barriers inserted for synchronization. const ORDERED = Self::INCLUSIVE.bits | Self::COLOR_TARGET.bits | Self::DEPTH_STENCIL_WRITE.bits | Self::STORAGE_READ.bits; //TODO: remove this - const UNINITIALIZED = 0xFFFF; + const UNINITIALIZED = 1 << 9; } } From de8410882f773fe80bf5c856908627ab0a6a5526 Mon Sep 17 00:00:00 2001 From: Connor Fitzgerald Date: Thu, 12 May 2022 14:21:43 -0400 Subject: [PATCH 19/35] temp18 - fixing unfixed things --- wgpu-core/src/track/texture.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wgpu-core/src/track/texture.rs b/wgpu-core/src/track/texture.rs index 45efada285..b51bbe4826 100644 --- a/wgpu-core/src/track/texture.rs +++ b/wgpu-core/src/track/texture.rs @@ -392,7 +392,7 @@ impl TextureTracker { id: TextureId, selector: TextureSelector, new_state: TextureUses, - ) -> Option<(&'a Texture, Option>)> { + ) -> Option<(&'a Texture, Drain>)> { let texture = storage.get(id).ok()?; let (index32, epoch, _) = id.unzip(); @@ -420,7 +420,7 @@ impl TextureTracker { .unwrap(); } - Some((texture, self.temp.pop())) + Some((texture, self.temp.drain(..))) } pub fn change_states_tracker( From a0c3494a083b73379c8a6a2ae899746d1faddca9 Mon Sep 17 00:00:00 2001 From: Connor Fitzgerald Date: Thu, 12 May 2022 23:53:12 -0400 Subject: [PATCH 20/35] temp19 - hmmm --- wgpu-core/src/device/queue.rs | 4 +- wgpu-core/src/track/mod.rs | 5 + wgpu-core/src/track/range.rs | 443 +++++++++++++++--------------- wgpu-core/src/track/texture.rs | 167 ++++++++--- wgpu-hal/examples/halmark/main.rs | 2 +- wgpu-hal/src/lib.rs | 25 +- wgpu-hal/src/vulkan/conv.rs | 4 +- 7 files changed, 388 insertions(+), 262 deletions(-) diff --git a/wgpu-core/src/device/queue.rs b/wgpu-core/src/device/queue.rs index d5699edd6d..c2ccd4fd92 100644 --- a/wgpu-core/src/device/queue.rs +++ b/wgpu-core/src/device/queue.rs @@ -697,7 +697,7 @@ impl Global { id, None, ref_count, - hal::TextureUses::empty(), + hal::TextureUses::PRESENT, ) .unwrap(); }; @@ -856,7 +856,7 @@ impl Global { id::Valid(id), None, &ref_count, - hal::TextureUses::empty(), + hal::TextureUses::PRESENT, ) .unwrap() }; diff --git a/wgpu-core/src/track/mod.rs b/wgpu-core/src/track/mod.rs index d8d37a4b41..f0868dd205 100644 --- a/wgpu-core/src/track/mod.rs +++ b/wgpu-core/src/track/mod.rs @@ -52,6 +52,11 @@ impl PendingTransition { ) -> hal::TextureBarrier<'a, A> { log::trace!("\ttexture -> {:?}", self); let texture = tex.inner.as_raw().expect("Texture is destroyed"); + + // These showing up in a barrier is always a bug + debug_assert_ne!(self.usage.start, hal::TextureUses::UNKNOWN); + debug_assert_ne!(self.usage.end, hal::TextureUses::UNKNOWN); + hal::TextureBarrier { texture, range: wgt::ImageSubresourceRange { diff --git a/wgpu-core/src/track/range.rs b/wgpu-core/src/track/range.rs index 8551a1dc11..a64821ec79 100644 --- a/wgpu-core/src/track/range.rs +++ b/wgpu-core/src/track/range.rs @@ -2,13 +2,7 @@ //TODO: consider getting rid of it. use smallvec::SmallVec; -use std::{ - cmp::Ordering, - fmt::Debug, - iter, - ops::Range, - slice::{Iter, IterMut}, -}; +use std::{cmp::Ordering, fmt::Debug, iter, ops::Range, slice::Iter}; /// Structure that keeps track of a I -> T mapping, /// optimized for a case where keys of the same values @@ -199,235 +193,256 @@ impl RangedStates { /// /// Each range in the returned iterator is a subset of a range in either /// `self` or `other`, and the value returned as a `Range` from `self` to `other`. - pub fn merge<'a>(&'a self, other: &'a mut Self, base: I) -> Merge<'a, I, T> { + pub fn merge<'a>(&'a self, other: &'a Self, base: I) -> Merge<'a, I, T> { Merge { base, - max: self.ranges.last().unwrap().0.end, - index_a: 0, - slice_a: &self.ranges, - index_b: 0, - slice_b: &mut other.ranges, + sa: self.ranges.iter().peekable(), + sb: other.ranges.iter().peekable(), } } } -pub(crate) struct MergeItem<'a, T>(pub T, pub &'a mut T); - /// A custom iterator that goes through two `RangedStates` and process a merge. #[derive(Debug)] pub(crate) struct Merge<'a, I, T> { base: I, - max: I, - index_a: usize, - slice_a: &'a [(Range, T)], - index_b: usize, - slice_b: &'a mut [(Range, T)], + sa: iter::Peekable, T)>>, + sb: iter::Peekable, T)>>, } -impl<'a, I: Ord + Copy, T: Copy + Debug> Iterator for Merge<'a, I, T> { - type Item = (Range, MergeItem<'a, T>); +impl<'a, I: Copy + Debug + Ord, T: Copy + Debug> Iterator for Merge<'a, I, T> { + type Item = (Range, Range>); fn next(&mut self) -> Option { - if self.base >= self.max { - return None; + match (self.sa.peek(), self.sb.peek()) { + // we have both streams + (Some(&&(ref ra, va)), Some(&&(ref rb, vb))) => { + let (range, usage) = if ra.start < self.base { + // in the middle of the left stream + let (end, end_value) = if self.base == rb.start { + // right stream is starting + debug_assert!(self.base < ra.end); + (rb.end, Some(vb)) + } else { + // right hasn't started yet + debug_assert!(self.base < rb.start); + (rb.start, None) + }; + (self.base..ra.end.min(end), Some(va)..end_value) + } else if rb.start < self.base { + // in the middle of the right stream + let (end, start_value) = if self.base == ra.start { + // left stream is starting + debug_assert!(self.base < rb.end); + (ra.end, Some(va)) + } else { + // left hasn't started yet + debug_assert!(self.base < ra.start); + (ra.start, None) + }; + (self.base..rb.end.min(end), start_value..Some(vb)) + } else { + // no active streams + match ra.start.cmp(&rb.start) { + // both are starting + Ordering::Equal => (ra.start..ra.end.min(rb.end), Some(va)..Some(vb)), + // only left is starting + Ordering::Less => (ra.start..rb.start.min(ra.end), Some(va)..None), + // only right is starting + Ordering::Greater => (rb.start..ra.start.min(rb.end), None..Some(vb)), + } + }; + self.base = range.end; + if ra.end == range.end { + let _ = self.sa.next(); + } + if rb.end == range.end { + let _ = self.sb.next(); + } + Some((range, usage)) + } + // only right stream + (None, Some(&&(ref rb, vb))) => { + let range = self.base.max(rb.start)..rb.end; + self.base = rb.end; + let _ = self.sb.next(); + Some((range, None..Some(vb))) + } + // only left stream + (Some(&&(ref ra, va)), None) => { + let range = self.base.max(ra.start)..ra.end; + self.base = ra.end; + let _ = self.sa.next(); + Some((range, Some(va)..None)) + } + // done + (None, None) => None, } + } +} - let a_end = self.slice_a[self.index_a].0.end; - let b_end = self.slice_b[self.index_b].0.end; +#[cfg(test)] +mod test { + //TODO: randomized/fuzzy testing + use super::RangedStates; + use std::{fmt::Debug, ops::Range}; + + fn easy_merge( + ra: &[(Range, T)], + rb: &[(Range, T)], + ) -> Vec<(Range, Range>)> { + RangedStates::from_slice(ra) + .merge(&RangedStates::from_slice(rb), 0) + .collect() + } - let a_item = self.slice_a[self.index_a].1; - let b_item = unsafe { &mut (*self.slice_b.as_mut_ptr().add(self.index_b)).1 }; + #[test] + fn sane_good() { + let rs = RangedStates::from_slice(&[(1..4, 9u8), (4..5, 9)]); + rs.check_sanity(); + } - match a_end.cmp(&b_end) { - Ordering::Less => { - self.index_a += 1; + #[test] + #[should_panic] + fn sane_empty() { + let rs = RangedStates::from_slice(&[(1..4, 9u8), (5..5, 9)]); + rs.check_sanity(); + } - let range = self.base..a_end; - self.base = a_end; + #[test] + #[should_panic] + fn sane_intersect() { + let rs = RangedStates::from_slice(&[(1..4, 9u8), (3..5, 9)]); + rs.check_sanity(); + } - Some((range, MergeItem(a_item, b_item))) - } - Ordering::Equal => { - self.index_a += 1; - self.index_b += 1; + #[test] + fn coalesce() { + let mut rs = RangedStates::from_slice(&[(1..4, 9u8), (4..5, 9), (5..7, 1), (8..9, 1)]); + rs.coalesce(); + rs.check_sanity(); + assert_eq!(rs.ranges.as_slice(), &[(1..5, 9), (5..7, 1), (8..9, 1),]); + } - let range = self.base..a_end; - self.base = a_end; + #[test] + fn query() { + let rs = RangedStates::from_slice(&[(1..4, 1u8), (5..7, 2)]); + assert_eq!(rs.query(&(0..1), |v| *v), None); + assert_eq!(rs.query(&(1..3), |v| *v), Some(Ok(1))); + assert_eq!(rs.query(&(1..6), |v| *v), Some(Err(()))); + } - Some((range, MergeItem(a_item, b_item))) - } - Ordering::Greater => { - self.index_b += 1; + #[test] + fn isolate() { + let rs = RangedStates::from_slice(&[(1..4, 9u8), (4..5, 9), (5..7, 1), (8..9, 1)]); + assert_eq!(&rs.sanely_isolated(4..5, 0), &[(4..5, 9u8),]); + assert_eq!( + &rs.sanely_isolated(0..6, 0), + &[(0..1, 0), (1..4, 9u8), (4..5, 9), (5..6, 1),] + ); + assert_eq!(&rs.sanely_isolated(8..10, 1), &[(8..9, 1), (9..10, 1),]); + assert_eq!( + &rs.sanely_isolated(6..9, 0), + &[(6..7, 1), (7..8, 0), (8..9, 1),] + ); + } - let range = self.base..b_end; - self.base = b_end; + #[test] + fn merge_same() { + assert_eq!( + &easy_merge(&[(1..4, 0u8),], &[(1..4, 2u8),],), + &[(1..4, Some(0)..Some(2)),] + ); + } - Some((range, MergeItem(a_item, b_item))) - } - } + #[test] + fn merge_empty() { + assert_eq!( + &easy_merge(&[(1..2, 0u8),], &[],), + &[(1..2, Some(0)..None),] + ); + assert_eq!( + &easy_merge(&[], &[(3..4, 1u8),],), + &[(3..4, None..Some(1)),] + ); + } + + #[test] + fn merge_separate() { + assert_eq!( + &easy_merge(&[(1..2, 0u8), (5..6, 1u8),], &[(2..4, 2u8),],), + &[ + (1..2, Some(0)..None), + (2..4, None..Some(2)), + (5..6, Some(1)..None), + ] + ); + } + + #[test] + fn merge_subset() { + assert_eq!( + &easy_merge(&[(1..6, 0u8),], &[(2..4, 2u8),],), + &[ + (1..2, Some(0)..None), + (2..4, Some(0)..Some(2)), + (4..6, Some(0)..None), + ] + ); + assert_eq!( + &easy_merge(&[(2..4, 0u8),], &[(1..4, 2u8),],), + &[(1..2, None..Some(2)), (2..4, Some(0)..Some(2)),] + ); } -} -// #[cfg(test)] -// mod test { -// //TODO: randomized/fuzzy testing -// use super::RangedStates; -// use std::{fmt::Debug, ops::Range}; - -// fn easy_merge( -// ra: &[(Range, T)], -// rb: &[(Range, T)], -// ) -> Vec<(Range, Range>)> { -// todo!() -// } - -// #[test] -// fn sane_good() { -// let rs = RangedStates::from_slice(&[(1..4, 9u8), (4..5, 9)]); -// rs.check_sanity(); -// } - -// #[test] -// #[should_panic] -// fn sane_empty() { -// let rs = RangedStates::from_slice(&[(1..4, 9u8), (5..5, 9)]); -// rs.check_sanity(); -// } - -// #[test] -// #[should_panic] -// fn sane_intersect() { -// let rs = RangedStates::from_slice(&[(1..4, 9u8), (3..5, 9)]); -// rs.check_sanity(); -// } - -// #[test] -// fn coalesce() { -// let mut rs = RangedStates::from_slice(&[(1..4, 9u8), (4..5, 9), (5..7, 1), (8..9, 1)]); -// rs.coalesce(); -// rs.check_sanity(); -// assert_eq!(rs.ranges.as_slice(), &[(1..5, 9), (5..7, 1), (8..9, 1),]); -// } - -// #[test] -// fn query() { -// let rs = RangedStates::from_slice(&[(1..4, 1u8), (5..7, 2)]); -// assert_eq!(rs.query(&(0..1), |v| *v), None); -// assert_eq!(rs.query(&(1..3), |v| *v), Some(Ok(1))); -// assert_eq!(rs.query(&(1..6), |v| *v), Some(Err(()))); -// } - -// #[test] -// fn isolate() { -// let rs = RangedStates::from_slice(&[(1..4, 9u8), (4..5, 9), (5..7, 1), (8..9, 1)]); -// assert_eq!(&rs.sanely_isolated(4..5, 0), &[(4..5, 9u8),]); -// assert_eq!( -// &rs.sanely_isolated(0..6, 0), -// &[(0..1, 0), (1..4, 9u8), (4..5, 9), (5..6, 1),] -// ); -// assert_eq!(&rs.sanely_isolated(8..10, 1), &[(8..9, 1), (9..10, 1),]); -// assert_eq!( -// &rs.sanely_isolated(6..9, 0), -// &[(6..7, 1), (7..8, 0), (8..9, 1),] -// ); -// } - -// #[test] -// fn merge_same() { -// assert_eq!( -// &easy_merge(&[(1..4, 0u8),], &[(1..4, 2u8),],), -// &[(1..4, Some(0)..Some(2)),] -// ); -// } - -// #[test] -// fn merge_empty() { -// assert_eq!( -// &easy_merge(&[(1..2, 0u8),], &[],), -// &[(1..2, Some(0)..None),] -// ); -// assert_eq!( -// &easy_merge(&[], &[(3..4, 1u8),],), -// &[(3..4, None..Some(1)),] -// ); -// } - -// #[test] -// fn merge_separate() { -// assert_eq!( -// &easy_merge(&[(1..2, 0u8), (5..6, 1u8),], &[(2..4, 2u8),],), -// &[ -// (1..2, Some(0)..None), -// (2..4, None..Some(2)), -// (5..6, Some(1)..None), -// ] -// ); -// } - -// #[test] -// fn merge_subset() { -// assert_eq!( -// &easy_merge(&[(1..6, 0u8),], &[(2..4, 2u8),],), -// &[ -// (1..2, Some(0)..None), -// (2..4, Some(0)..Some(2)), -// (4..6, Some(0)..None), -// ] -// ); -// assert_eq!( -// &easy_merge(&[(2..4, 0u8),], &[(1..4, 2u8),],), -// &[(1..2, None..Some(2)), (2..4, Some(0)..Some(2)),] -// ); -// } - -// #[test] -// fn merge_all() { -// assert_eq!( -// &easy_merge(&[(1..4, 0u8), (5..8, 1u8),], &[(2..6, 2u8), (7..9, 3u8),],), -// &[ -// (1..2, Some(0)..None), -// (2..4, Some(0)..Some(2)), -// (4..5, None..Some(2)), -// (5..6, Some(1)..Some(2)), -// (6..7, Some(1)..None), -// (7..8, Some(1)..Some(3)), -// (8..9, None..Some(3)), -// ] -// ); -// } - -// #[test] -// fn merge_complex() { -// assert_eq!( -// &easy_merge( -// &[ -// (0..8, 0u8), -// (8..9, 1), -// (9..16, 2), -// (16..17, 3), -// (17..118, 4), -// (118..119, 5), -// (119..124, 6), -// (124..125, 7), -// (125..512, 8), -// ], -// &[(15..16, 10u8), (51..52, 11), (126..127, 12),], -// ), -// &[ -// (0..8, Some(0)..None), -// (8..9, Some(1)..None), -// (9..15, Some(2)..None), -// (15..16, Some(2)..Some(10)), -// (16..17, Some(3)..None), -// (17..51, Some(4)..None), -// (51..52, Some(4)..Some(11)), -// (52..118, Some(4)..None), -// (118..119, Some(5)..None), -// (119..124, Some(6)..None), -// (124..125, Some(7)..None), -// (125..126, Some(8)..None), -// (126..127, Some(8)..Some(12)), -// (127..512, Some(8)..None), -// ] -// ); -// } -// } + #[test] + fn merge_all() { + assert_eq!( + &easy_merge(&[(1..4, 0u8), (5..8, 1u8),], &[(2..6, 2u8), (7..9, 3u8),],), + &[ + (1..2, Some(0)..None), + (2..4, Some(0)..Some(2)), + (4..5, None..Some(2)), + (5..6, Some(1)..Some(2)), + (6..7, Some(1)..None), + (7..8, Some(1)..Some(3)), + (8..9, None..Some(3)), + ] + ); + } + + #[test] + fn merge_complex() { + assert_eq!( + &easy_merge( + &[ + (0..8, 0u8), + (8..9, 1), + (9..16, 2), + (16..17, 3), + (17..118, 4), + (118..119, 5), + (119..124, 6), + (124..125, 7), + (125..512, 8), + ], + &[(15..16, 10u8), (51..52, 11), (126..127, 12),], + ), + &[ + (0..8, Some(0)..None), + (8..9, Some(1)..None), + (9..15, Some(2)..None), + (15..16, Some(2)..Some(10)), + (16..17, Some(3)..None), + (17..51, Some(4)..None), + (51..52, Some(4)..Some(11)), + (52..118, Some(4)..None), + (118..119, Some(5)..None), + (119..124, Some(6)..None), + (124..125, Some(7)..None), + (125..126, Some(8)..None), + (126..127, Some(8)..Some(12)), + (127..512, Some(8)..None), + ] + ); + } +} diff --git a/wgpu-core/src/track/texture.rs b/wgpu-core/src/track/texture.rs index b51bbe4826..c62f2f2ee3 100644 --- a/wgpu-core/src/track/texture.rs +++ b/wgpu-core/src/track/texture.rs @@ -4,8 +4,8 @@ use crate::{ id::{TextureId, TypedId, Valid}, resource::Texture, track::{ - invalid_resource_state, iterate_bitvec_indices, range::MergeItem, skip_barrier, - ResourceMetadata, ResourceUses, UsageConflict, + invalid_resource_state, iterate_bitvec_indices, skip_barrier, ResourceMetadata, + ResourceUses, UsageConflict, }, Epoch, LifeGuard, RefCount, }; @@ -54,7 +54,7 @@ impl ComplexTextureState { fn new(mip_level_count: u32, array_layer_count: u32) -> Self { Self { mips: iter::repeat_with(|| { - RangedStates::from_range(0..array_layer_count, TextureUses::UNINITIALIZED) + RangedStates::from_range(0..array_layer_count, TextureUses::UNKNOWN) }) .take(mip_level_count as usize) .collect(), @@ -70,7 +70,7 @@ impl ComplexTextureState { for (selector, desired_state) in state_iter { let mips = selector.mips.start as usize..selector.mips.end as usize; for mip in &mut complex.mips[mips] { - for (_, state) in mip.isolate(&selector.layers, TextureUses::UNINITIALIZED) { + for (_, state) in mip.isolate(&selector.layers, TextureUses::UNKNOWN) { *state = desired_state; } } @@ -447,7 +447,7 @@ impl TextureTracker { index32, index, LayeredStateProvider::TextureSet { - set: &tracker.end_set, + set: &tracker.start_set, }, ResourceMetadataProvider::Indirect { metadata: &tracker.metadata, @@ -610,6 +610,7 @@ where } } +#[derive(Debug)] enum SingleOrManyStates { Single(S), Many(M), @@ -725,6 +726,16 @@ unsafe fn texture_data_from_texture( (&texture.life_guard, &texture.full_range) } +// I think the trick to the barriers is that there are three different possible operations: + +// merge from a single state +// barrier from a single state into a double sided tracker +// update from a single state into a double sided tracker + +// recording rpasses is a merge(rpass, resource) +// cmd_buf <- rpass is a barrier(cmd_buf, rpass) + update(cmd_buf, rpass) +// device <- cmd_buf is a barrier(device, cmd_buff.start) + update(cmd_buf, cmd_buf.end) + #[inline(always)] unsafe fn state_combine( texture_data: Option<(&LifeGuard, &TextureSelector)>, @@ -750,7 +761,7 @@ unsafe fn state_combine( let new_state = set_state( texture_data, - start_state, + None, // We never need to set the start state here. end_state, index32, index, @@ -758,8 +769,22 @@ unsafe fn state_combine( barriers.is_none(), ); + let mut start_complex = None; + if let Some(start_state) = start_state { + let start_simple = *start_state.simple.get_unchecked(index); + + if start_simple == TextureUses::COMPLEX { + start_complex = Some(start_state.complex.get_mut(&index32).unwrap_unchecked()); + } else { + dbg!(&start_simple); + } + } else { + dbg!("no start state"); + } + merge_or_transition_state( texture_data.unwrap(), + start_complex, old_state, new_state, index32, @@ -860,14 +885,17 @@ struct PartialUsageConflict { new_state: TextureUses, } +// in single -> double or #[inline(always)] fn merge_or_transition_state( texture_data: (&LifeGuard, &TextureSelector), + start_complex: Option<&mut ComplexTextureState>, old_state: SingleOrManyStates, new_state: SingleOrManyStates<&mut TextureUses, &mut ComplexTextureState>, index32: u32, barriers: Option<&mut Vec>>, ) -> Result<(), PartialUsageConflict> { + dbg!(&start_complex, &old_state, &new_state); match (old_state, new_state) { (SingleOrManyStates::Single(old_simple), SingleOrManyStates::Single(new_simple)) => { match barriers { @@ -903,9 +931,10 @@ fn merge_or_transition_state( (SingleOrManyStates::Single(old_simple), SingleOrManyStates::Many(new_many)) => { for (mip_index, mips) in new_many.mips.iter_mut().enumerate() { let mip_index = mip_index as u32; - for (layer, new_state) in mips.iter_mut() { - if *new_state == TextureUses::UNINITIALIZED { + for (layers, new_state) in mips.iter_mut() { + if *new_state == TextureUses::UNKNOWN { *new_state = old_simple; + continue; } match barriers { Some(&mut ref mut barriers) => { @@ -918,7 +947,7 @@ fn merge_or_transition_state( id: index32, selector: TextureSelector { mips: mip_index..mip_index + 1, - layers: layer.clone(), + layers: layers.clone(), }, usage: old_simple..*new_state, }) @@ -930,7 +959,7 @@ fn merge_or_transition_state( return Err(PartialUsageConflict { selector: TextureSelector { mips: mip_index..mip_index + 1, - layers: layer.clone(), + layers: layers.clone(), }, current_state: old_simple, new_state: *new_state, @@ -948,9 +977,28 @@ fn merge_or_transition_state( Ok(()) } (SingleOrManyStates::Many(old_many), SingleOrManyStates::Single(new_single)) => { - for (mip_index, mips) in old_many.mips.iter().enumerate() { - let mip_index = mip_index as u32; - for &(ref layer, old_state) in mips.iter() { + for (mip_index_usize, mips) in old_many.mips.iter().enumerate() { + let mip_index = mip_index_usize as u32; + for &(ref layers, old_state) in mips.iter() { + // If this state is still unknown, at no point during this tracker have we had information about + // these states. This means that the starting side also has no idea what the state is. We need to + // go tell it about this new state. + if old_state == TextureUses::UNKNOWN { + // We only care if this is a two sided tracker, if this is a one sided tracker this block is unreachable. + + // We only care if the starting side of the two sided tracker is complex. If it is simple, there was + // no way for us to get an unknown in the old_state anyway as the simple usage would have carried over. + if let Some(&mut ref mut start_complex) = start_complex { + for (_, state) in + start_complex.mips[mip_index_usize].isolate(layers, *new_single) + { + *state = *new_single; + } + } + // We're the first usage, so no barrier to generate. + continue; + } + match barriers { Some(&mut ref mut barriers) => { if skip_barrier(old_state, *new_single) { @@ -962,7 +1010,7 @@ fn merge_or_transition_state( id: index32, selector: TextureSelector { mips: mip_index..mip_index + 1, - layers: layer.clone(), + layers: layers.clone(), }, usage: old_state..*new_single, }) @@ -971,6 +1019,11 @@ fn merge_or_transition_state( unreachable!(); } } + + // If this exists, we almost certainly caused chaos to it, let's clean up. + if let Some(&mut ref mut start_complex) = start_complex { + start_complex.mips[mip_index_usize].coalesce(); + } } } @@ -979,61 +1032,113 @@ fn merge_or_transition_state( (SingleOrManyStates::Many(mut old_complex), SingleOrManyStates::Many(new_complex)) => { debug_assert!(old_complex.mips.len() >= new_complex.mips.len()); - for (mip_id, (mip_old, mip_new)) in old_complex + let mut temp = Vec::new(); + + for (mip_index_usize, (mip_old, mip_new)) in old_complex .mips .iter_mut() .zip(&mut new_complex.mips) .enumerate() { - let level = mip_id as u32; + let mip_index = mip_index_usize as u32; + temp.extend(mip_old.merge(mip_new, 0)); - for (layers, states) in mip_old.merge(mip_new, 0) { + for (layers, states) in temp.drain(..) { match states { - MergeItem(start, end) => { - if *end == TextureUses::UNINITIALIZED { - *end = start; - } + Range { + start: Some(old_state), + end: Some(new_state), + } => { match barriers { Some(&mut ref mut barriers) => { - if skip_barrier(start, *end) { + // We only care if the starting side of the two sided tracker is complex. If it is simple, there was + // no way for us to get an unknown in the old_state anyway as the simple usage would have carried over. + if let Some(&mut ref mut start_complex) = start_complex { + // If we know about the state now, but the previous complex tracker doesn't know then + // at no point during this tracker have we had information about these states. This means + // that the starting side also has no idea what the state is. We need to go tell it about this new state. + if old_state == TextureUses::UNKNOWN + && new_state != TextureUses::UNKNOWN + { + for (_, state) in start_complex.mips[mip_index_usize] + .isolate(&layers, new_state) + { + *state = new_state; + } + // It's not our job to issue a barrier, this is the first state in the command buffer. + continue; + } + } + + // The tracker knows the state of this, but the new state has no idea. Let's tell the new state about it. + if old_state != TextureUses::UNKNOWN + && new_state == TextureUses::UNKNOWN + { + for (_, state) in mip_new.isolate(&layers, new_state) { + *state = old_state; + } + // There's no new state, so no barrier to generate. + continue; + } + + if old_state == TextureUses::UNKNOWN + && new_state == TextureUses::UNKNOWN + { + // We know nothing about what is going on here. continue; } - // TODO: Can't satisfy clippy here unless we modify - // `TextureSelector` to use `std::ops::RangeBounds`. + + if skip_barrier(old_state, new_state) { + continue; + } + #[allow(clippy::range_plus_one)] let pending = PendingTransition { id: index32, selector: TextureSelector { - mips: level..level + 1, + mips: mip_index..mip_index + 1, layers: layers.clone(), }, - usage: start..*end, + usage: old_state..new_state, }; barriers.push(pending); } None => { - let merged_state = start | *end; + let mut merged_state = old_state | new_state; + // If we have any state other than unknown, strip the unknown + let unknownless = merged_state - TextureUses::UNKNOWN; + if !unknownless.is_empty() { + merged_state = unknownless; + } if invalid_resource_state(merged_state) { return Err(PartialUsageConflict { selector: TextureSelector { - mips: level..level + 1, + mips: mip_index..mip_index + 1, layers: layers.clone(), }, - current_state: start, - new_state: *end, + current_state: old_state, + new_state, }); } - *end = merged_state; + for (_, state) in mip_new.isolate(&layers, new_state) { + *state = merged_state; + } } } } + _ => unreachable!(), }; } mip_new.coalesce(); + + // If this exists, we almost certainly caused chaos to it, let's clean up. + if let Some(&mut ref mut start_complex) = start_complex { + start_complex.mips[mip_index_usize].coalesce(); + } } Ok(()) diff --git a/wgpu-hal/examples/halmark/main.rs b/wgpu-hal/examples/halmark/main.rs index 1ed59325c1..ff021a889d 100644 --- a/wgpu-hal/examples/halmark/main.rs +++ b/wgpu-hal/examples/halmark/main.rs @@ -685,7 +685,7 @@ impl Example { let target_barrier1 = hal::TextureBarrier { texture: surface_tex.borrow(), range: wgt::ImageSubresourceRange::default(), - usage: hal::TextureUses::COLOR_TARGET..hal::TextureUses::empty(), + usage: hal::TextureUses::COLOR_TARGET..hal::TextureUses::PRESENT, }; unsafe { ctx.encoder.end_render_pass(); diff --git a/wgpu-hal/src/lib.rs b/wgpu-hal/src/lib.rs index 7918b297c7..dd33a9a57d 100644 --- a/wgpu-hal/src/lib.rs +++ b/wgpu-hal/src/lib.rs @@ -654,26 +654,27 @@ bitflags::bitflags! { bitflags::bitflags! { /// Similar to `wgt::TextureUsages` but for internal use. pub struct TextureUses: u16 { - const COPY_SRC = 1 << 0; - const COPY_DST = 1 << 1; - const RESOURCE = 1 << 2; - const COLOR_TARGET = 1 << 3; - const DEPTH_STENCIL_READ = 1 << 4; - const DEPTH_STENCIL_WRITE = 1 << 5; - const STORAGE_READ = 1 << 6; - const STORAGE_WRITE = 1 << 7; - const COMPLEX = 1 << 8; + const PRESENT = 1 << 0; + const COPY_SRC = 1 << 1; + const COPY_DST = 1 << 2; + const RESOURCE = 1 << 3; + const COLOR_TARGET = 1 << 4; + const DEPTH_STENCIL_READ = 1 << 5; + const DEPTH_STENCIL_WRITE = 1 << 6; + const STORAGE_READ = 1 << 7; + const STORAGE_WRITE = 1 << 8; /// The combination of usages that can be used together (read-only). const INCLUSIVE = Self::COPY_SRC.bits | Self::RESOURCE.bits | Self::DEPTH_STENCIL_READ.bits; /// The combination of exclusive usages (write-only and read-write). /// These usages may still show up with others, but can't automatically be combined. - const EXCLUSIVE = Self::COPY_DST.bits | Self::COLOR_TARGET.bits | Self::DEPTH_STENCIL_WRITE.bits | Self::STORAGE_READ.bits | Self::STORAGE_WRITE.bits; + const EXCLUSIVE = Self::COPY_DST.bits | Self::COLOR_TARGET.bits | Self::DEPTH_STENCIL_WRITE.bits | Self::STORAGE_READ.bits | Self::STORAGE_WRITE.bits | Self::PRESENT.bits; /// The combination of all usages that the are guaranteed to be be ordered by the hardware. /// If a usage is not ordered, then even if it doesn't change between draw calls, there /// still need to be pipeline barriers inserted for synchronization. const ORDERED = Self::INCLUSIVE.bits | Self::COLOR_TARGET.bits | Self::DEPTH_STENCIL_WRITE.bits | Self::STORAGE_READ.bits; - //TODO: remove this - const UNINITIALIZED = 1 << 9; + const COMPLEX = 1 << 9; + const UNINITIALIZED = 1 << 10; + const UNKNOWN = 1 << 11; } } diff --git a/wgpu-hal/src/vulkan/conv.rs b/wgpu-hal/src/vulkan/conv.rs index 727502aed8..56065dd7b1 100644 --- a/wgpu-hal/src/vulkan/conv.rs +++ b/wgpu-hal/src/vulkan/conv.rs @@ -199,7 +199,7 @@ pub fn derive_image_layout( vk::ImageLayout::DEPTH_STENCIL_ATTACHMENT_OPTIMAL } _ => { - if usage.is_empty() { + if usage == crate::TextureUses::PRESENT { vk::ImageLayout::PRESENT_SRC_KHR } else if is_color { vk::ImageLayout::GENERAL @@ -280,7 +280,7 @@ pub fn map_texture_usage_to_barrier( access |= vk::AccessFlags::SHADER_WRITE; } - if usage == crate::TextureUses::UNINITIALIZED || usage.is_empty() { + if usage == crate::TextureUses::UNINITIALIZED || usage == crate::TextureUses::PRESENT { ( vk::PipelineStageFlags::TOP_OF_PIPE, vk::AccessFlags::empty(), From 9fa9e3c25fb71636173d6a0525830ef8b593288b Mon Sep 17 00:00:00 2001 From: Connor Fitzgerald Date: Sat, 14 May 2022 00:37:23 -0400 Subject: [PATCH 21/35] temp20 - curious about line deltas --- wgpu-core/src/present.rs | 5 +- wgpu-core/src/track/mod.rs | 6 +- wgpu-core/src/track/range.rs | 15 +- wgpu-core/src/track/texture.rs | 654 +++++++++++++++++---------------- 4 files changed, 350 insertions(+), 330 deletions(-) diff --git a/wgpu-core/src/present.rs b/wgpu-core/src/present.rs index 6e62294d4e..f75b33d348 100644 --- a/wgpu-core/src/present.rs +++ b/wgpu-core/src/present.rs @@ -266,7 +266,10 @@ impl Global { // The texture ID got added to the device tracker by `submit()`, // and now we are moving it away. - log::debug!("Removing swapchain texture {:?} from the device tracker", texture_id.value); + log::debug!( + "Removing swapchain texture {:?} from the device tracker", + texture_id.value + ); device.trackers.lock().textures.remove(texture_id.value); let (texture, _) = hub.textures.unregister(texture_id.value.0, &mut token); diff --git a/wgpu-core/src/track/mod.rs b/wgpu-core/src/track/mod.rs index f0868dd205..888118d215 100644 --- a/wgpu-core/src/track/mod.rs +++ b/wgpu-core/src/track/mod.rs @@ -52,11 +52,11 @@ impl PendingTransition { ) -> hal::TextureBarrier<'a, A> { log::trace!("\ttexture -> {:?}", self); let texture = tex.inner.as_raw().expect("Texture is destroyed"); - + // These showing up in a barrier is always a bug debug_assert_ne!(self.usage.start, hal::TextureUses::UNKNOWN); debug_assert_ne!(self.usage.end, hal::TextureUses::UNKNOWN); - + hal::TextureBarrier { texture, range: wgt::ImageSubresourceRange { @@ -96,7 +96,7 @@ fn invalid_resource_state(state: T) -> bool { fn skip_barrier(old_state: T, new_state: T) -> bool { // If the state didn't change and all the usages are ordered, the hardware // will guarentee the order of accesses, so we do not need to issue a barrier at all - old_state == new_state && (old_state.all_ordered() || old_state.uninit()) + old_state == new_state && old_state.all_ordered() } fn resize_bitvec(vec: &mut BitVec, size: usize) { diff --git a/wgpu-core/src/track/range.rs b/wgpu-core/src/track/range.rs index a64821ec79..6329c0f2fd 100644 --- a/wgpu-core/src/track/range.rs +++ b/wgpu-core/src/track/range.rs @@ -14,7 +14,7 @@ pub(crate) struct RangedStates { ranges: SmallVec<[(Range, T); 1]>, } -impl RangedStates { +impl RangedStates { #[cfg_attr(not(test), allow(dead_code))] pub fn empty() -> Self { Self { @@ -36,7 +36,7 @@ impl RangedStates { } } - pub fn iter(&self) -> impl Iterator, T)> { + pub fn iter(&self) -> impl Iterator, T)> + Clone { self.ranges.iter() } @@ -119,6 +119,17 @@ impl RangedStates { result.map(Ok) } + pub fn iter_filter<'a>(&'a self, range: &'a Range) -> impl Iterator, &T)> + 'a { + self.ranges + .iter() + .filter(move |(inner, ..)| inner.end > range.start && inner.start < range.end) + .map(move |(inner, v)| { + let new_range = inner.start.max(range.start)..inner.end.min(range.end); + + (new_range, v) + }) + } + /// Split the storage ranges in such a way that there is a linear subset of /// them occupying exactly `index` range, which is returned mutably. /// diff --git a/wgpu-core/src/track/texture.rs b/wgpu-core/src/track/texture.rs index c62f2f2ee3..faedc6751c 100644 --- a/wgpu-core/src/track/texture.rs +++ b/wgpu-core/src/track/texture.rs @@ -80,7 +80,7 @@ impl ComplexTextureState { fn into_selector_state_iter( &self, - ) -> impl Iterator + '_ { + ) -> impl Iterator + Clone + '_ { self.mips.iter().enumerate().flat_map(|(mip, inner)| { let mip = mip as u32; { @@ -590,6 +590,7 @@ impl TextureTracker { } } +#[derive(Clone)] enum Either { Left(L), Right(R), @@ -610,7 +611,7 @@ where } } -#[derive(Debug)] +#[derive(Debug, Clone)] enum SingleOrManyStates { Single(S), Many(M), @@ -645,8 +646,10 @@ impl<'a> LayeredStateProvider<'a> { texture_data: Option<(&LifeGuard, &TextureSelector)>, index32: u32, index: usize, - ) -> SingleOrManyStates + 'a> - { + ) -> SingleOrManyStates< + TextureUses, + impl Iterator + Clone + 'a, + > { match self { LayeredStateProvider::KnownSingle { state } => SingleOrManyStates::Single(state), LayeredStateProvider::Selector { selector, state } => { @@ -737,121 +740,18 @@ unsafe fn texture_data_from_texture( // device <- cmd_buf is a barrier(device, cmd_buff.start) + update(cmd_buf, cmd_buf.end) #[inline(always)] -unsafe fn state_combine( - texture_data: Option<(&LifeGuard, &TextureSelector)>, - start_state: Option<&mut TextureStateSet>, - end_state: &mut TextureStateSet, - resource_metadata: &mut ResourceMetadata, - index32: u32, - index: usize, - state_provider: LayeredStateProvider<'_>, - metadata_provider: ResourceMetadataProvider<'_, A>, - barriers: Option<&mut Vec>>, -) -> Result<(), UsageConflict> { - let currently_owned = resource_metadata.owned.get(index).unwrap_unchecked(); - - if currently_owned { - let old_simple = end_state.simple.get_unchecked_mut(index); - - let old_state = if *old_simple == TextureUses::COMPLEX { - SingleOrManyStates::Many(end_state.complex.remove(&index32).unwrap_unchecked()) - } else { - SingleOrManyStates::Single(*old_simple) - }; - - let new_state = set_state( - texture_data, - None, // We never need to set the start state here. - end_state, - index32, - index, - state_provider, - barriers.is_none(), - ); - - let mut start_complex = None; - if let Some(start_state) = start_state { - let start_simple = *start_state.simple.get_unchecked(index); - - if start_simple == TextureUses::COMPLEX { - start_complex = Some(start_state.complex.get_mut(&index32).unwrap_unchecked()); - } else { - dbg!(&start_simple); - } - } else { - dbg!("no start state"); - } - - merge_or_transition_state( - texture_data.unwrap(), - start_complex, - old_state, - new_state, - index32, - barriers, - ) - .map_err(|partial| { - let epoch = metadata_provider.get_epoch(index); - UsageConflict::from_texture( - TextureId::zip(index32, epoch, A::VARIANT), - partial.selector, - partial.current_state, - partial.new_state, - ) - })?; - } else { - set_state( - texture_data, - start_state, - end_state, - index32, - index, - state_provider, - false, - ); - - let (epoch, ref_count) = metadata_provider.get_own(texture_data, index); - - resource_metadata.owned.set(index, true); - *resource_metadata.epochs.get_unchecked_mut(index) = epoch; - *resource_metadata.ref_counts.get_unchecked_mut(index) = Some(ref_count); - } - - Ok(()) -} - -#[inline(always)] -unsafe fn set_state<'a>( +unsafe fn insert<'a>( texture_data: Option<(&LifeGuard, &TextureSelector)>, start_state: Option<&mut TextureStateSet>, end_state: &'a mut TextureStateSet, index32: u32, index: usize, - state_provider: LayeredStateProvider, - merging_and_existing: bool, + state_provider: LayeredStateProvider<'_>, ) -> SingleOrManyStates<&'a mut TextureUses, &'a mut ComplexTextureState> { match state_provider.get_layers(texture_data, index32, index) { SingleOrManyStates::Single(state) => { let reference = end_state.simple.get_unchecked_mut(index); - if merging_and_existing && *reference == TextureUses::COMPLEX { - let full_range = texture_data.unwrap().1.clone(); - - let complex = ComplexTextureState::from_selector_state_iter( - full_range.clone(), - iter::once((full_range, state)), - ); - - *reference = state; - let new_state = end_state - .complex - .entry(index32) - .insert_entry(complex) - .into_mut(); - - return SingleOrManyStates::Many(new_state); - } - if let Some(start_state) = start_state { *start_state.simple.get_unchecked_mut(index) = state; } @@ -879,273 +779,379 @@ unsafe fn set_state<'a>( } } -struct PartialUsageConflict { - selector: TextureSelector, - current_state: TextureUses, - new_state: TextureUses, -} - -// in single -> double or -#[inline(always)] -fn merge_or_transition_state( +unsafe fn merge( texture_data: (&LifeGuard, &TextureSelector), - start_complex: Option<&mut ComplexTextureState>, - old_state: SingleOrManyStates, - new_state: SingleOrManyStates<&mut TextureUses, &mut ComplexTextureState>, + current_state_set: &mut TextureStateSet, index32: u32, - barriers: Option<&mut Vec>>, + index: usize, + state_provider: LayeredStateProvider<'_>, ) -> Result<(), PartialUsageConflict> { - dbg!(&start_complex, &old_state, &new_state); - match (old_state, new_state) { - (SingleOrManyStates::Single(old_simple), SingleOrManyStates::Single(new_simple)) => { - match barriers { - Some(barriers) => { - if skip_barrier(old_simple, *new_simple) { - return Ok(()); + let current_simple = current_state_set.simple.get_unchecked_mut(index); + let current_state = if *current_simple == TextureUses::COMPLEX { + SingleOrManyStates::Many( + current_state_set + .complex + .get_mut(&index32) + .unwrap_unchecked(), + ) + } else { + SingleOrManyStates::Single(current_simple) + }; + + let new_state = state_provider.get_layers(Some(texture_data), index32, index); + + match (current_state, new_state) { + (SingleOrManyStates::Single(current_simple), SingleOrManyStates::Single(new_simple)) => { + let merged_state = *current_simple | new_simple; + + if invalid_resource_state(merged_state) { + return Err(PartialUsageConflict { + selector: texture_data.1.clone(), + current_state: *current_simple, + new_state: new_simple, + }); + } + + *current_simple = merged_state; + } + (SingleOrManyStates::Single(current_simple), SingleOrManyStates::Many(new_many)) => { + let mut new_complex = ComplexTextureState::from_selector_state_iter( + texture_data.1.clone(), + iter::once((texture_data.1.clone(), *current_simple)), + ); + + for (selector, new_state) in new_many { + let merged_state = *current_simple | new_state; + + if invalid_resource_state(merged_state) { + return Err(PartialUsageConflict { + selector: selector.clone(), + current_state: *current_simple, + new_state: new_state, + }); + } + + for mip in + &mut new_complex.mips[selector.mips.start as usize..selector.mips.end as usize] + { + for (_, current_layer_state) in + mip.isolate(&selector.layers, TextureUses::UNKNOWN) + { + *current_layer_state = merged_state; } - #[allow(clippy::range_plus_one)] - barriers.push(PendingTransition { - id: index32, - selector: texture_data.1.clone(), - usage: old_simple..*new_simple, - }) + mip.coalesce(); } - None => { - let merged_state = old_simple | *new_simple; + } + + *current_simple = TextureUses::COMPLEX; + current_state_set.complex.insert(index32, new_complex); + } + (SingleOrManyStates::Many(current_complex), SingleOrManyStates::Single(new_simple)) => { + for (mip_id, mip) in current_complex.mips.iter_mut().enumerate() { + let mip_id = mip_id as u32; + + for (layers, current_layer_state) in mip.iter_mut() { + let merged_state = *current_layer_state | new_simple; + + // Once we remove unknown, this will never be empty, as simple states are never unknown. + let merged_state = merged_state - TextureUses::UNKNOWN; if invalid_resource_state(merged_state) { return Err(PartialUsageConflict { - selector: texture_data.1.clone(), - current_state: old_simple, - new_state: *new_simple, + selector: TextureSelector { + mips: mip_id..mip_id + 1, + layers: layers.clone(), + }, + current_state: *current_layer_state, + new_state: new_simple, }); } - *new_simple = merged_state; + *current_layer_state = merged_state; } - } - Ok(()) + mip.coalesce(); + } } - (SingleOrManyStates::Single(old_simple), SingleOrManyStates::Many(new_many)) => { - for (mip_index, mips) in new_many.mips.iter_mut().enumerate() { - let mip_index = mip_index as u32; - for (layers, new_state) in mips.iter_mut() { - if *new_state == TextureUses::UNKNOWN { - *new_state = old_simple; - continue; - } - match barriers { - Some(&mut ref mut barriers) => { - if skip_barrier(old_simple, *new_state) { - continue; - } + (SingleOrManyStates::Many(current_complex), SingleOrManyStates::Many(new_many)) => { + for (selector, new_state) in new_many { + for mip_id in selector.mips { + debug_assert!((mip_id as usize) < current_complex.mips.len()); + + let mip = current_complex.mips.get_unchecked_mut(mip_id as usize); + + for (layers, current_layer_state) in + mip.isolate(&selector.layers, TextureUses::UNKNOWN) + { + let merged_state = *current_layer_state | new_state; + let unknown_less = merged_state - TextureUses::UNKNOWN; + + if unknown_less.is_empty() { + // We know nothing about this state, lets just move on. + continue; + } - #[allow(clippy::range_plus_one)] - barriers.push(PendingTransition { - id: index32, + if invalid_resource_state(merged_state) { + return Err(PartialUsageConflict { selector: TextureSelector { - mips: mip_index..mip_index + 1, + mips: mip_id..mip_id + 1, layers: layers.clone(), }, - usage: old_simple..*new_state, - }) - } - None => { - let merged_state = old_simple | *new_state; - - if invalid_resource_state(merged_state) { - return Err(PartialUsageConflict { - selector: TextureSelector { - mips: mip_index..mip_index + 1, - layers: layers.clone(), - }, - current_state: old_simple, - new_state: *new_state, - }); - } - - *new_state = merged_state; + current_state: *current_layer_state, + new_state: new_state, + }); } + *current_layer_state = merged_state; } + + mip.coalesce(); } + } + } + } + Ok(()) +} - mips.coalesce(); +unsafe fn barrier( + texture_data: (&LifeGuard, &TextureSelector), + current_state_set: &TextureStateSet, + index32: u32, + index: usize, + state_provider: LayeredStateProvider<'_>, + barriers: &mut Vec>, +) { + let current_simple = *current_state_set.simple.get_unchecked(index); + let current_state = if current_simple == TextureUses::COMPLEX { + SingleOrManyStates::Many(current_state_set.complex.get(&index32).unwrap_unchecked()) + } else { + SingleOrManyStates::Single(current_simple) + }; + + let new_state = state_provider.get_layers(Some(texture_data), index32, index); + + match (current_state, new_state) { + (SingleOrManyStates::Single(current_simple), SingleOrManyStates::Single(new_simple)) => { + if skip_barrier(current_simple, new_simple) { + return; } - Ok(()) + barriers.push(PendingTransition { + id: index32, + selector: texture_data.1.clone(), + usage: current_simple..new_simple, + }) } - (SingleOrManyStates::Many(old_many), SingleOrManyStates::Single(new_single)) => { - for (mip_index_usize, mips) in old_many.mips.iter().enumerate() { - let mip_index = mip_index_usize as u32; - for &(ref layers, old_state) in mips.iter() { - // If this state is still unknown, at no point during this tracker have we had information about - // these states. This means that the starting side also has no idea what the state is. We need to - // go tell it about this new state. - if old_state == TextureUses::UNKNOWN { - // We only care if this is a two sided tracker, if this is a one sided tracker this block is unreachable. - - // We only care if the starting side of the two sided tracker is complex. If it is simple, there was - // no way for us to get an unknown in the old_state anyway as the simple usage would have carried over. - if let Some(&mut ref mut start_complex) = start_complex { - for (_, state) in - start_complex.mips[mip_index_usize].isolate(layers, *new_single) - { - *state = *new_single; - } - } - // We're the first usage, so no barrier to generate. + (SingleOrManyStates::Single(current_simple), SingleOrManyStates::Many(new_many)) => { + for (selector, new_state) in new_many { + if new_state == TextureUses::UNKNOWN { + continue; + } + + if skip_barrier(current_simple, new_state) { + continue; + } + + barriers.push(PendingTransition { + id: index32, + selector, + usage: current_simple..new_state, + }) + } + } + (SingleOrManyStates::Many(current_complex), SingleOrManyStates::Single(new_simple)) => { + for (mip_id, mip) in current_complex.mips.iter().enumerate() { + let mip_id = mip_id as u32; + + for (layers, current_layer_state) in mip.iter() { + if *current_layer_state == TextureUses::UNKNOWN { continue; } - match barriers { - Some(&mut ref mut barriers) => { - if skip_barrier(old_state, *new_single) { - continue; - } + if skip_barrier(*current_layer_state, new_simple) { + continue; + } - #[allow(clippy::range_plus_one)] - barriers.push(PendingTransition { - id: index32, - selector: TextureSelector { - mips: mip_index..mip_index + 1, - layers: layers.clone(), - }, - usage: old_state..*new_single, - }) + barriers.push(PendingTransition { + id: index32, + selector: TextureSelector { + mips: mip_id..mip_id + 1, + layers: layers.clone(), + }, + usage: *current_layer_state..new_simple, + }); + } + } + } + (SingleOrManyStates::Many(current_complex), SingleOrManyStates::Many(new_many)) => { + for (selector, new_state) in new_many { + for mip_id in selector.mips { + debug_assert!((mip_id as usize) < current_complex.mips.len()); + + let mip = current_complex.mips.get_unchecked(mip_id as usize); + + for (layers, current_layer_state) in mip.iter_filter(&selector.layers) { + if *current_layer_state == TextureUses::UNKNOWN + || new_state == TextureUses::UNKNOWN + { + continue; } - None => { - unreachable!(); + + if skip_barrier(*current_layer_state, new_state) { + continue; } + + barriers.push(PendingTransition { + id: index32, + selector: TextureSelector { + mips: mip_id..mip_id + 1, + layers, + }, + usage: *current_layer_state..new_state, + }); } + } + } + } + } +} + +unsafe fn update( + texture_data: (&LifeGuard, &TextureSelector), + start_state: &mut TextureStateSet, + current_state_set: &mut TextureStateSet, + index32: u32, + index: usize, + state_provider: LayeredStateProvider<'_>, +) { + let start_simple = *start_state.simple.get_unchecked(index); + let mut start_complex = None; + if start_simple == TextureUses::COMPLEX { + start_complex = Some(start_state.complex.get_mut(&index32).unwrap_unchecked()); + } + + let current_simple = current_state_set.simple.get_unchecked_mut(index); + let current_state = if *current_simple == TextureUses::COMPLEX { + SingleOrManyStates::Many( + current_state_set + .complex + .get_mut(&index32) + .unwrap_unchecked(), + ) + } else { + SingleOrManyStates::Single(current_simple) + }; + + let new_state = state_provider.get_layers(Some(texture_data), index32, index); + + match (current_state, new_state) { + (SingleOrManyStates::Single(current_simple), SingleOrManyStates::Single(new_simple)) => { + *current_simple = new_simple; + } + (SingleOrManyStates::Single(current_simple), SingleOrManyStates::Many(new_many)) => { + let mut new_complex = ComplexTextureState::from_selector_state_iter( + texture_data.1.clone(), + iter::once((texture_data.1.clone(), *current_simple)), + ); - // If this exists, we almost certainly caused chaos to it, let's clean up. - if let Some(&mut ref mut start_complex) = start_complex { - start_complex.mips[mip_index_usize].coalesce(); + for (selector, mut new_state) in new_many { + if new_state == TextureUses::UNKNOWN { + new_state = *current_simple; + } + for mip in + &mut new_complex.mips[selector.mips.start as usize..selector.mips.end as usize] + { + for (_, current_layer_state) in + mip.isolate(&selector.layers, TextureUses::UNKNOWN) + { + *current_layer_state = new_state; } + + mip.coalesce(); } } - Ok(()) + *current_simple = TextureUses::COMPLEX; + current_state_set.complex.insert(index32, new_complex); } - (SingleOrManyStates::Many(mut old_complex), SingleOrManyStates::Many(new_complex)) => { - debug_assert!(old_complex.mips.len() >= new_complex.mips.len()); + (SingleOrManyStates::Many(current_complex), SingleOrManyStates::Single(new_single)) => { + for (mip_id, mip) in current_complex.mips.iter().enumerate() { + for (layers, current_layer_state) in mip.iter() { + // If this state is unknown, that means that the start is _also_ unknown. + if *current_layer_state == TextureUses::UNKNOWN { + if let Some(&mut ref mut start_complex) = start_complex { + debug_assert!(mip_id < start_complex.mips.len()); - let mut temp = Vec::new(); + let start_mip = start_complex.mips.get_unchecked_mut(mip_id); - for (mip_index_usize, (mip_old, mip_new)) in old_complex - .mips - .iter_mut() - .zip(&mut new_complex.mips) - .enumerate() - { - let mip_index = mip_index_usize as u32; - temp.extend(mip_old.merge(mip_new, 0)); - - for (layers, states) in temp.drain(..) { - match states { - Range { - start: Some(old_state), - end: Some(new_state), - } => { - match barriers { - Some(&mut ref mut barriers) => { - // We only care if the starting side of the two sided tracker is complex. If it is simple, there was - // no way for us to get an unknown in the old_state anyway as the simple usage would have carried over. - if let Some(&mut ref mut start_complex) = start_complex { - // If we know about the state now, but the previous complex tracker doesn't know then - // at no point during this tracker have we had information about these states. This means - // that the starting side also has no idea what the state is. We need to go tell it about this new state. - if old_state == TextureUses::UNKNOWN - && new_state != TextureUses::UNKNOWN - { - for (_, state) in start_complex.mips[mip_index_usize] - .isolate(&layers, new_state) - { - *state = new_state; - } - // It's not our job to issue a barrier, this is the first state in the command buffer. - continue; - } - } - - // The tracker knows the state of this, but the new state has no idea. Let's tell the new state about it. - if old_state != TextureUses::UNKNOWN - && new_state == TextureUses::UNKNOWN - { - for (_, state) in mip_new.isolate(&layers, new_state) { - *state = old_state; - } - // There's no new state, so no barrier to generate. - continue; - } - - if old_state == TextureUses::UNKNOWN - && new_state == TextureUses::UNKNOWN - { - // We know nothing about what is going on here. - continue; - } - - if skip_barrier(old_state, new_state) { - continue; - } - - #[allow(clippy::range_plus_one)] - let pending = PendingTransition { - id: index32, - selector: TextureSelector { - mips: mip_index..mip_index + 1, - layers: layers.clone(), - }, - usage: old_state..new_state, - }; - - barriers.push(pending); - } - None => { - let mut merged_state = old_state | new_state; - // If we have any state other than unknown, strip the unknown - let unknownless = merged_state - TextureUses::UNKNOWN; - if !unknownless.is_empty() { - merged_state = unknownless; - } - - if invalid_resource_state(merged_state) { - return Err(PartialUsageConflict { - selector: TextureSelector { - mips: mip_index..mip_index + 1, - layers: layers.clone(), - }, - current_state: old_state, - new_state, - }); - } - - for (_, state) in mip_new.isolate(&layers, new_state) { - *state = merged_state; - } - } + for (_, current_start_state) in + start_mip.isolate(layers, TextureUses::UNKNOWN) + { + debug_assert_eq!(*current_start_state, TextureUses::UNKNOWN); + *current_start_state = new_single; } + + start_mip.coalesce(); } - _ => unreachable!(), - }; + } + } + } + + *current_state_set.simple.get_unchecked_mut(index) = new_single; + current_state_set + .complex + .remove(&index32) + .unwrap_unchecked(); + } + (SingleOrManyStates::Many(current_complex), SingleOrManyStates::Many(new_many)) => { + for (selector, new_state) in new_many { + if new_state == TextureUses::UNKNOWN { + // We know nothing new + continue; } - mip_new.coalesce(); + for mip_id in selector.mips { + let mip_id = mip_id as usize; + debug_assert!(mip_id < current_complex.mips.len()); + + let mip = current_complex.mips.get_unchecked_mut(mip_id); + + for (layers, current_layer_state) in + mip.isolate(&selector.layers, TextureUses::UNKNOWN) + { + if *current_layer_state == TextureUses::UNKNOWN + && new_state != TextureUses::UNKNOWN + { + if let Some(&mut ref mut start_complex) = start_complex { + debug_assert!(mip_id < start_complex.mips.len()); + + let start_mip = start_complex.mips.get_unchecked_mut(mip_id); + + for (_, current_start_state) in + start_mip.isolate(layers, TextureUses::UNKNOWN) + { + debug_assert_eq!(*current_start_state, TextureUses::UNKNOWN); + *current_start_state = new_state; + } + + start_mip.coalesce(); + } + } + } - // If this exists, we almost certainly caused chaos to it, let's clean up. - if let Some(&mut ref mut start_complex) = start_complex { - start_complex.mips[mip_index_usize].coalesce(); + mip.coalesce(); } } - - Ok(()) } } } +struct PartialUsageConflict { + selector: TextureSelector, + current_state: TextureUses, + new_state: TextureUses, +} + #[cfg(test)] mod test { use super::*; From ad627e41e8ba166478ee23467e311be1a96be5d1 Mon Sep 17 00:00:00 2001 From: Connor Fitzgerald Date: Sat, 14 May 2022 01:31:43 -0400 Subject: [PATCH 22/35] temp21 - working texture trackers --- wgpu-core/src/track/texture.rs | 616 +++++++++++++++++++-------------- 1 file changed, 357 insertions(+), 259 deletions(-) diff --git a/wgpu-core/src/track/texture.rs b/wgpu-core/src/track/texture.rs index faedc6751c..47af9ab424 100644 --- a/wgpu-core/src/track/texture.rs +++ b/wgpu-core/src/track/texture.rs @@ -218,9 +218,8 @@ impl TextureUsageScope { let index32 = index as u32; unsafe { - state_combine( - Some(texture_data_from_texture(storage, index32)), - None, + insert_or_merge( + texture_data_from_texture(storage, index32), &mut self.set, &mut self.metadata, index32, @@ -229,7 +228,6 @@ impl TextureUsageScope { ResourceMetadataProvider::Indirect { metadata: &scope.metadata, }, - None, )? }; } @@ -265,9 +263,8 @@ impl TextureUsageScope { self.debug_assert_in_bounds(index); - state_combine( - Some(texture_data_from_texture(storage, index32)), - None, + insert_or_merge( + texture_data_from_texture(storage, index32), &mut self.set, &mut self.metadata, index32, @@ -277,7 +274,6 @@ impl TextureUsageScope { epoch: epoch, ref_count: Cow::Borrowed(ref_count), }, - None, )?; Ok(()) @@ -368,7 +364,13 @@ impl TextureTracker { self.debug_assert_in_bounds(index); unsafe { - state_combine( + let currently_owned = self.metadata.owned.get(index).unwrap_unchecked(); + + if currently_owned { + panic!("Tried to init resource already tracked"); + } + + insert( None, Some(&mut self.start_set), &mut self.end_set, @@ -380,10 +382,8 @@ impl TextureTracker { epoch: epoch, ref_count: Cow::Owned(ref_count), }, - None, ) - .unwrap(); - } + }; } pub fn change_state<'a>( @@ -403,8 +403,8 @@ impl TextureTracker { self.debug_assert_in_bounds(index); unsafe { - state_combine( - Some(texture_data_from_texture(storage, index32)), + insert_or_barrier_update( + texture_data_from_texture(storage, index32), Some(&mut self.start_set), &mut self.end_set, &mut self.metadata, @@ -414,10 +414,10 @@ impl TextureTracker { selector, state: new_state, }, + None, ResourceMetadataProvider::Resource { epoch }, - Some(&mut self.temp), + &mut self.temp, ) - .unwrap(); } Some((texture, self.temp.drain(..))) @@ -439,8 +439,8 @@ impl TextureTracker { self.debug_assert_in_bounds(index); tracker.debug_assert_in_bounds(index); unsafe { - state_combine( - Some(texture_data_from_texture(storage, index32)), + insert_or_barrier_update( + texture_data_from_texture(storage, index32), Some(&mut self.start_set), &mut self.end_set, &mut self.metadata, @@ -449,12 +449,14 @@ impl TextureTracker { LayeredStateProvider::TextureSet { set: &tracker.start_set, }, + Some(LayeredStateProvider::TextureSet { + set: &tracker.end_set, + }), ResourceMetadataProvider::Indirect { metadata: &tracker.metadata, }, - Some(&mut self.temp), - ) - .unwrap(); + &mut self.temp, + ); } } } @@ -475,20 +477,20 @@ impl TextureTracker { self.debug_assert_in_bounds(index); scope.debug_assert_in_bounds(index); unsafe { - state_combine( - Some(texture_data_from_texture(storage, index32)), + insert_or_barrier_update( + texture_data_from_texture(storage, index32), Some(&mut self.start_set), &mut self.end_set, &mut self.metadata, index32, index, LayeredStateProvider::TextureSet { set: &scope.set }, + None, ResourceMetadataProvider::Indirect { metadata: &scope.metadata, }, - Some(&mut self.temp), - ) - .unwrap(); + &mut self.temp, + ); } } } @@ -512,20 +514,20 @@ impl TextureTracker { if !scope.metadata.owned.get(index).unwrap_unchecked() { continue; } - state_combine( - Some(texture_data_from_texture(storage, index32)), + insert_or_barrier_update( + texture_data_from_texture(storage, index32), Some(&mut self.start_set), &mut self.end_set, &mut self.metadata, index32, index, LayeredStateProvider::TextureSet { set: &scope.set }, + None, ResourceMetadataProvider::Indirect { metadata: &scope.metadata, }, - Some(&mut self.temp), - ) - .unwrap(); + &mut self.temp, + ); scope.metadata.reset(index); } @@ -617,6 +619,7 @@ enum SingleOrManyStates { Many(M), } +#[derive(Clone)] enum LayeredStateProvider<'a> { KnownSingle { state: TextureUses, @@ -740,14 +743,102 @@ unsafe fn texture_data_from_texture( // device <- cmd_buf is a barrier(device, cmd_buff.start) + update(cmd_buf, cmd_buf.end) #[inline(always)] -unsafe fn insert<'a>( +unsafe fn insert_or_merge( + texture_data: (&LifeGuard, &TextureSelector), + current_state_set: &mut TextureStateSet, + resource_metadata: &mut ResourceMetadata, + index32: u32, + index: usize, + state_provider: LayeredStateProvider<'_>, + metadata_provider: ResourceMetadataProvider<'_, A>, +) -> Result<(), UsageConflict> { + let currently_owned = resource_metadata.owned.get(index).unwrap_unchecked(); + + if !currently_owned { + insert( + Some(texture_data), + None, + current_state_set, + resource_metadata, + index32, + index, + state_provider, + metadata_provider, + ); + return Ok(()); + } + + merge( + texture_data, + current_state_set, + index32, + index, + state_provider, + metadata_provider, + ) +} + +#[inline(always)] +unsafe fn insert_or_barrier_update( + texture_data: (&LifeGuard, &TextureSelector), + start_state: Option<&mut TextureStateSet>, + current_state_set: &mut TextureStateSet, + resource_metadata: &mut ResourceMetadata, + index32: u32, + index: usize, + start_state_provider: LayeredStateProvider<'_>, + end_state_provider: Option>, + metadata_provider: ResourceMetadataProvider<'_, A>, + barriers: &mut Vec>, +) { + let currently_owned = resource_metadata.owned.get(index).unwrap_unchecked(); + + if !currently_owned { + insert( + Some(texture_data), + start_state, + current_state_set, + resource_metadata, + index32, + index, + start_state_provider, + metadata_provider, + ); + return; + } + + let update_state_provider = end_state_provider.unwrap_or_else(|| start_state_provider.clone()); + barrier( + texture_data, + current_state_set, + index32, + index, + start_state_provider, + barriers, + ); + + let start_state_set = start_state.unwrap(); + update( + texture_data, + start_state_set, + current_state_set, + index32, + index, + update_state_provider, + ) +} + +#[inline(always)] +unsafe fn insert( texture_data: Option<(&LifeGuard, &TextureSelector)>, start_state: Option<&mut TextureStateSet>, - end_state: &'a mut TextureStateSet, + end_state: &mut TextureStateSet, + resource_metadata: &mut ResourceMetadata, index32: u32, index: usize, state_provider: LayeredStateProvider<'_>, -) -> SingleOrManyStates<&'a mut TextureUses, &'a mut ComplexTextureState> { + metadata_provider: ResourceMetadataProvider<'_, A>, +) { match state_provider.get_layers(texture_data, index32, index) { SingleOrManyStates::Single(state) => { let reference = end_state.simple.get_unchecked_mut(index); @@ -756,7 +847,6 @@ unsafe fn insert<'a>( *start_state.simple.get_unchecked_mut(index) = state; } *reference = state; - SingleOrManyStates::Single(reference) } SingleOrManyStates::Many(state_iter) => { let full_range = texture_data.unwrap().1.clone(); @@ -768,24 +858,26 @@ unsafe fn insert<'a>( start_state.complex.insert(index32, complex.clone()); } *end_state.simple.get_unchecked_mut(index) = TextureUses::COMPLEX; - let new_state = end_state - .complex - .entry(index32) - .insert_entry(complex) - .into_mut(); - - SingleOrManyStates::Many(new_state) + end_state.complex.insert(index32, complex); } } + + let (epoch, ref_count) = metadata_provider.get_own(texture_data, index); + + resource_metadata.owned.set(index, true); + *resource_metadata.epochs.get_unchecked_mut(index) = epoch; + *resource_metadata.ref_counts.get_unchecked_mut(index) = Some(ref_count); } -unsafe fn merge( +#[inline(always)] +unsafe fn merge( texture_data: (&LifeGuard, &TextureSelector), current_state_set: &mut TextureStateSet, index32: u32, index: usize, state_provider: LayeredStateProvider<'_>, -) -> Result<(), PartialUsageConflict> { + metadata_provider: ResourceMetadataProvider<'_, A>, +) -> Result<(), UsageConflict> { let current_simple = current_state_set.simple.get_unchecked_mut(index); let current_state = if *current_simple == TextureUses::COMPLEX { SingleOrManyStates::Many( @@ -805,11 +897,12 @@ unsafe fn merge( let merged_state = *current_simple | new_simple; if invalid_resource_state(merged_state) { - return Err(PartialUsageConflict { - selector: texture_data.1.clone(), - current_state: *current_simple, - new_state: new_simple, - }); + return Err(UsageConflict::from_texture( + TextureId::zip(index32, metadata_provider.get_epoch(index), A::VARIANT), + texture_data.1.clone(), + *current_simple, + new_simple, + )); } *current_simple = merged_state; @@ -824,11 +917,12 @@ unsafe fn merge( let merged_state = *current_simple | new_state; if invalid_resource_state(merged_state) { - return Err(PartialUsageConflict { - selector: selector.clone(), - current_state: *current_simple, - new_state: new_state, - }); + return Err(UsageConflict::from_texture( + TextureId::zip(index32, metadata_provider.get_epoch(index), A::VARIANT), + selector.clone(), + *current_simple, + new_state, + )); } for mip in @@ -858,14 +952,15 @@ unsafe fn merge( let merged_state = merged_state - TextureUses::UNKNOWN; if invalid_resource_state(merged_state) { - return Err(PartialUsageConflict { - selector: TextureSelector { + return Err(UsageConflict::from_texture( + TextureId::zip(index32, metadata_provider.get_epoch(index), A::VARIANT), + TextureSelector { mips: mip_id..mip_id + 1, layers: layers.clone(), }, - current_state: *current_layer_state, - new_state: new_simple, - }); + *current_layer_state, + new_simple, + )); } *current_layer_state = merged_state; @@ -885,22 +980,27 @@ unsafe fn merge( mip.isolate(&selector.layers, TextureUses::UNKNOWN) { let merged_state = *current_layer_state | new_state; - let unknown_less = merged_state - TextureUses::UNKNOWN; + let merged_state = merged_state - TextureUses::UNKNOWN; - if unknown_less.is_empty() { + if merged_state.is_empty() { // We know nothing about this state, lets just move on. continue; } if invalid_resource_state(merged_state) { - return Err(PartialUsageConflict { - selector: TextureSelector { + return Err(UsageConflict::from_texture( + TextureId::zip( + index32, + metadata_provider.get_epoch(index), + A::VARIANT, + ), + TextureSelector { mips: mip_id..mip_id + 1, layers: layers.clone(), }, - current_state: *current_layer_state, - new_state: new_state, - }); + *current_layer_state, + new_state, + )); } *current_layer_state = merged_state; } @@ -913,6 +1013,7 @@ unsafe fn merge( Ok(()) } +#[inline(always)] unsafe fn barrier( texture_data: (&LifeGuard, &TextureSelector), current_state_set: &TextureStateSet, @@ -1016,18 +1117,19 @@ unsafe fn barrier( } } +#[inline(always)] unsafe fn update( texture_data: (&LifeGuard, &TextureSelector), - start_state: &mut TextureStateSet, + start_state_set: &mut TextureStateSet, current_state_set: &mut TextureStateSet, index32: u32, index: usize, state_provider: LayeredStateProvider<'_>, ) { - let start_simple = *start_state.simple.get_unchecked(index); + let start_simple = *start_state_set.simple.get_unchecked(index); let mut start_complex = None; if start_simple == TextureUses::COMPLEX { - start_complex = Some(start_state.complex.get_mut(&index32).unwrap_unchecked()); + start_complex = Some(start_state_set.complex.get_mut(&index32).unwrap_unchecked()); } let current_simple = current_state_set.simple.get_unchecked_mut(index); @@ -1137,6 +1239,8 @@ unsafe fn update( start_mip.coalesce(); } } + + *current_layer_state = new_state; } mip.coalesce(); @@ -1146,193 +1250,187 @@ unsafe fn update( } } -struct PartialUsageConflict { - selector: TextureSelector, - current_state: TextureUses, - new_state: TextureUses, -} - -#[cfg(test)] -mod test { - use super::*; - - struct TrackingTest { - life_guard: LifeGuard, - selector: TextureSelector, - set: TextureStateSet, - metadata: ResourceMetadata, - } - - fn setup(mips: Range, layers: Range, states: &[TextureUses]) -> TrackingTest { - let life_guard = LifeGuard::new("test"); - let selector = TextureSelector { mips, layers }; - - let mut set = TextureStateSet::new(); - let mut metadata = ResourceMetadata::::new(); - - set.set_size(states.len()); - metadata.set_size(states.len()); - - for &state in states { - set.simple[0] = state; - - metadata.owned.set(0, true); - metadata.ref_counts[0] = None; // Okay because we only read this through metadata provider - metadata.epochs[0] = 1; - } - - TrackingTest { - life_guard, - selector, - set, - metadata, - } - } - - #[test] - fn simple_transition() { - let mut test = setup(0..1, 0..1, &[TextureUses::UNINITIALIZED]); - - let mut barriers = Vec::new(); - unsafe { - let _ = state_combine( - Some((&test.life_guard, &test.selector)), - None, - &mut test.set, - &mut test.metadata, - 0, - 0, - LayeredStateProvider::KnownSingle { - state: TextureUses::RESOURCE, - }, - ResourceMetadataProvider::Direct { - epoch: 1, - ref_count: Cow::Borrowed(test.life_guard.ref_count.as_ref().unwrap()), - }, - Some(&mut barriers), - ); - }; - assert_eq!(test.set.simple, &[TextureUses::RESOURCE]); - assert_eq!( - barriers, - &[PendingTransition { - id: 0, - selector: test.selector, - usage: TextureUses::UNINITIALIZED..TextureUses::RESOURCE - }] - ); - } - - #[test] - fn simple_merger() { - let mut test = setup(0..1, 0..1, &[TextureUses::COPY_SRC]); - - unsafe { - let res = state_combine( - Some((&test.life_guard, &test.selector)), - None, - &mut test.set, - &mut test.metadata, - 0, - 0, - LayeredStateProvider::KnownSingle { - state: TextureUses::RESOURCE, - }, - ResourceMetadataProvider::Direct { - epoch: 1, - ref_count: Cow::Borrowed(test.life_guard.ref_count.as_ref().unwrap()), - }, - None, - ); - assert_eq!(res, Ok(())); - }; - assert_eq!( - test.set.simple, - &[TextureUses::COPY_SRC | TextureUses::RESOURCE] - ); - } - - #[test] - fn simple_to_complex_transition() { - let mut test = setup(0..1, 0..2, &[TextureUses::RESOURCE]); - - let transition_selector = TextureSelector { - mips: 0..1, - layers: 1..2, - }; - - let mut barriers = Vec::new(); - unsafe { - let _ = state_combine( - Some((&test.life_guard, &test.selector)), - None, - &mut test.set, - &mut test.metadata, - 0, - 0, - LayeredStateProvider::Selector { - selector: transition_selector.clone(), - state: TextureUses::STORAGE_WRITE, - }, - ResourceMetadataProvider::Direct { - epoch: 1, - ref_count: Cow::Borrowed(test.life_guard.ref_count.as_ref().unwrap()), - }, - Some(&mut barriers), - ); - }; - assert_eq!(test.set.simple, &[TextureUses::COMPLEX]); - assert_eq!( - barriers, - &[PendingTransition { - id: 0, - selector: transition_selector, - usage: TextureUses::RESOURCE..TextureUses::STORAGE_WRITE - }] - ); - } - - #[test] - fn simple_to_complex_merger() { - let mut test = setup(0..1, 0..2, &[TextureUses::RESOURCE]); - - let transition_selector = TextureSelector { - mips: 0..1, - layers: 1..2, - }; - - unsafe { - let res = state_combine( - Some((&test.life_guard, &test.selector)), - None, - &mut test.set, - &mut test.metadata, - 0, - 0, - LayeredStateProvider::Selector { - selector: transition_selector.clone(), - state: TextureUses::DEPTH_STENCIL_READ, - }, - ResourceMetadataProvider::Direct { - epoch: 1, - ref_count: Cow::Borrowed(test.life_guard.ref_count.as_ref().unwrap()), - }, - None, - ); - assert_eq!(res, Ok(())); - }; - assert_eq!(test.set.simple, &[TextureUses::COMPLEX]); - let complex = test.set.complex.get(&0).unwrap(); - let mip0: Vec<_> = complex.mips[0].iter().cloned().collect(); - assert_eq!( - mip0, - &[ - (0..1, TextureUses::RESOURCE), - ( - 1..2, - TextureUses::RESOURCE | TextureUses::DEPTH_STENCIL_READ - ) - ] - ); - } -} +// #[cfg(test)] +// mod test { +// use super::*; + +// struct TrackingTest { +// life_guard: LifeGuard, +// selector: TextureSelector, +// set: TextureStateSet, +// metadata: ResourceMetadata, +// } + +// fn setup(mips: Range, layers: Range, states: &[TextureUses]) -> TrackingTest { +// let life_guard = LifeGuard::new("test"); +// let selector = TextureSelector { mips, layers }; + +// let mut set = TextureStateSet::new(); +// let mut metadata = ResourceMetadata::::new(); + +// set.set_size(states.len()); +// metadata.set_size(states.len()); + +// for &state in states { +// set.simple[0] = state; + +// metadata.owned.set(0, true); +// metadata.ref_counts[0] = None; // Okay because we only read this through metadata provider +// metadata.epochs[0] = 1; +// } + +// TrackingTest { +// life_guard, +// selector, +// set, +// metadata, +// } +// } + +// #[test] +// fn simple_transition() { +// let mut test = setup(0..1, 0..1, &[TextureUses::UNINITIALIZED]); + +// let mut barriers = Vec::new(); +// unsafe { +// let _ = state_combine( +// Some((&test.life_guard, &test.selector)), +// None, +// &mut test.set, +// &mut test.metadata, +// 0, +// 0, +// LayeredStateProvider::KnownSingle { +// state: TextureUses::RESOURCE, +// }, +// ResourceMetadataProvider::Direct { +// epoch: 1, +// ref_count: Cow::Borrowed(test.life_guard.ref_count.as_ref().unwrap()), +// }, +// Some(&mut barriers), +// ); +// }; +// assert_eq!(test.set.simple, &[TextureUses::RESOURCE]); +// assert_eq!( +// barriers, +// &[PendingTransition { +// id: 0, +// selector: test.selector, +// usage: TextureUses::UNINITIALIZED..TextureUses::RESOURCE +// }] +// ); +// } + +// #[test] +// fn simple_merger() { +// let mut test = setup(0..1, 0..1, &[TextureUses::COPY_SRC]); + +// unsafe { +// let res = state_combine( +// Some((&test.life_guard, &test.selector)), +// None, +// &mut test.set, +// &mut test.metadata, +// 0, +// 0, +// LayeredStateProvider::KnownSingle { +// state: TextureUses::RESOURCE, +// }, +// ResourceMetadataProvider::Direct { +// epoch: 1, +// ref_count: Cow::Borrowed(test.life_guard.ref_count.as_ref().unwrap()), +// }, +// None, +// ); +// assert_eq!(res, Ok(())); +// }; +// assert_eq!( +// test.set.simple, +// &[TextureUses::COPY_SRC | TextureUses::RESOURCE] +// ); +// } + +// #[test] +// fn simple_to_complex_transition() { +// let mut test = setup(0..1, 0..2, &[TextureUses::RESOURCE]); + +// let transition_selector = TextureSelector { +// mips: 0..1, +// layers: 1..2, +// }; + +// let mut barriers = Vec::new(); +// unsafe { +// let _ = state_combine( +// Some((&test.life_guard, &test.selector)), +// None, +// &mut test.set, +// &mut test.metadata, +// 0, +// 0, +// LayeredStateProvider::Selector { +// selector: transition_selector.clone(), +// state: TextureUses::STORAGE_WRITE, +// }, +// ResourceMetadataProvider::Direct { +// epoch: 1, +// ref_count: Cow::Borrowed(test.life_guard.ref_count.as_ref().unwrap()), +// }, +// Some(&mut barriers), +// ); +// }; +// assert_eq!(test.set.simple, &[TextureUses::COMPLEX]); +// assert_eq!( +// barriers, +// &[PendingTransition { +// id: 0, +// selector: transition_selector, +// usage: TextureUses::RESOURCE..TextureUses::STORAGE_WRITE +// }] +// ); +// } + +// #[test] +// fn simple_to_complex_merger() { +// let mut test = setup(0..1, 0..2, &[TextureUses::RESOURCE]); + +// let transition_selector = TextureSelector { +// mips: 0..1, +// layers: 1..2, +// }; + +// unsafe { +// let res = state_combine( +// Some((&test.life_guard, &test.selector)), +// None, +// &mut test.set, +// &mut test.metadata, +// 0, +// 0, +// LayeredStateProvider::Selector { +// selector: transition_selector.clone(), +// state: TextureUses::DEPTH_STENCIL_READ, +// }, +// ResourceMetadataProvider::Direct { +// epoch: 1, +// ref_count: Cow::Borrowed(test.life_guard.ref_count.as_ref().unwrap()), +// }, +// None, +// ); +// assert_eq!(res, Ok(())); +// }; +// assert_eq!(test.set.simple, &[TextureUses::COMPLEX]); +// let complex = test.set.complex.get(&0).unwrap(); +// let mip0: Vec<_> = complex.mips[0].iter().cloned().collect(); +// assert_eq!( +// mip0, +// &[ +// (0..1, TextureUses::RESOURCE), +// ( +// 1..2, +// TextureUses::RESOURCE | TextureUses::DEPTH_STENCIL_READ +// ) +// ] +// ); +// } +// } From 64cc463afec98262d71ab189c936ac12304dd0e1 Mon Sep 17 00:00:00 2001 From: Connor Fitzgerald Date: Sun, 15 May 2022 04:07:27 -0400 Subject: [PATCH 23/35] temp22 - write merge/update/barrier for buffers --- wgpu-core/src/track/buffer.rs | 93 ++++++++++++++++++++++++++++++++-- wgpu-core/src/track/mod.rs | 50 +++++++++++++++++- wgpu-core/src/track/texture.rs | 52 ++----------------- 3 files changed, 142 insertions(+), 53 deletions(-) diff --git a/wgpu-core/src/track/buffer.rs b/wgpu-core/src/track/buffer.rs index 87c353834a..a44adde468 100644 --- a/wgpu-core/src/track/buffer.rs +++ b/wgpu-core/src/track/buffer.rs @@ -6,10 +6,10 @@ use crate::{ id::{BufferId, TypedId, Valid}, resource::Buffer, track::{ - invalid_resource_state, iterate_bitvec_indices, resize_bitvec, skip_barrier, ResourceUses, - UsageConflict, + invalid_resource_state, iterate_bitvec_indices, resize_bitvec, skip_barrier, + ResourceMetadata, ResourceMetadataProvider, ResourceUses, UsageConflict, }, - Epoch, RefCount, + Epoch, RefCount, LifeGuard, }; use bit_vec::BitVec; use hal::BufferUses; @@ -502,3 +502,90 @@ impl BufferTracker { false } } + +enum StateProvider<'a> { + Direct { state: BufferUses }, + Indirect { other: &'a [BufferUses] }, +} +impl StateProvider<'_> { + unsafe fn get_state(&self, index: usize) -> BufferUses { + match *self { + StateProvider::Direct { state } => state, + StateProvider::Indirect { other } => other[index], + } + } +} + +unsafe fn insert( + life_guard: Option<&LifeGuard>, + start_state: Option<&mut Vec>, + current_state: &mut Vec, + resource_metadata: &mut ResourceMetadata, + index: usize, + state_provider: StateProvider<'_>, + metadata_provider: ResourceMetadataProvider<'_, A>, +) { + let new_state = state_provider.get_state(index); + if let Some(&mut ref mut start_state) = start_state { + *start_state.get_unchecked_mut(index) = new_state; + } + *current_state.get_unchecked_mut(index) = new_state; + + let (epoch, ref_count) = metadata_provider.get_own(life_guard, index); + + resource_metadata.owned.set(index, true); + *resource_metadata.epochs.get_unchecked_mut(index) = epoch; + *resource_metadata.ref_counts.get_unchecked_mut(index) = Some(ref_count); +} + +unsafe fn merge( + current_states: &mut Vec, + id: BufferId, + index: usize, + state_provider: StateProvider<'_>, +) -> Result<(), UsageConflict> { + let current_state = current_states.get_unchecked_mut(index); + let new_state = state_provider.get_state(index); + + let merged_state = *current_state | new_state; + + if invalid_resource_state(merged_state) { + return Err(UsageConflict::from_buffer(id, *current_state, new_state)); + } + + *current_state = merged_state; + + Ok(()) +} + +unsafe fn barrier( + current_states: &mut Vec, + index32: u32, + index: usize, + state_provider: StateProvider<'_>, + barriers: &mut Vec>, +) { + let current_state = *current_states.get_unchecked(index); + let new_state = state_provider.get_state(index); + + if skip_barrier(current_state, new_state) { + return; + } + + barriers.push(PendingTransition { + id: index32, + selector: (), + usage: current_state..new_state, + }) +} + +unsafe fn update( + current_states: &mut Vec, + index: usize, + state_provider: StateProvider<'_>, +) { + let current_state = current_states.get_unchecked_mut(index); + let new_state = state_provider.get_state(index); + + *current_state = new_state; +} diff --git a/wgpu-core/src/track/mod.rs b/wgpu-core/src/track/mod.rs index 888118d215..f379f79b96 100644 --- a/wgpu-core/src/track/mod.rs +++ b/wgpu-core/src/track/mod.rs @@ -6,11 +6,11 @@ mod texture; use crate::{ binding_model, command, conv, hub, id::{self, TypedId}, - pipeline, resource, Epoch, RefCount, + pipeline, resource, Epoch, RefCount, LifeGuard, }; use bit_vec::BitVec; -use std::{fmt, marker::PhantomData, mem, num::NonZeroU32, ops}; +use std::{fmt, marker::PhantomData, mem, num::NonZeroU32, ops, borrow::Cow}; use thiserror::Error; pub(crate) use buffer::{BufferBindGroupState, BufferTracker, BufferUsageScope}; @@ -268,6 +268,52 @@ impl ResourceMetadata { } } +enum ResourceMetadataProvider<'a, A: hub::HalApi> { + Direct { + epoch: Epoch, + ref_count: Cow<'a, RefCount>, + }, + Indirect { + metadata: &'a ResourceMetadata, + }, + Resource { + epoch: Epoch, + }, +} +impl ResourceMetadataProvider<'_, A> { + unsafe fn get_own( + self, + life_guard: Option<&LifeGuard>, + index: usize, + ) -> (Epoch, RefCount) { + match self { + ResourceMetadataProvider::Direct { epoch, ref_count } => { + (epoch, ref_count.into_owned()) + } + ResourceMetadataProvider::Indirect { metadata } => ( + *metadata.epochs.get_unchecked(index), + metadata + .ref_counts + .get_unchecked(index) + .clone() + .unwrap_unchecked(), + ), + ResourceMetadataProvider::Resource { epoch } => { + (epoch, life_guard.unwrap().add_ref()) + } + } + } + unsafe fn get_epoch(self, index: usize) -> Epoch { + match self { + ResourceMetadataProvider::Direct { epoch, .. } + | ResourceMetadataProvider::Resource { epoch, .. } => epoch, + ResourceMetadataProvider::Indirect { metadata } => { + *metadata.epochs.get_unchecked(index) + } + } + } +} + pub(crate) struct BindGroupStates { pub buffers: BufferBindGroupState, pub textures: TextureBindGroupState, diff --git a/wgpu-core/src/track/texture.rs b/wgpu-core/src/track/texture.rs index 47af9ab424..2f789a3fb8 100644 --- a/wgpu-core/src/track/texture.rs +++ b/wgpu-core/src/track/texture.rs @@ -5,7 +5,7 @@ use crate::{ resource::Texture, track::{ invalid_resource_state, iterate_bitvec_indices, skip_barrier, ResourceMetadata, - ResourceUses, UsageConflict, + ResourceMetadataProvider, ResourceUses, UsageConflict, }, Epoch, LifeGuard, RefCount, }; @@ -677,52 +677,6 @@ impl<'a> LayeredStateProvider<'a> { } } -enum ResourceMetadataProvider<'a, A: hub::HalApi> { - Direct { - epoch: Epoch, - ref_count: Cow<'a, RefCount>, - }, - Indirect { - metadata: &'a ResourceMetadata, - }, - Resource { - epoch: Epoch, - }, -} -impl ResourceMetadataProvider<'_, A> { - unsafe fn get_own( - self, - texture_data: Option<(&LifeGuard, &TextureSelector)>, - index: usize, - ) -> (Epoch, RefCount) { - match self { - ResourceMetadataProvider::Direct { epoch, ref_count } => { - (epoch, ref_count.into_owned()) - } - ResourceMetadataProvider::Indirect { metadata } => ( - *metadata.epochs.get_unchecked(index), - metadata - .ref_counts - .get_unchecked(index) - .clone() - .unwrap_unchecked(), - ), - ResourceMetadataProvider::Resource { epoch } => { - (epoch, texture_data.unwrap().0.add_ref()) - } - } - } - unsafe fn get_epoch(self, index: usize) -> Epoch { - match self { - ResourceMetadataProvider::Direct { epoch, .. } - | ResourceMetadataProvider::Resource { epoch, .. } => epoch, - ResourceMetadataProvider::Indirect { metadata } => { - *metadata.epochs.get_unchecked(index) - } - } - } -} - #[inline(always)] unsafe fn texture_data_from_texture( storage: &hub::Storage, TextureId>, @@ -794,6 +748,7 @@ unsafe fn insert_or_barrier_update( let currently_owned = resource_metadata.owned.get(index).unwrap_unchecked(); if !currently_owned { + let update_state_provider = end_state_provider.unwrap_or_else(|| start_state_provider); insert( Some(texture_data), start_state, @@ -862,7 +817,8 @@ unsafe fn insert( } } - let (epoch, ref_count) = metadata_provider.get_own(texture_data, index); + let (epoch, ref_count) = + metadata_provider.get_own(texture_data.map(|(life_guard, _)| life_guard), index); resource_metadata.owned.set(index, true); *resource_metadata.epochs.get_unchecked_mut(index) = epoch; From 426e8e27a77db1534b3301b827956d4d02f64342 Mon Sep 17 00:00:00 2001 From: Connor Fitzgerald Date: Sun, 15 May 2022 18:38:02 -0400 Subject: [PATCH 24/35] temp23 - cleanup and buffers --- wgpu-core/src/command/bundle.rs | 74 +++-- wgpu-core/src/command/compute.rs | 68 ++--- wgpu-core/src/command/query.rs | 24 +- wgpu-core/src/command/render.rs | 161 +++++------ wgpu-core/src/device/mod.rs | 22 +- wgpu-core/src/id.rs | 2 +- wgpu-core/src/track/buffer.rs | 461 ++++++++++++++++--------------- wgpu-core/src/track/mod.rs | 14 +- wgpu-core/src/track/range.rs | 263 +----------------- wgpu-core/src/track/texture.rs | 8 +- 10 files changed, 403 insertions(+), 694 deletions(-) diff --git a/wgpu-core/src/command/bundle.rs b/wgpu-core/src/command/bundle.rs index 286f756f93..6d111ecc4d 100644 --- a/wgpu-core/src/command/bundle.rs +++ b/wgpu-core/src/command/bundle.rs @@ -230,14 +230,12 @@ impl RenderBundleEncoder { let offsets = &base.dynamic_offsets[..num_dynamic_offsets as usize]; base.dynamic_offsets = &base.dynamic_offsets[num_dynamic_offsets as usize..]; - let bind_group: &binding_model::BindGroup = unsafe { - state - .trackers - .bind_groups - .extend(&*bind_group_guard, bind_group_id) - .ok_or_else(|| RenderCommandError::InvalidBindGroup(bind_group_id)) - .map_pass_err(scope)? - }; + let bind_group: &binding_model::BindGroup = state + .trackers + .bind_groups + .extend(&*bind_group_guard, bind_group_id) + .ok_or_else(|| RenderCommandError::InvalidBindGroup(bind_group_id)) + .map_pass_err(scope)?; if bind_group.dynamic_binding_info.len() != offsets.len() { return Err(RenderCommandError::InvalidDynamicOffsetCount { actual: offsets.len(), @@ -280,13 +278,11 @@ impl RenderBundleEncoder { state.pipeline = Some(pipeline_id); - let pipeline: &pipeline::RenderPipeline = unsafe { - state - .trackers - .render_pipelines - .extend(&*pipeline_guard, pipeline_id) - .unwrap() - }; + let pipeline: &pipeline::RenderPipeline = state + .trackers + .render_pipelines + .extend(&*pipeline_guard, pipeline_id) + .unwrap(); self.context .check_compatible(&pipeline.pass_context) @@ -321,13 +317,11 @@ impl RenderBundleEncoder { size, } => { let scope = PassErrorScope::SetIndexBuffer(buffer_id); - let buffer: &resource::Buffer = unsafe { - state - .trackers - .buffers - .extend(&*buffer_guard, buffer_id, hal::BufferUses::INDEX) - .map_pass_err(scope)? - }; + let buffer: &resource::Buffer = state + .trackers + .buffers + .extend(&*buffer_guard, buffer_id, hal::BufferUses::INDEX) + .map_pass_err(scope)?; check_buffer_usage(buffer.usage, wgt::BufferUsages::INDEX) .map_pass_err(scope)?; @@ -350,13 +344,11 @@ impl RenderBundleEncoder { size, } => { let scope = PassErrorScope::SetVertexBuffer(buffer_id); - let buffer: &resource::Buffer = unsafe { - state - .trackers - .buffers - .extend(&*buffer_guard, buffer_id, hal::BufferUses::VERTEX) - .map_pass_err(scope)? - }; + let buffer: &resource::Buffer = state + .trackers + .buffers + .extend(&*buffer_guard, buffer_id, hal::BufferUses::VERTEX) + .map_pass_err(scope)?; check_buffer_usage(buffer.usage, wgt::BufferUsages::VERTEX) .map_pass_err(scope)?; @@ -477,13 +469,11 @@ impl RenderBundleEncoder { .require_downlevel_flags(wgt::DownlevelFlags::INDIRECT_EXECUTION) .map_pass_err(scope)?; - let buffer: &resource::Buffer = unsafe { - state - .trackers - .buffers - .extend(&*buffer_guard, buffer_id, hal::BufferUses::INDIRECT) - .map_pass_err(scope)? - }; + let buffer: &resource::Buffer = state + .trackers + .buffers + .extend(&*buffer_guard, buffer_id, hal::BufferUses::INDIRECT) + .map_pass_err(scope)?; check_buffer_usage(buffer.usage, wgt::BufferUsages::INDIRECT) .map_pass_err(scope)?; @@ -512,13 +502,11 @@ impl RenderBundleEncoder { .require_downlevel_flags(wgt::DownlevelFlags::INDIRECT_EXECUTION) .map_pass_err(scope)?; - let buffer: &resource::Buffer = unsafe { - state - .trackers - .buffers - .extend(&*buffer_guard, buffer_id, hal::BufferUses::INDIRECT) - .map_pass_err(scope)? - }; + let buffer: &resource::Buffer = state + .trackers + .buffers + .extend(&*buffer_guard, buffer_id, hal::BufferUses::INDIRECT) + .map_pass_err(scope)?; check_buffer_usage(buffer.usage, wgt::BufferUsages::INDIRECT) .map_pass_err(scope)?; diff --git a/wgpu-core/src/command/compute.rs b/wgpu-core/src/command/compute.rs index 81bcaf00f5..c2ce1e64ef 100644 --- a/wgpu-core/src/command/compute.rs +++ b/wgpu-core/src/command/compute.rs @@ -393,14 +393,12 @@ impl Global { ); dynamic_offset_count += num_dynamic_offsets as usize; - let bind_group: &BindGroup = unsafe { - cmd_buf - .trackers - .bind_groups - .extend(&*bind_group_guard, bind_group_id) - .ok_or_else(|| ComputePassErrorInner::InvalidBindGroup(bind_group_id)) - .map_pass_err(scope)? - }; + let bind_group: &BindGroup = cmd_buf + .trackers + .bind_groups + .extend(&*bind_group_guard, bind_group_id) + .ok_or_else(|| ComputePassErrorInner::InvalidBindGroup(bind_group_id)) + .map_pass_err(scope)?; bind_group .validate_dynamic_bindings(&temp_offsets, &cmd_buf.limits) .map_pass_err(scope)?; @@ -450,14 +448,12 @@ impl Global { state.pipeline = Some(pipeline_id); - let pipeline: &pipeline::ComputePipeline = unsafe { - cmd_buf - .trackers - .compute_pipelines - .extend(&*pipeline_guard, pipeline_id) - .ok_or_else(|| ComputePassErrorInner::InvalidPipeline(pipeline_id)) - .map_pass_err(scope)? - }; + let pipeline: &pipeline::ComputePipeline = cmd_buf + .trackers + .compute_pipelines + .extend(&*pipeline_guard, pipeline_id) + .ok_or_else(|| ComputePassErrorInner::InvalidPipeline(pipeline_id)) + .map_pass_err(scope)?; unsafe { raw.set_compute_pipeline(&pipeline.raw); @@ -605,13 +601,11 @@ impl Global { .require_downlevel_flags(wgt::DownlevelFlags::INDIRECT_EXECUTION) .map_pass_err(scope)?; - let indirect_buffer: &Buffer = unsafe { - state - .scope - .buffers - .extend(&*buffer_guard, buffer_id, hal::BufferUses::INDIRECT) - .map_pass_err(scope)? - }; + let indirect_buffer: &Buffer = state + .scope + .buffers + .extend(&*buffer_guard, buffer_id, hal::BufferUses::INDIRECT) + .map_pass_err(scope)?; check_buffer_usage(indirect_buffer.usage, wgt::BufferUsages::INDIRECT) .map_pass_err(scope)?; @@ -689,14 +683,12 @@ impl Global { } => { let scope = PassErrorScope::WriteTimestamp; - let query_set: &resource::QuerySet = unsafe { - cmd_buf - .trackers - .query_sets - .extend(&*query_set_guard, query_set_id) - .ok_or_else(|| ComputePassErrorInner::InvalidQuerySet(query_set_id)) - .map_pass_err(scope)? - }; + let query_set: &resource::QuerySet = cmd_buf + .trackers + .query_sets + .extend(&*query_set_guard, query_set_id) + .ok_or_else(|| ComputePassErrorInner::InvalidQuerySet(query_set_id)) + .map_pass_err(scope)?; query_set .validate_and_write_timestamp(raw, query_set_id, query_index, None) @@ -708,14 +700,12 @@ impl Global { } => { let scope = PassErrorScope::BeginPipelineStatisticsQuery; - let query_set: &resource::QuerySet = unsafe { - cmd_buf - .trackers - .query_sets - .extend(&*query_set_guard, query_set_id) - .ok_or_else(|| ComputePassErrorInner::InvalidQuerySet(query_set_id)) - .map_pass_err(scope)? - }; + let query_set: &resource::QuerySet = cmd_buf + .trackers + .query_sets + .extend(&*query_set_guard, query_set_id) + .ok_or_else(|| ComputePassErrorInner::InvalidQuerySet(query_set_id)) + .map_pass_err(scope)?; query_set .validate_and_begin_pipeline_statistics_query( diff --git a/wgpu-core/src/command/query.rs b/wgpu-core/src/command/query.rs index 60f1d64504..911b304820 100644 --- a/wgpu-core/src/command/query.rs +++ b/wgpu-core/src/command/query.rs @@ -296,13 +296,11 @@ impl Global { }); } - let query_set = unsafe { - cmd_buf - .trackers - .query_sets - .extend(&*query_set_guard, query_set_id) - .ok_or_else(|| QueryError::InvalidQuerySet(query_set_id))? - }; + let query_set = cmd_buf + .trackers + .query_sets + .extend(&*query_set_guard, query_set_id) + .ok_or_else(|| QueryError::InvalidQuerySet(query_set_id))?; query_set.validate_and_write_timestamp(raw_encoder, query_set_id, query_index, None)?; @@ -343,13 +341,11 @@ impl Global { return Err(QueryError::Resolve(ResolveError::BufferOffsetAlignment)); } - let query_set = unsafe { - cmd_buf - .trackers - .query_sets - .extend(&*query_set_guard, query_set_id) - .ok_or_else(|| QueryError::InvalidQuerySet(query_set_id))? - }; + let query_set = cmd_buf + .trackers + .query_sets + .extend(&*query_set_guard, query_set_id) + .ok_or_else(|| QueryError::InvalidQuerySet(query_set_id))?; let (dst_buffer, dst_pending) = cmd_buf .trackers diff --git a/wgpu-core/src/command/render.rs b/wgpu-core/src/command/render.rs index 4182795c30..ff4db69f96 100644 --- a/wgpu-core/src/command/render.rs +++ b/wgpu-core/src/command/render.rs @@ -697,13 +697,11 @@ impl<'a, A: HalApi> RenderPassInfo<'a, A> { let mut depth_stencil = None; if let Some(at) = depth_stencil_attachment { - let view = unsafe { - cmd_buf - .trackers - .views - .extend(&*view_guard, at.view) - .ok_or_else(|| RenderPassErrorInner::InvalidAttachment(at.view))? - }; + let view = cmd_buf + .trackers + .views + .extend(&*view_guard, at.view) + .ok_or_else(|| RenderPassErrorInner::InvalidAttachment(at.view))?; check_multiview(view)?; add_view(view, "depth")?; @@ -808,13 +806,11 @@ impl<'a, A: HalApi> RenderPassInfo<'a, A> { } for at in color_attachments { - let color_view = unsafe { - cmd_buf - .trackers - .views - .extend(&*view_guard, at.view) - .ok_or_else(|| RenderPassErrorInner::InvalidAttachment(at.view))? - }; + let color_view = cmd_buf + .trackers + .views + .extend(&*view_guard, at.view) + .ok_or_else(|| RenderPassErrorInner::InvalidAttachment(at.view))?; check_multiview(color_view)?; add_view(color_view, "color")?; @@ -840,13 +836,11 @@ impl<'a, A: HalApi> RenderPassInfo<'a, A> { let mut hal_resolve_target = None; if let Some(resolve_target) = at.resolve_target { - let resolve_view = unsafe { - cmd_buf - .trackers - .views - .extend(&*view_guard, resolve_target) - .ok_or_else(|| RenderPassErrorInner::InvalidAttachment(resolve_target))? - }; + let resolve_view = cmd_buf + .trackers + .views + .extend(&*view_guard, resolve_target) + .ok_or_else(|| RenderPassErrorInner::InvalidAttachment(resolve_target))?; check_multiview(resolve_view)?; if color_view.extent != resolve_view.extent { @@ -1159,14 +1153,12 @@ impl Global { ); dynamic_offset_count += num_dynamic_offsets as usize; - let bind_group = unsafe { - cmd_buf - .trackers - .bind_groups - .extend(&*bind_group_guard, bind_group_id) - .ok_or_else(|| RenderCommandError::InvalidBindGroup(bind_group_id)) - .map_pass_err(scope)? - }; + let bind_group = cmd_buf + .trackers + .bind_groups + .extend(&*bind_group_guard, bind_group_id) + .ok_or_else(|| RenderCommandError::InvalidBindGroup(bind_group_id)) + .map_pass_err(scope)?; bind_group .validate_dynamic_bindings(&temp_offsets, &cmd_buf.limits) .map_pass_err(scope)?; @@ -1225,14 +1217,12 @@ impl Global { let scope = PassErrorScope::SetPipelineRender(pipeline_id); state.pipeline = Some(pipeline_id); - let pipeline = unsafe { - cmd_buf - .trackers - .render_pipelines - .extend(&*render_pipeline_guard, pipeline_id) - .ok_or_else(|| RenderCommandError::InvalidPipeline(pipeline_id)) - .map_pass_err(scope)? - }; + let pipeline = cmd_buf + .trackers + .render_pipelines + .extend(&*render_pipeline_guard, pipeline_id) + .ok_or_else(|| RenderCommandError::InvalidPipeline(pipeline_id)) + .map_pass_err(scope)?; info.context .check_compatible(&pipeline.pass_context) @@ -1338,12 +1328,11 @@ impl Global { size, } => { let scope = PassErrorScope::SetIndexBuffer(buffer_id); - let buffer: &Buffer = unsafe { - info.usage_scope - .buffers - .extend(&*buffer_guard, buffer_id, hal::BufferUses::INDEX) - .map_pass_err(scope)? - }; + let buffer: &Buffer = info + .usage_scope + .buffers + .extend(&*buffer_guard, buffer_id, hal::BufferUses::INDEX) + .map_pass_err(scope)?; check_buffer_usage(buffer.usage, BufferUsages::INDEX) .map_pass_err(scope)?; let buf_raw = buffer @@ -1385,12 +1374,11 @@ impl Global { size, } => { let scope = PassErrorScope::SetVertexBuffer(buffer_id); - let buffer: &Buffer = unsafe { - info.usage_scope - .buffers - .extend(&*buffer_guard, buffer_id, hal::BufferUses::VERTEX) - .map_pass_err(scope)? - }; + let buffer: &Buffer = info + .usage_scope + .buffers + .extend(&*buffer_guard, buffer_id, hal::BufferUses::VERTEX) + .map_pass_err(scope)?; check_buffer_usage(buffer.usage, BufferUsages::VERTEX) .map_pass_err(scope)?; let buf_raw = buffer @@ -1643,12 +1631,11 @@ impl Global { .require_downlevel_flags(wgt::DownlevelFlags::INDIRECT_EXECUTION) .map_pass_err(scope)?; - let indirect_buffer: &Buffer = unsafe { - info.usage_scope - .buffers - .extend(&*buffer_guard, buffer_id, hal::BufferUses::INDIRECT) - .map_pass_err(scope)? - }; + let indirect_buffer: &Buffer = info + .usage_scope + .buffers + .extend(&*buffer_guard, buffer_id, hal::BufferUses::INDIRECT) + .map_pass_err(scope)?; check_buffer_usage(indirect_buffer.usage, BufferUsages::INDIRECT) .map_pass_err(scope)?; let indirect_raw = indirect_buffer @@ -1714,12 +1701,11 @@ impl Global { .require_downlevel_flags(wgt::DownlevelFlags::INDIRECT_EXECUTION) .map_pass_err(scope)?; - let indirect_buffer: &Buffer = unsafe { - info.usage_scope - .buffers - .extend(&*buffer_guard, buffer_id, hal::BufferUses::INDIRECT) - .map_pass_err(scope)? - }; + let indirect_buffer: &Buffer = info + .usage_scope + .buffers + .extend(&*buffer_guard, buffer_id, hal::BufferUses::INDIRECT) + .map_pass_err(scope)?; check_buffer_usage(indirect_buffer.usage, BufferUsages::INDIRECT) .map_pass_err(scope)?; let indirect_raw = indirect_buffer @@ -1728,12 +1714,11 @@ impl Global { .ok_or(RenderCommandError::DestroyedBuffer(buffer_id)) .map_pass_err(scope)?; - let count_buffer: &Buffer = unsafe { - info.usage_scope - .buffers - .extend(&*buffer_guard, count_buffer_id, hal::BufferUses::INDIRECT) - .map_pass_err(scope)? - }; + let count_buffer: &Buffer = info + .usage_scope + .buffers + .extend(&*buffer_guard, count_buffer_id, hal::BufferUses::INDIRECT) + .map_pass_err(scope)?; check_buffer_usage(count_buffer.usage, BufferUsages::INDIRECT) .map_pass_err(scope)?; let count_raw = count_buffer @@ -1835,14 +1820,12 @@ impl Global { } => { let scope = PassErrorScope::WriteTimestamp; - let query_set: &resource::QuerySet = unsafe { - cmd_buf - .trackers - .query_sets - .extend(&*query_set_guard, query_set_id) - .ok_or_else(|| RenderCommandError::InvalidQuerySet(query_set_id)) - .map_pass_err(scope)? - }; + let query_set: &resource::QuerySet = cmd_buf + .trackers + .query_sets + .extend(&*query_set_guard, query_set_id) + .ok_or_else(|| RenderCommandError::InvalidQuerySet(query_set_id)) + .map_pass_err(scope)?; query_set .validate_and_write_timestamp( @@ -1859,14 +1842,12 @@ impl Global { } => { let scope = PassErrorScope::BeginPipelineStatisticsQuery; - let query_set: &resource::QuerySet = unsafe { - cmd_buf - .trackers - .query_sets - .extend(&*query_set_guard, query_set_id) - .ok_or_else(|| RenderCommandError::InvalidQuerySet(query_set_id)) - .map_pass_err(scope)? - }; + let query_set: &resource::QuerySet = cmd_buf + .trackers + .query_sets + .extend(&*query_set_guard, query_set_id) + .ok_or_else(|| RenderCommandError::InvalidQuerySet(query_set_id)) + .map_pass_err(scope)?; query_set .validate_and_begin_pipeline_statistics_query( @@ -1886,14 +1867,12 @@ impl Global { } RenderCommand::ExecuteBundle(bundle_id) => { let scope = PassErrorScope::ExecuteBundle; - let bundle: &command::RenderBundle = unsafe { - cmd_buf - .trackers - .bundles - .extend(&*bundle_guard, bundle_id) - .ok_or_else(|| RenderCommandError::InvalidRenderBundle(bundle_id)) - .map_pass_err(scope)? - }; + let bundle: &command::RenderBundle = cmd_buf + .trackers + .bundles + .extend(&*bundle_guard, bundle_id) + .ok_or_else(|| RenderCommandError::InvalidRenderBundle(bundle_id)) + .map_pass_err(scope)?; info.context .check_compatible(&bundle.context) diff --git a/wgpu-core/src/device/mod.rs b/wgpu-core/src/device/mod.rs index 4cc111ed15..99f627916d 100644 --- a/wgpu-core/src/device/mod.rs +++ b/wgpu-core/src/device/mod.rs @@ -3693,7 +3693,7 @@ impl Global { let ref_count = view.life_guard.add_ref(); let id = fid.assign(view, &mut token); - unsafe { device.trackers.lock().views.init(id, ref_count) }; + device.trackers.lock().views.init(id, ref_count); return (id.0, None); }; @@ -3786,9 +3786,8 @@ impl Global { let ref_count = sampler.life_guard.add_ref(); let id = fid.assign(sampler, &mut token); - unsafe { - device.trackers.lock().samplers.init(id, ref_count); - } + device.trackers.lock().samplers.init(id, ref_count); + return (id.0, None); }; @@ -4048,7 +4047,7 @@ impl Global { let id = fid.assign(bind_group, &mut token); log::debug!("Bind group {:?}", id,); - unsafe { device.trackers.lock().bind_groups.init(id, ref_count) }; + device.trackers.lock().bind_groups.init(id, ref_count); return (id.0, None); }; @@ -4355,7 +4354,7 @@ impl Global { let ref_count = render_bundle.life_guard.add_ref(); let id = fid.assign(render_bundle, &mut token); - unsafe { device.trackers.lock().bundles.init(id, ref_count) }; + device.trackers.lock().bundles.init(id, ref_count); return (id.0, None); }; @@ -4430,9 +4429,7 @@ impl Global { let ref_count = query_set.life_guard.add_ref(); let id = fid.assign(query_set, &mut token); - unsafe { - device.trackers.lock().query_sets.init(id, ref_count); - } + device.trackers.lock().query_sets.init(id, ref_count); return (id.0, None); }; @@ -4523,9 +4520,8 @@ impl Global { let id = fid.assign(pipeline, &mut token); log::info!("Created render pipeline {:?} with {:?}", id, desc); - unsafe { - device.trackers.lock().render_pipelines.init(id, ref_count); - } + device.trackers.lock().render_pipelines.init(id, ref_count); + return (id.0, None); }; @@ -4661,7 +4657,7 @@ impl Global { let id = fid.assign(pipeline, &mut token); log::info!("Created compute pipeline {:?} with {:?}", id, desc); - unsafe { device.trackers.lock().compute_pipelines.init(id, ref_count) }; + device.trackers.lock().compute_pipelines.init(id, ref_count); return (id.0, None); }; diff --git a/wgpu-core/src/id.rs b/wgpu-core/src/id.rs index e7a19b229c..0955857235 100644 --- a/wgpu-core/src/id.rs +++ b/wgpu-core/src/id.rs @@ -64,7 +64,7 @@ impl From for Id { } impl Id { - #[cfg(test)] + #[allow(dead_code)] pub(crate) fn dummy(index: u32) -> Valid { Valid(Id::zip(index, 1, Backend::Empty)) } diff --git a/wgpu-core/src/track/buffer.rs b/wgpu-core/src/track/buffer.rs index a44adde468..6960a07654 100644 --- a/wgpu-core/src/track/buffer.rs +++ b/wgpu-core/src/track/buffer.rs @@ -1,4 +1,4 @@ -use std::{marker::PhantomData, vec::Drain}; +use std::{borrow::Cow, marker::PhantomData, vec::Drain}; use super::PendingTransition; use crate::{ @@ -6,12 +6,11 @@ use crate::{ id::{BufferId, TypedId, Valid}, resource::Buffer, track::{ - invalid_resource_state, iterate_bitvec_indices, resize_bitvec, skip_barrier, - ResourceMetadata, ResourceMetadataProvider, ResourceUses, UsageConflict, + invalid_resource_state, iterate_bitvec_indices, skip_barrier, ResourceMetadata, + ResourceMetadataProvider, ResourceUses, UsageConflict, }, - Epoch, RefCount, LifeGuard, + LifeGuard, RefCount, }; -use bit_vec::BitVec; use hal::BufferUses; impl ResourceUses for BufferUses { @@ -79,12 +78,7 @@ impl BufferBindGroupState { pub(crate) struct BufferUsageScope { state: Vec, - ref_counts: Vec>, - epochs: Vec, - - owned: BitVec, - - _phantom: PhantomData, + metadata: ResourceMetadata, } impl BufferUsageScope { @@ -92,44 +86,28 @@ impl BufferUsageScope { Self { state: Vec::new(), - ref_counts: Vec::new(), - epochs: Vec::new(), - - owned: BitVec::default(), - - _phantom: PhantomData, + metadata: ResourceMetadata::new(), } } fn debug_assert_in_bounds(&self, index: usize) { debug_assert!(index < self.state.len()); - debug_assert!(index < self.ref_counts.len()); - debug_assert!(index < self.epochs.len()); - debug_assert!(index < self.owned.len()); - - debug_assert!(if self.owned.get(index).unwrap() { - self.ref_counts[index].is_some() - } else { - true - }); + self.metadata.debug_assert_in_bounds(index); } pub fn set_size(&mut self, size: usize) { self.state.resize(size, BufferUses::empty()); - self.ref_counts.resize(size, None); - self.epochs.resize(size, u32::MAX); + self.metadata.set_size(size); + } - resize_bitvec(&mut self.owned, size); + fn allow_index(&mut self, index: usize) { + if index >= self.state.len() { + self.set_size(index + 1); + } } pub fn used(&self) -> impl Iterator> + '_ { - if !self.owned.is_empty() { - self.debug_assert_in_bounds(self.owned.len() - 1) - }; - iterate_bitvec_indices(&self.owned).map(move |index| { - let epoch = unsafe { *self.epochs.get_unchecked(index) }; - Valid(BufferId::zip(index as u32, epoch, A::VARIANT)) - }) + self.metadata.used() } pub unsafe fn extend_from_bind_group( @@ -140,7 +118,19 @@ impl BufferUsageScope { let (index32, epoch, _) = id.0.unzip(); let index = index32 as usize; - self.extend_inner(*id, index, epoch, ref_count, *state)?; + insert_or_merge( + None, + None, + &mut self.state, + &mut self.metadata, + index32, + index, + StateProvider::Direct { state: *state }, + ResourceMetadataProvider::Direct { + epoch, + ref_count: Cow::Borrowed(ref_count), + }, + )?; } Ok(()) @@ -152,25 +142,24 @@ impl BufferUsageScope { self.set_size(incoming_size); } - for index in iterate_bitvec_indices(&scope.owned) { + for index in iterate_bitvec_indices(&scope.metadata.owned) { self.debug_assert_in_bounds(index); scope.debug_assert_in_bounds(index); unsafe { - let ref_count = scope - .ref_counts - .get_unchecked(index) - .as_ref() - .unwrap_unchecked(); - let epoch = *scope.epochs.get_unchecked(index); - let new_state = *scope.state.get_unchecked(index); - - self.extend_inner( - Valid(BufferId::zip(index as u32, epoch, A::VARIANT)), + insert_or_merge( + None, + None, + &mut self.state, + &mut self.metadata, + index as u32, index, - epoch, - ref_count, - new_state, + StateProvider::Indirect { + state: &scope.state, + }, + ResourceMetadataProvider::Indirect { + metadata: &scope.metadata, + }, )?; }; } @@ -178,7 +167,7 @@ impl BufferUsageScope { Ok(()) } - pub unsafe fn extend<'a>( + pub fn extend<'a>( &mut self, storage: &'a hub::Storage, BufferId>, id: BufferId, @@ -191,48 +180,24 @@ impl BufferUsageScope { let (index32, epoch, _) = id.unzip(); let index = index32 as usize; - self.extend_inner( - Valid(id), - index, - epoch, - buffer.life_guard.ref_count.as_ref().unwrap(), - new_state, - )?; - - Ok(buffer) - } + self.allow_index(index); - unsafe fn extend_inner<'a>( - &mut self, - id: Valid, - index: usize, - epoch: u32, - ref_count: &RefCount, - new_state: BufferUses, - ) -> Result<(), UsageConflict> { self.debug_assert_in_bounds(index); - let currently_active = self.owned.get(index).unwrap_unchecked(); - if currently_active { - let current_state = *self.state.get_unchecked(index); - - let merged_state = current_state | new_state; - - if invalid_resource_state(merged_state) { - return Err(UsageConflict::from_buffer(id.0, current_state, new_state)); - } - - *self.state.get_unchecked_mut(index) = merged_state; + unsafe { + insert_or_merge( + Some(&buffer.life_guard), + None, + &mut self.state, + &mut self.metadata, + index32, + index, + StateProvider::Direct { state: new_state }, + ResourceMetadataProvider::Resource { epoch }, + )?; } - // We're the first to use this resource, let's add it. - *self.epochs.get_unchecked_mut(index) = epoch; - *self.ref_counts.get_unchecked_mut(index) = Some(ref_count.clone()); - self.owned.set(index, true); - - *self.state.get_unchecked_mut(index) = new_state; - - Ok(()) + Ok(buffer) } } @@ -240,13 +205,9 @@ pub(crate) struct BufferTracker { start: Vec, end: Vec, - epochs: Vec, - ref_counts: Vec>, - owned: BitVec, + metadata: ResourceMetadata, temp: Vec>, - - _phantom: PhantomData, } impl BufferTracker { pub fn new() -> Self { @@ -254,38 +215,23 @@ impl BufferTracker { start: Vec::new(), end: Vec::new(), - epochs: Vec::new(), - ref_counts: Vec::new(), - owned: BitVec::default(), + metadata: ResourceMetadata::new(), temp: Vec::new(), - - _phantom: PhantomData, } } fn debug_assert_in_bounds(&self, index: usize) { debug_assert!(index < self.start.len()); debug_assert!(index < self.end.len()); - debug_assert!(index < self.ref_counts.len()); - debug_assert!(index < self.epochs.len()); - debug_assert!(index < self.owned.len()); - - debug_assert!(if self.owned.get(index).unwrap() { - self.ref_counts[index].is_some() - } else { - true - }); + self.metadata.debug_assert_in_bounds(index); } pub fn set_size(&mut self, size: usize) { self.start.resize(size, BufferUses::empty()); self.end.resize(size, BufferUses::empty()); - self.epochs.resize(size, u32::MAX); - self.ref_counts.resize(size, None); - - resize_bitvec(&mut self.owned, size); + self.metadata.set_size(size); } fn allow_index(&mut self, index: usize) { @@ -295,13 +241,7 @@ impl BufferTracker { } pub fn used(&self) -> impl Iterator> + '_ { - if !self.owned.is_empty() { - self.debug_assert_in_bounds(self.owned.len() - 1) - }; - iterate_bitvec_indices(&self.owned).map(move |index| { - let epoch = unsafe { *self.epochs.get_unchecked(index) }; - Valid(BufferId::zip(index as u32, epoch, A::VARIANT)) - }) + self.metadata.used() } pub fn drain(&mut self) -> Drain> { @@ -317,13 +257,19 @@ impl BufferTracker { self.debug_assert_in_bounds(index); unsafe { - *self.start.get_unchecked_mut(index) = state; - *self.end.get_unchecked_mut(index) = state; - - *self.ref_counts.get_unchecked_mut(index) = Some(ref_count); - *self.epochs.get_unchecked_mut(index) = epoch; - - self.owned.set(index, true); + insert( + None, + Some(&mut self.start), + &mut self.end, + &mut self.metadata, + index, + StateProvider::Direct { state }, + None, + ResourceMetadataProvider::Direct { + epoch, + ref_count: Cow::Owned(ref_count), + }, + ) } } @@ -343,14 +289,22 @@ impl BufferTracker { self.debug_assert_in_bounds(index); unsafe { - self.transition_inner( + insert_or_barrier_update( + Some(&value.life_guard), + Some(&mut self.start), + &mut self.end, + &mut self.metadata, + index32, index, - epoch, - value.life_guard.ref_count.as_ref().unwrap(), - state, + StateProvider::Direct { state }, + None, + ResourceMetadataProvider::Resource { epoch }, + &mut self.temp, ) }; + debug_assert!(self.temp.len() <= 1); + Some((value, self.temp.pop())) } @@ -360,18 +314,27 @@ impl BufferTracker { self.set_size(incoming_size); } - for index in iterate_bitvec_indices(&tracker.owned) { + for index in iterate_bitvec_indices(&tracker.metadata.owned) { tracker.debug_assert_in_bounds(index); unsafe { - let ref_count = tracker - .ref_counts - .get_unchecked(index) - .as_ref() - .unwrap_unchecked(); - - let epoch = *tracker.epochs.get_unchecked(index); - - self.transition(&tracker.start, ref_count, index, epoch); + insert_or_barrier_update( + None, + Some(&mut self.start), + &mut self.end, + &mut self.metadata, + index as u32, + index, + StateProvider::Indirect { + state: &tracker.start, + }, + Some(StateProvider::Indirect { + state: &tracker.end, + }), + ResourceMetadataProvider::Indirect { + metadata: &tracker.metadata, + }, + &mut self.temp, + ) } } } @@ -382,18 +345,25 @@ impl BufferTracker { self.set_size(incoming_size); } - for index in iterate_bitvec_indices(&scope.owned) { + for index in iterate_bitvec_indices(&scope.metadata.owned) { scope.debug_assert_in_bounds(index); unsafe { - let ref_count = scope - .ref_counts - .get_unchecked(index) - .as_ref() - .unwrap_unchecked(); - - let epoch = *scope.epochs.get_unchecked(index); - - self.transition(&scope.state, ref_count, index, epoch); + insert_or_barrier_update( + None, + Some(&mut self.start), + &mut self.end, + &mut self.metadata, + index as u32, + index, + StateProvider::Indirect { + state: &scope.state, + }, + None, + ResourceMetadataProvider::Indirect { + metadata: &scope.metadata, + }, + &mut self.temp, + ) } } } @@ -414,61 +384,28 @@ impl BufferTracker { scope.debug_assert_in_bounds(index); - if !scope.owned.get(index).unwrap_unchecked() { + if !scope.metadata.owned.get(index).unwrap_unchecked() { continue; } - self.transition(&scope.state, ref_count, index, epoch); - - *scope.ref_counts.get_unchecked_mut(index) = None; - *scope.epochs.get_unchecked_mut(index) = u32::MAX; - scope.owned.set(index, false); - } - } - - unsafe fn transition( - &mut self, - incoming_set: &Vec, - ref_count: &RefCount, - index: usize, - epoch: u32, - ) { - let new_state = *incoming_set.get_unchecked(index); - - self.transition_inner(index, epoch, ref_count, new_state); - } - - unsafe fn transition_inner( - &mut self, - index: usize, - epoch: u32, - ref_count: &RefCount, - new_state: BufferUses, - ) { - self.debug_assert_in_bounds(index); - - let old_tracked = self.owned.get(index).unwrap_unchecked(); - let old_state = *self.end.get_unchecked(index); - - if old_tracked { - if skip_barrier(old_state, new_state) { - return; - } - - self.temp.push(PendingTransition { - id: index as u32, - selector: (), - usage: old_state..new_state, - }); - - *self.end.get_unchecked_mut(index) = new_state; - } else { - *self.start.get_unchecked_mut(index) = new_state; - *self.end.get_unchecked_mut(index) = new_state; - - *self.ref_counts.get_unchecked_mut(index) = Some(ref_count.clone()); - *self.epochs.get_unchecked_mut(index) = epoch; + insert_or_barrier_update( + None, + Some(&mut self.start), + &mut self.end, + &mut self.metadata, + index as u32, + index, + StateProvider::Indirect { + state: &scope.state, + }, + None, + ResourceMetadataProvider::Direct { + epoch, + ref_count: Cow::Borrowed(ref_count), + }, + &mut self.temp, + ); - self.owned.set(index, true); + scope.metadata.reset(index); } } @@ -476,23 +413,21 @@ impl BufferTracker { let (index32, epoch, _) = id.0.unzip(); let index = index32 as usize; - if index > self.owned.len() { + if index > self.metadata.owned.len() { return false; } self.debug_assert_in_bounds(index); unsafe { - if self.owned.get(index).unwrap_unchecked() { - let existing_epoch = self.epochs.get_unchecked_mut(index); - let existing_ref_count = self.ref_counts.get_unchecked_mut(index); + if self.metadata.owned.get(index).unwrap_unchecked() { + let existing_epoch = self.metadata.epochs.get_unchecked_mut(index); + let existing_ref_count = self.metadata.ref_counts.get_unchecked_mut(index); if *existing_epoch == epoch && existing_ref_count.as_mut().unwrap_unchecked().load() == 1 { - self.owned.set(index, false); - *existing_epoch = u32::MAX; - *existing_ref_count = None; + self.metadata.reset(index); return true; } @@ -503,33 +438,112 @@ impl BufferTracker { } } +#[derive(Debug, Clone)] enum StateProvider<'a> { Direct { state: BufferUses }, - Indirect { other: &'a [BufferUses] }, + Indirect { state: &'a [BufferUses] }, } impl StateProvider<'_> { unsafe fn get_state(&self, index: usize) -> BufferUses { match *self { StateProvider::Direct { state } => state, - StateProvider::Indirect { other } => other[index], + StateProvider::Indirect { state: other } => other[index], } } } -unsafe fn insert( +unsafe fn insert_or_merge( life_guard: Option<&LifeGuard>, - start_state: Option<&mut Vec>, - current_state: &mut Vec, + start_states: Option<&mut Vec>, + current_states: &mut Vec, resource_metadata: &mut ResourceMetadata, + index32: u32, index: usize, state_provider: StateProvider<'_>, metadata_provider: ResourceMetadataProvider<'_, A>, +) -> Result<(), UsageConflict> { + let currently_owned = resource_metadata.owned.get(index).unwrap_unchecked(); + + if !currently_owned { + insert( + life_guard, + start_states, + current_states, + resource_metadata, + index, + state_provider, + None, + metadata_provider, + ); + return Ok(()); + } + + merge( + current_states, + index32, + index, + state_provider, + metadata_provider, + ) +} + +unsafe fn insert_or_barrier_update( + life_guard: Option<&LifeGuard>, + start_states: Option<&mut Vec>, + current_states: &mut Vec, + resource_metadata: &mut ResourceMetadata, + index32: u32, + index: usize, + start_state_provider: StateProvider<'_>, + end_state_provider: Option>, + metadata_provider: ResourceMetadataProvider<'_, A>, + barriers: &mut Vec>, ) { - let new_state = state_provider.get_state(index); - if let Some(&mut ref mut start_state) = start_state { - *start_state.get_unchecked_mut(index) = new_state; + let currently_owned = resource_metadata.owned.get(index).unwrap_unchecked(); + + if !currently_owned { + insert( + life_guard, + start_states, + current_states, + resource_metadata, + index, + start_state_provider, + end_state_provider, + metadata_provider, + ); + return; + } + + let update_state_provider = end_state_provider.unwrap_or_else(|| start_state_provider.clone()); + barrier( + current_states, + index32, + index, + start_state_provider, + barriers, + ); + + let start_state_set = start_states.unwrap(); + update(start_state_set, index, update_state_provider); +} + +unsafe fn insert( + life_guard: Option<&LifeGuard>, + start_states: Option<&mut Vec>, + current_states: &mut Vec, + resource_metadata: &mut ResourceMetadata, + index: usize, + start_state_provider: StateProvider<'_>, + end_state_provider: Option>, + metadata_provider: ResourceMetadataProvider<'_, A>, +) { + let new_start_state = start_state_provider.get_state(index); + let new_end_state = end_state_provider.map_or(new_start_state, |p| p.get_state(index)); + if let Some(&mut ref mut start_state) = start_states { + *start_state.get_unchecked_mut(index) = new_start_state; } - *current_state.get_unchecked_mut(index) = new_state; + *current_states.get_unchecked_mut(index) = new_end_state; let (epoch, ref_count) = metadata_provider.get_own(life_guard, index); @@ -538,11 +552,12 @@ unsafe fn insert( *resource_metadata.ref_counts.get_unchecked_mut(index) = Some(ref_count); } -unsafe fn merge( +unsafe fn merge( current_states: &mut Vec, - id: BufferId, + index32: u32, index: usize, state_provider: StateProvider<'_>, + metadata_provider: ResourceMetadataProvider<'_, A>, ) -> Result<(), UsageConflict> { let current_state = current_states.get_unchecked_mut(index); let new_state = state_provider.get_state(index); @@ -550,7 +565,11 @@ unsafe fn merge( let merged_state = *current_state | new_state; if invalid_resource_state(merged_state) { - return Err(UsageConflict::from_buffer(id, *current_state, new_state)); + return Err(UsageConflict::from_buffer( + BufferId::zip(index32, metadata_provider.get_epoch(index), A::VARIANT), + *current_state, + new_state, + )); } *current_state = merged_state; diff --git a/wgpu-core/src/track/mod.rs b/wgpu-core/src/track/mod.rs index f379f79b96..3652b8781e 100644 --- a/wgpu-core/src/track/mod.rs +++ b/wgpu-core/src/track/mod.rs @@ -6,11 +6,11 @@ mod texture; use crate::{ binding_model, command, conv, hub, id::{self, TypedId}, - pipeline, resource, Epoch, RefCount, LifeGuard, + pipeline, resource, Epoch, LifeGuard, RefCount, }; use bit_vec::BitVec; -use std::{fmt, marker::PhantomData, mem, num::NonZeroU32, ops, borrow::Cow}; +use std::{borrow::Cow, fmt, marker::PhantomData, mem, num::NonZeroU32, ops}; use thiserror::Error; pub(crate) use buffer::{BufferBindGroupState, BufferTracker, BufferUsageScope}; @@ -281,11 +281,7 @@ enum ResourceMetadataProvider<'a, A: hub::HalApi> { }, } impl ResourceMetadataProvider<'_, A> { - unsafe fn get_own( - self, - life_guard: Option<&LifeGuard>, - index: usize, - ) -> (Epoch, RefCount) { + unsafe fn get_own(self, life_guard: Option<&LifeGuard>, index: usize) -> (Epoch, RefCount) { match self { ResourceMetadataProvider::Direct { epoch, ref_count } => { (epoch, ref_count.into_owned()) @@ -298,9 +294,7 @@ impl ResourceMetadataProvider<'_, A> { .clone() .unwrap_unchecked(), ), - ResourceMetadataProvider::Resource { epoch } => { - (epoch, life_guard.unwrap().add_ref()) - } + ResourceMetadataProvider::Resource { epoch } => (epoch, life_guard.unwrap().add_ref()), } } unsafe fn get_epoch(self, index: usize) -> Epoch { diff --git a/wgpu-core/src/track/range.rs b/wgpu-core/src/track/range.rs index 6329c0f2fd..890f8d5eab 100644 --- a/wgpu-core/src/track/range.rs +++ b/wgpu-core/src/track/range.rs @@ -2,7 +2,7 @@ //TODO: consider getting rid of it. use smallvec::SmallVec; -use std::{cmp::Ordering, fmt::Debug, iter, ops::Range, slice::Iter}; +use std::{fmt::Debug, iter, ops::Range}; /// Structure that keeps track of a I -> T mapping, /// optimized for a case where keys of the same values @@ -15,13 +15,6 @@ pub(crate) struct RangedStates { } impl RangedStates { - #[cfg_attr(not(test), allow(dead_code))] - pub fn empty() -> Self { - Self { - ranges: SmallVec::new(), - } - } - pub fn from_range(range: Range, value: T) -> Self { Self { ranges: iter::once((range, value)).collect(), @@ -44,26 +37,6 @@ impl RangedStates { self.ranges.iter_mut() } - pub fn into_iter(self) -> impl Iterator, T)> { - self.ranges.into_iter() - } - - /// Clear all the ranges. - pub fn clear(&mut self) { - self.ranges.clear(); - } - - /// Append a range. - /// - /// Assumes that the object is being constructed from a set of - /// ranges, and they are given in the ascending order of their keys. - pub fn append(&mut self, index: Range, value: T) { - if let Some(last) = self.ranges.last() { - debug_assert!(last.0.end <= index.start); - } - self.ranges.push((index, value)); - } - /// Check that all the ranges are non-intersecting and ordered. /// Panics otherwise. #[cfg(test)] @@ -98,28 +71,10 @@ impl RangedStates { } } - /// Check if all intersecting ranges have the same value, which is returned. - /// - /// Returns `None` if no intersections are detected. - /// Returns `Some(Err)` if the intersected values are inconsistent. - pub fn query( - &self, - index: &Range, - fun: impl Fn(&T) -> U, - ) -> Option> { - let mut result = None; - for &(ref range, ref value) in self.ranges.iter() { - if range.end > index.start && range.start < index.end { - let old = result.replace(fun(value)); - if old.is_some() && old != result { - return Some(Err(())); - } - } - } - result.map(Ok) - } - - pub fn iter_filter<'a>(&'a self, range: &'a Range) -> impl Iterator, &T)> + 'a { + pub fn iter_filter<'a>( + &'a self, + range: &'a Range, + ) -> impl Iterator, &T)> + 'a { self.ranges .iter() .filter(move |(inner, ..)| inner.end > range.start && inner.start < range.end) @@ -199,112 +154,12 @@ impl RangedStates { clone.check_sanity(); result } - - /// Produce an iterator that merges two instances together. - /// - /// Each range in the returned iterator is a subset of a range in either - /// `self` or `other`, and the value returned as a `Range` from `self` to `other`. - pub fn merge<'a>(&'a self, other: &'a Self, base: I) -> Merge<'a, I, T> { - Merge { - base, - sa: self.ranges.iter().peekable(), - sb: other.ranges.iter().peekable(), - } - } -} - -/// A custom iterator that goes through two `RangedStates` and process a merge. -#[derive(Debug)] -pub(crate) struct Merge<'a, I, T> { - base: I, - sa: iter::Peekable, T)>>, - sb: iter::Peekable, T)>>, -} - -impl<'a, I: Copy + Debug + Ord, T: Copy + Debug> Iterator for Merge<'a, I, T> { - type Item = (Range, Range>); - fn next(&mut self) -> Option { - match (self.sa.peek(), self.sb.peek()) { - // we have both streams - (Some(&&(ref ra, va)), Some(&&(ref rb, vb))) => { - let (range, usage) = if ra.start < self.base { - // in the middle of the left stream - let (end, end_value) = if self.base == rb.start { - // right stream is starting - debug_assert!(self.base < ra.end); - (rb.end, Some(vb)) - } else { - // right hasn't started yet - debug_assert!(self.base < rb.start); - (rb.start, None) - }; - (self.base..ra.end.min(end), Some(va)..end_value) - } else if rb.start < self.base { - // in the middle of the right stream - let (end, start_value) = if self.base == ra.start { - // left stream is starting - debug_assert!(self.base < rb.end); - (ra.end, Some(va)) - } else { - // left hasn't started yet - debug_assert!(self.base < ra.start); - (ra.start, None) - }; - (self.base..rb.end.min(end), start_value..Some(vb)) - } else { - // no active streams - match ra.start.cmp(&rb.start) { - // both are starting - Ordering::Equal => (ra.start..ra.end.min(rb.end), Some(va)..Some(vb)), - // only left is starting - Ordering::Less => (ra.start..rb.start.min(ra.end), Some(va)..None), - // only right is starting - Ordering::Greater => (rb.start..ra.start.min(rb.end), None..Some(vb)), - } - }; - self.base = range.end; - if ra.end == range.end { - let _ = self.sa.next(); - } - if rb.end == range.end { - let _ = self.sb.next(); - } - Some((range, usage)) - } - // only right stream - (None, Some(&&(ref rb, vb))) => { - let range = self.base.max(rb.start)..rb.end; - self.base = rb.end; - let _ = self.sb.next(); - Some((range, None..Some(vb))) - } - // only left stream - (Some(&&(ref ra, va)), None) => { - let range = self.base.max(ra.start)..ra.end; - self.base = ra.end; - let _ = self.sa.next(); - Some((range, Some(va)..None)) - } - // done - (None, None) => None, - } - } } #[cfg(test)] mod test { //TODO: randomized/fuzzy testing use super::RangedStates; - use std::{fmt::Debug, ops::Range}; - - fn easy_merge( - ra: &[(Range, T)], - rb: &[(Range, T)], - ) -> Vec<(Range, Range>)> { - RangedStates::from_slice(ra) - .merge(&RangedStates::from_slice(rb), 0) - .collect() - } #[test] fn sane_good() { @@ -334,14 +189,6 @@ mod test { assert_eq!(rs.ranges.as_slice(), &[(1..5, 9), (5..7, 1), (8..9, 1),]); } - #[test] - fn query() { - let rs = RangedStates::from_slice(&[(1..4, 1u8), (5..7, 2)]); - assert_eq!(rs.query(&(0..1), |v| *v), None); - assert_eq!(rs.query(&(1..3), |v| *v), Some(Ok(1))); - assert_eq!(rs.query(&(1..6), |v| *v), Some(Err(()))); - } - #[test] fn isolate() { let rs = RangedStates::from_slice(&[(1..4, 9u8), (4..5, 9), (5..7, 1), (8..9, 1)]); @@ -356,104 +203,4 @@ mod test { &[(6..7, 1), (7..8, 0), (8..9, 1),] ); } - - #[test] - fn merge_same() { - assert_eq!( - &easy_merge(&[(1..4, 0u8),], &[(1..4, 2u8),],), - &[(1..4, Some(0)..Some(2)),] - ); - } - - #[test] - fn merge_empty() { - assert_eq!( - &easy_merge(&[(1..2, 0u8),], &[],), - &[(1..2, Some(0)..None),] - ); - assert_eq!( - &easy_merge(&[], &[(3..4, 1u8),],), - &[(3..4, None..Some(1)),] - ); - } - - #[test] - fn merge_separate() { - assert_eq!( - &easy_merge(&[(1..2, 0u8), (5..6, 1u8),], &[(2..4, 2u8),],), - &[ - (1..2, Some(0)..None), - (2..4, None..Some(2)), - (5..6, Some(1)..None), - ] - ); - } - - #[test] - fn merge_subset() { - assert_eq!( - &easy_merge(&[(1..6, 0u8),], &[(2..4, 2u8),],), - &[ - (1..2, Some(0)..None), - (2..4, Some(0)..Some(2)), - (4..6, Some(0)..None), - ] - ); - assert_eq!( - &easy_merge(&[(2..4, 0u8),], &[(1..4, 2u8),],), - &[(1..2, None..Some(2)), (2..4, Some(0)..Some(2)),] - ); - } - - #[test] - fn merge_all() { - assert_eq!( - &easy_merge(&[(1..4, 0u8), (5..8, 1u8),], &[(2..6, 2u8), (7..9, 3u8),],), - &[ - (1..2, Some(0)..None), - (2..4, Some(0)..Some(2)), - (4..5, None..Some(2)), - (5..6, Some(1)..Some(2)), - (6..7, Some(1)..None), - (7..8, Some(1)..Some(3)), - (8..9, None..Some(3)), - ] - ); - } - - #[test] - fn merge_complex() { - assert_eq!( - &easy_merge( - &[ - (0..8, 0u8), - (8..9, 1), - (9..16, 2), - (16..17, 3), - (17..118, 4), - (118..119, 5), - (119..124, 6), - (124..125, 7), - (125..512, 8), - ], - &[(15..16, 10u8), (51..52, 11), (126..127, 12),], - ), - &[ - (0..8, Some(0)..None), - (8..9, Some(1)..None), - (9..15, Some(2)..None), - (15..16, Some(2)..Some(10)), - (16..17, Some(3)..None), - (17..51, Some(4)..None), - (51..52, Some(4)..Some(11)), - (52..118, Some(4)..None), - (118..119, Some(5)..None), - (119..124, Some(6)..None), - (124..125, Some(7)..None), - (125..126, Some(8)..None), - (126..127, Some(8)..Some(12)), - (127..512, Some(8)..None), - ] - ); - } } diff --git a/wgpu-core/src/track/texture.rs b/wgpu-core/src/track/texture.rs index 2f789a3fb8..785a2799bd 100644 --- a/wgpu-core/src/track/texture.rs +++ b/wgpu-core/src/track/texture.rs @@ -7,7 +7,7 @@ use crate::{ invalid_resource_state, iterate_bitvec_indices, skip_barrier, ResourceMetadata, ResourceMetadataProvider, ResourceUses, UsageConflict, }, - Epoch, LifeGuard, RefCount, + LifeGuard, RefCount, }; use hal::TextureUses; @@ -748,7 +748,7 @@ unsafe fn insert_or_barrier_update( let currently_owned = resource_metadata.owned.get(index).unwrap_unchecked(); if !currently_owned { - let update_state_provider = end_state_provider.unwrap_or_else(|| start_state_provider); + let insert_state_provider = end_state_provider.unwrap_or_else(|| start_state_provider); insert( Some(texture_data), start_state, @@ -756,7 +756,7 @@ unsafe fn insert_or_barrier_update( resource_metadata, index32, index, - start_state_provider, + insert_state_provider, metadata_provider, ); return; @@ -780,7 +780,7 @@ unsafe fn insert_or_barrier_update( index32, index, update_state_provider, - ) + ); } #[inline(always)] From a7c309d9bdcec83557d2fc7221998be6bc10eff2 Mon Sep 17 00:00:00 2001 From: Connor Fitzgerald Date: Sun, 15 May 2022 19:06:14 -0400 Subject: [PATCH 25/35] temp24 - clippy cleanup --- wgpu-core/src/command/bundle.rs | 2 +- wgpu-core/src/command/clear.rs | 2 +- wgpu-core/src/command/compute.rs | 8 +-- wgpu-core/src/command/draw.rs | 9 ++-- wgpu-core/src/command/query.rs | 6 +-- wgpu-core/src/command/render.rs | 16 +++--- wgpu-core/src/command/transfer.rs | 14 +++--- wgpu-core/src/device/mod.rs | 18 +++---- wgpu-core/src/device/queue.rs | 4 +- wgpu-core/src/instance.rs | 3 ++ wgpu-core/src/lib.rs | 2 + wgpu-core/src/track/buffer.rs | 24 ++++----- wgpu-core/src/track/mod.rs | 4 +- wgpu-core/src/track/range.rs | 4 +- wgpu-core/src/track/stateless.rs | 84 +++++++++++-------------------- wgpu-core/src/track/texture.rs | 59 +++++++++++----------- 16 files changed, 118 insertions(+), 141 deletions(-) diff --git a/wgpu-core/src/command/bundle.rs b/wgpu-core/src/command/bundle.rs index 6d111ecc4d..c2a39001f3 100644 --- a/wgpu-core/src/command/bundle.rs +++ b/wgpu-core/src/command/bundle.rs @@ -234,7 +234,7 @@ impl RenderBundleEncoder { .trackers .bind_groups .extend(&*bind_group_guard, bind_group_id) - .ok_or_else(|| RenderCommandError::InvalidBindGroup(bind_group_id)) + .ok_or(RenderCommandError::InvalidBindGroup(bind_group_id)) .map_pass_err(scope)?; if bind_group.dynamic_binding_info.len() != offsets.len() { return Err(RenderCommandError::InvalidDynamicOffsetCount { diff --git a/wgpu-core/src/command/clear.rs b/wgpu-core/src/command/clear.rs index a0a7450f07..5dea08168f 100644 --- a/wgpu-core/src/command/clear.rs +++ b/wgpu-core/src/command/clear.rs @@ -90,7 +90,7 @@ impl Global { .trackers .buffers .change_state(&*buffer_guard, dst, hal::BufferUses::COPY_DST) - .ok_or_else(|| ClearError::InvalidBuffer(dst))?; + .ok_or(ClearError::InvalidBuffer(dst))?; let dst_raw = dst_buffer .raw .as_ref() diff --git a/wgpu-core/src/command/compute.rs b/wgpu-core/src/command/compute.rs index c2ce1e64ef..84d6c22112 100644 --- a/wgpu-core/src/command/compute.rs +++ b/wgpu-core/src/command/compute.rs @@ -397,7 +397,7 @@ impl Global { .trackers .bind_groups .extend(&*bind_group_guard, bind_group_id) - .ok_or_else(|| ComputePassErrorInner::InvalidBindGroup(bind_group_id)) + .ok_or(ComputePassErrorInner::InvalidBindGroup(bind_group_id)) .map_pass_err(scope)?; bind_group .validate_dynamic_bindings(&temp_offsets, &cmd_buf.limits) @@ -452,7 +452,7 @@ impl Global { .trackers .compute_pipelines .extend(&*pipeline_guard, pipeline_id) - .ok_or_else(|| ComputePassErrorInner::InvalidPipeline(pipeline_id)) + .ok_or(ComputePassErrorInner::InvalidPipeline(pipeline_id)) .map_pass_err(scope)?; unsafe { @@ -687,7 +687,7 @@ impl Global { .trackers .query_sets .extend(&*query_set_guard, query_set_id) - .ok_or_else(|| ComputePassErrorInner::InvalidQuerySet(query_set_id)) + .ok_or(ComputePassErrorInner::InvalidQuerySet(query_set_id)) .map_pass_err(scope)?; query_set @@ -704,7 +704,7 @@ impl Global { .trackers .query_sets .extend(&*query_set_guard, query_set_id) - .ok_or_else(|| ComputePassErrorInner::InvalidQuerySet(query_set_id)) + .ok_or(ComputePassErrorInner::InvalidQuerySet(query_set_id)) .map_pass_err(scope)?; query_set diff --git a/wgpu-core/src/command/draw.rs b/wgpu-core/src/command/draw.rs index 5e96fcddec..2206f3f204 100644 --- a/wgpu-core/src/command/draw.rs +++ b/wgpu-core/src/command/draw.rs @@ -104,9 +104,12 @@ impl crate::error::PrettyError for RenderCommandError { Self::InvalidPipeline(id) => { fmt.render_pipeline_label(&id); } - Self::UsageConflict(_, ..) | Self::DestroyedBuffer(_) => { - // fmt.buffer_label(&id); - todo!() + Self::UsageConflict(UsageConflict::TextureInvalid { id }) => { + fmt.texture_label(&id); + } + Self::UsageConflict(UsageConflict::BufferInvalid { id }) + | Self::DestroyedBuffer(id) => { + fmt.buffer_label(&id); } _ => {} }; diff --git a/wgpu-core/src/command/query.rs b/wgpu-core/src/command/query.rs index 911b304820..4140ceb82d 100644 --- a/wgpu-core/src/command/query.rs +++ b/wgpu-core/src/command/query.rs @@ -300,7 +300,7 @@ impl Global { .trackers .query_sets .extend(&*query_set_guard, query_set_id) - .ok_or_else(|| QueryError::InvalidQuerySet(query_set_id))?; + .ok_or(QueryError::InvalidQuerySet(query_set_id))?; query_set.validate_and_write_timestamp(raw_encoder, query_set_id, query_index, None)?; @@ -345,13 +345,13 @@ impl Global { .trackers .query_sets .extend(&*query_set_guard, query_set_id) - .ok_or_else(|| QueryError::InvalidQuerySet(query_set_id))?; + .ok_or(QueryError::InvalidQuerySet(query_set_id))?; let (dst_buffer, dst_pending) = cmd_buf .trackers .buffers .change_state(&*buffer_guard, destination, hal::BufferUses::COPY_DST) - .ok_or_else(|| QueryError::InvalidBuffer(destination))?; + .ok_or(QueryError::InvalidBuffer(destination))?; let dst_barrier = dst_pending.map(|pending| pending.into_hal(dst_buffer)); if !dst_buffer.usage.contains(wgt::BufferUsages::COPY_DST) { diff --git a/wgpu-core/src/command/render.rs b/wgpu-core/src/command/render.rs index ff4db69f96..2b93fe9de6 100644 --- a/wgpu-core/src/command/render.rs +++ b/wgpu-core/src/command/render.rs @@ -701,7 +701,7 @@ impl<'a, A: HalApi> RenderPassInfo<'a, A> { .trackers .views .extend(&*view_guard, at.view) - .ok_or_else(|| RenderPassErrorInner::InvalidAttachment(at.view))?; + .ok_or(RenderPassErrorInner::InvalidAttachment(at.view))?; check_multiview(view)?; add_view(view, "depth")?; @@ -810,7 +810,7 @@ impl<'a, A: HalApi> RenderPassInfo<'a, A> { .trackers .views .extend(&*view_guard, at.view) - .ok_or_else(|| RenderPassErrorInner::InvalidAttachment(at.view))?; + .ok_or(RenderPassErrorInner::InvalidAttachment(at.view))?; check_multiview(color_view)?; add_view(color_view, "color")?; @@ -840,7 +840,7 @@ impl<'a, A: HalApi> RenderPassInfo<'a, A> { .trackers .views .extend(&*view_guard, resolve_target) - .ok_or_else(|| RenderPassErrorInner::InvalidAttachment(resolve_target))?; + .ok_or(RenderPassErrorInner::InvalidAttachment(resolve_target))?; check_multiview(resolve_view)?; if color_view.extent != resolve_view.extent { @@ -1157,7 +1157,7 @@ impl Global { .trackers .bind_groups .extend(&*bind_group_guard, bind_group_id) - .ok_or_else(|| RenderCommandError::InvalidBindGroup(bind_group_id)) + .ok_or(RenderCommandError::InvalidBindGroup(bind_group_id)) .map_pass_err(scope)?; bind_group .validate_dynamic_bindings(&temp_offsets, &cmd_buf.limits) @@ -1221,7 +1221,7 @@ impl Global { .trackers .render_pipelines .extend(&*render_pipeline_guard, pipeline_id) - .ok_or_else(|| RenderCommandError::InvalidPipeline(pipeline_id)) + .ok_or(RenderCommandError::InvalidPipeline(pipeline_id)) .map_pass_err(scope)?; info.context @@ -1824,7 +1824,7 @@ impl Global { .trackers .query_sets .extend(&*query_set_guard, query_set_id) - .ok_or_else(|| RenderCommandError::InvalidQuerySet(query_set_id)) + .ok_or(RenderCommandError::InvalidQuerySet(query_set_id)) .map_pass_err(scope)?; query_set @@ -1846,7 +1846,7 @@ impl Global { .trackers .query_sets .extend(&*query_set_guard, query_set_id) - .ok_or_else(|| RenderCommandError::InvalidQuerySet(query_set_id)) + .ok_or(RenderCommandError::InvalidQuerySet(query_set_id)) .map_pass_err(scope)?; query_set @@ -1871,7 +1871,7 @@ impl Global { .trackers .bundles .extend(&*bundle_guard, bundle_id) - .ok_or_else(|| RenderCommandError::InvalidRenderBundle(bundle_id)) + .ok_or(RenderCommandError::InvalidRenderBundle(bundle_id)) .map_pass_err(scope)?; info.context diff --git a/wgpu-core/src/command/transfer.rs b/wgpu-core/src/command/transfer.rs index 78a6943230..f29462025f 100644 --- a/wgpu-core/src/command/transfer.rs +++ b/wgpu-core/src/command/transfer.rs @@ -518,7 +518,7 @@ impl Global { .trackers .buffers .change_state(&*buffer_guard, source, hal::BufferUses::COPY_SRC) - .ok_or_else(|| TransferError::InvalidBuffer(source))?; + .ok_or(TransferError::InvalidBuffer(source))?; let src_raw = src_buffer .raw .as_ref() @@ -533,7 +533,7 @@ impl Global { .trackers .buffers .change_state(&*buffer_guard, destination, hal::BufferUses::COPY_DST) - .ok_or_else(|| TransferError::InvalidBuffer(destination))?; + .ok_or(TransferError::InvalidBuffer(destination))?; let dst_raw = dst_buffer .raw .as_ref() @@ -652,7 +652,7 @@ impl Global { .trackers .buffers .change_state(&*buffer_guard, source.buffer, hal::BufferUses::COPY_SRC) - .ok_or_else(|| TransferError::InvalidBuffer(source.buffer))?; + .ok_or(TransferError::InvalidBuffer(source.buffer))?; let src_raw = src_buffer .raw .as_ref() @@ -671,7 +671,7 @@ impl Global { dst_range, hal::TextureUses::COPY_DST, ) - .ok_or_else(|| TransferError::InvalidTexture(destination.texture))?; + .ok_or(TransferError::InvalidTexture(destination.texture))?; let dst_raw = dst_texture .inner .as_raw() @@ -784,7 +784,7 @@ impl Global { src_range, hal::TextureUses::COPY_SRC, ) - .ok_or_else(|| TransferError::InvalidTexture(source.texture))?; + .ok_or(TransferError::InvalidTexture(source.texture))?; let src_raw = src_texture .inner .as_raw() @@ -802,7 +802,7 @@ impl Global { destination.buffer, hal::BufferUses::COPY_DST, ) - .ok_or_else(|| TransferError::InvalidBuffer(destination.buffer))?; + .ok_or(TransferError::InvalidBuffer(destination.buffer))?; let dst_raw = dst_buffer .raw .as_ref() @@ -934,7 +934,7 @@ impl Global { src_range, hal::TextureUses::COPY_SRC, ) - .ok_or_else(|| TransferError::InvalidTexture(source.texture))?; + .ok_or(TransferError::InvalidTexture(source.texture))?; let src_raw = src_texture .inner .as_raw() diff --git a/wgpu-core/src/device/mod.rs b/wgpu-core/src/device/mod.rs index 99f627916d..fb08ce5bdb 100644 --- a/wgpu-core/src/device/mod.rs +++ b/wgpu-core/src/device/mod.rs @@ -328,7 +328,6 @@ impl Device { } impl Device { - #[allow(clippy::too_many_arguments)] pub(crate) fn new( open: hal::OpenDevice, adapter_id: Stored, @@ -1487,7 +1486,6 @@ impl Device { }) } - #[allow(clippy::too_many_arguments)] fn create_buffer_binding<'a>( bb: &binding_model::BufferBinding, binding: u32, @@ -1545,7 +1543,7 @@ impl Device { let buffer = used .buffers .extend(storage, bb.buffer_id, internal_use) - .ok_or_else(|| Error::InvalidBuffer(bb.buffer_id))?; + .ok_or(Error::InvalidBuffer(bb.buffer_id))?; check_buffer_usage(buffer.usage, pub_usage)?; let raw_buffer = buffer .raw @@ -1631,9 +1629,9 @@ impl Device { Some(view.selector.clone()), internal_use, ) - .ok_or_else(|| { - binding_model::CreateBindGroupError::InvalidTexture(view.parent_id.value.0) - })?; + .ok_or(binding_model::CreateBindGroupError::InvalidTexture( + view.parent_id.value.0, + ))?; check_texture_usage(texture.desc.usage, pub_usage)?; used_texture_ranges.push(TextureInitTrackerAction { @@ -1740,7 +1738,7 @@ impl Device { let sampler = used .samplers .extend(&*sampler_guard, id) - .ok_or_else(|| Error::InvalidSampler(id))?; + .ok_or(Error::InvalidSampler(id))?; // Allowed sampler values for filtering and comparison let (allowed_filtering, allowed_comparison) = match ty { @@ -1789,7 +1787,7 @@ impl Device { let sampler = used .samplers .extend(&*sampler_guard, id) - .ok_or_else(|| Error::InvalidSampler(id))?; + .ok_or(Error::InvalidSampler(id))?; hal_samplers.push(&sampler.raw); } @@ -1799,7 +1797,7 @@ impl Device { let view = used .views .extend(&*texture_view_guard, id) - .ok_or_else(|| Error::InvalidTextureView(id))?; + .ok_or(Error::InvalidTextureView(id))?; let (pub_usage, internal_use) = Self::texture_use_parameters( binding, decl, @@ -1830,7 +1828,7 @@ impl Device { let view = used .views .extend(&*texture_view_guard, id) - .ok_or_else(|| Error::InvalidTextureView(id))?; + .ok_or(Error::InvalidTextureView(id))?; let (pub_usage, internal_use) = Self::texture_use_parameters(binding, decl, view, "SampledTextureArray, ReadonlyStorageTextureArray or WriteonlyStorageTextureArray")?; diff --git a/wgpu-core/src/device/queue.rs b/wgpu-core/src/device/queue.rs index c2ccd4fd92..a90b6780e7 100644 --- a/wgpu-core/src/device/queue.rs +++ b/wgpu-core/src/device/queue.rs @@ -287,7 +287,7 @@ impl Global { let (dst, transition) = trackers .buffers .change_state(&*buffer_guard, buffer_id, hal::BufferUses::COPY_DST) - .ok_or_else(|| TransferError::InvalidBuffer(buffer_id))?; + .ok_or(TransferError::InvalidBuffer(buffer_id))?; let dst_raw = dst .raw .as_ref() @@ -479,7 +479,7 @@ impl Global { selector, hal::TextureUses::COPY_DST, ) - .ok_or_else(|| TransferError::InvalidTexture(destination.texture))?; + .ok_or(TransferError::InvalidTexture(destination.texture))?; let (hal_copy_size, array_layer_count) = validate_texture_copy_range(destination, &dst.desc, CopySide::Destination, size)?; diff --git a/wgpu-core/src/instance.rs b/wgpu-core/src/instance.rs index d93d5d3d08..78ed3ab48f 100644 --- a/wgpu-core/src/instance.rs +++ b/wgpu-core/src/instance.rs @@ -494,6 +494,9 @@ impl Global { } #[cfg(dx12)] + /// Safety: + /// + /// The visual must be valid and able to be used to make a swapchain with. pub unsafe fn instance_create_surface_from_visual( &self, visual: *mut std::ffi::c_void, diff --git a/wgpu-core/src/lib.rs b/wgpu-core/src/lib.rs index e3036ad0f2..19bf4f5e64 100644 --- a/wgpu-core/src/lib.rs +++ b/wgpu-core/src/lib.rs @@ -16,6 +16,8 @@ clippy::new_without_default, // Needless updates are more scaleable, easier to play with features. clippy::needless_update, + // Need many arguments for some core functions to be able to re-use code in many situations. + clippy::too_many_arguments, // For some reason `rustc` can warn about these in const generics even // though they are required. unused_braces, diff --git a/wgpu-core/src/track/buffer.rs b/wgpu-core/src/track/buffer.rs index 6960a07654..3ac8b45c9a 100644 --- a/wgpu-core/src/track/buffer.rs +++ b/wgpu-core/src/track/buffer.rs @@ -114,7 +114,7 @@ impl BufferUsageScope { &mut self, bind_group: &BufferBindGroupState, ) -> Result<(), UsageConflict> { - for (id, ref_count, state) in &bind_group.buffers { + for &(id, ref ref_count, state) in &bind_group.buffers { let (index32, epoch, _) = id.0.unzip(); let index = index32 as usize; @@ -125,7 +125,7 @@ impl BufferUsageScope { &mut self.metadata, index32, index, - StateProvider::Direct { state: *state }, + StateProvider::Direct { state }, ResourceMetadataProvider::Direct { epoch, ref_count: Cow::Borrowed(ref_count), @@ -378,7 +378,7 @@ impl BufferTracker { self.set_size(incoming_size); } - for (id, ref_count, _) in bind_group_state.buffers.iter() { + for &(id, ref ref_count, _) in bind_group_state.buffers.iter() { let (index32, epoch, _) = id.0.unzip(); let index = index32 as usize; @@ -454,8 +454,8 @@ impl StateProvider<'_> { unsafe fn insert_or_merge( life_guard: Option<&LifeGuard>, - start_states: Option<&mut Vec>, - current_states: &mut Vec, + start_states: Option<&mut [BufferUses]>, + current_states: &mut [BufferUses], resource_metadata: &mut ResourceMetadata, index32: u32, index: usize, @@ -489,8 +489,8 @@ unsafe fn insert_or_merge( unsafe fn insert_or_barrier_update( life_guard: Option<&LifeGuard>, - start_states: Option<&mut Vec>, - current_states: &mut Vec, + start_states: Option<&mut [BufferUses]>, + current_states: &mut [BufferUses], resource_metadata: &mut ResourceMetadata, index32: u32, index: usize, @@ -530,8 +530,8 @@ unsafe fn insert_or_barrier_update( unsafe fn insert( life_guard: Option<&LifeGuard>, - start_states: Option<&mut Vec>, - current_states: &mut Vec, + start_states: Option<&mut [BufferUses]>, + current_states: &mut [BufferUses], resource_metadata: &mut ResourceMetadata, index: usize, start_state_provider: StateProvider<'_>, @@ -553,7 +553,7 @@ unsafe fn insert( } unsafe fn merge( - current_states: &mut Vec, + current_states: &mut [BufferUses], index32: u32, index: usize, state_provider: StateProvider<'_>, @@ -578,7 +578,7 @@ unsafe fn merge( } unsafe fn barrier( - current_states: &mut Vec, + current_states: &mut [BufferUses], index32: u32, index: usize, state_provider: StateProvider<'_>, @@ -599,7 +599,7 @@ unsafe fn barrier( } unsafe fn update( - current_states: &mut Vec, + current_states: &mut [BufferUses], index: usize, state_provider: StateProvider<'_>, ) { diff --git a/wgpu-core/src/track/mod.rs b/wgpu-core/src/track/mod.rs index 3652b8781e..97257f985e 100644 --- a/wgpu-core/src/track/mod.rs +++ b/wgpu-core/src/track/mod.rs @@ -118,7 +118,7 @@ fn iterate_bitvec_indices(ownership: &BitVec) -> impl Iterator Self { Self::Texture { - id: id, + id, mip_levels: selector.mips, array_layers: selector.layers, invalid_use: InvalidUse { diff --git a/wgpu-core/src/track/range.rs b/wgpu-core/src/track/range.rs index 890f8d5eab..12c527fb2a 100644 --- a/wgpu-core/src/track/range.rs +++ b/wgpu-core/src/track/range.rs @@ -77,8 +77,8 @@ impl RangedStates { ) -> impl Iterator, &T)> + 'a { self.ranges .iter() - .filter(move |(inner, ..)| inner.end > range.start && inner.start < range.end) - .map(move |(inner, v)| { + .filter(move |&&(ref inner, ..)| inner.end > range.start && inner.start < range.end) + .map(move |&(ref inner, ref v)| { let new_range = inner.start.max(range.start)..inner.end.min(range.end); (new_range, v) diff --git a/wgpu-core/src/track/stateless.rs b/wgpu-core/src/track/stateless.rs index 96015b27c2..3ff1c0b820 100644 --- a/wgpu-core/src/track/stateless.rs +++ b/wgpu-core/src/track/stateless.rs @@ -1,12 +1,10 @@ use std::marker::PhantomData; -use bit_vec::BitVec; - use crate::{ hub, id::{TypedId, Valid}, - track::{iterate_bitvec_indices, resize_bitvec}, - Epoch, RefCount, + track::{iterate_bitvec_indices, ResourceMetadata}, + RefCount, }; pub(crate) struct StatelessBindGroupSate { @@ -44,57 +42,34 @@ impl StatelessBindGroupSate { } pub(crate) struct StatelessTracker { - ref_counts: Vec>, - epochs: Vec, - - owned: BitVec, + metadata: ResourceMetadata, - _phantom: PhantomData<(A, T, Id)>, + _phantom: PhantomData<(T, Id)>, } impl StatelessTracker { pub fn new() -> Self { Self { - ref_counts: Vec::new(), - epochs: Vec::new(), - - owned: BitVec::default(), + metadata: ResourceMetadata::new(), _phantom: PhantomData, } } fn debug_assert_in_bounds(&self, index: usize) { - debug_assert!(index < self.ref_counts.len()); - debug_assert!(index < self.epochs.len()); - debug_assert!(index < self.owned.len()); - - debug_assert!(if self.owned.get(index).unwrap() { - self.ref_counts[index].is_some() - } else { - true - }); + self.metadata.debug_assert_in_bounds(index); } pub fn set_size(&mut self, size: usize) { - self.epochs.resize(size, u32::MAX); - self.ref_counts.resize(size, None); - - resize_bitvec(&mut self.owned, size); + self.metadata.set_size(size); } pub fn used(&self) -> impl Iterator> + '_ { - if !self.owned.is_empty() { - self.debug_assert_in_bounds(self.owned.len() - 1) - }; - iterate_bitvec_indices(&self.owned).map(move |index| { - let epoch = unsafe { *self.epochs.get_unchecked(index) }; - Valid(Id::zip(index as u32, epoch, A::VARIANT)) - }) + self.metadata.used() } fn allow_index(&mut self, index: usize) { - if index >= self.owned.len() { + if index >= self.metadata.owned.len() { self.set_size(index + 1); } } @@ -108,9 +83,9 @@ impl StatelessTracker { self.debug_assert_in_bounds(index); unsafe { - *self.epochs.get_unchecked_mut(index) = epoch; - *self.ref_counts.get_unchecked_mut(index) = Some(ref_count); - self.owned.set(index, true); + *self.metadata.epochs.get_unchecked_mut(index) = epoch; + *self.metadata.ref_counts.get_unchecked_mut(index) = Some(ref_count); + self.metadata.owned.set(index, true); } } @@ -126,38 +101,39 @@ impl StatelessTracker { self.debug_assert_in_bounds(index); unsafe { - *self.epochs.get_unchecked_mut(index) = epoch; - *self.ref_counts.get_unchecked_mut(index) = Some(item.life_guard().add_ref()); - self.owned.set(index, true); + *self.metadata.epochs.get_unchecked_mut(index) = epoch; + *self.metadata.ref_counts.get_unchecked_mut(index) = Some(item.life_guard().add_ref()); + self.metadata.owned.set(index, true); } Some(item) } pub fn extend_from_tracker(&mut self, other: &Self) { - let incoming_size = other.owned.len(); - if incoming_size > self.owned.len() { + let incoming_size = other.metadata.owned.len(); + if incoming_size > self.metadata.owned.len() { self.set_size(incoming_size); } - for index in iterate_bitvec_indices(&other.owned) { + for index in iterate_bitvec_indices(&other.metadata.owned) { self.debug_assert_in_bounds(index); other.debug_assert_in_bounds(index); unsafe { - let previously_owned = self.owned.get(index).unwrap_unchecked(); + let previously_owned = self.metadata.owned.get(index).unwrap_unchecked(); if !previously_owned { - self.owned.set(index, true); + self.metadata.owned.set(index, true); let other_ref_count = other + .metadata .ref_counts .get_unchecked(index) .clone() .unwrap_unchecked(); - *self.ref_counts.get_unchecked_mut(index) = Some(other_ref_count); + *self.metadata.ref_counts.get_unchecked_mut(index) = Some(other_ref_count); - let epoch = *other.epochs.get_unchecked(index); - *self.epochs.get_unchecked_mut(index) = epoch; + let epoch = *other.metadata.epochs.get_unchecked(index); + *self.metadata.epochs.get_unchecked_mut(index) = epoch; } } } @@ -167,23 +143,21 @@ impl StatelessTracker { let (index32, epoch, _) = id.0.unzip(); let index = index32 as usize; - if index > self.owned.len() { + if index > self.metadata.owned.len() { return false; } self.debug_assert_in_bounds(index); unsafe { - if self.owned.get(index).unwrap_unchecked() { - let existing_epoch = self.epochs.get_unchecked_mut(index); - let existing_ref_count = self.ref_counts.get_unchecked_mut(index); + if self.metadata.owned.get(index).unwrap_unchecked() { + let existing_epoch = self.metadata.epochs.get_unchecked_mut(index); + let existing_ref_count = self.metadata.ref_counts.get_unchecked_mut(index); if *existing_epoch == epoch && existing_ref_count.as_mut().unwrap_unchecked().load() == 1 { - self.owned.set(index, false); - *existing_epoch = u32::MAX; - *existing_ref_count = None; + self.metadata.reset(index); return true; } diff --git a/wgpu-core/src/track/texture.rs b/wgpu-core/src/track/texture.rs index 785a2799bd..95f0366ae0 100644 --- a/wgpu-core/src/track/texture.rs +++ b/wgpu-core/src/track/texture.rs @@ -70,7 +70,7 @@ impl ComplexTextureState { for (selector, desired_state) in state_iter { let mips = selector.mips.start as usize..selector.mips.end as usize; for mip in &mut complex.mips[mips] { - for (_, state) in mip.isolate(&selector.layers, TextureUses::UNKNOWN) { + for &mut (_, ref mut state) in mip.isolate(&selector.layers, TextureUses::UNKNOWN) { *state = desired_state; } } @@ -78,19 +78,19 @@ impl ComplexTextureState { complex } - fn into_selector_state_iter( + fn to_selector_state_iter( &self, ) -> impl Iterator + Clone + '_ { self.mips.iter().enumerate().flat_map(|(mip, inner)| { let mip = mip as u32; { - inner.iter().map(move |(layers, inner)| { + inner.iter().map(move |&(ref layers, inner)| { ( TextureSelector { mips: mip..mip + 1, layers: layers.clone(), }, - *inner, + inner, ) }) } @@ -240,8 +240,8 @@ impl TextureUsageScope { storage: &hub::Storage, TextureId>, bind_group: &TextureBindGroupState, ) -> Result<(), UsageConflict> { - for (id, selector, ref_count, state) in &bind_group.textures { - self.extend_refcount(storage, *id, selector.clone(), ref_count, *state)?; + for &(id, ref selector, ref ref_count, state) in &bind_group.textures { + self.extend_refcount(storage, id, selector.clone(), ref_count, state)?; } Ok(()) @@ -271,7 +271,7 @@ impl TextureUsageScope { index, LayeredStateProvider::from_option(selector, new_state), ResourceMetadataProvider::Direct { - epoch: epoch, + epoch, ref_count: Cow::Borrowed(ref_count), }, )?; @@ -379,7 +379,7 @@ impl TextureTracker { index, LayeredStateProvider::KnownSingle { state: usage }, ResourceMetadataProvider::Direct { - epoch: epoch, + epoch, ref_count: Cow::Owned(ref_count), }, ) @@ -606,9 +606,9 @@ where type Item = D; fn next(&mut self) -> Option { - match self { - Either::Left(inner) => inner.next(), - Either::Right(inner) => inner.next(), + match *self { + Either::Left(ref mut inner) => inner.next(), + Either::Right(ref mut inner) => inner.next(), } } } @@ -635,10 +635,7 @@ enum LayeredStateProvider<'a> { impl<'a> LayeredStateProvider<'a> { fn from_option(selector: Option, state: TextureUses) -> Self { match selector { - Some(selector) => Self::Selector { - selector: selector, - state, - }, + Some(selector) => Self::Selector { selector, state }, None => Self::KnownSingle { state }, } } @@ -668,7 +665,7 @@ impl<'a> LayeredStateProvider<'a> { if new_state == TextureUses::COMPLEX { let new_complex = set.complex.get(&index32).unwrap_unchecked(); - SingleOrManyStates::Many(Either::Right(new_complex.into_selector_state_iter())) + SingleOrManyStates::Many(Either::Right(new_complex.to_selector_state_iter())) } else { SingleOrManyStates::Single(new_state) } @@ -748,7 +745,7 @@ unsafe fn insert_or_barrier_update( let currently_owned = resource_metadata.owned.get(index).unwrap_unchecked(); if !currently_owned { - let insert_state_provider = end_state_provider.unwrap_or_else(|| start_state_provider); + let insert_state_provider = end_state_provider.unwrap_or(start_state_provider); insert( Some(texture_data), start_state, @@ -875,7 +872,7 @@ unsafe fn merge( if invalid_resource_state(merged_state) { return Err(UsageConflict::from_texture( TextureId::zip(index32, metadata_provider.get_epoch(index), A::VARIANT), - selector.clone(), + selector, *current_simple, new_state, )); @@ -884,7 +881,7 @@ unsafe fn merge( for mip in &mut new_complex.mips[selector.mips.start as usize..selector.mips.end as usize] { - for (_, current_layer_state) in + for &mut (_, ref mut current_layer_state) in mip.isolate(&selector.layers, TextureUses::UNKNOWN) { *current_layer_state = merged_state; @@ -901,7 +898,7 @@ unsafe fn merge( for (mip_id, mip) in current_complex.mips.iter_mut().enumerate() { let mip_id = mip_id as u32; - for (layers, current_layer_state) in mip.iter_mut() { + for &mut (ref layers, ref mut current_layer_state) in mip.iter_mut() { let merged_state = *current_layer_state | new_simple; // Once we remove unknown, this will never be empty, as simple states are never unknown. @@ -932,7 +929,7 @@ unsafe fn merge( let mip = current_complex.mips.get_unchecked_mut(mip_id as usize); - for (layers, current_layer_state) in + for &mut (ref layers, ref mut current_layer_state) in mip.isolate(&selector.layers, TextureUses::UNKNOWN) { let merged_state = *current_layer_state | new_state; @@ -1020,12 +1017,12 @@ unsafe fn barrier( for (mip_id, mip) in current_complex.mips.iter().enumerate() { let mip_id = mip_id as u32; - for (layers, current_layer_state) in mip.iter() { - if *current_layer_state == TextureUses::UNKNOWN { + for &(ref layers, current_layer_state) in mip.iter() { + if current_layer_state == TextureUses::UNKNOWN { continue; } - if skip_barrier(*current_layer_state, new_simple) { + if skip_barrier(current_layer_state, new_simple) { continue; } @@ -1035,7 +1032,7 @@ unsafe fn barrier( mips: mip_id..mip_id + 1, layers: layers.clone(), }, - usage: *current_layer_state..new_simple, + usage: current_layer_state..new_simple, }); } } @@ -1119,7 +1116,7 @@ unsafe fn update( for mip in &mut new_complex.mips[selector.mips.start as usize..selector.mips.end as usize] { - for (_, current_layer_state) in + for &mut (_, ref mut current_layer_state) in mip.isolate(&selector.layers, TextureUses::UNKNOWN) { *current_layer_state = new_state; @@ -1134,15 +1131,15 @@ unsafe fn update( } (SingleOrManyStates::Many(current_complex), SingleOrManyStates::Single(new_single)) => { for (mip_id, mip) in current_complex.mips.iter().enumerate() { - for (layers, current_layer_state) in mip.iter() { + for &(ref layers, current_layer_state) in mip.iter() { // If this state is unknown, that means that the start is _also_ unknown. - if *current_layer_state == TextureUses::UNKNOWN { + if current_layer_state == TextureUses::UNKNOWN { if let Some(&mut ref mut start_complex) = start_complex { debug_assert!(mip_id < start_complex.mips.len()); let start_mip = start_complex.mips.get_unchecked_mut(mip_id); - for (_, current_start_state) in + for &mut (_, ref mut current_start_state) in start_mip.isolate(layers, TextureUses::UNKNOWN) { debug_assert_eq!(*current_start_state, TextureUses::UNKNOWN); @@ -1174,7 +1171,7 @@ unsafe fn update( let mip = current_complex.mips.get_unchecked_mut(mip_id); - for (layers, current_layer_state) in + for &mut (ref layers, ref mut current_layer_state) in mip.isolate(&selector.layers, TextureUses::UNKNOWN) { if *current_layer_state == TextureUses::UNKNOWN @@ -1185,7 +1182,7 @@ unsafe fn update( let start_mip = start_complex.mips.get_unchecked_mut(mip_id); - for (_, current_start_state) in + for &mut (_, ref mut current_start_state) in start_mip.isolate(layers, TextureUses::UNKNOWN) { debug_assert_eq!(*current_start_state, TextureUses::UNKNOWN); From dbe615a82b625a9bbf7f2d309c88638efeb352c3 Mon Sep 17 00:00:00 2001 From: Connor Fitzgerald Date: Sun, 15 May 2022 19:18:05 -0400 Subject: [PATCH 26/35] temp25 - Add inline markers on buffer functions --- wgpu-core/src/track/buffer.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/wgpu-core/src/track/buffer.rs b/wgpu-core/src/track/buffer.rs index 3ac8b45c9a..12806f1097 100644 --- a/wgpu-core/src/track/buffer.rs +++ b/wgpu-core/src/track/buffer.rs @@ -444,6 +444,7 @@ enum StateProvider<'a> { Indirect { state: &'a [BufferUses] }, } impl StateProvider<'_> { + #[inline(always)] unsafe fn get_state(&self, index: usize) -> BufferUses { match *self { StateProvider::Direct { state } => state, @@ -452,6 +453,7 @@ impl StateProvider<'_> { } } +#[inline(always)] unsafe fn insert_or_merge( life_guard: Option<&LifeGuard>, start_states: Option<&mut [BufferUses]>, @@ -487,6 +489,7 @@ unsafe fn insert_or_merge( ) } +#[inline(always)] unsafe fn insert_or_barrier_update( life_guard: Option<&LifeGuard>, start_states: Option<&mut [BufferUses]>, @@ -528,6 +531,7 @@ unsafe fn insert_or_barrier_update( update(start_state_set, index, update_state_provider); } +#[inline(always)] unsafe fn insert( life_guard: Option<&LifeGuard>, start_states: Option<&mut [BufferUses]>, @@ -552,6 +556,7 @@ unsafe fn insert( *resource_metadata.ref_counts.get_unchecked_mut(index) = Some(ref_count); } +#[inline(always)] unsafe fn merge( current_states: &mut [BufferUses], index32: u32, @@ -577,6 +582,7 @@ unsafe fn merge( Ok(()) } +#[inline(always)] unsafe fn barrier( current_states: &mut [BufferUses], index32: u32, @@ -598,6 +604,7 @@ unsafe fn barrier( }) } +#[inline(always)] unsafe fn update( current_states: &mut [BufferUses], index: usize, From 1bbff998081c4c26b2ab2f34521d40f5d710511e Mon Sep 17 00:00:00 2001 From: Connor Fitzgerald Date: Mon, 16 May 2022 18:23:25 -0400 Subject: [PATCH 27/35] temp26 - Fix buffer trackers --- wgpu-core/src/track/buffer.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/wgpu-core/src/track/buffer.rs b/wgpu-core/src/track/buffer.rs index 12806f1097..b62ecf4759 100644 --- a/wgpu-core/src/track/buffer.rs +++ b/wgpu-core/src/track/buffer.rs @@ -527,8 +527,7 @@ unsafe fn insert_or_barrier_update( barriers, ); - let start_state_set = start_states.unwrap(); - update(start_state_set, index, update_state_provider); + update(current_states, index, update_state_provider); } #[inline(always)] From aa85535d4222e7d96323ce061a010338e72ab00b Mon Sep 17 00:00:00 2001 From: Connor Fitzgerald Date: Mon, 16 May 2022 18:40:56 -0400 Subject: [PATCH 28/35] temp27 - fixed texture insert to handle both sides --- wgpu-core/src/track/texture.rs | 41 ++++++++++++++++++++++++++-------- 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/wgpu-core/src/track/texture.rs b/wgpu-core/src/track/texture.rs index 95f0366ae0..f4e5fe0246 100644 --- a/wgpu-core/src/track/texture.rs +++ b/wgpu-core/src/track/texture.rs @@ -378,6 +378,7 @@ impl TextureTracker { index32, index, LayeredStateProvider::KnownSingle { state: usage }, + None, ResourceMetadataProvider::Direct { epoch, ref_count: Cow::Owned(ref_count), @@ -714,6 +715,7 @@ unsafe fn insert_or_merge( index32, index, state_provider, + None, metadata_provider, ); return Ok(()); @@ -745,7 +747,6 @@ unsafe fn insert_or_barrier_update( let currently_owned = resource_metadata.owned.get(index).unwrap_unchecked(); if !currently_owned { - let insert_state_provider = end_state_provider.unwrap_or(start_state_provider); insert( Some(texture_data), start_state, @@ -753,7 +754,8 @@ unsafe fn insert_or_barrier_update( resource_metadata, index32, index, - insert_state_provider, + start_state_provider, + end_state_provider, metadata_provider, ); return; @@ -788,17 +790,19 @@ unsafe fn insert( resource_metadata: &mut ResourceMetadata, index32: u32, index: usize, - state_provider: LayeredStateProvider<'_>, + start_state_provider: LayeredStateProvider<'_>, + end_state_provider: Option>, metadata_provider: ResourceMetadataProvider<'_, A>, ) { - match state_provider.get_layers(texture_data, index32, index) { + let start_layers = start_state_provider.get_layers(texture_data, index32, index); + match start_layers { SingleOrManyStates::Single(state) => { - let reference = end_state.simple.get_unchecked_mut(index); - if let Some(start_state) = start_state { *start_state.simple.get_unchecked_mut(index) = state; } - *reference = state; + if end_state_provider.is_none() { + *end_state.simple.get_unchecked_mut(index) = state; + } } SingleOrManyStates::Many(state_iter) => { let full_range = texture_data.unwrap().1.clone(); @@ -809,8 +813,27 @@ unsafe fn insert( *start_state.simple.get_unchecked_mut(index) = TextureUses::COMPLEX; start_state.complex.insert(index32, complex.clone()); } - *end_state.simple.get_unchecked_mut(index) = TextureUses::COMPLEX; - end_state.complex.insert(index32, complex); + + if end_state_provider.is_none() { + *end_state.simple.get_unchecked_mut(index) = TextureUses::COMPLEX; + end_state.complex.insert(index32, complex); + } + } + } + + if let Some(end_state_provider) = end_state_provider { + match end_state_provider.get_layers(texture_data, index32, index) { + SingleOrManyStates::Single(state) => { + *end_state.simple.get_unchecked_mut(index) = state; + } + SingleOrManyStates::Many(state_iter) => { + let full_range = texture_data.unwrap().1.clone(); + + let complex = ComplexTextureState::from_selector_state_iter(full_range, state_iter); + + *end_state.simple.get_unchecked_mut(index) = TextureUses::COMPLEX; + end_state.complex.insert(index32, complex); + } } } From 65192417a36966ece8ae5687864f71894692e78b Mon Sep 17 00:00:00 2001 From: Connor Fitzgerald Date: Mon, 16 May 2022 23:08:27 -0400 Subject: [PATCH 29/35] temp28 - document tracker and usages --- wgpu-core/src/track/mod.rs | 85 +++++++++++++++ wgpu-core/src/track/texture.rs | 185 --------------------------------- wgpu-hal/src/lib.rs | 68 ++++++++---- 3 files changed, 132 insertions(+), 206 deletions(-) diff --git a/wgpu-core/src/track/mod.rs b/wgpu-core/src/track/mod.rs index 97257f985e..0e2834e496 100644 --- a/wgpu-core/src/track/mod.rs +++ b/wgpu-core/src/track/mod.rs @@ -1,3 +1,88 @@ +/*! Resource State and Lifetime Trackers + * + * These structures are responsible for keeping track of resource state, + * generating barriers where needed, and making sure resources are kept + * alive until the trackers die. + * + * ## General Architecture + * + * Tracking is some of the hottest code in the entire codebase, so the trackers + * are designed to be as cache efficient as possible. They store resource state + * in flat vectors, storing metadata SOA style, one vector per type of metadata. + * + * In wgpu, resource IDs are allocated and re-used, so will always be as low + * as reasonably possible. This allows us to use the ID as an index into an array. + * + * ## Statefulness + * + * There are two main types of trackers, stateful and stateless. + * + * Stateful trackers are for buffers and textures. They both have + * resource state attached to them which needs to be used to generate + * automatic synchronization. Because of the different requirements of + * buffers and textures, they have two separate tracking structures. + * + * Stateless trackers only store metadata and own the given resource. + * + * ## Use Case + * + * Within each type of tracker, the trackers are further split into 3 different + * use cases, Bind Group, Usage Scope, and a full Tracker. + * + * Bind Group trackers are just a list of different resources, their refcount, + * and how they are used. Textures are used via a selector and a usage type. + * Buffers by just a usage type. Stateless resources don't have a usage type. + * + * Usage Scope trackers are only for stateful resources. These trackers represent + * a single [`UsageScope`] in the spec. When a use is added to a usage scope, + * it is merged with all other uses of that resource in that scope. If there + * is a usage conflict, merging will fail and an error will be reported. + * + * Full trackers represent a before and after state of a resource. These + * are used for tracking on the device and on command buffers. The before + * state represents the state the resource is first used as in the command buffer, + * the after state is the state the command buffer leaves the resource in. + * These double ended buffers can then be used to generate the needed transitions + * between command buffers. + * + * ## Dense Datastructure with Sparse Data + * + * This tracking system is based on having completely dense data, but trackers do + * not always contain every resource. Some resources (or even most resources) go + * unused in any given command buffer. So to help speed up the process of iterating + * through possibly thousands of resources, we use a bit vector to represent if + * a resource is in the buffer or not. This allows us extremely efficient memory + * utilization, as well as being able to bail out of whole blocks of 32-64 resources + * with a single usize comparison with zero. In practice this means that merging + * partially resident buffers is extremely quick. + * + * The main advantage of this dense datastructure is that we can do merging + * of trackers in an extremely efficient fashion that results in us doing linear + * scans down a couple of buffers. CPUs and their caches absolutely eat this up. + * + * ## Stateful Resource Operations + * + * All operations on stateful trackers boil down to one of four operations: + * - `insert(tracker, new_state)` adds a resource with a given state to the tracker + * for the first time. + * - `merge(tracker, new_state)` merges this new state with the previous state, checking + * for usage conflicts. + * - `barrier(tracker, new_state)` compares the given state to the existing state and + * generates the needed barriers. + * - `update(tracker, new_state)` takes the given new state and overrides the old state. + * + * This allows us to compose the operations to form the various kinds of tracker merges + * that need to happen in the codebase. For each resource in the given merger, the following + * operation applies: + * + * UsageScope <- Resource = insert(scope, usage) OR merge(scope, usage) + * UsageScope <- UsageScope = insert(scope, scope) OR merge(scope, scope) + * CommandBuffer <- UsageScope = insert(buffer.start, buffer.end, scope) OR barrier(buffer.end, scope) + update(buffer.end, scope) + * Deivce <- CommandBuffer = insert(device.start, device.end, buffer.start, buffer.end) OR barrier(device.end, buffer.start) + update(device.end, buffer.end) + * + * [`UsageScope`]: https://gpuweb.github.io/gpuweb/#programming-model-synchronization +!*/ + mod buffer; mod range; mod stateless; diff --git a/wgpu-core/src/track/texture.rs b/wgpu-core/src/track/texture.rs index f4e5fe0246..7a290fc47e 100644 --- a/wgpu-core/src/track/texture.rs +++ b/wgpu-core/src/track/texture.rs @@ -1225,188 +1225,3 @@ unsafe fn update( } } } - -// #[cfg(test)] -// mod test { -// use super::*; - -// struct TrackingTest { -// life_guard: LifeGuard, -// selector: TextureSelector, -// set: TextureStateSet, -// metadata: ResourceMetadata, -// } - -// fn setup(mips: Range, layers: Range, states: &[TextureUses]) -> TrackingTest { -// let life_guard = LifeGuard::new("test"); -// let selector = TextureSelector { mips, layers }; - -// let mut set = TextureStateSet::new(); -// let mut metadata = ResourceMetadata::::new(); - -// set.set_size(states.len()); -// metadata.set_size(states.len()); - -// for &state in states { -// set.simple[0] = state; - -// metadata.owned.set(0, true); -// metadata.ref_counts[0] = None; // Okay because we only read this through metadata provider -// metadata.epochs[0] = 1; -// } - -// TrackingTest { -// life_guard, -// selector, -// set, -// metadata, -// } -// } - -// #[test] -// fn simple_transition() { -// let mut test = setup(0..1, 0..1, &[TextureUses::UNINITIALIZED]); - -// let mut barriers = Vec::new(); -// unsafe { -// let _ = state_combine( -// Some((&test.life_guard, &test.selector)), -// None, -// &mut test.set, -// &mut test.metadata, -// 0, -// 0, -// LayeredStateProvider::KnownSingle { -// state: TextureUses::RESOURCE, -// }, -// ResourceMetadataProvider::Direct { -// epoch: 1, -// ref_count: Cow::Borrowed(test.life_guard.ref_count.as_ref().unwrap()), -// }, -// Some(&mut barriers), -// ); -// }; -// assert_eq!(test.set.simple, &[TextureUses::RESOURCE]); -// assert_eq!( -// barriers, -// &[PendingTransition { -// id: 0, -// selector: test.selector, -// usage: TextureUses::UNINITIALIZED..TextureUses::RESOURCE -// }] -// ); -// } - -// #[test] -// fn simple_merger() { -// let mut test = setup(0..1, 0..1, &[TextureUses::COPY_SRC]); - -// unsafe { -// let res = state_combine( -// Some((&test.life_guard, &test.selector)), -// None, -// &mut test.set, -// &mut test.metadata, -// 0, -// 0, -// LayeredStateProvider::KnownSingle { -// state: TextureUses::RESOURCE, -// }, -// ResourceMetadataProvider::Direct { -// epoch: 1, -// ref_count: Cow::Borrowed(test.life_guard.ref_count.as_ref().unwrap()), -// }, -// None, -// ); -// assert_eq!(res, Ok(())); -// }; -// assert_eq!( -// test.set.simple, -// &[TextureUses::COPY_SRC | TextureUses::RESOURCE] -// ); -// } - -// #[test] -// fn simple_to_complex_transition() { -// let mut test = setup(0..1, 0..2, &[TextureUses::RESOURCE]); - -// let transition_selector = TextureSelector { -// mips: 0..1, -// layers: 1..2, -// }; - -// let mut barriers = Vec::new(); -// unsafe { -// let _ = state_combine( -// Some((&test.life_guard, &test.selector)), -// None, -// &mut test.set, -// &mut test.metadata, -// 0, -// 0, -// LayeredStateProvider::Selector { -// selector: transition_selector.clone(), -// state: TextureUses::STORAGE_WRITE, -// }, -// ResourceMetadataProvider::Direct { -// epoch: 1, -// ref_count: Cow::Borrowed(test.life_guard.ref_count.as_ref().unwrap()), -// }, -// Some(&mut barriers), -// ); -// }; -// assert_eq!(test.set.simple, &[TextureUses::COMPLEX]); -// assert_eq!( -// barriers, -// &[PendingTransition { -// id: 0, -// selector: transition_selector, -// usage: TextureUses::RESOURCE..TextureUses::STORAGE_WRITE -// }] -// ); -// } - -// #[test] -// fn simple_to_complex_merger() { -// let mut test = setup(0..1, 0..2, &[TextureUses::RESOURCE]); - -// let transition_selector = TextureSelector { -// mips: 0..1, -// layers: 1..2, -// }; - -// unsafe { -// let res = state_combine( -// Some((&test.life_guard, &test.selector)), -// None, -// &mut test.set, -// &mut test.metadata, -// 0, -// 0, -// LayeredStateProvider::Selector { -// selector: transition_selector.clone(), -// state: TextureUses::DEPTH_STENCIL_READ, -// }, -// ResourceMetadataProvider::Direct { -// epoch: 1, -// ref_count: Cow::Borrowed(test.life_guard.ref_count.as_ref().unwrap()), -// }, -// None, -// ); -// assert_eq!(res, Ok(())); -// }; -// assert_eq!(test.set.simple, &[TextureUses::COMPLEX]); -// let complex = test.set.complex.get(&0).unwrap(); -// let mip0: Vec<_> = complex.mips[0].iter().cloned().collect(); -// assert_eq!( -// mip0, -// &[ -// (0..1, TextureUses::RESOURCE), -// ( -// 1..2, -// TextureUses::RESOURCE | TextureUses::DEPTH_STENCIL_READ -// ) -// ] -// ); -// } -// } diff --git a/wgpu-hal/src/lib.rs b/wgpu-hal/src/lib.rs index d8752e6a10..05ecd8d60b 100644 --- a/wgpu-hal/src/lib.rs +++ b/wgpu-hal/src/lib.rs @@ -627,26 +627,39 @@ bitflags!( bitflags::bitflags! { /// Similar to `wgt::BufferUsages` but for internal use. pub struct BufferUses: u16 { + /// The argument to a read-only mapping. const MAP_READ = 1 << 0; + /// The argument to a write-only mapping. const MAP_WRITE = 1 << 1; + /// The source of a hardware copy. const COPY_SRC = 1 << 2; + /// The destination of a hardware copy. const COPY_DST = 1 << 3; + /// The index buffer used for drawing. const INDEX = 1 << 4; + /// A vertex buffer used for drawing. const VERTEX = 1 << 5; + /// A uniform buffer bound in a bind group. const UNIFORM = 1 << 6; + /// A read-only storage buffer used in a bind group. + /// + /// Can be combined with STORAGE_WRITE to create a read-write usage. const STORAGE_READ = 1 << 7; + /// A read-write or write only buffer used for binding + /// + /// Can be combined with STORAGE_READ to create a read-write usage. const STORAGE_WRITE = 1 << 8; + /// The indirect or count buffer in a indirect draw or dispatch. const INDIRECT = 1 << 9; - /// The combination of usages that can be used together (read-only). + /// The combination of states that a buffer may be in _at the same time_. const INCLUSIVE = Self::MAP_READ.bits | Self::COPY_SRC.bits | Self::INDEX.bits | Self::VERTEX.bits | Self::UNIFORM.bits | Self::STORAGE_READ.bits | Self::INDIRECT.bits; - /// The combination of exclusive usages (write-only and read-write). - /// These usages may still show up with others, but can't automatically be combined. + /// The combination of states that a buffer must exclusively be in. const EXCLUSIVE = Self::MAP_WRITE.bits | Self::COPY_DST.bits | Self::STORAGE_WRITE.bits; /// The combination of all usages that the are guaranteed to be be ordered by the hardware. - /// If a usage is not ordered, then even if it doesn't change between draw calls, there - /// still need to be pipeline barriers inserted for synchronization. + /// If a usage is ordered, then if the buffer state doesn't change between draw calls, there + /// are no barriers needed for synchronization. const ORDERED = Self::INCLUSIVE.bits | Self::MAP_WRITE.bits; } } @@ -654,26 +667,39 @@ bitflags::bitflags! { bitflags::bitflags! { /// Similar to `wgt::TextureUsages` but for internal use. pub struct TextureUses: u16 { - const PRESENT = 1 << 0; - const COPY_SRC = 1 << 1; - const COPY_DST = 1 << 2; - const RESOURCE = 1 << 3; - const COLOR_TARGET = 1 << 4; - const DEPTH_STENCIL_READ = 1 << 5; - const DEPTH_STENCIL_WRITE = 1 << 6; - const STORAGE_READ = 1 << 7; - const STORAGE_WRITE = 1 << 8; - /// The combination of usages that can be used together (read-only). + /// The texture is in unknown state. + const UNINITIALIZED = 1 << 0; + /// Ready to present image to the surface. + const PRESENT = 1 << 1; + /// The source of a hardware copy. + const COPY_SRC = 1 << 2; + /// The destination of a hardware copy. + const COPY_DST = 1 << 3; + /// Read-only sampled or fetched resource. + const RESOURCE = 1 << 4; + /// The color target of a renderpass. + const COLOR_TARGET = 1 << 5; + /// Read-only depth stencil usage. + const DEPTH_STENCIL_READ = 1 << 6; + /// Read-write depth stencil usage + const DEPTH_STENCIL_WRITE = 1 << 7; + /// Read-only storage buffer usage. Corresponds to a UAV in d3d, so is exclusive, despite being read only. + const STORAGE_READ = 1 << 8; + /// Read-write storage buffer usage. + const STORAGE_WRITE = 1 << 9; + /// The combination of states that a texture may be in _at the same time_. const INCLUSIVE = Self::COPY_SRC.bits | Self::RESOURCE.bits | Self::DEPTH_STENCIL_READ.bits; - /// The combination of exclusive usages (write-only and read-write). - /// These usages may still show up with others, but can't automatically be combined. + /// The combination of states that a texture must exclusively be in. const EXCLUSIVE = Self::COPY_DST.bits | Self::COLOR_TARGET.bits | Self::DEPTH_STENCIL_WRITE.bits | Self::STORAGE_READ.bits | Self::STORAGE_WRITE.bits | Self::PRESENT.bits; /// The combination of all usages that the are guaranteed to be be ordered by the hardware. - /// If a usage is not ordered, then even if it doesn't change between draw calls, there - /// still need to be pipeline barriers inserted for synchronization. + /// If a usage is ordered, then if the texture state doesn't change between draw calls, there + /// are no barriers needed for synchronization. const ORDERED = Self::INCLUSIVE.bits | Self::COLOR_TARGET.bits | Self::DEPTH_STENCIL_WRITE.bits | Self::STORAGE_READ.bits; - const COMPLEX = 1 << 9; - const UNINITIALIZED = 1 << 10; + + /// Flag used by the wgpu-core texture tracker to say a texture is in different states for every sub-resource + const COMPLEX = 1 << 10; + /// Flag used by the wgpu-core texture tracker to say that the tracker does not know the state of the sub-resource. + /// This is different from UNINITIALIZED as that says the tracker does know, but the texture has not been initialized. const UNKNOWN = 1 << 11; } } From 0b8ce175f118193b1c2716c00c5abed00c83973f Mon Sep 17 00:00:00 2001 From: Connor Fitzgerald Date: Tue, 17 May 2022 00:02:38 -0400 Subject: [PATCH 30/35] temp29 - document track/mod.rs --- wgpu-core/src/command/compute.rs | 2 +- wgpu-core/src/command/render.rs | 2 +- wgpu-core/src/track/buffer.rs | 4 - wgpu-core/src/track/mod.rs | 170 ++++++++++++++++++++++++++----- wgpu-core/src/track/texture.rs | 4 - 5 files changed, 149 insertions(+), 33 deletions(-) diff --git a/wgpu-core/src/command/compute.rs b/wgpu-core/src/command/compute.rs index a40d59bc12..a73fd0ef46 100644 --- a/wgpu-core/src/command/compute.rs +++ b/wgpu-core/src/command/compute.rs @@ -272,7 +272,7 @@ impl State { for id in self.binder.list_active() { unsafe { - base_trackers.extend_from_bind_group( + base_trackers.change_states_from_bind_group( texture_guard, &mut self.scope, &bind_group_guard[id].used, diff --git a/wgpu-core/src/command/render.rs b/wgpu-core/src/command/render.rs index 8a9d61451c..c22c206e5a 100644 --- a/wgpu-core/src/command/render.rs +++ b/wgpu-core/src/command/render.rs @@ -1932,7 +1932,7 @@ impl Global { .map_pass_err(scope)?; cmd_buf .trackers - .extend_from_render_bundle(&bundle.used) + .change_state_from_render_bundle(&bundle.used) .map_pass_err(scope)?; }; state.reset_bundle(); diff --git a/wgpu-core/src/track/buffer.rs b/wgpu-core/src/track/buffer.rs index b62ecf4759..3a37fd76c2 100644 --- a/wgpu-core/src/track/buffer.rs +++ b/wgpu-core/src/track/buffer.rs @@ -30,10 +30,6 @@ impl ResourceUses for BufferUses { fn any_exclusive(self) -> bool { self.intersects(Self::EXCLUSIVE) } - - fn uninit(self) -> bool { - false - } } pub(crate) struct BufferBindGroupState { diff --git a/wgpu-core/src/track/mod.rs b/wgpu-core/src/track/mod.rs index 0e2834e496..4cbcc3a0bd 100644 --- a/wgpu-core/src/track/mod.rs +++ b/wgpu-core/src/track/mod.rs @@ -10,6 +10,12 @@ * are designed to be as cache efficient as possible. They store resource state * in flat vectors, storing metadata SOA style, one vector per type of metadata. * + * A lot of the tracker code is deeply unsafe, using unchecked accesses all over + * to make performance as good as possible. However, for all unsafe accesses, there + * is a corresponding debug assert the checks if that access is valid. This helps + * get bugs caught fast, while still letting users not need to pay for the bounds + * checks. + * * In wgpu, resource IDs are allocated and re-used, so will always be as low * as reasonably possible. This allows us to use the ID as an index into an array. * @@ -142,48 +148,62 @@ impl PendingTransition { debug_assert_ne!(self.usage.start, hal::TextureUses::UNKNOWN); debug_assert_ne!(self.usage.end, hal::TextureUses::UNKNOWN); + let mip_count = self.selector.mips.end - self.selector.mips.start; + debug_assert_ne!(mip_count, 0); + let layer_count = self.selector.layers.end - self.selector.layers.start; + debug_assert_ne!(layer_count, 0); + hal::TextureBarrier { texture, range: wgt::ImageSubresourceRange { aspect: wgt::TextureAspect::All, base_mip_level: self.selector.mips.start, - mip_level_count: NonZeroU32::new(self.selector.mips.end - self.selector.mips.start), + mip_level_count: unsafe { Some(NonZeroU32::new_unchecked(mip_count)) }, base_array_layer: self.selector.layers.start, - array_layer_count: NonZeroU32::new( - self.selector.layers.end - self.selector.layers.start, - ), + array_layer_count: unsafe { Some(NonZeroU32::new_unchecked(layer_count)) }, }, usage: self.usage, } } } -pub trait ResourceUses: +/// The uses that a resource or subresource can be in. +pub(crate) trait ResourceUses: fmt::Debug + ops::BitAnd + ops::BitOr + PartialEq + Sized + Copy { + /// All flags that are exclusive. const EXCLUSIVE: Self; + /// The relevant resource ID type. type Id: Copy + fmt::Debug + TypedId; + /// The selector used by this resource. type Selector: fmt::Debug; + /// Turn the resource into a pile of bits. fn bits(self) -> u16; + /// Returns true if the all the uses are ordered. fn all_ordered(self) -> bool; + /// Returns true if any of the uses are exclusive. fn any_exclusive(self) -> bool; - fn uninit(self) -> bool; } +/// Returns true if the given states violates the usage scope rule +/// of any(inclusive) XOR one(exclusive) fn invalid_resource_state(state: T) -> bool { // Is power of two also means "is one bit set". We check for this as if // we're in any exclusive state, we must only be in a single state. state.any_exclusive() && !conv::is_power_of_two_u16(state.bits()) } +/// Returns true if the transition from one state to another does not require +/// a barrier. fn skip_barrier(old_state: T, new_state: T) -> bool { // If the state didn't change and all the usages are ordered, the hardware // will guarentee the order of accesses, so we do not need to issue a barrier at all old_state == new_state && old_state.all_ordered() } +/// Resizes the given bitvec to the given size. I'm not sure why this is hard to do but it is. fn resize_bitvec(vec: &mut BitVec, size: usize) { let owned_size_to_grow = size.checked_sub(vec.len()); if let Some(delta) = owned_size_to_grow { @@ -195,6 +215,9 @@ fn resize_bitvec(vec: &mut BitVec, size: usize) { } } +/// Produces an iterator that yields the indexes of all bits that are set in the bitvec. +/// +/// Will skip entire usize's worth of bits if they are all false. fn iterate_bitvec_indices(ownership: &BitVec) -> impl Iterator + '_ { const BITS_PER_BLOCK: usize = mem::size_of::() * 8; @@ -269,6 +292,7 @@ impl UsageConflict { } } +/// Pretty print helper that shows helpful descriptions of a conflicting usage. #[derive(Clone, Debug, PartialEq)] pub struct InvalidUse { current_state: T, @@ -285,6 +309,7 @@ impl fmt::Display for InvalidUse { let exclusive = current_exclusive | new_exclusive; + // The text starts with "tried to use X resource with {self}" write!( f, "conflicting usages. Current usage {current:?} and new usage {new:?}. \ @@ -294,6 +319,10 @@ impl fmt::Display for InvalidUse { } } +/// SOA container for storing metadata of a resource. +/// +/// This contins the ownership bitvec, the refcount of +/// the resource, and the epoch of the object's full ID. #[derive(Debug)] pub(crate) struct ResourceMetadata { owned: BitVec, @@ -320,6 +349,10 @@ impl ResourceMetadata { resize_bitvec(&mut self.owned, size); } + /// Ensures a given index is in bounds for all arrays and does + /// sanity checks of the presence of a refcount. + /// + /// In release mode this function is completely empty and is removed. fn debug_assert_in_bounds(&self, index: usize) { debug_assert!(index < self.owned.len()); debug_assert!(index < self.ref_counts.len()); @@ -332,10 +365,14 @@ impl ResourceMetadata { }); } + /// Returns true if the tracker owns no resources. + /// + /// This is a O(n) operation. fn is_empty(&self) -> bool { !self.owned.any() } + /// Returns ids for all resources we own. fn used(&self) -> impl Iterator> + '_ { if !self.owned.is_empty() { self.debug_assert_in_bounds(self.owned.len() - 1) @@ -346,6 +383,7 @@ impl ResourceMetadata { }) } + /// Resets the metadata for a given index to sane "invalid" values. unsafe fn reset(&mut self, index: usize) { *self.ref_counts.get_unchecked_mut(index) = None; *self.epochs.get_unchecked_mut(index) = u32::MAX; @@ -353,46 +391,72 @@ impl ResourceMetadata { } } +/// A source of resource metadata. +/// +/// This is used to abstract over the various places +/// trackers can get new resource metadata from. enum ResourceMetadataProvider<'a, A: hub::HalApi> { + /// Comes directly from explicit values. Direct { epoch: Epoch, ref_count: Cow<'a, RefCount>, }, - Indirect { - metadata: &'a ResourceMetadata, - }, - Resource { - epoch: Epoch, - }, + /// Comes from another metadata tracker. + Indirect { metadata: &'a ResourceMetadata }, + /// The epoch is given directly, but the life count comes from the resource itself. + Resource { epoch: Epoch }, } impl ResourceMetadataProvider<'_, A> { + /// Get the epoch and an owned refcount from this. + /// + /// # Safety + /// + /// - The index must be in bounds of the metadata tracker if this uses an indirect source. + /// - life_guard must be Some if this uses a Resource source. unsafe fn get_own(self, life_guard: Option<&LifeGuard>, index: usize) -> (Epoch, RefCount) { match self { ResourceMetadataProvider::Direct { epoch, ref_count } => { (epoch, ref_count.into_owned()) } - ResourceMetadataProvider::Indirect { metadata } => ( - *metadata.epochs.get_unchecked(index), - metadata - .ref_counts - .get_unchecked(index) - .clone() - .unwrap_unchecked(), - ), - ResourceMetadataProvider::Resource { epoch } => (epoch, life_guard.unwrap().add_ref()), + ResourceMetadataProvider::Indirect { metadata } => { + metadata.debug_assert_in_bounds(index); + ( + *metadata.epochs.get_unchecked(index), + metadata + .ref_counts + .get_unchecked(index) + .clone() + .unwrap_unchecked(), + ) + } + ResourceMetadataProvider::Resource { epoch } => { + debug_assert!(life_guard.is_some()); + (epoch, life_guard.unwrap_unchecked().add_ref()) + } } } + /// Get the epoch from this. + /// + /// # Safety + /// + /// - The index must be in bounds of the metadata tracker if this uses an indirect source. unsafe fn get_epoch(self, index: usize) -> Epoch { match self { ResourceMetadataProvider::Direct { epoch, .. } | ResourceMetadataProvider::Resource { epoch, .. } => epoch, ResourceMetadataProvider::Indirect { metadata } => { + metadata.debug_assert_in_bounds(index); *metadata.epochs.get_unchecked(index) } } } } +/// All the usages that a bind group contains. The uses are not deduplicated in any way +/// and may include conflicting uses. This is fully compliant by the WebGPU spec. +/// +/// All bind group states are sorted by their ID so that when adding to a tracker, +/// they are added in the most efficient order possible (assending order). pub(crate) struct BindGroupStates { pub buffers: BufferBindGroupState, pub textures: TextureBindGroupState, @@ -410,6 +474,7 @@ impl BindGroupStates { } } + /// Sort all uses. pub fn optimize(&mut self) { self.buffers.optimize(); self.textures.optimize(); @@ -418,6 +483,9 @@ impl BindGroupStates { } } +/// This is a render bundle specific usage scope. It includes stateless resources +/// that are not normally included in a usage scope, but are used by render bundles +/// and need to be owned by the render bundles. pub(crate) struct RenderBundleScope { pub buffers: BufferUsageScope, pub textures: TextureUsageScope, @@ -428,6 +496,7 @@ pub(crate) struct RenderBundleScope { } impl RenderBundleScope { + /// Create the render bundle scope and pull the maximum IDs from the hubs. pub fn new( buffers: &hub::Storage, id::BufferId>, textures: &hub::Storage, id::TextureId>, @@ -452,6 +521,15 @@ impl RenderBundleScope { value } + /// Merge the inner contents of a bind group into the render bundle tracker. + /// + /// Only stateful things are merged in here, all other resources are owned + /// indirectly by the bind group. + /// + /// # Safety + /// + /// The maximum ID given by each bind group resource must be less than the + /// length of the storage given at the call to `new`. pub unsafe fn extend_from_bind_group( &mut self, textures: &hub::Storage, id::TextureId>, @@ -465,6 +543,8 @@ impl RenderBundleScope { } } +/// A usage scope tracker. Only needs to store stateful resources as stateless +/// resources cannot possibly have a usage conflict. #[derive(Debug)] pub(crate) struct UsageScope { pub buffers: BufferUsageScope, @@ -472,6 +552,7 @@ pub(crate) struct UsageScope { } impl UsageScope { + /// Create the render bundle scope and pull the maximum IDs from the hubs. pub fn new( buffers: &hub::Storage, id::BufferId>, textures: &hub::Storage, id::TextureId>, @@ -487,6 +568,15 @@ impl UsageScope { value } + /// Merge the inner contents of a bind group into the usage scope. + /// + /// Only stateful things are merged in here, all other resources are owned + /// indirectly by the bind group. + /// + /// # Safety + /// + /// The maximum ID given by each bind group resource must be less than the + /// length of the storage given at the call to `new`. pub unsafe fn extend_from_bind_group( &mut self, textures: &hub::Storage, id::TextureId>, @@ -499,6 +589,15 @@ impl UsageScope { Ok(()) } + /// Merge the inner contents of a bind group into the usage scope. + /// + /// Only stateful things are merged in here, all other resources are owned + /// indirectly by a bind group or are merged directly into the command buffer tracker. + /// + /// # Safety + /// + /// The maximum ID given by each bind group resource must be less than the + /// length of the storage given at the call to `new`. pub unsafe fn extend_from_render_bundle( &mut self, textures: &hub::Storage, id::TextureId>, @@ -512,6 +611,7 @@ impl UsageScope { } } +/// A full double sided tracker used by CommandBuffers and the Device. pub(crate) struct Tracker { pub buffers: BufferTracker, pub textures: TextureTracker, @@ -539,6 +639,7 @@ impl Tracker { } } + /// Pull the maximum IDs from the hubs. pub fn set_size( &mut self, buffers: Option<&hub::Storage, id::BufferId>>, @@ -582,7 +683,23 @@ impl Tracker { }; } - pub unsafe fn extend_from_bind_group( + /// Uses the the indices of the resources used in a bind group to + /// remove them from the usage scope, and transition to the state + /// given by the usage scope. + /// + /// This is weird method used for compute dispatches. Bind groups are added + /// to the usage scope in order to combine the uses, then this method acts + /// as a sparse way of only changing the needed resoures based on which resources + /// the bind groups touched. + /// + /// Only stateful things are merged in here, all other resources are owned + /// indirectly by the bind group. + /// + /// # Safety + /// + /// The maximum ID given by each bind group resource must be less than the + /// value given to `set_size` + pub unsafe fn change_states_from_bind_group( &mut self, textures: &hub::Storage, id::TextureId>, scope: &mut UsageScope, @@ -594,7 +711,14 @@ impl Tracker { .change_states_bind_group(textures, &mut scope.textures, &bind_group.textures); } - pub unsafe fn extend_from_render_bundle( + /// Tracks the stateless resources from the given renderbundle. It is expected + /// that the stateful resources will get merged into a usage scope first. + /// + /// # Safety + /// + /// The maximum ID given by each bind group resource must be less than the + /// value given to `set_size` + pub unsafe fn change_state_from_render_bundle( &mut self, render_bundle: &RenderBundleScope, ) -> Result<(), UsageConflict> { diff --git a/wgpu-core/src/track/texture.rs b/wgpu-core/src/track/texture.rs index 7a290fc47e..6d2468833c 100644 --- a/wgpu-core/src/track/texture.rs +++ b/wgpu-core/src/track/texture.rs @@ -39,10 +39,6 @@ impl ResourceUses for TextureUses { fn any_exclusive(self) -> bool { self.intersects(Self::EXCLUSIVE) } - - fn uninit(self) -> bool { - self == Self::UNINITIALIZED - } } #[derive(Clone, Debug, Default, PartialEq)] From e79eb4b6707ed8aadb0f9fff0abb9164bf460baf Mon Sep 17 00:00:00 2001 From: Connor Fitzgerald Date: Wed, 18 May 2022 18:01:22 -0400 Subject: [PATCH 31/35] temp30 - convert STORAGE_WRITE to STORAGE_READ_WRITE --- wgpu-core/src/conv.rs | 4 ++-- wgpu-core/src/device/mod.rs | 8 ++++---- wgpu-core/src/track/mod.rs | 2 ++ wgpu-hal/src/dx12/command.rs | 4 ++-- wgpu-hal/src/dx12/conv.rs | 8 ++++---- wgpu-hal/src/dx12/device.rs | 9 ++++----- wgpu-hal/src/gles/command.rs | 12 ++++++++++-- wgpu-hal/src/gles/queue.rs | 8 ++++---- wgpu-hal/src/lib.rs | 16 ++++++---------- wgpu-hal/src/metal/conv.rs | 4 ++-- wgpu-hal/src/vulkan/conv.rs | 14 +++++++------- 11 files changed, 47 insertions(+), 42 deletions(-) diff --git a/wgpu-core/src/conv.rs b/wgpu-core/src/conv.rs index 253cd24d3f..83fb72fab6 100644 --- a/wgpu-core/src/conv.rs +++ b/wgpu-core/src/conv.rs @@ -55,7 +55,7 @@ pub fn map_buffer_usage(usage: wgt::BufferUsages) -> hal::BufferUses { usage.contains(wgt::BufferUsages::UNIFORM), ); u.set( - hal::BufferUses::STORAGE_READ | hal::BufferUses::STORAGE_WRITE, + hal::BufferUses::STORAGE_READ | hal::BufferUses::STORAGE_READ_WRITE, usage.contains(wgt::BufferUsages::STORAGE), ); u.set( @@ -83,7 +83,7 @@ pub fn map_texture_usage( usage.contains(wgt::TextureUsages::TEXTURE_BINDING), ); u.set( - hal::TextureUses::STORAGE_READ | hal::TextureUses::STORAGE_WRITE, + hal::TextureUses::STORAGE_READ | hal::TextureUses::STORAGE_READ_WRITE, usage.contains(wgt::TextureUsages::STORAGE_BINDING), ); let is_color = aspect.contains(hal::FormatAspects::COLOR); diff --git a/wgpu-core/src/device/mod.rs b/wgpu-core/src/device/mod.rs index d550d118f2..467926cf04 100644 --- a/wgpu-core/src/device/mod.rs +++ b/wgpu-core/src/device/mod.rs @@ -971,7 +971,7 @@ impl Device { wgt::TextureViewDimension::D3 => { hal::TextureUses::RESOURCE | hal::TextureUses::STORAGE_READ - | hal::TextureUses::STORAGE_WRITE + | hal::TextureUses::STORAGE_READ_WRITE } _ => hal::TextureUses::all(), }; @@ -1524,7 +1524,7 @@ impl Device { if read_only { hal::BufferUses::STORAGE_READ } else { - hal::BufferUses::STORAGE_READ | hal::BufferUses::STORAGE_WRITE + hal::BufferUses::STORAGE_READ_WRITE }, limits.max_storage_buffer_binding_size, ), @@ -2028,7 +2028,7 @@ impl Device { } let internal_use = match access { - wgt::StorageTextureAccess::WriteOnly => hal::TextureUses::STORAGE_WRITE, + wgt::StorageTextureAccess::WriteOnly => hal::TextureUses::STORAGE_READ_WRITE, wgt::StorageTextureAccess::ReadOnly => { if !view .format_features @@ -2048,7 +2048,7 @@ impl Device { return Err(Error::StorageReadNotSupported(view.desc.format)); } - hal::TextureUses::STORAGE_WRITE | hal::TextureUses::STORAGE_READ + hal::TextureUses::STORAGE_READ_WRITE } }; Ok((wgt::TextureUsages::STORAGE_BINDING, internal_use)) diff --git a/wgpu-core/src/track/mod.rs b/wgpu-core/src/track/mod.rs index 4cbcc3a0bd..1723cfbbef 100644 --- a/wgpu-core/src/track/mod.rs +++ b/wgpu-core/src/track/mod.rs @@ -413,6 +413,7 @@ impl ResourceMetadataProvider<'_, A> { /// /// - The index must be in bounds of the metadata tracker if this uses an indirect source. /// - life_guard must be Some if this uses a Resource source. + #[inline(always)] unsafe fn get_own(self, life_guard: Option<&LifeGuard>, index: usize) -> (Epoch, RefCount) { match self { ResourceMetadataProvider::Direct { epoch, ref_count } => { @@ -440,6 +441,7 @@ impl ResourceMetadataProvider<'_, A> { /// # Safety /// /// - The index must be in bounds of the metadata tracker if this uses an indirect source. + #[inline(always)] unsafe fn get_epoch(self, index: usize) -> Epoch { match self { ResourceMetadataProvider::Direct { epoch, .. } diff --git a/wgpu-hal/src/dx12/command.rs b/wgpu-hal/src/dx12/command.rs index 95546d213a..dc653911c5 100644 --- a/wgpu-hal/src/dx12/command.rs +++ b/wgpu-hal/src/dx12/command.rs @@ -287,7 +287,7 @@ impl crate::CommandEncoder for super::CommandEncoder { StateAfter: s1, }; self.temp.barriers.push(raw); - } else if barrier.usage.start == crate::BufferUses::STORAGE_WRITE { + } else if barrier.usage.start == crate::BufferUses::STORAGE_READ_WRITE { let mut raw = d3d12::D3D12_RESOURCE_BARRIER { Type: d3d12::D3D12_RESOURCE_BARRIER_TYPE_UAV, Flags: d3d12::D3D12_RESOURCE_BARRIER_FLAG_NONE, @@ -382,7 +382,7 @@ impl crate::CommandEncoder for super::CommandEncoder { } } } - } else if barrier.usage.start == crate::TextureUses::STORAGE_WRITE { + } else if barrier.usage.start == crate::TextureUses::STORAGE_READ_WRITE { let mut raw = d3d12::D3D12_RESOURCE_BARRIER { Type: d3d12::D3D12_RESOURCE_BARRIER_TYPE_UAV, Flags: d3d12::D3D12_RESOURCE_BARRIER_FLAG_NONE, diff --git a/wgpu-hal/src/dx12/conv.rs b/wgpu-hal/src/dx12/conv.rs index 6ea0339afb..8dfd40c92b 100644 --- a/wgpu-hal/src/dx12/conv.rs +++ b/wgpu-hal/src/dx12/conv.rs @@ -3,7 +3,7 @@ use winapi::um::{d3d12, d3dcommon}; pub fn map_buffer_usage_to_resource_flags(usage: crate::BufferUses) -> d3d12::D3D12_RESOURCE_FLAGS { let mut flags = 0; - if usage.contains(crate::BufferUses::STORAGE_WRITE) { + if usage.contains(crate::BufferUses::STORAGE_READ_WRITE) { flags |= d3d12::D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS; } flags @@ -33,7 +33,7 @@ pub fn map_texture_usage_to_resource_flags( flags |= d3d12::D3D12_RESOURCE_FLAG_DENY_SHADER_RESOURCE; } } - if usage.contains(crate::TextureUses::STORAGE_WRITE) { + if usage.contains(crate::TextureUses::STORAGE_READ_WRITE) { flags |= d3d12::D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS; } @@ -130,7 +130,7 @@ pub fn map_buffer_usage_to_state(usage: crate::BufferUses) -> d3d12::D3D12_RESOU if usage.intersects(Bu::VERTEX | Bu::UNIFORM) { state |= d3d12::D3D12_RESOURCE_STATE_VERTEX_AND_CONSTANT_BUFFER; } - if usage.intersects(Bu::STORAGE_WRITE) { + if usage.intersects(Bu::STORAGE_READ_WRITE) { state |= d3d12::D3D12_RESOURCE_STATE_UNORDERED_ACCESS; } else if usage.intersects(Bu::STORAGE_READ) { state |= d3d12::D3D12_RESOURCE_STATE_PIXEL_SHADER_RESOURCE @@ -170,7 +170,7 @@ pub fn map_texture_usage_to_state(usage: crate::TextureUses) -> d3d12::D3D12_RES if usage.intersects(Tu::DEPTH_STENCIL_WRITE) { state |= d3d12::D3D12_RESOURCE_STATE_DEPTH_WRITE; } - if usage.intersects(Tu::STORAGE_READ | Tu::STORAGE_WRITE) { + if usage.intersects(Tu::STORAGE_READ | Tu::STORAGE_READ_WRITE) { state |= d3d12::D3D12_RESOURCE_STATE_UNORDERED_ACCESS; } state diff --git a/wgpu-hal/src/dx12/device.rs b/wgpu-hal/src/dx12/device.rs index 2f92cab44c..c25f58c767 100644 --- a/wgpu-hal/src/dx12/device.rs +++ b/wgpu-hal/src/dx12/device.rs @@ -428,7 +428,7 @@ impl crate::Device for super::Device { || !desc.usage.intersects( crate::TextureUses::RESOURCE | crate::TextureUses::STORAGE_READ - | crate::TextureUses::STORAGE_WRITE, + | crate::TextureUses::STORAGE_READ_WRITE, ) { auxil::dxgi::conv::map_texture_format(desc.format) } else { @@ -516,10 +516,9 @@ impl crate::Device for super::Device { } else { None }, - handle_uav: if desc - .usage - .intersects(crate::TextureUses::STORAGE_READ | crate::TextureUses::STORAGE_WRITE) - { + handle_uav: if desc.usage.intersects( + crate::TextureUses::STORAGE_READ | crate::TextureUses::STORAGE_READ_WRITE, + ) { let raw_desc = view_desc.to_uav(); let handle = self.srv_uav_pool.lock().alloc_handle(); self.raw.CreateUnorderedAccessView( diff --git a/wgpu-hal/src/gles/command.rs b/wgpu-hal/src/gles/command.rs index e316844b66..5a4b50462e 100644 --- a/wgpu-hal/src/gles/command.rs +++ b/wgpu-hal/src/gles/command.rs @@ -230,7 +230,11 @@ impl crate::CommandEncoder for super::CommandEncoder { } for bar in barriers { // GLES only synchronizes storage -> anything explicitly - if !bar.usage.start.contains(crate::BufferUses::STORAGE_WRITE) { + if !bar + .usage + .start + .contains(crate::BufferUses::STORAGE_READ_WRITE) + { continue; } self.cmd_buffer @@ -253,7 +257,11 @@ impl crate::CommandEncoder for super::CommandEncoder { let mut combined_usage = crate::TextureUses::empty(); for bar in barriers { // GLES only synchronizes storage -> anything explicitly - if !bar.usage.start.contains(crate::TextureUses::STORAGE_WRITE) { + if !bar + .usage + .start + .contains(crate::TextureUses::STORAGE_READ_WRITE) + { continue; } // unlike buffers, there is no need for a concrete texture diff --git a/wgpu-hal/src/gles/queue.rs b/wgpu-hal/src/gles/queue.rs index c223c2a851..85c1c0758d 100644 --- a/wgpu-hal/src/gles/queue.rs +++ b/wgpu-hal/src/gles/queue.rs @@ -787,9 +787,9 @@ impl super::Queue { if usage.intersects(crate::BufferUses::MAP_READ | crate::BufferUses::MAP_WRITE) { flags |= glow::BUFFER_UPDATE_BARRIER_BIT; } - if usage - .intersects(crate::BufferUses::STORAGE_READ | crate::BufferUses::STORAGE_WRITE) - { + if usage.intersects( + crate::BufferUses::STORAGE_READ | crate::BufferUses::STORAGE_READ_WRITE, + ) { flags |= glow::SHADER_STORAGE_BARRIER_BIT; } gl.memory_barrier(flags); @@ -800,7 +800,7 @@ impl super::Queue { flags |= glow::TEXTURE_FETCH_BARRIER_BIT; } if usage.intersects( - crate::TextureUses::STORAGE_READ | crate::TextureUses::STORAGE_WRITE, + crate::TextureUses::STORAGE_READ | crate::TextureUses::STORAGE_READ_WRITE, ) { flags |= glow::SHADER_IMAGE_ACCESS_BARRIER_BIT; } diff --git a/wgpu-hal/src/lib.rs b/wgpu-hal/src/lib.rs index 05ecd8d60b..8702a9f92b 100644 --- a/wgpu-hal/src/lib.rs +++ b/wgpu-hal/src/lib.rs @@ -642,13 +642,9 @@ bitflags::bitflags! { /// A uniform buffer bound in a bind group. const UNIFORM = 1 << 6; /// A read-only storage buffer used in a bind group. - /// - /// Can be combined with STORAGE_WRITE to create a read-write usage. const STORAGE_READ = 1 << 7; - /// A read-write or write only buffer used for binding - /// - /// Can be combined with STORAGE_READ to create a read-write usage. - const STORAGE_WRITE = 1 << 8; + /// A read-write or write-only buffer used in a bind group. + const STORAGE_READ_WRITE = 1 << 8; /// The indirect or count buffer in a indirect draw or dispatch. const INDIRECT = 1 << 9; /// The combination of states that a buffer may be in _at the same time_. @@ -656,7 +652,7 @@ bitflags::bitflags! { Self::INDEX.bits | Self::VERTEX.bits | Self::UNIFORM.bits | Self::STORAGE_READ.bits | Self::INDIRECT.bits; /// The combination of states that a buffer must exclusively be in. - const EXCLUSIVE = Self::MAP_WRITE.bits | Self::COPY_DST.bits | Self::STORAGE_WRITE.bits; + const EXCLUSIVE = Self::MAP_WRITE.bits | Self::COPY_DST.bits | Self::STORAGE_READ_WRITE.bits; /// The combination of all usages that the are guaranteed to be be ordered by the hardware. /// If a usage is ordered, then if the buffer state doesn't change between draw calls, there /// are no barriers needed for synchronization. @@ -685,12 +681,12 @@ bitflags::bitflags! { const DEPTH_STENCIL_WRITE = 1 << 7; /// Read-only storage buffer usage. Corresponds to a UAV in d3d, so is exclusive, despite being read only. const STORAGE_READ = 1 << 8; - /// Read-write storage buffer usage. - const STORAGE_WRITE = 1 << 9; + /// Read-write or write-only storage buffer usage. + const STORAGE_READ_WRITE = 1 << 9; /// The combination of states that a texture may be in _at the same time_. const INCLUSIVE = Self::COPY_SRC.bits | Self::RESOURCE.bits | Self::DEPTH_STENCIL_READ.bits; /// The combination of states that a texture must exclusively be in. - const EXCLUSIVE = Self::COPY_DST.bits | Self::COLOR_TARGET.bits | Self::DEPTH_STENCIL_WRITE.bits | Self::STORAGE_READ.bits | Self::STORAGE_WRITE.bits | Self::PRESENT.bits; + const EXCLUSIVE = Self::COPY_DST.bits | Self::COLOR_TARGET.bits | Self::DEPTH_STENCIL_WRITE.bits | Self::STORAGE_READ.bits | Self::STORAGE_READ_WRITE.bits | Self::PRESENT.bits; /// The combination of all usages that the are guaranteed to be be ordered by the hardware. /// If a usage is ordered, then if the texture state doesn't change between draw calls, there /// are no barriers needed for synchronization. diff --git a/wgpu-hal/src/metal/conv.rs b/wgpu-hal/src/metal/conv.rs index f275a74ec9..67bf535b80 100644 --- a/wgpu-hal/src/metal/conv.rs +++ b/wgpu-hal/src/metal/conv.rs @@ -9,11 +9,11 @@ pub fn map_texture_usage(usage: crate::TextureUses) -> mtl::MTLTextureUsage { ); mtl_usage.set( mtl::MTLTextureUsage::ShaderRead, - usage.intersects(Tu::RESOURCE | Tu::DEPTH_STENCIL_READ | Tu::STORAGE_READ), + usage.intersects(Tu::RESOURCE | Tu::DEPTH_STENCIL_READ | Tu::STORAGE_READ | Tu::STORAGE_READ_WRITE), ); mtl_usage.set( mtl::MTLTextureUsage::ShaderWrite, - usage.intersects(Tu::STORAGE_WRITE), + usage.intersects(Tu::STORAGE_READ_WRITE), ); mtl_usage diff --git a/wgpu-hal/src/vulkan/conv.rs b/wgpu-hal/src/vulkan/conv.rs index 56065dd7b1..90eb11b047 100644 --- a/wgpu-hal/src/vulkan/conv.rs +++ b/wgpu-hal/src/vulkan/conv.rs @@ -229,7 +229,7 @@ pub fn map_texture_usage(usage: crate::TextureUses) -> vk::ImageUsageFlags { ) { flags |= vk::ImageUsageFlags::DEPTH_STENCIL_ATTACHMENT; } - if usage.intersects(crate::TextureUses::STORAGE_READ | crate::TextureUses::STORAGE_WRITE) { + if usage.intersects(crate::TextureUses::STORAGE_READ | crate::TextureUses::STORAGE_READ_WRITE) { flags |= vk::ImageUsageFlags::STORAGE; } flags @@ -275,9 +275,9 @@ pub fn map_texture_usage_to_barrier( stages |= shader_stages; access |= vk::AccessFlags::SHADER_READ; } - if usage.contains(crate::TextureUses::STORAGE_WRITE) { + if usage.contains(crate::TextureUses::STORAGE_READ_WRITE) { stages |= shader_stages; - access |= vk::AccessFlags::SHADER_WRITE; + access |= vk::AccessFlags::SHADER_READ | vk::AccessFlags::SHADER_WRITE; } if usage == crate::TextureUses::UNINITIALIZED || usage == crate::TextureUses::PRESENT { @@ -308,7 +308,7 @@ pub fn map_vk_image_usage(usage: vk::ImageUsageFlags) -> crate::TextureUses { bits |= crate::TextureUses::DEPTH_STENCIL_READ | crate::TextureUses::DEPTH_STENCIL_WRITE; } if usage.contains(vk::ImageUsageFlags::STORAGE) { - bits |= crate::TextureUses::STORAGE_READ | crate::TextureUses::STORAGE_WRITE; + bits |= crate::TextureUses::STORAGE_READ | crate::TextureUses::STORAGE_READ_WRITE; } bits } @@ -456,7 +456,7 @@ pub fn map_buffer_usage(usage: crate::BufferUses) -> vk::BufferUsageFlags { if usage.contains(crate::BufferUses::UNIFORM) { flags |= vk::BufferUsageFlags::UNIFORM_BUFFER; } - if usage.intersects(crate::BufferUses::STORAGE_READ | crate::BufferUses::STORAGE_WRITE) { + if usage.intersects(crate::BufferUses::STORAGE_READ | crate::BufferUses::STORAGE_READ_WRITE) { flags |= vk::BufferUsageFlags::STORAGE_BUFFER; } if usage.contains(crate::BufferUses::INDEX) { @@ -504,9 +504,9 @@ pub fn map_buffer_usage_to_barrier( stages |= shader_stages; access |= vk::AccessFlags::SHADER_READ; } - if usage.intersects(crate::BufferUses::STORAGE_WRITE) { + if usage.intersects(crate::BufferUses::STORAGE_READ_WRITE) { stages |= shader_stages; - access |= vk::AccessFlags::SHADER_WRITE; + access |= vk::AccessFlags::SHADER_READ | vk::AccessFlags::SHADER_WRITE; } if usage.contains(crate::BufferUses::INDEX) { stages |= vk::PipelineStageFlags::VERTEX_INPUT; From c7539e6328bd543b370b35e0da05cc6a0c6969bd Mon Sep 17 00:00:00 2001 From: Connor Fitzgerald Date: Wed, 18 May 2022 18:09:38 -0400 Subject: [PATCH 32/35] temp31 - Add some debug asserts to make sure we can't insert invalid states --- wgpu-core/src/track/buffer.rs | 6 ++++++ wgpu-core/src/track/texture.rs | 12 ++++++++++++ 2 files changed, 18 insertions(+) diff --git a/wgpu-core/src/track/buffer.rs b/wgpu-core/src/track/buffer.rs index 3a37fd76c2..b018ae3faf 100644 --- a/wgpu-core/src/track/buffer.rs +++ b/wgpu-core/src/track/buffer.rs @@ -539,6 +539,12 @@ unsafe fn insert( ) { let new_start_state = start_state_provider.get_state(index); let new_end_state = end_state_provider.map_or(new_start_state, |p| p.get_state(index)); + + // This should only ever happen with a wgpu bug, but let's just double + // check that resource states don't have any conflicts. + debug_assert_eq!(invalid_resource_state(new_start_state), false); + debug_assert_eq!(invalid_resource_state(new_end_state), false); + if let Some(&mut ref mut start_state) = start_states { *start_state.get_unchecked_mut(index) = new_start_state; } diff --git a/wgpu-core/src/track/texture.rs b/wgpu-core/src/track/texture.rs index 6d2468833c..69a0073438 100644 --- a/wgpu-core/src/track/texture.rs +++ b/wgpu-core/src/track/texture.rs @@ -64,6 +64,10 @@ impl ComplexTextureState { let mut complex = ComplexTextureState::new(full_range.mips.len() as u32, full_range.layers.len() as u32); for (selector, desired_state) in state_iter { + // This should only ever happen with a wgpu bug, but let's just double + // check that resource states don't have any conflicts. + debug_assert_eq!(invalid_resource_state(desired_state), false); + let mips = selector.mips.start as usize..selector.mips.end as usize; for mip in &mut complex.mips[mips] { for &mut (_, ref mut state) in mip.isolate(&selector.layers, TextureUses::UNKNOWN) { @@ -793,6 +797,10 @@ unsafe fn insert( let start_layers = start_state_provider.get_layers(texture_data, index32, index); match start_layers { SingleOrManyStates::Single(state) => { + // This should only ever happen with a wgpu bug, but let's just double + // check that resource states don't have any conflicts. + debug_assert_eq!(invalid_resource_state(state), false); + if let Some(start_state) = start_state { *start_state.simple.get_unchecked_mut(index) = state; } @@ -820,6 +828,10 @@ unsafe fn insert( if let Some(end_state_provider) = end_state_provider { match end_state_provider.get_layers(texture_data, index32, index) { SingleOrManyStates::Single(state) => { + // This should only ever happen with a wgpu bug, but let's just double + // check that resource states don't have any conflicts. + debug_assert_eq!(invalid_resource_state(state), false); + *end_state.simple.get_unchecked_mut(index) = state; } SingleOrManyStates::Many(state_iter) => { From 1b060b87ffa77211412a1f9efbe02fe2b647b01a Mon Sep 17 00:00:00 2001 From: Connor Fitzgerald Date: Wed, 18 May 2022 18:21:08 -0400 Subject: [PATCH 33/35] temp32 - clippy is opinionated sometimes --- wgpu-core/src/lib.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/wgpu-core/src/lib.rs b/wgpu-core/src/lib.rs index 19bf4f5e64..8d28a3df26 100644 --- a/wgpu-core/src/lib.rs +++ b/wgpu-core/src/lib.rs @@ -4,6 +4,8 @@ */ #![allow( + // It is much clearer to assert negative conditions with eq! false + clippy::bool_assert_comparison, // We use loops for getting early-out of scope without closures. clippy::never_loop, // We don't use syntax sugar where it's not necessary. From fb58fb9700d1363014bbc7478f950f3841cf9f23 Mon Sep 17 00:00:00 2001 From: Connor Fitzgerald Date: Mon, 23 May 2022 23:24:24 -0400 Subject: [PATCH 34/35] temp33 - renaming and documentation --- wgpu-core/src/command/bundle.rs | 14 +- wgpu-core/src/command/clear.rs | 4 +- wgpu-core/src/command/compute.rs | 14 +- wgpu-core/src/command/memory_init.rs | 2 +- wgpu-core/src/command/mod.rs | 8 +- wgpu-core/src/command/query.rs | 6 +- wgpu-core/src/command/render.rs | 38 +-- wgpu-core/src/command/transfer.rs | 16 +- wgpu-core/src/device/mod.rs | 66 +++-- wgpu-core/src/device/queue.rs | 14 +- wgpu-core/src/present.rs | 2 +- wgpu-core/src/track/buffer.rs | 216 ++++++++++++--- wgpu-core/src/track/mod.rs | 62 +++-- wgpu-core/src/track/stateless.rs | 58 +++- wgpu-core/src/track/texture.rs | 394 +++++++++++++++++++++------ 15 files changed, 688 insertions(+), 226 deletions(-) diff --git a/wgpu-core/src/command/bundle.rs b/wgpu-core/src/command/bundle.rs index 8773154847..738e33004e 100644 --- a/wgpu-core/src/command/bundle.rs +++ b/wgpu-core/src/command/bundle.rs @@ -233,7 +233,7 @@ impl RenderBundleEncoder { let bind_group: &binding_model::BindGroup = state .trackers .bind_groups - .extend(&*bind_group_guard, bind_group_id) + .add_single(&*bind_group_guard, bind_group_id) .ok_or(RenderCommandError::InvalidBindGroup(bind_group_id)) .map_pass_err(scope)?; if bind_group.dynamic_binding_info.len() != offsets.len() { @@ -267,7 +267,7 @@ impl RenderBundleEncoder { unsafe { state .trackers - .extend_from_bind_group(&*texture_guard, &bind_group.used) + .merge_bind_group(&*texture_guard, &bind_group.used) .map_pass_err(scope)? }; //Note: stateless trackers are not merged: the lifetime reference @@ -281,7 +281,7 @@ impl RenderBundleEncoder { let pipeline: &pipeline::RenderPipeline = state .trackers .render_pipelines - .extend(&*pipeline_guard, pipeline_id) + .add_single(&*pipeline_guard, pipeline_id) .ok_or(RenderCommandError::InvalidPipeline(pipeline_id)) .map_pass_err(scope)?; @@ -321,7 +321,7 @@ impl RenderBundleEncoder { let buffer: &resource::Buffer = state .trackers .buffers - .extend(&*buffer_guard, buffer_id, hal::BufferUses::INDEX) + .merge_single(&*buffer_guard, buffer_id, hal::BufferUses::INDEX) .map_pass_err(scope)?; check_buffer_usage(buffer.usage, wgt::BufferUsages::INDEX) .map_pass_err(scope)?; @@ -348,7 +348,7 @@ impl RenderBundleEncoder { let buffer: &resource::Buffer = state .trackers .buffers - .extend(&*buffer_guard, buffer_id, hal::BufferUses::VERTEX) + .merge_single(&*buffer_guard, buffer_id, hal::BufferUses::VERTEX) .map_pass_err(scope)?; check_buffer_usage(buffer.usage, wgt::BufferUsages::VERTEX) .map_pass_err(scope)?; @@ -473,7 +473,7 @@ impl RenderBundleEncoder { let buffer: &resource::Buffer = state .trackers .buffers - .extend(&*buffer_guard, buffer_id, hal::BufferUses::INDIRECT) + .merge_single(&*buffer_guard, buffer_id, hal::BufferUses::INDIRECT) .map_pass_err(scope)?; check_buffer_usage(buffer.usage, wgt::BufferUsages::INDIRECT) .map_pass_err(scope)?; @@ -506,7 +506,7 @@ impl RenderBundleEncoder { let buffer: &resource::Buffer = state .trackers .buffers - .extend(&*buffer_guard, buffer_id, hal::BufferUses::INDIRECT) + .merge_single(&*buffer_guard, buffer_id, hal::BufferUses::INDIRECT) .map_pass_err(scope)?; check_buffer_usage(buffer.usage, wgt::BufferUsages::INDIRECT) .map_pass_err(scope)?; diff --git a/wgpu-core/src/command/clear.rs b/wgpu-core/src/command/clear.rs index 5dea08168f..3e94843798 100644 --- a/wgpu-core/src/command/clear.rs +++ b/wgpu-core/src/command/clear.rs @@ -89,7 +89,7 @@ impl Global { let (dst_buffer, dst_pending) = cmd_buf .trackers .buffers - .change_state(&*buffer_guard, dst, hal::BufferUses::COPY_DST) + .set_single(&*buffer_guard, dst, hal::BufferUses::COPY_DST) .ok_or(ClearError::InvalidBuffer(dst))?; let dst_raw = dst_buffer .raw @@ -273,7 +273,7 @@ pub(crate) fn clear_texture( // // We could in theory distinguish these two scenarios in the internal clear_texture api in order to remove this check and call the cheaper change_replace_tracked whenever possible. let dst_barrier = texture_tracker - .change_state(storage, dst_texture_id.0, selector, clear_usage) + .set_single(storage, dst_texture_id.0, selector, clear_usage) .unwrap() .1 .map(|pending| pending.into_hal(dst_texture)); diff --git a/wgpu-core/src/command/compute.rs b/wgpu-core/src/command/compute.rs index a73fd0ef46..d81c726ad2 100644 --- a/wgpu-core/src/command/compute.rs +++ b/wgpu-core/src/command/compute.rs @@ -264,7 +264,7 @@ impl State { for id in self.binder.list_active() { unsafe { self.scope - .extend_from_bind_group(texture_guard, &bind_group_guard[id].used)? + .merge_bind_group(texture_guard, &bind_group_guard[id].used)? }; // Note: stateless trackers are not merged: the lifetime reference // is held to the bind group itself. @@ -272,7 +272,7 @@ impl State { for id in self.binder.list_active() { unsafe { - base_trackers.change_states_from_bind_group( + base_trackers.set_and_remove_from_usage_scope_sparse( texture_guard, &mut self.scope, &bind_group_guard[id].used, @@ -398,7 +398,7 @@ impl Global { let bind_group: &BindGroup = cmd_buf .trackers .bind_groups - .extend(&*bind_group_guard, bind_group_id) + .add_single(&*bind_group_guard, bind_group_id) .ok_or(ComputePassErrorInner::InvalidBindGroup(bind_group_id)) .map_pass_err(scope)?; bind_group @@ -453,7 +453,7 @@ impl Global { let pipeline: &pipeline::ComputePipeline = cmd_buf .trackers .compute_pipelines - .extend(&*pipeline_guard, pipeline_id) + .add_single(&*pipeline_guard, pipeline_id) .ok_or(ComputePassErrorInner::InvalidPipeline(pipeline_id)) .map_pass_err(scope)?; @@ -606,7 +606,7 @@ impl Global { let indirect_buffer: &Buffer = state .scope .buffers - .extend(&*buffer_guard, buffer_id, hal::BufferUses::INDIRECT) + .merge_single(&*buffer_guard, buffer_id, hal::BufferUses::INDIRECT) .map_pass_err(scope)?; check_buffer_usage(indirect_buffer.usage, wgt::BufferUsages::INDIRECT) .map_pass_err(scope)?; @@ -688,7 +688,7 @@ impl Global { let query_set: &resource::QuerySet = cmd_buf .trackers .query_sets - .extend(&*query_set_guard, query_set_id) + .add_single(&*query_set_guard, query_set_id) .ok_or(ComputePassErrorInner::InvalidQuerySet(query_set_id)) .map_pass_err(scope)?; @@ -705,7 +705,7 @@ impl Global { let query_set: &resource::QuerySet = cmd_buf .trackers .query_sets - .extend(&*query_set_guard, query_set_id) + .add_single(&*query_set_guard, query_set_id) .ok_or(ComputePassErrorInner::InvalidQuerySet(query_set_id)) .map_pass_err(scope)?; diff --git a/wgpu-core/src/command/memory_init.rs b/wgpu-core/src/command/memory_init.rs index b35a6a5e9e..d974575f91 100644 --- a/wgpu-core/src/command/memory_init.rs +++ b/wgpu-core/src/command/memory_init.rs @@ -205,7 +205,7 @@ impl BakedCommands { // However, we *know* that it is currently in use, so the tracker must already know about it. let transition = device_tracker .buffers - .change_state(buffer_guard, buffer_id, hal::BufferUses::COPY_DST) + .set_single(buffer_guard, buffer_id, hal::BufferUses::COPY_DST) .unwrap() .1; diff --git a/wgpu-core/src/command/mod.rs b/wgpu-core/src/command/mod.rs index 78896bc8cd..1559f87503 100644 --- a/wgpu-core/src/command/mod.rs +++ b/wgpu-core/src/command/mod.rs @@ -147,9 +147,9 @@ impl CommandBuffer { ) { profiling::scope!("insert_barriers"); - base.buffers.change_states_tracker(&head.buffers); + base.buffers.set_from_tracker(&head.buffers); base.textures - .change_states_tracker(&*texture_guard, &head.textures); + .set_from_tracker(&*texture_guard, &head.textures); Self::drain_barriers(raw, base, buffer_guard, texture_guard); } @@ -163,9 +163,9 @@ impl CommandBuffer { ) { profiling::scope!("insert_barriers"); - base.buffers.change_states_scope(&head.buffers); + base.buffers.set_from_usage_scope(&head.buffers); base.textures - .change_states_scope(&*texture_guard, &head.textures); + .set_from_usage_scope(&*texture_guard, &head.textures); Self::drain_barriers(raw, base, buffer_guard, texture_guard); } diff --git a/wgpu-core/src/command/query.rs b/wgpu-core/src/command/query.rs index 4140ceb82d..873a3031ac 100644 --- a/wgpu-core/src/command/query.rs +++ b/wgpu-core/src/command/query.rs @@ -299,7 +299,7 @@ impl Global { let query_set = cmd_buf .trackers .query_sets - .extend(&*query_set_guard, query_set_id) + .add_single(&*query_set_guard, query_set_id) .ok_or(QueryError::InvalidQuerySet(query_set_id))?; query_set.validate_and_write_timestamp(raw_encoder, query_set_id, query_index, None)?; @@ -344,13 +344,13 @@ impl Global { let query_set = cmd_buf .trackers .query_sets - .extend(&*query_set_guard, query_set_id) + .add_single(&*query_set_guard, query_set_id) .ok_or(QueryError::InvalidQuerySet(query_set_id))?; let (dst_buffer, dst_pending) = cmd_buf .trackers .buffers - .change_state(&*buffer_guard, destination, hal::BufferUses::COPY_DST) + .set_single(&*buffer_guard, destination, hal::BufferUses::COPY_DST) .ok_or(QueryError::InvalidBuffer(destination))?; let dst_barrier = dst_pending.map(|pending| pending.into_hal(dst_buffer)); diff --git a/wgpu-core/src/command/render.rs b/wgpu-core/src/command/render.rs index c22c206e5a..21cf30a341 100644 --- a/wgpu-core/src/command/render.rs +++ b/wgpu-core/src/command/render.rs @@ -700,7 +700,7 @@ impl<'a, A: HalApi> RenderPassInfo<'a, A> { let view = cmd_buf .trackers .views - .extend(&*view_guard, at.view) + .add_single(&*view_guard, at.view) .ok_or(RenderPassErrorInner::InvalidAttachment(at.view))?; check_multiview(view)?; add_view(view, "depth")?; @@ -809,7 +809,7 @@ impl<'a, A: HalApi> RenderPassInfo<'a, A> { let color_view = cmd_buf .trackers .views - .extend(&*view_guard, at.view) + .add_single(&*view_guard, at.view) .ok_or(RenderPassErrorInner::InvalidAttachment(at.view))?; check_multiview(color_view)?; add_view(color_view, "color")?; @@ -839,7 +839,7 @@ impl<'a, A: HalApi> RenderPassInfo<'a, A> { let resolve_view = cmd_buf .trackers .views - .extend(&*view_guard, resolve_target) + .add_single(&*view_guard, resolve_target) .ok_or(RenderPassErrorInner::InvalidAttachment(resolve_target))?; check_multiview(resolve_view)?; @@ -967,7 +967,7 @@ impl<'a, A: HalApi> RenderPassInfo<'a, A> { unsafe { self.usage_scope .textures - .extend_refcount( + .merge_single( &*texture_guard, ra.texture_id.value, Some(ra.selector.clone()), @@ -1159,7 +1159,7 @@ impl Global { let bind_group = cmd_buf .trackers .bind_groups - .extend(&*bind_group_guard, bind_group_id) + .add_single(&*bind_group_guard, bind_group_id) .ok_or(RenderCommandError::InvalidBindGroup(bind_group_id)) .map_pass_err(scope)?; bind_group @@ -1169,7 +1169,7 @@ impl Global { // merge the resource tracker in unsafe { info.usage_scope - .extend_from_bind_group(&*texture_guard, &bind_group.used) + .merge_bind_group(&*texture_guard, &bind_group.used) .map_pass_err(scope)?; } //Note: stateless trackers are not merged: the lifetime reference @@ -1223,7 +1223,7 @@ impl Global { let pipeline = cmd_buf .trackers .render_pipelines - .extend(&*render_pipeline_guard, pipeline_id) + .add_single(&*render_pipeline_guard, pipeline_id) .ok_or(RenderCommandError::InvalidPipeline(pipeline_id)) .map_pass_err(scope)?; @@ -1334,7 +1334,7 @@ impl Global { let buffer: &Buffer = info .usage_scope .buffers - .extend(&*buffer_guard, buffer_id, hal::BufferUses::INDEX) + .merge_single(&*buffer_guard, buffer_id, hal::BufferUses::INDEX) .map_pass_err(scope)?; check_buffer_usage(buffer.usage, BufferUsages::INDEX) .map_pass_err(scope)?; @@ -1380,7 +1380,7 @@ impl Global { let buffer: &Buffer = info .usage_scope .buffers - .extend(&*buffer_guard, buffer_id, hal::BufferUses::VERTEX) + .merge_single(&*buffer_guard, buffer_id, hal::BufferUses::VERTEX) .map_pass_err(scope)?; check_buffer_usage(buffer.usage, BufferUsages::VERTEX) .map_pass_err(scope)?; @@ -1637,7 +1637,7 @@ impl Global { let indirect_buffer: &Buffer = info .usage_scope .buffers - .extend(&*buffer_guard, buffer_id, hal::BufferUses::INDIRECT) + .merge_single(&*buffer_guard, buffer_id, hal::BufferUses::INDIRECT) .map_pass_err(scope)?; check_buffer_usage(indirect_buffer.usage, BufferUsages::INDIRECT) .map_pass_err(scope)?; @@ -1707,7 +1707,7 @@ impl Global { let indirect_buffer: &Buffer = info .usage_scope .buffers - .extend(&*buffer_guard, buffer_id, hal::BufferUses::INDIRECT) + .merge_single(&*buffer_guard, buffer_id, hal::BufferUses::INDIRECT) .map_pass_err(scope)?; check_buffer_usage(indirect_buffer.usage, BufferUsages::INDIRECT) .map_pass_err(scope)?; @@ -1720,7 +1720,11 @@ impl Global { let count_buffer: &Buffer = info .usage_scope .buffers - .extend(&*buffer_guard, count_buffer_id, hal::BufferUses::INDIRECT) + .merge_single( + &*buffer_guard, + count_buffer_id, + hal::BufferUses::INDIRECT, + ) .map_pass_err(scope)?; check_buffer_usage(count_buffer.usage, BufferUsages::INDIRECT) .map_pass_err(scope)?; @@ -1826,7 +1830,7 @@ impl Global { let query_set: &resource::QuerySet = cmd_buf .trackers .query_sets - .extend(&*query_set_guard, query_set_id) + .add_single(&*query_set_guard, query_set_id) .ok_or(RenderCommandError::InvalidQuerySet(query_set_id)) .map_pass_err(scope)?; @@ -1848,7 +1852,7 @@ impl Global { let query_set: &resource::QuerySet = cmd_buf .trackers .query_sets - .extend(&*query_set_guard, query_set_id) + .add_single(&*query_set_guard, query_set_id) .ok_or(RenderCommandError::InvalidQuerySet(query_set_id)) .map_pass_err(scope)?; @@ -1873,7 +1877,7 @@ impl Global { let bundle: &command::RenderBundle = cmd_buf .trackers .bundles - .extend(&*bundle_guard, bundle_id) + .add_single(&*bundle_guard, bundle_id) .ok_or(RenderCommandError::InvalidRenderBundle(bundle_id)) .map_pass_err(scope)?; @@ -1928,11 +1932,11 @@ impl Global { unsafe { info.usage_scope - .extend_from_render_bundle(&*texture_guard, &bundle.used) + .merge_render_bundle(&*texture_guard, &bundle.used) .map_pass_err(scope)?; cmd_buf .trackers - .change_state_from_render_bundle(&bundle.used) + .add_from_render_bundle(&bundle.used) .map_pass_err(scope)?; }; state.reset_bundle(); diff --git a/wgpu-core/src/command/transfer.rs b/wgpu-core/src/command/transfer.rs index c7ca6cde15..998263b44e 100644 --- a/wgpu-core/src/command/transfer.rs +++ b/wgpu-core/src/command/transfer.rs @@ -518,7 +518,7 @@ impl Global { let (src_buffer, src_pending) = cmd_buf .trackers .buffers - .change_state(&*buffer_guard, source, hal::BufferUses::COPY_SRC) + .set_single(&*buffer_guard, source, hal::BufferUses::COPY_SRC) .ok_or(TransferError::InvalidBuffer(source))?; let src_raw = src_buffer .raw @@ -533,7 +533,7 @@ impl Global { let (dst_buffer, dst_pending) = cmd_buf .trackers .buffers - .change_state(&*buffer_guard, destination, hal::BufferUses::COPY_DST) + .set_single(&*buffer_guard, destination, hal::BufferUses::COPY_DST) .ok_or(TransferError::InvalidBuffer(destination))?; let dst_raw = dst_buffer .raw @@ -652,7 +652,7 @@ impl Global { let (src_buffer, src_pending) = cmd_buf .trackers .buffers - .change_state(&*buffer_guard, source.buffer, hal::BufferUses::COPY_SRC) + .set_single(&*buffer_guard, source.buffer, hal::BufferUses::COPY_SRC) .ok_or(TransferError::InvalidBuffer(source.buffer))?; let src_raw = src_buffer .raw @@ -666,7 +666,7 @@ impl Global { let (dst_texture, dst_pending) = cmd_buf .trackers .textures - .change_state( + .set_single( &*texture_guard, destination.texture, dst_range, @@ -779,7 +779,7 @@ impl Global { let (src_texture, src_pending) = cmd_buf .trackers .textures - .change_state( + .set_single( &*texture_guard, source.texture, src_range, @@ -798,7 +798,7 @@ impl Global { let (dst_buffer, dst_pending) = cmd_buf .trackers .buffers - .change_state( + .set_single( &*buffer_guard, destination.buffer, hal::BufferUses::COPY_DST, @@ -929,7 +929,7 @@ impl Global { let (src_texture, src_pending) = cmd_buf .trackers .textures - .change_state( + .set_single( &*texture_guard, source.texture, src_range, @@ -954,7 +954,7 @@ impl Global { let (dst_texture, dst_pending) = cmd_buf .trackers .textures - .change_state( + .set_single( &*texture_guard, destination.texture, dst_range, diff --git a/wgpu-core/src/device/mod.rs b/wgpu-core/src/device/mod.rs index 467926cf04..8e883599a9 100644 --- a/wgpu-core/src/device/mod.rs +++ b/wgpu-core/src/device/mod.rs @@ -1542,7 +1542,7 @@ impl Device { let buffer = used .buffers - .extend(storage, bb.buffer_id, internal_use) + .add_single(storage, bb.buffer_id, internal_use) .ok_or(Error::InvalidBuffer(bb.buffer_id))?; check_buffer_usage(buffer.usage, pub_usage)?; let raw_buffer = buffer @@ -1622,7 +1622,7 @@ impl Device { // if it was deleted by the user. let texture = used .textures - .extend_with_refcount( + .add_single( texture_guard, view.parent_id.value.0, view.parent_id.ref_count.clone(), @@ -1737,7 +1737,7 @@ impl Device { wgt::BindingType::Sampler(ty) => { let sampler = used .samplers - .extend(&*sampler_guard, id) + .add_single(&*sampler_guard, id) .ok_or(Error::InvalidSampler(id))?; // Allowed sampler values for filtering and comparison @@ -1786,7 +1786,7 @@ impl Device { for &id in bindings_array.iter() { let sampler = used .samplers - .extend(&*sampler_guard, id) + .add_single(&*sampler_guard, id) .ok_or(Error::InvalidSampler(id))?; hal_samplers.push(&sampler.raw); } @@ -1796,7 +1796,7 @@ impl Device { Br::TextureView(id) => { let view = used .views - .extend(&*texture_view_guard, id) + .add_single(&*texture_view_guard, id) .ok_or(Error::InvalidTextureView(id))?; let (pub_usage, internal_use) = Self::texture_use_parameters( binding, @@ -1827,7 +1827,7 @@ impl Device { for &id in bindings_array.iter() { let view = used .views - .extend(&*texture_view_guard, id) + .add_single(&*texture_view_guard, id) .ok_or(Error::InvalidTextureView(id))?; let (pub_usage, internal_use) = Self::texture_use_parameters(binding, decl, view, @@ -3185,7 +3185,7 @@ impl Global { .trackers .lock() .buffers - .init(id, ref_count, buffer_use); + .insert_single(id, ref_count, buffer_use); return (id.0, None); }; @@ -3452,11 +3452,11 @@ impl Global { let id = fid.assign(texture, &mut token); log::info!("Created texture {:?} with {:?}", id, desc); - device - .trackers - .lock() - .textures - .init(id.0, ref_count, hal::TextureUses::UNINITIALIZED); + device.trackers.lock().textures.insert_single( + id.0, + ref_count, + hal::TextureUses::UNINITIALIZED, + ); return (id.0, None); }; @@ -3528,11 +3528,11 @@ impl Global { let id = fid.assign(texture, &mut token); log::info!("Created texture {:?} with {:?}", id, desc); - device - .trackers - .lock() - .textures - .init(id.0, ref_count, hal::TextureUses::UNINITIALIZED); + device.trackers.lock().textures.insert_single( + id.0, + ref_count, + hal::TextureUses::UNINITIALIZED, + ); return (id.0, None); }; @@ -3691,7 +3691,7 @@ impl Global { let ref_count = view.life_guard.add_ref(); let id = fid.assign(view, &mut token); - device.trackers.lock().views.init(id, ref_count); + device.trackers.lock().views.insert_single(id, ref_count); return (id.0, None); }; @@ -3784,7 +3784,7 @@ impl Global { let ref_count = sampler.life_guard.add_ref(); let id = fid.assign(sampler, &mut token); - device.trackers.lock().samplers.init(id, ref_count); + device.trackers.lock().samplers.insert_single(id, ref_count); return (id.0, None); }; @@ -4045,7 +4045,11 @@ impl Global { let id = fid.assign(bind_group, &mut token); log::debug!("Bind group {:?}", id,); - device.trackers.lock().bind_groups.init(id, ref_count); + device + .trackers + .lock() + .bind_groups + .insert_single(id, ref_count); return (id.0, None); }; @@ -4352,7 +4356,7 @@ impl Global { let ref_count = render_bundle.life_guard.add_ref(); let id = fid.assign(render_bundle, &mut token); - device.trackers.lock().bundles.init(id, ref_count); + device.trackers.lock().bundles.insert_single(id, ref_count); return (id.0, None); }; @@ -4427,7 +4431,11 @@ impl Global { let ref_count = query_set.life_guard.add_ref(); let id = fid.assign(query_set, &mut token); - device.trackers.lock().query_sets.init(id, ref_count); + device + .trackers + .lock() + .query_sets + .insert_single(id, ref_count); return (id.0, None); }; @@ -4518,7 +4526,11 @@ impl Global { let id = fid.assign(pipeline, &mut token); log::info!("Created render pipeline {:?} with {:?}", id, desc); - device.trackers.lock().render_pipelines.init(id, ref_count); + device + .trackers + .lock() + .render_pipelines + .insert_single(id, ref_count); return (id.0, None); }; @@ -4655,7 +4667,11 @@ impl Global { let id = fid.assign(pipeline, &mut token); log::info!("Created compute pipeline {:?} with {:?}", id, desc); - device.trackers.lock().compute_pipelines.init(id, ref_count); + device + .trackers + .lock() + .compute_pipelines + .insert_single(id, ref_count); return (id.0, None); }; @@ -5134,7 +5150,7 @@ impl Global { let mut trackers = device.trackers.lock(); trackers .buffers - .change_state(&*buffer_guard, buffer_id, internal_use); + .set_single(&*buffer_guard, buffer_id, internal_use); trackers.buffers.drain(); ret diff --git a/wgpu-core/src/device/queue.rs b/wgpu-core/src/device/queue.rs index a90b6780e7..e76255f772 100644 --- a/wgpu-core/src/device/queue.rs +++ b/wgpu-core/src/device/queue.rs @@ -286,7 +286,7 @@ impl Global { let mut trackers = device.trackers.lock(); let (dst, transition) = trackers .buffers - .change_state(&*buffer_guard, buffer_id, hal::BufferUses::COPY_DST) + .set_single(&*buffer_guard, buffer_id, hal::BufferUses::COPY_DST) .ok_or(TransferError::InvalidBuffer(buffer_id))?; let dst_raw = dst .raw @@ -473,7 +473,7 @@ impl Global { let (dst, transition) = trackers .textures - .change_state( + .set_single( &*texture_guard, destination.texture, selector, @@ -689,10 +689,10 @@ impl Global { device.temp_suspected.textures.push(id); } if should_extend { - let ref_count = cmdbuf.trackers.textures.get_ref_count(id); unsafe { + let ref_count = cmdbuf.trackers.textures.get_ref_count(id); used_surface_textures - .extend_refcount( + .merge_single( &*texture_guard, id, None, @@ -795,7 +795,7 @@ impl Global { }; trackers .textures - .change_states_scope(&*texture_guard, &used_surface_textures); + .set_from_usage_scope(&*texture_guard, &used_surface_textures); let texture_barriers = trackers.textures.drain().map(|pending| { let tex = unsafe { texture_guard.get_unchecked(pending.id) }; pending.into_hal(tex) @@ -851,7 +851,7 @@ impl Global { let ref_count = texture.life_guard.add_ref(); unsafe { used_surface_textures - .extend_refcount( + .merge_single( &*texture_guard, id::Valid(id), None, @@ -869,7 +869,7 @@ impl Global { trackers .textures - .change_states_scope(&*texture_guard, &used_surface_textures); + .set_from_usage_scope(&*texture_guard, &used_surface_textures); let texture_barriers = trackers.textures.drain().map(|pending| { let tex = unsafe { texture_guard.get_unchecked(pending.id) }; pending.into_hal(tex) diff --git a/wgpu-core/src/present.rs b/wgpu-core/src/present.rs index f75b33d348..8411d99d78 100644 --- a/wgpu-core/src/present.rs +++ b/wgpu-core/src/present.rs @@ -189,7 +189,7 @@ impl Global { { // register it in the device tracker as uninitialized let mut trackers = device.trackers.lock(); - let _ = trackers.textures.init( + let _ = trackers.textures.insert_single( id.0, ref_count.clone(), hal::TextureUses::UNINITIALIZED, diff --git a/wgpu-core/src/track/buffer.rs b/wgpu-core/src/track/buffer.rs index b018ae3faf..1651a54611 100644 --- a/wgpu-core/src/track/buffer.rs +++ b/wgpu-core/src/track/buffer.rs @@ -1,3 +1,10 @@ +/*! Buffer Trackers + * + * Buffers are represented by a single state for the whole resource, + * a 16 bit bitflag of buffer usages. Because there is only ever + * one subresource, they have no selector. +!*/ + use std::{borrow::Cow, marker::PhantomData, vec::Drain}; use super::PendingTransition; @@ -32,6 +39,7 @@ impl ResourceUses for BufferUses { } } +/// Stores all the buffers that a bind group stores. pub(crate) struct BufferBindGroupState { buffers: Vec<(Valid, RefCount, BufferUses)>, @@ -46,16 +54,22 @@ impl BufferBindGroupState { } } + /// Optimize the buffer bind group state by sorting it by ID. + /// + /// When this list of states is merged into a tracker, the memory + /// accesses will be in a constant assending order. pub(crate) fn optimize(&mut self) { self.buffers .sort_unstable_by_key(|&(id, _, _)| id.0.unzip().0); } + /// Returns a list of all buffers tracked. May contain duplicates. pub fn used(&self) -> impl Iterator> + '_ { self.buffers.iter().map(|&(id, _, _)| id) } - pub fn extend<'a>( + /// Adds the given resource with the given state. + pub fn add_single<'a>( &mut self, storage: &'a hub::Storage, BufferId>, id: BufferId, @@ -70,6 +84,7 @@ impl BufferBindGroupState { } } +/// Stores all buffer state within a single usage scope. #[derive(Debug)] pub(crate) struct BufferUsageScope { state: Vec, @@ -91,22 +106,40 @@ impl BufferUsageScope { self.metadata.debug_assert_in_bounds(index); } + /// Sets the size of all the vectors inside the tracker. + /// + /// Must be called with the highest possible Buffer ID before + /// all unsafe functions are called. pub fn set_size(&mut self, size: usize) { self.state.resize(size, BufferUses::empty()); self.metadata.set_size(size); } + /// Extend the vectors to let the given index be valid. fn allow_index(&mut self, index: usize) { if index >= self.state.len() { self.set_size(index + 1); } } + /// Returns a list of all buffers tracked. pub fn used(&self) -> impl Iterator> + '_ { self.metadata.used() } - pub unsafe fn extend_from_bind_group( + /// Merge the list of buffer states in the given bind group into this usage scope. + /// + /// If any of the resulting states is invalid, stops the merge and returns a usage + /// conflict with the details of the invalid state. + /// + /// Because bind groups do not check if the union of all their states is valid, + /// this method is allowed to return Err on the first bind group bound. + /// + /// # Safety + /// + /// [`Self::set_size`] must be called with the maximum possible Buffer ID before this + /// method is called. + pub unsafe fn merge_bind_group( &mut self, bind_group: &BufferBindGroupState, ) -> Result<(), UsageConflict> { @@ -121,7 +154,7 @@ impl BufferUsageScope { &mut self.metadata, index32, index, - StateProvider::Direct { state }, + BufferStateProvider::Direct { state }, ResourceMetadataProvider::Direct { epoch, ref_count: Cow::Borrowed(ref_count), @@ -132,7 +165,14 @@ impl BufferUsageScope { Ok(()) } - pub fn extend_from_scope(&mut self, scope: &Self) -> Result<(), UsageConflict> { + /// Merge the list of buffer states in the given usage scope into this UsageScope. + /// + /// If any of the resulting states is invalid, stops the merge and returns a usage + /// conflict with the details of the invalid state. + /// + /// If the given tracker uses IDs higher than the length of internal vectors, + /// the vectors will be extended. A call to set_size is not needed. + pub fn merge_usage_scope(&mut self, scope: &Self) -> Result<(), UsageConflict> { let incoming_size = scope.state.len(); if incoming_size > self.state.len() { self.set_size(incoming_size); @@ -150,7 +190,7 @@ impl BufferUsageScope { &mut self.metadata, index as u32, index, - StateProvider::Indirect { + BufferStateProvider::Indirect { state: &scope.state, }, ResourceMetadataProvider::Indirect { @@ -163,7 +203,14 @@ impl BufferUsageScope { Ok(()) } - pub fn extend<'a>( + /// Merge a single state into the UsageScope. + /// + /// If the resulting state is invalid, returns a usage + /// conflict with the details of the invalid state. + /// + /// If the ID is higher than the length of internal vectors, + /// the vectors will be extended. A call to set_size is not needed. + pub fn merge_single<'a>( &mut self, storage: &'a hub::Storage, BufferId>, id: BufferId, @@ -188,7 +235,7 @@ impl BufferUsageScope { &mut self.metadata, index32, index, - StateProvider::Direct { state: new_state }, + BufferStateProvider::Direct { state: new_state }, ResourceMetadataProvider::Resource { epoch }, )?; } @@ -197,6 +244,7 @@ impl BufferUsageScope { } } +/// Stores all buffer state within a command buffer or device. pub(crate) struct BufferTracker { start: Vec, end: Vec, @@ -223,6 +271,10 @@ impl BufferTracker { self.metadata.debug_assert_in_bounds(index); } + /// Sets the size of all the vectors inside the tracker. + /// + /// Must be called with the highest possible Buffer ID before + /// all unsafe functions are called. pub fn set_size(&mut self, size: usize) { self.start.resize(size, BufferUses::empty()); self.end.resize(size, BufferUses::empty()); @@ -230,21 +282,30 @@ impl BufferTracker { self.metadata.set_size(size); } + /// Extend the vectors to let the given index be valid. fn allow_index(&mut self, index: usize) { if index >= self.start.len() { self.set_size(index + 1); } } + /// Returns a list of all buffers tracked. pub fn used(&self) -> impl Iterator> + '_ { self.metadata.used() } - pub fn drain(&mut self) -> Drain> { + /// Drains all currently pending transitions. + pub fn drain(&mut self) -> Drain<'_, PendingTransition> { self.temp.drain(..) } - pub fn init(&mut self, id: Valid, ref_count: RefCount, state: BufferUses) { + /// Inserts a single buffer and its state into the resource tracker. + /// + /// If the resource already exists in the tracker, this will panic. + /// + /// If the ID is higher than the length of internal vectors, + /// the vectors will be extended. A call to set_size is not needed. + pub fn insert_single(&mut self, id: Valid, ref_count: RefCount, state: BufferUses) { let (index32, epoch, _) = id.0.unzip(); let index = index32 as usize; @@ -253,13 +314,19 @@ impl BufferTracker { self.debug_assert_in_bounds(index); unsafe { + let currently_owned = self.metadata.owned.get(index).unwrap_unchecked(); + + if currently_owned { + panic!("Tried to insert buffer already tracked"); + } + insert( None, Some(&mut self.start), &mut self.end, &mut self.metadata, index, - StateProvider::Direct { state }, + BufferStateProvider::Direct { state }, None, ResourceMetadataProvider::Direct { epoch, @@ -269,7 +336,14 @@ impl BufferTracker { } } - pub fn change_state<'a>( + /// Sets the state of a single buffer. + /// + /// If a transition is needed to get the buffer into the given state, that transition + /// is returned. No more than one transition is needed. + /// + /// If the ID is higher than the length of internal vectors, + /// the vectors will be extended. A call to set_size is not needed. + pub fn set_single<'a>( &mut self, storage: &'a hub::Storage, BufferId>, id: BufferId, @@ -292,7 +366,7 @@ impl BufferTracker { &mut self.metadata, index32, index, - StateProvider::Direct { state }, + BufferStateProvider::Direct { state }, None, ResourceMetadataProvider::Resource { epoch }, &mut self.temp, @@ -304,13 +378,22 @@ impl BufferTracker { Some((value, self.temp.pop())) } - pub fn change_states_tracker(&mut self, tracker: &Self) { + /// Sets the given state for all buffers in the given tracker. + /// + /// If a transition is needed to get the buffers into the needed state, + /// those transitions are stored within the tracker. A subsequent + /// call to [`Self::drain`] is needed to get those transitions. + /// + /// If the ID is higher than the length of internal vectors, + /// the vectors will be extended. A call to set_size is not needed. + pub fn set_from_tracker(&mut self, tracker: &Self) { let incoming_size = tracker.start.len(); if incoming_size > self.start.len() { self.set_size(incoming_size); } for index in iterate_bitvec_indices(&tracker.metadata.owned) { + self.debug_assert_in_bounds(index); tracker.debug_assert_in_bounds(index); unsafe { insert_or_barrier_update( @@ -320,10 +403,10 @@ impl BufferTracker { &mut self.metadata, index as u32, index, - StateProvider::Indirect { + BufferStateProvider::Indirect { state: &tracker.start, }, - Some(StateProvider::Indirect { + Some(BufferStateProvider::Indirect { state: &tracker.end, }), ResourceMetadataProvider::Indirect { @@ -335,13 +418,22 @@ impl BufferTracker { } } - pub fn change_states_scope(&mut self, scope: &BufferUsageScope) { + /// Sets the given state for all buffers in the given UsageScope. + /// + /// If a transition is needed to get the buffers into the needed state, + /// those transitions are stored within the tracker. A subsequent + /// call to [`Self::drain`] is needed to get those transitions. + /// + /// If the ID is higher than the length of internal vectors, + /// the vectors will be extended. A call to set_size is not needed. + pub fn set_from_usage_scope(&mut self, scope: &BufferUsageScope) { let incoming_size = scope.state.len(); if incoming_size > self.start.len() { self.set_size(incoming_size); } for index in iterate_bitvec_indices(&scope.metadata.owned) { + self.debug_assert_in_bounds(index); scope.debug_assert_in_bounds(index); unsafe { insert_or_barrier_update( @@ -351,7 +443,7 @@ impl BufferTracker { &mut self.metadata, index as u32, index, - StateProvider::Indirect { + BufferStateProvider::Indirect { state: &scope.state, }, None, @@ -364,7 +456,25 @@ impl BufferTracker { } } - pub unsafe fn change_states_bind_group( + /// Iterates through all buffers in the given bind group and adopts + /// the state given for those buffers in the UsageScope. It also + /// removes all touched buffers from the usage scope. + /// + /// If a transition is needed to get the buffers into the needed state, + /// those transitions are stored within the tracker. A subsequent + /// call to [`Self::drain`] is needed to get those transitions. + /// + /// This is a really funky method used by Compute Passes to generate + /// barriers after a call to dispatch without needing to iterate + /// over all elements in the usage scope. We use each the + /// bind group as a source of which IDs to look at. The bind groups + /// must have first been added to the usage scope. + /// + /// # Safety + /// + /// [`Self::set_size`] must be called with the maximum possible Buffer ID before this + /// method is called. + pub unsafe fn set_and_remove_from_usage_scope_sparse( &mut self, scope: &mut BufferUsageScope, bind_group_state: &BufferBindGroupState, @@ -390,7 +500,7 @@ impl BufferTracker { &mut self.metadata, index as u32, index, - StateProvider::Indirect { + BufferStateProvider::Indirect { state: &scope.state, }, None, @@ -405,6 +515,13 @@ impl BufferTracker { } } + /// Removes the given resource from the tracker iff we have the last reference to the + /// resource and the epoch matches. + /// + /// Returns true if the resource was removed. + /// + /// If the ID is higher than the length of internal vectors, + /// false will be returned. pub fn remove_abandoned(&mut self, id: Valid) -> bool { let (index32, epoch, _) = id.0.unzip(); let index = index32 as usize; @@ -434,21 +551,41 @@ impl BufferTracker { } } +/// Source of Buffer State. #[derive(Debug, Clone)] -enum StateProvider<'a> { +enum BufferStateProvider<'a> { + /// Get a state that was provided directly. Direct { state: BufferUses }, + /// Get a state from an an array of states. Indirect { state: &'a [BufferUses] }, } -impl StateProvider<'_> { +impl BufferStateProvider<'_> { + /// Gets the state from the provider, given a resource ID index. + /// + /// # Safety + /// + /// Index must be in bounds for the indirect source iff this is in the indirect state. #[inline(always)] unsafe fn get_state(&self, index: usize) -> BufferUses { match *self { - StateProvider::Direct { state } => state, - StateProvider::Indirect { state: other } => other[index], + BufferStateProvider::Direct { state } => state, + BufferStateProvider::Indirect { state } => { + debug_assert!(index < state.len()); + *state.get_unchecked(index) + } } } } +/// Does an insertion operation if the index isn't tracked +/// in the current metadata, otherwise merges the given state +/// with the current state. If the merging would cause +/// a conflict, returns that usage conflict. +/// +/// # Safety +/// +/// Indexes must be valid indexes into all arrays passed in +/// to this function, either directly or via metadata or provider structs. #[inline(always)] unsafe fn insert_or_merge( life_guard: Option<&LifeGuard>, @@ -457,7 +594,7 @@ unsafe fn insert_or_merge( resource_metadata: &mut ResourceMetadata, index32: u32, index: usize, - state_provider: StateProvider<'_>, + state_provider: BufferStateProvider<'_>, metadata_provider: ResourceMetadataProvider<'_, A>, ) -> Result<(), UsageConflict> { let currently_owned = resource_metadata.owned.get(index).unwrap_unchecked(); @@ -485,6 +622,23 @@ unsafe fn insert_or_merge( ) } +/// If the resource isn't tracked +/// - Inserts the given resource. +/// - Uses the `start_state_provider` to populate `start_states` +/// - Uses either `end_state_provider` or `start_state_provider` +/// to populate `current_states`. +/// If the resource is tracked +/// - Inserts barriers from the state in `current_states` +/// to the state provided by `start_state_provider`. +/// - Updates the `current_states` with either the state from +/// `end_state_provider` or `start_state_provider`. +/// +/// Any barriers are added to the barrier vector. +/// +/// # Safety +/// +/// Indexes must be valid indexes into all arrays passed in +/// to this function, either directly or via metadata or provider structs. #[inline(always)] unsafe fn insert_or_barrier_update( life_guard: Option<&LifeGuard>, @@ -493,8 +647,8 @@ unsafe fn insert_or_barrier_update( resource_metadata: &mut ResourceMetadata, index32: u32, index: usize, - start_state_provider: StateProvider<'_>, - end_state_provider: Option>, + start_state_provider: BufferStateProvider<'_>, + end_state_provider: Option>, metadata_provider: ResourceMetadataProvider<'_, A>, barriers: &mut Vec>, ) { @@ -533,8 +687,8 @@ unsafe fn insert( current_states: &mut [BufferUses], resource_metadata: &mut ResourceMetadata, index: usize, - start_state_provider: StateProvider<'_>, - end_state_provider: Option>, + start_state_provider: BufferStateProvider<'_>, + end_state_provider: Option>, metadata_provider: ResourceMetadataProvider<'_, A>, ) { let new_start_state = start_state_provider.get_state(index); @@ -562,7 +716,7 @@ unsafe fn merge( current_states: &mut [BufferUses], index32: u32, index: usize, - state_provider: StateProvider<'_>, + state_provider: BufferStateProvider<'_>, metadata_provider: ResourceMetadataProvider<'_, A>, ) -> Result<(), UsageConflict> { let current_state = current_states.get_unchecked_mut(index); @@ -588,7 +742,7 @@ unsafe fn barrier( current_states: &mut [BufferUses], index32: u32, index: usize, - state_provider: StateProvider<'_>, + state_provider: BufferStateProvider<'_>, barriers: &mut Vec>, ) { let current_state = *current_states.get_unchecked(index); @@ -609,7 +763,7 @@ unsafe fn barrier( unsafe fn update( current_states: &mut [BufferUses], index: usize, - state_provider: StateProvider<'_>, + state_provider: BufferStateProvider<'_>, ) { let current_state = current_states.get_unchecked_mut(index); let new_state = state_provider.get_state(index); diff --git a/wgpu-core/src/track/mod.rs b/wgpu-core/src/track/mod.rs index 1723cfbbef..04316495fc 100644 --- a/wgpu-core/src/track/mod.rs +++ b/wgpu-core/src/track/mod.rs @@ -476,7 +476,10 @@ impl BindGroupStates { } } - /// Sort all uses. + /// Optimize the bind group states by sorting them by ID. + /// + /// When this list of states is merged into a tracker, the memory + /// accesses will be in a constant assending order. pub fn optimize(&mut self) { self.buffers.optimize(); self.textures.optimize(); @@ -532,14 +535,14 @@ impl RenderBundleScope { /// /// The maximum ID given by each bind group resource must be less than the /// length of the storage given at the call to `new`. - pub unsafe fn extend_from_bind_group( + pub unsafe fn merge_bind_group( &mut self, textures: &hub::Storage, id::TextureId>, bind_group: &BindGroupStates, ) -> Result<(), UsageConflict> { - self.buffers.extend_from_bind_group(&bind_group.buffers)?; + self.buffers.merge_bind_group(&bind_group.buffers)?; self.textures - .extend_from_bind_group(textures, &bind_group.textures)?; + .merge_bind_group(textures, &bind_group.textures)?; Ok(()) } @@ -579,14 +582,14 @@ impl UsageScope { /// /// The maximum ID given by each bind group resource must be less than the /// length of the storage given at the call to `new`. - pub unsafe fn extend_from_bind_group( + pub unsafe fn merge_bind_group( &mut self, textures: &hub::Storage, id::TextureId>, bind_group: &BindGroupStates, ) -> Result<(), UsageConflict> { - self.buffers.extend_from_bind_group(&bind_group.buffers)?; + self.buffers.merge_bind_group(&bind_group.buffers)?; self.textures - .extend_from_bind_group(textures, &bind_group.textures)?; + .merge_bind_group(textures, &bind_group.textures)?; Ok(()) } @@ -600,14 +603,14 @@ impl UsageScope { /// /// The maximum ID given by each bind group resource must be less than the /// length of the storage given at the call to `new`. - pub unsafe fn extend_from_render_bundle( + pub unsafe fn merge_render_bundle( &mut self, textures: &hub::Storage, id::TextureId>, render_bundle: &RenderBundleScope, ) -> Result<(), UsageConflict> { - self.buffers.extend_from_scope(&render_bundle.buffers)?; + self.buffers.merge_usage_scope(&render_bundle.buffers)?; self.textures - .extend_from_scope(textures, &render_bundle.textures)?; + .merge_usage_scope(textures, &render_bundle.textures)?; Ok(()) } @@ -685,14 +688,19 @@ impl Tracker { }; } - /// Uses the the indices of the resources used in a bind group to - /// remove them from the usage scope, and transition to the state - /// given by the usage scope. + /// Iterates through all resources in the given bind group and adopts + /// the state given for those resources in the UsageScope. It also + /// removes all touched resources from the usage scope. + /// + /// If a transition is needed to get the resources into the needed state, + /// those transitions are stored within the tracker. A subsequent + /// call to [`Self::drain`] is needed to get those transitions. /// - /// This is weird method used for compute dispatches. Bind groups are added - /// to the usage scope in order to combine the uses, then this method acts - /// as a sparse way of only changing the needed resoures based on which resources - /// the bind groups touched. + /// This is a really funky method used by Compute Passes to generate + /// barriers after a call to dispatch without needing to iterate + /// over all elements in the usage scope. We use each the + /// bind group as a source of which IDs to look at. The bind groups + /// must have first been added to the usage scope. /// /// Only stateful things are merged in here, all other resources are owned /// indirectly by the bind group. @@ -701,16 +709,19 @@ impl Tracker { /// /// The maximum ID given by each bind group resource must be less than the /// value given to `set_size` - pub unsafe fn change_states_from_bind_group( + pub unsafe fn set_and_remove_from_usage_scope_sparse( &mut self, textures: &hub::Storage, id::TextureId>, scope: &mut UsageScope, bind_group: &BindGroupStates, ) { self.buffers - .change_states_bind_group(&mut scope.buffers, &bind_group.buffers); - self.textures - .change_states_bind_group(textures, &mut scope.textures, &bind_group.textures); + .set_and_remove_from_usage_scope_sparse(&mut scope.buffers, &bind_group.buffers); + self.textures.set_and_remove_from_usage_scope_sparse( + textures, + &mut scope.textures, + &bind_group.textures, + ); } /// Tracks the stateless resources from the given renderbundle. It is expected @@ -720,16 +731,15 @@ impl Tracker { /// /// The maximum ID given by each bind group resource must be less than the /// value given to `set_size` - pub unsafe fn change_state_from_render_bundle( + pub unsafe fn add_from_render_bundle( &mut self, render_bundle: &RenderBundleScope, ) -> Result<(), UsageConflict> { self.bind_groups - .extend_from_tracker(&render_bundle.bind_groups); + .add_from_tracker(&render_bundle.bind_groups); self.render_pipelines - .extend_from_tracker(&render_bundle.render_pipelines); - self.query_sets - .extend_from_tracker(&render_bundle.query_sets); + .add_from_tracker(&render_bundle.render_pipelines); + self.query_sets.add_from_tracker(&render_bundle.query_sets); Ok(()) } diff --git a/wgpu-core/src/track/stateless.rs b/wgpu-core/src/track/stateless.rs index 3ff1c0b820..a8051bb3ce 100644 --- a/wgpu-core/src/track/stateless.rs +++ b/wgpu-core/src/track/stateless.rs @@ -1,3 +1,9 @@ +/*! Stateless Trackers + * + * Stateless trackers don't have any state, so make no + * distinction between a usage scope and a full tracker. +!*/ + use std::marker::PhantomData; use crate::{ @@ -7,6 +13,7 @@ use crate::{ RefCount, }; +/// Stores all the resources that a bind group stores. pub(crate) struct StatelessBindGroupSate { resources: Vec<(Valid, RefCount)>, @@ -22,16 +29,22 @@ impl StatelessBindGroupSate { } } + /// Optimize the buffer bind group state by sorting it by ID. + /// + /// When this list of states is merged into a tracker, the memory + /// accesses will be in a constant assending order. pub(crate) fn optimize(&mut self) { self.resources .sort_unstable_by_key(|&(id, _)| id.0.unzip().0); } + /// Returns a list of all resources tracked. May contain duplicates. pub fn used(&self) -> impl Iterator> + '_ { self.resources.iter().map(|&(id, _)| id) } - pub fn extend<'a>(&mut self, storage: &'a hub::Storage, id: Id) -> Option<&'a T> { + /// Adds the given resource. + pub fn add_single<'a>(&mut self, storage: &'a hub::Storage, id: Id) -> Option<&'a T> { let resource = storage.get(id).ok()?; self.resources @@ -41,6 +54,7 @@ impl StatelessBindGroupSate { } } +/// Stores all resource state within a command buffer or device. pub(crate) struct StatelessTracker { metadata: ResourceMetadata, @@ -60,21 +74,33 @@ impl StatelessTracker { self.metadata.debug_assert_in_bounds(index); } + /// Sets the size of all the vectors inside the tracker. + /// + /// Must be called with the highest possible Resource ID of this type + /// before all unsafe functions are called. pub fn set_size(&mut self, size: usize) { self.metadata.set_size(size); } - pub fn used(&self) -> impl Iterator> + '_ { - self.metadata.used() - } - + /// Extend the vectors to let the given index be valid. fn allow_index(&mut self, index: usize) { if index >= self.metadata.owned.len() { self.set_size(index + 1); } } - pub fn init(&mut self, id: Valid, ref_count: RefCount) { + /// Returns a list of all resources tracked. + pub fn used(&self) -> impl Iterator> + '_ { + self.metadata.used() + } + + /// Inserts a single resource into the resource tracker. + /// + /// If the resource already exists in the tracker, it will be overwritten. + /// + /// If the ID is higher than the length of internal vectors, + /// the vectors will be extended. A call to set_size is not needed. + pub fn insert_single(&mut self, id: Valid, ref_count: RefCount) { let (index32, epoch, _) = id.0.unzip(); let index = index32 as usize; @@ -89,8 +115,11 @@ impl StatelessTracker { } } - /// Requires set_size to be called - pub fn extend<'a>(&mut self, storage: &'a hub::Storage, id: Id) -> Option<&'a T> { + /// Adds the given resource to the tracker. + /// + /// If the ID is higher than the length of internal vectors, + /// the vectors will be extended. A call to set_size is not needed. + pub fn add_single<'a>(&mut self, storage: &'a hub::Storage, id: Id) -> Option<&'a T> { let item = storage.get(id).ok()?; let (index32, epoch, _) = id.unzip(); @@ -109,7 +138,11 @@ impl StatelessTracker { Some(item) } - pub fn extend_from_tracker(&mut self, other: &Self) { + /// Adds the given resources from the given tracker. + /// + /// If the ID is higher than the length of internal vectors, + /// the vectors will be extended. A call to set_size is not needed. + pub fn add_from_tracker(&mut self, other: &Self) { let incoming_size = other.metadata.owned.len(); if incoming_size > self.metadata.owned.len() { self.set_size(incoming_size); @@ -139,6 +172,13 @@ impl StatelessTracker { } } + /// Removes the given resource from the tracker iff we have the last reference to the + /// resource and the epoch matches. + /// + /// Returns true if the resource was removed. + /// + /// If the ID is higher than the length of internal vectors, + /// false will be returned. pub fn remove_abandoned(&mut self, id: Valid) -> bool { let (index32, epoch, _) = id.0.unzip(); let index = index32 as usize; diff --git a/wgpu-core/src/track/texture.rs b/wgpu-core/src/track/texture.rs index 69a0073438..d78bf670e6 100644 --- a/wgpu-core/src/track/texture.rs +++ b/wgpu-core/src/track/texture.rs @@ -1,3 +1,24 @@ +/*! Texture Trackers + * + * Texture trackers are signifigantly more complicated than + * the buffer trackers because textures can be in a "complex" + * state where each individual subresource can potentially be + * in a different state from every other subtresource. These + * complex states are stored seperately from the simple states + * because they are signifignatly more difficult to track and + * most resources spend the vast majority of their lives in + * simple states. + * + * There are two special texture usages: `UNKNOWN` and `UNINITIALIZED`. + * - `UNKNOWN` is only used in complex states and is used to signify + * that the complex state does not know anything about those subresources. + * It cannot leak into transitions, it is invalid to transition into UNKNOWN + * state. + * - `UNINITIALIZED` is used in both simple and complex states to mean the texture + * is known to be in some undefined state. Any transition away from UNINITIALIZED + * will treat the contents as junk. +!*/ + use super::{range::RangedStates, PendingTransition}; use crate::{ hub, @@ -16,6 +37,7 @@ use naga::FastHashMap; use std::{borrow::Cow, iter, marker::PhantomData, ops::Range, vec::Drain}; +/// Specifies a particular set of subresources in a texture. #[derive(Clone, Debug, PartialEq, Eq)] pub struct TextureSelector { pub mips: Range, @@ -41,12 +63,18 @@ impl ResourceUses for TextureUses { } } +/// Represents the complex state of textures where every subresource is potentially +/// in a different state. #[derive(Clone, Debug, Default, PartialEq)] struct ComplexTextureState { mips: ArrayVec, { hal::MAX_MIP_LEVELS as usize }>, } impl ComplexTextureState { + /// Creates complex texture state for the given sizes. + /// + /// This state will be initialized with the UNKNOWN state, a special state + /// which means the trakcer knows nothing about the state. fn new(mip_level_count: u32, array_layer_count: u32) -> Self { Self { mips: iter::repeat_with(|| { @@ -57,19 +85,36 @@ impl ComplexTextureState { } } - fn from_selector_state_iter( + /// Initialize a complex state from a selector representing the full size of the texture + /// and an iterator of a selector and a texture use, specifying a usage for a specific + /// set of subresources. + /// + /// [`Self::to_selector_state_iter`] can be used to create such an iterator. + /// + /// # Safety + /// + /// All selectors in the iterator must be inside of the full_range selector. + /// + /// The full range selector must have mips and layers start at 0. + unsafe fn from_selector_state_iter( full_range: TextureSelector, state_iter: impl Iterator, ) -> Self { + debug_assert_eq!(full_range.layers.start, 0); + debug_assert_eq!(full_range.mips.start, 0); + let mut complex = ComplexTextureState::new(full_range.mips.len() as u32, full_range.layers.len() as u32); for (selector, desired_state) in state_iter { + debug_assert!(selector.layers.end <= full_range.layers.end); + debug_assert!(selector.mips.end <= full_range.mips.end); + // This should only ever happen with a wgpu bug, but let's just double // check that resource states don't have any conflicts. debug_assert_eq!(invalid_resource_state(desired_state), false); let mips = selector.mips.start as usize..selector.mips.end as usize; - for mip in &mut complex.mips[mips] { + for mip in complex.mips.get_unchecked_mut(mips) { for &mut (_, ref mut state) in mip.isolate(&selector.layers, TextureUses::UNKNOWN) { *state = desired_state; } @@ -78,6 +123,9 @@ impl ComplexTextureState { complex } + /// Convert a complex state into an iterator over all states stored. + /// + /// [`Self::from_selector_state_iter`] can be used to consume such an iterator. fn to_selector_state_iter( &self, ) -> impl Iterator + Clone + '_ { @@ -98,7 +146,7 @@ impl ComplexTextureState { } } -// TODO: This representation could be optimized in a couple ways, but keep it simple for now. +/// Stores all the textures that a bind group stores. pub(crate) struct TextureBindGroupState { textures: Vec<( Valid, @@ -118,16 +166,22 @@ impl TextureBindGroupState { } } + /// Optimize the texture bind group state by sorting it by ID. + /// + /// When this list of states is merged into a tracker, the memory + /// accesses will be in a constant assending order. pub(crate) fn optimize(&mut self) { self.textures .sort_unstable_by_key(|&(id, _, _, _)| id.0.unzip().0); } + /// Returns a list of all buffers tracked. May contain duplicates. pub fn used(&self) -> impl Iterator> + '_ { self.textures.iter().map(|&(id, _, _, _)| id) } - pub fn extend_with_refcount<'a>( + /// Adds the given resource with the given state. + pub fn add_single<'a>( &mut self, storage: &'a hub::Storage, TextureId>, id: TextureId, @@ -143,6 +197,7 @@ impl TextureBindGroupState { } } +/// Container for corresponding simple and complex texture states. #[derive(Debug)] pub(crate) struct TextureStateSet { simple: Vec, @@ -161,6 +216,7 @@ impl TextureStateSet { } } +/// Stores all texture state within a single usage scope. #[derive(Debug)] pub(crate) struct TextureUsageScope { set: TextureStateSet, @@ -191,20 +247,35 @@ impl TextureUsageScope { }); } + /// Sets the size of all the vectors inside the tracker. + /// + /// Must be called with the highest possible Texture ID before + /// all unsafe functions are called. pub fn set_size(&mut self, size: usize) { self.set.set_size(size); self.metadata.set_size(size); } + /// Returns a list of all textures tracked. pub fn used(&self) -> impl Iterator> + '_ { self.metadata.used() } + /// Returns true if the tracker owns no resources. + /// + /// This is a O(n) operation. pub(crate) fn is_empty(&self) -> bool { self.metadata.is_empty() } - pub fn extend_from_scope( + /// Merge the list of texture states in the given usage scope into this UsageScope. + /// + /// If any of the resulting states is invalid, stops the merge and returns a usage + /// conflict with the details of the invalid state. + /// + /// If the given tracker uses IDs higher than the length of internal vectors, + /// the vectors will be extended. A call to set_size is not needed. + pub fn merge_usage_scope( &mut self, storage: &hub::Storage, TextureId>, scope: &Self, @@ -217,6 +288,9 @@ impl TextureUsageScope { for index in iterate_bitvec_indices(&scope.metadata.owned) { let index32 = index as u32; + self.debug_assert_in_bounds(index); + scope.debug_assert_in_bounds(index); + unsafe { insert_or_merge( texture_data_from_texture(storage, index32), @@ -224,7 +298,7 @@ impl TextureUsageScope { &mut self.metadata, index32, index, - LayeredStateProvider::TextureSet { set: &scope.set }, + TextureStateProvider::TextureSet { set: &scope.set }, ResourceMetadataProvider::Indirect { metadata: &scope.metadata, }, @@ -235,22 +309,44 @@ impl TextureUsageScope { Ok(()) } - pub unsafe fn extend_from_bind_group( + /// Merge the list of texture states in the given bind group into this usage scope. + /// + /// If any of the resulting states is invalid, stops the merge and returns a usage + /// conflict with the details of the invalid state. + /// + /// Because bind groups do not check if the union of all their states is valid, + /// this method is allowed to return Err on the first bind group bound. + /// + /// # Safety + /// + /// [`Self::set_size`] must be called with the maximum possible Buffer ID before this + /// method is called. + pub unsafe fn merge_bind_group( &mut self, storage: &hub::Storage, TextureId>, bind_group: &TextureBindGroupState, ) -> Result<(), UsageConflict> { for &(id, ref selector, ref ref_count, state) in &bind_group.textures { - self.extend_refcount(storage, id, selector.clone(), ref_count, state)?; + self.merge_single(storage, id, selector.clone(), ref_count, state)?; } Ok(()) } + /// Merge a single state into the UsageScope. + /// + /// If the resulting state is invalid, returns a usage + /// conflict with the details of the invalid state. + /// /// # Safety /// - /// `id` must be a valid ID and have an ID value less than the last call to set_size. - pub unsafe fn extend_refcount( + /// Unlike other trackers whose merge_single is safe, this method is only + /// called where there is already other unsafe tracking functions active, + /// so we can prove this unsafe "for free". + /// + /// [`Self::set_size`] must be called with the maximum possible Buffer ID before this + /// method is called. + pub unsafe fn merge_single( &mut self, storage: &hub::Storage, TextureId>, id: Valid, @@ -269,7 +365,7 @@ impl TextureUsageScope { &mut self.metadata, index32, index, - LayeredStateProvider::from_option(selector, new_state), + TextureStateProvider::from_option(selector, new_state), ResourceMetadataProvider::Direct { epoch, ref_count: Cow::Borrowed(ref_count), @@ -280,13 +376,13 @@ impl TextureUsageScope { } } +/// Stores all texture state within a command buffer or device. pub(crate) struct TextureTracker { start_set: TextureStateSet, end_set: TextureStateSet, metadata: ResourceMetadata, - /// Temporary storage for collecting transitions. temp: Vec>, _phantom: PhantomData, @@ -327,6 +423,10 @@ impl TextureTracker { }); } + /// Sets the size of all the vectors inside the tracker. + /// + /// Must be called with the highest possible Texture ID before + /// all unsafe functions are called. pub fn set_size(&mut self, size: usize) { self.start_set.set_size(size); self.end_set.set_size(size); @@ -334,28 +434,51 @@ impl TextureTracker { self.metadata.set_size(size); } + /// Extend the vectors to let the given index be valid. fn allow_index(&mut self, index: usize) { if index >= self.start_set.simple.len() { self.set_size(index + 1); } } + /// Returns a list of all textures tracked. pub fn used(&self) -> impl Iterator> + '_ { self.metadata.used() } + /// Drains all currently pending transitions. pub fn drain(&mut self) -> Drain> { self.temp.drain(..) } - pub fn get_ref_count(&self, id: Valid) -> &RefCount { + /// Get the refcount of the given resource. + /// + /// # Safety + /// + /// [`Self::set_size`] must be called with the maximum possible Buffer ID before this + /// method is called. + /// + /// The resource must be tracked by this tracker. + pub unsafe fn get_ref_count(&self, id: Valid) -> &RefCount { let (index32, _, _) = id.0.unzip(); let index = index32 as usize; - self.metadata.ref_counts[index].as_ref().unwrap() + self.debug_assert_in_bounds(index); + + self.metadata + .ref_counts + .get_unchecked(index) + .as_ref() + .unwrap_unchecked() } - pub fn init(&mut self, id: TextureId, ref_count: RefCount, usage: TextureUses) { + /// Inserts a single texture and a state into the resource tracker. + /// + /// If the resource already exists in the tracker, this will panic. + /// + /// If the ID is higher than the length of internal vectors, + /// the vectors will be extended. A call to set_size is not needed. + pub fn insert_single(&mut self, id: TextureId, ref_count: RefCount, usage: TextureUses) { let (index32, epoch, _) = id.unzip(); let index = index32 as usize; @@ -367,7 +490,7 @@ impl TextureTracker { let currently_owned = self.metadata.owned.get(index).unwrap_unchecked(); if currently_owned { - panic!("Tried to init resource already tracked"); + panic!("Tried to insert texture already tracked"); } insert( @@ -377,7 +500,7 @@ impl TextureTracker { &mut self.metadata, index32, index, - LayeredStateProvider::KnownSingle { state: usage }, + TextureStateProvider::KnownSingle { state: usage }, None, ResourceMetadataProvider::Direct { epoch, @@ -387,13 +510,20 @@ impl TextureTracker { }; } - pub fn change_state<'a>( + /// Sets the state of a single texture. + /// + /// If a transition is needed to get the texture into the given state, that transition + /// is returned. + /// + /// If the ID is higher than the length of internal vectors, + /// the vectors will be extended. A call to set_size is not needed. + pub fn set_single<'a>( &mut self, storage: &'a hub::Storage, TextureId>, id: TextureId, selector: TextureSelector, new_state: TextureUses, - ) -> Option<(&'a Texture, Drain>)> { + ) -> Option<(&'a Texture, Drain<'_, PendingTransition>)> { let texture = storage.get(id).ok()?; let (index32, epoch, _) = id.unzip(); @@ -411,7 +541,7 @@ impl TextureTracker { &mut self.metadata, index32, index, - LayeredStateProvider::Selector { + TextureStateProvider::Selector { selector, state: new_state, }, @@ -424,7 +554,15 @@ impl TextureTracker { Some((texture, self.temp.drain(..))) } - pub fn change_states_tracker( + /// Sets the given state for all texture in the given tracker. + /// + /// If a transition is needed to get the texture into the needed state, + /// those transitions are stored within the tracker. A subsequent + /// call to [`Self::drain`] is needed to get those transitions. + /// + /// If the ID is higher than the length of internal vectors, + /// the vectors will be extended. A call to set_size is not needed. + pub fn set_from_tracker( &mut self, storage: &hub::Storage, TextureId>, tracker: &Self, @@ -447,10 +585,10 @@ impl TextureTracker { &mut self.metadata, index32, index, - LayeredStateProvider::TextureSet { + TextureStateProvider::TextureSet { set: &tracker.start_set, }, - Some(LayeredStateProvider::TextureSet { + Some(TextureStateProvider::TextureSet { set: &tracker.end_set, }), ResourceMetadataProvider::Indirect { @@ -462,7 +600,15 @@ impl TextureTracker { } } - pub fn change_states_scope( + /// Sets the given state for all textures in the given UsageScope. + /// + /// If a transition is needed to get the textures into the needed state, + /// those transitions are stored within the tracker. A subsequent + /// call to [`Self::drain`] is needed to get those transitions. + /// + /// If the ID is higher than the length of internal vectors, + /// the vectors will be extended. A call to set_size is not needed. + pub fn set_from_usage_scope( &mut self, storage: &hub::Storage, TextureId>, scope: &TextureUsageScope, @@ -485,7 +631,7 @@ impl TextureTracker { &mut self.metadata, index32, index, - LayeredStateProvider::TextureSet { set: &scope.set }, + TextureStateProvider::TextureSet { set: &scope.set }, None, ResourceMetadataProvider::Indirect { metadata: &scope.metadata, @@ -496,7 +642,25 @@ impl TextureTracker { } } - pub unsafe fn change_states_bind_group( + /// Iterates through all textures in the given bind group and adopts + /// the state given for those textures in the UsageScope. It also + /// removes all touched textures from the usage scope. + /// + /// If a transition is needed to get the textures into the needed state, + /// those transitions are stored within the tracker. A subsequent + /// call to [`Self::drain`] is needed to get those transitions. + /// + /// This is a really funky method used by Compute Passes to generate + /// barriers after a call to dispatch without needing to iterate + /// over all elements in the usage scope. We use each the + /// bind group as a source of which IDs to look at. The bind groups + /// must have first been added to the usage scope. + /// + /// # Safety + /// + /// [`Self::set_size`] must be called with the maximum possible Buffer ID before this + /// method is called. + pub unsafe fn set_and_remove_from_usage_scope_sparse( &mut self, storage: &hub::Storage, TextureId>, scope: &mut TextureUsageScope, @@ -522,7 +686,7 @@ impl TextureTracker { &mut self.metadata, index32, index, - LayeredStateProvider::TextureSet { set: &scope.set }, + TextureStateProvider::TextureSet { set: &scope.set }, None, ResourceMetadataProvider::Indirect { metadata: &scope.metadata, @@ -534,6 +698,12 @@ impl TextureTracker { } } + /// Unconditionally removes the given resource from the tracker. + /// + /// Returns true if the resource was removed. + /// + /// If the ID is higher than the length of internal vectors, + /// false will be returned. pub fn remove(&mut self, id: Valid) -> bool { let (index32, epoch, _) = id.0.unzip(); let index = index32 as usize; @@ -561,6 +731,13 @@ impl TextureTracker { false } + /// Removes the given resource from the tracker iff we have the last reference to the + /// resource and the epoch matches. + /// + /// Returns true if the resource was removed. + /// + /// If the ID is higher than the length of internal vectors, + /// false will be returned. pub fn remove_abandoned(&mut self, id: Valid) -> bool { let (index32, epoch, _) = id.0.unzip(); let index = index32 as usize; @@ -593,13 +770,14 @@ impl TextureTracker { } } +/// An iterator adapter that can store two different iterator types. #[derive(Clone)] -enum Either { +enum EitherIter { Left(L), Right(R), } -impl Iterator for Either +impl Iterator for EitherIter where L: Iterator, R: Iterator, @@ -608,32 +786,36 @@ where fn next(&mut self) -> Option { match *self { - Either::Left(ref mut inner) => inner.next(), - Either::Right(ref mut inner) => inner.next(), + EitherIter::Left(ref mut inner) => inner.next(), + EitherIter::Right(ref mut inner) => inner.next(), } } } +/// Container that signifies storing both different things +/// if there is a single state or many different states +/// involved in the operation. #[derive(Debug, Clone)] enum SingleOrManyStates { Single(S), Many(M), } +/// A source of texture state. #[derive(Clone)] -enum LayeredStateProvider<'a> { - KnownSingle { - state: TextureUses, - }, +enum TextureStateProvider<'a> { + /// Comes directly from a single state. + KnownSingle { state: TextureUses }, + /// Comes from a selector and a single state. Selector { selector: TextureSelector, state: TextureUses, }, - TextureSet { - set: &'a TextureStateSet, - }, + /// Comes from another texture set. + TextureSet { set: &'a TextureStateSet }, } -impl<'a> LayeredStateProvider<'a> { +impl<'a> TextureStateProvider<'a> { + /// Convenience function turning Option into this enum. fn from_option(selector: Option, state: TextureUses) -> Self { match selector { Some(selector) => Self::Selector { selector, state }, @@ -641,8 +823,17 @@ impl<'a> LayeredStateProvider<'a> { } } + /// Get the state provided by this. + /// + /// # Panics + /// + /// Panics if texture_data is None and this uses a Selector source. + /// + /// # Safety + /// + /// - The index must be in bounds of the state set if this uses an TextureSet source. #[inline(always)] - unsafe fn get_layers( + unsafe fn get_state( self, texture_data: Option<(&LifeGuard, &TextureSelector)>, index32: u32, @@ -652,21 +843,27 @@ impl<'a> LayeredStateProvider<'a> { impl Iterator + Clone + 'a, > { match self { - LayeredStateProvider::KnownSingle { state } => SingleOrManyStates::Single(state), - LayeredStateProvider::Selector { selector, state } => { + TextureStateProvider::KnownSingle { state } => SingleOrManyStates::Single(state), + TextureStateProvider::Selector { selector, state } => { + // We check if the selector given is actually for the full resource, + // and if it is we promote to a simple state. This allows upstream + // code to specify selectors willy nilly, and all that are really + // single states are promoted here. if *texture_data.unwrap().1 == selector { SingleOrManyStates::Single(state) } else { - SingleOrManyStates::Many(Either::Left(iter::once((selector, state)))) + SingleOrManyStates::Many(EitherIter::Left(iter::once((selector, state)))) } } - LayeredStateProvider::TextureSet { set } => { + TextureStateProvider::TextureSet { set } => { let new_state = *set.simple.get_unchecked(index); if new_state == TextureUses::COMPLEX { let new_complex = set.complex.get(&index32).unwrap_unchecked(); - SingleOrManyStates::Many(Either::Right(new_complex.to_selector_state_iter())) + SingleOrManyStates::Many(EitherIter::Right( + new_complex.to_selector_state_iter(), + )) } else { SingleOrManyStates::Single(new_state) } @@ -675,6 +872,8 @@ impl<'a> LayeredStateProvider<'a> { } } +/// Helper function that gets what is needed from the texture storage +/// out of the texture storage. #[inline(always)] unsafe fn texture_data_from_texture( storage: &hub::Storage, TextureId>, @@ -684,16 +883,15 @@ unsafe fn texture_data_from_texture( (&texture.life_guard, &texture.full_range) } -// I think the trick to the barriers is that there are three different possible operations: - -// merge from a single state -// barrier from a single state into a double sided tracker -// update from a single state into a double sided tracker - -// recording rpasses is a merge(rpass, resource) -// cmd_buf <- rpass is a barrier(cmd_buf, rpass) + update(cmd_buf, rpass) -// device <- cmd_buf is a barrier(device, cmd_buff.start) + update(cmd_buf, cmd_buf.end) - +/// Does an insertion operation if the index isn't tracked +/// in the current metadata, otherwise merges the given state +/// with the current state. If the merging would cause +/// a conflict, returns that usage conflict. +/// +/// # Safety +/// +/// Indexes must be valid indexes into all arrays passed in +/// to this function, either directly or via metadata or provider structs. #[inline(always)] unsafe fn insert_or_merge( texture_data: (&LifeGuard, &TextureSelector), @@ -701,7 +899,7 @@ unsafe fn insert_or_merge( resource_metadata: &mut ResourceMetadata, index32: u32, index: usize, - state_provider: LayeredStateProvider<'_>, + state_provider: TextureStateProvider<'_>, metadata_provider: ResourceMetadataProvider<'_, A>, ) -> Result<(), UsageConflict> { let currently_owned = resource_metadata.owned.get(index).unwrap_unchecked(); @@ -731,6 +929,23 @@ unsafe fn insert_or_merge( ) } +/// If the resource isn't tracked +/// - Inserts the given resource. +/// - Uses the `start_state_provider` to populate `start_states` +/// - Uses either `end_state_provider` or `start_state_provider` +/// to populate `current_states`. +/// If the resource is tracked +/// - Inserts barriers from the state in `current_states` +/// to the state provided by `start_state_provider`. +/// - Updates the `current_states` with either the state from +/// `end_state_provider` or `start_state_provider`. +/// +/// Any barriers are added to the barrier vector. +/// +/// # Safety +/// +/// Indexes must be valid indexes into all arrays passed in +/// to this function, either directly or via metadata or provider structs. #[inline(always)] unsafe fn insert_or_barrier_update( texture_data: (&LifeGuard, &TextureSelector), @@ -739,8 +954,8 @@ unsafe fn insert_or_barrier_update( resource_metadata: &mut ResourceMetadata, index32: u32, index: usize, - start_state_provider: LayeredStateProvider<'_>, - end_state_provider: Option>, + start_state_provider: TextureStateProvider<'_>, + end_state_provider: Option>, metadata_provider: ResourceMetadataProvider<'_, A>, barriers: &mut Vec>, ) { @@ -790,11 +1005,11 @@ unsafe fn insert( resource_metadata: &mut ResourceMetadata, index32: u32, index: usize, - start_state_provider: LayeredStateProvider<'_>, - end_state_provider: Option>, + start_state_provider: TextureStateProvider<'_>, + end_state_provider: Option>, metadata_provider: ResourceMetadataProvider<'_, A>, ) { - let start_layers = start_state_provider.get_layers(texture_data, index32, index); + let start_layers = start_state_provider.get_state(texture_data, index32, index); match start_layers { SingleOrManyStates::Single(state) => { // This should only ever happen with a wgpu bug, but let's just double @@ -804,6 +1019,8 @@ unsafe fn insert( if let Some(start_state) = start_state { *start_state.simple.get_unchecked_mut(index) = state; } + + // We only need to insert ourselves the end state if there is no end state provider. if end_state_provider.is_none() { *end_state.simple.get_unchecked_mut(index) = state; } @@ -818,6 +1035,7 @@ unsafe fn insert( start_state.complex.insert(index32, complex.clone()); } + // We only need to insert ourselves the end state if there is no end state provider. if end_state_provider.is_none() { *end_state.simple.get_unchecked_mut(index) = TextureUses::COMPLEX; end_state.complex.insert(index32, complex); @@ -826,12 +1044,14 @@ unsafe fn insert( } if let Some(end_state_provider) = end_state_provider { - match end_state_provider.get_layers(texture_data, index32, index) { + match end_state_provider.get_state(texture_data, index32, index) { SingleOrManyStates::Single(state) => { // This should only ever happen with a wgpu bug, but let's just double // check that resource states don't have any conflicts. debug_assert_eq!(invalid_resource_state(state), false); + // We only need to insert into the end, as there is guarenteed to be + // a start state provider. *end_state.simple.get_unchecked_mut(index) = state; } SingleOrManyStates::Many(state_iter) => { @@ -839,6 +1059,8 @@ unsafe fn insert( let complex = ComplexTextureState::from_selector_state_iter(full_range, state_iter); + // We only need to insert into the end, as there is guarenteed to be + // a start state provider. *end_state.simple.get_unchecked_mut(index) = TextureUses::COMPLEX; end_state.complex.insert(index32, complex); } @@ -859,7 +1081,7 @@ unsafe fn merge( current_state_set: &mut TextureStateSet, index32: u32, index: usize, - state_provider: LayeredStateProvider<'_>, + state_provider: TextureStateProvider<'_>, metadata_provider: ResourceMetadataProvider<'_, A>, ) -> Result<(), UsageConflict> { let current_simple = current_state_set.simple.get_unchecked_mut(index); @@ -874,7 +1096,7 @@ unsafe fn merge( SingleOrManyStates::Single(current_simple) }; - let new_state = state_provider.get_layers(Some(texture_data), index32, index); + let new_state = state_provider.get_state(Some(texture_data), index32, index); match (current_state, new_state) { (SingleOrManyStates::Single(current_simple), SingleOrManyStates::Single(new_simple)) => { @@ -892,6 +1114,8 @@ unsafe fn merge( *current_simple = merged_state; } (SingleOrManyStates::Single(current_simple), SingleOrManyStates::Many(new_many)) => { + // Because we are now demoting this simple state to a complex state, we actually need to make a whole + // new complex state for us to use as there wasn't one before. let mut new_complex = ComplexTextureState::from_selector_state_iter( texture_data.1.clone(), iter::once((texture_data.1.clone(), *current_simple)), @@ -1003,7 +1227,7 @@ unsafe fn barrier( current_state_set: &TextureStateSet, index32: u32, index: usize, - state_provider: LayeredStateProvider<'_>, + state_provider: TextureStateProvider<'_>, barriers: &mut Vec>, ) { let current_simple = *current_state_set.simple.get_unchecked(index); @@ -1013,7 +1237,7 @@ unsafe fn barrier( SingleOrManyStates::Single(current_simple) }; - let new_state = state_provider.get_layers(Some(texture_data), index32, index); + let new_state = state_provider.get_state(Some(texture_data), index32, index); match (current_state, new_state) { (SingleOrManyStates::Single(current_simple), SingleOrManyStates::Single(new_simple)) => { @@ -1101,6 +1325,7 @@ unsafe fn barrier( } } +#[allow(clippy::needless_option_as_deref)] // we use this for reborrowing Option<&mut T> #[inline(always)] unsafe fn update( texture_data: (&LifeGuard, &TextureSelector), @@ -1108,9 +1333,13 @@ unsafe fn update( current_state_set: &mut TextureStateSet, index32: u32, index: usize, - state_provider: LayeredStateProvider<'_>, + state_provider: TextureStateProvider<'_>, ) { let start_simple = *start_state_set.simple.get_unchecked(index); + + // We only ever need to update the start state here if the state is complex. + // + // If the state is simple, the first insert to the tracker would cover it. let mut start_complex = None; if start_simple == TextureUses::COMPLEX { start_complex = Some(start_state_set.complex.get_mut(&index32).unwrap_unchecked()); @@ -1128,13 +1357,15 @@ unsafe fn update( SingleOrManyStates::Single(current_simple) }; - let new_state = state_provider.get_layers(Some(texture_data), index32, index); + let new_state = state_provider.get_state(Some(texture_data), index32, index); match (current_state, new_state) { (SingleOrManyStates::Single(current_simple), SingleOrManyStates::Single(new_simple)) => { *current_simple = new_simple; } (SingleOrManyStates::Single(current_simple), SingleOrManyStates::Many(new_many)) => { + // Because we are now demoting this simple state to a complex state, we actually need to make a whole + // new complex state for us to use as there wasn't one before. let mut new_complex = ComplexTextureState::from_selector_state_iter( texture_data.1.clone(), iter::once((texture_data.1.clone(), *current_simple)), @@ -1208,20 +1439,27 @@ unsafe fn update( if *current_layer_state == TextureUses::UNKNOWN && new_state != TextureUses::UNKNOWN { - if let Some(&mut ref mut start_complex) = start_complex { - debug_assert!(mip_id < start_complex.mips.len()); + // We now know something about this subresource that we didn't before + // so we should go back and update the start state. + + // We know we must have starter state be complex, otherwise we would know + // about this state. + debug_assert!(start_complex.is_some()); - let start_mip = start_complex.mips.get_unchecked_mut(mip_id); + let start_complex = start_complex.as_deref_mut().unwrap_unchecked(); - for &mut (_, ref mut current_start_state) in - start_mip.isolate(layers, TextureUses::UNKNOWN) - { - debug_assert_eq!(*current_start_state, TextureUses::UNKNOWN); - *current_start_state = new_state; - } + debug_assert!(mip_id < start_complex.mips.len()); + + let start_mip = start_complex.mips.get_unchecked_mut(mip_id); - start_mip.coalesce(); + for &mut (_, ref mut current_start_state) in + start_mip.isolate(layers, TextureUses::UNKNOWN) + { + debug_assert_eq!(*current_start_state, TextureUses::UNKNOWN); + *current_start_state = new_state; } + + start_mip.coalesce(); } *current_layer_state = new_state; From f5d8e5d617f53e2df40e25e013421a371aa864e8 Mon Sep 17 00:00:00 2001 From: Connor Fitzgerald Date: Tue, 24 May 2022 01:42:48 -0400 Subject: [PATCH 35/35] temp34 - logging --- wgpu-core/src/command/render.rs | 2 +- wgpu-core/src/present.rs | 2 +- wgpu-core/src/track/buffer.rs | 8 +++++- wgpu-core/src/track/mod.rs | 2 -- wgpu-core/src/track/texture.rs | 44 +++++++++++++++++++++++++++++++-- wgpu-hal/src/vulkan/device.rs | 2 +- 6 files changed, 52 insertions(+), 8 deletions(-) diff --git a/wgpu-core/src/command/render.rs b/wgpu-core/src/command/render.rs index 21cf30a341..7ef45aa0a6 100644 --- a/wgpu-core/src/command/render.rs +++ b/wgpu-core/src/command/render.rs @@ -1944,7 +1944,7 @@ impl Global { } } - log::trace!("Merging {:?} with the render pass", encoder_id); + log::trace!("Merging renderpass into cmd_buf {:?}", encoder_id); let (trackers, pending_discard_init_fixups) = info.finish(raw, &*texture_guard).map_pass_err(init_scope)?; diff --git a/wgpu-core/src/present.rs b/wgpu-core/src/present.rs index 8411d99d78..fa7a34bf9d 100644 --- a/wgpu-core/src/present.rs +++ b/wgpu-core/src/present.rs @@ -189,7 +189,7 @@ impl Global { { // register it in the device tracker as uninitialized let mut trackers = device.trackers.lock(); - let _ = trackers.textures.insert_single( + trackers.textures.insert_single( id.0, ref_count.clone(), hal::TextureUses::UNINITIALIZED, diff --git a/wgpu-core/src/track/buffer.rs b/wgpu-core/src/track/buffer.rs index 1651a54611..7770b42308 100644 --- a/wgpu-core/src/track/buffer.rs +++ b/wgpu-core/src/track/buffer.rs @@ -699,6 +699,8 @@ unsafe fn insert( debug_assert_eq!(invalid_resource_state(new_start_state), false); debug_assert_eq!(invalid_resource_state(new_end_state), false); + log::trace!("\tbuf {index}: insert {new_start_state:?}..{new_end_state:?}"); + if let Some(&mut ref mut start_state) = start_states { *start_state.get_unchecked_mut(index) = new_start_state; } @@ -732,6 +734,8 @@ unsafe fn merge( )); } + log::trace!("\tbuf {index32}: merge {current_state:?} + {new_state:?}"); + *current_state = merged_state; Ok(()) @@ -756,7 +760,9 @@ unsafe fn barrier( id: index32, selector: (), usage: current_state..new_state, - }) + }); + + log::trace!("\tbuf {index32}: transition {current_state:?} -> {new_state:?}"); } #[inline(always)] diff --git a/wgpu-core/src/track/mod.rs b/wgpu-core/src/track/mod.rs index 04316495fc..8d38ff8e72 100644 --- a/wgpu-core/src/track/mod.rs +++ b/wgpu-core/src/track/mod.rs @@ -126,7 +126,6 @@ impl PendingTransition { self, buf: &'a resource::Buffer, ) -> hal::BufferBarrier<'a, A> { - log::trace!("\tbuffer -> {:?}", self); let buffer = buf.raw.as_ref().expect("Buffer is destroyed"); hal::BufferBarrier { buffer, @@ -141,7 +140,6 @@ impl PendingTransition { self, tex: &'a resource::Texture, ) -> hal::TextureBarrier<'a, A> { - log::trace!("\ttexture -> {:?}", self); let texture = tex.inner.as_raw().expect("Texture is destroyed"); // These showing up in a barrier is always a bug diff --git a/wgpu-core/src/track/texture.rs b/wgpu-core/src/track/texture.rs index d78bf670e6..0f157c8f15 100644 --- a/wgpu-core/src/track/texture.rs +++ b/wgpu-core/src/track/texture.rs @@ -1016,6 +1016,8 @@ unsafe fn insert( // check that resource states don't have any conflicts. debug_assert_eq!(invalid_resource_state(state), false); + log::trace!("\ttex {index32}: insert start {state:?}"); + if let Some(start_state) = start_state { *start_state.simple.get_unchecked_mut(index) = state; } @@ -1030,6 +1032,8 @@ unsafe fn insert( let complex = ComplexTextureState::from_selector_state_iter(full_range, state_iter); + log::trace!("\ttex {index32}: insert start {complex:?}"); + if let Some(start_state) = start_state { *start_state.simple.get_unchecked_mut(index) = TextureUses::COMPLEX; start_state.complex.insert(index32, complex.clone()); @@ -1050,6 +1054,8 @@ unsafe fn insert( // check that resource states don't have any conflicts. debug_assert_eq!(invalid_resource_state(state), false); + log::trace!("\ttex {index32}: insert end {state:?}"); + // We only need to insert into the end, as there is guarenteed to be // a start state provider. *end_state.simple.get_unchecked_mut(index) = state; @@ -1059,6 +1065,8 @@ unsafe fn insert( let complex = ComplexTextureState::from_selector_state_iter(full_range, state_iter); + log::trace!("\ttex {index32}: insert end {complex:?}"); + // We only need to insert into the end, as there is guarenteed to be // a start state provider. *end_state.simple.get_unchecked_mut(index) = TextureUses::COMPLEX; @@ -1102,6 +1110,8 @@ unsafe fn merge( (SingleOrManyStates::Single(current_simple), SingleOrManyStates::Single(new_simple)) => { let merged_state = *current_simple | new_simple; + log::trace!("\ttex {index32}: merge simple {current_simple:?} + {new_simple:?}"); + if invalid_resource_state(merged_state) { return Err(UsageConflict::from_texture( TextureId::zip(index32, metadata_provider.get_epoch(index), A::VARIANT), @@ -1124,6 +1134,10 @@ unsafe fn merge( for (selector, new_state) in new_many { let merged_state = *current_simple | new_state; + log::trace!( + "\ttex {index32}: merge {selector:?} {current_simple:?} + {new_state:?}" + ); + if invalid_resource_state(merged_state) { return Err(UsageConflict::from_texture( TextureId::zip(index32, metadata_provider.get_epoch(index), A::VARIANT), @@ -1159,6 +1173,11 @@ unsafe fn merge( // Once we remove unknown, this will never be empty, as simple states are never unknown. let merged_state = merged_state - TextureUses::UNKNOWN; + log::trace!( + "\ttex {index32}: merge mip {mip_id} layers {layers:?} \ + {current_layer_state:?} + {new_simple:?}" + ); + if invalid_resource_state(merged_state) { return Err(UsageConflict::from_texture( TextureId::zip(index32, metadata_provider.get_epoch(index), A::VARIANT), @@ -1195,6 +1214,11 @@ unsafe fn merge( continue; } + log::trace!( + "\ttex {index32}: merge mip {mip_id} layers {layers:?} \ + {current_layer_state:?} + {new_state:?}" + ); + if invalid_resource_state(merged_state) { return Err(UsageConflict::from_texture( TextureId::zip( @@ -1245,11 +1269,13 @@ unsafe fn barrier( return; } + log::trace!("\ttex {index32}: transition simple {current_simple:?} -> {new_simple:?}"); + barriers.push(PendingTransition { id: index32, selector: texture_data.1.clone(), usage: current_simple..new_simple, - }) + }); } (SingleOrManyStates::Single(current_simple), SingleOrManyStates::Many(new_many)) => { for (selector, new_state) in new_many { @@ -1261,11 +1287,15 @@ unsafe fn barrier( continue; } + log::trace!( + "\ttex {index32}: transition {selector:?} {current_simple:?} -> {new_state:?}" + ); + barriers.push(PendingTransition { id: index32, selector, usage: current_simple..new_state, - }) + }); } } (SingleOrManyStates::Many(current_complex), SingleOrManyStates::Single(new_simple)) => { @@ -1281,6 +1311,11 @@ unsafe fn barrier( continue; } + log::trace!( + "\ttex {index32}: transition mip {mip_id} layers {layers:?} \ + {current_layer_state:?} -> {new_simple:?}" + ); + barriers.push(PendingTransition { id: index32, selector: TextureSelector { @@ -1310,6 +1345,11 @@ unsafe fn barrier( continue; } + log::trace!( + "\ttex {index32}: transition mip {mip_id} layers {layers:?} \ + {current_layer_state:?} -> {new_state:?}" + ); + barriers.push(PendingTransition { id: index32, selector: TextureSelector { diff --git a/wgpu-hal/src/vulkan/device.rs b/wgpu-hal/src/vulkan/device.rs index 24f992953f..3dd6b8b31b 100644 --- a/wgpu-hal/src/vulkan/device.rs +++ b/wgpu-hal/src/vulkan/device.rs @@ -1413,7 +1413,7 @@ impl crate::Device for super::Device { unsafe fn destroy_shader_module(&self, module: super::ShaderModule) { match module { super::ShaderModule::Raw(raw) => { - let _ = self.shared.raw.destroy_shader_module(raw, None); + self.shared.raw.destroy_shader_module(raw, None); } super::ShaderModule::Intermediate { .. } => {} }