From 40b136636ab4a4c234b753be75c2abd8650374b4 Mon Sep 17 00:00:00 2001 From: TheRawMeatball Date: Sat, 23 Oct 2021 16:44:06 +0300 Subject: [PATCH] Ptrify everything --- crates/bevy_ecs/macros/src/lib.rs | 22 +- crates/bevy_ecs/src/bundle.rs | 12 +- crates/bevy_ecs/src/lib.rs | 3 +- crates/bevy_ecs/src/ptr.rs | 104 +++- crates/bevy_ecs/src/query/fetch.rs | 577 +++++++++++------- crates/bevy_ecs/src/query/filter.rs | 212 ++++--- crates/bevy_ecs/src/query/iter.rs | 107 ++-- crates/bevy_ecs/src/query/state.rs | 91 +-- crates/bevy_ecs/src/reflect.rs | 6 +- crates/bevy_ecs/src/storage/blob_vec.rs | 79 ++- crates/bevy_ecs/src/storage/sparse_set.rs | 9 +- crates/bevy_ecs/src/storage/table.rs | 49 +- crates/bevy_ecs/src/system/mod.rs | 42 +- crates/bevy_ecs/src/system/query.rs | 59 +- crates/bevy_ecs/src/system/system_param.rs | 29 +- crates/bevy_ecs/src/world/entity_ref.rs | 98 +-- crates/bevy_ecs/src/world/mod.rs | 56 +- .../src/render_graph/nodes/pass_node.rs | 4 +- crates/bevy_ui/src/flex/mod.rs | 10 +- 19 files changed, 949 insertions(+), 620 deletions(-) diff --git a/crates/bevy_ecs/macros/src/lib.rs b/crates/bevy_ecs/macros/src/lib.rs index c7d30025484f8..35416d472dcfe 100644 --- a/crates/bevy_ecs/macros/src/lib.rs +++ b/crates/bevy_ecs/macros/src/lib.rs @@ -126,17 +126,17 @@ pub fn derive_bundle(input: TokenStream) -> TokenStream { self.#field.get_components(&mut func); }); field_from_components.push(quote! { - #field: <#field_type as #ecs_path::bundle::Bundle>::from_components(&mut func), + #field: <#field_type as #ecs_path::bundle::Bundle>::from_components(ctx, &mut func), }); } else { field_component_ids.push(quote! { component_ids.push(components.init_component::<#field_type>(storages)); }); field_get_components.push(quote! { - #ecs_path::ptr::OwningPtr::make(self.#field, func); + #ecs_path::ptr::OwningPtr::make(self.#field, &mut func); }); field_from_components.push(quote! { - #field: func().inner().cast::<#field_type>().read(), + #field: func(ctx).inner().cast::<#field_type>().read(), }); } } @@ -158,9 +158,9 @@ pub fn derive_bundle(input: TokenStream) -> TokenStream { } #[allow(unused_variables, unused_mut, non_snake_case)] - unsafe fn from_components<'a, F>(func: F) -> Self + unsafe fn from_components(ctx: &mut T, mut func: F) -> Self where - F: FnMut() -> #ecs_path::ptr::OwningPtr<'a> + 'a + F: FnMut(&mut T) -> #ecs_path::ptr::OwningPtr<'_> { Self { #(#field_from_components)* @@ -211,20 +211,20 @@ pub fn impl_query_set(_input: TokenStream) -> TokenStream { let query_fn_mut = &query_fn_muts[0..query_count]; tokens.extend(TokenStream::from(quote! { impl<'w, 's, #(#query: WorldQuery + 'static,)* #(#filter: WorldQuery + 'static,)*> SystemParam for QuerySet<'w, 's, (#(QueryState<#query, #filter>,)*)> - where #(#filter::Fetch: FilterFetch,)* + where #(for<'x, 'y> QueryFetch<'x, 'y, #filter>: FilterFetch<'x, 'y>,)* { type Fetch = QuerySetState<(#(QueryState<#query, #filter>,)*)>; } - // SAFE: All Queries are constrained to ReadOnlyFetch, so World is only read + // SAFE: All Queries are constrained to ReadOnlyQuery, so World is only read unsafe impl<#(#query: WorldQuery + 'static,)* #(#filter: WorldQuery + 'static,)*> ReadOnlySystemParamFetch for QuerySetState<(#(QueryState<#query, #filter>,)*)> - where #(#query::Fetch: ReadOnlyFetch,)* #(#filter::Fetch: FilterFetch,)* + where #(#query: ReadOnlyQuery,)* #(for<'x, 'y> QueryFetch<'x, 'y, #filter>: FilterFetch<'x, 'y>,)* { } // SAFE: Relevant query ComponentId and ArchetypeComponentId access is applied to SystemMeta. If any QueryState conflicts // with any prior access, a panic will occur. unsafe impl<#(#query: WorldQuery + 'static,)* #(#filter: WorldQuery + 'static,)*> SystemParamState for QuerySetState<(#(QueryState<#query, #filter>,)*)> - where #(#filter::Fetch: FilterFetch,)* + where #(for<'x, 'y> QueryFetch<'x, 'y, #filter>: FilterFetch<'x, 'y>,)* { type Config = (); fn init(world: &mut World, system_meta: &mut SystemMeta, config: Self::Config) -> Self { @@ -264,7 +264,7 @@ pub fn impl_query_set(_input: TokenStream) -> TokenStream { } impl<'w, 's, #(#query: WorldQuery + 'static,)* #(#filter: WorldQuery + 'static,)*> SystemParamFetch<'w, 's> for QuerySetState<(#(QueryState<#query, #filter>,)*)> - where #(#filter::Fetch: FilterFetch,)* + where #(for<'x, 'y> QueryFetch<'x, 'y, #filter>: FilterFetch<'x, 'y>,)* { type Item = QuerySet<'w, 's, (#(QueryState<#query, #filter>,)*)>; @@ -285,7 +285,7 @@ pub fn impl_query_set(_input: TokenStream) -> TokenStream { } impl<'w, 's, #(#query: WorldQuery,)* #(#filter: WorldQuery,)*> QuerySet<'w, 's, (#(QueryState<#query, #filter>,)*)> - where #(#filter::Fetch: FilterFetch,)* + where #(for<'x, 'y> QueryFetch<'x, 'y, #filter>: FilterFetch<'x, 'y>,)* { #(#query_fn_mut)* } diff --git a/crates/bevy_ecs/src/bundle.rs b/crates/bevy_ecs/src/bundle.rs index 572f7881f90b9..fc3354b7f2355 100644 --- a/crates/bevy_ecs/src/bundle.rs +++ b/crates/bevy_ecs/src/bundle.rs @@ -86,9 +86,9 @@ pub unsafe trait Bundle: Send + Sync + 'static { /// # Safety /// Caller must return data for each component in the bundle, in the order of this bundle's /// Components - unsafe fn from_components<'a, F>(func: F) -> Self + unsafe fn from_components(ctx: &mut T, func: F) -> Self where - F: FnMut() -> OwningPtr<'a> + 'a, + F: FnMut(&mut T) -> OwningPtr<'_>, Self: Sized; /// Calls `func` on each value, in the order of this bundle's Components. This will @@ -108,13 +108,13 @@ macro_rules! tuple_impl { #[allow(unused_variables, unused_mut)] #[allow(clippy::unused_unit)] - unsafe fn from_components<'a, F>(func: F) -> Self + unsafe fn from_components(ctx: &mut T, mut func: F) -> Self where - F: FnMut() -> OwningPtr<'a> + 'a + F: FnMut(&mut T) -> OwningPtr<'_> { #[allow(non_snake_case)] let ($(mut $name,)*) = ( - $(func().inner().cast::<$name>(),)* + $(func(ctx).inner().cast::<$name>(),)* ); ($($name.read(),)*) } @@ -124,7 +124,7 @@ macro_rules! tuple_impl { #[allow(non_snake_case)] let ($(mut $name,)*) = self; $( - OwningPtr::make($name, func); + OwningPtr::make($name, &mut func); )* } } diff --git a/crates/bevy_ecs/src/lib.rs b/crates/bevy_ecs/src/lib.rs index 4726d69f5b026..a27514df84b54 100644 --- a/crates/bevy_ecs/src/lib.rs +++ b/crates/bevy_ecs/src/lib.rs @@ -42,6 +42,7 @@ pub mod prelude { #[cfg(test)] mod tests { use crate as bevy_ecs; + use crate::query::QueryFetch; use crate::{ bundle::Bundle, component::{Component, ComponentId}, @@ -887,7 +888,7 @@ mod tests { fn get_filtered(world: &mut World) -> Vec where - F::Fetch: FilterFetch, + for<'x, 'y> QueryFetch<'x, 'y, F>: FilterFetch<'x, 'y>, { world .query_filtered::() diff --git a/crates/bevy_ecs/src/ptr.rs b/crates/bevy_ecs/src/ptr.rs index 9e91ebb6f2341..9458f1c024fa9 100644 --- a/crates/bevy_ecs/src/ptr.rs +++ b/crates/bevy_ecs/src/ptr.rs @@ -1,16 +1,77 @@ use std::{marker::PhantomData, mem::MaybeUninit, ptr::NonNull}; +/// Type-erased pointer into memory. Guaranteed to be correctly aligned, non-null and safe to read for a particular type. #[derive(Copy, Clone)] pub struct Ptr<'a>(NonNull, PhantomData<&'a u8>); -#[derive(Copy, Clone)] + +/// Type-erased pointer into memory. Guaranteed to be correctly aligned, non-null and safe to modify for a particular type. pub struct PtrMut<'a>(NonNull, PhantomData<&'a mut u8>); +/// Type-erased pointer into memory. Guaranteed to be correctly aligned, non-null and safe to move out of for a particular type. pub struct OwningPtr<'a>(NonNull, PhantomData<&'a mut u8>); +pub struct ThinSlicePtr<'a, T> { + ptr: NonNull, + #[cfg(debug_assertions)] + len: usize, + _marker: PhantomData<&'a [T]>, +} + +impl Clone for ThinSlicePtr<'_, T> { + fn clone(&self) -> Self { + Self { + ptr: self.ptr, + #[cfg(debug_assertions)] + len: self.len, + _marker: PhantomData, + } + } +} +impl Copy for ThinSlicePtr<'_, T> {} + +impl<'a, T> ThinSlicePtr<'a, T> { + pub fn new(slice: &'a [T]) -> Self { + unsafe { + Self { + ptr: NonNull::new_unchecked(slice.as_ptr() as *mut _), + #[cfg(debug_assertions)] + len: slice.len(), + _marker: PhantomData, + } + } + } + + /// # Safety + /// ptr must be valid for the returned lifetime. + pub unsafe fn new_raw(ptr: NonNull, #[cfg(debug_assertions)] len: usize) -> Self { + Self { + ptr, + #[cfg(debug_assertions)] + len, + _marker: PhantomData, + } + } + + /// # Safety + /// index must not be out of bounds + pub unsafe fn index(self, index: usize) -> &'a T { + debug_assert!(index < self.len); + &*self.ptr.as_ptr().add(index) + } + + /// # Safety + /// index must not be out of bounds, and the same element must not be mutably accessed twice. + pub unsafe fn index_mut(self, index: usize) -> &'a mut T { + debug_assert!(index < self.len); + &mut *self.ptr.as_ptr().add(index) + } +} + macro_rules! impl_ptr { ($ptr:ident) => { impl $ptr<'_> { - /// Safety: the offset cannot make the existing ptr null, or take it out of bounds for it's allocation. + /// # Safety + /// the offset cannot make the existing ptr null, or take it out of bounds for it's allocation. pub unsafe fn offset(self, count: isize) -> Self { Self( NonNull::new_unchecked(self.0.as_ptr().offset(count)), @@ -18,7 +79,8 @@ macro_rules! impl_ptr { ) } - /// Safety: the offset cannot make the existing ptr null, or take it out of bounds for it's allocation. + /// # Safety + /// the offset cannot make the existing ptr null, or take it out of bounds for it's allocation. pub unsafe fn add(self, count: usize) -> Self { Self( NonNull::new_unchecked(self.0.as_ptr().add(count)), @@ -26,58 +88,62 @@ macro_rules! impl_ptr { ) } + /// # Safety + /// the lifetime for the returned item must not exceed the lifetime `inner` is valid for pub unsafe fn new(inner: NonNull) -> Self { Self(inner, PhantomData) } - - pub unsafe fn inner_nonnull(self) -> NonNull { - self.0 - } } }; } impl_ptr!(Ptr); impl<'a> Ptr<'a> { - pub unsafe fn inner(self) -> *const u8 { - self.0.as_ptr() as *const _ + /// # Safety + /// another PtrMut for the same Ptr shouldn't be created until the first is dropped. + pub unsafe fn assert_unique(self) -> PtrMut<'a> { + PtrMut(self.0, PhantomData) } + /// # Safety + /// Must point to a valid T pub unsafe fn deref(self) -> &'a T { - &*self.inner().cast() + &*self.0.as_ptr().cast() } } impl_ptr!(PtrMut); impl<'a> PtrMut<'a> { - pub unsafe fn inner(self) -> *mut u8 { - self.0.as_ptr() + pub fn inner(self) -> NonNull { + self.0 } + /// # Safety + /// must have right to drop or move out of PtrMut, and current PtrMut should not be accessed again unless it's written to again. pub unsafe fn promote(self) -> OwningPtr<'a> { OwningPtr(self.0, PhantomData) } + /// # Safety + /// Must point to a valid T pub unsafe fn deref_mut(self) -> &'a mut T { - &mut *self.inner().cast() + &mut *self.inner().as_ptr().cast() } } impl_ptr!(OwningPtr); impl<'a> OwningPtr<'a> { - pub unsafe fn inner(self) -> *mut u8 { + pub fn inner(self) -> *mut u8 { self.0.as_ptr() } pub fn make) -> R, R>(val: T, f: F) -> R { - let temp = MaybeUninit::new(val); + let mut temp = MaybeUninit::new(val); let ptr = unsafe { NonNull::new_unchecked(temp.as_mut_ptr().cast::()) }; f(Self(ptr, PhantomData)) } + /// # Safety + /// must point to a valid T. pub unsafe fn read(self) -> T { self.inner().cast::().read() } - - pub unsafe fn deref_mut(self) -> &'a mut T { - &mut *self.inner().cast() - } } diff --git a/crates/bevy_ecs/src/query/fetch.rs b/crates/bevy_ecs/src/query/fetch.rs index 60084ff442d0e..76b01b49b7517 100644 --- a/crates/bevy_ecs/src/query/fetch.rs +++ b/crates/bevy_ecs/src/query/fetch.rs @@ -3,16 +3,13 @@ use crate::{ change_detection::Ticks, component::{Component, ComponentId, ComponentStorage, ComponentTicks, StorageType}, entity::Entity, + ptr::ThinSlicePtr, query::{Access, FilteredAccess}, storage::{ComponentSparseSet, Table, Tables}, world::{Mut, World}, }; use bevy_ecs_macros::all_tuples; -use std::{ - cell::UnsafeCell, - marker::PhantomData, - ptr::{self, NonNull}, -}; +use std::{cell::UnsafeCell, marker::PhantomData}; /// Types that can be queried from a [`World`]. /// @@ -41,13 +38,28 @@ use std::{ /// /// [`Or`]: crate::query::Or pub trait WorldQuery { - type Fetch: for<'world, 'state> Fetch<'world, 'state, State = Self::State>; + /// Returns true if (and only if) every table of every archetype matched by Self::FetchInit::Fetch contains + /// all of the matched components. This is used to select a more efficient "table iterator" + /// for "dense" queries. If this returns true, [`Fetch::set_table`] and [`Fetch::table_fetch`] + /// will be called for iterators. If this returns false, [`Fetch::set_archetype`] and + /// [`Fetch::archetype_fetch`] will be called for iterators. + const IS_DENSE: bool; + + type FetchInit: for<'world, 'state> FetchInit<'world, 'state, State = Self::State>; type State: FetchState; + + fn shrink<'wlong: 'wshort, 'slong: 'sshort, 'wshort, 'sshort>( + item: QueryItem<'wlong, 'slong, Self>, + ) -> QueryItem<'wshort, 'sshort, Self>; } -pub trait Fetch<'world, 'state>: Sized { - type Item; +pub type QueryFetch<'w, 's, Q> = <::FetchInit as FetchInit<'w, 's>>::Fetch; +pub type QueryItem<'w, 's, Q> = <::FetchInit as FetchInit<'w, 's>>::Item; + +pub trait FetchInit<'world, 'state> { type State: FetchState; + type Fetch: Fetch<'world, 'state, State = Self::State, Item = Self::Item>; + type Item; /// Creates a new instance of this fetch. /// @@ -55,19 +67,17 @@ pub trait Fetch<'world, 'state>: Sized { /// /// `state` must have been initialized (via [FetchState::init]) using the same `world` passed in /// to this function. - unsafe fn init( - world: &World, - state: &Self::State, + unsafe fn fetch_init( + world: &'world World, + state: &'state Self::State, last_change_tick: u32, change_tick: u32, - ) -> Self; + ) -> Self::Fetch; +} - /// Returns true if (and only if) every table of every archetype matched by this Fetch contains - /// all of the matched components. This is used to select a more efficient "table iterator" - /// for "dense" queries. If this returns true, [`Fetch::set_table`] and [`Fetch::table_fetch`] - /// will be called for iterators. If this returns false, [`Fetch::set_archetype`] and - /// [`Fetch::archetype_fetch`] will be called for iterators. - const IS_DENSE: bool; +pub trait Fetch<'world, 'state>: Sized { + type Item; + type State: FetchState; /// Adjusts internal state to account for the next [`Archetype`]. This will always be called on /// archetypes that match this [`Fetch`]. @@ -76,7 +86,12 @@ pub trait Fetch<'world, 'state>: Sized { /// /// `archetype` and `tables` must be from the [`World`] [`Fetch::init`] was called on. `state` must /// be the [Self::State] this was initialized with. - unsafe fn set_archetype(&mut self, state: &Self::State, archetype: &Archetype, tables: &Tables); + unsafe fn set_archetype( + &mut self, + state: &'state Self::State, + archetype: &'world Archetype, + tables: &'world Tables, + ); /// Adjusts internal state to account for the next [`Table`]. This will always be called on tables /// that match this [`Fetch`]. @@ -85,7 +100,7 @@ pub trait Fetch<'world, 'state>: Sized { /// /// `table` must be from the [`World`] [`Fetch::init`] was called on. `state` must be the /// [Self::State] this was initialized with. - unsafe fn set_table(&mut self, state: &Self::State, table: &Table); + unsafe fn set_table(&mut self, state: &'state Self::State, table: &'world Table); /// Fetch [`Self::Item`] for the given `archetype_index` in the current [`Archetype`]. This must /// always be called after [`Fetch::set_archetype`] with an `archetype_index` in the range of @@ -128,22 +143,30 @@ pub unsafe trait FetchState: Send + Sync + Sized { fn matches_table(&self, table: &Table) -> bool; } -/// A fetch that is read only. This must only be implemented for read-only fetches. -pub unsafe trait ReadOnlyFetch {} +/// A query that is read only. This must only be implemented for read-only queries. +pub unsafe trait ReadOnlyQuery: WorldQuery {} impl WorldQuery for Entity { - type Fetch = EntityFetch; + const IS_DENSE: bool = true; + + type FetchInit = Self; type State = EntityState; + + fn shrink<'wlong: 'wshort, 'slong: 'sshort, 'wshort, 'sshort>( + item: QueryItem<'wlong, 'slong, Self>, + ) -> QueryItem<'wshort, 'sshort, Self> { + item + } } /// The [`Fetch`] of [`Entity`]. #[derive(Clone)] -pub struct EntityFetch { - entities: *const Entity, +pub struct EntityFetch<'w> { + entities: Option>, } /// SAFETY: access is read only -unsafe impl ReadOnlyFetch for EntityFetch {} +unsafe impl ReadOnlyQuery for Entity {} /// The [`FetchState`] of [`Entity`]. pub struct EntityState; @@ -174,52 +197,73 @@ unsafe impl FetchState for EntityState { } } -impl<'w, 's> Fetch<'w, 's> for EntityFetch { - type Item = Entity; +impl<'w> FetchInit<'w, '_> for Entity { type State = EntityState; + type Fetch = EntityFetch<'w>; + type Item = Entity; - const IS_DENSE: bool = true; - - unsafe fn init( - _world: &World, + unsafe fn fetch_init( + _world: &'w World, _state: &Self::State, _last_change_tick: u32, _change_tick: u32, - ) -> Self { - Self { - entities: std::ptr::null::(), - } + ) -> EntityFetch<'w> { + EntityFetch { entities: None } } +} + +impl<'w, 's> Fetch<'w, 's> for EntityFetch<'w> { + type Item = Entity; + type State = EntityState; #[inline] unsafe fn set_archetype( &mut self, _state: &Self::State, - archetype: &Archetype, + archetype: &'w Archetype, _tables: &Tables, ) { - self.entities = archetype.entities().as_ptr(); + self.entities = Some(ThinSlicePtr::new(archetype.entities())); } #[inline] - unsafe fn set_table(&mut self, _state: &Self::State, table: &Table) { - self.entities = table.entities().as_ptr(); + unsafe fn set_table(&mut self, _state: &Self::State, table: &'w Table) { + self.entities = Some(ThinSlicePtr::new(table.entities())); } #[inline] unsafe fn table_fetch(&mut self, table_row: usize) -> Self::Item { - *self.entities.add(table_row) + *self + .entities + .unwrap_or_else(|| std::hint::unreachable_unchecked()) + .index(table_row) } #[inline] unsafe fn archetype_fetch(&mut self, archetype_index: usize) -> Self::Item { - *self.entities.add(archetype_index) + *self + .entities + .unwrap_or_else(|| std::hint::unreachable_unchecked()) + .index(archetype_index) } } impl WorldQuery for &T { - type Fetch = ReadFetch; + const IS_DENSE: bool = { + match T::Storage::STORAGE_TYPE { + StorageType::Table => true, + StorageType::SparseSet => false, + } + }; + + type FetchInit = Self; type State = ReadState; + + fn shrink<'wlong: 'wshort, 'slong: 'sshort, 'wshort, 'sshort>( + item: QueryItem<'wlong, 'slong, Self>, + ) -> QueryItem<'wshort, 'sshort, Self> { + item + } } /// The [`FetchState`] of `&T`. @@ -269,14 +313,16 @@ unsafe impl FetchState for ReadState { } /// The [`Fetch`] of `&T`. -pub struct ReadFetch { - table_components: NonNull, - entity_table_rows: *const usize, - entities: *const Entity, - sparse_set: *const ComponentSparseSet, +pub struct ReadFetch<'w, T> { + // T::Storage = TableStorage + table_components: Option>, + entity_table_rows: Option>, + // T::Storage = SparseStorage + entities: Option>, + sparse_set: Option<&'w ComponentSparseSet>, } -impl Clone for ReadFetch { +impl Clone for ReadFetch<'_, T> { fn clone(&self) -> Self { Self { table_components: self.table_components, @@ -288,112 +334,127 @@ impl Clone for ReadFetch { } /// SAFETY: access is read only -unsafe impl ReadOnlyFetch for ReadFetch {} +unsafe impl ReadOnlyQuery for &T {} -impl<'w, 's, T: Component> Fetch<'w, 's> for ReadFetch { - type Item = &'w T; +impl<'w, T: Component> FetchInit<'w, '_> for &T { + type Fetch = ReadFetch<'w, T>; type State = ReadState; + type Item = &'w T; - const IS_DENSE: bool = { - match T::Storage::STORAGE_TYPE { - StorageType::Table => true, - StorageType::SparseSet => false, - } - }; - - unsafe fn init( - world: &World, + unsafe fn fetch_init( + world: &'w World, state: &Self::State, _last_change_tick: u32, _change_tick: u32, - ) -> Self { - let mut value = Self { - table_components: NonNull::dangling(), - entities: ptr::null::(), - entity_table_rows: ptr::null::(), - sparse_set: ptr::null::(), - }; - if T::Storage::STORAGE_TYPE == StorageType::SparseSet { - value.sparse_set = world - .storages() - .sparse_sets - .get(state.component_id) - .unwrap(); - } - value + ) -> ReadFetch<'w, T> { + ReadFetch { + table_components: None, + entity_table_rows: None, + entities: None, + sparse_set: (T::Storage::STORAGE_TYPE == StorageType::SparseSet).then(|| { + world + .storages() + .sparse_sets + .get(state.component_id) + .unwrap() + }), + } } +} + +impl<'w, 's, T: Component> Fetch<'w, 's> for ReadFetch<'w, T> { + type Item = &'w T; + type State = ReadState; #[inline] unsafe fn set_archetype( &mut self, state: &Self::State, - archetype: &Archetype, - tables: &Tables, + archetype: &'w Archetype, + tables: &'w Tables, ) { match T::Storage::STORAGE_TYPE { StorageType::Table => { - self.entity_table_rows = archetype.entity_table_rows().as_ptr(); + self.entity_table_rows = Some(ThinSlicePtr::new(archetype.entity_table_rows())); let column = tables[archetype.table_id()] .get_column(state.component_id) .unwrap(); - self.table_components = column.get_data_ptr().inner_nonnull().cast::(); + self.table_components = Some(column.get_data_slice()); } - StorageType::SparseSet => self.entities = archetype.entities().as_ptr(), + StorageType::SparseSet => self.entities = Some(ThinSlicePtr::new(archetype.entities())), } } #[inline] - unsafe fn set_table(&mut self, state: &Self::State, table: &Table) { - self.table_components = table - .get_column_mut(state.component_id) - .unwrap() - .get_data_ptr_mut() - .inner_nonnull() - .cast::(); + unsafe fn set_table(&mut self, state: &Self::State, table: &'w Table) { + self.table_components = Some( + table + .get_column(state.component_id) + .unwrap() + .get_data_slice(), + ); } #[inline] unsafe fn archetype_fetch(&mut self, archetype_index: usize) -> Self::Item { match T::Storage::STORAGE_TYPE { StorageType::Table => { - let table_row = *self.entity_table_rows.add(archetype_index); - &*self.table_components.as_ptr().add(table_row) + let (entity_table_rows, table_components) = self + .entity_table_rows + .zip(self.table_components) + .unwrap_or_else(|| std::hint::unreachable_unchecked()); + let table_row = *entity_table_rows.index(archetype_index); + table_components.index(table_row) } StorageType::SparseSet => { - let entity = *self.entities.add(archetype_index); - &*(*self.sparse_set) - .get(entity) - .unwrap() - .inner_nonnull() - .cast::() - .as_ptr() + let (entities, sparse_set) = self + .entities + .zip(self.sparse_set) + .unwrap_or_else(|| std::hint::unreachable_unchecked()); + let entity = *entities.index(archetype_index); + sparse_set.get(entity).unwrap().deref::() } } } #[inline] unsafe fn table_fetch(&mut self, table_row: usize) -> Self::Item { - &*self.table_components.as_ptr().add(table_row) + self.table_components + .unwrap_or_else(|| std::hint::unreachable_unchecked()) + .index(table_row) } } impl WorldQuery for &mut T { - type Fetch = WriteFetch; + const IS_DENSE: bool = { + match T::Storage::STORAGE_TYPE { + StorageType::Table => true, + StorageType::SparseSet => false, + } + }; + + type FetchInit = Self; type State = WriteState; + + fn shrink<'wlong: 'wshort, 'slong: 'sshort, 'wshort, 'sshort>( + item: QueryItem<'wlong, 'slong, Self>, + ) -> QueryItem<'wshort, 'sshort, Self> { + item + } } /// The [`Fetch`] of `&mut T`. -pub struct WriteFetch { - table_components: NonNull, - table_ticks: *const UnsafeCell, - entities: *const Entity, - entity_table_rows: *const usize, - sparse_set: *const ComponentSparseSet, +pub struct WriteFetch<'w, T> { + table_components: Option>, + table_ticks: Option>>, + entities: Option>, + entity_table_rows: Option>, + sparse_set: Option<&'w ComponentSparseSet>, last_change_tick: u32, change_tick: u32, } -impl Clone for WriteFetch { +impl Clone for WriteFetch<'_, T> { fn clone(&self) -> Self { Self { table_components: self.table_components, @@ -453,91 +514,94 @@ unsafe impl FetchState for WriteState { } } -impl<'w, 's, T: Component> Fetch<'w, 's> for WriteFetch { - type Item = Mut<'w, T>; +impl<'w, T: Component> FetchInit<'w, '_> for &mut T { type State = WriteState; + type Fetch = WriteFetch<'w, T>; + type Item = Mut<'w, T>; - const IS_DENSE: bool = { - match T::Storage::STORAGE_TYPE { - StorageType::Table => true, - StorageType::SparseSet => false, - } - }; - - unsafe fn init( - world: &World, + unsafe fn fetch_init( + world: &'w World, state: &Self::State, last_change_tick: u32, change_tick: u32, - ) -> Self { - let mut value = Self { - table_components: NonNull::dangling(), - entities: ptr::null::(), - entity_table_rows: ptr::null::(), - sparse_set: ptr::null::(), - table_ticks: ptr::null::>(), + ) -> Self::Fetch { + Self::Fetch { + table_components: None, + entities: None, + entity_table_rows: None, + sparse_set: (T::Storage::STORAGE_TYPE == StorageType::SparseSet).then(|| { + world + .storages() + .sparse_sets + .get(state.component_id) + .unwrap() + }), + table_ticks: None, last_change_tick, change_tick, - }; - if T::Storage::STORAGE_TYPE == StorageType::SparseSet { - value.sparse_set = world - .storages() - .sparse_sets - .get(state.component_id) - .unwrap(); } - value } +} +impl<'w, 's, T: Component> Fetch<'w, 's> for WriteFetch<'w, T> { + type Item = Mut<'w, T>; + type State = WriteState; #[inline] unsafe fn set_archetype( &mut self, state: &Self::State, - archetype: &Archetype, - tables: &Tables, + archetype: &'w Archetype, + tables: &'w Tables, ) { match T::Storage::STORAGE_TYPE { StorageType::Table => { - self.entity_table_rows = archetype.entity_table_rows().as_ptr(); + self.entity_table_rows = Some(ThinSlicePtr::new(archetype.entity_table_rows())); let column = tables[archetype.table_id()] .get_column(state.component_id) .unwrap(); - self.table_components = column.get_data_ptr().inner_nonnull().cast::(); - self.table_ticks = column.get_ticks_ptr(); + self.table_components = Some(column.get_data_slice()); + self.table_ticks = Some(ThinSlicePtr::new(column.get_ticks())); } - StorageType::SparseSet => self.entities = archetype.entities().as_ptr(), + StorageType::SparseSet => self.entities = Some(ThinSlicePtr::new(archetype.entities())), } } #[inline] - unsafe fn set_table(&mut self, state: &Self::State, table: &Table) { + unsafe fn set_table(&mut self, state: &Self::State, table: &'w Table) { let column = table.get_column(state.component_id).unwrap(); - self.table_components = column.get_data_ptr().inner_nonnull().cast::(); - self.table_ticks = column.get_ticks_ptr(); + self.table_components = Some(column.get_data_slice()); + self.table_ticks = Some(ThinSlicePtr::new(column.get_ticks())); } #[inline] unsafe fn archetype_fetch(&mut self, archetype_index: usize) -> Self::Item { match T::Storage::STORAGE_TYPE { StorageType::Table => { - let table_row = *self.entity_table_rows.add(archetype_index); + let (entity_table_rows, (table_components, table_ticks)) = self + .entity_table_rows + .zip(self.table_components.zip(self.table_ticks)) + .unwrap_or_else(|| std::hint::unreachable_unchecked()); + let table_row = *entity_table_rows.index(archetype_index); Mut { - value: &mut *self.table_components.as_ptr().add(table_row), + value: table_components.index_mut(table_row), ticks: Ticks { - component_ticks: &mut *(&*self.table_ticks.add(table_row)).get(), + component_ticks: &mut *table_ticks.index(table_row).get(), change_tick: self.change_tick, last_change_tick: self.last_change_tick, }, } } StorageType::SparseSet => { - let entity = *self.entities.add(archetype_index); - let (component, component_ticks) = - (*self.sparse_set).get_with_ticks(entity).unwrap(); + let (entities, sparse_set) = self + .entities + .zip(self.sparse_set) + .unwrap_or_else(|| std::hint::unreachable_unchecked()); + let entity = *entities.index(archetype_index); + let (component, component_ticks) = sparse_set.get_with_ticks(entity).unwrap(); Mut { - value: &mut *component.inner_nonnull().cast::().as_ptr(), + value: component.assert_unique().deref_mut(), ticks: Ticks { - component_ticks: &mut *component_ticks, + component_ticks: &mut *component_ticks.get(), change_tick: self.change_tick, last_change_tick: self.last_change_tick, }, @@ -548,10 +612,14 @@ impl<'w, 's, T: Component> Fetch<'w, 's> for WriteFetch { #[inline] unsafe fn table_fetch(&mut self, table_row: usize) -> Self::Item { + let (table_components, table_ticks) = self + .table_components + .zip(self.table_ticks) + .unwrap_or_else(|| std::hint::unreachable_unchecked()); Mut { - value: &mut *self.table_components.as_ptr().add(table_row), + value: table_components.index_mut(table_row), ticks: Ticks { - component_ticks: &mut *(&*self.table_ticks.add(table_row)).get(), + component_ticks: &mut *(table_ticks.index(table_row)).get(), change_tick: self.change_tick, last_change_tick: self.last_change_tick, }, @@ -560,8 +628,16 @@ impl<'w, 's, T: Component> Fetch<'w, 's> for WriteFetch { } impl WorldQuery for Option { - type Fetch = OptionFetch; + const IS_DENSE: bool = T::IS_DENSE; + + type FetchInit = Option; type State = OptionState; + + fn shrink<'wlong: 'wshort, 'slong: 'sshort, 'wshort, 'sshort>( + item: QueryItem<'wlong, 'slong, Self>, + ) -> QueryItem<'wshort, 'sshort, Self> { + item.map(T::shrink) + } } /// The [`Fetch`] of `Option`. @@ -572,7 +648,7 @@ pub struct OptionFetch { } /// SAFETY: OptionFetch is read only because T is read only -unsafe impl ReadOnlyFetch for OptionFetch {} +unsafe impl ReadOnlyQuery for Option {} /// The [`FetchState`] of `Option`. pub struct OptionState { @@ -612,30 +688,34 @@ unsafe impl FetchState for OptionState { } } -impl<'w, 's, T: Fetch<'w, 's>> Fetch<'w, 's> for OptionFetch { - type Item = Option; +impl<'w, 's, T: FetchInit<'w, 's>> FetchInit<'w, 's> for Option { type State = OptionState; + type Fetch = OptionFetch; + type Item = Option; - const IS_DENSE: bool = T::IS_DENSE; - - unsafe fn init( - world: &World, - state: &Self::State, + unsafe fn fetch_init( + world: &'w World, + state: &'s Self::State, last_change_tick: u32, change_tick: u32, - ) -> Self { - Self { - fetch: T::init(world, &state.state, last_change_tick, change_tick), + ) -> Self::Fetch { + Self::Fetch { + fetch: T::fetch_init(world, &state.state, last_change_tick, change_tick), matches: false, } } +} + +impl<'w, 's, T: Fetch<'w, 's>> Fetch<'w, 's> for OptionFetch { + type Item = Option; + type State = OptionState; #[inline] unsafe fn set_archetype( &mut self, - state: &Self::State, - archetype: &Archetype, - tables: &Tables, + state: &'s Self::State, + archetype: &'w Archetype, + tables: &'w Tables, ) { self.matches = state.state.matches_archetype(archetype); if self.matches { @@ -644,7 +724,7 @@ impl<'w, 's, T: Fetch<'w, 's>> Fetch<'w, 's> for OptionFetch { } #[inline] - unsafe fn set_table(&mut self, state: &Self::State, table: &Table) { + unsafe fn set_table(&mut self, state: &'s Self::State, table: &'w Table) { self.matches = state.state.matches_table(table); if self.matches { self.fetch.set_table(&state.state, table); @@ -735,8 +815,21 @@ impl ChangeTrackers { } impl WorldQuery for ChangeTrackers { - type Fetch = ChangeTrackersFetch; + const IS_DENSE: bool = { + match T::Storage::STORAGE_TYPE { + StorageType::Table => true, + StorageType::SparseSet => false, + } + }; + + type FetchInit = Self; type State = ChangeTrackersState; + + fn shrink<'wlong: 'wshort, 'slong: 'sshort, 'wshort, 'sshort>( + item: QueryItem<'wlong, 'slong, Self>, + ) -> QueryItem<'wshort, 'sshort, Self> { + item + } } /// The [`FetchState`] of [`ChangeTrackers`]. @@ -786,17 +879,17 @@ unsafe impl FetchState for ChangeTrackersState { } /// The [`Fetch`] of [`ChangeTrackers`]. -pub struct ChangeTrackersFetch { - table_ticks: *const ComponentTicks, - entity_table_rows: *const usize, - entities: *const Entity, - sparse_set: *const ComponentSparseSet, +pub struct ChangeTrackersFetch<'w, T> { + table_ticks: Option>>, + entity_table_rows: Option>, + entities: Option>, + sparse_set: Option<&'w ComponentSparseSet>, marker: PhantomData, last_change_tick: u32, change_tick: u32, } -impl Clone for ChangeTrackersFetch { +impl Clone for ChangeTrackersFetch<'_, T> { fn clone(&self) -> Self { Self { table_ticks: self.table_ticks, @@ -811,87 +904,99 @@ impl Clone for ChangeTrackersFetch { } /// SAFETY: access is read only -unsafe impl ReadOnlyFetch for ChangeTrackersFetch {} +unsafe impl ReadOnlyQuery for ChangeTrackers {} -impl<'w, 's, T: Component> Fetch<'w, 's> for ChangeTrackersFetch { - type Item = ChangeTrackers; +impl<'w, T: Component> FetchInit<'w, '_> for ChangeTrackers { type State = ChangeTrackersState; + type Fetch = ChangeTrackersFetch<'w, T>; + type Item = ChangeTrackers; - const IS_DENSE: bool = { - match T::Storage::STORAGE_TYPE { - StorageType::Table => true, - StorageType::SparseSet => false, - } - }; - - unsafe fn init( - world: &World, + unsafe fn fetch_init( + world: &'w World, state: &Self::State, last_change_tick: u32, change_tick: u32, - ) -> Self { - let mut value = Self { - table_ticks: ptr::null::(), - entities: ptr::null::(), - entity_table_rows: ptr::null::(), - sparse_set: ptr::null::(), + ) -> ChangeTrackersFetch<'w, T> { + ChangeTrackersFetch { + table_ticks: None, + entities: None, + entity_table_rows: None, + sparse_set: (T::Storage::STORAGE_TYPE == StorageType::SparseSet).then(|| { + world + .storages() + .sparse_sets + .get(state.component_id) + .unwrap() + }), marker: PhantomData, last_change_tick, change_tick, - }; - if T::Storage::STORAGE_TYPE == StorageType::SparseSet { - value.sparse_set = world - .storages() - .sparse_sets - .get(state.component_id) - .unwrap(); } - value } +} + +impl<'w, 's, T: Component> Fetch<'w, 's> for ChangeTrackersFetch<'w, T> { + type Item = ChangeTrackers; + type State = ChangeTrackersState; #[inline] unsafe fn set_archetype( &mut self, state: &Self::State, - archetype: &Archetype, - tables: &Tables, + archetype: &'w Archetype, + tables: &'w Tables, ) { match T::Storage::STORAGE_TYPE { StorageType::Table => { - self.entity_table_rows = archetype.entity_table_rows().as_ptr(); + self.entity_table_rows = Some(ThinSlicePtr::new(archetype.entity_table_rows())); let column = tables[archetype.table_id()] .get_column(state.component_id) .unwrap(); - self.table_ticks = column.get_ticks_const_ptr(); + self.table_ticks = Some(ThinSlicePtr::new(column.get_ticks())); } - StorageType::SparseSet => self.entities = archetype.entities().as_ptr(), + StorageType::SparseSet => self.entities = Some(ThinSlicePtr::new(archetype.entities())), } } #[inline] - unsafe fn set_table(&mut self, state: &Self::State, table: &Table) { - self.table_ticks = table - .get_column(state.component_id) - .unwrap() - .get_ticks_const_ptr(); + unsafe fn set_table(&mut self, state: &Self::State, table: &'w Table) { + self.table_ticks = Some(ThinSlicePtr::new( + table.get_column(state.component_id).unwrap().get_ticks(), + )); } #[inline] unsafe fn archetype_fetch(&mut self, archetype_index: usize) -> Self::Item { match T::Storage::STORAGE_TYPE { StorageType::Table => { - let table_row = *self.entity_table_rows.add(archetype_index); + let table_row = *self + .entity_table_rows + .unwrap_or_else(|| std::hint::unreachable_unchecked()) + .index(archetype_index); ChangeTrackers { - component_ticks: (&*self.table_ticks.add(table_row)).clone(), + component_ticks: (&*self + .table_ticks + .unwrap_or_else(|| std::hint::unreachable_unchecked()) + .index(table_row) + .get()) + .clone(), marker: PhantomData, last_change_tick: self.last_change_tick, change_tick: self.change_tick, } } StorageType::SparseSet => { - let entity = *self.entities.add(archetype_index); + let entity = *self + .entities + .unwrap_or_else(|| std::hint::unreachable_unchecked()) + .index(archetype_index); ChangeTrackers { - component_ticks: (&*self.sparse_set).get_ticks(entity).cloned().unwrap(), + component_ticks: self + .sparse_set + .unwrap_or_else(|| std::hint::unreachable_unchecked()) + .get_ticks(entity) + .cloned() + .unwrap(), marker: PhantomData, last_change_tick: self.last_change_tick, change_tick: self.change_tick, @@ -903,7 +1008,12 @@ impl<'w, 's, T: Component> Fetch<'w, 's> for ChangeTrackersFetch { #[inline] unsafe fn table_fetch(&mut self, table_row: usize) -> Self::Item { ChangeTrackers { - component_ticks: (&*self.table_ticks.add(table_row)).clone(), + component_ticks: (&*self + .table_ticks + .unwrap_or_else(|| std::hint::unreachable_unchecked()) + .index(table_row) + .get()) + .clone(), marker: PhantomData, last_change_tick: self.last_change_tick, change_tick: self.change_tick, @@ -913,28 +1023,34 @@ impl<'w, 's, T: Component> Fetch<'w, 's> for ChangeTrackersFetch { macro_rules! impl_tuple_fetch { ($(($name: ident, $state: ident)),*) => { + #[allow(unused_variables)] #[allow(non_snake_case)] - impl<'w, 's, $($name: Fetch<'w, 's>),*> Fetch<'w, 's> for ($($name,)*) { - type Item = ($($name::Item,)*); + impl<'w, 's, $($name: FetchInit<'w, 's>),*> FetchInit<'w, 's> for ($($name,)*) { type State = ($($name::State,)*); + type Fetch = ($($name::Fetch,)*); + type Item = ($($name::Item,)*); #[allow(clippy::unused_unit)] - unsafe fn init(_world: &World, state: &Self::State, _last_change_tick: u32, _change_tick: u32) -> Self { + unsafe fn fetch_init(_world: &'w World, state: &'s Self::State, _last_change_tick: u32, _change_tick: u32) -> Self::Fetch { let ($($name,)*) = state; - ($($name::init(_world, $name, _last_change_tick, _change_tick),)*) + ($($name::fetch_init(_world, $name, _last_change_tick, _change_tick),)*) } + } - const IS_DENSE: bool = true $(&& $name::IS_DENSE)*; + #[allow(non_snake_case)] + impl<'w, 's, $($name: Fetch<'w, 's>),*> Fetch<'w, 's> for ($($name,)*) { + type Item = ($($name::Item,)*); + type State = ($($name::State,)*); #[inline] - unsafe fn set_archetype(&mut self, _state: &Self::State, _archetype: &Archetype, _tables: &Tables) { + unsafe fn set_archetype(&mut self, _state: &'s Self::State, _archetype: &'w Archetype, _tables: &'w Tables) { let ($($name,)*) = self; let ($($state,)*) = _state; $($name.set_archetype($state, _archetype, _tables);)* } #[inline] - unsafe fn set_table(&mut self, _state: &Self::State, _table: &Table) { + unsafe fn set_table(&mut self, _state: &'s Self::State, _table: &'w Table) { let ($($name,)*) = self; let ($($state,)*) = _state; $($name.set_table($state, _table);)* @@ -984,13 +1100,26 @@ macro_rules! impl_tuple_fetch { } } + #[allow(non_snake_case)] + #[allow(clippy::unused_unit)] impl<$($name: WorldQuery),*> WorldQuery for ($($name,)*) { - type Fetch = ($($name::Fetch,)*); + const IS_DENSE: bool = true $(&& $name::IS_DENSE)*; + + type FetchInit = ($($name::FetchInit,)*); type State = ($($name::State,)*); + + fn shrink<'wlong: 'wshort, 'slong: 'sshort, 'wshort, 'sshort>( + item: QueryItem<'wlong, 'slong, Self>, + ) -> QueryItem<'wshort, 'sshort, Self> { + let ($($name,)*) = item; + ($( + $name::shrink($name), + )*) + } } /// SAFETY: each item in the tuple is read only - unsafe impl<$($name: ReadOnlyFetch),*> ReadOnlyFetch for ($($name,)*) {} + unsafe impl<$($name: ReadOnlyQuery),*> ReadOnlyQuery for ($($name,)*) {} }; } diff --git a/crates/bevy_ecs/src/query/filter.rs b/crates/bevy_ecs/src/query/filter.rs index bbd8ec567a985..e5e46395665e1 100644 --- a/crates/bevy_ecs/src/query/filter.rs +++ b/crates/bevy_ecs/src/query/filter.rs @@ -2,16 +2,17 @@ use crate::{ archetype::{Archetype, ArchetypeComponentId}, component::{Component, ComponentId, ComponentStorage, ComponentTicks, StorageType}, entity::Entity, - query::{Access, Fetch, FetchState, FilteredAccess, WorldQuery}, + ptr::ThinSlicePtr, + query::{Access, Fetch, FetchInit, FetchState, FilteredAccess, WorldQuery}, storage::{ComponentSparseSet, Table, Tables}, world::World, }; use bevy_ecs_macros::all_tuples; -use std::{cell::UnsafeCell, marker::PhantomData, ptr}; +use std::{cell::UnsafeCell, marker::PhantomData}; /// Extension trait for [`Fetch`] containing methods used by query filters. /// This trait exists to allow "short circuit" behaviors for relevant query filter fetches. -pub trait FilterFetch: for<'w, 's> Fetch<'w, 's> { +pub trait FilterFetch<'w, 's>: Fetch<'w, 's> { /// # Safety /// /// Must always be called _after_ [`Fetch::set_archetype`]. `archetype_index` must be in the range @@ -25,9 +26,9 @@ pub trait FilterFetch: for<'w, 's> Fetch<'w, 's> { unsafe fn table_filter_fetch(&mut self, table_row: usize) -> bool; } -impl FilterFetch for T +impl<'w, 's, T> FilterFetch<'w, 's> for T where - T: for<'w, 's> Fetch<'w, 's, Item = bool>, + T: Fetch<'w, 's, Item = bool>, { #[inline] unsafe fn archetype_filter_fetch(&mut self, archetype_index: usize) -> bool { @@ -70,8 +71,21 @@ where pub struct With(PhantomData); impl WorldQuery for With { - type Fetch = WithFetch; + const IS_DENSE: bool = { + match T::Storage::STORAGE_TYPE { + StorageType::Table => true, + StorageType::SparseSet => false, + } + }; + + type FetchInit = Self; type State = WithState; + + fn shrink<'wlong: 'wshort, 'slong: 'sshort, 'wshort, 'sshort>( + item: super::QueryItem<'wlong, 'slong, Self>, + ) -> super::QueryItem<'wshort, 'sshort, Self> { + item + } } /// The [`Fetch`] of [`With`]. @@ -117,27 +131,26 @@ unsafe impl FetchState for WithState { } } -impl<'w, 's, T: Component> Fetch<'w, 's> for WithFetch { - type Item = bool; +impl FetchInit<'_, '_> for With { type State = WithState; + type Fetch = WithFetch; + type Item = bool; - unsafe fn init( + unsafe fn fetch_init( _world: &World, _state: &Self::State, _last_change_tick: u32, _change_tick: u32, - ) -> Self { - Self { + ) -> Self::Fetch { + Self::Fetch { marker: PhantomData, } } +} - const IS_DENSE: bool = { - match T::Storage::STORAGE_TYPE { - StorageType::Table => true, - StorageType::SparseSet => false, - } - }; +impl<'w, 's, T: Component> Fetch<'w, 's> for WithFetch { + type Item = bool; + type State = WithState; #[inline] unsafe fn set_table(&mut self, _state: &Self::State, _table: &Table) {} @@ -189,8 +202,21 @@ impl<'w, 's, T: Component> Fetch<'w, 's> for WithFetch { pub struct Without(PhantomData); impl WorldQuery for Without { - type Fetch = WithoutFetch; + const IS_DENSE: bool = { + match T::Storage::STORAGE_TYPE { + StorageType::Table => true, + StorageType::SparseSet => false, + } + }; + + type FetchInit = Self; type State = WithoutState; + + fn shrink<'wlong: 'wshort, 'slong: 'sshort, 'wshort, 'sshort>( + item: super::QueryItem<'wlong, 'slong, Self>, + ) -> super::QueryItem<'wshort, 'sshort, Self> { + item + } } /// The [`Fetch`] of [`Without`]. @@ -236,27 +262,26 @@ unsafe impl FetchState for WithoutState { } } -impl<'w, 's, T: Component> Fetch<'w, 's> for WithoutFetch { - type Item = bool; +impl FetchInit<'_, '_> for Without { type State = WithoutState; + type Fetch = WithoutFetch; + type Item = bool; - unsafe fn init( + unsafe fn fetch_init( _world: &World, _state: &Self::State, _last_change_tick: u32, _change_tick: u32, - ) -> Self { - Self { + ) -> Self::Fetch { + WithoutFetch { marker: PhantomData, } } +} - const IS_DENSE: bool = { - match T::Storage::STORAGE_TYPE { - StorageType::Table => true, - StorageType::SparseSet => false, - } - }; +impl<'w, 's, T: Component> Fetch<'w, 's> for WithoutFetch { + type Item = bool; + type State = WithoutState; #[inline] unsafe fn set_table(&mut self, _state: &Self::State, _table: &Table) {} @@ -314,16 +339,17 @@ impl<'w, 's, T: Component> Fetch<'w, 's> for WithoutFetch { pub struct Or(pub T); /// The [`Fetch`] of [`Or`]. -pub struct OrFetch { +pub struct OrFetch<'w, 's, T: FilterFetch<'w, 's>> { fetch: T, matches: bool, + _marker: PhantomData<(&'w (), &'s ())>, } macro_rules! impl_query_filter_tuple { ($(($filter: ident, $state: ident)),*) => { #[allow(unused_variables)] #[allow(non_snake_case)] - impl<'a, $($filter: FilterFetch),*> FilterFetch for ($($filter,)*) { + impl<'w, 's, $($filter: FilterFetch<'w, 's>),*> FilterFetch<'w, 's> for ($($filter,)*) { #[inline] unsafe fn table_filter_fetch(&mut self, table_row: usize) -> bool { let ($($filter,)*) = self; @@ -337,32 +363,50 @@ macro_rules! impl_query_filter_tuple { } } + #[allow(unused_variables)] + #[allow(non_snake_case)] impl<$($filter: WorldQuery),*> WorldQuery for Or<($($filter,)*)> - where $($filter::Fetch: FilterFetch),* + where $(for<'w, 's> <$filter::FetchInit as FetchInit<'w, 's>>::Fetch: FilterFetch<'w, 's>),* { - type Fetch = Or<($(OrFetch<$filter::Fetch>,)*)>; + const IS_DENSE: bool = true $(&& $filter::IS_DENSE)*; + + type FetchInit = Or<($($filter::FetchInit,)*)>; type State = Or<($($filter::State,)*)>; - } + fn shrink<'wlong: 'wshort, 'slong: 'sshort, 'wshort, 'sshort>( + item: super::QueryItem<'wlong, 'slong, Self>, + ) -> super::QueryItem<'wshort, 'sshort, Self> { + item + } + } #[allow(unused_variables)] #[allow(non_snake_case)] - impl<'w, 's, $($filter: FilterFetch),*> Fetch<'w, 's> for Or<($(OrFetch<$filter>,)*)> { - type State = Or<($(<$filter as Fetch<'w, 's>>::State,)*)>; + impl<'w, 's, $($filter: FetchInit<'w, 's>),*> FetchInit<'w, 's> for Or<($($filter,)*)> + where $($filter::Fetch: FilterFetch<'w, 's>),* + { + type Fetch = Or<($(OrFetch<'w, 's, $filter::Fetch>,)*)>; + type State = Or<($($filter::State,)*)>; type Item = bool; - unsafe fn init(world: &World, state: &Self::State, last_change_tick: u32, change_tick: u32) -> Self { + unsafe fn fetch_init(world: &'w World, state: &'s Self::State, last_change_tick: u32, change_tick: u32) -> Self::Fetch { let ($($filter,)*) = &state.0; Or(($(OrFetch { - fetch: $filter::init(world, $filter, last_change_tick, change_tick), + fetch: $filter::fetch_init(world, $filter, last_change_tick, change_tick), matches: false, + _marker: PhantomData, },)*)) } + } - const IS_DENSE: bool = true $(&& $filter::IS_DENSE)*; + #[allow(unused_variables)] + #[allow(non_snake_case)] + impl<'w, 's, $($filter: FilterFetch<'w, 's>),*> Fetch<'w, 's> for Or<($(OrFetch<'w, 's, $filter>,)*)> { + type State = Or<($(<$filter as Fetch<'w, 's>>::State,)*)>; + type Item = bool; #[inline] - unsafe fn set_table(&mut self, state: &Self::State, table: &Table) { + unsafe fn set_table(&mut self, state: &'s Self::State, table: &'w Table) { let ($($filter,)*) = &mut self.0; let ($($state,)*) = &state.0; $( @@ -374,7 +418,7 @@ macro_rules! impl_query_filter_tuple { } #[inline] - unsafe fn set_archetype(&mut self, state: &Self::State, archetype: &Archetype, tables: &Tables) { + unsafe fn set_archetype(&mut self, state: &'s Self::State, archetype: &'w Archetype, tables: &'w Tables) { let ($($filter,)*) = &mut self.0; let ($($state,)*) = &state.0; $( @@ -445,12 +489,12 @@ macro_rules! impl_tick_filter { pub struct $name(PhantomData); $(#[$fetch_meta])* - pub struct $fetch_name { - table_ticks: *const UnsafeCell, - entity_table_rows: *const usize, + pub struct $fetch_name<'w, T> { + table_ticks: Option>>, + entity_table_rows: Option>, marker: PhantomData, - entities: *const Entity, - sparse_set: *const ComponentSparseSet, + entities: Option>, + sparse_set: Option<&'w ComponentSparseSet>, last_change_tick: u32, change_tick: u32, } @@ -462,8 +506,21 @@ macro_rules! impl_tick_filter { } impl WorldQuery for $name { - type Fetch = $fetch_name; + const IS_DENSE: bool = { + match T::Storage::STORAGE_TYPE { + StorageType::Table => true, + StorageType::SparseSet => false, + } + }; + + type FetchInit = Self; type State = $state_name; + + fn shrink<'wlong: 'wshort, 'slong: 'sshort, 'wshort, 'sshort>( + item: super::QueryItem<'wlong, 'slong, Self>, + ) -> super::QueryItem<'wshort, 'sshort, Self> { + item + } } @@ -505,68 +562,63 @@ macro_rules! impl_tick_filter { } } - impl<'w, 's, T: Component> Fetch<'w, 's> for $fetch_name { + impl<'w, T: Component> FetchInit<'w, '_> for $name { type State = $state_name; + type Fetch = $fetch_name<'w, T>; type Item = bool; - unsafe fn init(world: &World, state: &Self::State, last_change_tick: u32, change_tick: u32) -> Self { - let mut value = Self { - table_ticks: ptr::null::>(), - entities: ptr::null::(), - entity_table_rows: ptr::null::(), - sparse_set: ptr::null::(), + unsafe fn fetch_init(world: &'w World, state: &Self::State, last_change_tick: u32, change_tick: u32) -> Self::Fetch { + Self::Fetch { + table_ticks: None, + entities: None, + entity_table_rows: None, + sparse_set: (T::Storage::STORAGE_TYPE == StorageType::SparseSet).then(|| world + .storages() + .sparse_sets + .get(state.component_id).unwrap()), marker: PhantomData, last_change_tick, change_tick, - }; - if T::Storage::STORAGE_TYPE == StorageType::SparseSet { - value.sparse_set = world - .storages() - .sparse_sets - .get(state.component_id).unwrap(); } - value } + } - const IS_DENSE: bool = { - match T::Storage::STORAGE_TYPE { - StorageType::Table => true, - StorageType::SparseSet => false, - } - }; + impl<'w, 's, T: Component> Fetch<'w, 's> for $fetch_name<'w, T> { + type State = $state_name; + type Item = bool; - unsafe fn set_table(&mut self, state: &Self::State, table: &Table) { - self.table_ticks = table + unsafe fn set_table(&mut self, state: &Self::State, table: &'w Table) { + self.table_ticks = Some(ThinSlicePtr::new(table .get_column(state.component_id).unwrap() - .get_ticks_ptr(); + .get_ticks())); } - unsafe fn set_archetype(&mut self, state: &Self::State, archetype: &Archetype, tables: &Tables) { + unsafe fn set_archetype(&mut self, state: &Self::State, archetype: &'w Archetype, tables: &'w Tables) { match T::Storage::STORAGE_TYPE { StorageType::Table => { - self.entity_table_rows = archetype.entity_table_rows().as_ptr(); + self.entity_table_rows = Some(ThinSlicePtr::new(archetype.entity_table_rows())); let table = &tables[archetype.table_id()]; - self.table_ticks = table + self.table_ticks = Some(ThinSlicePtr::new(table .get_column(state.component_id).unwrap() - .get_ticks_ptr(); + .get_ticks())); } - StorageType::SparseSet => self.entities = archetype.entities().as_ptr(), + StorageType::SparseSet => self.entities = Some(ThinSlicePtr::new(archetype.entities())), } } unsafe fn table_fetch(&mut self, table_row: usize) -> bool { - $is_detected(&*(&*self.table_ticks.add(table_row)).get(), self.last_change_tick, self.change_tick) + $is_detected(&*(&*self.table_ticks.unwrap_or_else(|| std::hint::unreachable_unchecked()).index(table_row)).get(), self.last_change_tick, self.change_tick) } unsafe fn archetype_fetch(&mut self, archetype_index: usize) -> bool { match T::Storage::STORAGE_TYPE { StorageType::Table => { - let table_row = *self.entity_table_rows.add(archetype_index); - $is_detected(&*(&*self.table_ticks.add(table_row)).get(), self.last_change_tick, self.change_tick) + let table_row = *self.entity_table_rows.unwrap_or_else(|| std::hint::unreachable_unchecked()).index(archetype_index); + $is_detected(&*(&*self.table_ticks.unwrap_or_else(|| std::hint::unreachable_unchecked()).index(table_row)).get(), self.last_change_tick, self.change_tick) } StorageType::SparseSet => { - let entity = *self.entities.add(archetype_index); - let ticks = (&*self.sparse_set).get_ticks(entity).cloned().unwrap(); + let entity = *self.entities.unwrap_or_else(|| std::hint::unreachable_unchecked()).index(archetype_index); + let ticks = self.sparse_set.unwrap_or_else(|| std::hint::unreachable_unchecked()).get_ticks(entity).cloned().unwrap(); $is_detected(&ticks, self.last_change_tick, self.change_tick) } } diff --git a/crates/bevy_ecs/src/query/iter.rs b/crates/bevy_ecs/src/query/iter.rs index 5199c971a81bb..92a8d7450173a 100644 --- a/crates/bevy_ecs/src/query/iter.rs +++ b/crates/bevy_ecs/src/query/iter.rs @@ -1,18 +1,20 @@ use crate::{ archetype::{ArchetypeId, Archetypes}, - query::{Fetch, FilterFetch, QueryState, ReadOnlyFetch, WorldQuery}, + query::{Fetch, FilterFetch, QueryState, ReadOnlyQuery, WorldQuery}, storage::{TableId, Tables}, world::World, }; use std::mem::MaybeUninit; +use super::{FetchInit, QueryFetch, QueryItem}; + /// An [`Iterator`] over query results of a [`Query`](crate::system::Query). /// /// This struct is created by the [`Query::iter`](crate::system::Query::iter) and /// [`Query::iter_mut`](crate::system::Query::iter_mut) methods. pub struct QueryIter<'w, 's, Q: WorldQuery, F: WorldQuery> where - F::Fetch: FilterFetch, + for<'x, 'y> QueryFetch<'x, 'y, F>: FilterFetch<'x, 'y>, { tables: &'w Tables, archetypes: &'w Archetypes, @@ -20,15 +22,15 @@ where world: &'w World, table_id_iter: std::slice::Iter<'s, TableId>, archetype_id_iter: std::slice::Iter<'s, ArchetypeId>, - fetch: Q::Fetch, - filter: F::Fetch, + fetch: QueryFetch<'w, 's, Q>, + filter: QueryFetch<'w, 's, F>, current_len: usize, current_index: usize, } impl<'w, 's, Q: WorldQuery, F: WorldQuery> QueryIter<'w, 's, Q, F> where - F::Fetch: FilterFetch, + for<'x, 'y> QueryFetch<'x, 'y, F>: FilterFetch<'x, 'y>, { /// # Safety /// This does not check for mutable query correctness. To be safe, make sure mutable queries @@ -41,13 +43,13 @@ where last_change_tick: u32, change_tick: u32, ) -> Self { - let fetch = ::init( + let fetch = >::fetch_init( world, &query_state.fetch_state, last_change_tick, change_tick, ); - let filter = ::init( + let filter = >::fetch_init( world, &query_state.filter_state, last_change_tick, @@ -74,7 +76,7 @@ where // NOTE: this mimics the behavior of `QueryIter::next()`, except that it // never gets a `Self::Item`. unsafe { - if Q::Fetch::IS_DENSE && F::Fetch::IS_DENSE { + if Q::IS_DENSE && F::IS_DENSE { loop { if self.current_index == self.current_len { let table_id = match self.table_id_iter.next() { @@ -127,9 +129,9 @@ where impl<'w, 's, Q: WorldQuery, F: WorldQuery> Iterator for QueryIter<'w, 's, Q, F> where - F::Fetch: FilterFetch, + for<'x, 'y> QueryFetch<'x, 'y, F>: FilterFetch<'x, 'y>, { - type Item = >::Item; + type Item = QueryItem<'w, 's, Q>; // NOTE: If you are changing query iteration code, remember to update the following places, where relevant: // QueryIter, QueryIterationCursor, QueryState::for_each_unchecked_manual, QueryState::par_for_each_unchecked_manual @@ -137,7 +139,7 @@ where #[inline(always)] fn next(&mut self) -> Option { unsafe { - if Q::Fetch::IS_DENSE && F::Fetch::IS_DENSE { + if Q::IS_DENSE && F::IS_DENSE { loop { if self.current_index == self.current_len { let table_id = self.table_id_iter.next()?; @@ -209,18 +211,18 @@ where pub struct QueryCombinationIter<'w, 's, Q: WorldQuery, F: WorldQuery, const K: usize> where - F::Fetch: FilterFetch, + for<'x, 'y> QueryFetch<'x, 'y, F>: FilterFetch<'x, 'y>, { tables: &'w Tables, archetypes: &'w Archetypes, query_state: &'s QueryState, world: &'w World, - cursors: [QueryIterationCursor<'s, Q, F>; K], + cursors: [QueryIterationCursor<'w, 's, Q, F>; K], } impl<'w, 's, Q: WorldQuery, F: WorldQuery, const K: usize> QueryCombinationIter<'w, 's, Q, F, K> where - F::Fetch: FilterFetch, + for<'x, 'y> QueryFetch<'x, 'y, F>: FilterFetch<'x, 'y>, { /// # Safety /// This does not check for mutable query correctness. To be safe, make sure mutable queries @@ -237,7 +239,7 @@ where // There is no FromIterator on arrays, so instead initialize it manually with MaybeUninit // TODO: use MaybeUninit::uninit_array if it stabilizes - let mut cursors: [MaybeUninit>; K] = + let mut cursors: [MaybeUninit>; K] = MaybeUninit::uninit().assume_init(); for (i, cursor) in cursors.iter_mut().enumerate() { match i { @@ -257,8 +259,8 @@ where } // TODO: use MaybeUninit::array_assume_init if it stabilizes - let cursors: [QueryIterationCursor<'s, Q, F>; K] = - (&cursors as *const _ as *const [QueryIterationCursor<'s, Q, F>; K]).read(); + let cursors: [QueryIterationCursor<'w, 's, Q, F>; K] = + (&cursors as *const _ as *const [QueryIterationCursor<'w, 's, Q, F>; K]).read(); QueryCombinationIter { world, @@ -275,12 +277,10 @@ where /// references to the same component, leading to unique reference aliasing. ///. /// It is always safe for shared access. - unsafe fn fetch_next_aliased_unchecked<'a>( - &mut self, - ) -> Option<[>::Item; K]> + unsafe fn fetch_next_aliased_unchecked(&mut self) -> Option<[QueryItem<'w, 's, Q>; K]> where - Q::Fetch: Clone, - F::Fetch: Clone, + for<'x, 'y> QueryFetch<'x, 'y, Q>: Clone, + for<'x, 'y> QueryFetch<'x, 'y, F>: Clone, { if K == 0 { return None; @@ -307,7 +307,7 @@ where } // TODO: use MaybeUninit::uninit_array if it stabilizes - let mut values: [MaybeUninit<>::Item>; K] = + let mut values: [MaybeUninit>; K] = MaybeUninit::uninit().assume_init(); for (value, cursor) in values.iter_mut().zip(&mut self.cursors) { @@ -315,36 +315,41 @@ where } // TODO: use MaybeUninit::array_assume_init if it stabilizes - let values: [>::Item; K] = - (&values as *const _ as *const [>::Item; K]).read(); + let values: [QueryItem<'w, 's, Q>; K] = + (&values as *const _ as *const [QueryItem<'w, 's, Q>; K]).read(); Some(values) } /// Get next combination of queried components #[inline] - pub fn fetch_next(&mut self) -> Option<[>::Item; K]> + pub fn fetch_next<'a>(&'a mut self) -> Option<[QueryItem<'a, 's, Q>; K]> where - Q::Fetch: Clone, - F::Fetch: Clone, + 'w: 'a, + for<'x, 'y> QueryFetch<'x, 'y, Q>: Clone, + for<'x, 'y> QueryFetch<'x, 'y, F>: Clone, { // safety: we are limiting the returned reference to self, // making sure this method cannot be called multiple times without getting rid // of any previously returned unique references first, thus preventing aliasing. - unsafe { self.fetch_next_aliased_unchecked() } + unsafe { + self.fetch_next_aliased_unchecked() + .map(|array| array.map(Q::shrink)) + } } } // Iterator type is intentionally implemented only for read-only access. // Doing so for mutable references would be unsound, because calling `next` // multiple times would allow multiple owned references to the same data to exist. -impl<'w, 's, Q: WorldQuery, F: WorldQuery, const K: usize> Iterator - for QueryCombinationIter<'w, 's, Q, F, K> +impl<'w, 's, Q, F, const K: usize> Iterator for QueryCombinationIter<'w, 's, Q, F, K> where - Q::Fetch: Clone + ReadOnlyFetch, - F::Fetch: Clone + FilterFetch + ReadOnlyFetch, + for<'x, 'y> QueryFetch<'x, 'y, Q>: Clone, + for<'x, 'y> QueryFetch<'x, 'y, F>: Clone + FilterFetch<'x, 'y>, + Q: WorldQuery + ReadOnlyQuery, + F: WorldQuery + ReadOnlyQuery, { - type Item = [>::Item; K]; + type Item = [QueryItem<'w, 's, Q>; K]; #[inline] fn next(&mut self) -> Option { @@ -398,19 +403,19 @@ impl<'w, 's, Q: WorldQuery> ExactSizeIterator for QueryIter<'w, 's, Q, ()> { } } -struct QueryIterationCursor<'s, Q: WorldQuery, F: WorldQuery> { +struct QueryIterationCursor<'w, 's, Q: WorldQuery, F: WorldQuery> { table_id_iter: std::slice::Iter<'s, TableId>, archetype_id_iter: std::slice::Iter<'s, ArchetypeId>, - fetch: Q::Fetch, - filter: F::Fetch, + fetch: QueryFetch<'w, 's, Q>, + filter: QueryFetch<'w, 's, F>, current_len: usize, current_index: usize, } -impl<'s, Q: WorldQuery, F: WorldQuery> Clone for QueryIterationCursor<'s, Q, F> +impl<'w, 's, Q: WorldQuery, F: WorldQuery> Clone for QueryIterationCursor<'w, 's, Q, F> where - Q::Fetch: Clone, - F::Fetch: Clone, + for<'x, 'y> QueryFetch<'x, 'y, Q>: Clone, + for<'x, 'y> QueryFetch<'x, 'y, F>: Clone, { fn clone(&self) -> Self { Self { @@ -424,12 +429,12 @@ where } } -impl<'s, Q: WorldQuery, F: WorldQuery> QueryIterationCursor<'s, Q, F> +impl<'w, 's, Q: WorldQuery, F: WorldQuery> QueryIterationCursor<'w, 's, Q, F> where - F::Fetch: FilterFetch, + for<'x, 'y> QueryFetch<'x, 'y, F>: FilterFetch<'x, 'y>, { unsafe fn init_empty( - world: &World, + world: &'w World, query_state: &'s QueryState, last_change_tick: u32, change_tick: u32, @@ -442,18 +447,18 @@ where } unsafe fn init( - world: &World, + world: &'w World, query_state: &'s QueryState, last_change_tick: u32, change_tick: u32, ) -> Self { - let fetch = ::init( + let fetch = >::fetch_init( world, &query_state.fetch_state, last_change_tick, change_tick, ); - let filter = ::init( + let filter = >::fetch_init( world, &query_state.filter_state, last_change_tick, @@ -471,9 +476,9 @@ where /// retrieve item returned from most recent `next` call again. #[inline] - unsafe fn peek_last<'w>(&mut self) -> Option<>::Item> { + unsafe fn peek_last(&mut self) -> Option> { if self.current_index > 0 { - if Q::Fetch::IS_DENSE && F::Fetch::IS_DENSE { + if Q::IS_DENSE && F::IS_DENSE { Some(self.fetch.table_fetch(self.current_index - 1)) } else { Some(self.fetch.archetype_fetch(self.current_index - 1)) @@ -487,13 +492,13 @@ where // QueryIter, QueryIterationCursor, QueryState::for_each_unchecked_manual, QueryState::par_for_each_unchecked_manual // We can't currently reuse QueryIterationCursor in QueryIter for performance reasons. See #1763 for context. #[inline(always)] - unsafe fn next<'w>( + unsafe fn next( &mut self, tables: &'w Tables, archetypes: &'w Archetypes, query_state: &'s QueryState, - ) -> Option<>::Item> { - if Q::Fetch::IS_DENSE && F::Fetch::IS_DENSE { + ) -> Option> { + if Q::IS_DENSE && F::IS_DENSE { loop { if self.current_index == self.current_len { let table_id = self.table_id_iter.next()?; diff --git a/crates/bevy_ecs/src/query/state.rs b/crates/bevy_ecs/src/query/state.rs index 6ecca3b870d72..3a0ac17f41c3b 100644 --- a/crates/bevy_ecs/src/query/state.rs +++ b/crates/bevy_ecs/src/query/state.rs @@ -4,7 +4,7 @@ use crate::{ entity::Entity, query::{ Access, Fetch, FetchState, FilterFetch, FilteredAccess, QueryCombinationIter, QueryIter, - ReadOnlyFetch, WorldQuery, + ReadOnlyQuery, WorldQuery, }, storage::TableId, world::{World, WorldId}, @@ -13,10 +13,12 @@ use bevy_tasks::TaskPool; use fixedbitset::FixedBitSet; use thiserror::Error; +use super::{FetchInit, QueryFetch, QueryItem}; + /// Provides scoped access to a [`World`] state according to a given [`WorldQuery`] and query filter. pub struct QueryState where - F::Fetch: FilterFetch, + for<'x, 'y> QueryFetch<'x, 'y, F>: FilterFetch<'x, 'y>, { world_id: WorldId, pub(crate) archetype_generation: ArchetypeGeneration, @@ -34,7 +36,7 @@ where impl QueryState where - F::Fetch: FilterFetch, + for<'x, 'y> QueryFetch<'x, 'y, F>: FilterFetch<'x, 'y>, { /// Creates a new [`QueryState`] from a given [`World`] and inherits the result of `world.id()`. pub fn new(world: &mut World) -> Self { @@ -134,9 +136,9 @@ where &'s mut self, world: &'w World, entity: Entity, - ) -> Result<>::Item, QueryEntityError> + ) -> Result, QueryEntityError> where - Q::Fetch: ReadOnlyFetch, + Q: ReadOnlyQuery, { // SAFETY: query is read only unsafe { self.get_unchecked(world, entity) } @@ -148,7 +150,7 @@ where &'s mut self, world: &'w mut World, entity: Entity, - ) -> Result<>::Item, QueryEntityError> { + ) -> Result, QueryEntityError> { // SAFETY: query has unique world access unsafe { self.get_unchecked(world, entity) } } @@ -164,7 +166,7 @@ where &'s mut self, world: &'w World, entity: Entity, - ) -> Result<>::Item, QueryEntityError> { + ) -> Result, QueryEntityError> { self.validate_world_and_update_archetypes(world); self.get_unchecked_manual( world, @@ -187,7 +189,7 @@ where entity: Entity, last_change_tick: u32, change_tick: u32, - ) -> Result<>::Item, QueryEntityError> { + ) -> Result, QueryEntityError> { let location = world .entities .get(entity) @@ -199,10 +201,18 @@ where return Err(QueryEntityError::QueryDoesNotMatch); } let archetype = &world.archetypes[location.archetype_id]; - let mut fetch = - ::init(world, &self.fetch_state, last_change_tick, change_tick); - let mut filter = - ::init(world, &self.filter_state, last_change_tick, change_tick); + let mut fetch = ::fetch_init( + world, + &self.fetch_state, + last_change_tick, + change_tick, + ); + let mut filter = ::fetch_init( + world, + &self.filter_state, + last_change_tick, + change_tick, + ); fetch.set_archetype(&self.fetch_state, archetype, &world.storages().tables); filter.set_archetype(&self.filter_state, archetype, &world.storages().tables); @@ -219,7 +229,7 @@ where #[inline] pub fn iter<'w, 's>(&'s mut self, world: &'w World) -> QueryIter<'w, 's, Q, F> where - Q::Fetch: ReadOnlyFetch, + Q: ReadOnlyQuery, { // SAFETY: query is read only unsafe { self.iter_unchecked(world) } @@ -248,7 +258,7 @@ where world: &'w World, ) -> QueryCombinationIter<'w, 's, Q, F, K> where - Q::Fetch: ReadOnlyFetch, + Q: ReadOnlyQuery, { // SAFE: query is read only unsafe { self.iter_combinations_unchecked(world) } @@ -350,12 +360,9 @@ where /// /// This can only be called for read-only queries, see [`Self::for_each_mut`] for write-queries. #[inline] - pub fn for_each<'w, 's>( - &'s mut self, - world: &'w World, - func: impl FnMut(>::Item), - ) where - Q::Fetch: ReadOnlyFetch, + pub fn for_each<'w, 's>(&'s mut self, world: &'w World, func: impl FnMut(QueryItem<'w, 's, Q>)) + where + Q: ReadOnlyQuery, { // SAFETY: query is read only unsafe { @@ -369,7 +376,7 @@ where pub fn for_each_mut<'w, 's>( &'s mut self, world: &'w mut World, - func: impl FnMut(>::Item), + func: impl FnMut(QueryItem<'w, 's, Q>), ) { // SAFETY: query has unique world access unsafe { @@ -390,7 +397,7 @@ where pub unsafe fn for_each_unchecked<'w, 's>( &'s mut self, world: &'w World, - func: impl FnMut(>::Item), + func: impl FnMut(QueryItem<'w, 's, Q>), ) { self.validate_world_and_update_archetypes(world); self.for_each_unchecked_manual( @@ -411,9 +418,9 @@ where world: &'w World, task_pool: &TaskPool, batch_size: usize, - func: impl Fn(>::Item) + Send + Sync + Clone, + func: impl Fn(QueryItem<'w, 's, Q>) + Send + Sync + Clone, ) where - Q::Fetch: ReadOnlyFetch, + Q: ReadOnlyQuery, { // SAFETY: query is read only unsafe { @@ -428,7 +435,7 @@ where world: &'w mut World, task_pool: &TaskPool, batch_size: usize, - func: impl Fn(>::Item) + Send + Sync + Clone, + func: impl Fn(QueryItem<'w, 's, Q>) + Send + Sync + Clone, ) { // SAFETY: query has unique world access unsafe { @@ -450,7 +457,7 @@ where world: &'w World, task_pool: &TaskPool, batch_size: usize, - func: impl Fn(>::Item) + Send + Sync + Clone, + func: impl Fn(QueryItem<'w, 's, Q>) + Send + Sync + Clone, ) { self.validate_world_and_update_archetypes(world); self.par_for_each_unchecked_manual( @@ -476,17 +483,25 @@ where pub(crate) unsafe fn for_each_unchecked_manual<'w, 's>( &'s self, world: &'w World, - mut func: impl FnMut(>::Item), + mut func: impl FnMut(QueryItem<'w, 's, Q>), last_change_tick: u32, change_tick: u32, ) { // NOTE: If you are changing query iteration code, remember to update the following places, where relevant: // QueryIter, QueryIterationCursor, QueryState::for_each_unchecked_manual, QueryState::par_for_each_unchecked_manual - let mut fetch = - ::init(world, &self.fetch_state, last_change_tick, change_tick); - let mut filter = - ::init(world, &self.filter_state, last_change_tick, change_tick); - if Q::Fetch::IS_DENSE && F::Fetch::IS_DENSE { + let mut fetch = ::fetch_init( + world, + &self.fetch_state, + last_change_tick, + change_tick, + ); + let mut filter = ::fetch_init( + world, + &self.filter_state, + last_change_tick, + change_tick, + ); + if Q::IS_DENSE && F::IS_DENSE { let tables = &world.storages().tables; for table_id in self.matched_table_ids.iter() { let table = &tables[*table_id]; @@ -534,14 +549,14 @@ where world: &'w World, task_pool: &TaskPool, batch_size: usize, - func: impl Fn(>::Item) + Send + Sync + Clone, + func: impl Fn(QueryItem<'w, 's, Q>) + Send + Sync + Clone, last_change_tick: u32, change_tick: u32, ) { // NOTE: If you are changing query iteration code, remember to update the following places, where relevant: // QueryIter, QueryIterationCursor, QueryState::for_each_unchecked_manual, QueryState::par_for_each_unchecked_manual task_pool.scope(|scope| { - if Q::Fetch::IS_DENSE && F::Fetch::IS_DENSE { + if Q::IS_DENSE && F::IS_DENSE { let tables = &world.storages().tables; for table_id in self.matched_table_ids.iter() { let table = &tables[*table_id]; @@ -549,13 +564,13 @@ where while offset < table.len() { let func = func.clone(); scope.spawn(async move { - let mut fetch = ::init( + let mut fetch = ::fetch_init( world, &self.fetch_state, last_change_tick, change_tick, ); - let mut filter = ::init( + let mut filter = ::fetch_init( world, &self.filter_state, last_change_tick, @@ -585,13 +600,13 @@ where while offset < archetype.len() { let func = func.clone(); scope.spawn(async move { - let mut fetch = ::init( + let mut fetch = ::fetch_init( world, &self.fetch_state, last_change_tick, change_tick, ); - let mut filter = ::init( + let mut filter = ::fetch_init( world, &self.filter_state, last_change_tick, diff --git a/crates/bevy_ecs/src/reflect.rs b/crates/bevy_ecs/src/reflect.rs index a967c60449c8a..7cccb4c5f9543 100644 --- a/crates/bevy_ecs/src/reflect.rs +++ b/crates/bevy_ecs/src/reflect.rs @@ -101,10 +101,8 @@ impl FromType for ReflectComponent { .entity_mut(destination_entity) .insert(destination_component); }, - reflect_component: |world, entity| { - world - .get_entity(entity)? - .get::() + reflect_component: |world, entity| unsafe { + crate::world::get::(world, entity, world.get_entity(entity)?.location()) .map(|c| c as &dyn Reflect) }, reflect_component_mut: |world, entity| unsafe { diff --git a/crates/bevy_ecs/src/storage/blob_vec.rs b/crates/bevy_ecs/src/storage/blob_vec.rs index 924dd2b7f5722..b4618034cb84a 100644 --- a/crates/bevy_ecs/src/storage/blob_vec.rs +++ b/crates/bevy_ecs/src/storage/blob_vec.rs @@ -3,11 +3,12 @@ use std::{ ptr::NonNull, }; -use crate::ptr::{OwningPtr, Ptr, PtrMut}; +use crate::ptr::{OwningPtr, Ptr, PtrMut, ThinSlicePtr}; pub struct BlobVec { item_layout: Layout, capacity: usize, + /// Number of elements, not bytes len: usize, data: NonNull, swap_scratch: NonNull, @@ -86,7 +87,7 @@ impl BlobVec { std::alloc::alloc(new_layout) } else { std::alloc::realloc( - self.get_ptr_mut().inner(), + self.get_ptr_mut().inner().as_ptr(), array_layout(&self.item_layout, self.capacity) .expect("array layout should be valid"), new_layout.size(), @@ -105,7 +106,7 @@ impl BlobVec { pub unsafe fn initialize_unchecked(&mut self, index: usize, value: OwningPtr<'_>) { debug_assert!(index < self.len()); let ptr = self.get_unchecked_mut(index); - std::ptr::copy_nonoverlapping(value.inner(), ptr.inner(), self.item_layout.size()); + std::ptr::copy_nonoverlapping(value.inner(), ptr.inner().as_ptr(), self.item_layout.size()); } /// # Safety @@ -114,11 +115,13 @@ impl BlobVec { pub unsafe fn replace_unchecked(&mut self, index: usize, value: OwningPtr<'_>) { debug_assert!(index < self.len()); let len = self.len; + let drop = self.drop; // to ensure we don't observe the empty slot in case of drop panic - self.len = index; let ptr = self.get_unchecked_mut(index); - (self.drop)(ptr.promote()); - std::ptr::copy_nonoverlapping(value.inner(), ptr.inner(), self.item_layout.size()); + let ptr = ptr.inner(); + self.len = index; + (drop)(OwningPtr::new(ptr)); + std::ptr::copy_nonoverlapping(value.inner(), ptr.as_ptr(), self.item_layout.size()); self.len = len; } @@ -157,13 +160,13 @@ impl BlobVec { let last = self.len - 1; let swap_scratch = self.swap_scratch.as_ptr(); std::ptr::copy_nonoverlapping( - self.get_unchecked_mut(index).inner(), + self.get_unchecked_mut(index).inner().as_ptr(), swap_scratch, self.item_layout.size(), ); std::ptr::copy( - self.get_unchecked_mut(last).inner(), - self.get_unchecked_mut(index).inner(), + self.get_unchecked_mut(last).inner().as_ptr(), + self.get_unchecked_mut(index).inner().as_ptr(), self.item_layout.size(), ); self.len -= 1; @@ -175,14 +178,15 @@ impl BlobVec { #[inline] pub unsafe fn swap_remove_and_drop_unchecked(&mut self, index: usize) { debug_assert!(index < self.len()); + let drop = self.drop; let value = self.swap_remove_and_forget_unchecked(index); - (self.drop)(value) + (drop)(value) } /// # Safety /// It is the caller's responsibility to ensure that `index` is < self.len() #[inline] - pub unsafe fn get_unchecked(&mut self, index: usize) -> Ptr<'_> { + pub unsafe fn get_unchecked(&self, index: usize) -> Ptr<'_> { debug_assert!(index < self.len()); self.get_ptr().add(index * self.item_layout.size()) } @@ -192,25 +196,37 @@ impl BlobVec { #[inline] pub unsafe fn get_unchecked_mut(&mut self, index: usize) -> PtrMut<'_> { debug_assert!(index < self.len()); - self.get_ptr_mut().add(index * self.item_layout.size()) + let layout_size = self.item_layout.size(); + self.get_ptr_mut().add(index * layout_size) } - /// Gets a pointer to the start of the vec - /// - /// # Safety - /// must ensure rust mutability rules are not violated + /// Gets a [Ptr] to the start of the vec #[inline] - pub unsafe fn get_ptr(&self) -> Ptr<'_> { - Ptr::new(self.data) + pub fn get_ptr(&self) -> Ptr<'_> { + // SAFE: the inner data will remain valid for as long as 'self. + unsafe { Ptr::new(self.data) } } - /// Gets a pointer to the start of the vec + /// Gets a [ThinSlicePtr] for the vec. /// /// # Safety - /// must ensure rust mutability rules are not violated + /// The type `T` must be the type of the items in this BlobVec. + pub unsafe fn get_thin_slice(&self) -> ThinSlicePtr<'_, T> { + // SAFE: the inner data will remain valid for as long as 'self. + ThinSlicePtr::new_raw( + self.data.cast(), + #[cfg(debug_assertions)] + { + self.len + }, + ) + } + + /// Gets a [PtrMut] to the start of the vec #[inline] - pub unsafe fn get_ptr_mut(&mut self) -> PtrMut<'_> { - PtrMut::new(self.data) + pub fn get_ptr_mut(&mut self) -> PtrMut<'_> { + // SAFE: the inner data will remain valid for as long as 'self. + unsafe { PtrMut::new(self.data) } } pub fn clear(&mut self) { @@ -222,11 +238,10 @@ impl BlobVec { unsafe { // NOTE: this doesn't use self.get_unchecked(i) because the debug_assert on index // will panic here due to self.len being set to 0 - let ptr = self - .get_ptr_mut() - .add(i * self.item_layout.size()) - .promote(); - (self.drop)(ptr); + let drop = self.drop; + let layout_size = self.item_layout.size(); + let ptr = self.get_ptr_mut().add(i * layout_size).promote(); + (drop)(ptr); } } } @@ -239,7 +254,7 @@ impl Drop for BlobVec { array_layout(&self.item_layout, self.capacity).expect("array layout should be valid"); if array_layout.size() > 0 { unsafe { - std::alloc::dealloc(self.get_ptr_mut().inner(), array_layout); + std::alloc::dealloc(self.get_ptr_mut().inner().as_ptr(), array_layout); std::alloc::dealloc(self.swap_scratch.as_ptr(), self.item_layout); } } @@ -315,7 +330,7 @@ mod tests { /// # Safety /// /// `blob_vec` must have a layout that matches Layout::new::() - unsafe fn push(blob_vec: &mut BlobVec, mut value: T) { + unsafe fn push(blob_vec: &mut BlobVec, value: T) { let index = blob_vec.push_uninit(); OwningPtr::make(value, |ptr| { blob_vec.initialize_unchecked(index, ptr); @@ -337,7 +352,11 @@ mod tests { /// at the given `index` unsafe fn get_mut(blob_vec: &mut BlobVec, index: usize) -> &mut T { assert!(index < blob_vec.len()); - &mut *blob_vec.get_unchecked_mut(index).inner().cast::() + &mut *blob_vec + .get_unchecked_mut(index) + .inner() + .as_ptr() + .cast::() } #[test] diff --git a/crates/bevy_ecs/src/storage/sparse_set.rs b/crates/bevy_ecs/src/storage/sparse_set.rs index 43a95a4b2790d..9c08f6863cbb3 100644 --- a/crates/bevy_ecs/src/storage/sparse_set.rs +++ b/crates/bevy_ecs/src/storage/sparse_set.rs @@ -166,12 +166,15 @@ impl ComponentSparseSet { /// # Safety /// ensure the same entity is not accessed twice at the same time #[inline] - pub unsafe fn get_with_ticks(&self, entity: Entity) -> Option<(Ptr<'_>, *mut ComponentTicks)> { + pub unsafe fn get_with_ticks( + &self, + entity: Entity, + ) -> Option<(Ptr<'_>, &UnsafeCell)> { let dense_index = *self.sparse.get(entity)?; // SAFE: if the sparse index points to something in the dense vec, it exists Some(( self.dense.get_unchecked(dense_index), - self.ticks.get_unchecked(dense_index).get(), + self.ticks.get_unchecked(dense_index), )) } @@ -186,7 +189,7 @@ impl ComponentSparseSet { /// it exists). It is the caller's responsibility to drop the returned ptr (if Some is /// returned). pub fn remove_and_forget(&mut self, entity: Entity) -> Option> { - self.sparse.remove(entity).map(|dense_index| { + self.sparse.remove(entity).map(move |dense_index| { self.ticks.swap_remove(dense_index); self.entities.swap_remove(dense_index); let is_last = dense_index == self.dense.len() - 1; diff --git a/crates/bevy_ecs/src/storage/table.rs b/crates/bevy_ecs/src/storage/table.rs index baeef5e6820de..dc4c57b4685a1 100644 --- a/crates/bevy_ecs/src/storage/table.rs +++ b/crates/bevy_ecs/src/storage/table.rs @@ -1,7 +1,7 @@ use crate::{ component::{ComponentId, ComponentInfo, ComponentTicks, Components}, entity::Entity, - ptr::{OwningPtr, Ptr, PtrMut}, + ptr::{OwningPtr, Ptr, PtrMut, ThinSlicePtr}, storage::{BlobVec, SparseSet}, }; use bevy_utils::{AHasher, HashMap}; @@ -133,39 +133,38 @@ impl Column { self.ticks.reserve_exact(additional); } - /// # Safety - /// must ensure rust mutability rules are not violated #[inline] - pub unsafe fn get_data_ptr(&self) -> Ptr<'_> { + pub fn get_data_ptr(&self) -> Ptr<'_> { self.data.get_ptr() } /// # Safety - /// must ensure rust mutability rules are not violated - #[inline] - pub unsafe fn get_data_ptr_mut(&self) -> PtrMut<'_> { - self.data.get_ptr_mut() + /// The type `T` must be the type of the items in this column. + pub unsafe fn get_data_slice(&self) -> ThinSlicePtr<'_, T> { + self.data.get_thin_slice() } #[inline] - pub fn get_ticks_ptr(&self) -> *const UnsafeCell { - self.ticks.as_ptr() + pub fn get_ticks(&self) -> &[UnsafeCell] { + &self.ticks } + /// # Safety + /// - index must be in-bounds + /// - no other reference to the data of the same row can exist at the same time #[inline] - pub fn get_ticks_const_ptr(&self) -> *const ComponentTicks { - // cast is valid, because UnsafeCell is repr(transparent) - self.get_ticks_ptr() as *const ComponentTicks + pub unsafe fn get_data_unchecked(&self, row: usize) -> Ptr<'_> { + debug_assert!(row < self.data.len()); + self.data.get_unchecked(row) } /// # Safety /// - index must be in-bounds /// - no other reference to the data of the same row can exist at the same time - /// - pointer cannot be dereferenced after mutable reference to this `Column` was live #[inline] - pub unsafe fn get_data_unchecked(&self, row: usize) -> Ptr<'_> { + pub unsafe fn get_data_unchecked_mut(&mut self, row: usize) -> PtrMut<'_> { debug_assert!(row < self.data.len()); - self.data.get_unchecked(row) + self.data.get_unchecked_mut(row) } /// # Safety @@ -179,11 +178,18 @@ impl Column { /// # Safety /// - index must be in-bounds /// - no other reference to the ticks of the same row can exist at the same time - /// - pointer cannot be dereferenced after mutable reference to this column was live #[inline] - pub unsafe fn get_ticks_mut_ptr_unchecked(&self, row: usize) -> *mut ComponentTicks { + pub unsafe fn get_ticks_mut_unchecked(&self, row: usize) -> &mut ComponentTicks { + debug_assert!(row < self.ticks.len()); + &mut *self.ticks.get_unchecked(row).get() + } + + /// # Safety + /// - index must be in-bounds + #[inline] + pub unsafe fn get_ticks_mut_ptr_unchecked(&self, row: usize) -> &UnsafeCell { debug_assert!(row < self.ticks.len()); - self.ticks.get_unchecked(row).get() + self.ticks.get_unchecked(row) } pub fn clear(&mut self) { @@ -265,8 +271,9 @@ impl Table { let is_last = row == self.entities.len() - 1; let new_row = new_table.allocate(self.entities.swap_remove(row)); for column in self.columns.values_mut() { + let component_id = column.component_id; let (data, ticks) = column.swap_remove_and_forget_unchecked(row); - if let Some(new_column) = new_table.get_column_mut(column.component_id) { + if let Some(new_column) = new_table.get_column_mut(component_id) { new_column.initialize(new_row, data, ticks); } } @@ -550,7 +557,7 @@ mod tests { // SAFE: we allocate and immediately set data afterwards unsafe { let row = table.allocate(*entity); - let mut value = row; + let value = row; OwningPtr::make(value, |value_ptr| { table .get_column_mut(component_id) diff --git a/crates/bevy_ecs/src/system/mod.rs b/crates/bevy_ecs/src/system/mod.rs index a0523bc6fe9d0..a123d871b3a01 100644 --- a/crates/bevy_ecs/src/system/mod.rs +++ b/crates/bevy_ecs/src/system/mod.rs @@ -787,8 +787,9 @@ mod tests { } } -/// ```compile_fail +/// ```compile_fail E0499 /// use bevy_ecs::prelude::*; +/// #[derive(Component)] /// struct A(usize); /// fn system(mut query: Query<&mut A>, e: Res) { /// let mut iter = query.iter_mut(); @@ -802,11 +803,12 @@ mod tests { /// } /// ``` #[allow(unused)] -#[cfg(doc)] +#[cfg(doctest)] fn system_query_iter_lifetime_safety_test() {} -/// ```compile_fail +/// ```compile_fail E0499 /// use bevy_ecs::prelude::*; +/// #[derive(Component)] /// struct A(usize); /// fn system(mut query: Query<&mut A>, e: Res) { /// let mut a1 = query.get_mut(*e).unwrap(); @@ -816,11 +818,12 @@ fn system_query_iter_lifetime_safety_test() {} /// } /// ``` #[allow(unused)] -#[cfg(doc)] +#[cfg(doctest)] fn system_query_get_lifetime_safety_test() {} -/// ```compile_fail +/// ```compile_fail E0499 /// use bevy_ecs::prelude::*; +/// #[derive(Component)] /// struct A(usize); /// fn query_set(mut queries: QuerySet<(QueryState<&mut A>, QueryState<&A>)>, e: Res) { /// let mut q2 = queries.q0(); @@ -836,11 +839,12 @@ fn system_query_get_lifetime_safety_test() {} /// } /// ``` #[allow(unused)] -#[cfg(doc)] +#[cfg(doctest)] fn system_query_set_iter_lifetime_safety_test() {} -/// ```compile_fail +/// ```compile_fail E0499 /// use bevy_ecs::prelude::*; +/// #[derive(Component)] /// struct A(usize); /// fn query_set(mut queries: QuerySet<(QueryState<&mut A>, QueryState<&A>)>, e: Res) { /// let q1 = queries.q1(); @@ -856,11 +860,12 @@ fn system_query_set_iter_lifetime_safety_test() {} /// } /// ``` #[allow(unused)] -#[cfg(doc)] +#[cfg(doctest)] fn system_query_set_iter_flip_lifetime_safety_test() {} -/// ```compile_fail +/// ```compile_fail E0499 /// use bevy_ecs::prelude::*; +/// #[derive(Component)] /// struct A(usize); /// fn query_set(mut queries: QuerySet<(QueryState<&mut A>, QueryState<&A>)>, e: Res) { /// let mut q2 = queries.q0(); @@ -874,11 +879,12 @@ fn system_query_set_iter_flip_lifetime_safety_test() {} /// } /// ``` #[allow(unused)] -#[cfg(doc)] +#[cfg(doctest)] fn system_query_set_get_lifetime_safety_test() {} -/// ```compile_fail +/// ```compile_fail E0499 /// use bevy_ecs::prelude::*; +/// #[derive(Component)] /// struct A(usize); /// fn query_set(mut queries: QuerySet<(QueryState<&mut A>, QueryState<&A>)>, e: Res) { /// let q1 = queries.q1(); @@ -891,13 +897,15 @@ fn system_query_set_get_lifetime_safety_test() {} /// } /// ``` #[allow(unused)] -#[cfg(doc)] +#[cfg(doctest)] fn system_query_set_get_flip_lifetime_safety_test() {} -/// ```compile_fail +/// ```compile_fail E0502 /// use bevy_ecs::prelude::*; /// use bevy_ecs::system::SystemState; +/// #[derive(Component)] /// struct A(usize); +/// #[derive(Component)] /// struct B(usize); /// struct State { /// state_r: SystemState>, @@ -918,13 +926,15 @@ fn system_query_set_get_flip_lifetime_safety_test() {} /// } /// ``` #[allow(unused)] -#[cfg(doc)] +#[cfg(doctest)] fn system_state_get_lifetime_safety_test() {} -/// ```compile_fail +/// ```compile_fail E0502 /// use bevy_ecs::prelude::*; /// use bevy_ecs::system::SystemState; +/// #[derive(Component)] /// struct A(usize); +/// #[derive(Component)] /// struct B(usize); /// struct State { /// state_r: SystemState>, @@ -943,5 +953,5 @@ fn system_state_get_lifetime_safety_test() {} /// } /// ``` #[allow(unused)] -#[cfg(doc)] +#[cfg(doctest)] fn system_state_iter_lifetime_safety_test() {} diff --git a/crates/bevy_ecs/src/system/query.rs b/crates/bevy_ecs/src/system/query.rs index 6aaf39bfbaa23..c2f16c01e3998 100644 --- a/crates/bevy_ecs/src/system/query.rs +++ b/crates/bevy_ecs/src/system/query.rs @@ -2,8 +2,8 @@ use crate::{ component::Component, entity::Entity, query::{ - Fetch, FilterFetch, QueryCombinationIter, QueryEntityError, QueryIter, QueryState, - ReadOnlyFetch, WorldQuery, + FilterFetch, QueryCombinationIter, QueryEntityError, QueryFetch, QueryItem, QueryIter, + QueryState, ReadOnlyQuery, WorldQuery, }, world::{Mut, World}, }; @@ -242,7 +242,7 @@ use thiserror::Error; /// (or `iter_mut.next()`) to only get the first query result. pub struct Query<'world, 'state, Q: WorldQuery, F: WorldQuery = ()> where - F::Fetch: FilterFetch, + for<'x, 'y> QueryFetch<'x, 'y, F>: FilterFetch<'x, 'y>, { pub(crate) world: &'world World, pub(crate) state: &'state QueryState, @@ -252,7 +252,7 @@ where impl<'w, 's, Q: WorldQuery, F: WorldQuery> Query<'w, 's, Q, F> where - F::Fetch: FilterFetch, + for<'x, 'y> QueryFetch<'x, 'y, F>: FilterFetch<'x, 'y>, { /// Creates a new query. /// @@ -301,7 +301,7 @@ where #[inline] pub fn iter(&'s self) -> QueryIter<'w, 's, Q, F> where - Q::Fetch: ReadOnlyFetch, + Q: ReadOnlyQuery, { // SAFE: system runs without conflicts with other systems. // same-system queries have runtime borrow checks when they conflict @@ -351,7 +351,7 @@ where #[inline] pub fn iter_combinations(&self) -> QueryCombinationIter<'_, '_, Q, F, K> where - Q::Fetch: ReadOnlyFetch, + Q: ReadOnlyQuery, { // SAFE: system runs without conflicts with other systems. // same-system queries have runtime borrow checks when they conflict @@ -458,9 +458,9 @@ where /// # report_names_system.system(); /// ``` #[inline] - pub fn for_each(&'s self, f: impl FnMut(>::Item)) + pub fn for_each(&'s self, f: impl FnMut(QueryItem<'w, 's, Q>)) where - Q::Fetch: ReadOnlyFetch, + Q: ReadOnlyQuery, { // SAFE: system runs without conflicts with other systems. // same-system queries have runtime borrow checks when they conflict @@ -496,7 +496,7 @@ where /// # gravity_system.system(); /// ``` #[inline] - pub fn for_each_mut<'a>(&'a mut self, f: impl FnMut(>::Item)) { + pub fn for_each_mut<'a>(&'a mut self, f: impl FnMut(QueryItem<'a, 'a, Q>)) { // SAFE: system runs without conflicts with other systems. same-system queries have runtime // borrow checks when they conflict unsafe { @@ -518,9 +518,9 @@ where &'s self, task_pool: &TaskPool, batch_size: usize, - f: impl Fn(>::Item) + Send + Sync + Clone, + f: impl Fn(QueryItem<'w, 's, Q>) + Send + Sync + Clone, ) where - Q::Fetch: ReadOnlyFetch, + Q: ReadOnlyQuery, { // SAFE: system runs without conflicts with other systems. same-system queries have runtime // borrow checks when they conflict @@ -542,7 +542,7 @@ where &'a mut self, task_pool: &TaskPool, batch_size: usize, - f: impl Fn(>::Item) + Send + Sync + Clone, + f: impl Fn(QueryItem<'a, 'a, Q>) + Send + Sync + Clone, ) { // SAFE: system runs without conflicts with other systems. same-system queries have runtime // borrow checks when they conflict @@ -590,12 +590,9 @@ where /// # print_selected_character_name_system.system(); /// ``` #[inline] - pub fn get( - &'s self, - entity: Entity, - ) -> Result<>::Item, QueryEntityError> + pub fn get(&'s self, entity: Entity) -> Result, QueryEntityError> where - Q::Fetch: ReadOnlyFetch, + Q: ReadOnlyQuery, { // SAFE: system runs without conflicts with other systems. // same-system queries have runtime borrow checks when they conflict @@ -634,10 +631,7 @@ where /// # poison_system.system(); /// ``` #[inline] - pub fn get_mut( - &mut self, - entity: Entity, - ) -> Result<::Item, QueryEntityError> { + pub fn get_mut(&mut self, entity: Entity) -> Result, QueryEntityError> { // SAFE: system runs without conflicts with other systems. // same-system queries have runtime borrow checks when they conflict unsafe { @@ -663,7 +657,7 @@ where pub unsafe fn get_unchecked( &self, entity: Entity, - ) -> Result<::Item, QueryEntityError> { + ) -> Result, QueryEntityError> { // SEMI-SAFE: system runs without conflicts with other systems. // same-system queries have runtime borrow checks when they conflict self.state @@ -717,9 +711,10 @@ where .archetype_component_access .has_read(archetype_component) { - entity_ref - .get::() - .ok_or(QueryComponentError::MissingComponent) + unsafe { + crate::world::get(world, entity, entity_ref.location()) + .ok_or(QueryComponentError::MissingComponent) + } } else { Err(QueryComponentError::MissingReadAccess) } @@ -824,9 +819,9 @@ where /// Panics if the number of query results is not exactly one. Use /// [`get_single`](Self::get_single) to return a `Result` instead of panicking. #[track_caller] - pub fn single(&'s self) -> >::Item + pub fn single(&'s self) -> QueryItem<'w, 's, Q> where - Q::Fetch: ReadOnlyFetch, + Q: ReadOnlyQuery, { self.get_single().unwrap() } @@ -863,9 +858,9 @@ where /// } /// # player_scoring_system.system(); /// ``` - pub fn get_single(&'s self) -> Result<>::Item, QuerySingleError> + pub fn get_single(&'s self) -> Result, QuerySingleError> where - Q::Fetch: ReadOnlyFetch, + Q: ReadOnlyQuery, { let mut query = self.iter(); let first = query.next(); @@ -905,7 +900,7 @@ where /// Panics if the number of query results is not exactly one. Use /// [`get_single_mut`](Self::get_single_mut) to return a `Result` instead of panicking. #[track_caller] - pub fn single_mut(&mut self) -> >::Item { + pub fn single_mut(&mut self) -> QueryItem<'_, '_, Q> { self.get_single_mut().unwrap() } @@ -931,9 +926,7 @@ where /// } /// # regenerate_player_health_system.system(); /// ``` - pub fn get_single_mut( - &mut self, - ) -> Result<>::Item, QuerySingleError> { + pub fn get_single_mut(&mut self) -> Result, QuerySingleError> { let mut query = self.iter_mut(); let first = query.next(); let extra = query.next().is_some(); diff --git a/crates/bevy_ecs/src/system/system_param.rs b/crates/bevy_ecs/src/system/system_param.rs index 5e12c4104085c..3ceab1384bf26 100644 --- a/crates/bevy_ecs/src/system/system_param.rs +++ b/crates/bevy_ecs/src/system/system_param.rs @@ -6,7 +6,8 @@ use crate::{ component::{Component, ComponentId, ComponentTicks, Components}, entity::{Entities, Entity}, query::{ - FilterFetch, FilteredAccess, FilteredAccessSet, QueryState, ReadOnlyFetch, WorldQuery, + FilterFetch, FilteredAccess, FilteredAccessSet, QueryFetch, QueryState, ReadOnlyQuery, + WorldQuery, }, system::{CommandQueue, Commands, Query, SystemMeta}, world::{FromWorld, World}, @@ -100,7 +101,7 @@ pub trait SystemParamFetch<'world, 'state>: SystemParamState { impl<'w, 's, Q: WorldQuery + 'static, F: WorldQuery + 'static> SystemParam for Query<'w, 's, Q, F> where - F::Fetch: FilterFetch, + for<'x, 'y> QueryFetch<'x, 'y, F>: FilterFetch<'x, 'y>, { type Fetch = QueryState; } @@ -108,8 +109,8 @@ where // SAFE: QueryState is constrained to read-only fetches, so it only reads World. unsafe impl ReadOnlySystemParamFetch for QueryState where - Q::Fetch: ReadOnlyFetch, - F::Fetch: FilterFetch, + Q: ReadOnlyQuery, + for<'x, 'y> QueryFetch<'x, 'y, F>: FilterFetch<'x, 'y>, { } @@ -117,7 +118,7 @@ where // this QueryState conflicts with any prior access, a panic will occur. unsafe impl SystemParamState for QueryState where - F::Fetch: FilterFetch, + for<'x, 'y> QueryFetch<'x, 'y, F>: FilterFetch<'x, 'y>, { type Config = (); @@ -153,7 +154,7 @@ where impl<'w, 's, Q: WorldQuery + 'static, F: WorldQuery + 'static> SystemParamFetch<'w, 's> for QueryState where - F::Fetch: FilterFetch, + for<'x, 'y> QueryFetch<'x, 'y, F>: FilterFetch<'x, 'y>, { type Item = Query<'w, 's, Q, F>; @@ -328,7 +329,7 @@ impl<'w, 's, T: Resource> SystemParamFetch<'w, 's> for ResState { ) }); Res { - value: &*column.get_data_ptr().inner_nonnull().cast::().as_ptr(), + value: column.get_data_ptr().deref::(), ticks: column.get_ticks_unchecked(0), last_change_tick: system_meta.last_change_tick, change_tick, @@ -370,7 +371,7 @@ impl<'w, 's, T: Resource> SystemParamFetch<'w, 's> for OptionResState { world .get_populated_resource_column(state.0.component_id) .map(|column| Res { - value: &*column.get_data_ptr().inner_nonnull().cast::().as_ptr(), + value: column.get_data_ptr().deref::(), ticks: column.get_ticks_unchecked(0), last_change_tick: system_meta.last_change_tick, change_tick, @@ -818,7 +819,7 @@ impl<'w, 's, T: 'static> SystemParamFetch<'w, 's> for NonSendState { }); NonSend { - value: &*column.get_data_ptr().inner_nonnull().cast::().as_ptr(), + value: column.get_data_ptr().deref::(), ticks: column.get_ticks_unchecked(0).clone(), last_change_tick: system_meta.last_change_tick, change_tick, @@ -861,7 +862,7 @@ impl<'w, 's, T: 'static> SystemParamFetch<'w, 's> for OptionNonSendState { world .get_populated_resource_column(state.0.component_id) .map(|column| NonSend { - value: &*column.get_data_ptr().inner_nonnull().cast::().as_ptr(), + value: column.get_data_ptr().deref::(), ticks: column.get_ticks_unchecked(0).clone(), last_change_tick: system_meta.last_change_tick, change_tick, @@ -937,9 +938,9 @@ impl<'w, 's, T: 'static> SystemParamFetch<'w, 's> for NonSendMutState { ) }); NonSendMut { - value: &mut *column.get_data_ptr().inner_nonnull().cast::().as_ptr(), + value: column.get_data_ptr().assert_unique().deref_mut::(), ticks: Ticks { - component_ticks: &mut *column.get_ticks_mut_ptr_unchecked(0), + component_ticks: column.get_ticks_mut_unchecked(0), last_change_tick: system_meta.last_change_tick, change_tick, }, @@ -979,9 +980,9 @@ impl<'w, 's, T: 'static> SystemParamFetch<'w, 's> for OptionNonSendMutState { world .get_populated_resource_column(state.0.component_id) .map(|column| NonSendMut { - value: &mut *column.get_data_ptr().inner_nonnull().cast::().as_ptr(), + value: column.get_data_ptr().assert_unique().deref_mut::(), ticks: Ticks { - component_ticks: &mut *column.get_ticks_mut_ptr_unchecked(0), + component_ticks: column.get_ticks_mut_unchecked(0), last_change_tick: system_meta.last_change_tick, change_tick, }, diff --git a/crates/bevy_ecs/src/world/entity_ref.rs b/crates/bevy_ecs/src/world/entity_ref.rs index 45ab979ff8039..b1c03623c9d3f 100644 --- a/crates/bevy_ecs/src/world/entity_ref.rs +++ b/crates/bevy_ecs/src/world/entity_ref.rs @@ -4,11 +4,11 @@ use crate::{ change_detection::Ticks, component::{Component, ComponentId, ComponentTicks, Components, StorageType}, entity::{Entities, Entity, EntityLocation}, - ptr::OwningPtr, + ptr::{OwningPtr, Ptr}, storage::{SparseSet, Storages}, world::{Mut, World}, }; -use std::any::TypeId; +use std::{any::TypeId, cell::UnsafeCell}; pub struct EntityRef<'w> { world: &'w World, @@ -62,12 +62,9 @@ impl<'w> EntityRef<'w> { } #[inline] - pub fn get(&self) -> Option<&'w T> { + pub fn get(&self) -> Option<&T> { // SAFE: entity location is valid and returned component is of type T - unsafe { - get_component_with_type(self.world, TypeId::of::(), self.entity, self.location) - .map(|value| &*value.cast::()) - } + unsafe { get(self.world, self.entity, self.location) } } /// # Safety @@ -81,9 +78,9 @@ impl<'w> EntityRef<'w> { ) -> Option> { get_component_and_ticks_with_type(self.world, TypeId::of::(), self.entity, self.location) .map(|(value, ticks)| Mut { - value: &mut *value.cast::(), + value: value.assert_unique().deref_mut::(), ticks: Ticks { - component_ticks: &mut *ticks, + component_ticks: &mut *ticks.get(), last_change_tick, change_tick, }, @@ -144,46 +141,28 @@ impl<'w> EntityMut<'w> { } #[inline] - pub fn get(&self) -> Option<&'w T> { + pub fn get(&self) -> Option<&T> { // SAFE: entity location is valid and returned component is of type T - unsafe { - get_component_with_type(self.world, TypeId::of::(), self.entity, self.location) - .map(|value| &*value.cast::()) - } + unsafe { get(self.world, self.entity, self.location) } } #[inline] - pub fn get_mut(&mut self) -> Option> { + pub fn get_mut(&mut self) -> Option> { // SAFE: world access is unique, entity location is valid, and returned component is of type // T - unsafe { - get_component_and_ticks_with_type( - self.world, - TypeId::of::(), - self.entity, - self.location, - ) - .map(|(value, ticks)| Mut { - value: &mut *value.cast::(), - ticks: Ticks { - component_ticks: &mut *ticks, - last_change_tick: self.world.last_change_tick(), - change_tick: self.world.change_tick(), - }, - }) - } + unsafe { get_mut(self.world, self.entity, self.location) } } /// # Safety /// This allows aliased mutability. You must make sure this call does not result in multiple /// mutable references to the same component #[inline] - pub unsafe fn get_unchecked_mut(&self) -> Option> { + pub unsafe fn get_unchecked_mut(&self) -> Option> { get_component_and_ticks_with_type(self.world, TypeId::of::(), self.entity, self.location) .map(|(value, ticks)| Mut { - value: &mut *value.cast::(), + value: value.assert_unique().deref_mut::(), ticks: Ticks { - component_ticks: &mut *ticks, + component_ticks: &mut *ticks.get(), last_change_tick: self.world.last_change_tick(), change_tick: self.world.read_change_tick(), }, @@ -243,7 +222,7 @@ impl<'w> EntityMut<'w> { // SAFE: bundle components are iterated in order, which guarantees that the component type // matches let result = unsafe { - T::from_components(|| { + T::from_components(storages, |storages| { let component_id = bundle_components.next().unwrap(); // SAFE: entity location is valid and table row is removed below take_component( @@ -474,7 +453,7 @@ unsafe fn get_component( component_id: ComponentId, entity: Entity, location: EntityLocation, -) -> Option<*mut u8> { +) -> Option> { let archetype = &world.archetypes[location.archetype_id]; // SAFE: component_id exists and is therefore valid let component_info = world.components.get_info_unchecked(component_id); @@ -503,7 +482,7 @@ unsafe fn get_component_and_ticks( component_id: ComponentId, entity: Entity, location: EntityLocation, -) -> Option<(*mut u8, *mut ComponentTicks)> { +) -> Option<(Ptr<'_>, &UnsafeCell)> { let archetype = &world.archetypes[location.archetype_id]; let component_info = world.components.get_info_unchecked(component_id); match component_info.storage_type() { @@ -551,12 +530,12 @@ unsafe fn take_component<'a>( removed_components.push(entity); match component_info.storage_type() { StorageType::Table => { - let table = &storages.tables[archetype.table_id()]; + let table = &mut storages.tables[archetype.table_id()]; // SAFE: archetypes will always point to valid columns - let components = table.get_column(component_id).unwrap(); + let components = table.get_column_mut(component_id).unwrap(); let table_row = archetype.entity_table_row(location.index); // SAFE: archetypes only store valid table_rows and the stored component type is T - components.get_data_unchecked(table_row) + components.get_data_unchecked_mut(table_row).promote() } StorageType::SparseSet => storages .sparse_sets @@ -574,7 +553,7 @@ unsafe fn get_component_with_type( type_id: TypeId, entity: Entity, location: EntityLocation, -) -> Option<*mut u8> { +) -> Option> { let component_id = world.components.get_id(type_id)?; get_component(world, component_id, entity, location) } @@ -586,7 +565,7 @@ pub(crate) unsafe fn get_component_and_ticks_with_type( type_id: TypeId, entity: Entity, location: EntityLocation, -) -> Option<(*mut u8, *mut ComponentTicks)> { +) -> Option<(Ptr<'_>, &UnsafeCell)> { let component_id = world.components.get_id(type_id)?; get_component_and_ticks(world, component_id, entity, location) } @@ -724,6 +703,41 @@ fn sorted_remove(source: &mut Vec, remove: &[T]) { }) } +// SAFETY: EntityLocation must be valid +#[inline] +pub(crate) unsafe fn get( + world: &World, + entity: Entity, + location: EntityLocation, +) -> Option<&T> { + // SAFE: entity location is valid and returned component is of type T + get_component_with_type(world, TypeId::of::(), entity, location) + .map(|value| value.deref::()) +} + +// SAFETY: EntityLocation must be valid +#[inline] +pub(crate) unsafe fn get_mut( + world: &mut World, + entity: Entity, + location: EntityLocation, +) -> Option> { + // SAFE: world access is unique, entity location is valid, and returned component is of type + // T + let change_tick = world.change_tick(); + let last_change_tick = world.last_change_tick(); + get_component_and_ticks_with_type(world, TypeId::of::(), entity, location).map( + |(value, ticks)| Mut { + value: value.assert_unique().deref_mut::(), + ticks: Ticks { + component_ticks: &mut *ticks.get(), + last_change_tick, + change_tick, + }, + }, + ) +} + #[cfg(test)] mod tests { #[test] diff --git a/crates/bevy_ecs/src/world/mod.rs b/crates/bevy_ecs/src/world/mod.rs index a770ba16f4aee..444c655da980f 100644 --- a/crates/bevy_ecs/src/world/mod.rs +++ b/crates/bevy_ecs/src/world/mod.rs @@ -13,7 +13,8 @@ use crate::{ change_detection::Ticks, component::{Component, ComponentId, ComponentTicks, Components, StorageType}, entity::{AllocAtWithoutReplacement, Entities, Entity}, - query::{FilterFetch, QueryState, WorldQuery}, + ptr::OwningPtr, + query::{FilterFetch, QueryFetch, QueryState, WorldQuery}, storage::{Column, SparseSet, Storages}, system::Resource, }; @@ -192,7 +193,8 @@ impl World { /// .insert(Position { x: 0.0, y: 0.0 }) /// .id(); /// - /// let position = world.entity(entity).get::().unwrap(); + /// let entity = world.entity(entity); + /// let position = entity.get::().unwrap(); /// assert_eq!(position.x, 0.0); /// ``` #[inline] @@ -220,7 +222,8 @@ impl World { /// .insert(Position { x: 0.0, y: 0.0 }) /// .id(); /// - /// let mut position = world.entity_mut(entity).get_mut::().unwrap(); + /// let mut entity = world.entity_mut(entity); + /// let mut position = entity.get_mut::().unwrap(); /// position.x = 1.0; /// ``` #[inline] @@ -332,7 +335,7 @@ impl World { /// .insert_bundle((Num(1), Label("hello"))) // add a bundle of components /// .id(); /// - /// let position = world.entity(entity).get::().unwrap(); + /// let position = world.get::(entity).unwrap(); /// assert_eq!(position.x, 0.0); /// ``` pub fn spawn(&mut self) -> EntityMut { @@ -409,7 +412,7 @@ impl World { /// ``` #[inline] pub fn get(&self, entity: Entity) -> Option<&T> { - self.get_entity(entity)?.get() + unsafe { get(self, entity, self.get_entity(entity)?.location()) } } /// Retrieves a mutable reference to the given `entity`'s [Component] of the given type. @@ -432,7 +435,7 @@ impl World { /// ``` #[inline] pub fn get_mut(&mut self, entity: Entity) -> Option> { - self.get_entity_mut(entity)?.get_mut() + unsafe { get_mut(self, entity, self.get_entity(entity)?.location()) } } /// Despawns the given `entity`, if it exists. This will also remove all of the entity's @@ -538,7 +541,7 @@ impl World { /// ``` #[inline] pub fn query(&mut self) -> QueryState { - QueryState::new(self) + self.query_filtered::() } /// Returns [QueryState] for the given filtered [WorldQuery], which is used to efficiently @@ -563,7 +566,7 @@ impl World { #[inline] pub fn query_filtered(&mut self) -> QueryState where - F::Fetch: FilterFetch, + for<'x, 'y> QueryFetch<'x, 'y, F>: FilterFetch<'x, 'y>, { QueryState::new(self) } @@ -908,6 +911,9 @@ impl World { /// assert_eq!(world.get_resource::().unwrap().0, 2); /// ``` pub fn resource_scope(&mut self, f: impl FnOnce(&mut World, Mut) -> U) -> U { + let last_change_tick = self.last_change_tick(); + let change_tick = self.change_tick(); + let component_id = self .components .get_resource_id(TypeId::of::()) @@ -925,25 +931,31 @@ impl World { // the ptr value / drop is called when T is dropped unsafe { column.swap_remove_and_forget_unchecked(0) } }; + let mut val = unsafe { ptr.read::() }; // SAFE: pointer is of type T let value = Mut { - value: unsafe { &mut *ptr.deref_mut::() }, + value: &mut val, ticks: Ticks { component_ticks: &mut ticks, - last_change_tick: self.last_change_tick(), - change_tick: self.change_tick(), + last_change_tick, + change_tick, }, }; let result = f(self, value); + assert!(self.get_resource::().is_none()); + let resource_archetype = self.archetypes.resource_mut(); let unique_components = resource_archetype.unique_components_mut(); let column = unique_components .get_mut(component_id) .unwrap_or_else(|| panic!("resource does not exist: {}", std::any::type_name::())); - unsafe { - // SAFE: pointer is of type T - column.push(ptr, ticks); - } + + OwningPtr::make(val, |ptr| { + unsafe { + // SAFE: pointer is of type T + column.push(ptr, ticks); + } + }); result } @@ -968,9 +980,9 @@ impl World { ) -> Option> { let column = self.get_populated_resource_column(component_id)?; Some(Mut { - value: column.get_data_ptr_mut().deref_mut(), + value: column.get_data_ptr().assert_unique().deref_mut(), ticks: Ticks { - component_ticks: &mut *column.get_ticks_mut_ptr_unchecked(0), + component_ticks: column.get_ticks_mut_unchecked(0), last_change_tick: self.last_change_tick(), change_tick: self.read_change_tick(), }, @@ -1003,17 +1015,17 @@ impl World { /// # Safety /// `component_id` must be valid and correspond to a resource component of type T #[inline] - unsafe fn insert_resource_with_id(&mut self, component_id: ComponentId, mut value: T) { + unsafe fn insert_resource_with_id(&mut self, component_id: ComponentId, value: T) { let change_tick = self.change_tick(); let column = self.initialize_resource_internal(component_id); if column.is_empty() { // SAFE: column is of type T and has been allocated above - let data = (&mut value as *mut T).cast::(); - std::mem::forget(value); - column.push(data, ComponentTicks::new(change_tick)); + OwningPtr::make(value, |ptr| { + column.push(ptr, ComponentTicks::new(change_tick)); + }); } else { // SAFE: column is of type T and has already been allocated - *column.get_data_unchecked(0).cast::() = value; + *column.get_data_unchecked_mut(0).deref_mut::() = value; column.get_ticks_unchecked_mut(0).set_changed(change_tick); } } diff --git a/crates/bevy_render/src/render_graph/nodes/pass_node.rs b/crates/bevy_render/src/render_graph/nodes/pass_node.rs index 73cf62e8f113a..56ebf66ca7040 100644 --- a/crates/bevy_render/src/render_graph/nodes/pass_node.rs +++ b/crates/bevy_render/src/render_graph/nodes/pass_node.rs @@ -11,7 +11,7 @@ use crate::{ }; use bevy_asset::{Assets, Handle}; use bevy_ecs::{ - query::{QueryState, ReadOnlyFetch, WorldQuery}, + query::{QueryState, ReadOnlyQuery, WorldQuery}, world::{Mut, World}, }; use bevy_utils::{tracing::debug, HashMap}; @@ -117,7 +117,7 @@ impl PassNode { impl Node for PassNode where - Q::Fetch: ReadOnlyFetch, + Q: ReadOnlyQuery, { fn input(&self) -> &[ResourceSlotInfo] { &self.inputs diff --git a/crates/bevy_ui/src/flex/mod.rs b/crates/bevy_ui/src/flex/mod.rs index 1e6a2a7b92860..069fa6b6e7293 100644 --- a/crates/bevy_ui/src/flex/mod.rs +++ b/crates/bevy_ui/src/flex/mod.rs @@ -4,7 +4,7 @@ use crate::{CalculatedSize, ControlNode, Node, Style}; use bevy_app::EventReader; use bevy_ecs::{ entity::Entity, - query::{Changed, FilterFetch, With, Without, WorldQuery}, + query::{Changed, FilterFetch, QueryFetch, With, Without, WorldQuery}, system::{Query, Res, ResMut}, }; use bevy_log::warn; @@ -275,7 +275,11 @@ pub fn flex_node_system( full_node_query, ); } else { - update_changed(&mut *flex_surface, logical_to_physical_factor, node_query); + update_changed::<(With, Changed