From c47d6e526227334d0eb8c25d5f3f8fae70cf992d Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Fri, 26 Jul 2024 08:25:27 +0200 Subject: [PATCH 01/81] add new backend --- Cargo.lock | 21 ++ .../crates/turbo-tasks-backend/Cargo.toml | 32 +++ turbopack/crates/turbo-tasks-backend/build.rs | 5 + .../turbo-tasks-backend/src/backend/mod.rs | 37 ++++ .../src/backend/operation/mod.rs | 1 + .../src/backend/storage.rs | 34 ++++ .../crates/turbo-tasks-backend/src/data.rs | 184 ++++++++++++++++++ .../crates/turbo-tasks-backend/src/lib.rs | 3 + .../turbo-tasks-backend/src/utils/bi_map.rs | 50 +++++ .../src/utils/chunked_vec.rs | 76 ++++++++ .../turbo-tasks-backend/src/utils/mod.rs | 2 + .../crates/turbo-tasks-macros/src/lib.rs | 7 + .../turbo-tasks-macros/src/with_key_macro.rs | 179 +++++++++++++++++ turbopack/crates/turbo-tasks/src/keyed.rs | 7 + turbopack/crates/turbo-tasks/src/lib.rs | 4 +- 15 files changed, 641 insertions(+), 1 deletion(-) create mode 100644 turbopack/crates/turbo-tasks-backend/Cargo.toml create mode 100644 turbopack/crates/turbo-tasks-backend/build.rs create mode 100644 turbopack/crates/turbo-tasks-backend/src/backend/mod.rs create mode 100644 turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs create mode 100644 turbopack/crates/turbo-tasks-backend/src/backend/storage.rs create mode 100644 turbopack/crates/turbo-tasks-backend/src/data.rs create mode 100644 turbopack/crates/turbo-tasks-backend/src/lib.rs create mode 100644 turbopack/crates/turbo-tasks-backend/src/utils/bi_map.rs create mode 100644 turbopack/crates/turbo-tasks-backend/src/utils/chunked_vec.rs create mode 100644 turbopack/crates/turbo-tasks-backend/src/utils/mod.rs create mode 100644 turbopack/crates/turbo-tasks-macros/src/with_key_macro.rs create mode 100644 turbopack/crates/turbo-tasks/src/keyed.rs diff --git a/Cargo.lock b/Cargo.lock index 5e0c19e4c6cb3..0d645183aa63b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8568,6 +8568,27 @@ dependencies = [ "unsize", ] +[[package]] +name = "turbo-tasks-backend" +version = "0.1.0" +dependencies = [ + "anyhow", + "auto-hash-map", + "dashmap", + "once_cell", + "parking_lot", + "rustc-hash", + "serde", + "smallvec", + "tokio", + "tracing", + "turbo-prehash", + "turbo-tasks", + "turbo-tasks-build", + "turbo-tasks-hash", + "turbo-tasks-malloc", +] + [[package]] name = "turbo-tasks-build" version = "0.1.0" diff --git a/turbopack/crates/turbo-tasks-backend/Cargo.toml b/turbopack/crates/turbo-tasks-backend/Cargo.toml new file mode 100644 index 0000000000000..8b21d629dfd3d --- /dev/null +++ b/turbopack/crates/turbo-tasks-backend/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "turbo-tasks-backend" +version = "0.1.0" +description = "TBD" +license = "MPL-2.0" +edition = "2021" +autobenches = false + +[lib] +bench = false + +[lints] +workspace = true + +[dependencies] +anyhow = { workspace = true } +auto-hash-map = { workspace = true } +dashmap = { workspace = true } +once_cell = { workspace = true } +parking_lot = { workspace = true } +rustc-hash = { workspace = true } +serde = { workspace = true } +smallvec = { workspace = true } +tokio = { workspace = true } +tracing = { workspace = true } +turbo-prehash = { workspace = true } +turbo-tasks = { workspace = true } +turbo-tasks-hash = { workspace = true } +turbo-tasks-malloc = { workspace = true, default-features = false } + +[build-dependencies] +turbo-tasks-build = { workspace = true } diff --git a/turbopack/crates/turbo-tasks-backend/build.rs b/turbopack/crates/turbo-tasks-backend/build.rs new file mode 100644 index 0000000000000..1673efed59cce --- /dev/null +++ b/turbopack/crates/turbo-tasks-backend/build.rs @@ -0,0 +1,5 @@ +use turbo_tasks_build::generate_register; + +fn main() { + generate_register(); +} diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs new file mode 100644 index 0000000000000..c4b918a446dee --- /dev/null +++ b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs @@ -0,0 +1,37 @@ +mod operation; +mod storage; + +use std::{collections::VecDeque, sync::Arc}; + +use parking_lot::Mutex; +use turbo_tasks::{backend::CachedTaskType, TaskId}; + +use self::{operation::Operation, storage::Storage}; +use crate::{ + data::{CachedDataItem, CachedDataUpdate}, + utils::{bi_map::BiMap, chunked_vec::ChunkedVec}, +}; + +pub struct TurboTasksBackend { + persisted_task_cache_log: Mutex, TaskId)>>, + task_cache: BiMap, TaskId>, + persisted_storage_log: Mutex>, + storage: Storage, + operations: Mutex>>, +} + +impl TurboTasksBackend { + pub fn new() -> Self { + Self { + persisted_task_cache_log: Mutex::new(ChunkedVec::new()), + task_cache: BiMap::new(), + persisted_storage_log: Mutex::new(ChunkedVec::new()), + storage: Storage::new(), + operations: Mutex::new(VecDeque::new()), + } + } + + fn run_operation(&self, operation: Box) { + self.operations.lock().push_back(operation); + } +} diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs new file mode 100644 index 0000000000000..b3b434511bf3f --- /dev/null +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs @@ -0,0 +1 @@ +pub trait Operation {} diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs b/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs new file mode 100644 index 0000000000000..957dabd2db994 --- /dev/null +++ b/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs @@ -0,0 +1,34 @@ +use auto_hash_map::AutoMap; +use dashmap::DashMap; +use turbo_tasks::Keyed; + +enum PersistanceState { + /// We know that all state of the object is only in the cache and nothing is + /// stored in the persistent cache. + CacheOnly, + /// We know that some state of the object is stored in the persistent cache. + Persisted, + /// We have never checked the persistent cache for the state of the object. + Unknown, +} + +struct InnerStorage { + map: AutoMap, + persistance_state: PersistanceState, +} + +pub struct Storage { + map: DashMap>, +} + +impl Storage +where + K: Eq + std::hash::Hash + Clone, + T: Keyed, +{ + pub fn new() -> Self { + Self { + map: DashMap::new(), + } + } +} diff --git a/turbopack/crates/turbo-tasks-backend/src/data.rs b/turbopack/crates/turbo-tasks-backend/src/data.rs new file mode 100644 index 0000000000000..d4a032e966627 --- /dev/null +++ b/turbopack/crates/turbo-tasks-backend/src/data.rs @@ -0,0 +1,184 @@ +use turbo_tasks::{util::SharedError, CellId, SharedReference, TaskId}; + +#[derive(Debug, Copy, Clone)] +pub struct CellRef { + task: TaskId, + cell: CellId, +} + +#[derive(Debug, Copy, Clone)] +pub enum OutputValue { + Cell(CellRef), + Output(TaskId), + Error, +} +impl OutputValue { + fn is_transient(&self) -> bool { + match self { + OutputValue::Cell(cell) => cell.task.is_transient(), + OutputValue::Output(task) => task.is_transient(), + OutputValue::Error => false, + } + } +} + +#[derive(Debug, Copy, Clone)] +pub enum RootType { + RootTask, + OnceTask, + ReadingStronglyConsistent, +} + +#[derive(Debug, Copy, Clone)] +pub enum InProgressState { + Scheduled { clean: bool }, + InProgress { clean: bool, stale: bool }, +} + +#[turbo_tasks::with_key] +#[derive(Debug, Clone)] +pub enum CachedDataItem { + // Output + Output { + value: OutputValue, + }, + Collectible { + collectible: CellRef, + value: (), + }, + + // State + Dirty { + value: (), + }, + DirtyWhenPersisted { + value: (), + }, + + // Children + Child { + task: TaskId, + value: (), + }, + + // Cells + CellData { + cell: CellId, + value: SharedReference, + }, + + // Dependencies + OutputDependency { + target: TaskId, + value: (), + }, + CellDependency { + target: CellRef, + value: (), + }, + + // Dependent + OutputDependent { + task: TaskId, + value: (), + }, + CellDependent { + cell: CellId, + task: TaskId, + value: (), + }, + + // Aggregation Graph + AggregationNumber { + value: u32, + }, + Follower { + task: TaskId, + value: (), + }, + Upper { + task: TaskId, + value: (), + }, + + // Aggregated Data + AggregatedDirtyTask { + task: TaskId, + value: (), + }, + AggregatedCollectible { + collectible: CellRef, + value: (), + }, + AggregatedUnfinishedTasks { + value: u32, + }, + + // Transient Root Type + AggregateRootType { + value: RootType, + }, + + // Transient In Progress state + InProgress { + value: InProgressState, + }, + OutdatedCollectible { + collectible: CellRef, + value: (), + }, + OutdatedOutputDependency { + target: TaskId, + value: (), + }, + OutdatedCellDependency { + target: CellRef, + value: (), + }, + + // Transient Error State + Error { + value: SharedError, + }, +} + +impl CachedDataItem { + pub fn is_persistent(&self) -> bool { + match self { + CachedDataItem::Output { value } => value.is_transient(), + CachedDataItem::Collectible { collectible, .. } => !collectible.task.is_transient(), + CachedDataItem::Dirty { .. } => true, + CachedDataItem::DirtyWhenPersisted { .. } => true, + CachedDataItem::Child { task, .. } => !task.is_transient(), + CachedDataItem::CellData { .. } => true, + CachedDataItem::OutputDependency { target, .. } => !target.is_transient(), + CachedDataItem::CellDependency { target, .. } => !target.task.is_transient(), + CachedDataItem::OutputDependent { task, .. } => !task.is_transient(), + CachedDataItem::CellDependent { task, .. } => !task.is_transient(), + CachedDataItem::AggregationNumber { .. } => true, + CachedDataItem::Follower { task, .. } => !task.is_transient(), + CachedDataItem::Upper { task, .. } => !task.is_transient(), + CachedDataItem::AggregatedDirtyTask { task, .. } => !task.is_transient(), + CachedDataItem::AggregatedCollectible { collectible, .. } => { + !collectible.task.is_transient() + } + CachedDataItem::AggregatedUnfinishedTasks { .. } => true, + CachedDataItem::AggregateRootType { .. } => false, + CachedDataItem::InProgress { .. } => false, + CachedDataItem::OutdatedCollectible { .. } => false, + CachedDataItem::OutdatedOutputDependency { .. } => false, + CachedDataItem::OutdatedCellDependency { .. } => false, + CachedDataItem::Error { .. } => false, + } + } +} + +trait IsDefault { + fn is_default(&self) -> bool; +} + +pub struct CachedDataUpdate { + pub task: TaskId, + pub key: CachedDataItemKey, + pub value: Option, +} diff --git a/turbopack/crates/turbo-tasks-backend/src/lib.rs b/turbopack/crates/turbo-tasks-backend/src/lib.rs new file mode 100644 index 0000000000000..5f67a0c86f697 --- /dev/null +++ b/turbopack/crates/turbo-tasks-backend/src/lib.rs @@ -0,0 +1,3 @@ +mod backend; +mod data; +mod utils; diff --git a/turbopack/crates/turbo-tasks-backend/src/utils/bi_map.rs b/turbopack/crates/turbo-tasks-backend/src/utils/bi_map.rs new file mode 100644 index 0000000000000..97392f52b8196 --- /dev/null +++ b/turbopack/crates/turbo-tasks-backend/src/utils/bi_map.rs @@ -0,0 +1,50 @@ +use std::{borrow::Borrow, hash::Hash}; + +use dashmap::{mapref::entry::Entry, DashMap}; + +pub struct BiMap { + forward: DashMap, + reverse: DashMap, +} + +impl BiMap +where + K: Eq + Hash + Clone, + V: Eq + Hash + Clone, +{ + pub fn new() -> Self { + Self { + forward: DashMap::new(), + reverse: DashMap::new(), + } + } + + pub fn lookup_forward(&self, key: &Q) -> Option + where + K: Borrow, + Q: Hash + Eq, + { + self.forward.get(key).map(|v| v.value().clone()) + } + + pub fn lookup_reverse(&self, key: &Q) -> Option + where + V: Borrow, + Q: Hash + Eq, + { + self.reverse.get(key).map(|v| v.value().clone()) + } + + pub fn try_insert(&self, key: K, value: V) -> Result<(), V> { + match self.forward.entry(key) { + Entry::Occupied(e) => Err(e.get().clone()), + Entry::Vacant(e) => { + let e = e.insert_entry(value.clone()); + let key = e.key().clone(); + drop(e); + self.reverse.insert(value, key); + Ok(()) + } + } + } +} diff --git a/turbopack/crates/turbo-tasks-backend/src/utils/chunked_vec.rs b/turbopack/crates/turbo-tasks-backend/src/utils/chunked_vec.rs new file mode 100644 index 0000000000000..46292f79e5e72 --- /dev/null +++ b/turbopack/crates/turbo-tasks-backend/src/utils/chunked_vec.rs @@ -0,0 +1,76 @@ +pub struct ChunkedVec { + chunks: Vec>, +} + +impl ChunkedVec { + pub fn new() -> Self { + Self { chunks: Vec::new() } + } + + pub fn len(&self) -> usize { + if let Some(last) = self.chunks.last() { + let free = last.capacity() - self.len(); + cummulative_chunk_size(self.chunks.len() - 1) - free + } else { + 0 + } + } + + pub fn push(&mut self, item: T) { + if let Some(chunk) = self.chunks.last_mut() { + if chunk.len() < chunk.capacity() { + chunk.push(item); + return; + } + } + let mut chunk = Vec::with_capacity(chunk_size(self.chunks.len())); + chunk.push(item); + self.chunks.push(chunk); + } + + pub fn into_iter(self) -> impl Iterator { + let len = self.len(); + ExactSizeIter { + iter: self.chunks.into_iter().flat_map(|chunk| chunk.into_iter()), + len, + } + } + + pub fn iter(&self) -> impl Iterator { + ExactSizeIter { + iter: self.chunks.iter().flat_map(|chunk| chunk.iter()), + len: self.len(), + } + } +} + +fn chunk_size(chunk_index: usize) -> usize { + 8 << chunk_index +} + +fn cummulative_chunk_size(chunk_index: usize) -> usize { + (8 << (chunk_index + 1)) - 8 +} + +struct ExactSizeIter { + iter: I, + len: usize, +} + +impl Iterator for ExactSizeIter { + type Item = I::Item; + + fn next(&mut self) -> Option { + self.iter.next().inspect(|_| self.len -= 1) + } + + fn size_hint(&self) -> (usize, Option) { + (self.len, Some(self.len)) + } +} + +impl ExactSizeIterator for ExactSizeIter { + fn len(&self) -> usize { + self.len + } +} diff --git a/turbopack/crates/turbo-tasks-backend/src/utils/mod.rs b/turbopack/crates/turbo-tasks-backend/src/utils/mod.rs new file mode 100644 index 0000000000000..54b7d067e843b --- /dev/null +++ b/turbopack/crates/turbo-tasks-backend/src/utils/mod.rs @@ -0,0 +1,2 @@ +pub mod bi_map; +pub mod chunked_vec; diff --git a/turbopack/crates/turbo-tasks-macros/src/lib.rs b/turbopack/crates/turbo-tasks-macros/src/lib.rs index 8e6358c47ed40..ddeaa735b30a2 100644 --- a/turbopack/crates/turbo-tasks-macros/src/lib.rs +++ b/turbopack/crates/turbo-tasks-macros/src/lib.rs @@ -11,6 +11,7 @@ mod primitive_macro; mod value_impl_macro; mod value_macro; mod value_trait_macro; +mod with_key_macro; extern crate proc_macro; @@ -100,6 +101,12 @@ pub fn value_impl(args: TokenStream, input: TokenStream) -> TokenStream { value_impl_macro::value_impl(args, input) } +#[proc_macro_error] +#[proc_macro_attribute] +pub fn with_key(_args: TokenStream, input: TokenStream) -> TokenStream { + with_key_macro::with_key(input) +} + #[allow_internal_unstable(min_specialization, into_future, trivial_bounds)] #[proc_macro_error] #[proc_macro] diff --git a/turbopack/crates/turbo-tasks-macros/src/with_key_macro.rs b/turbopack/crates/turbo-tasks-macros/src/with_key_macro.rs new file mode 100644 index 0000000000000..127c681998258 --- /dev/null +++ b/turbopack/crates/turbo-tasks-macros/src/with_key_macro.rs @@ -0,0 +1,179 @@ +use proc_macro::TokenStream; +use quote::quote; +use syn::{parse_macro_input, Ident, ItemEnum}; + +pub fn with_key(input: TokenStream) -> TokenStream { + let input = parse_macro_input!(input as ItemEnum); + + let attrs = &input.attrs; + let ident = &input.ident; + let vis = &input.vis; + let key_name = Ident::new(&format!("{}Key", input.ident), input.ident.span()); + let value_name = Ident::new(&format!("{}Value", input.ident), input.ident.span()); + + let variant_names = input + .variants + .iter() + .map(|variant| &variant.ident) + .collect::>(); + + let key_fields = input + .variants + .iter() + .map(|variant| { + variant + .fields + .iter() + .filter(|field| { + let Some(ident) = &field.ident else { + return false; + }; + ident != "value" + }) + .collect::>() + }) + .collect::>(); + + let value_fields = input + .variants + .iter() + .map(|variant| { + variant + .fields + .iter() + .filter(|field| { + let Some(ident) = &field.ident else { + return false; + }; + ident == "value" + }) + .collect::>() + }) + .collect::>(); + + let key_decl = field_declarations(&key_fields); + let key_pat = patterns(&key_fields); + let key_clone_fields = clone_fields(&key_fields); + + let value_decl = field_declarations(&value_fields); + let value_pat = patterns(&value_fields); + let value_clone_fields = clone_fields(&value_fields); + + quote! { + #input + + impl turbo_tasks::Keyed for #ident { + type Key = #key_name; + type Value = #value_name; + + fn key(&self) -> #key_name { + match self { + #( + #ident::#variant_names { #key_pat .. } => #key_name::#variant_names { #key_clone_fields }, + )* + } + } + + fn value(&self) -> #value_name { + match self { + #( + #ident::#variant_names { #value_pat .. } => #value_name::#variant_names { #value_clone_fields }, + )* + } + } + + fn from_key_and_value(key: #key_name, value: #value_name) -> Self { + match (key, value) { + #( + (#key_name::#variant_names { #key_pat }, #value_name::#variant_names { #value_pat }) => #ident::#variant_names { #key_pat #value_pat }, + )* + _ => panic!("Invalid key and value combination"), + } + } + } + + #(#attrs)* + #vis enum #key_name { + #( + #variant_names { + #key_decl + }, + )* + } + + #(#attrs)* + #vis enum #value_name { + #( + #variant_names { + #value_decl + }, + )* + } + } + .into() +} + +fn patterns(fields: &[Vec<&syn::Field>]) -> Vec { + let variant_pat = fields + .iter() + .map(|fields| { + let pat = fields + .iter() + .map(|field| { + let ident = field.ident.as_ref().unwrap(); + quote! { + #ident + } + }) + .collect::>(); + quote! { + #(#pat,)* + } + }) + .collect::>(); + variant_pat +} + +fn clone_fields(fields: &[Vec<&syn::Field>]) -> Vec { + let variant_pat = fields + .iter() + .map(|fields| { + let pat = fields + .iter() + .map(|field| { + let ident = field.ident.as_ref().unwrap(); + quote! { + #ident: #ident.clone() + } + }) + .collect::>(); + quote! { + #(#pat,)* + } + }) + .collect::>(); + variant_pat +} + +fn field_declarations(fields: &[Vec<&syn::Field>]) -> Vec { + fields + .iter() + .map(|fields| { + let fields = fields + .iter() + .map(|field| { + let ty = &field.ty; + let ident = field.ident.as_ref().unwrap(); + let attrs = &field.attrs; + quote! { + #(#attrs)* + #ident: #ty + } + }) + .collect::>(); + quote! { + #(#fields),* + } + }) + .collect::>() +} diff --git a/turbopack/crates/turbo-tasks/src/keyed.rs b/turbopack/crates/turbo-tasks/src/keyed.rs new file mode 100644 index 0000000000000..46a1cf7f6b9bf --- /dev/null +++ b/turbopack/crates/turbo-tasks/src/keyed.rs @@ -0,0 +1,7 @@ +pub trait Keyed { + type Key; + type Value; + fn key(&self) -> Self::Key; + fn value(&self) -> Self::Value; + fn from_key_and_value(key: Self::Key, value: Self::Value) -> Self; +} diff --git a/turbopack/crates/turbo-tasks/src/lib.rs b/turbopack/crates/turbo-tasks/src/lib.rs index b65a3ae423a27..f7e216df68c2b 100644 --- a/turbopack/crates/turbo-tasks/src/lib.rs +++ b/turbopack/crates/turbo-tasks/src/lib.rs @@ -49,6 +49,7 @@ mod id; mod id_factory; mod invalidation; mod join_iter_ext; +mod keyed; #[doc(hidden)] pub mod macro_helpers; mod magic_any; @@ -91,6 +92,7 @@ pub use invalidation::{ InvalidationReasonSet, Invalidator, }; pub use join_iter_ext::{JoinIterExt, TryFlatJoinIterExt, TryJoinIterExt}; +pub use keyed::Keyed; pub use magic_any::MagicAny; pub use manager::{ dynamic_call, dynamic_this_call, emit, mark_dirty_when_persisted, mark_finished, mark_stateful, @@ -107,7 +109,7 @@ pub use serialization_invalidation::SerializationInvalidator; pub use state::{State, TransientState}; pub use task::{task_input::TaskInput, SharedReference}; pub use trait_ref::{IntoTraitRef, TraitRef}; -pub use turbo_tasks_macros::{function, value_impl, value_trait, TaskInput}; +pub use turbo_tasks_macros::{function, value_impl, value_trait, with_key, TaskInput}; pub use value::{TransientInstance, TransientValue, Value}; pub use value_type::{TraitMethod, TraitType, ValueType}; pub use vc::{ From c33edc102fe7d3e588c7aa9f13f27ef1445a3ddc Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Mon, 29 Jul 2024 10:52:22 +0200 Subject: [PATCH 02/81] new backend --- Cargo.lock | 1 + .../crates/turbo-tasks-backend/Cargo.toml | 1 + .../turbo-tasks-backend/src/backend/mod.rs | 312 +++++++++++++++++- .../src/backend/operation/connect_child.rs | 52 +++ .../src/backend/operation/mod.rs | 176 +++++++++- .../src/backend/storage.rs | 85 ++++- .../crates/turbo-tasks-backend/src/data.rs | 53 ++- .../crates/turbo-tasks-backend/src/lib.rs | 2 + .../src/utils/external_locks.rs | 61 ++++ .../turbo-tasks-backend/src/utils/mod.rs | 2 + .../src/utils/ptr_eq_arc.rs | 47 +++ .../key_value_pair_macro.rs} | 19 +- .../turbo-tasks-macros/src/derive/mod.rs | 2 + .../crates/turbo-tasks-macros/src/lib.rs | 12 +- .../src/{keyed.rs => key_value_pair.rs} | 5 +- turbopack/crates/turbo-tasks/src/lib.rs | 6 +- 16 files changed, 791 insertions(+), 45 deletions(-) create mode 100644 turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs create mode 100644 turbopack/crates/turbo-tasks-backend/src/utils/external_locks.rs create mode 100644 turbopack/crates/turbo-tasks-backend/src/utils/ptr_eq_arc.rs rename turbopack/crates/turbo-tasks-macros/src/{with_key_macro.rs => derive/key_value_pair_macro.rs} (90%) rename turbopack/crates/turbo-tasks/src/{keyed.rs => key_value_pair.rs} (53%) diff --git a/Cargo.lock b/Cargo.lock index 0d645183aa63b..ed033fc7f2de4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8573,6 +8573,7 @@ name = "turbo-tasks-backend" version = "0.1.0" dependencies = [ "anyhow", + "async-trait", "auto-hash-map", "dashmap", "once_cell", diff --git a/turbopack/crates/turbo-tasks-backend/Cargo.toml b/turbopack/crates/turbo-tasks-backend/Cargo.toml index 8b21d629dfd3d..cd38a6bb31794 100644 --- a/turbopack/crates/turbo-tasks-backend/Cargo.toml +++ b/turbopack/crates/turbo-tasks-backend/Cargo.toml @@ -14,6 +14,7 @@ workspace = true [dependencies] anyhow = { workspace = true } +async-trait = { workspace = true } auto-hash-map = { workspace = true } dashmap = { workspace = true } once_cell = { workspace = true } diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs index c4b918a446dee..9f495fbf27a21 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs @@ -1,37 +1,331 @@ mod operation; mod storage; -use std::{collections::VecDeque, sync::Arc}; +use std::{ + borrow::Cow, + collections::HashSet, + future::Future, + hash::BuildHasherDefault, + pin::Pin, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, + time::Duration, +}; -use parking_lot::Mutex; -use turbo_tasks::{backend::CachedTaskType, TaskId}; +use anyhow::Result; +use auto_hash_map::{AutoMap, AutoSet}; +use parking_lot::{Condvar, Mutex}; +use rustc_hash::FxHasher; +use turbo_tasks::{ + backend::{ + Backend, BackendJobId, CachedTaskType, CellContent, TaskExecutionSpec, TransientTaskType, + TypedCellContent, + }, + event::EventListener, + util::IdFactoryWithReuse, + CellId, FunctionId, RawVc, ReadConsistency, TaskId, TraitTypeId, TurboTasksBackendApi, + ValueTypeId, TRANSIENT_TASK_BIT, +}; -use self::{operation::Operation, storage::Storage}; +use self::{ + operation::{AnyOperation, ExecuteContext, Operation}, + storage::Storage, +}; use crate::{ data::{CachedDataItem, CachedDataUpdate}, - utils::{bi_map::BiMap, chunked_vec::ChunkedVec}, + utils::{bi_map::BiMap, chunked_vec::ChunkedVec, ptr_eq_arc::PtrEqArc}, }; +const SNAPSHOT_REQUESTED_BIT: usize = 1 << 63; + +struct SnapshotRequest { + snapshot_requested: bool, + suspended_operations: HashSet>, +} + +impl SnapshotRequest { + fn new() -> Self { + Self { + snapshot_requested: false, + suspended_operations: HashSet::new(), + } + } +} + pub struct TurboTasksBackend { + persisted_task_id_factory: IdFactoryWithReuse, + transient_task_id_factory: IdFactoryWithReuse, + persisted_task_cache_log: Mutex, TaskId)>>, task_cache: BiMap, TaskId>, + persisted_storage_log: Mutex>, storage: Storage, - operations: Mutex>>, + + /// Number of executing operations + Highest bit is set when snapshot is + /// requested. When that bit is set, operations should pause until the + /// snapshot is completed. When the bit is set and in progress counter + /// reaches zero, `operations_completed_when_snapshot_requested` is + /// triggered. + in_progress_operations: AtomicUsize, + + snapshot_request: Mutex, + /// Condition Variable that is triggered when `in_progress_operations` + /// reaches zero while snapshot is requested. All operations are either + /// completed or suspended. + operations_suspended: Condvar, + /// Condition Variable that is triggered when a snapshot is completed and + /// operations can continue. + snapshot_completed: Condvar, } impl TurboTasksBackend { pub fn new() -> Self { Self { + persisted_task_id_factory: IdFactoryWithReuse::new(1, (TRANSIENT_TASK_BIT - 1) as u64), + transient_task_id_factory: IdFactoryWithReuse::new( + TRANSIENT_TASK_BIT as u64, + u32::MAX as u64, + ), persisted_task_cache_log: Mutex::new(ChunkedVec::new()), task_cache: BiMap::new(), persisted_storage_log: Mutex::new(ChunkedVec::new()), storage: Storage::new(), - operations: Mutex::new(VecDeque::new()), + in_progress_operations: AtomicUsize::new(0), + snapshot_request: Mutex::new(SnapshotRequest::new()), + operations_suspended: Condvar::new(), + snapshot_completed: Condvar::new(), } } - fn run_operation(&self, operation: Box) { - self.operations.lock().push_back(operation); + fn run_operation( + &self, + operation: impl Operation, + turbo_tasks: &dyn TurboTasksBackendApi, + ) { + operation.execute(ExecuteContext::new(self, turbo_tasks)); + } + + fn operation_suspend_point(&self, suspend: impl FnOnce() -> AnyOperation) { + if (self.in_progress_operations.load(Ordering::Relaxed) & SNAPSHOT_REQUESTED_BIT) != 0 { + let operation = Arc::new(suspend()); + let mut snapshot_request = self.snapshot_request.lock(); + if snapshot_request.snapshot_requested { + snapshot_request + .suspended_operations + .insert(operation.clone().into()); + let value = self.in_progress_operations.fetch_sub(1, Ordering::AcqRel); + assert!((value & SNAPSHOT_REQUESTED_BIT) != 0); + if value == SNAPSHOT_REQUESTED_BIT { + self.operations_suspended.notify_all(); + } + self.snapshot_completed + .wait_while(&mut snapshot_request, |snapshot_request| { + snapshot_request.snapshot_requested + }); + self.in_progress_operations.fetch_add(1, Ordering::AcqRel); + snapshot_request + .suspended_operations + .remove(&operation.into()); + } + } + } +} + +// Operations +impl TurboTasksBackend { + pub fn connect_child( + &self, + parent_task: TaskId, + child_task: TaskId, + turbo_tasks: &dyn TurboTasksBackendApi, + ) { + self.run_operation( + operation::ConnectChildOperation::new(parent_task, child_task), + turbo_tasks, + ); + } +} + +impl Backend for TurboTasksBackend { + fn get_or_create_persistent_task( + &self, + task_type: CachedTaskType, + parent_task: TaskId, + turbo_tasks: &dyn TurboTasksBackendApi, + ) -> TaskId { + if let Some(task_id) = self.task_cache.lookup_forward(&task_type) { + self.connect_child(parent_task, task_id, turbo_tasks); + return task_id; + } + + let task_type = Arc::new(task_type); + let task_id = self.persisted_task_id_factory.get(); + if let Err(existing_task_id) = self.task_cache.try_insert(task_type.clone(), task_id) { + // Safety: We just created the id and failed to insert it. + unsafe { + self.persisted_task_id_factory.reuse(task_id); + } + self.connect_child(parent_task, existing_task_id, turbo_tasks); + return existing_task_id; + } + self.persisted_task_cache_log + .lock() + .push((task_type, task_id)); + + self.connect_child(parent_task, task_id, turbo_tasks); + + task_id + } + + fn invalidate_task(&self, _: TaskId, _: &dyn TurboTasksBackendApi) { + todo!() + } + fn invalidate_tasks(&self, _: &[TaskId], _: &dyn TurboTasksBackendApi) { + todo!() + } + fn invalidate_tasks_set( + &self, + _: &AutoSet, 2>, + _: &dyn TurboTasksBackendApi, + ) { + todo!() + } + fn get_task_description(&self, _: TaskId) -> std::string::String { + todo!() + } + fn try_get_function_id(&self, _: TaskId) -> Option { + todo!() + } + type TaskState = (); + fn new_task_state(&self, _task: TaskId) -> Self::TaskState { + () + } + fn try_start_task_execution( + &self, + _: TaskId, + _: &dyn TurboTasksBackendApi, + ) -> std::option::Option> { + todo!() + } + fn task_execution_result( + &self, + _: TaskId, + _: Result, std::option::Option>>, + _: &dyn TurboTasksBackendApi, + ) { + todo!() + } + fn task_execution_completed( + &self, + _: TaskId, + _: Duration, + _: usize, + _: &AutoMap, 8>, + _: bool, + _: &dyn TurboTasksBackendApi, + ) -> bool { + todo!() + } + fn run_backend_job( + &self, + _: BackendJobId, + _: &dyn TurboTasksBackendApi, + ) -> Pin + Send + 'static)>> { + todo!() + } + fn try_read_task_output( + &self, + _: TaskId, + _: TaskId, + _: ReadConsistency, + _: &dyn TurboTasksBackendApi, + ) -> Result, turbo_tasks::Error> { + todo!() + } + fn try_read_task_output_untracked( + &self, + _: TaskId, + _: ReadConsistency, + _: &dyn TurboTasksBackendApi, + ) -> Result, turbo_tasks::Error> { + todo!() + } + fn try_read_task_cell( + &self, + _: TaskId, + _: CellId, + _: TaskId, + _: &dyn TurboTasksBackendApi, + ) -> Result, turbo_tasks::Error> { + todo!() + } + fn try_read_task_cell_untracked( + &self, + _: TaskId, + _: CellId, + _: &dyn TurboTasksBackendApi, + ) -> Result, turbo_tasks::Error> { + todo!() + } + fn read_task_collectibles( + &self, + _: TaskId, + _: TraitTypeId, + _: TaskId, + _: &dyn TurboTasksBackendApi, + ) -> AutoMap, 1> { + todo!() + } + fn emit_collectible( + &self, + _: TraitTypeId, + _: RawVc, + _: TaskId, + _: &dyn TurboTasksBackendApi, + ) { + todo!() + } + fn unemit_collectible( + &self, + _: TraitTypeId, + _: RawVc, + _: u32, + _: TaskId, + _: &dyn TurboTasksBackendApi, + ) { + todo!() + } + fn update_task_cell( + &self, + _: TaskId, + _: CellId, + _: CellContent, + _: &dyn TurboTasksBackendApi, + ) { + todo!() + } + fn get_or_create_transient_task( + &self, + _: CachedTaskType, + _: TaskId, + _: &dyn TurboTasksBackendApi, + ) -> TaskId { + todo!() + } + fn connect_task(&self, _: TaskId, _: TaskId, _: &dyn TurboTasksBackendApi) { + todo!() + } + fn create_transient_task( + &self, + _: TransientTaskType, + _: &dyn TurboTasksBackendApi, + ) -> TaskId { + todo!() + } + fn dispose_root_task(&self, _: TaskId, _: &dyn TurboTasksBackendApi) { + todo!() } } diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs new file mode 100644 index 0000000000000..11f5f5c7ed4a6 --- /dev/null +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs @@ -0,0 +1,52 @@ +use serde::{Deserialize, Serialize}; +use turbo_tasks::TaskId; + +use super::{ExecuteContext, Operation}; +use crate::{data::CachedDataItem, suspend_point}; + +#[derive(Serialize, Deserialize, Clone, Default)] +pub enum ConnectChildOperation { + AddChild { + parent_task: TaskId, + child_task: TaskId, + }, + #[default] + Done, + // TODO Add aggregated edge +} + +impl ConnectChildOperation { + pub fn new(parent_task: TaskId, child_task: TaskId) -> Self { + ConnectChildOperation::AddChild { + parent_task, + child_task, + } + } +} + +impl Operation for ConnectChildOperation { + fn execute(mut self, ctx: ExecuteContext<'_>) { + loop { + match self { + ConnectChildOperation::AddChild { + parent_task, + child_task, + } => { + let mut parent_task = ctx.task(parent_task); + if parent_task.add(CachedDataItem::Child { + task: child_task, + value: (), + }) { + // TODO add aggregated edge + suspend_point!(self, ctx, ConnectChildOperation::Done); + } else { + suspend_point!(self, ctx, ConnectChildOperation::Done); + } + } + ConnectChildOperation::Done => { + return; + } + } + } + } +} diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs index b3b434511bf3f..232ea7a493380 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs @@ -1 +1,175 @@ -pub trait Operation {} +mod connect_child; + +use serde::{Deserialize, Serialize}; +use turbo_tasks::{KeyValuePair, TaskId, TurboTasksBackendApi}; + +use super::{storage::StorageWriteGuard, TurboTasksBackend}; +use crate::data::{CachedDataItem, CachedDataItemKey, CachedDataItemValue, CachedDataUpdate}; + +pub trait Operation: Serialize + for<'de> Deserialize<'de> { + fn execute(self, ctx: ExecuteContext<'_>); +} + +pub struct ExecuteContext<'a> { + backend: &'a TurboTasksBackend, + turbo_tasks: &'a dyn TurboTasksBackendApi, +} + +impl<'a> ExecuteContext<'a> { + pub fn new( + backend: &'a TurboTasksBackend, + turbo_tasks: &'a dyn TurboTasksBackendApi, + ) -> Self { + Self { + backend, + turbo_tasks, + } + } + + pub fn task(&self, task_id: TaskId) -> TaskGuard<'a> { + TaskGuard { + task: self.backend.storage.access_mut(task_id), + task_id, + backend: self.backend, + } + } + + pub fn operation_suspend_point(&self, f: impl FnOnce() -> AnyOperation) { + self.backend.operation_suspend_point(f) + } +} + +pub struct TaskGuard<'a> { + task_id: TaskId, + task: StorageWriteGuard<'a, TaskId, CachedDataItem>, + backend: &'a TurboTasksBackend, +} + +impl<'a> TaskGuard<'a> { + fn new( + task_id: TaskId, + task: StorageWriteGuard<'a, TaskId, CachedDataItem>, + backend: &'a TurboTasksBackend, + ) -> Self { + Self { + task_id, + task, + backend, + } + } + + pub fn add(&mut self, item: CachedDataItem) -> bool { + if !item.is_persistent() { + self.task.add(item) + } else { + if self.task.add(item.clone()) { + let (key, value) = item.into_key_and_value(); + self.backend + .persisted_storage_log + .lock() + .push(CachedDataUpdate { + key, + task: self.task_id, + value: Some(value), + }); + true + } else { + false + } + } + } + + pub fn upsert(&mut self, item: CachedDataItem) -> Option { + let (key, value) = item.into_key_and_value(); + if !key.is_persistent() { + self.task + .upsert(CachedDataItem::from_key_and_value(key, value)) + } else if value.is_persistent() { + let old = self.task.upsert(CachedDataItem::from_key_and_value( + key.clone(), + value.clone(), + )); + self.backend + .persisted_storage_log + .lock() + .push(CachedDataUpdate { + key, + task: self.task_id, + value: Some(value), + }); + old + } else { + let item = CachedDataItem::from_key_and_value(key.clone(), value); + if let Some(old) = self.task.upsert(item) { + if old.is_persistent() { + self.backend + .persisted_storage_log + .lock() + .push(CachedDataUpdate { + key, + task: self.task_id, + value: None, + }); + } + Some(old) + } else { + None + } + } + } + + pub fn remove(&mut self, key: &CachedDataItemKey) -> Option { + let old_value = self.task.remove(key); + if let Some(value) = old_value { + if key.is_persistent() && value.is_persistent() { + let key = key.clone(); + self.backend + .persisted_storage_log + .lock() + .push(CachedDataUpdate { + key, + task: self.task_id, + value: None, + }); + } + Some(value) + } else { + None + } + } +} + +macro_rules! impl_operation { + ($name:ident $type_path:path) => { + impl From<$type_path> for AnyOperation { + fn from(op: $type_path) -> Self { + AnyOperation::$name(op) + } + } + + pub use $type_path; + }; +} + +#[derive(Serialize, Deserialize)] +pub enum AnyOperation { + ConnectChild(connect_child::ConnectChildOperation), +} + +impl Operation for AnyOperation { + fn execute(self, ctx: ExecuteContext<'_>) { + match self { + AnyOperation::ConnectChild(op) => op.execute(ctx), + } + } +} + +impl_operation!(ConnectChild connect_child::ConnectChildOperation); + +#[macro_export(local_inner_macros)] +macro_rules! suspend_point { + ($self:expr, $ctx:expr, $op:expr) => { + $self = $op; + $ctx.operation_suspend_point(|| $self.clone().into()); + }; +} diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs b/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs index 957dabd2db994..d06d9dc0cb839 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs @@ -1,6 +1,13 @@ -use auto_hash_map::AutoMap; -use dashmap::DashMap; -use turbo_tasks::Keyed; +use std::{ + hash::{BuildHasherDefault, Hash}, + ops::{Deref, DerefMut}, + thread::available_parallelism, +}; + +use auto_hash_map::{map::Entry, AutoMap}; +use dashmap::{mapref::one::RefMut, DashMap}; +use rustc_hash::FxHasher; +use turbo_tasks::KeyValuePair; enum PersistanceState { /// We know that all state of the object is only in the cache and nothing is @@ -12,23 +19,83 @@ enum PersistanceState { Unknown, } -struct InnerStorage { +pub struct InnerStorage { + // TODO consider adding some inline storage map: AutoMap, persistance_state: PersistanceState, } -pub struct Storage { - map: DashMap>, +impl InnerStorage { + fn new() -> Self { + Self { + map: AutoMap::new(), + persistance_state: PersistanceState::Unknown, + } + } + + pub fn add(&mut self, item: T) -> bool { + let (key, value) = item.into_key_and_value(); + match self.map.entry(key) { + Entry::Occupied(_) => false, + Entry::Vacant(e) => { + e.insert(value); + true + } + } + } + + pub fn upsert(&mut self, item: T) -> Option { + let (key, value) = item.into_key_and_value(); + self.map.insert(key, value) + } + + pub fn remove(&mut self, key: &T::Key) -> Option { + self.map.remove(key) + } +} + +pub struct Storage { + map: DashMap, BuildHasherDefault>, } -impl Storage +impl Storage where K: Eq + std::hash::Hash + Clone, - T: Keyed, + T: KeyValuePair, { pub fn new() -> Self { Self { - map: DashMap::new(), + map: DashMap::with_capacity_and_hasher_and_shard_amount( + 1024 * 1024, + Default::default(), + available_parallelism().map_or(4, |v| v.get()) * 64, + ), } } + + pub fn access_mut(&self, key: K) -> StorageWriteGuard<'_, K, T> { + let inner = match self.map.entry(key) { + dashmap::mapref::entry::Entry::Occupied(e) => e.into_ref(), + dashmap::mapref::entry::Entry::Vacant(e) => e.insert(InnerStorage::new()), + }; + StorageWriteGuard { inner } + } +} + +pub struct StorageWriteGuard<'a, K, T: KeyValuePair> { + inner: RefMut<'a, K, InnerStorage, BuildHasherDefault>, +} + +impl<'a, K: Eq + Hash, T: KeyValuePair> Deref for StorageWriteGuard<'a, K, T> { + type Target = InnerStorage; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +impl<'a, K: Eq + Hash, T: KeyValuePair> DerefMut for StorageWriteGuard<'a, K, T> { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.inner + } } diff --git a/turbopack/crates/turbo-tasks-backend/src/data.rs b/turbopack/crates/turbo-tasks-backend/src/data.rs index d4a032e966627..76e7df4f9e64e 100644 --- a/turbopack/crates/turbo-tasks-backend/src/data.rs +++ b/turbopack/crates/turbo-tasks-backend/src/data.rs @@ -1,12 +1,12 @@ -use turbo_tasks::{util::SharedError, CellId, SharedReference, TaskId}; +use turbo_tasks::{util::SharedError, CellId, KeyValuePair, SharedReference, TaskId}; -#[derive(Debug, Copy, Clone)] +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] pub struct CellRef { task: TaskId, cell: CellId, } -#[derive(Debug, Copy, Clone)] +#[derive(Debug, Copy, Clone, PartialEq, Eq)] pub enum OutputValue { Cell(CellRef), Output(TaskId), @@ -22,21 +22,20 @@ impl OutputValue { } } -#[derive(Debug, Copy, Clone)] +#[derive(Debug, Copy, Clone, PartialEq, Eq)] pub enum RootType { RootTask, OnceTask, ReadingStronglyConsistent, } -#[derive(Debug, Copy, Clone)] +#[derive(Debug, Copy, Clone, PartialEq, Eq)] pub enum InProgressState { Scheduled { clean: bool }, InProgress { clean: bool, stale: bool }, } -#[turbo_tasks::with_key] -#[derive(Debug, Clone)] +#[derive(Debug, Clone, KeyValuePair)] pub enum CachedDataItem { // Output Output { @@ -173,6 +172,46 @@ impl CachedDataItem { } } +impl CachedDataItemKey { + pub fn is_persistent(&self) -> bool { + match self { + CachedDataItemKey::Output { .. } => true, + CachedDataItemKey::Collectible { collectible, .. } => !collectible.task.is_transient(), + CachedDataItemKey::Dirty { .. } => true, + CachedDataItemKey::DirtyWhenPersisted { .. } => true, + CachedDataItemKey::Child { task, .. } => !task.is_transient(), + CachedDataItemKey::CellData { .. } => true, + CachedDataItemKey::OutputDependency { target, .. } => !target.is_transient(), + CachedDataItemKey::CellDependency { target, .. } => !target.task.is_transient(), + CachedDataItemKey::OutputDependent { task, .. } => !task.is_transient(), + CachedDataItemKey::CellDependent { task, .. } => !task.is_transient(), + CachedDataItemKey::AggregationNumber { .. } => true, + CachedDataItemKey::Follower { task, .. } => !task.is_transient(), + CachedDataItemKey::Upper { task, .. } => !task.is_transient(), + CachedDataItemKey::AggregatedDirtyTask { task, .. } => !task.is_transient(), + CachedDataItemKey::AggregatedCollectible { collectible, .. } => { + !collectible.task.is_transient() + } + CachedDataItemKey::AggregatedUnfinishedTasks { .. } => true, + CachedDataItemKey::AggregateRootType { .. } => false, + CachedDataItemKey::InProgress { .. } => false, + CachedDataItemKey::OutdatedCollectible { .. } => false, + CachedDataItemKey::OutdatedOutputDependency { .. } => false, + CachedDataItemKey::OutdatedCellDependency { .. } => false, + CachedDataItemKey::Error { .. } => false, + } + } +} + +impl CachedDataItemValue { + pub fn is_persistent(&self) -> bool { + match self { + CachedDataItemValue::Output { value } => !value.is_transient(), + _ => true, + } + } +} + trait IsDefault { fn is_default(&self) -> bool; } diff --git a/turbopack/crates/turbo-tasks-backend/src/lib.rs b/turbopack/crates/turbo-tasks-backend/src/lib.rs index 5f67a0c86f697..5fc95b21a44d9 100644 --- a/turbopack/crates/turbo-tasks-backend/src/lib.rs +++ b/turbopack/crates/turbo-tasks-backend/src/lib.rs @@ -1,3 +1,5 @@ mod backend; mod data; mod utils; + +pub use backend::TurboTasksBackend; diff --git a/turbopack/crates/turbo-tasks-backend/src/utils/external_locks.rs b/turbopack/crates/turbo-tasks-backend/src/utils/external_locks.rs new file mode 100644 index 0000000000000..ddbfa54a75479 --- /dev/null +++ b/turbopack/crates/turbo-tasks-backend/src/utils/external_locks.rs @@ -0,0 +1,61 @@ +use std::{ + collections::{hash_map::Entry, HashMap}, + hash::Hash, + mem::transmute, + sync::Arc, +}; + +use parking_lot::{Mutex, MutexGuard}; + +pub struct ExternalLocks { + locks: HashMap>>, +} + +impl ExternalLocks { + pub fn new() -> Self { + Self { + locks: HashMap::new(), + } + } + + pub fn lock(&mut self, key: K) -> ExternalLockGuard { + let mutex = match self.locks.entry(key) { + Entry::Occupied(e) => e.get().clone(), + Entry::Vacant(e) => { + let lock = Arc::new(Mutex::new(())); + e.insert(lock.clone()); + if self.locks.len().is_power_of_two() { + let to_remove = self + .locks + .iter() + .filter_map(|(k, v)| { + if Arc::strong_count(v) == 1 { + Some(k.clone()) + } else { + None + } + }) + .collect::>(); + to_remove.into_iter().for_each(|k| { + self.locks.remove(&k); + }); + if self.locks.capacity() > self.locks.len() * 3 { + self.locks.shrink_to_fit(); + } + } + lock + } + }; + let guard = mutex.lock(); + // Safety: We know that the guard is valid for the lifetime of the lock as we + // keep the lock + let guard = unsafe { transmute::, MutexGuard<'static, _>>(guard) }; + ExternalLockGuard { lock: mutex, guard } + } +} + +pub struct ExternalLockGuard { + // Safety: guard must be before lock as it is dropped first + guard: MutexGuard<'static, ()>, + lock: Arc>, +} diff --git a/turbopack/crates/turbo-tasks-backend/src/utils/mod.rs b/turbopack/crates/turbo-tasks-backend/src/utils/mod.rs index 54b7d067e843b..84e4a29fdb11a 100644 --- a/turbopack/crates/turbo-tasks-backend/src/utils/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/utils/mod.rs @@ -1,2 +1,4 @@ pub mod bi_map; pub mod chunked_vec; +pub mod external_locks; +pub mod ptr_eq_arc; diff --git a/turbopack/crates/turbo-tasks-backend/src/utils/ptr_eq_arc.rs b/turbopack/crates/turbo-tasks-backend/src/utils/ptr_eq_arc.rs new file mode 100644 index 0000000000000..87543c0399031 --- /dev/null +++ b/turbopack/crates/turbo-tasks-backend/src/utils/ptr_eq_arc.rs @@ -0,0 +1,47 @@ +use std::{ + hash::{Hash, Hasher}, + ops::Deref, + sync::Arc, +}; + +pub struct PtrEqArc(Arc); + +impl PtrEqArc { + pub fn new(value: T) -> Self { + Self(Arc::new(value)) + } +} + +impl From> for PtrEqArc { + fn from(value: Arc) -> Self { + Self(value) + } +} + +impl Deref for PtrEqArc { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl Clone for PtrEqArc { + fn clone(&self) -> Self { + Self(self.0.clone()) + } +} + +impl PartialEq for PtrEqArc { + fn eq(&self, other: &Self) -> bool { + Arc::ptr_eq(&self.0, &other.0) + } +} + +impl Eq for PtrEqArc {} + +impl Hash for PtrEqArc { + fn hash(&self, state: &mut H) { + Arc::as_ptr(&self.0).hash(state) + } +} diff --git a/turbopack/crates/turbo-tasks-macros/src/with_key_macro.rs b/turbopack/crates/turbo-tasks-macros/src/derive/key_value_pair_macro.rs similarity index 90% rename from turbopack/crates/turbo-tasks-macros/src/with_key_macro.rs rename to turbopack/crates/turbo-tasks-macros/src/derive/key_value_pair_macro.rs index 127c681998258..3026d73f5dcd3 100644 --- a/turbopack/crates/turbo-tasks-macros/src/with_key_macro.rs +++ b/turbopack/crates/turbo-tasks-macros/src/derive/key_value_pair_macro.rs @@ -2,10 +2,9 @@ use proc_macro::TokenStream; use quote::quote; use syn::{parse_macro_input, Ident, ItemEnum}; -pub fn with_key(input: TokenStream) -> TokenStream { +pub fn derive_key_value_pair(input: TokenStream) -> TokenStream { let input = parse_macro_input!(input as ItemEnum); - let attrs = &input.attrs; let ident = &input.ident; let vis = &input.vis; let key_name = Ident::new(&format!("{}Key", input.ident), input.ident.span()); @@ -60,9 +59,7 @@ pub fn with_key(input: TokenStream) -> TokenStream { let value_clone_fields = clone_fields(&value_fields); quote! { - #input - - impl turbo_tasks::Keyed for #ident { + impl turbo_tasks::KeyValuePair for #ident { type Key = #key_name; type Value = #value_name; @@ -90,9 +87,17 @@ pub fn with_key(input: TokenStream) -> TokenStream { _ => panic!("Invalid key and value combination"), } } + + fn into_key_and_value(self) -> (#key_name, #value_name) { + match self { + #( + #ident::#variant_names { #key_pat #value_pat } => (#key_name::#variant_names { #key_pat }, #value_name::#variant_names { #value_pat }), + )* + } + } } - #(#attrs)* + #[derive(Debug, Clone, PartialEq, Eq, Hash)] #vis enum #key_name { #( #variant_names { @@ -101,7 +106,7 @@ pub fn with_key(input: TokenStream) -> TokenStream { )* } - #(#attrs)* + #[derive(Debug, Clone)] #vis enum #value_name { #( #variant_names { diff --git a/turbopack/crates/turbo-tasks-macros/src/derive/mod.rs b/turbopack/crates/turbo-tasks-macros/src/derive/mod.rs index d48323309096e..d8c507574ab3b 100644 --- a/turbopack/crates/turbo-tasks-macros/src/derive/mod.rs +++ b/turbopack/crates/turbo-tasks-macros/src/derive/mod.rs @@ -1,4 +1,5 @@ mod deterministic_hash_macro; +mod key_value_pair_macro; mod resolved_value_macro; mod task_input_macro; mod trace_raw_vcs_macro; @@ -6,6 +7,7 @@ mod value_debug_format_macro; mod value_debug_macro; pub use deterministic_hash_macro::derive_deterministic_hash; +pub use key_value_pair_macro::derive_key_value_pair; pub use resolved_value_macro::derive_resolved_value; use syn::{spanned::Spanned, Attribute, Meta, MetaList, NestedMeta}; pub use task_input_macro::derive_task_input; diff --git a/turbopack/crates/turbo-tasks-macros/src/lib.rs b/turbopack/crates/turbo-tasks-macros/src/lib.rs index ddeaa735b30a2..35dc25c0c0d68 100644 --- a/turbopack/crates/turbo-tasks-macros/src/lib.rs +++ b/turbopack/crates/turbo-tasks-macros/src/lib.rs @@ -11,7 +11,6 @@ mod primitive_macro; mod value_impl_macro; mod value_macro; mod value_trait_macro; -mod with_key_macro; extern crate proc_macro; @@ -48,6 +47,11 @@ pub fn derive_task_input(input: TokenStream) -> TokenStream { derive::derive_task_input(input) } +#[proc_macro_derive(KeyValuePair)] +pub fn derive_key_value_pair(input: TokenStream) -> TokenStream { + derive::derive_key_value_pair(input) +} + #[allow_internal_unstable(min_specialization, into_future, trivial_bounds)] #[proc_macro_error] #[proc_macro_attribute] @@ -101,12 +105,6 @@ pub fn value_impl(args: TokenStream, input: TokenStream) -> TokenStream { value_impl_macro::value_impl(args, input) } -#[proc_macro_error] -#[proc_macro_attribute] -pub fn with_key(_args: TokenStream, input: TokenStream) -> TokenStream { - with_key_macro::with_key(input) -} - #[allow_internal_unstable(min_specialization, into_future, trivial_bounds)] #[proc_macro_error] #[proc_macro] diff --git a/turbopack/crates/turbo-tasks/src/keyed.rs b/turbopack/crates/turbo-tasks/src/key_value_pair.rs similarity index 53% rename from turbopack/crates/turbo-tasks/src/keyed.rs rename to turbopack/crates/turbo-tasks/src/key_value_pair.rs index 46a1cf7f6b9bf..6aceaea04d4f7 100644 --- a/turbopack/crates/turbo-tasks/src/keyed.rs +++ b/turbopack/crates/turbo-tasks/src/key_value_pair.rs @@ -1,7 +1,8 @@ -pub trait Keyed { - type Key; +pub trait KeyValuePair { + type Key: PartialEq + Eq + std::hash::Hash; type Value; fn key(&self) -> Self::Key; fn value(&self) -> Self::Value; fn from_key_and_value(key: Self::Key, value: Self::Value) -> Self; + fn into_key_and_value(self) -> (Self::Key, Self::Value); } diff --git a/turbopack/crates/turbo-tasks/src/lib.rs b/turbopack/crates/turbo-tasks/src/lib.rs index f7e216df68c2b..558f37cf35d8f 100644 --- a/turbopack/crates/turbo-tasks/src/lib.rs +++ b/turbopack/crates/turbo-tasks/src/lib.rs @@ -49,7 +49,7 @@ mod id; mod id_factory; mod invalidation; mod join_iter_ext; -mod keyed; +mod key_value_pair; #[doc(hidden)] pub mod macro_helpers; mod magic_any; @@ -92,7 +92,7 @@ pub use invalidation::{ InvalidationReasonSet, Invalidator, }; pub use join_iter_ext::{JoinIterExt, TryFlatJoinIterExt, TryJoinIterExt}; -pub use keyed::Keyed; +pub use key_value_pair::KeyValuePair; pub use magic_any::MagicAny; pub use manager::{ dynamic_call, dynamic_this_call, emit, mark_dirty_when_persisted, mark_finished, mark_stateful, @@ -109,7 +109,7 @@ pub use serialization_invalidation::SerializationInvalidator; pub use state::{State, TransientState}; pub use task::{task_input::TaskInput, SharedReference}; pub use trait_ref::{IntoTraitRef, TraitRef}; -pub use turbo_tasks_macros::{function, value_impl, value_trait, with_key, TaskInput}; +pub use turbo_tasks_macros::{function, value_impl, value_trait, KeyValuePair, TaskInput}; pub use value::{TransientInstance, TransientValue, Value}; pub use value_type::{TraitMethod, TraitType, ValueType}; pub use vc::{ From c55395c00f696a4c01cbed66578765f8a8f3a590 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Fri, 2 Aug 2024 12:11:35 +0200 Subject: [PATCH 03/81] add more backend method implementations --- .../turbo-tasks-backend/src/backend/mod.rs | 31 ++++++++++++++----- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs index 9f495fbf27a21..0e27c367746b8 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs @@ -183,26 +183,40 @@ impl Backend for TurboTasksBackend { fn invalidate_task(&self, _: TaskId, _: &dyn TurboTasksBackendApi) { todo!() } - fn invalidate_tasks(&self, _: &[TaskId], _: &dyn TurboTasksBackendApi) { - todo!() + + fn invalidate_tasks(&self, tasks: &[TaskId], turbo_tasks: &dyn TurboTasksBackendApi) { + for task in tasks { + self.invalidate_task(*task, turbo_tasks); + } } + fn invalidate_tasks_set( &self, - _: &AutoSet, 2>, - _: &dyn TurboTasksBackendApi, + tasks: &AutoSet, 2>, + turbo_tasks: &dyn TurboTasksBackendApi, ) { - todo!() + for task in tasks { + self.invalidate_task(*task, turbo_tasks); + } } - fn get_task_description(&self, _: TaskId) -> std::string::String { - todo!() + + fn get_task_description(&self, task: TaskId) -> std::string::String { + let task_type = self + .task_cache + .lookup_reverse(&task) + .expect("Task not found"); + task_type.to_string() } + fn try_get_function_id(&self, _: TaskId) -> Option { todo!() } + type TaskState = (); fn new_task_state(&self, _task: TaskId) -> Self::TaskState { () } + fn try_start_task_execution( &self, _: TaskId, @@ -210,6 +224,7 @@ impl Backend for TurboTasksBackend { ) -> std::option::Option> { todo!() } + fn task_execution_result( &self, _: TaskId, @@ -218,6 +233,7 @@ impl Backend for TurboTasksBackend { ) { todo!() } + fn task_execution_completed( &self, _: TaskId, @@ -229,6 +245,7 @@ impl Backend for TurboTasksBackend { ) -> bool { todo!() } + fn run_backend_job( &self, _: BackendJobId, From 0409755852cdde51b29d6846213f68c2f01ff1cc Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Wed, 7 Aug 2024 11:59:40 +0200 Subject: [PATCH 04/81] very basic operation working --- Cargo.lock | 1 + .../crates/turbo-tasks-backend/Cargo.toml | 1 + .../src/backend/helpers/mod.rs | 1 + .../src/backend/helpers/schedule.rs | 1 + .../turbo-tasks-backend/src/backend/mod.rs | 320 +++++++++++++++--- .../src/backend/operation/connect_child.rs | 37 +- .../src/backend/operation/invalidate.rs | 93 +++++ .../src/backend/operation/mod.rs | 102 +++++- .../src/backend/operation/update_cell.rs | 40 +++ .../src/backend/operation/update_output.rs | 90 +++++ .../src/backend/storage.rs | 63 +++- .../crates/turbo-tasks-backend/src/data.rs | 38 ++- .../crates/turbo-tasks-backend/tests/basic.rs | 1 + .../turbo-tasks-backend/tests/test_config.trs | 3 + .../crates/turbo-tasks-memory/tests/basic.rs | 1 + .../crates/turbo-tasks-testing/tests/basic.rs | 31 ++ turbopack/crates/turbo-tasks/src/backend.rs | 81 +++++ 17 files changed, 804 insertions(+), 100 deletions(-) create mode 100644 turbopack/crates/turbo-tasks-backend/src/backend/helpers/mod.rs create mode 100644 turbopack/crates/turbo-tasks-backend/src/backend/helpers/schedule.rs create mode 100644 turbopack/crates/turbo-tasks-backend/src/backend/operation/invalidate.rs create mode 100644 turbopack/crates/turbo-tasks-backend/src/backend/operation/update_cell.rs create mode 100644 turbopack/crates/turbo-tasks-backend/src/backend/operation/update_output.rs create mode 120000 turbopack/crates/turbo-tasks-backend/tests/basic.rs create mode 100644 turbopack/crates/turbo-tasks-backend/tests/test_config.trs create mode 120000 turbopack/crates/turbo-tasks-memory/tests/basic.rs create mode 100644 turbopack/crates/turbo-tasks-testing/tests/basic.rs diff --git a/Cargo.lock b/Cargo.lock index ed033fc7f2de4..6fc9d0841197b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8588,6 +8588,7 @@ dependencies = [ "turbo-tasks-build", "turbo-tasks-hash", "turbo-tasks-malloc", + "turbo-tasks-testing", ] [[package]] diff --git a/turbopack/crates/turbo-tasks-backend/Cargo.toml b/turbopack/crates/turbo-tasks-backend/Cargo.toml index cd38a6bb31794..8feb087ae4f0f 100644 --- a/turbopack/crates/turbo-tasks-backend/Cargo.toml +++ b/turbopack/crates/turbo-tasks-backend/Cargo.toml @@ -28,6 +28,7 @@ turbo-prehash = { workspace = true } turbo-tasks = { workspace = true } turbo-tasks-hash = { workspace = true } turbo-tasks-malloc = { workspace = true, default-features = false } +turbo-tasks-testing = { workspace = true } [build-dependencies] turbo-tasks-build = { workspace = true } diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/helpers/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/helpers/mod.rs new file mode 100644 index 0000000000000..8b137891791fe --- /dev/null +++ b/turbopack/crates/turbo-tasks-backend/src/backend/helpers/mod.rs @@ -0,0 +1 @@ + diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/helpers/schedule.rs b/turbopack/crates/turbo-tasks-backend/src/backend/helpers/schedule.rs new file mode 100644 index 0000000000000..8b137891791fe --- /dev/null +++ b/turbopack/crates/turbo-tasks-backend/src/backend/helpers/schedule.rs @@ -0,0 +1 @@ + diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs index 0e27c367746b8..973d9df655b47 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs @@ -1,3 +1,4 @@ +mod helpers; mod operation; mod storage; @@ -16,25 +17,32 @@ use std::{ use anyhow::Result; use auto_hash_map::{AutoMap, AutoSet}; +use dashmap::DashMap; use parking_lot::{Condvar, Mutex}; use rustc_hash::FxHasher; +use smallvec::smallvec; use turbo_tasks::{ backend::{ Backend, BackendJobId, CachedTaskType, CellContent, TaskExecutionSpec, TransientTaskType, TypedCellContent, }, event::EventListener, + registry, util::IdFactoryWithReuse, CellId, FunctionId, RawVc, ReadConsistency, TaskId, TraitTypeId, TurboTasksBackendApi, ValueTypeId, TRANSIENT_TASK_BIT, }; use self::{ - operation::{AnyOperation, ExecuteContext, Operation}, + operation::{AnyOperation, ExecuteContext}, storage::Storage, }; use crate::{ - data::{CachedDataItem, CachedDataUpdate}, + data::{ + CachedDataItem, CachedDataItemKey, CachedDataItemValue, CachedDataUpdate, InProgressState, + OutputValue, + }, + get, remove, utils::{bi_map::BiMap, chunked_vec::ChunkedVec, ptr_eq_arc::PtrEqArc}, }; @@ -60,6 +68,7 @@ pub struct TurboTasksBackend { persisted_task_cache_log: Mutex, TaskId)>>, task_cache: BiMap, TaskId>, + transient_tasks: DashMap>>, persisted_storage_log: Mutex>, storage: Storage, @@ -91,6 +100,7 @@ impl TurboTasksBackend { ), persisted_task_cache_log: Mutex::new(ChunkedVec::new()), task_cache: BiMap::new(), + transient_tasks: DashMap::new(), persisted_storage_log: Mutex::new(ChunkedVec::new()), storage: Storage::new(), in_progress_operations: AtomicUsize::new(0), @@ -100,12 +110,15 @@ impl TurboTasksBackend { } } - fn run_operation( - &self, - operation: impl Operation, - turbo_tasks: &dyn TurboTasksBackendApi, - ) { - operation.execute(ExecuteContext::new(self, turbo_tasks)); + fn execute_context<'a>( + &'a self, + turbo_tasks: &'a dyn TurboTasksBackendApi, + ) -> ExecuteContext<'a> { + ExecuteContext::new(self, turbo_tasks) + } + + fn suspending_requested(&self) -> bool { + (self.in_progress_operations.load(Ordering::Relaxed) & SNAPSHOT_REQUESTED_BIT) != 0 } fn operation_suspend_point(&self, suspend: impl FnOnce() -> AnyOperation) { @@ -142,11 +155,31 @@ impl TurboTasksBackend { child_task: TaskId, turbo_tasks: &dyn TurboTasksBackendApi, ) { - self.run_operation( - operation::ConnectChildOperation::new(parent_task, child_task), - turbo_tasks, + operation::ConnectChildOperation::run( + parent_task, + child_task, + self.execute_context(turbo_tasks), + ); + } + + pub fn update_cell( + &self, + task_id: TaskId, + cell: CellId, + content: CellContent, + turbo_tasks: &dyn TurboTasksBackendApi, + ) { + operation::UpdateCellOperation::run( + task_id, + cell, + content, + self.execute_context(turbo_tasks), ); } + + pub fn invalidate(&self, task_id: TaskId, turbo_tasks: &dyn TurboTasksBackendApi) { + operation::InvalidateOperation::run(smallvec![task_id], self.execute_context(turbo_tasks)); + } } impl Backend for TurboTasksBackend { @@ -180,14 +213,15 @@ impl Backend for TurboTasksBackend { task_id } - fn invalidate_task(&self, _: TaskId, _: &dyn TurboTasksBackendApi) { - todo!() + fn invalidate_task(&self, task_id: TaskId, turbo_tasks: &dyn TurboTasksBackendApi) { + self.invalidate(task_id, turbo_tasks); } fn invalidate_tasks(&self, tasks: &[TaskId], turbo_tasks: &dyn TurboTasksBackendApi) { - for task in tasks { - self.invalidate_task(*task, turbo_tasks); - } + operation::InvalidateOperation::run( + tasks.iter().copied().collect(), + self.execute_context(turbo_tasks), + ); } fn invalidate_tasks_set( @@ -195,9 +229,10 @@ impl Backend for TurboTasksBackend { tasks: &AutoSet, 2>, turbo_tasks: &dyn TurboTasksBackendApi, ) { - for task in tasks { - self.invalidate_task(*task, turbo_tasks); - } + operation::InvalidateOperation::run( + tasks.iter().copied().collect(), + self.execute_context(turbo_tasks), + ); } fn get_task_description(&self, task: TaskId) -> std::string::String { @@ -219,31 +254,158 @@ impl Backend for TurboTasksBackend { fn try_start_task_execution( &self, - _: TaskId, - _: &dyn TurboTasksBackendApi, - ) -> std::option::Option> { - todo!() + task_id: TaskId, + turbo_tasks: &dyn TurboTasksBackendApi, + ) -> Option> { + { + let ctx = self.execute_context(turbo_tasks); + let mut task = ctx.task(task_id); + let Some(InProgressState::Scheduled { + clean, + done_event, + start_event, + }) = remove!(task, InProgress) + else { + return None; + }; + task.add(CachedDataItem::InProgress { + value: InProgressState::InProgress { + clean, + stale: false, + done_event, + }, + }); + start_event.notify(usize::MAX); + } + let (span, future) = if let Some(task_type) = self.task_cache.lookup_reverse(&task_id) { + match &*task_type { + CachedTaskType::Native { fn_type, this, arg } => ( + registry::get_function(*fn_type).span(), + registry::get_function(*fn_type).execute(*this, &**arg), + ), + CachedTaskType::ResolveNative { fn_type, .. } => { + let span = registry::get_function(*fn_type).resolve_span(); + let turbo_tasks = turbo_tasks.pin(); + ( + span, + Box::pin(async move { + let CachedTaskType::ResolveNative { fn_type, this, arg } = &*task_type + else { + unreachable!() + }; + CachedTaskType::run_resolve_native( + *fn_type, + *this, + &**arg, + task_id.persistence(), + turbo_tasks, + ) + .await + }) as Pin + Send + '_>>, + ) + } + CachedTaskType::ResolveTrait { + trait_type, + method_name, + .. + } => { + let span = registry::get_trait(*trait_type).resolve_span(&**method_name); + let turbo_tasks = turbo_tasks.pin(); + ( + span, + Box::pin(async move { + let CachedTaskType::ResolveTrait { + trait_type, + method_name, + this, + arg, + } = &*task_type + else { + unreachable!() + }; + CachedTaskType::run_resolve_trait( + *trait_type, + method_name.clone(), + *this, + &**arg, + task_id.persistence(), + turbo_tasks, + ) + .await + }) as Pin + Send + '_>>, + ) + } + } + } else if let Some(task_type) = self.transient_tasks.get(&task_id) { + let task_type = task_type.clone(); + let span = tracing::trace_span!("turbo_tasks::root_task"); + let future = Box::pin(async move { + let mut task_type = task_type.lock().await; + match &mut *task_type { + TransientTaskType::Root(f) => { + let future = f(); + drop(task_type); + future.await + } + TransientTaskType::Once(future) => future.await, + } + }) as Pin + Send + '_>>; + (span, future) + } else { + return None; + }; + Some(TaskExecutionSpec { future, span }) } fn task_execution_result( &self, - _: TaskId, - _: Result, std::option::Option>>, - _: &dyn TurboTasksBackendApi, + task_id: TaskId, + result: Result, Option>>, + turbo_tasks: &dyn TurboTasksBackendApi, ) { - todo!() + operation::UpdateOutputOperation::run(task_id, result, self.execute_context(turbo_tasks)); } fn task_execution_completed( &self, - _: TaskId, - _: Duration, - _: usize, - _: &AutoMap, 8>, - _: bool, - _: &dyn TurboTasksBackendApi, + task_id: TaskId, + duration: Duration, + memory_usage: usize, + cell_counters: &AutoMap, 8>, + stateful: bool, + turbo_tasks: &dyn TurboTasksBackendApi, ) -> bool { - todo!() + let ctx = self.execute_context(turbo_tasks); + let mut task = ctx.task(task_id); + let Some(CachedDataItemValue::InProgress { value: in_progress }) = + task.remove(&CachedDataItemKey::InProgress {}) + else { + panic!("Task execution completed, but task is not in progress"); + }; + let InProgressState::InProgress { + done_event, + clean, + stale, + } = in_progress + else { + panic!("Task execution completed, but task is not in progress"); + }; + + // TODO handle cell counters + + if stale { + task.add(CachedDataItem::InProgress { + value: InProgressState::InProgress { + clean: false, + stale: false, + done_event, + }, + }); + } else { + done_event.notify(usize::MAX); + } + + stale } fn run_backend_job( @@ -259,16 +421,45 @@ impl Backend for TurboTasksBackend { _: TaskId, _: ReadConsistency, _: &dyn TurboTasksBackendApi, - ) -> Result, turbo_tasks::Error> { + ) -> Result> { todo!() } fn try_read_task_output_untracked( &self, - _: TaskId, - _: ReadConsistency, - _: &dyn TurboTasksBackendApi, - ) -> Result, turbo_tasks::Error> { - todo!() + task_id: TaskId, + consistency: ReadConsistency, + turbo_tasks: &dyn TurboTasksBackendApi, + ) -> Result> { + let ctx = self.execute_context(turbo_tasks); + let task = ctx.task(task_id); + + if let Some(in_progress) = get!(task, InProgress) { + match in_progress { + InProgressState::Scheduled { done_event, .. } + | InProgressState::InProgress { done_event, .. } => { + let listener = done_event.listen(); + return Ok(Err(listener)); + } + } + } + + if matches!(consistency, ReadConsistency::Strong) { + todo!("Handle strongly consistent read"); + } + + if let Some(output) = get!(task, Output) { + match output { + OutputValue::Cell(cell) => return Ok(Ok(RawVc::TaskCell(cell.task, cell.cell))), + OutputValue::Output(task) => return Ok(Ok(RawVc::TaskOutput(*task))), + OutputValue::Error | OutputValue::Panic => { + if let Some(error) = get!(task, Error) { + return Err(error.clone().into()); + } + } + } + } + + todo!("Output is not available, recompute task"); } fn try_read_task_cell( &self, @@ -276,17 +467,26 @@ impl Backend for TurboTasksBackend { _: CellId, _: TaskId, _: &dyn TurboTasksBackendApi, - ) -> Result, turbo_tasks::Error> { + ) -> Result> { todo!() } + fn try_read_task_cell_untracked( &self, - _: TaskId, - _: CellId, - _: &dyn TurboTasksBackendApi, - ) -> Result, turbo_tasks::Error> { - todo!() + task_id: TaskId, + cell: CellId, + turbo_tasks: &dyn TurboTasksBackendApi, + ) -> Result> { + let ctx = self.execute_context(turbo_tasks); + let task = ctx.task(task_id); + if let Some(content) = get!(task, CellData { cell }) { + return Ok(Ok( + CellContent(Some(content.clone())).into_typed(cell.type_id) + )); + } + todo!("Cell is not available, recompute task or error"); } + fn read_task_collectibles( &self, _: TaskId, @@ -296,6 +496,7 @@ impl Backend for TurboTasksBackend { ) -> AutoMap, 1> { todo!() } + fn emit_collectible( &self, _: TraitTypeId, @@ -305,6 +506,7 @@ impl Backend for TurboTasksBackend { ) { todo!() } + fn unemit_collectible( &self, _: TraitTypeId, @@ -315,15 +517,17 @@ impl Backend for TurboTasksBackend { ) { todo!() } + fn update_task_cell( &self, - _: TaskId, - _: CellId, - _: CellContent, - _: &dyn TurboTasksBackendApi, + task_id: TaskId, + cell: CellId, + content: CellContent, + turbo_tasks: &dyn TurboTasksBackendApi, ) { - todo!() + self.update_cell(task_id, cell, content, turbo_tasks); } + fn get_or_create_transient_task( &self, _: CachedTaskType, @@ -337,10 +541,18 @@ impl Backend for TurboTasksBackend { } fn create_transient_task( &self, - _: TransientTaskType, - _: &dyn TurboTasksBackendApi, + task_type: TransientTaskType, + turbo_tasks: &dyn TurboTasksBackendApi, ) -> TaskId { - todo!() + let task_id = self.transient_task_id_factory.get(); + self.transient_tasks + .insert(task_id, Arc::new(tokio::sync::Mutex::new(task_type))); + { + let mut task = self.storage.access_mut(task_id); + task.add(CachedDataItem::new_scheduled(task_id)); + } + turbo_tasks.schedule(task_id); + task_id } fn dispose_root_task(&self, _: TaskId, _: &dyn TurboTasksBackendApi) { todo!() diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs index 11f5f5c7ed4a6..636f5edb05f2d 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs @@ -2,24 +2,25 @@ use serde::{Deserialize, Serialize}; use turbo_tasks::TaskId; use super::{ExecuteContext, Operation}; -use crate::{data::CachedDataItem, suspend_point}; +use crate::data::CachedDataItem; #[derive(Serialize, Deserialize, Clone, Default)] pub enum ConnectChildOperation { - AddChild { - parent_task: TaskId, - child_task: TaskId, - }, + Todo, #[default] Done, // TODO Add aggregated edge } impl ConnectChildOperation { - pub fn new(parent_task: TaskId, child_task: TaskId) -> Self { - ConnectChildOperation::AddChild { - parent_task, - child_task, + pub fn run(parent_task: TaskId, child_task: TaskId, ctx: ExecuteContext<'_>) { + let mut parent_task = ctx.task(parent_task); + if parent_task.add(CachedDataItem::Child { + task: child_task, + value: (), + }) { + // TODO add aggregated edge + ConnectChildOperation::Todo.execute(ctx); } } } @@ -27,21 +28,11 @@ impl ConnectChildOperation { impl Operation for ConnectChildOperation { fn execute(mut self, ctx: ExecuteContext<'_>) { loop { + ctx.operation_suspend_point(&self); match self { - ConnectChildOperation::AddChild { - parent_task, - child_task, - } => { - let mut parent_task = ctx.task(parent_task); - if parent_task.add(CachedDataItem::Child { - task: child_task, - value: (), - }) { - // TODO add aggregated edge - suspend_point!(self, ctx, ConnectChildOperation::Done); - } else { - suspend_point!(self, ctx, ConnectChildOperation::Done); - } + ConnectChildOperation::Todo => { + self = ConnectChildOperation::Done; + continue; } ConnectChildOperation::Done => { return; diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/invalidate.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/invalidate.rs new file mode 100644 index 0000000000000..1dbfd76424fc6 --- /dev/null +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/invalidate.rs @@ -0,0 +1,93 @@ +use serde::{Deserialize, Serialize}; +use smallvec::SmallVec; +use turbo_tasks::TaskId; + +use super::{ExecuteContext, Operation}; +use crate::{ + data::{CachedDataItem, InProgressState}, + get, remove, +}; + +#[derive(Serialize, Deserialize, Clone, Default)] +pub enum InvalidateOperation { + // TODO DetermineActiveness + MakeDirty { + task_ids: SmallVec<[TaskId; 4]>, + }, + // TODO Add to dirty tasks list + #[default] + Done, +} + +impl InvalidateOperation { + pub fn run(task_ids: SmallVec<[TaskId; 4]>, ctx: ExecuteContext<'_>) { + InvalidateOperation::MakeDirty { task_ids }.execute(ctx) + } +} + +impl Operation for InvalidateOperation { + fn execute(self, ctx: ExecuteContext<'_>) { + loop { + ctx.operation_suspend_point(&self); + match self { + InvalidateOperation::MakeDirty { task_ids } => { + for task_id in task_ids { + let mut task = ctx.task(task_id); + let in_progress = match get!(task, InProgress) { + Some(InProgressState::Scheduled { clean, .. }) => { + if *clean { + let Some(InProgressState::Scheduled { + clean: _, + done_event, + start_event, + }) = remove!(task, InProgress) + else { + unreachable!(); + }; + task.insert(CachedDataItem::InProgress { + value: InProgressState::Scheduled { + clean: false, + done_event, + start_event, + }, + }); + } + true + } + Some(InProgressState::InProgress { clean, stale, .. }) => { + if *clean || !*stale { + let Some(InProgressState::InProgress { + clean: _, + stale: _, + done_event, + }) = remove!(task, InProgress) + else { + unreachable!(); + }; + task.insert(CachedDataItem::InProgress { + value: InProgressState::InProgress { + clean: false, + stale: true, + done_event, + }, + }); + } + true + } + None => false, + }; + if task.add(CachedDataItem::Dirty { value: () }) { + if !in_progress && task.add(CachedDataItem::new_scheduled(task_id)) { + ctx.turbo_tasks.schedule(task_id) + } + } + } + return; + } + InvalidateOperation::Done => { + return; + } + } + } + } +} diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs index 232ea7a493380..13f2ee7e7a351 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs @@ -1,4 +1,9 @@ mod connect_child; +mod invalidate; +mod update_cell; +mod update_output; + +use std::mem::take; use serde::{Deserialize, Serialize}; use turbo_tasks::{KeyValuePair, TaskId, TurboTasksBackendApi}; @@ -6,13 +11,21 @@ use turbo_tasks::{KeyValuePair, TaskId, TurboTasksBackendApi}; use super::{storage::StorageWriteGuard, TurboTasksBackend}; use crate::data::{CachedDataItem, CachedDataItemKey, CachedDataItemValue, CachedDataUpdate}; -pub trait Operation: Serialize + for<'de> Deserialize<'de> { +pub trait Operation: + Serialize + + for<'de> Deserialize<'de> + + Default + + TryFrom + + Into +{ fn execute(self, ctx: ExecuteContext<'_>); } +#[derive(Clone, Copy)] pub struct ExecuteContext<'a> { backend: &'a TurboTasksBackend, turbo_tasks: &'a dyn TurboTasksBackendApi, + parent: Option<(&'a AnyOperation, &'a ExecuteContext<'a>)>, } impl<'a> ExecuteContext<'a> { @@ -23,6 +36,7 @@ impl<'a> ExecuteContext<'a> { Self { backend, turbo_tasks, + parent: None, } } @@ -34,8 +48,41 @@ impl<'a> ExecuteContext<'a> { } } - pub fn operation_suspend_point(&self, f: impl FnOnce() -> AnyOperation) { - self.backend.operation_suspend_point(f) + pub fn operation_suspend_point>(&self, op: &T) { + if self.parent.is_some() { + self.backend.operation_suspend_point(|| { + let mut nested = Vec::new(); + nested.push(op.clone().into()); + let mut cur = self; + while let Some((op, parent_ctx)) = cur.parent { + nested.push(op.clone()); + cur = parent_ctx; + } + AnyOperation::Nested(nested) + }); + } else { + self.backend.operation_suspend_point(|| op.clone().into()); + } + } + + pub fn suspending_requested(&self) -> bool { + self.backend.suspending_requested() + } + + pub fn run_operation( + &self, + parent_op_ref: &mut impl Operation, + run: impl FnOnce(ExecuteContext<'_>), + ) { + let parent_op = take(parent_op_ref); + let parent_op: AnyOperation = parent_op.into(); + let inner_ctx = ExecuteContext { + backend: self.backend, + turbo_tasks: self.turbo_tasks, + parent: Some((&parent_op, self)), + }; + run(inner_ctx); + *parent_op_ref = parent_op.try_into().unwrap(); } } @@ -79,13 +126,13 @@ impl<'a> TaskGuard<'a> { } } - pub fn upsert(&mut self, item: CachedDataItem) -> Option { + pub fn insert(&mut self, item: CachedDataItem) -> Option { let (key, value) = item.into_key_and_value(); if !key.is_persistent() { self.task - .upsert(CachedDataItem::from_key_and_value(key, value)) + .insert(CachedDataItem::from_key_and_value(key, value)) } else if value.is_persistent() { - let old = self.task.upsert(CachedDataItem::from_key_and_value( + let old = self.task.insert(CachedDataItem::from_key_and_value( key.clone(), value.clone(), )); @@ -100,7 +147,7 @@ impl<'a> TaskGuard<'a> { old } else { let item = CachedDataItem::from_key_and_value(key.clone(), value); - if let Some(old) = self.task.upsert(item) { + if let Some(old) = self.task.insert(item) { if old.is_persistent() { self.backend .persisted_storage_log @@ -137,6 +184,14 @@ impl<'a> TaskGuard<'a> { None } } + + pub fn get(&self, key: &CachedDataItemKey) -> Option<&CachedDataItemValue> { + self.task.get(key) + } + + pub fn iter(&self) -> impl Iterator { + self.task.iter() + } } macro_rules! impl_operation { @@ -147,29 +202,44 @@ macro_rules! impl_operation { } } + impl TryFrom for $type_path { + type Error = (); + + fn try_from(op: AnyOperation) -> Result { + match op { + AnyOperation::$name(op) => Ok(op), + _ => Err(()), + } + } + } + pub use $type_path; }; } -#[derive(Serialize, Deserialize)] +#[derive(Serialize, Deserialize, Clone)] pub enum AnyOperation { ConnectChild(connect_child::ConnectChildOperation), + Invalidate(invalidate::InvalidateOperation), + Nested(Vec), } -impl Operation for AnyOperation { +impl AnyOperation { fn execute(self, ctx: ExecuteContext<'_>) { match self { AnyOperation::ConnectChild(op) => op.execute(ctx), + AnyOperation::Invalidate(op) => op.execute(ctx), + AnyOperation::Nested(ops) => { + for op in ops { + op.execute(ctx); + } + } } } } impl_operation!(ConnectChild connect_child::ConnectChildOperation); +impl_operation!(Invalidate invalidate::InvalidateOperation); -#[macro_export(local_inner_macros)] -macro_rules! suspend_point { - ($self:expr, $ctx:expr, $op:expr) => { - $self = $op; - $ctx.operation_suspend_point(|| $self.clone().into()); - }; -} +pub use update_cell::UpdateCellOperation; +pub use update_output::UpdateOutputOperation; diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/update_cell.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/update_cell.rs new file mode 100644 index 0000000000000..664f3fdc3cfbd --- /dev/null +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/update_cell.rs @@ -0,0 +1,40 @@ +use turbo_tasks::{backend::CellContent, CellId, TaskId}; + +use super::{ExecuteContext, InvalidateOperation}; +use crate::data::{CachedDataItem, CachedDataItemKey}; + +pub struct UpdateCellOperation; + +impl UpdateCellOperation { + pub fn run(task: TaskId, cell: CellId, content: CellContent, ctx: ExecuteContext<'_>) { + let mut task = ctx.task(task); + let old_content = if let CellContent(Some(new_content)) = content { + task.insert(CachedDataItem::CellData { + cell, + value: new_content, + }) + } else { + task.remove(&CachedDataItemKey::CellData { cell }) + }; + + let dependent = task + .iter() + .filter_map(|(key, _)| { + if let CachedDataItemKey::CellDependent { + cell: dependent_cell, + task, + } = *key + { + (dependent_cell == cell).then_some(task) + } else { + None + } + }) + .collect(); + + drop(task); + drop(old_content); + + InvalidateOperation::run(dependent, ctx); + } +} diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/update_output.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/update_output.rs new file mode 100644 index 0000000000000..ac3f7fba5514d --- /dev/null +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/update_output.rs @@ -0,0 +1,90 @@ +use std::borrow::Cow; + +use anyhow::{anyhow, Result}; +use turbo_tasks::{util::SharedError, RawVc, TaskId}; + +use super::{ExecuteContext, InvalidateOperation}; +use crate::data::{CachedDataItem, CachedDataItemKey, CachedDataItemValue, CellRef, OutputValue}; + +pub struct UpdateOutputOperation; + +impl UpdateOutputOperation { + pub fn run( + task_id: TaskId, + output: Result, Option>>, + ctx: ExecuteContext<'_>, + ) { + let mut task = ctx.task(task_id); + let old_error = task.remove(&CachedDataItemKey::Error {}); + let current_output = task.get(&CachedDataItemKey::Output {}); + let output_value = match output { + Ok(Ok(RawVc::TaskOutput(output_task_id))) => { + if let Some(CachedDataItemValue::Output { + value: OutputValue::Output(current_task_id), + }) = current_output + { + if *current_task_id == output_task_id { + return; + } + } + OutputValue::Output(output_task_id) + } + Ok(Ok(RawVc::TaskCell(output_task_id, cell))) => { + if let Some(CachedDataItemValue::Output { + value: + OutputValue::Cell(CellRef { + task: current_task_id, + cell: current_cell, + }), + }) = current_output + { + if *current_task_id == output_task_id && *current_cell == cell { + return; + } + } + OutputValue::Cell(CellRef { + task: output_task_id, + cell, + }) + } + Ok(Ok(RawVc::LocalCell(_, _))) => { + panic!("LocalCell must not be output of a task"); + } + Ok(Ok(RawVc::LocalOutput(_, _))) => { + panic!("LocalOutput must not be output of a task"); + } + Ok(Err(err)) => { + task.insert(CachedDataItem::Error { + value: SharedError::new(err), + }); + OutputValue::Error + } + Err(panic) => { + task.insert(CachedDataItem::Error { + value: SharedError::new(anyhow!("Panic: {:?}", panic)), + }); + OutputValue::Panic + } + }; + let old_content = task.insert(CachedDataItem::Output { + value: output_value, + }); + + let dependent = task + .iter() + .filter_map(|(key, _)| { + if let CachedDataItemKey::OutputDependent { task } = *key { + Some(task) + } else { + None + } + }) + .collect(); + + drop(task); + drop(old_content); + drop(old_error); + + InvalidateOperation::run(dependent, ctx); + } +} diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs b/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs index d06d9dc0cb839..c8f9702585096 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs @@ -1,5 +1,6 @@ use std::{ hash::{BuildHasherDefault, Hash}, + mem::take, ops::{Deref, DerefMut}, thread::available_parallelism, }; @@ -44,7 +45,7 @@ impl InnerStorage { } } - pub fn upsert(&mut self, item: T) -> Option { + pub fn insert(&mut self, item: T) -> Option { let (key, value) = item.into_key_and_value(); self.map.insert(key, value) } @@ -52,6 +53,30 @@ impl InnerStorage { pub fn remove(&mut self, key: &T::Key) -> Option { self.map.remove(key) } + + pub fn get(&self, key: &T::Key) -> Option<&T::Value> { + self.map.get(key) + } + + pub fn iter(&self) -> impl Iterator { + self.map.iter() + } +} + +impl InnerStorage +where + T::Value: PartialEq, +{ + pub fn has(&self, item: &mut T) -> bool { + let (key, value) = take(item).into_key_and_value(); + let result = if let Some(stored_value) = self.map.get(&key) { + *stored_value == value + } else { + false + }; + *item = T::from_key_and_value(key, value); + result + } } pub struct Storage { @@ -99,3 +124,39 @@ impl<'a, K: Eq + Hash, T: KeyValuePair> DerefMut for StorageWriteGuard<'a, K, T> &mut self.inner } } + +#[macro_export] +macro_rules! get { + ($task:ident, $key:ident $input:tt) => { + if let Some($crate::data::CachedDataItemValue::$key { value }) = $task.get(&$crate::data::CachedDataItemKey::$key $input).as_ref() { + Some(value) + } else { + None + } + }; + ($task:ident, $key:ident) => { + if let Some($crate::data::CachedDataItemValue::$key { value }) = $task.get(&$crate::data::CachedDataItemKey::$key {}).as_ref() { + Some(value) + } else { + None + } + }; +} + +#[macro_export] +macro_rules! remove { + ($task:ident, $key:ident $input:tt) => { + if let Some($crate::data::CachedDataItemValue::$key { value }) = $task.remove(&$crate::data::CachedDataItemKey::$key $input) { + Some(value) + } else { + None + } + }; + ($task:ident, $key:ident) => { + if let Some($crate::data::CachedDataItemValue::$key { value }) = $task.remove(&$crate::data::CachedDataItemKey::$key {}) { + Some(value) + } else { + None + } + }; +} diff --git a/turbopack/crates/turbo-tasks-backend/src/data.rs b/turbopack/crates/turbo-tasks-backend/src/data.rs index 76e7df4f9e64e..c217ae5d58c9e 100644 --- a/turbopack/crates/turbo-tasks-backend/src/data.rs +++ b/turbopack/crates/turbo-tasks-backend/src/data.rs @@ -1,9 +1,9 @@ -use turbo_tasks::{util::SharedError, CellId, KeyValuePair, SharedReference, TaskId}; +use turbo_tasks::{event::Event, util::SharedError, CellId, KeyValuePair, SharedReference, TaskId}; #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] pub struct CellRef { - task: TaskId, - cell: CellId, + pub task: TaskId, + pub cell: CellId, } #[derive(Debug, Copy, Clone, PartialEq, Eq)] @@ -11,6 +11,7 @@ pub enum OutputValue { Cell(CellRef), Output(TaskId), Error, + Panic, } impl OutputValue { fn is_transient(&self) -> bool { @@ -18,6 +19,7 @@ impl OutputValue { OutputValue::Cell(cell) => cell.task.is_transient(), OutputValue::Output(task) => task.is_transient(), OutputValue::Error => false, + OutputValue::Panic => false, } } } @@ -29,10 +31,24 @@ pub enum RootType { ReadingStronglyConsistent, } -#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[derive(Debug)] pub enum InProgressState { - Scheduled { clean: bool }, - InProgress { clean: bool, stale: bool }, + Scheduled { + clean: bool, + done_event: Event, + start_event: Event, + }, + InProgress { + clean: bool, + stale: bool, + done_event: Event, + }, +} + +impl Clone for InProgressState { + fn clone(&self) -> Self { + panic!("InProgressState cannot be cloned"); + } } #[derive(Debug, Clone, KeyValuePair)] @@ -170,6 +186,16 @@ impl CachedDataItem { CachedDataItem::Error { .. } => false, } } + + pub fn new_scheduled(task_id: TaskId) -> Self { + CachedDataItem::InProgress { + value: InProgressState::Scheduled { + clean: false, + done_event: Event::new(move || format!("{} done_event", task_id)), + start_event: Event::new(move || format!("{} start_event", task_id)), + }, + } + } } impl CachedDataItemKey { diff --git a/turbopack/crates/turbo-tasks-backend/tests/basic.rs b/turbopack/crates/turbo-tasks-backend/tests/basic.rs new file mode 120000 index 0000000000000..d2c98272f0102 --- /dev/null +++ b/turbopack/crates/turbo-tasks-backend/tests/basic.rs @@ -0,0 +1 @@ +../../turbo-tasks-testing/tests/basic.rs \ No newline at end of file diff --git a/turbopack/crates/turbo-tasks-backend/tests/test_config.trs b/turbopack/crates/turbo-tasks-backend/tests/test_config.trs new file mode 100644 index 0000000000000..7387c44aaf3dd --- /dev/null +++ b/turbopack/crates/turbo-tasks-backend/tests/test_config.trs @@ -0,0 +1,3 @@ +|_name, _initial| { + turbo_tasks::TurboTasks::new(turbo_tasks_backend::TurboTasksBackend::new()) +} diff --git a/turbopack/crates/turbo-tasks-memory/tests/basic.rs b/turbopack/crates/turbo-tasks-memory/tests/basic.rs new file mode 120000 index 0000000000000..d2c98272f0102 --- /dev/null +++ b/turbopack/crates/turbo-tasks-memory/tests/basic.rs @@ -0,0 +1 @@ +../../turbo-tasks-testing/tests/basic.rs \ No newline at end of file diff --git a/turbopack/crates/turbo-tasks-testing/tests/basic.rs b/turbopack/crates/turbo-tasks-testing/tests/basic.rs new file mode 100644 index 0000000000000..04c949db97132 --- /dev/null +++ b/turbopack/crates/turbo-tasks-testing/tests/basic.rs @@ -0,0 +1,31 @@ +#![feature(arbitrary_self_types)] + +use anyhow::Result; +use turbo_tasks::Vc; +use turbo_tasks_testing::{register, run, Registration}; + +static REGISTRATION: Registration = register!(); + +#[tokio::test] +async fn basic() { + run(®ISTRATION, || async { + // let input = Value { value: 42 }.cell(); + // let output = func(input); + // assert_eq!(output.await?.value, 42); + + anyhow::Ok(()) + }) + .await + .unwrap() +} + +#[turbo_tasks::value] +struct Value { + value: u32, +} + +#[turbo_tasks::function] +async fn func(input: Vc) -> Result> { + let value = input.await?.value; + Ok(Value { value }.cell()) +} diff --git a/turbopack/crates/turbo-tasks/src/backend.rs b/turbopack/crates/turbo-tasks/src/backend.rs index d87ebe638048a..9f7ca0dc41540 100644 --- a/turbopack/crates/turbo-tasks/src/backend.rs +++ b/turbopack/crates/turbo-tasks/src/backend.rs @@ -92,13 +92,94 @@ impl Display for CachedTaskType { } mod ser { + use std::any::Any; + use serde::{ + de::{self}, ser::{SerializeSeq, SerializeTuple}, Deserialize, Deserializer, Serialize, Serializer, }; use super::*; + impl Serialize for TypedCellContent { + fn serialize(&self, serializer: S) -> std::result::Result + where + S: Serializer, + { + let value_type = registry::get_value_type(self.0); + let serializable = if let Some(value) = &self.1 .0 { + value_type.any_as_serializable(&value.0) + } else { + None + }; + let mut state = serializer.serialize_tuple(3)?; + state.serialize_element(registry::get_value_type_global_name(self.0))?; + if let Some(serializable) = serializable { + state.serialize_element(&true)?; + state.serialize_element(serializable)?; + } else { + state.serialize_element(&false)?; + state.serialize_element(&())?; + } + state.end() + } + } + + impl<'de> Deserialize<'de> for TypedCellContent { + fn deserialize(deserializer: D) -> std::result::Result + where + D: Deserializer<'de>, + { + struct Visitor; + + impl<'de> serde::de::Visitor<'de> for Visitor { + type Value = TypedCellContent; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + write!(formatter, "a valid TypedCellContent") + } + + fn visit_seq(self, mut seq: A) -> std::result::Result + where + A: de::SeqAccess<'de>, + { + let value_type = seq + .next_element()? + .ok_or_else(|| de::Error::invalid_length(0, &self))?; + let value_type = registry::get_value_type_id_by_global_name(value_type) + .ok_or_else(|| de::Error::custom("Unknown value type"))?; + let has_value: bool = seq + .next_element()? + .ok_or_else(|| de::Error::invalid_length(1, &self))?; + if has_value { + let seed = registry::get_value_type(value_type) + .get_any_deserialize_seed() + .ok_or_else(|| { + de::Error::custom("Value type doesn't support deserialization") + })?; + let value = seq + .next_element_seed(seed)? + .ok_or_else(|| de::Error::invalid_length(2, &self))?; + let arc: triomphe::Arc = + todo!("convert Box to Arc"); + Ok(TypedCellContent( + value_type, + CellContent(Some(SharedReference(arc))), + )) + } else { + let () = seq + .next_element()? + .ok_or_else(|| de::Error::invalid_length(2, &self))?; + Ok(TypedCellContent(value_type, CellContent(None))) + } + } + } + + deserializer.deserialize_tuple(2, Visitor) + } + } + enum FunctionAndArg<'a> { Owned { fn_type: FunctionId, From f6002fcd2573a320efd3386b11998a6b837775c9 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Wed, 7 Aug 2024 13:19:38 +0200 Subject: [PATCH 05/81] add own cells reading --- .../turbo-tasks-backend/src/backend/mod.rs | 19 +++++++++++++++++-- .../src/backend/operation/invalidate.rs | 1 + .../src/backend/operation/mod.rs | 1 + .../crates/turbo-tasks-testing/tests/basic.rs | 2 +- 4 files changed, 20 insertions(+), 3 deletions(-) diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs index 973d9df655b47..f018be74c4b18 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs @@ -309,7 +309,7 @@ impl Backend for TurboTasksBackend { method_name, .. } => { - let span = registry::get_trait(*trait_type).resolve_span(&**method_name); + let span = registry::get_trait(*trait_type).resolve_span(method_name); let turbo_tasks = turbo_tasks.pin(); ( span, @@ -484,7 +484,22 @@ impl Backend for TurboTasksBackend { CellContent(Some(content.clone())).into_typed(cell.type_id) )); } - todo!("Cell is not available, recompute task or error"); + todo!("Cell {cell:?} from {task_id:?} is not available, recompute task or error"); + } + + fn try_read_own_task_cell_untracked( + &self, + task_id: TaskId, + cell: CellId, + turbo_tasks: &dyn TurboTasksBackendApi, + ) -> Result { + let ctx = self.execute_context(turbo_tasks); + let task = ctx.task(task_id); + if let Some(content) = get!(task, CellData { cell }) { + Ok(CellContent(Some(content.clone())).into_typed(cell.type_id)) + } else { + Ok(CellContent(None).into_typed(cell.type_id)) + } } fn read_task_collectibles( diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/invalidate.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/invalidate.rs index 1dbfd76424fc6..8e307bc3fc3ef 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/invalidate.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/invalidate.rs @@ -76,6 +76,7 @@ impl Operation for InvalidateOperation { } None => false, }; + #[allow(clippy::collapsible_if)] if task.add(CachedDataItem::Dirty { value: () }) { if !in_progress && task.add(CachedDataItem::new_scheduled(task_id)) { ctx.turbo_tasks.schedule(task_id) diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs index 13f2ee7e7a351..b2b632578a796 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs @@ -109,6 +109,7 @@ impl<'a> TaskGuard<'a> { if !item.is_persistent() { self.task.add(item) } else { + #[allow(clippy::collapsible_if)] if self.task.add(item.clone()) { let (key, value) = item.into_key_and_value(); self.backend diff --git a/turbopack/crates/turbo-tasks-testing/tests/basic.rs b/turbopack/crates/turbo-tasks-testing/tests/basic.rs index 04c949db97132..16ee41db2ae0b 100644 --- a/turbopack/crates/turbo-tasks-testing/tests/basic.rs +++ b/turbopack/crates/turbo-tasks-testing/tests/basic.rs @@ -9,7 +9,7 @@ static REGISTRATION: Registration = register!(); #[tokio::test] async fn basic() { run(®ISTRATION, || async { - // let input = Value { value: 42 }.cell(); + let input = Value { value: 42 }.cell(); // let output = func(input); // assert_eq!(output.await?.value, 42); From 06be6c6b2851445b183095b7953d29a58db72879 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Wed, 7 Aug 2024 13:23:59 +0200 Subject: [PATCH 06/81] call transient tasks --- .../turbo-tasks-backend/src/backend/mod.rs | 35 ++++++++++++++----- .../crates/turbo-tasks-testing/tests/basic.rs | 2 +- 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs index f018be74c4b18..71c994329a7b9 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs @@ -213,6 +213,33 @@ impl Backend for TurboTasksBackend { task_id } + fn get_or_create_transient_task( + &self, + task_type: CachedTaskType, + parent_task: TaskId, + turbo_tasks: &dyn TurboTasksBackendApi, + ) -> TaskId { + if let Some(task_id) = self.task_cache.lookup_forward(&task_type) { + self.connect_child(parent_task, task_id, turbo_tasks); + return task_id; + } + + let task_type = Arc::new(task_type); + let task_id = self.transient_task_id_factory.get(); + if let Err(existing_task_id) = self.task_cache.try_insert(task_type, task_id) { + // Safety: We just created the id and failed to insert it. + unsafe { + self.transient_task_id_factory.reuse(task_id); + } + self.connect_child(parent_task, existing_task_id, turbo_tasks); + return existing_task_id; + } + + self.connect_child(parent_task, task_id, turbo_tasks); + + task_id + } + fn invalidate_task(&self, task_id: TaskId, turbo_tasks: &dyn TurboTasksBackendApi) { self.invalidate(task_id, turbo_tasks); } @@ -543,14 +570,6 @@ impl Backend for TurboTasksBackend { self.update_cell(task_id, cell, content, turbo_tasks); } - fn get_or_create_transient_task( - &self, - _: CachedTaskType, - _: TaskId, - _: &dyn TurboTasksBackendApi, - ) -> TaskId { - todo!() - } fn connect_task(&self, _: TaskId, _: TaskId, _: &dyn TurboTasksBackendApi) { todo!() } diff --git a/turbopack/crates/turbo-tasks-testing/tests/basic.rs b/turbopack/crates/turbo-tasks-testing/tests/basic.rs index 16ee41db2ae0b..b8312b9b2aac0 100644 --- a/turbopack/crates/turbo-tasks-testing/tests/basic.rs +++ b/turbopack/crates/turbo-tasks-testing/tests/basic.rs @@ -10,7 +10,7 @@ static REGISTRATION: Registration = register!(); async fn basic() { run(®ISTRATION, || async { let input = Value { value: 42 }.cell(); - // let output = func(input); + let output = func(input); // assert_eq!(output.await?.value, 42); anyhow::Ok(()) From 26875176ce97b7f18e4289170b40a98b135d9648 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Wed, 7 Aug 2024 13:25:29 +0200 Subject: [PATCH 07/81] call persistent tasks --- turbopack/crates/turbo-tasks-testing/tests/basic.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/turbopack/crates/turbo-tasks-testing/tests/basic.rs b/turbopack/crates/turbo-tasks-testing/tests/basic.rs index b8312b9b2aac0..491b4bbc10ff7 100644 --- a/turbopack/crates/turbo-tasks-testing/tests/basic.rs +++ b/turbopack/crates/turbo-tasks-testing/tests/basic.rs @@ -9,9 +9,12 @@ static REGISTRATION: Registration = register!(); #[tokio::test] async fn basic() { run(®ISTRATION, || async { + let output1 = func_without_args(); + // assert_eq!(output1.await?.value, 123); + let input = Value { value: 42 }.cell(); - let output = func(input); - // assert_eq!(output.await?.value, 42); + let output2 = func(input); + // assert_eq!(output2.await?.value, 42); anyhow::Ok(()) }) @@ -29,3 +32,9 @@ async fn func(input: Vc) -> Result> { let value = input.await?.value; Ok(Value { value }.cell()) } + +#[turbo_tasks::function] +async fn func_without_args() -> Result> { + let value = 123; + Ok(Value { value }.cell()) +} From c0a3dae1a6e98f68bbbacf149d7785cfeef7a7ff Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Wed, 7 Aug 2024 19:23:48 +0200 Subject: [PATCH 08/81] run tests multiple times to test caching --- turbopack/crates/turbo-tasks-testing/tests/basic.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/turbopack/crates/turbo-tasks-testing/tests/basic.rs b/turbopack/crates/turbo-tasks-testing/tests/basic.rs index 491b4bbc10ff7..84a56237e3193 100644 --- a/turbopack/crates/turbo-tasks-testing/tests/basic.rs +++ b/turbopack/crates/turbo-tasks-testing/tests/basic.rs @@ -10,11 +10,11 @@ static REGISTRATION: Registration = register!(); async fn basic() { run(®ISTRATION, || async { let output1 = func_without_args(); - // assert_eq!(output1.await?.value, 123); + assert_eq!(output1.await?.value, 123); let input = Value { value: 42 }.cell(); let output2 = func(input); - // assert_eq!(output2.await?.value, 42); + assert_eq!(output2.await?.value, 42); anyhow::Ok(()) }) From 814e4e652ff65938cfa9c46b2b788bc8b00b441b Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Wed, 7 Aug 2024 19:25:22 +0200 Subject: [PATCH 09/81] add cell/output reading and creating cached tasks --- .../turbo-tasks-backend/src/backend/mod.rs | 223 ++++++++++++------ .../src/backend/operation/connect_child.rs | 23 +- .../src/backend/operation/invalidate.rs | 4 +- .../src/backend/operation/mod.rs | 37 ++- 4 files changed, 204 insertions(+), 83 deletions(-) diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs index 71c994329a7b9..64624dd88a17c 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs @@ -39,8 +39,8 @@ use self::{ }; use crate::{ data::{ - CachedDataItem, CachedDataItemKey, CachedDataItemValue, CachedDataUpdate, InProgressState, - OutputValue, + CachedDataItem, CachedDataItemKey, CachedDataItemValue, CachedDataUpdate, CellRef, + InProgressState, OutputValue, }, get, remove, utils::{bi_map::BiMap, chunked_vec::ChunkedVec, ptr_eq_arc::PtrEqArc}, @@ -129,7 +129,7 @@ impl TurboTasksBackend { snapshot_request .suspended_operations .insert(operation.clone().into()); - let value = self.in_progress_operations.fetch_sub(1, Ordering::AcqRel); + let value = self.in_progress_operations.fetch_sub(1, Ordering::AcqRel) - 1; assert!((value & SNAPSHOT_REQUESTED_BIT) != 0); if value == SNAPSHOT_REQUESTED_BIT { self.operations_suspended.notify_all(); @@ -145,11 +145,46 @@ impl TurboTasksBackend { } } } + + pub(crate) fn start_operation(&self) -> OperationGuard<'_> { + let fetch_add = self.in_progress_operations.fetch_add(1, Ordering::AcqRel); + if (fetch_add & SNAPSHOT_REQUESTED_BIT) != 0 { + let mut snapshot_request = self.snapshot_request.lock(); + if snapshot_request.snapshot_requested { + let value = self.in_progress_operations.fetch_sub(1, Ordering::AcqRel) - 1; + if value == SNAPSHOT_REQUESTED_BIT { + self.operations_suspended.notify_all(); + } + self.snapshot_completed + .wait_while(&mut snapshot_request, |snapshot_request| { + snapshot_request.snapshot_requested + }); + self.in_progress_operations.fetch_add(1, Ordering::AcqRel); + } + } + OperationGuard { backend: self } + } +} + +pub(crate) struct OperationGuard<'a> { + backend: &'a TurboTasksBackend, +} + +impl<'a> Drop for OperationGuard<'a> { + fn drop(&mut self) { + let fetch_sub = self + .backend + .in_progress_operations + .fetch_sub(1, Ordering::AcqRel); + if fetch_sub - 1 == SNAPSHOT_REQUESTED_BIT { + self.backend.operations_suspended.notify_all(); + } + } } // Operations impl TurboTasksBackend { - pub fn connect_child( + fn connect_child( &self, parent_task: TaskId, child_task: TaskId, @@ -162,23 +197,95 @@ impl TurboTasksBackend { ); } - pub fn update_cell( + fn try_read_task_output( &self, task_id: TaskId, - cell: CellId, - content: CellContent, + reader: Option, + consistency: ReadConsistency, turbo_tasks: &dyn TurboTasksBackendApi, - ) { - operation::UpdateCellOperation::run( - task_id, - cell, - content, - self.execute_context(turbo_tasks), - ); + ) -> Result> { + let ctx = self.execute_context(turbo_tasks); + let mut task = ctx.task(task_id); + + if let Some(in_progress) = get!(task, InProgress) { + match in_progress { + InProgressState::Scheduled { done_event, .. } + | InProgressState::InProgress { done_event, .. } => { + let listener = done_event.listen(); + return Ok(Err(listener)); + } + } + } + + if matches!(consistency, ReadConsistency::Strong) { + todo!("Handle strongly consistent read: {task:#?}"); + } + + if let Some(output) = get!(task, Output) { + let result = match output { + OutputValue::Cell(cell) => Some(Ok(Ok(RawVc::TaskCell(cell.task, cell.cell)))), + OutputValue::Output(task) => Some(Ok(Ok(RawVc::TaskOutput(*task)))), + OutputValue::Error | OutputValue::Panic => { + if let Some(error) = get!(task, Error) { + Some(Err(error.clone().into())) + } else { + None + } + } + }; + if let Some(result) = result { + if let Some(reader) = reader { + task.add(CachedDataItem::OutputDependent { + task: reader, + value: (), + }); + drop(task); + + let mut reader_task = ctx.task(reader); + reader_task.add(CachedDataItem::OutputDependency { + target: task_id, + value: (), + }); + } + + return result; + } + } + + todo!("Output of is not available, recompute task: {task:#?}"); } - pub fn invalidate(&self, task_id: TaskId, turbo_tasks: &dyn TurboTasksBackendApi) { - operation::InvalidateOperation::run(smallvec![task_id], self.execute_context(turbo_tasks)); + fn try_read_task_cell( + &self, + task_id: TaskId, + reader: Option, + cell: CellId, + turbo_tasks: &dyn TurboTasksBackendApi, + ) -> Result> { + let ctx = self.execute_context(turbo_tasks); + let mut task = ctx.task(task_id); + if let Some(content) = get!(task, CellData { cell }) { + let content = content.clone(); + if let Some(reader) = reader { + task.add(CachedDataItem::CellDependent { + cell, + task: reader, + value: (), + }); + drop(task); + let mut reader_task = ctx.task(reader); + reader_task.add(CachedDataItem::CellDependency { + target: CellRef { + task: task_id, + cell, + }, + value: (), + }); + } + return Ok(Ok(CellContent(Some(content)).into_typed(cell.type_id))); + } + + todo!("Cell {cell:?} is not available, recompute task or error: {task:#?}"); } } @@ -241,7 +348,7 @@ impl Backend for TurboTasksBackend { } fn invalidate_task(&self, task_id: TaskId, turbo_tasks: &dyn TurboTasksBackendApi) { - self.invalidate(task_id, turbo_tasks); + operation::InvalidateOperation::run(smallvec![task_id], self.execute_context(turbo_tasks)); } fn invalidate_tasks(&self, tasks: &[TaskId], turbo_tasks: &dyn TurboTasksBackendApi) { @@ -287,12 +394,16 @@ impl Backend for TurboTasksBackend { { let ctx = self.execute_context(turbo_tasks); let mut task = ctx.task(task_id); - let Some(InProgressState::Scheduled { + let Some(in_progress) = remove!(task, InProgress) else { + return None; + }; + let InProgressState::Scheduled { clean, done_event, start_event, - }) = remove!(task, InProgress) + } = in_progress else { + task.add(CachedDataItem::InProgress { value: in_progress }); return None; }; task.add(CachedDataItem::InProgress { @@ -407,7 +518,7 @@ impl Backend for TurboTasksBackend { let Some(CachedDataItemValue::InProgress { value: in_progress }) = task.remove(&CachedDataItemKey::InProgress {}) else { - panic!("Task execution completed, but task is not in progress"); + panic!("Task execution completed, but task is not in progress: {task:#?}"); }; let InProgressState::InProgress { done_event, @@ -415,7 +526,7 @@ impl Backend for TurboTasksBackend { stale, } = in_progress else { - panic!("Task execution completed, but task is not in progress"); + panic!("Task execution completed, but task is not in progress: {task:#?}"); }; // TODO handle cell counters @@ -442,60 +553,34 @@ impl Backend for TurboTasksBackend { ) -> Pin + Send + 'static)>> { todo!() } + fn try_read_task_output( &self, - _: TaskId, - _: TaskId, - _: ReadConsistency, - _: &dyn TurboTasksBackendApi, + task_id: TaskId, + reader: TaskId, + consistency: ReadConsistency, + turbo_tasks: &dyn TurboTasksBackendApi, ) -> Result> { - todo!() + self.try_read_task_output(task_id, Some(reader), consistency, turbo_tasks) } + fn try_read_task_output_untracked( &self, task_id: TaskId, consistency: ReadConsistency, turbo_tasks: &dyn TurboTasksBackendApi, ) -> Result> { - let ctx = self.execute_context(turbo_tasks); - let task = ctx.task(task_id); - - if let Some(in_progress) = get!(task, InProgress) { - match in_progress { - InProgressState::Scheduled { done_event, .. } - | InProgressState::InProgress { done_event, .. } => { - let listener = done_event.listen(); - return Ok(Err(listener)); - } - } - } - - if matches!(consistency, ReadConsistency::Strong) { - todo!("Handle strongly consistent read"); - } - - if let Some(output) = get!(task, Output) { - match output { - OutputValue::Cell(cell) => return Ok(Ok(RawVc::TaskCell(cell.task, cell.cell))), - OutputValue::Output(task) => return Ok(Ok(RawVc::TaskOutput(*task))), - OutputValue::Error | OutputValue::Panic => { - if let Some(error) = get!(task, Error) { - return Err(error.clone().into()); - } - } - } - } - - todo!("Output is not available, recompute task"); + self.try_read_task_output(task_id, None, consistency, turbo_tasks) } + fn try_read_task_cell( &self, - _: TaskId, - _: CellId, - _: TaskId, - _: &dyn TurboTasksBackendApi, + task_id: TaskId, + cell: CellId, + reader: TaskId, + turbo_tasks: &dyn TurboTasksBackendApi, ) -> Result> { - todo!() + self.try_read_task_cell(task_id, Some(reader), cell, turbo_tasks) } fn try_read_task_cell_untracked( @@ -504,14 +589,7 @@ impl Backend for TurboTasksBackend { cell: CellId, turbo_tasks: &dyn TurboTasksBackendApi, ) -> Result> { - let ctx = self.execute_context(turbo_tasks); - let task = ctx.task(task_id); - if let Some(content) = get!(task, CellData { cell }) { - return Ok(Ok( - CellContent(Some(content.clone())).into_typed(cell.type_id) - )); - } - todo!("Cell {cell:?} from {task_id:?} is not available, recompute task or error"); + self.try_read_task_cell(task_id, None, cell, turbo_tasks) } fn try_read_own_task_cell_untracked( @@ -567,7 +645,12 @@ impl Backend for TurboTasksBackend { content: CellContent, turbo_tasks: &dyn TurboTasksBackendApi, ) { - self.update_cell(task_id, cell, content, turbo_tasks); + operation::UpdateCellOperation::run( + task_id, + cell, + content, + self.execute_context(turbo_tasks), + ); } fn connect_task(&self, _: TaskId, _: TaskId, _: &dyn TurboTasksBackendApi) { diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs index 636f5edb05f2d..c2f6b92206d63 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs @@ -6,7 +6,9 @@ use crate::data::CachedDataItem; #[derive(Serialize, Deserialize, Clone, Default)] pub enum ConnectChildOperation { - Todo, + ScheduleTask { + task_id: TaskId, + }, #[default] Done, // TODO Add aggregated edge @@ -20,19 +22,28 @@ impl ConnectChildOperation { value: (), }) { // TODO add aggregated edge - ConnectChildOperation::Todo.execute(ctx); + // TODO check for active + ConnectChildOperation::ScheduleTask { + task_id: child_task, + } + .execute(&ctx); } } } impl Operation for ConnectChildOperation { - fn execute(mut self, ctx: ExecuteContext<'_>) { + fn execute(mut self, ctx: &ExecuteContext<'_>) { loop { ctx.operation_suspend_point(&self); match self { - ConnectChildOperation::Todo => { - self = ConnectChildOperation::Done; - continue; + ConnectChildOperation::ScheduleTask { task_id } => { + { + let mut task = ctx.task(task_id); + task.add(CachedDataItem::new_scheduled(task_id)); + } + ctx.schedule(task_id); + + return; } ConnectChildOperation::Done => { return; diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/invalidate.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/invalidate.rs index 8e307bc3fc3ef..5ffcdb05ebdb7 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/invalidate.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/invalidate.rs @@ -21,12 +21,12 @@ pub enum InvalidateOperation { impl InvalidateOperation { pub fn run(task_ids: SmallVec<[TaskId; 4]>, ctx: ExecuteContext<'_>) { - InvalidateOperation::MakeDirty { task_ids }.execute(ctx) + InvalidateOperation::MakeDirty { task_ids }.execute(&ctx) } } impl Operation for InvalidateOperation { - fn execute(self, ctx: ExecuteContext<'_>) { + fn execute(self, ctx: &ExecuteContext<'_>) { loop { ctx.operation_suspend_point(&self); match self { diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs index b2b632578a796..ffb97991d9885 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs @@ -3,13 +3,19 @@ mod invalidate; mod update_cell; mod update_output; -use std::mem::take; +use std::{ + fmt::{Debug, Formatter}, + mem::take, +}; use serde::{Deserialize, Serialize}; use turbo_tasks::{KeyValuePair, TaskId, TurboTasksBackendApi}; use super::{storage::StorageWriteGuard, TurboTasksBackend}; -use crate::data::{CachedDataItem, CachedDataItemKey, CachedDataItemValue, CachedDataUpdate}; +use crate::{ + backend::OperationGuard, + data::{CachedDataItem, CachedDataItemKey, CachedDataItemValue, CachedDataUpdate}, +}; pub trait Operation: Serialize @@ -18,13 +24,14 @@ pub trait Operation: + TryFrom + Into { - fn execute(self, ctx: ExecuteContext<'_>); + fn execute(self, ctx: &ExecuteContext<'_>); } -#[derive(Clone, Copy)] pub struct ExecuteContext<'a> { backend: &'a TurboTasksBackend, turbo_tasks: &'a dyn TurboTasksBackendApi, + #[allow(dead_code)] + operation_guard: Option>, parent: Option<(&'a AnyOperation, &'a ExecuteContext<'a>)>, } @@ -36,6 +43,7 @@ impl<'a> ExecuteContext<'a> { Self { backend, turbo_tasks, + operation_guard: Some(backend.start_operation()), parent: None, } } @@ -48,6 +56,10 @@ impl<'a> ExecuteContext<'a> { } } + pub fn schedule(&self, task_id: TaskId) { + self.turbo_tasks.schedule(task_id); + } + pub fn operation_suspend_point>(&self, op: &T) { if self.parent.is_some() { self.backend.operation_suspend_point(|| { @@ -79,6 +91,7 @@ impl<'a> ExecuteContext<'a> { let inner_ctx = ExecuteContext { backend: self.backend, turbo_tasks: self.turbo_tasks, + operation_guard: None, parent: Some((&parent_op, self)), }; run(inner_ctx); @@ -92,6 +105,20 @@ pub struct TaskGuard<'a> { backend: &'a TurboTasksBackend, } +impl<'a> Debug for TaskGuard<'a> { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut d = f.debug_struct("TaskGuard"); + d.field("task_id", &self.task_id); + if let Some(task_type) = self.backend.task_cache.lookup_reverse(&self.task_id) { + d.field("task_type", &task_type); + }; + for (key, value) in self.task.iter() { + d.field(&format!("{:?}", key), &value); + } + d.finish() + } +} + impl<'a> TaskGuard<'a> { fn new( task_id: TaskId, @@ -226,7 +253,7 @@ pub enum AnyOperation { } impl AnyOperation { - fn execute(self, ctx: ExecuteContext<'_>) { + fn execute(self, ctx: &ExecuteContext<'_>) { match self { AnyOperation::ConnectChild(op) => op.execute(ctx), AnyOperation::Invalidate(op) => op.execute(ctx), From 8e067c910de58382ce0c5147b9672c04086d9937 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Fri, 9 Aug 2024 11:15:42 +0200 Subject: [PATCH 10/81] remove unused stuff --- .../turbo-tasks-backend/src/backend/mod.rs | 10 ++- .../src/backend/operation/connect_child.rs | 2 +- .../src/backend/operation/mod.rs | 12 ---- .../crates/turbo-tasks-backend/src/data.rs | 10 +-- .../src/utils/external_locks.rs | 61 ------------------- .../turbo-tasks-backend/src/utils/mod.rs | 1 - 6 files changed, 11 insertions(+), 85 deletions(-) delete mode 100644 turbopack/crates/turbo-tasks-backend/src/utils/external_locks.rs diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs index 64624dd88a17c..e7e23064fe857 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs @@ -507,8 +507,8 @@ impl Backend for TurboTasksBackend { fn task_execution_completed( &self, task_id: TaskId, - duration: Duration, - memory_usage: usize, + _duration: Duration, + _memory_usage: usize, cell_counters: &AutoMap, 8>, stateful: bool, turbo_tasks: &dyn TurboTasksBackendApi, @@ -522,7 +522,7 @@ impl Backend for TurboTasksBackend { }; let InProgressState::InProgress { done_event, - clean, + clean: _, stale, } = in_progress else { @@ -530,6 +530,10 @@ impl Backend for TurboTasksBackend { }; // TODO handle cell counters + let _ = cell_counters; + + // TODO handle stateful + let _ = stateful; if stale { task.add(CachedDataItem::InProgress { diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs index c2f6b92206d63..f73dcb11bdb9d 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs @@ -32,7 +32,7 @@ impl ConnectChildOperation { } impl Operation for ConnectChildOperation { - fn execute(mut self, ctx: &ExecuteContext<'_>) { + fn execute(self, ctx: &ExecuteContext<'_>) { loop { ctx.operation_suspend_point(&self); match self { diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs index ffb97991d9885..3e99aa907d6e5 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs @@ -120,18 +120,6 @@ impl<'a> Debug for TaskGuard<'a> { } impl<'a> TaskGuard<'a> { - fn new( - task_id: TaskId, - task: StorageWriteGuard<'a, TaskId, CachedDataItem>, - backend: &'a TurboTasksBackend, - ) -> Self { - Self { - task_id, - task, - backend, - } - } - pub fn add(&mut self, item: CachedDataItem) -> bool { if !item.is_persistent() { self.task.add(item) diff --git a/turbopack/crates/turbo-tasks-backend/src/data.rs b/turbopack/crates/turbo-tasks-backend/src/data.rs index c217ae5d58c9e..d9e5ffe60b3e1 100644 --- a/turbopack/crates/turbo-tasks-backend/src/data.rs +++ b/turbopack/crates/turbo-tasks-backend/src/data.rs @@ -26,9 +26,9 @@ impl OutputValue { #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub enum RootType { - RootTask, - OnceTask, - ReadingStronglyConsistent, + _RootTask, + _OnceTask, + _ReadingStronglyConsistent, } #[derive(Debug)] @@ -238,10 +238,6 @@ impl CachedDataItemValue { } } -trait IsDefault { - fn is_default(&self) -> bool; -} - pub struct CachedDataUpdate { pub task: TaskId, pub key: CachedDataItemKey, diff --git a/turbopack/crates/turbo-tasks-backend/src/utils/external_locks.rs b/turbopack/crates/turbo-tasks-backend/src/utils/external_locks.rs deleted file mode 100644 index ddbfa54a75479..0000000000000 --- a/turbopack/crates/turbo-tasks-backend/src/utils/external_locks.rs +++ /dev/null @@ -1,61 +0,0 @@ -use std::{ - collections::{hash_map::Entry, HashMap}, - hash::Hash, - mem::transmute, - sync::Arc, -}; - -use parking_lot::{Mutex, MutexGuard}; - -pub struct ExternalLocks { - locks: HashMap>>, -} - -impl ExternalLocks { - pub fn new() -> Self { - Self { - locks: HashMap::new(), - } - } - - pub fn lock(&mut self, key: K) -> ExternalLockGuard { - let mutex = match self.locks.entry(key) { - Entry::Occupied(e) => e.get().clone(), - Entry::Vacant(e) => { - let lock = Arc::new(Mutex::new(())); - e.insert(lock.clone()); - if self.locks.len().is_power_of_two() { - let to_remove = self - .locks - .iter() - .filter_map(|(k, v)| { - if Arc::strong_count(v) == 1 { - Some(k.clone()) - } else { - None - } - }) - .collect::>(); - to_remove.into_iter().for_each(|k| { - self.locks.remove(&k); - }); - if self.locks.capacity() > self.locks.len() * 3 { - self.locks.shrink_to_fit(); - } - } - lock - } - }; - let guard = mutex.lock(); - // Safety: We know that the guard is valid for the lifetime of the lock as we - // keep the lock - let guard = unsafe { transmute::, MutexGuard<'static, _>>(guard) }; - ExternalLockGuard { lock: mutex, guard } - } -} - -pub struct ExternalLockGuard { - // Safety: guard must be before lock as it is dropped first - guard: MutexGuard<'static, ()>, - lock: Arc>, -} diff --git a/turbopack/crates/turbo-tasks-backend/src/utils/mod.rs b/turbopack/crates/turbo-tasks-backend/src/utils/mod.rs index 84e4a29fdb11a..3ad1e0475ec86 100644 --- a/turbopack/crates/turbo-tasks-backend/src/utils/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/utils/mod.rs @@ -1,4 +1,3 @@ pub mod bi_map; pub mod chunked_vec; -pub mod external_locks; pub mod ptr_eq_arc; From 6cb3e5d39825a54a444066a8ffe78812b52bde06 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Fri, 9 Aug 2024 11:19:25 +0200 Subject: [PATCH 11/81] clippy --- .../src/backend/operation/mod.rs | 41 ++++++------------- 1 file changed, 12 insertions(+), 29 deletions(-) diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs index 3e99aa907d6e5..d112c41e3914a 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs @@ -123,22 +123,19 @@ impl<'a> TaskGuard<'a> { pub fn add(&mut self, item: CachedDataItem) -> bool { if !item.is_persistent() { self.task.add(item) + } else if self.task.add(item.clone()) { + let (key, value) = item.into_key_and_value(); + self.backend + .persisted_storage_log + .lock() + .push(CachedDataUpdate { + key, + task: self.task_id, + value: Some(value), + }); + true } else { - #[allow(clippy::collapsible_if)] - if self.task.add(item.clone()) { - let (key, value) = item.into_key_and_value(); - self.backend - .persisted_storage_log - .lock() - .push(CachedDataUpdate { - key, - task: self.task_id, - value: Some(value), - }); - true - } else { - false - } + false } } @@ -240,20 +237,6 @@ pub enum AnyOperation { Nested(Vec), } -impl AnyOperation { - fn execute(self, ctx: &ExecuteContext<'_>) { - match self { - AnyOperation::ConnectChild(op) => op.execute(ctx), - AnyOperation::Invalidate(op) => op.execute(ctx), - AnyOperation::Nested(ops) => { - for op in ops { - op.execute(ctx); - } - } - } - } -} - impl_operation!(ConnectChild connect_child::ConnectChildOperation); impl_operation!(Invalidate invalidate::InvalidateOperation); From 6e5c56541d47921501facfadeb04df32475748a7 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Fri, 9 Aug 2024 11:19:54 +0200 Subject: [PATCH 12/81] remove things not yet used (persistence) --- .../turbo-tasks-backend/src/backend/storage.rs | 12 ------------ turbopack/crates/turbo-tasks-backend/src/data.rs | 4 ++++ 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs b/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs index c8f9702585096..368be96745400 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs @@ -10,27 +10,15 @@ use dashmap::{mapref::one::RefMut, DashMap}; use rustc_hash::FxHasher; use turbo_tasks::KeyValuePair; -enum PersistanceState { - /// We know that all state of the object is only in the cache and nothing is - /// stored in the persistent cache. - CacheOnly, - /// We know that some state of the object is stored in the persistent cache. - Persisted, - /// We have never checked the persistent cache for the state of the object. - Unknown, -} - pub struct InnerStorage { // TODO consider adding some inline storage map: AutoMap, - persistance_state: PersistanceState, } impl InnerStorage { fn new() -> Self { Self { map: AutoMap::new(), - persistance_state: PersistanceState::Unknown, } } diff --git a/turbopack/crates/turbo-tasks-backend/src/data.rs b/turbopack/crates/turbo-tasks-backend/src/data.rs index d9e5ffe60b3e1..4800dec0a3690 100644 --- a/turbopack/crates/turbo-tasks-backend/src/data.rs +++ b/turbopack/crates/turbo-tasks-backend/src/data.rs @@ -239,7 +239,11 @@ impl CachedDataItemValue { } pub struct CachedDataUpdate { + // TODO persistence + #[allow(dead_code)] pub task: TaskId, + #[allow(dead_code)] pub key: CachedDataItemKey, + #[allow(dead_code)] pub value: Option, } From 06da3b876bb0e684d1feca642784d8c002d0e719 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Fri, 9 Aug 2024 11:24:42 +0200 Subject: [PATCH 13/81] clippy --- .../turbo-tasks-backend/src/backend/mod.rs | 16 ++++++++-------- .../src/backend/operation/connect_child.rs | 5 +++-- .../src/backend/operation/invalidate.rs | 5 +++-- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs index e7e23064fe857..4847cddcad276 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs @@ -90,6 +90,12 @@ pub struct TurboTasksBackend { snapshot_completed: Condvar, } +impl Default for TurboTasksBackend { + fn default() -> Self { + Self::new() + } +} + impl TurboTasksBackend { pub fn new() -> Self { Self { @@ -226,11 +232,7 @@ impl TurboTasksBackend { OutputValue::Cell(cell) => Some(Ok(Ok(RawVc::TaskCell(cell.task, cell.cell)))), OutputValue::Output(task) => Some(Ok(Ok(RawVc::TaskOutput(*task)))), OutputValue::Error | OutputValue::Panic => { - if let Some(error) = get!(task, Error) { - Some(Err(error.clone().into())) - } else { - None - } + get!(task, Error).map(|error| Err(error.clone().into())) } }; if let Some(result) = result { @@ -394,9 +396,7 @@ impl Backend for TurboTasksBackend { { let ctx = self.execute_context(turbo_tasks); let mut task = ctx.task(task_id); - let Some(in_progress) = remove!(task, InProgress) else { - return None; - }; + let in_progress = remove!(task, InProgress)?; let InProgressState::Scheduled { clean, done_event, diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs index f73dcb11bdb9d..8bd3e39b7ce59 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs @@ -32,7 +32,7 @@ impl ConnectChildOperation { } impl Operation for ConnectChildOperation { - fn execute(self, ctx: &ExecuteContext<'_>) { + fn execute(mut self, ctx: &ExecuteContext<'_>) { loop { ctx.operation_suspend_point(&self); match self { @@ -43,7 +43,8 @@ impl Operation for ConnectChildOperation { } ctx.schedule(task_id); - return; + self = ConnectChildOperation::Done; + continue; } ConnectChildOperation::Done => { return; diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/invalidate.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/invalidate.rs index 5ffcdb05ebdb7..e817da6a4933a 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/invalidate.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/invalidate.rs @@ -26,7 +26,7 @@ impl InvalidateOperation { } impl Operation for InvalidateOperation { - fn execute(self, ctx: &ExecuteContext<'_>) { + fn execute(mut self, ctx: &ExecuteContext<'_>) { loop { ctx.operation_suspend_point(&self); match self { @@ -83,7 +83,8 @@ impl Operation for InvalidateOperation { } } } - return; + self = InvalidateOperation::Done; + continue; } InvalidateOperation::Done => { return; From 9cd0f1a6e3f30d6e45610d50d40609891f31ec36 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Fri, 9 Aug 2024 13:58:50 +0200 Subject: [PATCH 14/81] add more tests for new backend --- Cargo.lock | 2 ++ turbopack/crates/turbo-tasks-backend/Cargo.toml | 2 ++ turbopack/crates/turbo-tasks-backend/tests/all_in_one.rs | 1 + turbopack/crates/turbo-tasks-backend/tests/call_types.rs | 1 + turbopack/crates/turbo-tasks-backend/tests/debug.rs | 1 + 5 files changed, 7 insertions(+) create mode 120000 turbopack/crates/turbo-tasks-backend/tests/all_in_one.rs create mode 120000 turbopack/crates/turbo-tasks-backend/tests/call_types.rs create mode 120000 turbopack/crates/turbo-tasks-backend/tests/debug.rs diff --git a/Cargo.lock b/Cargo.lock index 6fc9d0841197b..f921856101c2e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8576,8 +8576,10 @@ dependencies = [ "async-trait", "auto-hash-map", "dashmap", + "indexmap 1.9.3", "once_cell", "parking_lot", + "rand", "rustc-hash", "serde", "smallvec", diff --git a/turbopack/crates/turbo-tasks-backend/Cargo.toml b/turbopack/crates/turbo-tasks-backend/Cargo.toml index 8feb087ae4f0f..ff458ac2808b2 100644 --- a/turbopack/crates/turbo-tasks-backend/Cargo.toml +++ b/turbopack/crates/turbo-tasks-backend/Cargo.toml @@ -17,8 +17,10 @@ anyhow = { workspace = true } async-trait = { workspace = true } auto-hash-map = { workspace = true } dashmap = { workspace = true } +indexmap = { workspace = true } once_cell = { workspace = true } parking_lot = { workspace = true } +rand = { workspace = true } rustc-hash = { workspace = true } serde = { workspace = true } smallvec = { workspace = true } diff --git a/turbopack/crates/turbo-tasks-backend/tests/all_in_one.rs b/turbopack/crates/turbo-tasks-backend/tests/all_in_one.rs new file mode 120000 index 0000000000000..391ab595a93e2 --- /dev/null +++ b/turbopack/crates/turbo-tasks-backend/tests/all_in_one.rs @@ -0,0 +1 @@ +../../turbo-tasks-testing/tests/all_in_one.rs \ No newline at end of file diff --git a/turbopack/crates/turbo-tasks-backend/tests/call_types.rs b/turbopack/crates/turbo-tasks-backend/tests/call_types.rs new file mode 120000 index 0000000000000..b20501cd53c79 --- /dev/null +++ b/turbopack/crates/turbo-tasks-backend/tests/call_types.rs @@ -0,0 +1 @@ +../../turbo-tasks-testing/tests/call_types.rs \ No newline at end of file diff --git a/turbopack/crates/turbo-tasks-backend/tests/debug.rs b/turbopack/crates/turbo-tasks-backend/tests/debug.rs new file mode 120000 index 0000000000000..ee7aea7eab52f --- /dev/null +++ b/turbopack/crates/turbo-tasks-backend/tests/debug.rs @@ -0,0 +1 @@ +../../turbo-tasks-testing/tests/debug.rs \ No newline at end of file From aff4e9575fcd533c9eab333acd3c0a80aa353f27 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Tue, 13 Aug 2024 07:26:05 +0200 Subject: [PATCH 15/81] add support for connect_child --- .../turbo-tasks-backend/src/backend/mod.rs | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs index 4847cddcad276..0911cb18fdc44 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs @@ -18,6 +18,8 @@ use std::{ use anyhow::Result; use auto_hash_map::{AutoMap, AutoSet}; use dashmap::DashMap; +pub use operation::AnyOperation; +use operation::ConnectChildOperation; use parking_lot::{Condvar, Mutex}; use rustc_hash::FxHasher; use smallvec::smallvec; @@ -33,10 +35,7 @@ use turbo_tasks::{ ValueTypeId, TRANSIENT_TASK_BIT, }; -use self::{ - operation::{AnyOperation, ExecuteContext}, - storage::Storage, -}; +use self::{operation::ExecuteContext, storage::Storage}; use crate::{ data::{ CachedDataItem, CachedDataItemKey, CachedDataItemValue, CachedDataUpdate, CellRef, @@ -657,9 +656,15 @@ impl Backend for TurboTasksBackend { ); } - fn connect_task(&self, _: TaskId, _: TaskId, _: &dyn TurboTasksBackendApi) { - todo!() + fn connect_task( + &self, + task: TaskId, + parent_task: TaskId, + turbo_tasks: &dyn TurboTasksBackendApi, + ) { + ConnectChildOperation::run(parent_task, task, self.execute_context(turbo_tasks)); } + fn create_transient_task( &self, task_type: TransientTaskType, From c230ba08543dd104ab0aba792179305147cd0812 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Fri, 9 Aug 2024 16:05:13 +0200 Subject: [PATCH 16/81] set root type on root tasks --- turbopack/crates/turbo-tasks-backend/src/backend/mod.rs | 7 ++++++- turbopack/crates/turbo-tasks-backend/src/data.rs | 4 ++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs index 0911cb18fdc44..66d27ac811345 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs @@ -39,7 +39,7 @@ use self::{operation::ExecuteContext, storage::Storage}; use crate::{ data::{ CachedDataItem, CachedDataItemKey, CachedDataItemValue, CachedDataUpdate, CellRef, - InProgressState, OutputValue, + InProgressState, OutputValue, RootType, }, get, remove, utils::{bi_map::BiMap, chunked_vec::ChunkedVec, ptr_eq_arc::PtrEqArc}, @@ -671,11 +671,16 @@ impl Backend for TurboTasksBackend { turbo_tasks: &dyn TurboTasksBackendApi, ) -> TaskId { let task_id = self.transient_task_id_factory.get(); + let root_type = match task_type { + TransientTaskType::Root(_) => RootType::RootTask, + TransientTaskType::Once(_) => RootType::OnceTask, + }; self.transient_tasks .insert(task_id, Arc::new(tokio::sync::Mutex::new(task_type))); { let mut task = self.storage.access_mut(task_id); task.add(CachedDataItem::new_scheduled(task_id)); + task.add(CachedDataItem::AggregateRootType { value: root_type }); } turbo_tasks.schedule(task_id); task_id diff --git a/turbopack/crates/turbo-tasks-backend/src/data.rs b/turbopack/crates/turbo-tasks-backend/src/data.rs index 4800dec0a3690..05a10bc3981d5 100644 --- a/turbopack/crates/turbo-tasks-backend/src/data.rs +++ b/turbopack/crates/turbo-tasks-backend/src/data.rs @@ -26,8 +26,8 @@ impl OutputValue { #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub enum RootType { - _RootTask, - _OnceTask, + RootTask, + OnceTask, _ReadingStronglyConsistent, } From 486752a397c11af984e5246bf01fc0b06cd488af Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Mon, 12 Aug 2024 15:36:17 +0200 Subject: [PATCH 17/81] remove old edges when task recomputation completes --- .../turbo-tasks-backend/src/backend/mod.rs | 125 ++++++++++++++++-- .../backend/operation/cleanup_old_edges.rs | 96 ++++++++++++++ .../src/backend/operation/mod.rs | 8 ++ .../src/backend/storage.rs | 4 + .../crates/turbo-tasks-backend/src/data.rs | 9 +- 5 files changed, 233 insertions(+), 9 deletions(-) create mode 100644 turbopack/crates/turbo-tasks-backend/src/backend/operation/cleanup_old_edges.rs diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs index 66d27ac811345..745e27bf2559e 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs @@ -19,7 +19,7 @@ use anyhow::Result; use auto_hash_map::{AutoMap, AutoSet}; use dashmap::DashMap; pub use operation::AnyOperation; -use operation::ConnectChildOperation; +use operation::{CleanupOldEdgesOperation, ConnectChildOperation, OutdatedEdge}; use parking_lot::{Condvar, Mutex}; use rustc_hash::FxHasher; use smallvec::smallvec; @@ -275,13 +275,16 @@ impl TurboTasksBackend { }); drop(task); let mut reader_task = ctx.task(reader); - reader_task.add(CachedDataItem::CellDependency { - target: CellRef { - task: task_id, - cell, - }, - value: (), - }); + let target = CellRef { + task: task_id, + cell, + }; + if reader_task + .remove(&CachedDataItemKey::OutdatedCellDependency { target }) + .is_none() + { + reader_task.add(CachedDataItem::CellDependency { target, value: () }); + } } return Ok(Ok(CellContent(Some(content)).into_typed(cell.type_id))); } @@ -412,8 +415,95 @@ impl Backend for TurboTasksBackend { done_event, }, }); + + // Make all current children outdated (remove left-over outdated children) + enum Child { + Current(TaskId), + Outdated(TaskId), + } + let children = task + .iter() + .filter_map(|(key, _)| match *key { + CachedDataItemKey::Child { task } => Some(Child::Current(task)), + CachedDataItemKey::OutdatedChild { task } => Some(Child::Outdated(task)), + _ => None, + }) + .collect::>(); + for child in children { + match child { + Child::Current(child) => { + task.add(CachedDataItem::OutdatedChild { + task: child, + value: (), + }); + } + Child::Outdated(child) => { + if !task.has_key(&CachedDataItemKey::Child { task: child }) { + task.remove(&CachedDataItemKey::OutdatedChild { task: child }); + } + } + } + } + + // Make all dependencies outdated + enum Dep { + CurrentCell(CellRef), + CurrentOutput(TaskId), + OutdatedCell(CellRef), + OutdatedOutput(TaskId), + } + let dependencies = task + .iter() + .filter_map(|(key, _)| match *key { + CachedDataItemKey::CellDependency { target } => Some(Dep::CurrentCell(target)), + CachedDataItemKey::OutputDependency { target } => { + Some(Dep::CurrentOutput(target)) + } + CachedDataItemKey::OutdatedCellDependency { target } => { + Some(Dep::OutdatedCell(target)) + } + CachedDataItemKey::OutdatedOutputDependency { target } => { + Some(Dep::OutdatedOutput(target)) + } + _ => None, + }) + .collect::>(); + for dep in dependencies { + match dep { + Dep::CurrentCell(cell) => { + task.add(CachedDataItem::OutdatedCellDependency { + target: cell, + value: (), + }); + } + Dep::CurrentOutput(output) => { + task.add(CachedDataItem::OutdatedOutputDependency { + target: output, + value: (), + }); + } + Dep::OutdatedCell(cell) => { + if !task.has_key(&CachedDataItemKey::CellDependency { target: cell }) { + task.remove(&CachedDataItemKey::OutdatedCellDependency { + target: cell, + }); + } + } + Dep::OutdatedOutput(output) => { + if !task.has_key(&CachedDataItemKey::OutputDependency { target: output }) { + task.remove(&CachedDataItemKey::OutdatedOutputDependency { + target: output, + }); + } + } + } + } + + // TODO: Make all collectibles outdated + start_event.notify(usize::MAX); } + let (span, future) = if let Some(task_type) = self.task_cache.lookup_reverse(&task_id) { match &*task_type { CachedTaskType::Native { fn_type, this, arg } => ( @@ -542,8 +632,27 @@ impl Backend for TurboTasksBackend { done_event, }, }); + drop(task); + drop(ctx); } else { + let old_edges = task + .iter() + .filter_map(|(key, _)| match *key { + CachedDataItemKey::OutdatedChild { task } => Some(OutdatedEdge::Child(task)), + CachedDataItemKey::OutdatedCellDependency { target } => { + Some(OutdatedEdge::CellDependency(target)) + } + CachedDataItemKey::OutdatedOutputDependency { target } => { + Some(OutdatedEdge::OutputDependency(target)) + } + _ => None, + }) + .collect::>(); + done_event.notify(usize::MAX); + drop(task); + + CleanupOldEdgesOperation::run(task_id, old_edges, ctx); } stale diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/cleanup_old_edges.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/cleanup_old_edges.rs new file mode 100644 index 0000000000000..e9eb3738d84ee --- /dev/null +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/cleanup_old_edges.rs @@ -0,0 +1,96 @@ +use serde::{Deserialize, Serialize}; +use turbo_tasks::TaskId; + +use super::{ExecuteContext, Operation}; +use crate::data::{CachedDataItemKey, CellRef}; + +#[derive(Serialize, Deserialize, Clone, Default)] +pub enum CleanupOldEdgesOperation { + RemoveEdges { + task_id: TaskId, + outdated: Vec, + }, + #[default] + Done, + // TODO Add aggregated edge +} + +#[derive(Serialize, Deserialize, Clone)] +pub enum OutdatedEdge { + Child(TaskId), + CellDependency(CellRef), + OutputDependency(TaskId), +} + +impl CleanupOldEdgesOperation { + pub fn run(task_id: TaskId, outdated: Vec, ctx: ExecuteContext<'_>) { + CleanupOldEdgesOperation::RemoveEdges { task_id, outdated }.execute(&ctx); + } +} + +impl Operation for CleanupOldEdgesOperation { + fn execute(mut self, ctx: &ExecuteContext<'_>) { + loop { + ctx.operation_suspend_point(&self); + match self { + CleanupOldEdgesOperation::RemoveEdges { + task_id, + ref mut outdated, + } => { + if let Some(edge) = outdated.pop() { + match edge { + OutdatedEdge::Child(child_id) => { + let mut task = ctx.task(task_id); + task.remove(&CachedDataItemKey::Child { task: child_id }); + // TODO remove aggregated edge + } + OutdatedEdge::CellDependency(CellRef { + task: cell_task_id, + cell, + }) => { + { + let mut task = ctx.task(cell_task_id); + task.remove(&CachedDataItemKey::CellDependent { + cell, + task: task_id, + }); + } + { + let mut task = ctx.task(task_id); + task.remove(&CachedDataItemKey::CellDependency { + target: CellRef { + task: cell_task_id, + cell, + }, + }); + } + } + OutdatedEdge::OutputDependency(output_task_id) => { + { + let mut task = ctx.task(output_task_id); + task.remove(&CachedDataItemKey::OutputDependent { + task: task_id, + }); + } + { + let mut task = ctx.task(task_id); + task.remove(&CachedDataItemKey::OutputDependency { + target: output_task_id, + }); + } + } + } + } + + if outdated.is_empty() { + self = CleanupOldEdgesOperation::Done; + } + continue; + } + CleanupOldEdgesOperation::Done => { + return; + } + } + } + } +} diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs index d112c41e3914a..39b47fbd6cc0e 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs @@ -1,3 +1,4 @@ +mod cleanup_old_edges; mod connect_child; mod invalidate; mod update_cell; @@ -202,6 +203,10 @@ impl<'a> TaskGuard<'a> { self.task.get(key) } + pub fn has_key(&self, key: &CachedDataItemKey) -> bool { + self.task.has_key(key) + } + pub fn iter(&self) -> impl Iterator { self.task.iter() } @@ -234,11 +239,14 @@ macro_rules! impl_operation { pub enum AnyOperation { ConnectChild(connect_child::ConnectChildOperation), Invalidate(invalidate::InvalidateOperation), + CleanupOldEdges(cleanup_old_edges::CleanupOldEdgesOperation), Nested(Vec), } impl_operation!(ConnectChild connect_child::ConnectChildOperation); impl_operation!(Invalidate invalidate::InvalidateOperation); +impl_operation!(CleanupOldEdges cleanup_old_edges::CleanupOldEdgesOperation); +pub use cleanup_old_edges::OutdatedEdge; pub use update_cell::UpdateCellOperation; pub use update_output::UpdateOutputOperation; diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs b/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs index 368be96745400..5549582696352 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs @@ -46,6 +46,10 @@ impl InnerStorage { self.map.get(key) } + pub fn has_key(&self, key: &T::Key) -> bool { + self.map.contains_key(key) + } + pub fn iter(&self) -> impl Iterator { self.map.iter() } diff --git a/turbopack/crates/turbo-tasks-backend/src/data.rs b/turbopack/crates/turbo-tasks-backend/src/data.rs index 05a10bc3981d5..3e1daf0272c2c 100644 --- a/turbopack/crates/turbo-tasks-backend/src/data.rs +++ b/turbopack/crates/turbo-tasks-backend/src/data.rs @@ -1,6 +1,7 @@ +use serde::{Deserialize, Serialize}; use turbo_tasks::{event::Event, util::SharedError, CellId, KeyValuePair, SharedReference, TaskId}; -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq, Serialize, Deserialize)] pub struct CellRef { pub task: TaskId, pub cell: CellId, @@ -150,6 +151,10 @@ pub enum CachedDataItem { target: CellRef, value: (), }, + OutdatedChild { + task: TaskId, + value: (), + }, // Transient Error State Error { @@ -183,6 +188,7 @@ impl CachedDataItem { CachedDataItem::OutdatedCollectible { .. } => false, CachedDataItem::OutdatedOutputDependency { .. } => false, CachedDataItem::OutdatedCellDependency { .. } => false, + CachedDataItem::OutdatedChild { .. } => false, CachedDataItem::Error { .. } => false, } } @@ -224,6 +230,7 @@ impl CachedDataItemKey { CachedDataItemKey::OutdatedCollectible { .. } => false, CachedDataItemKey::OutdatedOutputDependency { .. } => false, CachedDataItemKey::OutdatedCellDependency { .. } => false, + CachedDataItemKey::OutdatedChild { .. } => false, CachedDataItemKey::Error { .. } => false, } } From 63925828b2a81e3f7ee104ed075c07ea3373dcae Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Wed, 14 Aug 2024 18:29:15 +0200 Subject: [PATCH 18/81] add try_get_function_id --- .../turbo-tasks-backend/src/backend/mod.rs | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs index 745e27bf2559e..7a3577fdaf6dd 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs @@ -291,6 +291,13 @@ impl TurboTasksBackend { todo!("Cell {cell:?} is not available, recompute task or error: {task:#?}"); } + + fn lookup_task_type(&self, task_id: TaskId) -> Option> { + if let Some(task_type) = self.task_cache.lookup_reverse(&task_id) { + return Some(task_type); + } + None + } } impl Backend for TurboTasksBackend { @@ -374,15 +381,16 @@ impl Backend for TurboTasksBackend { } fn get_task_description(&self, task: TaskId) -> std::string::String { - let task_type = self - .task_cache - .lookup_reverse(&task) - .expect("Task not found"); + let task_type = self.lookup_task_type(task).expect("Task not found"); task_type.to_string() } - fn try_get_function_id(&self, _: TaskId) -> Option { - todo!() + fn try_get_function_id(&self, task_id: TaskId) -> Option { + self.lookup_task_type(task_id) + .and_then(|task_type| match &*task_type { + CachedTaskType::Native { fn_type, .. } => Some(*fn_type), + _ => None, + }) } type TaskState = (); @@ -504,7 +512,7 @@ impl Backend for TurboTasksBackend { start_event.notify(usize::MAX); } - let (span, future) = if let Some(task_type) = self.task_cache.lookup_reverse(&task_id) { + let (span, future) = if let Some(task_type) = self.lookup_task_type(task_id) { match &*task_type { CachedTaskType::Native { fn_type, this, arg } => ( registry::get_function(*fn_type).span(), From 432c69cc05f337b4ffc89da1e3622faf42183c53 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Mon, 12 Aug 2024 11:02:18 +0200 Subject: [PATCH 19/81] move backend impl into type alias --- crates/napi/src/next_api/project.rs | 13 ++++++------- crates/napi/src/next_api/utils.rs | 12 +++++++----- crates/napi/src/turbotrace.rs | 9 +++++---- 3 files changed, 18 insertions(+), 16 deletions(-) diff --git a/crates/napi/src/next_api/project.rs b/crates/napi/src/next_api/project.rs index 65143fd3ff11c..aed5d0024b475 100644 --- a/crates/napi/src/next_api/project.rs +++ b/crates/napi/src/next_api/project.rs @@ -24,7 +24,6 @@ use tracing::Instrument; use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter, Registry}; use turbo_tasks::{Completion, RcStr, ReadRef, TransientInstance, TurboTasks, UpdateInfo, Vc}; use turbo_tasks_fs::{DiskFileSystem, FileContent, FileSystem, FileSystemPath}; -use turbo_tasks_memory::MemoryBackend; use turbopack_core::{ diagnostics::PlainDiagnostic, error::PrettyPrintError, @@ -44,7 +43,7 @@ use url::Url; use super::{ endpoint::ExternalEndpoint, utils::{ - get_diagnostics, get_issues, subscribe, NapiDiagnostic, NapiIssue, RootTask, + get_diagnostics, get_issues, subscribe, NapiDiagnostic, NapiIssue, NextBackend, RootTask, TurbopackResult, VcArc, }, }; @@ -273,7 +272,7 @@ impl From for DefineEnv { } pub struct ProjectInstance { - turbo_tasks: Arc>, + turbo_tasks: Arc>, container: Vc, exit_receiver: tokio::sync::Mutex>, } @@ -338,7 +337,7 @@ pub async fn project_new( subscriber.init(); } - let turbo_tasks = TurboTasks::new(MemoryBackend::new( + let turbo_tasks = TurboTasks::new(NextBackend::new( turbo_engine_options .memory_limit .map(|m| m as usize) @@ -502,7 +501,7 @@ impl NapiRoute { fn from_route( pathname: String, value: Route, - turbo_tasks: &Arc>, + turbo_tasks: &Arc>, ) -> Self { let convert_endpoint = |endpoint: Vc>| { Some(External::new(ExternalEndpoint(VcArc::new( @@ -569,7 +568,7 @@ struct NapiMiddleware { impl NapiMiddleware { fn from_middleware( value: &Middleware, - turbo_tasks: &Arc>, + turbo_tasks: &Arc>, ) -> Result { Ok(NapiMiddleware { endpoint: External::new(ExternalEndpoint(VcArc::new( @@ -589,7 +588,7 @@ struct NapiInstrumentation { impl NapiInstrumentation { fn from_instrumentation( value: &Instrumentation, - turbo_tasks: &Arc>, + turbo_tasks: &Arc>, ) -> Result { Ok(NapiInstrumentation { node_js: External::new(ExternalEndpoint(VcArc::new( diff --git a/crates/napi/src/next_api/utils.rs b/crates/napi/src/next_api/utils.rs index 62cefa34b7af5..6f01afed43885 100644 --- a/crates/napi/src/next_api/utils.rs +++ b/crates/napi/src/next_api/utils.rs @@ -19,22 +19,24 @@ use turbopack_core::{ use crate::util::log_internal_error_and_inform; +pub type NextBackend = MemoryBackend; + /// A helper type to hold both a Vc operation and the TurboTasks root process. /// Without this, we'd need to pass both individually all over the place #[derive(Clone)] pub struct VcArc { - turbo_tasks: Arc>, + turbo_tasks: Arc>, /// The Vc. Must be resolved, otherwise you are referencing an inactive /// operation. vc: T, } impl VcArc { - pub fn new(turbo_tasks: Arc>, vc: T) -> Self { + pub fn new(turbo_tasks: Arc>, vc: T) -> Self { Self { turbo_tasks, vc } } - pub fn turbo_tasks(&self) -> &Arc> { + pub fn turbo_tasks(&self) -> &Arc> { &self.turbo_tasks } } @@ -57,7 +59,7 @@ pub fn serde_enum_to_string(value: &T) -> Result { /// The root of our turbopack computation. pub struct RootTask { #[allow(dead_code)] - turbo_tasks: Arc>, + turbo_tasks: Arc>, #[allow(dead_code)] task_id: Option, } @@ -301,7 +303,7 @@ impl ToNapiValue for TurbopackResult { } pub fn subscribe> + Send, V: ToNapiValue>( - turbo_tasks: Arc>, + turbo_tasks: Arc>, func: JsFunction, handler: impl 'static + Sync + Send + Clone + Fn() -> F, mapper: impl 'static + Sync + Send + FnMut(ThreadSafeCallContext) -> napi::Result>, diff --git a/crates/napi/src/turbotrace.rs b/crates/napi/src/turbotrace.rs index faf0c63db3764..f3281ad3030d4 100644 --- a/crates/napi/src/turbotrace.rs +++ b/crates/napi/src/turbotrace.rs @@ -3,15 +3,16 @@ use std::sync::Arc; use napi::bindgen_prelude::*; use node_file_trace::{start, Args}; use turbo_tasks::TurboTasks; -use turbo_tasks_memory::MemoryBackend; use turbopack::{ module_options::{EcmascriptOptionsContext, ModuleOptionsContext}, resolve_options_context::ResolveOptionsContext, }; +use crate::next_api::utils::NextBackend; + #[napi] -pub fn create_turbo_tasks(memory_limit: Option) -> External>> { - let turbo_tasks = TurboTasks::new(MemoryBackend::new( +pub fn create_turbo_tasks(memory_limit: Option) -> External>> { + let turbo_tasks = TurboTasks::new(NextBackend::new( memory_limit.map(|m| m as usize).unwrap_or(usize::MAX), )); External::new_with_size_hint( @@ -23,7 +24,7 @@ pub fn create_turbo_tasks(memory_limit: Option) -> External>>>, + turbo_tasks: Option>>>, ) -> napi::Result> { let args: Args = serde_json::from_slice(options.as_ref())?; let turbo_tasks = turbo_tasks.map(|t| t.clone()); From 7fa108671242fc7e69e5b7301671412429748e12 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Wed, 4 Sep 2024 09:23:43 +0200 Subject: [PATCH 20/81] add new backend feature --- Cargo.lock | 1 + Cargo.toml | 1 + crates/napi/Cargo.toml | 3 + crates/napi/src/next_api/project.rs | 58 +++++++------- crates/napi/src/next_api/utils.rs | 22 +++++- crates/napi/src/turbotrace.rs | 25 +++--- .../next/src/build/collect-build-traces.ts | 1 + packages/next/src/build/index.ts | 1 + .../next/src/build/swc/generated-native.d.ts | 5 +- packages/next/src/build/swc/index.ts | 8 +- packages/next/src/build/swc/types.ts | 10 ++- .../src/server/dev/hot-reloader-turbopack.ts | 1 + test/development/basic/next-rs-api.test.ts | 14 ++-- turbopack/crates/node-file-trace/src/lib.rs | 76 ++----------------- turbopack/crates/node-file-trace/src/main.rs | 7 +- 15 files changed, 106 insertions(+), 127 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f921856101c2e..ff2b16f37ba01 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4226,6 +4226,7 @@ dependencies = [ "tracing-chrome", "tracing-subscriber", "turbo-tasks", + "turbo-tasks-backend", "turbo-tasks-build", "turbo-tasks-fs", "turbo-tasks-malloc", diff --git a/Cargo.toml b/Cargo.toml index d4a9d2675d31a..4412bf582a2dd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -45,6 +45,7 @@ swc-ast-explorer = { path = "turbopack/crates/turbopack-swc-ast-explorer" } turbo-prehash = { path = "turbopack/crates/turbo-prehash" } turbo-tasks-malloc = { path = "turbopack/crates/turbo-tasks-malloc", default-features = false } turbo-tasks = { path = "turbopack/crates/turbo-tasks" } +turbo-tasks-backend = { path = "turbopack/crates/turbo-tasks-backend" } turbo-tasks-build = { path = "turbopack/crates/turbo-tasks-build" } turbo-tasks-bytes = { path = "turbopack/crates/turbo-tasks-bytes" } turbo-tasks-env = { path = "turbopack/crates/turbo-tasks-env" } diff --git a/crates/napi/Cargo.toml b/crates/napi/Cargo.toml index e7d428bbeb4fb..604dd24188771 100644 --- a/crates/napi/Cargo.toml +++ b/crates/napi/Cargo.toml @@ -37,6 +37,8 @@ __internal_dhat-heap = ["dhat"] # effectively does nothing. __internal_dhat-ad-hoc = ["dhat"] +new-backend = ["dep:turbo-tasks-backend"] + # Enable specific tls features per-target. [target.'cfg(all(target_os = "windows", target_arch = "aarch64"))'.dependencies] next-core = { workspace = true, features = ["native-tls"] } @@ -105,6 +107,7 @@ lightningcss-napi = { workspace = true } tokio = { workspace = true, features = ["full"] } turbo-tasks = { workspace = true } turbo-tasks-memory = { workspace = true } +turbo-tasks-backend = { workspace = true, optional = true } turbo-tasks-fs = { workspace = true } next-api = { workspace = true } next-build = { workspace = true } diff --git a/crates/napi/src/next_api/project.rs b/crates/napi/src/next_api/project.rs index aed5d0024b475..df20d341d7094 100644 --- a/crates/napi/src/next_api/project.rs +++ b/crates/napi/src/next_api/project.rs @@ -1,4 +1,4 @@ -use std::{io::Write, path::PathBuf, sync::Arc, thread, time::Duration}; +use std::{path::PathBuf, sync::Arc, thread, time::Duration}; use anyhow::{anyhow, bail, Context, Result}; use napi::{ @@ -43,8 +43,8 @@ use url::Url; use super::{ endpoint::ExternalEndpoint, utils::{ - get_diagnostics, get_issues, subscribe, NapiDiagnostic, NapiIssue, NextBackend, RootTask, - TurbopackResult, VcArc, + create_turbo_tasks, get_diagnostics, get_issues, subscribe, NapiDiagnostic, NapiIssue, + NextBackend, RootTask, TurbopackResult, VcArc, }, }; use crate::register; @@ -98,7 +98,7 @@ pub struct NapiProjectOptions { /// next.config's distDir. Project initialization occurs eariler than /// deserializing next.config, so passing it as separate option. - pub dist_dir: Option, + pub dist_dir: String, /// Filesystem watcher options. pub watch: NapiWatchOptions, @@ -308,10 +308,7 @@ pub async fn project_new( let subscriber = Registry::default(); let subscriber = subscriber.with(EnvFilter::builder().parse(trace).unwrap()); - let dist_dir = options - .dist_dir - .as_ref() - .map_or_else(|| ".next".to_string(), |d| d.to_string()); + let dist_dir = options.dist_dir.clone(); let internal_dir = PathBuf::from(&options.project_path).join(dist_dir); std::fs::create_dir_all(&internal_dir) @@ -337,27 +334,30 @@ pub async fn project_new( subscriber.init(); } - let turbo_tasks = TurboTasks::new(NextBackend::new( - turbo_engine_options - .memory_limit - .map(|m| m as usize) - .unwrap_or(usize::MAX), - )); - let stats_path = std::env::var_os("NEXT_TURBOPACK_TASK_STATISTICS"); - if let Some(stats_path) = stats_path { - let task_stats = turbo_tasks.backend().task_statistics().enable().clone(); - exit.on_exit(async move { - tokio::task::spawn_blocking(move || { - let mut file = std::fs::File::create(&stats_path) - .with_context(|| format!("failed to create or open {stats_path:?}"))?; - serde_json::to_writer(&file, &task_stats) - .context("failed to serialize or write task statistics")?; - file.flush().context("failed to flush file") - }) - .await - .unwrap() - .unwrap(); - }); + let memory_limit = turbo_engine_options + .memory_limit + .map(|m| m as usize) + .unwrap_or(usize::MAX); + let turbo_tasks = create_turbo_tasks(PathBuf::from(&options.dist_dir), memory_limit)?; + #[cfg(not(feature = "new-backend"))] + { + use std::io::Write; + let stats_path = std::env::var_os("NEXT_TURBOPACK_TASK_STATISTICS"); + if let Some(stats_path) = stats_path { + let task_stats = turbo_tasks.backend().task_statistics().enable().clone(); + exit.on_exit(async move { + tokio::task::spawn_blocking(move || { + let mut file = std::fs::File::create(&stats_path) + .with_context(|| format!("failed to create or open {stats_path:?}"))?; + serde_json::to_writer(&file, &task_stats) + .context("failed to serialize or write task statistics")?; + file.flush().context("failed to flush file") + }) + .await + .unwrap() + .unwrap(); + }); + } } let options: ProjectOptions = options.into(); let container = turbo_tasks diff --git a/crates/napi/src/next_api/utils.rs b/crates/napi/src/next_api/utils.rs index 6f01afed43885..a5c4284c73c66 100644 --- a/crates/napi/src/next_api/utils.rs +++ b/crates/napi/src/next_api/utils.rs @@ -1,4 +1,4 @@ -use std::{collections::HashMap, future::Future, ops::Deref, sync::Arc}; +use std::{collections::HashMap, future::Future, ops::Deref, path::PathBuf, sync::Arc}; use anyhow::{anyhow, Context, Result}; use napi::{ @@ -9,7 +9,6 @@ use napi::{ use serde::Serialize; use turbo_tasks::{ReadRef, TaskId, TryJoinIterExt, TurboTasks, Vc}; use turbo_tasks_fs::FileContent; -use turbo_tasks_memory::MemoryBackend; use turbopack_core::{ diagnostics::{Diagnostic, DiagnosticContextExt, PlainDiagnostic}, error::PrettyPrintError, @@ -19,7 +18,24 @@ use turbopack_core::{ use crate::util::log_internal_error_and_inform; -pub type NextBackend = MemoryBackend; +#[cfg(not(feature = "new-backend"))] +pub type NextBackend = turbo_tasks_memory::MemoryBackend; +#[cfg(feature = "new-backend")] +pub type NextBackend = turbo_tasks_backend::TurboTasksBackend; + +#[allow(unused_variables, reason = "feature-gated")] +pub fn create_turbo_tasks( + output_path: PathBuf, + memory_limit: usize, +) -> Result>> { + #[cfg(not(feature = "new-backend"))] + let backend = TurboTasks::new(turbo_tasks_memory::MemoryBackend::new(memory_limit)); + #[cfg(feature = "new-backend")] + let backend = TurboTasks::new(turbo_tasks_backend::TurboTasksBackend::new(Arc::new( + turbo_tasks_backend::LmdbBackingStorage::new(&output_path.join("cache/turbopack"))?, + ))); + Ok(backend) +} /// A helper type to hold both a Vc operation and the TurboTasks root process. /// Without this, we'd need to pass both individually all over the place diff --git a/crates/napi/src/turbotrace.rs b/crates/napi/src/turbotrace.rs index f3281ad3030d4..fef534c04b550 100644 --- a/crates/napi/src/turbotrace.rs +++ b/crates/napi/src/turbotrace.rs @@ -1,4 +1,4 @@ -use std::sync::Arc; +use std::{path::PathBuf, sync::Arc}; use napi::bindgen_prelude::*; use node_file_trace::{start, Args}; @@ -8,29 +8,28 @@ use turbopack::{ resolve_options_context::ResolveOptionsContext, }; -use crate::next_api::utils::NextBackend; +use crate::next_api::utils::{self, NextBackend}; #[napi] -pub fn create_turbo_tasks(memory_limit: Option) -> External>> { - let turbo_tasks = TurboTasks::new(NextBackend::new( - memory_limit.map(|m| m as usize).unwrap_or(usize::MAX), - )); - External::new_with_size_hint( - turbo_tasks, - memory_limit.map(|u| u as usize).unwrap_or(usize::MAX), - ) +pub fn create_turbo_tasks( + output_path: String, + memory_limit: Option, +) -> External>> { + let limit = memory_limit.map(|u| u as usize).unwrap_or(usize::MAX); + let turbo_tasks = utils::create_turbo_tasks(PathBuf::from(&output_path), limit) + .expect("Failed to create TurboTasks"); + External::new_with_size_hint(turbo_tasks, limit) } #[napi] pub async fn run_turbo_tracing( options: Buffer, - turbo_tasks: Option>>>, + turbo_tasks: External>>, ) -> napi::Result> { let args: Args = serde_json::from_slice(options.as_ref())?; - let turbo_tasks = turbo_tasks.map(|t| t.clone()); let files = start( Arc::new(args), - turbo_tasks.as_ref(), + turbo_tasks.clone(), Some(ModuleOptionsContext { ecmascript: EcmascriptOptionsContext { enable_types: true, diff --git a/packages/next/src/build/collect-build-traces.ts b/packages/next/src/build/collect-build-traces.ts index 498aced6a9922..75639427ed0cf 100644 --- a/packages/next/src/build/collect-build-traces.ts +++ b/packages/next/src/build/collect-build-traces.ts @@ -119,6 +119,7 @@ export async function collectBuildTraces({ let turbotraceOutputPath: string | undefined let turbotraceFiles: string[] | undefined turboTasksForTrace = bindings.turbo.createTurboTasks( + distDir, (config.experimental.turbotrace?.memoryLimit ?? TURBO_TRACE_DEFAULT_MEMORY_LIMIT) * 1024 * diff --git a/packages/next/src/build/index.ts b/packages/next/src/build/index.ts index 1f2ab770fea3f..5eba2e12fde4f 100644 --- a/packages/next/src/build/index.ts +++ b/packages/next/src/build/index.ts @@ -1379,6 +1379,7 @@ export default async function build( config.experimental?.turbo?.root || config.outputFileTracingRoot || dir, + distDir, nextConfig: config, jsConfig: await getTurbopackJsConfig(dir, config), watch: { diff --git a/packages/next/src/build/swc/generated-native.d.ts b/packages/next/src/build/swc/generated-native.d.ts index 0921b0b0c2e2a..2c9def68d3b7b 100644 --- a/packages/next/src/build/swc/generated-native.d.ts +++ b/packages/next/src/build/swc/generated-native.d.ts @@ -103,7 +103,7 @@ export interface NapiProjectOptions { * next.config's distDir. Project initialization occurs eariler than * deserializing next.config, so passing it as separate option. */ - distDir?: string + distDir: string /** Filesystem watcher options. */ watch: NapiWatchOptions /** The contents of next.config.js, serialized to JSON. */ @@ -363,11 +363,12 @@ export interface NapiRewrite { missing?: Array } export function createTurboTasks( + outputPath: string, memoryLimit?: number | undefined | null ): ExternalObject export function runTurboTracing( options: Buffer, - turboTasks?: ExternalObject | undefined | null + turboTasks: ExternalObject ): Promise> export function getTargetTriple(): string export function initHeapProfiler(): ExternalObject diff --git a/packages/next/src/build/swc/index.ts b/packages/next/src/build/swc/index.ts index 06eb571c85afd..0b35aee09bd5c 100644 --- a/packages/next/src/build/swc/index.ts +++ b/packages/next/src/build/swc/index.ts @@ -1054,6 +1054,7 @@ async function loadWasm(importPath = '') { Log.error('Wasm binding does not support trace yet') }, createTurboTasks: function ( + _outputPath: string, _memoryLimit?: number | undefined ): ExternalObject { throw new Error( @@ -1234,8 +1235,11 @@ function loadNative(importPath?: string) { turboTasks ) }, - createTurboTasks(memoryLimit?: number): ExternalObject { - return bindings.createTurboTasks(memoryLimit) + createTurboTasks( + outputPath: string, + memoryLimit?: number + ): ExternalObject { + return bindings.createTurboTasks(outputPath, memoryLimit) }, createProject: bindingToApi(customBindings ?? bindings, false), startTurbopackTraceServer(traceFilePath) { diff --git a/packages/next/src/build/swc/types.ts b/packages/next/src/build/swc/types.ts index 69d99816e1645..76eaf9a02e112 100644 --- a/packages/next/src/build/swc/types.ts +++ b/packages/next/src/build/swc/types.ts @@ -6,7 +6,10 @@ export interface Binding { isWasm: boolean turbo: { startTrace(options: any, turboTasks: ExternalObject): any - createTurboTasks(memoryLimit?: number): ExternalObject + createTurboTasks( + outputPath: string, + memoryLimit?: number + ): ExternalObject createProject( options: ProjectOptions, turboEngineOptions?: TurboEngineOptions @@ -320,6 +323,11 @@ export interface ProjectOptions { */ projectPath: string + /** + * The path to the .next directory. + */ + distDir: string + /** * The next.config.js contents. */ diff --git a/packages/next/src/server/dev/hot-reloader-turbopack.ts b/packages/next/src/server/dev/hot-reloader-turbopack.ts index 56068bb00d2ab..f35251289c010 100644 --- a/packages/next/src/server/dev/hot-reloader-turbopack.ts +++ b/packages/next/src/server/dev/hot-reloader-turbopack.ts @@ -150,6 +150,7 @@ export async function createHotReloaderTurbopack( opts.nextConfig.experimental.turbo?.root || opts.nextConfig.outputFileTracingRoot || dir, + distDir, nextConfig: opts.nextConfig, jsConfig: await getTurbopackJsConfig(dir, nextConfig), watch: { diff --git a/test/development/basic/next-rs-api.test.ts b/test/development/basic/next-rs-api.test.ts index 71e084d64080d..8d57bbc23b95f 100644 --- a/test/development/basic/next-rs-api.test.ts +++ b/test/development/basic/next-rs-api.test.ts @@ -190,6 +190,12 @@ describe('next.rs api', () => { console.log(next.testDir) const nextConfig = await loadConfig(PHASE_DEVELOPMENT_SERVER, next.testDir) const bindings = await loadBindings() + const distDir = path.join( + process.env.NEXT_SKIP_ISOLATE + ? path.resolve(__dirname, '../../..') + : next.testDir, + '.next' + ) project = await bindings.turbo.createProject({ env: {}, jsConfig: { @@ -197,6 +203,7 @@ describe('next.rs api', () => { }, nextConfig: nextConfig, projectPath: next.testDir, + distDir, rootPath: process.env.NEXT_SKIP_ISOLATE ? path.resolve(__dirname, '../../..') : next.testDir, @@ -209,12 +216,7 @@ describe('next.rs api', () => { clientRouterFilters: undefined, config: nextConfig, dev: true, - distDir: path.join( - process.env.NEXT_SKIP_ISOLATE - ? path.resolve(__dirname, '../../..') - : next.testDir, - '.next' - ), + distDir: distDir, fetchCacheKeyPrefix: undefined, hasRewrites: false, middlewareMatchers: undefined, diff --git a/turbopack/crates/node-file-trace/src/lib.rs b/turbopack/crates/node-file-trace/src/lib.rs index 5b6b09befbe36..387c95240ac2b 100644 --- a/turbopack/crates/node-file-trace/src/lib.rs +++ b/turbopack/crates/node-file-trace/src/lib.rs @@ -28,7 +28,6 @@ use turbo_tasks::{ use turbo_tasks_fs::{ glob::Glob, DirectoryEntry, DiskFileSystem, FileSystem, FileSystemPath, ReadGlobResult, }; -use turbo_tasks_memory::MemoryBackend; use turbopack::{ emit_asset, emit_with_completion, module_options::ModuleOptionsContext, rebase::RebasedAsset, ModuleAssetContext, @@ -177,7 +176,7 @@ fn default_output_directory() -> String { } impl Args { - fn common(&self) -> &CommonArgs { + pub fn common(&self) -> &CommonArgs { match self { Args::Print { common, .. } | Args::Annotate { common, .. } @@ -310,78 +309,16 @@ fn process_input(dir: &Path, context_directory: &str, input: &[String]) -> Resul .collect() } -pub async fn start( +pub async fn start( args: Arc, - turbo_tasks: Option<&Arc>>, + turbo_tasks: Arc>, module_options: Option, resolve_options: Option, ) -> Result> { register(); - let &CommonArgs { - memory_limit, - #[cfg(feature = "persistent_cache")] - cache: CacheArgs { - ref cache, - ref cache_fully, - }, - .. - } = args.common(); - #[cfg(feature = "persistent_cache")] - if let Some(cache) = cache { - use tokio::time::timeout; - use turbo_tasks_memory::MemoryBackendWithPersistedGraph; - use turbo_tasks_rocksdb::RocksDbPersistedGraph; - - run( - &args, - || { - let start = Instant::now(); - let backend = MemoryBackendWithPersistedGraph::new( - RocksDbPersistedGraph::new(cache).unwrap(), - ); - let tt = TurboTasks::new(backend); - let elapsed = start.elapsed(); - println!("restored cache {}", FormatDuration(elapsed)); - tt - }, - |tt, _, duration| async move { - let mut start = Instant::now(); - if *cache_fully { - tt.wait_background_done().await; - tt.stop_and_wait().await; - let elapsed = start.elapsed(); - println!("flushed cache {}", FormatDuration(elapsed)); - } else { - let background_timeout = - std::cmp::max(duration / 5, Duration::from_millis(100)); - let timed_out = timeout(background_timeout, tt.wait_background_done()) - .await - .is_err(); - tt.stop_and_wait().await; - let elapsed = start.elapsed(); - if timed_out { - println!("flushed cache partially {}", FormatDuration(elapsed)); - } else { - println!("flushed cache completely {}", FormatDuration(elapsed)); - } - } - start = Instant::now(); - drop(tt); - let elapsed = start.elapsed(); - println!("writing cache {}", FormatDuration(elapsed)); - }, - ) - .await; - return; - } - run( - args.clone(), - || { - turbo_tasks.cloned().unwrap_or_else(|| { - TurboTasks::new(MemoryBackend::new(memory_limit.unwrap_or(usize::MAX))) - }) - }, + args, + turbo_tasks, |_, _, _| async move {}, module_options, resolve_options, @@ -391,7 +328,7 @@ pub async fn start( async fn run>( args: Arc, - create_tt: impl Fn() -> Arc>, + tt: Arc>, final_finish: impl FnOnce(Arc>, TaskId, Duration) -> F, module_options: Option, resolve_options: Option, @@ -459,7 +396,6 @@ async fn run>( matches!(&*args, Args::Annotate { .. }) || matches!(&*args, Args::Print { .. }); let (sender, mut receiver) = channel(1); let dir = current_dir().unwrap(); - let tt = create_tt(); let module_options = TransientInstance::new(module_options.unwrap_or_default()); let resolve_options = TransientInstance::new(resolve_options.unwrap_or_default()); let log_options = TransientInstance::new(LogOptions { diff --git a/turbopack/crates/node-file-trace/src/main.rs b/turbopack/crates/node-file-trace/src/main.rs index 2e435166363da..77767706c2df9 100644 --- a/turbopack/crates/node-file-trace/src/main.rs +++ b/turbopack/crates/node-file-trace/src/main.rs @@ -5,6 +5,8 @@ use std::sync::Arc; use anyhow::Result; use clap::Parser; use node_file_trace::{start, Args}; +use turbo_tasks::TurboTasks; +use turbo_tasks_memory::MemoryBackend; #[global_allocator] static ALLOC: turbo_tasks_malloc::TurboMalloc = turbo_tasks_malloc::TurboMalloc; @@ -15,7 +17,10 @@ async fn main() -> Result<()> { console_subscriber::init(); let args = Arc::new(Args::parse()); let should_print = matches!(&*args, Args::Print { .. }); - let result = start(args, None, None, None).await?; + let turbo_tasks = TurboTasks::new(MemoryBackend::new( + args.common().memory_limit.unwrap_or(usize::MAX), + )); + let result = start(args, turbo_tasks, None, None).await?; if should_print { for file in result.iter() { println!("{}", file); From ec3fd548129beddc92512758246cbf2f500a1723 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Tue, 13 Aug 2024 07:18:26 +0200 Subject: [PATCH 21/81] initial aggregation update --- .../backend/operation/aggregation_update.rs | 98 +++++++++++++++++++ .../src/backend/operation/connect_child.rs | 41 ++++++-- .../src/backend/operation/mod.rs | 1 + 3 files changed, 134 insertions(+), 6 deletions(-) create mode 100644 turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs new file mode 100644 index 0000000000000..9b01fc286cac8 --- /dev/null +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs @@ -0,0 +1,98 @@ +use serde::{Deserialize, Serialize}; +use turbo_tasks::TaskId; + +use super::ExecuteContext; +use crate::data::{CachedDataItem, CachedDataItemKey}; + +#[derive(Serialize, Deserialize, Clone)] +pub enum AggregationUpdateJob { + InnerHasNewFollower { + upper_ids: Vec, + new_follower_id: TaskId, + new_follower_data: (), + }, +} + +#[derive(Default, Serialize, Deserialize, Clone)] +pub struct AggregationUpdateQueue { + jobs: Vec, +} + +impl AggregationUpdateQueue { + pub fn new() -> Self { + Self { jobs: Vec::new() } + } + + pub fn push(&mut self, job: AggregationUpdateJob) { + self.jobs.push(job); + } + + pub fn process(&mut self, ctx: &ExecuteContext<'_>) -> bool { + if let Some(job) = self.jobs.pop() { + match job { + AggregationUpdateJob::InnerHasNewFollower { + mut upper_ids, + new_follower_id, + new_follower_data, + } => { + upper_ids.retain(|&upper_id| { + // let mut upper = ctx.task(upper_id); + // TODO decide if it should be an inner or follower + // TODO for now: always inner + + // TODO add new_follower_data + // TODO propagate change to all uppers + + // TODO return true for inner, false for follower + true + }); + let children; + let data; + { + let mut follower = ctx.task(new_follower_id); + upper_ids.retain(|&upper_id| { + if follower.add(CachedDataItem::Upper { + task: upper_id, + value: (), + }) { + // It's a new upper + true + } else { + // It's already an upper + false + } + }); + if !upper_ids.is_empty() { + // TODO get data + data = (); + children = follower + .iter() + .filter_map(|(key, _)| match *key { + CachedDataItemKey::Child { task } => Some(task), + _ => None, + }) + .collect::>(); + } else { + data = Default::default(); + children = Default::default(); + } + } + for upper_id in upper_ids.iter() { + // TODO add data to upper + } + for child_id in children { + let child = ctx.task(child_id); + // TODO get child data + self.jobs.push(AggregationUpdateJob::InnerHasNewFollower { + upper_ids: upper_ids.clone(), + new_follower_id: child_id, + new_follower_data: (), + }) + } + } + } + } + + self.jobs.is_empty() + } +} diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs index 8bd3e39b7ce59..d08eee204de11 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs @@ -1,11 +1,18 @@ use serde::{Deserialize, Serialize}; use turbo_tasks::TaskId; -use super::{ExecuteContext, Operation}; -use crate::data::CachedDataItem; +use super::{ + aggregation_update::{AggregationUpdateJob, AggregationUpdateQueue}, + ExecuteContext, Operation, +}; +use crate::data::{CachedDataItem, CachedDataItemKey}; #[derive(Serialize, Deserialize, Clone, Default)] pub enum ConnectChildOperation { + UpdateAggregation { + task_id: TaskId, + aggregation_update: AggregationUpdateQueue, + }, ScheduleTask { task_id: TaskId, }, @@ -21,10 +28,23 @@ impl ConnectChildOperation { task: child_task, value: (), }) { - // TODO add aggregated edge - // TODO check for active - ConnectChildOperation::ScheduleTask { + // Update the task aggregation + let upper_ids = parent_task + .iter() + .filter_map(|(key, _)| match *key { + CachedDataItemKey::Upper { task } => Some(task), + _ => None, + }) + .collect::>(); + let mut queue = AggregationUpdateQueue::new(); + queue.push(AggregationUpdateJob::InnerHasNewFollower { + upper_ids, + new_follower_id: child_task, + new_follower_data: (), + }); + ConnectChildOperation::UpdateAggregation { task_id: child_task, + aggregation_update: queue, } .execute(&ctx); } @@ -36,6 +56,15 @@ impl Operation for ConnectChildOperation { loop { ctx.operation_suspend_point(&self); match self { + ConnectChildOperation::UpdateAggregation { + task_id, + ref mut aggregation_update, + } => { + if aggregation_update.process(ctx) { + // TODO check for active + self = ConnectChildOperation::ScheduleTask { task_id } + } + } ConnectChildOperation::ScheduleTask { task_id } => { { let mut task = ctx.task(task_id); @@ -44,8 +73,8 @@ impl Operation for ConnectChildOperation { ctx.schedule(task_id); self = ConnectChildOperation::Done; - continue; } + ConnectChildOperation::Done => { return; } diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs index 39b47fbd6cc0e..71230eaab83b0 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs @@ -1,3 +1,4 @@ +mod aggregation_update; mod cleanup_old_edges; mod connect_child; mod invalidate; From 584c8e337bac02cfa8505fedab641dd5bb21b8d0 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Tue, 13 Aug 2024 13:29:50 +0200 Subject: [PATCH 22/81] more aggregation operations --- .../turbo-tasks-backend/src/backend/mod.rs | 3 + .../backend/operation/aggregation_update.rs | 272 ++++++++++++++++-- .../backend/operation/cleanup_old_edges.rs | 36 ++- .../src/backend/operation/connect_child.rs | 38 +-- .../src/backend/operation/invalidate.rs | 112 +++++--- .../src/backend/operation/mod.rs | 51 ++++ .../src/backend/storage.rs | 110 +++++++ .../crates/turbo-tasks-backend/src/data.rs | 33 ++- .../src/derive/key_value_pair_macro.rs | 4 +- 9 files changed, 557 insertions(+), 102 deletions(-) diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs index 7a3577fdaf6dd..f818544c26823 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs @@ -657,6 +657,8 @@ impl Backend for TurboTasksBackend { }) .collect::>(); + task.remove(&CachedDataItemKey::Dirty {}); + done_event.notify(usize::MAX); drop(task); @@ -798,6 +800,7 @@ impl Backend for TurboTasksBackend { let mut task = self.storage.access_mut(task_id); task.add(CachedDataItem::new_scheduled(task_id)); task.add(CachedDataItem::AggregateRootType { value: root_type }); + task.add(CachedDataItem::AggregationNumber { value: u32::MAX }); } turbo_tasks.schedule(task_id); task_id diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs index 9b01fc286cac8..72fa2a37705fb 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs @@ -1,18 +1,166 @@ +use std::{collections::HashMap, ops::Add}; + use serde::{Deserialize, Serialize}; use turbo_tasks::TaskId; -use super::ExecuteContext; -use crate::data::{CachedDataItem, CachedDataItemKey}; +use super::{ExecuteContext, TaskGuard}; +use crate::{ + data::{CachedDataItem, CachedDataItemKey}, + get, get_many, update, update_count, +}; #[derive(Serialize, Deserialize, Clone)] pub enum AggregationUpdateJob { InnerHasNewFollower { upper_ids: Vec, new_follower_id: TaskId, - new_follower_data: (), + }, + InnerHasNewFollowers { + upper_ids: Vec, + new_follower_ids: Vec, + }, + InnerLostFollower { + upper_ids: Vec, + lost_follower_id: TaskId, + }, + AggregatedDataUpdate { + upper_ids: Vec, + update: AggregatedDataUpdate, + }, + DataUpdate { + task_id: TaskId, + update: AggregatedDataUpdate, + }, + ScheduleWhenDirty { + task_ids: Vec, }, } +#[derive(Default, Serialize, Deserialize, Clone)] +pub struct AggregatedDataUpdate { + unfinished: i32, + dirty_tasks_update: HashMap, + // TODO collectibles +} + +impl AggregatedDataUpdate { + fn from_task(task: &mut TaskGuard<'_>) -> Self { + let aggregation = get!(task, AggregationNumber); + if aggregation.is_some() { + let unfinished = get!(task, AggregatedUnfinishedTasks); + let dirty_tasks_update = task + .iter() + .filter_map(|(key, _)| match *key { + CachedDataItemKey::AggregatedDirtyTask { task } => Some((task, 1)), + _ => None, + }) + .collect(); + Self { + unfinished: unfinished.copied().unwrap_or(0) as i32, + dirty_tasks_update, + } + } else { + let dirty = get!(task, Dirty); + if dirty.is_some() { + Self::dirty_task(task.id()) + } else { + Self::default() + } + } + } + + fn apply( + &self, + task: &mut TaskGuard<'_>, + queue: &mut AggregationUpdateQueue, + ) -> AggregatedDataUpdate { + let Self { + unfinished, + dirty_tasks_update, + } = self; + let mut result = Self::default(); + if *unfinished != 0 { + update!(task, AggregatedUnfinishedTasks, |old: Option| { + let old = old.unwrap_or(0); + let new = (old as i32 + *unfinished) as u32; + if new == 0 { + result.unfinished = -1; + None + } else { + if old > 0 { + result.unfinished = 1; + } + Some(new) + } + }); + } + if !dirty_tasks_update.is_empty() { + let mut task_to_schedule = Vec::new(); + let root_type = get!(task, AggregateRootType).copied(); + for (task_id, count) in dirty_tasks_update { + update!( + task, + AggregatedDirtyTask { task: *task_id }, + |old: Option| { + let old = old.unwrap_or(0); + if old == 0 { + if root_type.is_some() { + task_to_schedule.push(*task_id); + } + } + let new = (old as i32 + *count) as u32; + if new == 0 { + result.dirty_tasks_update.insert(*task_id, -1); + None + } else { + if old > 0 { + result.dirty_tasks_update.insert(*task_id, 1); + } + Some(new) + } + } + ); + } + if !task_to_schedule.is_empty() { + queue.push(AggregationUpdateJob::ScheduleWhenDirty { + task_ids: task_to_schedule, + }) + } + } + result + } + + fn is_empty(&self) -> bool { + let Self { + unfinished, + dirty_tasks_update, + } = self; + *unfinished == 0 && dirty_tasks_update.is_empty() + } + + pub fn dirty_task(task_id: TaskId) -> Self { + Self { + unfinished: 1, + dirty_tasks_update: HashMap::from([(task_id, 1)]), + } + } +} + +impl Add for AggregatedDataUpdate { + type Output = Self; + + fn add(self, rhs: Self) -> Self::Output { + let mut dirty_tasks_update = self.dirty_tasks_update; + for (task, count) in rhs.dirty_tasks_update { + *dirty_tasks_update.entry(task).or_default() += count; + } + Self { + unfinished: self.unfinished + rhs.unfinished, + dirty_tasks_update, + } + } +} + #[derive(Default, Serialize, Deserialize, Clone)] pub struct AggregationUpdateQueue { jobs: Vec, @@ -23,6 +171,10 @@ impl AggregationUpdateQueue { Self { jobs: Vec::new() } } + pub fn is_empty(&self) -> bool { + self.jobs.is_empty() + } + pub fn push(&mut self, job: AggregationUpdateJob) { self.jobs.push(job); } @@ -30,12 +182,33 @@ impl AggregationUpdateQueue { pub fn process(&mut self, ctx: &ExecuteContext<'_>) -> bool { if let Some(job) = self.jobs.pop() { match job { + AggregationUpdateJob::InnerHasNewFollowers { + upper_ids, + mut new_follower_ids, + } => { + if let Some(new_follower_id) = new_follower_ids.pop() { + if new_follower_ids.is_empty() { + self.jobs.push(AggregationUpdateJob::InnerHasNewFollower { + upper_ids, + new_follower_id, + }); + } else { + self.jobs.push(AggregationUpdateJob::InnerHasNewFollowers { + upper_ids: upper_ids.clone(), + new_follower_ids, + }); + self.jobs.push(AggregationUpdateJob::InnerHasNewFollower { + upper_ids, + new_follower_id, + }); + } + } + } AggregationUpdateJob::InnerHasNewFollower { mut upper_ids, new_follower_id, - new_follower_data, } => { - upper_ids.retain(|&upper_id| { + upper_ids.retain(|&_upper_id| { // let mut upper = ctx.task(upper_id); // TODO decide if it should be an inner or follower // TODO for now: always inner @@ -46,15 +219,12 @@ impl AggregationUpdateQueue { // TODO return true for inner, false for follower true }); - let children; + let children: Vec; let data; { let mut follower = ctx.task(new_follower_id); upper_ids.retain(|&upper_id| { - if follower.add(CachedDataItem::Upper { - task: upper_id, - value: (), - }) { + if update_count!(follower, Upper { task: upper_id }, 1) { // It's a new upper true } else { @@ -63,31 +233,79 @@ impl AggregationUpdateQueue { } }); if !upper_ids.is_empty() { - // TODO get data - data = (); - children = follower - .iter() - .filter_map(|(key, _)| match *key { - CachedDataItemKey::Child { task } => Some(task), - _ => None, - }) - .collect::>(); + data = AggregatedDataUpdate::from_task(&mut follower); + children = get_many!(follower, Child { task } => task); } else { data = Default::default(); children = Default::default(); } } for upper_id in upper_ids.iter() { - // TODO add data to upper + // add data to upper + let mut upper = ctx.task(*upper_id); + let diff = data.apply(&mut upper, self); + if !diff.is_empty() { + let upper_ids = get_many!(upper, Upper { task } => task); + self.jobs.push(AggregationUpdateJob::AggregatedDataUpdate { + upper_ids, + update: diff, + }) + } } - for child_id in children { - let child = ctx.task(child_id); - // TODO get child data - self.jobs.push(AggregationUpdateJob::InnerHasNewFollower { + if !children.is_empty() { + self.jobs.push(AggregationUpdateJob::InnerHasNewFollowers { upper_ids: upper_ids.clone(), - new_follower_id: child_id, - new_follower_data: (), - }) + new_follower_ids: children, + }); + } + } + AggregationUpdateJob::InnerLostFollower { + upper_ids, + lost_follower_id, + } => { + for upper_id in upper_ids { + let mut upper = ctx.task(upper_id); + upper.remove(&CachedDataItemKey::Upper { + task: lost_follower_id, + }); + let diff = AggregatedDataUpdate::dirty_task(lost_follower_id); + let upper_ids = get_many!(upper, Upper { task } => task); + self.jobs.push(AggregationUpdateJob::AggregatedDataUpdate { + upper_ids, + update: diff, + }); + } + } + AggregationUpdateJob::AggregatedDataUpdate { upper_ids, update } => { + for upper_id in upper_ids { + let mut upper = ctx.task(upper_id); + let diff = update.apply(&mut upper, self); + if !diff.is_empty() { + let upper_ids = get_many!(upper, Upper { task } => task); + self.jobs.push(AggregationUpdateJob::AggregatedDataUpdate { + upper_ids, + update: diff, + }); + } + } + } + AggregationUpdateJob::DataUpdate { task_id, update } => { + let mut task = ctx.task(task_id); + let diff = update.apply(&mut task, self); + if !diff.is_empty() { + let upper_ids = get_many!(task, Upper { task } => task); + self.jobs.push(AggregationUpdateJob::AggregatedDataUpdate { + upper_ids, + update: diff, + }); + } + } + AggregationUpdateJob::ScheduleWhenDirty { task_ids } => { + for task_id in task_ids { + let mut task = ctx.task(task_id); + if task.add(CachedDataItem::new_scheduled(task_id)) { + ctx.turbo_tasks.schedule(task_id); + } } } } diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/cleanup_old_edges.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/cleanup_old_edges.rs index e9eb3738d84ee..e4dc93f6a6c2c 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/cleanup_old_edges.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/cleanup_old_edges.rs @@ -1,14 +1,26 @@ +use std::mem::take; + use serde::{Deserialize, Serialize}; use turbo_tasks::TaskId; -use super::{ExecuteContext, Operation}; -use crate::data::{CachedDataItemKey, CellRef}; +use super::{ + aggregation_update::{AggregationUpdateJob, AggregationUpdateQueue}, + ExecuteContext, Operation, +}; +use crate::{ + data::{CachedDataItemKey, CellRef}, + get_many, +}; #[derive(Serialize, Deserialize, Clone, Default)] pub enum CleanupOldEdgesOperation { RemoveEdges { task_id: TaskId, outdated: Vec, + queue: AggregationUpdateQueue, + }, + AggregationUpdate { + queue: AggregationUpdateQueue, }, #[default] Done, @@ -24,7 +36,12 @@ pub enum OutdatedEdge { impl CleanupOldEdgesOperation { pub fn run(task_id: TaskId, outdated: Vec, ctx: ExecuteContext<'_>) { - CleanupOldEdgesOperation::RemoveEdges { task_id, outdated }.execute(&ctx); + CleanupOldEdgesOperation::RemoveEdges { + task_id, + outdated, + queue: AggregationUpdateQueue::new(), + } + .execute(&ctx); } } @@ -36,13 +53,18 @@ impl Operation for CleanupOldEdgesOperation { CleanupOldEdgesOperation::RemoveEdges { task_id, ref mut outdated, + ref mut queue, } => { if let Some(edge) = outdated.pop() { match edge { OutdatedEdge::Child(child_id) => { let mut task = ctx.task(task_id); task.remove(&CachedDataItemKey::Child { task: child_id }); - // TODO remove aggregated edge + let upper_ids = get_many!(task, Upper { task } => task); + queue.push(AggregationUpdateJob::InnerLostFollower { + upper_ids, + lost_follower_id: child_id, + }); } OutdatedEdge::CellDependency(CellRef { task: cell_task_id, @@ -83,9 +105,13 @@ impl Operation for CleanupOldEdgesOperation { } if outdated.is_empty() { + self = CleanupOldEdgesOperation::AggregationUpdate { queue: take(queue) }; + } + } + CleanupOldEdgesOperation::AggregationUpdate { ref mut queue } => { + if queue.process(ctx) { self = CleanupOldEdgesOperation::Done; } - continue; } CleanupOldEdgesOperation::Done => { return; diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs index d08eee204de11..5fb5847fe2eb3 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs @@ -5,7 +5,10 @@ use super::{ aggregation_update::{AggregationUpdateJob, AggregationUpdateQueue}, ExecuteContext, Operation, }; -use crate::data::{CachedDataItem, CachedDataItemKey}; +use crate::{ + data::{CachedDataItem, CachedDataItemKey}, + get, get_many, +}; #[derive(Serialize, Deserialize, Clone, Default)] pub enum ConnectChildOperation { @@ -22,28 +25,29 @@ pub enum ConnectChildOperation { } impl ConnectChildOperation { - pub fn run(parent_task: TaskId, child_task: TaskId, ctx: ExecuteContext<'_>) { - let mut parent_task = ctx.task(parent_task); + pub fn run(parent_task_id: TaskId, child_task_id: TaskId, ctx: ExecuteContext<'_>) { + let mut parent_task = ctx.task(parent_task_id); if parent_task.add(CachedDataItem::Child { - task: child_task, + task: child_task_id, value: (), }) { // Update the task aggregation - let upper_ids = parent_task - .iter() - .filter_map(|(key, _)| match *key { - CachedDataItemKey::Upper { task } => Some(task), - _ => None, - }) - .collect::>(); let mut queue = AggregationUpdateQueue::new(); - queue.push(AggregationUpdateJob::InnerHasNewFollower { - upper_ids, - new_follower_id: child_task, - new_follower_data: (), - }); + if get!(parent_task, AggregationNumber).is_some() { + queue.push(AggregationUpdateJob::InnerHasNewFollower { + upper_ids: vec![parent_task_id], + new_follower_id: child_task_id, + }); + } else { + let upper_ids = get_many!(parent_task, Upper { task } => task); + queue.push(AggregationUpdateJob::InnerHasNewFollower { + upper_ids, + new_follower_id: child_task_id, + }); + } + drop(parent_task); ConnectChildOperation::UpdateAggregation { - task_id: child_task, + task_id: child_task_id, aggregation_update: queue, } .execute(&ctx); diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/invalidate.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/invalidate.rs index e817da6a4933a..07fb67e5fe180 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/invalidate.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/invalidate.rs @@ -2,10 +2,13 @@ use serde::{Deserialize, Serialize}; use smallvec::SmallVec; use turbo_tasks::TaskId; -use super::{ExecuteContext, Operation}; +use super::{ + aggregation_update::{AggregatedDataUpdate, AggregationUpdateJob, AggregationUpdateQueue}, + ExecuteContext, Operation, +}; use crate::{ data::{CachedDataItem, InProgressState}, - get, remove, + get, update, }; #[derive(Serialize, Deserialize, Clone, Default)] @@ -14,6 +17,9 @@ pub enum InvalidateOperation { MakeDirty { task_ids: SmallVec<[TaskId; 4]>, }, + AggregationUpdate { + queue: AggregationUpdateQueue, + }, // TODO Add to dirty tasks list #[default] Done, @@ -31,61 +37,75 @@ impl Operation for InvalidateOperation { ctx.operation_suspend_point(&self); match self { InvalidateOperation::MakeDirty { task_ids } => { + let mut queue = AggregationUpdateQueue::new(); for task_id in task_ids { let mut task = ctx.task(task_id); - let in_progress = match get!(task, InProgress) { - Some(InProgressState::Scheduled { clean, .. }) => { - if *clean { - let Some(InProgressState::Scheduled { - clean: _, - done_event, - start_event, - }) = remove!(task, InProgress) - else { - unreachable!(); - }; - task.insert(CachedDataItem::InProgress { - value: InProgressState::Scheduled { - clean: false, - done_event, - start_event, - }, - }); + + if task.add(CachedDataItem::Dirty { value: () }) { + let in_progress = match get!(task, InProgress) { + Some(InProgressState::Scheduled { clean, .. }) => { + if *clean { + update!(task, InProgress, |in_progress| { + let Some(InProgressState::Scheduled { + clean: _, + done_event, + start_event, + }) = in_progress + else { + unreachable!(); + }; + Some(InProgressState::Scheduled { + clean: false, + done_event, + start_event, + }) + }); + } + true } - true - } - Some(InProgressState::InProgress { clean, stale, .. }) => { - if *clean || !*stale { - let Some(InProgressState::InProgress { - clean: _, - stale: _, - done_event, - }) = remove!(task, InProgress) - else { - unreachable!(); - }; - task.insert(CachedDataItem::InProgress { - value: InProgressState::InProgress { - clean: false, - stale: true, - done_event, - }, - }); + Some(InProgressState::InProgress { clean, stale, .. }) => { + if *clean || !*stale { + update!(task, InProgress, |in_progress| { + let Some(InProgressState::InProgress { + clean: _, + stale: _, + done_event, + }) = in_progress + else { + unreachable!(); + }; + Some(InProgressState::InProgress { + clean: false, + stale: true, + done_event, + }) + }); + } + true } - true - } - None => false, - }; - #[allow(clippy::collapsible_if)] - if task.add(CachedDataItem::Dirty { value: () }) { + None => false, + }; if !in_progress && task.add(CachedDataItem::new_scheduled(task_id)) { ctx.turbo_tasks.schedule(task_id) } + queue.push(AggregationUpdateJob::DataUpdate { + task_id, + update: AggregatedDataUpdate::dirty_task(task_id), + }) } } - self = InvalidateOperation::Done; + if queue.is_empty() { + self = InvalidateOperation::Done + } else { + self = InvalidateOperation::AggregationUpdate { queue } + } continue; } + InvalidateOperation::AggregationUpdate { ref mut queue } => { + if queue.process(ctx) { + self = InvalidateOperation::Done + } + } InvalidateOperation::Done => { return; } diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs index 71230eaab83b0..e9057c081bccf 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs @@ -122,6 +122,10 @@ impl<'a> Debug for TaskGuard<'a> { } impl<'a> TaskGuard<'a> { + pub fn id(&self) -> TaskId { + self.task_id + } + pub fn add(&mut self, item: CachedDataItem) -> bool { if !item.is_persistent() { self.task.add(item) @@ -180,6 +184,53 @@ impl<'a> TaskGuard<'a> { } } + pub fn update( + &mut self, + key: &CachedDataItemKey, + update: impl FnOnce(Option) -> Option, + ) { + if !key.is_persistent() { + self.task.update(key, update); + return; + } + let Self { + task, + task_id, + backend, + } = self; + let mut add_persisting_item = false; + task.update(key, |old| { + let old_persistent = old.as_ref().map(|old| old.is_persistent()).unwrap_or(false); + let new = update(old); + let new_persistent = new.as_ref().map(|new| new.is_persistent()).unwrap_or(false); + + match (old_persistent, new_persistent) { + (false, false) => {} + (true, false) => { + add_persisting_item = true; + backend.persisted_storage_log.lock().push(CachedDataUpdate { + key: key.clone(), + task: *task_id, + value: None, + }); + } + (_, true) => { + add_persisting_item = true; + backend.persisted_storage_log.lock().push(CachedDataUpdate { + key: key.clone(), + task: *task_id, + value: new.clone(), + }); + } + } + + new + }); + if add_persisting_item { + // TODO task.persistance_state.add_persisting_item(); + } + } + pub fn remove(&mut self, key: &CachedDataItemKey) -> Option { let old_value = self.task.remove(key); if let Some(value) = old_value { diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs b/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs index 5549582696352..b8e5544ec4c7c 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs @@ -55,6 +55,31 @@ impl InnerStorage { } } +impl InnerStorage +where + T::Value: Default, + T::Key: Clone, +{ + pub fn update( + &mut self, + key: &T::Key, + update: impl FnOnce(Option) -> Option, + ) { + if let Some(value) = self.map.get_mut(key) { + let v = take(value); + if let Some(v) = update(Some(v)) { + *value = v; + } else { + self.map.remove(key); + } + } else { + if let Some(v) = update(None) { + self.map.insert(key.clone(), v); + } + } + } +} + impl InnerStorage where T::Value: PartialEq, @@ -135,6 +160,91 @@ macro_rules! get { }; } +#[macro_export] +macro_rules! get_many { + ($task:ident, $key:ident $input:tt => $value:ident) => { + $task + .iter() + .filter_map(|(key, _)| match *key { + CachedDataItemKey::$key $input => Some($value), + _ => None, + }) + .collect() + }; + ($task:ident, $key1:ident $input1:tt => $value1:ident, $key2:ident $input2:tt => $value2:ident) => { + $task + .iter() + .filter_map(|(key, _)| match *key { + CachedDataItemKey::$key1 $input1 => Some($value1), + CachedDataItemKey::$key2 $input2 => Some($value2), + _ => None, + }) + .collect() + }; +} + +#[macro_export] +macro_rules! update { + ($task:ident, $key:ident $input:tt, $update:expr) => { + #[allow(unused_mut)] + match $update { + mut update => $task.update(&$crate::data::CachedDataItemKey::$key $input, |old| { + update(old.and_then(|old| { + if let $crate::data::CachedDataItemValue::$key { value } = old { + Some(value) + } else { + None + } + })) + .map(|new| $crate::data::CachedDataItemValue::$key { value: new }) + }) + } + }; + ($task:ident, $key:ident, $update:expr) => { + #[allow(unused_mut)] + match $update { + mut update => $task.update(&$crate::data::CachedDataItemKey::$key {}, |old| { + update(old.and_then(|old| { + if let $crate::data::CachedDataItemValue::$key { value } = old { + Some(value) + } else { + None + } + })) + .map(|new| $crate::data::CachedDataItemValue::$key { value: new }) + }) + } + }; +} + +#[macro_export] +macro_rules! update_count { + ($task:ident, $key:ident $input:tt, $update:expr) => { + match $update { + update => { + let mut state_change = false; + $crate::update!($task, $key $input, |old: Option| { + if old.is_none() { + state_change = true; + } + let old = old.unwrap_or(0); + let new = old as i32 + update; + if new == 0 { + state_change = true; + None + } else { + Some(new as u32) + } + }); + state_change + } + } + }; + ($task:ident, $key:ident, $update:expr) => { + $crate::update_count!($task, $key {}, $update) + }; +} + #[macro_export] macro_rules! remove { ($task:ident, $key:ident $input:tt) => { diff --git a/turbopack/crates/turbo-tasks-backend/src/data.rs b/turbopack/crates/turbo-tasks-backend/src/data.rs index 3e1daf0272c2c..6a091d7486336 100644 --- a/turbopack/crates/turbo-tasks-backend/src/data.rs +++ b/turbopack/crates/turbo-tasks-backend/src/data.rs @@ -1,5 +1,7 @@ use serde::{Deserialize, Serialize}; -use turbo_tasks::{event::Event, util::SharedError, CellId, KeyValuePair, SharedReference, TaskId}; +use turbo_tasks::{ + event::Event, util::SharedError, CellId, KeyValuePair, SharedReference, TaskId, ValueTypeId, +}; #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq, Serialize, Deserialize)] pub struct CellRef { @@ -7,7 +9,13 @@ pub struct CellRef { pub cell: CellId, } -#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq, Serialize, Deserialize)] +pub struct CollectiblesRef { + pub task: TaskId, + pub collectible_type: ValueTypeId, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize)] pub enum OutputValue { Cell(CellRef), Output(TaskId), @@ -92,6 +100,10 @@ pub enum CachedDataItem { target: CellRef, value: (), }, + CollectiblesDependency { + target: CollectiblesRef, + value: (), + }, // Dependent OutputDependent { @@ -103,6 +115,11 @@ pub enum CachedDataItem { task: TaskId, value: (), }, + CollectiblesDependent { + collectibles_type: ValueTypeId, + task: TaskId, + value: (), + }, // Aggregation Graph AggregationNumber { @@ -110,21 +127,21 @@ pub enum CachedDataItem { }, Follower { task: TaskId, - value: (), + value: u32, }, Upper { task: TaskId, - value: (), + value: u32, }, // Aggregated Data AggregatedDirtyTask { task: TaskId, - value: (), + value: u32, }, AggregatedCollectible { collectible: CellRef, - value: (), + value: u32, }, AggregatedUnfinishedTasks { value: u32, @@ -173,8 +190,10 @@ impl CachedDataItem { CachedDataItem::CellData { .. } => true, CachedDataItem::OutputDependency { target, .. } => !target.is_transient(), CachedDataItem::CellDependency { target, .. } => !target.task.is_transient(), + CachedDataItem::CollectiblesDependency { target, .. } => !target.task.is_transient(), CachedDataItem::OutputDependent { task, .. } => !task.is_transient(), CachedDataItem::CellDependent { task, .. } => !task.is_transient(), + CachedDataItem::CollectiblesDependent { task, .. } => !task.is_transient(), CachedDataItem::AggregationNumber { .. } => true, CachedDataItem::Follower { task, .. } => !task.is_transient(), CachedDataItem::Upper { task, .. } => !task.is_transient(), @@ -215,8 +234,10 @@ impl CachedDataItemKey { CachedDataItemKey::CellData { .. } => true, CachedDataItemKey::OutputDependency { target, .. } => !target.is_transient(), CachedDataItemKey::CellDependency { target, .. } => !target.task.is_transient(), + CachedDataItemKey::CollectiblesDependency { target, .. } => !target.task.is_transient(), CachedDataItemKey::OutputDependent { task, .. } => !task.is_transient(), CachedDataItemKey::CellDependent { task, .. } => !task.is_transient(), + CachedDataItemKey::CollectiblesDependent { task, .. } => !task.is_transient(), CachedDataItemKey::AggregationNumber { .. } => true, CachedDataItemKey::Follower { task, .. } => !task.is_transient(), CachedDataItemKey::Upper { task, .. } => !task.is_transient(), diff --git a/turbopack/crates/turbo-tasks-macros/src/derive/key_value_pair_macro.rs b/turbopack/crates/turbo-tasks-macros/src/derive/key_value_pair_macro.rs index 3026d73f5dcd3..a3a0d192ec9a2 100644 --- a/turbopack/crates/turbo-tasks-macros/src/derive/key_value_pair_macro.rs +++ b/turbopack/crates/turbo-tasks-macros/src/derive/key_value_pair_macro.rs @@ -106,13 +106,15 @@ pub fn derive_key_value_pair(input: TokenStream) -> TokenStream { )* } - #[derive(Debug, Clone)] + #[derive(Debug, Clone, Default)] #vis enum #value_name { #( #variant_names { #value_decl }, )* + #[default] + Reserved, } } .into() From b7316a1282f84bdaa74fa6082f627c3a2eb93397 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Thu, 15 Aug 2024 11:28:30 +0200 Subject: [PATCH 23/81] remove exceeding cells --- .../turbo-tasks-backend/src/backend/mod.rs | 111 +++++++++++++++-- .../backend/operation/cleanup_old_edges.rs | 5 + .../src/backend/operation/invalidate.rs | 112 +++++++++--------- .../src/backend/storage.rs | 9 ++ .../crates/turbo-tasks-backend/src/data.rs | 54 ++++++++- 5 files changed, 227 insertions(+), 64 deletions(-) diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs index f818544c26823..38191351498c1 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs @@ -4,7 +4,7 @@ mod storage; use std::{ borrow::Cow, - collections::HashSet, + collections::{HashMap, HashSet}, future::Future, hash::BuildHasherDefault, pin::Pin, @@ -15,7 +15,7 @@ use std::{ time::Duration, }; -use anyhow::Result; +use anyhow::{bail, Result}; use auto_hash_map::{AutoMap, AutoSet}; use dashmap::DashMap; pub use operation::AnyOperation; @@ -39,9 +39,9 @@ use self::{operation::ExecuteContext, storage::Storage}; use crate::{ data::{ CachedDataItem, CachedDataItemKey, CachedDataItemValue, CachedDataUpdate, CellRef, - InProgressState, OutputValue, RootType, + InProgressCellState, InProgressState, OutputValue, RootType, }, - get, remove, + get, get_many, remove, utils::{bi_map::BiMap, chunked_vec::ChunkedVec, ptr_eq_arc::PtrEqArc}, }; @@ -253,7 +253,14 @@ impl TurboTasksBackend { } } - todo!("Output of is not available, recompute task: {task:#?}"); + // Output doesn't exist. We need to schedule the task to compute it. + let dirty = task.has_key(&CachedDataItemKey::Dirty {}); + let (item, listener) = CachedDataItem::new_scheduled_with_listener(task_id, !dirty); + if task.add(item) { + turbo_tasks.schedule(task_id); + } + + Ok(Err(listener)) } fn try_read_task_cell( @@ -289,7 +296,45 @@ impl TurboTasksBackend { return Ok(Ok(CellContent(Some(content)).into_typed(cell.type_id))); } - todo!("Cell {cell:?} is not available, recompute task or error: {task:#?}"); + // Check cell index range (cell might not exist at all) + let Some(max_id) = get!( + task, + CellTypeMaxIndex { + cell_type: cell.type_id + } + ) else { + bail!( + "Cell {cell:?} no longer exists in task {task_id:?} (no cell of this type exists)" + ); + }; + if cell.index > *max_id { + bail!("Cell {cell:?} no longer exists in task {task_id:?} (index out of bounds)"); + } + + // Cell should exist, but data was dropped or is not serializable. We need to recompute the + // task the get the cell content. + + // Register event listener for cell computation + if let Some(in_progress) = get!(task, InProgressCell { cell }) { + // Someone else is already computing the cell + let listener = in_progress.event.listen(); + return Ok(Err(listener)); + } + + // We create the event and potentially schedule the task + let in_progress = InProgressCellState::new(task_id, cell); + let listener = in_progress.event.listen(); + task.add(CachedDataItem::InProgressCell { + cell, + value: in_progress, + }); + + // Schedule the task + if task.add(CachedDataItem::new_scheduled(task_id)) { + turbo_tasks.schedule(task_id); + } + + Ok(Err(listener)) } fn lookup_task_type(&self, task_id: TaskId) -> Option> { @@ -626,9 +671,6 @@ impl Backend for TurboTasksBackend { panic!("Task execution completed, but task is not in progress: {task:#?}"); }; - // TODO handle cell counters - let _ = cell_counters; - // TODO handle stateful let _ = stateful; @@ -643,6 +685,48 @@ impl Backend for TurboTasksBackend { drop(task); drop(ctx); } else { + // handle cell counters: update max index and remove cells that are no longer used + let mut removed_cells = HashMap::new(); + let mut old_counters: HashMap<_, _> = + get_many!(task, CellTypeMaxIndex { cell_type } max_index => (cell_type, max_index)); + for (&cell_type, &max_index) in cell_counters.iter() { + if let Some(old_max_index) = old_counters.remove(&cell_type) { + if old_max_index != max_index { + task.insert(CachedDataItem::CellTypeMaxIndex { + cell_type, + value: max_index, + }); + if old_max_index > max_index { + removed_cells.insert(cell_type, max_index + 1..=old_max_index); + } + } + } else { + task.add(CachedDataItem::CellTypeMaxIndex { + cell_type, + value: max_index, + }); + } + } + for (cell_type, old_max_index) in old_counters { + task.remove(&CachedDataItemKey::CellTypeMaxIndex { cell_type }); + removed_cells.insert(cell_type, 0..=old_max_index); + } + let mut removed_data = Vec::new(); + for (&cell_type, range) in removed_cells.iter() { + for index in range.clone() { + removed_data.extend( + task.remove(&CachedDataItemKey::CellData { + cell: CellId { + type_id: cell_type, + index, + }, + }) + .into_iter(), + ); + } + } + + // find all outdated data items (removed cells, outdated edges) let old_edges = task .iter() .filter_map(|(key, _)| match *key { @@ -653,6 +737,13 @@ impl Backend for TurboTasksBackend { CachedDataItemKey::OutdatedOutputDependency { target } => { Some(OutdatedEdge::OutputDependency(target)) } + CachedDataItemKey::CellDependent { cell, task } + if removed_cells + .get(&cell.type_id) + .map_or(false, |range| range.contains(&cell.index)) => + { + Some(OutdatedEdge::RemovedCellDependent(task)) + } _ => None, }) .collect::>(); @@ -663,6 +754,8 @@ impl Backend for TurboTasksBackend { drop(task); CleanupOldEdgesOperation::run(task_id, old_edges, ctx); + + drop(removed_data) } stale diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/cleanup_old_edges.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/cleanup_old_edges.rs index e4dc93f6a6c2c..ecd71c9df6fff 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/cleanup_old_edges.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/cleanup_old_edges.rs @@ -5,6 +5,7 @@ use turbo_tasks::TaskId; use super::{ aggregation_update::{AggregationUpdateJob, AggregationUpdateQueue}, + invalidate::make_task_dirty, ExecuteContext, Operation, }; use crate::{ @@ -32,6 +33,7 @@ pub enum OutdatedEdge { Child(TaskId), CellDependency(CellRef), OutputDependency(TaskId), + RemovedCellDependent(TaskId), } impl CleanupOldEdgesOperation { @@ -101,6 +103,9 @@ impl Operation for CleanupOldEdgesOperation { }); } } + OutdatedEdge::RemovedCellDependent(task_id) => { + make_task_dirty(task_id, queue, ctx); + } } } diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/invalidate.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/invalidate.rs index 07fb67e5fe180..c4c42415c2e32 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/invalidate.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/invalidate.rs @@ -39,60 +39,7 @@ impl Operation for InvalidateOperation { InvalidateOperation::MakeDirty { task_ids } => { let mut queue = AggregationUpdateQueue::new(); for task_id in task_ids { - let mut task = ctx.task(task_id); - - if task.add(CachedDataItem::Dirty { value: () }) { - let in_progress = match get!(task, InProgress) { - Some(InProgressState::Scheduled { clean, .. }) => { - if *clean { - update!(task, InProgress, |in_progress| { - let Some(InProgressState::Scheduled { - clean: _, - done_event, - start_event, - }) = in_progress - else { - unreachable!(); - }; - Some(InProgressState::Scheduled { - clean: false, - done_event, - start_event, - }) - }); - } - true - } - Some(InProgressState::InProgress { clean, stale, .. }) => { - if *clean || !*stale { - update!(task, InProgress, |in_progress| { - let Some(InProgressState::InProgress { - clean: _, - stale: _, - done_event, - }) = in_progress - else { - unreachable!(); - }; - Some(InProgressState::InProgress { - clean: false, - stale: true, - done_event, - }) - }); - } - true - } - None => false, - }; - if !in_progress && task.add(CachedDataItem::new_scheduled(task_id)) { - ctx.turbo_tasks.schedule(task_id) - } - queue.push(AggregationUpdateJob::DataUpdate { - task_id, - update: AggregatedDataUpdate::dirty_task(task_id), - }) - } + make_task_dirty(task_id, &mut queue, ctx); } if queue.is_empty() { self = InvalidateOperation::Done @@ -113,3 +60,60 @@ impl Operation for InvalidateOperation { } } } + +pub fn make_task_dirty(task_id: TaskId, queue: &mut AggregationUpdateQueue, ctx: &ExecuteContext) { + let mut task = ctx.task(task_id); + + if task.add(CachedDataItem::Dirty { value: () }) { + let in_progress = match get!(task, InProgress) { + Some(InProgressState::Scheduled { clean, .. }) => { + if *clean { + update!(task, InProgress, |in_progress| { + let Some(InProgressState::Scheduled { + clean: _, + done_event, + start_event, + }) = in_progress + else { + unreachable!(); + }; + Some(InProgressState::Scheduled { + clean: false, + done_event, + start_event, + }) + }); + } + true + } + Some(InProgressState::InProgress { clean, stale, .. }) => { + if *clean || !*stale { + update!(task, InProgress, |in_progress| { + let Some(InProgressState::InProgress { + clean: _, + stale: _, + done_event, + }) = in_progress + else { + unreachable!(); + }; + Some(InProgressState::InProgress { + clean: false, + stale: true, + done_event, + }) + }); + } + true + } + None => false, + }; + if !in_progress && task.add(CachedDataItem::new_scheduled(task_id)) { + ctx.turbo_tasks.schedule(task_id) + } + queue.push(AggregationUpdateJob::DataUpdate { + task_id, + update: AggregatedDataUpdate::dirty_task(task_id), + }) + } +} diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs b/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs index b8e5544ec4c7c..8474658b0278f 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs @@ -171,6 +171,15 @@ macro_rules! get_many { }) .collect() }; + ($task:ident, $key:ident $input:tt $value_ident:ident => $value:expr) => { + $task + .iter() + .filter_map(|(key, value)| match (key, value) { + (&CachedDataItemKey::$key $input, &CachedDataItemValue::$key { value: $value_ident }) => Some($value), + _ => None, + }) + .collect() + }; ($task:ident, $key1:ident $input1:tt => $value1:ident, $key2:ident $input2:tt => $value2:ident) => { $task .iter() diff --git a/turbopack/crates/turbo-tasks-backend/src/data.rs b/turbopack/crates/turbo-tasks-backend/src/data.rs index 6a091d7486336..bdf783bc33531 100644 --- a/turbopack/crates/turbo-tasks-backend/src/data.rs +++ b/turbopack/crates/turbo-tasks-backend/src/data.rs @@ -1,6 +1,8 @@ use serde::{Deserialize, Serialize}; use turbo_tasks::{ - event::Event, util::SharedError, CellId, KeyValuePair, SharedReference, TaskId, ValueTypeId, + event::{Event, EventListener}, + util::SharedError, + CellId, KeyValuePair, SharedReference, TaskId, ValueTypeId, }; #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq, Serialize, Deserialize)] @@ -43,11 +45,13 @@ pub enum RootType { #[derive(Debug)] pub enum InProgressState { Scheduled { + // TODO remove in favor of Dirty clean: bool, done_event: Event, start_event: Event, }, InProgress { + // TODO remove in favor of Dirty clean: bool, stale: bool, done_event: Event, @@ -60,6 +64,27 @@ impl Clone for InProgressState { } } +#[derive(Debug)] +pub struct InProgressCellState { + pub event: Event, +} + +impl Clone for InProgressCellState { + fn clone(&self) -> Self { + panic!("InProgressCell cannot be cloned"); + } +} + +impl InProgressCellState { + pub fn new(task_id: TaskId, cell: CellId) -> Self { + InProgressCellState { + event: Event::new(move || { + format!("InProgressCellState::event ({} {:?})", task_id, cell) + }), + } + } +} + #[derive(Debug, Clone, KeyValuePair)] pub enum CachedDataItem { // Output @@ -90,6 +115,10 @@ pub enum CachedDataItem { cell: CellId, value: SharedReference, }, + CellTypeMaxIndex { + cell_type: ValueTypeId, + value: u32, + }, // Dependencies OutputDependency { @@ -156,6 +185,10 @@ pub enum CachedDataItem { InProgress { value: InProgressState, }, + InProgressCell { + cell: CellId, + value: InProgressCellState, + }, OutdatedCollectible { collectible: CellRef, value: (), @@ -188,6 +221,7 @@ impl CachedDataItem { CachedDataItem::DirtyWhenPersisted { .. } => true, CachedDataItem::Child { task, .. } => !task.is_transient(), CachedDataItem::CellData { .. } => true, + CachedDataItem::CellTypeMaxIndex { .. } => true, CachedDataItem::OutputDependency { target, .. } => !target.is_transient(), CachedDataItem::CellDependency { target, .. } => !target.task.is_transient(), CachedDataItem::CollectiblesDependency { target, .. } => !target.task.is_transient(), @@ -204,6 +238,7 @@ impl CachedDataItem { CachedDataItem::AggregatedUnfinishedTasks { .. } => true, CachedDataItem::AggregateRootType { .. } => false, CachedDataItem::InProgress { .. } => false, + CachedDataItem::InProgressCell { .. } => false, CachedDataItem::OutdatedCollectible { .. } => false, CachedDataItem::OutdatedOutputDependency { .. } => false, CachedDataItem::OutdatedCellDependency { .. } => false, @@ -221,6 +256,21 @@ impl CachedDataItem { }, } } + + pub fn new_scheduled_with_listener(task_id: TaskId, clean: bool) -> (Self, EventListener) { + let done_event = Event::new(move || format!("{} done_event", task_id)); + let listener = done_event.listen(); + ( + CachedDataItem::InProgress { + value: InProgressState::Scheduled { + clean, + done_event, + start_event: Event::new(move || format!("{} start_event", task_id)), + }, + }, + listener, + ) + } } impl CachedDataItemKey { @@ -232,6 +282,7 @@ impl CachedDataItemKey { CachedDataItemKey::DirtyWhenPersisted { .. } => true, CachedDataItemKey::Child { task, .. } => !task.is_transient(), CachedDataItemKey::CellData { .. } => true, + CachedDataItemKey::CellTypeMaxIndex { .. } => true, CachedDataItemKey::OutputDependency { target, .. } => !target.is_transient(), CachedDataItemKey::CellDependency { target, .. } => !target.task.is_transient(), CachedDataItemKey::CollectiblesDependency { target, .. } => !target.task.is_transient(), @@ -248,6 +299,7 @@ impl CachedDataItemKey { CachedDataItemKey::AggregatedUnfinishedTasks { .. } => true, CachedDataItemKey::AggregateRootType { .. } => false, CachedDataItemKey::InProgress { .. } => false, + CachedDataItemKey::InProgressCell { .. } => false, CachedDataItemKey::OutdatedCollectible { .. } => false, CachedDataItemKey::OutdatedOutputDependency { .. } => false, CachedDataItemKey::OutdatedCellDependency { .. } => false, From 9feec203f19b9f6ff17a7f520ece5d09bce1f961 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Thu, 15 Aug 2024 11:54:20 +0200 Subject: [PATCH 24/81] remove clean flag in favor of Dirty --- .../turbo-tasks-backend/src/backend/mod.rs | 13 ++------- .../backend/operation/aggregation_update.rs | 6 +++-- .../src/backend/operation/invalidate.rs | 27 +++---------------- .../crates/turbo-tasks-backend/src/data.rs | 8 +----- 4 files changed, 10 insertions(+), 44 deletions(-) diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs index 38191351498c1..e44b594574470 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs @@ -254,8 +254,7 @@ impl TurboTasksBackend { } // Output doesn't exist. We need to schedule the task to compute it. - let dirty = task.has_key(&CachedDataItemKey::Dirty {}); - let (item, listener) = CachedDataItem::new_scheduled_with_listener(task_id, !dirty); + let (item, listener) = CachedDataItem::new_scheduled_with_listener(task_id); if task.add(item) { turbo_tasks.schedule(task_id); } @@ -453,7 +452,6 @@ impl Backend for TurboTasksBackend { let mut task = ctx.task(task_id); let in_progress = remove!(task, InProgress)?; let InProgressState::Scheduled { - clean, done_event, start_event, } = in_progress @@ -463,7 +461,6 @@ impl Backend for TurboTasksBackend { }; task.add(CachedDataItem::InProgress { value: InProgressState::InProgress { - clean, stale: false, done_event, }, @@ -662,12 +659,7 @@ impl Backend for TurboTasksBackend { else { panic!("Task execution completed, but task is not in progress: {task:#?}"); }; - let InProgressState::InProgress { - done_event, - clean: _, - stale, - } = in_progress - else { + let InProgressState::InProgress { done_event, stale } = in_progress else { panic!("Task execution completed, but task is not in progress: {task:#?}"); }; @@ -677,7 +669,6 @@ impl Backend for TurboTasksBackend { if stale { task.add(CachedDataItem::InProgress { value: InProgressState::InProgress { - clean: false, stale: false, done_event, }, diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs index 72fa2a37705fb..c2bc541636c4f 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs @@ -303,8 +303,10 @@ impl AggregationUpdateQueue { AggregationUpdateJob::ScheduleWhenDirty { task_ids } => { for task_id in task_ids { let mut task = ctx.task(task_id); - if task.add(CachedDataItem::new_scheduled(task_id)) { - ctx.turbo_tasks.schedule(task_id); + if task.has_key(&CachedDataItemKey::Dirty {}) { + if task.add(CachedDataItem::new_scheduled(task_id)) { + ctx.turbo_tasks.schedule(task_id); + } } } } diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/invalidate.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/invalidate.rs index c4c42415c2e32..248c1274abd0a 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/invalidate.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/invalidate.rs @@ -66,31 +66,10 @@ pub fn make_task_dirty(task_id: TaskId, queue: &mut AggregationUpdateQueue, ctx: if task.add(CachedDataItem::Dirty { value: () }) { let in_progress = match get!(task, InProgress) { - Some(InProgressState::Scheduled { clean, .. }) => { - if *clean { - update!(task, InProgress, |in_progress| { - let Some(InProgressState::Scheduled { - clean: _, - done_event, - start_event, - }) = in_progress - else { - unreachable!(); - }; - Some(InProgressState::Scheduled { - clean: false, - done_event, - start_event, - }) - }); - } - true - } - Some(InProgressState::InProgress { clean, stale, .. }) => { - if *clean || !*stale { + Some(InProgressState::InProgress { stale, .. }) => { + if !*stale { update!(task, InProgress, |in_progress| { let Some(InProgressState::InProgress { - clean: _, stale: _, done_event, }) = in_progress @@ -98,7 +77,6 @@ pub fn make_task_dirty(task_id: TaskId, queue: &mut AggregationUpdateQueue, ctx: unreachable!(); }; Some(InProgressState::InProgress { - clean: false, stale: true, done_event, }) @@ -106,6 +84,7 @@ pub fn make_task_dirty(task_id: TaskId, queue: &mut AggregationUpdateQueue, ctx: } true } + Some(_) => true, None => false, }; if !in_progress && task.add(CachedDataItem::new_scheduled(task_id)) { diff --git a/turbopack/crates/turbo-tasks-backend/src/data.rs b/turbopack/crates/turbo-tasks-backend/src/data.rs index bdf783bc33531..360b730c9faea 100644 --- a/turbopack/crates/turbo-tasks-backend/src/data.rs +++ b/turbopack/crates/turbo-tasks-backend/src/data.rs @@ -45,14 +45,10 @@ pub enum RootType { #[derive(Debug)] pub enum InProgressState { Scheduled { - // TODO remove in favor of Dirty - clean: bool, done_event: Event, start_event: Event, }, InProgress { - // TODO remove in favor of Dirty - clean: bool, stale: bool, done_event: Event, }, @@ -250,20 +246,18 @@ impl CachedDataItem { pub fn new_scheduled(task_id: TaskId) -> Self { CachedDataItem::InProgress { value: InProgressState::Scheduled { - clean: false, done_event: Event::new(move || format!("{} done_event", task_id)), start_event: Event::new(move || format!("{} start_event", task_id)), }, } } - pub fn new_scheduled_with_listener(task_id: TaskId, clean: bool) -> (Self, EventListener) { + pub fn new_scheduled_with_listener(task_id: TaskId) -> (Self, EventListener) { let done_event = Event::new(move || format!("{} done_event", task_id)); let listener = done_event.listen(); ( CachedDataItem::InProgress { value: InProgressState::Scheduled { - clean, done_event, start_event: Event::new(move || format!("{} start_event", task_id)), }, From a76d0ec51c371e97c692ef45ca3494df66a5477b Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Thu, 15 Aug 2024 14:53:22 +0200 Subject: [PATCH 25/81] avoid marking once tasks as stale --- .../turbo-tasks-backend/src/backend/mod.rs | 93 ++++++++++++++----- .../src/backend/operation/invalidate.rs | 6 +- .../crates/turbo-tasks-backend/src/data.rs | 1 + turbopack/crates/turbo-tasks/src/backend.rs | 2 +- 4 files changed, 75 insertions(+), 27 deletions(-) diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs index e44b594574470..c88b5b5109a13 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs @@ -25,8 +25,8 @@ use rustc_hash::FxHasher; use smallvec::smallvec; use turbo_tasks::{ backend::{ - Backend, BackendJobId, CachedTaskType, CellContent, TaskExecutionSpec, TransientTaskType, - TypedCellContent, + Backend, BackendJobId, CachedTaskType, CellContent, TaskExecutionSpec, TransientTaskRoot, + TransientTaskType, TypedCellContent, }, event::EventListener, registry, @@ -61,13 +61,30 @@ impl SnapshotRequest { } } +pub enum TransientTask { + /// A root task that will track dependencies and re-execute when + /// dependencies change. Task will eventually settle to the correct + /// execution. + /// Always active. Automatically scheduled. + Root(TransientTaskRoot), + + // TODO implement these strongly consistency + /// A single root task execution. It won't track dependencies. + /// Task will definitely include all invalidations that happened before the + /// start of the task. It may or may not include invalidations that + /// happened after that. It may see these invalidations partially + /// applied. + /// Active until done. Automatically scheduled. + Once(tokio::sync::Mutex> + Send + 'static>>>), +} + pub struct TurboTasksBackend { persisted_task_id_factory: IdFactoryWithReuse, transient_task_id_factory: IdFactoryWithReuse, persisted_task_cache_log: Mutex, TaskId)>>, task_cache: BiMap, TaskId>, - transient_tasks: DashMap>>, + transient_tasks: DashMap>, persisted_storage_log: Mutex>, storage: Storage, @@ -447,6 +464,20 @@ impl Backend for TurboTasksBackend { task_id: TaskId, turbo_tasks: &dyn TurboTasksBackendApi, ) -> Option> { + enum TaskType { + Cached(Arc), + Transient(Arc), + } + let (task_type, once_task) = if let Some(task_type) = self.lookup_task_type(task_id) { + (TaskType::Cached(task_type), false) + } else if let Some(task_type) = self.transient_tasks.get(&task_id) { + ( + TaskType::Transient(task_type.clone()), + matches!(**task_type, TransientTask::Once(_)), + ) + } else { + return None; + }; { let ctx = self.execute_context(turbo_tasks); let mut task = ctx.task(task_id); @@ -462,6 +493,7 @@ impl Backend for TurboTasksBackend { task.add(CachedDataItem::InProgress { value: InProgressState::InProgress { stale: false, + once_task, done_event, }, }); @@ -554,8 +586,8 @@ impl Backend for TurboTasksBackend { start_event.notify(usize::MAX); } - let (span, future) = if let Some(task_type) = self.lookup_task_type(task_id) { - match &*task_type { + let (span, future) = match task_type { + TaskType::Cached(task_type) => match &*task_type { CachedTaskType::Native { fn_type, this, arg } => ( registry::get_function(*fn_type).span(), registry::get_function(*fn_type).execute(*this, &**arg), @@ -612,24 +644,24 @@ impl Backend for TurboTasksBackend { }) as Pin + Send + '_>>, ) } - } - } else if let Some(task_type) = self.transient_tasks.get(&task_id) { - let task_type = task_type.clone(); - let span = tracing::trace_span!("turbo_tasks::root_task"); - let future = Box::pin(async move { - let mut task_type = task_type.lock().await; - match &mut *task_type { - TransientTaskType::Root(f) => { - let future = f(); - drop(task_type); - future.await + }, + TaskType::Transient(task_type) => { + let task_type = task_type.clone(); + let span = tracing::trace_span!("turbo_tasks::root_task"); + let future = Box::pin(async move { + match &*task_type { + TransientTask::Root(f) => { + let future = f(); + future.await + } + TransientTask::Once(future) => { + let mut mutex_guard = future.lock().await; + (&mut *mutex_guard).await + } } - TransientTaskType::Once(future) => future.await, - } - }) as Pin + Send + '_>>; - (span, future) - } else { - return None; + }) as Pin + Send + '_>>; + (span, future) + } }; Some(TaskExecutionSpec { future, span }) } @@ -659,7 +691,12 @@ impl Backend for TurboTasksBackend { else { panic!("Task execution completed, but task is not in progress: {task:#?}"); }; - let InProgressState::InProgress { done_event, stale } = in_progress else { + let InProgressState::InProgress { + done_event, + once_task, + stale, + } = in_progress + else { panic!("Task execution completed, but task is not in progress: {task:#?}"); }; @@ -670,6 +707,7 @@ impl Backend for TurboTasksBackend { task.add(CachedDataItem::InProgress { value: InProgressState::InProgress { stale: false, + once_task, done_event, }, }); @@ -878,8 +916,13 @@ impl Backend for TurboTasksBackend { TransientTaskType::Root(_) => RootType::RootTask, TransientTaskType::Once(_) => RootType::OnceTask, }; - self.transient_tasks - .insert(task_id, Arc::new(tokio::sync::Mutex::new(task_type))); + self.transient_tasks.insert( + task_id, + Arc::new(match task_type { + TransientTaskType::Root(f) => TransientTask::Root(f), + TransientTaskType::Once(f) => TransientTask::Once(tokio::sync::Mutex::new(f)), + }), + ); { let mut task = self.storage.access_mut(task_id); task.add(CachedDataItem::new_scheduled(task_id)); diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/invalidate.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/invalidate.rs index 248c1274abd0a..02cfa2b8e633d 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/invalidate.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/invalidate.rs @@ -66,11 +66,14 @@ pub fn make_task_dirty(task_id: TaskId, queue: &mut AggregationUpdateQueue, ctx: if task.add(CachedDataItem::Dirty { value: () }) { let in_progress = match get!(task, InProgress) { - Some(InProgressState::InProgress { stale, .. }) => { + Some(InProgressState::InProgress { + stale, once_task, .. + }) if !once_task => { if !*stale { update!(task, InProgress, |in_progress| { let Some(InProgressState::InProgress { stale: _, + once_task, done_event, }) = in_progress else { @@ -78,6 +81,7 @@ pub fn make_task_dirty(task_id: TaskId, queue: &mut AggregationUpdateQueue, ctx: }; Some(InProgressState::InProgress { stale: true, + once_task, done_event, }) }); diff --git a/turbopack/crates/turbo-tasks-backend/src/data.rs b/turbopack/crates/turbo-tasks-backend/src/data.rs index 360b730c9faea..7d3acfa57d268 100644 --- a/turbopack/crates/turbo-tasks-backend/src/data.rs +++ b/turbopack/crates/turbo-tasks-backend/src/data.rs @@ -50,6 +50,7 @@ pub enum InProgressState { }, InProgress { stale: bool, + once_task: bool, done_event: Event, }, } diff --git a/turbopack/crates/turbo-tasks/src/backend.rs b/turbopack/crates/turbo-tasks/src/backend.rs index 9f7ca0dc41540..4f771c60f283b 100644 --- a/turbopack/crates/turbo-tasks/src/backend.rs +++ b/turbopack/crates/turbo-tasks/src/backend.rs @@ -27,7 +27,7 @@ use crate::{ TraitTypeId, ValueTypeId, VcRead, VcValueTrait, VcValueType, }; -type TransientTaskRoot = +pub type TransientTaskRoot = Box Pin> + Send>> + Send + Sync>; pub enum TransientTaskType { From 4d32f661564026fa5d5de61285e19de28aedbb74 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Fri, 16 Aug 2024 07:18:57 +0200 Subject: [PATCH 26/81] notify in progress cells --- .../src/backend/operation/update_cell.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/update_cell.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/update_cell.rs index 664f3fdc3cfbd..2eaeb48faea25 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/update_cell.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/update_cell.rs @@ -1,7 +1,10 @@ use turbo_tasks::{backend::CellContent, CellId, TaskId}; use super::{ExecuteContext, InvalidateOperation}; -use crate::data::{CachedDataItem, CachedDataItemKey}; +use crate::{ + data::{CachedDataItem, CachedDataItemKey}, + remove, +}; pub struct UpdateCellOperation; @@ -17,6 +20,10 @@ impl UpdateCellOperation { task.remove(&CachedDataItemKey::CellData { cell }) }; + if let Some(in_progress) = remove!(task, InProgressCell { cell }) { + in_progress.event.notify(usize::MAX); + } + let dependent = task .iter() .filter_map(|(key, _)| { From 909f92434ef91346da4cb0d0121bc05a07ad90e8 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Fri, 16 Aug 2024 07:19:54 +0200 Subject: [PATCH 27/81] asset adding, note on listener --- .../turbo-tasks-backend/src/backend/mod.rs | 56 +++++++++++-------- .../src/backend/operation/mod.rs | 6 ++ .../crates/turbo-tasks-backend/src/data.rs | 7 ++- 3 files changed, 43 insertions(+), 26 deletions(-) diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs index c88b5b5109a13..b3a99e87294e3 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs @@ -233,7 +233,8 @@ impl TurboTasksBackend { match in_progress { InProgressState::Scheduled { done_event, .. } | InProgressState::InProgress { done_event, .. } => { - let listener = done_event.listen(); + let listener = done_event + .listen_with_note(move || format!("try_read_task_output from {reader:?}")); return Ok(Err(listener)); } } @@ -253,17 +254,22 @@ impl TurboTasksBackend { }; if let Some(result) = result { if let Some(reader) = reader { - task.add(CachedDataItem::OutputDependent { + let _ = task.add(CachedDataItem::OutputDependent { task: reader, value: (), }); drop(task); let mut reader_task = ctx.task(reader); - reader_task.add(CachedDataItem::OutputDependency { - target: task_id, - value: (), - }); + if reader_task + .remove(&CachedDataItemKey::OutdatedOutputDependency { target: task_id }) + .is_none() + { + let _ = reader_task.add(CachedDataItem::OutputDependency { + target: task_id, + value: (), + }); + } } return result; @@ -271,10 +277,11 @@ impl TurboTasksBackend { } // Output doesn't exist. We need to schedule the task to compute it. - let (item, listener) = CachedDataItem::new_scheduled_with_listener(task_id); - if task.add(item) { - turbo_tasks.schedule(task_id); - } + let (item, listener) = CachedDataItem::new_scheduled_with_listener(task_id, move || { + format!("try_read_task_output (recompute) from {reader:?}") + }); + task.add_new(item); + turbo_tasks.schedule(task_id); Ok(Err(listener)) } @@ -291,12 +298,13 @@ impl TurboTasksBackend { if let Some(content) = get!(task, CellData { cell }) { let content = content.clone(); if let Some(reader) = reader { - task.add(CachedDataItem::CellDependent { + let _ = task.add(CachedDataItem::CellDependent { cell, task: reader, value: (), }); drop(task); + let mut reader_task = ctx.task(reader); let target = CellRef { task: task_id, @@ -306,7 +314,7 @@ impl TurboTasksBackend { .remove(&CachedDataItemKey::OutdatedCellDependency { target }) .is_none() { - reader_task.add(CachedDataItem::CellDependency { target, value: () }); + let _ = reader_task.add(CachedDataItem::CellDependency { target, value: () }); } } return Ok(Ok(CellContent(Some(content)).into_typed(cell.type_id))); @@ -340,12 +348,12 @@ impl TurboTasksBackend { // We create the event and potentially schedule the task let in_progress = InProgressCellState::new(task_id, cell); let listener = in_progress.event.listen(); - task.add(CachedDataItem::InProgressCell { + task.add_new(CachedDataItem::InProgressCell { cell, value: in_progress, }); - // Schedule the task + // Schedule the task, if not already scheduled if task.add(CachedDataItem::new_scheduled(task_id)) { turbo_tasks.schedule(task_id); } @@ -487,10 +495,10 @@ impl Backend for TurboTasksBackend { start_event, } = in_progress else { - task.add(CachedDataItem::InProgress { value: in_progress }); + task.add_new(CachedDataItem::InProgress { value: in_progress }); return None; }; - task.add(CachedDataItem::InProgress { + task.add_new(CachedDataItem::InProgress { value: InProgressState::InProgress { stale: false, once_task, @@ -514,7 +522,7 @@ impl Backend for TurboTasksBackend { for child in children { match child { Child::Current(child) => { - task.add(CachedDataItem::OutdatedChild { + let _ = task.add(CachedDataItem::OutdatedChild { task: child, value: (), }); @@ -553,13 +561,13 @@ impl Backend for TurboTasksBackend { for dep in dependencies { match dep { Dep::CurrentCell(cell) => { - task.add(CachedDataItem::OutdatedCellDependency { + let _ = task.add(CachedDataItem::OutdatedCellDependency { target: cell, value: (), }); } Dep::CurrentOutput(output) => { - task.add(CachedDataItem::OutdatedOutputDependency { + let _ = task.add(CachedDataItem::OutdatedOutputDependency { target: output, value: (), }); @@ -704,7 +712,7 @@ impl Backend for TurboTasksBackend { let _ = stateful; if stale { - task.add(CachedDataItem::InProgress { + task.add_new(CachedDataItem::InProgress { value: InProgressState::InProgress { stale: false, once_task, @@ -730,7 +738,7 @@ impl Backend for TurboTasksBackend { } } } else { - task.add(CachedDataItem::CellTypeMaxIndex { + task.add_new(CachedDataItem::CellTypeMaxIndex { cell_type, value: max_index, }); @@ -925,9 +933,9 @@ impl Backend for TurboTasksBackend { ); { let mut task = self.storage.access_mut(task_id); - task.add(CachedDataItem::new_scheduled(task_id)); - task.add(CachedDataItem::AggregateRootType { value: root_type }); - task.add(CachedDataItem::AggregationNumber { value: u32::MAX }); + let _ = task.add(CachedDataItem::new_scheduled(task_id)); + let _ = task.add(CachedDataItem::AggregateRootType { value: root_type }); + let _ = task.add(CachedDataItem::AggregationNumber { value: u32::MAX }); } turbo_tasks.schedule(task_id); task_id diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs index e9057c081bccf..dd1ff755fa7fb 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs @@ -126,6 +126,7 @@ impl<'a> TaskGuard<'a> { self.task_id } + #[must_use] pub fn add(&mut self, item: CachedDataItem) -> bool { if !item.is_persistent() { self.task.add(item) @@ -145,6 +146,11 @@ impl<'a> TaskGuard<'a> { } } + pub fn add_new(&mut self, item: CachedDataItem) { + let added = self.add(item); + assert!(added, "Item already exists"); + } + pub fn insert(&mut self, item: CachedDataItem) -> Option { let (key, value) = item.into_key_and_value(); if !key.is_persistent() { diff --git a/turbopack/crates/turbo-tasks-backend/src/data.rs b/turbopack/crates/turbo-tasks-backend/src/data.rs index 7d3acfa57d268..4dfb6186d9035 100644 --- a/turbopack/crates/turbo-tasks-backend/src/data.rs +++ b/turbopack/crates/turbo-tasks-backend/src/data.rs @@ -253,9 +253,12 @@ impl CachedDataItem { } } - pub fn new_scheduled_with_listener(task_id: TaskId) -> (Self, EventListener) { + pub fn new_scheduled_with_listener( + task_id: TaskId, + note: impl Fn() -> String + Sync + Send + 'static, + ) -> (Self, EventListener) { let done_event = Event::new(move || format!("{} done_event", task_id)); - let listener = done_event.listen(); + let listener = done_event.listen_with_note(note); ( CachedDataItem::InProgress { value: InProgressState::Scheduled { From 20d49578a8cdafe8b9cdc9c4a0aca0646e8e8665 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Fri, 16 Aug 2024 07:46:26 +0200 Subject: [PATCH 28/81] remove helpers --- turbopack/crates/turbo-tasks-backend/src/backend/helpers/mod.rs | 1 - .../crates/turbo-tasks-backend/src/backend/helpers/schedule.rs | 1 - turbopack/crates/turbo-tasks-backend/src/backend/mod.rs | 1 - 3 files changed, 3 deletions(-) delete mode 100644 turbopack/crates/turbo-tasks-backend/src/backend/helpers/mod.rs delete mode 100644 turbopack/crates/turbo-tasks-backend/src/backend/helpers/schedule.rs diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/helpers/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/helpers/mod.rs deleted file mode 100644 index 8b137891791fe..0000000000000 --- a/turbopack/crates/turbo-tasks-backend/src/backend/helpers/mod.rs +++ /dev/null @@ -1 +0,0 @@ - diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/helpers/schedule.rs b/turbopack/crates/turbo-tasks-backend/src/backend/helpers/schedule.rs deleted file mode 100644 index 8b137891791fe..0000000000000 --- a/turbopack/crates/turbo-tasks-backend/src/backend/helpers/schedule.rs +++ /dev/null @@ -1 +0,0 @@ - diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs index b3a99e87294e3..fc298305c361c 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs @@ -1,4 +1,3 @@ -mod helpers; mod operation; mod storage; From dd7834a9f08480df496adc2b6b8bddc8071cd433 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Fri, 16 Aug 2024 07:47:36 +0200 Subject: [PATCH 29/81] event listen note, remove start event --- .../turbo-tasks-backend/src/backend/mod.rs | 69 ++++++++++++++----- .../backend/operation/aggregation_update.rs | 3 +- .../src/backend/operation/connect_child.rs | 12 +++- .../src/backend/operation/invalidate.rs | 6 +- .../crates/turbo-tasks-backend/src/data.rs | 15 ++-- 5 files changed, 74 insertions(+), 31 deletions(-) diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs index fc298305c361c..c8ad6e2175a29 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs @@ -232,8 +232,14 @@ impl TurboTasksBackend { match in_progress { InProgressState::Scheduled { done_event, .. } | InProgressState::InProgress { done_event, .. } => { - let listener = done_event - .listen_with_note(move || format!("try_read_task_output from {reader:?}")); + let reader_desc = reader.map(|r| self.get_task_desc_fn(r)); + let listener = done_event.listen_with_note(move || { + if let Some(reader_desc) = reader_desc.as_ref() { + format!("try_read_task_output from {}", reader_desc()) + } else { + format!("try_read_task_output (untracked)") + } + }); return Ok(Err(listener)); } } @@ -275,10 +281,18 @@ impl TurboTasksBackend { } } + let reader_desc = reader.map(|r| self.get_task_desc_fn(r)); + let note = move || { + if let Some(reader_desc) = reader_desc.as_ref() { + format!("try_read_task_cell (recompute) from {}", reader_desc()) + } else { + format!("try_read_task_cell (recompute, untracked)") + } + }; + // Output doesn't exist. We need to schedule the task to compute it. - let (item, listener) = CachedDataItem::new_scheduled_with_listener(task_id, move || { - format!("try_read_task_output (recompute) from {reader:?}") - }); + let (item, listener) = + CachedDataItem::new_scheduled_with_listener(self.get_task_desc_fn(task_id), note); task.add_new(item); turbo_tasks.schedule(task_id); @@ -337,23 +351,35 @@ impl TurboTasksBackend { // Cell should exist, but data was dropped or is not serializable. We need to recompute the // task the get the cell content. + let reader_desc = reader.map(|r| self.get_task_desc_fn(r)); + let note = move || { + if let Some(reader_desc) = reader_desc.as_ref() { + format!("try_read_task_cell from {}", reader_desc()) + } else { + format!("try_read_task_cell (untracked)") + } + }; + // Register event listener for cell computation if let Some(in_progress) = get!(task, InProgressCell { cell }) { // Someone else is already computing the cell - let listener = in_progress.event.listen(); + let listener = in_progress.event.listen_with_note(note); return Ok(Err(listener)); } // We create the event and potentially schedule the task let in_progress = InProgressCellState::new(task_id, cell); - let listener = in_progress.event.listen(); + + let listener = in_progress.event.listen_with_note(note); task.add_new(CachedDataItem::InProgressCell { cell, value: in_progress, }); // Schedule the task, if not already scheduled - if task.add(CachedDataItem::new_scheduled(task_id)) { + if task.add(CachedDataItem::new_scheduled( + self.get_task_desc_fn(task_id), + )) { turbo_tasks.schedule(task_id); } @@ -366,6 +392,17 @@ impl TurboTasksBackend { } None } + + // TODO feature flag that for hanging detection only + fn get_task_desc_fn(&self, task_id: TaskId) -> impl Fn() -> String + Send + Sync + 'static { + let task_type = self.lookup_task_type(task_id); + move || { + task_type.as_ref().map_or_else( + || format!("{task_id:?} transient"), + |task_type| format!("{task_id:?} {task_type}"), + ) + } + } } impl Backend for TurboTasksBackend { @@ -489,11 +526,7 @@ impl Backend for TurboTasksBackend { let ctx = self.execute_context(turbo_tasks); let mut task = ctx.task(task_id); let in_progress = remove!(task, InProgress)?; - let InProgressState::Scheduled { - done_event, - start_event, - } = in_progress - else { + let InProgressState::Scheduled { done_event } = in_progress else { task.add_new(CachedDataItem::InProgress { value: in_progress }); return None; }; @@ -589,8 +622,6 @@ impl Backend for TurboTasksBackend { } // TODO: Make all collectibles outdated - - start_event.notify(usize::MAX); } let (span, future) = match task_type { @@ -932,9 +963,13 @@ impl Backend for TurboTasksBackend { ); { let mut task = self.storage.access_mut(task_id); - let _ = task.add(CachedDataItem::new_scheduled(task_id)); - let _ = task.add(CachedDataItem::AggregateRootType { value: root_type }); let _ = task.add(CachedDataItem::AggregationNumber { value: u32::MAX }); + let _ = task.add(CachedDataItem::AggregateRootType { value: root_type }); + let _ = task.add(CachedDataItem::new_scheduled(move || match root_type { + RootType::RootTask => "Root Task".to_string(), + RootType::OnceTask => "Once Task".to_string(), + _ => unreachable!(), + })); } turbo_tasks.schedule(task_id); task_id diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs index c2bc541636c4f..27244cade747f 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs @@ -302,9 +302,10 @@ impl AggregationUpdateQueue { } AggregationUpdateJob::ScheduleWhenDirty { task_ids } => { for task_id in task_ids { + let description = ctx.backend.get_task_desc_fn(task_id); let mut task = ctx.task(task_id); if task.has_key(&CachedDataItemKey::Dirty {}) { - if task.add(CachedDataItem::new_scheduled(task_id)) { + if task.add(CachedDataItem::new_scheduled(description)) { ctx.turbo_tasks.schedule(task_id); } } diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs index 5fb5847fe2eb3..5bde34aa32921 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs @@ -70,11 +70,19 @@ impl Operation for ConnectChildOperation { } } ConnectChildOperation::ScheduleTask { task_id } => { + let mut should_schedule; { let mut task = ctx.task(task_id); - task.add(CachedDataItem::new_scheduled(task_id)); + should_schedule = !task.has_key(&CachedDataItemKey::Output {}); + if should_schedule { + should_schedule = task.add(CachedDataItem::new_scheduled( + task.backend.get_task_desc_fn(task_id), + )); + } + } + if should_schedule { + ctx.schedule(task_id); } - ctx.schedule(task_id); self = ConnectChildOperation::Done; } diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/invalidate.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/invalidate.rs index 02cfa2b8e633d..66637422b1c39 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/invalidate.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/invalidate.rs @@ -91,7 +91,11 @@ pub fn make_task_dirty(task_id: TaskId, queue: &mut AggregationUpdateQueue, ctx: Some(_) => true, None => false, }; - if !in_progress && task.add(CachedDataItem::new_scheduled(task_id)) { + if !in_progress + && task.add(CachedDataItem::new_scheduled( + task.backend.get_task_desc_fn(task_id), + )) + { ctx.turbo_tasks.schedule(task_id) } queue.push(AggregationUpdateJob::DataUpdate { diff --git a/turbopack/crates/turbo-tasks-backend/src/data.rs b/turbopack/crates/turbo-tasks-backend/src/data.rs index 4dfb6186d9035..db0c2b04b14d1 100644 --- a/turbopack/crates/turbo-tasks-backend/src/data.rs +++ b/turbopack/crates/turbo-tasks-backend/src/data.rs @@ -46,7 +46,6 @@ pub enum RootType { pub enum InProgressState { Scheduled { done_event: Event, - start_event: Event, }, InProgress { stale: bool, @@ -244,27 +243,23 @@ impl CachedDataItem { } } - pub fn new_scheduled(task_id: TaskId) -> Self { + pub fn new_scheduled(description: impl Fn() -> String + Sync + Send + 'static) -> Self { CachedDataItem::InProgress { value: InProgressState::Scheduled { - done_event: Event::new(move || format!("{} done_event", task_id)), - start_event: Event::new(move || format!("{} start_event", task_id)), + done_event: Event::new(move || format!("{} done_event", description())), }, } } pub fn new_scheduled_with_listener( - task_id: TaskId, + description: impl Fn() -> String + Sync + Send + 'static, note: impl Fn() -> String + Sync + Send + 'static, ) -> (Self, EventListener) { - let done_event = Event::new(move || format!("{} done_event", task_id)); + let done_event = Event::new(move || format!("{} done_event", description())); let listener = done_event.listen_with_note(note); ( CachedDataItem::InProgress { - value: InProgressState::Scheduled { - done_event, - start_event: Event::new(move || format!("{} start_event", task_id)), - }, + value: InProgressState::Scheduled { done_event }, }, listener, ) From cabc0f7fa1cc3b0b51c854471ebfc44371e4d5c8 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Wed, 4 Sep 2024 12:05:03 +0200 Subject: [PATCH 30/81] improve once task handling --- .../turbo-tasks-backend/src/backend/mod.rs | 25 +++++++++---------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs index c8ad6e2175a29..d87d005eaa51b 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs @@ -6,6 +6,7 @@ use std::{ collections::{HashMap, HashSet}, future::Future, hash::BuildHasherDefault, + mem::take, pin::Pin, sync::{ atomic::{AtomicUsize, Ordering}, @@ -74,7 +75,7 @@ pub enum TransientTask { /// happened after that. It may see these invalidations partially /// applied. /// Active until done. Automatically scheduled. - Once(tokio::sync::Mutex> + Send + 'static>>>), + Once(Mutex> + Send + 'static>>>>), } pub struct TurboTasksBackend { @@ -686,18 +687,16 @@ impl Backend for TurboTasksBackend { TaskType::Transient(task_type) => { let task_type = task_type.clone(); let span = tracing::trace_span!("turbo_tasks::root_task"); - let future = Box::pin(async move { - match &*task_type { - TransientTask::Root(f) => { - let future = f(); - future.await - } - TransientTask::Once(future) => { - let mut mutex_guard = future.lock().await; - (&mut *mutex_guard).await - } + let future = match &*task_type { + TransientTask::Root(f) => { + let future = f(); + future } - }) as Pin + Send + '_>>; + TransientTask::Once(future_mutex) => { + let future = take(&mut *future_mutex.lock())?; + future + } + }; (span, future) } }; @@ -958,7 +957,7 @@ impl Backend for TurboTasksBackend { task_id, Arc::new(match task_type { TransientTaskType::Root(f) => TransientTask::Root(f), - TransientTaskType::Once(f) => TransientTask::Once(tokio::sync::Mutex::new(f)), + TransientTaskType::Once(f) => TransientTask::Once(Mutex::new(Some(f))), }), ); { From 42b630cd747876723a8ca3d96361d10b3e8b35e9 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Fri, 16 Aug 2024 12:48:12 +0200 Subject: [PATCH 31/81] fix notify when recomputing values --- .../src/backend/operation/update_cell.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/update_cell.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/update_cell.rs index 2eaeb48faea25..bb301010c5900 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/update_cell.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/update_cell.rs @@ -24,6 +24,17 @@ impl UpdateCellOperation { in_progress.event.notify(usize::MAX); } + let recomputed = old_content.is_none() && !task.has_key(&CachedDataItemKey::Dirty {}); + + if recomputed { + // Task wasn't invalidated, so we just recompute, so the content has not actually + // changed (At least we have to assume that tasks are deterministic and + // pure). + drop(task); + drop(old_content); + return; + } + let dependent = task .iter() .filter_map(|(key, _)| { From 816f4c2965f5c2719976085a0758df33227d271d Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Fri, 16 Aug 2024 12:48:34 +0200 Subject: [PATCH 32/81] remove from dirty list again --- .../src/backend/operation/aggregation_update.rs | 7 +++++++ .../src/backend/operation/cleanup_old_edges.rs | 9 +++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs index 27244cade747f..4b51befac43c6 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs @@ -144,6 +144,13 @@ impl AggregatedDataUpdate { dirty_tasks_update: HashMap::from([(task_id, 1)]), } } + + pub fn no_longer_dirty_task(task_id: TaskId) -> Self { + Self { + unfinished: -1, + dirty_tasks_update: HashMap::from([(task_id, -1)]), + } + } } impl Add for AggregatedDataUpdate { diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/cleanup_old_edges.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/cleanup_old_edges.rs index ecd71c9df6fff..1ab8c9ebc3281 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/cleanup_old_edges.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/cleanup_old_edges.rs @@ -4,7 +4,7 @@ use serde::{Deserialize, Serialize}; use turbo_tasks::TaskId; use super::{ - aggregation_update::{AggregationUpdateJob, AggregationUpdateQueue}, + aggregation_update::{AggregatedDataUpdate, AggregationUpdateJob, AggregationUpdateQueue}, invalidate::make_task_dirty, ExecuteContext, Operation, }; @@ -38,10 +38,15 @@ pub enum OutdatedEdge { impl CleanupOldEdgesOperation { pub fn run(task_id: TaskId, outdated: Vec, ctx: ExecuteContext<'_>) { + let mut queue = AggregationUpdateQueue::new(); + queue.push(AggregationUpdateJob::DataUpdate { + task_id, + update: AggregatedDataUpdate::no_longer_dirty_task(task_id), + }); CleanupOldEdgesOperation::RemoveEdges { task_id, outdated, - queue: AggregationUpdateQueue::new(), + queue, } .execute(&ctx); } From 54b3b70824a2eaf1b328bcb5b6a2c5dd61339967 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Fri, 16 Aug 2024 12:48:53 +0200 Subject: [PATCH 33/81] fixup --- turbopack/crates/turbo-tasks-backend/src/backend/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs index d87d005eaa51b..c7c6ad368d4b8 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs @@ -285,9 +285,9 @@ impl TurboTasksBackend { let reader_desc = reader.map(|r| self.get_task_desc_fn(r)); let note = move || { if let Some(reader_desc) = reader_desc.as_ref() { - format!("try_read_task_cell (recompute) from {}", reader_desc()) + format!("try_read_task_output (recompute) from {}", reader_desc()) } else { - format!("try_read_task_cell (recompute, untracked)") + format!("try_read_task_output (recompute, untracked)") } }; From 16522519b6f89c9ec61bb1adb1695856dd70b737 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Sat, 17 Aug 2024 10:05:17 +0200 Subject: [PATCH 34/81] fix hanging of stale tasks --- turbopack/crates/turbo-tasks-backend/src/backend/mod.rs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs index c7c6ad368d4b8..777f341e753a3 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs @@ -742,11 +742,7 @@ impl Backend for TurboTasksBackend { if stale { task.add_new(CachedDataItem::InProgress { - value: InProgressState::InProgress { - stale: false, - once_task, - done_event, - }, + value: InProgressState::Scheduled { done_event }, }); drop(task); drop(ctx); @@ -816,9 +812,10 @@ impl Backend for TurboTasksBackend { task.remove(&CachedDataItemKey::Dirty {}); - done_event.notify(usize::MAX); drop(task); + done_event.notify(usize::MAX); + CleanupOldEdgesOperation::run(task_id, old_edges, ctx); drop(removed_data) From 66de1c2b3a23e5af8361f47ccd96a2963ab4e422 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Sat, 17 Aug 2024 12:19:19 +0200 Subject: [PATCH 35/81] avoid race condition --- turbopack/crates/turbo-tasks-backend/src/utils/bi_map.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/turbopack/crates/turbo-tasks-backend/src/utils/bi_map.rs b/turbopack/crates/turbo-tasks-backend/src/utils/bi_map.rs index 97392f52b8196..e5be1c583a648 100644 --- a/turbopack/crates/turbo-tasks-backend/src/utils/bi_map.rs +++ b/turbopack/crates/turbo-tasks-backend/src/utils/bi_map.rs @@ -41,8 +41,8 @@ where Entry::Vacant(e) => { let e = e.insert_entry(value.clone()); let key = e.key().clone(); - drop(e); self.reverse.insert(value, key); + drop(e); Ok(()) } } From 82d036ca4dd3eef2e8282a64ee68c2a339e82f0f Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Sat, 17 Aug 2024 12:21:44 +0200 Subject: [PATCH 36/81] remove outdated child --- .../turbo-tasks-backend/src/backend/operation/connect_child.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs index 5bde34aa32921..48070a59584d3 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs @@ -27,6 +27,9 @@ pub enum ConnectChildOperation { impl ConnectChildOperation { pub fn run(parent_task_id: TaskId, child_task_id: TaskId, ctx: ExecuteContext<'_>) { let mut parent_task = ctx.task(parent_task_id); + parent_task.remove(&CachedDataItemKey::OutdatedChild { + task: child_task_id, + }); if parent_task.add(CachedDataItem::Child { task: child_task_id, value: (), From bf3843db82916ef596c6be193e2f9bf234e6f28d Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Sat, 17 Aug 2024 22:18:08 +0200 Subject: [PATCH 37/81] transient tasks are scheduled by the manager --- turbopack/crates/turbo-tasks-backend/src/backend/mod.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs index 777f341e753a3..819731d05a1c9 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs @@ -967,7 +967,6 @@ impl Backend for TurboTasksBackend { _ => unreachable!(), })); } - turbo_tasks.schedule(task_id); task_id } fn dispose_root_task(&self, _: TaskId, _: &dyn TurboTasksBackendApi) { From 2cfe0a20543717ed8dcf2a03d1ebf6c186793f7b Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Mon, 2 Sep 2024 20:43:07 +0200 Subject: [PATCH 38/81] improve aggregation implementation --- .../turbo-tasks-backend/src/backend/mod.rs | 4 +- .../backend/operation/aggregation_update.rs | 379 +++++++++++++----- .../backend/operation/cleanup_old_edges.rs | 17 +- .../src/backend/operation/connect_child.rs | 16 +- .../src/backend/storage.rs | 18 +- .../crates/turbo-tasks-backend/src/data.rs | 1 + 6 files changed, 331 insertions(+), 104 deletions(-) diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs index 819731d05a1c9..5c90aab09c981 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs @@ -810,13 +810,13 @@ impl Backend for TurboTasksBackend { }) .collect::>(); - task.remove(&CachedDataItemKey::Dirty {}); + let was_dirty = task.remove(&CachedDataItemKey::Dirty {}).is_some(); drop(task); done_event.notify(usize::MAX); - CleanupOldEdgesOperation::run(task_id, old_edges, ctx); + CleanupOldEdgesOperation::run(task_id, old_edges, was_dirty, ctx); drop(removed_data) } diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs index 4b51befac43c6..fc8df3b25d193 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs @@ -1,4 +1,7 @@ -use std::{collections::HashMap, ops::Add}; +use std::{ + collections::{hash_map::Entry, HashMap, VecDeque}, + ops::Add, +}; use serde::{Deserialize, Serialize}; use turbo_tasks::TaskId; @@ -6,11 +9,25 @@ use turbo_tasks::TaskId; use super::{ExecuteContext, TaskGuard}; use crate::{ data::{CachedDataItem, CachedDataItemKey}, - get, get_many, update, update_count, + get, get_many, iter_many, update, update_count, }; +const LEAF_NUMBER: u32 = 8; + +pub fn is_aggregating_node(aggregation_number: u32) -> bool { + aggregation_number >= LEAF_NUMBER +} + +pub fn is_root_node(aggregation_number: u32) -> bool { + aggregation_number == u32::MAX +} + #[derive(Serialize, Deserialize, Clone)] pub enum AggregationUpdateJob { + UpdateAggregationNumber { + task_id: TaskId, + aggregation_number: u32, + }, InnerHasNewFollower { upper_ids: Vec, new_follower_id: TaskId, @@ -23,6 +40,10 @@ pub enum AggregationUpdateJob { upper_ids: Vec, lost_follower_id: TaskId, }, + InnerLostFollowers { + upper_ids: Vec, + lost_follower_ids: Vec, + }, AggregatedDataUpdate { upper_ids: Vec, update: AggregatedDataUpdate, @@ -34,6 +55,10 @@ pub enum AggregationUpdateJob { ScheduleWhenDirty { task_ids: Vec, }, + BalanceEdge { + upper_id: TaskId, + task_id: TaskId, + }, } #[derive(Default, Serialize, Deserialize, Clone)] @@ -45,28 +70,38 @@ pub struct AggregatedDataUpdate { impl AggregatedDataUpdate { fn from_task(task: &mut TaskGuard<'_>) -> Self { - let aggregation = get!(task, AggregationNumber); - if aggregation.is_some() { - let unfinished = get!(task, AggregatedUnfinishedTasks); - let dirty_tasks_update = task + let aggregation = get!(task, AggregationNumber).copied().unwrap_or_default(); + let dirty = get!(task, Dirty).is_some(); + if is_aggregating_node(aggregation) { + let mut unfinished = get!(task, AggregatedUnfinishedTasks).copied().unwrap_or(0); + let mut dirty_tasks_update = task .iter() .filter_map(|(key, _)| match *key { CachedDataItemKey::AggregatedDirtyTask { task } => Some((task, 1)), _ => None, }) - .collect(); + .collect::>(); + if dirty { + unfinished += 1; + dirty_tasks_update.insert(task.id(), 1); + } Self { - unfinished: unfinished.copied().unwrap_or(0) as i32, + unfinished: unfinished as i32, dirty_tasks_update, } + } else if dirty { + Self::dirty_task(task.id()) } else { - let dirty = get!(task, Dirty); - if dirty.is_some() { - Self::dirty_task(task.id()) - } else { - Self::default() - } + Self::default() + } + } + + fn invert(mut self) -> Self { + self.unfinished = -self.unfinished; + for value in self.dirty_tasks_update.values_mut() { + *value = -*value; } + self } fn apply( @@ -82,12 +117,14 @@ impl AggregatedDataUpdate { if *unfinished != 0 { update!(task, AggregatedUnfinishedTasks, |old: Option| { let old = old.unwrap_or(0); - let new = (old as i32 + *unfinished) as u32; + let new = old as i32 + *unfinished; + debug_assert!(new >= 0); + let new = new as u32; if new == 0 { result.unfinished = -1; None } else { - if old > 0 { + if old <= 0 && new > 0 { result.unfinished = 1; } Some(new) @@ -103,17 +140,17 @@ impl AggregatedDataUpdate { AggregatedDirtyTask { task: *task_id }, |old: Option| { let old = old.unwrap_or(0); - if old == 0 { - if root_type.is_some() { - task_to_schedule.push(*task_id); - } - } - let new = (old as i32 + *count) as u32; + let new = old as i32 + *count; + debug_assert!(new >= 0); + let new = new as u32; if new == 0 { result.dirty_tasks_update.insert(*task_id, -1); None } else { - if old > 0 { + if old <= 0 && new > 0 { + if root_type.is_some() { + task_to_schedule.push(*task_id); + } result.dirty_tasks_update.insert(*task_id, 1); } Some(new) @@ -159,7 +196,20 @@ impl Add for AggregatedDataUpdate { fn add(self, rhs: Self) -> Self::Output { let mut dirty_tasks_update = self.dirty_tasks_update; for (task, count) in rhs.dirty_tasks_update { - *dirty_tasks_update.entry(task).or_default() += count; + match dirty_tasks_update.entry(task) { + Entry::Occupied(mut entry) => { + let value = entry.get_mut(); + *value += count; + if *value == 0 { + entry.remove(); + } + } + Entry::Vacant(entry) => { + if count != 0 { + entry.insert(count); + } + } + } } Self { unfinished: self.unfinished + rhs.unfinished, @@ -170,12 +220,14 @@ impl Add for AggregatedDataUpdate { #[derive(Default, Serialize, Deserialize, Clone)] pub struct AggregationUpdateQueue { - jobs: Vec, + jobs: VecDeque, } impl AggregationUpdateQueue { pub fn new() -> Self { - Self { jobs: Vec::new() } + Self { + jobs: VecDeque::new(), + } } pub fn is_empty(&self) -> bool { @@ -183,31 +235,70 @@ impl AggregationUpdateQueue { } pub fn push(&mut self, job: AggregationUpdateJob) { - self.jobs.push(job); + self.jobs.push_back(job); } pub fn process(&mut self, ctx: &ExecuteContext<'_>) -> bool { - if let Some(job) = self.jobs.pop() { + if let Some(job) = self.jobs.pop_back() { match job { + AggregationUpdateJob::UpdateAggregationNumber { + task_id, + aggregation_number, + } => { + let mut task = ctx.task(task_id); + let old = get!(task, AggregationNumber).copied().unwrap_or_default(); + if old < aggregation_number { + task.insert(CachedDataItem::AggregationNumber { + value: aggregation_number, + }); + + if !is_aggregating_node(old) && is_aggregating_node(aggregation_number) { + // When converted from leaf to aggregating node, all children become + // followers + let children: Vec<_> = get_many!(task, Child { task } => task); + for child_id in children { + task.add_new(CachedDataItem::Follower { + task: child_id, + value: 1, + }); + } + } + + if is_aggregating_node(aggregation_number) { + // followers might become inner nodes when the aggregation number is + // increased + let followers = iter_many!(task, Follower { task } => task); + for follower_id in followers { + self.jobs.push_back(AggregationUpdateJob::BalanceEdge { + upper_id: task_id, + task_id: follower_id, + }); + } + } + } + } AggregationUpdateJob::InnerHasNewFollowers { upper_ids, mut new_follower_ids, } => { if let Some(new_follower_id) = new_follower_ids.pop() { if new_follower_ids.is_empty() { - self.jobs.push(AggregationUpdateJob::InnerHasNewFollower { - upper_ids, - new_follower_id, - }); + self.jobs + .push_front(AggregationUpdateJob::InnerHasNewFollower { + upper_ids, + new_follower_id, + }); } else { - self.jobs.push(AggregationUpdateJob::InnerHasNewFollowers { - upper_ids: upper_ids.clone(), - new_follower_ids, - }); - self.jobs.push(AggregationUpdateJob::InnerHasNewFollower { - upper_ids, - new_follower_id, - }); + self.jobs + .push_front(AggregationUpdateJob::InnerHasNewFollowers { + upper_ids: upper_ids.clone(), + new_follower_ids, + }); + self.jobs + .push_front(AggregationUpdateJob::InnerHasNewFollower { + upper_ids, + new_follower_id, + }); } } } @@ -215,20 +306,29 @@ impl AggregationUpdateQueue { mut upper_ids, new_follower_id, } => { - upper_ids.retain(|&_upper_id| { - // let mut upper = ctx.task(upper_id); - // TODO decide if it should be an inner or follower - // TODO for now: always inner + let follower_aggregation_number = { + let follower = ctx.task(new_follower_id); + get!(follower, AggregationNumber) + .copied() + .unwrap_or_default() + }; + let mut upper_ids_as_follower = Vec::new(); + upper_ids.retain(|&upper_id| { + let upper = ctx.task(upper_id); + // decide if it should be an inner or follower + let upper_aggregation_number = + get!(upper, AggregationNumber).copied().unwrap_or_default(); - // TODO add new_follower_data - // TODO propagate change to all uppers - - // TODO return true for inner, false for follower - true + if upper_aggregation_number < follower_aggregation_number { + // It's a follower of the upper node + upper_ids_as_follower.push(upper_id); + false + } else { + // It's an inner node, continue with the list + true + } }); - let children: Vec; - let data; - { + if !upper_ids.is_empty() { let mut follower = ctx.task(new_follower_id); upper_ids.retain(|&upper_id| { if update_count!(follower, Upper { task: upper_id }, 1) { @@ -240,47 +340,137 @@ impl AggregationUpdateQueue { } }); if !upper_ids.is_empty() { - data = AggregatedDataUpdate::from_task(&mut follower); - children = get_many!(follower, Child { task } => task); + let data = AggregatedDataUpdate::from_task(&mut follower); + let children: Vec<_> = get_many!(follower, Child { task } => task); + drop(follower); + + for upper_id in upper_ids.iter() { + // add data to upper + let mut upper = ctx.task(*upper_id); + let diff = data.apply(&mut upper, self); + if !diff.is_empty() { + let upper_ids = get_many!(upper, Upper { task } => task); + self.jobs.push_back( + AggregationUpdateJob::AggregatedDataUpdate { + upper_ids, + update: diff, + }, + ) + } + } + if !children.is_empty() { + self.jobs + .push_back(AggregationUpdateJob::InnerHasNewFollowers { + upper_ids: upper_ids.clone(), + new_follower_ids: children, + }); + } } else { - data = Default::default(); - children = Default::default(); + drop(follower); } } - for upper_id in upper_ids.iter() { - // add data to upper - let mut upper = ctx.task(*upper_id); - let diff = data.apply(&mut upper, self); - if !diff.is_empty() { - let upper_ids = get_many!(upper, Upper { task } => task); - self.jobs.push(AggregationUpdateJob::AggregatedDataUpdate { - upper_ids, - update: diff, - }) + for upper_id in upper_ids_as_follower { + let mut upper = ctx.task(upper_id); + if update_count!( + upper, + Follower { + task: new_follower_id + }, + 1 + ) { + self.jobs + .push_back(AggregationUpdateJob::InnerHasNewFollower { + upper_ids: vec![upper_id], + new_follower_id, + }) } } - if !children.is_empty() { - self.jobs.push(AggregationUpdateJob::InnerHasNewFollowers { - upper_ids: upper_ids.clone(), - new_follower_ids: children, - }); + } + AggregationUpdateJob::InnerLostFollowers { + upper_ids, + mut lost_follower_ids, + } => { + if let Some(lost_follower_id) = lost_follower_ids.pop() { + if lost_follower_ids.is_empty() { + self.jobs + .push_front(AggregationUpdateJob::InnerLostFollower { + upper_ids, + lost_follower_id, + }); + } else { + self.jobs + .push_front(AggregationUpdateJob::InnerLostFollowers { + upper_ids: upper_ids.clone(), + lost_follower_ids, + }); + self.jobs + .push_front(AggregationUpdateJob::InnerLostFollower { + upper_ids, + lost_follower_id, + }); + } } } AggregationUpdateJob::InnerLostFollower { - upper_ids, + mut upper_ids, lost_follower_id, } => { - for upper_id in upper_ids { + let mut follower = ctx.task(lost_follower_id); + let mut upper_ids_as_follower = Vec::new(); + upper_ids.retain(|&upper_id| { + if update_count!(follower, Upper { task: upper_id }, -1) { + // It was an inner + true + } else { + // It is a follower + upper_ids_as_follower.push(upper_id); + false + } + }); + if !upper_ids.is_empty() { + let data = AggregatedDataUpdate::from_task(&mut follower).invert(); + let children: Vec<_> = get_many!(follower, Child { task } => task); + drop(follower); + + for upper_id in upper_ids.iter() { + // remove data from upper + let mut upper = ctx.task(*upper_id); + let diff = data.apply(&mut upper, self); + if !diff.is_empty() { + let upper_ids = get_many!(upper, Upper { task } => task); + self.jobs + .push_back(AggregationUpdateJob::AggregatedDataUpdate { + upper_ids, + update: diff, + }) + } + } + if !children.is_empty() { + self.jobs + .push_back(AggregationUpdateJob::InnerLostFollowers { + upper_ids: upper_ids.clone(), + lost_follower_ids: children, + }); + } + } else { + drop(follower); + } + + for upper_id in upper_ids_as_follower { let mut upper = ctx.task(upper_id); - upper.remove(&CachedDataItemKey::Upper { - task: lost_follower_id, - }); - let diff = AggregatedDataUpdate::dirty_task(lost_follower_id); - let upper_ids = get_many!(upper, Upper { task } => task); - self.jobs.push(AggregationUpdateJob::AggregatedDataUpdate { - upper_ids, - update: diff, - }); + if update_count!( + upper, + Follower { + task: lost_follower_id + }, + -1 + ) { + self.jobs + .push_back(AggregationUpdateJob::InnerLostFollower { + upper_ids: vec![upper_id], + lost_follower_id, + }) + } } } AggregationUpdateJob::AggregatedDataUpdate { upper_ids, update } => { @@ -289,10 +479,11 @@ impl AggregationUpdateQueue { let diff = update.apply(&mut upper, self); if !diff.is_empty() { let upper_ids = get_many!(upper, Upper { task } => task); - self.jobs.push(AggregationUpdateJob::AggregatedDataUpdate { - upper_ids, - update: diff, - }); + self.jobs + .push_back(AggregationUpdateJob::AggregatedDataUpdate { + upper_ids, + update: diff, + }); } } } @@ -301,10 +492,11 @@ impl AggregationUpdateQueue { let diff = update.apply(&mut task, self); if !diff.is_empty() { let upper_ids = get_many!(task, Upper { task } => task); - self.jobs.push(AggregationUpdateJob::AggregatedDataUpdate { - upper_ids, - update: diff, - }); + self.jobs + .push_back(AggregationUpdateJob::AggregatedDataUpdate { + upper_ids, + update: diff, + }); } } AggregationUpdateJob::ScheduleWhenDirty { task_ids } => { @@ -318,6 +510,13 @@ impl AggregationUpdateQueue { } } } + AggregationUpdateJob::BalanceEdge { + upper_id: _, + task_id: _, + } => { + // TODO: implement + // Ignore for now + } } } diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/cleanup_old_edges.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/cleanup_old_edges.rs index 1ab8c9ebc3281..faaf54b6203fa 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/cleanup_old_edges.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/cleanup_old_edges.rs @@ -37,12 +37,19 @@ pub enum OutdatedEdge { } impl CleanupOldEdgesOperation { - pub fn run(task_id: TaskId, outdated: Vec, ctx: ExecuteContext<'_>) { + pub fn run( + task_id: TaskId, + outdated: Vec, + was_dirty: bool, + ctx: ExecuteContext<'_>, + ) { let mut queue = AggregationUpdateQueue::new(); - queue.push(AggregationUpdateJob::DataUpdate { - task_id, - update: AggregatedDataUpdate::no_longer_dirty_task(task_id), - }); + if was_dirty { + queue.push(AggregationUpdateJob::DataUpdate { + task_id, + update: AggregatedDataUpdate::no_longer_dirty_task(task_id), + }); + } CleanupOldEdgesOperation::RemoveEdges { task_id, outdated, diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs index 48070a59584d3..7d459a867c5e1 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs @@ -2,7 +2,9 @@ use serde::{Deserialize, Serialize}; use turbo_tasks::TaskId; use super::{ - aggregation_update::{AggregationUpdateJob, AggregationUpdateQueue}, + aggregation_update::{ + is_aggregating_node, is_root_node, AggregationUpdateJob, AggregationUpdateQueue, + }, ExecuteContext, Operation, }; use crate::{ @@ -21,7 +23,6 @@ pub enum ConnectChildOperation { }, #[default] Done, - // TODO Add aggregated edge } impl ConnectChildOperation { @@ -36,7 +37,16 @@ impl ConnectChildOperation { }) { // Update the task aggregation let mut queue = AggregationUpdateQueue::new(); - if get!(parent_task, AggregationNumber).is_some() { + let parent_aggregation = get!(parent_task, AggregationNumber) + .copied() + .unwrap_or_default(); + if !is_root_node(parent_aggregation) { + queue.push(AggregationUpdateJob::UpdateAggregationNumber { + task_id: child_task_id, + aggregation_number: parent_aggregation + 1, + }); + } + if is_aggregating_node(parent_aggregation) { queue.push(AggregationUpdateJob::InnerHasNewFollower { upper_ids: vec![parent_task_id], new_follower_id: child_task_id, diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs b/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs index 8474658b0278f..b3256ca7d722e 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs @@ -161,7 +161,7 @@ macro_rules! get { } #[macro_export] -macro_rules! get_many { +macro_rules! iter_many { ($task:ident, $key:ident $input:tt => $value:ident) => { $task .iter() @@ -169,7 +169,6 @@ macro_rules! get_many { CachedDataItemKey::$key $input => Some($value), _ => None, }) - .collect() }; ($task:ident, $key:ident $input:tt $value_ident:ident => $value:expr) => { $task @@ -178,7 +177,6 @@ macro_rules! get_many { (&CachedDataItemKey::$key $input, &CachedDataItemValue::$key { value: $value_ident }) => Some($value), _ => None, }) - .collect() }; ($task:ident, $key1:ident $input1:tt => $value1:ident, $key2:ident $input2:tt => $value2:ident) => { $task @@ -188,7 +186,19 @@ macro_rules! get_many { CachedDataItemKey::$key2 $input2 => Some($value2), _ => None, }) - .collect() + }; +} + +#[macro_export] +macro_rules! get_many { + ($task:ident, $key:ident $input:tt => $value:ident) => { + $crate::iter_many!($task, $key $input => $value).collect() + }; + ($task:ident, $key:ident $input:tt $value_ident:ident => $value:expr) => { + $crate::iter_many!($task, $key $input $value_ident => $value).collect() + }; + ($task:ident, $key1:ident $input1:tt => $value1:ident, $key2:ident $input2:tt => $value2:ident) => { + $crate::iter_many!($task, $key1 $input1 => $value1, $key2 $input2 => $value2).collect() }; } diff --git a/turbopack/crates/turbo-tasks-backend/src/data.rs b/turbopack/crates/turbo-tasks-backend/src/data.rs index db0c2b04b14d1..207949cfac33f 100644 --- a/turbopack/crates/turbo-tasks-backend/src/data.rs +++ b/turbopack/crates/turbo-tasks-backend/src/data.rs @@ -311,6 +311,7 @@ impl CachedDataItemValue { } } +#[derive(Debug)] pub struct CachedDataUpdate { // TODO persistence #[allow(dead_code)] From 308bcd069e1a1b2b545a42027e6ae7b99a0a6224 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Tue, 3 Sep 2024 09:13:27 +0200 Subject: [PATCH 39/81] fixup --- .../src/backend/operation/aggregation_update.rs | 4 +++- .../src/backend/operation/connect_child.rs | 7 ++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs index fc8df3b25d193..454a54f1f4fc5 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs @@ -319,7 +319,9 @@ impl AggregationUpdateQueue { let upper_aggregation_number = get!(upper, AggregationNumber).copied().unwrap_or_default(); - if upper_aggregation_number < follower_aggregation_number { + if !is_root_node(upper_aggregation_number) + && upper_aggregation_number <= follower_aggregation_number + { // It's a follower of the upper node upper_ids_as_follower.push(upper_id); false diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs index 7d459a867c5e1..d31b0766a52f5 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs @@ -40,7 +40,12 @@ impl ConnectChildOperation { let parent_aggregation = get!(parent_task, AggregationNumber) .copied() .unwrap_or_default(); - if !is_root_node(parent_aggregation) { + if parent_task_id.is_transient() && !child_task_id.is_transient() { + queue.push(AggregationUpdateJob::UpdateAggregationNumber { + task_id: child_task_id, + aggregation_number: u32::MAX, + }); + } else if !is_root_node(parent_aggregation) { queue.push(AggregationUpdateJob::UpdateAggregationNumber { task_id: child_task_id, aggregation_number: parent_aggregation + 1, From 8f502e6dda8273de98ec697795dc2969af5c7cdc Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Tue, 3 Sep 2024 10:27:13 +0200 Subject: [PATCH 40/81] strong reads --- .../turbo-tasks-backend/src/backend/mod.rs | 62 +++++++++++++++++-- .../backend/operation/aggregation_update.rs | 32 +++++++--- .../src/backend/operation/mod.rs | 3 + .../crates/turbo-tasks-backend/src/data.rs | 39 +++++++++--- 4 files changed, 116 insertions(+), 20 deletions(-) diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs index 5c90aab09c981..438c7b1e6dff8 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs @@ -19,7 +19,10 @@ use anyhow::{bail, Result}; use auto_hash_map::{AutoMap, AutoSet}; use dashmap::DashMap; pub use operation::AnyOperation; -use operation::{CleanupOldEdgesOperation, ConnectChildOperation, OutdatedEdge}; +use operation::{ + is_root_node, AggregationUpdateJob, AggregationUpdateQueue, CleanupOldEdgesOperation, + ConnectChildOperation, OutdatedEdge, +}; use parking_lot::{Condvar, Mutex}; use rustc_hash::FxHasher; use smallvec::smallvec; @@ -39,7 +42,7 @@ use self::{operation::ExecuteContext, storage::Storage}; use crate::{ data::{ CachedDataItem, CachedDataItemKey, CachedDataItemValue, CachedDataUpdate, CellRef, - InProgressCellState, InProgressState, OutputValue, RootType, + InProgressCellState, InProgressState, OutputValue, RootState, RootType, }, get, get_many, remove, utils::{bi_map::BiMap, chunked_vec::ChunkedVec, ptr_eq_arc::PtrEqArc}, @@ -247,7 +250,56 @@ impl TurboTasksBackend { } if matches!(consistency, ReadConsistency::Strong) { - todo!("Handle strongly consistent read: {task:#?}"); + // Ensure it's an root node + loop { + let aggregation_number = get!(task, AggregationNumber).copied().unwrap_or_default(); + if is_root_node(aggregation_number) { + break; + } + drop(task); + AggregationUpdateQueue::run( + AggregationUpdateJob::UpdateAggregationNumber { + task_id, + aggregation_number: u32::MAX, + }, + &ctx, + ); + task = ctx.task(task_id); + } + + // Check the dirty count of the root node + let dirty_tasks = get!(task, AggregatedDirtyTaskCount) + .copied() + .unwrap_or_default(); + let root = get!(task, AggregateRoot); + if dirty_tasks > 0 { + // When there are dirty task, subscribe to the all_clean_event + let root = if let Some(root) = root { + root + } else { + // If we don't have a root state, add one + // This also makes sure all tasks stay active, so we need to remove that after + // reading again + task.add_new(CachedDataItem::AggregateRoot { + value: RootState::new(RootType::ReadingStronglyConsistent), + }); + get!(task, AggregateRoot).unwrap() + }; + let listener = root.all_clean_event.listen_with_note(move || { + format!( + "try_read_task_output (strongly consistent) from {:?}", + reader + ) + }); + return Ok(Err(listener)); + } else { + // When there ain't dirty tasks, remove the reading strongly consistent root type + if let Some(root) = root { + if matches!(root.ty, RootType::ReadingStronglyConsistent) { + task.remove(&CachedDataItemKey::AggregateRoot {}); + } + } + } } if let Some(output) = get!(task, Output) { @@ -960,7 +1012,9 @@ impl Backend for TurboTasksBackend { { let mut task = self.storage.access_mut(task_id); let _ = task.add(CachedDataItem::AggregationNumber { value: u32::MAX }); - let _ = task.add(CachedDataItem::AggregateRootType { value: root_type }); + let _ = task.add(CachedDataItem::AggregateRoot { + value: RootState::new(root_type), + }); let _ = task.add(CachedDataItem::new_scheduled(move || match root_type { RootType::RootTask => "Root Task".to_string(), RootType::OnceTask => "Once Task".to_string(), diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs index 454a54f1f4fc5..668366689d2ef 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs @@ -6,7 +6,7 @@ use std::{ use serde::{Deserialize, Serialize}; use turbo_tasks::TaskId; -use super::{ExecuteContext, TaskGuard}; +use super::{ExecuteContext, Operation, TaskGuard}; use crate::{ data::{CachedDataItem, CachedDataItemKey}, get, get_many, iter_many, update, update_count, @@ -73,7 +73,7 @@ impl AggregatedDataUpdate { let aggregation = get!(task, AggregationNumber).copied().unwrap_or_default(); let dirty = get!(task, Dirty).is_some(); if is_aggregating_node(aggregation) { - let mut unfinished = get!(task, AggregatedUnfinishedTasks).copied().unwrap_or(0); + let mut unfinished = get!(task, AggregatedDirtyTaskCount).copied().unwrap_or(0); let mut dirty_tasks_update = task .iter() .filter_map(|(key, _)| match *key { @@ -115,7 +115,7 @@ impl AggregatedDataUpdate { } = self; let mut result = Self::default(); if *unfinished != 0 { - update!(task, AggregatedUnfinishedTasks, |old: Option| { + update!(task, AggregatedDirtyTaskCount, |old: Option| { let old = old.unwrap_or(0); let new = old as i32 + *unfinished; debug_assert!(new >= 0); @@ -133,7 +133,7 @@ impl AggregatedDataUpdate { } if !dirty_tasks_update.is_empty() { let mut task_to_schedule = Vec::new(); - let root_type = get!(task, AggregateRootType).copied(); + let root = get!(task, AggregateRoot).is_some(); for (task_id, count) in dirty_tasks_update { update!( task, @@ -148,7 +148,7 @@ impl AggregatedDataUpdate { None } else { if old <= 0 && new > 0 { - if root_type.is_some() { + if root { task_to_schedule.push(*task_id); } result.dirty_tasks_update.insert(*task_id, 1); @@ -226,7 +226,7 @@ pub struct AggregationUpdateQueue { impl AggregationUpdateQueue { pub fn new() -> Self { Self { - jobs: VecDeque::new(), + jobs: VecDeque::with_capacity(8), } } @@ -238,8 +238,14 @@ impl AggregationUpdateQueue { self.jobs.push_back(job); } + pub fn run(job: AggregationUpdateJob, ctx: &ExecuteContext<'_>) { + let mut queue = Self::new(); + queue.push(job); + queue.execute(ctx); + } + pub fn process(&mut self, ctx: &ExecuteContext<'_>) -> bool { - if let Some(job) = self.jobs.pop_back() { + if let Some(job) = self.jobs.pop_front() { match job { AggregationUpdateJob::UpdateAggregationNumber { task_id, @@ -525,3 +531,15 @@ impl AggregationUpdateQueue { self.jobs.is_empty() } } + +impl Operation for AggregationUpdateQueue { + fn execute(mut self, ctx: &ExecuteContext<'_>) { + let _span = tracing::trace_span!("aggregation update queue").entered(); + loop { + ctx.operation_suspend_point(&self); + if self.process(ctx) { + return; + } + } + } +} diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs index dd1ff755fa7fb..26141fafed9cc 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs @@ -298,13 +298,16 @@ pub enum AnyOperation { ConnectChild(connect_child::ConnectChildOperation), Invalidate(invalidate::InvalidateOperation), CleanupOldEdges(cleanup_old_edges::CleanupOldEdgesOperation), + AggregationUpdate(aggregation_update::AggregationUpdateQueue), Nested(Vec), } impl_operation!(ConnectChild connect_child::ConnectChildOperation); impl_operation!(Invalidate invalidate::InvalidateOperation); impl_operation!(CleanupOldEdges cleanup_old_edges::CleanupOldEdgesOperation); +impl_operation!(AggregationUpdate aggregation_update::AggregationUpdateQueue); +pub use aggregation_update::{is_root_node, AggregationUpdateJob}; pub use cleanup_old_edges::OutdatedEdge; pub use update_cell::UpdateCellOperation; pub use update_output::UpdateOutputOperation; diff --git a/turbopack/crates/turbo-tasks-backend/src/data.rs b/turbopack/crates/turbo-tasks-backend/src/data.rs index 207949cfac33f..a9173e5db55ee 100644 --- a/turbopack/crates/turbo-tasks-backend/src/data.rs +++ b/turbopack/crates/turbo-tasks-backend/src/data.rs @@ -35,11 +35,32 @@ impl OutputValue { } } -#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[derive(Debug)] +pub struct RootState { + pub ty: RootType, + pub all_clean_event: Event, +} + +impl RootState { + pub fn new(ty: RootType) -> Self { + Self { + ty, + all_clean_event: Event::new(|| "RootState::all_clean_event".to_string()), + } + } +} + +#[derive(Debug, Clone, Copy)] pub enum RootType { RootTask, OnceTask, - _ReadingStronglyConsistent, + ReadingStronglyConsistent, +} + +impl Clone for RootState { + fn clone(&self) -> Self { + panic!("RootState cannot be cloned"); + } } #[derive(Debug)] @@ -168,13 +189,13 @@ pub enum CachedDataItem { collectible: CellRef, value: u32, }, - AggregatedUnfinishedTasks { + AggregatedDirtyTaskCount { value: u32, }, // Transient Root Type - AggregateRootType { - value: RootType, + AggregateRoot { + value: RootState, }, // Transient In Progress state @@ -231,8 +252,8 @@ impl CachedDataItem { CachedDataItem::AggregatedCollectible { collectible, .. } => { !collectible.task.is_transient() } - CachedDataItem::AggregatedUnfinishedTasks { .. } => true, - CachedDataItem::AggregateRootType { .. } => false, + CachedDataItem::AggregatedDirtyTaskCount { .. } => true, + CachedDataItem::AggregateRoot { .. } => false, CachedDataItem::InProgress { .. } => false, CachedDataItem::InProgressCell { .. } => false, CachedDataItem::OutdatedCollectible { .. } => false, @@ -289,8 +310,8 @@ impl CachedDataItemKey { CachedDataItemKey::AggregatedCollectible { collectible, .. } => { !collectible.task.is_transient() } - CachedDataItemKey::AggregatedUnfinishedTasks { .. } => true, - CachedDataItemKey::AggregateRootType { .. } => false, + CachedDataItemKey::AggregatedDirtyTaskCount { .. } => true, + CachedDataItemKey::AggregateRoot { .. } => false, CachedDataItemKey::InProgress { .. } => false, CachedDataItemKey::InProgressCell { .. } => false, CachedDataItemKey::OutdatedCollectible { .. } => false, From 7e2de2f7789a5a6cf53b24acc9da6857dc71e7d9 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Thu, 5 Sep 2024 01:26:42 +0200 Subject: [PATCH 41/81] improve task aggregation --- Cargo.lock | 1 + Cargo.toml | 1 + .../crates/turbo-tasks-backend/Cargo.toml | 3 +- .../backend/operation/aggregation_update.rs | 192 +++++++++++++++--- .../src/backend/operation/connect_child.rs | 4 +- .../src/backend/operation/mod.rs | 16 ++ .../src/backend/storage.rs | 20 +- .../src/utils/dash_map_multi.rs | 166 +++++++++++++++ .../turbo-tasks-backend/src/utils/mod.rs | 1 + 9 files changed, 371 insertions(+), 33 deletions(-) create mode 100644 turbopack/crates/turbo-tasks-backend/src/utils/dash_map_multi.rs diff --git a/Cargo.lock b/Cargo.lock index ff2b16f37ba01..00410fb4a3e07 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8577,6 +8577,7 @@ dependencies = [ "async-trait", "auto-hash-map", "dashmap", + "hashbrown 0.14.5", "indexmap 1.9.3", "once_cell", "parking_lot", diff --git a/Cargo.toml b/Cargo.toml index 4412bf582a2dd..098e39e555720 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -144,6 +144,7 @@ dunce = "1.0.3" either = "1.9.0" futures = "0.3.26" futures-retry = "0.6.0" +hashbrown = "0.14.5" httpmock = { version = "0.6.8", default-features = false } image = { version = "0.25.0", default-features = false } indexmap = "1.9.2" diff --git a/turbopack/crates/turbo-tasks-backend/Cargo.toml b/turbopack/crates/turbo-tasks-backend/Cargo.toml index ff458ac2808b2..01419f4d06e13 100644 --- a/turbopack/crates/turbo-tasks-backend/Cargo.toml +++ b/turbopack/crates/turbo-tasks-backend/Cargo.toml @@ -16,7 +16,8 @@ workspace = true anyhow = { workspace = true } async-trait = { workspace = true } auto-hash-map = { workspace = true } -dashmap = { workspace = true } +dashmap = { workspace = true, features = ["raw-api"]} +hashbrown = { workspace = true } indexmap = { workspace = true } once_cell = { workspace = true } parking_lot = { workspace = true } diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs index 668366689d2ef..3f26af76aa9e7 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs @@ -9,10 +9,10 @@ use turbo_tasks::TaskId; use super::{ExecuteContext, Operation, TaskGuard}; use crate::{ data::{CachedDataItem, CachedDataItemKey}, - get, get_many, iter_many, update, update_count, + get, get_many, iter_many, remove, update, update_count, }; -const LEAF_NUMBER: u32 = 8; +const LEAF_NUMBER: u32 = 16; pub fn is_aggregating_node(aggregation_number: u32) -> bool { aggregation_number >= LEAF_NUMBER @@ -22,6 +22,25 @@ pub fn is_root_node(aggregation_number: u32) -> bool { aggregation_number == u32::MAX } +fn get_followers_with_aggregation_number( + task: &TaskGuard<'_>, + aggregation_number: u32, +) -> Vec { + if is_aggregating_node(aggregation_number) { + get_many!(task, Follower { task } => task) + } else { + get_many!(task, Child { task } => task) + } +} + +fn get_followers(task: &TaskGuard<'_>) -> Vec { + get_followers_with_aggregation_number(task, get_aggregation_number(task)) +} + +pub fn get_aggregation_number(task: &TaskGuard<'_>) -> u32 { + get!(task, AggregationNumber).copied().unwrap_or_default() +} + #[derive(Serialize, Deserialize, Clone)] pub enum AggregationUpdateJob { UpdateAggregationNumber { @@ -70,7 +89,7 @@ pub struct AggregatedDataUpdate { impl AggregatedDataUpdate { fn from_task(task: &mut TaskGuard<'_>) -> Self { - let aggregation = get!(task, AggregationNumber).copied().unwrap_or_default(); + let aggregation = get_aggregation_number(task); let dirty = get!(task, Dirty).is_some(); if is_aggregating_node(aggregation) { let mut unfinished = get!(task, AggregatedDirtyTaskCount).copied().unwrap_or(0); @@ -130,6 +149,11 @@ impl AggregatedDataUpdate { Some(new) } }); + if result.unfinished == -1 { + if let Some(root_state) = get!(task, AggregateRoot) { + root_state.all_clean_event.notify(usize::MAX); + } + } } if !dirty_tasks_update.is_empty() { let mut task_to_schedule = Vec::new(); @@ -252,7 +276,7 @@ impl AggregationUpdateQueue { aggregation_number, } => { let mut task = ctx.task(task_id); - let old = get!(task, AggregationNumber).copied().unwrap_or_default(); + let old = get_aggregation_number(&task); if old < aggregation_number { task.insert(CachedDataItem::AggregationNumber { value: aggregation_number, @@ -280,6 +304,16 @@ impl AggregationUpdateQueue { task_id: follower_id, }); } + } else { + let children = iter_many!(task, Child { task } => task); + for child_id in children { + self.jobs.push_back( + AggregationUpdateJob::UpdateAggregationNumber { + task_id: child_id, + aggregation_number: aggregation_number + 1, + }, + ); + } } } } @@ -314,16 +348,13 @@ impl AggregationUpdateQueue { } => { let follower_aggregation_number = { let follower = ctx.task(new_follower_id); - get!(follower, AggregationNumber) - .copied() - .unwrap_or_default() + get_aggregation_number(&follower) }; let mut upper_ids_as_follower = Vec::new(); upper_ids.retain(|&upper_id| { let upper = ctx.task(upper_id); // decide if it should be an inner or follower - let upper_aggregation_number = - get!(upper, AggregationNumber).copied().unwrap_or_default(); + let upper_aggregation_number = get_aggregation_number(&upper); if !is_root_node(upper_aggregation_number) && upper_aggregation_number <= follower_aggregation_number @@ -349,7 +380,7 @@ impl AggregationUpdateQueue { }); if !upper_ids.is_empty() { let data = AggregatedDataUpdate::from_task(&mut follower); - let children: Vec<_> = get_many!(follower, Child { task } => task); + let children: Vec<_> = get_followers(&follower); drop(follower); for upper_id in upper_ids.iter() { @@ -437,7 +468,7 @@ impl AggregationUpdateQueue { }); if !upper_ids.is_empty() { let data = AggregatedDataUpdate::from_task(&mut follower).invert(); - let children: Vec<_> = get_many!(follower, Child { task } => task); + let children: Vec<_> = get_followers(&follower); drop(follower); for upper_id in upper_ids.iter() { @@ -486,12 +517,14 @@ impl AggregationUpdateQueue { let mut upper = ctx.task(upper_id); let diff = update.apply(&mut upper, self); if !diff.is_empty() { - let upper_ids = get_many!(upper, Upper { task } => task); - self.jobs - .push_back(AggregationUpdateJob::AggregatedDataUpdate { - upper_ids, - update: diff, - }); + let upper_ids: Vec<_> = get_many!(upper, Upper { task } => task); + if !upper_ids.is_empty() { + self.jobs + .push_back(AggregationUpdateJob::AggregatedDataUpdate { + upper_ids, + update: diff, + }); + } } } } @@ -499,12 +532,14 @@ impl AggregationUpdateQueue { let mut task = ctx.task(task_id); let diff = update.apply(&mut task, self); if !diff.is_empty() { - let upper_ids = get_many!(task, Upper { task } => task); - self.jobs - .push_back(AggregationUpdateJob::AggregatedDataUpdate { - upper_ids, - update: diff, - }); + let upper_ids: Vec<_> = get_many!(task, Upper { task } => task); + if !upper_ids.is_empty() { + self.jobs + .push_back(AggregationUpdateJob::AggregatedDataUpdate { + upper_ids, + update: diff, + }); + } } } AggregationUpdateJob::ScheduleWhenDirty { task_ids } => { @@ -518,12 +553,111 @@ impl AggregationUpdateQueue { } } } - AggregationUpdateJob::BalanceEdge { - upper_id: _, - task_id: _, - } => { - // TODO: implement - // Ignore for now + AggregationUpdateJob::BalanceEdge { upper_id, task_id } => { + let (mut upper, mut task) = ctx.task_pair(upper_id, task_id); + let upper_aggregation_number = get_aggregation_number(&upper); + let task_aggregation_number = get_aggregation_number(&task); + + let should_be_inner = is_root_node(upper_aggregation_number) + || upper_aggregation_number > task_aggregation_number; + let should_be_follower = task_aggregation_number > upper_aggregation_number; + + if should_be_inner { + // remove all follower edges + let count = remove!(upper, Follower { task: task_id }).unwrap_or_default(); + if count > 0 { + // notify uppers about lost follower + let upper_ids: Vec<_> = get_many!(upper, Upper { task } => task); + if !upper_ids.is_empty() { + self.jobs + .push_back(AggregationUpdateJob::InnerLostFollower { + upper_ids, + lost_follower_id: task_id, + }); + } + + // Add the same amount of upper edges + if update_count!(task, Upper { task: upper_id }, count as i32) { + // When this is a new inner node, update aggregated data and + // followers + let data = AggregatedDataUpdate::from_task(&mut task); + let followers = get_followers(&task); + let diff = data.apply(&mut upper, self); + + let upper_ids: Vec<_> = get_many!(upper, Upper { task } => task); + if !upper_ids.is_empty() { + if !diff.is_empty() { + // Notify uppers about changed aggregated data + self.jobs.push_back( + AggregationUpdateJob::AggregatedDataUpdate { + upper_ids: upper_ids.clone(), + update: diff, + }, + ); + } + if !followers.is_empty() { + self.jobs.push_back( + AggregationUpdateJob::InnerHasNewFollowers { + upper_ids, + new_follower_ids: followers, + }, + ); + } + } + } + } + } else if should_be_follower { + // Remove the upper edge + let count = remove!(task, Upper { task: upper_id }).unwrap_or_default(); + if count > 0 { + // Add the same amount of follower edges + if update_count!(upper, Follower { task: task_id }, count as i32) { + // notify uppers about new follower + let upper_ids: Vec<_> = get_many!(upper, Upper { task } => task); + if !upper_ids.is_empty() { + self.jobs.push_back( + AggregationUpdateJob::InnerHasNewFollower { + upper_ids, + new_follower_id: task_id, + }, + ); + } + } + + // Since this is no longer an inner node, update the aggregated data and + // followers + let data = AggregatedDataUpdate::from_task(&mut task).invert(); + let followers = get_followers(&task); + let diff = data.apply(&mut upper, self); + let upper_ids: Vec<_> = get_many!(upper, Upper { task } => task); + if !upper_ids.is_empty() { + if !diff.is_empty() { + self.jobs.push_back( + AggregationUpdateJob::AggregatedDataUpdate { + upper_ids: upper_ids.clone(), + update: diff, + }, + ); + } + if !followers.is_empty() { + self.jobs + .push_back(AggregationUpdateJob::InnerLostFollowers { + upper_ids, + lost_follower_ids: followers, + }); + } + } + } + } else { + // both nodes have the same aggregation number + // We need to change the aggregation number of the task + let new_aggregation_number = upper_aggregation_number + 1; + self.jobs + .push_back(AggregationUpdateJob::UpdateAggregationNumber { + task_id, + aggregation_number: new_aggregation_number, + }); + } } } } diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs index d31b0766a52f5..be8db5749a175 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs @@ -12,6 +12,8 @@ use crate::{ get, get_many, }; +const AGGREGATION_NUMBER_BUFFER_SPACE: u32 = 2; + #[derive(Serialize, Deserialize, Clone, Default)] pub enum ConnectChildOperation { UpdateAggregation { @@ -48,7 +50,7 @@ impl ConnectChildOperation { } else if !is_root_node(parent_aggregation) { queue.push(AggregationUpdateJob::UpdateAggregationNumber { task_id: child_task_id, - aggregation_number: parent_aggregation + 1, + aggregation_number: parent_aggregation + AGGREGATION_NUMBER_BUFFER_SPACE + 1, }); } if is_aggregating_node(parent_aggregation) { diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs index 26141fafed9cc..c9848b9e35c57 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs @@ -58,6 +58,22 @@ impl<'a> ExecuteContext<'a> { } } + pub fn task_pair(&self, task_id1: TaskId, task_id2: TaskId) -> (TaskGuard<'a>, TaskGuard<'a>) { + let (task1, task2) = self.backend.storage.access_pair_mut(task_id1, task_id2); + ( + TaskGuard { + task: task1, + task_id: task_id1, + backend: self.backend, + }, + TaskGuard { + task: task2, + task_id: task_id2, + backend: self.backend, + }, + ) + } + pub fn schedule(&self, task_id: TaskId) { self.turbo_tasks.schedule(task_id); } diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs b/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs index b3256ca7d722e..a6c4a5dd8c200 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs @@ -6,10 +6,12 @@ use std::{ }; use auto_hash_map::{map::Entry, AutoMap}; -use dashmap::{mapref::one::RefMut, DashMap}; +use dashmap::DashMap; use rustc_hash::FxHasher; use turbo_tasks::KeyValuePair; +use crate::utils::dash_map_multi::{get_multiple_mut, RefMut}; + pub struct InnerStorage { // TODO consider adding some inline storage map: AutoMap, @@ -120,7 +122,21 @@ where dashmap::mapref::entry::Entry::Occupied(e) => e.into_ref(), dashmap::mapref::entry::Entry::Vacant(e) => e.insert(InnerStorage::new()), }; - StorageWriteGuard { inner } + StorageWriteGuard { + inner: inner.into(), + } + } + + pub fn access_pair_mut( + &self, + key1: K, + key2: K, + ) -> (StorageWriteGuard<'_, K, T>, StorageWriteGuard<'_, K, T>) { + let (a, b) = get_multiple_mut(&self.map, key1, key2, || InnerStorage::new()); + ( + StorageWriteGuard { inner: a }, + StorageWriteGuard { inner: b }, + ) } } diff --git a/turbopack/crates/turbo-tasks-backend/src/utils/dash_map_multi.rs b/turbopack/crates/turbo-tasks-backend/src/utils/dash_map_multi.rs new file mode 100644 index 0000000000000..ead6317a0262e --- /dev/null +++ b/turbopack/crates/turbo-tasks-backend/src/utils/dash_map_multi.rs @@ -0,0 +1,166 @@ +use std::{ + hash::{BuildHasher, Hash}, + ops::{Deref, DerefMut}, + sync::Arc, +}; + +use dashmap::{DashMap, RwLockWriteGuard, SharedValue}; +use hashbrown::{hash_map, HashMap}; + +pub enum RefMut<'a, K, V, S> +where + S: BuildHasher, +{ + Base(dashmap::mapref::one::RefMut<'a, K, V, S>), + Simple { + #[allow(dead_code)] + guard: RwLockWriteGuard<'a, HashMap, S>>, + key: *const K, + value: *mut V, + }, + Shared { + #[allow(dead_code)] + guard: Arc, S>>>, + key: *const K, + value: *mut V, + }, +} + +unsafe impl<'a, K: Eq + Hash + Sync, V: Sync, S: BuildHasher> Send for RefMut<'a, K, V, S> {} +unsafe impl<'a, K: Eq + Hash + Sync, V: Sync, S: BuildHasher> Sync for RefMut<'a, K, V, S> {} + +impl<'a, K: Eq + Hash, V, S: BuildHasher> RefMut<'a, K, V, S> { + pub fn key(&self) -> &K { + self.pair().0 + } + + pub fn value(&self) -> &V { + self.pair().1 + } + + pub fn value_mut(&mut self) -> &mut V { + self.pair_mut().1 + } + + pub fn pair(&self) -> (&K, &V) { + match self { + RefMut::Base(r) => r.pair(), + &RefMut::Simple { key, value, .. } => unsafe { (&*key, &*value) }, + &RefMut::Shared { key, value, .. } => unsafe { (&*key, &*value) }, + } + } + + pub fn pair_mut(&mut self) -> (&K, &mut V) { + match self { + RefMut::Base(r) => r.pair_mut(), + &mut RefMut::Simple { key, value, .. } => unsafe { (&*key, &mut *value) }, + &mut RefMut::Shared { key, value, .. } => unsafe { (&*key, &mut *value) }, + } + } +} + +impl<'a, K: Eq + Hash, V, S: BuildHasher> Deref for RefMut<'a, K, V, S> { + type Target = V; + + fn deref(&self) -> &V { + self.value() + } +} + +impl<'a, K: Eq + Hash, V, S: BuildHasher> DerefMut for RefMut<'a, K, V, S> { + fn deref_mut(&mut self) -> &mut V { + self.value_mut() + } +} + +impl<'a, K, V, S> From> for RefMut<'a, K, V, S> +where + K: Hash + Eq, + S: BuildHasher, +{ + fn from(r: dashmap::mapref::one::RefMut<'a, K, V, S>) -> Self { + RefMut::Base(r) + } +} + +pub fn get_multiple_mut( + map: &DashMap, + key1: K, + key2: K, + insert_with: impl Fn() -> V, +) -> (RefMut<'_, K, V, S>, RefMut<'_, K, V, S>) +where + K: Hash + Eq + Clone, + S: BuildHasher + Clone, +{ + let s1 = map.determine_map(&key1); + let s2 = map.determine_map(&key2); + let shards = map.shards(); + if s1 == s2 { + let mut guard = shards[s1].write(); + let e1 = guard + .raw_entry_mut() + .from_key(&key1) + .or_insert_with(|| (key1.clone(), SharedValue::new(insert_with()))); + let mut key1_ptr = e1.0 as *const K; + let mut value1_ptr = e1.1.get_mut() as *mut V; + let key2_ptr; + let value2_ptr; + match guard.raw_entry_mut().from_key(&key2) { + hash_map::RawEntryMut::Occupied(e) => { + let e2 = e.into_key_value(); + key2_ptr = e2.0 as *const K; + value2_ptr = e2.1.get_mut() as *mut V; + } + hash_map::RawEntryMut::Vacant(e) => { + let e2 = e.insert(key2.clone(), SharedValue::new(insert_with())); + key2_ptr = e2.0 as *const K; + value2_ptr = e2.1.get_mut() as *mut V; + // inserting a new entry might invalidate the pointers of the first entry + let e1 = guard.get_key_value_mut(&key1).unwrap(); + key1_ptr = e1.0 as *const K; + value1_ptr = e1.1.get_mut() as *mut V; + } + } + let guard = Arc::new(guard); + ( + RefMut::Shared { + guard: guard.clone(), + key: key1_ptr, + value: value1_ptr, + }, + RefMut::Shared { + guard, + key: key2_ptr, + value: value2_ptr, + }, + ) + } else { + let mut guard1 = shards[s1].write(); + let mut guard2 = shards[s2].write(); + let e1 = guard1 + .raw_entry_mut() + .from_key(&key1) + .or_insert_with(|| (key1, SharedValue::new(insert_with()))); + let key1 = e1.0 as *const K; + let value1 = e1.1.get_mut() as *mut V; + let e2 = guard2 + .raw_entry_mut() + .from_key(&key2) + .or_insert_with(|| (key2, SharedValue::new(insert_with()))); + let key2 = e2.0 as *const K; + let value2 = e2.1.get_mut() as *mut V; + ( + RefMut::Simple { + guard: guard1, + key: key1, + value: value1, + }, + RefMut::Simple { + guard: guard2, + key: key2, + value: value2, + }, + ) + } +} diff --git a/turbopack/crates/turbo-tasks-backend/src/utils/mod.rs b/turbopack/crates/turbo-tasks-backend/src/utils/mod.rs index 3ad1e0475ec86..676e3b809b388 100644 --- a/turbopack/crates/turbo-tasks-backend/src/utils/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/utils/mod.rs @@ -1,3 +1,4 @@ pub mod bi_map; pub mod chunked_vec; +pub mod dash_map_multi; pub mod ptr_eq_arc; From 446caa6259690ccb45ee6235ed4e41534a2dfe7c Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Tue, 3 Sep 2024 16:04:06 +0200 Subject: [PATCH 42/81] remove scheduling on invalidation --- .../src/backend/operation/invalidate.rs | 38 +------------------ 1 file changed, 1 insertion(+), 37 deletions(-) diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/invalidate.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/invalidate.rs index 66637422b1c39..106d3e4d5a3ab 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/invalidate.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/invalidate.rs @@ -6,10 +6,7 @@ use super::{ aggregation_update::{AggregatedDataUpdate, AggregationUpdateJob, AggregationUpdateQueue}, ExecuteContext, Operation, }; -use crate::{ - data::{CachedDataItem, InProgressState}, - get, update, -}; +use crate::data::CachedDataItem; #[derive(Serialize, Deserialize, Clone, Default)] pub enum InvalidateOperation { @@ -65,39 +62,6 @@ pub fn make_task_dirty(task_id: TaskId, queue: &mut AggregationUpdateQueue, ctx: let mut task = ctx.task(task_id); if task.add(CachedDataItem::Dirty { value: () }) { - let in_progress = match get!(task, InProgress) { - Some(InProgressState::InProgress { - stale, once_task, .. - }) if !once_task => { - if !*stale { - update!(task, InProgress, |in_progress| { - let Some(InProgressState::InProgress { - stale: _, - once_task, - done_event, - }) = in_progress - else { - unreachable!(); - }; - Some(InProgressState::InProgress { - stale: true, - once_task, - done_event, - }) - }); - } - true - } - Some(_) => true, - None => false, - }; - if !in_progress - && task.add(CachedDataItem::new_scheduled( - task.backend.get_task_desc_fn(task_id), - )) - { - ctx.turbo_tasks.schedule(task_id) - } queue.push(AggregationUpdateJob::DataUpdate { task_id, update: AggregatedDataUpdate::dirty_task(task_id), From 703b017dce98a789dc6ca2d2492b3dfd234b9ca4 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Tue, 3 Sep 2024 16:04:22 +0200 Subject: [PATCH 43/81] add recompute test case --- turbopack/crates/turbo-tasks-backend/tests/recompute.rs | 1 + 1 file changed, 1 insertion(+) create mode 120000 turbopack/crates/turbo-tasks-backend/tests/recompute.rs diff --git a/turbopack/crates/turbo-tasks-backend/tests/recompute.rs b/turbopack/crates/turbo-tasks-backend/tests/recompute.rs new file mode 120000 index 0000000000000..5c35fb81af4e3 --- /dev/null +++ b/turbopack/crates/turbo-tasks-backend/tests/recompute.rs @@ -0,0 +1 @@ +../../turbo-tasks-testing/tests/recompute.rs \ No newline at end of file From fcb7401f0c001dd7fc2e0f082aff5553e6ce0182 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Tue, 3 Sep 2024 16:11:53 +0200 Subject: [PATCH 44/81] more test cases --- turbopack/crates/turbo-tasks-backend/tests/generics.rs | 1 + turbopack/crates/turbo-tasks-backend/tests/local_cell.rs | 1 + turbopack/crates/turbo-tasks-backend/tests/read_ref_cell.rs | 1 + turbopack/crates/turbo-tasks-backend/tests/trait_ref_cell.rs | 1 + 4 files changed, 4 insertions(+) create mode 120000 turbopack/crates/turbo-tasks-backend/tests/generics.rs create mode 120000 turbopack/crates/turbo-tasks-backend/tests/local_cell.rs create mode 120000 turbopack/crates/turbo-tasks-backend/tests/read_ref_cell.rs create mode 120000 turbopack/crates/turbo-tasks-backend/tests/trait_ref_cell.rs diff --git a/turbopack/crates/turbo-tasks-backend/tests/generics.rs b/turbopack/crates/turbo-tasks-backend/tests/generics.rs new file mode 120000 index 0000000000000..526d71f58d8ba --- /dev/null +++ b/turbopack/crates/turbo-tasks-backend/tests/generics.rs @@ -0,0 +1 @@ +../../turbo-tasks-testing/tests/generics.rs \ No newline at end of file diff --git a/turbopack/crates/turbo-tasks-backend/tests/local_cell.rs b/turbopack/crates/turbo-tasks-backend/tests/local_cell.rs new file mode 120000 index 0000000000000..9249e3399052e --- /dev/null +++ b/turbopack/crates/turbo-tasks-backend/tests/local_cell.rs @@ -0,0 +1 @@ +../../turbo-tasks-testing/tests/local_cell.rs \ No newline at end of file diff --git a/turbopack/crates/turbo-tasks-backend/tests/read_ref_cell.rs b/turbopack/crates/turbo-tasks-backend/tests/read_ref_cell.rs new file mode 120000 index 0000000000000..4e1719dfefec2 --- /dev/null +++ b/turbopack/crates/turbo-tasks-backend/tests/read_ref_cell.rs @@ -0,0 +1 @@ +../../turbo-tasks-testing/tests/read_ref_cell.rs \ No newline at end of file diff --git a/turbopack/crates/turbo-tasks-backend/tests/trait_ref_cell.rs b/turbopack/crates/turbo-tasks-backend/tests/trait_ref_cell.rs new file mode 120000 index 0000000000000..026eed7f3b50f --- /dev/null +++ b/turbopack/crates/turbo-tasks-backend/tests/trait_ref_cell.rs @@ -0,0 +1 @@ +../../turbo-tasks-testing/tests/trait_ref_cell.rs \ No newline at end of file From e6f55b468ddf6aff32c28f76f18e2ff44cb6b007 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Wed, 4 Sep 2024 09:07:13 +0200 Subject: [PATCH 45/81] aggregation improvements --- .../backend/operation/aggregation_update.rs | 199 ++++++++++-------- .../backend/operation/cleanup_old_edges.rs | 22 +- .../src/backend/operation/connect_child.rs | 4 +- .../src/backend/storage.rs | 36 ++-- .../crates/turbo-tasks-backend/src/data.rs | 10 +- 5 files changed, 151 insertions(+), 120 deletions(-) diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs index 3f26af76aa9e7..71d39e7663013 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs @@ -27,7 +27,7 @@ fn get_followers_with_aggregation_number( aggregation_number: u32, ) -> Vec { if is_aggregating_node(aggregation_number) { - get_many!(task, Follower { task } => task) + get_many!(task, Follower { task } count if count > 0 => task) } else { get_many!(task, Child { task } => task) } @@ -37,11 +37,19 @@ fn get_followers(task: &TaskGuard<'_>) -> Vec { get_followers_with_aggregation_number(task, get_aggregation_number(task)) } +pub fn get_uppers(task: &TaskGuard<'_>) -> Vec { + get_many!(task, Upper { task } count if count > 0 => task) +} + +fn iter_uppers<'a>(task: &'a TaskGuard<'a>) -> impl Iterator + 'a { + iter_many!(task, Upper { task } count if count > 0 => task) +} + pub fn get_aggregation_number(task: &TaskGuard<'_>) -> u32 { get!(task, AggregationNumber).copied().unwrap_or_default() } -#[derive(Serialize, Deserialize, Clone)] +#[derive(Serialize, Deserialize, Clone, Debug)] pub enum AggregationUpdateJob { UpdateAggregationNumber { task_id: TaskId, @@ -80,9 +88,9 @@ pub enum AggregationUpdateJob { }, } -#[derive(Default, Serialize, Deserialize, Clone)] +#[derive(Default, Serialize, Deserialize, Clone, Debug)] pub struct AggregatedDataUpdate { - unfinished: i32, + dirty_task_count: i32, dirty_tasks_update: HashMap, // TODO collectibles } @@ -92,7 +100,7 @@ impl AggregatedDataUpdate { let aggregation = get_aggregation_number(task); let dirty = get!(task, Dirty).is_some(); if is_aggregating_node(aggregation) { - let mut unfinished = get!(task, AggregatedDirtyTaskCount).copied().unwrap_or(0); + let mut dirty_task_count = get!(task, AggregatedDirtyTaskCount).copied().unwrap_or(0); let mut dirty_tasks_update = task .iter() .filter_map(|(key, _)| match *key { @@ -101,11 +109,11 @@ impl AggregatedDataUpdate { }) .collect::>(); if dirty { - unfinished += 1; + dirty_task_count += 1; dirty_tasks_update.insert(task.id(), 1); } Self { - unfinished: unfinished as i32, + dirty_task_count: dirty_task_count as i32, dirty_tasks_update, } } else if dirty { @@ -116,7 +124,7 @@ impl AggregatedDataUpdate { } fn invert(mut self) -> Self { - self.unfinished = -self.unfinished; + self.dirty_task_count = -self.dirty_task_count; for value in self.dirty_tasks_update.values_mut() { *value = -*value; } @@ -129,27 +137,22 @@ impl AggregatedDataUpdate { queue: &mut AggregationUpdateQueue, ) -> AggregatedDataUpdate { let Self { - unfinished, + dirty_task_count, dirty_tasks_update, } = self; let mut result = Self::default(); - if *unfinished != 0 { - update!(task, AggregatedDirtyTaskCount, |old: Option| { + if *dirty_task_count != 0 { + update!(task, AggregatedDirtyTaskCount, |old: Option| { let old = old.unwrap_or(0); - let new = old as i32 + *unfinished; - debug_assert!(new >= 0); - let new = new as u32; - if new == 0 { - result.unfinished = -1; - None - } else { - if old <= 0 && new > 0 { - result.unfinished = 1; - } - Some(new) + let new = old + *dirty_task_count; + if old <= 0 && new > 0 { + result.dirty_task_count = 1; + } else if old > 0 && new <= 0 { + result.dirty_task_count = -1; } + (new != 0).then_some(new) }); - if result.unfinished == -1 { + if result.dirty_task_count == -1 { if let Some(root_state) = get!(task, AggregateRoot) { root_state.all_clean_event.notify(usize::MAX); } @@ -162,23 +165,18 @@ impl AggregatedDataUpdate { update!( task, AggregatedDirtyTask { task: *task_id }, - |old: Option| { + |old: Option| { let old = old.unwrap_or(0); - let new = old as i32 + *count; - debug_assert!(new >= 0); - let new = new as u32; - if new == 0 { - result.dirty_tasks_update.insert(*task_id, -1); - None - } else { - if old <= 0 && new > 0 { - if root { - task_to_schedule.push(*task_id); - } - result.dirty_tasks_update.insert(*task_id, 1); + let new = old + *count; + if old <= 0 && new > 0 { + if root { + task_to_schedule.push(*task_id); } - Some(new) + result.dirty_tasks_update.insert(*task_id, 1); + } else if old > 0 && new <= 0 { + result.dirty_tasks_update.insert(*task_id, -1); } + (new != 0).then_some(new) } ); } @@ -193,22 +191,22 @@ impl AggregatedDataUpdate { fn is_empty(&self) -> bool { let Self { - unfinished, + dirty_task_count, dirty_tasks_update, } = self; - *unfinished == 0 && dirty_tasks_update.is_empty() + *dirty_task_count == 0 && dirty_tasks_update.is_empty() } pub fn dirty_task(task_id: TaskId) -> Self { Self { - unfinished: 1, + dirty_task_count: 1, dirty_tasks_update: HashMap::from([(task_id, 1)]), } } pub fn no_longer_dirty_task(task_id: TaskId) -> Self { Self { - unfinished: -1, + dirty_task_count: -1, dirty_tasks_update: HashMap::from([(task_id, -1)]), } } @@ -236,7 +234,7 @@ impl Add for AggregatedDataUpdate { } } Self { - unfinished: self.unfinished + rhs.unfinished, + dirty_task_count: self.dirty_task_count + rhs.dirty_task_count, dirty_tasks_update, } } @@ -297,13 +295,21 @@ impl AggregationUpdateQueue { if is_aggregating_node(aggregation_number) { // followers might become inner nodes when the aggregation number is // increased - let followers = iter_many!(task, Follower { task } => task); + let followers = + iter_many!(task, Follower { task } count if count > 0 => task); for follower_id in followers { self.jobs.push_back(AggregationUpdateJob::BalanceEdge { upper_id: task_id, task_id: follower_id, }); } + let uppers = iter_uppers(&task); + for upper_id in uppers { + self.jobs.push_back(AggregationUpdateJob::BalanceEdge { + upper_id, + task_id, + }); + } } else { let children = iter_many!(task, Child { task } => task); for child_id in children { @@ -383,18 +389,20 @@ impl AggregationUpdateQueue { let children: Vec<_> = get_followers(&follower); drop(follower); - for upper_id in upper_ids.iter() { - // add data to upper - let mut upper = ctx.task(*upper_id); - let diff = data.apply(&mut upper, self); - if !diff.is_empty() { - let upper_ids = get_many!(upper, Upper { task } => task); - self.jobs.push_back( - AggregationUpdateJob::AggregatedDataUpdate { - upper_ids, - update: diff, - }, - ) + if !data.is_empty() { + for upper_id in upper_ids.iter() { + // add data to upper + let mut upper = ctx.task(*upper_id); + let diff = data.apply(&mut upper, self); + if !diff.is_empty() { + let upper_ids = get_uppers(&upper); + self.jobs.push_back( + AggregationUpdateJob::AggregatedDataUpdate { + upper_ids, + update: diff, + }, + ) + } } } if !children.is_empty() { @@ -471,17 +479,20 @@ impl AggregationUpdateQueue { let children: Vec<_> = get_followers(&follower); drop(follower); - for upper_id in upper_ids.iter() { - // remove data from upper - let mut upper = ctx.task(*upper_id); - let diff = data.apply(&mut upper, self); - if !diff.is_empty() { - let upper_ids = get_many!(upper, Upper { task } => task); - self.jobs - .push_back(AggregationUpdateJob::AggregatedDataUpdate { - upper_ids, - update: diff, - }) + if !data.is_empty() { + for upper_id in upper_ids.iter() { + // remove data from upper + let mut upper = ctx.task(*upper_id); + let diff = data.apply(&mut upper, self); + if !diff.is_empty() { + let upper_ids = get_uppers(&upper); + self.jobs.push_back( + AggregationUpdateJob::AggregatedDataUpdate { + upper_ids, + update: diff, + }, + ) + } } } if !children.is_empty() { @@ -517,7 +528,7 @@ impl AggregationUpdateQueue { let mut upper = ctx.task(upper_id); let diff = update.apply(&mut upper, self); if !diff.is_empty() { - let upper_ids: Vec<_> = get_many!(upper, Upper { task } => task); + let upper_ids = get_uppers(&upper); if !upper_ids.is_empty() { self.jobs .push_back(AggregationUpdateJob::AggregatedDataUpdate { @@ -529,17 +540,14 @@ impl AggregationUpdateQueue { } } AggregationUpdateJob::DataUpdate { task_id, update } => { - let mut task = ctx.task(task_id); - let diff = update.apply(&mut task, self); - if !diff.is_empty() { - let upper_ids: Vec<_> = get_many!(task, Upper { task } => task); - if !upper_ids.is_empty() { - self.jobs - .push_back(AggregationUpdateJob::AggregatedDataUpdate { - upper_ids, - update: diff, - }); - } + let task = ctx.task(task_id); + let upper_ids: Vec<_> = get_uppers(&task); + if !upper_ids.is_empty() { + self.jobs + .push_back(AggregationUpdateJob::AggregatedDataUpdate { + upper_ids, + update: update.clone(), + }); } } AggregationUpdateJob::ScheduleWhenDirty { task_ids } => { @@ -565,16 +573,13 @@ impl AggregationUpdateQueue { if should_be_inner { // remove all follower edges let count = remove!(upper, Follower { task: task_id }).unwrap_or_default(); - if count > 0 { - // notify uppers about lost follower - let upper_ids: Vec<_> = get_many!(upper, Upper { task } => task); - if !upper_ids.is_empty() { - self.jobs - .push_back(AggregationUpdateJob::InnerLostFollower { - upper_ids, - lost_follower_id: task_id, - }); - } + if count < 0 { + upper.add_new(CachedDataItem::Follower { + task: task_id, + value: count, + }) + } else if count > 0 { + let upper_ids = get_uppers(&upper); // Add the same amount of upper edges if update_count!(task, Upper { task: upper_id }, count as i32) { @@ -584,7 +589,6 @@ impl AggregationUpdateQueue { let followers = get_followers(&task); let diff = data.apply(&mut upper, self); - let upper_ids: Vec<_> = get_many!(upper, Upper { task } => task); if !upper_ids.is_empty() { if !diff.is_empty() { // Notify uppers about changed aggregated data @@ -598,26 +602,36 @@ impl AggregationUpdateQueue { if !followers.is_empty() { self.jobs.push_back( AggregationUpdateJob::InnerHasNewFollowers { - upper_ids, + upper_ids: upper_ids.clone(), new_follower_ids: followers, }, ); } } } + + // notify uppers about lost follower + if !upper_ids.is_empty() { + self.jobs + .push_back(AggregationUpdateJob::InnerLostFollower { + upper_ids, + lost_follower_id: task_id, + }); + } } } else if should_be_follower { // Remove the upper edge let count = remove!(task, Upper { task: upper_id }).unwrap_or_default(); if count > 0 { + let upper_ids: Vec<_> = get_uppers(&upper); + // Add the same amount of follower edges if update_count!(upper, Follower { task: task_id }, count as i32) { // notify uppers about new follower - let upper_ids: Vec<_> = get_many!(upper, Upper { task } => task); if !upper_ids.is_empty() { self.jobs.push_back( AggregationUpdateJob::InnerHasNewFollower { - upper_ids, + upper_ids: upper_ids.clone(), new_follower_id: task_id, }, ); @@ -629,7 +643,6 @@ impl AggregationUpdateQueue { let data = AggregatedDataUpdate::from_task(&mut task).invert(); let followers = get_followers(&task); let diff = data.apply(&mut upper, self); - let upper_ids: Vec<_> = get_many!(upper, Upper { task } => task); if !upper_ids.is_empty() { if !diff.is_empty() { self.jobs.push_back( diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/cleanup_old_edges.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/cleanup_old_edges.rs index faaf54b6203fa..04a266c98d150 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/cleanup_old_edges.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/cleanup_old_edges.rs @@ -4,7 +4,10 @@ use serde::{Deserialize, Serialize}; use turbo_tasks::TaskId; use super::{ - aggregation_update::{AggregatedDataUpdate, AggregationUpdateJob, AggregationUpdateQueue}, + aggregation_update::{ + get_aggregation_number, get_uppers, is_aggregating_node, AggregatedDataUpdate, + AggregationUpdateJob, AggregationUpdateQueue, + }, invalidate::make_task_dirty, ExecuteContext, Operation, }; @@ -74,11 +77,18 @@ impl Operation for CleanupOldEdgesOperation { OutdatedEdge::Child(child_id) => { let mut task = ctx.task(task_id); task.remove(&CachedDataItemKey::Child { task: child_id }); - let upper_ids = get_many!(task, Upper { task } => task); - queue.push(AggregationUpdateJob::InnerLostFollower { - upper_ids, - lost_follower_id: child_id, - }); + if is_aggregating_node(get_aggregation_number(&task)) { + queue.push(AggregationUpdateJob::InnerLostFollower { + upper_ids: vec![task_id], + lost_follower_id: child_id, + }); + } else { + let upper_ids = get_uppers(&task); + queue.push(AggregationUpdateJob::InnerLostFollower { + upper_ids, + lost_follower_id: child_id, + }); + } } OutdatedEdge::CellDependency(CellRef { task: cell_task_id, diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs index be8db5749a175..7fc8107ea5471 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs @@ -3,7 +3,7 @@ use turbo_tasks::TaskId; use super::{ aggregation_update::{ - is_aggregating_node, is_root_node, AggregationUpdateJob, AggregationUpdateQueue, + get_uppers, is_aggregating_node, is_root_node, AggregationUpdateJob, AggregationUpdateQueue, }, ExecuteContext, Operation, }; @@ -59,7 +59,7 @@ impl ConnectChildOperation { new_follower_id: child_task_id, }); } else { - let upper_ids = get_many!(parent_task, Upper { task } => task); + let upper_ids = get_uppers(&parent_task); queue.push(AggregationUpdateJob::InnerHasNewFollower { upper_ids, new_follower_id: child_task_id, diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs b/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs index a6c4a5dd8c200..78a7ab9e23b90 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs @@ -182,7 +182,7 @@ macro_rules! iter_many { $task .iter() .filter_map(|(key, _)| match *key { - CachedDataItemKey::$key $input => Some($value), + $crate::data::CachedDataItemKey::$key $input => Some($value), _ => None, }) }; @@ -190,7 +190,15 @@ macro_rules! iter_many { $task .iter() .filter_map(|(key, value)| match (key, value) { - (&CachedDataItemKey::$key $input, &CachedDataItemValue::$key { value: $value_ident }) => Some($value), + (&$crate::data::CachedDataItemKey::$key $input, &$crate::data::CachedDataItemValue::$key { value: $value_ident }) => Some($value), + _ => None, + }) + }; + ($task:ident, $key:ident $input:tt $value_ident:ident if $cond:expr => $value:expr) => { + $task + .iter() + .filter_map(|(key, value)| match (key, value) { + (&$crate::data::CachedDataItemKey::$key $input, &$crate::data::CachedDataItemValue::$key { value: $value_ident }) if $cond => Some($value), _ => None, }) }; @@ -198,8 +206,8 @@ macro_rules! iter_many { $task .iter() .filter_map(|(key, _)| match *key { - CachedDataItemKey::$key1 $input1 => Some($value1), - CachedDataItemKey::$key2 $input2 => Some($value2), + $crate::data::CachedDataItemKey::$key1 $input1 => Some($value1), + $crate::data::CachedDataItemKey::$key2 $input2 => Some($value2), _ => None, }) }; @@ -213,6 +221,9 @@ macro_rules! get_many { ($task:ident, $key:ident $input:tt $value_ident:ident => $value:expr) => { $crate::iter_many!($task, $key $input $value_ident => $value).collect() }; + ($task:ident, $key:ident $input:tt $value_ident:ident if $cond:expr => $value:expr) => { + $crate::iter_many!($task, $key $input $value_ident if $cond => $value).collect() + }; ($task:ident, $key1:ident $input1:tt => $value1:ident, $key2:ident $input2:tt => $value2:ident) => { $crate::iter_many!($task, $key1 $input1 => $value1, $key2 $input2 => $value2).collect() }; @@ -258,17 +269,14 @@ macro_rules! update_count { match $update { update => { let mut state_change = false; - $crate::update!($task, $key $input, |old: Option| { - if old.is_none() { - state_change = true; - } - let old = old.unwrap_or(0); - let new = old as i32 + update; - if new == 0 { - state_change = true; - None + $crate::update!($task, $key $input, |old: Option| { + if let Some(old) = old { + let new = old + update; + state_change = old <= 0 && new > 0 || old > 0 && new <= 0; + (new != 0).then_some(new) } else { - Some(new as u32) + state_change = update > 0; + (update != 0).then_some(update) } }); state_change diff --git a/turbopack/crates/turbo-tasks-backend/src/data.rs b/turbopack/crates/turbo-tasks-backend/src/data.rs index a9173e5db55ee..a33d8263acf3f 100644 --- a/turbopack/crates/turbo-tasks-backend/src/data.rs +++ b/turbopack/crates/turbo-tasks-backend/src/data.rs @@ -173,24 +173,24 @@ pub enum CachedDataItem { }, Follower { task: TaskId, - value: u32, + value: i32, }, Upper { task: TaskId, - value: u32, + value: i32, }, // Aggregated Data AggregatedDirtyTask { task: TaskId, - value: u32, + value: i32, }, AggregatedCollectible { collectible: CellRef, - value: u32, + value: i32, }, AggregatedDirtyTaskCount { - value: u32, + value: i32, }, // Transient Root Type From 12f26c9919465328460c9432472a41332e1c2b43 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Fri, 6 Sep 2024 12:15:43 +0200 Subject: [PATCH 46/81] aggregation fixes --- .../backend/operation/aggregation_update.rs | 166 ++++++++---------- 1 file changed, 70 insertions(+), 96 deletions(-) diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs index 71d39e7663013..ee3a826678285 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs @@ -298,27 +298,22 @@ impl AggregationUpdateQueue { let followers = iter_many!(task, Follower { task } count if count > 0 => task); for follower_id in followers { - self.jobs.push_back(AggregationUpdateJob::BalanceEdge { + self.push(AggregationUpdateJob::BalanceEdge { upper_id: task_id, task_id: follower_id, }); } let uppers = iter_uppers(&task); for upper_id in uppers { - self.jobs.push_back(AggregationUpdateJob::BalanceEdge { - upper_id, - task_id, - }); + self.push(AggregationUpdateJob::BalanceEdge { upper_id, task_id }); } } else { let children = iter_many!(task, Child { task } => task); for child_id in children { - self.jobs.push_back( - AggregationUpdateJob::UpdateAggregationNumber { - task_id: child_id, - aggregation_number: aggregation_number + 1, - }, - ); + self.push(AggregationUpdateJob::UpdateAggregationNumber { + task_id: child_id, + aggregation_number: aggregation_number + 1, + }); } } } @@ -396,21 +391,18 @@ impl AggregationUpdateQueue { let diff = data.apply(&mut upper, self); if !diff.is_empty() { let upper_ids = get_uppers(&upper); - self.jobs.push_back( - AggregationUpdateJob::AggregatedDataUpdate { - upper_ids, - update: diff, - }, - ) + self.push(AggregationUpdateJob::AggregatedDataUpdate { + upper_ids, + update: diff, + }) } } } if !children.is_empty() { - self.jobs - .push_back(AggregationUpdateJob::InnerHasNewFollowers { - upper_ids: upper_ids.clone(), - new_follower_ids: children, - }); + self.push(AggregationUpdateJob::InnerHasNewFollowers { + upper_ids: upper_ids.clone(), + new_follower_ids: children, + }); } } else { drop(follower); @@ -425,11 +417,10 @@ impl AggregationUpdateQueue { }, 1 ) { - self.jobs - .push_back(AggregationUpdateJob::InnerHasNewFollower { - upper_ids: vec![upper_id], - new_follower_id, - }) + self.push(AggregationUpdateJob::InnerHasNewFollower { + upper_ids: vec![upper_id], + new_follower_id, + }) } } } @@ -486,21 +477,18 @@ impl AggregationUpdateQueue { let diff = data.apply(&mut upper, self); if !diff.is_empty() { let upper_ids = get_uppers(&upper); - self.jobs.push_back( - AggregationUpdateJob::AggregatedDataUpdate { - upper_ids, - update: diff, - }, - ) + self.push(AggregationUpdateJob::AggregatedDataUpdate { + upper_ids, + update: diff, + }) } } } if !children.is_empty() { - self.jobs - .push_back(AggregationUpdateJob::InnerLostFollowers { - upper_ids: upper_ids.clone(), - lost_follower_ids: children, - }); + self.push(AggregationUpdateJob::InnerLostFollowers { + upper_ids: upper_ids.clone(), + lost_follower_ids: children, + }); } } else { drop(follower); @@ -515,11 +503,10 @@ impl AggregationUpdateQueue { }, -1 ) { - self.jobs - .push_back(AggregationUpdateJob::InnerLostFollower { - upper_ids: vec![upper_id], - lost_follower_id, - }) + self.push(AggregationUpdateJob::InnerLostFollower { + upper_ids: vec![upper_id], + lost_follower_id, + }) } } } @@ -530,11 +517,10 @@ impl AggregationUpdateQueue { if !diff.is_empty() { let upper_ids = get_uppers(&upper); if !upper_ids.is_empty() { - self.jobs - .push_back(AggregationUpdateJob::AggregatedDataUpdate { - upper_ids, - update: diff, - }); + self.push(AggregationUpdateJob::AggregatedDataUpdate { + upper_ids, + update: diff, + }); } } } @@ -543,11 +529,10 @@ impl AggregationUpdateQueue { let task = ctx.task(task_id); let upper_ids: Vec<_> = get_uppers(&task); if !upper_ids.is_empty() { - self.jobs - .push_back(AggregationUpdateJob::AggregatedDataUpdate { - upper_ids, - update: update.clone(), - }); + self.push(AggregationUpdateJob::AggregatedDataUpdate { + upper_ids, + update: update.clone(), + }); } } AggregationUpdateJob::ScheduleWhenDirty { task_ids } => { @@ -592,31 +577,26 @@ impl AggregationUpdateQueue { if !upper_ids.is_empty() { if !diff.is_empty() { // Notify uppers about changed aggregated data - self.jobs.push_back( - AggregationUpdateJob::AggregatedDataUpdate { - upper_ids: upper_ids.clone(), - update: diff, - }, - ); - } - if !followers.is_empty() { - self.jobs.push_back( - AggregationUpdateJob::InnerHasNewFollowers { - upper_ids: upper_ids.clone(), - new_follower_ids: followers, - }, - ); + self.push(AggregationUpdateJob::AggregatedDataUpdate { + upper_ids: upper_ids.clone(), + update: diff, + }); } } + if !followers.is_empty() { + self.push(AggregationUpdateJob::InnerHasNewFollowers { + upper_ids: vec![upper_id], + new_follower_ids: followers, + }); + } } // notify uppers about lost follower if !upper_ids.is_empty() { - self.jobs - .push_back(AggregationUpdateJob::InnerLostFollower { - upper_ids, - lost_follower_id: task_id, - }); + self.push(AggregationUpdateJob::InnerLostFollower { + upper_ids, + lost_follower_id: task_id, + }); } } } else if should_be_follower { @@ -629,12 +609,10 @@ impl AggregationUpdateQueue { if update_count!(upper, Follower { task: task_id }, count as i32) { // notify uppers about new follower if !upper_ids.is_empty() { - self.jobs.push_back( - AggregationUpdateJob::InnerHasNewFollower { - upper_ids: upper_ids.clone(), - new_follower_id: task_id, - }, - ); + self.push(AggregationUpdateJob::InnerHasNewFollower { + upper_ids: upper_ids.clone(), + new_follower_id: task_id, + }); } } @@ -645,31 +623,27 @@ impl AggregationUpdateQueue { let diff = data.apply(&mut upper, self); if !upper_ids.is_empty() { if !diff.is_empty() { - self.jobs.push_back( - AggregationUpdateJob::AggregatedDataUpdate { - upper_ids: upper_ids.clone(), - update: diff, - }, - ); - } - if !followers.is_empty() { - self.jobs - .push_back(AggregationUpdateJob::InnerLostFollowers { - upper_ids, - lost_follower_ids: followers, - }); + self.push(AggregationUpdateJob::AggregatedDataUpdate { + upper_ids: upper_ids.clone(), + update: diff, + }); } } + if !followers.is_empty() { + self.push(AggregationUpdateJob::InnerLostFollowers { + upper_ids: vec![upper_id], + lost_follower_ids: followers, + }); + } } } else { // both nodes have the same aggregation number // We need to change the aggregation number of the task let new_aggregation_number = upper_aggregation_number + 1; - self.jobs - .push_back(AggregationUpdateJob::UpdateAggregationNumber { - task_id, - aggregation_number: new_aggregation_number, - }); + self.push(AggregationUpdateJob::UpdateAggregationNumber { + task_id, + aggregation_number: new_aggregation_number, + }); } } } From efba41bf3aadb6536584128735b32c1f071cc50c Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Fri, 6 Sep 2024 15:26:30 +0200 Subject: [PATCH 47/81] clippy --- .../turbo-tasks-backend/src/backend/mod.rs | 29 ++++---- .../backend/operation/aggregation_update.rs | 71 +++++++++---------- .../backend/operation/cleanup_old_edges.rs | 5 +- .../src/backend/operation/connect_child.rs | 2 +- .../src/backend/storage.rs | 6 +- .../crates/turbo-tasks-backend/src/data.rs | 1 + 6 files changed, 52 insertions(+), 62 deletions(-) diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs index 438c7b1e6dff8..e3a3d3068f5f7 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs @@ -64,6 +64,9 @@ impl SnapshotRequest { } } +type TransientOnceTask = + Mutex> + Send + 'static>>>>; + pub enum TransientTask { /// A root task that will track dependencies and re-execute when /// dependencies change. Task will eventually settle to the correct @@ -78,7 +81,7 @@ pub enum TransientTask { /// happened after that. It may see these invalidations partially /// applied. /// Active until done. Automatically scheduled. - Once(Mutex> + Send + 'static>>>>), + Once(TransientOnceTask), } pub struct TurboTasksBackend { @@ -241,7 +244,7 @@ impl TurboTasksBackend { if let Some(reader_desc) = reader_desc.as_ref() { format!("try_read_task_output from {}", reader_desc()) } else { - format!("try_read_task_output (untracked)") + "try_read_task_output (untracked)".to_string() } }); return Ok(Err(listener)); @@ -339,7 +342,7 @@ impl TurboTasksBackend { if let Some(reader_desc) = reader_desc.as_ref() { format!("try_read_task_output (recompute) from {}", reader_desc()) } else { - format!("try_read_task_output (recompute, untracked)") + "try_read_task_output (recompute, untracked)".to_string() } }; @@ -409,7 +412,7 @@ impl TurboTasksBackend { if let Some(reader_desc) = reader_desc.as_ref() { format!("try_read_task_cell from {}", reader_desc()) } else { - format!("try_read_task_cell (untracked)") + "try_read_task_cell (untracked)".to_string() } }; @@ -552,9 +555,7 @@ impl Backend for TurboTasksBackend { } type TaskState = (); - fn new_task_state(&self, _task: TaskId) -> Self::TaskState { - () - } + fn new_task_state(&self, _task: TaskId) -> Self::TaskState {} fn try_start_task_execution( &self, @@ -740,14 +741,8 @@ impl Backend for TurboTasksBackend { let task_type = task_type.clone(); let span = tracing::trace_span!("turbo_tasks::root_task"); let future = match &*task_type { - TransientTask::Root(f) => { - let future = f(); - future - } - TransientTask::Once(future_mutex) => { - let future = take(&mut *future_mutex.lock())?; - future - } + TransientTask::Root(f) => f(), + TransientTask::Once(future_mutex) => take(&mut *future_mutex.lock())?, }; (span, future) } @@ -782,7 +777,7 @@ impl Backend for TurboTasksBackend { }; let InProgressState::InProgress { done_event, - once_task, + once_task: _, stale, } = in_progress else { @@ -995,7 +990,7 @@ impl Backend for TurboTasksBackend { fn create_transient_task( &self, task_type: TransientTaskType, - turbo_tasks: &dyn TurboTasksBackendApi, + _turbo_tasks: &dyn TurboTasksBackendApi, ) -> TaskId { let task_id = self.transient_task_id_factory.get(); let root_type = match task_type { diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs index ee3a826678285..fe1740945455c 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs @@ -113,7 +113,7 @@ impl AggregatedDataUpdate { dirty_tasks_update.insert(task.id(), 1); } Self { - dirty_task_count: dirty_task_count as i32, + dirty_task_count, dirty_tasks_update, } } else if dirty { @@ -539,6 +539,7 @@ impl AggregationUpdateQueue { for task_id in task_ids { let description = ctx.backend.get_task_desc_fn(task_id); let mut task = ctx.task(task_id); + #[allow(clippy::collapsible_if, reason = "readablility")] if task.has_key(&CachedDataItemKey::Dirty {}) { if task.add(CachedDataItem::new_scheduled(description)) { ctx.turbo_tasks.schedule(task_id); @@ -558,46 +559,46 @@ impl AggregationUpdateQueue { if should_be_inner { // remove all follower edges let count = remove!(upper, Follower { task: task_id }).unwrap_or_default(); - if count < 0 { - upper.add_new(CachedDataItem::Follower { + match count.cmp(&0) { + std::cmp::Ordering::Less => upper.add_new(CachedDataItem::Follower { task: task_id, value: count, - }) - } else if count > 0 { - let upper_ids = get_uppers(&upper); - - // Add the same amount of upper edges - if update_count!(task, Upper { task: upper_id }, count as i32) { - // When this is a new inner node, update aggregated data and - // followers - let data = AggregatedDataUpdate::from_task(&mut task); - let followers = get_followers(&task); - let diff = data.apply(&mut upper, self); + }), + std::cmp::Ordering::Greater => { + let upper_ids = get_uppers(&upper); + + // Add the same amount of upper edges + if update_count!(task, Upper { task: upper_id }, count) { + // When this is a new inner node, update aggregated data and + // followers + let data = AggregatedDataUpdate::from_task(&mut task); + let followers = get_followers(&task); + let diff = data.apply(&mut upper, self); - if !upper_ids.is_empty() { - if !diff.is_empty() { + if !upper_ids.is_empty() && !diff.is_empty() { // Notify uppers about changed aggregated data self.push(AggregationUpdateJob::AggregatedDataUpdate { upper_ids: upper_ids.clone(), update: diff, }); } + if !followers.is_empty() { + self.push(AggregationUpdateJob::InnerHasNewFollowers { + upper_ids: vec![upper_id], + new_follower_ids: followers, + }); + } } - if !followers.is_empty() { - self.push(AggregationUpdateJob::InnerHasNewFollowers { - upper_ids: vec![upper_id], - new_follower_ids: followers, + + // notify uppers about lost follower + if !upper_ids.is_empty() { + self.push(AggregationUpdateJob::InnerLostFollower { + upper_ids, + lost_follower_id: task_id, }); } } - - // notify uppers about lost follower - if !upper_ids.is_empty() { - self.push(AggregationUpdateJob::InnerLostFollower { - upper_ids, - lost_follower_id: task_id, - }); - } + std::cmp::Ordering::Equal => {} } } else if should_be_follower { // Remove the upper edge @@ -606,7 +607,7 @@ impl AggregationUpdateQueue { let upper_ids: Vec<_> = get_uppers(&upper); // Add the same amount of follower edges - if update_count!(upper, Follower { task: task_id }, count as i32) { + if update_count!(upper, Follower { task: task_id }, count) { // notify uppers about new follower if !upper_ids.is_empty() { self.push(AggregationUpdateJob::InnerHasNewFollower { @@ -621,13 +622,11 @@ impl AggregationUpdateQueue { let data = AggregatedDataUpdate::from_task(&mut task).invert(); let followers = get_followers(&task); let diff = data.apply(&mut upper, self); - if !upper_ids.is_empty() { - if !diff.is_empty() { - self.push(AggregationUpdateJob::AggregatedDataUpdate { - upper_ids: upper_ids.clone(), - update: diff, - }); - } + if !upper_ids.is_empty() && !diff.is_empty() { + self.push(AggregationUpdateJob::AggregatedDataUpdate { + upper_ids: upper_ids.clone(), + update: diff, + }); } if !followers.is_empty() { self.push(AggregationUpdateJob::InnerLostFollowers { diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/cleanup_old_edges.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/cleanup_old_edges.rs index 04a266c98d150..d2886d1dab7aa 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/cleanup_old_edges.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/cleanup_old_edges.rs @@ -11,10 +11,7 @@ use super::{ invalidate::make_task_dirty, ExecuteContext, Operation, }; -use crate::{ - data::{CachedDataItemKey, CellRef}, - get_many, -}; +use crate::data::{CachedDataItemKey, CellRef}; #[derive(Serialize, Deserialize, Clone, Default)] pub enum CleanupOldEdgesOperation { diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs index 7fc8107ea5471..c6684a458e075 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs @@ -9,7 +9,7 @@ use super::{ }; use crate::{ data::{CachedDataItem, CachedDataItemKey}, - get, get_many, + get, }; const AGGREGATION_NUMBER_BUFFER_SPACE: u32 = 2; diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs b/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs index 78a7ab9e23b90..e94fbf6b3602d 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs @@ -74,10 +74,8 @@ where } else { self.map.remove(key); } - } else { - if let Some(v) = update(None) { - self.map.insert(key.clone(), v); - } + } else if let Some(v) = update(None) { + self.map.insert(key.clone(), v); } } } diff --git a/turbopack/crates/turbo-tasks-backend/src/data.rs b/turbopack/crates/turbo-tasks-backend/src/data.rs index a33d8263acf3f..1744f064f6daa 100644 --- a/turbopack/crates/turbo-tasks-backend/src/data.rs +++ b/turbopack/crates/turbo-tasks-backend/src/data.rs @@ -70,6 +70,7 @@ pub enum InProgressState { }, InProgress { stale: bool, + #[allow(dead_code)] once_task: bool, done_event: Event, }, From 3ddc905522eaca2818e011dbbda09977b1cad4cf Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Fri, 6 Sep 2024 15:51:08 +0200 Subject: [PATCH 48/81] shared_amount need to be a power to two --- turbopack/crates/turbo-tasks-backend/src/backend/storage.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs b/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs index e94fbf6b3602d..4b8cb62b22ad3 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs @@ -106,11 +106,13 @@ where T: KeyValuePair, { pub fn new() -> Self { + let shard_amount = + (available_parallelism().map_or(4, |v| v.get()) * 64).next_power_of_two(); Self { map: DashMap::with_capacity_and_hasher_and_shard_amount( 1024 * 1024, Default::default(), - available_parallelism().map_or(4, |v| v.get()) * 64, + shard_amount, ), } } From 89ca6b8166e41781c42710012a7d50ef6c6d8a25 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Fri, 20 Sep 2024 08:50:57 +0200 Subject: [PATCH 49/81] fix concurrent data update --- .../turbo-tasks-backend/src/backend/mod.rs | 14 +++- .../backend/operation/aggregation_update.rs | 73 ++++++++++++------- .../backend/operation/cleanup_old_edges.rs | 13 +--- .../src/backend/operation/invalidate.rs | 8 +- .../src/backend/operation/mod.rs | 2 +- .../tests/emptied_cells.rs | 1 + 6 files changed, 66 insertions(+), 45 deletions(-) create mode 120000 turbopack/crates/turbo-tasks-backend/tests/emptied_cells.rs diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs index e3a3d3068f5f7..1a268fd6ef7de 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs @@ -20,8 +20,8 @@ use auto_hash_map::{AutoMap, AutoSet}; use dashmap::DashMap; pub use operation::AnyOperation; use operation::{ - is_root_node, AggregationUpdateJob, AggregationUpdateQueue, CleanupOldEdgesOperation, - ConnectChildOperation, OutdatedEdge, + is_root_node, AggregatedDataUpdate, AggregationUpdateJob, AggregationUpdateQueue, + CleanupOldEdgesOperation, ConnectChildOperation, OutdatedEdge, }; use parking_lot::{Condvar, Mutex}; use rustc_hash::FxHasher; @@ -858,12 +858,20 @@ impl Backend for TurboTasksBackend { .collect::>(); let was_dirty = task.remove(&CachedDataItemKey::Dirty {}).is_some(); + let data_update = was_dirty + .then(|| { + AggregationUpdateJob::data_update( + &mut task, + AggregatedDataUpdate::no_longer_dirty_task(task_id), + ) + }) + .flatten(); drop(task); done_event.notify(usize::MAX); - CleanupOldEdgesOperation::run(task_id, old_edges, was_dirty, ctx); + CleanupOldEdgesOperation::run(task_id, old_edges, data_update, ctx); drop(removed_data) } diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs index fe1740945455c..058f63ddff9e5 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs @@ -75,10 +75,6 @@ pub enum AggregationUpdateJob { upper_ids: Vec, update: AggregatedDataUpdate, }, - DataUpdate { - task_id: TaskId, - update: AggregatedDataUpdate, - }, ScheduleWhenDirty { task_ids: Vec, }, @@ -88,6 +84,23 @@ pub enum AggregationUpdateJob { }, } +impl AggregationUpdateJob { + pub fn data_update( + task: &mut TaskGuard<'_>, + update: AggregatedDataUpdate, + ) -> Option { + let upper_ids: Vec<_> = get_uppers(&task); + if !upper_ids.is_empty() { + Some(AggregationUpdateJob::AggregatedDataUpdate { + upper_ids, + update: update.clone(), + }) + } else { + None + } + } +} + #[derive(Default, Serialize, Deserialize, Clone, Debug)] pub struct AggregatedDataUpdate { dirty_task_count: i32, @@ -260,6 +273,10 @@ impl AggregationUpdateQueue { self.jobs.push_back(job); } + pub fn extend(&mut self, jobs: impl IntoIterator) { + self.jobs.extend(jobs); + } + pub fn run(job: AggregationUpdateJob, ctx: &ExecuteContext<'_>) { let mut queue = Self::new(); queue.push(job); @@ -454,20 +471,29 @@ impl AggregationUpdateQueue { lost_follower_id, } => { let mut follower = ctx.task(lost_follower_id); - let mut upper_ids_as_follower = Vec::new(); + let mut follower_in_upper_ids = Vec::new(); upper_ids.retain(|&upper_id| { - if update_count!(follower, Upper { task: upper_id }, -1) { - // It was an inner - true - } else { - // It is a follower - upper_ids_as_follower.push(upper_id); - false - } + let mut keep_upper = false; + update!(follower, Upper { task: upper_id }, |old| { + let Some(old) = old else { + follower_in_upper_ids.push(upper_id); + return None; + }; + if old < 0 { + follower_in_upper_ids.push(upper_id); + return Some(old); + } + if old == 1 { + keep_upper = true; + return None; + } + Some(old - 1) + }); + keep_upper }); if !upper_ids.is_empty() { let data = AggregatedDataUpdate::from_task(&mut follower).invert(); - let children: Vec<_> = get_followers(&follower); + let followers: Vec<_> = get_followers(&follower); drop(follower); if !data.is_empty() { @@ -484,17 +510,17 @@ impl AggregationUpdateQueue { } } } - if !children.is_empty() { + if !followers.is_empty() { self.push(AggregationUpdateJob::InnerLostFollowers { upper_ids: upper_ids.clone(), - lost_follower_ids: children, + lost_follower_ids: followers, }); } } else { drop(follower); } - for upper_id in upper_ids_as_follower { + for upper_id in follower_in_upper_ids { let mut upper = ctx.task(upper_id); if update_count!( upper, @@ -503,8 +529,9 @@ impl AggregationUpdateQueue { }, -1 ) { + let upper_ids = get_uppers(&upper); self.push(AggregationUpdateJob::InnerLostFollower { - upper_ids: vec![upper_id], + upper_ids, lost_follower_id, }) } @@ -525,16 +552,6 @@ impl AggregationUpdateQueue { } } } - AggregationUpdateJob::DataUpdate { task_id, update } => { - let task = ctx.task(task_id); - let upper_ids: Vec<_> = get_uppers(&task); - if !upper_ids.is_empty() { - self.push(AggregationUpdateJob::AggregatedDataUpdate { - upper_ids, - update: update.clone(), - }); - } - } AggregationUpdateJob::ScheduleWhenDirty { task_ids } => { for task_id in task_ids { let description = ctx.backend.get_task_desc_fn(task_id); diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/cleanup_old_edges.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/cleanup_old_edges.rs index d2886d1dab7aa..4c102950c63f7 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/cleanup_old_edges.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/cleanup_old_edges.rs @@ -5,8 +5,8 @@ use turbo_tasks::TaskId; use super::{ aggregation_update::{ - get_aggregation_number, get_uppers, is_aggregating_node, AggregatedDataUpdate, - AggregationUpdateJob, AggregationUpdateQueue, + get_aggregation_number, get_uppers, is_aggregating_node, AggregationUpdateJob, + AggregationUpdateQueue, }, invalidate::make_task_dirty, ExecuteContext, Operation, @@ -40,16 +40,11 @@ impl CleanupOldEdgesOperation { pub fn run( task_id: TaskId, outdated: Vec, - was_dirty: bool, + data_update: Option, ctx: ExecuteContext<'_>, ) { let mut queue = AggregationUpdateQueue::new(); - if was_dirty { - queue.push(AggregationUpdateJob::DataUpdate { - task_id, - update: AggregatedDataUpdate::no_longer_dirty_task(task_id), - }); - } + queue.extend(data_update); CleanupOldEdgesOperation::RemoveEdges { task_id, outdated, diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/invalidate.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/invalidate.rs index 106d3e4d5a3ab..bf748050da068 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/invalidate.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/invalidate.rs @@ -62,9 +62,9 @@ pub fn make_task_dirty(task_id: TaskId, queue: &mut AggregationUpdateQueue, ctx: let mut task = ctx.task(task_id); if task.add(CachedDataItem::Dirty { value: () }) { - queue.push(AggregationUpdateJob::DataUpdate { - task_id, - update: AggregatedDataUpdate::dirty_task(task_id), - }) + queue.extend(AggregationUpdateJob::data_update( + &mut task, + AggregatedDataUpdate::dirty_task(task_id), + )); } } diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs index c9848b9e35c57..2d6c6c0f11350 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs @@ -323,7 +323,7 @@ impl_operation!(Invalidate invalidate::InvalidateOperation); impl_operation!(CleanupOldEdges cleanup_old_edges::CleanupOldEdgesOperation); impl_operation!(AggregationUpdate aggregation_update::AggregationUpdateQueue); -pub use aggregation_update::{is_root_node, AggregationUpdateJob}; +pub use aggregation_update::{is_root_node, AggregatedDataUpdate, AggregationUpdateJob}; pub use cleanup_old_edges::OutdatedEdge; pub use update_cell::UpdateCellOperation; pub use update_output::UpdateOutputOperation; diff --git a/turbopack/crates/turbo-tasks-backend/tests/emptied_cells.rs b/turbopack/crates/turbo-tasks-backend/tests/emptied_cells.rs new file mode 120000 index 0000000000000..9070c4d0b4dcc --- /dev/null +++ b/turbopack/crates/turbo-tasks-backend/tests/emptied_cells.rs @@ -0,0 +1 @@ +../../turbo-tasks-testing/tests/emptied_cells.rs \ No newline at end of file From ee0064d080a2b7ab2ac0b6360f45de91a5f043db Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Fri, 20 Sep 2024 09:39:36 +0200 Subject: [PATCH 50/81] add more test cases --- turbopack/crates/turbo-tasks-backend/tests/detached.rs | 1 + turbopack/crates/turbo-tasks-backend/tests/resolved_vc.rs | 1 + 2 files changed, 2 insertions(+) create mode 120000 turbopack/crates/turbo-tasks-backend/tests/detached.rs create mode 120000 turbopack/crates/turbo-tasks-backend/tests/resolved_vc.rs diff --git a/turbopack/crates/turbo-tasks-backend/tests/detached.rs b/turbopack/crates/turbo-tasks-backend/tests/detached.rs new file mode 120000 index 0000000000000..e726e54a7881e --- /dev/null +++ b/turbopack/crates/turbo-tasks-backend/tests/detached.rs @@ -0,0 +1 @@ +../../turbo-tasks-testing/tests/detached.rs \ No newline at end of file diff --git a/turbopack/crates/turbo-tasks-backend/tests/resolved_vc.rs b/turbopack/crates/turbo-tasks-backend/tests/resolved_vc.rs new file mode 120000 index 0000000000000..601c7f0fc0008 --- /dev/null +++ b/turbopack/crates/turbo-tasks-backend/tests/resolved_vc.rs @@ -0,0 +1 @@ +../../turbo-tasks-testing/tests/resolved_vc.rs \ No newline at end of file From f805046826493f340674a193858602523564ca70 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Fri, 20 Sep 2024 09:40:43 +0200 Subject: [PATCH 51/81] run scope_stress with persistent cache too --- .../crates/turbo-tasks-testing/src/lib.rs | 2 +- .../crates/turbo-tasks-testing/src/run.rs | 17 +++++++++++--- .../turbo-tasks-testing/tests/scope_stress.rs | 22 +++++++------------ 3 files changed, 23 insertions(+), 18 deletions(-) diff --git a/turbopack/crates/turbo-tasks-testing/src/lib.rs b/turbopack/crates/turbo-tasks-testing/src/lib.rs index 5ba43c8e3fb88..370194d94d151 100644 --- a/turbopack/crates/turbo-tasks-testing/src/lib.rs +++ b/turbopack/crates/turbo-tasks-testing/src/lib.rs @@ -24,7 +24,7 @@ use turbo_tasks::{ TaskPersistence, TraitTypeId, TurboTasksApi, TurboTasksCallApi, }; -pub use crate::run::{run, run_without_cache_check, Registration}; +pub use crate::run::{run, run_with_tt, run_without_cache_check, Registration}; enum Task { Spawned(Event), diff --git a/turbopack/crates/turbo-tasks-testing/src/run.rs b/turbopack/crates/turbo-tasks-testing/src/run.rs index cf7f8e8e785e8..8de9199b79097 100644 --- a/turbopack/crates/turbo-tasks-testing/src/run.rs +++ b/turbopack/crates/turbo-tasks-testing/src/run.rs @@ -86,6 +86,17 @@ pub async fn run( registration: &Registration, fut: impl Fn() -> F + Send + 'static, ) -> Result<()> +where + F: Future> + Send + 'static, + T: Debug + PartialEq + Eq + TraceRawVcs + Send + 'static, +{ + run_with_tt(registration, move |tt| run_once(tt, fut())).await +} + +pub async fn run_with_tt( + registration: &Registration, + fut: impl Fn(Arc) -> F + Send + 'static, +) -> Result<()> where F: Future> + Send + 'static, T: Debug + PartialEq + Eq + TraceRawVcs + Send + 'static, @@ -95,14 +106,14 @@ where let name = closure_to_name(&fut); let tt = registration.create_turbo_tasks(&name, true); println!("Run #1 (without cache)"); - let first = run_once(tt.clone(), fut()).await?; + let first = fut(tt.clone()).await?; println!("Run #2 (with memory cache, same TurboTasks instance)"); - let second = run_once(tt.clone(), fut()).await?; + let second = fut(tt.clone()).await?; assert_eq!(first, second); tt.stop_and_wait().await; let tt = registration.create_turbo_tasks(&name, false); println!("Run #3 (with persistent cache if available, new TurboTasks instance)"); - let third = run_once(tt.clone(), fut()).await?; + let third = fut(tt.clone()).await?; tt.stop_and_wait().await; assert_eq!(first, third); Ok(()) diff --git a/turbopack/crates/turbo-tasks-testing/tests/scope_stress.rs b/turbopack/crates/turbo-tasks-testing/tests/scope_stress.rs index c1b50136400ad..553c598b28b8d 100644 --- a/turbopack/crates/turbo-tasks-testing/tests/scope_stress.rs +++ b/turbopack/crates/turbo-tasks-testing/tests/scope_stress.rs @@ -2,22 +2,16 @@ use anyhow::Result; use turbo_tasks::{run_once, Completion, TryJoinIterExt, Vc}; -use turbo_tasks_testing::{register, Registration}; +use turbo_tasks_testing::{register, run_with_tt, Registration}; static REGISTRATION: Registration = register!(); -#[test] -fn rectangle_stress() { - REGISTRATION.ensure_registered(); - let rt = tokio::runtime::Builder::new_multi_thread() - .enable_all() - .build() - .unwrap(); - rt.block_on(async { - let tt = REGISTRATION.create_turbo_tasks("scope_stress_rectangle_stress", true); - let size = std::env::var("TURBOPACK_TEST_RECTANGLE_STRESS_SIZE") - .map(|size| size.parse().unwrap()) - .unwrap_or(50); +#[tokio::test(flavor = "multi_thread")] +async fn rectangle_stress() -> Result<()> { + let size = std::env::var("TURBOPACK_TEST_RECTANGLE_STRESS_SIZE") + .map(|size| size.parse().unwrap()) + .unwrap_or(50); + run_with_tt(®ISTRATION, move |tt| async move { (0..size) .map(|a| (a, size - 1)) .chain((0..size - 1).map(|b| (size - 1, b))) @@ -33,8 +27,8 @@ fn rectangle_stress() { }) .try_join() .await - .unwrap(); }) + .await } /// This fills a rectagle from (0, 0) to (a, b) by From 44bb3ec705e459263bdb9e0fe825963c7cef0046 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Thu, 5 Sep 2024 18:19:35 +0200 Subject: [PATCH 52/81] avoid flagging once tasks as dirty --- .../src/backend/operation/invalidate.rs | 4 ++++ .../src/backend/operation/mod.rs | 13 ++++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/invalidate.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/invalidate.rs index bf748050da068..dd38b3afe08fd 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/invalidate.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/invalidate.rs @@ -59,6 +59,10 @@ impl Operation for InvalidateOperation { } pub fn make_task_dirty(task_id: TaskId, queue: &mut AggregationUpdateQueue, ctx: &ExecuteContext) { + if ctx.is_once_task(task_id) { + return; + } + let mut task = ctx.task(task_id); if task.add(CachedDataItem::Dirty { value: () }) { diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs index 2d6c6c0f11350..3e6c7968a80c3 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs @@ -15,7 +15,7 @@ use turbo_tasks::{KeyValuePair, TaskId, TurboTasksBackendApi}; use super::{storage::StorageWriteGuard, TurboTasksBackend}; use crate::{ - backend::OperationGuard, + backend::{OperationGuard, TransientTask}, data::{CachedDataItem, CachedDataItemKey, CachedDataItemValue, CachedDataUpdate}, }; @@ -58,6 +58,17 @@ impl<'a> ExecuteContext<'a> { } } + pub fn is_once_task(&self, task_id: TaskId) -> bool { + if !task_id.is_transient() { + return false; + } + if let Some(ty) = self.backend.transient_tasks.get(&task_id) { + matches!(**ty, TransientTask::Once(_)) + } else { + false + } + } + pub fn task_pair(&self, task_id1: TaskId, task_id2: TaskId) -> (TaskGuard<'a>, TaskGuard<'a>) { let (task1, task2) = self.backend.storage.access_pair_mut(task_id1, task_id2); ( From 46f244c52e600ba4f7706e38dac9092f42704c28 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Mon, 23 Sep 2024 10:37:11 +0200 Subject: [PATCH 53/81] refactor dirty task list to be cached in subgraphs and propagate less often --- .../turbo-tasks-backend/src/backend/mod.rs | 52 ++--- .../backend/operation/aggregation_update.rs | 193 ++++++++---------- .../src/backend/operation/connect_child.rs | 45 ++-- .../src/backend/operation/invalidate.rs | 25 ++- .../crates/turbo-tasks-backend/src/data.rs | 23 ++- 5 files changed, 169 insertions(+), 169 deletions(-) diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs index 1a268fd6ef7de..62b335fbbddac 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs @@ -41,8 +41,8 @@ use turbo_tasks::{ use self::{operation::ExecuteContext, storage::Storage}; use crate::{ data::{ - CachedDataItem, CachedDataItemKey, CachedDataItemValue, CachedDataUpdate, CellRef, - InProgressCellState, InProgressState, OutputValue, RootState, RootType, + ActiveType, CachedDataItem, CachedDataItemKey, CachedDataItemValue, CachedDataUpdate, + CellRef, InProgressCellState, InProgressState, OutputValue, RootState, }, get, get_many, remove, utils::{bi_map::BiMap, chunked_vec::ChunkedVec, ptr_eq_arc::PtrEqArc}, @@ -271,7 +271,7 @@ impl TurboTasksBackend { } // Check the dirty count of the root node - let dirty_tasks = get!(task, AggregatedDirtyTaskCount) + let dirty_tasks = get!(task, AggregatedDirtyContainerCount) .copied() .unwrap_or_default(); let root = get!(task, AggregateRoot); @@ -280,11 +280,11 @@ impl TurboTasksBackend { let root = if let Some(root) = root { root } else { - // If we don't have a root state, add one - // This also makes sure all tasks stay active, so we need to remove that after - // reading again + // If we don't have a root state, add one. This also makes sure all tasks stay + // active and this task won't stale. CachedActiveUntilClean + // is automatically removed when this task is clean. task.add_new(CachedDataItem::AggregateRoot { - value: RootState::new(RootType::ReadingStronglyConsistent), + value: RootState::new(ActiveType::CachedActiveUntilClean), }); get!(task, AggregateRoot).unwrap() }; @@ -295,13 +295,6 @@ impl TurboTasksBackend { ) }); return Ok(Err(listener)); - } else { - // When there ain't dirty tasks, remove the reading strongly consistent root type - if let Some(root) = root { - if matches!(root.ty, RootType::ReadingStronglyConsistent) { - task.remove(&CachedDataItemKey::AggregateRoot {}); - } - } } } @@ -858,14 +851,27 @@ impl Backend for TurboTasksBackend { .collect::>(); let was_dirty = task.remove(&CachedDataItemKey::Dirty {}).is_some(); - let data_update = was_dirty - .then(|| { + let data_update = if was_dirty { + let dirty_containers = get!(task, AggregatedDirtyContainerCount) + .copied() + .unwrap_or_default(); + if dirty_containers == 0 { + if let Some(root_state) = get!(task, AggregateRoot) { + root_state.all_clean_event.notify(usize::MAX); + if matches!(root_state.ty, ActiveType::CachedActiveUntilClean) { + task.remove(&CachedDataItemKey::AggregateRoot {}); + } + } AggregationUpdateJob::data_update( &mut task, - AggregatedDataUpdate::no_longer_dirty_task(task_id), + AggregatedDataUpdate::no_longer_dirty_container(task_id), ) - }) - .flatten(); + } else { + None + } + } else { + None + }; drop(task); @@ -1002,8 +1008,8 @@ impl Backend for TurboTasksBackend { ) -> TaskId { let task_id = self.transient_task_id_factory.get(); let root_type = match task_type { - TransientTaskType::Root(_) => RootType::RootTask, - TransientTaskType::Once(_) => RootType::OnceTask, + TransientTaskType::Root(_) => ActiveType::RootTask, + TransientTaskType::Once(_) => ActiveType::OnceTask, }; self.transient_tasks.insert( task_id, @@ -1019,8 +1025,8 @@ impl Backend for TurboTasksBackend { value: RootState::new(root_type), }); let _ = task.add(CachedDataItem::new_scheduled(move || match root_type { - RootType::RootTask => "Root Task".to_string(), - RootType::OnceTask => "Once Task".to_string(), + ActiveType::RootTask => "Root Task".to_string(), + ActiveType::OnceTask => "Once Task".to_string(), _ => unreachable!(), })); } diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs index 058f63ddff9e5..490995d6d9c02 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs @@ -1,14 +1,11 @@ -use std::{ - collections::{hash_map::Entry, HashMap, VecDeque}, - ops::Add, -}; +use std::collections::VecDeque; use serde::{Deserialize, Serialize}; use turbo_tasks::TaskId; use super::{ExecuteContext, Operation, TaskGuard}; use crate::{ - data::{CachedDataItem, CachedDataItemKey}, + data::{ActiveType, CachedDataItem, CachedDataItemKey, RootState}, get, get_many, iter_many, remove, update, update_count, }; @@ -75,7 +72,7 @@ pub enum AggregationUpdateJob { upper_ids: Vec, update: AggregatedDataUpdate, }, - ScheduleWhenDirty { + FindAndScheduleDirty { task_ids: Vec, }, BalanceEdge { @@ -103,42 +100,31 @@ impl AggregationUpdateJob { #[derive(Default, Serialize, Deserialize, Clone, Debug)] pub struct AggregatedDataUpdate { - dirty_task_count: i32, - dirty_tasks_update: HashMap, + dirty_container_update: Option<(TaskId, i32)>, // TODO collectibles } impl AggregatedDataUpdate { fn from_task(task: &mut TaskGuard<'_>) -> Self { let aggregation = get_aggregation_number(task); - let dirty = get!(task, Dirty).is_some(); + let mut dirty = get!(task, Dirty).is_some(); if is_aggregating_node(aggregation) { - let mut dirty_task_count = get!(task, AggregatedDirtyTaskCount).copied().unwrap_or(0); - let mut dirty_tasks_update = task - .iter() - .filter_map(|(key, _)| match *key { - CachedDataItemKey::AggregatedDirtyTask { task } => Some((task, 1)), - _ => None, - }) - .collect::>(); - if dirty { - dirty_task_count += 1; - dirty_tasks_update.insert(task.id(), 1); - } - Self { - dirty_task_count, - dirty_tasks_update, + let dirty_container_count = get!(task, AggregatedDirtyContainerCount) + .copied() + .unwrap_or(0); + if dirty_container_count > 0 { + dirty = true; } - } else if dirty { - Self::dirty_task(task.id()) + } + if dirty { + Self::dirty_container(task.id()) } else { Self::default() } } fn invert(mut self) -> Self { - self.dirty_task_count = -self.dirty_task_count; - for value in self.dirty_tasks_update.values_mut() { + if let Some((_, value)) = self.dirty_container_update.as_mut() { *value = -*value; } self @@ -150,53 +136,62 @@ impl AggregatedDataUpdate { queue: &mut AggregationUpdateQueue, ) -> AggregatedDataUpdate { let Self { - dirty_task_count, - dirty_tasks_update, + dirty_container_update, } = self; let mut result = Self::default(); - if *dirty_task_count != 0 { - update!(task, AggregatedDirtyTaskCount, |old: Option| { + if let Some((dirty_container_id, count)) = dirty_container_update { + let mut added = false; + let mut removed = false; + update!( + task, + AggregatedDirtyContainer { + task: *dirty_container_id + }, + |old: Option| { + let old = old.unwrap_or(0); + let new = old + *count; + if old <= 0 && new > 0 { + added = true; + } else if old > 0 && new <= 0 { + removed = true; + } + (new != 0).then_some(new) + } + ); + let mut count_update = 0; + if added { + if task.has_key(&CachedDataItemKey::AggregateRoot {}) { + queue.push(AggregationUpdateJob::FindAndScheduleDirty { + task_ids: vec![*dirty_container_id], + }) + } + count_update += 1; + } else if removed { + count_update -= 1; + } + let dirty = task.has_key(&CachedDataItemKey::Dirty {}); + let task_id = task.id(); + update!(task, AggregatedDirtyContainerCount, |old: Option| { let old = old.unwrap_or(0); - let new = old + *dirty_task_count; - if old <= 0 && new > 0 { - result.dirty_task_count = 1; - } else if old > 0 && new <= 0 { - result.dirty_task_count = -1; + let new = old + count_update; + if !dirty { + if old <= 0 && new > 0 { + result.dirty_container_update = Some((task_id, 1)); + } else if old > 0 && new <= 0 { + result.dirty_container_update = Some((task_id, -1)); + } } (new != 0).then_some(new) }); - if result.dirty_task_count == -1 { + if let Some((_, count)) = result.dirty_container_update.as_ref() { if let Some(root_state) = get!(task, AggregateRoot) { - root_state.all_clean_event.notify(usize::MAX); - } - } - } - if !dirty_tasks_update.is_empty() { - let mut task_to_schedule = Vec::new(); - let root = get!(task, AggregateRoot).is_some(); - for (task_id, count) in dirty_tasks_update { - update!( - task, - AggregatedDirtyTask { task: *task_id }, - |old: Option| { - let old = old.unwrap_or(0); - let new = old + *count; - if old <= 0 && new > 0 { - if root { - task_to_schedule.push(*task_id); - } - result.dirty_tasks_update.insert(*task_id, 1); - } else if old > 0 && new <= 0 { - result.dirty_tasks_update.insert(*task_id, -1); + if *count < 0 { + root_state.all_clean_event.notify(usize::MAX); + if matches!(root_state.ty, ActiveType::CachedActiveUntilClean) { + task.remove(&CachedDataItemKey::AggregateRoot {}); } - (new != 0).then_some(new) } - ); - } - if !task_to_schedule.is_empty() { - queue.push(AggregationUpdateJob::ScheduleWhenDirty { - task_ids: task_to_schedule, - }) + } } } result @@ -204,51 +199,20 @@ impl AggregatedDataUpdate { fn is_empty(&self) -> bool { let Self { - dirty_task_count, - dirty_tasks_update, + dirty_container_update, } = self; - *dirty_task_count == 0 && dirty_tasks_update.is_empty() + dirty_container_update.is_none() } - pub fn dirty_task(task_id: TaskId) -> Self { + pub fn dirty_container(task_id: TaskId) -> Self { Self { - dirty_task_count: 1, - dirty_tasks_update: HashMap::from([(task_id, 1)]), + dirty_container_update: Some((task_id, 1)), } } - pub fn no_longer_dirty_task(task_id: TaskId) -> Self { + pub fn no_longer_dirty_container(task_id: TaskId) -> Self { Self { - dirty_task_count: -1, - dirty_tasks_update: HashMap::from([(task_id, -1)]), - } - } -} - -impl Add for AggregatedDataUpdate { - type Output = Self; - - fn add(self, rhs: Self) -> Self::Output { - let mut dirty_tasks_update = self.dirty_tasks_update; - for (task, count) in rhs.dirty_tasks_update { - match dirty_tasks_update.entry(task) { - Entry::Occupied(mut entry) => { - let value = entry.get_mut(); - *value += count; - if *value == 0 { - entry.remove(); - } - } - Entry::Vacant(entry) => { - if count != 0 { - entry.insert(count); - } - } - } - } - Self { - dirty_task_count: self.dirty_task_count + rhs.dirty_task_count, - dirty_tasks_update, + dirty_container_update: Some((task_id, -1)), } } } @@ -552,16 +516,33 @@ impl AggregationUpdateQueue { } } } - AggregationUpdateJob::ScheduleWhenDirty { task_ids } => { - for task_id in task_ids { - let description = ctx.backend.get_task_desc_fn(task_id); + AggregationUpdateJob::FindAndScheduleDirty { mut task_ids } => { + let popped = task_ids.pop(); + if !task_ids.is_empty() { + self.push(AggregationUpdateJob::FindAndScheduleDirty { task_ids }); + } + if let Some(task_id) = popped { let mut task = ctx.task(task_id); #[allow(clippy::collapsible_if, reason = "readablility")] if task.has_key(&CachedDataItemKey::Dirty {}) { + let description = ctx.backend.get_task_desc_fn(task_id); if task.add(CachedDataItem::new_scheduled(description)) { ctx.turbo_tasks.schedule(task_id); } } + if is_aggregating_node(get_aggregation_number(&task)) { + if !task.has_key(&CachedDataItemKey::AggregateRoot {}) { + task.insert(CachedDataItem::AggregateRoot { + value: RootState::new(ActiveType::CachedActiveUntilClean), + }); + } + let dirty_containers: Vec<_> = get_many!(task, AggregatedDirtyContainer { task } count if count > 0 => task); + if !dirty_containers.is_empty() { + self.push(AggregationUpdateJob::FindAndScheduleDirty { + task_ids: dirty_containers, + }); + } + } } } AggregationUpdateJob::BalanceEdge { upper_id, task_id } => { diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs index c6684a458e075..225fbc70dafb0 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs @@ -17,12 +17,8 @@ const AGGREGATION_NUMBER_BUFFER_SPACE: u32 = 2; #[derive(Serialize, Deserialize, Clone, Default)] pub enum ConnectChildOperation { UpdateAggregation { - task_id: TaskId, aggregation_update: AggregationUpdateQueue, }, - ScheduleTask { - task_id: TaskId, - }, #[default] Done, } @@ -37,6 +33,12 @@ impl ConnectChildOperation { task: child_task_id, value: (), }) { + // When task is added to a AggregateRoot is need to be scheduled, + // indirect connections are handled by the aggregation update. + let mut should_schedule = false; + if parent_task.has_key(&CachedDataItemKey::AggregateRoot {}) { + should_schedule = true; + } // Update the task aggregation let mut queue = AggregationUpdateQueue::new(); let parent_aggregation = get!(parent_task, AggregationNumber) @@ -66,8 +68,20 @@ impl ConnectChildOperation { }); } drop(parent_task); + + { + let mut task = ctx.task(child_task_id); + should_schedule = should_schedule || !task.has_key(&CachedDataItemKey::Output {}); + if should_schedule { + let description = ctx.backend.get_task_desc_fn(child_task_id); + should_schedule = task.add(CachedDataItem::new_scheduled(description)); + } + } + if should_schedule { + ctx.schedule(child_task_id); + } + ConnectChildOperation::UpdateAggregation { - task_id: child_task_id, aggregation_update: queue, } .execute(&ctx); @@ -81,30 +95,11 @@ impl Operation for ConnectChildOperation { ctx.operation_suspend_point(&self); match self { ConnectChildOperation::UpdateAggregation { - task_id, ref mut aggregation_update, } => { if aggregation_update.process(ctx) { - // TODO check for active - self = ConnectChildOperation::ScheduleTask { task_id } - } - } - ConnectChildOperation::ScheduleTask { task_id } => { - let mut should_schedule; - { - let mut task = ctx.task(task_id); - should_schedule = !task.has_key(&CachedDataItemKey::Output {}); - if should_schedule { - should_schedule = task.add(CachedDataItem::new_scheduled( - task.backend.get_task_desc_fn(task_id), - )); - } + self = ConnectChildOperation::Done } - if should_schedule { - ctx.schedule(task_id); - } - - self = ConnectChildOperation::Done; } ConnectChildOperation::Done => { diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/invalidate.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/invalidate.rs index dd38b3afe08fd..b293b82df7cb1 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/invalidate.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/invalidate.rs @@ -6,7 +6,10 @@ use super::{ aggregation_update::{AggregatedDataUpdate, AggregationUpdateJob, AggregationUpdateQueue}, ExecuteContext, Operation, }; -use crate::data::CachedDataItem; +use crate::{ + data::{CachedDataItem, CachedDataItemKey}, + get, +}; #[derive(Serialize, Deserialize, Clone, Default)] pub enum InvalidateOperation { @@ -66,9 +69,21 @@ pub fn make_task_dirty(task_id: TaskId, queue: &mut AggregationUpdateQueue, ctx: let mut task = ctx.task(task_id); if task.add(CachedDataItem::Dirty { value: () }) { - queue.extend(AggregationUpdateJob::data_update( - &mut task, - AggregatedDataUpdate::dirty_task(task_id), - )); + let dirty_container = get!(task, AggregatedDirtyContainerCount) + .copied() + .unwrap_or_default(); + if dirty_container == 0 { + queue.extend(AggregationUpdateJob::data_update( + &mut task, + AggregatedDataUpdate::dirty_container(task_id), + )); + } + let root = task.has_key(&CachedDataItemKey::AggregateRoot {}); + if root { + let description = ctx.backend.get_task_desc_fn(task_id); + if task.add(CachedDataItem::new_scheduled(description)) { + ctx.schedule(task_id); + } + } } } diff --git a/turbopack/crates/turbo-tasks-backend/src/data.rs b/turbopack/crates/turbo-tasks-backend/src/data.rs index 1744f064f6daa..2a235390034a7 100644 --- a/turbopack/crates/turbo-tasks-backend/src/data.rs +++ b/turbopack/crates/turbo-tasks-backend/src/data.rs @@ -37,12 +37,12 @@ impl OutputValue { #[derive(Debug)] pub struct RootState { - pub ty: RootType, + pub ty: ActiveType, pub all_clean_event: Event, } impl RootState { - pub fn new(ty: RootType) -> Self { + pub fn new(ty: ActiveType) -> Self { Self { ty, all_clean_event: Event::new(|| "RootState::all_clean_event".to_string()), @@ -51,10 +51,13 @@ impl RootState { } #[derive(Debug, Clone, Copy)] -pub enum RootType { +pub enum ActiveType { RootTask, OnceTask, - ReadingStronglyConsistent, + /// The aggregated task graph was scheduled because it has reached an AggregatedRoot while + /// propagating the dirty container or is read strongly consistent. This state is reset when + /// all this sub graph becomes clean again. + CachedActiveUntilClean, } impl Clone for RootState { @@ -182,7 +185,7 @@ pub enum CachedDataItem { }, // Aggregated Data - AggregatedDirtyTask { + AggregatedDirtyContainer { task: TaskId, value: i32, }, @@ -190,7 +193,7 @@ pub enum CachedDataItem { collectible: CellRef, value: i32, }, - AggregatedDirtyTaskCount { + AggregatedDirtyContainerCount { value: i32, }, @@ -249,11 +252,11 @@ impl CachedDataItem { CachedDataItem::AggregationNumber { .. } => true, CachedDataItem::Follower { task, .. } => !task.is_transient(), CachedDataItem::Upper { task, .. } => !task.is_transient(), - CachedDataItem::AggregatedDirtyTask { task, .. } => !task.is_transient(), + CachedDataItem::AggregatedDirtyContainer { task, .. } => !task.is_transient(), CachedDataItem::AggregatedCollectible { collectible, .. } => { !collectible.task.is_transient() } - CachedDataItem::AggregatedDirtyTaskCount { .. } => true, + CachedDataItem::AggregatedDirtyContainerCount { .. } => true, CachedDataItem::AggregateRoot { .. } => false, CachedDataItem::InProgress { .. } => false, CachedDataItem::InProgressCell { .. } => false, @@ -307,11 +310,11 @@ impl CachedDataItemKey { CachedDataItemKey::AggregationNumber { .. } => true, CachedDataItemKey::Follower { task, .. } => !task.is_transient(), CachedDataItemKey::Upper { task, .. } => !task.is_transient(), - CachedDataItemKey::AggregatedDirtyTask { task, .. } => !task.is_transient(), + CachedDataItemKey::AggregatedDirtyContainer { task, .. } => !task.is_transient(), CachedDataItemKey::AggregatedCollectible { collectible, .. } => { !collectible.task.is_transient() } - CachedDataItemKey::AggregatedDirtyTaskCount { .. } => true, + CachedDataItemKey::AggregatedDirtyContainerCount { .. } => true, CachedDataItemKey::AggregateRoot { .. } => false, CachedDataItemKey::InProgress { .. } => false, CachedDataItemKey::InProgressCell { .. } => false, From b00979ac8e5e993755755bfee6ed9d3565d70046 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Mon, 23 Sep 2024 13:04:46 +0200 Subject: [PATCH 54/81] fix deadlock --- .../turbo-tasks-backend/src/utils/dash_map_multi.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/turbopack/crates/turbo-tasks-backend/src/utils/dash_map_multi.rs b/turbopack/crates/turbo-tasks-backend/src/utils/dash_map_multi.rs index ead6317a0262e..06537df822965 100644 --- a/turbopack/crates/turbo-tasks-backend/src/utils/dash_map_multi.rs +++ b/turbopack/crates/turbo-tasks-backend/src/utils/dash_map_multi.rs @@ -136,8 +136,16 @@ where }, ) } else { - let mut guard1 = shards[s1].write(); - let mut guard2 = shards[s2].write(); + let (mut guard1, mut guard2) = loop { + let g1 = shards[s1].write(); + if let Some(g2) = shards[s2].try_write() { + break (g1, g2); + } + let g2 = shards[s2].write(); + if let Some(g1) = shards[s1].try_write() { + break (g1, g2); + } + }; let e1 = guard1 .raw_entry_mut() .from_key(&key1) From 0164bfce661c4d4721e453b597c0dcb04d172e5a Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Mon, 23 Sep 2024 13:04:46 +0200 Subject: [PATCH 55/81] fix deadlock --- .../src/utils/dash_map_multi.rs | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/turbopack/crates/turbo-tasks-backend/src/utils/dash_map_multi.rs b/turbopack/crates/turbo-tasks-backend/src/utils/dash_map_multi.rs index 06537df822965..8f886625c965f 100644 --- a/turbopack/crates/turbo-tasks-backend/src/utils/dash_map_multi.rs +++ b/turbopack/crates/turbo-tasks-backend/src/utils/dash_map_multi.rs @@ -137,13 +137,17 @@ where ) } else { let (mut guard1, mut guard2) = loop { - let g1 = shards[s1].write(); - if let Some(g2) = shards[s2].try_write() { - break (g1, g2); + { + let g1 = shards[s1].write(); + if let Some(g2) = shards[s2].try_write() { + break (g1, g2); + } } - let g2 = shards[s2].write(); - if let Some(g1) = shards[s1].try_write() { - break (g1, g2); + { + let g2 = shards[s2].write(); + if let Some(g1) = shards[s1].try_write() { + break (g1, g2); + } } }; let e1 = guard1 From c515a8e318c3000997385d60e58c64121923b186 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Mon, 23 Sep 2024 15:17:46 +0200 Subject: [PATCH 56/81] print timing when running test cases --- turbopack/crates/turbo-tasks-testing/src/run.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/turbopack/crates/turbo-tasks-testing/src/run.rs b/turbopack/crates/turbo-tasks-testing/src/run.rs index 8de9199b79097..5b2681421a196 100644 --- a/turbopack/crates/turbo-tasks-testing/src/run.rs +++ b/turbopack/crates/turbo-tasks-testing/src/run.rs @@ -106,15 +106,25 @@ where let name = closure_to_name(&fut); let tt = registration.create_turbo_tasks(&name, true); println!("Run #1 (without cache)"); + let start = std::time::Instant::now(); let first = fut(tt.clone()).await?; + println!("Run #1 took {:?}", start.elapsed()); println!("Run #2 (with memory cache, same TurboTasks instance)"); + let start = std::time::Instant::now(); let second = fut(tt.clone()).await?; + println!("Run #2 took {:?}", start.elapsed()); assert_eq!(first, second); + let start = std::time::Instant::now(); tt.stop_and_wait().await; + println!("Stopping TurboTasks took {:?}", start.elapsed()); let tt = registration.create_turbo_tasks(&name, false); println!("Run #3 (with persistent cache if available, new TurboTasks instance)"); + let start = std::time::Instant::now(); let third = fut(tt.clone()).await?; + println!("Run #3 took {:?}", start.elapsed()); + let start = std::time::Instant::now(); tt.stop_and_wait().await; + println!("Stopping TurboTasks took {:?}", start.elapsed()); assert_eq!(first, third); Ok(()) } From 87c45c0451a6e4aee0ab171b2f98c28f84d20bb9 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Mon, 23 Sep 2024 17:52:41 +0200 Subject: [PATCH 57/81] little refactor --- .../src/backend/operation/connect_child.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs index 225fbc70dafb0..8afbaf8e9f345 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs @@ -3,7 +3,7 @@ use turbo_tasks::TaskId; use super::{ aggregation_update::{ - get_uppers, is_aggregating_node, is_root_node, AggregationUpdateJob, AggregationUpdateQueue, + get_uppers, is_aggregating_node, AggregationUpdateJob, AggregationUpdateQueue, }, ExecuteContext, Operation, }; @@ -44,18 +44,19 @@ impl ConnectChildOperation { let parent_aggregation = get!(parent_task, AggregationNumber) .copied() .unwrap_or_default(); + let is_aggregating_node = is_aggregating_node(parent_aggregation); if parent_task_id.is_transient() && !child_task_id.is_transient() { queue.push(AggregationUpdateJob::UpdateAggregationNumber { task_id: child_task_id, aggregation_number: u32::MAX, }); - } else if !is_root_node(parent_aggregation) { + } else if !is_aggregating_node { queue.push(AggregationUpdateJob::UpdateAggregationNumber { task_id: child_task_id, aggregation_number: parent_aggregation + AGGREGATION_NUMBER_BUFFER_SPACE + 1, }); } - if is_aggregating_node(parent_aggregation) { + if is_aggregating_node { queue.push(AggregationUpdateJob::InnerHasNewFollower { upper_ids: vec![parent_task_id], new_follower_id: child_task_id, From db27a4006de15cd91f8f72238f50fe873a4cb76d Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Mon, 23 Sep 2024 18:46:44 +0200 Subject: [PATCH 58/81] add indicies for faster access --- Cargo.lock | 1 + .../crates/turbo-tasks-backend/Cargo.toml | 1 + .../src/backend/indexed.rs | 4 + .../turbo-tasks-backend/src/backend/mod.rs | 86 +++++++--- .../src/backend/operation/mod.rs | 26 ++- .../src/backend/operation/update_cell.rs | 22 +-- .../src/backend/operation/update_output.rs | 16 +- .../src/backend/storage.rs | 157 +++++++++++++----- .../crates/turbo-tasks-backend/src/data.rs | 66 ++++++++ 9 files changed, 288 insertions(+), 91 deletions(-) create mode 100644 turbopack/crates/turbo-tasks-backend/src/backend/indexed.rs diff --git a/Cargo.lock b/Cargo.lock index 00410fb4a3e07..2c36d4aab63b4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8577,6 +8577,7 @@ dependencies = [ "async-trait", "auto-hash-map", "dashmap", + "either", "hashbrown 0.14.5", "indexmap 1.9.3", "once_cell", diff --git a/turbopack/crates/turbo-tasks-backend/Cargo.toml b/turbopack/crates/turbo-tasks-backend/Cargo.toml index 01419f4d06e13..f51271f10f948 100644 --- a/turbopack/crates/turbo-tasks-backend/Cargo.toml +++ b/turbopack/crates/turbo-tasks-backend/Cargo.toml @@ -17,6 +17,7 @@ anyhow = { workspace = true } async-trait = { workspace = true } auto-hash-map = { workspace = true } dashmap = { workspace = true, features = ["raw-api"]} +either = { workspace = true } hashbrown = { workspace = true } indexmap = { workspace = true } once_cell = { workspace = true } diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/indexed.rs b/turbopack/crates/turbo-tasks-backend/src/backend/indexed.rs new file mode 100644 index 0000000000000..b5598224c2eef --- /dev/null +++ b/turbopack/crates/turbo-tasks-backend/src/backend/indexed.rs @@ -0,0 +1,4 @@ +pub trait Indexed { + type Index: Clone + PartialEq + Eq + std::hash::Hash; + fn index(&self) -> Self::Index; +} diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs index 62b335fbbddac..c831594cc46e8 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs @@ -1,3 +1,4 @@ +pub mod indexed; mod operation; mod storage; @@ -41,8 +42,8 @@ use turbo_tasks::{ use self::{operation::ExecuteContext, storage::Storage}; use crate::{ data::{ - ActiveType, CachedDataItem, CachedDataItemKey, CachedDataItemValue, CachedDataUpdate, - CellRef, InProgressCellState, InProgressState, OutputValue, RootState, + ActiveType, CachedDataItem, CachedDataItemIndex, CachedDataItemKey, CachedDataItemValue, + CachedDataUpdate, CellRef, InProgressCellState, InProgressState, OutputValue, RootState, }, get, get_many, remove, utils::{bi_map::BiMap, chunked_vec::ChunkedVec, ptr_eq_arc::PtrEqArc}, @@ -591,7 +592,7 @@ impl Backend for TurboTasksBackend { Outdated(TaskId), } let children = task - .iter() + .iter(CachedDataItemIndex::Children) .filter_map(|(key, _)| match *key { CachedDataItemKey::Child { task } => Some(Child::Current(task)), CachedDataItemKey::OutdatedChild { task } => Some(Child::Outdated(task)), @@ -622,7 +623,7 @@ impl Backend for TurboTasksBackend { OutdatedOutput(TaskId), } let dependencies = task - .iter() + .iter(CachedDataItemIndex::Dependencies) .filter_map(|(key, _)| match *key { CachedDataItemKey::CellDependency { target } => Some(Dep::CurrentCell(target)), CachedDataItemKey::OutputDependency { target } => { @@ -829,26 +830,63 @@ impl Backend for TurboTasksBackend { } // find all outdated data items (removed cells, outdated edges) - let old_edges = task - .iter() - .filter_map(|(key, _)| match *key { - CachedDataItemKey::OutdatedChild { task } => Some(OutdatedEdge::Child(task)), - CachedDataItemKey::OutdatedCellDependency { target } => { - Some(OutdatedEdge::CellDependency(target)) - } - CachedDataItemKey::OutdatedOutputDependency { target } => { - Some(OutdatedEdge::OutputDependency(target)) - } - CachedDataItemKey::CellDependent { cell, task } - if removed_cells - .get(&cell.type_id) - .map_or(false, |range| range.contains(&cell.index)) => - { - Some(OutdatedEdge::RemovedCellDependent(task)) - } - _ => None, - }) - .collect::>(); + let old_edges = if task.is_indexed() { + task.iter(CachedDataItemIndex::Children) + .filter_map(|(key, _)| match *key { + CachedDataItemKey::OutdatedChild { task } => { + Some(OutdatedEdge::Child(task)) + } + _ => None, + }) + .chain( + task.iter(CachedDataItemIndex::Dependencies) + .filter_map(|(key, _)| match *key { + CachedDataItemKey::OutdatedCellDependency { target } => { + Some(OutdatedEdge::CellDependency(target)) + } + CachedDataItemKey::OutdatedOutputDependency { target } => { + Some(OutdatedEdge::OutputDependency(target)) + } + _ => None, + }), + ) + .chain( + task.iter(CachedDataItemIndex::CellDependent) + .filter_map(|(key, _)| match *key { + CachedDataItemKey::CellDependent { cell, task } + if removed_cells + .get(&cell.type_id) + .map_or(false, |range| range.contains(&cell.index)) => + { + Some(OutdatedEdge::RemovedCellDependent(task)) + } + _ => None, + }), + ) + .collect::>() + } else { + task.iter_all() + .filter_map(|(key, _)| match *key { + CachedDataItemKey::OutdatedChild { task } => { + Some(OutdatedEdge::Child(task)) + } + CachedDataItemKey::OutdatedCellDependency { target } => { + Some(OutdatedEdge::CellDependency(target)) + } + CachedDataItemKey::OutdatedOutputDependency { target } => { + Some(OutdatedEdge::OutputDependency(target)) + } + CachedDataItemKey::CellDependent { cell, task } + if removed_cells + .get(&cell.type_id) + .map_or(false, |range| range.contains(&cell.index)) => + { + Some(OutdatedEdge::RemovedCellDependent(task)) + } + _ => None, + }) + .collect::>() + }; let was_dirty = task.remove(&CachedDataItemKey::Dirty {}).is_some(); let data_update = if was_dirty { diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs index 3e6c7968a80c3..d3f42e6c5b444 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs @@ -16,7 +16,10 @@ use turbo_tasks::{KeyValuePair, TaskId, TurboTasksBackendApi}; use super::{storage::StorageWriteGuard, TurboTasksBackend}; use crate::{ backend::{OperationGuard, TransientTask}, - data::{CachedDataItem, CachedDataItemKey, CachedDataItemValue, CachedDataUpdate}, + data::{ + CachedDataItem, CachedDataItemIndex, CachedDataItemKey, CachedDataItemValue, + CachedDataUpdate, + }, }; pub trait Operation: @@ -141,7 +144,7 @@ impl<'a> Debug for TaskGuard<'a> { if let Some(task_type) = self.backend.task_cache.lookup_reverse(&self.task_id) { d.field("task_type", &task_type); }; - for (key, value) in self.task.iter() { + for (key, value) in self.task.iter_all() { d.field(&format!("{:?}", key), &value); } d.finish() @@ -159,6 +162,7 @@ impl<'a> TaskGuard<'a> { self.task.add(item) } else if self.task.add(item.clone()) { let (key, value) = item.into_key_and_value(); + // TODO task.persistance_state.add_persisting_item(); self.backend .persisted_storage_log .lock() @@ -188,6 +192,7 @@ impl<'a> TaskGuard<'a> { key.clone(), value.clone(), )); + // TODO task.persistance_state.add_persisting_item(); self.backend .persisted_storage_log .lock() @@ -201,6 +206,7 @@ impl<'a> TaskGuard<'a> { let item = CachedDataItem::from_key_and_value(key.clone(), value); if let Some(old) = self.task.insert(item) { if old.is_persistent() { + // TODO task.persistance_state.add_persisting_item(); self.backend .persisted_storage_log .lock() @@ -269,6 +275,7 @@ impl<'a> TaskGuard<'a> { if let Some(value) = old_value { if key.is_persistent() && value.is_persistent() { let key = key.clone(); + // TODO task.persistance_state.add_persisting_item(); self.backend .persisted_storage_log .lock() @@ -292,8 +299,19 @@ impl<'a> TaskGuard<'a> { self.task.has_key(key) } - pub fn iter(&self) -> impl Iterator { - self.task.iter() + pub fn is_indexed(&self) -> bool { + self.task.is_indexed() + } + + pub fn iter( + &self, + index: CachedDataItemIndex, + ) -> impl Iterator { + self.task.iter(Some(index)) + } + + pub fn iter_all(&self) -> impl Iterator { + self.task.iter_all() } } diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/update_cell.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/update_cell.rs index bb301010c5900..c973a991a03be 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/update_cell.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/update_cell.rs @@ -3,7 +3,7 @@ use turbo_tasks::{backend::CellContent, CellId, TaskId}; use super::{ExecuteContext, InvalidateOperation}; use crate::{ data::{CachedDataItem, CachedDataItemKey}, - remove, + get_many, remove, }; pub struct UpdateCellOperation; @@ -35,20 +35,12 @@ impl UpdateCellOperation { return; } - let dependent = task - .iter() - .filter_map(|(key, _)| { - if let CachedDataItemKey::CellDependent { - cell: dependent_cell, - task, - } = *key - { - (dependent_cell == cell).then_some(task) - } else { - None - } - }) - .collect(); + let dependent = get_many!( + task, + CellDependent { cell: dependent_cell, task } _value + if dependent_cell == cell + => task + ); drop(task); drop(old_content); diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/update_output.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/update_output.rs index ac3f7fba5514d..f563580b4203d 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/update_output.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/update_output.rs @@ -4,7 +4,10 @@ use anyhow::{anyhow, Result}; use turbo_tasks::{util::SharedError, RawVc, TaskId}; use super::{ExecuteContext, InvalidateOperation}; -use crate::data::{CachedDataItem, CachedDataItemKey, CachedDataItemValue, CellRef, OutputValue}; +use crate::{ + data::{CachedDataItem, CachedDataItemKey, CachedDataItemValue, CellRef, OutputValue}, + get_many, +}; pub struct UpdateOutputOperation; @@ -70,16 +73,7 @@ impl UpdateOutputOperation { value: output_value, }); - let dependent = task - .iter() - .filter_map(|(key, _)| { - if let CachedDataItemKey::OutputDependent { task } = *key { - Some(task) - } else { - None - } - }) - .collect(); + let dependent = get_many!(task, OutputDependent { task } _value => task); drop(task); drop(old_content); diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs b/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs index 4b8cb62b22ad3..f9bf64045df55 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs @@ -7,26 +7,79 @@ use std::{ use auto_hash_map::{map::Entry, AutoMap}; use dashmap::DashMap; +use either::Either; use rustc_hash::FxHasher; use turbo_tasks::KeyValuePair; +use super::indexed::Indexed; use crate::utils::dash_map_multi::{get_multiple_mut, RefMut}; -pub struct InnerStorage { - // TODO consider adding some inline storage - map: AutoMap, +const INDEX_THRESHOLD: usize = 1024; + +pub enum InnerStorage +where + T::Key: Indexed, +{ + Plain { + // TODO use FxHasher + map: AutoMap, + }, + Indexed { + // TODO use FxHasher + map: AutoMap<::Index, AutoMap>, + }, } -impl InnerStorage { +impl InnerStorage +where + T::Key: Indexed, +{ fn new() -> Self { - Self { + Self::Plain { map: AutoMap::new(), } } + fn check_threshold(&mut self) { + let InnerStorage::Plain { map: plain_map } = self else { + return; + }; + if plain_map.len() >= INDEX_THRESHOLD { + let mut map: AutoMap<::Index, AutoMap> = + AutoMap::new(); + for (key, value) in take(plain_map).into_iter() { + let index = key.index(); + map.entry(index).or_default().insert(key, value); + } + *self = InnerStorage::Indexed { map }; + } + } + + fn map_mut(&mut self, key: &T::Key) -> &mut AutoMap { + self.check_threshold(); + match self { + InnerStorage::Plain { map, .. } => map, + InnerStorage::Indexed { map, .. } => map.entry(key.index()).or_default(), + } + } + + fn map(&self, key: &T::Key) -> Option<&AutoMap> { + match self { + InnerStorage::Plain { map, .. } => Some(map), + InnerStorage::Indexed { map, .. } => map.get(&key.index()), + } + } + + fn index_map(&self, index: ::Index) -> Option<&AutoMap> { + match self { + InnerStorage::Plain { map, .. } => Some(map), + InnerStorage::Indexed { map, .. } => map.get(&index), + } + } + pub fn add(&mut self, item: T) -> bool { let (key, value) = item.into_key_and_value(); - match self.map.entry(key) { + match self.map_mut(&key).entry(key) { Entry::Occupied(_) => false, Entry::Vacant(e) => { e.insert(value); @@ -37,28 +90,50 @@ impl InnerStorage { pub fn insert(&mut self, item: T) -> Option { let (key, value) = item.into_key_and_value(); - self.map.insert(key, value) + self.map_mut(&key).insert(key, value) } pub fn remove(&mut self, key: &T::Key) -> Option { - self.map.remove(key) + self.map_mut(key).remove(key) } pub fn get(&self, key: &T::Key) -> Option<&T::Value> { - self.map.get(key) + self.map(key).and_then(|m| m.get(key)) } pub fn has_key(&self, key: &T::Key) -> bool { - self.map.contains_key(key) + self.map(key) + .map(|m| m.contains_key(key)) + .unwrap_or_default() } - pub fn iter(&self) -> impl Iterator { - self.map.iter() + pub fn is_indexed(&self) -> bool { + matches!(self, InnerStorage::Indexed { .. }) + } + + pub fn iter( + &self, + index: ::Index, + ) -> impl Iterator { + self.index_map(index) + .map(|m| m.iter()) + .into_iter() + .flatten() + } + + pub fn iter_all(&self) -> impl Iterator { + match self { + InnerStorage::Plain { map, .. } => Either::Left(map.iter()), + InnerStorage::Indexed { map, .. } => { + Either::Right(map.iter().flat_map(|(_, m)| m.iter())) + } + } } } impl InnerStorage where + T::Key: Indexed, T::Value: Default, T::Key: Clone, { @@ -67,26 +142,28 @@ where key: &T::Key, update: impl FnOnce(Option) -> Option, ) { - if let Some(value) = self.map.get_mut(key) { + let map = self.map_mut(key); + if let Some(value) = map.get_mut(key) { let v = take(value); if let Some(v) = update(Some(v)) { *value = v; } else { - self.map.remove(key); + map.remove(key); } } else if let Some(v) = update(None) { - self.map.insert(key.clone(), v); + map.insert(key.clone(), v); } } } impl InnerStorage where + T::Key: Indexed, T::Value: PartialEq, { pub fn has(&self, item: &mut T) -> bool { let (key, value) = take(item).into_key_and_value(); - let result = if let Some(stored_value) = self.map.get(&key) { + let result = if let Some(stored_value) = self.get(&key) { *stored_value == value } else { false @@ -96,14 +173,18 @@ where } } -pub struct Storage { +pub struct Storage +where + T::Key: Indexed, +{ map: DashMap, BuildHasherDefault>, } -impl Storage +impl Storage where - K: Eq + std::hash::Hash + Clone, T: KeyValuePair, + T::Key: Indexed, + K: Eq + std::hash::Hash + Clone, { pub fn new() -> Self { let shard_amount = @@ -140,11 +221,20 @@ where } } -pub struct StorageWriteGuard<'a, K, T: KeyValuePair> { +pub struct StorageWriteGuard<'a, K, T> +where + T: KeyValuePair, + T::Key: Indexed, +{ inner: RefMut<'a, K, InnerStorage, BuildHasherDefault>, } -impl<'a, K: Eq + Hash, T: KeyValuePair> Deref for StorageWriteGuard<'a, K, T> { +impl<'a, K, T> Deref for StorageWriteGuard<'a, K, T> +where + T: KeyValuePair, + T::Key: Indexed, + K: Eq + Hash, +{ type Target = InnerStorage; fn deref(&self) -> &Self::Target { @@ -152,7 +242,12 @@ impl<'a, K: Eq + Hash, T: KeyValuePair> Deref for StorageWriteGuard<'a, K, T> { } } -impl<'a, K: Eq + Hash, T: KeyValuePair> DerefMut for StorageWriteGuard<'a, K, T> { +impl<'a, K, T> DerefMut for StorageWriteGuard<'a, K, T> +where + T: KeyValuePair, + T::Key: Indexed, + K: Eq + Hash, +{ fn deref_mut(&mut self) -> &mut Self::Target { &mut self.inner } @@ -180,7 +275,7 @@ macro_rules! get { macro_rules! iter_many { ($task:ident, $key:ident $input:tt => $value:ident) => { $task - .iter() + .iter($crate::data::indicies::$key) .filter_map(|(key, _)| match *key { $crate::data::CachedDataItemKey::$key $input => Some($value), _ => None, @@ -188,7 +283,7 @@ macro_rules! iter_many { }; ($task:ident, $key:ident $input:tt $value_ident:ident => $value:expr) => { $task - .iter() + .iter($crate::data::indicies::$key) .filter_map(|(key, value)| match (key, value) { (&$crate::data::CachedDataItemKey::$key $input, &$crate::data::CachedDataItemValue::$key { value: $value_ident }) => Some($value), _ => None, @@ -196,21 +291,12 @@ macro_rules! iter_many { }; ($task:ident, $key:ident $input:tt $value_ident:ident if $cond:expr => $value:expr) => { $task - .iter() + .iter($crate::data::indicies::$key) .filter_map(|(key, value)| match (key, value) { (&$crate::data::CachedDataItemKey::$key $input, &$crate::data::CachedDataItemValue::$key { value: $value_ident }) if $cond => Some($value), _ => None, }) }; - ($task:ident, $key1:ident $input1:tt => $value1:ident, $key2:ident $input2:tt => $value2:ident) => { - $task - .iter() - .filter_map(|(key, _)| match *key { - $crate::data::CachedDataItemKey::$key1 $input1 => Some($value1), - $crate::data::CachedDataItemKey::$key2 $input2 => Some($value2), - _ => None, - }) - }; } #[macro_export] @@ -224,9 +310,6 @@ macro_rules! get_many { ($task:ident, $key:ident $input:tt $value_ident:ident if $cond:expr => $value:expr) => { $crate::iter_many!($task, $key $input $value_ident if $cond => $value).collect() }; - ($task:ident, $key1:ident $input1:tt => $value1:ident, $key2:ident $input2:tt => $value2:ident) => { - $crate::iter_many!($task, $key1 $input1 => $value1, $key2 $input2 => $value2).collect() - }; } #[macro_export] diff --git a/turbopack/crates/turbo-tasks-backend/src/data.rs b/turbopack/crates/turbo-tasks-backend/src/data.rs index 2a235390034a7..f448795174fb4 100644 --- a/turbopack/crates/turbo-tasks-backend/src/data.rs +++ b/turbopack/crates/turbo-tasks-backend/src/data.rs @@ -5,6 +5,8 @@ use turbo_tasks::{ CellId, KeyValuePair, SharedReference, TaskId, ValueTypeId, }; +use crate::backend::indexed::Indexed; + #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq, Serialize, Deserialize)] pub struct CellRef { pub task: TaskId, @@ -327,6 +329,70 @@ impl CachedDataItemKey { } } +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum CachedDataItemIndex { + Children, + Follower, + Upper, + AggregatedDirtyContainer, + CellData, + CellTypeMaxIndex, + CellDependent, + OutputDependent, + Dependencies, +} + +#[allow(non_upper_case_globals, dead_code)] +pub mod indicies { + use super::CachedDataItemIndex; + + pub const Child: CachedDataItemIndex = CachedDataItemIndex::Children; + pub const OutdatedChild: CachedDataItemIndex = CachedDataItemIndex::Children; + pub const Follower: CachedDataItemIndex = CachedDataItemIndex::Follower; + pub const Upper: CachedDataItemIndex = CachedDataItemIndex::Upper; + pub const AggregatedDirtyContainer: CachedDataItemIndex = + CachedDataItemIndex::AggregatedDirtyContainer; + pub const CellData: CachedDataItemIndex = CachedDataItemIndex::CellData; + pub const CellTypeMaxIndex: CachedDataItemIndex = CachedDataItemIndex::CellTypeMaxIndex; + pub const CellDependent: CachedDataItemIndex = CachedDataItemIndex::CellDependent; + pub const OutputDependent: CachedDataItemIndex = CachedDataItemIndex::OutputDependent; + pub const OutputDependency: CachedDataItemIndex = CachedDataItemIndex::Dependencies; + pub const CellDependency: CachedDataItemIndex = CachedDataItemIndex::Dependencies; + pub const OutdatedOutputDependency: CachedDataItemIndex = CachedDataItemIndex::Dependencies; + pub const OutdatedCellDependency: CachedDataItemIndex = CachedDataItemIndex::Dependencies; +} + +impl Indexed for CachedDataItemKey { + type Index = Option; + + fn index(&self) -> Option { + match self { + CachedDataItemKey::Child { .. } => Some(CachedDataItemIndex::Children), + CachedDataItemKey::OutdatedChild { .. } => Some(CachedDataItemIndex::Children), + CachedDataItemKey::Follower { .. } => Some(CachedDataItemIndex::Follower), + CachedDataItemKey::Upper { .. } => Some(CachedDataItemIndex::Upper), + CachedDataItemKey::AggregatedDirtyContainer { .. } => { + Some(CachedDataItemIndex::AggregatedDirtyContainer) + } + CachedDataItemKey::CellData { .. } => Some(CachedDataItemIndex::CellData), + CachedDataItemKey::CellTypeMaxIndex { .. } => { + Some(CachedDataItemIndex::CellTypeMaxIndex) + } + CachedDataItemKey::CellDependent { .. } => Some(CachedDataItemIndex::CellDependent), + CachedDataItemKey::OutputDependent { .. } => Some(CachedDataItemIndex::OutputDependent), + CachedDataItemKey::OutputDependency { .. } => Some(CachedDataItemIndex::Dependencies), + CachedDataItemKey::CellDependency { .. } => Some(CachedDataItemIndex::Dependencies), + CachedDataItemKey::OutdatedOutputDependency { .. } => { + Some(CachedDataItemIndex::Dependencies) + } + CachedDataItemKey::OutdatedCellDependency { .. } => { + Some(CachedDataItemIndex::Dependencies) + } + _ => None, + } + } +} + impl CachedDataItemValue { pub fn is_persistent(&self) -> bool { match self { From 4895244a47241e8912846c87b06da5436fa1593f Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Mon, 23 Sep 2024 16:23:55 +0200 Subject: [PATCH 59/81] optimize aggregation number based on number of children --- .../turbo-tasks-backend/src/backend/mod.rs | 22 ++++++--- .../backend/operation/aggregation_update.rs | 49 ++++++++++++++----- .../src/backend/operation/connect_child.rs | 44 +++++++++++++++-- .../src/backend/operation/mod.rs | 4 +- .../crates/turbo-tasks-backend/src/data.rs | 9 +++- 5 files changed, 104 insertions(+), 24 deletions(-) diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs index c831594cc46e8..70b92fb9629a2 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs @@ -21,8 +21,8 @@ use auto_hash_map::{AutoMap, AutoSet}; use dashmap::DashMap; pub use operation::AnyOperation; use operation::{ - is_root_node, AggregatedDataUpdate, AggregationUpdateJob, AggregationUpdateQueue, - CleanupOldEdgesOperation, ConnectChildOperation, OutdatedEdge, + get_aggregation_number, is_root_node, AggregatedDataUpdate, AggregationUpdateJob, + AggregationUpdateQueue, CleanupOldEdgesOperation, ConnectChildOperation, OutdatedEdge, }; use parking_lot::{Condvar, Mutex}; use rustc_hash::FxHasher; @@ -42,8 +42,9 @@ use turbo_tasks::{ use self::{operation::ExecuteContext, storage::Storage}; use crate::{ data::{ - ActiveType, CachedDataItem, CachedDataItemIndex, CachedDataItemKey, CachedDataItemValue, - CachedDataUpdate, CellRef, InProgressCellState, InProgressState, OutputValue, RootState, + ActiveType, AggregationNumber, CachedDataItem, CachedDataItemIndex, CachedDataItemKey, + CachedDataItemValue, CachedDataUpdate, CellRef, InProgressCellState, InProgressState, + OutputValue, RootState, }, get, get_many, remove, utils::{bi_map::BiMap, chunked_vec::ChunkedVec, ptr_eq_arc::PtrEqArc}, @@ -256,7 +257,7 @@ impl TurboTasksBackend { if matches!(consistency, ReadConsistency::Strong) { // Ensure it's an root node loop { - let aggregation_number = get!(task, AggregationNumber).copied().unwrap_or_default(); + let aggregation_number = get_aggregation_number(&task); if is_root_node(aggregation_number) { break; } @@ -264,7 +265,8 @@ impl TurboTasksBackend { AggregationUpdateQueue::run( AggregationUpdateJob::UpdateAggregationNumber { task_id, - aggregation_number: u32::MAX, + base_aggregation_number: u32::MAX, + distance: None, }, &ctx, ); @@ -1058,7 +1060,13 @@ impl Backend for TurboTasksBackend { ); { let mut task = self.storage.access_mut(task_id); - let _ = task.add(CachedDataItem::AggregationNumber { value: u32::MAX }); + let _ = task.add(CachedDataItem::AggregationNumber { + value: AggregationNumber { + base: u32::MAX, + distance: 0, + effective: u32::MAX, + }, + }); let _ = task.add(CachedDataItem::AggregateRoot { value: RootState::new(root_type), }); diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs index 490995d6d9c02..d2b798279f611 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs @@ -1,11 +1,11 @@ -use std::collections::VecDeque; +use std::{cmp::max, collections::VecDeque, num::NonZeroU32}; use serde::{Deserialize, Serialize}; use turbo_tasks::TaskId; use super::{ExecuteContext, Operation, TaskGuard}; use crate::{ - data::{ActiveType, CachedDataItem, CachedDataItemKey, RootState}, + data::{ActiveType, AggregationNumber, CachedDataItem, CachedDataItemKey, RootState}, get, get_many, iter_many, remove, update, update_count, }; @@ -43,14 +43,17 @@ fn iter_uppers<'a>(task: &'a TaskGuard<'a>) -> impl Iterator + 'a } pub fn get_aggregation_number(task: &TaskGuard<'_>) -> u32 { - get!(task, AggregationNumber).copied().unwrap_or_default() + get!(task, AggregationNumber) + .map(|a| a.effective) + .unwrap_or_default() } #[derive(Serialize, Deserialize, Clone, Debug)] pub enum AggregationUpdateJob { UpdateAggregationNumber { task_id: TaskId, - aggregation_number: u32, + base_aggregation_number: u32, + distance: Option, }, InnerHasNewFollower { upper_ids: Vec, @@ -252,13 +255,35 @@ impl AggregationUpdateQueue { match job { AggregationUpdateJob::UpdateAggregationNumber { task_id, - aggregation_number, + base_aggregation_number, + distance: base_effective_distance, } => { let mut task = ctx.task(task_id); - let old = get_aggregation_number(&task); - if old < aggregation_number { + let current = get!(task, AggregationNumber).copied().unwrap_or_default(); + // The wanted new distance is either the provided one or the old distance + let distance = base_effective_distance.map_or(current.distance, |d| d.get()); + // The base aggregation number can only increase + let base_aggregation_number = max(current.base, base_aggregation_number); + let old = current.effective; + // The new target effecive aggregation number is base + distance + let aggregation_number = base_aggregation_number.saturating_add(distance); + if old >= aggregation_number { + if base_aggregation_number != current.base && distance != current.distance { + task.insert(CachedDataItem::AggregationNumber { + value: AggregationNumber { + base: base_aggregation_number, + distance, + effective: old, + }, + }); + } + } else { task.insert(CachedDataItem::AggregationNumber { - value: aggregation_number, + value: AggregationNumber { + base: base_aggregation_number, + distance, + effective: aggregation_number, + }, }); if !is_aggregating_node(old) && is_aggregating_node(aggregation_number) { @@ -293,7 +318,8 @@ impl AggregationUpdateQueue { for child_id in children { self.push(AggregationUpdateJob::UpdateAggregationNumber { task_id: child_id, - aggregation_number: aggregation_number + 1, + base_aggregation_number: aggregation_number + 1, + distance: None, }); } } @@ -636,10 +662,11 @@ impl AggregationUpdateQueue { } else { // both nodes have the same aggregation number // We need to change the aggregation number of the task - let new_aggregation_number = upper_aggregation_number + 1; + let current = get!(task, AggregationNumber).copied().unwrap_or_default(); self.push(AggregationUpdateJob::UpdateAggregationNumber { task_id, - aggregation_number: new_aggregation_number, + base_aggregation_number: current.base + 1, + distance: None, }); } } diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs index 8afbaf8e9f345..a915fa776347b 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs @@ -1,3 +1,5 @@ +use std::{cmp::max, num::NonZeroU32}; + use serde::{Deserialize, Serialize}; use turbo_tasks::TaskId; @@ -8,7 +10,8 @@ use super::{ ExecuteContext, Operation, }; use crate::{ - data::{CachedDataItem, CachedDataItemKey}, + backend::operation::is_root_node, + data::{CachedDataItem, CachedDataItemIndex, CachedDataItemKey}, get, }; @@ -41,19 +44,52 @@ impl ConnectChildOperation { } // Update the task aggregation let mut queue = AggregationUpdateQueue::new(); - let parent_aggregation = get!(parent_task, AggregationNumber) + + // Compute new parent aggregation number based on the number of children + let current_parent_aggregation = get!(parent_task, AggregationNumber) .copied() .unwrap_or_default(); + let parent_aggregation = if is_root_node(current_parent_aggregation.base) { + u32::MAX + } else { + let children_count = parent_task + .iter(CachedDataItemIndex::Children) + .filter(|(k, _)| { + matches!( + *k, + CachedDataItemKey::Child { .. } + | CachedDataItemKey::OutdatedChild { .. } + ) + }) + .count(); + let target_distance = children_count.ilog2() as u32 * 2; + let parent_aggregation = current_parent_aggregation + .base + .saturating_add(target_distance); + if target_distance != current_parent_aggregation.distance { + queue.push(AggregationUpdateJob::UpdateAggregationNumber { + task_id: parent_task_id, + base_aggregation_number: 0, + distance: NonZeroU32::new(target_distance), + }) + } + max(current_parent_aggregation.effective, parent_aggregation) + }; + + // Update child aggregation number based on parent aggregation number let is_aggregating_node = is_aggregating_node(parent_aggregation); if parent_task_id.is_transient() && !child_task_id.is_transient() { queue.push(AggregationUpdateJob::UpdateAggregationNumber { task_id: child_task_id, - aggregation_number: u32::MAX, + base_aggregation_number: u32::MAX, + distance: None, }); } else if !is_aggregating_node { queue.push(AggregationUpdateJob::UpdateAggregationNumber { task_id: child_task_id, - aggregation_number: parent_aggregation + AGGREGATION_NUMBER_BUFFER_SPACE + 1, + base_aggregation_number: parent_aggregation + .saturating_add(AGGREGATION_NUMBER_BUFFER_SPACE), + distance: None, }); } if is_aggregating_node { diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs index d3f42e6c5b444..6d3f13d1bc7ee 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs @@ -352,7 +352,9 @@ impl_operation!(Invalidate invalidate::InvalidateOperation); impl_operation!(CleanupOldEdges cleanup_old_edges::CleanupOldEdgesOperation); impl_operation!(AggregationUpdate aggregation_update::AggregationUpdateQueue); -pub use aggregation_update::{is_root_node, AggregatedDataUpdate, AggregationUpdateJob}; +pub use aggregation_update::{ + get_aggregation_number, is_root_node, AggregatedDataUpdate, AggregationUpdateJob, +}; pub use cleanup_old_edges::OutdatedEdge; pub use update_cell::UpdateCellOperation; pub use update_output::UpdateOutputOperation; diff --git a/turbopack/crates/turbo-tasks-backend/src/data.rs b/turbopack/crates/turbo-tasks-backend/src/data.rs index f448795174fb4..390ac1c093d16 100644 --- a/turbopack/crates/turbo-tasks-backend/src/data.rs +++ b/turbopack/crates/turbo-tasks-backend/src/data.rs @@ -108,6 +108,13 @@ impl InProgressCellState { } } +#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize)] +pub struct AggregationNumber { + pub base: u32, + pub distance: u32, + pub effective: u32, +} + #[derive(Debug, Clone, KeyValuePair)] pub enum CachedDataItem { // Output @@ -175,7 +182,7 @@ pub enum CachedDataItem { // Aggregation Graph AggregationNumber { - value: u32, + value: AggregationNumber, }, Follower { task: TaskId, From 741cf1ee90bd9ed50f99fed69c40ea61fb4dd5ef Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Mon, 23 Sep 2024 15:17:29 +0200 Subject: [PATCH 60/81] add performance test case --- .../turbo-tasks-backend/tests/performance.rs | 1 + .../turbo-tasks-memory/tests/performance.rs | 1 + .../turbo-tasks-testing/tests/performance.rs | 100 ++++++++++++++++++ 3 files changed, 102 insertions(+) create mode 120000 turbopack/crates/turbo-tasks-backend/tests/performance.rs create mode 120000 turbopack/crates/turbo-tasks-memory/tests/performance.rs create mode 100644 turbopack/crates/turbo-tasks-testing/tests/performance.rs diff --git a/turbopack/crates/turbo-tasks-backend/tests/performance.rs b/turbopack/crates/turbo-tasks-backend/tests/performance.rs new file mode 120000 index 0000000000000..23ff275bf1de5 --- /dev/null +++ b/turbopack/crates/turbo-tasks-backend/tests/performance.rs @@ -0,0 +1 @@ +../../turbo-tasks-testing/tests/performance.rs \ No newline at end of file diff --git a/turbopack/crates/turbo-tasks-memory/tests/performance.rs b/turbopack/crates/turbo-tasks-memory/tests/performance.rs new file mode 120000 index 0000000000000..23ff275bf1de5 --- /dev/null +++ b/turbopack/crates/turbo-tasks-memory/tests/performance.rs @@ -0,0 +1 @@ +../../turbo-tasks-testing/tests/performance.rs \ No newline at end of file diff --git a/turbopack/crates/turbo-tasks-testing/tests/performance.rs b/turbopack/crates/turbo-tasks-testing/tests/performance.rs new file mode 100644 index 0000000000000..5dbd561bb285d --- /dev/null +++ b/turbopack/crates/turbo-tasks-testing/tests/performance.rs @@ -0,0 +1,100 @@ +#![feature(arbitrary_self_types)] + +use std::time::Duration; + +use turbo_tasks::Vc; +use turbo_tasks_testing::{register, run, Registration}; + +static REGISTRATION: Registration = register!(); + +const COUNT1: u32 = 100; +const COUNT2: u32 = 2000; + +#[tokio::test] +async fn many_calls_to_many_children() { + run(®ISTRATION, || async { + // The first call will actually execute many_children and its children. + let start = std::time::Instant::now(); + calls_many_children(0).strongly_consistent().await?; + println!("Initial call took {:?}", start.elapsed()); + + // The second call will connect to the cached many_children, but it would be ok if that's + // not yet optimized. + let start = std::time::Instant::now(); + calls_many_children(1).strongly_consistent().await?; + println!("Second call took {:?}", start.elapsed()); + + // Susbsequent calls should be very fast. + let start = std::time::Instant::now(); + for i in 2..COUNT1 { + calls_many_children(i).strongly_consistent().await?; + } + let subsequent = start.elapsed(); + println!( + "First {} subsequent calls took {:?}", + COUNT1 - 2, + subsequent + ); + + let start = std::time::Instant::now(); + for i in COUNT1..COUNT1 * 2 - 2 { + calls_many_children(i).strongly_consistent().await?; + } + let subsequent2 = start.elapsed(); + println!( + "Another {} subsequent calls took {:?}", + COUNT1 - 2, + subsequent2 + ); + + let start = std::time::Instant::now(); + calls_many_children(COUNT1 - 1) + .strongly_consistent() + .await?; + let final_call = start.elapsed(); + println!("Final call took {:?}", final_call); + + assert!( + subsequent2 * 2 < subsequent * 3, + "Performance should not regress with more calls" + ); + + assert!( + subsequent < Duration::from_micros(100) * (COUNT1 - 2), + "Each call should be less than 100µs" + ); + + assert!( + subsequent2 < Duration::from_micros(100) * (COUNT1 - 2), + "Each call should be less than 100µs" + ); + + anyhow::Ok(()) + }) + .await + .unwrap() +} + +#[turbo_tasks::value] +struct Value { + value: u32, +} + +#[turbo_tasks::function] +async fn calls_many_children(_i: u32) -> Vc<()> { + let _ = many_children(); + Vc::cell(()) +} + +#[turbo_tasks::function] +fn many_children() -> Vc<()> { + for i in 0..COUNT2 { + let _ = many_children_inner(i); + } + Vc::cell(()) +} + +#[turbo_tasks::function] +fn many_children_inner(_i: u32) -> Vc<()> { + Vc::cell(()) +} From eb1d2e7bfb47e2c2194d4126a3965c52e48f3e5b Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Thu, 26 Sep 2024 13:35:51 +0200 Subject: [PATCH 61/81] refactor aggregation update into separate functions --- .../backend/operation/aggregation_update.rs | 733 ++++++++++-------- 1 file changed, 393 insertions(+), 340 deletions(-) diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs index d2b798279f611..7c95a8e760a54 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs @@ -258,72 +258,12 @@ impl AggregationUpdateQueue { base_aggregation_number, distance: base_effective_distance, } => { - let mut task = ctx.task(task_id); - let current = get!(task, AggregationNumber).copied().unwrap_or_default(); - // The wanted new distance is either the provided one or the old distance - let distance = base_effective_distance.map_or(current.distance, |d| d.get()); - // The base aggregation number can only increase - let base_aggregation_number = max(current.base, base_aggregation_number); - let old = current.effective; - // The new target effecive aggregation number is base + distance - let aggregation_number = base_aggregation_number.saturating_add(distance); - if old >= aggregation_number { - if base_aggregation_number != current.base && distance != current.distance { - task.insert(CachedDataItem::AggregationNumber { - value: AggregationNumber { - base: base_aggregation_number, - distance, - effective: old, - }, - }); - } - } else { - task.insert(CachedDataItem::AggregationNumber { - value: AggregationNumber { - base: base_aggregation_number, - distance, - effective: aggregation_number, - }, - }); - - if !is_aggregating_node(old) && is_aggregating_node(aggregation_number) { - // When converted from leaf to aggregating node, all children become - // followers - let children: Vec<_> = get_many!(task, Child { task } => task); - for child_id in children { - task.add_new(CachedDataItem::Follower { - task: child_id, - value: 1, - }); - } - } - - if is_aggregating_node(aggregation_number) { - // followers might become inner nodes when the aggregation number is - // increased - let followers = - iter_many!(task, Follower { task } count if count > 0 => task); - for follower_id in followers { - self.push(AggregationUpdateJob::BalanceEdge { - upper_id: task_id, - task_id: follower_id, - }); - } - let uppers = iter_uppers(&task); - for upper_id in uppers { - self.push(AggregationUpdateJob::BalanceEdge { upper_id, task_id }); - } - } else { - let children = iter_many!(task, Child { task } => task); - for child_id in children { - self.push(AggregationUpdateJob::UpdateAggregationNumber { - task_id: child_id, - base_aggregation_number: aggregation_number + 1, - distance: None, - }); - } - } - } + self.update_aggregation_number( + ctx, + task_id, + base_effective_distance, + base_aggregation_number, + ); } AggregationUpdateJob::InnerHasNewFollowers { upper_ids, @@ -351,85 +291,10 @@ impl AggregationUpdateQueue { } } AggregationUpdateJob::InnerHasNewFollower { - mut upper_ids, + upper_ids, new_follower_id, } => { - let follower_aggregation_number = { - let follower = ctx.task(new_follower_id); - get_aggregation_number(&follower) - }; - let mut upper_ids_as_follower = Vec::new(); - upper_ids.retain(|&upper_id| { - let upper = ctx.task(upper_id); - // decide if it should be an inner or follower - let upper_aggregation_number = get_aggregation_number(&upper); - - if !is_root_node(upper_aggregation_number) - && upper_aggregation_number <= follower_aggregation_number - { - // It's a follower of the upper node - upper_ids_as_follower.push(upper_id); - false - } else { - // It's an inner node, continue with the list - true - } - }); - if !upper_ids.is_empty() { - let mut follower = ctx.task(new_follower_id); - upper_ids.retain(|&upper_id| { - if update_count!(follower, Upper { task: upper_id }, 1) { - // It's a new upper - true - } else { - // It's already an upper - false - } - }); - if !upper_ids.is_empty() { - let data = AggregatedDataUpdate::from_task(&mut follower); - let children: Vec<_> = get_followers(&follower); - drop(follower); - - if !data.is_empty() { - for upper_id in upper_ids.iter() { - // add data to upper - let mut upper = ctx.task(*upper_id); - let diff = data.apply(&mut upper, self); - if !diff.is_empty() { - let upper_ids = get_uppers(&upper); - self.push(AggregationUpdateJob::AggregatedDataUpdate { - upper_ids, - update: diff, - }) - } - } - } - if !children.is_empty() { - self.push(AggregationUpdateJob::InnerHasNewFollowers { - upper_ids: upper_ids.clone(), - new_follower_ids: children, - }); - } - } else { - drop(follower); - } - } - for upper_id in upper_ids_as_follower { - let mut upper = ctx.task(upper_id); - if update_count!( - upper, - Follower { - task: new_follower_id - }, - 1 - ) { - self.push(AggregationUpdateJob::InnerHasNewFollower { - upper_ids: vec![upper_id], - new_follower_id, - }) - } - } + self.inner_has_new_follower(ctx, new_follower_id, upper_ids); } AggregationUpdateJob::InnerLostFollowers { upper_ids, @@ -457,223 +322,411 @@ impl AggregationUpdateQueue { } } AggregationUpdateJob::InnerLostFollower { - mut upper_ids, + upper_ids, lost_follower_id, } => { - let mut follower = ctx.task(lost_follower_id); - let mut follower_in_upper_ids = Vec::new(); - upper_ids.retain(|&upper_id| { - let mut keep_upper = false; - update!(follower, Upper { task: upper_id }, |old| { - let Some(old) = old else { - follower_in_upper_ids.push(upper_id); - return None; - }; - if old < 0 { - follower_in_upper_ids.push(upper_id); - return Some(old); - } - if old == 1 { - keep_upper = true; - return None; - } - Some(old - 1) - }); - keep_upper - }); - if !upper_ids.is_empty() { - let data = AggregatedDataUpdate::from_task(&mut follower).invert(); - let followers: Vec<_> = get_followers(&follower); - drop(follower); - - if !data.is_empty() { - for upper_id in upper_ids.iter() { - // remove data from upper - let mut upper = ctx.task(*upper_id); - let diff = data.apply(&mut upper, self); - if !diff.is_empty() { - let upper_ids = get_uppers(&upper); - self.push(AggregationUpdateJob::AggregatedDataUpdate { - upper_ids, - update: diff, - }) - } - } + self.inner_lost_follower(ctx, lost_follower_id, upper_ids); + } + AggregationUpdateJob::AggregatedDataUpdate { upper_ids, update } => { + self.stats.data_update += 1; + self.aggregated_data_update(upper_ids, ctx, update); + } + AggregationUpdateJob::FindAndScheduleDirty { task_ids } => { + self.stats.find_and_schedule_dirty += 1; + self.find_and_schedule_dirty(task_ids, ctx); + } + AggregationUpdateJob::BalanceEdge { upper_id, task_id } => { + self.stats.balance += 1; + self.balance_edge(ctx, upper_id, task_id); + } + } + } + + self.jobs.is_empty() + } + + fn balance_edge(&mut self, ctx: &ExecuteContext, upper_id: TaskId, task_id: TaskId) { + let (mut upper, mut task) = ctx.task_pair(upper_id, task_id, TaskDataCategory::Meta); + let upper_aggregation_number = get_aggregation_number(&upper); + let task_aggregation_number = get_aggregation_number(&task); + + let should_be_inner = is_root_node(upper_aggregation_number) + || upper_aggregation_number > task_aggregation_number; + let should_be_follower = task_aggregation_number > upper_aggregation_number; + + if should_be_inner { + // remove all follower edges + let count = remove!(upper, Follower { task: task_id }).unwrap_or_default(); + match count.cmp(&0) { + std::cmp::Ordering::Less => upper.add_new(CachedDataItem::Follower { + task: task_id, + value: count, + }), + std::cmp::Ordering::Greater => { + let upper_ids = get_uppers(&upper); + + // Add the same amount of upper edges + if update_count!(task, Upper { task: upper_id }, count) { + // When this is a new inner node, update aggregated data and + // followers + let data = AggregatedDataUpdate::from_task(&mut task); + let followers = get_followers(&task); + let diff = data.apply(&mut upper, self); + + if !upper_ids.is_empty() && !diff.is_empty() { + // Notify uppers about changed aggregated data + self.push(AggregationUpdateJob::AggregatedDataUpdate { + upper_ids: upper_ids.clone(), + update: diff, + }); } if !followers.is_empty() { - self.push(AggregationUpdateJob::InnerLostFollowers { - upper_ids: upper_ids.clone(), - lost_follower_ids: followers, + self.push(AggregationUpdateJob::InnerHasNewFollowers { + upper_ids: vec![upper_id], + new_follower_ids: followers, }); } - } else { - drop(follower); } - for upper_id in follower_in_upper_ids { - let mut upper = ctx.task(upper_id); - if update_count!( - upper, - Follower { - task: lost_follower_id - }, - -1 - ) { - let upper_ids = get_uppers(&upper); - self.push(AggregationUpdateJob::InnerLostFollower { - upper_ids, - lost_follower_id, - }) - } + // notify uppers about lost follower + if !upper_ids.is_empty() { + self.push(AggregationUpdateJob::InnerLostFollower { + upper_ids, + lost_follower_id: task_id, + }); } } - AggregationUpdateJob::AggregatedDataUpdate { upper_ids, update } => { - for upper_id in upper_ids { - let mut upper = ctx.task(upper_id); - let diff = update.apply(&mut upper, self); - if !diff.is_empty() { - let upper_ids = get_uppers(&upper); - if !upper_ids.is_empty() { - self.push(AggregationUpdateJob::AggregatedDataUpdate { - upper_ids, - update: diff, - }); - } - } + std::cmp::Ordering::Equal => {} + } + } else if should_be_follower { + // Remove the upper edge + let count = remove!(task, Upper { task: upper_id }).unwrap_or_default(); + if count > 0 { + let upper_ids: Vec<_> = get_uppers(&upper); + + // Add the same amount of follower edges + if update_count!(upper, Follower { task: task_id }, count) { + // notify uppers about new follower + if !upper_ids.is_empty() { + self.push(AggregationUpdateJob::InnerHasNewFollower { + upper_ids: upper_ids.clone(), + new_follower_id: task_id, + }); } } - AggregationUpdateJob::FindAndScheduleDirty { mut task_ids } => { - let popped = task_ids.pop(); - if !task_ids.is_empty() { - self.push(AggregationUpdateJob::FindAndScheduleDirty { task_ids }); - } - if let Some(task_id) = popped { - let mut task = ctx.task(task_id); - #[allow(clippy::collapsible_if, reason = "readablility")] - if task.has_key(&CachedDataItemKey::Dirty {}) { - let description = ctx.backend.get_task_desc_fn(task_id); - if task.add(CachedDataItem::new_scheduled(description)) { - ctx.turbo_tasks.schedule(task_id); - } - } - if is_aggregating_node(get_aggregation_number(&task)) { - if !task.has_key(&CachedDataItemKey::AggregateRoot {}) { - task.insert(CachedDataItem::AggregateRoot { - value: RootState::new(ActiveType::CachedActiveUntilClean), - }); - } - let dirty_containers: Vec<_> = get_many!(task, AggregatedDirtyContainer { task } count if count > 0 => task); - if !dirty_containers.is_empty() { - self.push(AggregationUpdateJob::FindAndScheduleDirty { - task_ids: dirty_containers, - }); - } - } + + // Since this is no longer an inner node, update the aggregated data and + // followers + let data = AggregatedDataUpdate::from_task(&mut task).invert(); + let followers = get_followers(&task); + let diff = data.apply(&mut upper, self); + if !upper_ids.is_empty() && !diff.is_empty() { + self.push(AggregationUpdateJob::AggregatedDataUpdate { + upper_ids: upper_ids.clone(), + update: diff, + }); + } + if !followers.is_empty() { + self.push(AggregationUpdateJob::InnerLostFollowers { + upper_ids: vec![upper_id], + lost_follower_ids: followers, + }); + } + } + } else { + // both nodes have the same aggregation number + // We need to change the aggregation number of the task + let current = get!(task, AggregationNumber).copied().unwrap_or_default(); + self.push(AggregationUpdateJob::UpdateAggregationNumber { + task_id, + base_aggregation_number: current.base + 1, + distance: None, + }); + } + } + + fn find_and_schedule_dirty(&mut self, mut task_ids: Vec, ctx: &ExecuteContext) { + let popped = task_ids.pop(); + if !task_ids.is_empty() { + self.push(AggregationUpdateJob::FindAndScheduleDirty { task_ids }); + } + if let Some(task_id) = popped { + let mut task = ctx.task(task_id, TaskDataCategory::Meta); + #[allow(clippy::collapsible_if, reason = "readablility")] + if task.has_key(&CachedDataItemKey::Dirty {}) { + let description = ctx.backend.get_task_desc_fn(task_id); + if task.add(CachedDataItem::new_scheduled(description)) { + ctx.turbo_tasks.schedule(task_id); + } + } + if is_aggregating_node(get_aggregation_number(&task)) { + if !task.has_key(&CachedDataItemKey::AggregateRoot {}) { + task.insert(CachedDataItem::AggregateRoot { + value: RootState::new(ActiveType::CachedActiveUntilClean), + }); + } + let dirty_containers: Vec<_> = + get_many!(task, AggregatedDirtyContainer { task } count if count > 0 => task); + if !dirty_containers.is_empty() { + self.push(AggregationUpdateJob::FindAndScheduleDirty { + task_ids: dirty_containers, + }); + } + } + } + } + + fn aggregated_data_update( + &mut self, + upper_ids: Vec, + ctx: &ExecuteContext, + update: AggregatedDataUpdate, + ) { + for upper_id in upper_ids { + let mut upper = ctx.task(upper_id, TaskDataCategory::Meta); + let diff = update.apply(&mut upper, self); + if !diff.is_empty() { + let upper_ids = get_uppers(&upper); + if !upper_ids.is_empty() { + self.push(AggregationUpdateJob::AggregatedDataUpdate { + upper_ids, + update: diff, + }); + } + } + } + } + + fn inner_lost_follower( + &mut self, + ctx: &ExecuteContext, + lost_follower_id: TaskId, + mut upper_ids: Vec, + ) { + let mut follower = ctx.task(lost_follower_id, TaskDataCategory::Meta); + let mut follower_in_upper_ids = Vec::new(); + upper_ids.retain(|&upper_id| { + let mut keep_upper = false; + update!(follower, Upper { task: upper_id }, |old| { + let Some(old) = old else { + follower_in_upper_ids.push(upper_id); + return None; + }; + if old < 0 { + follower_in_upper_ids.push(upper_id); + return Some(old); + } + if old == 1 { + keep_upper = true; + return None; + } + Some(old - 1) + }); + keep_upper + }); + if !upper_ids.is_empty() { + let data = AggregatedDataUpdate::from_task(&mut follower).invert(); + let followers: Vec<_> = get_followers(&follower); + drop(follower); + + if !data.is_empty() { + for upper_id in upper_ids.iter() { + // remove data from upper + let mut upper = ctx.task(*upper_id, TaskDataCategory::Meta); + let diff = data.apply(&mut upper, self); + if !diff.is_empty() { + let upper_ids = get_uppers(&upper); + self.push(AggregationUpdateJob::AggregatedDataUpdate { + upper_ids, + update: diff, + }) } } - AggregationUpdateJob::BalanceEdge { upper_id, task_id } => { - let (mut upper, mut task) = ctx.task_pair(upper_id, task_id); - let upper_aggregation_number = get_aggregation_number(&upper); - let task_aggregation_number = get_aggregation_number(&task); - - let should_be_inner = is_root_node(upper_aggregation_number) - || upper_aggregation_number > task_aggregation_number; - let should_be_follower = task_aggregation_number > upper_aggregation_number; - - if should_be_inner { - // remove all follower edges - let count = remove!(upper, Follower { task: task_id }).unwrap_or_default(); - match count.cmp(&0) { - std::cmp::Ordering::Less => upper.add_new(CachedDataItem::Follower { - task: task_id, - value: count, - }), - std::cmp::Ordering::Greater => { - let upper_ids = get_uppers(&upper); - - // Add the same amount of upper edges - if update_count!(task, Upper { task: upper_id }, count) { - // When this is a new inner node, update aggregated data and - // followers - let data = AggregatedDataUpdate::from_task(&mut task); - let followers = get_followers(&task); - let diff = data.apply(&mut upper, self); - - if !upper_ids.is_empty() && !diff.is_empty() { - // Notify uppers about changed aggregated data - self.push(AggregationUpdateJob::AggregatedDataUpdate { - upper_ids: upper_ids.clone(), - update: diff, - }); - } - if !followers.is_empty() { - self.push(AggregationUpdateJob::InnerHasNewFollowers { - upper_ids: vec![upper_id], - new_follower_ids: followers, - }); - } - } - - // notify uppers about lost follower - if !upper_ids.is_empty() { - self.push(AggregationUpdateJob::InnerLostFollower { - upper_ids, - lost_follower_id: task_id, - }); - } - } - std::cmp::Ordering::Equal => {} - } - } else if should_be_follower { - // Remove the upper edge - let count = remove!(task, Upper { task: upper_id }).unwrap_or_default(); - if count > 0 { - let upper_ids: Vec<_> = get_uppers(&upper); - - // Add the same amount of follower edges - if update_count!(upper, Follower { task: task_id }, count) { - // notify uppers about new follower - if !upper_ids.is_empty() { - self.push(AggregationUpdateJob::InnerHasNewFollower { - upper_ids: upper_ids.clone(), - new_follower_id: task_id, - }); - } - } - - // Since this is no longer an inner node, update the aggregated data and - // followers - let data = AggregatedDataUpdate::from_task(&mut task).invert(); - let followers = get_followers(&task); - let diff = data.apply(&mut upper, self); - if !upper_ids.is_empty() && !diff.is_empty() { - self.push(AggregationUpdateJob::AggregatedDataUpdate { - upper_ids: upper_ids.clone(), - update: diff, - }); - } - if !followers.is_empty() { - self.push(AggregationUpdateJob::InnerLostFollowers { - upper_ids: vec![upper_id], - lost_follower_ids: followers, - }); - } + } + if !followers.is_empty() { + self.push(AggregationUpdateJob::InnerLostFollowers { + upper_ids: upper_ids.clone(), + lost_follower_ids: followers, + }); + } + } else { + drop(follower); + } + + for upper_id in follower_in_upper_ids { + let mut upper = ctx.task(upper_id, TaskDataCategory::Meta); + if update_count!( + upper, + Follower { + task: lost_follower_id + }, + -1 + ) { + let upper_ids = get_uppers(&upper); + self.push(AggregationUpdateJob::InnerLostFollower { + upper_ids, + lost_follower_id, + }) + } + } + } + + fn inner_has_new_follower( + &mut self, + ctx: &ExecuteContext, + new_follower_id: TaskId, + mut upper_ids: Vec, + ) { + let follower_aggregation_number = { + let follower = ctx.task(new_follower_id, TaskDataCategory::Meta); + get_aggregation_number(&follower) + }; + let mut upper_ids_as_follower = Vec::new(); + upper_ids.retain(|&upper_id| { + let upper = ctx.task(upper_id, TaskDataCategory::Meta); + // decide if it should be an inner or follower + let upper_aggregation_number = get_aggregation_number(&upper); + + if !is_root_node(upper_aggregation_number) + && upper_aggregation_number <= follower_aggregation_number + { + // It's a follower of the upper node + upper_ids_as_follower.push(upper_id); + false + } else { + // It's an inner node, continue with the list + true + } + }); + if !upper_ids.is_empty() { + let mut follower = ctx.task(new_follower_id, TaskDataCategory::Meta); + upper_ids.retain(|&upper_id| { + if update_count!(follower, Upper { task: upper_id }, 1) { + // It's a new upper + true + } else { + // It's already an upper + false + } + }); + if !upper_ids.is_empty() { + let data = AggregatedDataUpdate::from_task(&mut follower); + let children: Vec<_> = get_followers(&follower); + drop(follower); + + if !data.is_empty() { + for upper_id in upper_ids.iter() { + // add data to upper + let mut upper = ctx.task(*upper_id, TaskDataCategory::Meta); + let diff = data.apply(&mut upper, self); + if !diff.is_empty() { + let upper_ids = get_uppers(&upper); + self.push(AggregationUpdateJob::AggregatedDataUpdate { + upper_ids, + update: diff, + }) } - } else { - // both nodes have the same aggregation number - // We need to change the aggregation number of the task - let current = get!(task, AggregationNumber).copied().unwrap_or_default(); - self.push(AggregationUpdateJob::UpdateAggregationNumber { - task_id, - base_aggregation_number: current.base + 1, - distance: None, - }); } } + if !children.is_empty() { + self.push(AggregationUpdateJob::InnerHasNewFollowers { + upper_ids: upper_ids.clone(), + new_follower_ids: children, + }); + } + } else { + drop(follower); } } + for upper_id in upper_ids_as_follower { + let mut upper = ctx.task(upper_id, TaskDataCategory::Meta); + if update_count!( + upper, + Follower { + task: new_follower_id + }, + 1 + ) { + self.push(AggregationUpdateJob::InnerHasNewFollower { + upper_ids: vec![upper_id], + new_follower_id, + }) + } + } + } - self.jobs.is_empty() + fn update_aggregation_number( + &mut self, + ctx: &ExecuteContext, + task_id: TaskId, + base_effective_distance: Option>, + base_aggregation_number: u32, + ) { + let mut task = ctx.task(task_id, TaskDataCategory::Meta); + let current = get!(task, AggregationNumber).copied().unwrap_or_default(); + // The wanted new distance is either the provided one or the old distance + let distance = base_effective_distance.map_or(current.distance, |d| d.get()); + // The base aggregation number can only increase + let base_aggregation_number = max(current.base, base_aggregation_number); + let old = current.effective; + // The new target effecive aggregation number is base + distance + let aggregation_number = base_aggregation_number.saturating_add(distance); + if old >= aggregation_number { + if base_aggregation_number != current.base && distance != current.distance { + task.insert(CachedDataItem::AggregationNumber { + value: AggregationNumber { + base: base_aggregation_number, + distance, + effective: old, + }, + }); + } + } else { + task.insert(CachedDataItem::AggregationNumber { + value: AggregationNumber { + base: base_aggregation_number, + distance, + effective: aggregation_number, + }, + }); + + if !is_aggregating_node(old) && is_aggregating_node(aggregation_number) { + // When converted from leaf to aggregating node, all children become + // followers + let children: Vec<_> = get_many!(task, Child { task } => task); + for child_id in children { + task.add_new(CachedDataItem::Follower { + task: child_id, + value: 1, + }); + } + } + + if is_aggregating_node(aggregation_number) { + // followers might become inner nodes when the aggregation number is + // increased + let followers = iter_many!(task, Follower { task } count if count > 0 => task); + for follower_id in followers { + self.push(AggregationUpdateJob::BalanceEdge { + upper_id: task_id, + task_id: follower_id, + }); + } + let uppers = iter_uppers(&task); + for upper_id in uppers { + self.push(AggregationUpdateJob::BalanceEdge { upper_id, task_id }); + } + } else { + let children = iter_many!(task, Child { task } => task); + for child_id in children { + self.push(AggregationUpdateJob::UpdateAggregationNumber { + task_id: child_id, + base_aggregation_number: aggregation_number + 1, + distance: None, + }); + } + } + } } } From 3c640b37724f6d5db1d2c3c160aa6118b26325c0 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Thu, 26 Sep 2024 13:37:55 +0200 Subject: [PATCH 62/81] more efficent new follower handling --- .../backend/operation/aggregation_update.rs | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs index 7c95a8e760a54..66be1d6d0049d 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs @@ -638,20 +638,21 @@ impl AggregationUpdateQueue { drop(follower); } } - for upper_id in upper_ids_as_follower { - let mut upper = ctx.task(upper_id, TaskDataCategory::Meta); - if update_count!( + upper_ids_as_follower.retain(|&upper_id| { + let upper = ctx.task(upper_id, TaskDataCategory::Meta); + update_count!( upper, Follower { task: new_follower_id }, 1 - ) { - self.push(AggregationUpdateJob::InnerHasNewFollower { - upper_ids: vec![upper_id], - new_follower_id, - }) - } + ) + }); + if !upper_ids_as_follower.is_empty() { + self.push(AggregationUpdateJob::InnerHasNewFollower { + upper_ids: upper_ids_as_follower, + new_follower_id, + }); } } From 3bb7cabfd6c97100fff9798515f2e758ecf254b2 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Thu, 26 Sep 2024 14:06:49 +0200 Subject: [PATCH 63/81] more efficient new follower handling --- .../backend/operation/aggregation_update.rs | 200 +++++++++++++++--- .../src/backend/operation/connect_child.rs | 4 +- 2 files changed, 171 insertions(+), 33 deletions(-) diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs index 66be1d6d0049d..3bc76187f556a 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs @@ -55,11 +55,15 @@ pub enum AggregationUpdateJob { base_aggregation_number: u32, distance: Option, }, - InnerHasNewFollower { + InnerOfUppersHasNewFollower { upper_ids: Vec, new_follower_id: TaskId, }, - InnerHasNewFollowers { + InnerOfUpperHasNewFollowers { + upper_id: TaskId, + new_follower_ids: Vec, + }, + InnerOfUppersHasNewFollowers { upper_ids: Vec, new_follower_ids: Vec, }, @@ -265,36 +269,71 @@ impl AggregationUpdateQueue { base_aggregation_number, ); } - AggregationUpdateJob::InnerHasNewFollowers { - upper_ids, + AggregationUpdateJob::InnerOfUppersHasNewFollowers { + mut upper_ids, mut new_follower_ids, } => { - if let Some(new_follower_id) = new_follower_ids.pop() { - if new_follower_ids.is_empty() { - self.jobs - .push_front(AggregationUpdateJob::InnerHasNewFollower { - upper_ids, - new_follower_id, - }); - } else { - self.jobs - .push_front(AggregationUpdateJob::InnerHasNewFollowers { - upper_ids: upper_ids.clone(), - new_follower_ids, - }); - self.jobs - .push_front(AggregationUpdateJob::InnerHasNewFollower { - upper_ids, - new_follower_id, - }); + if upper_ids.len() > new_follower_ids.len() { + if let Some(new_follower_id) = new_follower_ids.pop() { + if new_follower_ids.is_empty() { + self.jobs.push_front( + AggregationUpdateJob::InnerOfUppersHasNewFollower { + upper_ids, + new_follower_id, + }, + ); + } else { + self.jobs.push_front( + AggregationUpdateJob::InnerOfUppersHasNewFollowers { + upper_ids: upper_ids.clone(), + new_follower_ids, + }, + ); + self.jobs.push_front( + AggregationUpdateJob::InnerOfUppersHasNewFollower { + upper_ids, + new_follower_id, + }, + ); + } + } + } else { + if let Some(upper_id) = upper_ids.pop() { + if upper_ids.is_empty() { + self.jobs.push_front( + AggregationUpdateJob::InnerOfUpperHasNewFollowers { + upper_id, + new_follower_ids, + }, + ); + } else { + self.jobs.push_front( + AggregationUpdateJob::InnerOfUppersHasNewFollowers { + upper_ids, + new_follower_ids: new_follower_ids.clone(), + }, + ); + self.jobs.push_front( + AggregationUpdateJob::InnerOfUpperHasNewFollowers { + upper_id, + new_follower_ids, + }, + ); + } } } } - AggregationUpdateJob::InnerHasNewFollower { + AggregationUpdateJob::InnerOfUppersHasNewFollower { upper_ids, new_follower_id, } => { - self.inner_has_new_follower(ctx, new_follower_id, upper_ids); + self.inner_of_uppers_has_new_follower(ctx, new_follower_id, upper_ids); + } + AggregationUpdateJob::InnerOfUpperHasNewFollowers { + upper_id, + new_follower_ids, + } => { + self.inner_of_upper_has_new_followers(ctx, new_follower_ids, upper_id); } AggregationUpdateJob::InnerLostFollowers { upper_ids, @@ -381,8 +420,8 @@ impl AggregationUpdateQueue { }); } if !followers.is_empty() { - self.push(AggregationUpdateJob::InnerHasNewFollowers { - upper_ids: vec![upper_id], + self.push(AggregationUpdateJob::InnerOfUpperHasNewFollowers { + upper_id, new_follower_ids: followers, }); } @@ -408,7 +447,7 @@ impl AggregationUpdateQueue { if update_count!(upper, Follower { task: task_id }, count) { // notify uppers about new follower if !upper_ids.is_empty() { - self.push(AggregationUpdateJob::InnerHasNewFollower { + self.push(AggregationUpdateJob::InnerOfUppersHasNewFollower { upper_ids: upper_ids.clone(), new_follower_id: task_id, }); @@ -571,7 +610,7 @@ impl AggregationUpdateQueue { } } - fn inner_has_new_follower( + fn inner_of_uppers_has_new_follower( &mut self, ctx: &ExecuteContext, new_follower_id: TaskId, @@ -629,7 +668,7 @@ impl AggregationUpdateQueue { } } if !children.is_empty() { - self.push(AggregationUpdateJob::InnerHasNewFollowers { + self.push(AggregationUpdateJob::InnerOfUppersHasNewFollowers { upper_ids: upper_ids.clone(), new_follower_ids: children, }); @@ -639,7 +678,7 @@ impl AggregationUpdateQueue { } } upper_ids_as_follower.retain(|&upper_id| { - let upper = ctx.task(upper_id, TaskDataCategory::Meta); + let mut upper = ctx.task(upper_id, TaskDataCategory::Meta); update_count!( upper, Follower { @@ -649,13 +688,112 @@ impl AggregationUpdateQueue { ) }); if !upper_ids_as_follower.is_empty() { - self.push(AggregationUpdateJob::InnerHasNewFollower { + self.push(AggregationUpdateJob::InnerOfUppersHasNewFollower { upper_ids: upper_ids_as_follower, new_follower_id, }); } } + fn inner_of_upper_has_new_followers( + &mut self, + ctx: &ExecuteContext, + new_follower_ids: Vec, + upper_id: TaskId, + ) { + let mut followers_with_aggregation_number = new_follower_ids + .into_iter() + .map(|new_follower_id| { + let follower = ctx.task(new_follower_id, TaskDataCategory::Meta); + (new_follower_id, get_aggregation_number(&follower)) + }) + .collect::>(); + + let mut followers_of_upper = Vec::new(); + { + let upper = ctx.task(upper_id, TaskDataCategory::Meta); + // decide if it should be an inner or follower + let upper_aggregation_number = get_aggregation_number(&upper); + + if !is_root_node(upper_aggregation_number) { + followers_with_aggregation_number.retain( + |(follower_id, follower_aggregation_number)| { + if upper_aggregation_number <= *follower_aggregation_number { + // It's a follower of the upper node + followers_of_upper.push(*follower_id); + false + } else { + // It's an inner node, continue with the list + true + } + }, + ); + } + } + + let mut upper_data_updates = Vec::new(); + let mut upper_new_followers = Vec::new(); + for (follower_id, _) in followers_with_aggregation_number { + let mut follower = ctx.task(follower_id, TaskDataCategory::Meta); + if update_count!(follower, Upper { task: upper_id }, 1) { + // It's a new upper + let data = AggregatedDataUpdate::from_task(&mut follower); + let children: Vec<_> = get_followers(&follower); + drop(follower); + + if !data.is_empty() { + upper_data_updates.push(data); + } + upper_new_followers.extend(children); + } + } + if !upper_new_followers.is_empty() { + self.push(AggregationUpdateJob::InnerOfUpperHasNewFollowers { + upper_id, + new_follower_ids: upper_new_followers, + }); + } + if !upper_data_updates.is_empty() { + // add data to upper + let mut upper = ctx.task(upper_id, TaskDataCategory::Meta); + let diffs = upper_data_updates + .into_iter() + .filter_map(|data| { + let diff = data.apply(&mut upper, self); + (!diff.is_empty()).then_some(diff) + }) + .collect::>(); + let mut iter = diffs.into_iter(); + if let Some(mut diff) = iter.next() { + let upper_ids = get_uppers(&upper); + drop(upper); + // TODO merge AggregatedDataUpdate + for next_diff in iter { + self.push(AggregationUpdateJob::AggregatedDataUpdate { + upper_ids: upper_ids.clone(), + update: diff, + }); + diff = next_diff; + } + self.push(AggregationUpdateJob::AggregatedDataUpdate { + upper_ids, + update: diff, + }); + } + } + if !followers_of_upper.is_empty() { + let mut upper = ctx.task(upper_id, TaskDataCategory::Meta); + followers_of_upper + .retain(|follower_id| update_count!(upper, Follower { task: *follower_id }, 1)); + if !followers_of_upper.is_empty() { + self.push(AggregationUpdateJob::InnerOfUpperHasNewFollowers { + upper_id, + new_follower_ids: followers_of_upper, + }); + } + } + } + fn update_aggregation_number( &mut self, ctx: &ExecuteContext, diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs index a915fa776347b..1dddcf70408e1 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs @@ -93,13 +93,13 @@ impl ConnectChildOperation { }); } if is_aggregating_node { - queue.push(AggregationUpdateJob::InnerHasNewFollower { + queue.push(AggregationUpdateJob::InnerOfUppersHasNewFollower { upper_ids: vec![parent_task_id], new_follower_id: child_task_id, }); } else { let upper_ids = get_uppers(&parent_task); - queue.push(AggregationUpdateJob::InnerHasNewFollower { + queue.push(AggregationUpdateJob::InnerOfUppersHasNewFollower { upper_ids, new_follower_id: child_task_id, }); From 1eaf8dbed7d7b5a5b0b65280c18f91d5c148a45e Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Thu, 26 Sep 2024 17:27:48 +0200 Subject: [PATCH 64/81] fixup! refactor aggregation update into separate functions --- .../backend/operation/aggregation_update.rs | 37 +++++++++---------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs index 3bc76187f556a..f937b7c77816d 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs @@ -367,15 +367,12 @@ impl AggregationUpdateQueue { self.inner_lost_follower(ctx, lost_follower_id, upper_ids); } AggregationUpdateJob::AggregatedDataUpdate { upper_ids, update } => { - self.stats.data_update += 1; self.aggregated_data_update(upper_ids, ctx, update); } AggregationUpdateJob::FindAndScheduleDirty { task_ids } => { - self.stats.find_and_schedule_dirty += 1; self.find_and_schedule_dirty(task_ids, ctx); } AggregationUpdateJob::BalanceEdge { upper_id, task_id } => { - self.stats.balance += 1; self.balance_edge(ctx, upper_id, task_id); } } @@ -385,7 +382,7 @@ impl AggregationUpdateQueue { } fn balance_edge(&mut self, ctx: &ExecuteContext, upper_id: TaskId, task_id: TaskId) { - let (mut upper, mut task) = ctx.task_pair(upper_id, task_id, TaskDataCategory::Meta); + let (mut upper, mut task) = ctx.task_pair(upper_id, task_id); let upper_aggregation_number = get_aggregation_number(&upper); let task_aggregation_number = get_aggregation_number(&task); @@ -490,7 +487,7 @@ impl AggregationUpdateQueue { self.push(AggregationUpdateJob::FindAndScheduleDirty { task_ids }); } if let Some(task_id) = popped { - let mut task = ctx.task(task_id, TaskDataCategory::Meta); + let mut task = ctx.task(task_id); #[allow(clippy::collapsible_if, reason = "readablility")] if task.has_key(&CachedDataItemKey::Dirty {}) { let description = ctx.backend.get_task_desc_fn(task_id); @@ -522,7 +519,7 @@ impl AggregationUpdateQueue { update: AggregatedDataUpdate, ) { for upper_id in upper_ids { - let mut upper = ctx.task(upper_id, TaskDataCategory::Meta); + let mut upper = ctx.task(upper_id); let diff = update.apply(&mut upper, self); if !diff.is_empty() { let upper_ids = get_uppers(&upper); @@ -542,7 +539,7 @@ impl AggregationUpdateQueue { lost_follower_id: TaskId, mut upper_ids: Vec, ) { - let mut follower = ctx.task(lost_follower_id, TaskDataCategory::Meta); + let mut follower = ctx.task(lost_follower_id); let mut follower_in_upper_ids = Vec::new(); upper_ids.retain(|&upper_id| { let mut keep_upper = false; @@ -571,7 +568,7 @@ impl AggregationUpdateQueue { if !data.is_empty() { for upper_id in upper_ids.iter() { // remove data from upper - let mut upper = ctx.task(*upper_id, TaskDataCategory::Meta); + let mut upper = ctx.task(*upper_id); let diff = data.apply(&mut upper, self); if !diff.is_empty() { let upper_ids = get_uppers(&upper); @@ -593,7 +590,7 @@ impl AggregationUpdateQueue { } for upper_id in follower_in_upper_ids { - let mut upper = ctx.task(upper_id, TaskDataCategory::Meta); + let mut upper = ctx.task(upper_id); if update_count!( upper, Follower { @@ -617,12 +614,12 @@ impl AggregationUpdateQueue { mut upper_ids: Vec, ) { let follower_aggregation_number = { - let follower = ctx.task(new_follower_id, TaskDataCategory::Meta); + let follower = ctx.task(new_follower_id); get_aggregation_number(&follower) }; let mut upper_ids_as_follower = Vec::new(); upper_ids.retain(|&upper_id| { - let upper = ctx.task(upper_id, TaskDataCategory::Meta); + let upper = ctx.task(upper_id); // decide if it should be an inner or follower let upper_aggregation_number = get_aggregation_number(&upper); @@ -638,7 +635,7 @@ impl AggregationUpdateQueue { } }); if !upper_ids.is_empty() { - let mut follower = ctx.task(new_follower_id, TaskDataCategory::Meta); + let mut follower = ctx.task(new_follower_id); upper_ids.retain(|&upper_id| { if update_count!(follower, Upper { task: upper_id }, 1) { // It's a new upper @@ -656,7 +653,7 @@ impl AggregationUpdateQueue { if !data.is_empty() { for upper_id in upper_ids.iter() { // add data to upper - let mut upper = ctx.task(*upper_id, TaskDataCategory::Meta); + let mut upper = ctx.task(*upper_id); let diff = data.apply(&mut upper, self); if !diff.is_empty() { let upper_ids = get_uppers(&upper); @@ -678,7 +675,7 @@ impl AggregationUpdateQueue { } } upper_ids_as_follower.retain(|&upper_id| { - let mut upper = ctx.task(upper_id, TaskDataCategory::Meta); + let mut upper = ctx.task(upper_id); update_count!( upper, Follower { @@ -704,14 +701,14 @@ impl AggregationUpdateQueue { let mut followers_with_aggregation_number = new_follower_ids .into_iter() .map(|new_follower_id| { - let follower = ctx.task(new_follower_id, TaskDataCategory::Meta); + let follower = ctx.task(new_follower_id); (new_follower_id, get_aggregation_number(&follower)) }) .collect::>(); let mut followers_of_upper = Vec::new(); { - let upper = ctx.task(upper_id, TaskDataCategory::Meta); + let upper = ctx.task(upper_id); // decide if it should be an inner or follower let upper_aggregation_number = get_aggregation_number(&upper); @@ -734,7 +731,7 @@ impl AggregationUpdateQueue { let mut upper_data_updates = Vec::new(); let mut upper_new_followers = Vec::new(); for (follower_id, _) in followers_with_aggregation_number { - let mut follower = ctx.task(follower_id, TaskDataCategory::Meta); + let mut follower = ctx.task(follower_id); if update_count!(follower, Upper { task: upper_id }, 1) { // It's a new upper let data = AggregatedDataUpdate::from_task(&mut follower); @@ -755,7 +752,7 @@ impl AggregationUpdateQueue { } if !upper_data_updates.is_empty() { // add data to upper - let mut upper = ctx.task(upper_id, TaskDataCategory::Meta); + let mut upper = ctx.task(upper_id); let diffs = upper_data_updates .into_iter() .filter_map(|data| { @@ -782,7 +779,7 @@ impl AggregationUpdateQueue { } } if !followers_of_upper.is_empty() { - let mut upper = ctx.task(upper_id, TaskDataCategory::Meta); + let mut upper = ctx.task(upper_id); followers_of_upper .retain(|follower_id| update_count!(upper, Follower { task: *follower_id }, 1)); if !followers_of_upper.is_empty() { @@ -801,7 +798,7 @@ impl AggregationUpdateQueue { base_effective_distance: Option>, base_aggregation_number: u32, ) { - let mut task = ctx.task(task_id, TaskDataCategory::Meta); + let mut task = ctx.task(task_id); let current = get!(task, AggregationNumber).copied().unwrap_or_default(); // The wanted new distance is either the provided one or the old distance let distance = base_effective_distance.map_or(current.distance, |d| d.get()); From f9826b7d2933edc58f1ebf72b1c186c1de12e485 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Wed, 2 Oct 2024 08:49:23 +0200 Subject: [PATCH 65/81] remove todo --- turbopack/crates/turbo-tasks-backend/src/backend/storage.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs b/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs index f9bf64045df55..d6f684c25651e 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs @@ -21,11 +21,9 @@ where T::Key: Indexed, { Plain { - // TODO use FxHasher map: AutoMap, }, Indexed { - // TODO use FxHasher map: AutoMap<::Index, AutoMap>, }, } From 9e11d4536bb13dcc7df53438a7b934771fa8816c Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Fri, 9 Aug 2024 09:03:54 +0200 Subject: [PATCH 66/81] serialization fixup --- turbopack/crates/turbo-tasks/src/backend.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/turbopack/crates/turbo-tasks/src/backend.rs b/turbopack/crates/turbo-tasks/src/backend.rs index 4f771c60f283b..cb47e6ac2a8b7 100644 --- a/turbopack/crates/turbo-tasks/src/backend.rs +++ b/turbopack/crates/turbo-tasks/src/backend.rs @@ -161,8 +161,7 @@ mod ser { let value = seq .next_element_seed(seed)? .ok_or_else(|| de::Error::invalid_length(2, &self))?; - let arc: triomphe::Arc = - todo!("convert Box to Arc"); + let arc = triomphe::Arc::::from(value); Ok(TypedCellContent( value_type, CellContent(Some(SharedReference(arc))), From eb464f755312b2d45b2af24204466480d57511ba Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Wed, 2 Oct 2024 11:30:02 +0200 Subject: [PATCH 67/81] improve macro visibility, method name and long types --- .../turbo-tasks-backend/src/backend/mod.rs | 2 +- .../backend/operation/aggregation_update.rs | 2 +- .../src/backend/operation/connect_child.rs | 3 +- .../src/backend/operation/invalidate.rs | 2 +- .../src/backend/operation/update_cell.rs | 2 +- .../src/backend/operation/update_output.rs | 2 +- .../src/backend/storage.rs | 55 ++++++++++--------- 7 files changed, 34 insertions(+), 34 deletions(-) diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs index 70b92fb9629a2..bf2b7bb5320d0 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs @@ -41,12 +41,12 @@ use turbo_tasks::{ use self::{operation::ExecuteContext, storage::Storage}; use crate::{ + backend::storage::{get, get_many, remove}, data::{ ActiveType, AggregationNumber, CachedDataItem, CachedDataItemIndex, CachedDataItemKey, CachedDataItemValue, CachedDataUpdate, CellRef, InProgressCellState, InProgressState, OutputValue, RootState, }, - get, get_many, remove, utils::{bi_map::BiMap, chunked_vec::ChunkedVec, ptr_eq_arc::PtrEqArc}, }; diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs index f937b7c77816d..a2f80a4350de9 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs @@ -5,8 +5,8 @@ use turbo_tasks::TaskId; use super::{ExecuteContext, Operation, TaskGuard}; use crate::{ + backend::storage::{get, get_many, iter_many, remove, update, update_count}, data::{ActiveType, AggregationNumber, CachedDataItem, CachedDataItemKey, RootState}, - get, get_many, iter_many, remove, update, update_count, }; const LEAF_NUMBER: u32 = 16; diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs index 1dddcf70408e1..17f0efd36a7af 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs @@ -10,9 +10,8 @@ use super::{ ExecuteContext, Operation, }; use crate::{ - backend::operation::is_root_node, + backend::{operation::is_root_node, storage::get}, data::{CachedDataItem, CachedDataItemIndex, CachedDataItemKey}, - get, }; const AGGREGATION_NUMBER_BUFFER_SPACE: u32 = 2; diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/invalidate.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/invalidate.rs index b293b82df7cb1..95b07549cc488 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/invalidate.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/invalidate.rs @@ -7,8 +7,8 @@ use super::{ ExecuteContext, Operation, }; use crate::{ + backend::storage::get, data::{CachedDataItem, CachedDataItemKey}, - get, }; #[derive(Serialize, Deserialize, Clone, Default)] diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/update_cell.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/update_cell.rs index c973a991a03be..223fd9e6d6550 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/update_cell.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/update_cell.rs @@ -2,8 +2,8 @@ use turbo_tasks::{backend::CellContent, CellId, TaskId}; use super::{ExecuteContext, InvalidateOperation}; use crate::{ + backend::storage::{get_many, remove}, data::{CachedDataItem, CachedDataItemKey}, - get_many, remove, }; pub struct UpdateCellOperation; diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/update_output.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/update_output.rs index f563580b4203d..e722bc241caf8 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/update_output.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/update_output.rs @@ -5,8 +5,8 @@ use turbo_tasks::{util::SharedError, RawVc, TaskId}; use super::{ExecuteContext, InvalidateOperation}; use crate::{ + backend::storage::get_many, data::{CachedDataItem, CachedDataItemKey, CachedDataItemValue, CellRef, OutputValue}, - get_many, }; pub struct UpdateOutputOperation; diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs b/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs index d6f684c25651e..24691c44323dd 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs @@ -16,16 +16,17 @@ use crate::utils::dash_map_multi::{get_multiple_mut, RefMut}; const INDEX_THRESHOLD: usize = 1024; +type IndexedMap = AutoMap< + <::Key as Indexed>::Index, + AutoMap<::Key, ::Value>, +>; + pub enum InnerStorage where T::Key: Indexed, { - Plain { - map: AutoMap, - }, - Indexed { - map: AutoMap<::Index, AutoMap>, - }, + Plain { map: AutoMap }, + Indexed { map: IndexedMap }, } impl InnerStorage @@ -43,8 +44,7 @@ where return; }; if plain_map.len() >= INDEX_THRESHOLD { - let mut map: AutoMap<::Index, AutoMap> = - AutoMap::new(); + let mut map: IndexedMap = AutoMap::new(); for (key, value) in take(plain_map).into_iter() { let index = key.index(); map.entry(index).or_default().insert(key, value); @@ -53,7 +53,7 @@ where } } - fn map_mut(&mut self, key: &T::Key) -> &mut AutoMap { + fn get_map_mut(&mut self, key: &T::Key) -> &mut AutoMap { self.check_threshold(); match self { InnerStorage::Plain { map, .. } => map, @@ -61,7 +61,7 @@ where } } - fn map(&self, key: &T::Key) -> Option<&AutoMap> { + fn get_map(&self, key: &T::Key) -> Option<&AutoMap> { match self { InnerStorage::Plain { map, .. } => Some(map), InnerStorage::Indexed { map, .. } => map.get(&key.index()), @@ -77,7 +77,7 @@ where pub fn add(&mut self, item: T) -> bool { let (key, value) = item.into_key_and_value(); - match self.map_mut(&key).entry(key) { + match self.get_map_mut(&key).entry(key) { Entry::Occupied(_) => false, Entry::Vacant(e) => { e.insert(value); @@ -88,19 +88,19 @@ where pub fn insert(&mut self, item: T) -> Option { let (key, value) = item.into_key_and_value(); - self.map_mut(&key).insert(key, value) + self.get_map_mut(&key).insert(key, value) } pub fn remove(&mut self, key: &T::Key) -> Option { - self.map_mut(key).remove(key) + self.get_map_mut(key).remove(key) } pub fn get(&self, key: &T::Key) -> Option<&T::Value> { - self.map(key).and_then(|m| m.get(key)) + self.get_map(key).and_then(|m| m.get(key)) } pub fn has_key(&self, key: &T::Key) -> bool { - self.map(key) + self.get_map(key) .map(|m| m.contains_key(key)) .unwrap_or_default() } @@ -140,7 +140,7 @@ where key: &T::Key, update: impl FnOnce(Option) -> Option, ) { - let map = self.map_mut(key); + let map = self.get_map_mut(key); if let Some(value) = map.get_mut(key) { let v = take(value); if let Some(v) = update(Some(v)) { @@ -251,7 +251,6 @@ where } } -#[macro_export] macro_rules! get { ($task:ident, $key:ident $input:tt) => { if let Some($crate::data::CachedDataItemValue::$key { value }) = $task.get(&$crate::data::CachedDataItemKey::$key $input).as_ref() { @@ -269,7 +268,6 @@ macro_rules! get { }; } -#[macro_export] macro_rules! iter_many { ($task:ident, $key:ident $input:tt => $value:ident) => { $task @@ -297,20 +295,18 @@ macro_rules! iter_many { }; } -#[macro_export] macro_rules! get_many { ($task:ident, $key:ident $input:tt => $value:ident) => { - $crate::iter_many!($task, $key $input => $value).collect() + $crate::backend::storage::iter_many!($task, $key $input => $value).collect() }; ($task:ident, $key:ident $input:tt $value_ident:ident => $value:expr) => { - $crate::iter_many!($task, $key $input $value_ident => $value).collect() + $crate::backend::storage::iter_many!($task, $key $input $value_ident => $value).collect() }; ($task:ident, $key:ident $input:tt $value_ident:ident if $cond:expr => $value:expr) => { - $crate::iter_many!($task, $key $input $value_ident if $cond => $value).collect() + $crate::backend::storage::iter_many!($task, $key $input $value_ident if $cond => $value).collect() }; } -#[macro_export] macro_rules! update { ($task:ident, $key:ident $input:tt, $update:expr) => { #[allow(unused_mut)] @@ -344,13 +340,12 @@ macro_rules! update { }; } -#[macro_export] macro_rules! update_count { ($task:ident, $key:ident $input:tt, $update:expr) => { match $update { update => { let mut state_change = false; - $crate::update!($task, $key $input, |old: Option| { + $crate::backend::storage::update!($task, $key $input, |old: Option| { if let Some(old) = old { let new = old + update; state_change = old <= 0 && new > 0 || old > 0 && new <= 0; @@ -365,11 +360,10 @@ macro_rules! update_count { } }; ($task:ident, $key:ident, $update:expr) => { - $crate::update_count!($task, $key {}, $update) + $crate::backend::storage::update_count!($task, $key {}, $update) }; } -#[macro_export] macro_rules! remove { ($task:ident, $key:ident $input:tt) => { if let Some($crate::data::CachedDataItemValue::$key { value }) = $task.remove(&$crate::data::CachedDataItemKey::$key $input) { @@ -386,3 +380,10 @@ macro_rules! remove { } }; } + +pub(crate) use get; +pub(crate) use get_many; +pub(crate) use iter_many; +pub(crate) use remove; +pub(crate) use update; +pub(crate) use update_count; From f79cf89bba665b11ea1d98153f079a69c5068966 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Wed, 2 Oct 2024 11:46:29 +0200 Subject: [PATCH 68/81] improve macro readablility --- .../src/backend/storage.rs | 26 +++---------------- 1 file changed, 3 insertions(+), 23 deletions(-) diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs b/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs index 24691c44323dd..292cfa1d140be 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs @@ -260,11 +260,7 @@ macro_rules! get { } }; ($task:ident, $key:ident) => { - if let Some($crate::data::CachedDataItemValue::$key { value }) = $task.get(&$crate::data::CachedDataItemKey::$key {}).as_ref() { - Some(value) - } else { - None - } + $crate::backend::storage::get!($task, $key {}) }; } @@ -324,19 +320,7 @@ macro_rules! update { } }; ($task:ident, $key:ident, $update:expr) => { - #[allow(unused_mut)] - match $update { - mut update => $task.update(&$crate::data::CachedDataItemKey::$key {}, |old| { - update(old.and_then(|old| { - if let $crate::data::CachedDataItemValue::$key { value } = old { - Some(value) - } else { - None - } - })) - .map(|new| $crate::data::CachedDataItemValue::$key { value: new }) - }) - } + $crate::backend::storage::update!($task, $key {}, $update) }; } @@ -373,11 +357,7 @@ macro_rules! remove { } }; ($task:ident, $key:ident) => { - if let Some($crate::data::CachedDataItemValue::$key { value }) = $task.remove(&$crate::data::CachedDataItemKey::$key {}) { - Some(value) - } else { - None - } + $crate::backend::storage::remove!($task, $key {}) }; } From 6d5dee1151b5bc6d94a8bf52b04f1a1808a2881c Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Wed, 2 Oct 2024 11:46:45 +0200 Subject: [PATCH 69/81] clippy --- .../src/backend/operation/aggregation_update.rs | 6 ++++-- .../src/backend/operation/connect_child.rs | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs index a2f80a4350de9..3b2165e86cc4f 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs @@ -93,7 +93,7 @@ impl AggregationUpdateJob { task: &mut TaskGuard<'_>, update: AggregatedDataUpdate, ) -> Option { - let upper_ids: Vec<_> = get_uppers(&task); + let upper_ids: Vec<_> = get_uppers(task); if !upper_ids.is_empty() { Some(AggregationUpdateJob::AggregatedDataUpdate { upper_ids, @@ -272,7 +272,9 @@ impl AggregationUpdateQueue { AggregationUpdateJob::InnerOfUppersHasNewFollowers { mut upper_ids, mut new_follower_ids, - } => { + } => + { + #[allow(clippy::collapsible_if, reason = "readablility")] if upper_ids.len() > new_follower_ids.len() { if let Some(new_follower_id) = new_follower_ids.pop() { if new_follower_ids.is_empty() { diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs index 17f0efd36a7af..c16c2a61be0cc 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs @@ -61,7 +61,7 @@ impl ConnectChildOperation { ) }) .count(); - let target_distance = children_count.ilog2() as u32 * 2; + let target_distance = children_count.ilog2() * 2; let parent_aggregation = current_parent_aggregation .base .saturating_add(target_distance); From a9f19c6f4f1da4dc5037df87fc8f4f8f6a6f1d9d Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Wed, 2 Oct 2024 11:47:01 +0200 Subject: [PATCH 70/81] readablilty --- turbopack/crates/turbo-tasks-backend/src/backend/mod.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs index bf2b7bb5320d0..5e2e7f0052bb0 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs @@ -66,13 +66,14 @@ impl SnapshotRequest { } } -type TransientOnceTask = +type TransientTaskOnce = Mutex> + Send + 'static>>>>; pub enum TransientTask { /// A root task that will track dependencies and re-execute when /// dependencies change. Task will eventually settle to the correct /// execution. + /// /// Always active. Automatically scheduled. Root(TransientTaskRoot), @@ -82,8 +83,9 @@ pub enum TransientTask { /// start of the task. It may or may not include invalidations that /// happened after that. It may see these invalidations partially /// applied. + /// /// Active until done. Automatically scheduled. - Once(TransientOnceTask), + Once(TransientTaskOnce), } pub struct TurboTasksBackend { @@ -152,7 +154,7 @@ impl TurboTasksBackend { } fn operation_suspend_point(&self, suspend: impl FnOnce() -> AnyOperation) { - if (self.in_progress_operations.load(Ordering::Relaxed) & SNAPSHOT_REQUESTED_BIT) != 0 { + if self.suspending_requested() { let operation = Arc::new(suspend()); let mut snapshot_request = self.snapshot_request.lock(); if snapshot_request.snapshot_requested { From 8bb77ac31cb975df422008ec1a9510489756e8fc Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Wed, 2 Oct 2024 11:52:26 +0200 Subject: [PATCH 71/81] add documentation --- turbopack/crates/turbo-tasks-backend/src/utils/bi_map.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/turbopack/crates/turbo-tasks-backend/src/utils/bi_map.rs b/turbopack/crates/turbo-tasks-backend/src/utils/bi_map.rs index e5be1c583a648..a55757d85d4b4 100644 --- a/turbopack/crates/turbo-tasks-backend/src/utils/bi_map.rs +++ b/turbopack/crates/turbo-tasks-backend/src/utils/bi_map.rs @@ -2,6 +2,10 @@ use std::{borrow::Borrow, hash::Hash}; use dashmap::{mapref::entry::Entry, DashMap}; +/// A bidirectional [`DashMap`] that allows lookup by key or value. +/// +/// As keys and values are stored twice, they should be small types, such as +/// [`Arc`][`std::sync::Arc`]. pub struct BiMap { forward: DashMap, reverse: DashMap, From 1c6566911ad9fdbbe1d4d031bf884f2fd9c58a8d Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Wed, 2 Oct 2024 12:15:12 +0200 Subject: [PATCH 72/81] review cleanups --- .../src/utils/dash_map_multi.rs | 14 ++++++-------- .../turbo-tasks-backend/src/utils/ptr_eq_arc.rs | 2 +- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/turbopack/crates/turbo-tasks-backend/src/utils/dash_map_multi.rs b/turbopack/crates/turbo-tasks-backend/src/utils/dash_map_multi.rs index 8f886625c965f..042b7f971760b 100644 --- a/turbopack/crates/turbo-tasks-backend/src/utils/dash_map_multi.rs +++ b/turbopack/crates/turbo-tasks-backend/src/utils/dash_map_multi.rs @@ -13,14 +13,12 @@ where { Base(dashmap::mapref::one::RefMut<'a, K, V, S>), Simple { - #[allow(dead_code)] - guard: RwLockWriteGuard<'a, HashMap, S>>, + _guard: RwLockWriteGuard<'a, HashMap, S>>, key: *const K, value: *mut V, }, Shared { - #[allow(dead_code)] - guard: Arc, S>>>, + _guard: Arc, S>>>, key: *const K, value: *mut V, }, @@ -125,12 +123,12 @@ where let guard = Arc::new(guard); ( RefMut::Shared { - guard: guard.clone(), + _guard: guard.clone(), key: key1_ptr, value: value1_ptr, }, RefMut::Shared { - guard, + _guard: guard, key: key2_ptr, value: value2_ptr, }, @@ -164,12 +162,12 @@ where let value2 = e2.1.get_mut() as *mut V; ( RefMut::Simple { - guard: guard1, + _guard: guard1, key: key1, value: value1, }, RefMut::Simple { - guard: guard2, + _guard: guard2, key: key2, value: value2, }, diff --git a/turbopack/crates/turbo-tasks-backend/src/utils/ptr_eq_arc.rs b/turbopack/crates/turbo-tasks-backend/src/utils/ptr_eq_arc.rs index 87543c0399031..7889de80e05ee 100644 --- a/turbopack/crates/turbo-tasks-backend/src/utils/ptr_eq_arc.rs +++ b/turbopack/crates/turbo-tasks-backend/src/utils/ptr_eq_arc.rs @@ -19,7 +19,7 @@ impl From> for PtrEqArc { } impl Deref for PtrEqArc { - type Target = T; + type Target = Arc; fn deref(&self) -> &Self::Target { &self.0 From d5ec3bf0e381d2b2a01608ac2d2763a0ebc69e9f Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Wed, 2 Oct 2024 12:15:24 +0200 Subject: [PATCH 73/81] add documentation --- turbopack/crates/turbo-tasks-macros/src/lib.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/turbopack/crates/turbo-tasks-macros/src/lib.rs b/turbopack/crates/turbo-tasks-macros/src/lib.rs index 35dc25c0c0d68..613b2da8ff789 100644 --- a/turbopack/crates/turbo-tasks-macros/src/lib.rs +++ b/turbopack/crates/turbo-tasks-macros/src/lib.rs @@ -47,6 +47,11 @@ pub fn derive_task_input(input: TokenStream) -> TokenStream { derive::derive_task_input(input) } +/// Derives the `turbo_tasks::KeyValuePair` trait for a enum. Each variant need to have a `value` +/// field which becomes part of the value enum and all remaining fields become part of the key. +/// Assuming the enum is called `Abc` it exposes `AbcKey` and `AbcValue` types for it too. The key +/// enum will have `Debug, Clone, PartialEq, Eq, Hash` derived and the value enum will have `Debug, +/// Clone` derived. It's expected that all fields implement these traits. #[proc_macro_derive(KeyValuePair)] pub fn derive_key_value_pair(input: TokenStream) -> TokenStream { derive::derive_key_value_pair(input) From 47c19dc2d3e2639c374d4679abfae41a90dac05a Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Wed, 2 Oct 2024 12:26:48 +0200 Subject: [PATCH 74/81] use BITS --- turbopack/crates/turbo-tasks-backend/src/backend/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs index 5e2e7f0052bb0..ea9098200ef7b 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs @@ -50,7 +50,7 @@ use crate::{ utils::{bi_map::BiMap, chunked_vec::ChunkedVec, ptr_eq_arc::PtrEqArc}, }; -const SNAPSHOT_REQUESTED_BIT: usize = 1 << 63; +const SNAPSHOT_REQUESTED_BIT: usize = 1 << (usize::BITS - 1); struct SnapshotRequest { snapshot_requested: bool, From 7308a3641cb62cc9f3cff3fc62dad8cc70e02d4f Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Wed, 2 Oct 2024 12:29:51 +0200 Subject: [PATCH 75/81] clippy --- .../src/backend/operation/aggregation_update.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs index 3b2165e86cc4f..b9b299ac868b2 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs @@ -272,9 +272,7 @@ impl AggregationUpdateQueue { AggregationUpdateJob::InnerOfUppersHasNewFollowers { mut upper_ids, mut new_follower_ids, - } => - { - #[allow(clippy::collapsible_if, reason = "readablility")] + } => { if upper_ids.len() > new_follower_ids.len() { if let Some(new_follower_id) = new_follower_ids.pop() { if new_follower_ids.is_empty() { @@ -300,6 +298,7 @@ impl AggregationUpdateQueue { } } } else { + #[allow(clippy::collapsible_if, reason = "readablility")] if let Some(upper_id) = upper_ids.pop() { if upper_ids.is_empty() { self.jobs.push_front( From a4454519a52c1d1836df25fe82da9e4d444519ba Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Wed, 2 Oct 2024 21:04:39 +0200 Subject: [PATCH 76/81] comment --- turbopack/crates/turbo-tasks/src/backend.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/turbopack/crates/turbo-tasks/src/backend.rs b/turbopack/crates/turbo-tasks/src/backend.rs index cb47e6ac2a8b7..b77c606270f82 100644 --- a/turbopack/crates/turbo-tasks/src/backend.rs +++ b/turbopack/crates/turbo-tasks/src/backend.rs @@ -34,15 +34,18 @@ pub enum TransientTaskType { /// A root task that will track dependencies and re-execute when /// dependencies change. Task will eventually settle to the correct /// execution. + /// /// Always active. Automatically scheduled. Root(TransientTaskRoot), // TODO implement these strongly consistency /// A single root task execution. It won't track dependencies. + /// /// Task will definitely include all invalidations that happened before the /// start of the task. It may or may not include invalidations that /// happened after that. It may see these invalidations partially /// applied. + /// /// Active until done. Automatically scheduled. Once(Pin> + Send + 'static>>), } From 68c4bedb7e059d373f5314a488060d635baad292 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Fri, 4 Oct 2024 05:36:14 +0200 Subject: [PATCH 77/81] remove unneeded `let _ = ` --- turbopack/crates/turbo-tasks-backend/src/backend/mod.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs index ea9098200ef7b..243f661a56908 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs @@ -1062,17 +1062,17 @@ impl Backend for TurboTasksBackend { ); { let mut task = self.storage.access_mut(task_id); - let _ = task.add(CachedDataItem::AggregationNumber { + task.add(CachedDataItem::AggregationNumber { value: AggregationNumber { base: u32::MAX, distance: 0, effective: u32::MAX, }, }); - let _ = task.add(CachedDataItem::AggregateRoot { + task.add(CachedDataItem::AggregateRoot { value: RootState::new(root_type), }); - let _ = task.add(CachedDataItem::new_scheduled(move || match root_type { + task.add(CachedDataItem::new_scheduled(move || match root_type { ActiveType::RootTask => "Root Task".to_string(), ActiveType::OnceTask => "Once Task".to_string(), _ => unreachable!(), From b466e86463022ce915e15abbe68a3de7b2e4d9e7 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Fri, 4 Oct 2024 05:36:54 +0200 Subject: [PATCH 78/81] remove allow dead_code --- .../turbo-tasks-backend/src/backend/operation/mod.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs index 6d3f13d1bc7ee..132511369ff9b 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs @@ -35,8 +35,7 @@ pub trait Operation: pub struct ExecuteContext<'a> { backend: &'a TurboTasksBackend, turbo_tasks: &'a dyn TurboTasksBackendApi, - #[allow(dead_code)] - operation_guard: Option>, + _operation_guard: Option>, parent: Option<(&'a AnyOperation, &'a ExecuteContext<'a>)>, } @@ -48,7 +47,7 @@ impl<'a> ExecuteContext<'a> { Self { backend, turbo_tasks, - operation_guard: Some(backend.start_operation()), + _operation_guard: Some(backend.start_operation()), parent: None, } } @@ -123,7 +122,7 @@ impl<'a> ExecuteContext<'a> { let inner_ctx = ExecuteContext { backend: self.backend, turbo_tasks: self.turbo_tasks, - operation_guard: None, + _operation_guard: None, parent: Some((&parent_op, self)), }; run(inner_ctx); From fbcd760948c49e3c3ecc0571a99767fad49d42d0 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Fri, 4 Oct 2024 06:07:54 +0200 Subject: [PATCH 79/81] improve imports --- .../turbo-tasks-backend/src/backend/mod.rs | 16 +++++++++------- .../src/backend/operation/aggregation_update.rs | 6 ++++-- .../src/backend/operation/cleanup_old_edges.rs | 16 +++++++++------- .../src/backend/operation/connect_child.rs | 16 +++++++++------- .../src/backend/operation/invalidate.rs | 14 +++++++++----- .../src/backend/operation/mod.rs | 15 ++++++++------- .../src/backend/operation/update_cell.rs | 6 ++++-- .../src/backend/operation/update_output.rs | 6 ++++-- .../turbo-tasks-backend/src/backend/storage.rs | 6 ++++-- turbopack/crates/turbo-tasks-backend/src/lib.rs | 2 +- 10 files changed, 61 insertions(+), 42 deletions(-) diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs index 243f661a56908..c8a9697a52d58 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs @@ -19,11 +19,6 @@ use std::{ use anyhow::{bail, Result}; use auto_hash_map::{AutoMap, AutoSet}; use dashmap::DashMap; -pub use operation::AnyOperation; -use operation::{ - get_aggregation_number, is_root_node, AggregatedDataUpdate, AggregationUpdateJob, - AggregationUpdateQueue, CleanupOldEdgesOperation, ConnectChildOperation, OutdatedEdge, -}; use parking_lot::{Condvar, Mutex}; use rustc_hash::FxHasher; use smallvec::smallvec; @@ -39,9 +34,16 @@ use turbo_tasks::{ ValueTypeId, TRANSIENT_TASK_BIT, }; -use self::{operation::ExecuteContext, storage::Storage}; +pub use self::operation::AnyOperation; use crate::{ - backend::storage::{get, get_many, remove}, + backend::{ + operation::{ + get_aggregation_number, is_root_node, AggregatedDataUpdate, AggregationUpdateJob, + AggregationUpdateQueue, CleanupOldEdgesOperation, ConnectChildOperation, + ExecuteContext, OutdatedEdge, + }, + storage::{get, get_many, remove, Storage}, + }, data::{ ActiveType, AggregationNumber, CachedDataItem, CachedDataItemIndex, CachedDataItemKey, CachedDataItemValue, CachedDataUpdate, CellRef, InProgressCellState, InProgressState, diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs index b9b299ac868b2..b019caf3090e8 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/aggregation_update.rs @@ -3,9 +3,11 @@ use std::{cmp::max, collections::VecDeque, num::NonZeroU32}; use serde::{Deserialize, Serialize}; use turbo_tasks::TaskId; -use super::{ExecuteContext, Operation, TaskGuard}; use crate::{ - backend::storage::{get, get_many, iter_many, remove, update, update_count}, + backend::{ + operation::{ExecuteContext, Operation, TaskGuard}, + storage::{get, get_many, iter_many, remove, update, update_count}, + }, data::{ActiveType, AggregationNumber, CachedDataItem, CachedDataItemKey, RootState}, }; diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/cleanup_old_edges.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/cleanup_old_edges.rs index 4c102950c63f7..66a65ea95acb4 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/cleanup_old_edges.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/cleanup_old_edges.rs @@ -3,15 +3,17 @@ use std::mem::take; use serde::{Deserialize, Serialize}; use turbo_tasks::TaskId; -use super::{ - aggregation_update::{ - get_aggregation_number, get_uppers, is_aggregating_node, AggregationUpdateJob, - AggregationUpdateQueue, +use crate::{ + backend::operation::{ + aggregation_update::{ + get_aggregation_number, get_uppers, is_aggregating_node, AggregationUpdateJob, + AggregationUpdateQueue, + }, + invalidate::make_task_dirty, + ExecuteContext, Operation, }, - invalidate::make_task_dirty, - ExecuteContext, Operation, + data::{CachedDataItemKey, CellRef}, }; -use crate::data::{CachedDataItemKey, CellRef}; #[derive(Serialize, Deserialize, Clone, Default)] pub enum CleanupOldEdgesOperation { diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs index c16c2a61be0cc..98b07e9978cdf 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/connect_child.rs @@ -3,14 +3,16 @@ use std::{cmp::max, num::NonZeroU32}; use serde::{Deserialize, Serialize}; use turbo_tasks::TaskId; -use super::{ - aggregation_update::{ - get_uppers, is_aggregating_node, AggregationUpdateJob, AggregationUpdateQueue, - }, - ExecuteContext, Operation, -}; use crate::{ - backend::{operation::is_root_node, storage::get}, + backend::{ + operation::{ + aggregation_update::{ + get_uppers, is_aggregating_node, AggregationUpdateJob, AggregationUpdateQueue, + }, + is_root_node, ExecuteContext, Operation, + }, + storage::get, + }, data::{CachedDataItem, CachedDataItemIndex, CachedDataItemKey}, }; diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/invalidate.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/invalidate.rs index 95b07549cc488..71c58a97ec92d 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/invalidate.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/invalidate.rs @@ -2,12 +2,16 @@ use serde::{Deserialize, Serialize}; use smallvec::SmallVec; use turbo_tasks::TaskId; -use super::{ - aggregation_update::{AggregatedDataUpdate, AggregationUpdateJob, AggregationUpdateQueue}, - ExecuteContext, Operation, -}; use crate::{ - backend::storage::get, + backend::{ + operation::{ + aggregation_update::{ + AggregatedDataUpdate, AggregationUpdateJob, AggregationUpdateQueue, + }, + ExecuteContext, Operation, + }, + storage::get, + }, data::{CachedDataItem, CachedDataItemKey}, }; diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs index 132511369ff9b..f1f0952861d8a 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs @@ -13,9 +13,8 @@ use std::{ use serde::{Deserialize, Serialize}; use turbo_tasks::{KeyValuePair, TaskId, TurboTasksBackendApi}; -use super::{storage::StorageWriteGuard, TurboTasksBackend}; use crate::{ - backend::{OperationGuard, TransientTask}, + backend::{storage::StorageWriteGuard, OperationGuard, TransientTask, TurboTasksBackend}, data::{ CachedDataItem, CachedDataItemIndex, CachedDataItemKey, CachedDataItemValue, CachedDataUpdate, @@ -351,9 +350,11 @@ impl_operation!(Invalidate invalidate::InvalidateOperation); impl_operation!(CleanupOldEdges cleanup_old_edges::CleanupOldEdgesOperation); impl_operation!(AggregationUpdate aggregation_update::AggregationUpdateQueue); -pub use aggregation_update::{ - get_aggregation_number, is_root_node, AggregatedDataUpdate, AggregationUpdateJob, +pub use self::{ + aggregation_update::{ + get_aggregation_number, is_root_node, AggregatedDataUpdate, AggregationUpdateJob, + }, + cleanup_old_edges::OutdatedEdge, + update_cell::UpdateCellOperation, + update_output::UpdateOutputOperation, }; -pub use cleanup_old_edges::OutdatedEdge; -pub use update_cell::UpdateCellOperation; -pub use update_output::UpdateOutputOperation; diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/update_cell.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/update_cell.rs index 223fd9e6d6550..b7b4e185839bc 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/update_cell.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/update_cell.rs @@ -1,8 +1,10 @@ use turbo_tasks::{backend::CellContent, CellId, TaskId}; -use super::{ExecuteContext, InvalidateOperation}; use crate::{ - backend::storage::{get_many, remove}, + backend::{ + operation::{ExecuteContext, InvalidateOperation}, + storage::{get_many, remove}, + }, data::{CachedDataItem, CachedDataItemKey}, }; diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/update_output.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/update_output.rs index e722bc241caf8..efcee8546d8f5 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/update_output.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/update_output.rs @@ -3,9 +3,11 @@ use std::borrow::Cow; use anyhow::{anyhow, Result}; use turbo_tasks::{util::SharedError, RawVc, TaskId}; -use super::{ExecuteContext, InvalidateOperation}; use crate::{ - backend::storage::get_many, + backend::{ + operation::{ExecuteContext, InvalidateOperation}, + storage::get_many, + }, data::{CachedDataItem, CachedDataItemKey, CachedDataItemValue, CellRef, OutputValue}, }; diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs b/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs index 292cfa1d140be..7a3006624c93c 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/storage.rs @@ -11,8 +11,10 @@ use either::Either; use rustc_hash::FxHasher; use turbo_tasks::KeyValuePair; -use super::indexed::Indexed; -use crate::utils::dash_map_multi::{get_multiple_mut, RefMut}; +use crate::{ + backend::indexed::Indexed, + utils::dash_map_multi::{get_multiple_mut, RefMut}, +}; const INDEX_THRESHOLD: usize = 1024; diff --git a/turbopack/crates/turbo-tasks-backend/src/lib.rs b/turbopack/crates/turbo-tasks-backend/src/lib.rs index 5fc95b21a44d9..c7b5fe1832ad1 100644 --- a/turbopack/crates/turbo-tasks-backend/src/lib.rs +++ b/turbopack/crates/turbo-tasks-backend/src/lib.rs @@ -2,4 +2,4 @@ mod backend; mod data; mod utils; -pub use backend::TurboTasksBackend; +pub use self::backend::TurboTasksBackend; From d2c34b6ab68035ac48e364b020bff1c2feb8cc88 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Fri, 4 Oct 2024 08:49:01 +0200 Subject: [PATCH 80/81] fix scope_stress test case --- turbopack/crates/turbo-tasks-testing/tests/scope_stress.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/turbopack/crates/turbo-tasks-testing/tests/scope_stress.rs b/turbopack/crates/turbo-tasks-testing/tests/scope_stress.rs index 553c598b28b8d..7c59f372b460f 100644 --- a/turbopack/crates/turbo-tasks-testing/tests/scope_stress.rs +++ b/turbopack/crates/turbo-tasks-testing/tests/scope_stress.rs @@ -26,7 +26,8 @@ async fn rectangle_stress() -> Result<()> { } }) .try_join() - .await + .await?; + Ok(()) }) .await } From e8d0378a7c1251410b844a8216d36d5c49903cdf Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Fri, 4 Oct 2024 09:05:32 +0200 Subject: [PATCH 81/81] add stress test for dash_map multi locking --- .../src/utils/dash_map_multi.rs | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/turbopack/crates/turbo-tasks-backend/src/utils/dash_map_multi.rs b/turbopack/crates/turbo-tasks-backend/src/utils/dash_map_multi.rs index 042b7f971760b..65d630499fcdf 100644 --- a/turbopack/crates/turbo-tasks-backend/src/utils/dash_map_multi.rs +++ b/turbopack/crates/turbo-tasks-backend/src/utils/dash_map_multi.rs @@ -174,3 +174,47 @@ where ) } } + +#[cfg(test)] +mod tests { + use std::thread::scope; + + use rand::prelude::SliceRandom; + + use super::*; + + #[test] + fn stress_deadlock() { + const N: usize = 100000; + const THREADS: usize = 20; + + let map = DashMap::with_shard_amount(4); + let indicies = (0..THREADS) + .map(|_| { + let mut vec = (0..N).collect::>(); + vec.shuffle(&mut rand::thread_rng()); + vec + }) + .collect::>(); + let map = ↦ + scope(|s| { + for indicies in indicies { + s.spawn(|| { + for i in indicies { + let (mut a, mut b) = get_multiple_mut(map, i, i + 1, || 0); + *a += 1; + *b += 1; + } + }); + } + }); + let value = *map.get(&0).unwrap(); + assert_eq!(value, THREADS); + for i in 1..N { + let value = *map.get(&i).unwrap(); + assert_eq!(value, THREADS * 2); + } + let value = *map.get(&N).unwrap(); + assert_eq!(value, THREADS); + } +}