From c6015eb9df75900a6dec3244d460389ca046fe93 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Mon, 12 Aug 2024 15:36:17 +0200 Subject: [PATCH] remove old edges when task recomputation completes --- .../turbo-tasks-backend/src/backend/mod.rs | 125 ++++++++++++++++-- .../backend/operation/cleanup_old_edges.rs | 96 ++++++++++++++ .../src/backend/operation/mod.rs | 4 + .../crates/turbo-tasks-backend/src/data.rs | 6 + 4 files changed, 223 insertions(+), 8 deletions(-) create mode 100644 turbopack/crates/turbo-tasks-backend/src/backend/operation/cleanup_old_edges.rs diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs index 1208b0bdc8fc3e..e0618aaa19cd50 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs @@ -19,7 +19,7 @@ use anyhow::Result; use auto_hash_map::{AutoMap, AutoSet}; use dashmap::DashMap; pub use operation::AnyOperation; -use operation::ConnectChildOperation; +use operation::{CleanupOldEdgesOperation, ConnectChildOperation, OutdatedEdge}; use parking_lot::{Condvar, Mutex}; use rustc_hash::FxHasher; use smallvec::smallvec; @@ -277,13 +277,16 @@ impl TurboTasksBackend { }); drop(task); let mut reader_task = ctx.task(reader); - reader_task.add(CachedDataItem::CellDependency { - target: CellRef { - task: task_id, - cell, - }, - value: (), - }); + let target = CellRef { + task: task_id, + cell, + }; + if reader_task + .remove(&CachedDataItemKey::OutdatedCellDependency { target }) + .is_none() + { + reader_task.add(CachedDataItem::CellDependency { target, value: () }); + } } return Ok(Ok(CellContent(Some(content)).into_typed(cell.type_id))); } @@ -413,8 +416,95 @@ impl Backend for TurboTasksBackend { done_event, }, }); + + // Make all current children outdated (remove left-over outdated children) + enum Child { + Current(TaskId), + Outdated(TaskId), + } + let children = task + .iter() + .filter_map(|(key, _)| match *key { + CachedDataItemKey::Child { task } => Some(Child::Current(task)), + CachedDataItemKey::OutdatedChild { task } => Some(Child::Outdated(task)), + _ => None, + }) + .collect::>(); + for child in children { + match child { + Child::Current(child) => { + task.add(CachedDataItem::OutdatedChild { + task: child, + value: (), + }); + } + Child::Outdated(child) => { + if !task.has_key(&CachedDataItemKey::Child { task: child }) { + task.remove(&CachedDataItemKey::OutdatedChild { task: child }); + } + } + } + } + + // Make all dependencies outdated + enum Dep { + CurrentCell(CellRef), + CurrentOutput(TaskId), + OutdatedCell(CellRef), + OutdatedOutput(TaskId), + } + let dependencies = task + .iter() + .filter_map(|(key, _)| match *key { + CachedDataItemKey::CellDependency { target } => Some(Dep::CurrentCell(target)), + CachedDataItemKey::OutputDependency { target } => { + Some(Dep::CurrentOutput(target)) + } + CachedDataItemKey::OutdatedCellDependency { target } => { + Some(Dep::OutdatedCell(target)) + } + CachedDataItemKey::OutdatedOutputDependency { target } => { + Some(Dep::OutdatedOutput(target)) + } + _ => None, + }) + .collect::>(); + for dep in dependencies { + match dep { + Dep::CurrentCell(cell) => { + task.add(CachedDataItem::OutdatedCellDependency { + target: cell, + value: (), + }); + } + Dep::CurrentOutput(output) => { + task.add(CachedDataItem::OutdatedOutputDependency { + target: output, + value: (), + }); + } + Dep::OutdatedCell(cell) => { + if !task.has_key(&CachedDataItemKey::CellDependency { target: cell }) { + task.remove(&CachedDataItemKey::OutdatedCellDependency { + target: cell, + }); + } + } + Dep::OutdatedOutput(output) => { + if !task.has_key(&CachedDataItemKey::OutputDependency { target: output }) { + task.remove(&CachedDataItemKey::OutdatedOutputDependency { + target: output, + }); + } + } + } + } + + // TODO: Make all collectibles outdated + start_event.notify(usize::MAX); } + let (span, future) = if let Some(task_type) = self.task_cache.lookup_reverse(&task_id) { match &*task_type { CachedTaskType::Native { fn_type, this, arg } => ( @@ -536,8 +626,27 @@ impl Backend for TurboTasksBackend { done_event, }, }); + drop(task); + drop(ctx); } else { + let old_edges = task + .iter() + .filter_map(|(key, _)| match *key { + CachedDataItemKey::OutdatedChild { task } => Some(OutdatedEdge::Child(task)), + CachedDataItemKey::OutdatedCellDependency { target } => { + Some(OutdatedEdge::CellDependency(target)) + } + CachedDataItemKey::OutdatedOutputDependency { target } => { + Some(OutdatedEdge::OutputDependency(target)) + } + _ => None, + }) + .collect::>(); + done_event.notify(usize::MAX); + drop(task); + + CleanupOldEdgesOperation::run(task_id, old_edges, ctx); } stale diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/cleanup_old_edges.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/cleanup_old_edges.rs new file mode 100644 index 00000000000000..e9eb3738d84eed --- /dev/null +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/cleanup_old_edges.rs @@ -0,0 +1,96 @@ +use serde::{Deserialize, Serialize}; +use turbo_tasks::TaskId; + +use super::{ExecuteContext, Operation}; +use crate::data::{CachedDataItemKey, CellRef}; + +#[derive(Serialize, Deserialize, Clone, Default)] +pub enum CleanupOldEdgesOperation { + RemoveEdges { + task_id: TaskId, + outdated: Vec, + }, + #[default] + Done, + // TODO Add aggregated edge +} + +#[derive(Serialize, Deserialize, Clone)] +pub enum OutdatedEdge { + Child(TaskId), + CellDependency(CellRef), + OutputDependency(TaskId), +} + +impl CleanupOldEdgesOperation { + pub fn run(task_id: TaskId, outdated: Vec, ctx: ExecuteContext<'_>) { + CleanupOldEdgesOperation::RemoveEdges { task_id, outdated }.execute(&ctx); + } +} + +impl Operation for CleanupOldEdgesOperation { + fn execute(mut self, ctx: &ExecuteContext<'_>) { + loop { + ctx.operation_suspend_point(&self); + match self { + CleanupOldEdgesOperation::RemoveEdges { + task_id, + ref mut outdated, + } => { + if let Some(edge) = outdated.pop() { + match edge { + OutdatedEdge::Child(child_id) => { + let mut task = ctx.task(task_id); + task.remove(&CachedDataItemKey::Child { task: child_id }); + // TODO remove aggregated edge + } + OutdatedEdge::CellDependency(CellRef { + task: cell_task_id, + cell, + }) => { + { + let mut task = ctx.task(cell_task_id); + task.remove(&CachedDataItemKey::CellDependent { + cell, + task: task_id, + }); + } + { + let mut task = ctx.task(task_id); + task.remove(&CachedDataItemKey::CellDependency { + target: CellRef { + task: cell_task_id, + cell, + }, + }); + } + } + OutdatedEdge::OutputDependency(output_task_id) => { + { + let mut task = ctx.task(output_task_id); + task.remove(&CachedDataItemKey::OutputDependent { + task: task_id, + }); + } + { + let mut task = ctx.task(task_id); + task.remove(&CachedDataItemKey::OutputDependency { + target: output_task_id, + }); + } + } + } + } + + if outdated.is_empty() { + self = CleanupOldEdgesOperation::Done; + } + continue; + } + CleanupOldEdgesOperation::Done => { + return; + } + } + } + } +} diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs index d112c41e3914a4..ef4c2399b0004f 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/operation/mod.rs @@ -1,3 +1,4 @@ +mod cleanup_old_edges; mod connect_child; mod invalidate; mod update_cell; @@ -234,11 +235,14 @@ macro_rules! impl_operation { pub enum AnyOperation { ConnectChild(connect_child::ConnectChildOperation), Invalidate(invalidate::InvalidateOperation), + CleanupOldEdges(cleanup_old_edges::CleanupOldEdgesOperation), Nested(Vec), } impl_operation!(ConnectChild connect_child::ConnectChildOperation); impl_operation!(Invalidate invalidate::InvalidateOperation); +impl_operation!(CleanupOldEdges cleanup_old_edges::CleanupOldEdgesOperation); +pub use cleanup_old_edges::OutdatedEdge; pub use update_cell::UpdateCellOperation; pub use update_output::UpdateOutputOperation; diff --git a/turbopack/crates/turbo-tasks-backend/src/data.rs b/turbopack/crates/turbo-tasks-backend/src/data.rs index 05a10bc3981d52..e6c774397de021 100644 --- a/turbopack/crates/turbo-tasks-backend/src/data.rs +++ b/turbopack/crates/turbo-tasks-backend/src/data.rs @@ -150,6 +150,10 @@ pub enum CachedDataItem { target: CellRef, value: (), }, + OutdatedChild { + task: TaskId, + value: (), + }, // Transient Error State Error { @@ -183,6 +187,7 @@ impl CachedDataItem { CachedDataItem::OutdatedCollectible { .. } => false, CachedDataItem::OutdatedOutputDependency { .. } => false, CachedDataItem::OutdatedCellDependency { .. } => false, + CachedDataItem::OutdatedChild { .. } => false, CachedDataItem::Error { .. } => false, } } @@ -224,6 +229,7 @@ impl CachedDataItemKey { CachedDataItemKey::OutdatedCollectible { .. } => false, CachedDataItemKey::OutdatedOutputDependency { .. } => false, CachedDataItemKey::OutdatedCellDependency { .. } => false, + CachedDataItemKey::OutdatedChild { .. } => false, CachedDataItemKey::Error { .. } => false, } }