From 6dc8cce7883fe3f936526786830c15275ed4e4ec Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Fri, 6 Dec 2024 15:32:07 +0100 Subject: [PATCH 1/6] replace EnvLayer with a faster filter --- crates/napi/src/next_api/project.rs | 5 +- crates/next-build-test/src/main.rs | 9 ++-- turbopack/crates/turbopack-cli/src/main.rs | 5 +- .../turbopack-trace-utils/src/filter_layer.rs | 47 +++++++++++++++++++ .../crates/turbopack-trace-utils/src/lib.rs | 1 + 5 files changed, 59 insertions(+), 8 deletions(-) create mode 100644 turbopack/crates/turbopack-trace-utils/src/filter_layer.rs diff --git a/crates/napi/src/next_api/project.rs b/crates/napi/src/next_api/project.rs index 919a0e16c16c2..a84e0806aa020 100644 --- a/crates/napi/src/next_api/project.rs +++ b/crates/napi/src/next_api/project.rs @@ -22,7 +22,7 @@ use once_cell::sync::Lazy; use rand::Rng; use tokio::{io::AsyncWriteExt, time::Instant}; use tracing::Instrument; -use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter, Registry}; +use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, Registry}; use turbo_rcstr::RcStr; use turbo_tasks::{ get_effects, Completion, Effects, ReadRef, ResolvedVc, TransientInstance, UpdateInfo, Vc, @@ -41,6 +41,7 @@ use turbopack_core::{ use turbopack_ecmascript_hmr_protocol::{ClientUpdateInstruction, ResourceIdentifier}; use turbopack_trace_utils::{ exit::{ExitHandler, ExitReceiver}, + filter_layer::FilterLayer, raw_trace::RawTraceLayer, trace_writer::TraceWriter, }; @@ -317,7 +318,7 @@ pub async fn project_new( let subscriber = Registry::default(); - let subscriber = subscriber.with(EnvFilter::builder().parse(trace).unwrap()); + let subscriber = subscriber.with(FilterLayer::try_new(trace).unwrap()); let dist_dir = options.dist_dir.clone(); let internal_dir = PathBuf::from(&options.project_path).join(dist_dir); diff --git a/crates/next-build-test/src/main.rs b/crates/next-build-test/src/main.rs index c796e49bb5e6e..2063a46d0f0e9 100644 --- a/crates/next-build-test/src/main.rs +++ b/crates/next-build-test/src/main.rs @@ -6,11 +6,13 @@ use next_core::tracing_presets::{ TRACING_NEXT_OVERVIEW_TARGETS, TRACING_NEXT_TARGETS, TRACING_NEXT_TURBOPACK_TARGETS, TRACING_NEXT_TURBO_TASKS_TARGETS, }; -use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter, Registry}; +use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, Registry}; use turbo_tasks::TurboTasks; use turbo_tasks_malloc::TurboMalloc; use turbo_tasks_memory::MemoryBackend; -use turbopack_trace_utils::{exit::ExitGuard, raw_trace::RawTraceLayer, trace_writer::TraceWriter}; +use turbopack_trace_utils::{ + exit::ExitGuard, filter_layer::FilterLayer, raw_trace::RawTraceLayer, trace_writer::TraceWriter, +}; #[global_allocator] static ALLOC: TurboMalloc = TurboMalloc; @@ -99,8 +101,7 @@ fn main() { let subscriber = Registry::default(); - let subscriber = - subscriber.with(EnvFilter::builder().parse(trace).unwrap()); + let subscriber = subscriber.with(FilterLayer::try_new(trace).unwrap()); let trace_file = "trace.log"; let trace_writer = std::fs::File::create(trace_file).unwrap(); let (trace_writer, guard) = TraceWriter::new(trace_writer); diff --git a/turbopack/crates/turbopack-cli/src/main.rs b/turbopack/crates/turbopack-cli/src/main.rs index 0129194f88bd0..a28749662b8a1 100644 --- a/turbopack/crates/turbopack-cli/src/main.rs +++ b/turbopack/crates/turbopack-cli/src/main.rs @@ -5,11 +5,12 @@ use std::path::Path; use anyhow::{Context, Result}; use clap::Parser; -use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter, Registry}; +use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, Registry}; use turbo_tasks_malloc::TurboMalloc; use turbopack_cli::{arguments::Arguments, register}; use turbopack_trace_utils::{ exit::ExitHandler, + filter_layer::FilterLayer, raw_trace::RawTraceLayer, trace_writer::TraceWriter, tracing_presets::{ @@ -55,7 +56,7 @@ async fn main_inner(args: Arguments) -> Result<()> { let subscriber = Registry::default(); - let subscriber = subscriber.with(EnvFilter::builder().parse(trace).unwrap()); + let subscriber = subscriber.with(FilterLayer::try_new(&trace).unwrap()); let internal_dir = args .dir() diff --git a/turbopack/crates/turbopack-trace-utils/src/filter_layer.rs b/turbopack/crates/turbopack-trace-utils/src/filter_layer.rs new file mode 100644 index 0000000000000..de2850da49563 --- /dev/null +++ b/turbopack/crates/turbopack-trace-utils/src/filter_layer.rs @@ -0,0 +1,47 @@ +use std::{collections::HashMap, str::FromStr}; + +use anyhow::{Context, Result}; +use tracing::{level_filters::LevelFilter, Subscriber}; +use tracing_subscriber::Layer; + +pub struct FilterLayer { + config: HashMap, +} + +impl FilterLayer { + pub fn try_new(config: &str) -> Result { + let mut map = HashMap::new(); + for entry in config.split(',') { + if entry.is_empty() { + continue; + } + let mut parts = entry.splitn(2, '='); + let target = parts.next().unwrap(); + let level = parts + .next() + .context("Invalid filter syntax, expected `target=level`")?; + let level = LevelFilter::from_str(level).unwrap(); + map.insert(target.to_string(), level); + } + Ok(Self { config: map }) + } +} + +impl Layer for FilterLayer { + fn enabled( + &self, + metadata: &tracing::Metadata<'_>, + _ctx: tracing_subscriber::layer::Context<'_, S>, + ) -> bool { + let target = metadata.target().split("::").next().unwrap(); + let Some(filter) = self.config.get(target) else { + return false; + }; + let level = metadata.level(); + return level <= filter; + } + + fn max_level_hint(&self) -> Option { + self.config.values().copied().min() + } +} diff --git a/turbopack/crates/turbopack-trace-utils/src/lib.rs b/turbopack/crates/turbopack-trace-utils/src/lib.rs index 984fc6b9778d9..a000ef3bbbabb 100644 --- a/turbopack/crates/turbopack-trace-utils/src/lib.rs +++ b/turbopack/crates/turbopack-trace-utils/src/lib.rs @@ -6,6 +6,7 @@ #![feature(arbitrary_self_types_pointers)] pub mod exit; +pub mod filter_layer; mod flavor; pub mod raw_trace; pub mod trace_writer; From 849d2e79baccabaad833b0315a8c160284511c83 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Fri, 6 Dec 2024 15:36:03 +0100 Subject: [PATCH 2/6] add global level --- .../turbopack-trace-utils/src/filter_layer.rs | 32 ++++++++++++------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/turbopack/crates/turbopack-trace-utils/src/filter_layer.rs b/turbopack/crates/turbopack-trace-utils/src/filter_layer.rs index de2850da49563..0c82a4da30a59 100644 --- a/turbopack/crates/turbopack-trace-utils/src/filter_layer.rs +++ b/turbopack/crates/turbopack-trace-utils/src/filter_layer.rs @@ -1,29 +1,36 @@ use std::{collections::HashMap, str::FromStr}; -use anyhow::{Context, Result}; +use anyhow::Result; use tracing::{level_filters::LevelFilter, Subscriber}; use tracing_subscriber::Layer; pub struct FilterLayer { config: HashMap, + global_level: LevelFilter, } impl FilterLayer { - pub fn try_new(config: &str) -> Result { - let mut map = HashMap::new(); - for entry in config.split(',') { + pub fn try_new(input: &str) -> Result { + let mut config = HashMap::new(); + let mut global_level = LevelFilter::OFF; + for entry in input.split(',') { if entry.is_empty() { continue; } let mut parts = entry.splitn(2, '='); let target = parts.next().unwrap(); - let level = parts - .next() - .context("Invalid filter syntax, expected `target=level`")?; + let level = parts.next().unwrap_or("trace"); let level = LevelFilter::from_str(level).unwrap(); - map.insert(target.to_string(), level); + if target == "*" { + global_level = level; + } else { + config.insert(target.to_string(), level); + } } - Ok(Self { config: map }) + Ok(Self { + config, + global_level, + }) } } @@ -33,10 +40,11 @@ impl Layer for FilterLayer { metadata: &tracing::Metadata<'_>, _ctx: tracing_subscriber::layer::Context<'_, S>, ) -> bool { + if self.config.is_empty() { + return true; + } let target = metadata.target().split("::").next().unwrap(); - let Some(filter) = self.config.get(target) else { - return false; - }; + let filter = self.config.get(target).unwrap_or(&self.global_level); let level = metadata.level(); return level <= filter; } From bbc12920196699cc52abd7e48f60b245945ffbeb Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Fri, 6 Dec 2024 15:38:22 +0100 Subject: [PATCH 3/6] use fx hash map --- Cargo.lock | 1 + turbopack/crates/turbopack-trace-utils/Cargo.toml | 1 + turbopack/crates/turbopack-trace-utils/src/filter_layer.rs | 7 ++++--- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 29cdeb27ed348..d8fdb388cba86 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9262,6 +9262,7 @@ dependencies = [ "crossbeam-channel", "once_cell", "postcard", + "rustc-hash 1.1.0", "serde", "tokio", "tracing", diff --git a/turbopack/crates/turbopack-trace-utils/Cargo.toml b/turbopack/crates/turbopack-trace-utils/Cargo.toml index d0c0a11f276c2..8135dbec5ef2e 100644 --- a/turbopack/crates/turbopack-trace-utils/Cargo.toml +++ b/turbopack/crates/turbopack-trace-utils/Cargo.toml @@ -16,6 +16,7 @@ anyhow = { workspace = true } crossbeam-channel = { workspace = true } once_cell = { workspace = true } postcard = { workspace = true, features = ["alloc", "use-std"] } +rustc-hash = { workspace = true } serde = { workspace = true, features = ["derive"] } tokio = { workspace = true, features = ["macros", "signal", "sync", "rt"] } tracing = { workspace = true } diff --git a/turbopack/crates/turbopack-trace-utils/src/filter_layer.rs b/turbopack/crates/turbopack-trace-utils/src/filter_layer.rs index 0c82a4da30a59..516277bac25e8 100644 --- a/turbopack/crates/turbopack-trace-utils/src/filter_layer.rs +++ b/turbopack/crates/turbopack-trace-utils/src/filter_layer.rs @@ -1,17 +1,18 @@ -use std::{collections::HashMap, str::FromStr}; +use std::{collections::HashMap, hash::BuildHasherDefault, str::FromStr}; use anyhow::Result; +use rustc_hash::FxHasher; use tracing::{level_filters::LevelFilter, Subscriber}; use tracing_subscriber::Layer; pub struct FilterLayer { - config: HashMap, + config: HashMap>, global_level: LevelFilter, } impl FilterLayer { pub fn try_new(input: &str) -> Result { - let mut config = HashMap::new(); + let mut config = HashMap::default(); let mut global_level = LevelFilter::OFF; for entry in input.split(',') { if entry.is_empty() { From 29b2ec054c86faab761f1624a1be2649b20d2939 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Fri, 6 Dec 2024 18:48:20 +0100 Subject: [PATCH 4/6] fixup --- crates/napi/src/next_api/project.rs | 2 +- crates/next-build-test/src/main.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/napi/src/next_api/project.rs b/crates/napi/src/next_api/project.rs index a84e0806aa020..ab02ac5839e2f 100644 --- a/crates/napi/src/next_api/project.rs +++ b/crates/napi/src/next_api/project.rs @@ -318,7 +318,7 @@ pub async fn project_new( let subscriber = Registry::default(); - let subscriber = subscriber.with(FilterLayer::try_new(trace).unwrap()); + let subscriber = subscriber.with(FilterLayer::try_new(&trace).unwrap()); let dist_dir = options.dist_dir.clone(); let internal_dir = PathBuf::from(&options.project_path).join(dist_dir); diff --git a/crates/next-build-test/src/main.rs b/crates/next-build-test/src/main.rs index 2063a46d0f0e9..59084ed7f5527 100644 --- a/crates/next-build-test/src/main.rs +++ b/crates/next-build-test/src/main.rs @@ -101,7 +101,7 @@ fn main() { let subscriber = Registry::default(); - let subscriber = subscriber.with(FilterLayer::try_new(trace).unwrap()); + let subscriber = subscriber.with(FilterLayer::try_new(&trace).unwrap()); let trace_file = "trace.log"; let trace_writer = std::fs::File::create(trace_file).unwrap(); let (trace_writer, guard) = TraceWriter::new(trace_writer); From 386304c535eeb2d8abefa417817524e7c81ba63f Mon Sep 17 00:00:00 2001 From: Niklas Mischkulnig <4586894+mischnic@users.noreply.github.com> Date: Sat, 7 Dec 2024 09:53:42 +0100 Subject: [PATCH 5/6] Clippy --- turbopack/crates/turbopack-trace-utils/src/filter_layer.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/turbopack/crates/turbopack-trace-utils/src/filter_layer.rs b/turbopack/crates/turbopack-trace-utils/src/filter_layer.rs index 516277bac25e8..bc2227b32288e 100644 --- a/turbopack/crates/turbopack-trace-utils/src/filter_layer.rs +++ b/turbopack/crates/turbopack-trace-utils/src/filter_layer.rs @@ -47,7 +47,7 @@ impl Layer for FilterLayer { let target = metadata.target().split("::").next().unwrap(); let filter = self.config.get(target).unwrap_or(&self.global_level); let level = metadata.level(); - return level <= filter; + level <= filter } fn max_level_hint(&self) -> Option { From 4f944cb50de1dc1af161587ea5c21c0b0b4263cf Mon Sep 17 00:00:00 2001 From: Niklas Mischkulnig <4586894+mischnic@users.noreply.github.com> Date: Sat, 7 Dec 2024 09:53:57 +0100 Subject: [PATCH 6/6] Update turbopack/crates/turbopack-trace-utils/src/filter_layer.rs --- turbopack/crates/turbopack-trace-utils/src/filter_layer.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/turbopack/crates/turbopack-trace-utils/src/filter_layer.rs b/turbopack/crates/turbopack-trace-utils/src/filter_layer.rs index bc2227b32288e..d283538ff10e2 100644 --- a/turbopack/crates/turbopack-trace-utils/src/filter_layer.rs +++ b/turbopack/crates/turbopack-trace-utils/src/filter_layer.rs @@ -47,7 +47,7 @@ impl Layer for FilterLayer { let target = metadata.target().split("::").next().unwrap(); let filter = self.config.get(target).unwrap_or(&self.global_level); let level = metadata.level(); - level <= filter + level <= filter } fn max_level_hint(&self) -> Option {