diff --git a/Cargo.lock b/Cargo.lock index 9a2bfa6f2..581995cd5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1944,14 +1944,21 @@ dependencies = [ "insta", "marzano-cli", "marzano-gritmodule", + "marzano-util", "ntest", + "opentelemetry", + "opentelemetry-otlp", + "opentelemetry_sdk", "predicates", "rayon", "regex", + "reqwest", "serde_json", "tempfile", "tokio", "tracing", + "tracing-opentelemetry", + "tracing-subscriber", ] [[package]] @@ -2018,6 +2025,7 @@ dependencies = [ "tokio", "tracing", "tracing-opentelemetry", + "tracing-subscriber", "trim-margin", "uuid", ] @@ -2086,6 +2094,7 @@ dependencies = [ "serde_yaml", "tempfile", "tokio", + "tracing", "tree-sitter-facade-sg", ] diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index 7aab3140b..823ddaefe 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -71,6 +71,7 @@ opentelemetry_sdk = { version = "0.21.1", optional = true, features = [ tracing-opentelemetry = { version = "0.22.0", optional = true, default-features = false } tracing = { version = "0.1.40", default-features = false, features = [] } +tracing-subscriber = { version = "0.3", default-features = false, optional = true } [target.'cfg(not(windows))'.dependencies] openssl = { version = "0.10", features = ["vendored"] } @@ -87,6 +88,7 @@ default = [ "external_functions", "updater", "workflows_v2", + "grit_tracing", # "remote_workflows", # "server", # "remote_redis", @@ -109,6 +111,7 @@ server = [ "remote_pubsub", "ai_builtins", "dep:cli_server", + "grit_tracing" ] updater = [] grit_tracing = [ @@ -116,6 +119,7 @@ grit_tracing = [ "dep:opentelemetry", "dep:opentelemetry_sdk", "dep:tracing-opentelemetry", + "dep:tracing-subscriber", "marzano-core/grit_tracing", ] external_functions = ["marzano-core/external_functions"] diff --git a/crates/cli/src/commands/apply_pattern.rs b/crates/cli/src/commands/apply_pattern.rs index 1e3fd9e1b..f6fc20aaa 100644 --- a/crates/cli/src/commands/apply_pattern.rs +++ b/crates/cli/src/commands/apply_pattern.rs @@ -7,6 +7,7 @@ use tracing::instrument; #[cfg(feature = "grit_tracing")] use tracing::span; #[cfg(feature = "grit_tracing")] +#[allow(unused_imports)] use tracing_opentelemetry::OpenTelemetrySpanExt as _; use grit_util::Position; @@ -191,12 +192,17 @@ pub(crate) async fn run_apply_pattern( // Get the current directory let cwd = std::env::current_dir().unwrap(); + #[cfg(feature = "grit_tracing")] + let module_resolution = span!(tracing::Level::INFO, "module_resolution",).entered(); + // Construct a resolver let resolver = GritModuleResolver::new(cwd.to_str().unwrap()); let current_repo_root = marzano_gritmodule::fetcher::LocalRepo::from_dir(&cwd) .await .map(|repo| repo.root()) .transpose()?; + #[cfg(feature = "grit_tracing")] + module_resolution.exit(); let mut emitter = create_emitter( &format, @@ -213,6 +219,9 @@ pub(crate) async fn run_apply_pattern( extract_filter_ranges(&shared, current_repo_root.as_ref()) ); + #[cfg(feature = "grit_tracing")] + let span_libs = span!(tracing::Level::INFO, "prep_libs",).entered(); + let (my_input, lang) = if let Some(pattern_libs) = pattern_libs { ( ApplyInput { @@ -223,6 +232,9 @@ pub(crate) async fn run_apply_pattern( lang, ) } else { + #[cfg(feature = "grit_tracing")] + let stdlib_download_span = span!(tracing::Level::INFO, "stdlib_download",).entered(); + let mod_dir = find_grit_modules_dir(cwd.clone()).await; if !env::var("GRIT_DOWNLOADS_DISABLED") .unwrap_or_else(|_| "false".to_owned()) @@ -236,6 +248,9 @@ pub(crate) async fn run_apply_pattern( ); } + #[cfg(feature = "grit_tracing")] + stdlib_download_span.exit(); + let warn_uncommitted = !arg.dry_run && !arg.force && has_uncommitted_changes(cwd.clone()).await; if warn_uncommitted { @@ -254,7 +269,11 @@ pub(crate) async fn run_apply_pattern( } } + #[cfg(feature = "grit_tracing")] + let grit_file_discovery = span!(tracing::Level::INFO, "grit_file_discovery",).entered(); + let pattern_libs = flushable_unwrap!(emitter, get_grit_files_from_cwd().await); + let (mut lang, pattern_body) = if pattern.ends_with(".grit") || pattern.ends_with(".md") { match fs::read_to_string(pattern.clone()).await { Ok(pb) => { @@ -330,6 +349,8 @@ pub(crate) async fn run_apply_pattern( emitter, pattern_libs.get_language_directory_or_default(lang) ); + #[cfg(feature = "grit_tracing")] + grit_file_discovery.exit(); ( ApplyInput { pattern_body, @@ -352,6 +373,8 @@ pub(crate) async fn run_apply_pattern( return Ok(()); } + #[cfg(feature = "grit_tracing")] + let collect_name = span!(tracing::Level::INFO, "collect_name",).entered(); let current_name = if is_pattern_name(&pattern) { Some(pattern.trim_end_matches("()").to_string()) } else { @@ -361,11 +384,17 @@ pub(crate) async fn run_apply_pattern( .find(|(_, body)| body.trim() == pattern.trim()) .map(|(name, _)| name.clone()) }; + #[cfg(feature = "grit_tracing")] + collect_name.exit(); + let pattern: crate::resolver::RichPattern<'_> = flushable_unwrap!( emitter, resolver.make_pattern(&my_input.pattern_body, current_name) ); + #[cfg(feature = "grit_tracing")] + span_libs.exit(); + let CompilationResult { problem: compiled, compilation_warnings, diff --git a/crates/cli/src/commands/filters.rs b/crates/cli/src/commands/filters.rs index c292a307c..b53731cd2 100644 --- a/crates/cli/src/commands/filters.rs +++ b/crates/cli/src/commands/filters.rs @@ -26,6 +26,7 @@ pub struct SharedFilterArgs { pub(crate) only_in_diff: Option>, } +#[tracing::instrument] pub(crate) fn extract_filter_ranges( args: &SharedFilterArgs, root: Option<&PathBuf>, diff --git a/crates/cli/src/commands/mod.rs b/crates/cli/src/commands/mod.rs index 112dbabb5..0cae7a6d6 100644 --- a/crates/cli/src/commands/mod.rs +++ b/crates/cli/src/commands/mod.rs @@ -29,6 +29,32 @@ pub(crate) mod workflows; #[cfg(feature = "workflows_v2")] pub(crate) mod workflows_list; +use crate::error::GoodError; + +#[cfg(feature = "grit_tracing")] +use marzano_util::base64; +#[cfg(feature = "grit_tracing")] +use opentelemetry::{global, KeyValue}; +#[cfg(feature = "grit_tracing")] +use opentelemetry_otlp::WithExportConfig; +#[cfg(feature = "grit_tracing")] +use opentelemetry_sdk::propagation::TraceContextPropagator; +#[cfg(feature = "grit_tracing")] +use opentelemetry_sdk::trace::Tracer; +#[cfg(feature = "grit_tracing")] +use opentelemetry_sdk::{trace, Resource}; +#[cfg(feature = "grit_tracing")] +use std::collections::HashMap; +#[cfg(feature = "grit_tracing")] +use tracing::Instrument; +#[cfg(feature = "grit_tracing")] +use tracing::{event, span, Level}; +#[cfg(feature = "grit_tracing")] +#[allow(unused_imports)] +use tracing_subscriber::prelude::*; +#[cfg(feature = "grit_tracing")] +use tracing_subscriber::{layer::SubscriberExt, EnvFilter, Registry}; + #[cfg(feature = "docgen")] pub(crate) mod docgen; mod filters; @@ -266,7 +292,7 @@ fn write_analytics_event( } #[instrument] -pub async fn run_command() -> Result<()> { +async fn run_command() -> Result<()> { let app = App::parse(); // Use this *only* for analytics, not for any other purpose. let analytics_args = std::env::args().collect::>(); @@ -394,3 +420,138 @@ pub async fn run_command() -> Result<()> { res } + +#[cfg(feature = "grit_tracing")] +fn get_otel_key(env_name: &str) -> Option { + match std::env::var(env_name) { + Ok(key) => { + if key.is_empty() { + None + } else { + Some(key) + } + } + Err(_) => None, + } +} + +#[cfg(feature = "grit_tracing")] +fn get_otel_setup() -> Result> { + let mut exporter = opentelemetry_otlp::new_exporter() + .http() + .with_http_client(reqwest::Client::default()) + .with_timeout(std::time::Duration::from_millis(500)); + + let grafana_key = get_otel_key("GRAFANA_OTEL_KEY"); + let honeycomb_key = get_otel_key("HONEYCOMB_OTEL_KEY"); + let baselime_key = get_otel_key("BASELIME_OTEL_KEY"); + let hyperdx_key = get_otel_key("HYPERDX_OTEL_KEY"); + + match (grafana_key, honeycomb_key, baselime_key, hyperdx_key) { + (None, None, None, None) => { + #[cfg(feature = "server")] + eprintln!("No OTLP key found, tracing will be disabled"); + return Ok(None); + } + (Some(grafana_key), _, _, _) => { + let instance_id = "665534"; + let encoded = + base64::encode_from_string(format!("{}:{}", instance_id, grafana_key).as_str())?; + exporter = exporter + .with_endpoint("https://otlp-gateway-prod-us-central-0.grafana.net/otlp") + .with_headers(HashMap::from([( + "Authorization".into(), + format!("Basic {}", encoded), + )])); + eprintln!("Using Grafana OTLP key"); + } + (_, Some(honeycomb_key), _, _) => { + exporter = exporter + .with_endpoint("https://api.honeycomb.io") + .with_headers(HashMap::from([("x-honeycomb-team".into(), honeycomb_key)])); + eprintln!("Using Honeycomb OTLP key"); + } + (_, _, Some(baselime_key), _) => { + exporter = exporter + .with_endpoint("https://otel.baselime.io/v1/") + .with_headers(HashMap::from([ + ("x-api-key".into(), baselime_key), + ("x-baselime-dataset".into(), "otel".into()), + ])); + eprintln!("Using Baselime OTLP key"); + } + (_, _, _, Some(hyperdx_key)) => { + exporter = exporter + .with_endpoint("https://in-otel.hyperdx.io") + .with_headers(HashMap::from([("authorization".into(), hyperdx_key)])); + eprintln!("Using HyperDX OTLP key"); + } + } + + let env = get_otel_key("GRIT_DEPLOYMENT_ENV").unwrap_or_else(|| "prod".to_string()); + + let tracer = opentelemetry_otlp::new_pipeline() + .tracing() + .with_exporter(exporter) + .with_trace_config( + trace::config().with_resource(Resource::new(vec![KeyValue::new( + "service.name", + format!("{}_grit_marzano", env), + )])), + ) + .install_batch(opentelemetry_sdk::runtime::Tokio)?; + + Ok(Some(tracer)) +} + +pub async fn run_command_with_tracing() -> Result<()> { + #[cfg(feature = "grit_tracing")] + { + let tracer = get_otel_setup()?; + + if let Some(tracer) = tracer { + let env_filter = EnvFilter::try_from_default_env() + .unwrap_or(EnvFilter::new("TRACE")) + // We don't want to trace the tracing library itself + .add_directive("hyper::proto=off".parse().unwrap()); + + let telemetry = tracing_opentelemetry::layer().with_tracer(tracer); + let subscriber = Registry::default().with(env_filter).with(telemetry); + + global::set_text_map_propagator(TraceContextPropagator::new()); + tracing::subscriber::set_global_default(subscriber) + .expect("setting tracing default failed"); + + let root_span = span!(Level::INFO, "grit_marzano.cli_command",); + + let res = async move { + event!(Level::INFO, "starting the CLI!"); + + let res = run_command().await; + + event!(Level::INFO, "ending the CLI!"); + + res + } + .instrument(root_span) + .await; + + opentelemetry::global::shutdown_tracer_provider(); + + return res; + } + } + let subscriber = tracing::subscriber::NoSubscriber::new(); + tracing::subscriber::set_global_default(subscriber).expect("setting tracing default failed"); + + let res = run_command().await; + if let Err(ref e) = res { + if let Some(good) = e.downcast_ref::() { + if let Some(msg) = &good.message { + println!("{}", msg); + } + std::process::exit(1); + } + } + res +} diff --git a/crates/cli/src/resolver.rs b/crates/cli/src/resolver.rs index 0b5412a9d..c5fc89f7e 100644 --- a/crates/cli/src/resolver.rs +++ b/crates/cli/src/resolver.rs @@ -92,6 +92,7 @@ pub async fn get_grit_files_from(cwd: Option) -> Result Result { let cwd = std::env::current_dir()?; get_grit_files_from(Some(cwd)).await diff --git a/crates/cli/src/updater.rs b/crates/cli/src/updater.rs index 973a1fb5c..a910dd5b7 100644 --- a/crates/cli/src/updater.rs +++ b/crates/cli/src/updater.rs @@ -184,8 +184,8 @@ pub struct Updater { access_token: Option, } -#[allow(dead_code)] impl Updater { + #[tracing::instrument] pub async fn from_current_bin() -> Result { let current_bin = std::env::current_exe()?; let install_path = current_bin @@ -255,16 +255,6 @@ impl Updater { } } - pub fn get_log_file(&self, app: SupportedApp) -> Result { - let log_path = self - .bin_path - .parent() - .unwrap() - .join(format!("{}.log", app.get_bin_name())); - let log_file = std::fs::File::create(log_path).unwrap(); - Ok(log_file) - } - pub async fn check_for_update(&mut self) -> Result { if self.binaries.is_empty() { return Ok(false); diff --git a/crates/cli_bin/Cargo.toml b/crates/cli_bin/Cargo.toml index 1a0833110..27f59b26a 100644 --- a/crates/cli_bin/Cargo.toml +++ b/crates/cli_bin/Cargo.toml @@ -14,8 +14,22 @@ publish = false [dependencies] anyhow = { version = "1.0.70" } marzano-cli = { path = "../cli", default-features = false } +marzano-util = { path = "../util", default-features = false } tokio = { version = "1", features = ["full"] } +reqwest = { version = "0.11", features = ["json", "stream"] } + tracing = { version = "0.1.40", default-features = false } +opentelemetry-otlp = { version = "0.14.0", optional = true, features = [ + "http-proto", + "reqwest-client", +] } +opentelemetry = { version = "0.21.0", optional = true } +opentelemetry_sdk = { version = "0.21.1", optional = true, features = [ + "rt-tokio", +] } +tracing-opentelemetry = { version = "0.22.0", optional = true, default-features = false } +tracing-subscriber = { version = "0.3", default-features = false, optional = true } + [dev-dependencies] serde_json = "1.0.96" @@ -30,9 +44,14 @@ assert_cmd = "2.0.12" marzano-gritmodule = { path = "../gritmodule" } [features] -default = ["marzano-cli/default"] +default = ["marzano-cli/default", "grit_tracing"] grit_tracing = [ "marzano-cli/grit_tracing", + "opentelemetry", + "opentelemetry-otlp", + "opentelemetry_sdk", + "tracing-opentelemetry", + "tracing-subscriber", ] docgen = ["marzano-cli/docgen"] diff --git a/crates/cli_bin/src/main.rs b/crates/cli_bin/src/main.rs index 8ca82b88e..15df963a3 100644 --- a/crates/cli_bin/src/main.rs +++ b/crates/cli_bin/src/main.rs @@ -1,157 +1,8 @@ -#![cfg_attr(not(test), warn(unused_crate_dependencies))] -use marzano_cli::commands::run_command; -use marzano_cli::error::GoodError; -// We always instrument - -#[cfg(feature = "grit_tracing")] -use marzano_util::base64; -#[cfg(feature = "grit_tracing")] -use opentelemetry::{global, KeyValue}; -#[cfg(feature = "grit_tracing")] -use opentelemetry_otlp::WithExportConfig; -#[cfg(feature = "grit_tracing")] -use opentelemetry_sdk::propagation::TraceContextPropagator; -#[cfg(feature = "grit_tracing")] -use opentelemetry_sdk::trace::Tracer; -#[cfg(feature = "grit_tracing")] -use opentelemetry_sdk::{trace, Resource}; -#[cfg(feature = "grit_tracing")] -use std::collections::HashMap; -#[cfg(feature = "grit_tracing")] -use tracing::Instrument; -#[cfg(feature = "grit_tracing")] -use tracing::{event, span, Level}; -#[cfg(feature = "grit_tracing")] -#[allow(unused_imports)] -use tracing_subscriber::prelude::*; -#[cfg(feature = "grit_tracing")] -use tracing_subscriber::{layer::SubscriberExt, EnvFilter, Registry}; +use marzano_cli::commands::run_command_with_tracing; use anyhow::Result; -#[cfg(feature = "workflows_v2")] -mod workflows; - -#[cfg(feature = "grit_tracing")] -fn get_otel_key(env_name: &str) -> Option { - match std::env::var(env_name) { - Ok(key) => { - if key.is_empty() { - None - } else { - Some(key) - } - } - Err(_) => None, - } -} - -#[cfg(feature = "grit_tracing")] -fn get_otel_setup() -> Result { - use anyhow::bail; - - let mut exporter = opentelemetry_otlp::new_exporter() - .http() - .with_http_client(reqwest::Client::default()) - .with_timeout(std::time::Duration::from_millis(500)); - - let grafana_key = get_otel_key("GRAFANA_OTEL_KEY"); - let honeycomb_key = get_otel_key("HONEYCOMB_OTEL_KEY"); - let baselime_key = get_otel_key("BASELIME_OTEL_KEY"); - - match (grafana_key, honeycomb_key, baselime_key) { - (None, None, None) => bail!("no OTLP key found"), - (Some(grafana_key), None, None) => { - let instance_id = "665534"; - let encoded = - base64::encode_from_string(format!("{}:{}", instance_id, grafana_key).as_str())?; - exporter = exporter - .with_endpoint("https://otlp-gateway-prod-us-central-0.grafana.net/otlp") - .with_headers(HashMap::from([( - "Authorization".into(), - format!("Basic {}", encoded), - )])); - } - (None, Some(honeycomb_key), None) => { - exporter = exporter - .with_endpoint("https://api.honeycomb.io") - .with_headers(HashMap::from([("x-honeycomb-team".into(), honeycomb_key)])); - } - (None, None, Some(baselime_key)) => { - exporter = exporter - .with_endpoint("https://otel.baselime.io/v1/") - .with_headers(HashMap::from([ - ("x-api-key".into(), baselime_key), - ("x-baselime-dataset".into(), "otel".into()), - ])); - } - _ => bail!("multiple OTLP keys found"), - } - - let tracer = opentelemetry_otlp::new_pipeline() - .tracing() - .with_exporter(exporter) - .with_trace_config( - trace::config().with_resource(Resource::new(vec![KeyValue::new( - "service.name", - "grit_marzano", - )])), - ) - .install_batch(opentelemetry_sdk::runtime::Tokio)?; - - Ok(tracer) -} #[tokio::main] async fn main() -> Result<()> { - #[cfg(feature = "grit_tracing")] - { - let tracer = get_otel_setup()?; - - let env_filter = EnvFilter::try_from_default_env() - .unwrap_or(EnvFilter::new("TRACE")) - // We don't want to trace the tracing library itself - .add_directive("hyper::proto=off".parse().unwrap()); - - let telemetry = tracing_opentelemetry::layer().with_tracer(tracer); - let subscriber = Registry::default().with(env_filter).with(telemetry); - - global::set_text_map_propagator(TraceContextPropagator::new()); - tracing::subscriber::set_global_default(subscriber) - .expect("setting tracing default failed"); - - let root_span = span!(Level::INFO, "grit_marzano.cli_command",); - - let res = async move { - event!(Level::INFO, "starting the CLI!"); - - let res = run_command().await; - - event!(Level::INFO, "ending the CLI!"); - - res - } - .instrument(root_span) - .await; - - opentelemetry::global::shutdown_tracer_provider(); - - return Ok(()); - } - #[cfg(not(feature = "grit_tracing"))] - { - let subscriber = tracing::subscriber::NoSubscriber::new(); - tracing::subscriber::set_global_default(subscriber) - .expect("setting tracing default failed"); - - let res = run_command().await; - if let Err(ref e) = res { - if let Some(good) = e.downcast_ref::() { - if let Some(msg) = &good.message { - println!("{}", msg); - } - std::process::exit(1); - } - } - return res; - } + run_command_with_tracing().await } diff --git a/crates/cli_bin/tests/plumbing.rs b/crates/cli_bin/tests/plumbing.rs index 53851662d..05f35e9a7 100644 --- a/crates/cli_bin/tests/plumbing.rs +++ b/crates/cli_bin/tests/plumbing.rs @@ -150,9 +150,6 @@ fn checks_patterns_round_trip() -> Result<()> { let result = cmd.output()?; - // no stderr - assert_eq!(String::from_utf8(result.stderr)?, ""); - // Result must be successful assert!(result.status.success()); @@ -173,8 +170,6 @@ fn checks_patterns_without_samples() -> Result<()> { cmd.write_stdin(String::from_utf8(config.into())?); let result = cmd.output()?; - // no stderr - assert_eq!(String::from_utf8(result.stderr)?, ""); // Result must be successful assert!(result.status.success()); @@ -225,9 +220,6 @@ fn checks_multifile_patterns() -> Result<()> { cmd.write_stdin(String::from_utf8(output.stdout)?); let result = cmd.output()?; - // no stderr - assert_eq!(String::from_utf8(result.stderr)?, ""); - assert!(result.status.success()); Ok(()) diff --git a/crates/core/src/problem.rs b/crates/core/src/problem.rs index 8cbd26ed2..1e860c7b5 100644 --- a/crates/core/src/problem.rs +++ b/crates/core/src/problem.rs @@ -41,6 +41,8 @@ use std::{ }; use std::{fmt::Debug, str::FromStr}; use tracing::{event, Level}; +#[cfg(feature = "grit_tracing")] +use tracing_opentelemetry::OpenTelemetrySpanExt; #[derive(Debug)] pub struct Problem { @@ -336,7 +338,7 @@ impl Problem { self.execute_shared(files, context, tx, cache) } - #[cfg_attr(feature = "grit_tracing", instrument(skip_all))] + #[cfg_attr(feature = "grit_tracing", tracing::instrument(skip_all))] pub(crate) fn execute_shared( &self, files: Vec, @@ -345,7 +347,7 @@ impl Problem { cache: &impl GritCache, ) { #[cfg(feature = "grit_tracing")] - let parent_span = span!(Level::INFO, "execute_shared_body",).entered(); + let parent_span = tracing::span!(Level::INFO, "execute_shared_body",).entered(); #[cfg(feature = "grit_tracing")] let parent_cx = parent_span.context(); diff --git a/crates/gritmodule/Cargo.toml b/crates/gritmodule/Cargo.toml index ad8c0a9f4..5dd5e73c7 100644 --- a/crates/gritmodule/Cargo.toml +++ b/crates/gritmodule/Cargo.toml @@ -33,6 +33,7 @@ log = { version = "0.4.19" } reqwest = { version = "0.11.22", features = ["blocking", "json"] } ignore = { version = "0.4.20" } homedir = { version = "0.2.1" } +tracing = { version = "0.1.40", default-features = false, features = [] } [dev-dependencies] insta = { version = "1.30.0", features = ["yaml"] } diff --git a/crates/gritmodule/src/patterns_directory.rs b/crates/gritmodule/src/patterns_directory.rs index bbf55c040..2e13d3ee6 100644 --- a/crates/gritmodule/src/patterns_directory.rs +++ b/crates/gritmodule/src/patterns_directory.rs @@ -147,6 +147,7 @@ impl PatternsDirectory { } } + #[tracing::instrument] fn get_language_and_universal_directory( &self, language: PatternLanguage,