From 1be3c9818677620be52a5f6d6232ecb77365e1b6 Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Tue, 7 May 2024 18:43:34 -0700 Subject: [PATCH 01/14] fix: repair tracing and add hyperdx --- crates/cli/Cargo.toml | 2 ++ crates/cli_bin/Cargo.toml | 21 ++++++++++++++++++++- crates/cli_bin/src/main.rs | 16 +++++++++++----- crates/core/src/problem.rs | 6 ++++-- 4 files changed, 37 insertions(+), 8 deletions(-) diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index 7aab3140b..cab0fd4eb 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -71,6 +71,7 @@ opentelemetry_sdk = { version = "0.21.1", optional = true, features = [ tracing-opentelemetry = { version = "0.22.0", optional = true, default-features = false } tracing = { version = "0.1.40", default-features = false, features = [] } +tracing-subscriber = { version = "0.3", default-features = false, optional = true } [target.'cfg(not(windows))'.dependencies] openssl = { version = "0.10", features = ["vendored"] } @@ -116,6 +117,7 @@ grit_tracing = [ "dep:opentelemetry", "dep:opentelemetry_sdk", "dep:tracing-opentelemetry", + "dep:tracing-subscriber", "marzano-core/grit_tracing", ] external_functions = ["marzano-core/external_functions"] diff --git a/crates/cli_bin/Cargo.toml b/crates/cli_bin/Cargo.toml index 1a0833110..27f59b26a 100644 --- a/crates/cli_bin/Cargo.toml +++ b/crates/cli_bin/Cargo.toml @@ -14,8 +14,22 @@ publish = false [dependencies] anyhow = { version = "1.0.70" } marzano-cli = { path = "../cli", default-features = false } +marzano-util = { path = "../util", default-features = false } tokio = { version = "1", features = ["full"] } +reqwest = { version = "0.11", features = ["json", "stream"] } + tracing = { version = "0.1.40", default-features = false } +opentelemetry-otlp = { version = "0.14.0", optional = true, features = [ + "http-proto", + "reqwest-client", +] } +opentelemetry = { version = "0.21.0", optional = true } +opentelemetry_sdk = { version = "0.21.1", optional = true, features = [ + "rt-tokio", +] } +tracing-opentelemetry = { version = "0.22.0", optional = true, default-features = false } +tracing-subscriber = { version = "0.3", default-features = false, optional = true } + [dev-dependencies] serde_json = "1.0.96" @@ -30,9 +44,14 @@ assert_cmd = "2.0.12" marzano-gritmodule = { path = "../gritmodule" } [features] -default = ["marzano-cli/default"] +default = ["marzano-cli/default", "grit_tracing"] grit_tracing = [ "marzano-cli/grit_tracing", + "opentelemetry", + "opentelemetry-otlp", + "opentelemetry_sdk", + "tracing-opentelemetry", + "tracing-subscriber", ] docgen = ["marzano-cli/docgen"] diff --git a/crates/cli_bin/src/main.rs b/crates/cli_bin/src/main.rs index 8ca82b88e..a98cdfa42 100644 --- a/crates/cli_bin/src/main.rs +++ b/crates/cli_bin/src/main.rs @@ -57,10 +57,11 @@ fn get_otel_setup() -> Result { let grafana_key = get_otel_key("GRAFANA_OTEL_KEY"); let honeycomb_key = get_otel_key("HONEYCOMB_OTEL_KEY"); let baselime_key = get_otel_key("BASELIME_OTEL_KEY"); + let hyperdx_key = get_otel_key("HYPERDX_OTEL_KEY"); - match (grafana_key, honeycomb_key, baselime_key) { - (None, None, None) => bail!("no OTLP key found"), - (Some(grafana_key), None, None) => { + match (grafana_key, honeycomb_key, baselime_key, hyperdx_key) { + (None, None, None, None) => bail!("no OTLP key found"), + (Some(grafana_key), None, None, None) => { let instance_id = "665534"; let encoded = base64::encode_from_string(format!("{}:{}", instance_id, grafana_key).as_str())?; @@ -71,12 +72,12 @@ fn get_otel_setup() -> Result { format!("Basic {}", encoded), )])); } - (None, Some(honeycomb_key), None) => { + (None, Some(honeycomb_key), None, None) => { exporter = exporter .with_endpoint("https://api.honeycomb.io") .with_headers(HashMap::from([("x-honeycomb-team".into(), honeycomb_key)])); } - (None, None, Some(baselime_key)) => { + (None, None, Some(baselime_key), None) => { exporter = exporter .with_endpoint("https://otel.baselime.io/v1/") .with_headers(HashMap::from([ @@ -84,6 +85,11 @@ fn get_otel_setup() -> Result { ("x-baselime-dataset".into(), "otel".into()), ])); } + (None, None, None, Some(hyperdx_key)) => { + exporter = exporter + .with_endpoint("https://in-otel.hyperdx.io") + .with_headers(HashMap::from([("authorization".into(), hyperdx_key)])); + } _ => bail!("multiple OTLP keys found"), } diff --git a/crates/core/src/problem.rs b/crates/core/src/problem.rs index 8cbd26ed2..1e860c7b5 100644 --- a/crates/core/src/problem.rs +++ b/crates/core/src/problem.rs @@ -41,6 +41,8 @@ use std::{ }; use std::{fmt::Debug, str::FromStr}; use tracing::{event, Level}; +#[cfg(feature = "grit_tracing")] +use tracing_opentelemetry::OpenTelemetrySpanExt; #[derive(Debug)] pub struct Problem { @@ -336,7 +338,7 @@ impl Problem { self.execute_shared(files, context, tx, cache) } - #[cfg_attr(feature = "grit_tracing", instrument(skip_all))] + #[cfg_attr(feature = "grit_tracing", tracing::instrument(skip_all))] pub(crate) fn execute_shared( &self, files: Vec, @@ -345,7 +347,7 @@ impl Problem { cache: &impl GritCache, ) { #[cfg(feature = "grit_tracing")] - let parent_span = span!(Level::INFO, "execute_shared_body",).entered(); + let parent_span = tracing::span!(Level::INFO, "execute_shared_body",).entered(); #[cfg(feature = "grit_tracing")] let parent_cx = parent_span.context(); From 1fc502525db185e16b23b89a23a9a7e6db5ce32b Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Tue, 7 May 2024 19:04:38 -0700 Subject: [PATCH 02/14] [skip ci] yolo --- crates/cli/src/commands/apply_pattern.rs | 16 +++++ crates/cli/src/commands/filters.rs | 1 + crates/cli/src/updater.rs | 12 +--- crates/cli_bin/src/main.rs | 76 ++++++++++++------------ 4 files changed, 57 insertions(+), 48 deletions(-) diff --git a/crates/cli/src/commands/apply_pattern.rs b/crates/cli/src/commands/apply_pattern.rs index 1e3fd9e1b..83e40fe69 100644 --- a/crates/cli/src/commands/apply_pattern.rs +++ b/crates/cli/src/commands/apply_pattern.rs @@ -191,12 +191,17 @@ pub(crate) async fn run_apply_pattern( // Get the current directory let cwd = std::env::current_dir().unwrap(); + #[cfg(feature = "grit_tracing")] + let module_resolution = span!(tracing::Level::INFO, "module_resolution",).entered(); + // Construct a resolver let resolver = GritModuleResolver::new(cwd.to_str().unwrap()); let current_repo_root = marzano_gritmodule::fetcher::LocalRepo::from_dir(&cwd) .await .map(|repo| repo.root()) .transpose()?; + #[cfg(feature = "grit_tracing")] + module_resolution.exit(); let mut emitter = create_emitter( &format, @@ -213,6 +218,9 @@ pub(crate) async fn run_apply_pattern( extract_filter_ranges(&shared, current_repo_root.as_ref()) ); + #[cfg(feature = "grit_tracing")] + let span_libs = span!(tracing::Level::INFO, "build_libs",).entered(); + let (my_input, lang) = if let Some(pattern_libs) = pattern_libs { ( ApplyInput { @@ -223,6 +231,9 @@ pub(crate) async fn run_apply_pattern( lang, ) } else { + #[cfg(feature = "grit_tracing")] + let stdlib_download_span = span!(tracing::Level::INFO, "download_modules",).entered(); + let mod_dir = find_grit_modules_dir(cwd.clone()).await; if !env::var("GRIT_DOWNLOADS_DISABLED") .unwrap_or_else(|_| "false".to_owned()) @@ -236,6 +247,9 @@ pub(crate) async fn run_apply_pattern( ); } + #[cfg(feature = "grit_tracing")] + stdlib_download_span.exit(); + let warn_uncommitted = !arg.dry_run && !arg.force && has_uncommitted_changes(cwd.clone()).await; if warn_uncommitted { @@ -365,6 +379,8 @@ pub(crate) async fn run_apply_pattern( emitter, resolver.make_pattern(&my_input.pattern_body, current_name) ); + #[cfg(feature = "grit_tracing")] + span_libs.exit(); let CompilationResult { problem: compiled, diff --git a/crates/cli/src/commands/filters.rs b/crates/cli/src/commands/filters.rs index c292a307c..b53731cd2 100644 --- a/crates/cli/src/commands/filters.rs +++ b/crates/cli/src/commands/filters.rs @@ -26,6 +26,7 @@ pub struct SharedFilterArgs { pub(crate) only_in_diff: Option>, } +#[tracing::instrument] pub(crate) fn extract_filter_ranges( args: &SharedFilterArgs, root: Option<&PathBuf>, diff --git a/crates/cli/src/updater.rs b/crates/cli/src/updater.rs index 973a1fb5c..a910dd5b7 100644 --- a/crates/cli/src/updater.rs +++ b/crates/cli/src/updater.rs @@ -184,8 +184,8 @@ pub struct Updater { access_token: Option, } -#[allow(dead_code)] impl Updater { + #[tracing::instrument] pub async fn from_current_bin() -> Result { let current_bin = std::env::current_exe()?; let install_path = current_bin @@ -255,16 +255,6 @@ impl Updater { } } - pub fn get_log_file(&self, app: SupportedApp) -> Result { - let log_path = self - .bin_path - .parent() - .unwrap() - .join(format!("{}.log", app.get_bin_name())); - let log_file = std::fs::File::create(log_path).unwrap(); - Ok(log_file) - } - pub async fn check_for_update(&mut self) -> Result { if self.binaries.is_empty() { return Ok(false); diff --git a/crates/cli_bin/src/main.rs b/crates/cli_bin/src/main.rs index a98cdfa42..9c6b59fb7 100644 --- a/crates/cli_bin/src/main.rs +++ b/crates/cli_bin/src/main.rs @@ -46,7 +46,7 @@ fn get_otel_key(env_name: &str) -> Option { } #[cfg(feature = "grit_tracing")] -fn get_otel_setup() -> Result { +fn get_otel_setup() -> Result> { use anyhow::bail; let mut exporter = opentelemetry_otlp::new_exporter() @@ -60,7 +60,11 @@ fn get_otel_setup() -> Result { let hyperdx_key = get_otel_key("HYPERDX_OTEL_KEY"); match (grafana_key, honeycomb_key, baselime_key, hyperdx_key) { - (None, None, None, None) => bail!("no OTLP key found"), + (None, None, None, None) => { + // NOTE: we don't include tracing in released builds, so this won't appear + eprintln!("No OTLP key found, tracing will be disabled"); + return Ok(None); + } (Some(grafana_key), None, None, None) => { let instance_id = "665534"; let encoded = @@ -104,7 +108,7 @@ fn get_otel_setup() -> Result { ) .install_batch(opentelemetry_sdk::runtime::Tokio)?; - Ok(tracer) + Ok(Some(tracer)) } #[tokio::main] @@ -113,51 +117,49 @@ async fn main() -> Result<()> { { let tracer = get_otel_setup()?; - let env_filter = EnvFilter::try_from_default_env() - .unwrap_or(EnvFilter::new("TRACE")) - // We don't want to trace the tracing library itself - .add_directive("hyper::proto=off".parse().unwrap()); + if let Some(tracer) = tracer { + let env_filter = EnvFilter::try_from_default_env() + .unwrap_or(EnvFilter::new("TRACE")) + // We don't want to trace the tracing library itself + .add_directive("hyper::proto=off".parse().unwrap()); - let telemetry = tracing_opentelemetry::layer().with_tracer(tracer); - let subscriber = Registry::default().with(env_filter).with(telemetry); + let telemetry = tracing_opentelemetry::layer().with_tracer(tracer); + let subscriber = Registry::default().with(env_filter).with(telemetry); - global::set_text_map_propagator(TraceContextPropagator::new()); - tracing::subscriber::set_global_default(subscriber) - .expect("setting tracing default failed"); + global::set_text_map_propagator(TraceContextPropagator::new()); + tracing::subscriber::set_global_default(subscriber) + .expect("setting tracing default failed"); - let root_span = span!(Level::INFO, "grit_marzano.cli_command",); + let root_span = span!(Level::INFO, "grit_marzano.cli_command",); - let res = async move { - event!(Level::INFO, "starting the CLI!"); + let res = async move { + event!(Level::INFO, "starting the CLI!"); - let res = run_command().await; + let res = run_command().await; - event!(Level::INFO, "ending the CLI!"); + event!(Level::INFO, "ending the CLI!"); - res - } - .instrument(root_span) - .await; + res + } + .instrument(root_span) + .await; - opentelemetry::global::shutdown_tracer_provider(); + opentelemetry::global::shutdown_tracer_provider(); - return Ok(()); + return Ok(()); + } } - #[cfg(not(feature = "grit_tracing"))] - { - let subscriber = tracing::subscriber::NoSubscriber::new(); - tracing::subscriber::set_global_default(subscriber) - .expect("setting tracing default failed"); - - let res = run_command().await; - if let Err(ref e) = res { - if let Some(good) = e.downcast_ref::() { - if let Some(msg) = &good.message { - println!("{}", msg); - } - std::process::exit(1); + let subscriber = tracing::subscriber::NoSubscriber::new(); + tracing::subscriber::set_global_default(subscriber).expect("setting tracing default failed"); + + let res = run_command().await; + if let Err(ref e) = res { + if let Some(good) = e.downcast_ref::() { + if let Some(msg) = &good.message { + println!("{}", msg); } + std::process::exit(1); } - return res; } + return res; } From 7baed91e49a00bed5e4c9ec9293031366d586daf Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Tue, 7 May 2024 19:14:54 -0700 Subject: [PATCH 03/14] run the tests --- crates/cli/src/commands/apply_pattern.rs | 16 ++++++++++++++-- crates/cli/src/resolver.rs | 1 + crates/gritmodule/Cargo.toml | 1 + crates/gritmodule/src/patterns_directory.rs | 1 + 4 files changed, 17 insertions(+), 2 deletions(-) diff --git a/crates/cli/src/commands/apply_pattern.rs b/crates/cli/src/commands/apply_pattern.rs index 83e40fe69..aaed820b1 100644 --- a/crates/cli/src/commands/apply_pattern.rs +++ b/crates/cli/src/commands/apply_pattern.rs @@ -219,7 +219,7 @@ pub(crate) async fn run_apply_pattern( ); #[cfg(feature = "grit_tracing")] - let span_libs = span!(tracing::Level::INFO, "build_libs",).entered(); + let span_libs = span!(tracing::Level::INFO, "prep_libs",).entered(); let (my_input, lang) = if let Some(pattern_libs) = pattern_libs { ( @@ -232,7 +232,7 @@ pub(crate) async fn run_apply_pattern( ) } else { #[cfg(feature = "grit_tracing")] - let stdlib_download_span = span!(tracing::Level::INFO, "download_modules",).entered(); + let stdlib_download_span = span!(tracing::Level::INFO, "stdlib_download",).entered(); let mod_dir = find_grit_modules_dir(cwd.clone()).await; if !env::var("GRIT_DOWNLOADS_DISABLED") @@ -268,7 +268,11 @@ pub(crate) async fn run_apply_pattern( } } + #[cfg(feature = "grit_tracing")] + let grit_file_discovery = span!(tracing::Level::INFO, "grit_file_discovery",).entered(); + let pattern_libs = flushable_unwrap!(emitter, get_grit_files_from_cwd().await); + let (mut lang, pattern_body) = if pattern.ends_with(".grit") || pattern.ends_with(".md") { match fs::read_to_string(pattern.clone()).await { Ok(pb) => { @@ -344,6 +348,8 @@ pub(crate) async fn run_apply_pattern( emitter, pattern_libs.get_language_directory_or_default(lang) ); + #[cfg(feature = "grit_tracing")] + grit_file_discovery.exit(); ( ApplyInput { pattern_body, @@ -366,6 +372,8 @@ pub(crate) async fn run_apply_pattern( return Ok(()); } + #[cfg(feature = "grit_tracing")] + let collect_name = span!(tracing::Level::INFO, "collect_name",).entered(); let current_name = if is_pattern_name(&pattern) { Some(pattern.trim_end_matches("()").to_string()) } else { @@ -375,10 +383,14 @@ pub(crate) async fn run_apply_pattern( .find(|(_, body)| body.trim() == pattern.trim()) .map(|(name, _)| name.clone()) }; + #[cfg(feature = "grit_tracing")] + collect_name.exit(); + let pattern: crate::resolver::RichPattern<'_> = flushable_unwrap!( emitter, resolver.make_pattern(&my_input.pattern_body, current_name) ); + #[cfg(feature = "grit_tracing")] span_libs.exit(); diff --git a/crates/cli/src/resolver.rs b/crates/cli/src/resolver.rs index 0b5412a9d..c5fc89f7e 100644 --- a/crates/cli/src/resolver.rs +++ b/crates/cli/src/resolver.rs @@ -92,6 +92,7 @@ pub async fn get_grit_files_from(cwd: Option) -> Result Result { let cwd = std::env::current_dir()?; get_grit_files_from(Some(cwd)).await diff --git a/crates/gritmodule/Cargo.toml b/crates/gritmodule/Cargo.toml index ad8c0a9f4..5dd5e73c7 100644 --- a/crates/gritmodule/Cargo.toml +++ b/crates/gritmodule/Cargo.toml @@ -33,6 +33,7 @@ log = { version = "0.4.19" } reqwest = { version = "0.11.22", features = ["blocking", "json"] } ignore = { version = "0.4.20" } homedir = { version = "0.2.1" } +tracing = { version = "0.1.40", default-features = false, features = [] } [dev-dependencies] insta = { version = "1.30.0", features = ["yaml"] } diff --git a/crates/gritmodule/src/patterns_directory.rs b/crates/gritmodule/src/patterns_directory.rs index bbf55c040..2e13d3ee6 100644 --- a/crates/gritmodule/src/patterns_directory.rs +++ b/crates/gritmodule/src/patterns_directory.rs @@ -147,6 +147,7 @@ impl PatternsDirectory { } } + #[tracing::instrument] fn get_language_and_universal_directory( &self, language: PatternLanguage, From 1992fdcae9bcfaabaffb6ad7cbbcdf30cf3cd814 Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Tue, 7 May 2024 19:17:16 -0700 Subject: [PATCH 04/14] run the tests From de544fe5ed2b202bf2c4de076fba32178c9b4739 Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Tue, 7 May 2024 19:23:42 -0700 Subject: [PATCH 05/14] clippy --- Cargo.lock | 9 +++++++++ crates/cli/src/commands/apply_pattern.rs | 1 + crates/cli_bin/src/main.rs | 2 +- 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 9a2bfa6f2..581995cd5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1944,14 +1944,21 @@ dependencies = [ "insta", "marzano-cli", "marzano-gritmodule", + "marzano-util", "ntest", + "opentelemetry", + "opentelemetry-otlp", + "opentelemetry_sdk", "predicates", "rayon", "regex", + "reqwest", "serde_json", "tempfile", "tokio", "tracing", + "tracing-opentelemetry", + "tracing-subscriber", ] [[package]] @@ -2018,6 +2025,7 @@ dependencies = [ "tokio", "tracing", "tracing-opentelemetry", + "tracing-subscriber", "trim-margin", "uuid", ] @@ -2086,6 +2094,7 @@ dependencies = [ "serde_yaml", "tempfile", "tokio", + "tracing", "tree-sitter-facade-sg", ] diff --git a/crates/cli/src/commands/apply_pattern.rs b/crates/cli/src/commands/apply_pattern.rs index aaed820b1..f6fc20aaa 100644 --- a/crates/cli/src/commands/apply_pattern.rs +++ b/crates/cli/src/commands/apply_pattern.rs @@ -7,6 +7,7 @@ use tracing::instrument; #[cfg(feature = "grit_tracing")] use tracing::span; #[cfg(feature = "grit_tracing")] +#[allow(unused_imports)] use tracing_opentelemetry::OpenTelemetrySpanExt as _; use grit_util::Position; diff --git a/crates/cli_bin/src/main.rs b/crates/cli_bin/src/main.rs index 9c6b59fb7..9ae1e6ce9 100644 --- a/crates/cli_bin/src/main.rs +++ b/crates/cli_bin/src/main.rs @@ -132,7 +132,7 @@ async fn main() -> Result<()> { let root_span = span!(Level::INFO, "grit_marzano.cli_command",); - let res = async move { + let _res = async move { event!(Level::INFO, "starting the CLI!"); let res = run_command().await; From fd63e9f57d208e37f2bb89ae299a8361871e1e25 Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Tue, 7 May 2024 19:25:31 -0700 Subject: [PATCH 06/14] default tracing --- crates/cli/Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index cab0fd4eb..89a97d6a6 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -88,6 +88,7 @@ default = [ "external_functions", "updater", "workflows_v2", + "grit_tracing", # "remote_workflows", # "server", # "remote_redis", From 7d27e7c56761eff1b353b77485b78581b9bfa37b Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Tue, 7 May 2024 19:42:02 -0700 Subject: [PATCH 07/14] run the tests --- crates/cli/Cargo.toml | 1 + crates/cli/src/commands/mod.rs | 163 ++++++++++++++++++++++++++++++++- crates/cli_bin/src/main.rs | 160 +------------------------------- 3 files changed, 165 insertions(+), 159 deletions(-) diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index 89a97d6a6..823ddaefe 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -111,6 +111,7 @@ server = [ "remote_pubsub", "ai_builtins", "dep:cli_server", + "grit_tracing" ] updater = [] grit_tracing = [ diff --git a/crates/cli/src/commands/mod.rs b/crates/cli/src/commands/mod.rs index 112dbabb5..242334b35 100644 --- a/crates/cli/src/commands/mod.rs +++ b/crates/cli/src/commands/mod.rs @@ -29,6 +29,32 @@ pub(crate) mod workflows; #[cfg(feature = "workflows_v2")] pub(crate) mod workflows_list; +use crate::error::GoodError; + +#[cfg(feature = "grit_tracing")] +use marzano_util::base64; +#[cfg(feature = "grit_tracing")] +use opentelemetry::{global, KeyValue}; +#[cfg(feature = "grit_tracing")] +use opentelemetry_otlp::WithExportConfig; +#[cfg(feature = "grit_tracing")] +use opentelemetry_sdk::propagation::TraceContextPropagator; +#[cfg(feature = "grit_tracing")] +use opentelemetry_sdk::trace::Tracer; +#[cfg(feature = "grit_tracing")] +use opentelemetry_sdk::{trace, Resource}; +#[cfg(feature = "grit_tracing")] +use std::collections::HashMap; +#[cfg(feature = "grit_tracing")] +use tracing::Instrument; +#[cfg(feature = "grit_tracing")] +use tracing::{event, span, Level}; +#[cfg(feature = "grit_tracing")] +#[allow(unused_imports)] +use tracing_subscriber::prelude::*; +#[cfg(feature = "grit_tracing")] +use tracing_subscriber::{layer::SubscriberExt, EnvFilter, Registry}; + #[cfg(feature = "docgen")] pub(crate) mod docgen; mod filters; @@ -266,7 +292,7 @@ fn write_analytics_event( } #[instrument] -pub async fn run_command() -> Result<()> { +async fn run_command() -> Result<()> { let app = App::parse(); // Use this *only* for analytics, not for any other purpose. let analytics_args = std::env::args().collect::>(); @@ -394,3 +420,138 @@ pub async fn run_command() -> Result<()> { res } + +#[cfg(feature = "grit_tracing")] +fn get_otel_key(env_name: &str) -> Option { + match std::env::var(env_name) { + Ok(key) => { + if key.is_empty() { + None + } else { + Some(key) + } + } + Err(_) => None, + } +} + +#[cfg(feature = "grit_tracing")] +fn get_otel_setup() -> Result> { + use anyhow::bail; + + let mut exporter = opentelemetry_otlp::new_exporter() + .http() + .with_http_client(reqwest::Client::default()) + .with_timeout(std::time::Duration::from_millis(500)); + + let grafana_key = get_otel_key("GRAFANA_OTEL_KEY"); + let honeycomb_key = get_otel_key("HONEYCOMB_OTEL_KEY"); + let baselime_key = get_otel_key("BASELIME_OTEL_KEY"); + let hyperdx_key = get_otel_key("HYPERDX_OTEL_KEY"); + + match (grafana_key, honeycomb_key, baselime_key, hyperdx_key) { + (None, None, None, None) => { + // NOTE: we don't include tracing in released builds, so this won't appear + eprintln!("No OTLP key found, tracing will be disabled"); + return Ok(None); + } + (Some(grafana_key), _, _, _) => { + let instance_id = "665534"; + let encoded = + base64::encode_from_string(format!("{}:{}", instance_id, grafana_key).as_str())?; + exporter = exporter + .with_endpoint("https://otlp-gateway-prod-us-central-0.grafana.net/otlp") + .with_headers(HashMap::from([( + "Authorization".into(), + format!("Basic {}", encoded), + )])); + eprintln!("Using Grafana OTLP key"); + } + (_, Some(honeycomb_key), _, _) => { + exporter = exporter + .with_endpoint("https://api.honeycomb.io") + .with_headers(HashMap::from([("x-honeycomb-team".into(), honeycomb_key)])); + eprintln!("Using Honeycomb OTLP key"); + } + (_, _, Some(baselime_key), _) => { + exporter = exporter + .with_endpoint("https://otel.baselime.io/v1/") + .with_headers(HashMap::from([ + ("x-api-key".into(), baselime_key), + ("x-baselime-dataset".into(), "otel".into()), + ])); + eprintln!("Using Baselime OTLP key"); + } + (_, _, _, Some(hyperdx_key)) => { + exporter = exporter + .with_endpoint("https://in-otel.hyperdx.io") + .with_headers(HashMap::from([("authorization".into(), hyperdx_key)])); + eprintln!("Using HyperDX OTLP key"); + } + } + + let tracer = opentelemetry_otlp::new_pipeline() + .tracing() + .with_exporter(exporter) + .with_trace_config( + trace::config().with_resource(Resource::new(vec![KeyValue::new( + "service.name", + "grit_marzano", + )])), + ) + .install_batch(opentelemetry_sdk::runtime::Tokio)?; + + Ok(Some(tracer)) +} + +pub async fn run_command_with_tracing() -> Result<()> { + #[cfg(feature = "grit_tracing")] + { + let tracer = get_otel_setup()?; + + if let Some(tracer) = tracer { + let env_filter = EnvFilter::try_from_default_env() + .unwrap_or(EnvFilter::new("TRACE")) + // We don't want to trace the tracing library itself + .add_directive("hyper::proto=off".parse().unwrap()); + + let telemetry = tracing_opentelemetry::layer().with_tracer(tracer); + let subscriber = Registry::default().with(env_filter).with(telemetry); + + global::set_text_map_propagator(TraceContextPropagator::new()); + tracing::subscriber::set_global_default(subscriber) + .expect("setting tracing default failed"); + + let root_span = span!(Level::INFO, "grit_marzano.cli_command",); + + let _res = async move { + event!(Level::INFO, "starting the CLI!"); + + let res = run_command().await; + + event!(Level::INFO, "ending the CLI!"); + + res + } + .instrument(root_span) + .await; + + opentelemetry::global::shutdown_tracer_provider(); + + return Ok(()); + } + } + let subscriber = tracing::subscriber::NoSubscriber::new(); + tracing::subscriber::set_global_default(subscriber).expect("setting tracing default failed"); + + let res = run_command().await; + if let Err(ref e) = res { + if let Some(good) = e.downcast_ref::() { + if let Some(msg) = &good.message { + println!("{}", msg); + } + std::process::exit(1); + } + } + return res; +} diff --git a/crates/cli_bin/src/main.rs b/crates/cli_bin/src/main.rs index 9ae1e6ce9..3370ec05a 100644 --- a/crates/cli_bin/src/main.rs +++ b/crates/cli_bin/src/main.rs @@ -1,165 +1,9 @@ -#![cfg_attr(not(test), warn(unused_crate_dependencies))] -use marzano_cli::commands::run_command; +use marzano_cli::commands::run_command_with_tracing; use marzano_cli::error::GoodError; -// We always instrument - -#[cfg(feature = "grit_tracing")] -use marzano_util::base64; -#[cfg(feature = "grit_tracing")] -use opentelemetry::{global, KeyValue}; -#[cfg(feature = "grit_tracing")] -use opentelemetry_otlp::WithExportConfig; -#[cfg(feature = "grit_tracing")] -use opentelemetry_sdk::propagation::TraceContextPropagator; -#[cfg(feature = "grit_tracing")] -use opentelemetry_sdk::trace::Tracer; -#[cfg(feature = "grit_tracing")] -use opentelemetry_sdk::{trace, Resource}; -#[cfg(feature = "grit_tracing")] -use std::collections::HashMap; -#[cfg(feature = "grit_tracing")] -use tracing::Instrument; -#[cfg(feature = "grit_tracing")] -use tracing::{event, span, Level}; -#[cfg(feature = "grit_tracing")] -#[allow(unused_imports)] -use tracing_subscriber::prelude::*; -#[cfg(feature = "grit_tracing")] -use tracing_subscriber::{layer::SubscriberExt, EnvFilter, Registry}; use anyhow::Result; -#[cfg(feature = "workflows_v2")] -mod workflows; - -#[cfg(feature = "grit_tracing")] -fn get_otel_key(env_name: &str) -> Option { - match std::env::var(env_name) { - Ok(key) => { - if key.is_empty() { - None - } else { - Some(key) - } - } - Err(_) => None, - } -} - -#[cfg(feature = "grit_tracing")] -fn get_otel_setup() -> Result> { - use anyhow::bail; - - let mut exporter = opentelemetry_otlp::new_exporter() - .http() - .with_http_client(reqwest::Client::default()) - .with_timeout(std::time::Duration::from_millis(500)); - - let grafana_key = get_otel_key("GRAFANA_OTEL_KEY"); - let honeycomb_key = get_otel_key("HONEYCOMB_OTEL_KEY"); - let baselime_key = get_otel_key("BASELIME_OTEL_KEY"); - let hyperdx_key = get_otel_key("HYPERDX_OTEL_KEY"); - - match (grafana_key, honeycomb_key, baselime_key, hyperdx_key) { - (None, None, None, None) => { - // NOTE: we don't include tracing in released builds, so this won't appear - eprintln!("No OTLP key found, tracing will be disabled"); - return Ok(None); - } - (Some(grafana_key), None, None, None) => { - let instance_id = "665534"; - let encoded = - base64::encode_from_string(format!("{}:{}", instance_id, grafana_key).as_str())?; - exporter = exporter - .with_endpoint("https://otlp-gateway-prod-us-central-0.grafana.net/otlp") - .with_headers(HashMap::from([( - "Authorization".into(), - format!("Basic {}", encoded), - )])); - } - (None, Some(honeycomb_key), None, None) => { - exporter = exporter - .with_endpoint("https://api.honeycomb.io") - .with_headers(HashMap::from([("x-honeycomb-team".into(), honeycomb_key)])); - } - (None, None, Some(baselime_key), None) => { - exporter = exporter - .with_endpoint("https://otel.baselime.io/v1/") - .with_headers(HashMap::from([ - ("x-api-key".into(), baselime_key), - ("x-baselime-dataset".into(), "otel".into()), - ])); - } - (None, None, None, Some(hyperdx_key)) => { - exporter = exporter - .with_endpoint("https://in-otel.hyperdx.io") - .with_headers(HashMap::from([("authorization".into(), hyperdx_key)])); - } - _ => bail!("multiple OTLP keys found"), - } - - let tracer = opentelemetry_otlp::new_pipeline() - .tracing() - .with_exporter(exporter) - .with_trace_config( - trace::config().with_resource(Resource::new(vec![KeyValue::new( - "service.name", - "grit_marzano", - )])), - ) - .install_batch(opentelemetry_sdk::runtime::Tokio)?; - - Ok(Some(tracer)) -} #[tokio::main] async fn main() -> Result<()> { - #[cfg(feature = "grit_tracing")] - { - let tracer = get_otel_setup()?; - - if let Some(tracer) = tracer { - let env_filter = EnvFilter::try_from_default_env() - .unwrap_or(EnvFilter::new("TRACE")) - // We don't want to trace the tracing library itself - .add_directive("hyper::proto=off".parse().unwrap()); - - let telemetry = tracing_opentelemetry::layer().with_tracer(tracer); - let subscriber = Registry::default().with(env_filter).with(telemetry); - - global::set_text_map_propagator(TraceContextPropagator::new()); - tracing::subscriber::set_global_default(subscriber) - .expect("setting tracing default failed"); - - let root_span = span!(Level::INFO, "grit_marzano.cli_command",); - - let _res = async move { - event!(Level::INFO, "starting the CLI!"); - - let res = run_command().await; - - event!(Level::INFO, "ending the CLI!"); - - res - } - .instrument(root_span) - .await; - - opentelemetry::global::shutdown_tracer_provider(); - - return Ok(()); - } - } - let subscriber = tracing::subscriber::NoSubscriber::new(); - tracing::subscriber::set_global_default(subscriber).expect("setting tracing default failed"); - - let res = run_command().await; - if let Err(ref e) = res { - if let Some(good) = e.downcast_ref::() { - if let Some(msg) = &good.message { - println!("{}", msg); - } - std::process::exit(1); - } - } - return res; + run_command_with_tracing().await } From 5d973faff55d1e3d7fcacd81db3a4ba81916cff4 Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Tue, 7 May 2024 19:44:32 -0700 Subject: [PATCH 08/14] wow that works --- crates/cli/src/commands/mod.rs | 4 +--- crates/cli_bin/src/main.rs | 1 - 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/crates/cli/src/commands/mod.rs b/crates/cli/src/commands/mod.rs index 242334b35..0f47b237b 100644 --- a/crates/cli/src/commands/mod.rs +++ b/crates/cli/src/commands/mod.rs @@ -437,8 +437,6 @@ fn get_otel_key(env_name: &str) -> Option { #[cfg(feature = "grit_tracing")] fn get_otel_setup() -> Result> { - use anyhow::bail; - let mut exporter = opentelemetry_otlp::new_exporter() .http() .with_http_client(reqwest::Client::default()) @@ -553,5 +551,5 @@ pub async fn run_command_with_tracing() -> Result<()> { std::process::exit(1); } } - return res; + res } diff --git a/crates/cli_bin/src/main.rs b/crates/cli_bin/src/main.rs index 3370ec05a..15df963a3 100644 --- a/crates/cli_bin/src/main.rs +++ b/crates/cli_bin/src/main.rs @@ -1,5 +1,4 @@ use marzano_cli::commands::run_command_with_tracing; -use marzano_cli::error::GoodError; use anyhow::Result; From b581571e36cfbc4041ecf9b7884be4a24dc6956f Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Tue, 7 May 2024 20:12:07 -0700 Subject: [PATCH 09/14] drop two stderr --- crates/cli_bin/tests/plumbing.rs | 8 -------- 1 file changed, 8 deletions(-) diff --git a/crates/cli_bin/tests/plumbing.rs b/crates/cli_bin/tests/plumbing.rs index 53851662d..05f35e9a7 100644 --- a/crates/cli_bin/tests/plumbing.rs +++ b/crates/cli_bin/tests/plumbing.rs @@ -150,9 +150,6 @@ fn checks_patterns_round_trip() -> Result<()> { let result = cmd.output()?; - // no stderr - assert_eq!(String::from_utf8(result.stderr)?, ""); - // Result must be successful assert!(result.status.success()); @@ -173,8 +170,6 @@ fn checks_patterns_without_samples() -> Result<()> { cmd.write_stdin(String::from_utf8(config.into())?); let result = cmd.output()?; - // no stderr - assert_eq!(String::from_utf8(result.stderr)?, ""); // Result must be successful assert!(result.status.success()); @@ -225,9 +220,6 @@ fn checks_multifile_patterns() -> Result<()> { cmd.write_stdin(String::from_utf8(output.stdout)?); let result = cmd.output()?; - // no stderr - assert_eq!(String::from_utf8(result.stderr)?, ""); - assert!(result.status.success()); Ok(()) From 59e1ef1ec9ad7564e7d43188125e10a443bb3843 Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Tue, 7 May 2024 20:12:55 -0700 Subject: [PATCH 10/14] run the tests From fd3a9dad7bb0d829eea1bb8b46b7d8603afcd776 Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Tue, 7 May 2024 20:44:18 -0700 Subject: [PATCH 11/14] just like that --- crates/cli/src/commands/mod.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/cli/src/commands/mod.rs b/crates/cli/src/commands/mod.rs index 0f47b237b..e89ecdf59 100644 --- a/crates/cli/src/commands/mod.rs +++ b/crates/cli/src/commands/mod.rs @@ -449,7 +449,7 @@ fn get_otel_setup() -> Result> { match (grafana_key, honeycomb_key, baselime_key, hyperdx_key) { (None, None, None, None) => { - // NOTE: we don't include tracing in released builds, so this won't appear + #[cfg(feature = "server")] eprintln!("No OTLP key found, tracing will be disabled"); return Ok(None); } @@ -522,7 +522,7 @@ pub async fn run_command_with_tracing() -> Result<()> { let root_span = span!(Level::INFO, "grit_marzano.cli_command",); - let _res = async move { + let res = async move { event!(Level::INFO, "starting the CLI!"); let res = run_command().await; @@ -536,7 +536,7 @@ pub async fn run_command_with_tracing() -> Result<()> { opentelemetry::global::shutdown_tracer_provider(); - return Ok(()); + return res; } } let subscriber = tracing::subscriber::NoSubscriber::new(); From 53ce9a93614d9a5b0e0b8cc78e6d4cfc7b87be30 Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Tue, 7 May 2024 23:11:29 -0700 Subject: [PATCH 12/14] inject env --- crates/cli/src/commands/mod.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/cli/src/commands/mod.rs b/crates/cli/src/commands/mod.rs index e89ecdf59..0cae7a6d6 100644 --- a/crates/cli/src/commands/mod.rs +++ b/crates/cli/src/commands/mod.rs @@ -488,13 +488,15 @@ fn get_otel_setup() -> Result> { } } + let env = get_otel_key("GRIT_DEPLOYMENT_ENV").unwrap_or_else(|| "prod".to_string()); + let tracer = opentelemetry_otlp::new_pipeline() .tracing() .with_exporter(exporter) .with_trace_config( trace::config().with_resource(Resource::new(vec![KeyValue::new( "service.name", - "grit_marzano", + format!("{}_grit_marzano", env), )])), ) .install_batch(opentelemetry_sdk::runtime::Tokio)?; From 622c1f17cb36394e35714530fd00cf989fc01ce7 Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Tue, 7 May 2024 23:21:25 -0700 Subject: [PATCH 13/14] Update crates/cli/src/commands/mod.rs Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- crates/cli/src/commands/mod.rs | 36 +++++++++++++--------------------- 1 file changed, 14 insertions(+), 22 deletions(-) diff --git a/crates/cli/src/commands/mod.rs b/crates/cli/src/commands/mod.rs index 0cae7a6d6..5c9f3c9bf 100644 --- a/crates/cli/src/commands/mod.rs +++ b/crates/cli/src/commands/mod.rs @@ -32,28 +32,20 @@ pub(crate) mod workflows_list; use crate::error::GoodError; #[cfg(feature = "grit_tracing")] -use marzano_util::base64; -#[cfg(feature = "grit_tracing")] -use opentelemetry::{global, KeyValue}; -#[cfg(feature = "grit_tracing")] -use opentelemetry_otlp::WithExportConfig; -#[cfg(feature = "grit_tracing")] -use opentelemetry_sdk::propagation::TraceContextPropagator; -#[cfg(feature = "grit_tracing")] -use opentelemetry_sdk::trace::Tracer; -#[cfg(feature = "grit_tracing")] -use opentelemetry_sdk::{trace, Resource}; -#[cfg(feature = "grit_tracing")] -use std::collections::HashMap; -#[cfg(feature = "grit_tracing")] -use tracing::Instrument; -#[cfg(feature = "grit_tracing")] -use tracing::{event, span, Level}; -#[cfg(feature = "grit_tracing")] -#[allow(unused_imports)] -use tracing_subscriber::prelude::*; -#[cfg(feature = "grit_tracing")] -use tracing_subscriber::{layer::SubscriberExt, EnvFilter, Registry}; +{ + use marzano_util::base64; + use opentelemetry::{global, KeyValue}; + use opentelemetry_otlp::WithExportConfig; + use opentelemetry_sdk::propagation::TraceContextPropagator; + use opentelemetry_sdk::trace::Tracer; + use opentelemetry_sdk::{trace, Resource}; + use std::collections::HashMap; + use tracing::Instrument; + use tracing::{event, span, Level}; + #[allow(unused_imports)] + use tracing_subscriber::prelude::*; + use tracing_subscriber::{layer::SubscriberExt, EnvFilter, Registry}; +} #[cfg(feature = "docgen")] pub(crate) mod docgen; From 2b93ca61d3ba6a70d855b9230a3a2af3eeeb4de7 Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Tue, 7 May 2024 23:28:28 -0700 Subject: [PATCH 14/14] Revert "Update crates/cli/src/commands/mod.rs" This reverts commit 622c1f17cb36394e35714530fd00cf989fc01ce7. --- crates/cli/src/commands/mod.rs | 36 +++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/crates/cli/src/commands/mod.rs b/crates/cli/src/commands/mod.rs index 5c9f3c9bf..0cae7a6d6 100644 --- a/crates/cli/src/commands/mod.rs +++ b/crates/cli/src/commands/mod.rs @@ -32,20 +32,28 @@ pub(crate) mod workflows_list; use crate::error::GoodError; #[cfg(feature = "grit_tracing")] -{ - use marzano_util::base64; - use opentelemetry::{global, KeyValue}; - use opentelemetry_otlp::WithExportConfig; - use opentelemetry_sdk::propagation::TraceContextPropagator; - use opentelemetry_sdk::trace::Tracer; - use opentelemetry_sdk::{trace, Resource}; - use std::collections::HashMap; - use tracing::Instrument; - use tracing::{event, span, Level}; - #[allow(unused_imports)] - use tracing_subscriber::prelude::*; - use tracing_subscriber::{layer::SubscriberExt, EnvFilter, Registry}; -} +use marzano_util::base64; +#[cfg(feature = "grit_tracing")] +use opentelemetry::{global, KeyValue}; +#[cfg(feature = "grit_tracing")] +use opentelemetry_otlp::WithExportConfig; +#[cfg(feature = "grit_tracing")] +use opentelemetry_sdk::propagation::TraceContextPropagator; +#[cfg(feature = "grit_tracing")] +use opentelemetry_sdk::trace::Tracer; +#[cfg(feature = "grit_tracing")] +use opentelemetry_sdk::{trace, Resource}; +#[cfg(feature = "grit_tracing")] +use std::collections::HashMap; +#[cfg(feature = "grit_tracing")] +use tracing::Instrument; +#[cfg(feature = "grit_tracing")] +use tracing::{event, span, Level}; +#[cfg(feature = "grit_tracing")] +#[allow(unused_imports)] +use tracing_subscriber::prelude::*; +#[cfg(feature = "grit_tracing")] +use tracing_subscriber::{layer::SubscriberExt, EnvFilter, Registry}; #[cfg(feature = "docgen")] pub(crate) mod docgen;