Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: repair tracing and add hyperdx #318

Merged
merged 14 commits into from
May 8, 2024
9 changes: 9 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 4 additions & 0 deletions crates/cli/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@ opentelemetry_sdk = { version = "0.21.1", optional = true, features = [
tracing-opentelemetry = { version = "0.22.0", optional = true, default-features = false }

tracing = { version = "0.1.40", default-features = false, features = [] }
tracing-subscriber = { version = "0.3", default-features = false, optional = true }

[target.'cfg(not(windows))'.dependencies]
openssl = { version = "0.10", features = ["vendored"] }
Expand All @@ -87,6 +88,7 @@ default = [
"external_functions",
"updater",
"workflows_v2",
"grit_tracing",
# "remote_workflows",
# "server",
# "remote_redis",
Expand All @@ -109,13 +111,15 @@ server = [
"remote_pubsub",
"ai_builtins",
"dep:cli_server",
"grit_tracing"
]
updater = []
grit_tracing = [
"dep:opentelemetry-otlp",
"dep:opentelemetry",
"dep:opentelemetry_sdk",
"dep:tracing-opentelemetry",
"dep:tracing-subscriber",
"marzano-core/grit_tracing",
]
external_functions = ["marzano-core/external_functions"]
Expand Down
29 changes: 29 additions & 0 deletions crates/cli/src/commands/apply_pattern.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ use tracing::instrument;
#[cfg(feature = "grit_tracing")]
use tracing::span;
#[cfg(feature = "grit_tracing")]
#[allow(unused_imports)]
use tracing_opentelemetry::OpenTelemetrySpanExt as _;

use grit_util::Position;
Expand Down Expand Up @@ -191,12 +192,17 @@ pub(crate) async fn run_apply_pattern(
// Get the current directory
let cwd = std::env::current_dir().unwrap();

#[cfg(feature = "grit_tracing")]
let module_resolution = span!(tracing::Level::INFO, "module_resolution",).entered();

// Construct a resolver
let resolver = GritModuleResolver::new(cwd.to_str().unwrap());
let current_repo_root = marzano_gritmodule::fetcher::LocalRepo::from_dir(&cwd)
.await
.map(|repo| repo.root())
.transpose()?;
#[cfg(feature = "grit_tracing")]
module_resolution.exit();

let mut emitter = create_emitter(
&format,
Expand All @@ -213,6 +219,9 @@ pub(crate) async fn run_apply_pattern(
extract_filter_ranges(&shared, current_repo_root.as_ref())
);

#[cfg(feature = "grit_tracing")]
let span_libs = span!(tracing::Level::INFO, "prep_libs",).entered();

let (my_input, lang) = if let Some(pattern_libs) = pattern_libs {
(
ApplyInput {
Expand All @@ -223,6 +232,9 @@ pub(crate) async fn run_apply_pattern(
lang,
)
} else {
#[cfg(feature = "grit_tracing")]
let stdlib_download_span = span!(tracing::Level::INFO, "stdlib_download",).entered();

let mod_dir = find_grit_modules_dir(cwd.clone()).await;
if !env::var("GRIT_DOWNLOADS_DISABLED")
.unwrap_or_else(|_| "false".to_owned())
Expand All @@ -236,6 +248,9 @@ pub(crate) async fn run_apply_pattern(
);
}

#[cfg(feature = "grit_tracing")]
stdlib_download_span.exit();

let warn_uncommitted =
!arg.dry_run && !arg.force && has_uncommitted_changes(cwd.clone()).await;
if warn_uncommitted {
Expand All @@ -254,7 +269,11 @@ pub(crate) async fn run_apply_pattern(
}
}

#[cfg(feature = "grit_tracing")]
let grit_file_discovery = span!(tracing::Level::INFO, "grit_file_discovery",).entered();

let pattern_libs = flushable_unwrap!(emitter, get_grit_files_from_cwd().await);

let (mut lang, pattern_body) = if pattern.ends_with(".grit") || pattern.ends_with(".md") {
match fs::read_to_string(pattern.clone()).await {
Ok(pb) => {
Expand Down Expand Up @@ -330,6 +349,8 @@ pub(crate) async fn run_apply_pattern(
emitter,
pattern_libs.get_language_directory_or_default(lang)
);
#[cfg(feature = "grit_tracing")]
grit_file_discovery.exit();
(
ApplyInput {
pattern_body,
Expand All @@ -352,6 +373,8 @@ pub(crate) async fn run_apply_pattern(
return Ok(());
}

#[cfg(feature = "grit_tracing")]
let collect_name = span!(tracing::Level::INFO, "collect_name",).entered();
let current_name = if is_pattern_name(&pattern) {
Some(pattern.trim_end_matches("()").to_string())
} else {
Expand All @@ -361,11 +384,17 @@ pub(crate) async fn run_apply_pattern(
.find(|(_, body)| body.trim() == pattern.trim())
.map(|(name, _)| name.clone())
};
#[cfg(feature = "grit_tracing")]
collect_name.exit();

let pattern: crate::resolver::RichPattern<'_> = flushable_unwrap!(
emitter,
resolver.make_pattern(&my_input.pattern_body, current_name)
);

#[cfg(feature = "grit_tracing")]
span_libs.exit();

let CompilationResult {
problem: compiled,
compilation_warnings,
Expand Down
1 change: 1 addition & 0 deletions crates/cli/src/commands/filters.rs
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ pub struct SharedFilterArgs {
pub(crate) only_in_diff: Option<Option<String>>,
}

#[tracing::instrument]
pub(crate) fn extract_filter_ranges(
args: &SharedFilterArgs,
root: Option<&PathBuf>,
Expand Down
163 changes: 162 additions & 1 deletion crates/cli/src/commands/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,32 @@ pub(crate) mod workflows;
#[cfg(feature = "workflows_v2")]
pub(crate) mod workflows_list;

use crate::error::GoodError;

#[cfg(feature = "grit_tracing")]
use marzano_util::base64;
#[cfg(feature = "grit_tracing")]
use opentelemetry::{global, KeyValue};
#[cfg(feature = "grit_tracing")]
use opentelemetry_otlp::WithExportConfig;
#[cfg(feature = "grit_tracing")]
use opentelemetry_sdk::propagation::TraceContextPropagator;
#[cfg(feature = "grit_tracing")]
use opentelemetry_sdk::trace::Tracer;
#[cfg(feature = "grit_tracing")]
use opentelemetry_sdk::{trace, Resource};
#[cfg(feature = "grit_tracing")]
use std::collections::HashMap;
#[cfg(feature = "grit_tracing")]
use tracing::Instrument;
#[cfg(feature = "grit_tracing")]
use tracing::{event, span, Level};
#[cfg(feature = "grit_tracing")]
#[allow(unused_imports)]
use tracing_subscriber::prelude::*;
#[cfg(feature = "grit_tracing")]
use tracing_subscriber::{layer::SubscriberExt, EnvFilter, Registry};
morgante marked this conversation as resolved.
Show resolved Hide resolved

#[cfg(feature = "docgen")]
pub(crate) mod docgen;
mod filters;
Expand Down Expand Up @@ -266,7 +292,7 @@ fn write_analytics_event(
}

#[instrument]
pub async fn run_command() -> Result<()> {
async fn run_command() -> Result<()> {
let app = App::parse();
// Use this *only* for analytics, not for any other purpose.
let analytics_args = std::env::args().collect::<Vec<_>>();
Expand Down Expand Up @@ -394,3 +420,138 @@ pub async fn run_command() -> Result<()> {

res
}

#[cfg(feature = "grit_tracing")]
fn get_otel_key(env_name: &str) -> Option<String> {
match std::env::var(env_name) {
Ok(key) => {
if key.is_empty() {
None
} else {
Some(key)
}
}
Err(_) => None,
}
}

#[cfg(feature = "grit_tracing")]
fn get_otel_setup() -> Result<Option<Tracer>> {
let mut exporter = opentelemetry_otlp::new_exporter()
.http()
.with_http_client(reqwest::Client::default())
.with_timeout(std::time::Duration::from_millis(500));

let grafana_key = get_otel_key("GRAFANA_OTEL_KEY");
let honeycomb_key = get_otel_key("HONEYCOMB_OTEL_KEY");
let baselime_key = get_otel_key("BASELIME_OTEL_KEY");
let hyperdx_key = get_otel_key("HYPERDX_OTEL_KEY");

match (grafana_key, honeycomb_key, baselime_key, hyperdx_key) {
(None, None, None, None) => {
#[cfg(feature = "server")]
eprintln!("No OTLP key found, tracing will be disabled");
return Ok(None);
}
(Some(grafana_key), _, _, _) => {
let instance_id = "665534";
let encoded =
base64::encode_from_string(format!("{}:{}", instance_id, grafana_key).as_str())?;
exporter = exporter
.with_endpoint("https://otlp-gateway-prod-us-central-0.grafana.net/otlp")
.with_headers(HashMap::from([(
"Authorization".into(),
format!("Basic {}", encoded),
)]));
eprintln!("Using Grafana OTLP key");
}
(_, Some(honeycomb_key), _, _) => {
exporter = exporter
.with_endpoint("https://api.honeycomb.io")
.with_headers(HashMap::from([("x-honeycomb-team".into(), honeycomb_key)]));
eprintln!("Using Honeycomb OTLP key");
}
(_, _, Some(baselime_key), _) => {
exporter = exporter
.with_endpoint("https://otel.baselime.io/v1/")
.with_headers(HashMap::from([
("x-api-key".into(), baselime_key),
("x-baselime-dataset".into(), "otel".into()),
]));
eprintln!("Using Baselime OTLP key");
}
(_, _, _, Some(hyperdx_key)) => {
exporter = exporter
.with_endpoint("https://in-otel.hyperdx.io")
.with_headers(HashMap::from([("authorization".into(), hyperdx_key)]));
eprintln!("Using HyperDX OTLP key");
}
}

let env = get_otel_key("GRIT_DEPLOYMENT_ENV").unwrap_or_else(|| "prod".to_string());

let tracer = opentelemetry_otlp::new_pipeline()
.tracing()
.with_exporter(exporter)
.with_trace_config(
trace::config().with_resource(Resource::new(vec![KeyValue::new(
"service.name",
format!("{}_grit_marzano", env),
)])),
)
.install_batch(opentelemetry_sdk::runtime::Tokio)?;

Ok(Some(tracer))
}

pub async fn run_command_with_tracing() -> Result<()> {
#[cfg(feature = "grit_tracing")]
{
let tracer = get_otel_setup()?;

if let Some(tracer) = tracer {
let env_filter = EnvFilter::try_from_default_env()
.unwrap_or(EnvFilter::new("TRACE"))
// We don't want to trace the tracing library itself
.add_directive("hyper::proto=off".parse().unwrap());

let telemetry = tracing_opentelemetry::layer().with_tracer(tracer);
let subscriber = Registry::default().with(env_filter).with(telemetry);

global::set_text_map_propagator(TraceContextPropagator::new());
tracing::subscriber::set_global_default(subscriber)
.expect("setting tracing default failed");

let root_span = span!(Level::INFO, "grit_marzano.cli_command",);

let res = async move {
event!(Level::INFO, "starting the CLI!");

let res = run_command().await;

event!(Level::INFO, "ending the CLI!");

res
}
.instrument(root_span)
.await;

opentelemetry::global::shutdown_tracer_provider();

return res;
}
}
let subscriber = tracing::subscriber::NoSubscriber::new();
tracing::subscriber::set_global_default(subscriber).expect("setting tracing default failed");

let res = run_command().await;
if let Err(ref e) = res {
if let Some(good) = e.downcast_ref::<GoodError>() {
if let Some(msg) = &good.message {
println!("{}", msg);
}
std::process::exit(1);
}
}
res
}
1 change: 1 addition & 0 deletions crates/cli/src/resolver.rs
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,7 @@ pub async fn get_grit_files_from(cwd: Option<PathBuf>) -> Result<PatternsDirecto
}
}

#[tracing::instrument]
pub async fn get_grit_files_from_cwd() -> Result<PatternsDirectory> {
let cwd = std::env::current_dir()?;
get_grit_files_from(Some(cwd)).await
Expand Down
12 changes: 1 addition & 11 deletions crates/cli/src/updater.rs
Original file line number Diff line number Diff line change
Expand Up @@ -184,8 +184,8 @@ pub struct Updater {
access_token: Option<String>,
}

#[allow(dead_code)]
impl Updater {
#[tracing::instrument]
pub async fn from_current_bin() -> Result<Self> {
let current_bin = std::env::current_exe()?;
let install_path = current_bin
Expand Down Expand Up @@ -255,16 +255,6 @@ impl Updater {
}
}

pub fn get_log_file(&self, app: SupportedApp) -> Result<std::fs::File> {
let log_path = self
.bin_path
.parent()
.unwrap()
.join(format!("{}.log", app.get_bin_name()));
let log_file = std::fs::File::create(log_path).unwrap();
Ok(log_file)
}

pub async fn check_for_update(&mut self) -> Result<bool> {
if self.binaries.is_empty() {
return Ok(false);
Expand Down
Loading
Loading