Refactoring, move layer creation to telemetry::capturing

query-engine/core/src/telemetry/capturing/helpers.rs

@@ -0,0 +1,12 @@
+use tracing::Metadata;
+
+/// Filters-in spans and events that are statically determined to be relevant for capturing
+/// Dynamic filtering will be done by the [`crate::capturer::Processor`]
+pub fn span_and_event_filter(meta: &Metadata) -> bool {
+    if meta.fields().iter().any(|f| f.name() == "user_facing") {
+        return true;
+    }
+
+    // relevant quaint connector spans and events
+    meta.target() == "quaint::connector::metrics"
+}

query-engine/core/src/telemetry/capturing/mod.rs

@@ -127,34 +127,55 @@ pub use self::settings::Settings;
 use self::capturer::Processor;
 use once_cell::sync::Lazy;
 use opentelemetry::{global, sdk, trace};
+use query_engine_metrics::MetricRegistry;
+use tracing::subscriber;
+use tracing_subscriber::{
+    filter::filter_fn, layer::Layered, prelude::__tracing_subscriber_SubscriberExt, Layer, Registry,
+};
 
 static PROCESSOR: Lazy<capturer::Processor> = Lazy::new(Processor::default);
-static TRACER: Lazy<sdk::trace::Tracer> = Lazy::new(setup_and_install_tracer_globally);
 
 /// Creates a new capturer, which is configured to export traces and log events happening during a
 /// particular request
 pub fn capturer(trace_id: trace::TraceId, settings: Settings) -> Capturer {
     Capturer::new(PROCESSOR.to_owned(), trace_id, settings)
 }
 
-/// Returns a clone to the global tracer used when capturing telemetry in the response
-pub fn tracer() -> &'static sdk::trace::Tracer {
-    &TRACER
-}
-
-/// Installs an opentelemetry tracer globally, which is configured to proecss
-/// spans and export them to global exporter.
-fn setup_and_install_tracer_globally() -> sdk::trace::Tracer {
+/// Adds a capturing layer to the given subscriber and installs the transformed subscriber as the
+/// global, default subscriber
+pub fn install_capturing_layer(
+    subscriber: Layered<Option<MetricRegistry>, Layered<Box<dyn Layer<Registry> + Send + Sync>, Registry>>,
+    log_queries: bool,
+) {
+    // set a trace context propagator, so that the trace context is propagated via the
+    // `traceparent` header from other systems
     global::set_text_map_propagator(sdk::propagation::TraceContextPropagator::new());
-
-    let provider_builder = sdk::trace::TracerProvider::builder().with_span_processor(PROCESSOR.to_owned());
-    let provider = provider_builder.build();
+    // create a tracer provider that is configured to use our custom processor to process spans
+    let provider = sdk::trace::TracerProvider::builder()
+        .with_span_processor(PROCESSOR.to_owned())
+        .build();
+    // create a tracer out of the provider
     let tracer = opentelemetry::trace::TracerProvider::tracer(&provider, "opentelemetry");
-
+    // set the provider as the global provider
     global::set_tracer_provider(provider);
-    tracer
+
+    // create a layer that will filter initial events and spans based on the log level configuration
+    // from the environment and a specific filter to discard things that we are not interested in
+    // from a capturiong perspective
+    let telemetry_layer = tracing_opentelemetry::layer()
+        .with_tracer(tracer)
+        .with_filter(crate::helpers::env_filter(
+            log_queries,
+            crate::helpers::QueryEngineLogLevel::FromEnv,
+        ))
+        .with_filter(filter_fn(helpers::span_and_event_filter));
+    // decorate the given subscriber (more layers were added before this one) with the telemetry layer
+    let subscriber = subscriber.with(telemetry_layer);
+    // and finally set the subscriber as the global, default subscriber
+    subscriber::set_global_default(subscriber).unwrap();
 }
 
 mod capturer;
+mod helpers;
 mod settings;
 pub mod storage;

query-engine/core/src/telemetry/helpers.rs

@@ -114,10 +114,6 @@ pub fn user_facing_span_only_filter(meta: &Metadata) -> bool {
         return false;
     }
 
-    user_facing_filter(meta)
-}
-
-pub fn user_facing_filter(meta: &Metadata) -> bool {
     if *SHOW_ALL_TRACES {
         return true;
     }
@@ -126,5 +122,7 @@ pub fn user_facing_filter(meta: &Metadata) -> bool {
         return true;
     }
 
+    // spans describing a quaint query.
+    // TODO: should this span be made user_facing in quaint?
     meta.target() == "quaint::connector::metrics" && meta.name() == "quaint:query"
 }

query-engine/query-engine/src/logger.rs

@@ -87,7 +87,6 @@ impl Logger {
     pub fn install(&self) -> LoggerResult<()> {
         let filter = telemetry::helpers::env_filter(self.log_queries, telemetry::helpers::QueryEngineLogLevel::FromEnv);
         let is_user_trace = filter_fn(telemetry::helpers::user_facing_span_only_filter);
-        let is_user_trace_or_event = filter_fn(telemetry::helpers::user_facing_filter);
 
         let fmt_layer = match self.log_format {
             LogFormat::Text => {
@@ -106,17 +105,8 @@ impl Logger {
 
         match self.tracing_config {
             TracingConfig::Captured => {
-                // Capturing is enabled, it overrides otel exporting.
-                let tracer = telemetry::capturing::tracer().to_owned();
-                let telemetry_layer = tracing_opentelemetry::layer()
-                    .with_tracer(tracer)
-                    .with_filter(is_user_trace_or_event)
-                    .with_filter(telemetry::helpers::env_filter(
-                        self.log_queries,
-                        telemetry::helpers::QueryEngineLogLevel::FromEnv,
-                    ));
-                let subscriber = subscriber.with(telemetry_layer);
-                subscriber::set_global_default(subscriber)?;
+                let log_queries = self.log_queries;
+                telemetry::capturing::install_capturing_layer(subscriber, log_queries)
             }
             TracingConfig::Http(ref endpoint) => {
                 // Opentelemetry is enabled, but capturing is disabled, there's an endpoint to export