Add unwind information eviction

This rather large commit adds unwind information eviction. This can be
triggered in two ways: if an unwind information bucket is full, object
file that last appeared in the profiles will be evicted to leave space
for the new unwind info. There's also a new flag
(`--max-native-unwind-info-size-mb`) that will be used as a rough limit
for the size of BPF maps used to store unwind information.

By default there's no limit on the unwind info stored in BPF maps.

Additionally, there's a new event to notify of addresses don't have
unwind information as now this can happen after an eviction.

Several other changes were made too, such as a bugfis on how the unwind
info pages were being deleted, and the addition of
`--enable-deadlock-detector` to start a thread that checks for deadlocks
in `parking_lot`'s locking facilities.

Future changes
==============

Something that stood out during the development of this feature is that
the generation of compact unwind information from .eh_frame data might
be called more frequently. This is perhaps something worth caching on
disk, but that'll happen in another PR.

Test Plan
=========

Ran for a while without issues, also with lower memory limits set.

Cargo.toml

@@ -61,6 +61,7 @@ thiserror = { workspace = true }
 errno = { workspace = true }
 procfs = { workspace = true }
 nix = { workspace = true, features = ["user"] }
+parking_lot = { version = "0.12.3", features = ["deadlock_detection"] }
 
 [dev-dependencies]
 assert_cmd = { version = "2.0.16" }

src/bpf/profiler.bpf.c

@@ -192,26 +192,21 @@ find_page(mapping_t *mapping, u64 object_relative_pc, u64 *left, u64 *right) {
   return NULL;
 }
 
-
-static __always_inline void event_new_process(struct bpf_perf_event_data *ctx, int per_process_id) {
-  Event event = {
-      .type = EVENT_NEW_PROCESS,
-      .pid = per_process_id,
-  };
-
-  bool *is_rate_limited = bpf_map_lookup_elem(&rate_limits, &event);
+static __always_inline void send_event(Event *event, struct bpf_perf_event_data *ctx) {
+  bool *is_rate_limited = bpf_map_lookup_elem(&rate_limits, event);
   if (is_rate_limited != NULL && *is_rate_limited) {
-    LOG("[debug] event_new_process was rate limited");
+    LOG("[debug] send_event was rate limited");
     return;
   }
 
-  if (bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, &event, sizeof(Event)) < 0) {
+  if (bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, event, sizeof(Event)) < 0) {
     bump_unwind_error_sending_new_process_event();
   }
 
-  LOG("[debug] event_new_process event sent");
+  LOG("[debug] event type %d sent", event->type);
   bool rate_limited = true;
-  bpf_map_update_elem(&rate_limits, &event, &rate_limited, BPF_ANY);
+
+  bpf_map_update_elem(&rate_limits, event, &rate_limited, BPF_ANY);
 }
 
 // Kernel addresses have the top bits set.
@@ -398,7 +393,12 @@ int dwarf_unwind(struct bpf_perf_event_data *ctx) {
     u64 right = 0;
     void *inner = find_page(mapping, object_relative_pc_high, &left, &right);
     if (inner == NULL) {
-      // TODO: add counter
+      Event event = {
+          .type = EVENT_NEED_UNWIND_INFO,
+          .pid = per_process_id,
+          .address = unwind_state->ip,
+      };
+      send_event(&event, ctx);
       return 1;
     }
 
@@ -672,7 +672,11 @@ int on_event(struct bpf_perf_event_data *ctx) {
     return 0;
   }
 
-  event_new_process(ctx, per_process_id);
+  Event event = {
+      .type = EVENT_NEW_PROCESS,
+      .pid = per_process_id,
+  };
+  send_event(&event, ctx);
   return 0;
 }
 

src/bpf/profiler.h

@@ -184,12 +184,13 @@ typedef struct {
 
 enum event_type {
   EVENT_NEW_PROCESS = 1,
-  // EVENT_NEED_UNWIND_INFO = 2, need a way to signal of new loaded mappings
+  EVENT_NEED_UNWIND_INFO = 2,
 };
 
 typedef struct {
   enum event_type type;
   int pid; // use right name here (tgid?)
+  u64 address;
 } Event;
 
 enum program {

src/cli/args.rs

@@ -136,4 +136,12 @@ pub(crate) struct CliArgs {
     pub(crate) symbolizer: Symbolizer,
     #[arg(long, default_value_t, value_enum)]
     pub(crate) debug_info_backend: DebugInfoBackend,
+    #[arg(
+        long,
+        default_value_t = ProfilerConfig::default().max_native_unwind_info_size_mb,
+        help = "approximate max size in megabytes used for the BPF maps that hold unwind information"
+    )]
+    pub(crate) max_native_unwind_info_size_mb: i32,
+    #[arg(long, help = "enable parking_lot's deadlock detector")]
+    pub(crate) enable_deadlock_detector: bool,
 }

src/cli/main.rs

@@ -55,10 +55,28 @@ fn panic_thread_hook() {
     }));
 }
 
+/// Starts `parking_lot`'s deadlock detector.
+fn start_deadlock_detector() {
+    std::thread::spawn(move || loop {
+        std::thread::sleep(std::time::Duration::from_secs(1));
+        for deadlock in parking_lot::deadlock::check_deadlock() {
+            for deadlock in deadlock {
+                eprintln!(
+                    "Found a deadlock! {}: {:?}",
+                    deadlock.thread_id(),
+                    deadlock.backtrace()
+                );
+            }
+        }
+    });
+}
+
 fn main() -> Result<(), Box<dyn Error>> {
     panic_thread_hook();
-
     let args = CliArgs::parse();
+    if args.enable_deadlock_detector {
+        start_deadlock_detector();
+    }
 
     if let Some(path) = args.show_unwind_info {
         show_unwind_info(&path);
@@ -277,7 +295,7 @@ mod tests {
         cmd.arg("--help");
         cmd.assert().success();
         let actual = String::from_utf8(cmd.unwrap().stdout).unwrap();
-        insta::assert_yaml_snapshot!(actual, @r#""Usage: lightswitch [OPTIONS]\n\nOptions:\n      --pids <PIDS>\n          Specific PIDs to profile\n\n      --tids <TIDS>\n          Specific TIDs to profile (these can be outside the PIDs selected above)\n\n      --show-unwind-info <PATH_TO_BINARY>\n          Show unwind info for given binary\n\n      --show-info <PATH_TO_BINARY>\n          Show build ID for given binary\n\n  -D, --duration <DURATION>\n          How long this agent will run in seconds\n          \n          [default: 18446744073709551615]\n\n      --libbpf-debug\n          Enable libbpf logs. This includes the BPF verifier output\n\n      --bpf-logging\n          Enable BPF programs logging\n\n      --logging <LOGGING>\n          Set lightswitch's logging level\n          \n          [default: info]\n          [possible values: trace, debug, info, warn, error]\n\n      --sample-freq <SAMPLE_FREQ_IN_HZ>\n          Per-CPU Sampling Frequency in Hz\n          \n          [default: 19]\n\n      --profile-format <PROFILE_FORMAT>\n          Output file for Flame Graph in SVG format\n          \n          [default: flame-graph]\n          [possible values: none, flame-graph, pprof]\n\n      --profile-path <PROFILE_PATH>\n          Path for the generated profile\n\n      --profile-name <PROFILE_NAME>\n          Name for the generated profile\n\n      --sender <SENDER>\n          Where to write the profile\n          \n          [default: local-disk]\n\n          Possible values:\n          - none:       Discard the profile. Used for kernel tests\n          - local-disk\n          - remote\n\n      --server-url <SERVER_URL>\n          \n\n      --perf-buffer-bytes <PERF_BUFFER_BYTES>\n          Size of each profiler perf buffer, in bytes (must be a power of 2)\n          \n          [default: 524288]\n\n      --mapsize-info\n          Print eBPF map sizes after creation\n\n      --mapsize-stacks <MAPSIZE_STACKS>\n          max number of individual stacks to capture before aggregation\n          \n          [default: 100000]\n\n      --mapsize-aggregated-stacks <MAPSIZE_AGGREGATED_STACKS>\n          max number of unique stacks after aggregation\n          \n          [default: 10000]\n\n      --mapsize-rate-limits <MAPSIZE_RATE_LIMITS>\n          max number of rate limit entries\n          \n          [default: 5000]\n\n      --exclude-self\n          Do not profile the profiler (myself)\n\n      --symbolizer <SYMBOLIZER>\n          [default: local]\n          [possible values: local, none]\n\n      --debug-info-backend <DEBUG_INFO_BACKEND>\n          [default: none]\n          [possible values: none, copy, remote]\n\n  -h, --help\n          Print help (see a summary with '-h')\n""#);
+        insta::assert_yaml_snapshot!(actual, @r#""Usage: lightswitch [OPTIONS]\n\nOptions:\n      --pids <PIDS>\n          Specific PIDs to profile\n\n      --tids <TIDS>\n          Specific TIDs to profile (these can be outside the PIDs selected above)\n\n      --show-unwind-info <PATH_TO_BINARY>\n          Show unwind info for given binary\n\n      --show-info <PATH_TO_BINARY>\n          Show build ID for given binary\n\n  -D, --duration <DURATION>\n          How long this agent will run in seconds\n          \n          [default: 18446744073709551615]\n\n      --libbpf-debug\n          Enable libbpf logs. This includes the BPF verifier output\n\n      --bpf-logging\n          Enable BPF programs logging\n\n      --logging <LOGGING>\n          Set lightswitch's logging level\n          \n          [default: info]\n          [possible values: trace, debug, info, warn, error]\n\n      --sample-freq <SAMPLE_FREQ_IN_HZ>\n          Per-CPU Sampling Frequency in Hz\n          \n          [default: 19]\n\n      --profile-format <PROFILE_FORMAT>\n          Output file for Flame Graph in SVG format\n          \n          [default: flame-graph]\n          [possible values: none, flame-graph, pprof]\n\n      --profile-path <PROFILE_PATH>\n          Path for the generated profile\n\n      --profile-name <PROFILE_NAME>\n          Name for the generated profile\n\n      --sender <SENDER>\n          Where to write the profile\n          \n          [default: local-disk]\n\n          Possible values:\n          - none:       Discard the profile. Used for kernel tests\n          - local-disk\n          - remote\n\n      --server-url <SERVER_URL>\n          \n\n      --perf-buffer-bytes <PERF_BUFFER_BYTES>\n          Size of each profiler perf buffer, in bytes (must be a power of 2)\n          \n          [default: 524288]\n\n      --mapsize-info\n          Print eBPF map sizes after creation\n\n      --mapsize-stacks <MAPSIZE_STACKS>\n          max number of individual stacks to capture before aggregation\n          \n          [default: 100000]\n\n      --mapsize-aggregated-stacks <MAPSIZE_AGGREGATED_STACKS>\n          max number of unique stacks after aggregation\n          \n          [default: 10000]\n\n      --mapsize-rate-limits <MAPSIZE_RATE_LIMITS>\n          max number of rate limit entries\n          \n          [default: 5000]\n\n      --exclude-self\n          Do not profile the profiler (myself)\n\n      --symbolizer <SYMBOLIZER>\n          [default: local]\n          [possible values: local, none]\n\n      --debug-info-backend <DEBUG_INFO_BACKEND>\n          [default: none]\n          [possible values: none, copy, remote]\n\n      --max-native-unwind-info-size-mb <MAX_NATIVE_UNWIND_INFO_SIZE_MB>\n          approximate max size in megabytes used for the BPF maps that hold unwind information\n          \n          [default: 2147483647]\n\n      --enable-deadlock-detector\n          enable parking_lot's deadlock detector\n\n  -h, --help\n          Print help (see a summary with '-h')\n""#);
     }
 
     #[rstest]