Skip to content

Commit

Permalink
Add option to output local variables during dump
Browse files Browse the repository at this point in the history
This adds the ability to show the local variables for each frame when showing
the stack trace with the dump command. Currently we only support str/float/int
/bool/nonetype/list/tuple/dict objects - everything else will just show the
type and address.  (#77).
Dictionary processing also currently only works for python 3.6+, everything else
will show a 'dict' object instead.

This also adds the ability to show the thead names for each thread during dump.
Since this relies on a bunch of dictionary lookups, this also only works for
python 3.6+ (#47)
  • Loading branch information
benfred committed Oct 6, 2019
1 parent 1842a68 commit e161a53
Show file tree
Hide file tree
Showing 18 changed files with 1,806 additions and 332 deletions.
13 changes: 12 additions & 1 deletion generate_bindings.py
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,7 @@ def calculate_pyruntime_offsets(cpython_path, version, configure=False):

def extract_bindings(cpython_path, version, configure=False):
print("Generating bindings for python %s from repo at %s" % (version, cpython_path))

ret = os.system(f"""
cd {cpython_path}
git checkout {version}
Expand All @@ -108,6 +109,7 @@ def extract_bindings(cpython_path, version, configure=False):
cat Include/Python.h > bindgen_input.h
cat Include/frameobject.h >> bindgen_input.h
cat Objects/dict-common.h >> bindgen_input.h
echo '#define Py_BUILD_CORE 1\n' >> bindgen_input.h
cat Include/internal/pycore_pystate.h >> bindgen_input.h
Expand All @@ -124,6 +126,15 @@ def extract_bindings(cpython_path, version, configure=False):
--whitelist-type PyUnicodeObject \
--whitelist-type PyCompactUnicodeObject \
--whitelist-type PyStringObject \
--whitelist-type PyTupleObject \
--whitelist-type PyListObject \
--whitelist-type PyLongObject \
--whitelist-type PyFloatObject \
--whitelist-type PyDictObject \
--whitelist-type PyDictKeysObject \
--whitelist-type PyDictKeyEntry \
--whitelist-type PyObject \
--whitelist-type PyTypeObject \
-- -I . -I ./Include -I ./Include/internal
""")
if ret:
Expand Down Expand Up @@ -177,7 +188,7 @@ def extract_bindings(cpython_path, version, configure=False):
sys.exit(1)

if args.all:
versions = ['v3.7.0', 'v3.6.6', 'v3.5.5', 'v3.4.8', 'v3.3.7', 'v3.2.6', 'v2.7.15']
versions = ['v3.8.0b4', 'v3.7.0', 'v3.6.6', 'v3.5.5', 'v3.4.8', 'v3.3.7', 'v3.2.6', 'v2.7.15']
else:
versions = args.versions
if not versions:
Expand Down
9 changes: 8 additions & 1 deletion src/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,8 @@ pub struct Config {
pub hide_progess: bool,
#[doc(hidden)]
pub dump_json: bool,
#[doc(hidden)]
pub dump_locals: bool,
}

arg_enum!{
Expand Down Expand Up @@ -69,7 +71,7 @@ impl Default for Config {
non_blocking: false, show_line_numbers: false, sampling_rate: 100,
duration: RecordDuration::Unlimited, native: false,
gil_only: false, include_idle: false, include_thread_ids: false,
hide_progess: false, dump_json: false}
hide_progess: false, dump_json: false, dump_locals: false}
}
}

Expand Down Expand Up @@ -171,6 +173,10 @@ impl Config {
let dump = clap::SubCommand::with_name("dump")
.about("Dumps stack traces for a target program to stdout")
.arg(pid.clone().required(true))
.arg(Arg::with_name("locals")
.short("l")
.long("locals")
.help("Show local variables for each frame"))
.arg(Arg::with_name("json")
.short("j")
.long("json")
Expand Down Expand Up @@ -241,6 +247,7 @@ impl Config {
config.native = matches.occurrences_of("native") > 0;
config.hide_progess = matches.occurrences_of("hideprogress") > 0;
config.dump_json = matches.occurrences_of("json") > 0;
config.dump_locals = matches.occurrences_of("locals") > 0;

// disable native profiling if invalidly asked for
if config.native && config.non_blocking {
Expand Down
129 changes: 129 additions & 0 deletions src/dump.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
use std::collections::HashMap;

use console::style;
use failure::Error;

use crate::config::Config;
use crate::python_bindings::{v3_6_6, v3_7_0, v3_8_0};
use crate::python_interpreters::{InterpreterState, Object, TypeObject};
use crate::python_spy::PythonSpy;
use crate::python_data_access::{copy_string, copy_long, stringify_pyobject, DictIterator};

use crate::version::Version;

use remoteprocess::ProcessMemory;

pub fn print_traces(process: &mut PythonSpy, config: &Config) -> Result<(), Error> {
if config.dump_json {
let traces = process.get_stack_traces()?;
println!("{}", serde_json::to_string_pretty(&traces)?);
return Ok(())
}

// try getting the threadnames, but don't sweat it if we can't. Since this relies on dictionary
// processing we only handle py3.6+ right now, and this doesn't work at all if the
// threading module isn't imported in the target program
let thread_names = match process.version {
Version{major: 3, minor: 6, ..} => thread_name_lookup::<v3_6_6::_is>(process).ok(),
Version{major: 3, minor: 7, ..} => thread_name_lookup::<v3_7_0::_is>(process).ok(),
Version{major: 3, minor: 8, ..} => thread_name_lookup::<v3_8_0::_is>(process).ok(),
_ => None
};

println!("Process {}: {}",
style(process.pid).bold().yellow(),
process.process.cmdline()?.join(" "));

println!("Python v{} ({})\n",
style(&process.version).bold(),
style(process.process.exe()?).dim());

let traces = process.get_stack_traces()?;

for trace in traces.iter().rev() {
let thread_id = trace.format_threadid();
let thread_name = match thread_names.as_ref() {
Some(names) => names.get(&trace.thread_id),
None => None
};
match thread_name {
Some(name) => {
println!("Thread {} ({}): \"{}\"", style(thread_id).bold().yellow(), trace.status_str(), name);
}
None => {
println!("Thread {} ({})", style(thread_id).bold().yellow(), trace.status_str());
}
};

for frame in &trace.frames {
let filename = match &frame.short_filename { Some(f) => &f, None => &frame.filename };
if frame.line != 0 {
println!(" {} ({}:{})", style(&frame.name).green(), style(&filename).cyan(), style(frame.line).dim());
} else {
println!(" {} ({})", style(&frame.name).green(), style(&filename).cyan());
}

if let Some(locals) = &frame.locals {
let mut shown_args = false;
let mut shown_locals = false;
for local in locals {
if local.arg && !shown_args {
println!(" {}:", style("Arguments:").dim());
shown_args = true;
} else if !local.arg && !shown_locals {
println!(" {}:", style("Locals:").dim());
shown_locals = true;
}

let value = stringify_pyobject(&process.process, &process.version, local.addr, 128)?;
println!(" {}: {}", local.name, value);
}
}
}
}
Ok(())
}

/// Returns a hashmap of threadid: threadname, by inspecting the '_active' variable in the
/// 'threading' module.
fn thread_name_lookup<I: InterpreterState>(spy: &PythonSpy) -> Result<HashMap<u64, String>, Error> {
let mut ret = HashMap::new();
let process = &spy.process;
let interp: I = process.copy_struct(spy.interpreter_address)?;
for entry in DictIterator::from(process, interp.modules() as usize)? {
let (key, value) = entry?;
let module_name = copy_string(key as *const I::StringObject, process)?;
if module_name == "threading" {
let module: I::Object = process.copy_struct(value)?;
let module_type = process.copy_pointer(module.ob_type())?;
let dictptr: usize = process.copy_struct(value + module_type.dictoffset() as usize)?;
for i in DictIterator::from(process, dictptr)? {
let (key, value) = i?;
let name = copy_string(key as *const I::StringObject, process)?;
if name == "_active" {
for i in DictIterator::from(process, value)? {
let (key, value) = i?;
let (threadid, _) = copy_long(process, key)?;

let thread: I::Object = process.copy_struct(value)?;
let thread_type = process.copy_pointer(thread.ob_type())?;
let thread_dict_addr: usize = process.copy_struct(value + thread_type.dictoffset() as usize)?;

for i in DictIterator::from(process, thread_dict_addr)? {
let (key, value) = i?;
let varname = copy_string(key as *const I::StringObject, process)?;
if varname == "_name" {
let threadname = copy_string(value as *const I::StringObject, process)?;
ret.insert(threadid as u64, threadname);
break;
}
}
}
break;
}
}
break;
}
}
Ok(ret)
}
3 changes: 2 additions & 1 deletion src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ mod native_stack_trace;
mod python_bindings;
mod python_interpreters;
mod python_spy;
mod python_data_access;
mod stack_trace;
mod utils;
mod version;
Expand All @@ -66,4 +67,4 @@ pub use config::Config;
pub use stack_trace::StackTrace;
pub use stack_trace::Frame;
pub use remoteprocess::Pid;

pub use python_data_access::stringify_pyobject;
54 changes: 5 additions & 49 deletions src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ extern crate serde_json;
extern crate remoteprocess;

mod config;
mod dump;
mod binary_parser;
#[cfg(unwind)]
mod cython;
Expand All @@ -41,6 +42,7 @@ mod native_stack_trace;
mod python_bindings;
mod python_interpreters;
mod python_spy;
mod python_data_access;
mod stack_trace;
mod console_viewer;
mod flamegraph;
Expand All @@ -61,52 +63,6 @@ use stack_trace::{StackTrace, Frame};
use console_viewer::ConsoleViewer;
use config::{Config, FileFormat, RecordDuration};

fn format_trace_threadid(trace: &StackTrace) -> String {
// native threadids in osx are kinda useless, use the pthread id instead
#[cfg(target_os="macos")]
return format!("{:#X}", trace.thread_id);

// otherwise use the native threadid if given
#[cfg(not(target_os="macos"))]
match trace.os_thread_id {
Some(tid) => format!("{}", tid),
None => format!("{:#X}", trace.thread_id)
}
}

fn print_traces(process: &mut PythonSpy, config: &Config) -> Result<(), Error> {
if config.dump_json {
let traces = process.get_stack_traces()?;
println!("{}", serde_json::to_string_pretty(&traces)?);
return Ok(())
}

use console::style;
println!("Process {}: {}",
style(process.pid).bold().yellow(),
process.process.cmdline()?.join(" "));

println!("Python v{} ({})\n",
style(&process.version).bold(),
style(process.process.exe()?).dim());

let traces = process.get_stack_traces()?;

for trace in traces.iter().rev() {
let thread_id = format_trace_threadid(&trace);
println!("Thread {} ({})", style(thread_id).bold().yellow(), trace.status_str());
for frame in &trace.frames {
let filename = match &frame.short_filename { Some(f) => &f, None => &frame.filename };
if frame.line != 0 {
println!("\t {} ({}:{})", style(&frame.name).green(), style(&filename).cyan(), style(frame.line).dim());
} else {
println!("\t {} ({})", style(&frame.name).green(), style(&filename).cyan());
}
}
}
Ok(())
}

fn process_exitted(process: &remoteprocess::Process) -> bool {
process.exe().is_err()
}
Expand Down Expand Up @@ -270,10 +226,10 @@ fn record_samples(process: &mut PythonSpy, config: &Config) -> Result<(), Error>
}

if config.include_thread_ids {
let threadid = format_trace_threadid(&trace);
let threadid = trace.format_threadid();
trace.frames.push(Frame{name: format!("thread {}", threadid),
filename: String::from(""),
module: None, short_filename: None, line: 0});
module: None, short_filename: None, line: 0, locals: None});
}

output.increment(&trace)?;
Expand Down Expand Up @@ -343,7 +299,7 @@ fn record_samples(process: &mut PythonSpy, config: &Config) -> Result<(), Error>
fn run_spy_command(process: &mut PythonSpy, config: &config::Config) -> Result<(), Error> {
match config.command.as_ref() {
"dump" => {
print_traces(process, config)?;
dump::print_traces(process, config)?;
},
"record" => {
record_samples(process, config)?;
Expand Down
Loading

0 comments on commit e161a53

Please sign in to comment.