Skip to content

Commit

Permalink
Merge pull request #176 from benfred/localvars
Browse files Browse the repository at this point in the history
Add option to output local variables during dump
  • Loading branch information
benfred authored Oct 7, 2019
2 parents 503f3c6 + 24f966f commit ec8252a
Show file tree
Hide file tree
Showing 19 changed files with 1,817 additions and 335 deletions.
13 changes: 12 additions & 1 deletion generate_bindings.py
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,7 @@ def calculate_pyruntime_offsets(cpython_path, version, configure=False):

def extract_bindings(cpython_path, version, configure=False):
print("Generating bindings for python %s from repo at %s" % (version, cpython_path))

ret = os.system(f"""
cd {cpython_path}
git checkout {version}
Expand All @@ -108,6 +109,7 @@ def extract_bindings(cpython_path, version, configure=False):
cat Include/Python.h > bindgen_input.h
cat Include/frameobject.h >> bindgen_input.h
cat Objects/dict-common.h >> bindgen_input.h
echo '#define Py_BUILD_CORE 1\n' >> bindgen_input.h
cat Include/internal/pycore_pystate.h >> bindgen_input.h
Expand All @@ -124,6 +126,15 @@ def extract_bindings(cpython_path, version, configure=False):
--whitelist-type PyUnicodeObject \
--whitelist-type PyCompactUnicodeObject \
--whitelist-type PyStringObject \
--whitelist-type PyTupleObject \
--whitelist-type PyListObject \
--whitelist-type PyLongObject \
--whitelist-type PyFloatObject \
--whitelist-type PyDictObject \
--whitelist-type PyDictKeysObject \
--whitelist-type PyDictKeyEntry \
--whitelist-type PyObject \
--whitelist-type PyTypeObject \
-- -I . -I ./Include -I ./Include/internal
""")
if ret:
Expand Down Expand Up @@ -177,7 +188,7 @@ def extract_bindings(cpython_path, version, configure=False):
sys.exit(1)

if args.all:
versions = ['v3.7.0', 'v3.6.6', 'v3.5.5', 'v3.4.8', 'v3.3.7', 'v3.2.6', 'v2.7.15']
versions = ['v3.8.0b4', 'v3.7.0', 'v3.6.6', 'v3.5.5', 'v3.4.8', 'v3.3.7', 'v3.2.6', 'v2.7.15']
else:
versions = args.versions
if not versions:
Expand Down
9 changes: 8 additions & 1 deletion src/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,8 @@ pub struct Config {
pub hide_progess: bool,
#[doc(hidden)]
pub dump_json: bool,
#[doc(hidden)]
pub dump_locals: bool,
}

arg_enum!{
Expand Down Expand Up @@ -69,7 +71,7 @@ impl Default for Config {
non_blocking: false, show_line_numbers: false, sampling_rate: 100,
duration: RecordDuration::Unlimited, native: false,
gil_only: false, include_idle: false, include_thread_ids: false,
hide_progess: false, dump_json: false}
hide_progess: false, dump_json: false, dump_locals: false}
}
}

Expand Down Expand Up @@ -171,6 +173,10 @@ impl Config {
let dump = clap::SubCommand::with_name("dump")
.about("Dumps stack traces for a target program to stdout")
.arg(pid.clone().required(true))
.arg(Arg::with_name("locals")
.short("l")
.long("locals")
.help("Show local variables for each frame"))
.arg(Arg::with_name("json")
.short("j")
.long("json")
Expand Down Expand Up @@ -241,6 +247,7 @@ impl Config {
config.native = matches.occurrences_of("native") > 0;
config.hide_progess = matches.occurrences_of("hideprogress") > 0;
config.dump_json = matches.occurrences_of("json") > 0;
config.dump_locals = matches.occurrences_of("locals") > 0;

// disable native profiling if invalidly asked for
if config.native && config.non_blocking {
Expand Down
129 changes: 129 additions & 0 deletions src/dump.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
use std::collections::HashMap;

use console::style;
use failure::Error;

use crate::config::Config;
use crate::python_bindings::{v3_6_6, v3_7_0, v3_8_0};
use crate::python_interpreters::{InterpreterState, Object, TypeObject};
use crate::python_spy::PythonSpy;
use crate::python_data_access::{copy_string, copy_long, stringify_pyobject, DictIterator};

use crate::version::Version;

use remoteprocess::ProcessMemory;

pub fn print_traces(process: &mut PythonSpy, config: &Config) -> Result<(), Error> {
if config.dump_json {
let traces = process.get_stack_traces()?;
println!("{}", serde_json::to_string_pretty(&traces)?);
return Ok(())
}

// try getting the threadnames, but don't sweat it if we can't. Since this relies on dictionary
// processing we only handle py3.6+ right now, and this doesn't work at all if the
// threading module isn't imported in the target program
let thread_names = match process.version {
Version{major: 3, minor: 6, ..} => thread_name_lookup::<v3_6_6::_is>(process).ok(),
Version{major: 3, minor: 7, ..} => thread_name_lookup::<v3_7_0::_is>(process).ok(),
Version{major: 3, minor: 8, ..} => thread_name_lookup::<v3_8_0::_is>(process).ok(),
_ => None
};

println!("Process {}: {}",
style(process.pid).bold().yellow(),
process.process.cmdline()?.join(" "));

println!("Python v{} ({})\n",
style(&process.version).bold(),
style(process.process.exe()?).dim());

let traces = process.get_stack_traces()?;

for trace in traces.iter().rev() {
let thread_id = trace.format_threadid();
let thread_name = match thread_names.as_ref() {
Some(names) => names.get(&trace.thread_id),
None => None
};
match thread_name {
Some(name) => {
println!("Thread {} ({}): \"{}\"", style(thread_id).bold().yellow(), trace.status_str(), name);
}
None => {
println!("Thread {} ({})", style(thread_id).bold().yellow(), trace.status_str());
}
};

for frame in &trace.frames {
let filename = match &frame.short_filename { Some(f) => &f, None => &frame.filename };
if frame.line != 0 {
println!(" {} ({}:{})", style(&frame.name).green(), style(&filename).cyan(), style(frame.line).dim());
} else {
println!(" {} ({})", style(&frame.name).green(), style(&filename).cyan());
}

if let Some(locals) = &frame.locals {
let mut shown_args = false;
let mut shown_locals = false;
for local in locals {
if local.arg && !shown_args {
println!(" {}:", style("Arguments:").dim());
shown_args = true;
} else if !local.arg && !shown_locals {
println!(" {}:", style("Locals:").dim());
shown_locals = true;
}

let value = stringify_pyobject(&process.process, &process.version, local.addr, 128)?;
println!(" {}: {}", local.name, value);
}
}
}
}
Ok(())
}

/// Returns a hashmap of threadid: threadname, by inspecting the '_active' variable in the
/// 'threading' module.
fn thread_name_lookup<I: InterpreterState>(spy: &PythonSpy) -> Result<HashMap<u64, String>, Error> {
let mut ret = HashMap::new();
let process = &spy.process;
let interp: I = process.copy_struct(spy.interpreter_address)?;
for entry in DictIterator::from(process, interp.modules() as usize)? {
let (key, value) = entry?;
let module_name = copy_string(key as *const I::StringObject, process)?;
if module_name == "threading" {
let module: I::Object = process.copy_struct(value)?;
let module_type = process.copy_pointer(module.ob_type())?;
let dictptr: usize = process.copy_struct(value + module_type.dictoffset() as usize)?;
for i in DictIterator::from(process, dictptr)? {
let (key, value) = i?;
let name = copy_string(key as *const I::StringObject, process)?;
if name == "_active" {
for i in DictIterator::from(process, value)? {
let (key, value) = i?;
let (threadid, _) = copy_long(process, key)?;

let thread: I::Object = process.copy_struct(value)?;
let thread_type = process.copy_pointer(thread.ob_type())?;
let thread_dict_addr: usize = process.copy_struct(value + thread_type.dictoffset() as usize)?;

for i in DictIterator::from(process, thread_dict_addr)? {
let (key, value) = i?;
let varname = copy_string(key as *const I::StringObject, process)?;
if varname == "_name" {
let threadname = copy_string(value as *const I::StringObject, process)?;
ret.insert(threadid as u64, threadname);
break;
}
}
}
break;
}
}
break;
}
}
Ok(ret)
}
3 changes: 2 additions & 1 deletion src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ mod native_stack_trace;
mod python_bindings;
mod python_interpreters;
mod python_spy;
mod python_data_access;
mod stack_trace;
mod utils;
mod version;
Expand All @@ -66,4 +67,4 @@ pub use config::Config;
pub use stack_trace::StackTrace;
pub use stack_trace::Frame;
pub use remoteprocess::Pid;

pub use python_data_access::stringify_pyobject;
54 changes: 5 additions & 49 deletions src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ extern crate serde_json;
extern crate remoteprocess;

mod config;
mod dump;
mod binary_parser;
#[cfg(unwind)]
mod cython;
Expand All @@ -41,6 +42,7 @@ mod native_stack_trace;
mod python_bindings;
mod python_interpreters;
mod python_spy;
mod python_data_access;
mod stack_trace;
mod console_viewer;
mod flamegraph;
Expand All @@ -61,52 +63,6 @@ use stack_trace::{StackTrace, Frame};
use console_viewer::ConsoleViewer;
use config::{Config, FileFormat, RecordDuration};

fn format_trace_threadid(trace: &StackTrace) -> String {
// native threadids in osx are kinda useless, use the pthread id instead
#[cfg(target_os="macos")]
return format!("{:#X}", trace.thread_id);

// otherwise use the native threadid if given
#[cfg(not(target_os="macos"))]
match trace.os_thread_id {
Some(tid) => format!("{}", tid),
None => format!("{:#X}", trace.thread_id)
}
}

fn print_traces(process: &mut PythonSpy, config: &Config) -> Result<(), Error> {
if config.dump_json {
let traces = process.get_stack_traces()?;
println!("{}", serde_json::to_string_pretty(&traces)?);
return Ok(())
}

use console::style;
println!("Process {}: {}",
style(process.pid).bold().yellow(),
process.process.cmdline()?.join(" "));

println!("Python v{} ({})\n",
style(&process.version).bold(),
style(process.process.exe()?).dim());

let traces = process.get_stack_traces()?;

for trace in traces.iter().rev() {
let thread_id = format_trace_threadid(&trace);
println!("Thread {} ({})", style(thread_id).bold().yellow(), trace.status_str());
for frame in &trace.frames {
let filename = match &frame.short_filename { Some(f) => &f, None => &frame.filename };
if frame.line != 0 {
println!("\t {} ({}:{})", style(&frame.name).green(), style(&filename).cyan(), style(frame.line).dim());
} else {
println!("\t {} ({})", style(&frame.name).green(), style(&filename).cyan());
}
}
}
Ok(())
}

fn process_exitted(process: &remoteprocess::Process) -> bool {
process.exe().is_err()
}
Expand Down Expand Up @@ -270,10 +226,10 @@ fn record_samples(process: &mut PythonSpy, config: &Config) -> Result<(), Error>
}

if config.include_thread_ids {
let threadid = format_trace_threadid(&trace);
let threadid = trace.format_threadid();
trace.frames.push(Frame{name: format!("thread {}", threadid),
filename: String::from(""),
module: None, short_filename: None, line: 0});
module: None, short_filename: None, line: 0, locals: None});
}

output.increment(&trace)?;
Expand Down Expand Up @@ -343,7 +299,7 @@ fn record_samples(process: &mut PythonSpy, config: &Config) -> Result<(), Error>
fn run_spy_command(process: &mut PythonSpy, config: &config::Config) -> Result<(), Error> {
match config.command.as_ref() {
"dump" => {
print_traces(process, config)?;
dump::print_traces(process, config)?;
},
"record" => {
record_samples(process, config)?;
Expand Down
6 changes: 3 additions & 3 deletions src/native_stack_trace.rs
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ impl NativeStack {
// if we can't symbolicate, just insert a stub here.
merged.push(Frame{filename: "?".to_owned(),
name: format!("0x{:x}", addr),
line: 0, short_filename: None, module: None});
line: 0, short_filename: None, module: None, locals: None});
});

if symbolicated_count == 1 {
Expand Down Expand Up @@ -232,11 +232,11 @@ impl NativeStack {
return None;
}
let name = cython::demangle(&name).to_owned();
Some(Frame{filename, line, name, short_filename: None, module: Some(frame.module.clone())})
Some(Frame{filename, line, name, short_filename: None, module: Some(frame.module.clone()), locals: None})
},
None => {
Some(Frame{filename: frame.module.clone(),
name: format!("0x{:x}", frame.addr),
name: format!("0x{:x}", frame.addr), locals: None,
line: 0, short_filename: None, module: Some(frame.module.clone())})
}
}
Expand Down
Loading

0 comments on commit ec8252a

Please sign in to comment.