Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Dev state merging loadsave #1357

Merged
merged 2 commits into from
Jan 17, 2019
Merged

Dev state merging loadsave #1357

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
10e1f6e
load/save/replace as needed by state merging
feliam Jan 17, 2019
dd5c220
WIP Move merging to a plugin
feliam Jan 17, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
119 changes: 91 additions & 28 deletions manticore/core/executor.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -164,8 +164,75 @@ def choice(self, state_ids):
else:
return None



def Merger(Plugin):
def load_state(self, state_id, delete=False):
return self.manticore._executor._load_state(state_id, delete)

def delete_state(self, state_id):
return self.manticore._executor._delete_state(state_id)

def replace_state(self, state_id, state):
return self.manticore._executor._replace_state(state_id, state)

def did_enqueue_state_callback(self, state, state_id):
# when a new state is addded to the list we save it so we do not have
# to repload all states when try to merges last PC
with self.locked_context('cpu_stateid_dict') as cpu_stateid_dict:
# as we may be riunning in a different process we need to access this
# on a lock and over shared memory like this
l = cpu_stateid_dict.get(state.cpu.PC, list())
l.append(state_id)
cpu_stateid_dict[state.cpu.PC] = l


def will_load_state(self, current_state_id):
# When a state is loaded for exploration lets check if we can find it a
# mate for merging
with self.locked_context('cpu_stateid_dict') as cpu_stateid_dict:
# we get the lock and get a copy of the shared context
merged_state = self.load_state(current_state_id)
states_at_pc = cpu_stateid_dict.get(merged_state.cpu.PC, [])

# lets remove ourself from the list of waiting states
assert current_state_id in states_at_pc
del states_at_pc[current_state_id]

#Iterate over all remaining states that are waiting for exploration
#at the same PC
merged_ids = []
for new_state_id in states_at_pc:
new_state = self.load_state(new_state_id)
(exp_merged_state, exp_new_state, merged_constraint) = merge_constraints(merged_state.constraints, new_state.constraints)
is_mergeable, reason = is_merge_possible(merged_state, new_state, merged_constraint)

if is_mergeable:
#Ok we'll merge it!
merged_state = merge(merged_state, new_state, exp_merged_state, exp_new_state, merged_constraint)

#lets remove the vestigial links to the old state
self.delete_state(new_state_id)
self._states.remove(new_state_id) # we are locked under locked context

merged_ids.append(new_state_id)
is_mergeable = "succeeded"
else:
is_mergeable = "failed because of " + reason
debug_string = "at PC = " + hex(current_state.cpu.PC) + \
", merge " + is_mergeable + " for state id = " + \
str(current_state_id) + " and " + str(new_state_id)
print(debug_string)

for i in merged_ids:
states_at_pc.remove(i)

cpu_stateid_dict[current_state.cpu.PC] = states_at_pc

#Ok so we have merged current_state_id with {merged_ids}
#And removed all merged_ids from everywhere

#UGLY we are replacing a state_id. This may be breaking caches in
#the future
self.replace_state(current_state_id, merged_state)


class Executor(Eventful):
Expand Down Expand Up @@ -223,7 +290,6 @@ def __init__(self, initial=None, store=None, policy='random', context=None, **kw
else:
if initial is not None:
self.add(initial)
self.cpu_stateid_dict = {}

def __del__(self):
self.manager.shutdown()
Expand Down Expand Up @@ -265,10 +331,6 @@ def enqueue(self, state):
'''
# save the state to secondary storage
state_id = self._workspace.save_state(state)
if state.cpu.PC in self.cpu_stateid_dict:
self.cpu_stateid_dict[state.cpu.PC].append(state_id)
else:
self.cpu_stateid_dict[state.cpu.PC] = [state_id]
self.put(state_id)
self._publish('did_enqueue_state', state_id, state)
return state_id
Expand Down Expand Up @@ -350,6 +412,28 @@ def get(self):
del self._states[self._states.index(state_id)]
return state_id

@sync
def _load_state(self, state_id, delete=False):
if state_id not in self._states:
raise Exception("State does not exist")
if delete:
del self._states[self._states.index(state_id)]
return self._workspace.load_state(new_state_id, delete=delete)

@sync
def _delete_state(self, state_id):
if state_id not in self._states:
raise Exception("State does not exist")
del self._states[self._states.index(state_id)]
return self._workspace.rm(state_id)

@sync
def _replace_state(self, state_id, state):
if state_id not in self._states:
raise Exception("State id does not exist")
self._workspace.rm(state_id)
self._workspace.save_state(state, state_id)

def list(self):
''' Returns the list of states ids currently queued '''
return list(self._states)
Expand Down Expand Up @@ -444,27 +528,6 @@ def run(self):
if current_state_id is not None:
self._publish('will_load_state', current_state_id)
current_state = self._workspace.load_state(current_state_id)
merged_state = current_state
if len(self.cpu_stateid_dict[current_state.cpu.PC]) > 1:
for new_state_id in self.cpu_stateid_dict[current_state.cpu.PC]:
if current_state_id != new_state_id:
new_state = self._workspace.load_state(new_state_id, delete=False)
(exp_merged_state, exp_new_state, merged_constraint) = merge_constraints(merged_state.constraints, new_state.constraints)
is_mergeable, reason = is_merge_possible(merged_state, new_state, merged_constraint)
if is_mergeable:
merged_state = merge(merged_state, new_state, exp_merged_state, exp_new_state, merged_constraint)
self._workspace.load_state(new_state_id, delete=True)
self._states.remove(new_state_id)
self.cpu_stateid_dict[current_state.cpu.PC].remove(new_state_id)
is_mergeable = "succeeded"
else:
is_mergeable = "failed because of " + reason
debug_string = "at PC = " + hex(current_state.cpu.PC) + \
", merge " + is_mergeable + " for state id = " + \
str(current_state_id) + " and " + str(new_state_id)
print(debug_string)
self.cpu_stateid_dict[current_state.cpu.PC].remove(current_state_id)
current_state = merged_state
self.forward_events_from(current_state, True)
self._publish('did_load_state', current_state, current_state_id)
logger.info("load state %r", current_state_id)
Expand Down