Skip to content

Commit

Permalink
add many missing backedges getting lost during incremental deserializ…
Browse files Browse the repository at this point in the history 
…ation

we need to restore not only backedges directly to the new methods,
but also to any methods whos backedges have not been inferred
but which might propagate invalidation changes to the new code

fix #21141
  • Loading branch information
@vtjnash
vtjnash committed Apr 6, 2017
1 parent 1d203b7 commit 78ce6c1
Showing 1 changed file with 92 additions and 33 deletions.
125 changes: 92 additions & 33 deletions src/dump.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,10 @@ static arraylist_t reinit_list;
// (only used by the incremental serializer in MODE_MODULE)
static jl_array_t *serializer_worklist;

// inverse of backedges tree
// (only used by the incremental serializer in MODE_MODULE)
htable_t edges_map;

// list of modules being deserialized with __init__ methods
// (not used in MODE_AST)
jl_array_t *jl_module_init_order;
Expand Down Expand Up @@ -1152,30 +1156,42 @@ static void jl_serialize_missing_backedges_to_mod(jl_serializer_state *s, jl_met
size_t i, l = jl_array_len(backedges);
for (i = 1; i < l; i += 2) {
jl_method_instance_t *caller = (jl_method_instance_t*)jl_array_ptr_ref(backedges, i);
if (caller->max_world == ~(size_t)0 && module_in_worklist(caller->def->module)) {
jl_serialize_value(s, caller);
jl_serialize_value(s, jl_array_ptr_ref(backedges, i - 1));
if (caller->max_world == ~(size_t)0) {
jl_value_t *missing_callee = jl_array_ptr_ref(backedges, i - 1);
jl_array_t **edges = (jl_array_t**)ptrhash_bp(&edges_map, (void*)caller);
if (*edges == HT_NOTFOUND)
*edges = jl_alloc_vec_any(0);
jl_array_ptr_1d_push(*edges, missing_callee);
}
}
}
}

static int jl_serialize_backedges_to_mod(jl_typemap_entry_t *ml, void *closure)
// the intent of this function is to invert the backedges tree
static void serialize_backedges(jl_method_instance_t *callee)
{
jl_serializer_state *s = (jl_serializer_state*)closure;
jl_method_instance_t *callee = ml->func.linfo;
jl_array_t *backedges = callee->backedges;
if (backedges) {
assert(callee->max_world == ~(size_t)0);
size_t i, l = jl_array_len(backedges);
for (i = 0; i < l; i++) {
jl_method_instance_t *caller = (jl_method_instance_t*)jl_array_ptr_ref(backedges, i);
if (caller->max_world == ~(size_t)0 && module_in_worklist(caller->def->module)) {
jl_serialize_value(s, caller);
jl_serialize_value(s, callee);
if (caller->max_world == ~(size_t)0) {
jl_array_t **edges = (jl_array_t**)ptrhash_bp(&edges_map, caller);
if (*edges == HT_NOTFOUND)
*edges = jl_alloc_vec_any(0);
jl_array_ptr_1d_push(*edges, (jl_value_t*)callee);
}
}
}
}


static int jl_serialize_backedges_to_mod(jl_typemap_entry_t *ml, void *closure)
{
(void)(jl_serializer_state*)closure;
jl_method_instance_t *callee = ml->func.linfo;
serialize_backedges(callee);
return 1;
}

Expand Down Expand Up @@ -1233,6 +1249,36 @@ static void jl_serialize_lambdas_from_mod(jl_serializer_state *s, jl_module_t *m
}
}

static void jl_serialize_backedges(jl_serializer_state *s)
{
arraylist_t worklist;
arraylist_new(&worklist, 0);
size_t i;
void **table = edges_map.table;
for (i = 0; i < edges_map.size; i += 2) {
jl_method_instance_t *caller = (jl_method_instance_t*)table[i];
jl_array_t *callee = (jl_array_t*)table[i + 1];
if (callee != HT_NOTFOUND && module_in_worklist(caller->def->module)) {
arraylist_push(&worklist, (void*)caller);
}
}
while (worklist.len) {
jl_method_instance_t *caller = (jl_method_instance_t*)arraylist_pop(&worklist);
jl_array_t **pcallee = (jl_array_t**)ptrhash_bp(&edges_map, (void*)caller),
*callee = *pcallee;
if (callee != HT_NOTFOUND) {
jl_serialize_value(s, caller);
jl_serialize_value(s, callee);
*pcallee = HT_NOTFOUND;
for (i = 0; i < jl_array_len(callee); i++) {
jl_value_t *c = jl_array_ptr_ref(callee, i);
if (jl_is_method_instance(c))
arraylist_push(&worklist, (void*)c);
}
}
}
}

// serialize information about all of the modules accessible directly from Main
static void write_mod_list(ios_t *s)
{
Expand Down Expand Up @@ -2084,7 +2130,7 @@ typedef struct _linkedlist_t {
};
struct {
jl_method_instance_t *caller;
jl_value_t *callee;
jl_array_t *callee;
};
} def[100];
size_t count;
Expand Down Expand Up @@ -2118,12 +2164,11 @@ static void jl_insert_methods(linkedlist_t *list)
while (list) {
size_t i;
for (i = 0; i < list->count; i++) {
if (jl_is_method(list->def[i].meth)) {
jl_method_t *meth = list->def[i].meth;
jl_datatype_t *gf = jl_first_argument_datatype((jl_value_t*)meth->sig);
assert(jl_is_datatype(gf) && gf->name->mt);
jl_method_table_insert(gf->name->mt, meth, list->def[i].simpletype);
}
assert(jl_is_method(list->def[i].meth));
jl_method_t *meth = list->def[i].meth;
jl_datatype_t *gf = jl_first_argument_datatype((jl_value_t*)meth->sig);
assert(jl_is_datatype(gf) && gf->name->mt);
jl_method_table_insert(gf->name->mt, meth, list->def[i].simpletype);
}
list = list->next;
}
Expand All @@ -2133,23 +2178,30 @@ void jl_method_instance_delete(jl_method_instance_t *mi);
static void jl_insert_backedges(linkedlist_t *list)
{
while (list) {
size_t i;
size_t i, j;
for (i = 0; i < list->count; i++) {
if (!jl_is_method(list->def[i].meth)) {
jl_method_instance_t *caller = list->def[i].caller;
assert(jl_is_method_instance(caller));
jl_value_t *callee = list->def[i].callee;
if (jl_is_method_instance(callee)) {
jl_method_instance_t *callee_mi = (jl_method_instance_t*)callee;
if (callee_mi->max_world == ~(size_t)0)
jl_method_instance_add_backedge(callee_mi, caller);
else
jl_method_instance_delete(caller);
}
else {
jl_datatype_t *gf = jl_first_argument_datatype(callee);
assert(jl_is_datatype(gf) && gf->name->mt);
jl_method_table_add_backedge(gf->name->mt, callee, (jl_value_t*)caller);
jl_method_instance_t *caller = list->def[i].caller;
assert(jl_is_method_instance(caller));
jl_array_t *callees = list->def[i].callee;
assert(jl_is_array(callees));
if (!caller->inferred || module_in_worklist(caller->def->module)) {
for (j = 0; j < jl_array_len(callees); j++) {
jl_value_t *callee = jl_array_ptr_ref(callees, j);
if (jl_is_method_instance(callee)) {
jl_method_instance_t *callee_mi = (jl_method_instance_t*)callee;
if (callee_mi->max_world == ~(size_t)0) {
jl_method_instance_add_backedge(callee_mi, caller);
}
else if (caller->min_world == jl_world_counter) {
jl_method_instance_delete(caller);
break;
}
}
else {
jl_datatype_t *gf = jl_first_argument_datatype(callee);
assert(jl_is_datatype(gf) && gf->name->mt);
jl_method_table_add_backedge(gf->name->mt, callee, (jl_value_t*)caller);
}
}
}
}
Expand Down Expand Up @@ -2800,6 +2852,7 @@ JL_DLLEXPORT int jl_save_incremental(const char *fname, jl_array_t *worklist)
// best to keep it early (before any actual initialization)

arraylist_new(&reinit_list, 0);
htable_new(&edges_map, 0);
htable_new(&backref_table, 5000);
ptrhash_put(&backref_table, jl_main_module, (char*)HT_NOTFOUND + 1);
backref_table_numel = 1;
Expand All @@ -2814,10 +2867,13 @@ JL_DLLEXPORT int jl_save_incremental(const char *fname, jl_array_t *worklist)
jl_serialize_value(&s, worklist);
jl_serialize_lambdas_from_mod(&s, jl_main_module);
jl_serialize_value(&s, NULL); // signal end of lambdas
jl_serialize_backedges(&s);
jl_serialize_value(&s, NULL); // signal end of backedges
jl_finalize_serializer(&s); // done with f
serializer_worklist = NULL;

jl_gc_enable(en);
htable_reset(&edges_map, 0);
htable_reset(&backref_table, 0);
arraylist_free(&reinit_list);
ios_close(&f);
Expand Down Expand Up @@ -3167,6 +3223,8 @@ static jl_value_t *_jl_restore_incremental(ios_t *f)
// get list of external generic functions
linkedlist_t external_methods;
jl_deserialize_methods_from_mod(&s, &external_methods);
linkedlist_t external_backedges;
jl_deserialize_methods_from_mod(&s, &external_backedges);

arraylist_t *tracee_list = NULL;
if (jl_newmeth_tracer)
Expand All @@ -3178,8 +3236,9 @@ static jl_value_t *_jl_restore_incremental(ios_t *f)
init_order = jl_finalize_deserializer(&s, tracee_list); // done with f and s (needs to be after recache types)
jl_insert_methods(&external_methods); // hook up methods of external generic functions (needs to be after recache types)
jl_recache_other(&dependent_worlds); // make all of the other objects identities correct (needs to be after insert methods)
jl_insert_backedges(&external_methods); // restore external backedges (needs to be after recache other)
jl_insert_backedges(&external_backedges); // restore external backedges (needs to be after recache other)
free_linkedlist(external_methods.next);
free_linkedlist(external_backedges.next);
serializer_worklist = NULL;

arraylist_free(&flagref_list);
Expand Down

0 comments on commit 78ce6c1

Please sign in to comment.