From 7b16106fa4cc3400f32279106d48f7c39cc3e819 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 6 Apr 2017 12:57:39 -0400 Subject: [PATCH 1/4] fix llvm 3.3 build --- src/cgutils.cpp | 12 ++++-------- src/codegen.cpp | 2 ++ 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/cgutils.cpp b/src/cgutils.cpp index bdc1de56ae06c..b030270e7b2c3 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -931,14 +931,8 @@ static inline void maybe_mark_argument_dereferenceable(Argument *A, jl_value_t * if (!size) { return; } - llvm::AttrBuilder Attrs; - Attrs.addDereferenceableAttr(size); -#if JL_LLVM_VERSION >= 50000 - A->addAttr(llvm::AttributeList::get(jl_LLVMContext, - A->getArgNo() + 1, Attrs)); -#else - A->addAttr(llvm::AttributeSet::get(jl_LLVMContext, - A->getArgNo() + 1, Attrs)); +#if JL_LLVM_VERSION >= 30700 + A->getParent()->addDereferenceableAttr(A->getArgNo() + 1, size); #endif } @@ -946,11 +940,13 @@ static inline Instruction *maybe_mark_load_dereferenceable(Instruction *LI, bool if (!size) { return LI; } +#if JL_LLVM_VERSION >= 30700 llvm::SmallVector OPs; OPs.push_back(ConstantAsMetadata::get(ConstantInt::get(T_int64, size))); LI->setMetadata(can_be_null ? "dereferenceable_or_null" : "dereferenceable", MDNode::get(jl_LLVMContext, OPs)); +#endif return LI; } diff --git a/src/codegen.cpp b/src/codegen.cpp index a0f0a6e168fc1..7a6b7e3983867 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -5033,7 +5033,9 @@ static jl_returninfo_t get_specsig_function(Module *M, const std::string &name, continue; if (ty->isAggregateType()) { // aggregate types are passed by pointer attributes = attributes.addAttribute(jl_LLVMContext, fsig.size() + 1, Attribute::NoCapture); +#if JL_LLVM_VERSION >= 30500 attributes = attributes.addAttribute(jl_LLVMContext, fsig.size() + 1, Attribute::ReadOnly); +#endif ty = PointerType::get(ty, 0); } fsig.push_back(ty); From a52919962d527029ebed75b713ef9fa40815ed6e Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 6 Apr 2017 13:38:49 -0400 Subject: [PATCH 2/4] delete false pretense of support of max_world on a Method --- base/inference.jl | 2 +- base/methodshow.jl | 2 +- base/reflection.jl | 2 +- base/replutil.jl | 3 --- src/dump.c | 7 ------- src/gf.c | 11 +++++------ src/jltypes.c | 8 +++----- src/julia.h | 1 - src/method.c | 3 +-- 9 files changed, 12 insertions(+), 27 deletions(-) diff --git a/base/inference.jl b/base/inference.jl index 039496b3f89ca..9840772355935 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -2391,7 +2391,7 @@ function finalize_backedges(frame::InferenceState) end function code_for_method(method::Method, atypes::ANY, sparams::SimpleVector, world::UInt, preexisting::Bool=false) - if world < min_world(method) || world > max_world(method) + if world < min_world(method) return nothing end if method.isstaged && !isleaftype(atypes) diff --git a/base/methodshow.jl b/base/methodshow.jl index 8ae16639561a6..6429133a0ebe9 100644 --- a/base/methodshow.jl +++ b/base/methodshow.jl @@ -68,7 +68,7 @@ end function kwarg_decl(m::Method, kwtype::DataType) sig = rewrap_unionall(Tuple{kwtype, Core.AnyVector, unwrap_unionall(m.sig).parameters...}, m.sig) - kwli = ccall(:jl_methtable_lookup, Any, (Any, Any, UInt), kwtype.name.mt, sig, max_world(m)) + kwli = ccall(:jl_methtable_lookup, Any, (Any, Any, UInt), kwtype.name.mt, sig, typemax(UInt)) if kwli !== nothing kwli = kwli::Method src = uncompressed_ast(kwli, kwli.source) diff --git a/base/reflection.jl b/base/reflection.jl index 44fad1bd4601c..3d4b35c7bc3f4 100644 --- a/base/reflection.jl +++ b/base/reflection.jl @@ -1002,6 +1002,6 @@ has_bottom_parameter(t::TypeVar) = has_bottom_parameter(t.ub) has_bottom_parameter(::Any) = false min_world(m::Method) = reinterpret(UInt, m.min_world) -max_world(m::Method) = reinterpret(UInt, m.max_world) +max_world(m::Method) = typemax(UInt) min_world(m::Core.MethodInstance) = reinterpret(UInt, m.min_world) max_world(m::Core.MethodInstance) = reinterpret(UInt, m.max_world) diff --git a/base/replutil.jl b/base/replutil.jl index 3827db8bd278a..33e4f461d081d 100644 --- a/base/replutil.jl +++ b/base/replutil.jl @@ -576,9 +576,6 @@ function show_method_candidates(io::IO, ex::MethodError, kwargs::Vector=Any[]) if ex.world < min_world(method) print(buf, " (method too new to be called from this world context.)") end - if ex.world > max_world(method) - print(buf, " (method deleted before this world age.)") - end # TODO: indicate if it's in the wrong world push!(lines, (buf, right_matches)) end diff --git a/src/dump.c b/src/dump.c index 3df2965305ed6..1684477413617 100644 --- a/src/dump.c +++ b/src/dump.c @@ -924,10 +924,6 @@ static void jl_serialize_value_(jl_serializer_state *s, jl_value_t *v, int as_li write_int32(s->s, m->line); if (s->mode != MODE_MODULE) { write_int32(s->s, m->min_world); - write_int32(s->s, m->max_world); - } - else { - assert(m->max_world == ~(size_t)0 && "method replacement cannot be handled by incremental serializer"); } if (external_mt) jl_serialize_value(s, jl_nothing); @@ -1667,11 +1663,9 @@ static jl_value_t *jl_deserialize_value_method(jl_serializer_state *s, jl_value_ m->line = read_int32(s->s); if (s->mode != MODE_MODULE) { m->min_world = read_int32(s->s); - m->max_world = read_int32(s->s); } else { m->min_world = jl_world_counter; - m->max_world = ~(size_t)0; } m->ambig = jl_deserialize_value(s, (jl_value_t**)&m->ambig); jl_gc_wb(m, m->ambig); @@ -2204,7 +2198,6 @@ static jl_value_t *read_verify_mod_list(ios_t *s) "Requiring \"%s\" did not define a corresponding module.", name); } if (!jl_is_module(m)) { - ios_close(s); return jl_get_exceptionf(jl_errorexception_type, "Invalid module path (%s does not name a module).", name); } diff --git a/src/gf.c b/src/gf.c index de023ba2163ea..7ed522fe6e74e 100644 --- a/src/gf.c +++ b/src/gf.c @@ -142,7 +142,7 @@ static int8_t jl_cachearg_offset(jl_methtable_t *mt) // get or create the MethodInstance for a specialization JL_DLLEXPORT jl_method_instance_t *jl_specializations_get_linfo(jl_method_t *m, jl_value_t *type, jl_svec_t *sparams, size_t world) { - assert(world >= m->min_world && world <= m->max_world && "typemap lookup is corrupted"); + assert(world >= m->min_world && "typemap lookup is corrupted"); JL_LOCK(&m->writelock); jl_typemap_entry_t *sf = jl_typemap_assoc_by_type(m->specializations, (jl_tupletype_t*)type, NULL, 1, /*subtype*/0, /*offs*/0, world); @@ -163,8 +163,7 @@ JL_DLLEXPORT jl_method_instance_t *jl_specializations_get_linfo(jl_method_t *m, li->min_world = world; } if (world == jl_world_counter) { - assert(m->max_world == ~(size_t)0 && "method validity shouldn't be scheduled to terminate at a fixed future age"); - li->max_world = m->max_world; + li->max_world = ~(size_t)0; } else { li->max_world = world; @@ -350,7 +349,7 @@ JL_DLLEXPORT jl_method_instance_t* jl_set_method_inferred( } else { JL_LOCK(&li->def->writelock); - assert(min_world >= li->def->min_world && max_world <= li->def->max_world); + assert(min_world >= li->def->min_world); int isinferred = jl_is_rettype_inferred(li); if (!isinferred && li->min_world >= min_world && li->max_world <= max_world) { // expand the current (uninferred) entry to cover the full inferred range @@ -917,7 +916,7 @@ static jl_method_instance_t *cache_method(jl_methtable_t *mt, union jl_typemap_t } size_t min_valid = definition->min_world; - size_t max_valid = definition->max_world; + size_t max_valid = ~(size_t)0; int cache_with_orig = 0; jl_svec_t* guardsigs = jl_emptysvec; jl_tupletype_t *origtype = type; // backup the prior value of `type` @@ -1375,7 +1374,7 @@ JL_DLLEXPORT void jl_method_table_insert(jl_methtable_t *mt, jl_method_t *method JL_LOCK(&mt->writelock); jl_typemap_entry_t *newentry = jl_typemap_insert(&mt->defs, (jl_value_t*)mt, (jl_tupletype_t*)type, simpletype, jl_emptysvec, (jl_value_t*)method, 0, &method_defs, - method->min_world, method->max_world, &oldvalue); + method->min_world, ~(size_t)0, &oldvalue); if (oldvalue) { method->ambig = ((jl_method_t*)oldvalue)->ambig; method_overwrite(newentry, (jl_method_t*)oldvalue); diff --git a/src/jltypes.c b/src/jltypes.c index 089a3407e0d4b..c5526a59abe17 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -1882,14 +1882,13 @@ void jl_init_types(void) jl_method_type = jl_new_datatype(jl_symbol("Method"), jl_any_type, jl_emptysvec, - jl_svec(20, + jl_svec(19, jl_symbol("name"), jl_symbol("module"), jl_symbol("file"), jl_symbol("line"), jl_symbol("sig"), jl_symbol("min_world"), - jl_symbol("max_world"), jl_symbol("ambig"), jl_symbol("specializations"), jl_symbol("sparam_syms"), @@ -1903,14 +1902,13 @@ void jl_init_types(void) jl_symbol("isva"), jl_symbol("isstaged"), jl_symbol("pure")), - jl_svec(20, + jl_svec(19, jl_sym_type, jl_module_type, jl_sym_type, jl_int32_type, jl_type_type, jl_long_type, - jl_long_type, jl_any_type, // Union{Array, Void} jl_any_type, // TypeMap jl_simplevector_type, @@ -1924,7 +1922,7 @@ void jl_init_types(void) jl_bool_type, jl_bool_type, jl_bool_type), - 0, 1, 10); + 0, 1, 9); jl_method_instance_type = jl_new_datatype(jl_symbol("MethodInstance"), diff --git a/src/julia.h b/src/julia.h index f5efda7e378c7..db2fda638b455 100644 --- a/src/julia.h +++ b/src/julia.h @@ -231,7 +231,6 @@ typedef struct _jl_method_t { // method's type signature. redundant with TypeMapEntry->specTypes jl_value_t *sig; size_t min_world; - size_t max_world; // list of potentially-ambiguous methods (nothing = none, Vector{Any} of Methods otherwise) jl_value_t *ambig; diff --git a/src/method.c b/src/method.c index e48158b9727ee..65caef3a10986 100644 --- a/src/method.c +++ b/src/method.c @@ -362,7 +362,7 @@ jl_method_instance_t *jl_get_specialized(jl_method_t *m, jl_value_t *types, jl_s new_linfo->specTypes = types; new_linfo->sparam_vals = sp; new_linfo->min_world = m->min_world; - new_linfo->max_world = m->max_world; + new_linfo->max_world = ~(size_t)0; return new_linfo; } @@ -435,7 +435,6 @@ JL_DLLEXPORT jl_method_t *jl_new_method_uninit(void) m->nargs = 0; m->traced = 0; m->min_world = 1; - m->max_world = ~(size_t)0; JL_MUTEX_INIT(&m->writelock); return m; } From 5e709e46458f8a1cc566d75718b385a63ac9d917 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 6 Apr 2017 16:20:21 -0400 Subject: [PATCH 3/4] improve correctness of incremental precompile deserialized edges previously we just assumed that the user didn't load any other code now this actually keeps track of which method would have been visible in the precompilation process --- src/dump.c | 104 +++++++++++++++++++++++++++++++++++++++++---------- src/gf.c | 9 ++++- src/julia.h | 1 + src/module.c | 4 +- 4 files changed, 97 insertions(+), 21 deletions(-) diff --git a/src/dump.c b/src/dump.c index 1684477413617..007e75860f3ac 100644 --- a/src/dump.c +++ b/src/dump.c @@ -1833,6 +1833,7 @@ static jl_value_t *jl_deserialize_value_module(jl_serializer_state *s) } m->istopmod = read_uint8(s->s); m->uuid = read_uint64(s->s); + m->primary_world = jl_world_counter; m->counter = read_int32(s->s); return (jl_value_t*)m; } @@ -2128,6 +2129,7 @@ static void jl_insert_methods(linkedlist_t *list) } } +void jl_method_instance_delete(jl_method_instance_t *mi); static void jl_insert_backedges(linkedlist_t *list) { while (list) { @@ -2138,7 +2140,11 @@ static void jl_insert_backedges(linkedlist_t *list) assert(jl_is_method_instance(caller)); jl_value_t *callee = list->def[i].callee; if (jl_is_method_instance(callee)) { - jl_method_instance_add_backedge((jl_method_instance_t*)callee, caller); + jl_method_instance_t *callee_mi = (jl_method_instance_t*)callee; + if (callee_mi->max_world == ~(size_t)0) + jl_method_instance_add_backedge(callee_mi, caller); + else + jl_method_instance_delete(caller); } else { jl_datatype_t *gf = jl_first_argument_datatype(callee); @@ -2161,7 +2167,12 @@ static void free_linkedlist(linkedlist_t *list) } } -static jl_value_t *read_verify_mod_list(ios_t *s) +static int size_isgreater(const void *a, const void *b) +{ + return *(size_t*)b - *(size_t*)a; +} + +static jl_value_t *read_verify_mod_list(ios_t *s, arraylist_t *dependent_worlds) { if (!jl_main_module->uuid) { return jl_get_exceptionf(jl_errorexception_type, @@ -2205,6 +2216,8 @@ static jl_value_t *read_verify_mod_list(ios_t *s) return jl_get_exceptionf(jl_errorexception_type, "Module %s uuid did not match cache file.", name); } + if (m->primary_world > jl_main_module->primary_world) + arraylist_push(dependent_worlds, (void*)m->primary_world); } } @@ -2381,6 +2394,7 @@ static void jl_save_system_image_to_stream(ios_t *f) NULL, NULL, jl_get_ptls_states() }; + write_int32(f, jl_world_counter); // orphan old Base module if present jl_base_module = (jl_module_t*)jl_get_global(jl_main_module, jl_symbol("Base")); @@ -2427,7 +2441,6 @@ static void jl_save_system_image_to_stream(ios_t *f) write_int32(f, jl_get_t_uid_ctr()); write_int32(f, jl_get_gs_ctr()); - write_int32(f, jl_world_counter); jl_finalize_serializer(&s); // done with f and s htable_reset(&backref_table, 0); @@ -2498,6 +2511,7 @@ static void jl_restore_system_image_from_stream(ios_t *f) NULL, NULL, jl_get_ptls_states() }; + jl_world_counter = read_int32(f); jl_main_module = (jl_module_t*)jl_deserialize_value(&s, NULL); jl_top_module = (jl_module_t*)jl_deserialize_value(&s, NULL); @@ -2516,7 +2530,7 @@ static void jl_restore_system_image_from_stream(ios_t *f) jl_module_type->name->mt = (jl_methtable_t*)jl_deserialize_value(&s, NULL); intptr_t i; - for(i=0; i < builtin_typenames.len; i++) { + for (i = 0; i < builtin_typenames.len; i++) { jl_typename_t *tn = (jl_typename_t*)builtin_typenames.items[i]; tn->cache = (jl_svec_t*)jl_deserialize_value(&s, NULL); jl_gc_wb(tn, tn->cache); tn->linearcache = (jl_svec_t*)jl_deserialize_value(&s, NULL); jl_gc_wb(tn, tn->linearcache); @@ -2538,7 +2552,6 @@ static void jl_restore_system_image_from_stream(ios_t *f) int uid_ctr = read_int32(f); int gs_ctr = read_int32(f); - jl_world_counter = read_int32(f); jl_module_init_order = jl_finalize_deserializer(&s, NULL); // done with s and f jl_set_t_uid_ctr(uid_ctr); @@ -2992,26 +3005,72 @@ static void jl_update_backref_list(jl_value_t *old, jl_value_t *_new, size_t sta } } -jl_method_t *jl_recache_method(jl_method_t *m, size_t start) +static size_t lowerbound_dependent_world_set(size_t world, arraylist_t *dependent_worlds) +{ + size_t i, l = dependent_worlds->len; + for (i = 0; i < l; i++) { + size_t depworld = (size_t)dependent_worlds->items[i]; + if (depworld <= world) + return depworld; + } + return jl_main_module->primary_world; +} + +// repeated look up older methods until we come to one that existed +// at the time this module was serialized +static jl_method_t *jl_lookup_method_worldset(jl_methtable_t *mt, jl_datatype_t *sig, arraylist_t *dependent_worlds) +{ + size_t world = jl_world_counter; + jl_method_t *_new; + while (1) { + _new = (jl_method_t*)jl_methtable_lookup(mt, sig, world); + assert(_new && jl_is_method(_new)); + if (_new->min_world == jl_world_counter) + return _new; + if (_new->min_world <= jl_main_module->primary_world) + return _new; + world = lowerbound_dependent_world_set(_new->min_world, dependent_worlds); + if (world == _new->min_world) + return _new; + } +} + +// repeated look up older methods until we come to one that was valid +// at the time this module was serialized +static jl_method_instance_t *jl_lookup_methodinstance_worldset(jl_method_t *m, jl_datatype_t *argtypes, jl_svec_t *env, arraylist_t *dependent_worlds) +{ + size_t world = jl_world_counter; + jl_method_instance_t *_new; + while (1) { + _new = jl_specializations_get_linfo(m, (jl_value_t*)argtypes, env, world); + if (_new->min_world == jl_world_counter) + return _new; + if (_new->min_world <= jl_main_module->primary_world) + return _new; + world = lowerbound_dependent_world_set(_new->min_world, dependent_worlds); + if (world == _new->min_world) + return _new; + } +} + +static jl_method_t *jl_recache_method(jl_method_t *m, size_t start, arraylist_t *dependent_worlds) { jl_datatype_t *sig = (jl_datatype_t*)m->sig; jl_datatype_t *ftype = jl_first_argument_datatype((jl_value_t*)sig); jl_methtable_t *mt = ftype->name->mt; jl_set_typeof(m, (void*)(intptr_t)0x30); // invalidate the old value to help catch errors - jl_method_t *_new = (jl_method_t*)jl_methtable_lookup(mt, sig, /*TODO*/jl_world_counter); - assert(_new && jl_is_method(_new)); + jl_method_t *_new = jl_lookup_method_worldset(mt, sig, dependent_worlds); jl_update_backref_list((jl_value_t*)m, (jl_value_t*)_new, start); return _new; } -jl_method_instance_t *jl_recache_method_instance(jl_method_instance_t *li, size_t start) +static jl_method_instance_t *jl_recache_method_instance(jl_method_instance_t *li, size_t start, arraylist_t *dependent_worlds) { jl_datatype_t *sig = (jl_datatype_t*)li->def; assert(jl_is_datatype(sig) || jl_is_unionall(sig)); jl_datatype_t *ftype = jl_first_argument_datatype((jl_value_t*)sig); jl_methtable_t *mt = ftype->name->mt; - jl_method_t *m = (jl_method_t*)jl_methtable_lookup(mt, sig, /*TODO*/jl_world_counter); - assert(m && jl_is_method(m)); + jl_method_t *m = jl_lookup_method_worldset(mt, sig, dependent_worlds); jl_datatype_t *argtypes = (jl_datatype_t*)li->specTypes; jl_set_typeof(li, (void*)(intptr_t)0x40); // invalidate the old value to help catch errors @@ -3020,12 +3079,12 @@ jl_method_instance_t *jl_recache_method_instance(jl_method_instance_t *li, size_ //assert(ti != jl_bottom_type); (void)ti; if (ti == jl_bottom_type) env = jl_emptysvec; // the intersection may fail now if the type system had made an incorrect subtype env in the past - jl_method_instance_t *_new = jl_specializations_get_linfo(m, (jl_value_t*)argtypes, env, /*TODO*/jl_world_counter); + jl_method_instance_t *_new = jl_lookup_methodinstance_worldset(m, argtypes, env, dependent_worlds); jl_update_backref_list((jl_value_t*)li, (jl_value_t*)_new, start); return _new; } -static void jl_recache_other(void) +static void jl_recache_other(arraylist_t *dependent_worlds) { size_t i = 0; while (i < flagref_list.len) { @@ -3035,11 +3094,11 @@ static void jl_recache_other(void) i += 2; if (jl_is_method(o)) { // lookup the real Method based on the placeholder sig - _new = (jl_value_t*)jl_recache_method((jl_method_t*)o, i); + _new = (jl_value_t*)jl_recache_method((jl_method_t*)o, i, dependent_worlds); } else if (jl_is_method_instance(o)) { // lookup the real MethodInstance based on the placeholder specTypes - _new = (jl_value_t*)jl_recache_method_instance((jl_method_instance_t*)o, i); + _new = (jl_value_t*)jl_recache_method_instance((jl_method_instance_t*)o, i, dependent_worlds); } else { abort(); @@ -3075,9 +3134,14 @@ static jl_value_t *_jl_restore_incremental(ios_t *f) ios_skip(f, deplen); } + // list of world counters of incremental dependencies + arraylist_t dependent_worlds; + arraylist_new(&dependent_worlds, 0); + // verify that the system state is valid - jl_value_t *verify_error = read_verify_mod_list(f); + jl_value_t *verify_error = read_verify_mod_list(f, &dependent_worlds); if (verify_error) { + arraylist_free(&dependent_worlds); ios_close(f); return verify_error; } @@ -3086,6 +3150,7 @@ static jl_value_t *_jl_restore_incremental(ios_t *f) arraylist_new(&backref_list, 4000); arraylist_push(&backref_list, jl_main_module); arraylist_new(&flagref_list, 0); + qsort(dependent_worlds.items, dependent_worlds.len, sizeof(size_t), size_isgreater); int en = jl_gc_enable(0); ++jl_world_counter; // reserve a world age for the deserialization @@ -3112,17 +3177,18 @@ static jl_value_t *_jl_restore_incremental(ios_t *f) jl_recache_types(); // make all of the types identities correct init_order = jl_finalize_deserializer(&s, tracee_list); // done with f and s (needs to be after recache types) jl_insert_methods(&external_methods); // hook up methods of external generic functions (needs to be after recache types) - jl_recache_other(); // make all of the other objects identities correct (needs to be after insert methods) + jl_recache_other(&dependent_worlds); // make all of the other objects identities correct (needs to be after insert methods) jl_insert_backedges(&external_methods); // restore external backedges (needs to be after recache other) free_linkedlist(external_methods.next); serializer_worklist = NULL; - JL_GC_PUSH2(&init_order, &restored); - jl_gc_enable(en); arraylist_free(&flagref_list); arraylist_free(&backref_list); + arraylist_free(&dependent_worlds); ios_close(f); + JL_GC_PUSH2(&init_order, &restored); + jl_gc_enable(en); if (tracee_list) { jl_methtable_t *mt; while ((mt = (jl_methtable_t*)arraylist_pop(tracee_list)) != NULL) diff --git a/src/gf.c b/src/gf.c index 7ed522fe6e74e..31adb4e020b6a 100644 --- a/src/gf.c +++ b/src/gf.c @@ -1246,7 +1246,7 @@ static void invalidate_method_instance(jl_method_instance_t *replaced, size_t ma jl_array_t *backedges = replaced->backedges; if (replaced->max_world > max_world) { // recurse to all backedges to update their valid range also - assert(replaced->min_world <= max_world && "attempting to set invalid world constraints"); + assert(replaced->min_world - 1 <= max_world && "attempting to set invalid world constraints"); if (JL_DEBUG_METHOD_INVALIDATION) { int d0 = depth; while (d0-- > 0) @@ -1361,6 +1361,13 @@ JL_DLLEXPORT void jl_method_table_add_backedge(jl_methtable_t *mt, jl_value_t *t JL_UNLOCK(&mt->writelock); } +void jl_method_instance_delete(jl_method_instance_t *mi) +{ + invalidate_method_instance(mi, mi->min_world - 1, 0); + if (JL_DEBUG_METHOD_INVALIDATION) + jl_uv_puts(JL_STDOUT, "<<<\n", 4); +} + JL_DLLEXPORT void jl_method_table_insert(jl_methtable_t *mt, jl_method_t *method, jl_tupletype_t *simpletype) { assert(jl_is_method(method)); diff --git a/src/julia.h b/src/julia.h index db2fda638b455..f9c520c0d4c59 100644 --- a/src/julia.h +++ b/src/julia.h @@ -405,6 +405,7 @@ typedef struct _jl_module_t { arraylist_t usings; // modules with all bindings potentially imported uint8_t istopmod; uint64_t uuid; + size_t primary_world; uint32_t counter; } jl_module_t; diff --git a/src/module.c b/src/module.c index 4225ce5eaa0f9..ec64c58dcc866 100644 --- a/src/module.c +++ b/src/module.c @@ -29,7 +29,9 @@ JL_DLLEXPORT jl_module_t *jl_new_module(jl_sym_t *name) m->istopmod = 0; static unsigned int mcounter; // simple counter backup, in case hrtime is not incrementing m->uuid = jl_hrtime() + (++mcounter); - if (!m->uuid) m->uuid++; // uuid 0 is invalid + if (!m->uuid) + m->uuid++; // uuid 0 is invalid + m->primary_world = 0; m->counter = 0; htable_new(&m->bindings, 0); arraylist_new(&m->usings, 0); From 8d16ddda75cfe26fdf8e1a9fea589d8170fe9585 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 6 Apr 2017 19:06:18 -0400 Subject: [PATCH 4/4] add many missing backedges getting lost during incremental deserialization we need to restore not only backedges directly to the new methods, but also to any methods whos backedges have not been inferred but which might propagate invalidation changes to the new code fix #21141 --- src/dump.c | 153 +++++++++++++++++++++++++++++++++++++----------- test/compile.jl | 47 ++++++++++++++- 2 files changed, 164 insertions(+), 36 deletions(-) diff --git a/src/dump.c b/src/dump.c index 007e75860f3ac..d482cd3cf5cdc 100644 --- a/src/dump.c +++ b/src/dump.c @@ -62,6 +62,10 @@ static arraylist_t reinit_list; // (only used by the incremental serializer in MODE_MODULE) static jl_array_t *serializer_worklist; +// inverse of backedges tree +// (only used by the incremental serializer in MODE_MODULE) +htable_t edges_map; + // list of modules being deserialized with __init__ methods // (not used in MODE_AST) jl_array_t *jl_module_init_order; @@ -1152,30 +1156,42 @@ static void jl_serialize_missing_backedges_to_mod(jl_serializer_state *s, jl_met size_t i, l = jl_array_len(backedges); for (i = 1; i < l; i += 2) { jl_method_instance_t *caller = (jl_method_instance_t*)jl_array_ptr_ref(backedges, i); - if (caller->max_world == ~(size_t)0 && module_in_worklist(caller->def->module)) { - jl_serialize_value(s, caller); - jl_serialize_value(s, jl_array_ptr_ref(backedges, i - 1)); + if (caller->max_world == ~(size_t)0) { + jl_value_t *missing_callee = jl_array_ptr_ref(backedges, i - 1); + jl_array_t **edges = (jl_array_t**)ptrhash_bp(&edges_map, (void*)caller); + if (*edges == HT_NOTFOUND) + *edges = jl_alloc_vec_any(0); + jl_array_ptr_1d_push(*edges, missing_callee); } } } } -static int jl_serialize_backedges_to_mod(jl_typemap_entry_t *ml, void *closure) +// the intent of this function is to invert the backedges tree +static void serialize_backedges(jl_method_instance_t *callee) { - jl_serializer_state *s = (jl_serializer_state*)closure; - jl_method_instance_t *callee = ml->func.linfo; jl_array_t *backedges = callee->backedges; if (backedges) { assert(callee->max_world == ~(size_t)0); size_t i, l = jl_array_len(backedges); for (i = 0; i < l; i++) { jl_method_instance_t *caller = (jl_method_instance_t*)jl_array_ptr_ref(backedges, i); - if (caller->max_world == ~(size_t)0 && module_in_worklist(caller->def->module)) { - jl_serialize_value(s, caller); - jl_serialize_value(s, callee); + if (caller->max_world == ~(size_t)0) { + jl_array_t **edges = (jl_array_t**)ptrhash_bp(&edges_map, caller); + if (*edges == HT_NOTFOUND) + *edges = jl_alloc_vec_any(0); + jl_array_ptr_1d_push(*edges, (jl_value_t*)callee); } } } +} + + +static int jl_serialize_backedges_to_mod(jl_typemap_entry_t *ml, void *closure) +{ + (void)(jl_serializer_state*)closure; + jl_method_instance_t *callee = ml->func.linfo; + serialize_backedges(callee); return 1; } @@ -1233,6 +1249,36 @@ static void jl_serialize_lambdas_from_mod(jl_serializer_state *s, jl_module_t *m } } +static void jl_serialize_backedges(jl_serializer_state *s) +{ + arraylist_t worklist; + arraylist_new(&worklist, 0); + size_t i; + void **table = edges_map.table; + for (i = 0; i < edges_map.size; i += 2) { + jl_method_instance_t *caller = (jl_method_instance_t*)table[i]; + jl_array_t *callee = (jl_array_t*)table[i + 1]; + if (callee != HT_NOTFOUND && module_in_worklist(caller->def->module)) { + arraylist_push(&worklist, (void*)caller); + } + } + while (worklist.len) { + jl_method_instance_t *caller = (jl_method_instance_t*)arraylist_pop(&worklist); + jl_array_t **pcallee = (jl_array_t**)ptrhash_bp(&edges_map, (void*)caller), + *callee = *pcallee; + if (callee != HT_NOTFOUND) { + jl_serialize_value(s, caller); + jl_serialize_value(s, callee); + *pcallee = HT_NOTFOUND; + for (i = 0; i < jl_array_len(callee); i++) { + jl_value_t *c = jl_array_ptr_ref(callee, i); + if (jl_is_method_instance(c)) + arraylist_push(&worklist, (void*)c); + } + } + } +} + // serialize information about all of the modules accessible directly from Main static void write_mod_list(ios_t *s) { @@ -2084,7 +2130,7 @@ typedef struct _linkedlist_t { }; struct { jl_method_instance_t *caller; - jl_value_t *callee; + jl_array_t *callee; }; } def[100]; size_t count; @@ -2118,12 +2164,11 @@ static void jl_insert_methods(linkedlist_t *list) while (list) { size_t i; for (i = 0; i < list->count; i++) { - if (jl_is_method(list->def[i].meth)) { - jl_method_t *meth = list->def[i].meth; - jl_datatype_t *gf = jl_first_argument_datatype((jl_value_t*)meth->sig); - assert(jl_is_datatype(gf) && gf->name->mt); - jl_method_table_insert(gf->name->mt, meth, list->def[i].simpletype); - } + assert(jl_is_method(list->def[i].meth)); + jl_method_t *meth = list->def[i].meth; + jl_datatype_t *gf = jl_first_argument_datatype((jl_value_t*)meth->sig); + assert(jl_is_datatype(gf) && gf->name->mt); + jl_method_table_insert(gf->name->mt, meth, list->def[i].simpletype); } list = list->next; } @@ -2133,23 +2178,56 @@ void jl_method_instance_delete(jl_method_instance_t *mi); static void jl_insert_backedges(linkedlist_t *list) { while (list) { - size_t i; + size_t i, j; for (i = 0; i < list->count; i++) { - if (!jl_is_method(list->def[i].meth)) { - jl_method_instance_t *caller = list->def[i].caller; - assert(jl_is_method_instance(caller)); - jl_value_t *callee = list->def[i].callee; - if (jl_is_method_instance(callee)) { - jl_method_instance_t *callee_mi = (jl_method_instance_t*)callee; - if (callee_mi->max_world == ~(size_t)0) - jl_method_instance_add_backedge(callee_mi, caller); - else - jl_method_instance_delete(caller); - } - else { - jl_datatype_t *gf = jl_first_argument_datatype(callee); - assert(jl_is_datatype(gf) && gf->name->mt); - jl_method_table_add_backedge(gf->name->mt, callee, (jl_value_t*)caller); + jl_method_instance_t *caller = list->def[i].caller; + assert(jl_is_method_instance(caller)); + jl_array_t *callees = list->def[i].callee; + assert(jl_is_array(callees)); + if (!caller->inferred || module_in_worklist(caller->def->module)) { + for (j = 0; j < jl_array_len(callees); j++) { + jl_value_t *callee = jl_array_ptr_ref(callees, j); + if (jl_is_method_instance(callee)) { + jl_method_instance_t *callee_mi = (jl_method_instance_t*)callee; + if (!module_in_worklist(callee_mi->def->module)) { + // verify that this MethodInstance is still the correct lookup for this callee sig + jl_value_t *sig = callee_mi->specTypes; + jl_method_t *m = NULL; + if (jl_is_datatype(sig)) { + jl_datatype_t *ftype = jl_first_argument_datatype(sig); + jl_methtable_t *mt = ftype->name->mt; + jl_typemap_entry_t *entry = jl_typemap_assoc_by_type( + mt->defs, (jl_datatype_t*)sig, + NULL, /*inexact*/0, /*subtype*/1, 0, jl_world_counter); + if (entry != NULL) + m = entry->func.method; + } + if (m != callee_mi->def) { + // probably no good, just invalidate everything now + jl_method_instance_delete(caller); + break; + } + } + if (callee_mi->max_world == ~(size_t)0) { + // if this callee is still valid, just add a backedge + jl_method_instance_add_backedge(callee_mi, caller); + } + else if (caller->min_world == jl_world_counter) { + // if this caller was just added, delete everything associated with it + jl_method_instance_delete(caller); + break; + } + else { + // the caller must have been something pre-existing + // assume that it'll take care of deleting the deserialized code + // whenever we process it here + } + } + else { + jl_datatype_t *gf = jl_first_argument_datatype(callee); + assert(jl_is_datatype(gf) && gf->name->mt); + jl_method_table_add_backedge(gf->name->mt, callee, (jl_value_t*)caller); + } } } } @@ -2800,6 +2878,7 @@ JL_DLLEXPORT int jl_save_incremental(const char *fname, jl_array_t *worklist) // best to keep it early (before any actual initialization) arraylist_new(&reinit_list, 0); + htable_new(&edges_map, 0); htable_new(&backref_table, 5000); ptrhash_put(&backref_table, jl_main_module, (char*)HT_NOTFOUND + 1); backref_table_numel = 1; @@ -2814,10 +2893,13 @@ JL_DLLEXPORT int jl_save_incremental(const char *fname, jl_array_t *worklist) jl_serialize_value(&s, worklist); jl_serialize_lambdas_from_mod(&s, jl_main_module); jl_serialize_value(&s, NULL); // signal end of lambdas + jl_serialize_backedges(&s); + jl_serialize_value(&s, NULL); // signal end of backedges jl_finalize_serializer(&s); // done with f serializer_worklist = NULL; jl_gc_enable(en); + htable_reset(&edges_map, 0); htable_reset(&backref_table, 0); arraylist_free(&reinit_list); ios_close(&f); @@ -3016,7 +3098,7 @@ static size_t lowerbound_dependent_world_set(size_t world, arraylist_t *dependen return jl_main_module->primary_world; } -// repeated look up older methods until we come to one that existed +// repeatedly look up older methods until we come to one that existed // at the time this module was serialized static jl_method_t *jl_lookup_method_worldset(jl_methtable_t *mt, jl_datatype_t *sig, arraylist_t *dependent_worlds) { @@ -3167,6 +3249,8 @@ static jl_value_t *_jl_restore_incremental(ios_t *f) // get list of external generic functions linkedlist_t external_methods; jl_deserialize_methods_from_mod(&s, &external_methods); + linkedlist_t external_backedges; + jl_deserialize_methods_from_mod(&s, &external_backedges); arraylist_t *tracee_list = NULL; if (jl_newmeth_tracer) @@ -3178,8 +3262,9 @@ static jl_value_t *_jl_restore_incremental(ios_t *f) init_order = jl_finalize_deserializer(&s, tracee_list); // done with f and s (needs to be after recache types) jl_insert_methods(&external_methods); // hook up methods of external generic functions (needs to be after recache types) jl_recache_other(&dependent_worlds); // make all of the other objects identities correct (needs to be after insert methods) - jl_insert_backedges(&external_methods); // restore external backedges (needs to be after recache other) + jl_insert_backedges(&external_backedges); // restore external backedges (needs to be after recache other) free_linkedlist(external_methods.next); + free_linkedlist(external_backedges.next); serializer_worklist = NULL; arraylist_free(&flagref_list); diff --git a/test/compile.jl b/test/compile.jl index 91b0f1e4192f8..0e058899537f5 100644 --- a/test/compile.jl +++ b/test/compile.jl @@ -3,6 +3,7 @@ using Base.Test Foo_module = :Foo4b3a94a1a081a8cb +Foo2_module = :F2oo4b3a94a1a081a8cb FooBase_module = :FooBase4b3a94a1a081a8cb @eval module ConflictingBindings export $Foo_module, $FooBase_module @@ -21,6 +22,7 @@ insert!(LOAD_PATH, 1, dir) insert!(Base.LOAD_CACHE_PATH, 1, dir) try Foo_file = joinpath(dir, "$Foo_module.jl") + Foo2_file = joinpath(dir, "$Foo2_module.jl") FooBase_file = joinpath(dir, "$FooBase_module.jl") write(FooBase_file, @@ -30,12 +32,23 @@ try module $FooBase_module end """) + write(Foo2_file, + """ + __precompile__(true) + + module $Foo2_module + export override + override(x::Integer) = 2 + override(x::AbstractFloat) = Float64(override(1)) + end + """) write(Foo_file, """ __precompile__(true) module $Foo_module using $FooBase_module + import $Foo2_module: $Foo2_module, override # test that docs get reconnected @doc "foo function" foo(x) = x + 1 @@ -109,6 +122,9 @@ try ccall(:jl_specializations_get_linfo, Ref{Core.MethodInstance}, (Any, Any, Any, UInt), some_method, Tuple{typeof(Base.include), String}, Core.svec(), typemax(UInt)) end + + g() = override(1.0) + Base.Test.@test g() === 2.0 # compile this end """) @test_throws ErrorException Core.kwfunc(Base.nothing) # make sure `nothing` didn't have a kwfunc (which would invalidate the attempted test) @@ -116,9 +132,29 @@ try # Issue #12623 @test __precompile__(true) === nothing + # Issue #21307 + Base.require(Foo2_module) + @eval let Foo2_module = $(QuoteNode(Foo2_module)), # use @eval to see the results of loading the compile + Foo = getfield(Main, Foo2_module) + Foo.override(::Int) = 'a' + Foo.override(::Float32) = 'b' + end + Base.require(Foo_module) - cachefile = joinpath(dir, "$Foo_module.ji") + @eval let Foo_module = $(QuoteNode(Foo_module)), # use @eval to see the results of loading the compile + Foo = getfield(Main, Foo_module) + @test Foo.foo(17) == 18 + @test Foo.Bar.bar(17) == 19 + + # Issue #21307 + @test Foo.g() === 97.0 + @test Foo.override(1.0e0) == Float64('a') + @test Foo.override(1.0f0) == 'b' + @test Foo.override(UInt(1)) == 2 + end + + cachefile = joinpath(dir, "$Foo_module.ji") # use _require_from_serialized to ensure that the test fails if # the module doesn't reload from the image: @test_warn "WARNING: replacing module Foo4b3a94a1a081a8cb.\nWARNING: Method definition " begin @@ -129,6 +165,7 @@ try @test_throws MethodError Foo.foo(17) # world shouldn't be visible yet end @eval let Foo_module = $(QuoteNode(Foo_module)), # use @eval to see the results of loading the compile + Foo2_module = $(QuoteNode(Foo2_module)), FooBase_module = $(QuoteNode(FooBase_module)), Foo = getfield(Main, Foo_module), dir = $(QuoteNode(dir)), @@ -137,6 +174,12 @@ try @test Foo.foo(17) == 18 @test Foo.Bar.bar(17) == 19 + # Issue #21307 + @test Foo.g() === 97.0 + @test Foo.override(1.0e0) == Float64('a') + @test Foo.override(1.0f0) == 'b' + @test Foo.override(UInt(1)) == 2 + # issue #12284: @test stringmime("text/plain", Base.Docs.doc(Foo.foo)) == "foo function\n" @test stringmime("text/plain", Base.Docs.doc(Foo.Bar.bar)) == "bar function\n" @@ -147,7 +190,7 @@ try modules, deps1 = Base.cache_dependencies(cachefile) @test sort(modules) == Any[(s, Base.module_uuid(getfield(Foo, s))) for s in - [:Base, :Core, FooBase_module, :Main]] + [:Base, :Core, Foo2_module, FooBase_module, :Main]] @test deps == deps1 @test current_task()(0x01, 0x4000, 0x30031234) == 2