diff --git a/.travis.yml b/.travis.yml index 0bc14582f..7b8f906f5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,7 +19,7 @@ notifications: jobs: include: - stage: "Documentation" - julia: 1.3 + julia: nightly os: linux script: - julia --project=docs/ -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd())); diff --git a/Project.toml b/Project.toml index a108580f7..2cd5c4d43 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "SnoopCompile" uuid = "aa65fe97-06da-5843-b5b1-d5d13cad87d2" author = ["Tim Holy "] -version = "1.3.0" +version = "1.4.0" [deps] OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" diff --git a/docs/src/reference.md b/docs/src/reference.md index 6222b1426..4c5b66802 100644 --- a/docs/src/reference.md +++ b/docs/src/reference.md @@ -3,9 +3,13 @@ ```@docs @snoopi @snoopc +@snoopr SnoopCompile.parcel SnoopCompile.write SnoopCompile.read SnoopCompile.format_userimg timesum +invalidation_trees +filtermod +findcaller ``` diff --git a/docs/src/snoopr.md b/docs/src/snoopr.md index b908e335f..7770757fb 100644 --- a/docs/src/snoopr.md +++ b/docs/src/snoopr.md @@ -156,6 +156,34 @@ MethodInstance for pointer(::String, ::Integer) (1027 children) Many nodes in this tree have multiple "child" branches. +## Filtering invalidations + +Some methods trigger widespread invalidation. +If you don't have time to fix all of them, you might want to focus on a specific set of invalidations. +For instance, you might be the author of `PkgA` and you've noted that loading `PkgB` invalidates a lot of `PkgA`'s code. +In that case, you might want to find just those invalidations triggered in your package. +You can find them with [`filtermod`](@ref): + +``` +trees = invalidation_trees(@snoopr using PkgB) +ftrees = filtermod(PkgA, trees) +``` + +`filtermod` only selects trees where the root method was defined in the specified module. + +A more selective yet exhaustive tool is [`findcaller`](@ref), which allows you to find the path through the trees to a particular method: + +``` +f(data) # run once to force compilation +m = @which f(data) +using SnoopCompile +trees = invalidation_trees(@snoopr using SomePkg) +invs = findcaller(m, trees) +``` + +When you don't know which method to choose, but know an operation that got slowed down by loading `SomePkg`, you can use `@snoopi` to find methods that needed to be recompiled. See [`findcaller`](@ref) for further details. + + ## Avoiding or fixing invalidations Invalidations occur in situations like our `call2f(c64)` example, where we changed our mind about what value `f` should return for `Float64`. diff --git a/src/invalidations.jl b/src/invalidations.jl index 539fb3fa4..4b7103630 100644 --- a/src/invalidations.jl +++ b/src/invalidations.jl @@ -1,4 +1,4 @@ -export @snoopr, invalidation_trees, filtermod +export @snoopr, invalidation_trees, filtermod, findcaller dummy() = nothing dummy() @@ -169,13 +169,42 @@ function Base.show(io::IO, invalidations::MethodInvalidations) iscompact && print(io, ';') end -# `list` is in RPN format, with the "reason" coming after the items -# Here is a brief summary of the cause and resulting entries -# delete_method: -# [zero or more (mi, "invalidate_mt_cache") pairs..., zero or more (depth1 tree, loctag) pairs..., method, loctag] with loctag = "jl_method_table_disable" -# method insertion: -# [zero or more (depth0 tree, sig) pairs..., same info as with delete_method except loctag = "jl_method_table_insert"] +""" + trees = invalidation_trees(list) + +Parse `list`, as captured by [`@snoopr`](@ref), into a set of invalidation trees, where parents nodes +were called by their children. + +# Example + +```julia +julia> f(x::Int) = 1 +f (generic function with 1 method) + +julia> f(x::Bool) = 2 +f (generic function with 2 methods) + +julia> applyf(container) = f(container[1]) +applyf (generic function with 1 method) +julia> callapplyf(container) = applyf(container) +callapplyf (generic function with 1 method) + +julia> c = Any[1] +1-element Array{Any,1}: + 1 + +julia> callapplyf(c) +1 + +julia> trees = invalidation_trees(@snoopr f(::AbstractFloat) = 3) +1-element Array{SnoopCompile.MethodInvalidations,1}: + insert f(::AbstractFloat) in Main at REPL[36]:1 invalidated: + mt_backedges: 1: signature Tuple{typeof(f),Any} triggered MethodInstance for applyf(::Array{Any,1}) (1 children) more specific +``` + +See the documentation for further details. +""" function invalidation_trees(list) function checkreason(reason, loctag) if loctag == "jl_method_table_disable" @@ -281,7 +310,115 @@ function filtermod(mod::Module, invs::MethodInvalidations) return MethodInvalidations(invs.method, invs.reason, mt_backedges, backedges, copy(invs.mt_cache)) end +""" + invs = findcaller(method::Method, trees) + +Find a path through `trees` that reaches `method`. Returns a single `MethodInvalidations` object. + +# Examples + +Suppose you know that loading package `SomePkg` triggers invalidation of `f(data)`. +You can find the specific source of invalidation as follows: + +``` +f(data) # run once to force compilation +m = @which f(data) +using SnoopCompile +trees = invalidation_trees(@snoopr using SomePkg) +invs = findcaller(m, trees) +``` + +If you don't know which method to look for, but know some operation that has had added latency, +you can look for methods using `@snoopi`. For example, suppose that loading `SomePkg` makes the +next `using` statement slow. You can find the source of trouble with + +``` +julia> using SnoopCompile +julia> trees = invalidation_trees(@snoopr using SomePkg); + +julia> tinf = @snoopi using SomePkg # this second `using` will need to recompile code invalidated above +1-element Array{Tuple{Float64,Core.MethodInstance},1}: + (0.08518409729003906, MethodInstance for require(::Module, ::Symbol)) + +julia> m = tinf[1][2].def +require(into::Module, mod::Symbol) in Base at loading.jl:887 + +julia> findcaller(m, trees) +insert ==(x, y::SomeType) in SomeOtherPkg at /path/to/code:100 invalidated: + backedges: 1: superseding ==(x, y) in Base at operators.jl:83 with MethodInstance for ==(::Symbol, ::Any) (16 children) more specific +``` +""" +function findcaller(meth::Method, trees::AbstractVector{MethodInvalidations}) + for tree in trees + ret = findcaller(meth, tree) + ret === nothing || return ret + end + return nothing +end + +function findcaller(meth::Method, invs::MethodInvalidations) + function newtree(vectree) + root0 = pop!(vectree) + root = InstanceTree(root0.mi, root0.depth) + return newtree!(root, vectree) + end + function newtree!(parent, vectree) + isempty(vectree) && return getroot(parent) + child = pop!(vectree) + newp = InstanceTree(child.mi, parent, child.depth) + push!(parent.children, newp) + return newtree!(newp, vectree) + end + + for (sig, node) in invs.mt_backedges + ret = findcaller(meth, node) + ret === nothing && continue + return MethodInvalidations(invs.method, invs.reason, [Pair{Any,InstanceTree}(sig, newtree(ret))], InstanceTree[], copy(invs.mt_cache)) + end + for node in invs.backedges + ret = findcaller(meth, node) + ret === nothing && continue + return MethodInvalidations(invs.method, invs.reason, Pair{Any,InstanceTree}[], [newtree(ret)], copy(invs.mt_cache)) + end + return nothing +end + +function findcaller(meth::Method, tree::InstanceTree) + meth === tree.mi.def && return [tree] + for child in tree.children + ret = findcaller(meth, child) + if ret !== nothing + push!(ret, tree) + return ret + end + end + return nothing +end + +""" + list = @snoopr expr + +Capture method cache invalidations triggered by evaluating `expr`. +`list` is a sequence of invalidated `Core.MethodInstance`s together with "explanations," consisting +of integers (encoding depth) and strings (documenting the source of an invalidation). + +Unless you are working at a low level, you essentially always want to pass `list` +directly to [`invalidation_trees`](@ref). + +# Extended help + +`list` is in a format where the "reason" comes after the items. +Method deletion results in the sequence + + [zero or more (mi, "invalidate_mt_cache") pairs..., zero or more (depth1 tree, loctag) pairs..., method, loctag] with loctag = "jl_method_table_disable" + +where `mi` means a `MethodInstance`. `depth1` means a sequence starting at `depth=1`. + +Method insertion results in the sequence + + [zero or more (depth0 tree, sig) pairs..., same info as with delete_method except loctag = "jl_method_table_insert"] +""" macro snoopr(expr) quote local invalidations = ccall(:jl_debug_method_invalidation, Any, (Cint,), 1) diff --git a/test/snoopr.jl b/test/snoopr.jl index 661936f12..06348b6e2 100644 --- a/test/snoopr.jl +++ b/test/snoopr.jl @@ -5,6 +5,22 @@ f(x::Int) = 1 f(x::Bool) = 2 applyf(container) = f(container[1]) callapplyf(container) = applyf(container) + +# "multi-caller". Mimics invalidations triggered by defining ==(::SomeType, ::Any) +mc(x, y) = false +mc(x::Int, y::Int) = x === y +mc(x::Symbol, y::Symbol) = x === y +function mcc(container, y) + x = container[1] + return mc(x, y) +end +function mcc(container, y, extra) + x = container[1] + return mc(x, y) + extra +end +mccc1(container, y) = mcc(container, y) +mccc2(container, y) = mcc(container, y, 10) + end @testset "@snoopr" begin @@ -104,4 +120,23 @@ end ftree = only(ftrees) @test ftree.backedges == tree.backedges @test isempty(ftree.mt_backedges) + + cai = Any[1] + cas = Any[:sym] + @test SnooprTests.mccc1(cai, 1) + @test !SnooprTests.mccc1(cai, 2) + @test !SnooprTests.mccc1(cas, 1) + @test SnooprTests.mccc2(cai, 1) == 11 + @test SnooprTests.mccc2(cai, 2) == 10 + @test SnooprTests.mccc2(cas, 1) == 10 + trees = invalidation_trees(@snoopr SnooprTests.mc(x::AbstractFloat, y::Int) = x == y) + node = only(trees).backedges[1] + @test length(node.children) == 2 + m = which(SnooprTests.mccc1, (Any, Any)) + ft = findcaller(m, trees) + fnode = only(ft.backedges) + while !isempty(fnode.children) + fnode = only(fnode.children) + end + @test fnode.mi.def === m end