Merge pull request #245 from timholy/teh/codetracking

Integrate with CodeTracking

.travis.yml

@@ -6,6 +6,7 @@ os:
   - osx
 julia:
   - 1.0
+  - 1.1
   - nightly
 notifications:
   email: false

REQUIRE

@@ -1,2 +1,3 @@
 julia 1.0
 OrderedCollections
+CodeTracking

appveyor.yml

@@ -1,6 +1,7 @@
 environment:
   matrix:
   - julia_version: 1
+  - julia_version: 1.1
   - julia_version: nightly
 
 platform:

src/Revise.jl

@@ -4,7 +4,8 @@ using FileWatching, REPL, Distributed, UUIDs
 import LibGit2
 using Base: PkgId
 
-using OrderedCollections: OrderedDict
+using OrderedCollections, CodeTracking
+using CodeTracking: PkgFiles, basedir, srcfiles
 
 export revise, includet, MethodSummary
 
@@ -330,7 +331,7 @@ function init_watching(pkgdata::PkgData, files)
     udirs = Set{String}()
     for file in files
         dir, basename = splitdir(file)
-        dirfull = joinpath(pkgdata.path, dir)
+        dirfull = joinpath(basedir(pkgdata), dir)
         haskey(watched_files, dirfull) || (watched_files[dirfull] = WatchList())
         push!(watched_files[dirfull], basename)
         if watching_files[]
@@ -341,7 +342,7 @@ function init_watching(pkgdata::PkgData, files)
         end
     end
     for dir in udirs
-        dirfull = joinpath(pkgdata.path, dir)
+        dirfull = joinpath(basedir(pkgdata), dir)
         updatetime!(watched_files[dirfull])
         if !watching_files[]
             dwatcher = Rescheduler(revise_dir_queued, (pkgdata, dir))
@@ -362,7 +363,7 @@ This is generally called via a [`Rescheduler`](@ref).
 @noinline function revise_dir_queued(pkgdata::PkgData, dirname)
     dirname0 = dirname
     if !isabspath(dirname)
-        dirname = joinpath(pkgdata.path, dirname)
+        dirname = joinpath(basedir(pkgdata), dirname)
     end
     if !isdir(dirname)
         sleep(0.1)   # in case git has done a delete/replace cycle
@@ -376,7 +377,7 @@ This is generally called via a [`Rescheduler`](@ref).
     latestfiles, stillwatching = watch_files_via_dir(dirname)  # will block here until file(s) change
     for file in latestfiles
         key = joinpath(dirname0, file)
-        if haskey(pkgdata.fileinfos, key)  # issue #228
+        if hasfile(pkgdata, key)  # issue #228
             push!(revision_queue, (pkgdata, key))
         end
     end
@@ -395,7 +396,7 @@ This is used only on platforms (like BSD) which cannot use [`revise_dir_queued`]
 function revise_file_queued(pkgdata::PkgData, file)
     file0 = file
     if !isabspath(file)
-        file = joinpath(pkgdata.path, file)
+        file = joinpath(basedir(pkgdata), file)
     end
     if !isfile(file)
         sleep(0.1)  # in case git has done a delete/replace cycle
@@ -428,20 +429,20 @@ It then deletes any removed methods and re-evaluates any changed expressions.
 `Revise.pkgdatas[id].fileinfos`.
 """
 function revise_file_now(pkgdata::PkgData, file)
-    pkgdict = pkgdata.fileinfos
-    if !haskey(pkgdict, file)
+    i = fileindex(pkgdata, file)
+    if i === nothing
         println("Revise is currently tracking the following files in $(pkgdata.id): ", keys(pkgdict))
         error(file, " is not currently being tracked.")
     end
     maybe_parse_from_cache!(pkgdata, file)
-    fi = pkgdict[file]
+    fi = fileinfo(pkgdata, i)
     fmref = fi.fm
-    filep = normpath(joinpath(pkgdata.path, file))
+    filep = normpath(joinpath(basedir(pkgdata), file))
     topmod = first(keys(fi.fm))
     fmnew = parse_source(filep, topmod)
     if fmnew != nothing
         fmrep = eval_revised(fmnew, fmref)
-        pkgdict[file] = FileInfo(fmrep, fi)
+        pkgdata.fileinfos[i] = FileInfo(fmrep, fi)
     end
     nothing
 end
@@ -467,8 +468,14 @@ function instantiate_sigs!(fmm::FMMaps, def::RelocatableExpr, sig::RelocatableEx
     sigts = Any[sigt2methsig(sigt) for sigt in sigts]
     # Insert into the maps
     fmm.defmap[def] = (sigts, 0)
+    defex = convert(Expr, def)
     for sigt in sigts
         fmm.sigtmap[sigt] = def
+        # Also add to CodeTracking
+        meth = whichtt(sigt)
+        if isa(meth, Method)
+            CodeTracking.method_info[sigt] = (LineNumberNode(Int(meth.line), meth.file), defex)
+        end
     end
     return def
 end
@@ -526,8 +533,9 @@ to propagate an updated macro definition, or to force recompiling generated func
 function revise(mod::Module)
     mod == Main && error("cannot revise(Main)")
     id = PkgId(mod)
-    pkgdict = pkgdatas[id].fileinfos
-    for (file, fi) in pkgdict
+    pkgdata = pkgdatas[id]
+    for (i, file) in enumerate(srcfiles(pkgdata))
+        fi = fileinfo(pkgdata, i)
         for (mod,fmm) in fi.fm
             for def in keys(fmm.defmap)
                 Core.eval(mod, convert(Expr, def))
@@ -556,7 +564,7 @@ function track(mod::Module, file::AbstractString)
             pkgdatas[id] = PkgData(id, pathof(mod))
         end
         pkgdata = pkgdatas[id]
-        pkgdata.fileinfos[file] = FileInfo(fm)
+        push!(pkgdata, file=>FileInfo(fm))
         init_watching(pkgdata, (file,))
     end
 end
@@ -658,14 +666,14 @@ function get_def(method::Method; modified_files=revision_queue)
     id === nothing && return nothing
     pkgdata = pkgdatas[id]
     filename = relpath(filename, pkgdata)
-    if haskey(pkgdata.fileinfos, filename)
+    if hasfile(pkgdata, filename)
         def = get_def(method, pkgdata, filename)
         def !== nothing && return def
     end
     # Perhaps this is a method that was defined via a macro defined in a different module.
     # E.g., https://github.com/timholy/Rebugger.jl/issues/3
     # Try to find the actual file in the same package/module
-    for file in keys(pkgdata.fileinfos)
+    for file in srcfiles(pkgdata)
         def = get_def(method, pkgdata, file)
         def !== nothing && return def
     end
@@ -675,43 +683,37 @@ end
 
 function get_def(method, pkgdata, filename)
     maybe_parse_from_cache!(pkgdata, filename)
-    fi = pkgdata.fileinfos[filename]
+    fi = fileinfo(pkgdata, filename)
     fmm = get(fi.fm, method.module, nothing)
     fmm === nothing && return nothing
     map = fmm.sigtmap
     haskey(map, method.sig) && return copy(map[method.sig])
     return nothing
 end
 
-function get_tracked_id(mod::Module; modified_files=revision_queue)
-    id = PkgId(mod)
+function get_tracked_id(id::PkgId; modified_files=revision_queue)
     # Methods from Base or the stdlibs may require that we start tracking
     if !haskey(pkgdatas, id)
-        recipemod = mod
-        if id.name == "Base"
-            recipemod = Base
-        elseif id.name == "Core"
-            recipemod = Core.Compiler
-        elseif Symbol(id.name) ∈ stdlib_names
-            prevmod = recipemod
-            while String(nameof(recipemod)) != id.name
-                recipemod = parentmodule(recipemod)
-                recipemod == prevmod && break
-                prevmod = recipemod
-            end
-        else
-            @warn "$id not found, not able to track"
-            return nothing
-        end
-        @info "tracking $recipemod"
-        track(recipemod; modified_files=modified_files)
+        recipe = id.name === "Compiler" ? :Compiler : Symbol(id.name)
+        _track(id, recipe; modified_files=modified_files)
+        @info "tracking $recipe"
         if !haskey(pkgdatas, id)
-            @warn "despite tracking $recipemod, $id was not found"
+            @warn "despite tracking $recipe, $id was not found"
             return nothing
         end
     end
     return id
 end
+get_tracked_id(mod::Module; modified_files=revision_queue) =
+    get_tracked_id(PkgId(mod); modified_files=modified_files)
+
+function get_expressions(id::PkgId, filename)
+    get_tracked_id(id)
+    pkgdata = pkgdatas[id]
+    maybe_parse_from_cache!(pkgdata, filename)
+    fi = fileinfo(pkgdata, filename)
+    return fi.fm
+end
 
 function fix_line_statements!(ex::Expr, file::Symbol, line_offset::Int=0)
     if ex.head == :line
@@ -859,6 +861,12 @@ function __init__()
     end
     # Correct line numbers for code moving around
     Base.update_stackframes_callback[] = update_stacktrace_lineno!
+    # Populate CodeTracking data for dependencies
+    parse_pkg_files(PkgId(CodeTracking))
+    parse_pkg_files(PkgId(OrderedCollections))
+    # Set the lookup callbacks
+    CodeTracking.method_lookup_callback[] = get_def
+    CodeTracking.expressions_callback[] = get_expressions
 
     # Watch the manifest file for changes
     mfile = manifest_file()

src/exprutils.jl

@@ -129,6 +129,12 @@ function sigt2methsig(sig)
     return methsig
 end
 
+function whichtt(tt)
+    m = ccall(:jl_gf_invoke_lookup, Any, (Any, UInt), tt, typemax(UInt))
+    m === nothing && return nothing
+    return m.func::Method
+end
+
 """
     callex = get_callexpr(sigex::ExLike)