hook up the file system to the registry parsing

src/Registry/Registry.jl

@@ -3,8 +3,8 @@ module Registry
 import ..Pkg
 using ..Pkg: depots1, printpkgstyle, DEFAULT_IO, isdir_nothrow, pathrepr, pkg_server,
              GitTools, OFFLINE_MODE, UPDATED_REGISTRY_THIS_SESSION
-using ..Pkg.PlatformEngines: download_verify_unpack, download
-using UUIDs, LibGit2
+using ..Pkg.PlatformEngines: download_verify_unpack, download, download_verify, exe7z
+using UUIDs, LibGit2, TOML
 
 include("tar_filesystem.jl")
 include("registry_instance.jl")
@@ -150,6 +150,8 @@ function registry_use_pkg_server(url)
     end
 end
 
+registry_read_from_tarball() = true
+
 function check_registry_state(reg, url)
     reg_currently_uses_pkg_server = reg.tree_info !== nothing
     reg_should_use_pkg_server = registry_use_pkg_server(url)
@@ -171,66 +173,84 @@ function download_registries(io::IO, regs::Vector{RegistrySpec}, depot::String=d
         if reg.path !== nothing && reg.url !== nothing
             Pkg.Types.pkgerror("ambiguous registry specification; both url and path is set.")
         end
-        # clone to tmpdir first
-        mktempdir() do tmp
-            url, registry_urls = pkg_server_registry_url(reg.uuid, registry_urls)
-            if reg.path !== nothing && reg.linked == true # symlink to local source
-                registry = Registry.RegistryInstance(reg.path)
-                regpath = joinpath(depot, "registries", registry.name)
-                printpkgstyle(io, :Symlinking, "registry from `$(Base.contractuser(reg.path))`")
-                isdir(dirname(regpath)) || mkpath(dirname(regpath))
-                symlink(reg.path, regpath)
-                isfile(joinpath(regpath, "Registry.toml")) || Pkg.Types.pkgerror("no `Registry.toml` file in linked registry.")
-                registry = Registry.RegistryInstance(regpath)
-                printpkgstyle(io, :Symlinked, "registry `$(Base.contractuser(registry.name))` to `$(Base.contractuser(regpath))`")
-                return
-            elseif registry_use_pkg_server(url)
-                # download from Pkg server
-                try
-                    download_verify_unpack(url, nothing, tmp, ignore_existence = true, io = io)
-                catch err
-                    Pkg.Types.pkgerror("could not download $url")
-                end
-                tree_info_file = joinpath(tmp, ".tree_info.toml")
-                hash = pkg_server_url_hash(url)
-                write(tree_info_file, "git-tree-sha1 = " * repr(string(hash)))
-            elseif reg.path !== nothing # copy from local source
-                printpkgstyle(io, :Copying, "registry from `$(Base.contractuser(reg.path))`")
-                isfile(joinpath(reg.path, "Registry.toml")) || Pkg.Types.pkgerror("no `Registry.toml` file in source directory.")
-                registry = Registry.RegistryInstance(reg.path)
-                regpath = joinpath(depot, "registries", registry.name)
-                cp(reg.path, regpath; force=true) # has to be cp given we're copying
-                printpkgstyle(io, :Copied, "registry `$(Base.contractuser(registry.name))` to `$(Base.contractuser(regpath))`")
-                return
-            elseif reg.url !== nothing # clone from url
-                repo = GitTools.clone(io, reg.url, tmp; header = "registry from $(repr(reg.url))")
-                LibGit2.close(repo)
-            else
-                Pkg.Types.pkgerror("no path or url specified for registry")
+        url, registry_urls = pkg_server_registry_url(reg.uuid, registry_urls)
+        if url !== nothing && registry_read_from_tarball()
+            tmp = tempname()
+            try
+                download_verify(url, nothing, tmp)
+            catch err
+                Pkg.Types.pkgerror("could not download $url")
             end
-            # verify that the clone looks like a registry
-            if !isfile(joinpath(tmp, "Registry.toml"))
-                Pkg.Types.pkgerror("no `Registry.toml` file in cloned registry.")
+            tmpdir = mktempdir()
+            read(`$(exe7z()) x $tmp -o$tmpdir`)
+            regpath = joinpath(depot, "registries", reg.name)
+            Base.rm(regpath; recursive=true, force=true)
+            mkdir(regpath)
+            mv(only(readdir(tmpdir; join=true)), joinpath(regpath, reg.name * ".tar"); force=true)
+            hash = pkg_server_url_hash(url)
+            reg_info = Dict("uuid" => string(reg.uuid), "tree_info" => string(hash), "filename" => reg.name * ".tar")
+            open(joinpath(regpath, ".registry_info.toml"), "w") do io
+                TOML.print(io, reg_info)
             end
-            registry = Registry.RegistryInstance(tmp)
-            regpath = joinpath(depot, "registries", registry.name)
-            # copy to `depot`
-            ispath(dirname(regpath)) || mkpath(dirname(regpath))
-            if isfile(joinpath(regpath, "Registry.toml"))
-                existing_registry = Registry.RegistryInstance(regpath)
-                if registry.uuid == existing_registry.uuid
-                    println(io,
-                            "registry `$(registry.name)` already exist in `$(Base.contractuser(regpath))`.")
+        else
+            mktempdir() do tmp
+                if reg.path !== nothing && reg.linked == true # symlink to local source
+                    registry = Registry.RegistryInstance(reg.path; parse_packages=false)
+                    regpath = joinpath(depot, "registries", registry.name)
+                    printpkgstyle(io, :Symlinking, "registry from `$(Base.contractuser(reg.path))`")
+                    isdir(dirname(regpath)) || mkpath(dirname(regpath))
+                    symlink(reg.path, regpath)
+                    isfile(joinpath(regpath, "Registry.toml")) || Pkg.Types.pkgerror("no `Registry.toml` file in linked registry.")
+                    registry = Registry.RegistryInstance(regpath; parse_packages=false)
+                    printpkgstyle(io, :Symlinked, "registry `$(Base.contractuser(registry.name))` to `$(Base.contractuser(regpath))`")
+                    return
+                elseif registry_use_pkg_server(url)
+                    # download from Pkg server
+                    try
+                        download_verify_unpack(url, nothing, tmp, ignore_existence = true, io = io)
+                    catch err
+                        Pkg.Types.pkgerror("could not download $url")
+                    end
+                    tree_info_file = joinpath(tmp, ".tree_info.toml")
+                    hash = pkg_server_url_hash(url)
+                    write(tree_info_file, "git-tree-sha1 = " * repr(string(hash)))
+                elseif reg.path !== nothing # copy from local source
+                    printpkgstyle(io, :Copying, "registry from `$(Base.contractuser(reg.path))`")
+                    isfile(joinpath(reg.path, "Registry.toml")) || Pkg.Types.pkgerror("no `Registry.toml` file in source directory.")
+                    registry = Registry.RegistryInstance(reg.path; parse_packages=false)
+                    regpath = joinpath(depot, "registries", registry.name)
+                    cp(reg.path, regpath; force=true) # has to be cp given we're copying
+                    printpkgstyle(io, :Copied, "registry `$(Base.contractuser(registry.name))` to `$(Base.contractuser(regpath))`")
+                    return
+                elseif reg.url !== nothing # clone from url
+                    repo = GitTools.clone(io, reg.url, tmp; header = "registry from $(repr(reg.url))")
+                    LibGit2.close(repo)
                 else
-                    throw(Pkg.Types.PkgError("registry `$(registry.name)=\"$(registry.uuid)\"` conflicts with " *
-                        "existing registry `$(existing_registry.name)=\"$(existing_registry.uuid)\"`. " *
-                        "To install it you can clone it manually into e.g. " *
-                        "`$(Base.contractuser(joinpath(depot, "registries", registry.name*"-2")))`."))
+                    Pkg.Types.pkgerror("no path or url specified for registry")
+                end
+                # verify that the clone looks like a registry
+                if !isfile(joinpath(tmp, "Registry.toml"))
+                    Pkg.Types.pkgerror("no `Registry.toml` file in cloned registry.")
+                end
+                registry = Registry.RegistryInstance(tmp; parse_packages=false)
+                regpath = joinpath(depot, "registries", registry.name)
+                # copy to `depot`
+                ispath(dirname(regpath)) || mkpath(dirname(regpath))
+                if isfile(joinpath(regpath, "Registry.toml"))
+                    existing_registry = Registry.RegistryInstance(regpath; parse_packages=false)
+                    if registry.uuid == existing_registry.uuid
+                        println(io,
+                                "registry `$(registry.name)` already exist in `$(Base.contractuser(regpath))`.")
+                    else
+                        # if the dir doesn't exist, or exists but doesn't contain a Registry.toml
+                        mv(tmp, regpath, force=true)
+                        printpkgstyle(io, :Added, "registry `$(registry.name)` to `$(Base.contractuser(regpath))`")
+                    end
+                elseif registry_use_pkg_server(url) || reg.linked !== true
+                    # if the dir doesn't exist, or exists but doesn't contain a Registry.toml
+                    mv(tmp, regpath, force=true)
+                    printpkgstyle(io, :Added, "registry `$(registry.name)` to `$(Base.contractuser(regpath))`")
                 end
-            elseif registry_use_pkg_server(url) || reg.linked !== true
-                # if the dir doesn't exist, or exists but doesn't contain a Registry.toml
-                mv(tmp, regpath, force=true)
-                printpkgstyle(io, :Added, "registry `$(registry.name)` to `$(Base.contractuser(regpath))`")
             end
         end
     end

src/Registry/registry_instance.jl

@@ -1,16 +1,31 @@
 using Base: UUID, SHA1
 using TOML
+using Tar
 using ..Versions: VersionSpec, VersionRange
 using ..LazilyInitializedFields
 
 # The content of a registry is assumed to be constant during the
 # lifetime of a `Registry`. Create a new `Registry` if you want to have
 # a new view on the current registry.
 
+
+# TODO, need to remake the cache here since it operates on paths...
+
 # See loading.jl
 const TOML_CACHE = Base.TOMLCache(TOML.Parser(), Dict{String, Dict{String, Any}}())
 const TOML_LOCK = ReentrantLock()
-parsefile(toml_file::AbstractString) = Base.parsed_toml(toml_file, TOML_CACHE, TOML_LOCK)
+_parsefile(toml_file::AbstractString) = Base.parsed_toml(toml_file, TOML_CACHE, TOML_LOCK)
+function parsefile(fs::Union{InMemoryFileSystem, Nothing}, folder::AbstractString, file::AbstractString)
+    if fs === nothing
+        return _parsefile(joinpath(folder, file))
+    else
+        content = readfile(fs, file)
+        return TOML.Internals.parse(TOML.Parser(content; filepath=file))
+    end
+end
+
+custom_isfile(fs::Union{InMemoryFileSystem, Nothing}, folder::AbstractString, file::AbstractString) =
+    fs === nothing ? isfile(joinpath(folder, file)) : haskey(fs.d, file)
 
 # Info about each version of a package
 @lazy mutable struct VersionInfo
@@ -113,6 +128,7 @@ end
     name::String
     uuid::UUID
 
+    fs::Union{InMemoryFileSystem, Nothing}
     # Version.toml / (Compat.toml / Deps.toml):
     @lazy info::PkgInfo
 end
@@ -122,24 +138,23 @@ registry_info(pkg::PkgEntry) = init_package_info!(pkg)
 function init_package_info!(pkg::PkgEntry)
     # Already uncompressed the info for this package, return early
     @isinit(pkg.info) && return pkg.info
-    path = joinpath(pkg.registry_path, pkg.path)
+    path = pkg.registry_path
 
-    path_package = joinpath(path, "Package.toml")
-    d_p = parsefile(path_package)
+    d_p = parsefile(pkg.fs, pkg.registry_path, joinpath(pkg.path, "Package.toml"))
     name = d_p["name"]::String
     name != pkg.name && error("inconsistend name in Registry.toml and Package.toml for pkg at $(path)")
     repo = get(d_p, "repo", nothing)::Union{Nothing, String}
     subdir = get(d_p, "subdir", nothing)::Union{Nothing, String}
 
     # Versions.toml
-    path_vers = joinpath(path, "Versions.toml")
-    d_v = isfile(path_vers) ? parsefile(path_vers) : Dict{String, Any}()
+    d_v = custom_isfile(pkg.fs, pkg.registry_path, joinpath(pkg.path, "Versions.toml")) ? 
+        parsefile(pkg.fs, pkg.registry_path, joinpath(pkg.path, "Versions.toml")) : Dict{String, Any}()
     version_info = Dict{VersionNumber, VersionInfo}(VersionNumber(k) =>
         VersionInfo(SHA1(v["git-tree-sha1"]::String), get(v, "yanked", false)::Bool) for (k, v) in d_v)
 
     # Compat.toml
-    compat_file = joinpath(path, "Compat.toml")
-    compat_data_toml = isfile(compat_file) ? parsefile(compat_file) : Dict{String, Any}()
+    compat_data_toml = custom_isfile(pkg.fs, pkg.registry_path, joinpath(pkg.path, "Compat.toml")) ? 
+        parsefile(pkg.fs, pkg.registry_path, joinpath(pkg.path, "Compat.toml")) : Dict{String, Any}()
     # The Compat.toml file might have string or vector values
     compat_data_toml = convert(Dict{String, Dict{String, Union{String, Vector{String}}}}, compat_data_toml)
     compat = Dict{VersionRange, Dict{String, VersionSpec}}()
@@ -150,8 +165,8 @@ function init_package_info!(pkg::PkgEntry)
     end
 
     # Deps.toml
-    deps_file = joinpath(path, "Deps.toml")
-    deps_data_toml = isfile(deps_file) ? parsefile(deps_file) : Dict{String, Any}()
+    deps_data_toml = custom_isfile(pkg.fs, pkg.registry_path, joinpath(pkg.path, "Deps.toml")) ?
+        parsefile(pkg.fs, pkg.registry_path, joinpath(pkg.path, "Deps.toml")) : Dict{String, Any}()
     # But the Deps.toml only have strings as values
     deps_data_toml = convert(Dict{String, Dict{String, String}}, deps_data_toml)
     deps = Dict{VersionRange, Dict{String, UUID}}()
@@ -168,7 +183,6 @@ function init_package_info!(pkg::PkgEntry)
     return pkg.info
 end
 
-
 struct RegistryInstance
     path::String
     name::String
@@ -178,14 +192,15 @@ struct RegistryInstance
     description::Union{String, Nothing}
     pkgs::Dict{UUID, PkgEntry}
     tree_info::Union{Base.SHA1, Nothing}
+    fs::Union{Nothing, InMemoryFileSystem}
     # various caches
     name_to_uuids::Dict{String, Vector{UUID}}
 end
 
-const REGISTRY_CACHE = Dict{Tuple{String, UUID}, Tuple{Base.SHA1, RegistryInstance}}()
+const REGISTRY_CACHE = Dict{String, Tuple{Base.SHA1, RegistryInstance}}()
 
-function get_cached_registry(path, uuid::UUID, tree_info::Base.SHA1)
-    v = get(REGISTRY_CACHE, (path, uuid), nothing)
+function get_cached_registry(path, tree_info::Base.SHA1)
+    v = get(REGISTRY_CACHE, path, nothing)
     if v !== nothing
         cached_tree_info, reg = v
         if cached_tree_info == tree_info
@@ -199,44 +214,57 @@ end
 
 
 function RegistryInstance(path::AbstractString)
-    d = parsefile(joinpath(path, "Registry.toml"))
-    tree_info_file = joinpath(path, ".tree_info.toml")
-    tree_info = if isfile(tree_info_file)
-        Base.SHA1(parsefile(tree_info_file)["git-tree-sha1"]::String)
+    compressed_file = nothing
+    if isfile(joinpath(path, ".registry_info.toml"))
+        d_reg_info = parsefile(nothing, path, ".registry_info.toml")
+        compressed_file = d_reg_info["filename"]::String
+        tree_info = Base.SHA1(d_reg_info["tree_info"]::String)
     else
-        nothing
+        tree_info_file = joinpath(path, ".tree_info.toml")
+        tree_info = if isfile(tree_info_file)
+            Base.SHA1(parsefile(nothing, path, ".tree_info.toml")["git-tree-sha1"]::String)
+        else
+            nothing
+        end
     end
-    reg_uuid = UUID(d["uuid"]::String)
-
     # Reuse an existing cached registry if it exists for this content
     if tree_info !== nothing
-        reg = get_cached_registry(path, reg_uuid, tree_info)
+        reg = get_cached_registry(path, tree_info)
         if reg isa RegistryInstance
             return reg
         end
     end
+
+    fs = if compressed_file !== nothing
+        create_inmemory_filesystem(joinpath(path, compressed_file))
+    else
+        nothing
+    end
+
+    d = parsefile(fs, path, "Registry.toml")
     pkgs = Dict{UUID, PkgEntry}()
     for (uuid, info) in d["packages"]::Dict{String, Any}
         uuid = UUID(uuid::String)
         info::Dict{String, Any}
         name = info["name"]::String
         pkgpath = info["path"]::String
-        pkg = PkgEntry(pkgpath, path, name, uuid, uninit)
+        pkg = PkgEntry(pkgpath, path, name, uuid, fs, uninit)
         pkgs[uuid] = pkg
     end
     reg = RegistryInstance(
         path,
         d["name"]::String,
-        reg_uuid,
+        UUID(d["uuid"]::String),
         get(d, "url", nothing)::Union{String, Nothing},
         get(d, "repo", nothing)::Union{String, Nothing},
         get(d, "description", nothing)::Union{String, Nothing},
         pkgs,
         tree_info,
+        fs,
         Dict{String, UUID}(),
     )
     if tree_info !== nothing
-        REGISTRY_CACHE[(path, reg_uuid)] = (tree_info, reg)
+        REGISTRY_CACHE[path] = (tree_info, reg)
     end
     return reg
 end
@@ -276,9 +304,11 @@ function reachable_registries(; depots::Union{String, Vector{String}}=Base.DEPOT
         reg_dir = joinpath(d, "registries")
         isdir(reg_dir) || continue
         for name in readdir(reg_dir)
-            file = joinpath(reg_dir, name, "Registry.toml")
-            isfile(file) || continue
-            push!(registries, RegistryInstance(joinpath(reg_dir, name)))
+            file_unpacked = joinpath(reg_dir, name, "Registry.toml")
+            file_packed = joinpath(reg_dir, name, ".registry_info.toml")
+            if isfile(file_unpacked) || isfile(file_packed)
+                push!(registries, RegistryInstance(joinpath(reg_dir, name)))
+            end
         end
     end
     return registries

src/Registry/tar_filesystem.jl

@@ -8,7 +8,7 @@ end
 
 mutable struct InMemoryFileSystem
     d::Dict{String, InMemoryFile}
-    tarball_io::IOBuffer
+    tarball_io::IOStream
     out_io::IOBuffer
     buf::Vector{UInt8}
 end
@@ -22,7 +22,7 @@ function readfile(ref::InMemoryFileSystem, path::AbstractString)
 end
 
 function create_inmemory_filesystem(path::AbstractString)
-    tarball = IOBuffer(read(`$(Pkg.PlatformEngines.exe7z()) x $path -so`))
+    tarball = open(path, "r")
     buf = Vector{UInt8}(undef, Tar.DEFAULT_BUFFER_SIZE)
     system = InMemoryFileSystem(Dict{String, InMemoryFile}(), tarball, IOBuffer(), buf)
     Tar.arg_read(tarball) do tar