hook up the file system to the registry parsing

src/Registry/Registry.jl

@@ -145,6 +145,8 @@ function registry_use_pkg_server(url)
     end
 end
 
+registry_read_from_tarball() = true
+
 function check_registry_state(reg, url)
     reg_currently_uses_pkg_server = reg.tree_info !== nothing
     reg_should_use_pkg_server = registry_use_pkg_server(url)
@@ -167,50 +169,59 @@ function download_registries(io::IO, regs::Vector{RegistrySpec}, depot::String=d
             Pkg.Types.pkgerror("ambiguous registry specification; both url and path is set.")
         end
         # clone to tmpdir first
-        mktempdir() do tmp
-            url, registry_urls = pkg_server_registry_url(reg.uuid, registry_urls)
-            if registry_use_pkg_server(url)
-                # download from Pkg server
-                try
-                    download_verify_unpack(url, nothing, tmp, ignore_existence = true, io = io)
-                catch err
-                    Pkg.Types.pkgerror("could not download $url")
-                end
-                tree_info_file = joinpath(tmp, ".tree_info.toml")
-                hash = pkg_server_url_hash(url)
-                write(tree_info_file, "git-tree-sha1 = " * repr(string(hash)))
-            elseif reg.path !== nothing # copy from local source
-                printpkgstyle(io, :Copying, "registry from `$(Base.contractuser(reg.path))`")
-                mv(reg.path, tmp; force=true)
-            elseif reg.url !== nothing # clone from url
-                repo = GitTools.clone(io, reg.url, tmp; header = "registry from $(repr(reg.url))")
-                LibGit2.close(repo)
-            else
-                Pkg.Types.pkgerror("no path or url specified for registry")
-            end
-            # verify that the clone looks like a registry
-            if !isfile(joinpath(tmp, "Registry.toml"))
-                Pkg.Types.pkgerror("no `Registry.toml` file in cloned registry.")
+        if registry_read_from_tarball()
+            regpath = joinpath(depot, "registries", registry.uuid, ".tar.gz")
+            try
+                download_verify(url, nothing, regpath, ignore_existence = true)
+            catch err
+                Pkg.Types.pkgerror("could not download $url")
             end
-            registry = Registry.RegistryInstance(tmp; parse_packages=false)
-            # copy to `depot`
-            regpath = joinpath(depot, "registries", registry.name)
-            ispath(dirname(regpath)) || mkpath(dirname(regpath))
-            if isfile(joinpath(regpath, "Registry.toml"))
-                existing_registry = Registry.RegistryInstance(regpath; parse_packages=false)
-                if registry.uuid == existing_registry.uuid
-                    println(io,
-                            "registry `$(registry.name)` already exist in `$(Base.contractuser(regpath))`.")
+        else
+            mktempdir() do tmp
+                url, registry_urls = pkg_server_registry_url(reg.uuid, registry_urls)
+                if registry_use_pkg_server(url)
+                    # download from Pkg server
+                    try
+                        download_verify_unpack(url, nothing, tmp, ignore_existence = true, io = io)
+                    catch err
+                        Pkg.Types.pkgerror("could not download $url")
+                    end
+                    tree_info_file = joinpath(tmp, ".tree_info.toml")
+                    hash = pkg_server_url_hash(url)
+                    write(tree_info_file, "git-tree-sha1 = " * repr(string(hash)))
+                elseif reg.path !== nothing # copy from local source
+                    printpkgstyle(io, :Copying, "registry from `$(Base.contractuser(reg.path))`")
+                    mv(reg.path, tmp; force=true)
+                elseif reg.url !== nothing # clone from url
+                    repo = GitTools.clone(io, reg.url, tmp; header = "registry from $(repr(reg.url))")
+                    LibGit2.close(repo)
+                else
+                    Pkg.Types.pkgerror("no path or url specified for registry")
+                end
+                # verify that the clone looks like a registry
+                if !isfile(joinpath(tmp, "Registry.toml"))
+                    Pkg.Types.pkgerror("no `Registry.toml` file in cloned registry.")
+                end
+                registry = Registry.RegistryInstance(tmp; parse_packages=false)
+                # copy to `depot`
+                regpath = joinpath(depot, "registries", registry.name)
+                ispath(dirname(regpath)) || mkpath(dirname(regpath))
+                if isfile(joinpath(regpath, "Registry.toml"))
+                    existing_registry = Registry.RegistryInstance(regpath; parse_packages=false)
+                    if registry.uuid == existing_registry.uuid
+                        println(io,
+                                "registry `$(registry.name)` already exist in `$(Base.contractuser(regpath))`.")
+                    else
+                        throw(Pkg.Types.PkgError("registry `$(registry.name)=\"$(registry.uuid)\"` conflicts with " *
+                            "existing registry `$(existing_registry.name)=\"$(existing_registry.uuid)\"`. " *
+                            "To install it you can clone it manually into e.g. " *
+                            "`$(Base.contractuser(joinpath(depot, "registries", registry.name*"-2")))`."))
+                    end
                 else
-                    throw(Pkg.Types.PkgError("registry `$(registry.name)=\"$(registry.uuid)\"` conflicts with " *
-                        "existing registry `$(existing_registry.name)=\"$(existing_registry.uuid)\"`. " *
-                        "To install it you can clone it manually into e.g. " *
-                        "`$(Base.contractuser(joinpath(depot, "registries", registry.name*"-2")))`."))
+                    # if the dir doesn't exist, or exists but doesn't contain a Registry.toml
+                    mv(tmp, regpath, force=true)
+                    printpkgstyle(io, :Added, "registry `$(registry.name)` to `$(Base.contractuser(regpath))`")
                 end
-            else
-                # if the dir doesn't exist, or exists but doesn't contain a Registry.toml
-                mv(tmp, regpath, force=true)
-                printpkgstyle(io, :Added, "registry `$(registry.name)` to `$(Base.contractuser(regpath))`")
             end
         end
     end

src/Registry/registry_instance.jl

@@ -1,16 +1,31 @@
 using Base: UUID, SHA1
 using TOML
+using Tar
 using ..Versions: VersionSpec, VersionRange
 using ..LazilyInitializedFields
 
 # The content of a registry is assumed to be constant during the
 # lifetime of a `Registry`. Create a new `Registry` if you want to have
 # a new view on the current registry.
 
+
+# TODO, need to remake the cache here since it operates on paths...
+
 # See loading.jl
 const TOML_CACHE = Base.TOMLCache(TOML.Parser(), Dict{String, Dict{String, Any}}())
 const TOML_LOCK = ReentrantLock()
-parsefile(toml_file::AbstractString) = Base.parsed_toml(toml_file, TOML_CACHE, TOML_LOCK)
+_parsefile(toml_file::AbstractString) = Base.parsed_toml(toml_file, TOML_CACHE, TOML_LOCK)
+function parsefile(fs::Union{InMemoryFileSystem, Nothing}, folder::AbstractString, file::AbstractString)
+    if fs === nothing
+        return _parsefile(joinpath(folder, file))
+    else
+        content = readfile(fs, file)
+        return TOML.Internals.parse(TOML.Parser(content; filepath=file))
+    end
+end
+
+custom_isfile(fs::Union{InMemoryFileSystem, Nothing}, folder::AbstractString, file::AbstractString) =
+    fs === nothing ? isfile(joinpath(folder, file)) : haskey(fs.d, file)
 
 # Info about each version of a package
 @lazy mutable struct VersionInfo
@@ -113,6 +128,7 @@ end
     name::String
     uuid::UUID
 
+    fs::Union{InMemoryFileSystem, Nothing}
     # Version.toml / (Compat.toml / Deps.toml):
     @lazy info::PkgInfo
 end
@@ -122,24 +138,23 @@ registry_info(pkg::PkgEntry) = init_package_info!(pkg)
 function init_package_info!(pkg::PkgEntry)
     # Already uncompressed the info for this package, return early
     @isinit(pkg.info) && return pkg.info
-    path = joinpath(pkg.registry_path, pkg.path)
+    path = pkg.registry_path
 
-    path_package = joinpath(path, "Package.toml")
-    d_p = parsefile(path_package)
+    d_p = parsefile(pkg.fs, pkg.registry_path, joinpath(pkg.path, "Package.toml"))
     name = d_p["name"]::String
     name != pkg.name && error("inconsistend name in Registry.toml and Package.toml for pkg at $(path)")
     repo = get(d_p, "repo", nothing)::Union{Nothing, String}
     subdir = get(d_p, "subdir", nothing)::Union{Nothing, String}
 
     # Versions.toml
-    path_vers = joinpath(path, "Versions.toml")
-    d_v = isfile(path_vers) ? parsefile(path_vers) : Dict{String, Any}()
+    d_v = custom_isfile(pkg.fs, pkg.registry_path, joinpath(pkg.path, "Versions.toml")) ? 
+        parsefile(pkg.fs, pkg.registry_path, joinpath(pkg.path, "Versions.toml")) : Dict{String, Any}()
     version_info = Dict{VersionNumber, VersionInfo}(VersionNumber(k) =>
         VersionInfo(SHA1(v["git-tree-sha1"]::String), get(v, "yanked", false)::Bool) for (k, v) in d_v)
 
     # Compat.toml
-    compat_file = joinpath(path, "Compat.toml")
-    compat_data_toml = isfile(compat_file) ? parsefile(compat_file) : Dict{String, Any}()
+    compat_data_toml = custom_isfile(pkg.fs, pkg.registry_path, joinpath(pkg.path, "Compat.toml")) ? 
+        parsefile(pkg.fs, pkg.registry_path, joinpath(pkg.path, "Compat.toml")) : Dict{String, Any}()
     # The Compat.toml file might have string or vector values
     compat_data_toml = convert(Dict{String, Dict{String, Union{String, Vector{String}}}}, compat_data_toml)
     compat = Dict{VersionRange, Dict{String, VersionSpec}}()
@@ -150,8 +165,8 @@ function init_package_info!(pkg::PkgEntry)
     end
 
     # Deps.toml
-    deps_file = joinpath(path, "Deps.toml")
-    deps_data_toml = isfile(deps_file) ? parsefile(deps_file) : Dict{String, Any}()
+    deps_data_toml = custom_isfile(pkg.fs, pkg.registry_path, joinpath(pkg.path, "Deps.toml")) ?
+        parsefile(pkg.fs, pkg.registry_path, joinpath(pkg.path, "Deps.toml")) : Dict{String, Any}()
     # But the Deps.toml only have strings as values
     deps_data_toml = convert(Dict{String, Dict{String, String}}, deps_data_toml)
     deps = Dict{VersionRange, Dict{String, UUID}}()
@@ -169,6 +184,12 @@ function init_package_info!(pkg::PkgEntry)
 end
 
 
+mutable struct InMemoryRegistry
+    d::Dict{String, String}
+    name::String
+end
+readfile(reg::InMemoryRegistry, path::AbstractString) = reg.d[joinpath(reg.name, path)] 
+
 struct RegistryInstance
     path::String
     name::String
@@ -178,26 +199,27 @@ struct RegistryInstance
     description::Union{String, Nothing}
     pkgs::Dict{UUID, PkgEntry}
     tree_info::Union{Base.SHA1, Nothing}
+    fs::Union{Nothing, InMemoryFileSystem}
     # various caches
     name_to_uuids::Dict{String, Vector{UUID}}
 end
 
 function RegistryInstance(path::AbstractString; parse_packages::Bool=true)
-    d = parsefile(joinpath(path, "Registry.toml"))
+    fs = isfile(path) ? create_inmemory_filesystem(path) : nothing
+    d = parsefile(fs, path, "Registry.toml")
     pkgs = Dict{UUID, PkgEntry}()
     if parse_packages
         for (uuid, info) in d["packages"]::Dict{String, Any}
             uuid = UUID(uuid::String)
             info::Dict{String, Any}
             name = info["name"]::String
             pkgpath = info["path"]::String
-            pkg = PkgEntry(pkgpath, path, name, uuid, uninit)
+            pkg = PkgEntry(pkgpath, path, name, uuid, fs, uninit)
             pkgs[uuid] = pkg
         end
     end
-    tree_info_file = joinpath(path, ".tree_info.toml")
-    tree_info = if isfile(tree_info_file)
-        Base.SHA1(parsefile(tree_info_file)["git-tree-sha1"]::String)
+    tree_info = if custom_isfile(fs, path, ".tree_info.toml")
+        Base.SHA1(parsefile(fs, path, ".tree_info.toml")["git-tree-sha1"]::String)
     else
         nothing
     end
@@ -210,6 +232,7 @@ function RegistryInstance(path::AbstractString; parse_packages::Bool=true)
         get(d, "description", nothing)::Union{String, Nothing},
         pkgs,
         tree_info,
+        fs,
         Dict{String, UUID}(),
     )
 end