read the registry from the compressed tarball instead of extracting

CHANGELOG.md

@@ -1,8 +1,10 @@
 Pkg v1.7 Release Notes
 ======================
 
+- Regisries downloaded from the Pkg Server (not git) are no longer uncompressed into files but instead read directly from the compressed tarball into memory. This improves performance on
+  filesystems which do not handle a large number of files well. To turn this feature off, set the environment variable `JULIA_PKG_UNPACK_REGISTRY=true`.
 - It is now possible to use an external `git` executable instead of the default libgit2 library for 
   the downloads that happen via the Git protocol by setting the environment variable `JULIA_PKG_USE_CLI_GIT=true`.
-- Registries downloaded from Pkg Server (not git) is now assumed to be immutable. Manual changes to their files might not be picked up by a running Pkg session.
+- Registries downloaded from the Pkg Server (not git) is now assumed to be immutable. Manual changes to their files might not be picked up by a running Pkg session.
 - Adding packages by folder name in the REPL mode now requires a prepending a `./` to the folder name package folder is in the current folder, e.g. `add ./Package` is required instead of `add Pacakge`. This is to avoid confusion between the package name `Package` and the local directory `Package`.
 - `rm`, `pin`, and `free` now support the `--all` option, and the api variants gain the `all_pkgs::Bool` kwarg, to perform the operation on all packages within the project or manifest, depending on the mode of the operation.

src/Registry/Registry.jl

@@ -3,8 +3,8 @@ module Registry
 import ..Pkg
 using ..Pkg: depots1, printpkgstyle, DEFAULT_IO, isdir_nothrow, pathrepr, pkg_server,
              GitTools, OFFLINE_MODE, UPDATED_REGISTRY_THIS_SESSION
-using ..Pkg.PlatformEngines: download_verify_unpack, download
-using UUIDs, LibGit2
+using ..Pkg.PlatformEngines: download_verify_unpack, download, download_verify, exe7z
+using UUIDs, LibGit2, TOML
 
 include("registry_instance.jl")
 
@@ -19,7 +19,7 @@ mutable struct RegistrySpec
 end
 RegistrySpec(name::String) = RegistrySpec(name = name)
 RegistrySpec(;name::Union{String,Nothing}=nothing, uuid::Union{String,UUID,Nothing}=nothing,
-              url::Union{String,Nothing}=nothing, path::Union{String,Nothing}=nothing, linked::Union{Bool,Nothing}=nothing) =
+url::Union{String,Nothing}=nothing, path::Union{String,Nothing}=nothing, linked::Union{Bool,Nothing}=nothing) =
     RegistrySpec(name, isa(uuid, String) ? UUID(uuid) : uuid, url, path, linked)
 
 """
@@ -141,17 +141,16 @@ function populate_known_registries_with_urls!(registries::Vector{RegistrySpec})
     end
 end
 
-function registry_use_pkg_server(url)
-    if url === nothing
-        return false
-    else
-        return haskey(ENV, "JULIA_PKG_SERVER")
-    end
+function registry_use_pkg_server()
+    get(ENV, "JULIA_PKG_SERVER", nothing) !== ""
 end
 
-function check_registry_state(reg, url)
+registry_read_from_tarball() = 
+    registry_use_pkg_server() && get(ENV, "JULIA_PKG_UNPACK_REGISTRY", "") == "true"
+
+function check_registry_state(reg)
     reg_currently_uses_pkg_server = reg.tree_info !== nothing
-    reg_should_use_pkg_server = registry_use_pkg_server(url)
+    reg_should_use_pkg_server = registry_use_pkg_server()
     if reg_currently_uses_pkg_server && !reg_should_use_pkg_server
         msg = string(
             "Your registry may be outdated. We recommend that you run the ",
@@ -170,66 +169,88 @@ function download_registries(io::IO, regs::Vector{RegistrySpec}, depot::String=d
         if reg.path !== nothing && reg.url !== nothing
             Pkg.Types.pkgerror("ambiguous registry specification; both url and path is set.")
         end
-        # clone to tmpdir first
-        mktempdir() do tmp
-            url, registry_urls = pkg_server_registry_url(reg.uuid, registry_urls)
-            if reg.path !== nothing && reg.linked == true # symlink to local source
-                registry = Registry.RegistryInstance(reg.path)
-                regpath = joinpath(depot, "registries", registry.name)
-                printpkgstyle(io, :Symlinking, "registry from `$(Base.contractuser(reg.path))`")
-                isdir(dirname(regpath)) || mkpath(dirname(regpath))
-                symlink(reg.path, regpath)
-                isfile(joinpath(regpath, "Registry.toml")) || Pkg.Types.pkgerror("no `Registry.toml` file in linked registry.")
-                registry = Registry.RegistryInstance(regpath)
-                printpkgstyle(io, :Symlinked, "registry `$(Base.contractuser(registry.name))` to `$(Base.contractuser(regpath))`")
-                return
-            elseif registry_use_pkg_server(url)
-                # download from Pkg server
-                try
-                    download_verify_unpack(url, nothing, tmp, ignore_existence = true, io = io)
-                catch err
-                    Pkg.Types.pkgerror("could not download $url")
-                end
-                tree_info_file = joinpath(tmp, ".tree_info.toml")
-                hash = pkg_server_url_hash(url)
-                write(tree_info_file, "git-tree-sha1 = " * repr(string(hash)))
-            elseif reg.path !== nothing # copy from local source
-                printpkgstyle(io, :Copying, "registry from `$(Base.contractuser(reg.path))`")
-                isfile(joinpath(reg.path, "Registry.toml")) || Pkg.Types.pkgerror("no `Registry.toml` file in source directory.")
-                registry = Registry.RegistryInstance(reg.path)
-                regpath = joinpath(depot, "registries", registry.name)
-                cp(reg.path, regpath; force=true) # has to be cp given we're copying
-                printpkgstyle(io, :Copied, "registry `$(Base.contractuser(registry.name))` to `$(Base.contractuser(regpath))`")
-                return
-            elseif reg.url !== nothing # clone from url
-                repo = GitTools.clone(io, reg.url, tmp; header = "registry from $(repr(reg.url))")
-                LibGit2.close(repo)
-            else
-                Pkg.Types.pkgerror("no path or url specified for registry")
+        url, registry_urls = pkg_server_registry_url(reg.uuid, registry_urls)
+        if url !== nothing && registry_read_from_tarball()
+            tmp = tempname()
+            try
+                download_verify(url, nothing, tmp)
+            catch err
+                Pkg.Types.pkgerror("could not download $url")
             end
-            # verify that the clone looks like a registry
-            if !isfile(joinpath(tmp, "Registry.toml"))
-                Pkg.Types.pkgerror("no `Registry.toml` file in cloned registry.")
+            if reg.name === nothing
+                # Need to look up the registry name here
+                reg_unc = uncompress_registry(tmp)
+                reg.name = TOML.parse(reg_unc["Registry.toml"])["name"]::String
             end
-            registry = Registry.RegistryInstance(tmp)
-            regpath = joinpath(depot, "registries", registry.name)
-            # copy to `depot`
-            ispath(dirname(regpath)) || mkpath(dirname(regpath))
-            if isfile(joinpath(regpath, "Registry.toml"))
-                existing_registry = Registry.RegistryInstance(regpath)
-                if registry.uuid == existing_registry.uuid
-                    println(io,
-                            "registry `$(registry.name)` already exist in `$(Base.contractuser(regpath))`.")
+            regpath = joinpath(depot, "registries", reg.name)
+            Base.rm(regpath; recursive=true, force=true)
+            mkpath(regpath)
+            mv(tmp, joinpath(regpath, reg.name * ".tar.gz"); force=true)
+            hash = pkg_server_url_hash(url)
+            reg_info = Dict("uuid" => string(reg.uuid), "git-tree-sha1" => string(hash), "filename" => reg.name * ".tar.gz")
+            open(joinpath(regpath, ".registry_info.toml"), "w") do io
+                TOML.print(io, reg_info)
+            end
+        else
+            mktempdir() do tmp
+                if reg.path !== nothing && reg.linked == true # symlink to local source
+                    registry = Registry.RegistryInstance(reg.path)
+                    regpath = joinpath(depot, "registries", registry.name)
+                    printpkgstyle(io, :Symlinking, "registry from `$(Base.contractuser(reg.path))`")
+                    isdir(dirname(regpath)) || mkpath(dirname(regpath))
+                    symlink(reg.path, regpath)
+                    isfile(joinpath(regpath, "Registry.toml")) || Pkg.Types.pkgerror("no `Registry.toml` file in linked registry.")
+                    registry = Registry.RegistryInstance(regpath)
+                    printpkgstyle(io, :Symlinked, "registry `$(Base.contractuser(registry.name))` to `$(Base.contractuser(regpath))`")
+                    return
+                elseif url !== nothing && registry_use_pkg_server()
+                    # download from Pkg server
+                    try
+                        download_verify_unpack(url, nothing, tmp, ignore_existence = true, io = io)
+                    catch err
+                        Pkg.Types.pkgerror("could not download $url")
+                    end
+                    tree_info_file = joinpath(tmp, ".tree_info.toml")
+                    hash = pkg_server_url_hash(url)
+                    write(tree_info_file, "git-tree-sha1 = " * repr(string(hash)))
+                elseif reg.path !== nothing # copy from local source
+                    printpkgstyle(io, :Copying, "registry from `$(Base.contractuser(reg.path))`")
+                    isfile(joinpath(reg.path, "Registry.toml")) || Pkg.Types.pkgerror("no `Registry.toml` file in source directory.")
+                    registry = Registry.RegistryInstance(reg.path)
+                    regpath = joinpath(depot, "registries", registry.name)
+                    cp(reg.path, regpath; force=true) # has to be cp given we're copying
+                    printpkgstyle(io, :Copied, "registry `$(Base.contractuser(registry.name))` to `$(Base.contractuser(regpath))`")
+                    return
+                elseif reg.url !== nothing # clone from url
+                    repo = GitTools.clone(io, reg.url, tmp; header = "registry from $(repr(reg.url))")
+                    LibGit2.close(repo)
                 else
-                    throw(Pkg.Types.PkgError("registry `$(registry.name)=\"$(registry.uuid)\"` conflicts with " *
-                        "existing registry `$(existing_registry.name)=\"$(existing_registry.uuid)\"`. " *
-                        "To install it you can clone it manually into e.g. " *
-                        "`$(Base.contractuser(joinpath(depot, "registries", registry.name*"-2")))`."))
+                    Pkg.Types.pkgerror("no path or url specified for registry")
+                end
+                # verify that the clone looks like a registry
+                if !isfile(joinpath(tmp, "Registry.toml"))
+                    Pkg.Types.pkgerror("no `Registry.toml` file in cloned registry.")
+                end
+                registry = Registry.RegistryInstance(tmp)
+                regpath = joinpath(depot, "registries", registry.name)
+                # copy to `depot`
+                ispath(dirname(regpath)) || mkpath(dirname(regpath))
+                if isfile(joinpath(regpath, "Registry.toml"))
+                    existing_registry = Registry.RegistryInstance(regpath)
+                    if registry.uuid == existing_registry.uuid
+                        println(io,
+                                "registry `$(registry.name)` already exist in `$(Base.contractuser(regpath))`.")
+                    else
+                        throw(Pkg.Types.PkgError("registry `$(registry.name)=\"$(registry.uuid)\"` conflicts with " *
+                            "existing registry `$(existing_registry.name)=\"$(existing_registry.uuid)\"`. " *
+                            "To install it you can clone it manually into e.g. " *
+                            "`$(Base.contractuser(joinpath(depot, "registries", registry.name*"-2")))`."))
+                    end
+                elseif (url !== nothing && registry_use_pkg_server()) || reg.linked !== true
+                    # if the dir doesn't exist, or exists but doesn't contain a Registry.toml
+                    mv(tmp, regpath, force=true)
+                    printpkgstyle(io, :Added, "registry `$(registry.name)` to `$(Base.contractuser(regpath))`")
                 end
-            elseif registry_use_pkg_server(url) || reg.linked !== true
-                # if the dir doesn't exist, or exists but doesn't contain a Registry.toml
-                mv(tmp, regpath, force=true)
-                printpkgstyle(io, :Added, "registry `$(registry.name)` to `$(Base.contractuser(regpath))`")
             end
         end
     end
@@ -332,20 +353,40 @@ function update(regs::Vector{RegistrySpec} = RegistrySpec[]; io::IO=DEFAULT_IO[]
                 printpkgstyle(io, :Updating, "registry at " * regpath)
                 old_hash = reg.tree_info
                 url, registry_urls = pkg_server_registry_url(reg.uuid, registry_urls)
-                check_registry_state(reg, url)
+                if url !== nothing
+                    check_registry_state(reg)
+                end
                 if url !== nothing && (new_hash = pkg_server_url_hash(url)) != old_hash
+                    # TODO: update faster by using a diff, if available
+                    # TODO: DRY with the code in `download_default_registries`
                     let new_hash = new_hash
-                        # TODO: update faster by using a diff, if available
-                        mktempdir() do tmp
+                        if registry_read_from_tarball()
+                            tmp = tempname()
                             try
-                                download_verify_unpack(url, nothing, tmp, ignore_existence = true, io=io)
+                                download_verify(url, nothing, tmp)
                             catch err
                                 @error "could not download $url" exception=err
                             end
-                            tree_info_file = joinpath(tmp, ".tree_info.toml")
+                            Base.rm(reg.path; recursive=true, force=true)
+                            mkpath(reg.path)
+                            mv(tmp, joinpath(reg.path, reg.name * ".tar.gz"))
                             hash = pkg_server_url_hash(url)
-                            write(tree_info_file, "git-tree-sha1 = " * repr(string(new_hash)))
-                            mv(tmp, reg.path, force=true)
+                            reg_info = Dict("uuid" => string(reg.uuid), "git-tree-sha1" => string(hash), "filename" => reg.name * ".tar.gz")
+                            open(joinpath(reg.path, ".registry_info.toml"), "w") do io
+                                TOML.print(io, reg_info)
+                            end
+                        else
+                            mktempdir() do tmp
+                                try
+                                    download_verify_unpack(url, nothing, tmp, ignore_existence = true, io=io)
+                                catch err
+                                    @error "could not download $url" exception=err
+                                end
+                                tree_info_file = joinpath(tmp, ".tree_info.toml")
+                                hash = pkg_server_url_hash(url)
+                                write(tree_info_file, "git-tree-sha1 = " * repr(string(new_hash)))
+                                mv(tmp, reg.path, force=true)
+                            end
                         end
                     end
                 end