Skip to content

Commit

Permalink
read the registry from the compressed tarball instead of extracting
Browse files Browse the repository at this point in the history
  • Loading branch information
@KristofferC
KristofferC committed Mar 31, 2021
1 parent 3c9188f commit 9cf03f5
Show file tree
Hide file tree
Showing 3 changed files with 201 additions and 104 deletions.
189 changes: 115 additions & 74 deletions src/Registry/Registry.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@ module Registry
import ..Pkg
using ..Pkg: depots1, printpkgstyle, DEFAULT_IO, isdir_nothrow, pathrepr, pkg_server,
GitTools, OFFLINE_MODE, UPDATED_REGISTRY_THIS_SESSION
using ..Pkg.PlatformEngines: download_verify_unpack, download
using UUIDs, LibGit2
using ..Pkg.PlatformEngines: download_verify_unpack, download, download_verify, exe7z
using UUIDs, LibGit2, TOML

include("registry_instance.jl")

Expand All @@ -19,7 +19,7 @@ mutable struct RegistrySpec
end
RegistrySpec(name::String) = RegistrySpec(name = name)
RegistrySpec(;name::Union{String,Nothing}=nothing, uuid::Union{String,UUID,Nothing}=nothing,
url::Union{String,Nothing}=nothing, path::Union{String,Nothing}=nothing, linked::Union{Bool,Nothing}=nothing) =
url::Union{String,Nothing}=nothing, path::Union{String,Nothing}=nothing, linked::Union{Bool,Nothing}=nothing) =
RegistrySpec(name, isa(uuid, String) ? UUID(uuid) : uuid, url, path, linked)

"""
Expand Down Expand Up @@ -141,17 +141,16 @@ function populate_known_registries_with_urls!(registries::Vector{RegistrySpec})
end
end

function registry_use_pkg_server(url)
if url === nothing
return false
else
return haskey(ENV, "JULIA_PKG_SERVER")
end
function registry_use_pkg_server()
get(ENV, "JULIA_PKG_SERVER", nothing) !== ""
end

function check_registry_state(reg, url)
registry_read_from_tarball() =
registry_use_pkg_server() && !haskey(ENV, "JULIA_PKG_UNPACK_REGISTRY")

function check_registry_state(reg)
reg_currently_uses_pkg_server = reg.tree_info !== nothing
reg_should_use_pkg_server = registry_use_pkg_server(url)
reg_should_use_pkg_server = registry_use_pkg_server()
if reg_currently_uses_pkg_server && !reg_should_use_pkg_server
msg = string(
"Your registry may be outdated. We recommend that you run the ",
Expand All @@ -170,66 +169,88 @@ function download_registries(io::IO, regs::Vector{RegistrySpec}, depot::String=d
if reg.path !== nothing && reg.url !== nothing
Pkg.Types.pkgerror("ambiguous registry specification; both url and path is set.")
end
# clone to tmpdir first
mktempdir() do tmp
url, registry_urls = pkg_server_registry_url(reg.uuid, registry_urls)
if reg.path !== nothing && reg.linked == true # symlink to local source
registry = Registry.RegistryInstance(reg.path)
regpath = joinpath(depot, "registries", registry.name)
printpkgstyle(io, :Symlinking, "registry from `$(Base.contractuser(reg.path))`")
isdir(dirname(regpath)) || mkpath(dirname(regpath))
symlink(reg.path, regpath)
isfile(joinpath(regpath, "Registry.toml")) || Pkg.Types.pkgerror("no `Registry.toml` file in linked registry.")
registry = Registry.RegistryInstance(regpath)
printpkgstyle(io, :Symlinked, "registry `$(Base.contractuser(registry.name))` to `$(Base.contractuser(regpath))`")
return
elseif registry_use_pkg_server(url)
# download from Pkg server
try
download_verify_unpack(url, nothing, tmp, ignore_existence = true, io = io)
catch err
Pkg.Types.pkgerror("could not download $url")
end
tree_info_file = joinpath(tmp, ".tree_info.toml")
hash = pkg_server_url_hash(url)
write(tree_info_file, "git-tree-sha1 = " * repr(string(hash)))
elseif reg.path !== nothing # copy from local source
printpkgstyle(io, :Copying, "registry from `$(Base.contractuser(reg.path))`")
isfile(joinpath(reg.path, "Registry.toml")) || Pkg.Types.pkgerror("no `Registry.toml` file in source directory.")
registry = Registry.RegistryInstance(reg.path)
regpath = joinpath(depot, "registries", registry.name)
cp(reg.path, regpath; force=true) # has to be cp given we're copying
printpkgstyle(io, :Copied, "registry `$(Base.contractuser(registry.name))` to `$(Base.contractuser(regpath))`")
return
elseif reg.url !== nothing # clone from url
repo = GitTools.clone(io, reg.url, tmp; header = "registry from $(repr(reg.url))")
LibGit2.close(repo)
else
Pkg.Types.pkgerror("no path or url specified for registry")
url, registry_urls = pkg_server_registry_url(reg.uuid, registry_urls)
if url !== nothing && registry_read_from_tarball()
tmp = tempname()
try
download_verify(url, nothing, tmp)
catch err
Pkg.Types.pkgerror("could not download $url")
end
# verify that the clone looks like a registry
if !isfile(joinpath(tmp, "Registry.toml"))
Pkg.Types.pkgerror("no `Registry.toml` file in cloned registry.")
if reg.name === nothing
# Need to look up the registry name here
reg_unc = uncompress_registry(tmp)
reg.name = TOML.parse(reg_unc["Registry.toml"])["name"]::String
end
registry = Registry.RegistryInstance(tmp)
regpath = joinpath(depot, "registries", registry.name)
# copy to `depot`
ispath(dirname(regpath)) || mkpath(dirname(regpath))
if isfile(joinpath(regpath, "Registry.toml"))
existing_registry = Registry.RegistryInstance(regpath)
if registry.uuid == existing_registry.uuid
println(io,
"registry `$(registry.name)` already exist in `$(Base.contractuser(regpath))`.")
regpath = joinpath(depot, "registries", reg.name)
Base.rm(regpath; recursive=true, force=true)
mkpath(regpath)
mv(tmp, joinpath(regpath, reg.name * ".tar.gz"); force=true)
hash = pkg_server_url_hash(url)
reg_info = Dict("uuid" => string(reg.uuid), "git-tree-sha1" => string(hash), "filename" => reg.name * ".tar.gz")
open(joinpath(regpath, ".registry_info.toml"), "w") do io
TOML.print(io, reg_info)
end
else
mktempdir() do tmp
if reg.path !== nothing && reg.linked == true # symlink to local source
registry = Registry.RegistryInstance(reg.path)
regpath = joinpath(depot, "registries", registry.name)
printpkgstyle(io, :Symlinking, "registry from `$(Base.contractuser(reg.path))`")
isdir(dirname(regpath)) || mkpath(dirname(regpath))
symlink(reg.path, regpath)
isfile(joinpath(regpath, "Registry.toml")) || Pkg.Types.pkgerror("no `Registry.toml` file in linked registry.")
registry = Registry.RegistryInstance(regpath)
printpkgstyle(io, :Symlinked, "registry `$(Base.contractuser(registry.name))` to `$(Base.contractuser(regpath))`")
return
elseif url !== nothing && registry_use_pkg_server()
# download from Pkg server
try
download_verify_unpack(url, nothing, tmp, ignore_existence = true, io = io)
catch err
Pkg.Types.pkgerror("could not download $url")
end
tree_info_file = joinpath(tmp, ".tree_info.toml")
hash = pkg_server_url_hash(url)
write(tree_info_file, "git-tree-sha1 = " * repr(string(hash)))
elseif reg.path !== nothing # copy from local source
printpkgstyle(io, :Copying, "registry from `$(Base.contractuser(reg.path))`")
isfile(joinpath(reg.path, "Registry.toml")) || Pkg.Types.pkgerror("no `Registry.toml` file in source directory.")
registry = Registry.RegistryInstance(reg.path)
regpath = joinpath(depot, "registries", registry.name)
cp(reg.path, regpath; force=true) # has to be cp given we're copying
printpkgstyle(io, :Copied, "registry `$(Base.contractuser(registry.name))` to `$(Base.contractuser(regpath))`")
return
elseif reg.url !== nothing # clone from url
repo = GitTools.clone(io, reg.url, tmp; header = "registry from $(repr(reg.url))")
LibGit2.close(repo)
else
throw(Pkg.Types.PkgError("registry `$(registry.name)=\"$(registry.uuid)\"` conflicts with " *
"existing registry `$(existing_registry.name)=\"$(existing_registry.uuid)\"`. " *
"To install it you can clone it manually into e.g. " *
"`$(Base.contractuser(joinpath(depot, "registries", registry.name*"-2")))`."))
Pkg.Types.pkgerror("no path or url specified for registry")
end
# verify that the clone looks like a registry
if !isfile(joinpath(tmp, "Registry.toml"))
Pkg.Types.pkgerror("no `Registry.toml` file in cloned registry.")
end
registry = Registry.RegistryInstance(tmp)
regpath = joinpath(depot, "registries", registry.name)
# copy to `depot`
ispath(dirname(regpath)) || mkpath(dirname(regpath))
if isfile(joinpath(regpath, "Registry.toml"))
existing_registry = Registry.RegistryInstance(regpath)
if registry.uuid == existing_registry.uuid
println(io,
"registry `$(registry.name)` already exist in `$(Base.contractuser(regpath))`.")
else
throw(Pkg.Types.PkgError("registry `$(registry.name)=\"$(registry.uuid)\"` conflicts with " *
"existing registry `$(existing_registry.name)=\"$(existing_registry.uuid)\"`. " *
"To install it you can clone it manually into e.g. " *
"`$(Base.contractuser(joinpath(depot, "registries", registry.name*"-2")))`."))
end
elseif (url !== nothing && registry_use_pkg_server()) || reg.linked !== true
# if the dir doesn't exist, or exists but doesn't contain a Registry.toml
mv(tmp, regpath, force=true)
printpkgstyle(io, :Added, "registry `$(registry.name)` to `$(Base.contractuser(regpath))`")
end
elseif registry_use_pkg_server(url) || reg.linked !== true
# if the dir doesn't exist, or exists but doesn't contain a Registry.toml
mv(tmp, regpath, force=true)
printpkgstyle(io, :Added, "registry `$(registry.name)` to `$(Base.contractuser(regpath))`")
end
end
end
Expand Down Expand Up @@ -332,20 +353,40 @@ function update(regs::Vector{RegistrySpec} = RegistrySpec[]; io::IO=DEFAULT_IO[]
printpkgstyle(io, :Updating, "registry at " * regpath)
old_hash = reg.tree_info
url, registry_urls = pkg_server_registry_url(reg.uuid, registry_urls)
check_registry_state(reg, url)
if url !== nothing
check_registry_state(reg)
end
if url !== nothing && (new_hash = pkg_server_url_hash(url)) != old_hash
# TODO: update faster by using a diff, if available
# TODO: DRY with the code in `download_default_registries`
let new_hash = new_hash
# TODO: update faster by using a diff, if available
mktempdir() do tmp
if registry_read_from_tarball()
tmp = tempname()
try
download_verify_unpack(url, nothing, tmp, ignore_existence = true, io=io)
download_verify(url, nothing, tmp)
catch err
@error "could not download $url" exception=err
end
tree_info_file = joinpath(tmp, ".tree_info.toml")
Base.rm(reg.path; recursive=true, force=true)
mkpath(reg.path)
mv(tmp, joinpath(reg.path, reg.name * ".tar.gz"))
hash = pkg_server_url_hash(url)
write(tree_info_file, "git-tree-sha1 = " * repr(string(new_hash)))
mv(tmp, reg.path, force=true)
reg_info = Dict("uuid" => string(reg.uuid), "git-tree-sha1" => string(hash), "filename" => reg.name * ".tar.gz")
open(joinpath(reg.path, ".registry_info.toml"), "w") do io
TOML.print(io, reg_info)
end
else
mktempdir() do tmp
try
download_verify_unpack(url, nothing, tmp, ignore_existence = true, io=io)
catch err
@error "could not download $url" exception=err
end
tree_info_file = joinpath(tmp, ".tree_info.toml")
hash = pkg_server_url_hash(url)
write(tree_info_file, "git-tree-sha1 = " * repr(string(new_hash)))
mv(tmp, reg.path, force=true)
end
end
end
end
Expand Down
Loading

0 comments on commit 9cf03f5

Please sign in to comment.