diff --git a/src/Registry/Registry.jl b/src/Registry/Registry.jl index 7c95bc8c4a..769a861842 100644 --- a/src/Registry/Registry.jl +++ b/src/Registry/Registry.jl @@ -3,8 +3,8 @@ module Registry import ..Pkg using ..Pkg: depots1, printpkgstyle, DEFAULT_IO, isdir_nothrow, pathrepr, pkg_server, GitTools, OFFLINE_MODE, UPDATED_REGISTRY_THIS_SESSION -using ..Pkg.PlatformEngines: download_verify_unpack, download -using UUIDs, LibGit2 +using ..Pkg.PlatformEngines: download_verify_unpack, download, download_verify, exe7z +using UUIDs, LibGit2, TOML include("registry_instance.jl") @@ -19,7 +19,7 @@ mutable struct RegistrySpec end RegistrySpec(name::String) = RegistrySpec(name = name) RegistrySpec(;name::Union{String,Nothing}=nothing, uuid::Union{String,UUID,Nothing}=nothing, - url::Union{String,Nothing}=nothing, path::Union{String,Nothing}=nothing, linked::Union{Bool,Nothing}=nothing) = +url::Union{String,Nothing}=nothing, path::Union{String,Nothing}=nothing, linked::Union{Bool,Nothing}=nothing) = RegistrySpec(name, isa(uuid, String) ? UUID(uuid) : uuid, url, path, linked) """ @@ -141,17 +141,16 @@ function populate_known_registries_with_urls!(registries::Vector{RegistrySpec}) end end -function registry_use_pkg_server(url) - if url === nothing - return false - else - return haskey(ENV, "JULIA_PKG_SERVER") - end +function registry_use_pkg_server() + get(ENV, "JULIA_PKG_SERVER", nothing) !== "" end -function check_registry_state(reg, url) +registry_read_from_tarball() = + registry_use_pkg_server() && !haskey(ENV, "JULIA_PKG_UNPACK_REGISTRY") + +function check_registry_state(reg) reg_currently_uses_pkg_server = reg.tree_info !== nothing - reg_should_use_pkg_server = registry_use_pkg_server(url) + reg_should_use_pkg_server = registry_use_pkg_server() if reg_currently_uses_pkg_server && !reg_should_use_pkg_server msg = string( "Your registry may be outdated. We recommend that you run the ", @@ -170,66 +169,88 @@ function download_registries(io::IO, regs::Vector{RegistrySpec}, depot::String=d if reg.path !== nothing && reg.url !== nothing Pkg.Types.pkgerror("ambiguous registry specification; both url and path is set.") end - # clone to tmpdir first - mktempdir() do tmp - url, registry_urls = pkg_server_registry_url(reg.uuid, registry_urls) - if reg.path !== nothing && reg.linked == true # symlink to local source - registry = Registry.RegistryInstance(reg.path) - regpath = joinpath(depot, "registries", registry.name) - printpkgstyle(io, :Symlinking, "registry from `$(Base.contractuser(reg.path))`") - isdir(dirname(regpath)) || mkpath(dirname(regpath)) - symlink(reg.path, regpath) - isfile(joinpath(regpath, "Registry.toml")) || Pkg.Types.pkgerror("no `Registry.toml` file in linked registry.") - registry = Registry.RegistryInstance(regpath) - printpkgstyle(io, :Symlinked, "registry `$(Base.contractuser(registry.name))` to `$(Base.contractuser(regpath))`") - return - elseif registry_use_pkg_server(url) - # download from Pkg server - try - download_verify_unpack(url, nothing, tmp, ignore_existence = true, io = io) - catch err - Pkg.Types.pkgerror("could not download $url") - end - tree_info_file = joinpath(tmp, ".tree_info.toml") - hash = pkg_server_url_hash(url) - write(tree_info_file, "git-tree-sha1 = " * repr(string(hash))) - elseif reg.path !== nothing # copy from local source - printpkgstyle(io, :Copying, "registry from `$(Base.contractuser(reg.path))`") - isfile(joinpath(reg.path, "Registry.toml")) || Pkg.Types.pkgerror("no `Registry.toml` file in source directory.") - registry = Registry.RegistryInstance(reg.path) - regpath = joinpath(depot, "registries", registry.name) - cp(reg.path, regpath; force=true) # has to be cp given we're copying - printpkgstyle(io, :Copied, "registry `$(Base.contractuser(registry.name))` to `$(Base.contractuser(regpath))`") - return - elseif reg.url !== nothing # clone from url - repo = GitTools.clone(io, reg.url, tmp; header = "registry from $(repr(reg.url))") - LibGit2.close(repo) - else - Pkg.Types.pkgerror("no path or url specified for registry") + url, registry_urls = pkg_server_registry_url(reg.uuid, registry_urls) + if url !== nothing && registry_read_from_tarball() + tmp = tempname() + try + download_verify(url, nothing, tmp) + catch err + Pkg.Types.pkgerror("could not download $url") end - # verify that the clone looks like a registry - if !isfile(joinpath(tmp, "Registry.toml")) - Pkg.Types.pkgerror("no `Registry.toml` file in cloned registry.") + if reg.name === nothing + # Need to look up the registry name here + reg_unc = uncompress_registry(tmp) + reg.name = TOML.parse(reg_unc["Registry.toml"])["name"]::String end - registry = Registry.RegistryInstance(tmp) - regpath = joinpath(depot, "registries", registry.name) - # copy to `depot` - ispath(dirname(regpath)) || mkpath(dirname(regpath)) - if isfile(joinpath(regpath, "Registry.toml")) - existing_registry = Registry.RegistryInstance(regpath) - if registry.uuid == existing_registry.uuid - println(io, - "registry `$(registry.name)` already exist in `$(Base.contractuser(regpath))`.") + regpath = joinpath(depot, "registries", reg.name) + Base.rm(regpath; recursive=true, force=true) + mkpath(regpath) + mv(tmp, joinpath(regpath, reg.name * ".tar.gz"); force=true) + hash = pkg_server_url_hash(url) + reg_info = Dict("uuid" => string(reg.uuid), "git-tree-sha1" => string(hash), "filename" => reg.name * ".tar.gz") + open(joinpath(regpath, ".registry_info.toml"), "w") do io + TOML.print(io, reg_info) + end + else + mktempdir() do tmp + if reg.path !== nothing && reg.linked == true # symlink to local source + registry = Registry.RegistryInstance(reg.path) + regpath = joinpath(depot, "registries", registry.name) + printpkgstyle(io, :Symlinking, "registry from `$(Base.contractuser(reg.path))`") + isdir(dirname(regpath)) || mkpath(dirname(regpath)) + symlink(reg.path, regpath) + isfile(joinpath(regpath, "Registry.toml")) || Pkg.Types.pkgerror("no `Registry.toml` file in linked registry.") + registry = Registry.RegistryInstance(regpath) + printpkgstyle(io, :Symlinked, "registry `$(Base.contractuser(registry.name))` to `$(Base.contractuser(regpath))`") + return + elseif url !== nothing && registry_use_pkg_server() + # download from Pkg server + try + download_verify_unpack(url, nothing, tmp, ignore_existence = true, io = io) + catch err + Pkg.Types.pkgerror("could not download $url") + end + tree_info_file = joinpath(tmp, ".tree_info.toml") + hash = pkg_server_url_hash(url) + write(tree_info_file, "git-tree-sha1 = " * repr(string(hash))) + elseif reg.path !== nothing # copy from local source + printpkgstyle(io, :Copying, "registry from `$(Base.contractuser(reg.path))`") + isfile(joinpath(reg.path, "Registry.toml")) || Pkg.Types.pkgerror("no `Registry.toml` file in source directory.") + registry = Registry.RegistryInstance(reg.path) + regpath = joinpath(depot, "registries", registry.name) + cp(reg.path, regpath; force=true) # has to be cp given we're copying + printpkgstyle(io, :Copied, "registry `$(Base.contractuser(registry.name))` to `$(Base.contractuser(regpath))`") + return + elseif reg.url !== nothing # clone from url + repo = GitTools.clone(io, reg.url, tmp; header = "registry from $(repr(reg.url))") + LibGit2.close(repo) else - throw(Pkg.Types.PkgError("registry `$(registry.name)=\"$(registry.uuid)\"` conflicts with " * - "existing registry `$(existing_registry.name)=\"$(existing_registry.uuid)\"`. " * - "To install it you can clone it manually into e.g. " * - "`$(Base.contractuser(joinpath(depot, "registries", registry.name*"-2")))`.")) + Pkg.Types.pkgerror("no path or url specified for registry") + end + # verify that the clone looks like a registry + if !isfile(joinpath(tmp, "Registry.toml")) + Pkg.Types.pkgerror("no `Registry.toml` file in cloned registry.") + end + registry = Registry.RegistryInstance(tmp) + regpath = joinpath(depot, "registries", registry.name) + # copy to `depot` + ispath(dirname(regpath)) || mkpath(dirname(regpath)) + if isfile(joinpath(regpath, "Registry.toml")) + existing_registry = Registry.RegistryInstance(regpath) + if registry.uuid == existing_registry.uuid + println(io, + "registry `$(registry.name)` already exist in `$(Base.contractuser(regpath))`.") + else + throw(Pkg.Types.PkgError("registry `$(registry.name)=\"$(registry.uuid)\"` conflicts with " * + "existing registry `$(existing_registry.name)=\"$(existing_registry.uuid)\"`. " * + "To install it you can clone it manually into e.g. " * + "`$(Base.contractuser(joinpath(depot, "registries", registry.name*"-2")))`.")) + end + elseif (url !== nothing && registry_use_pkg_server()) || reg.linked !== true + # if the dir doesn't exist, or exists but doesn't contain a Registry.toml + mv(tmp, regpath, force=true) + printpkgstyle(io, :Added, "registry `$(registry.name)` to `$(Base.contractuser(regpath))`") end - elseif registry_use_pkg_server(url) || reg.linked !== true - # if the dir doesn't exist, or exists but doesn't contain a Registry.toml - mv(tmp, regpath, force=true) - printpkgstyle(io, :Added, "registry `$(registry.name)` to `$(Base.contractuser(regpath))`") end end end @@ -332,20 +353,40 @@ function update(regs::Vector{RegistrySpec} = RegistrySpec[]; io::IO=DEFAULT_IO[] printpkgstyle(io, :Updating, "registry at " * regpath) old_hash = reg.tree_info url, registry_urls = pkg_server_registry_url(reg.uuid, registry_urls) - check_registry_state(reg, url) + if url !== nothing + check_registry_state(reg) + end if url !== nothing && (new_hash = pkg_server_url_hash(url)) != old_hash + # TODO: update faster by using a diff, if available + # TODO: DRY with the code in `download_default_registries` let new_hash = new_hash - # TODO: update faster by using a diff, if available - mktempdir() do tmp + if registry_read_from_tarball() + tmp = tempname() try - download_verify_unpack(url, nothing, tmp, ignore_existence = true, io=io) + download_verify(url, nothing, tmp) catch err @error "could not download $url" exception=err end - tree_info_file = joinpath(tmp, ".tree_info.toml") + Base.rm(reg.path; recursive=true, force=true) + mkpath(reg.path) + mv(tmp, joinpath(reg.path, reg.name * ".tar.gz")) hash = pkg_server_url_hash(url) - write(tree_info_file, "git-tree-sha1 = " * repr(string(new_hash))) - mv(tmp, reg.path, force=true) + reg_info = Dict("uuid" => string(reg.uuid), "git-tree-sha1" => string(hash), "filename" => reg.name * ".tar.gz") + open(joinpath(reg.path, ".registry_info.toml"), "w") do io + TOML.print(io, reg_info) + end + else + mktempdir() do tmp + try + download_verify_unpack(url, nothing, tmp, ignore_existence = true, io=io) + catch err + @error "could not download $url" exception=err + end + tree_info_file = joinpath(tmp, ".tree_info.toml") + hash = pkg_server_url_hash(url) + write(tree_info_file, "git-tree-sha1 = " * repr(string(new_hash))) + mv(tmp, reg.path, force=true) + end end end end diff --git a/src/Registry/registry_instance.jl b/src/Registry/registry_instance.jl index e54f748734..bc5c372558 100644 --- a/src/Registry/registry_instance.jl +++ b/src/Registry/registry_instance.jl @@ -1,5 +1,6 @@ using Base: UUID, SHA1 using TOML +using Tar using ..Versions: VersionSpec, VersionRange using ..LazilyInitializedFields @@ -7,10 +8,31 @@ using ..LazilyInitializedFields # lifetime of a `Registry`. Create a new `Registry` if you want to have # a new view on the current registry. + +# TODO, need to remake the cache here since it operates on paths... + +function to_tar_path_format(file::AbstractString) + @static if Sys.iswindows() + file = replace(file, "\\" => "/") + end + return file +end + # See loading.jl const TOML_CACHE = Base.TOMLCache(TOML.Parser(), Dict{String, Dict{String, Any}}()) const TOML_LOCK = ReentrantLock() -parsefile(toml_file::AbstractString) = Base.parsed_toml(toml_file, TOML_CACHE, TOML_LOCK) +_parsefile(toml_file::AbstractString) = Base.parsed_toml(toml_file, TOML_CACHE, TOML_LOCK) +function parsefile(in_memory_registry::Union{Dict, Nothing}, folder::AbstractString, file::AbstractString) + if in_memory_registry === nothing + return _parsefile(joinpath(folder, file)) + else + content = in_memory_registry[to_tar_path_format(file)] + return TOML.Internals.parse(TOML.Parser(content; filepath=file)) + end +end + +custom_isfile(in_memory_registry::Union{Dict, Nothing}, folder::AbstractString, file::AbstractString) = + in_memory_registry === nothing ? isfile(joinpath(folder, file)) : haskey(in_memory_registry, to_tar_path_format(file)) # Info about each version of a package @lazy mutable struct VersionInfo @@ -113,6 +135,7 @@ end name::String uuid::UUID + in_memory_registry::Union{Dict{String, String}, Nothing} # Version.toml / (Compat.toml / Deps.toml): @lazy info::PkgInfo end @@ -122,24 +145,23 @@ registry_info(pkg::PkgEntry) = init_package_info!(pkg) function init_package_info!(pkg::PkgEntry) # Already uncompressed the info for this package, return early @isinit(pkg.info) && return pkg.info - path = joinpath(pkg.registry_path, pkg.path) + path = pkg.registry_path - path_package = joinpath(path, "Package.toml") - d_p = parsefile(path_package) + d_p = parsefile(pkg.in_memory_registry, pkg.registry_path, joinpath(pkg.path, "Package.toml")) name = d_p["name"]::String name != pkg.name && error("inconsistend name in Registry.toml and Package.toml for pkg at $(path)") repo = get(d_p, "repo", nothing)::Union{Nothing, String} subdir = get(d_p, "subdir", nothing)::Union{Nothing, String} # Versions.toml - path_vers = joinpath(path, "Versions.toml") - d_v = isfile(path_vers) ? parsefile(path_vers) : Dict{String, Any}() + d_v = custom_isfile(pkg.in_memory_registry, pkg.registry_path, joinpath(pkg.path, "Versions.toml")) ? + parsefile(pkg.in_memory_registry, pkg.registry_path, joinpath(pkg.path, "Versions.toml")) : Dict{String, Any}() version_info = Dict{VersionNumber, VersionInfo}(VersionNumber(k) => VersionInfo(SHA1(v["git-tree-sha1"]::String), get(v, "yanked", false)::Bool) for (k, v) in d_v) # Compat.toml - compat_file = joinpath(path, "Compat.toml") - compat_data_toml = isfile(compat_file) ? parsefile(compat_file) : Dict{String, Any}() + compat_data_toml = custom_isfile(pkg.in_memory_registry, pkg.registry_path, joinpath(pkg.path, "Compat.toml")) ? + parsefile(pkg.in_memory_registry, pkg.registry_path, joinpath(pkg.path, "Compat.toml")) : Dict{String, Any}() # The Compat.toml file might have string or vector values compat_data_toml = convert(Dict{String, Dict{String, Union{String, Vector{String}}}}, compat_data_toml) compat = Dict{VersionRange, Dict{String, VersionSpec}}() @@ -150,8 +172,8 @@ function init_package_info!(pkg::PkgEntry) end # Deps.toml - deps_file = joinpath(path, "Deps.toml") - deps_data_toml = isfile(deps_file) ? parsefile(deps_file) : Dict{String, Any}() + deps_data_toml = custom_isfile(pkg.in_memory_registry, pkg.registry_path, joinpath(pkg.path, "Deps.toml")) ? + parsefile(pkg.in_memory_registry, pkg.registry_path, joinpath(pkg.path, "Deps.toml")) : Dict{String, Any}() # But the Deps.toml only have strings as values deps_data_toml = convert(Dict{String, Dict{String, String}}, deps_data_toml) deps = Dict{VersionRange, Dict{String, UUID}}() @@ -169,6 +191,21 @@ function init_package_info!(pkg::PkgEntry) end +function uncompress_registry(tar_gz::AbstractString) + data = Dict{String, String}() + buf = Vector{UInt8}(undef, Tar.DEFAULT_BUFFER_SIZE) + io = IOBuffer() + open(`$(exe7z()) x $tar_gz -so`) do tar + Tar.read_tarball(x->true, tar; buf=buf) do hdr, _ + if hdr.type == :file + Tar.read_data(tar, io; size=hdr.size, buf=buf) + data[hdr.path] = String(take!(io)) + end + end + end + return data +end + struct RegistryInstance path::String name::String @@ -178,17 +215,22 @@ struct RegistryInstance description::Union{String, Nothing} pkgs::Dict{UUID, PkgEntry} tree_info::Union{Base.SHA1, Nothing} + in_memory_registry::Union{Nothing, Dict{String, String}} # various caches name_to_uuids::Dict{String, Vector{UUID}} end -const REGISTRY_CACHE = Dict{String, Tuple{Base.SHA1, RegistryInstance}}() +const REGISTRY_CACHE = Dict{String, Tuple{Base.SHA1, Bool, RegistryInstance}}() -function get_cached_registry(path::AbstractString, tree_info::Base.SHA1) +function get_cached_registry(path, tree_info::Base.SHA1, compressed::Bool) + if !isdir(path) + delete!(REGISTRY_CACHE, path) + return nothing + end v = get(REGISTRY_CACHE, path, nothing) if v !== nothing - cached_tree_info, reg = v - if cached_tree_info == tree_info + cached_tree_info, cached_compressed, reg = v + if cached_tree_info == tree_info && cached_compressed == compressed return reg end end @@ -197,46 +239,58 @@ function get_cached_registry(path::AbstractString, tree_info::Base.SHA1) return nothing end - function RegistryInstance(path::AbstractString) - d = parsefile(joinpath(path, "Registry.toml")) - tree_info_file = joinpath(path, ".tree_info.toml") - tree_info = if isfile(tree_info_file) - Base.SHA1(parsefile(tree_info_file)["git-tree-sha1"]::String) + compressed_file = nothing + if isfile(joinpath(path, ".registry_info.toml")) + d_reg_info = parsefile(nothing, path, ".registry_info.toml") + compressed_file = d_reg_info["filename"]::String + tree_info = Base.SHA1(d_reg_info["git-tree-sha1"]::String) else - nothing + tree_info_file = joinpath(path, ".tree_info.toml") + tree_info = if isfile(tree_info_file) + Base.SHA1(parsefile(nothing, path, ".tree_info.toml")["git-tree-sha1"]::String) + else + nothing + end end - reg_uuid = UUID(d["uuid"]::String) - # Reuse an existing cached registry if it exists for this content if tree_info !== nothing - reg = get_cached_registry(path, tree_info) + reg = get_cached_registry(path, tree_info, compressed_file !== nothing) if reg isa RegistryInstance return reg end end + + in_memory_registry = if compressed_file !== nothing + uncompress_registry(joinpath(path, compressed_file)) + else + nothing + end + + d = parsefile(in_memory_registry, path, "Registry.toml") pkgs = Dict{UUID, PkgEntry}() for (uuid, info) in d["packages"]::Dict{String, Any} uuid = UUID(uuid::String) info::Dict{String, Any} name = info["name"]::String pkgpath = info["path"]::String - pkg = PkgEntry(pkgpath, path, name, uuid, uninit) + pkg = PkgEntry(pkgpath, path, name, uuid, in_memory_registry, uninit) pkgs[uuid] = pkg end reg = RegistryInstance( path, d["name"]::String, - reg_uuid, + UUID(d["uuid"]::String), get(d, "url", nothing)::Union{String, Nothing}, get(d, "repo", nothing)::Union{String, Nothing}, get(d, "description", nothing)::Union{String, Nothing}, pkgs, tree_info, + in_memory_registry, Dict{String, UUID}(), ) if tree_info !== nothing - REGISTRY_CACHE[path] = (tree_info, reg) + REGISTRY_CACHE[path] = (tree_info, compressed_file !== nothing, reg) end return reg end @@ -276,9 +330,11 @@ function reachable_registries(; depots::Union{String, Vector{String}}=Base.DEPOT reg_dir = joinpath(d, "registries") isdir(reg_dir) || continue for name in readdir(reg_dir) - file = joinpath(reg_dir, name, "Registry.toml") - isfile(file) || continue - push!(registries, RegistryInstance(joinpath(reg_dir, name))) + file_unpacked = joinpath(reg_dir, name, "Registry.toml") + file_packed = joinpath(reg_dir, name, ".registry_info.toml") + if isfile(file_unpacked) || isfile(file_packed) + push!(registries, RegistryInstance(joinpath(reg_dir, name))) + end end end return registries diff --git a/test/utils.jl b/test/utils.jl index b4cd12c272..fdfafc0175 100644 --- a/test/utils.jl +++ b/test/utils.jl @@ -20,7 +20,7 @@ const GENERAL_UUID = UUID("23338594-aafe-5451-b93e-139f81909106") function init_reg() url, _ = Pkg.Registry.pkg_server_registry_url(GENERAL_UUID, nothing) mkpath(REGISTRY_DIR) - if Pkg.Registry.registry_use_pkg_server(url) + if Pkg.Registry.registry_use_pkg_server() @info "Downloading General registry from $url" Pkg.PlatformEngines.download_verify_unpack(url, nothing, REGISTRY_DIR, ignore_existence = true, io = stderr) tree_info_file = joinpath(REGISTRY_DIR, ".tree_info.toml")