Skip to content

Commit

Permalink
Merge pull request #2425 from IanButterworth/ib/backports_16
Browse files Browse the repository at this point in the history 
1.6.1 backports
  • Loading branch information
@KristofferC
KristofferC authored Apr 14, 2021
2 parents 05fa7f9 + 0d33deb commit c78a8be
Show file tree
Hide file tree
Showing 12 changed files with 278 additions and 190 deletions.
6 changes: 4 additions & 2 deletions .github/workflows/test.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@ jobs:
fail-fast: false
matrix:
julia-version:
# - '1.6'
- 'nightly'
- '1.6'
# - 'nightly'
julia-arch:
- 'x64'
- 'x86'
Expand Down Expand Up @@ -53,6 +53,8 @@ jobs:
env:
JULIA_PKG_SERVER: ${{ matrix.pkg-server }}
- uses: julia-actions/julia-processcoverage@v1
env:
JULIA_PKG_SERVER: ${{ matrix.pkg-server }}
- uses: codecov/codecov-action@v1
with:
file: lcov.info
Expand Down
202 changes: 115 additions & 87 deletions src/API.jl
View file

Large diffs are not rendered by default.

41 changes: 24 additions & 17 deletions src/Artifacts.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -286,7 +286,7 @@ end

"""
download_artifact(tree_hash::SHA1, tarball_url::String, tarball_hash::String;
verbose::Bool = false)
verbose::Bool = false, io::IO=DEFAULT_IO[])
Download/install an artifact into the artifact store. Returns `true` on success.
Expand All @@ -299,6 +299,7 @@ function download_artifact(
tarball_hash::Union{String, Nothing} = nothing;
verbose::Bool = false,
quiet_download::Bool = false,
io::IO=DEFAULT_IO[],
)
if artifact_exists(tree_hash)
return true
Expand All @@ -319,7 +320,7 @@ function download_artifact(
# the filesystem ACLs for executable permissions, which git tree hashes care about.
try
download_verify_unpack(tarball_url, tarball_hash, dest_dir, ignore_existence=true,
verbose=verbose, quiet_download=quiet_download)
verbose=verbose, quiet_download=quiet_download, io=io)
catch e
# Clean that destination directory out if something went wrong
rm(dest_dir; force=true, recursive=true)
Expand All @@ -339,7 +340,8 @@ function download_artifact(
# `create_artifact()` wrapper does, so we use that here.
calc_hash = try
create_artifact() do dir
download_verify_unpack(tarball_url, tarball_hash, dir, ignore_existence=true, verbose=verbose, quiet_download=quiet_download)
download_verify_unpack(tarball_url, tarball_hash, dir, ignore_existence=true, verbose=verbose,
quiet_download=quiet_download, io=io)
end
catch e
if isa(e, InterruptException)
Expand Down Expand Up @@ -379,7 +381,10 @@ end
"""
ensure_artifact_installed(name::String, artifacts_toml::String;
platform::AbstractPlatform = HostPlatform(),
pkg_uuid::Union{Base.UUID,Nothing}=nothing)
pkg_uuid::Union{Base.UUID,Nothing}=nothing,
verbose::Bool = false,
quiet_download::Bool = false,
io::IO=DEFAULT_IO[])
Ensures an artifact is installed, downloading it via the download information stored in
`artifacts_toml` if necessary. Throws an error if unable to install.
Expand All @@ -391,29 +396,31 @@ function ensure_artifact_installed(name::String, artifacts_toml::String;
platform::AbstractPlatform = HostPlatform(),
pkg_uuid::Union{Base.UUID,Nothing}=nothing,
verbose::Bool = false,
quiet_download::Bool = false)
quiet_download::Bool = false,
io::IO=DEFAULT_IO[])
meta = artifact_meta(name, artifacts_toml; pkg_uuid=pkg_uuid, platform=platform)
if meta === nothing
error("Cannot locate artifact '$(name)' in '$(artifacts_toml)'")
end

return ensure_artifact_installed(name, meta, artifacts_toml; platform=platform,
verbose=verbose, quiet_download=quiet_download)
verbose=verbose, quiet_download=quiet_download, io=io)
end

function ensure_artifact_installed(name::String, meta::Dict, artifacts_toml::String;
platform::AbstractPlatform = HostPlatform(),
verbose::Bool = false,
quiet_download::Bool = false)
quiet_download::Bool = false,
io::IO=DEFAULT_IO[])
hash = SHA1(meta["git-tree-sha1"])

if !artifact_exists(hash)
# first try downloading from Pkg server
# TODO: only do this if Pkg server knows about this package
if (server = pkg_server()) !== nothing
url = "$server/artifact/$hash"
download_success = with_show_download_info(name, quiet_download) do
download_artifact(hash, url; verbose=verbose, quiet_download=quiet_download)
download_success = with_show_download_info(io, name, quiet_download) do
download_artifact(hash, url; verbose=verbose, quiet_download=quiet_download, io=io)
end
download_success && return artifact_path(hash)
end
Expand All @@ -428,8 +435,8 @@ function ensure_artifact_installed(name::String, meta::Dict, artifacts_toml::Str
for entry in meta["download"]
url = entry["url"]
tarball_hash = entry["sha256"]
download_success = with_show_download_info(name, quiet_download) do
download_artifact(hash, url, tarball_hash; verbose=verbose, quiet_download=quiet_download)
download_success = with_show_download_info(io, name, quiet_download) do
download_artifact(hash, url, tarball_hash; verbose=verbose, quiet_download=quiet_download, io=io)
end
download_success && return artifact_path(hash)
end
Expand All @@ -439,9 +446,7 @@ function ensure_artifact_installed(name::String, meta::Dict, artifacts_toml::Str
end
end

function with_show_download_info(f, name, quiet_download)
# TODO: Use DEFAULT_IO?
io = stderr
function with_show_download_info(f, io, name, quiet_download)
fancyprint = can_fancyprint(io)
if !quiet_download
fancyprint && print_progress_bottom(io)
Expand All @@ -464,7 +469,8 @@ end
pkg_uuid = nothing,
include_lazy = false,
verbose = false,
quiet_download = false)
quiet_download = false,
io::IO=DEFAULT_IO[])
Installs all non-lazy artifacts from a given `(Julia)Artifacts.toml` file. `package_uuid` must
be provided to properly support overrides from `Overrides.toml` entries in depots.
Expand All @@ -479,7 +485,8 @@ function ensure_all_artifacts_installed(artifacts_toml::String;
pkg_uuid::Union{Nothing,Base.UUID} = nothing,
include_lazy::Bool = false,
verbose::Bool = false,
quiet_download::Bool = false)
quiet_download::Bool = false,
io::IO=DEFAULT_IO[])
if !isfile(artifacts_toml)
return
end
Expand All @@ -499,7 +506,7 @@ function ensure_all_artifacts_installed(artifacts_toml::String;

# Otherwise, let's try and install it!
ensure_artifact_installed(name, meta, artifacts_toml; platform=platform,
verbose=verbose, quiet_download=quiet_download)
verbose=verbose, quiet_download=quiet_download, io=io)
end
end

Expand Down
37 changes: 11 additions & 26 deletions src/MiniProgressBars.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,25 +21,6 @@ end
const NONINTERACTIVE_TIME_GRANULARITY = Ref(2.0)
const PROGRESS_BAR_PERCENTAGE_GRANULARITY = Ref(0.1)

function pretend_cursor()
io = stderr
bar = MiniProgressBar(; indent=2, header = "Progress", color = Base.info_color(),
percentage=false, always_reprint=true)
bar.max = 40
start_progress(io, bar)
sleep(0.5)
for i in 1:40
bar.current = i
print_progress_bottom(io)
println("Downloading ... $i")
show_progress(io, bar)
x = randstring(7)
sleep(0.01)
end
end_progress(io, bar)
print("Hello!")
end

function start_progress(io::IO, _::MiniProgressBar)
ansi_disablecursor = "\e[?25l"
print(io, ansi_disablecursor)
Expand Down Expand Up @@ -67,18 +48,22 @@ function show_progress(io::IO, p::MiniProgressBar)
end
p.prev = p.current
p.has_shown = true
n_filled = ceil(Int, p.width * perc / 100)
n_left = p.width - n_filled

progress_text = if p.percentage
@sprintf "%2.1f %%" perc
else
string(p.current, "/", p.max)
end

max_progress_width = max(0, min(displaysize(io)[2] - textwidth(p.header) - textwidth(progress_text) - 10 , p.width))
n_filled = ceil(Int, max_progress_width * perc / 100)
n_left = max_progress_width - n_filled
print(io, " "^p.indent)
printstyled(io, p.header, color=p.color, bold=true)
print(io, " [")
print(io, "="^n_filled, ">")
print(io, " "^n_left, "] ", )
if p.percentage
@printf io "%2.1f %%" perc
else
print(io, p.current, "/", p.max)
end
print(io, progress_text)
print(io, "\r")
end

Expand Down
40 changes: 22 additions & 18 deletions src/Operations.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@ import ..Artifacts: ensure_all_artifacts_installed, artifact_names, extract_all_
using Base.BinaryPlatforms
import ...Pkg
import ...Pkg: pkg_server
import ..Pkg: can_fancyprint
import ...Pkg: can_fancyprint, DEFAULT_IO
import ..Types: printpkgstyle

#########
# Utils #
Expand Down Expand Up @@ -565,7 +566,8 @@ end
function install_archive(
urls::Vector{Pair{String,Bool}},
hash::SHA1,
version_path::String
version_path::String;
io::IO=DEFAULT_IO[]
)::Bool
tmp_objects = String[]
url_success = false
Expand All @@ -574,7 +576,7 @@ function install_archive(
push!(tmp_objects, path) # for cleanup
url_success = true
try
PlatformEngines.download(url, path; verbose=false)
PlatformEngines.download(url, path; verbose=false, io=io)
catch e
e isa InterruptException && rethrow()
url_success = false
Expand Down Expand Up @@ -669,20 +671,22 @@ end
function download_artifacts(ctx::Context, pkgs::Vector{PackageSpec};
platform::AbstractPlatform=HostPlatform(),
julia_version = VERSION,
verbose::Bool=false)
verbose::Bool=false,
io::IO=DEFAULT_IO[])
# Filter out packages that have no source_path()
# pkg_roots = String[p for p in source_path.((ctx,), pkgs) if p !== nothing] # this runs up against inference limits?
pkg_roots = String[]
for pkg in pkgs
p = source_path(ctx, pkg)
p !== nothing && push!(pkg_roots, p)
end
return download_artifacts(ctx, pkg_roots; platform=platform, verbose=verbose)
return download_artifacts(ctx, pkg_roots; platform=platform, verbose=verbose, io=io)
end

function download_artifacts(ctx::Context, pkg_roots::Vector{String};
platform::AbstractPlatform=HostPlatform(),
verbose::Bool=false)
verbose::Bool=false,
io::IO=DEFAULT_IO[])
# List of Artifacts.toml files that we're going to download from
artifacts_tomls = String[]

Expand Down Expand Up @@ -776,7 +780,7 @@ function download_source(ctx::Context, pkgs::Vector{PackageSpec},
url = get_archive_url_for_version(repo_url, pkg.tree_hash)
url !== nothing && push!(archive_urls, url => false)
end
success = install_archive(archive_urls, pkg.tree_hash, path)
success = install_archive(archive_urls, pkg.tree_hash, path, io=ctx.io)
if success && readonly
set_readonly(path) # In add mode, files should be read-only
end
Expand Down Expand Up @@ -1101,17 +1105,15 @@ function rm(ctx::Context, pkgs::Vector{PackageSpec})
println(ctx.io, "No changes")
return
end
# only declare `compat` for direct dependencies
# only declare `compat` for remaining direct or `extra` dependencies
# `julia` is always an implicit direct dependency
filter!(ctx.env.project.compat) do (name, _)
name == "julia" || name in keys(ctx.env.project.deps)
name == "julia" || name in keys(ctx.env.project.deps) || name in keys(ctx.env.project.extras)
end
deps_names = append!(collect(keys(ctx.env.project.deps)),
collect(keys(ctx.env.project.extras)))
deps_names = union(keys(ctx.env.project.deps), keys(ctx.env.project.extras))
filter!(ctx.env.project.targets) do (target, deps)
!isempty(filter!(in(deps_names), deps))
end

# only keep reachable manifest entires
prune_manifest(ctx)
# update project & manifest
Expand Down Expand Up @@ -1232,7 +1234,7 @@ function add(ctx::Context, pkgs::Vector{PackageSpec}, new_git=UUID[];

# After downloading resolutionary packages, search for (Julia)Artifacts.toml files
# and ensure they are all downloaded and unpacked as well:
download_artifacts(ctx, pkgs; platform=platform)
download_artifacts(ctx, pkgs; platform=platform, io=ctx.io)

write_env(ctx.env) # write env before building
show_update(ctx)
Expand All @@ -1251,7 +1253,7 @@ function develop(ctx::Context, pkgs::Vector{PackageSpec}, new_git::Vector{UUID};
pkgs, deps_map = _resolve(ctx, pkgs, preserve)
update_manifest!(ctx, pkgs, deps_map)
new_apply = download_source(ctx, pkgs; readonly=true)
download_artifacts(ctx, pkgs; platform=platform)
download_artifacts(ctx, pkgs; platform=platform, io=ctx.io)
write_env(ctx.env) # write env before building
show_update(ctx)
build_versions(ctx, union(UUID[pkg.uuid for pkg in new_apply], new_git))
Expand Down Expand Up @@ -1316,7 +1318,7 @@ function up(ctx::Context, pkgs::Vector{PackageSpec}, level::UpgradeLevel)
deps_map = resolve_versions!(ctx, pkgs)
update_manifest!(ctx, pkgs, deps_map)
new_apply = download_source(ctx, pkgs)
download_artifacts(ctx, pkgs)
download_artifacts(ctx, pkgs; io=ctx.io)
write_env(ctx.env) # write env before building
show_update(ctx)
build_versions(ctx, union(UUID[pkg.uuid for pkg in new_apply], new_git))
Expand Down Expand Up @@ -1359,7 +1361,7 @@ function pin(ctx::Context, pkgs::Vector{PackageSpec})
update_manifest!(ctx, pkgs, deps_map)

new = download_source(ctx, pkgs)
download_artifacts(ctx, pkgs)
download_artifacts(ctx, pkgs; io=ctx.io)
write_env(ctx.env) # write env before building
show_update(ctx)
build_versions(ctx, UUID[pkg.uuid for pkg in new])
Expand Down Expand Up @@ -1396,7 +1398,7 @@ function free(ctx::Context, pkgs::Vector{PackageSpec})
pkgs, deps_map = _resolve(ctx, pkgs, PRESERVE_TIERED)
update_manifest!(ctx, pkgs, deps_map)
new = download_source(ctx, pkgs)
download_artifacts(ctx, new)
download_artifacts(ctx, new; io=ctx.io)
write_env(ctx.env) # write env before building
show_update(ctx)
build_versions(ctx, UUID[pkg.uuid for pkg in new])
Expand Down Expand Up @@ -1742,12 +1744,14 @@ print_single(ctx::Context, pkg::PackageSpec) = printstyled(ctx.io, stat_rep(pkg)

is_instantiated(::Nothing) = false
is_instantiated(x::PackageSpec) = x.version != VersionSpec() || is_stdlib(x.uuid)
# Compare an old and new node of the dependency graph and print a single line to summarize the change
function print_diff(ctx::Context, old::Union{Nothing,PackageSpec}, new::Union{Nothing,PackageSpec})
if !is_instantiated(old) && is_instantiated(new)
printstyled(ctx.io, "+ $(stat_rep(new))"; color=:light_green)
elseif !is_instantiated(new)
printstyled(ctx.io, "- $(stat_rep(old))"; color=:light_red)
elseif is_tracking_registry(old) && is_tracking_registry(new) && new.version isa VersionNumber && old.version isa VersionNumber
elseif is_tracking_registry(old) && is_tracking_registry(new) &&
new.version isa VersionNumber && old.version isa VersionNumber && new.version != old.version
if new.version > old.version
printstyled(ctx.io, "↑ $(stat_rep(old)) ⇒ $(stat_rep(new; name=false))"; color=:light_yellow)
else
Expand Down
Loading

0 comments on commit c78a8be

Please sign in to comment.