From e7e8ce38359330441b1340046add367761035f69 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Fri, 23 Feb 2024 19:25:09 +0100 Subject: [PATCH] do not start a new process for precompiling the test env (#3792) --- src/API.jl | 9 ++-- src/Operations.jl | 44 ++++-------------- .../SandboxFallback2/Manifest.toml | 28 ++++++++++- .../dev/Unregistered/Manifest.toml | 26 ++++++++++- .../dev/Unregistered/test/Manifest.toml | 46 ++++++++++++++++++- .../dev/Foo/test/Manifest.toml | 26 ++++++++++- 6 files changed, 132 insertions(+), 47 deletions(-) diff --git a/src/API.jl b/src/API.jl index 09ee968137..c9846ce263 100644 --- a/src/API.jl +++ b/src/API.jl @@ -1084,8 +1084,9 @@ end function precompile(ctx::Context, pkgs::Vector{PackageSpec}; internal_call::Bool=false, strict::Bool=false, warn_loaded = true, already_instantiated = false, timing::Bool = false, - _from_loading::Bool=false, kwargs...) + _from_loading::Bool=false, flags_cacheflags::Pair{Cmd, Base.CacheFlags}=(``=>Base.CacheFlags()), kwargs...) Context!(ctx; kwargs...) + flags, cacheflags = flags_cacheflags if !already_instantiated instantiate(ctx; allow_autoprecomp=false, kwargs...) @debug "precompile: instantiated" @@ -1508,7 +1509,7 @@ function precompile(ctx::Context, pkgs::Vector{PackageSpec}; internal_call::Bool circular = pkg in circular_deps is_stale = true - if !circular && (queued || (!suspended && (is_stale = !Base.isprecompiled(pkg; ignore_loaded=true, stale_cache, cachepaths, sourcepath)))) + if !circular && (queued || (!suspended && (is_stale = !Base.isprecompiled(pkg; ignore_loaded=true, flags=cacheflags, stale_cache, cachepaths, sourcepath)))) Base.acquire(parallel_limiter) is_direct_dep = pkg in direct_deps @@ -1535,7 +1536,7 @@ function precompile(ctx::Context, pkgs::Vector{PackageSpec}; internal_call::Bool t = @elapsed ret = maybe_cachefile_lock(io, print_lock, fancyprint, pkg, pkgspidlocked, hascolor) do Logging.with_logger(Logging.NullLogger()) do # The false here means we ignore loaded modules, so precompile for a fresh session - Base.compilecache(pkg, sourcepath, std_pipe, std_pipe, false) + Base.compilecache(pkg, sourcepath, std_pipe, std_pipe, false; flags) end end t_str = timing ? string(lpad(round(t * 1e3, digits = 1), 9), " ms") : "" @@ -1717,7 +1718,7 @@ function maybe_cachefile_lock(f, io::IO, print_lock::ReentrantLock, fancyprint:: # wait until the lock is available FileWatching.mkpidlock(pidfile; stale_age) do # double-check in case the other process crashed or the lock expired - if Base.isprecompiled(pkg; ignore_loaded=true) # don't use caches for this as the env state will have changed + if Base.isprecompiled(pkg; ignore_loaded=true, flags=cacheflags) # don't use caches for this as the env state will have changed return nothing # returning nothing indicates a process waited for another else delete!(pkgspidlocked, pkg) diff --git a/src/Operations.jl b/src/Operations.jl index a46a470fe3..42db0ec638 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -1676,29 +1676,16 @@ function free(ctx::Context, pkgs::Vector{PackageSpec}; err_if_free=true) end end -function gen_test_code(source_path::String; coverage, julia_args::Cmd, test_args::Cmd) +function gen_test_code(source_path::String; test_args::Cmd) test_file = testfile(source_path) - code = """ + return """ $(Base.load_path_setup_code(false)) cd($(repr(dirname(test_file)))) append!(empty!(ARGS), $(repr(test_args.exec))) include($(repr(test_file))) """ - return gen_subprocess_cmd(code, source_path; coverage, julia_args) end -function gen_test_precompile_code(source_path::String; coverage, julia_args::Cmd, test_args::Cmd) - code = """ - try using Pkg - catch - @warn "Pkg failed to load, skipping precompilation." - else - Pkg.activate($(repr(Base.active_project()))) - Pkg.precompile(warn_loaded = false) - end - """ - return gen_subprocess_cmd(code, source_path; coverage, julia_args) -end function get_threads_spec() if Threads.nthreads(:interactive) > 0 @@ -1708,7 +1695,7 @@ function get_threads_spec() end end -function gen_subprocess_cmd(code::String, source_path::String; coverage, julia_args) +function gen_subprocess_flags(source_path::String; coverage, julia_args) coverage_arg = if coverage isa Bool coverage ? string("@", source_path) : "none" elseif coverage isa AbstractString @@ -1717,7 +1704,6 @@ function gen_subprocess_cmd(code::String, source_path::String; coverage, julia_a throw(ArgumentError("coverage should be a boolean or a string.")) end return ``` - $(Base.julia_cmd()) --code-coverage=$(coverage_arg) --color=$(Base.have_color === nothing ? "auto" : Base.have_color ? "yes" : "no") --check-bounds=yes @@ -1728,7 +1714,6 @@ function gen_subprocess_cmd(code::String, source_path::String; coverage, julia_a --track-allocation=$(("none", "user", "all")[Base.JLOptions().malloc_log + 1]) --threads=$(get_threads_spec()) $(julia_args) - --eval $(code) ``` end @@ -1797,13 +1782,6 @@ function sandbox(fn::Function, ctx::Context, target::PackageSpec, target_path::S sandbox_project_override = Project() end end - # add Pkg so that the test environment sandbox subprocesses can be precompiled - Pkg_uuid = UUID(PkgUUID) - if get!(sandbox_project_override.deps, "Pkg", Pkg_uuid) != Pkg_uuid - @warn """ - A package called Pkg is declared as a dependency with a UUID that doesn't match the Pkg stdlib. - This may cause unexpected behavior""" - end Types.write_project(sandbox_project_override, tmp_project) # create merged manifest @@ -2001,23 +1979,19 @@ function test(ctx::Context, pkgs::Vector{PackageSpec}; test_fn !== nothing && test_fn() sandbox_ctx = Context(;io=ctx.io) status(sandbox_ctx.env, sandbox_ctx.registries; mode=PKGMODE_COMBINED, io=sandbox_ctx.io, ignore_indent = false, show_usagetips = false) + flags = gen_subprocess_flags(source_path; coverage, julia_args) if should_autoprecompile() - # Precompile in a child process with the test julia args to ensure native caches match test setup - cmd = gen_test_precompile_code(source_path; coverage, julia_args, test_args) - p, interrupted = subprocess_handler(cmd, ctx, sandbox_ctx, "Precompilation of test environment interrupted. Exiting the test precompilation process") - if !success(p) - if interrupted - return - else - printpkgstyle(ctx.io, :Testing, "Precompilation of test environment failed. Continuing to tests", color = Base.warn_color()) - end + cacheflags = Base.CacheFlags(parse(UInt8, read(`$(Base.julia_cmd()) $(flags) --eval 'show(ccall(:jl_cache_flags, UInt8, ()))'`, String))) + Pkg.activate(sandbox_ctx.env.project_file; #=io=devnull=#) do + Pkg.precompile(sandbox_ctx; io=sandbox_ctx.io, flags_cacheflags = flags => cacheflags) end end printpkgstyle(ctx.io, :Testing, "Running tests...") flush(ctx.io) - cmd = gen_test_code(source_path; coverage, julia_args, test_args) + code = gen_test_code(source_path; test_args) + cmd = `$(Base.julia_cmd()) $(flags) --eval $code` p, interrupted = subprocess_handler(cmd, ctx, sandbox_ctx, "Tests interrupted. Exiting the test process") if success(p) printpkgstyle(ctx.io, :Testing, pkg.name * " tests passed ") diff --git a/test/test_packages/SandboxFallback2/Manifest.toml b/test/test_packages/SandboxFallback2/Manifest.toml index 357c81d24b..2504c40307 100644 --- a/test/test_packages/SandboxFallback2/Manifest.toml +++ b/test/test_packages/SandboxFallback2/Manifest.toml @@ -1,21 +1,27 @@ # This file is machine-generated - editing it directly is not advised +julia_version = "1.12.0-DEV" manifest_format = "2.0" +project_hash = "feee85e6d0dd3410fc4ea7cf6e409c2cbcd76272" [[deps.Base64]] uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" +version = "1.11.0" [[deps.Dates]] deps = ["Printf"] uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" +version = "1.11.0" [[deps.Distributed]] deps = ["Random", "Serialization", "Sockets"] uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" +version = "1.11.0" [[deps.InteractiveUtils]] deps = ["Markdown"] uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" +version = "1.11.0" [[deps.JSON]] deps = ["Dates", "Distributed", "Mmap", "Sockets", "Test", "Unicode"] @@ -24,35 +30,53 @@ uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" version = "0.19.0" [[deps.Logging]] +deps = ["StyledStrings"] uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" +version = "1.11.0" [[deps.Markdown]] deps = ["Base64"] uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" +version = "1.11.0" [[deps.Mmap]] uuid = "a63ad114-7e13-5084-954f-fe012c677804" +version = "1.11.0" [[deps.Printf]] deps = ["Unicode"] uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" +version = "1.11.0" [[deps.Random]] -deps = ["Serialization"] +deps = ["SHA"] uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +version = "1.11.0" + +[[deps.SHA]] +uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" +version = "0.7.0" [[deps.Serialization]] uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" +version = "1.11.0" [[deps.Sockets]] uuid = "6462fe0b-24de-5631-8697-dd941f90decc" +version = "1.11.0" + +[[deps.StyledStrings]] +uuid = "f489334b-da3d-4c2e-b8f0-e476e12c162b" +version = "1.11.0" [[deps.Test]] -deps = ["Distributed", "InteractiveUtils", "Logging", "Random"] +deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +version = "1.11.0" [[deps.Unicode]] uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" +version = "1.11.0" [[deps.Unregistered]] deps = ["JSON"] diff --git a/test/test_packages/SandboxFallback2/dev/Unregistered/Manifest.toml b/test/test_packages/SandboxFallback2/dev/Unregistered/Manifest.toml index 3877ba13da..5b236e45fa 100644 --- a/test/test_packages/SandboxFallback2/dev/Unregistered/Manifest.toml +++ b/test/test_packages/SandboxFallback2/dev/Unregistered/Manifest.toml @@ -2,18 +2,22 @@ [[Base64]] uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" +version = "1.11.0" [[Dates]] deps = ["Printf"] uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" +version = "1.11.0" [[Distributed]] deps = ["Random", "Serialization", "Sockets"] uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" +version = "1.11.0" [[InteractiveUtils]] deps = ["Markdown"] uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" +version = "1.11.0" [[JSON]] deps = ["Dates", "Distributed", "Mmap", "Sockets", "Test", "Unicode"] @@ -22,32 +26,50 @@ uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" version = "0.20.0" [[Logging]] +deps = ["StyledStrings"] uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" +version = "1.11.0" [[Markdown]] deps = ["Base64"] uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" +version = "1.11.0" [[Mmap]] uuid = "a63ad114-7e13-5084-954f-fe012c677804" +version = "1.11.0" [[Printf]] deps = ["Unicode"] uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" +version = "1.11.0" [[Random]] -deps = ["Serialization"] +deps = ["SHA"] uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +version = "1.11.0" + +[[SHA]] +uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" +version = "0.7.0" [[Serialization]] uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" +version = "1.11.0" [[Sockets]] uuid = "6462fe0b-24de-5631-8697-dd941f90decc" +version = "1.11.0" + +[[StyledStrings]] +uuid = "f489334b-da3d-4c2e-b8f0-e476e12c162b" +version = "1.11.0" [[Test]] -deps = ["Distributed", "InteractiveUtils", "Logging", "Random"] +deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +version = "1.11.0" [[Unicode]] uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" +version = "1.11.0" diff --git a/test/test_packages/SandboxFallback2/dev/Unregistered/test/Manifest.toml b/test/test_packages/SandboxFallback2/dev/Unregistered/test/Manifest.toml index ac58105fd6..5b236e45fa 100644 --- a/test/test_packages/SandboxFallback2/dev/Unregistered/test/Manifest.toml +++ b/test/test_packages/SandboxFallback2/dev/Unregistered/test/Manifest.toml @@ -2,32 +2,74 @@ [[Base64]] uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" +version = "1.11.0" + +[[Dates]] +deps = ["Printf"] +uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" +version = "1.11.0" [[Distributed]] deps = ["Random", "Serialization", "Sockets"] uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" +version = "1.11.0" [[InteractiveUtils]] deps = ["Markdown"] uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" +version = "1.11.0" + +[[JSON]] +deps = ["Dates", "Distributed", "Mmap", "Sockets", "Test", "Unicode"] +git-tree-sha1 = "1f7a25b53ec67f5e9422f1f551ee216503f4a0fa" +uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" +version = "0.20.0" [[Logging]] +deps = ["StyledStrings"] uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" +version = "1.11.0" [[Markdown]] deps = ["Base64"] uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" +version = "1.11.0" + +[[Mmap]] +uuid = "a63ad114-7e13-5084-954f-fe012c677804" +version = "1.11.0" + +[[Printf]] +deps = ["Unicode"] +uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" +version = "1.11.0" [[Random]] -deps = ["Serialization"] +deps = ["SHA"] uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +version = "1.11.0" + +[[SHA]] +uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" +version = "0.7.0" [[Serialization]] uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" +version = "1.11.0" [[Sockets]] uuid = "6462fe0b-24de-5631-8697-dd941f90decc" +version = "1.11.0" + +[[StyledStrings]] +uuid = "f489334b-da3d-4c2e-b8f0-e476e12c162b" +version = "1.11.0" [[Test]] -deps = ["Distributed", "InteractiveUtils", "Logging", "Random"] +deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +version = "1.11.0" + +[[Unicode]] +uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" +version = "1.11.0" diff --git a/test/test_packages/Sandbox_PreserveTestDeps/dev/Foo/test/Manifest.toml b/test/test_packages/Sandbox_PreserveTestDeps/dev/Foo/test/Manifest.toml index 2f26504b60..fd9b579394 100644 --- a/test/test_packages/Sandbox_PreserveTestDeps/dev/Foo/test/Manifest.toml +++ b/test/test_packages/Sandbox_PreserveTestDeps/dev/Foo/test/Manifest.toml @@ -2,14 +2,17 @@ [[Base64]] uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" +version = "1.11.0" [[Dates]] deps = ["Printf"] uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" +version = "1.11.0" [[Distributed]] deps = ["Random", "Serialization", "Sockets"] uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" +version = "1.11.0" [[Example]] deps = ["Test"] @@ -20,6 +23,7 @@ version = "0.5.1" [[InteractiveUtils]] deps = ["Markdown"] uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" +version = "1.11.0" [[JSON]] deps = ["Dates", "Distributed", "Mmap", "Sockets", "Test", "Unicode"] @@ -28,32 +32,50 @@ uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" version = "0.20.0" [[Logging]] +deps = ["StyledStrings"] uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" +version = "1.11.0" [[Markdown]] deps = ["Base64"] uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" +version = "1.11.0" [[Mmap]] uuid = "a63ad114-7e13-5084-954f-fe012c677804" +version = "1.11.0" [[Printf]] deps = ["Unicode"] uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" +version = "1.11.0" [[Random]] -deps = ["Serialization"] +deps = ["SHA"] uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +version = "1.11.0" + +[[SHA]] +uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" +version = "0.7.0" [[Serialization]] uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" +version = "1.11.0" [[Sockets]] uuid = "6462fe0b-24de-5631-8697-dd941f90decc" +version = "1.11.0" + +[[StyledStrings]] +uuid = "f489334b-da3d-4c2e-b8f0-e476e12c162b" +version = "1.11.0" [[Test]] -deps = ["Distributed", "InteractiveUtils", "Logging", "Random"] +deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +version = "1.11.0" [[Unicode]] uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" +version = "1.11.0"