diff --git a/src/API.jl b/src/API.jl index c9846ce263..09ee968137 100644 --- a/src/API.jl +++ b/src/API.jl @@ -1084,9 +1084,8 @@ end function precompile(ctx::Context, pkgs::Vector{PackageSpec}; internal_call::Bool=false, strict::Bool=false, warn_loaded = true, already_instantiated = false, timing::Bool = false, - _from_loading::Bool=false, flags_cacheflags::Pair{Cmd, Base.CacheFlags}=(``=>Base.CacheFlags()), kwargs...) + _from_loading::Bool=false, kwargs...) Context!(ctx; kwargs...) - flags, cacheflags = flags_cacheflags if !already_instantiated instantiate(ctx; allow_autoprecomp=false, kwargs...) @debug "precompile: instantiated" @@ -1509,7 +1508,7 @@ function precompile(ctx::Context, pkgs::Vector{PackageSpec}; internal_call::Bool circular = pkg in circular_deps is_stale = true - if !circular && (queued || (!suspended && (is_stale = !Base.isprecompiled(pkg; ignore_loaded=true, flags=cacheflags, stale_cache, cachepaths, sourcepath)))) + if !circular && (queued || (!suspended && (is_stale = !Base.isprecompiled(pkg; ignore_loaded=true, stale_cache, cachepaths, sourcepath)))) Base.acquire(parallel_limiter) is_direct_dep = pkg in direct_deps @@ -1536,7 +1535,7 @@ function precompile(ctx::Context, pkgs::Vector{PackageSpec}; internal_call::Bool t = @elapsed ret = maybe_cachefile_lock(io, print_lock, fancyprint, pkg, pkgspidlocked, hascolor) do Logging.with_logger(Logging.NullLogger()) do # The false here means we ignore loaded modules, so precompile for a fresh session - Base.compilecache(pkg, sourcepath, std_pipe, std_pipe, false; flags) + Base.compilecache(pkg, sourcepath, std_pipe, std_pipe, false) end end t_str = timing ? string(lpad(round(t * 1e3, digits = 1), 9), " ms") : "" @@ -1718,7 +1717,7 @@ function maybe_cachefile_lock(f, io::IO, print_lock::ReentrantLock, fancyprint:: # wait until the lock is available FileWatching.mkpidlock(pidfile; stale_age) do # double-check in case the other process crashed or the lock expired - if Base.isprecompiled(pkg; ignore_loaded=true, flags=cacheflags) # don't use caches for this as the env state will have changed + if Base.isprecompiled(pkg; ignore_loaded=true) # don't use caches for this as the env state will have changed return nothing # returning nothing indicates a process waited for another else delete!(pkgspidlocked, pkg) diff --git a/src/Operations.jl b/src/Operations.jl index 42db0ec638..a46a470fe3 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -1676,16 +1676,29 @@ function free(ctx::Context, pkgs::Vector{PackageSpec}; err_if_free=true) end end -function gen_test_code(source_path::String; test_args::Cmd) +function gen_test_code(source_path::String; coverage, julia_args::Cmd, test_args::Cmd) test_file = testfile(source_path) - return """ + code = """ $(Base.load_path_setup_code(false)) cd($(repr(dirname(test_file)))) append!(empty!(ARGS), $(repr(test_args.exec))) include($(repr(test_file))) """ + return gen_subprocess_cmd(code, source_path; coverage, julia_args) end +function gen_test_precompile_code(source_path::String; coverage, julia_args::Cmd, test_args::Cmd) + code = """ + try using Pkg + catch + @warn "Pkg failed to load, skipping precompilation." + else + Pkg.activate($(repr(Base.active_project()))) + Pkg.precompile(warn_loaded = false) + end + """ + return gen_subprocess_cmd(code, source_path; coverage, julia_args) +end function get_threads_spec() if Threads.nthreads(:interactive) > 0 @@ -1695,7 +1708,7 @@ function get_threads_spec() end end -function gen_subprocess_flags(source_path::String; coverage, julia_args) +function gen_subprocess_cmd(code::String, source_path::String; coverage, julia_args) coverage_arg = if coverage isa Bool coverage ? string("@", source_path) : "none" elseif coverage isa AbstractString @@ -1704,6 +1717,7 @@ function gen_subprocess_flags(source_path::String; coverage, julia_args) throw(ArgumentError("coverage should be a boolean or a string.")) end return ``` + $(Base.julia_cmd()) --code-coverage=$(coverage_arg) --color=$(Base.have_color === nothing ? "auto" : Base.have_color ? "yes" : "no") --check-bounds=yes @@ -1714,6 +1728,7 @@ function gen_subprocess_flags(source_path::String; coverage, julia_args) --track-allocation=$(("none", "user", "all")[Base.JLOptions().malloc_log + 1]) --threads=$(get_threads_spec()) $(julia_args) + --eval $(code) ``` end @@ -1782,6 +1797,13 @@ function sandbox(fn::Function, ctx::Context, target::PackageSpec, target_path::S sandbox_project_override = Project() end end + # add Pkg so that the test environment sandbox subprocesses can be precompiled + Pkg_uuid = UUID(PkgUUID) + if get!(sandbox_project_override.deps, "Pkg", Pkg_uuid) != Pkg_uuid + @warn """ + A package called Pkg is declared as a dependency with a UUID that doesn't match the Pkg stdlib. + This may cause unexpected behavior""" + end Types.write_project(sandbox_project_override, tmp_project) # create merged manifest @@ -1979,19 +2001,23 @@ function test(ctx::Context, pkgs::Vector{PackageSpec}; test_fn !== nothing && test_fn() sandbox_ctx = Context(;io=ctx.io) status(sandbox_ctx.env, sandbox_ctx.registries; mode=PKGMODE_COMBINED, io=sandbox_ctx.io, ignore_indent = false, show_usagetips = false) - flags = gen_subprocess_flags(source_path; coverage, julia_args) if should_autoprecompile() - cacheflags = Base.CacheFlags(parse(UInt8, read(`$(Base.julia_cmd()) $(flags) --eval 'show(ccall(:jl_cache_flags, UInt8, ()))'`, String))) - Pkg.activate(sandbox_ctx.env.project_file; #=io=devnull=#) do - Pkg.precompile(sandbox_ctx; io=sandbox_ctx.io, flags_cacheflags = flags => cacheflags) + # Precompile in a child process with the test julia args to ensure native caches match test setup + cmd = gen_test_precompile_code(source_path; coverage, julia_args, test_args) + p, interrupted = subprocess_handler(cmd, ctx, sandbox_ctx, "Precompilation of test environment interrupted. Exiting the test precompilation process") + if !success(p) + if interrupted + return + else + printpkgstyle(ctx.io, :Testing, "Precompilation of test environment failed. Continuing to tests", color = Base.warn_color()) + end end end printpkgstyle(ctx.io, :Testing, "Running tests...") flush(ctx.io) - code = gen_test_code(source_path; test_args) - cmd = `$(Base.julia_cmd()) $(flags) --eval $code` + cmd = gen_test_code(source_path; coverage, julia_args, test_args) p, interrupted = subprocess_handler(cmd, ctx, sandbox_ctx, "Tests interrupted. Exiting the test process") if success(p) printpkgstyle(ctx.io, :Testing, pkg.name * " tests passed ") diff --git a/test/test_packages/SandboxFallback2/Manifest.toml b/test/test_packages/SandboxFallback2/Manifest.toml index 2504c40307..357c81d24b 100644 --- a/test/test_packages/SandboxFallback2/Manifest.toml +++ b/test/test_packages/SandboxFallback2/Manifest.toml @@ -1,27 +1,21 @@ # This file is machine-generated - editing it directly is not advised -julia_version = "1.12.0-DEV" manifest_format = "2.0" -project_hash = "feee85e6d0dd3410fc4ea7cf6e409c2cbcd76272" [[deps.Base64]] uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" -version = "1.11.0" [[deps.Dates]] deps = ["Printf"] uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" -version = "1.11.0" [[deps.Distributed]] deps = ["Random", "Serialization", "Sockets"] uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" -version = "1.11.0" [[deps.InteractiveUtils]] deps = ["Markdown"] uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" -version = "1.11.0" [[deps.JSON]] deps = ["Dates", "Distributed", "Mmap", "Sockets", "Test", "Unicode"] @@ -30,53 +24,35 @@ uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" version = "0.19.0" [[deps.Logging]] -deps = ["StyledStrings"] uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" -version = "1.11.0" [[deps.Markdown]] deps = ["Base64"] uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" -version = "1.11.0" [[deps.Mmap]] uuid = "a63ad114-7e13-5084-954f-fe012c677804" -version = "1.11.0" [[deps.Printf]] deps = ["Unicode"] uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" -version = "1.11.0" [[deps.Random]] -deps = ["SHA"] +deps = ["Serialization"] uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" -version = "1.11.0" - -[[deps.SHA]] -uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" -version = "0.7.0" [[deps.Serialization]] uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" -version = "1.11.0" [[deps.Sockets]] uuid = "6462fe0b-24de-5631-8697-dd941f90decc" -version = "1.11.0" - -[[deps.StyledStrings]] -uuid = "f489334b-da3d-4c2e-b8f0-e476e12c162b" -version = "1.11.0" [[deps.Test]] -deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] +deps = ["Distributed", "InteractiveUtils", "Logging", "Random"] uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" -version = "1.11.0" [[deps.Unicode]] uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" -version = "1.11.0" [[deps.Unregistered]] deps = ["JSON"] diff --git a/test/test_packages/SandboxFallback2/dev/Unregistered/Manifest.toml b/test/test_packages/SandboxFallback2/dev/Unregistered/Manifest.toml index 5b236e45fa..3877ba13da 100644 --- a/test/test_packages/SandboxFallback2/dev/Unregistered/Manifest.toml +++ b/test/test_packages/SandboxFallback2/dev/Unregistered/Manifest.toml @@ -2,22 +2,18 @@ [[Base64]] uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" -version = "1.11.0" [[Dates]] deps = ["Printf"] uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" -version = "1.11.0" [[Distributed]] deps = ["Random", "Serialization", "Sockets"] uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" -version = "1.11.0" [[InteractiveUtils]] deps = ["Markdown"] uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" -version = "1.11.0" [[JSON]] deps = ["Dates", "Distributed", "Mmap", "Sockets", "Test", "Unicode"] @@ -26,50 +22,32 @@ uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" version = "0.20.0" [[Logging]] -deps = ["StyledStrings"] uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" -version = "1.11.0" [[Markdown]] deps = ["Base64"] uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" -version = "1.11.0" [[Mmap]] uuid = "a63ad114-7e13-5084-954f-fe012c677804" -version = "1.11.0" [[Printf]] deps = ["Unicode"] uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" -version = "1.11.0" [[Random]] -deps = ["SHA"] +deps = ["Serialization"] uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" -version = "1.11.0" - -[[SHA]] -uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" -version = "0.7.0" [[Serialization]] uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" -version = "1.11.0" [[Sockets]] uuid = "6462fe0b-24de-5631-8697-dd941f90decc" -version = "1.11.0" - -[[StyledStrings]] -uuid = "f489334b-da3d-4c2e-b8f0-e476e12c162b" -version = "1.11.0" [[Test]] -deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] +deps = ["Distributed", "InteractiveUtils", "Logging", "Random"] uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" -version = "1.11.0" [[Unicode]] uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" -version = "1.11.0" diff --git a/test/test_packages/SandboxFallback2/dev/Unregistered/test/Manifest.toml b/test/test_packages/SandboxFallback2/dev/Unregistered/test/Manifest.toml index 5b236e45fa..ac58105fd6 100644 --- a/test/test_packages/SandboxFallback2/dev/Unregistered/test/Manifest.toml +++ b/test/test_packages/SandboxFallback2/dev/Unregistered/test/Manifest.toml @@ -2,74 +2,32 @@ [[Base64]] uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" -version = "1.11.0" - -[[Dates]] -deps = ["Printf"] -uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" -version = "1.11.0" [[Distributed]] deps = ["Random", "Serialization", "Sockets"] uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" -version = "1.11.0" [[InteractiveUtils]] deps = ["Markdown"] uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" -version = "1.11.0" - -[[JSON]] -deps = ["Dates", "Distributed", "Mmap", "Sockets", "Test", "Unicode"] -git-tree-sha1 = "1f7a25b53ec67f5e9422f1f551ee216503f4a0fa" -uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" -version = "0.20.0" [[Logging]] -deps = ["StyledStrings"] uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" -version = "1.11.0" [[Markdown]] deps = ["Base64"] uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" -version = "1.11.0" - -[[Mmap]] -uuid = "a63ad114-7e13-5084-954f-fe012c677804" -version = "1.11.0" - -[[Printf]] -deps = ["Unicode"] -uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" -version = "1.11.0" [[Random]] -deps = ["SHA"] +deps = ["Serialization"] uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" -version = "1.11.0" - -[[SHA]] -uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" -version = "0.7.0" [[Serialization]] uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" -version = "1.11.0" [[Sockets]] uuid = "6462fe0b-24de-5631-8697-dd941f90decc" -version = "1.11.0" - -[[StyledStrings]] -uuid = "f489334b-da3d-4c2e-b8f0-e476e12c162b" -version = "1.11.0" [[Test]] -deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] +deps = ["Distributed", "InteractiveUtils", "Logging", "Random"] uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" -version = "1.11.0" - -[[Unicode]] -uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" -version = "1.11.0" diff --git a/test/test_packages/Sandbox_PreserveTestDeps/dev/Foo/test/Manifest.toml b/test/test_packages/Sandbox_PreserveTestDeps/dev/Foo/test/Manifest.toml index fd9b579394..2f26504b60 100644 --- a/test/test_packages/Sandbox_PreserveTestDeps/dev/Foo/test/Manifest.toml +++ b/test/test_packages/Sandbox_PreserveTestDeps/dev/Foo/test/Manifest.toml @@ -2,17 +2,14 @@ [[Base64]] uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" -version = "1.11.0" [[Dates]] deps = ["Printf"] uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" -version = "1.11.0" [[Distributed]] deps = ["Random", "Serialization", "Sockets"] uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" -version = "1.11.0" [[Example]] deps = ["Test"] @@ -23,7 +20,6 @@ version = "0.5.1" [[InteractiveUtils]] deps = ["Markdown"] uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" -version = "1.11.0" [[JSON]] deps = ["Dates", "Distributed", "Mmap", "Sockets", "Test", "Unicode"] @@ -32,50 +28,32 @@ uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" version = "0.20.0" [[Logging]] -deps = ["StyledStrings"] uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" -version = "1.11.0" [[Markdown]] deps = ["Base64"] uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" -version = "1.11.0" [[Mmap]] uuid = "a63ad114-7e13-5084-954f-fe012c677804" -version = "1.11.0" [[Printf]] deps = ["Unicode"] uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" -version = "1.11.0" [[Random]] -deps = ["SHA"] +deps = ["Serialization"] uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" -version = "1.11.0" - -[[SHA]] -uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" -version = "0.7.0" [[Serialization]] uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" -version = "1.11.0" [[Sockets]] uuid = "6462fe0b-24de-5631-8697-dd941f90decc" -version = "1.11.0" - -[[StyledStrings]] -uuid = "f489334b-da3d-4c2e-b8f0-e476e12c162b" -version = "1.11.0" [[Test]] -deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] +deps = ["Distributed", "InteractiveUtils", "Logging", "Random"] uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" -version = "1.11.0" [[Unicode]] uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" -version = "1.11.0"