diff --git a/base/loading.jl b/base/loading.jl index 986efc1e1e5fb..fd8eba6bed460 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -611,6 +611,23 @@ function env_project_file(env::String)::Union{Bool,String} end end +function base_project(project_file) + base_dir = abspath(joinpath(dirname(project_file), "..")) + base_project_file = env_project_file(base_dir) + base_project_file isa String || return nothing + d = parsed_toml(base_project_file) + workspace = get(d, "workspace", nothing)::Union{Dict{String, Any}, Nothing} + if workspace === nothing + return nothing + end + projects = get(workspace, "projects", nothing)::Union{Vector{String}, Nothing, String} + projects === nothing && return nothing + if projects isa Vector && basename(dirname(project_file)) in projects + return base_project_file + end + return nothing +end + function project_deps_get(env::String, name::String)::Union{Nothing,PkgId} project_file = env_project_file(env) if project_file isa String @@ -622,21 +639,27 @@ function project_deps_get(env::String, name::String)::Union{Nothing,PkgId} return nothing end +function package_get(project_file, where::PkgId, name::String) + proj = project_file_name_uuid(project_file, where.name) + if proj == where + # if `where` matches the project, use [deps] section as manifest, and stop searching + pkg_uuid = explicit_project_deps_get(project_file, name) + return PkgId(pkg_uuid, name) + end + return nothing +end + function manifest_deps_get(env::String, where::PkgId, name::String)::Union{Nothing,PkgId} uuid = where.uuid @assert uuid !== nothing project_file = env_project_file(env) if project_file isa String - # first check if `where` names the Project itself - proj = project_file_name_uuid(project_file, where.name) - if proj == where - # if `where` matches the project, use [deps] section as manifest, and stop searching - pkg_uuid = explicit_project_deps_get(project_file, name) - return PkgId(pkg_uuid, name) - end + pkg = package_get(project_file, where, name) + pkg === nothing || return pkg d = parsed_toml(project_file) exts = get(d, "extensions", nothing)::Union{Dict{String, Any}, Nothing} if exts !== nothing + proj = project_file_name_uuid(project_file, where.name) # Check if `where` is an extension of the project if where.name in keys(exts) && where.uuid == uuid5(proj.uuid::UUID, where.name) # Extensions can load weak deps... @@ -726,6 +749,14 @@ function project_file_path(project_file::String) joinpath(dirname(project_file), get(d, "path", "")::String) end +function workspace_manifest(project_file) + base = base_project(project_file) + if base !== nothing + return project_file_manifest_path(base) + end + return nothing +end + # find project file's corresponding manifest file function project_file_manifest_path(project_file::String)::Union{Nothing,String} @lock require_lock begin @@ -736,6 +767,10 @@ function project_file_manifest_path(project_file::String)::Union{Nothing,String} end dir = abspath(dirname(project_file)) d = parsed_toml(project_file) + base_manifest = workspace_manifest(project_file) + if base_manifest !== nothing + return base_manifest + end explicit_manifest = get(d, "manifest", nothing)::Union{String, Nothing} manifest_path = nothing if explicit_manifest !== nothing @@ -3355,9 +3390,27 @@ function recursive_prefs_merge(base::Dict{String, Any}, overrides::Dict{String, return new_base end +function get_projects_workspace_to_root(project_file) + projects = String[project_file] + while true + project_file = base_project(project_file) + if project_file === nothing + return projects + end + push!(projects, project_file) + end +end + function get_preferences(uuid::Union{UUID,Nothing} = nothing) merged_prefs = Dict{String,Any}() - for env in reverse(load_path()) + loadpath = load_path() + projects_to_merge_prefs = String[] + append!(projects_to_merge_prefs, Iterators.drop(loadpath, 1)) + if length(loadpath) >= 1 + prepend!(projects_to_merge_prefs, get_projects_workspace_to_root(first(loadpath))) + end + + for env in reverse(projects_to_merge_prefs) project_toml = env_project_file(env) if !isa(project_toml, String) continue diff --git a/base/precompilation.jl b/base/precompilation.jl index 50b84ad829d0a..a823b2b579852 100644 --- a/base/precompilation.jl +++ b/base/precompilation.jl @@ -1,7 +1,8 @@ module Precompilation using Base: PkgId, UUID, SHA1, parsed_toml, project_file_name_uuid, project_names, - project_file_manifest_path, get_deps, preferences_names, isaccessibledir, isfile_casesensitive + project_file_manifest_path, get_deps, preferences_names, isaccessibledir, isfile_casesensitive, + base_project # This is currently only used for pkgprecompile but the plan is to use this in code loading in the future # see the `kc/codeloading2.0` branch @@ -59,6 +60,19 @@ function ExplicitEnv(envpath::String=Base.active_project()) delete!(project_deps, name) end + # This project might be a package, in that case, that is also a "dependency" + # of the project. + proj_name = get(project_d, "name", nothing)::Union{String, Nothing} + _proj_uuid = get(project_d, "uuid", nothing)::Union{String, Nothing} + proj_uuid = _proj_uuid === nothing ? nothing : UUID(_proj_uuid) + + project_is_package = proj_name !== nothing && proj_uuid !== nothing + if project_is_package + # TODO: Error on missing uuid? + project_deps[proj_name] = UUID(proj_uuid) + names[UUID(proj_uuid)] = proj_name + end + project_extensions = Dict{String, Vector{UUID}}() # Collect all extensions of the project for (name, triggers::Union{String, Vector{String}}) in get(Dict{String, Any}, project_d, "extensions")::Dict{String, Any} @@ -76,18 +90,6 @@ function ExplicitEnv(envpath::String=Base.active_project()) project_extensions[name] = uuids end - # This project might be a package, in that case, that is also a "dependency" - # of the project. - proj_name = get(project_d, "name", nothing)::Union{String, Nothing} - _proj_uuid = get(project_d, "uuid", nothing)::Union{String, Nothing} - proj_uuid = _proj_uuid === nothing ? nothing : UUID(_proj_uuid) - - if proj_name !== nothing && proj_uuid !== nothing - # TODO: Error on missing uuid? - project_deps[proj_name] = UUID(proj_uuid) - names[UUID(proj_uuid)] = proj_name - end - manifest = project_file_manifest_path(envpath) manifest_d = manifest === nothing ? Dict{String, Any}() : parsed_toml(manifest) @@ -355,8 +357,8 @@ function precompilepkgs(pkgs::Vector{String}=String[]; configs::Union{Config,Vector{Config}}=(``=>Base.CacheFlags()), io::IO=stderr, # asking for timing disables fancy mode, as timing is shown in non-fancy mode - fancyprint::Bool = can_fancyprint(io) && !timing - ) + fancyprint::Bool = can_fancyprint(io) && !timing, + manifest::Bool=false,) configs = configs isa Config ? [configs] : configs @@ -512,9 +514,15 @@ function precompilepkgs(pkgs::Vector{String}=String[]; end @debug "precompile: circular dep check done" - # if a list of packages is given, restrict to dependencies of given packages - if !isempty(pkgs) - function collect_all_deps(depsmap, dep, alldeps=Set{Base.PkgId}()) + if !manifest + if isempty(pkgs) + pkgs = [pkg.name for pkg in direct_deps] + target = "all packages" + else + target = join(pkgs, ", ") + end + # restrict to dependencies of given packages + function collect_all_deps(depsmap, dep, alldeps=Set{Base.PkgId}()) for _dep in depsmap[dep] if !(_dep in alldeps) push!(alldeps, _dep) @@ -544,13 +552,13 @@ function precompilepkgs(pkgs::Vector{String}=String[]; # TODO: actually handle packages from other envs in the stack return else - error("No direct dependencies outside of the sysimage found matching $(repr(pkgs))") + return end end - target = join(pkgs, ", ") else - target = "project" + target = "manifest" end + nconfigs = length(configs) if nconfigs == 1 if !isempty(only(configs)[1]) diff --git a/doc/src/manual/code-loading.md b/doc/src/manual/code-loading.md index 25ebf3475f34c..9e753d2a7de33 100644 --- a/doc/src/manual/code-loading.md +++ b/doc/src/manual/code-loading.md @@ -394,6 +394,28 @@ are stored in the manifest file in the section for that package. The dependency a package are the same as for its "parent" except that the listed extension dependencies are also considered as dependencies. +### [Workspaces](@id workspaces) + +A project file can define a workspace by giving a set of projects that is part of that workspace: + +```toml +[workspace] +projects = ["test", "benchmarks", "docs", "SomePackage"] +``` + +Each subfolder contains its own `Project.toml` file, which may include additional dependencies and compatibility constraints. In such cases, the package manager gathers all dependency information from all the projects in the workspace generating a single manifest file that combines the versions of all dependencies. + +Furthermore, workspaces can be "nested", meaning a project defining a workspace can also be part of another workspace. In this scenario, a single manifest file is still utilized, stored alongside the "root project" (the project that doesn't have another workspace including it). An example file structure could look like this: + +``` +Project.toml # projects = ["MyPackage"] +Manifest.toml +MyPackage/ + Project.toml # projects = ["test"] + test/ + Project.toml +``` + ### [Package/Environment Preferences](@id preferences) Preferences are dictionaries of metadata that influence package behavior within an environment. diff --git a/test/loading.jl b/test/loading.jl index 8ba2cf3026120..872ccd197f0f8 100644 --- a/test/loading.jl +++ b/test/loading.jl @@ -1550,3 +1550,61 @@ end rot13proj = joinpath(@__DIR__, "project", "Rot13") @test readchomp(`$(Base.julia_cmd()) --startup-file=no --project=$rot13proj -m Rot13 --project nowhere ABJURER`) == "--cebwrpg abjurer NOWHERE " end + +@testset "workspace loading" begin + old_load_path = copy(LOAD_PATH) + try + empty!(LOAD_PATH) + push!(LOAD_PATH, joinpath(@__DIR__, "project", "SubProject")) + @test Base.get_preferences()["value"] == 1 + @test Base.get_preferences()["x"] == 1 + + empty!(LOAD_PATH) + push!(LOAD_PATH, joinpath(@__DIR__, "project", "SubProject", "sub")) + id = Base.identify_package("Devved") + @test isfile(Base.locate_package(id)) + @test Base.identify_package("Devved2") === nothing + id3 = Base.identify_package("MyPkg") + @test isfile(Base.locate_package(id3)) + + empty!(LOAD_PATH) + push!(LOAD_PATH, joinpath(@__DIR__, "project", "SubProject", "PackageThatIsSub")) + id_pkg = Base.identify_package("PackageThatIsSub") + @test Base.identify_package(id_pkg, "Devved") === nothing + id_dev2 = Base.identify_package(id_pkg, "Devved2") + @test isfile(Base.locate_package(id_dev2)) + id_mypkg = Base.identify_package("MyPkg") + @test isfile(Base.locate_package(id_mypkg)) + id_dev = Base.identify_package(id_mypkg, "Devved") + @test isfile(Base.locate_package(id_dev)) + @test Base.get_preferences()["value"] == 2 + @test Base.get_preferences()["x"] == 1 + @test Base.get_preferences()["y"] == 2 + + empty!(LOAD_PATH) + push!(LOAD_PATH, joinpath(@__DIR__, "project", "SubProject", "PackageThatIsSub", "test")) + id_pkg = Base.identify_package("PackageThatIsSub") + @test isfile(Base.locate_package(id_pkg)) + @test Base.identify_package(id_pkg, "Devved") === nothing + id_dev2 = Base.identify_package(id_pkg, "Devved2") + @test isfile(Base.locate_package(id_dev2)) + id_mypkg = Base.identify_package("MyPkg") + @test isfile(Base.locate_package(id_mypkg)) + id_dev = Base.identify_package(id_mypkg, "Devved") + @test isfile(Base.locate_package(id_dev)) + @test Base.get_preferences()["value"] == 3 + @test Base.get_preferences()["x"] == 1 + @test Base.get_preferences()["y"] == 2 + @test Base.get_preferences()["z"] == 3 + + empty!(LOAD_PATH) + push!(LOAD_PATH, joinpath(@__DIR__, "project", "SubProject", "test")) + id_mypkg = Base.identify_package("MyPkg") + id_dev = Base.identify_package(id_mypkg, "Devved") + @test isfile(Base.locate_package(id_dev)) + @test Base.identify_package("Devved2") === nothing + + finally + copy!(LOAD_PATH, old_load_path) + end +end diff --git a/test/project/SubProject/Devved/Project.toml b/test/project/SubProject/Devved/Project.toml new file mode 100644 index 0000000000000..63088a132cb77 --- /dev/null +++ b/test/project/SubProject/Devved/Project.toml @@ -0,0 +1,3 @@ +name = "Devved" +uuid = "cbce3a6e-7a3d-4e84-8e6d-b87208df7599" +version = "0.1.0" diff --git a/test/project/SubProject/Devved/src/Devved.jl b/test/project/SubProject/Devved/src/Devved.jl new file mode 100644 index 0000000000000..f3eb267409ece --- /dev/null +++ b/test/project/SubProject/Devved/src/Devved.jl @@ -0,0 +1,5 @@ +module Devved + +greet() = print("Hello World!") + +end # module Devved diff --git a/test/project/SubProject/Devved2/Project.toml b/test/project/SubProject/Devved2/Project.toml new file mode 100644 index 0000000000000..c761630566116 --- /dev/null +++ b/test/project/SubProject/Devved2/Project.toml @@ -0,0 +1,3 @@ +name = "Devved2" +uuid = "08f74b90-50f5-462f-80b9-a72b1258a17b" +version = "0.1.0" diff --git a/test/project/SubProject/Devved2/src/Devved2.jl b/test/project/SubProject/Devved2/src/Devved2.jl new file mode 100644 index 0000000000000..9bd5df2793671 --- /dev/null +++ b/test/project/SubProject/Devved2/src/Devved2.jl @@ -0,0 +1,5 @@ +module Devved2 + +greet() = print("Hello World!") + +end # module Devved2 diff --git a/test/project/SubProject/Manifest.toml b/test/project/SubProject/Manifest.toml new file mode 100644 index 0000000000000..5d791a74652d4 --- /dev/null +++ b/test/project/SubProject/Manifest.toml @@ -0,0 +1,68 @@ +# This file is machine-generated - editing it directly is not advised + +julia_version = "1.12.0-DEV" +manifest_format = "2.0" +project_hash = "620b9377bc807ff657e6618c8ccc24887eb40285" + +[[deps.Base64]] +uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" +version = "1.11.0" + +[[deps.Devved]] +path = "Devved" +uuid = "cbce3a6e-7a3d-4e84-8e6d-b87208df7599" +version = "0.1.0" + +[[deps.Devved2]] +path = "Devved2" +uuid = "08f74b90-50f5-462f-80b9-a72b1258a17b" +version = "0.1.0" + +[[deps.InteractiveUtils]] +deps = ["Markdown"] +uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" +version = "1.11.0" + +[[deps.Logging]] +deps = ["StyledStrings"] +uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" +version = "1.11.0" + +[[deps.Markdown]] +deps = ["Base64"] +uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" +version = "1.11.0" + +[[deps.MyPkg]] +deps = ["Devved", "Devved2"] +path = "." +uuid = "0cafdeb2-d7a2-40d0-8d22-4411fcc2c4ee" +version = "0.0.0" + +[[deps.PackageThatIsSub]] +deps = ["Devved2", "MyPkg"] +path = "PackageThatIsSub" +uuid = "1efb588c-9412-4e40-90a4-710420bd84aa" +version = "0.1.0" + +[[deps.Random]] +deps = ["SHA"] +uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +version = "1.11.0" + +[[deps.SHA]] +uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" +version = "0.7.0" + +[[deps.Serialization]] +uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" +version = "1.11.0" + +[[deps.StyledStrings]] +uuid = "f489334b-da3d-4c2e-b8f0-e476e12c162b" +version = "1.11.0" + +[[deps.Test]] +deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] +uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +version = "1.11.0" diff --git a/test/project/SubProject/PackageThatIsSub/Project.toml b/test/project/SubProject/PackageThatIsSub/Project.toml new file mode 100644 index 0000000000000..e41dd998c5a1c --- /dev/null +++ b/test/project/SubProject/PackageThatIsSub/Project.toml @@ -0,0 +1,14 @@ +name = "PackageThatIsSub" +uuid = "1efb588c-9412-4e40-90a4-710420bd84aa" +version = "0.1.0" + +[workspace] +projects = ["test"] + +[deps] +Devved2 = "08f74b90-50f5-462f-80b9-a72b1258a17b" +MyPkg = "0cafdeb2-d7a2-40d0-8d22-4411fcc2c4ee" + +[preferences] +value = 2 +y = 2 diff --git a/test/project/SubProject/PackageThatIsSub/src/PackageThatIsSub.jl b/test/project/SubProject/PackageThatIsSub/src/PackageThatIsSub.jl new file mode 100644 index 0000000000000..7f9ea94ccb156 --- /dev/null +++ b/test/project/SubProject/PackageThatIsSub/src/PackageThatIsSub.jl @@ -0,0 +1,5 @@ +module PackageThatIsSub + +greet() = print("Hello World!") + +end # module PackageThatIsSub diff --git a/test/project/SubProject/PackageThatIsSub/test/Project.toml b/test/project/SubProject/PackageThatIsSub/test/Project.toml new file mode 100644 index 0000000000000..dc8186e2b735e --- /dev/null +++ b/test/project/SubProject/PackageThatIsSub/test/Project.toml @@ -0,0 +1,8 @@ +[deps] +MyPkg = "0cafdeb2-d7a2-40d0-8d22-4411fcc2c4ee" +PackageThatIsSub = "1efb588c-9412-4e40-90a4-710420bd84aa" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[preferences] +value = 3 +z = 3 diff --git a/test/project/SubProject/Project.toml b/test/project/SubProject/Project.toml new file mode 100644 index 0000000000000..dcb84d865ac85 --- /dev/null +++ b/test/project/SubProject/Project.toml @@ -0,0 +1,13 @@ +name = "MyPkg" +uuid = "0cafdeb2-d7a2-40d0-8d22-4411fcc2c4ee" + +[workspace] +projects = ["sub", "PackageThatIsSub", "test"] + +[deps] +Devved = "cbce3a6e-7a3d-4e84-8e6d-b87208df7599" +Devved2 = "08f74b90-50f5-462f-80b9-a72b1258a17b" + +[preferences] +value = 1 +x = 1 diff --git a/test/project/SubProject/src/MyPkg.jl b/test/project/SubProject/src/MyPkg.jl new file mode 100644 index 0000000000000..6d84954645d55 --- /dev/null +++ b/test/project/SubProject/src/MyPkg.jl @@ -0,0 +1,3 @@ +module MyPkg + +end diff --git a/test/project/SubProject/sub/Project.toml b/test/project/SubProject/sub/Project.toml new file mode 100644 index 0000000000000..50aa238e91d57 --- /dev/null +++ b/test/project/SubProject/sub/Project.toml @@ -0,0 +1,3 @@ +[deps] +Devved = "cbce3a6e-7a3d-4e84-8e6d-b87208df7599" +MyPkg = "0cafdeb2-d7a2-40d0-8d22-4411fcc2c4ee" diff --git a/test/project/SubProject/test/Project.toml b/test/project/SubProject/test/Project.toml new file mode 100644 index 0000000000000..b64312e4b1ee2 --- /dev/null +++ b/test/project/SubProject/test/Project.toml @@ -0,0 +1,4 @@ +[deps] +MyPkg = "0cafdeb2-d7a2-40d0-8d22-4411fcc2c4ee" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +Devved = "cbce3a6e-7a3d-4e84-8e6d-b87208df7599"