Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add code loading support for workspaces #53653

Merged
merged 21 commits into from
Mar 23, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 61 additions & 8 deletions base/loading.jl
Original file line number Diff line number Diff line change
Expand Up @@ -611,6 +611,23 @@ function env_project_file(env::String)::Union{Bool,String}
end
end

function base_project(project_file)
base_dir = abspath(joinpath(dirname(project_file), ".."))
base_project_file = env_project_file(base_dir)
base_project_file isa String || return nothing
d = parsed_toml(base_project_file)
workspace = get(d, "workspace", nothing)::Union{Dict{String, Any}, Nothing}
if workspace === nothing
return nothing
end
projects = get(workspace, "projects", nothing)::Union{Vector{String}, Nothing, String}
projects === nothing && return nothing
if projects isa Vector && basename(dirname(project_file)) in projects
return base_project_file
end
return nothing
end

function project_deps_get(env::String, name::String)::Union{Nothing,PkgId}
project_file = env_project_file(env)
if project_file isa String
Expand All @@ -622,21 +639,27 @@ function project_deps_get(env::String, name::String)::Union{Nothing,PkgId}
return nothing
end

function package_get(project_file, where::PkgId, name::String)
proj = project_file_name_uuid(project_file, where.name)
if proj == where
# if `where` matches the project, use [deps] section as manifest, and stop searching
pkg_uuid = explicit_project_deps_get(project_file, name)
return PkgId(pkg_uuid, name)
end
return nothing
end

function manifest_deps_get(env::String, where::PkgId, name::String)::Union{Nothing,PkgId}
uuid = where.uuid
@assert uuid !== nothing
project_file = env_project_file(env)
if project_file isa String
# first check if `where` names the Project itself
proj = project_file_name_uuid(project_file, where.name)
if proj == where
# if `where` matches the project, use [deps] section as manifest, and stop searching
pkg_uuid = explicit_project_deps_get(project_file, name)
return PkgId(pkg_uuid, name)
end
pkg = package_get(project_file, where, name)
pkg === nothing || return pkg
d = parsed_toml(project_file)
exts = get(d, "extensions", nothing)::Union{Dict{String, Any}, Nothing}
if exts !== nothing
proj = project_file_name_uuid(project_file, where.name)
# Check if `where` is an extension of the project
if where.name in keys(exts) && where.uuid == uuid5(proj.uuid::UUID, where.name)
# Extensions can load weak deps...
Expand Down Expand Up @@ -726,6 +749,14 @@ function project_file_path(project_file::String)
joinpath(dirname(project_file), get(d, "path", "")::String)
end

function workspace_manifest(project_file)
base = base_project(project_file)
if base !== nothing
return project_file_manifest_path(base)
end
return nothing
end

# find project file's corresponding manifest file
function project_file_manifest_path(project_file::String)::Union{Nothing,String}
@lock require_lock begin
Expand All @@ -736,6 +767,10 @@ function project_file_manifest_path(project_file::String)::Union{Nothing,String}
end
dir = abspath(dirname(project_file))
d = parsed_toml(project_file)
base_manifest = workspace_manifest(project_file)
if base_manifest !== nothing
return base_manifest
end
explicit_manifest = get(d, "manifest", nothing)::Union{String, Nothing}
manifest_path = nothing
if explicit_manifest !== nothing
Expand Down Expand Up @@ -3355,9 +3390,27 @@ function recursive_prefs_merge(base::Dict{String, Any}, overrides::Dict{String,
return new_base
end

function get_projects_workspace_to_root(project_file)
projects = String[project_file]
while true
project_file = base_project(project_file)
if project_file === nothing
return projects
end
push!(projects, project_file)
end
end

function get_preferences(uuid::Union{UUID,Nothing} = nothing)
merged_prefs = Dict{String,Any}()
for env in reverse(load_path())
loadpath = load_path()
projects_to_merge_prefs = String[]
append!(projects_to_merge_prefs, Iterators.drop(loadpath, 1))
if length(loadpath) >= 1
prepend!(projects_to_merge_prefs, get_projects_workspace_to_root(first(loadpath)))
end

for env in reverse(projects_to_merge_prefs)
project_toml = env_project_file(env)
if !isa(project_toml, String)
continue
Expand Down
50 changes: 29 additions & 21 deletions base/precompilation.jl
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
module Precompilation

using Base: PkgId, UUID, SHA1, parsed_toml, project_file_name_uuid, project_names,
project_file_manifest_path, get_deps, preferences_names, isaccessibledir, isfile_casesensitive
project_file_manifest_path, get_deps, preferences_names, isaccessibledir, isfile_casesensitive,
base_project

# This is currently only used for pkgprecompile but the plan is to use this in code loading in the future
# see the `kc/codeloading2.0` branch
Expand Down Expand Up @@ -59,6 +60,19 @@ function ExplicitEnv(envpath::String=Base.active_project())
delete!(project_deps, name)
end

# This project might be a package, in that case, that is also a "dependency"
# of the project.
proj_name = get(project_d, "name", nothing)::Union{String, Nothing}
_proj_uuid = get(project_d, "uuid", nothing)::Union{String, Nothing}
proj_uuid = _proj_uuid === nothing ? nothing : UUID(_proj_uuid)

project_is_package = proj_name !== nothing && proj_uuid !== nothing
if project_is_package
# TODO: Error on missing uuid?
project_deps[proj_name] = UUID(proj_uuid)
names[UUID(proj_uuid)] = proj_name
end

project_extensions = Dict{String, Vector{UUID}}()
# Collect all extensions of the project
for (name, triggers::Union{String, Vector{String}}) in get(Dict{String, Any}, project_d, "extensions")::Dict{String, Any}
Expand All @@ -76,18 +90,6 @@ function ExplicitEnv(envpath::String=Base.active_project())
project_extensions[name] = uuids
end

# This project might be a package, in that case, that is also a "dependency"
# of the project.
proj_name = get(project_d, "name", nothing)::Union{String, Nothing}
_proj_uuid = get(project_d, "uuid", nothing)::Union{String, Nothing}
proj_uuid = _proj_uuid === nothing ? nothing : UUID(_proj_uuid)

if proj_name !== nothing && proj_uuid !== nothing
# TODO: Error on missing uuid?
project_deps[proj_name] = UUID(proj_uuid)
names[UUID(proj_uuid)] = proj_name
end

manifest = project_file_manifest_path(envpath)
manifest_d = manifest === nothing ? Dict{String, Any}() : parsed_toml(manifest)

Expand Down Expand Up @@ -355,8 +357,8 @@ function precompilepkgs(pkgs::Vector{String}=String[];
configs::Union{Config,Vector{Config}}=(``=>Base.CacheFlags()),
io::IO=stderr,
# asking for timing disables fancy mode, as timing is shown in non-fancy mode
fancyprint::Bool = can_fancyprint(io) && !timing
)
fancyprint::Bool = can_fancyprint(io) && !timing,
manifest::Bool=false,)

configs = configs isa Config ? [configs] : configs

Expand Down Expand Up @@ -512,9 +514,15 @@ function precompilepkgs(pkgs::Vector{String}=String[];
end
@debug "precompile: circular dep check done"

# if a list of packages is given, restrict to dependencies of given packages
if !isempty(pkgs)
function collect_all_deps(depsmap, dep, alldeps=Set{Base.PkgId}())
if !manifest
if isempty(pkgs)
pkgs = [pkg.name for pkg in direct_deps]
target = "all packages"
else
target = join(pkgs, ", ")
end
# restrict to dependencies of given packages
function collect_all_deps(depsmap, dep, alldeps=Set{Base.PkgId}())
for _dep in depsmap[dep]
if !(_dep in alldeps)
push!(alldeps, _dep)
Expand Down Expand Up @@ -544,13 +552,13 @@ function precompilepkgs(pkgs::Vector{String}=String[];
# TODO: actually handle packages from other envs in the stack
return
else
error("No direct dependencies outside of the sysimage found matching $(repr(pkgs))")
return
end
end
target = join(pkgs, ", ")
else
target = "project"
target = "manifest"
end

nconfigs = length(configs)
if nconfigs == 1
if !isempty(only(configs)[1])
Expand Down
22 changes: 22 additions & 0 deletions doc/src/manual/code-loading.md
Original file line number Diff line number Diff line change
Expand Up @@ -394,6 +394,28 @@ are stored in the manifest file in the section for that package. The dependency
a package are the same as for its "parent" except that the listed extension dependencies are also considered as
dependencies.

### [Workspaces](@id workspaces)

A project file can define a workspace by giving a set of projects that is part of that workspace:

```toml
[workspace]
projects = ["test", "benchmarks", "docs", "SomePackage"]
```

Each subfolder contains its own `Project.toml` file, which may include additional dependencies and compatibility constraints. In such cases, the package manager gathers all dependency information from all the projects in the workspace generating a single manifest file that combines the versions of all dependencies.

Furthermore, workspaces can be "nested", meaning a project defining a workspace can also be part of another workspace. In this scenario, a single manifest file is still utilized, stored alongside the "root project" (the project that doesn't have another workspace including it). An example file structure could look like this:

```
Project.toml # projects = ["MyPackage"]
Manifest.toml
MyPackage/
Project.toml # projects = ["test"]
test/
Project.toml
```

### [Package/Environment Preferences](@id preferences)

Preferences are dictionaries of metadata that influence package behavior within an environment.
Expand Down
58 changes: 58 additions & 0 deletions test/loading.jl
Original file line number Diff line number Diff line change
Expand Up @@ -1550,3 +1550,61 @@ end
rot13proj = joinpath(@__DIR__, "project", "Rot13")
@test readchomp(`$(Base.julia_cmd()) --startup-file=no --project=$rot13proj -m Rot13 --project nowhere ABJURER`) == "--cebwrpg abjurer NOWHERE "
end

@testset "workspace loading" begin
old_load_path = copy(LOAD_PATH)
try
empty!(LOAD_PATH)
push!(LOAD_PATH, joinpath(@__DIR__, "project", "SubProject"))
@test Base.get_preferences()["value"] == 1
@test Base.get_preferences()["x"] == 1

empty!(LOAD_PATH)
push!(LOAD_PATH, joinpath(@__DIR__, "project", "SubProject", "sub"))
id = Base.identify_package("Devved")
@test isfile(Base.locate_package(id))
@test Base.identify_package("Devved2") === nothing
id3 = Base.identify_package("MyPkg")
@test isfile(Base.locate_package(id3))

empty!(LOAD_PATH)
push!(LOAD_PATH, joinpath(@__DIR__, "project", "SubProject", "PackageThatIsSub"))
id_pkg = Base.identify_package("PackageThatIsSub")
@test Base.identify_package(id_pkg, "Devved") === nothing
id_dev2 = Base.identify_package(id_pkg, "Devved2")
@test isfile(Base.locate_package(id_dev2))
id_mypkg = Base.identify_package("MyPkg")
@test isfile(Base.locate_package(id_mypkg))
id_dev = Base.identify_package(id_mypkg, "Devved")
@test isfile(Base.locate_package(id_dev))
@test Base.get_preferences()["value"] == 2
@test Base.get_preferences()["x"] == 1
@test Base.get_preferences()["y"] == 2

empty!(LOAD_PATH)
push!(LOAD_PATH, joinpath(@__DIR__, "project", "SubProject", "PackageThatIsSub", "test"))
id_pkg = Base.identify_package("PackageThatIsSub")
@test isfile(Base.locate_package(id_pkg))
@test Base.identify_package(id_pkg, "Devved") === nothing
id_dev2 = Base.identify_package(id_pkg, "Devved2")
@test isfile(Base.locate_package(id_dev2))
id_mypkg = Base.identify_package("MyPkg")
@test isfile(Base.locate_package(id_mypkg))
id_dev = Base.identify_package(id_mypkg, "Devved")
@test isfile(Base.locate_package(id_dev))
@test Base.get_preferences()["value"] == 3
@test Base.get_preferences()["x"] == 1
@test Base.get_preferences()["y"] == 2
@test Base.get_preferences()["z"] == 3

empty!(LOAD_PATH)
push!(LOAD_PATH, joinpath(@__DIR__, "project", "SubProject", "test"))
id_mypkg = Base.identify_package("MyPkg")
id_dev = Base.identify_package(id_mypkg, "Devved")
@test isfile(Base.locate_package(id_dev))
@test Base.identify_package("Devved2") === nothing

finally
copy!(LOAD_PATH, old_load_path)
end
end
3 changes: 3 additions & 0 deletions test/project/SubProject/Devved/Project.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
name = "Devved"
uuid = "cbce3a6e-7a3d-4e84-8e6d-b87208df7599"
version = "0.1.0"
5 changes: 5 additions & 0 deletions test/project/SubProject/Devved/src/Devved.jl
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
module Devved

greet() = print("Hello World!")

end # module Devved
3 changes: 3 additions & 0 deletions test/project/SubProject/Devved2/Project.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
name = "Devved2"
uuid = "08f74b90-50f5-462f-80b9-a72b1258a17b"
version = "0.1.0"
5 changes: 5 additions & 0 deletions test/project/SubProject/Devved2/src/Devved2.jl
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
module Devved2

greet() = print("Hello World!")

end # module Devved2
68 changes: 68 additions & 0 deletions test/project/SubProject/Manifest.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
# This file is machine-generated - editing it directly is not advised

julia_version = "1.12.0-DEV"
manifest_format = "2.0"
project_hash = "620b9377bc807ff657e6618c8ccc24887eb40285"

[[deps.Base64]]
uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
version = "1.11.0"

[[deps.Devved]]
path = "Devved"
uuid = "cbce3a6e-7a3d-4e84-8e6d-b87208df7599"
version = "0.1.0"

[[deps.Devved2]]
path = "Devved2"
uuid = "08f74b90-50f5-462f-80b9-a72b1258a17b"
version = "0.1.0"

[[deps.InteractiveUtils]]
deps = ["Markdown"]
uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
version = "1.11.0"

[[deps.Logging]]
deps = ["StyledStrings"]
uuid = "56ddb016-857b-54e1-b83d-db4d58db5568"
version = "1.11.0"

[[deps.Markdown]]
deps = ["Base64"]
uuid = "d6f4376e-aef5-505a-96c1-9c027394607a"
version = "1.11.0"

[[deps.MyPkg]]
deps = ["Devved", "Devved2"]
path = "."
uuid = "0cafdeb2-d7a2-40d0-8d22-4411fcc2c4ee"
version = "0.0.0"

[[deps.PackageThatIsSub]]
deps = ["Devved2", "MyPkg"]
path = "PackageThatIsSub"
uuid = "1efb588c-9412-4e40-90a4-710420bd84aa"
version = "0.1.0"

[[deps.Random]]
deps = ["SHA"]
uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
version = "1.11.0"

[[deps.SHA]]
uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"
version = "0.7.0"

[[deps.Serialization]]
uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
version = "1.11.0"

[[deps.StyledStrings]]
uuid = "f489334b-da3d-4c2e-b8f0-e476e12c162b"
version = "1.11.0"

[[deps.Test]]
deps = ["InteractiveUtils", "Logging", "Random", "Serialization"]
uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
version = "1.11.0"
Loading