From fe5dc952db472d8709e87d3501737481e86b53cc Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Fri, 1 Dec 2023 08:52:19 -0500 Subject: [PATCH 1/4] use .julia/scratchspaces instead --- Project.toml | 2 ++ docs/src/z10-for-end-users.md | 6 +++--- src/DataDeps.jl | 1 + src/locations.jl | 12 +++++++++++- 4 files changed, 17 insertions(+), 4 deletions(-) diff --git a/Project.toml b/Project.toml index 841d305..08f6ad3 100644 --- a/Project.toml +++ b/Project.toml @@ -8,12 +8,14 @@ HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3" Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" Reexport = "189a3867-3050-52da-a836-e630ba90ab69" SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce" +Scratch = "6c6a2e73-6563-6170-7368-637461726353" p7zip_jll = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" [compat] ExpectationStubs = "0.2" HTTP = "1" Reexport = "0.2, 1.0" +Scratch = "1" julia = "1.6" p7zip_jll = "16.2.0, 17" diff --git a/docs/src/z10-for-end-users.md b/docs/src/z10-for-end-users.md index 3fa7d42..bb40e04 100644 --- a/docs/src/z10-for-end-users.md +++ b/docs/src/z10-for-end-users.md @@ -7,8 +7,8 @@ Moving data is a great idea. DataDeps.jl is in favour of moving data. When data is automatically downloaded it will almost always go to the same location: the first (existent, writable) directory on your `DATADEPS_LOAD_PATH`. -Which by-default is `~/.julia/datadeps/`. -(If you delete this, it will go to another location). +Which by-default is DataDeps's scratch space under `~/.julia/scratchspaces/124859b0-ceae-595e-8997-d05f6a7a8dfe/datadeps`, +such that `Pkg.gc()` can automatically delete data if DataDeps is uninstalled. But you can move them from there to anywhere in the `DATADEPS_LOAD_PATH`. (See below) If you have a large chunk of data that everyone in your lab is using (e.g. a 1TB video corpora), @@ -47,7 +47,7 @@ You can (and should when desired) move things around between any folder in the l For the user **oxinabox** ```bash -/home/wheel/oxinabox/.julia/datadeps +/home/wheel/oxinabox/.julia/scratchspaces/124859b0-ceae-595e-8997-d05f6a7a8dfe/datadeps /home/wheel/oxinabox/datadeps /scratch/datadeps /staging/datadeps diff --git a/src/DataDeps.jl b/src/DataDeps.jl index 8ec41fd..d312251 100644 --- a/src/DataDeps.jl +++ b/src/DataDeps.jl @@ -4,6 +4,7 @@ using p7zip_jll using HTTP using Reexport +using Scratch @reexport using SHA export DataDep, ManualDataDep diff --git a/src/locations.jl b/src/locations.jl index 5a010ec..6be317f 100644 --- a/src/locations.jl +++ b/src/locations.jl @@ -2,7 +2,7 @@ ## Core path determining stuff const standard_loadpath = joinpath.([ - Base.DEPOT_PATH; homedir(); # Common all systems + homedir(); # Common all systems @static if Sys.iswindows() vcat(get.(Ref(ENV), @@ -15,6 +15,16 @@ const standard_loadpath = joinpath.([ "/usr/share", "/usr/local/share"] # Unix Filestructure end], "datadeps") +if VERSION > v"1.6.0-0" + # Scratch.jl works correctly with Julia 1.0 and above. + # However, Pkg's built-in garbage collection, i.e. Pkg.gc(), is only aware of + # scratchspaces for Julia 1.6 and above so use the more user accessible DEPOT/datadeps location + # for visibility to the user for space management + pushfirst!(standard_loadpath, @get_scratch!("datadeps")) +else + pushfirst!(standard_loadpath, joinpath(Base.DEPOT_PATH, "datadeps")) +end + # ensure at least something in the loadpath exists when instaleld mkpath(first(standard_loadpath)) From ee16c3dd113ce13f0178c06d60f580f3a610a156 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Fri, 1 Dec 2023 16:38:27 -0500 Subject: [PATCH 2/4] set during init --- src/DataDeps.jl | 6 ++++++ src/locations.jl | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/DataDeps.jl b/src/DataDeps.jl index d312251..c9a2f1d 100644 --- a/src/DataDeps.jl +++ b/src/DataDeps.jl @@ -30,6 +30,12 @@ include("preupload.jl") include("fetch_helpers.jl") include("post_fetch_helpers.jl") +# populated by __init__() +datadeps_scratch_dir = "" + +function __init__() + global datadeps_scratch_dir = @get_scratch!("datadeps") +end function _precompile_() ccall(:jl_generating_output, Cint, ()) == 1 || return nothing diff --git a/src/locations.jl b/src/locations.jl index 6be317f..a02f711 100644 --- a/src/locations.jl +++ b/src/locations.jl @@ -20,7 +20,7 @@ if VERSION > v"1.6.0-0" # However, Pkg's built-in garbage collection, i.e. Pkg.gc(), is only aware of # scratchspaces for Julia 1.6 and above so use the more user accessible DEPOT/datadeps location # for visibility to the user for space management - pushfirst!(standard_loadpath, @get_scratch!("datadeps")) + pushfirst!(standard_loadpath, datadeps_scratch_dir) else pushfirst!(standard_loadpath, joinpath(Base.DEPOT_PATH, "datadeps")) end From d114733314b66f3a044f90b6d0e1802c7ce9e785 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Wed, 13 Dec 2023 15:03:09 -0500 Subject: [PATCH 3/4] Update src/locations.jl --- src/locations.jl | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/locations.jl b/src/locations.jl index a02f711..9c2a22b 100644 --- a/src/locations.jl +++ b/src/locations.jl @@ -15,15 +15,7 @@ const standard_loadpath = joinpath.([ "/usr/share", "/usr/local/share"] # Unix Filestructure end], "datadeps") -if VERSION > v"1.6.0-0" - # Scratch.jl works correctly with Julia 1.0 and above. - # However, Pkg's built-in garbage collection, i.e. Pkg.gc(), is only aware of - # scratchspaces for Julia 1.6 and above so use the more user accessible DEPOT/datadeps location - # for visibility to the user for space management - pushfirst!(standard_loadpath, datadeps_scratch_dir) -else - pushfirst!(standard_loadpath, joinpath(Base.DEPOT_PATH, "datadeps")) -end +pushfirst!(standard_loadpath, joinpath(Base.DEPOT_PATH, "datadeps")) # ensure at least something in the loadpath exists when instaleld mkpath(first(standard_loadpath)) From 3c28d82ddedf98932d3cef95dd5df0b0e423bce0 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Wed, 13 Dec 2023 15:06:07 -0500 Subject: [PATCH 4/4] pushfirst to standard_loadpath during init --- src/DataDeps.jl | 1 + src/locations.jl | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/DataDeps.jl b/src/DataDeps.jl index c9a2f1d..aa3d495 100644 --- a/src/DataDeps.jl +++ b/src/DataDeps.jl @@ -35,6 +35,7 @@ datadeps_scratch_dir = "" function __init__() global datadeps_scratch_dir = @get_scratch!("datadeps") + pushfirst!(standard_loadpath, datadeps_scratch_dir) end function _precompile_() diff --git a/src/locations.jl b/src/locations.jl index 9c2a22b..22e5111 100644 --- a/src/locations.jl +++ b/src/locations.jl @@ -15,7 +15,8 @@ const standard_loadpath = joinpath.([ "/usr/share", "/usr/local/share"] # Unix Filestructure end], "datadeps") -pushfirst!(standard_loadpath, joinpath(Base.DEPOT_PATH, "datadeps")) +# NOTE: the scratchspace is pushed to the front during __init__() + # ensure at least something in the loadpath exists when instaleld mkpath(first(standard_loadpath))