From 03039d3ec45e13fd03d7d50197c61fa6893b6719 Mon Sep 17 00:00:00 2001 From: Morten Piibeleht Date: Mon, 11 Sep 2023 15:47:22 +1200 Subject: [PATCH 01/14] test: fix lineendings in tests on Windows --- test/documentertools.jl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/documentertools.jl b/test/documentertools.jl index 3b44a85d..bb55b497 100644 --- a/test/documentertools.jl +++ b/test/documentertools.jl @@ -3,6 +3,8 @@ import MultiDocumenter: DocumenterTools FIXTURES = joinpath(@__DIR__, "fixtures") +normalize_newlines(s::AbstractString) = replace(s, "\r\n" => "\n") + @testset "walkdocs" begin let fileinfos = DocumenterTools.FileInfo[] rs = DocumenterTools.walkdocs(joinpath(FIXTURES, "pre")) do fileinfo @@ -123,8 +125,8 @@ end canonical = "https://example.org/this-is-test", ) DocumenterTools.walkdocs(joinpath(FIXTURES, "post")) do fileinfo - post = read(fileinfo.fullpath, String) - changed = read(joinpath(out, fileinfo.relpath), String) + post = normalize_newlines(read(fileinfo.fullpath, String)) + changed = normalize_newlines(read(joinpath(out, fileinfo.relpath), String)) if changed != post @error "update_canonical_links: change and post not matching" out fileinfo end From ec641b04deaf5c1d2d7ff915a97151b571f29984 Mon Sep 17 00:00:00 2001 From: Morten Piibeleht Date: Mon, 11 Sep 2023 19:13:27 +1200 Subject: [PATCH 02/14] improve some symlink checks --- src/MultiDocumenter.jl | 6 ++++++ src/documentertools/canonical_urls.jl | 4 +++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/MultiDocumenter.jl b/src/MultiDocumenter.jl index 73d6d2ad..6099bdcc 100644 --- a/src/MultiDocumenter.jl +++ b/src/MultiDocumenter.jl @@ -74,6 +74,12 @@ function walk_outputs(f, root, docs::Vector{MultiDocRef}, dirs::Vector{String}) p = joinpath(root, ref.path) for dir in dirs dirpath = joinpath(p, dir) + # Symlinks do not really work on Windows. There are permissions problems and we can not + # actually traverse them. + if Sys.iswindows() && islink(dirpath) + @warn "[WINDOWS] Symlink encountered at $(dirpath) -- excluding from search index, since symlinks are not properly supported on Windows." + continue + end isdir(dirpath) || continue DocumenterTools.walkdocs(dirpath, DocumenterTools.isdochtml) do fileinfo f(relpath(dirname(fileinfo.fullpath), root), fileinfo.fullpath) diff --git a/src/documentertools/canonical_urls.jl b/src/documentertools/canonical_urls.jl index ea9ca040..22289514 100644 --- a/src/documentertools/canonical_urls.jl +++ b/src/documentertools/canonical_urls.jl @@ -101,7 +101,9 @@ function update_canonical_links(docs_directory::AbstractString; canonical::Abstr # We'll skip all files. This includes files such as index.html, which in this # directory will likely be the redirect. Also, links should be pointing to other # versions, so we'll skip them too. - if !isdir(path) || islink(path) + # Note: we need to check islink() first, because on windows, calling isdir() on a + # symlink can make it throw a permissions IOError... + if islink(path) || !isdir(path) continue end # Preview directory is should contain other Documenter directories, so we just add From 3fa1bc43c1682021ef0a9ce031e0ac8f2237b41f Mon Sep 17 00:00:00 2001 From: Morten Piibeleht Date: Tue, 12 Sep 2023 11:34:02 +1200 Subject: [PATCH 03/14] add some windows handling --- src/MultiDocumenter.jl | 29 ++++++++++++++++++++++++----- test/runtests.jl | 12 ++++++++++-- 2 files changed, 34 insertions(+), 7 deletions(-) diff --git a/src/MultiDocumenter.jl b/src/MultiDocumenter.jl index 0a8d41e2..551b9c65 100644 --- a/src/MultiDocumenter.jl +++ b/src/MultiDocumenter.jl @@ -74,13 +74,9 @@ function walk_outputs(f, root, docs::Vector{MultiDocRef}, dirs::Vector{String}) p = joinpath(root, ref.path) for dir in dirs dirpath = joinpath(p, dir) - # Symlinks do not really work on Windows. There are permissions problems and we can not - # actually traverse them. - if Sys.iswindows() && islink(dirpath) - @warn "[WINDOWS] Symlink encountered at $(dirpath) -- excluding from search index, since symlinks are not properly supported on Windows." + if !_windows_symlink_wrapper(isdir, dirpath) continue end - isdir(dirpath) || continue DocumenterTools.walkdocs(dirpath, DocumenterTools.isdochtml) do fileinfo f(relpath(dirname(fileinfo.fullpath), root), fileinfo.fullpath) end @@ -487,4 +483,27 @@ function inject_styles_and_global_navigation( end end +function _windows_symlink_wrapper(f::Base.Callable, path::AbstractString) + if Sys.iswindows() && islink(path) + if isinteractive() + @warn """ + A symlink was ignored for $(f) at $(path) + This only happens when running interactive on Windows.""" + return false + else + throw(SymlinkOnWindowsError()) + end + end + return f(path) +end + +struct SymlinkOnWindowsError <: Exception end +function Base.showerror(io::IO, err::SymlinkOnWindowsError) + print(io, """ + SymlinkOnWindowsError: this builds requires symlinks, but these are not properly supported in Windows + You can still run the build interactively for debugging/testing (i.e. in the REPL), but the + build will not exactly match the full build.""") + print(io, err.msg) +end + end diff --git a/test/runtests.jl b/test/runtests.jl index 5c3b8de9..a6bd2b02 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -107,8 +107,16 @@ MultiDocumenter.make( @testset "structure" begin @test isdir(outpath, "inf") @test !isdir(outpath, "inf", "previews") - @test isdir(outpath, "inf", "stable") - @test isfile(outpath, "inf", "stable", "index.html") + if Sys.iswindows() + # On Windows, symlinks are either kept as simple files, or are in fact + # symlinks, but then you would run into permission errors with isdir(). + # So we need to have platform-specific test logic here. + path = joinpath(outpath, "inf", "stable") + @test islink(path) || isdir(path) + else + @test isdir(outpath, "inf", "stable") + @test isfile(outpath, "inf", "stable", "index.html") + end @test read(joinpath(outpath, "inf", "index.html"), String) == """ From b9dedb5d5ba7c507faa389e965327b10e4bb1d85 Mon Sep 17 00:00:00 2001 From: Morten Piibeleht Date: Tue, 12 Sep 2023 11:45:56 +1200 Subject: [PATCH 04/14] formatting --- src/MultiDocumenter.jl | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/MultiDocumenter.jl b/src/MultiDocumenter.jl index 551b9c65..fc516281 100644 --- a/src/MultiDocumenter.jl +++ b/src/MultiDocumenter.jl @@ -499,10 +499,13 @@ end struct SymlinkOnWindowsError <: Exception end function Base.showerror(io::IO, err::SymlinkOnWindowsError) - print(io, """ - SymlinkOnWindowsError: this builds requires symlinks, but these are not properly supported in Windows - You can still run the build interactively for debugging/testing (i.e. in the REPL), but the - build will not exactly match the full build.""") + print( + io, + """ +SymlinkOnWindowsError: this builds requires symlinks, but these are not properly supported in Windows + You can still run the build interactively for debugging/testing (i.e. in the REPL), but the + build will not exactly match the full build.""", + ) print(io, err.msg) end From b1d995310804919b7ef68f2d69c1cbe1f80159f2 Mon Sep 17 00:00:00 2001 From: Morten Piibeleht Date: Tue, 12 Sep 2023 11:46:57 +1200 Subject: [PATCH 05/14] fix test --- test/runtests.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/runtests.jl b/test/runtests.jl index a6bd2b02..c7ccb5f0 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -112,7 +112,7 @@ MultiDocumenter.make( # symlinks, but then you would run into permission errors with isdir(). # So we need to have platform-specific test logic here. path = joinpath(outpath, "inf", "stable") - @test islink(path) || isdir(path) + @test islink(path) || isfile(path) else @test isdir(outpath, "inf", "stable") @test isfile(outpath, "inf", "stable", "index.html") From 7f8380ac46820c80d26bb74a0f55a86070491841 Mon Sep 17 00:00:00 2001 From: Morten Piibeleht Date: Tue, 12 Sep 2023 11:48:23 +1200 Subject: [PATCH 06/14] fix another test --- test/runtests.jl | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index c7ccb5f0..5b0bb795 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -151,9 +151,12 @@ MultiDocumenter.make( end @testset "canonical URLs" begin - index = read(joinpath(outpath, "inf", "stable", "index.html"), String) - canonical_href = "" - @test occursin(canonical_href, index) + # We can't traverse symlinks on Windows, so we ignore this case + if !Sys.iswindows() + index = read(joinpath(outpath, "inf", "stable", "index.html"), String) + canonical_href = "" + @test occursin(canonical_href, index) + end index = read(joinpath(outpath, "inf", "v1.6.0", "index.html"), String) canonical_href = "" From 2edfed6e994620086ba204a9713e33fded19ae04 Mon Sep 17 00:00:00 2001 From: Morten Piibeleht Date: Tue, 12 Sep 2023 11:49:55 +1200 Subject: [PATCH 07/14] ... --- test/runtests.jl | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index 5b0bb795..ca21f771 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -169,8 +169,12 @@ MultiDocumenter.make( @test !isempty(store_content) @test occursin("Infiltrator.jl", store_content) @test occursin("@infiltrate", store_content) - @test occursin("$(rootpath)inf/stable/", store_content) - @test occursin("$(rootpath)inf/stable/", store_content) + # We can't traverse symlinks on Windows, so stable/ things do not get + # written into the search index. + if !Sys.iswindows() + @test occursin("$(rootpath)inf/stable/", store_content) + @test occursin("$(rootpath)inf/stable/", store_content) + end @test !occursin("/inf/dev/", store_content) end From 8ca1b5ce6757f550705ac7dc344442bb99ec55c2 Mon Sep 17 00:00:00 2001 From: Morten Piibeleht Date: Tue, 12 Sep 2023 12:41:06 +1200 Subject: [PATCH 08/14] wip --- src/MultiDocumenter.jl | 30 +-------------------------- src/documentertools/canonical_urls.jl | 2 +- test/runtests.jl | 10 ++++++--- 3 files changed, 9 insertions(+), 33 deletions(-) diff --git a/src/MultiDocumenter.jl b/src/MultiDocumenter.jl index fc516281..b1237155 100644 --- a/src/MultiDocumenter.jl +++ b/src/MultiDocumenter.jl @@ -74,9 +74,7 @@ function walk_outputs(f, root, docs::Vector{MultiDocRef}, dirs::Vector{String}) p = joinpath(root, ref.path) for dir in dirs dirpath = joinpath(p, dir) - if !_windows_symlink_wrapper(isdir, dirpath) - continue - end + isdir(dirpath) || continue DocumenterTools.walkdocs(dirpath, DocumenterTools.isdochtml) do fileinfo f(relpath(dirname(fileinfo.fullpath), root), fileinfo.fullpath) end @@ -483,30 +481,4 @@ function inject_styles_and_global_navigation( end end -function _windows_symlink_wrapper(f::Base.Callable, path::AbstractString) - if Sys.iswindows() && islink(path) - if isinteractive() - @warn """ - A symlink was ignored for $(f) at $(path) - This only happens when running interactive on Windows.""" - return false - else - throw(SymlinkOnWindowsError()) - end - end - return f(path) -end - -struct SymlinkOnWindowsError <: Exception end -function Base.showerror(io::IO, err::SymlinkOnWindowsError) - print( - io, - """ -SymlinkOnWindowsError: this builds requires symlinks, but these are not properly supported in Windows - You can still run the build interactively for debugging/testing (i.e. in the REPL), but the - build will not exactly match the full build.""", - ) - print(io, err.msg) -end - end diff --git a/src/documentertools/canonical_urls.jl b/src/documentertools/canonical_urls.jl index 22289514..fae84d0c 100644 --- a/src/documentertools/canonical_urls.jl +++ b/src/documentertools/canonical_urls.jl @@ -102,7 +102,7 @@ function update_canonical_links(docs_directory::AbstractString; canonical::Abstr # directory will likely be the redirect. Also, links should be pointing to other # versions, so we'll skip them too. # Note: we need to check islink() first, because on windows, calling isdir() on a - # symlink can make it throw a permissions IOError... + # symlink can make it throw a permissions IOError. if islink(path) || !isdir(path) continue end diff --git a/test/runtests.jl b/test/runtests.jl index ca21f771..8e9479fc 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -170,12 +170,16 @@ MultiDocumenter.make( @test occursin("Infiltrator.jl", store_content) @test occursin("@infiltrate", store_content) # We can't traverse symlinks on Windows, so stable/ things do not get - # written into the search index. - if !Sys.iswindows() + # written into the search index. Instead, it looks like we write dev/ + if Sys.iswindows() + @test !occursin("$(rootpath)inf/stable/", store_content) + @test !occursin("$(rootpath)inf/stable/", store_content) + @test occursin("/inf/dev/", store_content) + else @test occursin("$(rootpath)inf/stable/", store_content) @test occursin("$(rootpath)inf/stable/", store_content) + @test !occursin("/inf/dev/", store_content) end - @test !occursin("/inf/dev/", store_content) end @testset "sitemap" begin From 22c9921f6b61e23ced7c3227363409ce0158b26a Mon Sep 17 00:00:00 2001 From: Morten Piibeleht Date: Tue, 12 Sep 2023 12:41:36 +1200 Subject: [PATCH 09/14] fix formatting --- src/MultiDocumenter.jl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/MultiDocumenter.jl b/src/MultiDocumenter.jl index b1237155..f5e9a551 100644 --- a/src/MultiDocumenter.jl +++ b/src/MultiDocumenter.jl @@ -249,7 +249,8 @@ function maybe_clone(docs::Vector{MultiDocRef}) `$(git()) clone --depth 1 $(doc.giturl) --branch $(doc.branch) --single-branch --no-tags $(doc.upstream)`, ) else - git_dir, git_worktree = abspath(joinpath(doc.upstream, ".git")), abspath(doc.upstream) + git_dir, git_worktree = + abspath(joinpath(doc.upstream, ".git")), abspath(doc.upstream) if !isdir(git_dir) @warn "Unable to update existing clone at $(doc.upstream): .git/ directory missing" continue @@ -266,7 +267,8 @@ function maybe_clone(docs::Vector{MultiDocRef}) catch e # We're only interested in catching `git` errors here isa(e, ProcessFailedException) || rethrow() - @error "Unable to update existing clone at $(doc.upstream)" exception = (e, catch_backtrace()) + @error "Unable to update existing clone at $(doc.upstream)" exception = + (e, catch_backtrace()) end end end From bb7a01e8d1d304b61050346906b45c5f4f453b78 Mon Sep 17 00:00:00 2001 From: Morten Piibeleht Date: Tue, 12 Sep 2023 12:52:57 +1200 Subject: [PATCH 10/14] disable MultiDocumenter on Windows --- src/MultiDocumenter.jl | 19 +++++++++++++++++++ test/runtests.jl | 2 ++ 2 files changed, 21 insertions(+) diff --git a/src/MultiDocumenter.jl b/src/MultiDocumenter.jl index f5e9a551..f5d69232 100644 --- a/src/MultiDocumenter.jl +++ b/src/MultiDocumenter.jl @@ -140,7 +140,26 @@ function make( canonical_domain::Union{AbstractString,Nothing} = nothing, sitemap::Bool = false, sitemap_filename::AbstractString = "sitemap.xml", + # This keyword is for internal test use only: + _override_windows_isinteractive_check::Bool = false, ) + if Sys.iswindows() && !isinteractive() + if _override_windows_isinteractive_check || isinteractive() + @warn """ + Running a MultiDocumenter build interactively in Windows. + This should only be used for development and testing, as it will lead to partial + and broken builds. See https://github.com/JuliaComputing/MultiDocumenter.jl/issues/70 + """ + else + msg = """ + MultiDocumenter deployments are disabled on Windows due to difficulties + with handling symlinks in documentation sources. + You _can_ test this build locally by running it interactively (i.e. in the REPL). + See also: https://github.com/JuliaComputing/MultiDocumenter.jl/issues/70 + """ + error(msg) + end + end if isnothing(canonical_domain) (sitemap === true) && throw(ArgumentError("When sitemap=true, canonical_domain must also be set")) diff --git a/test/runtests.jl b/test/runtests.jl index 8e9479fc..899e58be 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -100,6 +100,8 @@ MultiDocumenter.make( canonical_domain = "https://example.org/", sitemap = true, sitemap_filename = "sitemap-mydocs.xml", + # The following keyword is not standard: + _override_windows_isinteractive_check = Sys.iswindows(), ) @testset "MultiDocumenter.jl" begin From 5ae751ad32e067a5e5dbfde891f69f160986a457 Mon Sep 17 00:00:00 2001 From: Morten Piibeleht Date: Tue, 12 Sep 2023 13:01:51 +1200 Subject: [PATCH 11/14] fix test --- test/runtests.jl | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index 899e58be..846c29dd 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -132,8 +132,7 @@ MultiDocumenter.make( @testset "custom scripts" begin - index = read(joinpath(outpath, "inf", "stable", "index.html"), String) - + index = read(joinpath(outpath, "inf", "v1.6.4", "index.html"), String) @test occursin( """""", index, @@ -150,6 +149,27 @@ MultiDocumenter.make( """""", index, ) + + if !Sys.iswindows() + # Going through symlinks does not work on Windows + index = read(joinpath(outpath, "inf", "stable", "index.html"), String) + @test occursin( + """""", + index, + ) + @test occursin( + """""", + index, + ) + @test occursin( + """""", + index, + ) + @test occursin( + """""", + index, + ) + end end @testset "canonical URLs" begin From 4c99bca3e8f538993ba35697da15e0663ab9fef1 Mon Sep 17 00:00:00 2001 From: Morten Piibeleht Date: Tue, 12 Sep 2023 13:03:17 +1200 Subject: [PATCH 12/14] test that build errors on windows --- test/runtests.jl | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/test/runtests.jl b/test/runtests.jl index 846c29dd..5fc8de7d 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -84,6 +84,27 @@ docs = [ # giturl = "git@github.com:JuliaComputing/DataSets.jl.git", ), ] + +if Sys.iswindows() + @test_throws ErrorException MultiDocumenter.make( + outpath, + docs; + search_engine = MultiDocumenter.SearchConfig( + index_versions = ["stable", "dev"], + engine = MultiDocumenter.FlexSearch, + ), + custom_scripts = [ + "foo/bar.js", + "https://foo.com/bar.js", + Docs.HTML("const foo = 'bar';"), + ], + rootpath = rootpath, + canonical_domain = "https://example.org/", + sitemap = true, + sitemap_filename = "sitemap-mydocs.xml", + ) +end + MultiDocumenter.make( outpath, docs; From 2b53b59e881c04c81aefc46d8662a7c67b19f9c7 Mon Sep 17 00:00:00 2001 From: Morten Piibeleht Date: Tue, 12 Sep 2023 14:30:49 +1200 Subject: [PATCH 13/14] add comment, readme notes --- README.md | 4 ++++ test/runtests.jl | 5 ++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 0f5658f3..e175b06d 100644 --- a/README.md +++ b/README.md @@ -84,3 +84,7 @@ Check [`.github/workflows/deploy.yml`](.github/workflows/deploy.yml) and [`docs/ The result of that script is available at [https://juliacomputing.github.io/MultiDocumenter.jl/](https://juliacomputing.github.io/MultiDocumenter.jl/). You can of course also just push the output artefact directly to S3 or some other hosting service. + +> **Warning** +> MultiDocumenter sites can not be deployed on Windows right now. +> See [#70](https://github.com/JuliaComputing/MultiDocumenter.jl/issues/70). diff --git a/test/runtests.jl b/test/runtests.jl index 5fc8de7d..cfdae536 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -85,7 +85,10 @@ docs = [ ), ] -if Sys.iswindows() +# We do not support deploying docs on Windows at the moment, and MultiDocumenter +# should throw an error if it's being run on Windows (in a non-interactive session). +# See also: https://github.com/JuliaComputing/MultiDocumenter.jl/issues/70 +if Sys.iswindows() && !isinteractive() @test_throws ErrorException MultiDocumenter.make( outpath, docs; From b8d4ed228afd55b257af292e9180352339c37b8e Mon Sep 17 00:00:00 2001 From: Morten Piibeleht Date: Tue, 12 Sep 2023 14:33:26 +1200 Subject: [PATCH 14/14] expand readme note --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e175b06d..95c11d47 100644 --- a/README.md +++ b/README.md @@ -86,5 +86,7 @@ The result of that script is available at [https://juliacomputing.github.io/Mult You can of course also just push the output artefact directly to S3 or some other hosting service. > **Warning** -> MultiDocumenter sites can not be deployed on Windows right now. +> MultiDocumenter sites can not be deployed on Windows right now, and the `make()` function will throw an error. > See [#70](https://github.com/JuliaComputing/MultiDocumenter.jl/issues/70). +> +> It is still possible to develop and debug MultiDocumenter sites on Windows if the build script is run interactively (e.g. by `include`-ing it into a REPL session).