Merge pull request #180 from MichaelHatherly/mh/walkdir

Add `walkdir` from Base.

README.md

@@ -107,6 +107,8 @@ Currently, the `@compat` macro supports the following syntaxes:
 
 * `foreach`, similar to `map` but when the return value is not needed ([#13744](https://github.com/JuliaLang/julia/pull/13774)).
 
+* `walkdir`, returns an iterator that walks the directory tree of a directory. ([#13707](https://github.com/JuliaLang/julia/pull/13707))
+
 ## Renamed functions
 
 * `itrunc`, `iround`, `iceil`, `ifloor` are now accessed via `trunc(T, x)`, etc. ([#9133](https://github.com/JuliaLang/julia/pull/9133)).  Truncated conversions between integer types are now `n % T` ([#8646](https://github.com/JuliaLang/julia/issues/8646)).

src/Compat.jl

@@ -216,6 +216,49 @@ elseif VERSION < v"0.4.0-dev+6987"
     export pipeline
 end
 
+if VERSION < v"0.5.0-dev+961"
+    export walkdir
+
+    function walkdir(root; topdown=true, follow_symlinks=false, onerror=throw)
+        content = nothing
+        try
+            content = readdir(root)
+        catch err
+            isa(err, SystemError) || throw(err)
+            onerror(err)
+            #Need to return an empty task to skip the current root folder
+            return Task(()->())
+        end
+        dirs = Array(eltype(content), 0)
+        files = Array(eltype(content), 0)
+        for name in content
+            if isdir(joinpath(root, name))
+                push!(dirs, name)
+            else
+                push!(files, name)
+            end
+        end
+
+        function _it()
+            if topdown
+                produce(root, dirs, files)
+            end
+            for dir in dirs
+                path = joinpath(root,dir)
+                if follow_symlinks || !islink(path)
+                    for (root_l, dirs_l, files_l) in walkdir(path, topdown=topdown, follow_symlinks=follow_symlinks, onerror=onerror)
+                        produce(root_l, dirs_l, files_l)
+                    end
+                end
+            end
+            if !topdown
+                produce(root, dirs, files)
+            end
+        end
+        Task(_it)
+    end
+end
+
 function rewrite_dict(ex)
     length(ex.args) == 1 && return ex
 

test/runtests.jl

@@ -893,3 +893,102 @@ if VERSION >= v"0.4"
 end
 
 end
+
+# walkdir
+
+dirwalk = mktempdir()
+cd(dirwalk) do
+    for i=1:2
+        mkdir("sub_dir$i")
+        open("file$i", "w") do f end
+
+        mkdir(joinpath("sub_dir1", "subsub_dir$i"))
+        touch(joinpath("sub_dir1", "file$i"))
+    end
+    touch(joinpath("sub_dir2", "file_dir2"))
+    has_symlinks = @unix? true : (isdefined(Base, :WINDOWS_VISTA_VER) && Base.windows_version() >= Base.WINDOWS_VISTA_VER)
+    follow_symlink_vec = has_symlinks ? [true, false] : [false]
+    has_symlinks && symlink(abspath("sub_dir2"), joinpath("sub_dir1", "link"))
+    for follow_symlinks in follow_symlink_vec
+        task = walkdir(".", follow_symlinks=follow_symlinks)
+        root, dirs, files = consume(task)
+        @test root == "."
+        @test dirs == ["sub_dir1", "sub_dir2"]
+        @test files == ["file1", "file2"]
+
+        root, dirs, files = consume(task)
+        @test root == joinpath(".", "sub_dir1")
+        @test dirs == (has_symlinks ? ["link", "subsub_dir1", "subsub_dir2"] : ["subsub_dir1", "subsub_dir2"])
+        @test files == ["file1", "file2"]
+
+        root, dirs, files = consume(task)
+        if follow_symlinks
+            @test root == joinpath(".", "sub_dir1", "link")
+            @test dirs == []
+            @test files == ["file_dir2"]
+            root, dirs, files = consume(task)
+        end
+        for i=1:2
+            @test root == joinpath(".", "sub_dir1", "subsub_dir$i")
+            @test dirs == []
+            @test files == []
+            root, dirs, files = consume(task)
+        end
+
+        @test root == joinpath(".", "sub_dir2")
+        @test dirs == []
+        @test files == ["file_dir2"]
+    end
+
+    for follow_symlinks in follow_symlink_vec
+        task = walkdir(".", follow_symlinks=follow_symlinks, topdown=false)
+        root, dirs, files = consume(task)
+        if follow_symlinks
+            @test root == joinpath(".", "sub_dir1", "link")
+            @test dirs == []
+            @test files == ["file_dir2"]
+            root, dirs, files = consume(task)
+        end
+        for i=1:2
+            @test root == joinpath(".", "sub_dir1", "subsub_dir$i")
+            @test dirs == []
+            @test files == []
+            root, dirs, files = consume(task)
+        end
+        @test root == joinpath(".", "sub_dir1")
+        @test dirs ==  (has_symlinks ? ["link", "subsub_dir1", "subsub_dir2"] : ["subsub_dir1", "subsub_dir2"])
+        @test files == ["file1", "file2"]
+
+        root, dirs, files = consume(task)
+        @test root == joinpath(".", "sub_dir2")
+        @test dirs == []
+        @test files == ["file_dir2"]
+
+        root, dirs, files = consume(task)
+        @test root == "."
+        @test dirs == ["sub_dir1", "sub_dir2"]
+        @test files == ["file1", "file2"]
+    end
+    #test of error handling
+    task_error = walkdir(".")
+    task_noerror = walkdir(".", onerror=x->x)
+    root, dirs, files = consume(task_error)
+    @test root == "."
+    @test dirs == ["sub_dir1", "sub_dir2"]
+    @test files == ["file1", "file2"]
+
+    rm(joinpath("sub_dir1"), recursive=true)
+    @test_throws SystemError consume(task_error) # throws an error because sub_dir1 do not exist
+
+    root, dirs, files = consume(task_noerror)
+    @test root == "."
+    @test dirs == ["sub_dir1", "sub_dir2"]
+    @test files == ["file1", "file2"]
+
+    root, dirs, files = consume(task_noerror) # skips sub_dir1 as it no longer exist
+    @test root == joinpath(".", "sub_dir2")
+    @test dirs == []
+    @test files == ["file_dir2"]
+
+end
+rm(dirwalk, recursive=true)