diff --git a/spec/std/dir_spec.cr b/spec/std/dir_spec.cr index 60e20cea581f..6fb95a9efcd3 100644 --- a/spec/std/dir_spec.cr +++ b/spec/std/dir_spec.cr @@ -25,6 +25,20 @@ private def unset_tempdir(&) {% end %} end +{% if flag?(:win32) %} + private def make_hidden(path) + wstr = Crystal::System.to_wstr(path) + attributes = LibC.GetFileAttributesW(wstr) + LibC.SetFileAttributesW(wstr, attributes | LibC::FILE_ATTRIBUTE_HIDDEN) + end + + private def make_system(path) + wstr = Crystal::System.to_wstr(path) + attributes = LibC.GetFileAttributesW(wstr) + LibC.SetFileAttributesW(wstr, attributes | LibC::FILE_ATTRIBUTE_SYSTEM) + end +{% end %} + private def it_raises_on_null_byte(operation, &block) it "errors on #{operation}" do expect_raises(ArgumentError, "String contains null byte") do @@ -438,26 +452,86 @@ describe "Dir" do ].sort end - context "match_hidden: true" do - it "matches hidden files" do + context "match: :dot_files / match_hidden" do + it "matches dot files" do + Dir.glob("#{datapath}/dir/dots/**/*", match: :dot_files).sort.should eq [ + datapath("dir", "dots", ".dot.hidden"), + datapath("dir", "dots", ".hidden"), + datapath("dir", "dots", ".hidden", "f1.txt"), + ].sort Dir.glob("#{datapath}/dir/dots/**/*", match_hidden: true).sort.should eq [ datapath("dir", "dots", ".dot.hidden"), datapath("dir", "dots", ".hidden"), datapath("dir", "dots", ".hidden", "f1.txt"), ].sort end - end - context "match_hidden: false" do it "ignores hidden files" do - Dir.glob("#{datapath}/dir/dots/*", match_hidden: false).size.should eq 0 + Dir.glob("#{datapath}/dir/dots/*", match: :none).should be_empty + Dir.glob("#{datapath}/dir/dots/*", match_hidden: false).should be_empty end it "ignores hidden files recursively" do - Dir.glob("#{datapath}/dir/dots/**/*", match_hidden: false).size.should eq 0 + Dir.glob("#{datapath}/dir/dots/**/*", match: :none).should be_empty + Dir.glob("#{datapath}/dir/dots/**/*", match_hidden: false).should be_empty end end + {% if flag?(:win32) %} + it "respects `NativeHidden` and `OSHidden`" do + with_tempfile("glob-system-hidden") do |path| + FileUtils.mkdir_p(path) + + visible_txt = File.join(path, "visible.txt") + hidden_txt = File.join(path, "hidden.txt") + system_txt = File.join(path, "system.txt") + system_hidden_txt = File.join(path, "system_hidden.txt") + + File.write(visible_txt, "") + File.write(hidden_txt, "") + File.write(system_txt, "") + File.write(system_hidden_txt, "") + make_hidden(hidden_txt) + make_hidden(system_hidden_txt) + make_system(system_txt) + make_system(system_hidden_txt) + + visible_dir = File.join(path, "visible_dir") + hidden_dir = File.join(path, "hidden_dir") + system_dir = File.join(path, "system_dir") + system_hidden_dir = File.join(path, "system_hidden_dir") + + Dir.mkdir(visible_dir) + Dir.mkdir(hidden_dir) + Dir.mkdir(system_dir) + Dir.mkdir(system_hidden_dir) + make_hidden(hidden_dir) + make_hidden(system_hidden_dir) + make_system(system_dir) + make_system(system_hidden_dir) + + inside_visible = File.join(visible_dir, "inside.txt") + inside_hidden = File.join(hidden_dir, "inside.txt") + inside_system = File.join(system_dir, "inside.txt") + inside_system_hidden = File.join(system_hidden_dir, "inside.txt") + + File.write(inside_visible, "") + File.write(inside_hidden, "") + File.write(inside_system, "") + File.write(inside_system_hidden, "") + + expected = [visible_txt, visible_dir, inside_visible, system_txt, system_dir, inside_system].sort! + expected_hidden = (expected + [hidden_txt, hidden_dir, inside_hidden]).sort! + expected_system_hidden = (expected_hidden + [system_hidden_txt, system_hidden_dir, inside_system_hidden]).sort! + + Dir.glob("#{path}/**/*", match: :none).sort.should eq(expected) + Dir.glob("#{path}/**/*", match: :native_hidden).sort.should eq(expected_hidden) + Dir.glob("#{path}/**/*", match: :os_hidden).sort.should eq(expected) + Dir.glob("#{path}/**/*", match: File::MatchOptions[NativeHidden, OSHidden]).sort.should eq(expected_system_hidden) + end + end + {% end %} + context "with path" do expected = [ datapath("dir", "f1.txt"), diff --git a/src/crystal/system/dir.cr b/src/crystal/system/dir.cr index eef9f330fbab..9d66a2653bc0 100644 --- a/src/crystal/system/dir.cr +++ b/src/crystal/system/dir.cr @@ -4,17 +4,19 @@ module Crystal::System::Dir # # Information about a directory entry. # - # In particular we only care about the name and whether its - # a directory or not to improve the performance of Dir.glob - # by avoid having to call File.info on every directory entry. + # In particular we only care about the name, whether it's a directory, and + # whether any hidden file attributes are set to improve the performance of + # `Dir.glob` by not having to call `File.info` on every directory entry. # If dir is nil, the type is unknown. # In the future we might change Dir's API to expose these entries # with more info but right now it's not necessary. struct Entry - getter name - getter? dir + getter name : String + getter? dir : Bool? + getter? native_hidden : Bool + getter? os_hidden : Bool - def initialize(@name : String, @dir : Bool?) + def initialize(@name, @dir, @native_hidden, @os_hidden = false) end end diff --git a/src/crystal/system/unix/dir.cr b/src/crystal/system/unix/dir.cr index e2faa53b4482..ddaeb34f9eea 100644 --- a/src/crystal/system/unix/dir.cr +++ b/src/crystal/system/unix/dir.cr @@ -19,7 +19,11 @@ module Crystal::System::Dir when LibC::DT_UNKNOWN, LibC::DT_LNK then nil else false end - Entry.new(name, dir) + + # TODO: support `st_flags & UF_HIDDEN` on BSD-like systems: https://man.freebsd.org/cgi/man.cgi?query=stat&sektion=2 + # TODO: support hidden file attributes on macOS / HFS+: https://stackoverflow.com/a/15236292 + # (are these the same?) + Entry.new(name, dir, false) elsif Errno.value != Errno::NONE raise ::File::Error.from_errno("Error reading directory entries", file: path) else diff --git a/src/crystal/system/wasi/dir.cr b/src/crystal/system/wasi/dir.cr index c44390ac5bb2..5d64a2f3aecd 100644 --- a/src/crystal/system/wasi/dir.cr +++ b/src/crystal/system/wasi/dir.cr @@ -56,7 +56,7 @@ module Crystal::System::Dir else false end - Entry.new(name, is_dir) + Entry.new(name, is_dir, false) end def self.rewind(dir) : Nil diff --git a/src/crystal/system/win32/dir.cr b/src/crystal/system/win32/dir.cr index 7e426be33207..1c3502efc269 100644 --- a/src/crystal/system/win32/dir.cr +++ b/src/crystal/system/win32/dir.cr @@ -55,9 +55,11 @@ module Crystal::System::Dir def self.data_to_entry(data) name = String.from_utf16(data.cFileName.to_unsafe)[0] unless data.dwFileAttributes.bits_set?(LibC::FILE_ATTRIBUTE_REPARSE_POINT) && data.dwReserved0 == LibC::IO_REPARSE_TAG_SYMLINK - dir = (data.dwFileAttributes & LibC::FILE_ATTRIBUTE_DIRECTORY) != 0 + dir = data.dwFileAttributes.bits_set?(LibC::FILE_ATTRIBUTE_DIRECTORY) end - Entry.new(name, dir) + native_hidden = data.dwFileAttributes.bits_set?(LibC::FILE_ATTRIBUTE_HIDDEN) + os_hidden = native_hidden && data.dwFileAttributes.bits_set?(LibC::FILE_ATTRIBUTE_SYSTEM) + Entry.new(name, dir, native_hidden, os_hidden) end def self.rewind(dir : DirHandle) : Nil diff --git a/src/dir/glob.cr b/src/dir/glob.cr index 088480873879..69efb55cc9ee 100644 --- a/src/dir/glob.cr +++ b/src/dir/glob.cr @@ -4,30 +4,76 @@ class Dir # The pattern syntax is similar to shell filename globbing, see `File.match?` for details. # # NOTE: Path separator in patterns needs to be always `/`. The returned file names use system-specific path separators. - def self.[](*patterns : Path | String, match_hidden = false, follow_symlinks = false) : Array(String) - glob(patterns, match_hidden: match_hidden, follow_symlinks: follow_symlinks) + def self.[](*patterns : Path | String, match : File::MatchOptions = File::MatchOptions.glob_default, follow_symlinks : Bool = false) : Array(String) + glob(patterns, match: match, follow_symlinks: follow_symlinks) end # :ditto: - def self.[](patterns : Enumerable, match_hidden = false, follow_symlinks = false) : Array(String) - glob(patterns, match_hidden: match_hidden, follow_symlinks: follow_symlinks) + def self.[](patterns : Enumerable, match : File::MatchOptions = File::MatchOptions.glob_default, follow_symlinks : Bool = false) : Array(String) + glob(patterns, match: match, follow_symlinks: follow_symlinks) + end + + # :ditto: + # + # For compatibility, a falsey *match_hidden* argument is equivalent to passing + # `match: File::MatchOptions.glob_default`, and a truthy *match_hidden* is + # equivalent to + # `match: File::MatchOptions.glob_default | File::MatchOptions::DotFiles`. + @[Deprecated("Use the overload with a `match` parameter instead")] + def self.[](*patterns : Path | String, match_hidden, follow_symlinks = false) : Array(String) + glob(patterns, match: match_hidden_to_options(match_hidden), follow_symlinks: follow_symlinks) + end + + # :ditto: + # + # For compatibility, a falsey *match_hidden* argument is equivalent to passing + # `match: File::MatchOptions.glob_default`, and a truthy *match_hidden* is + # equivalent to + # `match: File::MatchOptions.glob_default | File::MatchOptions::DotFiles`. + @[Deprecated("Use the overload with a `match` parameter instead")] + def self.[](patterns : Enumerable, match_hidden, follow_symlinks = false) : Array(String) + glob(patterns, match: match_hidden_to_options(match_hidden), follow_symlinks: follow_symlinks) end # Returns an array of all files that match against any of *patterns*. # # The pattern syntax is similar to shell filename globbing, see `File.match?` for details. # - # If *match_hidden* is `true` the pattern will match hidden files and folders. - # # NOTE: Path separator in patterns needs to be always `/`. The returned file names use system-specific path separators. - def self.glob(*patterns : Path | String, match_hidden = false, follow_symlinks = false) : Array(String) - glob(patterns, match_hidden: match_hidden, follow_symlinks: follow_symlinks) + def self.glob(*patterns : Path | String, match : File::MatchOptions = File::MatchOptions.glob_default, follow_symlinks : Bool = false) : Array(String) + glob(patterns, match: match, follow_symlinks: follow_symlinks) end # :ditto: - def self.glob(patterns : Enumerable, match_hidden = false, follow_symlinks = false) : Array(String) + def self.glob(patterns : Enumerable, match : File::MatchOptions = File::MatchOptions.glob_default, follow_symlinks : Bool = false) : Array(String) paths = [] of String - glob(patterns, match_hidden: match_hidden, follow_symlinks: follow_symlinks) do |path| + glob(patterns, match: match, follow_symlinks: follow_symlinks) do |path| + paths << path + end + paths + end + + # :ditto: + # + # For compatibility, a falsey *match_hidden* argument is equivalent to passing + # `match: File::MatchOptions.glob_default`, and a truthy *match_hidden* is + # equivalent to + # `match: File::MatchOptions.glob_default | File::MatchOptions::DotFiles`. + @[Deprecated("Use the overload with a `match` parameter instead")] + def self.glob(*patterns : Path | String, match_hidden, follow_symlinks = false) : Array(String) + glob(patterns, match: match_hidden_to_options(match_hidden), follow_symlinks: follow_symlinks) + end + + # :ditto: + # + # For compatibility, a falsey *match_hidden* argument is equivalent to passing + # `match: File::MatchOptions.glob_default`, and a truthy *match_hidden* is + # equivalent to + # `match: File::MatchOptions.glob_default | File::MatchOptions::DotFiles`. + @[Deprecated("Use the overload with a `match` parameter instead")] + def self.glob(patterns : Enumerable, match_hidden, follow_symlinks = false) : Array(String) + paths = [] of String + glob(patterns, match: match_hidden_to_options(match_hidden), follow_symlinks: follow_symlinks) do |path| paths << path end paths @@ -37,22 +83,52 @@ class Dir # # The pattern syntax is similar to shell filename globbing, see `File.match?` for details. # - # If *match_hidden* is `true` the pattern will match hidden files and folders. - # # NOTE: Path separator in patterns needs to be always `/`. The returned file names use system-specific path separators. - def self.glob(*patterns : Path | String, match_hidden = false, follow_symlinks = false, &block : String -> _) - glob(patterns, match_hidden: match_hidden, follow_symlinks: follow_symlinks) do |path| + def self.glob(*patterns : Path | String, match : File::MatchOptions = File::MatchOptions.glob_default, follow_symlinks : Bool = false, &block : String -> _) + glob(patterns, match: match, follow_symlinks: follow_symlinks) do |path| + yield path + end + end + + # :ditto: + def self.glob(patterns : Enumerable, match : File::MatchOptions = File::MatchOptions.glob_default, follow_symlinks : Bool = false, &block : String -> _) + Globber.glob(patterns, match: match, follow_symlinks: follow_symlinks) do |path| yield path end end # :ditto: - def self.glob(patterns : Enumerable, match_hidden = false, follow_symlinks = false, &block : String -> _) - Globber.glob(patterns, match_hidden: match_hidden, follow_symlinks: follow_symlinks) do |path| + # + # For compatibility, a falsey *match_hidden* argument is equivalent to passing + # `match: File::MatchOptions.glob_default`, and a truthy *match_hidden* is + # equivalent to + # `match: File::MatchOptions.glob_default | File::MatchOptions::DotFiles`. + @[Deprecated("Use the overload with a `match` parameter instead")] + def self.glob(*patterns : Path | String, match_hidden, follow_symlinks = false, &block : String -> _) + glob(patterns, match: match_hidden_to_options(match_hidden), follow_symlinks: follow_symlinks) do |path| yield path end end + # :ditto: + # + # For compatibility, a falsey *match_hidden* argument is equivalent to passing + # `match: File::MatchOptions.glob_default`, and a truthy *match_hidden* is + # equivalent to + # `match: File::MatchOptions.glob_default | File::MatchOptions::DotFiles`. + @[Deprecated("Use the overload with a `match` parameter instead")] + def self.glob(patterns : Enumerable, match_hidden, follow_symlinks = false, &block : String -> _) + Globber.glob(patterns, match: match_hidden_to_options(match_hidden), follow_symlinks: follow_symlinks) do |path| + yield path + end + end + + private def self.match_hidden_to_options(match_hidden) + options = File::MatchOptions.glob_default + options |= File::MatchOptions::DotFiles if match_hidden + options + end + # :nodoc: module Globber record DirectoriesOnly @@ -72,7 +148,7 @@ class Dir end alias PatternType = DirectoriesOnly | ConstantEntry | EntryMatch | RecursiveDirectories | ConstantDirectory | RootDirectory | DirectoryMatch - def self.glob(patterns : Enumerable, *, match_hidden, follow_symlinks, &block : String -> _) + def self.glob(patterns : Enumerable, *, match, follow_symlinks, &block : String -> _) patterns.each do |pattern| if pattern.is_a?(Path) pattern = pattern.to_posix.to_s @@ -81,11 +157,11 @@ class Dir sequences.each do |sequence| if sequence.count(&.is_a?(RecursiveDirectories)) > 1 - run_tracking(sequence, match_hidden: match_hidden, follow_symlinks: follow_symlinks) do |match| + run_tracking(sequence, match: match, follow_symlinks: follow_symlinks) do |match| yield match end else - run(sequence, match_hidden: match_hidden, follow_symlinks: follow_symlinks) do |match| + run(sequence, match: match, follow_symlinks: follow_symlinks) do |match| yield match end end @@ -153,17 +229,17 @@ class Dir true end - private def self.run_tracking(sequence, match_hidden, follow_symlinks, &block : String -> _) + private def self.run_tracking(sequence, match, follow_symlinks, &block : String -> _) result_tracker = Set(String).new - run(sequence, match_hidden, follow_symlinks) do |result| + run(sequence, match, follow_symlinks) do |result| if result_tracker.add?(result) yield result end end end - private def self.run(sequence, match_hidden, follow_symlinks, &block : String -> _) + private def self.run(sequence, match, follow_symlinks, &block : String -> _) return if sequence.empty? path_stack = [] of Tuple(Int32, String?, Crystal::System::Dir::Entry?) @@ -195,7 +271,7 @@ class Dir in EntryMatch next if sequence[pos + 1]?.is_a?(RecursiveDirectories) each_child(path) do |entry| - next if !match_hidden && entry.name.starts_with?('.') + next unless matches_file?(entry, match) yield join(path, entry.name) if cmd.matches?(entry.name) end in DirectoryMatch @@ -255,7 +331,7 @@ class Dir if entry = read_entry(dir) next if entry.name.in?(".", "..") - next if !match_hidden && entry.name.starts_with?('.') + next unless matches_file?(entry, match) if dir_path.bytesize == 0 fullpath = entry.name @@ -340,5 +416,12 @@ class Dir # call File.info? which is really expensive. Crystal::System::Dir.next_entry(dir.@dir, dir.path) end + + private def self.matches_file?(entry, match) + return false if entry.name.starts_with?('.') && !match.dot_files? + return false if entry.native_hidden? && !match.native_hidden? + return false if entry.os_hidden? && !match.os_hidden? + true + end end end diff --git a/src/file.cr b/src/file.cr index 6875f51397b2..7092abb13648 100644 --- a/src/file.cr +++ b/src/file.cr @@ -85,6 +85,43 @@ class File < IO::FileDescriptor "/dev/null" {% end %} + # Options used to control the behavior of `Dir.glob`. + @[Flags] + enum MatchOptions + # Includes files whose name begins with a period (`.`). + DotFiles + + # Includes files which have a hidden attribute backed by the native + # filesystem. + # + # On Windows, this matches files that have the NTFS hidden attribute set. + # This option alone doesn't match files with _both_ the hidden and the + # system attributes, `OSHidden` must also be used. + # + # On other systems, this has no effect. + NativeHidden + + # Includes files which are considered hidden by operating system + # conventions (apart from `DotFiles`), but not by the filesystem. + # + # On Windows, this option alone has no effect. However, combining it with + # `NativeHidden` matches files that have both the NTFS hidden and system + # attributes set. Note that files with just the system attribute, but not + # the hidden attribute, are always matched regardless of this option or + # `NativeHidden`. + # + # On other systems, this has no effect. + OSHidden + + # Returns a suitable platform-specific default set of options for + # `Dir.glob` and `Dir.[]`. + # + # Currently this is always `NativeHidden | OSHidden`. + def self.glob_default + NativeHidden | OSHidden + end + end + include Crystal::System::File # This constructor is provided for subclasses to be able to initialize an diff --git a/src/lib_c/x86_64-windows-msvc/c/winnt.cr b/src/lib_c/x86_64-windows-msvc/c/winnt.cr index 0c7ceae716f3..9496f051a6a4 100644 --- a/src/lib_c/x86_64-windows-msvc/c/winnt.cr +++ b/src/lib_c/x86_64-windows-msvc/c/winnt.cr @@ -16,8 +16,10 @@ lib LibC INVALID_FILE_ATTRIBUTES = DWORD.new!(-1) FILE_ATTRIBUTE_DIRECTORY = 0x10 + FILE_ATTRIBUTE_HIDDEN = 0x2 FILE_ATTRIBUTE_READONLY = 0x1 FILE_ATTRIBUTE_REPARSE_POINT = 0x400 + FILE_ATTRIBUTE_SYSTEM = 0x4 FILE_APPEND_DATA = 0x00000004