From 8a509b667f65a3a1a4b9d95afcc377c4db5b7f63 Mon Sep 17 00:00:00 2001 From: Quinton Miller Date: Sun, 26 Mar 2023 18:21:30 +0800 Subject: [PATCH] Implement `File.readlink` on Windows (#13195) --- spec/std/dir_spec.cr | 31 +++-- spec/std/file_spec.cr | 46 +++++--- spec/std/file_utils_spec.cr | 2 +- src/crystal/system/win32/dir.cr | 4 +- src/crystal/system/win32/file.cr | 122 ++++++++++++++++---- src/crystal/system/win32/file_info.cr | 2 +- src/file.cr | 5 +- src/lib_c/x86_64-windows-msvc/c/errno.cr | 1 + src/lib_c/x86_64-windows-msvc/c/ioapiset.cr | 11 ++ src/lib_c/x86_64-windows-msvc/c/ntifs.cr | 37 ++++++ src/lib_c/x86_64-windows-msvc/c/winioctl.cr | 4 + src/lib_c/x86_64-windows-msvc/c/winnt.cr | 5 + 12 files changed, 217 insertions(+), 53 deletions(-) create mode 100644 src/lib_c/x86_64-windows-msvc/c/ntifs.cr create mode 100644 src/lib_c/x86_64-windows-msvc/c/winioctl.cr diff --git a/spec/std/dir_spec.cr b/spec/std/dir_spec.cr index 47d95806e72a..d7bfa6cdb19f 100644 --- a/spec/std/dir_spec.cr +++ b/spec/std/dir_spec.cr @@ -350,7 +350,7 @@ describe "Dir" do ].sort end - pending_win32 "matches symlinks" do + it "matches symlinks" do link = datapath("f1_link.txt") non_link = datapath("non_link.txt") @@ -369,19 +369,30 @@ describe "Dir" do end end - pending_win32 "matches symlink dir" do + it "matches symlink dir" do with_tempfile "symlink_dir" do |path| - Dir.mkdir_p(Path[path, "glob"]) target = Path[path, "target"] - Dir.mkdir_p(target) + non_link = target / "a.txt" + link_dir = Path[path, "glob", "dir"] - File.write(target / "a.txt", "") - File.symlink(target, Path[path, "glob", "dir"]) + Dir.mkdir_p(Path[path, "glob"]) + Dir.mkdir_p(target) - Dir.glob("#{path}/glob/*/a.txt").sort.should eq [] of String - Dir.glob("#{path}/glob/*/a.txt", follow_symlinks: true).sort.should eq [ - "#{path}/glob/dir/a.txt", - ] + File.write(non_link, "") + File.symlink(target, link_dir) + + begin + Dir.glob("#{path}/glob/*/a.txt").sort.should eq [] of String + Dir.glob("#{path}/glob/*/a.txt", follow_symlinks: true).sort.should eq [ + File.join(path, "glob", "dir", "a.txt"), + ] + ensure + # FIXME: `with_tempfile` will delete this symlink directory using + # `File.delete` otherwise, see #13194 + {% if flag?(:win32) %} + Dir.delete(link_dir) + {% end %} + end end end diff --git a/spec/std/file_spec.cr b/spec/std/file_spec.cr index 538fa6a165a2..a0cafb5cad39 100644 --- a/spec/std/file_spec.cr +++ b/spec/std/file_spec.cr @@ -226,7 +226,7 @@ describe "File" do it "creates a symbolic link" do in_path = datapath("test_file.txt") with_tempfile("test_file_link.txt") do |out_path| - File.symlink(File.real_path(in_path), out_path) + File.symlink(File.realpath(in_path), out_path) File.symlink?(out_path).should be_true File.same?(in_path, out_path, follow_symlinks: true).should be_true end @@ -234,11 +234,6 @@ describe "File" do end describe "symlink?" do - # TODO: this fails depending on how Git checks out the repository - pending_win32 "gives true" do - File.symlink?(datapath("symlink.txt")).should be_true - end - it "gives false" do File.symlink?(datapath("test_file.txt")).should be_false File.symlink?(datapath("unknown_file.txt")).should be_false @@ -254,7 +249,7 @@ describe "File" do end describe ".readlink" do - pending_win32 "reads link" do + it "reads link" do File.readlink(datapath("symlink.txt")).should eq "test_file.txt" end end @@ -393,17 +388,17 @@ describe "File" do end end - # See TODO in win32 Crystal::System::File.chmod - pending_win32 "follows symlinks" do + it "follows symlinks" do with_tempfile("chmod-destination.txt", "chmod-source.txt") do |source_path, target_path| File.write(source_path, "") - File.symlink(File.real_path(source_path), target_path) + File.symlink(File.realpath(source_path), target_path) File.symlink?(target_path).should be_true + File.chmod(source_path, 0o664) File.chmod(target_path, 0o444) - File.info(target_path).permissions.should eq(normalize_permissions(0o444, directory: false)) + File.info(source_path).permissions.should eq(normalize_permissions(0o444, directory: false)) end end @@ -431,10 +426,15 @@ describe "File" do info.type.should eq(File::Type::CharacterDevice) end - # TODO: this fails depending on how Git checks out the repository - pending_win32 "gets for a symlink" do - info = File.info(datapath("symlink.txt"), follow_symlinks: false) - info.type.should eq(File::Type::Symlink) + it "gets for a symlink" do + file_path = File.expand_path(datapath("test_file.txt")) + with_tempfile("symlink.txt") do |symlink_path| + File.symlink(file_path, symlink_path) + info = File.info(symlink_path, follow_symlinks: false) + info.type.should eq(File::Type::Symlink) + info = File.info(symlink_path, follow_symlinks: true) + info.type.should_not eq(File::Type::Symlink) + end end it "gets for open file" do @@ -603,8 +603,7 @@ describe "File" do end end - # TODO: see Crystal::System::File.realpath TODO - pending_win32 "expands paths of symlinks" do + it "expands paths of symlinks" do file_path = File.expand_path(datapath("test_file.txt")) with_tempfile("symlink.txt") do |symlink_path| File.symlink(file_path, symlink_path) @@ -613,6 +612,19 @@ describe "File" do real_symlink_path.should eq(real_file_path) end end + + it "expands multiple layers of symlinks" do + file_path = File.expand_path(datapath("test_file.txt")) + with_tempfile("symlink1.txt") do |symlink_path1| + with_tempfile("symlink2.txt") do |symlink_path2| + File.symlink(file_path, symlink_path1) + File.symlink(symlink_path1, symlink_path2) + real_symlink_path = File.realpath(symlink_path2) + real_file_path = File.realpath(file_path) + real_symlink_path.should eq(real_file_path) + end + end + end end describe "write" do diff --git a/spec/std/file_utils_spec.cr b/spec/std/file_utils_spec.cr index 54dd0bf4de7e..0d536fc7a97a 100644 --- a/spec/std/file_utils_spec.cr +++ b/spec/std/file_utils_spec.cr @@ -670,7 +670,7 @@ describe "FileUtils" do end end - pending_win32 "works with a nonexistent source" do + it "works with a nonexistent source" do with_tempfile("ln_s_src_missing", "ln_s_dst_missing") do |path1, path2| test_with_string_and_path(path1, path2) do |arg1, arg2| FileUtils.ln_s(arg1, arg2) diff --git a/src/crystal/system/win32/dir.cr b/src/crystal/system/win32/dir.cr index 80189b9f030b..fb35e9dafdd0 100644 --- a/src/crystal/system/win32/dir.cr +++ b/src/crystal/system/win32/dir.cr @@ -53,7 +53,9 @@ module Crystal::System::Dir def self.data_to_entry(data) name = String.from_utf16(data.cFileName.to_unsafe)[0] - dir = (data.dwFileAttributes & LibC::FILE_ATTRIBUTE_DIRECTORY) != 0 + unless data.dwFileAttributes.bits_set?(LibC::FILE_ATTRIBUTE_REPARSE_POINT) && data.dwReserved0 == LibC::IO_REPARSE_TAG_SYMLINK + dir = (data.dwFileAttributes & LibC::FILE_ATTRIBUTE_DIRECTORY) != 0 + end Entry.new(name, dir) end diff --git a/src/crystal/system/win32/file.cr b/src/crystal/system/win32/file.cr index 5c5842fcdb34..095b5fc45924 100644 --- a/src/crystal/system/win32/file.cr +++ b/src/crystal/system/win32/file.cr @@ -5,6 +5,8 @@ require "c/sys/utime" require "c/sys/stat" require "c/winbase" require "c/handleapi" +require "c/ntifs" +require "c/winioctl" module Crystal::System::File def self.open(filename : String, mode : String, perm : Int32 | ::File::Permissions) : LibC::Int @@ -113,8 +115,6 @@ module Crystal::System::File WinError::ERROR_INVALID_NAME, } - REPARSE_TAG_NAME_SURROGATE_MASK = 1 << 29 - private def self.check_not_found_error(message, path) error = WinError.value if NOT_FOUND_ERRORS.includes? error @@ -146,7 +146,7 @@ module Crystal::System::File raise RuntimeError.from_winerror("FindClose") end - if find_data.dwReserved0.bits_set? REPARSE_TAG_NAME_SURROGATE_MASK + if find_data.dwReserved0 == LibC::IO_REPARSE_TAG_SYMLINK return ::File::Info.new(find_data) end end @@ -179,7 +179,10 @@ module Crystal::System::File info?(path, follow_symlinks) || raise ::File::Error.from_winerror("Unable to get file info", file: path) end - def self.exists?(path) + def self.exists?(path, *, follow_symlinks = true) + if follow_symlinks + path = realpath?(path) || return false + end accessible?(path, 0) end @@ -210,7 +213,11 @@ module Crystal::System::File def self.chmod(path : String, mode : Int32 | ::File::Permissions) : Nil mode = ::File::Permissions.new(mode) unless mode.is_a? ::File::Permissions - # TODO: dereference symlinks + unless exists?(path, follow_symlinks: false) + raise ::File::Error.from_os_error("Error changing permissions", Errno::ENOENT, file: path) + end + + path = realpath(path) attributes = LibC.GetFileAttributesW(System.to_wstr(path)) if attributes == LibC::INVALID_FILE_ATTRIBUTES @@ -245,26 +252,37 @@ module Crystal::System::File end end - def self.realpath(path : String) : String - # TODO: read links using https://msdn.microsoft.com/en-us/library/windows/desktop/aa364571(v=vs.85).aspx - win_path = System.to_wstr(path) - - realpath = System.retry_wstr_buffer do |buffer, small_buf| - len = LibC.GetFullPathNameW(win_path, buffer.size, buffer, nil) - if 0 < len < buffer.size - break String.from_utf16(buffer[0, len]) - elsif small_buf && len > 0 - next len - else - raise ::File::Error.from_winerror("Error resolving real path", file: path) + private REALPATH_SYMLINK_LIMIT = 100 + + private def self.realpath?(path : String) : String? + REALPATH_SYMLINK_LIMIT.times do + win_path = System.to_wstr(path) + + realpath = System.retry_wstr_buffer do |buffer, small_buf| + len = LibC.GetFullPathNameW(win_path, buffer.size, buffer, nil) + if 0 < len < buffer.size + break String.from_utf16(buffer[0, len]) + elsif small_buf && len > 0 + next len + else + raise ::File::Error.from_winerror("Error resolving real path", file: path) + end + end + + if symlink_info = symlink_info?(realpath) + new_path, is_relative = symlink_info + path = is_relative ? ::File.expand_path(new_path, ::File.dirname(realpath)) : new_path + next end - end - unless exists? realpath - raise ::File::Error.from_os_error("Error resolving real path", Errno::ENOENT, file: path) + return exists?(realpath, follow_symlinks: false) ? realpath : nil end - realpath + raise ::File::Error.from_os_error("Too many symbolic links", Errno::ELOOP, file: path) + end + + def self.realpath(path : String) : String + realpath?(path) || raise ::File::Error.from_os_error("Error resolving real path", Errno::ENOENT, file: path) end def self.link(old_path : String, new_path : String) : Nil @@ -297,8 +315,68 @@ module Crystal::System::File end end + private def self.symlink_info?(path) + handle = LibC.CreateFileW( + System.to_wstr(path), + LibC::FILE_READ_ATTRIBUTES, + LibC::DEFAULT_SHARE_MODE, + nil, + LibC::OPEN_EXISTING, + LibC::FILE_FLAG_BACKUP_SEMANTICS | LibC::FILE_FLAG_OPEN_REPARSE_POINT, + LibC::HANDLE.null + ) + + return nil if handle == LibC::INVALID_HANDLE_VALUE + + begin + size = 0x40 + buf = Pointer(UInt8).malloc(size) + + while true + if LibC.DeviceIoControl(handle, LibC::FSCTL_GET_REPARSE_POINT, nil, 0, buf, size, out _, nil) != 0 + reparse_data = buf.as(LibC::REPARSE_DATA_BUFFER*) + if reparse_data.value.reparseTag == LibC::IO_REPARSE_TAG_SYMLINK + symlink_data = reparse_data.value.dummyUnionName.symbolicLinkReparseBuffer + path_buffer = reparse_data.value.dummyUnionName.symbolicLinkReparseBuffer.pathBuffer.to_unsafe.as(UInt8*) + is_relative = symlink_data.flags.bits_set?(LibC::SYMLINK_FLAG_RELATIVE) + + # the print name is not necessarily set; fall back to substitute + # name if unavailable + if (name_len = symlink_data.printNameLength) > 0 + name_ptr = path_buffer + symlink_data.printNameOffset + name = String.from_utf16(Slice.new(name_ptr, name_len).unsafe_slice_of(UInt16)) + return {name, is_relative} + end + + name_len = symlink_data.substituteNameLength + name_ptr = path_buffer + symlink_data.substituteNameOffset + name = String.from_utf16(Slice.new(name_ptr, name_len).unsafe_slice_of(UInt16)) + # remove the internal prefix for NT paths which shows up when e.g. + # creating a symbolic link with an absolute source + # TODO: support the other possible paths, for example see + # https://github.com/golang/go/blob/ab28b834c4a38bd2295ee43eca4f9e38c28d54a2/src/os/file_windows.go#L362 + if name.starts_with?(%q(\??\)) && name[5]? == ':' + name = name[4..] + end + return {name, is_relative} + else + raise ::File::Error.new("Not a symlink", file: path) + end + end + + return nil if WinError.value != WinError::ERROR_MORE_DATA || size == LibC::MAXIMUM_REPARSE_DATA_BUFFER_SIZE + size *= 2 + buf = buf.realloc(size) + end + ensure + LibC.CloseHandle(handle) + end + end + def self.readlink(path) : String - raise NotImplementedError.new("readlink") + info = symlink_info?(path) || raise ::File::Error.new("Cannot read link", file: path) + path, _is_relative = info + path end def self.rename(old_path : String, new_path : String) : ::File::Error? diff --git a/src/crystal/system/win32/file_info.cr b/src/crystal/system/win32/file_info.cr index 10a163c44644..ba5ed3007a88 100644 --- a/src/crystal/system/win32/file_info.cr +++ b/src/crystal/system/win32/file_info.cr @@ -54,7 +54,7 @@ module Crystal::System::FileInfo when LibC::FILE_TYPE_DISK # See: https://msdn.microsoft.com/en-us/library/windows/desktop/aa365511(v=vs.85).aspx if @file_attributes.dwFileAttributes.bits_set?(LibC::FILE_ATTRIBUTE_REPARSE_POINT) && - @reparse_tag.bits_set? File::REPARSE_TAG_NAME_SURROGATE_MASK + @reparse_tag == LibC::IO_REPARSE_TAG_SYMLINK ::File::Type::Symlink elsif @file_attributes.dwFileAttributes.bits_set? LibC::FILE_ATTRIBUTE_DIRECTORY ::File::Type::Directory diff --git a/src/file.cr b/src/file.cr index 3fd22017e1e0..070ccfc0445f 100644 --- a/src/file.cr +++ b/src/file.cr @@ -168,7 +168,10 @@ class File < IO::FileDescriptor Crystal::System::File.info(path.to_s, follow_symlinks) end - # Returns `true` if *path* exists else returns `false` + # Returns whether the file given by *path* exists. + # + # Symbolic links are dereferenced, posibly recursively. Returns `false` if a + # symbolic link refers to a non-existent file. # # ``` # File.delete("foo") if File.exists?("foo") diff --git a/src/lib_c/x86_64-windows-msvc/c/errno.cr b/src/lib_c/x86_64-windows-msvc/c/errno.cr index f5c795889f55..691a04fac013 100644 --- a/src/lib_c/x86_64-windows-msvc/c/errno.cr +++ b/src/lib_c/x86_64-windows-msvc/c/errno.cr @@ -46,6 +46,7 @@ lib LibC ECONNRESET = 108 EINPROGRESS = 112 EISCONN = 113 + ELOOP = 114 ENOPROTOOPT = 123 alias ErrnoT = Int diff --git a/src/lib_c/x86_64-windows-msvc/c/ioapiset.cr b/src/lib_c/x86_64-windows-msvc/c/ioapiset.cr index df42772fd20a..eaef34deaa3e 100644 --- a/src/lib_c/x86_64-windows-msvc/c/ioapiset.cr +++ b/src/lib_c/x86_64-windows-msvc/c/ioapiset.cr @@ -21,4 +21,15 @@ lib LibC fun CancelIo( hFile : HANDLE ) : BOOL + + fun DeviceIoControl( + hDevice : HANDLE, + dwIoControlCode : DWORD, + lpInBuffer : Void*, + nInBufferSize : DWORD, + lpOutBuffer : Void*, + nOutBufferSize : DWORD, + lpBytesReturned : DWORD*, + lpOverlapped : OVERLAPPED* + ) : BOOL end diff --git a/src/lib_c/x86_64-windows-msvc/c/ntifs.cr b/src/lib_c/x86_64-windows-msvc/c/ntifs.cr new file mode 100644 index 000000000000..67f51c5b388f --- /dev/null +++ b/src/lib_c/x86_64-windows-msvc/c/ntifs.cr @@ -0,0 +1,37 @@ +lib LibC + SYMLINK_FLAG_RELATIVE = 0x00000001 + + struct REPARSE_DATA_BUFFER_struct1 + substituteNameOffset : UShort + substituteNameLength : UShort + printNameOffset : UShort + printNameLength : UShort + flags : ULong + pathBuffer : WCHAR[1] + end + + struct REPARSE_DATA_BUFFER_struct2 + substituteNameOffset : UShort + substituteNameLength : UShort + printNameOffset : UShort + printNameLength : UShort + pathBuffer : WCHAR[1] + end + + struct REPARSE_DATA_BUFFER_struct3 + dataBuffer : UChar[1] + end + + union REPARSE_DATA_BUFFER_union + symbolicLinkReparseBuffer : REPARSE_DATA_BUFFER_struct1 + mountPointReparseBuffer : REPARSE_DATA_BUFFER_struct2 + genericReparseBuffer : REPARSE_DATA_BUFFER_struct3 + end + + struct REPARSE_DATA_BUFFER + reparseTag : ULong + reparseDataLength : UShort + reserved : UShort + dummyUnionName : REPARSE_DATA_BUFFER_union + end +end diff --git a/src/lib_c/x86_64-windows-msvc/c/winioctl.cr b/src/lib_c/x86_64-windows-msvc/c/winioctl.cr new file mode 100644 index 000000000000..b7917826bbe3 --- /dev/null +++ b/src/lib_c/x86_64-windows-msvc/c/winioctl.cr @@ -0,0 +1,4 @@ +lib LibC + FSCTL_SET_REPARSE_POINT = 0x000900A4 # CTL_CODE(FILE_DEVICE_FILE_SYSTEM, 41, METHOD_BUFFERED, FILE_SPECIAL_ACCESS) + FSCTL_GET_REPARSE_POINT = 0x000900A8 # CTL_CODE(FILE_DEVICE_FILE_SYSTEM, 42, METHOD_BUFFERED, FILE_ANY_ACCESS) +end diff --git a/src/lib_c/x86_64-windows-msvc/c/winnt.cr b/src/lib_c/x86_64-windows-msvc/c/winnt.cr index d0d419f4fa01..52cfab5ebf89 100644 --- a/src/lib_c/x86_64-windows-msvc/c/winnt.cr +++ b/src/lib_c/x86_64-windows-msvc/c/winnt.cr @@ -25,6 +25,11 @@ lib LibC FILE_READ_ATTRIBUTES = 0x80 FILE_WRITE_ATTRIBUTES = 0x0100 + MAXIMUM_REPARSE_DATA_BUFFER_SIZE = 0x4000 + + IO_REPARSE_TAG_SYMLINK = 0xA000000C_u32 + IO_REPARSE_TAG_AF_UNIX = 0x80000023_u32 + # Memory protection constants PAGE_READWRITE = 0x04 PAGE_GUARD = 0x100