feat(Windows platform): Allow building on Windows Stable Rust (#1560)

Signed-off-by: Ana Hobden <[email protected]>

.circleci/config.yml

@@ -20,7 +20,7 @@ install-rust-windows: &install-rust-windows
     shell: bash
     command: |
       curl https://sh.rustup.rs -sSf | sh -s -- -y \
-        --default-toolchain nightly-2019-11-19 \
+        --default-toolchain stable \
         --default-host x86_64-pc-windows-msvc
       # see https://github.com/rust-lang/cargo/issues/2078
       printf '[net]\ngit-fetch-with-cli = true\n' > "$HOME/.cargo/config"
@@ -136,7 +136,6 @@ jobs:
             AWS_SECRET_ACCESS_KEY: fake-aws-key
           command: |
             PATH="$HOME/.cargo/bin:/c/Strawberry/perl/bin:/c/Program Files/CMake/bin:$PATH"
-            rm rust-toolchain
             cargo test --release --no-default-features --features default-msvc -- --test-threads 1
 
   test-stable-kubernetes:

lib/file-source/Cargo.toml

@@ -14,6 +14,8 @@ scan_fmt = "0.2.3"
 tracing = "0.1.2"
 indexmap = {version = "1.0.2", features = ["serde-1"]}
 flate2 = "1.0.6"
+winapi = { version = "0.3", features = ["winioctl"] }
+libc =  "0.2"
 
 [dev-dependencies]
 quickcheck = "0.6"

lib/file-source/src/file_server.rs

@@ -4,14 +4,14 @@ use futures::{stream, Future, Sink, Stream};
 use glob::{glob, Pattern};
 use indexmap::IndexMap;
 use std::collections::{HashMap, HashSet};
-use std::fs;
+use std::fs::{self, File};
 use std::io::{self, Read, Seek, Write};
 use std::path::{Path, PathBuf};
 use std::sync::mpsc::RecvTimeoutError;
 use std::time;
 use tracing::field;
 
-use crate::metadata_ext::PortableMetadataExt;
+use crate::metadata_ext::PortableFileExt;
 
 /// `FileServer` is a Source which cooperatively schedules reads over files,
 /// converting the lines of said files into `LogLine` structures. As
@@ -396,9 +396,9 @@ impl Fingerprinter {
     ) -> Result<FileFingerprint, io::Error> {
         match *self {
             Fingerprinter::DevInode => {
-                let metadata = fs::metadata(path)?;
-                let dev = metadata.portable_dev();
-                let ino = metadata.portable_ino();
+                let file_handle = File::open(path)?;
+                let dev = file_handle.portable_dev()?;
+                let ino = file_handle.portable_ino()?;
                 buffer.clear();
                 buffer.write_all(&dev.to_be_bytes())?;
                 buffer.write_all(&ino.to_be_bytes())?;

lib/file-source/src/file_watcher.rs

@@ -1,14 +1,14 @@
 use crate::FilePosition;
 use flate2::bufread::MultiGzDecoder;
 use std::{
-    fs,
+    fs::{self, File},
     io::{self, BufRead, Seek},
     path::PathBuf,
     thread,
     time::{Duration, Instant, SystemTime},
 };
 
-use crate::metadata_ext::PortableMetadataExt;
+use crate::metadata_ext::PortableFileExt;
 
 /// The `FileWatcher` struct defines the polling based state machine which reads
 /// from a file path, transparently updating the underlying file descriptor when
@@ -41,6 +41,7 @@ impl FileWatcher {
         ignore_before: Option<SystemTime>,
     ) -> Result<FileWatcher, io::Error> {
         let f = fs::File::open(&path)?;
+        let (devno, ino) = (f.portable_dev()?, f.portable_ino()?);
         let metadata = f.metadata()?;
         let mut reader = io::BufReader::new(f);
 
@@ -87,17 +88,17 @@ impl FileWatcher {
             findable: true,
             reader,
             file_position,
-            devno: metadata.portable_dev(),
-            inode: metadata.portable_ino(),
+            devno: devno,
+            inode: ino,
             is_dead: false,
             last_read_attempt: ts.clone(),
             last_read_success: ts,
         })
     }
 
     pub fn update_path(&mut self, path: PathBuf) -> io::Result<()> {
-        let metadata = fs::metadata(&path)?;
-        if (metadata.portable_dev(), metadata.portable_ino()) != (self.devno, self.inode) {
+        let file_handle = File::open(&path)?;
+        if (file_handle.portable_dev()?, file_handle.portable_ino()?) != (self.devno, self.inode) {
             let mut reader = io::BufReader::new(fs::File::open(&path)?);
             let gzipped = is_gzipped(&mut reader)?;
             let new_reader: Box<dyn BufRead> = if gzipped {
@@ -111,8 +112,8 @@ impl FileWatcher {
                 Box::new(reader)
             };
             self.reader = new_reader;
-            self.devno = metadata.portable_dev();
-            self.inode = metadata.portable_ino();
+            self.devno = file_handle.portable_dev()?;
+            self.inode = file_handle.portable_ino()?;
         }
         self.path = path;
         Ok(())

lib/file-source/src/lib.rs

@@ -1,4 +1,3 @@
-#![cfg_attr(windows, feature(windows_by_handle))]
 #[macro_use]
 extern crate scan_fmt;
 #[macro_use]

lib/file-source/src/metadata_ext.rs

@@ -1,30 +1,111 @@
-use std::fs::Metadata;
+//! FIXME: A workaround to fix https://github.com/timberio/vector/issues/1480 resulting from https://github.com/rust-lang/rust/issues/63010
+//! Most of code is cribbed directly from the Rust stdlib and ported to work with winapi.
+//!
+//! In stdlib imported code, warnings are allowed.
+
+use std::fs::File;
 #[cfg(unix)]
 use std::os::unix::fs::MetadataExt;
 #[cfg(windows)]
-use std::os::windows::fs::MetadataExt;
+use std::{mem::zeroed, ptr};
+#[cfg(windows)]
+use winapi::shared::minwindef::DWORD;
+#[cfg(windows)]
+use winapi::um::{
+    fileapi::GetFileInformationByHandle, fileapi::BY_HANDLE_FILE_INFORMATION,
+    ioapiset::DeviceIoControl, winioctl::FSCTL_GET_REPARSE_POINT,
+    winnt::FILE_ATTRIBUTE_REPARSE_POINT, winnt::MAXIMUM_REPARSE_DATA_BUFFER_SIZE,
+};
+
+#[cfg(not(windows))]
+pub trait PortableFileExt {
+    fn portable_dev(&self) -> std::io::Result<u64>;
+    fn portable_ino(&self) -> std::io::Result<u64>;
+}
 
-pub trait PortableMetadataExt {
-    fn portable_dev(&self) -> u64;
-    fn portable_ino(&self) -> u64;
+#[cfg(windows)]
+pub trait PortableFileExt: std::os::windows::io::AsRawHandle {
+    fn portable_dev(&self) -> std::io::Result<u64>;
+    fn portable_ino(&self) -> std::io::Result<u64>;
+    // This code is from the Rust stdlib https://github.com/rust-lang/rust/blob/30ddb5a8c1e85916da0acdc665d6a16535a12dd6/src/libstd/sys/windows/fs.rs#L458-L478
+    #[allow(unused_assignments, unused_variables)]
+    fn reparse_point<'a>(
+        &self,
+        space: &'a mut [u8; MAXIMUM_REPARSE_DATA_BUFFER_SIZE as usize],
+    ) -> std::io::Result<(DWORD, &'a REPARSE_DATA_BUFFER)> {
+        unsafe {
+            let mut bytes = 0;
+            cvt({
+                DeviceIoControl(
+                    self.as_raw_handle(),
+                    FSCTL_GET_REPARSE_POINT,
+                    ptr::null_mut(),
+                    0,
+                    space.as_mut_ptr() as *mut _,
+                    space.len() as DWORD,
+                    &mut bytes,
+                    ptr::null_mut(),
+                )
+            })?;
+            Ok((bytes, &*(space.as_ptr() as *const REPARSE_DATA_BUFFER)))
+        }
+    }
+    // This code is from the Rust stdlib https://github.com/rust-lang/rust/blob/30ddb5a8c1e85916da0acdc665d6a16535a12dd6/src/libstd/sys/windows/fs.rs#L326-L351
+    #[allow(unused_assignments, unused_variables)]
+    fn get_file_info(&self) -> std::io::Result<BY_HANDLE_FILE_INFORMATION> {
+        unsafe {
+            let mut info: BY_HANDLE_FILE_INFORMATION = zeroed();
+            cvt(GetFileInformationByHandle(self.as_raw_handle(), &mut info))?;
+            let mut reparse_tag = 0;
+            if info.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT != 0 {
+                let mut b = [0; MAXIMUM_REPARSE_DATA_BUFFER_SIZE as usize];
+                if let Ok((_, buf)) = self.reparse_point(&mut b) {
+                    reparse_tag = buf.ReparseTag;
+                }
+            }
+            Ok(info)
+        }
+    }
 }
 
 #[cfg(unix)]
-impl PortableMetadataExt for Metadata {
-    fn portable_dev(&self) -> u64 {
-        self.dev()
+impl PortableFileExt for File {
+    fn portable_dev(&self) -> std::io::Result<u64> {
+        Ok(self.metadata()?.dev())
     }
-    fn portable_ino(&self) -> u64 {
-        self.ino()
+    fn portable_ino(&self) -> std::io::Result<u64> {
+        Ok(self.metadata()?.ino())
     }
 }
 
 #[cfg(windows)]
-impl PortableMetadataExt for Metadata {
-    fn portable_dev(&self) -> u64 {
-        self.volume_serial_number().unwrap_or(0u32) as u64
+impl PortableFileExt for File {
+    fn portable_dev(&self) -> std::io::Result<u64> {
+        let info = self.get_file_info()?;
+        Ok(info.dwVolumeSerialNumber.into())
     }
-    fn portable_ino(&self) -> u64 {
-        self.file_index().unwrap_or(0u64)
+    fn portable_ino(&self) -> std::io::Result<u64> {
+        let info = self.get_file_info()?;
+        Ok(info.nNumberOfLinks.into())
+    }
+}
+
+// This code is from the Rust stdlib https://github.com/rust-lang/rust/blob/a916ac22b9f7f1f0f7aba0a41a789b3ecd765018/src/libstd/sys/windows/c.rs#L380-L386
+#[cfg(windows)]
+#[allow(non_snake_case, non_camel_case_types)]
+pub struct REPARSE_DATA_BUFFER {
+    pub ReparseTag: libc::c_uint,
+    pub ReparseDataLength: libc::c_ushort,
+    pub Reserved: libc::c_ushort,
+    pub rest: (),
+}
+
+// This code is from the Rust stdlib  https://github.com/rust-lang/rust/blob/30ddb5a8c1e85916da0acdc665d6a16535a12dd6/src/libstd/sys/hermit/mod.rs#L141-L143
+#[cfg(windows)]
+pub fn cvt(result: i32) -> std::io::Result<usize> {
+    if result < 0 {
+        Err(std::io::Error::from_raw_os_error(-result))
+    } else {
+        Ok(result as usize)
     }
 }

website/docs/setup/installation/manual/from-source.md

@@ -295,7 +295,7 @@ Building steps:
 
 The steps to compile Vector on Windows are different from the ones for other operating systems.
 
-1. Install Rust using [`rustup`][urls.rustup]. It would install toolchain for `nightly-x86_64-pc-windows-msvc` target on 64-bit machines by default. If you don't have VC++ build tools, the installer would prompt you to install them.
+1. Install Rust using [`rustup`][urls.rustup]. If you don't have VC++ build tools, the installer would prompt you to install them.
 
 2. Install [Perl for Windows][urls.perl_windows].
 
@@ -338,7 +338,7 @@ The steps to compile Vector on Windows are different from the ones for other ope
 
     ```
     set RUSTFLAGS=-Ctarget-feature=+crt-static
-    cargo +nightly-x86_64-pc-windows-msvc build --no-default-features --features default-msvc --release
+    cargo build --no-default-features --features default-msvc --release
     ```
 
 6. After these steps a binary `vector.exe` in `target\release` would be created. It can be started by running

website/docs/setup/installation/manual/from-source.md.erb

@@ -223,7 +223,7 @@ Building steps:
 
 The steps to compile Vector on Windows are different from the ones for other operating systems.
 
-1. Install Rust using [`rustup`][urls.rustup]. It would install toolchain for `nightly-x86_64-pc-windows-msvc` target on 64-bit machines by default. If you don't have VC++ build tools, the installer would prompt you to install them.
+1. Install Rust using [`rustup`][urls.rustup]. If you don't have VC++ build tools, the installer would prompt you to install them.
 
 2. Install [Perl for Windows][urls.perl_windows].
 
@@ -266,7 +266,7 @@ The steps to compile Vector on Windows are different from the ones for other ope
 
     ```
     set RUSTFLAGS=-Ctarget-feature=+crt-static
-    cargo +nightly-x86_64-pc-windows-msvc build --no-default-features --features default-msvc --release
+    cargo build --no-default-features --features default-msvc --release
     ```
 
 6. After these steps a binary `vector.exe` in `target\release` would be created. It can be started by running