[WIP] Tests: Skip failing tests on windows

Sources/_InternalTestSupport/XCTSkipHelpers.swift

@@ -17,6 +17,18 @@ import XCTest
 import class Basics.AsyncProcess
 import struct TSCBasic.StringError
 
+public func skipOnWindowsAsTestCurrentlyFails(because reason: String? = nil) throws {
+    #if os(Windows)
+    let failureCause: String
+    if reason == nil {
+        failureCause = ""
+    } else {
+        failureCause = " because \(reason!.description)"
+    }
+    throw XCTSkip("Test fails on windows\(failureCause)")
+    #endif
+}
+
 extension Toolchain {
     package func skipUnlessAtLeastSwift6(
         file: StaticString = #file,

Tests/BasicsTests/AsyncProcessTests.swift

@@ -23,6 +23,11 @@ import func TSCBasic.withTemporaryFile
 import func TSCTestSupport.withCustomEnv
 
 final class AsyncProcessTests: XCTestCase {
+
+    override func setUp() async throws {
+        try skipOnWindowsAsTestCurrentlyFails()
+    }
+
     func testBasics() throws {
         do {
             let process = AsyncProcess(args: "echo", "hello")

Tests/BasicsTests/ConcurrencyHelpersTests.swift

@@ -14,9 +14,15 @@
 import TSCTestSupport
 import XCTest
 
+import _InternalTestSupport // for skipOnWindowsAsTestCurrentlyFails
+
 final class ConcurrencyHelpersTest: XCTestCase {
     let queue = DispatchQueue(label: "ConcurrencyHelpersTest", attributes: .concurrent)
 
+    override func setUpWithError() throws {
+        try skipOnWindowsAsTestCurrentlyFails()
+    }
+
     func testThreadSafeKeyValueStore() {
         for _ in 0 ..< 100 {
             let sync = DispatchGroup()

Tests/BasicsTests/Environment/EnvironmentTests.swift

@@ -15,6 +15,7 @@
 import Basics
 
 import XCTest
+import _InternalTestSupport // for skipOnWindowsAsTestCurrentlyFails()
 
 final class EnvironmentTests: XCTestCase {
     func test_init() {
@@ -35,10 +36,11 @@ final class EnvironmentTests: XCTestCase {
             "testKey": "TestValue2",
         ]
         let environment = Environment(dictionary)
-        XCTAssertEqual(environment["TestKey"], "TestValue")
         #if os(Windows)
+        XCTAssertEqual(environment["TestKey"], "TestValue2")
         XCTAssertEqual(environment.count, 1)
         #else
+        XCTAssertEqual(environment["TestKey"], "TestValue")
         XCTAssertEqual(environment.count, 2)
         #endif
     }
@@ -47,6 +49,7 @@ final class EnvironmentTests: XCTestCase {
         let dictionary = ["TestKey": "TestValue"]
         let environment = Environment(dictionary)
         XCTAssertEqual(environment["TestKey"], "TestValue")
+        XCTAssertEqual(environment.count, 1)
     }
 
     func path(_ components: String...) -> String {
@@ -99,10 +102,13 @@ final class EnvironmentTests: XCTestCase {
 
     /// Important: This test is inherently race-prone, if it is proven to be
     /// flaky, it should run in a singled threaded environment/removed entirely.
-    func test_current() {
+    func test_current() throws {
+        try skipOnWindowsAsTestCurrentlyFails(because: "ProcessInfo.processInfo.environment[\"PATH\"] return nil")
+
         XCTAssertEqual(
             Environment.current["PATH"],
-            ProcessInfo.processInfo.environment["PATH"])
+            ProcessInfo.processInfo.environment["PATH"]
+        )
     }
 
     /// Important: This test is inherently race-prone, if it is proven to be

Tests/BasicsTests/FileSystem/PathTests.swift

@@ -12,6 +12,8 @@ import Basics
 import Foundation
 import XCTest
 
+import _InternalTestSupport // for skipOnWindowsAsTestCurrentlyFails()
+
 #if os(Windows)
 private var windows: Bool { true }
 #else
@@ -54,28 +56,36 @@ class PathTests: XCTestCase {
         XCTAssertEqual(rel2.pathString, "~")  // `~` is not special
     }
 
-    func testRepeatedPathSeparators() {
+    func testRepeatedPathSeparators() throws {
+        try skipOnWindowsAsTestCurrentlyFails(because: "all assertions fail")
+
         XCTAssertEqual(AbsolutePath("/ab//cd//ef").pathString, windows ? #"\ab\cd\ef"# : "/ab/cd/ef")
         XCTAssertEqual(AbsolutePath("/ab///cd//ef").pathString, windows ? #"\ab\cd\ef"# : "/ab/cd/ef")
         XCTAssertEqual(RelativePath("ab//cd//ef").pathString, windows ? #"ab\cd\ef"# : "ab/cd/ef")
         XCTAssertEqual(RelativePath("ab//cd///ef").pathString, windows ? #"ab\cd\ef"# : "ab/cd/ef")
     }
 
-    func testTrailingPathSeparators() {
+    func testTrailingPathSeparators() throws {
+        try skipOnWindowsAsTestCurrentlyFails(because: "trailing path seperator is not removed from pathString")
+
         XCTAssertEqual(AbsolutePath("/ab/cd/ef/").pathString, windows ? #"\ab\cd\ef"# : "/ab/cd/ef")
         XCTAssertEqual(AbsolutePath("/ab/cd/ef//").pathString, windows ? #"\ab\cd\ef"# : "/ab/cd/ef")
         XCTAssertEqual(RelativePath("ab/cd/ef/").pathString, windows ? #"ab\cd\ef"# : "ab/cd/ef")
         XCTAssertEqual(RelativePath("ab/cd/ef//").pathString, windows ? #"ab\cd\ef"# : "ab/cd/ef")
     }
 
-    func testDotPathComponents() {
+    func testDotPathComponents() throws {
+        try skipOnWindowsAsTestCurrentlyFails()
+
         XCTAssertEqual(AbsolutePath("/ab/././cd//ef").pathString, "/ab/cd/ef")
         XCTAssertEqual(AbsolutePath("/ab/./cd//ef/.").pathString, "/ab/cd/ef")
         XCTAssertEqual(RelativePath("ab/./cd/././ef").pathString, "ab/cd/ef")
         XCTAssertEqual(RelativePath("ab/./cd/ef/.").pathString, "ab/cd/ef")
     }
 
-    func testDotDotPathComponents() {
+    func testDotDotPathComponents() throws {
+        try skipOnWindowsAsTestCurrentlyFails()
+
         XCTAssertEqual(AbsolutePath("/..").pathString, windows ? #"\"# : "/")
         XCTAssertEqual(AbsolutePath("/../../../../..").pathString, windows ? #"\"# : "/")
         XCTAssertEqual(AbsolutePath("/abc/..").pathString, windows ? #"\"# : "/")
@@ -91,7 +101,9 @@ class PathTests: XCTestCase {
         XCTAssertEqual(RelativePath("abc/..").pathString, ".")
     }
 
-    func testCombinationsAndEdgeCases() {
+    func testCombinationsAndEdgeCases() throws {
+        try skipOnWindowsAsTestCurrentlyFails()
+
         XCTAssertEqual(AbsolutePath("///").pathString, windows ? #"\"# : "/")
         XCTAssertEqual(AbsolutePath("/./").pathString, windows ? #"\"# : "/")
         XCTAssertEqual(RelativePath("").pathString, ".")
@@ -120,7 +132,9 @@ class PathTests: XCTestCase {
         XCTAssertEqual(RelativePath("a/../////../////./////").pathString, "..")
     }
 
-    func testDirectoryNameExtraction() {
+    func testDirectoryNameExtraction() throws {
+        try skipOnWindowsAsTestCurrentlyFails()
+
         XCTAssertEqual(AbsolutePath("/").dirname, windows ? #"\"# : "/")
         XCTAssertEqual(AbsolutePath("/a").dirname, windows ? #"\"# : "/")
         XCTAssertEqual(AbsolutePath("/./a").dirname, windows ? #"\"# : "/")
@@ -137,7 +151,9 @@ class PathTests: XCTestCase {
         XCTAssertEqual(RelativePath(".").dirname, ".")
     }
 
-    func testBaseNameExtraction() {
+    func testBaseNameExtraction() throws {
+        try skipOnWindowsAsTestCurrentlyFails()
+
         XCTAssertEqual(AbsolutePath("/").basename, windows ? #"\"# : "/")
         XCTAssertEqual(AbsolutePath("/a").basename, "a")
         XCTAssertEqual(AbsolutePath("/./a").basename, "a")
@@ -153,7 +169,9 @@ class PathTests: XCTestCase {
         XCTAssertEqual(RelativePath(".").basename, ".")
     }
 
-    func testBaseNameWithoutExt() {
+    func testBaseNameWithoutExt() throws{
+        try skipOnWindowsAsTestCurrentlyFails()
+
         XCTAssertEqual(AbsolutePath("/").basenameWithoutExt, windows ? #"\"# : "/")
         XCTAssertEqual(AbsolutePath("/a").basenameWithoutExt, "a")
         XCTAssertEqual(AbsolutePath("/./a").basenameWithoutExt, "a")
@@ -176,7 +194,9 @@ class PathTests: XCTestCase {
         XCTAssertEqual(RelativePath("abc.xyz.123").basenameWithoutExt, "abc.xyz")
     }
 
-    func testSuffixExtraction() {
+    func testSuffixExtraction() throws {
+        try skipOnWindowsAsTestCurrentlyFails(because: "expected nil is not the actual")
+
         XCTAssertEqual(RelativePath("a").suffix, nil)
         XCTAssertEqual(RelativePath("a").extension, nil)
         XCTAssertEqual(RelativePath("a.").suffix, nil)
@@ -201,7 +221,9 @@ class PathTests: XCTestCase {
         XCTAssertEqual(RelativePath(".a.foo.bar.baz").extension, "baz")
     }
 
-    func testParentDirectory() {
+    func testParentDirectory() throws {
+        try skipOnWindowsAsTestCurrentlyFails()
+
         XCTAssertEqual(AbsolutePath("/").parentDirectory, AbsolutePath("/"))
         XCTAssertEqual(AbsolutePath("/").parentDirectory.parentDirectory, AbsolutePath("/"))
         XCTAssertEqual(AbsolutePath("/bar").parentDirectory, AbsolutePath("/"))
@@ -210,7 +232,9 @@ class PathTests: XCTestCase {
     }
 
     @available(*, deprecated)
-    func testConcatenation() {
+    func testConcatenation() throws {
+        try skipOnWindowsAsTestCurrentlyFails()
+
         XCTAssertEqual(AbsolutePath(AbsolutePath("/"), RelativePath("")).pathString, windows ? #"\"# : "/")
         XCTAssertEqual(AbsolutePath(AbsolutePath("/"), RelativePath(".")).pathString, windows ? #"\"# : "/")
         XCTAssertEqual(AbsolutePath(AbsolutePath("/"), RelativePath("..")).pathString, windows ? #"\"# : "/")
@@ -247,7 +271,9 @@ class PathTests: XCTestCase {
         XCTAssertEqual(RelativePath("hello").appending(RelativePath("a/b/../c/d")).pathString, windows ? #"hello\a\c\d"# : "hello/a/c/d")
     }
 
-    func testPathComponents() {
+    func testPathComponents() throws {
+        try skipOnWindowsAsTestCurrentlyFails()
+
         XCTAssertEqual(AbsolutePath("/").components, ["/"])
         XCTAssertEqual(AbsolutePath("/.").components, ["/"])
         XCTAssertEqual(AbsolutePath("/..").components, ["/"])
@@ -275,7 +301,9 @@ class PathTests: XCTestCase {
         XCTAssertEqual(RelativePath("abc").components, ["abc"])
     }
 
-    func testRelativePathFromAbsolutePaths() {
+    func testRelativePathFromAbsolutePaths() throws {
+        try skipOnWindowsAsTestCurrentlyFails()
+
         XCTAssertEqual(AbsolutePath("/").relative(to: AbsolutePath("/")), RelativePath("."));
         XCTAssertEqual(AbsolutePath("/a/b/c/d").relative(to: AbsolutePath("/")), RelativePath("a/b/c/d"));
         XCTAssertEqual(AbsolutePath("/").relative(to: AbsolutePath("/a/b/c")), RelativePath("../../.."));
@@ -316,7 +344,9 @@ class PathTests: XCTestCase {
         XCTAssertTrue(AbsolutePath("/foo").isAncestor(of: AbsolutePath("/foo/bar")))
     }
 
-    func testAbsolutePathValidation() {
+    func testAbsolutePathValidation() throws {
+        try skipOnWindowsAsTestCurrentlyFails()
+
         XCTAssertNoThrow(try AbsolutePath(validating: "/a/b/c/d"))
 
         XCTAssertThrowsError(try AbsolutePath(validating: "~/a/b/d")) { error in
@@ -328,7 +358,9 @@ class PathTests: XCTestCase {
         }
     }
 
-    func testRelativePathValidation() {
+    func testRelativePathValidation() throws {
+        try skipOnWindowsAsTestCurrentlyFails()
+
         XCTAssertNoThrow(try RelativePath(validating: "a/b/c/d"))
 
         XCTAssertThrowsError(try RelativePath(validating: "/a/b/d")) { error in
@@ -342,6 +374,8 @@ class PathTests: XCTestCase {
     }
 
     func testCodable() throws {
+        try skipOnWindowsAsTestCurrentlyFails()
+
         struct Foo: Codable, Equatable {
             var path: AbsolutePath
         }

Tests/BasicsTests/FileSystem/VFSTests.swift

@@ -16,6 +16,8 @@ import XCTest
 
 import struct TSCBasic.ByteString
 
+import _InternalTestSupport // for skipOnWindowsAsTestCurrentlyFails
+
 func testWithTemporaryDirectory(
     function: StaticString = #function,
     body: @escaping (AbsolutePath) async throws -> Void
@@ -36,6 +38,8 @@ func testWithTemporaryDirectory(
 
 class VFSTests: XCTestCase {
     func testLocalBasics() throws {
+        try skipOnWindowsAsTestCurrentlyFails()
+
         // tiny PE binary from: https://archive.is/w01DO
         let contents: [UInt8] = [
           0x4d, 0x5a, 0x00, 0x00, 0x50, 0x45, 0x00, 0x00, 0x4c, 0x01, 0x01, 0x00,

Tests/BasicsTests/HTTPClientTests.swift

@@ -226,6 +226,8 @@ final class HTTPClientTests: XCTestCase {
     }
 
     func testExponentialBackoff() async throws {
+        try skipOnWindowsAsTestCurrentlyFails()
+
         let counter = SendableBox(0)
         let lastCall = SendableBox<Date>()
         let maxAttempts = 5

Tests/BasicsTests/Serialization/SerializedJSONTests.swift

@@ -12,9 +12,12 @@
 
 @testable import Basics
 import XCTest
+import _InternalTestSupport // for skipOnWindowsAsTestCurrentlyFails
 
 final class SerializedJSONTests: XCTestCase {
     func testPathInterpolation() throws {
+        try skipOnWindowsAsTestCurrentlyFails()
+
         var path = try AbsolutePath(validating: #"/test\backslashes"#)
         var json: SerializedJSON = "\(path)"
 

Tests/BuildTests/BuildPlanTests.swift

@@ -1999,6 +1999,8 @@ final class BuildPlanTests: XCTestCase {
     }
 
     func test_symbolGraphExtract_arguments() async throws {
+        try skipOnWindowsAsTestCurrentlyFails()
+
         // ModuleGraph:
         // .
         // ├── A (Swift)
@@ -4619,6 +4621,8 @@ final class BuildPlanTests: XCTestCase {
     }
 
     func testUserToolchainCompileFlags() async throws {
+        try skipOnWindowsAsTestCurrentlyFails()
+
         let fs = InMemoryFileSystem(
             emptyFiles:
             "/Pkg/Sources/exe/main.swift",
@@ -4882,6 +4886,8 @@ final class BuildPlanTests: XCTestCase {
     }
 
     func testUserToolchainWithToolsetCompileFlags() async throws {
+        try skipOnWindowsAsTestCurrentlyFails()
+
         let fileSystem = InMemoryFileSystem(
             emptyFiles:
             "/Pkg/Sources/exe/main.swift",
@@ -5050,6 +5056,8 @@ final class BuildPlanTests: XCTestCase {
     }
 
     func testUserToolchainWithSDKSearchPaths() async throws {
+        try skipOnWindowsAsTestCurrentlyFails()
+
         let fileSystem = InMemoryFileSystem(
             emptyFiles:
             "/Pkg/Sources/exe/main.swift",

Tests/BuildTests/BuildSystemDelegateTests.swift

@@ -31,6 +31,8 @@ final class BuildSystemDelegateTests: XCTestCase {
     }
 
     func testFilterNonFatalCodesignMessages() async throws {
+        try skipOnWindowsAsTestCurrentlyFails()
+
         try XCTSkipIf(!UserToolchain.default.supportsSDKDependentTests(), "skipping because test environment doesn't support this test")
         // Note: we can re-use the `TestableExe` fixture here since we just need an executable.
         try await fixture(name: "Miscellaneous/TestableExe") { fixturePath in

Tests/BuildTests/LLBuildManifestBuilderTests.swift

@@ -194,6 +194,8 @@ final class LLBuildManifestBuilderTests: XCTestCase {
 
     /// Verifies that two modules with the same name but different triples don't share same build manifest keys.
     func testToolsBuildTriple() async throws {
+        try skipOnWindowsAsTestCurrentlyFails()
+
         let (graph, fs, scope) = try macrosPackageGraph()
         let productsTriple = Triple.x86_64MacOS
         let toolsTriple = Triple.arm64Linux

Tests/BuildTests/PluginsBuildPlanTests.swift

@@ -18,6 +18,8 @@ import PackageModel
 
 final class PluginsBuildPlanTests: XCTestCase {
     func testBuildToolsDatabasePath() async throws {
+        try skipOnWindowsAsTestCurrentlyFails()
+
         try await fixture(name: "Miscellaneous/Plugins/MySourceGenPlugin") { fixturePath in
             let (stdout, _) = try await executeSwiftBuild(fixturePath)
             XCTAssertMatch(stdout, .contains("Build complete!"))