Skip to content

Commit

Permalink
Fix test output parser to be incremental and show failed tests (#505)
Browse files Browse the repository at this point in the history
  • Loading branch information
kateinoigakukun authored Nov 21, 2024
1 parent b91944c commit 4f7be4f
Show file tree
Hide file tree
Showing 7 changed files with 189 additions and 179 deletions.
117 changes: 21 additions & 96 deletions Sources/CartonHelpers/Process+run.swift
Original file line number Diff line number Diff line change
Expand Up @@ -17,21 +17,12 @@ import Dispatch
import Foundation

struct ProcessError: Error {
let stderr: String?
let stdout: String?
let exitCode: Int32
}

extension ProcessError: CustomStringConvertible {
var description: String {
var result = "Process failed with non-zero exit status"
if let stdout = stdout {
result += " and following output:\n\(stdout)"
}

if let stderr = stderr {
result += " and following error output:\n\(stderr)"
}
return result
return "Process failed with exit code \(exitCode)"
}
}

Expand All @@ -41,11 +32,10 @@ extension Foundation.Process {
_ arguments: [String],
environment: [String: String] = [:],
loadingMessage: String = "Running...",
parser: ProcessOutputParser? = nil,
_ terminal: InteractiveWriter
) async throws {
terminal.clearLine()
terminal.write("\(loadingMessage)\n", inColor: .yellow)
terminal.write("Running \(arguments.joined(separator: " "))\n")

if !environment.isEmpty {
terminal.write(environment.map { "\($0)=\($1)" }.joined(separator: " ") + " ")
Expand All @@ -54,91 +44,26 @@ extension Foundation.Process {
let processName = URL(fileURLWithPath: arguments[0]).lastPathComponent

do {
try await withCheckedThrowingContinuation {
(continuation: CheckedContinuation<(), Swift.Error>) in
DispatchQueue.global().async {
var stdoutBuffer = ""

let stdout: Process.OutputClosure = {
guard let string = String(data: Data($0), encoding: .utf8) else { return }
if parser != nil {
// Aggregate this for formatting later
stdoutBuffer += string
} else {
terminal.write(string)
}
}

var stderrBuffer = [UInt8]()

let stderr: Process.OutputClosure = {
stderrBuffer.append(contentsOf: $0)
}

let process = Process(
arguments: arguments,
environmentBlock: ProcessEnvironmentBlock(
ProcessInfo.processInfo.environment.merging(environment) { _, new in new }
),
outputRedirection: .stream(stdout: stdout, stderr: stderr),
startNewProcessGroup: true,
loggingHandler: {
terminal.write($0 + "\n")
}
)

let result = Result<ProcessResult, Swift.Error> {
try process.launch()
return try process.waitUntilExit()
}

switch result.map(\.exitStatus) {
case .success(.terminated(code: EXIT_SUCCESS)):
if let parser = parser {
if parser.parsingConditions.contains(.success) {
parser.parse(stdoutBuffer, terminal)
}
} else {
terminal.write(stdoutBuffer)
}
terminal.write(
"`\(processName)` process finished successfully\n",
inColor: .green,
bold: false
)
continuation.resume()

case let .failure(error):
continuation.resume(throwing: error)
default:
continuation.resume(
throwing: ProcessError(
stderr: String(data: Data(stderrBuffer), encoding: .utf8) ?? "",
stdout: stdoutBuffer
)
)
}
try await Process.checkNonZeroExit(
arguments: arguments,
environmentBlock: ProcessEnvironmentBlock(
ProcessInfo.processInfo.environment.merging(environment) { _, new in new }
),
loggingHandler: {
terminal.write($0 + "\n")
}
}
)
terminal.write(
"`\(processName)` process finished successfully\n",
inColor: .green,
bold: false
)
} catch {
let errorString = String(describing: error)
if errorString.isEmpty {
terminal.clearLine()
terminal.write(
"\(processName) process failed.\n\n",
inColor: .red
)
if let error = error as? ProcessError, let stdout = error.stdout {
if let parser = parser {
if parser.parsingConditions.contains(.failure) {
parser.parse(stdout, terminal)
}
} else {
terminal.write(stdout)
}
}
}

terminal.clearLine()
terminal.write(
"\(processName) process failed.\n\n",
inColor: .red
)
throw error
}
}
Expand Down
15 changes: 0 additions & 15 deletions Sources/CartonKit/Parsers/DiagnosticsParser.swift

This file was deleted.

7 changes: 6 additions & 1 deletion Sources/carton-frontend-slim/CartonFrontendTestCommand.swift
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,9 @@ struct CartonFrontendTestCommand: AsyncParsableCommand {
@Flag(help: "When running browser tests, run the browser in headless mode")
var headless: Bool = false

@Flag(help: "Enable verbose output")
var verbose: Bool = false

@Option(help: "Turn on runtime checks for various behavior.")
private var sanitize: SanitizeVariant?

Expand Down Expand Up @@ -195,6 +198,8 @@ struct CartonFrontendTestCommand: AsyncParsableCommand {
env[key] = parentEnv[key]
}
}
return TestRunnerOptions(env: env, listTestCases: list, testCases: testCases)
return TestRunnerOptions(
env: env, listTestCases: list, testCases: testCases,
testsParser: verbose ? RawTestsParser() : FancyTestsParser())
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ struct CommandTestRunner: TestRunner {
}

arguments += [testFilePath.pathString] + xctestArgs
try await Process.run(arguments, parser: TestsParser(), terminal)
try await runTestProcess(arguments, parser: options.testsParser, terminal)
}

func defaultWASIRuntime() throws -> String {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ struct JavaScriptTestRunner: TestRunner {
var arguments =
["node"] + nodeArguments + [pluginWorkDirectory.appending(component: testHarness).pathString]
options.applyXCTestArguments(to: &arguments)
try await Process.run(arguments, environment: options.env, parser: TestsParser(), terminal)
try await runTestProcess(
arguments, environment: options.env, parser: options.testsParser, terminal)
}
}
61 changes: 61 additions & 0 deletions Sources/carton-frontend-slim/TestRunners/TestRunner.swift
Original file line number Diff line number Diff line change
Expand Up @@ -12,13 +12,19 @@
// See the License for the specific language governing permissions and
// limitations under the License.

import CartonCore
import CartonHelpers
import Foundation

struct TestRunnerOptions {
/// The environment variables to pass to the test process.
let env: [String: String]
/// When specified, list all available test cases.
let listTestCases: Bool
/// Filter the test cases to run.
let testCases: [String]
/// The parser to use for the test output.
let testsParser: any TestsParser

func applyXCTestArguments(to arguments: inout [String]) {
if listTestCases {
Expand All @@ -32,3 +38,58 @@ struct TestRunnerOptions {
protocol TestRunner {
func run(options: TestRunnerOptions) async throws
}

struct LineStream {
var buffer: String = ""
let onLine: (String) -> Void

mutating func feed(_ bytes: [UInt8]) {
buffer += String(decoding: bytes, as: UTF8.self)
while let newlineIndex = buffer.firstIndex(of: "\n") {
let line = buffer[..<newlineIndex]
buffer.removeSubrange(buffer.startIndex...newlineIndex)
onLine(String(line))
}
}
}

extension TestRunner {
func runTestProcess(
_ arguments: [String],
environment: [String: String] = [:],
parser: any TestsParser,
_ terminal: InteractiveWriter
) async throws {
do {
terminal.clearLine()
let commandLine = arguments.map { "\"\($0)\"" }.joined(separator: " ")
terminal.write("Running \(commandLine)\n")

let (lines, continuation) = AsyncStream.makeStream(
of: String.self, bufferingPolicy: .unbounded
)
var lineStream = LineStream { line in
continuation.yield(line)
}
let process = Process(
arguments: arguments,
environmentBlock: ProcessEnvironmentBlock(
ProcessInfo.processInfo.environment.merging(environment) { _, new in new }
),
outputRedirection: .stream(
stdout: { bytes in
lineStream.feed(bytes)
}, stderr: { _ in },
redirectStderr: true
),
startNewProcessGroup: true
)
async let _ = parser.parse(lines, terminal)
try process.launch()
let result = try await process.waitUntilExit()
guard result.exitStatus == .terminated(code: 0) else {
throw ProcessResult.Error.nonZeroExit(result)
}
}
}
}
Loading

0 comments on commit 4f7be4f

Please sign in to comment.