Add basic reporting to the test runner

packages/patrol_cli/lib/src/features/drive/drive_command.dart

@@ -1,3 +1,4 @@
+import 'package:ansi_styles/extension.dart';
 import 'package:dispose_scope/dispose_scope.dart';
 import 'package:freezed_annotation/freezed_annotation.dart';
 import 'package:mason_logger/mason_logger.dart';
@@ -32,6 +33,8 @@ class DriveCommandConfig with _$DriveCommandConfig {
   }) = _DriveCommandConfig;
 }
 
+const _defaultRepeats = 1;
+
 class DriveCommand extends StagedCommand<DriveCommandConfig> {
   DriveCommand({
     required DeviceFinder deviceFinder,
@@ -114,7 +117,7 @@ class DriveCommand extends StagedCommand<DriveCommandConfig> {
         'repeat',
         abbr: 'n',
         help: 'Repeat the test n times.',
-        defaultsTo: '1',
+        defaultsTo: '$_defaultRepeats',
       );
   }
 
@@ -175,9 +178,9 @@ class DriveCommand extends StagedCommand<DriveCommandConfig> {
       throw const FormatException('`wait` argument is not an int');
     }
 
-    var repeat = 1;
+    final int repeat;
     try {
-      final repeatStr = argResults?['repeat'] as String? ?? '1';
+      final repeatStr = argResults?['repeat'] as String? ?? '$_defaultRepeats';
       repeat = int.parse(repeatStr);
     } on FormatException {
       throw const FormatException('`repeat` argument is not an int');
@@ -222,15 +225,12 @@ class DriveCommand extends StagedCommand<DriveCommandConfig> {
       dartDefines: config.dartDefines,
     );
 
-    var exitCode = 0;
-
     _testRunner
       ..repeats = config.repeat
       ..builder = (target, device) async {
         try {
           await _flutterTool.build(target, device);
         } catch (err) {
-          exitCode = 1;
           _logger
             ..err('$err')
             ..err(
@@ -242,8 +242,7 @@ class DriveCommand extends StagedCommand<DriveCommandConfig> {
       ..executor = (target, device) async {
         try {
           await _flutterTool.drive(target, device);
-        } on FlutterDriverFailedException catch (err) {
-          exitCode = 1;
+        } catch (err) {
           _logger
             ..err('$err')
             ..err(
@@ -276,8 +275,29 @@ class DriveCommand extends StagedCommand<DriveCommandConfig> {
       }
     }
 
-    await _testRunner.run();
+    final results = await _testRunner.run();
+
+    for (final res in results.targetRunResults) {
+      if (res.allRunsPassed) {
+        _logger.write(
+          '${' PASS '.bgGreen.black.bold} ${res.targetName} on ${res.device.id}\n',
+        );
+      } else if (res.allRunsFailed) {
+        _logger.write(
+          '${' FAIL '.bgRed.white.bold} ${res.targetName} on ${res.device.id}\n',
+        );
+      } else if (res.canceled) {
+        _logger.write(
+          '${' CANC '.bgGray.white.bold} ${res.targetName} on ${res.device.id}\n',
+        );
+      } else {
+        _logger.write(
+          '${' FLAK '.bgYellow.black.bold} ${res.targetName} on ${res.device.id}\n',
+        );
+      }
+    }
 
+    final exitCode = results.allSuccessful ? 0 : 1;
     return exitCode;
   }
 }

packages/patrol_cli/lib/src/features/drive/test_runner.dart

@@ -1,8 +1,65 @@
 import 'package:dispose_scope/dispose_scope.dart';
+import 'package:equatable/equatable.dart';
+import 'package:path/path.dart' show basename;
 import 'package:patrol_cli/src/features/drive/device.dart';
 
 typedef _Callback = Future<void> Function(String target, Device device);
 
+class TestRunnerResult with EquatableMixin {
+  const TestRunnerResult({required this.targetRunResults});
+
+  final List<TargetRunResult> targetRunResults;
+
+  bool get allSuccessful => targetRunResults
+      .every((e) => e.runs.every((run) => run == TargetRunStatus.passed));
+
+  @override
+  List<Object?> get props => [targetRunResults];
+}
+
+class TargetRunResult with EquatableMixin {
+  TargetRunResult({
+    required this.target,
+    required this.device,
+    required this.runs,
+  });
+
+  final String target;
+  final Device device;
+
+  final List<TargetRunStatus> runs;
+
+  String get targetName => basename(target);
+
+  bool get allRunsPassed => runs.every((run) => run == TargetRunStatus.passed);
+
+  bool get allRunsFailed => runs.every(
+        (run) =>
+            run == TargetRunStatus.failedToBuild ||
+            run == TargetRunStatus.failedToExecute,
+      );
+
+  /// True if at least 1 test run was canceled.
+  bool get canceled => runs.any((run) => run == TargetRunStatus.canceled);
+
+  int get passedRuns {
+    return runs.where((run) => run == TargetRunStatus.passed).length;
+  }
+
+  int get runsFailedToBuild {
+    return runs.where((run) => run == TargetRunStatus.failedToBuild).length;
+  }
+
+  int get runsFailedToExecute {
+    return runs.where((run) => run == TargetRunStatus.failedToExecute).length;
+  }
+
+  @override
+  List<Object?> get props => [target, device, runs];
+}
+
+enum TargetRunStatus { failedToBuild, failedToExecute, passed, canceled }
+
 /// Orchestrates running tests on devices.
 ///
 /// It maps running T test targets on D devices, resulting in T * D test runs.
@@ -78,7 +135,7 @@ class TestRunner extends Disposable {
   /// Tests are run sequentially on a single device, but many devices can be
   /// attached at the same time, and thus many tests can be running at the same
   /// time.
-  Future<void> run() async {
+  Future<TestRunnerResult> run() async {
     if (_running) {
       throw StateError('tests are already running');
     }
@@ -101,30 +158,45 @@ class TestRunner extends Disposable {
 
     _running = true;
 
+    final targetRunResults = <TargetRunResult>[];
+
     final testRunsOnAllDevices = <Future<void>>[];
     for (final device in _devices.values) {
       Future<void> runTestsOnDevice() async {
         for (final target in _targets) {
+          final targetRuns = <TargetRunStatus>[];
+          targetRunResults.add(
+            TargetRunResult(target: target, device: device, runs: targetRuns),
+          );
+
           if (_disposed) {
+            for (var i = 0; i < _repeats; i++) {
+              targetRuns.add(TargetRunStatus.canceled);
+            }
             continue;
           }
 
           try {
             await builder(target, device);
           } catch (_) {
+            targetRuns.add(TargetRunStatus.failedToBuild);
             continue;
           }
 
           for (var i = 0; i < _repeats; i++) {
             if (_disposed) {
+              targetRuns.add(TargetRunStatus.canceled);
               continue;
             }
 
             try {
               await executor(target, device);
             } catch (_) {
+              targetRuns.add(TargetRunStatus.failedToExecute);
               continue;
             }
+
+            targetRuns.add(TargetRunStatus.passed);
           }
         }
       }
@@ -135,6 +207,8 @@ class TestRunner extends Disposable {
 
     await Future.wait<void>(testRunsOnAllDevices);
     _running = false;
+
+    return TestRunnerResult(targetRunResults: targetRunResults);
   }
 
   @override

packages/patrol_cli/pubspec.lock

@@ -176,6 +176,13 @@ packages:
       url: "https://pub.dartlang.org"
     source: hosted
     version: "2.0.1"
+  equatable:
+    dependency: "direct main"
+    description:
+      name: equatable
+      url: "https://pub.dartlang.org"
+    source: hosted
+    version: "2.0.5"
   fake_async:
     dependency: "direct dev"
     description:

packages/patrol_cli/pubspec.yaml

@@ -16,6 +16,7 @@ dependencies:
   args: ^2.3.1
   collection: ^1.16.0
   dispose_scope: ^2.0.1
+  equatable: ^2.0.5
   file: ^6.1.4
   freezed_annotation: ^2.1.0
   http: ^0.13.5