Merge pull request #98 from matrix-org/paul/SYT-42

Two-phase "load then run" split (SYT-42)

lib/SyTest/Output/TAP.pm

@@ -10,8 +10,6 @@ STDOUT->autoflush(1);
 # File status
 sub run_file {}
 
-sub abort_file {}
-
 my $test_num;
 
 # General test status

lib/SyTest/Output/Term.pm

@@ -25,13 +25,6 @@ sub run_file
    print "${CYAN_B}Running $filename...${RESET}\n";
 }
 
-sub abort_file
-{
-   shift;
-   my ( $filename, $reason ) = @_;
-   print "${RED_B}ABORT${RESET} ${CYAN_B}$filename${RESET} because $reason\n";
-}
-
 # General test status
 sub enter_test
 {

run-tests.pl

@@ -337,8 +337,6 @@ sub fixture
 use constant { PROVEN => 1, PRESUMED => 2 };
 my %proven;
 
-our $SKIPPING;
-
 our $MORE_STUBS;
 
 sub maybe_stub
@@ -365,22 +363,33 @@ sub require_stub
    });
 }
 
+struct Test => [qw(
+   file name multi expect_fail critical proves requires check do timeout
+)];
+
+my @TESTS;
+
+sub _push_test
+{
+   my ( $filename, $multi, $name, %params ) = @_;
+
+   # We expect this test to fail if it's declared to be dependent on a bug that
+   # is not yet fixed
+   $params{expect_fail}++ if $params{bug} and not $FIXED_BUGS{ $params{bug} };
+
+   push @TESTS, Test( $filename, $name, $multi,
+      @params{qw( expect_fail critical proves requires check do timeout )} );
+}
+
 sub _run_test
 {
-   my ( $t, %params ) = @_;
+   my ( $t, $test ) = @_;
 
    undef @log_if_fail_lines;
 
    local $MORE_STUBS = [];
 
-   # If we're in skipping mode, just stop right now
-   if( $SKIPPING ) {
-      $proven{$_} = PRESUMED for @{ $params{proves} // [] };
-      $t->skipped++;
-      return;
-   }
-
-   my @requires = @{ $params{requires} || [] };
+   my @requires = @{ $test->requires // [] };
 
    my $f_start = Future->new;
    my @req_futures;
@@ -412,8 +421,8 @@ sub _run_test
          ->on_done( sub { @reqs = @_ } )
          ->on_fail( sub { die "fixture failed - $_[0]\n" } );
 
-      my $check = $params{check};
-      if( my $do = $params{do} ) {
+      my $check = $test->check;
+      if( my $do = $test->do ) {
          if( $check ) {
             $f_test = $f_test->then( sub {
                Future->wrap( $check->( @reqs ) )
@@ -444,14 +453,10 @@ sub _run_test
          });
       }
 
-      if( my $await = $params{await} ) {
-         die "TODO: 'await' now dead";
-      }
-
       Future->wait_any(
          $f_test,
 
-         $loop->delay_future( after => $params{timeout} // 10 )
+         $loop->delay_future( after => $test->timeout // 10 )
             ->then_fail( "Timed out waiting for test" )
       )->get;
 
@@ -472,88 +477,30 @@ sub _run_test
    } @requires )->get;
 
    if( $success ) {
-      $proven{$_} = PROVEN for @{ $params{proves} // [] };
+      $proven{$_} = PROVEN for @{ $test->proves // [] };
       $t->pass;
    }
    else {
       my $e = $@; chomp $e;
       $t->fail( $e );
    }
-
-   if( $t->failed ) {
-      $params{expect_fail} ? $expected_fail++ : $failed++;
-   }
 }
 
-sub test
-{
-   my ( $name, %params ) = @_;
+our $RUNNING_TEST;
 
-   # We expect this test to fail if it's declared to be dependent on a bug that
-   # is not yet fixed
-   $params{expect_fail}++ if $params{bug} and not $FIXED_BUGS{ $params{bug} };
-
-   my $t = $output->enter_test( $name, $params{expect_fail} );
-   _run_test( $t, %params );
-   $t->leave;
-
-   if( $t->failed ) {
-      $output->diag( $_ ) for @log_if_fail_lines;
-   }
-   if( $t->skipped ) {
-      $skipped_count++;
-   }
-
-   if( $t->failed ) {
-      no warnings 'exiting';
-
-      last TEST if $STOP_ON_FAIL and not $params{expect_fail};
-
-      warn( "This CRITICAL test has failed - bailing out\n" ), last TEST if $params{critical};
-   }
+sub pass
+{
+   my ( $testname ) = @_;
+   $RUNNING_TEST->ok( 1, $testname );
 }
 
+# A convenience for the otherwise-common pattern of
+#   ->on_done( sub { pass $message } )
+sub SyTest::pass_on_done
 {
-   our $RUNNING_TEST;
-
-   sub pass
-   {
-      my ( $testname ) = @_;
-      $RUNNING_TEST->ok( 1, $testname );
-   }
-
-   # A convenience for the otherwise-common pattern of
-   #   ->on_done( sub { pass $message } )
-   sub SyTest::pass_on_done
-   {
-      my $self = shift;
-      my ( $message ) = @_;
-      $self->on_done( sub { $RUNNING_TEST->ok( 1, $message ) } );
-   }
-
-   sub multi_test
-   {
-      my ( $name, %params ) = @_;
-
-      # We expect this test to fail if it's declared to be dependent on a bug that
-      # is not yet fixed
-      $params{expect_fail}++ if $params{bug} and not $FIXED_BUGS{ $params{bug} };
-
-      local $RUNNING_TEST = my $t = $output->enter_multi_test(
-          $name, $params{expect_fail} );
-      _run_test( $t, %params );
-      $t->leave;
-
-      if( $t->failed ) {
-         $output->diag( $_ ) for @log_if_fail_lines;
-      }
-      if( $t->skipped ) {
-         $skipped_count++;
-      }
-
-      no warnings 'exiting';
-      last TEST if $STOP_ON_FAIL and $t->failed and not $params{expect_fail};
-   }
+   my $self = shift;
+   my ( $message ) = @_;
+   $self->on_done( sub { $RUNNING_TEST->ok( 1, $message ) } );
 }
 
 my %only_files;
@@ -580,7 +527,10 @@ sub list_symbols
 
          return unless basename( $filename ) =~ m/\.pl$/;
 
-         $output->run_file( $filename );
+         no warnings 'once';
+
+         local *test       = sub { _push_test( $filename, 0, @_ ); };
+         local *multi_test = sub { _push_test( $filename, 1, @_ ); };
 
          # Slurp and eval() the file instead of do() because then lexical
          # environment such as strict/warnings will still apply
@@ -589,33 +539,13 @@ sub list_symbols
             local $/; <$fh>
          };
 
-         local $SKIPPING = 1 if %only_files and not exists $only_files{$filename};
-
          # Protect against symbolic leakage between test files by cleaning up
          # extra symbols in the 'main::' namespace
          my %was_symbs = map { $_ => 1 } list_symbols( "main" );
 
          # Tell eval what the filename is so we get nicer warnings/errors that
          # give the filename instead of (eval 123)
-         my $died_during_compile;
-
-         my $success = do {
-            local $SIG{__DIE__} = sub {
-               return if $^S;
-               $died_during_compile = 1 if !defined $^S;
-               die @_;
-            };
-
-            eval( "#line 1 $filename\n" . $code . "; 1" );
-         };
-
-         if( !$success ) {
-            die $@ if $died_during_compile;
-
-            chomp( my $e = $@ );
-            $output->abort_file( $filename, $e );
-            $failed++;
-         }
+         eval( "#line 1 $filename\n" . $code . "; 1" ) or die $@;
 
          {
             no strict 'refs';
@@ -633,6 +563,46 @@ sub list_symbols
    );
 }
 
+# Now run the tests
+my $prev_filename;
+foreach my $test ( @TESTS ) {
+   if( %only_files and not exists $only_files{ $test->file } ) {
+      $proven{$_} = PRESUMED for @{ $test->proves // [] };
+      $skipped_count++;
+      next;
+   }
+
+   if( !$prev_filename or $prev_filename ne $test->file ) {
+      $output->run_file( $prev_filename = $test->file );
+   }
+
+   my $m = $test->multi ? "enter_multi_test" : "enter_test";
+
+   my $t = $output->$m( $test->name, $test->expect_fail );
+   local $RUNNING_TEST = $t;
+
+   _run_test( $t, $test );
+
+   $t->leave;
+
+   if( $t->skipped ) {
+      $skipped_count++;
+   }
+
+   if( $t->failed ) {
+      $test->expect_fail ? $expected_fail++ : $failed++;
+
+      $output->diag( $_ ) for @log_if_fail_lines;
+
+      last if $STOP_ON_FAIL and not $test->expect_fail;
+
+      if( $test->critical ) {
+         warn "This CRITICAL test has failed - bailing out\n";
+         last;
+      }
+   }
+}
+
 if( $WAIT_AT_END ) {
    print STDERR "Waiting... (hit ENTER to end)\n";
    $loop->add( my $stdin = IO::Async::Stream->new_for_stdin( on_read => sub {} ) );