NEW Rerun failed features in ci

config/silverstripe.yml

@@ -8,6 +8,7 @@ parameters:
   silverstripe_extension.ajax_steps: ~
   silverstripe_extension.ajax_timeout: ~
   silverstripe_extension.admin_url: ~
+  silverstripe_extension.is_ci: ~
   silverstripe_extension.login_url: ~
   silverstripe_extension.screenshot_path: ~
   silverstripe_extension.module:

src/Extension.php

@@ -19,6 +19,9 @@
 use Symfony\Component\Console\Input\ArgvInput;
 use Symfony\Component\DependencyInjection\Definition;
 use Symfony\Component\DependencyInjection\Reference;
+use Behat\Behat\Tester\ServiceContainer\TesterExtension;
+use SilverStripe\BehatExtension\Utility\RerunTotalStatistics;
+use SilverStripe\BehatExtension\Utility\RerunRuntimeSuiteTester;
 
 /*
  * This file is part of the SilverStripe\BehatExtension
@@ -100,6 +103,22 @@ public function load(ContainerBuilder $container, array $config)
              $container->setParameter('silverstripe_extension.region_map', $config['region_map']);
         }
         $container->setParameter('silverstripe_extension.bootstrap_file', $config['bootstrap_file']);
+        $container->setParameter('silverstripe_extension.is_ci', $config['is_ci']);
+
+        // When running in CI, behat scenarios will occasionally sporadically fail
+        // Replaces services with custom implementations that will rerun failed features
+        // Note that features rather than scenarios need to be rerun to ensure that
+        // everything is setup and torn down correctly and that "Background" bits of
+        // feature fits are rerun
+        if ($config['is_ci']) {
+            $definition = new Definition(RerunRuntimeSuiteTester::class, array(
+                new Reference(TesterExtension::SPECIFICATION_TESTER_ID)
+            ));
+            $container->setDefinition(TesterExtension::SUITE_TESTER_ID, $definition);
+
+            $definition = new Definition(RerunTotalStatistics::class);
+            $container->setDefinition('output.pretty.statistics', $definition);
+        }
     }
 
     /**
@@ -144,6 +163,9 @@ public function configure(ArrayNodeDefinition $builder)
                     info('Number of seconds that @retry tags will retry for')->
                     defaultValue(2)->
                 end()->
+                scalarNode('is_ci')->
+                    defaultValue(false)->
+                end()->
                 arrayNode('ajax_steps')->
                     defaultValue(array(
                         'go to',

src/Utility/RerunRuntimeSuiteTester.php

@@ -0,0 +1,80 @@
+<?php
+
+namespace SilverStripe\BehatExtension\Utility;
+
+use Behat\Testwork\Environment\Environment;
+use Behat\Testwork\Specification\SpecificationIterator;
+use Behat\Testwork\Tester\Result\IntegerTestResult;
+use Behat\Testwork\Tester\Result\TestResult;
+use Behat\Testwork\Tester\Result\TestResults;
+use Behat\Testwork\Tester\Result\TestWithSetupResult;
+use Behat\Testwork\Tester\Setup\SuccessfulSetup;
+use Behat\Testwork\Tester\Setup\SuccessfulTeardown;
+use Behat\Testwork\Tester\SpecificationTester;
+use Behat\Testwork\Tester\SuiteTester;
+
+/**
+ * Copy paste of Behat\Testwork\Tester\Runtime\RuntimeSuiteTester which is a final class
+ *
+ * Reruns failed features
+ */
+class RerunRuntimeSuiteTester implements SuiteTester
+{
+    /**
+     * @var SpecificationTester
+     */
+    private $specTester;
+
+    /**
+     * Initializes tester.
+     *
+     * @param SpecificationTester $specTester
+     */
+    public function __construct(SpecificationTester $specTester)
+    {
+        $this->specTester = $specTester;
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function setUp(Environment $env, SpecificationIterator $iterator, $skip)
+    {
+        return new SuccessfulSetup();
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function test(Environment $env, SpecificationIterator $iterator, $skip = false)
+    {
+        $results = array();
+        foreach ($iterator as $specification) {
+            $setup = $this->specTester->setUp($env, $specification, $skip);
+            $localSkip = !$setup->isSuccessful() || $skip;
+            $testResult = $this->specTester->test($env, $specification, $localSkip);
+            $teardown = $this->specTester->tearDown($env, $specification, $localSkip, $testResult);
+
+            if (!$testResult->isPassed()) {
+                file_put_contents('php://stdout', 'Retrying specification' . PHP_EOL);
+                $setup = $this->specTester->setUp($env, $specification, $skip);
+                $localSkip = !$setup->isSuccessful() || $skip;
+                $testResult = $this->specTester->test($env, $specification, $localSkip);
+                $teardown = $this->specTester->tearDown($env, $specification, $localSkip, $testResult);
+            }
+
+            $integerResult = new IntegerTestResult($testResult->getResultCode());
+            $results[] = new TestWithSetupResult($setup, $integerResult, $teardown);
+        }
+
+        return new TestResults($results);
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function tearDown(Environment $env, SpecificationIterator $iterator, $skip, TestResult $result)
+    {
+        return new SuccessfulTeardown();
+    }
+}