NEW Add RetryScenarioTester

config/silverstripe.yml

@@ -8,6 +8,7 @@ parameters:
   silverstripe_extension.ajax_steps: ~
   silverstripe_extension.ajax_timeout: ~
   silverstripe_extension.admin_url: ~
+  silverstripe_extension.is_ci: ~
   silverstripe_extension.login_url: ~
   silverstripe_extension.screenshot_path: ~
   silverstripe_extension.module:

src/Extension.php

@@ -19,6 +19,8 @@
 use Symfony\Component\Console\Input\ArgvInput;
 use Symfony\Component\DependencyInjection\Definition;
 use Symfony\Component\DependencyInjection\Reference;
+use Behat\Behat\Tester\ServiceContainer\TesterExtension;
+use SilverStripe\BehatExtension\Utility\RetryScenarioTester;
 
 /*
  * This file is part of the SilverStripe\BehatExtension
@@ -100,6 +102,17 @@ public function load(ContainerBuilder $container, array $config)
              $container->setParameter('silverstripe_extension.region_map', $config['region_map']);
         }
         $container->setParameter('silverstripe_extension.bootstrap_file', $config['bootstrap_file']);
+        $container->setParameter('silverstripe_extension.is_ci', $config['is_ci']);
+
+        // When running in CI, behat scenarios will occasionally sporadically fail
+        // decorate the scenario tester with our own class so we can retry it once if it fails
+        if ($config['is_ci']) {
+            $definition = new Definition(RetryScenarioTester::class, [
+                new Reference(TesterExtension::SCENARIO_TESTER_ID),
+            ]);
+            $definition->addTag(TesterExtension::SCENARIO_TESTER_WRAPPER_TAG);
+            $container->setDefinition(TesterExtension::SCENARIO_TESTER_WRAPPER_TAG . '.silverstripe', $definition);
+        }
     }
 
     /**
@@ -144,6 +157,9 @@ public function configure(ArrayNodeDefinition $builder)
                     info('Number of seconds that @retry tags will retry for')->
                     defaultValue(2)->
                 end()->
+                scalarNode('is_ci')->
+                    defaultValue(false)->
+                end()->
                 arrayNode('ajax_steps')->
                     defaultValue(array(
                         'go to',

src/Utility/RetryScenarioTester.php

@@ -0,0 +1,46 @@
+<?php
+
+namespace SilverStripe\BehatExtension\Utility;
+
+use Behat\Behat\Tester\ScenarioTester;
+use Behat\Gherkin\Node\FeatureNode;
+use Behat\Gherkin\Node\ScenarioInterface;
+use Behat\Testwork\Environment\Environment;
+use Behat\Testwork\Tester\Result\TestResult;
+
+/**
+ * This tester will retry a scenario if it fails within the same test run.
+ * Note that this is different from the following retry solutions:
+ * - RetryableCallHandler using the @retry tag, which gives assertion an additional 3 seconds
+ * - RerunController, which uses the --rerun option to run only previously failed scenarios in a subsequent test run
+ */
+class RetryScenarioTester implements ScenarioTester
+{
+    private ScenarioTester $decoratedTester;
+
+    public function __construct(ScenarioTester $decoratedTester)
+    {
+        $this->decoratedTester = $decoratedTester;
+    }
+
+    public function setUp(Environment $env, FeatureNode $feature, ScenarioInterface $scenario, $skip)
+    {
+        return $this->decoratedTester->setUp($env, $feature, $scenario, $skip);
+    }
+
+    public function test(Environment $env, FeatureNode $feature, ScenarioInterface $scenario, $skip)
+    {
+        $result = $this->decoratedTester->test($env, $feature, $scenario, $skip);
+        if (!$skip && !$result->isPassed()) {
+            // Run the scenario again
+            file_put_contents('php://stdout', 'Retrying scenario: ' . $scenario->getTitle() . PHP_EOL);
+            $result = $this->decoratedTester->test($env, $feature, $scenario, $skip);
+        }
+        return $result;
+    }
+
+    public function tearDown(Environment $env, FeatureNode $feature, ScenarioInterface $scenario, $skip, TestResult $result)
+    {
+        return $this->decoratedTester->tearDown($env, $feature, $scenario, $skip, $result);
+    }
+}