Add timeout for suite and test execution

presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/cli/DurationConverter.java

@@ -0,0 +1,27 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.prestosql.tests.product.launcher.cli;
+
+import io.airlift.units.Duration;
+import picocli.CommandLine;
+
+public class DurationConverter
+        implements CommandLine.ITypeConverter<Duration>
+{
+    @Override
+    public Duration convert(String value)
+    {
+        return Duration.valueOf(value);
+    }
+}

presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/cli/SuiteRun.java

@@ -46,6 +46,7 @@
 
 import static com.google.common.base.MoreObjects.toStringHelper;
 import static io.airlift.units.Duration.nanosSince;
+import static io.airlift.units.Duration.succinctNanos;
 import static io.prestosql.tests.product.launcher.cli.Commands.runCommand;
 import static java.lang.String.format;
 import static java.util.Objects.requireNonNull;
@@ -104,6 +105,9 @@ public static class SuiteRunOptions
         @Option(names = "--logs-dir", paramLabel = "<dir>", description = "Location of the exported logs directory " + DEFAULT_VALUE, converter = OptionalPathConverter.class, defaultValue = "")
         public Optional<Path> logsDirBase;
 
+        @Option(names = "--timeout", paramLabel = "<timeout>", description = "Maximum duration of suite execution " + DEFAULT_VALUE, converter = DurationConverter.class, defaultValue = "2h")
+        public Duration timeout;
+
         public Module toModule()
         {
             return binder -> binder.bind(SuiteRunOptions.class).toInstance(this);
@@ -118,6 +122,7 @@ public static class Execution
         private final SuiteFactory suiteFactory;
         private final EnvironmentFactory environmentFactory;
         private final EnvironmentConfigFactory configFactory;
+        private final long suiteStartTime;
 
         @Inject
         public Execution(
@@ -132,6 +137,7 @@ public Execution(
             this.suiteFactory = requireNonNull(suiteFactory, "suiteFactory is null");
             this.environmentFactory = requireNonNull(environmentFactory, "environmentFactory is null");
             this.configFactory = requireNonNull(configFactory, "configFactory is null");
+            this.suiteStartTime = System.nanoTime();
         }
 
         @Override
@@ -155,29 +161,27 @@ public Integer call()
                         testRun.getTests(),
                         testRun.getExcludedTests());
             }
-
-            long suiteStartTime = System.nanoTime();
             ImmutableList.Builder<TestRunResult> results = ImmutableList.builder();
 
             for (int runId = 0; runId < suiteTestRuns.size(); runId++) {
                 results.add(executeSuiteTestRun(runId + 1, suiteName, suiteTestRuns.get(runId), environmentConfig));
             }
 
             List<TestRunResult> testRunsResults = results.build();
-            printTestRunsSummary(suiteStartTime, suiteName, testRunsResults);
+            printTestRunsSummary(suiteName, testRunsResults);
 
             return getFailedCount(testRunsResults) == 0 ? ExitCode.OK : ExitCode.SOFTWARE;
         }
 
-        private static int printTestRunsSummary(long startTime, String suiteName, List<TestRunResult> results)
+        private int printTestRunsSummary(String suiteName, List<TestRunResult> results)
         {
             long failedRuns = getFailedCount(results);
 
             if (failedRuns > 0) {
-                log.info("Suite %s failed in %s (%d passed, %d failed): ", suiteName, nanosSince(startTime), results.size() - failedRuns, failedRuns);
+                log.info("Suite %s failed in %s (%d passed, %d failed): ", suiteName, nanosSince(suiteStartTime), results.size() - failedRuns, failedRuns);
             }
             else {
-                log.info("Suite %s succeeded in %s: ", suiteName, nanosSince(startTime));
+                log.info("Suite %s succeeded in %s: ", suiteName, nanosSince(suiteStartTime));
             }
 
             results.stream()
@@ -210,8 +214,8 @@ private static void printTestRunSummary(TestRunResult result)
 
         public TestRunResult executeSuiteTestRun(int runId, String suiteName, SuiteTestRun suiteTestRun, EnvironmentConfig environmentConfig)
         {
-            log.info("Starting test run #%02d %s with config %s", runId, suiteTestRun, environmentConfig);
             TestRun.TestRunOptions testRunOptions = createTestRunOptions(runId, suiteName, suiteTestRun, environmentConfig, suiteRunOptions.logsDirBase);
+            log.info("Starting test run #%02d %s with config %s and remaining timeout %s", runId, suiteTestRun, environmentConfig, testRunOptions.timeout);
             log.info("Execute this test run using:\n%s test run %s", environmentOptions.launcherBin, OptionsPrinter.format(environmentOptions, testRunOptions));
 
             Stopwatch stopwatch = Stopwatch.createStarted();
@@ -245,9 +249,17 @@ private TestRun.TestRunOptions createTestRunOptions(int runId, String suiteName,
             String suiteRunId = suiteRunId(runId, suiteName, suiteTestRun, environmentConfig);
             testRunOptions.reportsDir = Paths.get("presto-product-tests/target/reports/" + suiteRunId);
             testRunOptions.logsDirBase = logsDirBase.map(dir -> dir.resolve(suiteRunId));
+            // Calculate remaining time
+            testRunOptions.timeout = remainingTimeout();
             return testRunOptions;
         }
 
+        private Duration remainingTimeout()
+        {
+            return succinctNanos(
+                    suiteRunOptions.timeout.roundTo(NANOSECONDS) - nanosSince(suiteStartTime).roundTo(NANOSECONDS));
+        }
+
         private static String suiteRunId(int runId, String suiteName, SuiteTestRun suiteTestRun, EnvironmentConfig environmentConfig)
         {
             return format("%s-%s-%s-%02d", suiteName, suiteTestRun.getEnvironmentName(), environmentConfig.getConfigName(), runId);

presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/cli/TestRun.java

@@ -17,6 +17,7 @@
 import com.google.common.collect.ImmutableList;
 import com.google.inject.Module;
 import io.airlift.log.Logger;
+import io.airlift.units.Duration;
 import io.prestosql.tests.product.launcher.Extensions;
 import io.prestosql.tests.product.launcher.LauncherModule;
 import io.prestosql.tests.product.launcher.env.DockerContainer;
@@ -27,6 +28,9 @@
 import io.prestosql.tests.product.launcher.env.EnvironmentOptions;
 import io.prestosql.tests.product.launcher.env.common.Standard;
 import io.prestosql.tests.product.launcher.testcontainers.ExistingNetwork;
+import net.jodah.failsafe.Failsafe;
+import net.jodah.failsafe.Timeout;
+import net.jodah.failsafe.TimeoutExceededException;
 import org.testcontainers.containers.Container;
 import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy;
 import picocli.CommandLine.ExitCode;
@@ -116,6 +120,9 @@ public static class TestRunOptions
         @Option(names = "--startup-retries", paramLabel = "<retries>", description = "Environment startup retries " + DEFAULT_VALUE, defaultValue = "5")
         public Integer startupRetries = 5;
 
+        @Option(names = "--timeout", paramLabel = "<timeout>", description = "Maximum duration of tests execution " + DEFAULT_VALUE, converter = DurationConverter.class, defaultValue = "2h")
+        public Duration timeout;
+
         @Parameters(paramLabel = "<argument>", description = "Test arguments")
         public List<String> testArguments;
 
@@ -135,6 +142,7 @@ public static class Execution
         private final List<String> testArguments;
         private final String environment;
         private final boolean attach;
+        private final Duration timeout;
         private final DockerContainer.OutputMode outputMode;
         private final int startupRetries;
         private final Path reportsDirBase;
@@ -151,6 +159,7 @@ public Execution(EnvironmentFactory environmentFactory, EnvironmentOptions envir
             this.testArguments = ImmutableList.copyOf(requireNonNull(testRunOptions.testArguments, "testOptions.testArguments is null"));
             this.environment = requireNonNull(testRunOptions.environment, "testRunOptions.environment is null");
             this.attach = testRunOptions.attach;
+            this.timeout = requireNonNull(testRunOptions.timeout, "testRunOptions.timeout is null");
             this.outputMode = requireNonNull(environmentOptions.output, "environmentOptions.output is null");
             this.startupRetries = testRunOptions.startupRetries;
             this.reportsDirBase = requireNonNull(testRunOptions.reportsDir, "testRunOptions.reportsDirBase is empty");
@@ -161,14 +170,27 @@ public Execution(EnvironmentFactory environmentFactory, EnvironmentOptions envir
         @Override
         public Integer call()
         {
-            try (Environment environment = startEnvironment()) {
-                return toIntExact(environment.awaitTestsCompletion());
+            try {
+                return Failsafe
+                        .with(Timeout.of(java.time.Duration.ofMillis(timeout.toMillis()))
+                                .withCancel(true))
+                        .get(() -> tryExecuteTests());
+            }
+            catch (TimeoutExceededException ignored) {
+                log.error("Test execution exceeded timeout of %s", timeout);
             }
             catch (Throwable e) {
                 // log failure (tersely) because cleanup may take some time
                 log.error("Failure: %s", getStackTraceAsString(e));
+            }
 
-                return ExitCode.SOFTWARE;
+            return ExitCode.SOFTWARE;
+        }
+
+        private Integer tryExecuteTests()
+        {
+            try (Environment environment = startEnvironment()) {
+                return toIntExact(environment.awaitTestsCompletion());
             }
         }
 

presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/Environment.java

@@ -158,6 +158,8 @@ public long awaitTestsCompletion()
             return exitCode;
         }
         catch (InterruptedException e) {
+            // Gracefully stop environment and trigger listeners
+            stop();
             Thread.currentThread().interrupt();
             throw new RuntimeException("Interrupted", e);
         }