diff --git a/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/cli/DurationConverter.java b/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/cli/DurationConverter.java new file mode 100644 index 000000000000..91b9024ccb8e --- /dev/null +++ b/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/cli/DurationConverter.java @@ -0,0 +1,27 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.tests.product.launcher.cli; + +import io.airlift.units.Duration; +import picocli.CommandLine; + +public class DurationConverter + implements CommandLine.ITypeConverter +{ + @Override + public Duration convert(String value) + { + return Duration.valueOf(value); + } +} diff --git a/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/cli/SuiteRun.java b/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/cli/SuiteRun.java index 1185b3a3d900..e87d3a996bfa 100644 --- a/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/cli/SuiteRun.java +++ b/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/cli/SuiteRun.java @@ -46,6 +46,7 @@ import static com.google.common.base.MoreObjects.toStringHelper; import static io.airlift.units.Duration.nanosSince; +import static io.airlift.units.Duration.succinctNanos; import static io.prestosql.tests.product.launcher.cli.Commands.runCommand; import static java.lang.String.format; import static java.util.Objects.requireNonNull; @@ -104,6 +105,9 @@ public static class SuiteRunOptions @Option(names = "--logs-dir", paramLabel = "", description = "Location of the exported logs directory " + DEFAULT_VALUE, converter = OptionalPathConverter.class, defaultValue = "") public Optional logsDirBase; + @Option(names = "--timeout", paramLabel = "", description = "Maximum duration of suite execution " + DEFAULT_VALUE, converter = DurationConverter.class, defaultValue = "2h") + public Duration timeout; + public Module toModule() { return binder -> binder.bind(SuiteRunOptions.class).toInstance(this); @@ -118,6 +122,7 @@ public static class Execution private final SuiteFactory suiteFactory; private final EnvironmentFactory environmentFactory; private final EnvironmentConfigFactory configFactory; + private final long suiteStartTime; @Inject public Execution( @@ -132,6 +137,7 @@ public Execution( this.suiteFactory = requireNonNull(suiteFactory, "suiteFactory is null"); this.environmentFactory = requireNonNull(environmentFactory, "environmentFactory is null"); this.configFactory = requireNonNull(configFactory, "configFactory is null"); + this.suiteStartTime = System.nanoTime(); } @Override @@ -155,8 +161,6 @@ public Integer call() testRun.getTests(), testRun.getExcludedTests()); } - - long suiteStartTime = System.nanoTime(); ImmutableList.Builder results = ImmutableList.builder(); for (int runId = 0; runId < suiteTestRuns.size(); runId++) { @@ -164,20 +168,20 @@ public Integer call() } List testRunsResults = results.build(); - printTestRunsSummary(suiteStartTime, suiteName, testRunsResults); + printTestRunsSummary(suiteName, testRunsResults); return getFailedCount(testRunsResults) == 0 ? ExitCode.OK : ExitCode.SOFTWARE; } - private static int printTestRunsSummary(long startTime, String suiteName, List results) + private int printTestRunsSummary(String suiteName, List results) { long failedRuns = getFailedCount(results); if (failedRuns > 0) { - log.info("Suite %s failed in %s (%d passed, %d failed): ", suiteName, nanosSince(startTime), results.size() - failedRuns, failedRuns); + log.info("Suite %s failed in %s (%d passed, %d failed): ", suiteName, nanosSince(suiteStartTime), results.size() - failedRuns, failedRuns); } else { - log.info("Suite %s succeeded in %s: ", suiteName, nanosSince(startTime)); + log.info("Suite %s succeeded in %s: ", suiteName, nanosSince(suiteStartTime)); } results.stream() @@ -210,8 +214,8 @@ private static void printTestRunSummary(TestRunResult result) public TestRunResult executeSuiteTestRun(int runId, String suiteName, SuiteTestRun suiteTestRun, EnvironmentConfig environmentConfig) { - log.info("Starting test run #%02d %s with config %s", runId, suiteTestRun, environmentConfig); TestRun.TestRunOptions testRunOptions = createTestRunOptions(runId, suiteName, suiteTestRun, environmentConfig, suiteRunOptions.logsDirBase); + log.info("Starting test run #%02d %s with config %s and remaining timeout %s", runId, suiteTestRun, environmentConfig, testRunOptions.timeout); log.info("Execute this test run using:\n%s test run %s", environmentOptions.launcherBin, OptionsPrinter.format(environmentOptions, testRunOptions)); Stopwatch stopwatch = Stopwatch.createStarted(); @@ -245,9 +249,17 @@ private TestRun.TestRunOptions createTestRunOptions(int runId, String suiteName, String suiteRunId = suiteRunId(runId, suiteName, suiteTestRun, environmentConfig); testRunOptions.reportsDir = Paths.get("presto-product-tests/target/reports/" + suiteRunId); testRunOptions.logsDirBase = logsDirBase.map(dir -> dir.resolve(suiteRunId)); + // Calculate remaining time + testRunOptions.timeout = remainingTimeout(); return testRunOptions; } + private Duration remainingTimeout() + { + return succinctNanos( + suiteRunOptions.timeout.roundTo(NANOSECONDS) - nanosSince(suiteStartTime).roundTo(NANOSECONDS)); + } + private static String suiteRunId(int runId, String suiteName, SuiteTestRun suiteTestRun, EnvironmentConfig environmentConfig) { return format("%s-%s-%s-%02d", suiteName, suiteTestRun.getEnvironmentName(), environmentConfig.getConfigName(), runId); diff --git a/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/cli/TestRun.java b/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/cli/TestRun.java index a8d2446e2746..77cbc6a929a9 100644 --- a/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/cli/TestRun.java +++ b/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/cli/TestRun.java @@ -17,6 +17,7 @@ import com.google.common.collect.ImmutableList; import com.google.inject.Module; import io.airlift.log.Logger; +import io.airlift.units.Duration; import io.prestosql.tests.product.launcher.Extensions; import io.prestosql.tests.product.launcher.LauncherModule; import io.prestosql.tests.product.launcher.env.DockerContainer; @@ -27,6 +28,9 @@ import io.prestosql.tests.product.launcher.env.EnvironmentOptions; import io.prestosql.tests.product.launcher.env.common.Standard; import io.prestosql.tests.product.launcher.testcontainers.ExistingNetwork; +import net.jodah.failsafe.Failsafe; +import net.jodah.failsafe.Timeout; +import net.jodah.failsafe.TimeoutExceededException; import org.testcontainers.containers.Container; import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy; import picocli.CommandLine.ExitCode; @@ -116,6 +120,9 @@ public static class TestRunOptions @Option(names = "--startup-retries", paramLabel = "", description = "Environment startup retries " + DEFAULT_VALUE, defaultValue = "5") public Integer startupRetries = 5; + @Option(names = "--timeout", paramLabel = "", description = "Maximum duration of tests execution " + DEFAULT_VALUE, converter = DurationConverter.class, defaultValue = "2h") + public Duration timeout; + @Parameters(paramLabel = "", description = "Test arguments") public List testArguments; @@ -135,6 +142,7 @@ public static class Execution private final List testArguments; private final String environment; private final boolean attach; + private final Duration timeout; private final DockerContainer.OutputMode outputMode; private final int startupRetries; private final Path reportsDirBase; @@ -151,6 +159,7 @@ public Execution(EnvironmentFactory environmentFactory, EnvironmentOptions envir this.testArguments = ImmutableList.copyOf(requireNonNull(testRunOptions.testArguments, "testOptions.testArguments is null")); this.environment = requireNonNull(testRunOptions.environment, "testRunOptions.environment is null"); this.attach = testRunOptions.attach; + this.timeout = requireNonNull(testRunOptions.timeout, "testRunOptions.timeout is null"); this.outputMode = requireNonNull(environmentOptions.output, "environmentOptions.output is null"); this.startupRetries = testRunOptions.startupRetries; this.reportsDirBase = requireNonNull(testRunOptions.reportsDir, "testRunOptions.reportsDirBase is empty"); @@ -161,14 +170,27 @@ public Execution(EnvironmentFactory environmentFactory, EnvironmentOptions envir @Override public Integer call() { - try (Environment environment = startEnvironment()) { - return toIntExact(environment.awaitTestsCompletion()); + try { + return Failsafe + .with(Timeout.of(java.time.Duration.ofMillis(timeout.toMillis())) + .withCancel(true)) + .get(() -> tryExecuteTests()); + } + catch (TimeoutExceededException ignored) { + log.error("Test execution exceeded timeout of %s", timeout); } catch (Throwable e) { // log failure (tersely) because cleanup may take some time log.error("Failure: %s", getStackTraceAsString(e)); + } - return ExitCode.SOFTWARE; + return ExitCode.SOFTWARE; + } + + private Integer tryExecuteTests() + { + try (Environment environment = startEnvironment()) { + return toIntExact(environment.awaitTestsCompletion()); } } diff --git a/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/Environment.java b/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/Environment.java index 5c18bcfe7685..c0c7a11087c3 100644 --- a/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/Environment.java +++ b/presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/Environment.java @@ -158,6 +158,8 @@ public long awaitTestsCompletion() return exitCode; } catch (InterruptedException e) { + // Gracefully stop environment and trigger listeners + stop(); Thread.currentThread().interrupt(); throw new RuntimeException("Interrupted", e); }