From ab205f6a56eaf0e8446a946003d6608b28058fc2 Mon Sep 17 00:00:00 2001 From: "Santiago M. Mola" Date: Fri, 20 Dec 2024 10:48:14 +0100 Subject: [PATCH] Check logs for errors at smoke tests cleanup (#8111) --- .../datadog/smoketest/ArmeriaSmokeTest.groovy | 2 - .../AsmStandaloneBillingSmokeTest.groovy | 6 +- .../CustomSystemLoaderSmokeTest.groovy | 8 +- .../springboot/controller/SsrfController.java | 5 +- .../AbstractIastServerSmokeTest.groovy | 10 +-- .../AbstractIastSpringBootTest.groovy | 66 ++++++++-------- .../smoketest/Java9ModulesSmokeTest.groovy | 6 ++ .../smoketest/AbstractModulesSmokeTest.groovy | 25 +++---- .../smoketest/LogInjectionSmokeTest.groovy | 15 ++++ .../smoketest/AbstractOSGiSmokeTest.groovy | 23 ++---- .../smoketest/QuarkusNativeSmokeTest.groovy | 2 - .../datadog/smoketest/QuarkusSmokeTest.groovy | 2 - ...SpringBootNativeInstrumentationTest.groovy | 18 ++--- .../SpringBootRabbitIntegrationTest.groovy | 8 ++ .../smoketest/IastSpringBootSmokeTest.groovy | 18 +++-- .../smoketest/AbstractSmokeTest.groovy | 1 + .../datadog/smoketest/ProcessManager.groovy | 75 ++++++++++++------- 17 files changed, 156 insertions(+), 134 deletions(-) diff --git a/dd-smoke-tests/armeria-grpc/src/test/groovy/datadog/smoketest/ArmeriaSmokeTest.groovy b/dd-smoke-tests/armeria-grpc/src/test/groovy/datadog/smoketest/ArmeriaSmokeTest.groovy index afe49bcd14e..3efb7a9cd1d 100644 --- a/dd-smoke-tests/armeria-grpc/src/test/groovy/datadog/smoketest/ArmeriaSmokeTest.groovy +++ b/dd-smoke-tests/armeria-grpc/src/test/groovy/datadog/smoketest/ArmeriaSmokeTest.groovy @@ -57,8 +57,6 @@ class ArmeriaSmokeTest extends AbstractServerSmokeTest { }) waitForTraceCount(totalInvocations) >= totalInvocations validateLogInjection() == totalInvocations - checkLogPostExit() - !logHasErrors } void doAndValidateRequest(int id) { diff --git a/dd-smoke-tests/asm-standalone-billing/src/test/groovy/datadog/smoketest/asmstandalonebilling/AsmStandaloneBillingSmokeTest.groovy b/dd-smoke-tests/asm-standalone-billing/src/test/groovy/datadog/smoketest/asmstandalonebilling/AsmStandaloneBillingSmokeTest.groovy index fe67336538d..f5d7761b676 100644 --- a/dd-smoke-tests/asm-standalone-billing/src/test/groovy/datadog/smoketest/asmstandalonebilling/AsmStandaloneBillingSmokeTest.groovy +++ b/dd-smoke-tests/asm-standalone-billing/src/test/groovy/datadog/smoketest/asmstandalonebilling/AsmStandaloneBillingSmokeTest.groovy @@ -64,10 +64,8 @@ class AsmStandaloneBillingSmokeTest extends AbstractAsmStandaloneBillingSmokeTes def computedStatsHeader = lastTraceRequestHeaders.get('Datadog-Client-Computed-Stats') assert computedStatsHeader != null && computedStatsHeader == 'true' - then:'metrics should be disabled' - checkLogPostExit { log -> - return log.contains('datadog.trace.agent.common.metrics.MetricsAggregatorFactory - tracer metrics disabled') - } + then: 'metrics should be disabled' + isLogPresent { it.contains('datadog.trace.agent.common.metrics.MetricsAggregatorFactory - tracer metrics disabled') } } void 'test _dd.p.appsec propagation for appsec event'() { diff --git a/dd-smoke-tests/custom-systemloader/src/test/groovy/datadog/smoketest/CustomSystemLoaderSmokeTest.groovy b/dd-smoke-tests/custom-systemloader/src/test/groovy/datadog/smoketest/CustomSystemLoaderSmokeTest.groovy index 4b476c4207d..ba8e64c8860 100644 --- a/dd-smoke-tests/custom-systemloader/src/test/groovy/datadog/smoketest/CustomSystemLoaderSmokeTest.groovy +++ b/dd-smoke-tests/custom-systemloader/src/test/groovy/datadog/smoketest/CustomSystemLoaderSmokeTest.groovy @@ -30,9 +30,12 @@ class CustomSystemLoaderSmokeTest extends AbstractSmokeTest { def "resource types loaded by custom system class-loader are transformed"() { when: testedProcess.waitFor(TIMEOUT_SECS, SECONDS) + + then: + testedProcess.exitValue() == 0 int loadedResources = 0 int transformedResources = 0 - checkLogPostExit { + forEachLogLine { String it -> if (it =~ /Loading sample.app.Resource[$]Test[1-3] from TestLoader/) { loadedResources++ } @@ -40,10 +43,7 @@ class CustomSystemLoaderSmokeTest extends AbstractSmokeTest { transformedResources++ } } - then: - testedProcess.exitValue() == 0 loadedResources == 3 transformedResources == 3 - !logHasErrors } } diff --git a/dd-smoke-tests/iast-util/src/main/java/datadog/smoketest/springboot/controller/SsrfController.java b/dd-smoke-tests/iast-util/src/main/java/datadog/smoketest/springboot/controller/SsrfController.java index 411e857f9db..64bcdc4ce66 100644 --- a/dd-smoke-tests/iast-util/src/main/java/datadog/smoketest/springboot/controller/SsrfController.java +++ b/dd-smoke-tests/iast-util/src/main/java/datadog/smoketest/springboot/controller/SsrfController.java @@ -80,7 +80,10 @@ public String okHttp2(@RequestParam(value = "url") final String url) { } catch (final Exception e) { } client.getDispatcher().getExecutorService().shutdown(); - client.getConnectionPool().evictAll(); + com.squareup.okhttp.ConnectionPool pool = client.getConnectionPool(); + if (pool != null) { + pool.evictAll(); + } return "ok"; } diff --git a/dd-smoke-tests/iast-util/src/testFixtures/groovy/datadog/smoketest/AbstractIastServerSmokeTest.groovy b/dd-smoke-tests/iast-util/src/testFixtures/groovy/datadog/smoketest/AbstractIastServerSmokeTest.groovy index c6b200197a5..b335e85b33f 100644 --- a/dd-smoke-tests/iast-util/src/testFixtures/groovy/datadog/smoketest/AbstractIastServerSmokeTest.groovy +++ b/dd-smoke-tests/iast-util/src/testFixtures/groovy/datadog/smoketest/AbstractIastServerSmokeTest.groovy @@ -54,10 +54,7 @@ abstract class AbstractIastServerSmokeTest extends AbstractServerSmokeTest { try { processTestLogLines(closure) } catch (TimeoutException toe) { - checkLogPostExit(closure) - if (!found) { - throw new AssertionError("No matching tainted found. Tainteds found: ${new JsonBuilder(tainteds).toPrettyString()}") - } + assert found, "No matching tainted found. Tainteds found: ${new JsonBuilder(tainteds).toPrettyString()}" } } @@ -83,10 +80,7 @@ abstract class AbstractIastServerSmokeTest extends AbstractServerSmokeTest { try { processTestLogLines(closure) } catch (TimeoutException toe) { - checkLogPostExit(closure) - if (!found) { - throw new AssertionError("No matching vulnerability found. Vulnerabilities found: ${new JsonBuilder(vulnerabilities).toPrettyString()}") - } + assert found, "No matching vulnerability found. Vulnerabilities found: ${new JsonBuilder(vulnerabilities).toPrettyString()}" } } diff --git a/dd-smoke-tests/iast-util/src/testFixtures/groovy/datadog/smoketest/AbstractIastSpringBootTest.groovy b/dd-smoke-tests/iast-util/src/testFixtures/groovy/datadog/smoketest/AbstractIastSpringBootTest.groovy index 8dbe33e52cb..65e46c6c444 100644 --- a/dd-smoke-tests/iast-util/src/testFixtures/groovy/datadog/smoketest/AbstractIastSpringBootTest.groovy +++ b/dd-smoke-tests/iast-util/src/testFixtures/groovy/datadog/smoketest/AbstractIastSpringBootTest.groovy @@ -39,33 +39,22 @@ abstract class AbstractIastSpringBootTest extends AbstractIastServerSmokeTest { ] } - void 'IAST subsystem starts'() { - given: 'an initial request has succeeded' - String url = "http://localhost:${httpPort}/greeting" - def request = new Request.Builder().url(url).get().build() - client.newCall(request).execute() + @Override + boolean isErrorLog(String log) { + if (log.contains('no such algorithm: DES for provider SUN')) { + return false + } - when: 'logs are read' - String startMsg = null - String errorMsg = null - checkLogPostExit { - if (it.contains('Not starting IAST subsystem')) { - errorMsg = it - } - if (it.contains('IAST is starting')) { - startMsg = it - } - // Check that there's no logged exception about missing classes from Datadog. - // We had this problem before with JDK9StackWalker. - if (it.contains('java.lang.ClassNotFoundException: datadog/')) { - errorMsg = it - } + if (super.isErrorLog(log) || log.contains('Not starting IAST subsystem')) { + return true + } + // Check that there's no logged exception about missing classes from Datadog. + // We had this problem before with JDK9StackWalker. + if (log.contains('java.lang.ClassNotFoundException: datadog/')) { + return true } - then: 'there are no errors in the log and IAST has started' - errorMsg == null - startMsg != null - !logHasErrors + return false } void 'default home page without errors'() { @@ -82,9 +71,6 @@ abstract class AbstractIastSpringBootTest extends AbstractIastServerSmokeTest { responseBodyStr.contains('Sup Dawg') response.body().contentType().toString().contains('text/plain') response.code() == 200 - - checkLogPostExit() - !logHasErrors } void 'Multipart Request parameters'() { @@ -329,13 +315,21 @@ abstract class AbstractIastSpringBootTest extends AbstractIastServerSmokeTest { def request = new Request.Builder().url(url).get().build() when: 'ensure the controller is loaded' - client.newCall(request).execute() + def resp = client.newCall(request).execute() - then: 'a vulnerability pops in the logs (startup traces might not always be available)' - hasVulnerabilityInLogs { vul -> - vul.type == 'WEAK_HASH' && - vul.evidence.value == 'SHA1' && - vul.location.spanId > 0 + then: + resp.code() == 200 + resp.close() + + and: 'a vulnerability pops in the logs (startup traces might not always be available)' + boolean found = false + isLogPresent { String log -> + def vulns = parseVulnerabilitiesLog(log) + vulns.any { vul -> + vul.type == 'WEAK_HASH' && + vul.evidence.value == 'SHA1' && + vul.location.spanId > 0 + } } } @@ -1060,8 +1054,10 @@ abstract class AbstractIastSpringBootTest extends AbstractIastServerSmokeTest { then: response.successful - hasVulnerabilityInLogs { vul -> - vul.type == 'SESSION_REWRITING' + // Vulnerability may have been detected in a previous request instead, check the full logs. + isLogPresent { String log -> + def vulns = parseVulnerabilitiesLog(log) + vulns.any { it.type == 'SESSION_REWRITING' } } } diff --git a/dd-smoke-tests/java9-modules/src/test/groovy/datadog/smoketest/Java9ModulesSmokeTest.groovy b/dd-smoke-tests/java9-modules/src/test/groovy/datadog/smoketest/Java9ModulesSmokeTest.groovy index d5d57f9b45e..63e0bcdb944 100644 --- a/dd-smoke-tests/java9-modules/src/test/groovy/datadog/smoketest/Java9ModulesSmokeTest.groovy +++ b/dd-smoke-tests/java9-modules/src/test/groovy/datadog/smoketest/Java9ModulesSmokeTest.groovy @@ -23,6 +23,12 @@ class Java9ModulesSmokeTest extends AbstractSmokeTest { processBuilder.directory(new File(buildDirectory)) } + @Override + boolean isErrorLog(String line) { + // FIXME: Too many bootstrap errors. + return false + } + def "Module application runs correctly"() { expect: assert testedProcess.waitFor(TIMEOUT_SECS, SECONDS) diff --git a/dd-smoke-tests/jboss-modules/src/test/groovy/datadog/smoketest/AbstractModulesSmokeTest.groovy b/dd-smoke-tests/jboss-modules/src/test/groovy/datadog/smoketest/AbstractModulesSmokeTest.groovy index bb80b54f473..a596bca3bcd 100644 --- a/dd-smoke-tests/jboss-modules/src/test/groovy/datadog/smoketest/AbstractModulesSmokeTest.groovy +++ b/dd-smoke-tests/jboss-modules/src/test/groovy/datadog/smoketest/AbstractModulesSmokeTest.groovy @@ -28,27 +28,20 @@ abstract class AbstractModulesSmokeTest extends AbstractSmokeTest { return processBuilder } + @Override + boolean isErrorLog(String log) { + super.isErrorLog(log) || log.contains("Cannot resolve type description") || log.contains("Instrumentation muzzled") + } + def "example application runs without errors"() { when: testedProcess.waitFor() - boolean instrumentedMessageClient = false - checkLogPostExit { - // check for additional OSGi class-loader issues - if (it.contains("Cannot resolve type description") || - it.contains("Instrumentation muzzled")) { - println it - logHasErrors = true - } - if (it.contains("Transformed - instrumentation.target.class=datadog.smoketest.jbossmodules.client.MessageClient")) { - println it - instrumentedMessageClient = true - } - } - then: + then: 'MessageClient is transformed' testedProcess.exitValue() == 0 - instrumentedMessageClient - !logHasErrors + processTestLogLines { + it.contains("Transformed - instrumentation.target.class=datadog.smoketest.jbossmodules.client.MessageClient") + } } @Override diff --git a/dd-smoke-tests/log-injection/src/test/groovy/datadog/smoketest/LogInjectionSmokeTest.groovy b/dd-smoke-tests/log-injection/src/test/groovy/datadog/smoketest/LogInjectionSmokeTest.groovy index c8c4823563f..e36b922ab78 100644 --- a/dd-smoke-tests/log-injection/src/test/groovy/datadog/smoketest/LogInjectionSmokeTest.groovy +++ b/dd-smoke-tests/log-injection/src/test/groovy/datadog/smoketest/LogInjectionSmokeTest.groovy @@ -104,6 +104,21 @@ abstract class LogInjectionSmokeTest extends AbstractSmokeTest { return processBuilder } + @Override + boolean isErrorLog(String log) { + // Exclude some errors that we consistently get because of the logging setups used here: + if (log.contains('no applicable action for [immediateFlush]')) { + return false + } + if (log.contains('JSONLayout contains an invalid element or attribute')) { + return false + } + if (log.contains('JSONLayout has no parameter that matches element')) { + return false + } + return super.isErrorLog(log) + } + @Override def logLevel() { return "debug" diff --git a/dd-smoke-tests/osgi/src/test/groovy/datadog/smoketest/AbstractOSGiSmokeTest.groovy b/dd-smoke-tests/osgi/src/test/groovy/datadog/smoketest/AbstractOSGiSmokeTest.groovy index e0cc379f2a7..784cb7ae53d 100644 --- a/dd-smoke-tests/osgi/src/test/groovy/datadog/smoketest/AbstractOSGiSmokeTest.groovy +++ b/dd-smoke-tests/osgi/src/test/groovy/datadog/smoketest/AbstractOSGiSmokeTest.groovy @@ -37,27 +37,20 @@ abstract class AbstractOSGiSmokeTest extends AbstractSmokeTest { abstract List frameworkArguments() + @Override + boolean isErrorLog(String log) { + super.isErrorLog(log) || log.contains("Cannot resolve type description") || log.contains("Instrumentation muzzled") + } + def "example application runs without errors"() { when: testedProcess.waitFor() - boolean instrumentedMessageClient = false - checkLogPostExit { - // check for additional OSGi class-loader issues - if (it.contains("Cannot resolve type description") || - it.contains("Instrumentation muzzled")) { - println it - logHasErrors = true - } - if (it.contains("Transformed - instrumentation.target.class=datadog.smoketest.osgi.client.MessageClient")) { - println it - instrumentedMessageClient = true - } - } then: testedProcess.exitValue() == 0 - instrumentedMessageClient - !logHasErrors + processTestLogLines { + it.contains("Transformed - instrumentation.target.class=datadog.smoketest.osgi.client.MessageClient") + } } @Override diff --git a/dd-smoke-tests/quarkus-native/src/test/groovy/datadog/smoketest/QuarkusNativeSmokeTest.groovy b/dd-smoke-tests/quarkus-native/src/test/groovy/datadog/smoketest/QuarkusNativeSmokeTest.groovy index 849e8e92865..c152964f185 100644 --- a/dd-smoke-tests/quarkus-native/src/test/groovy/datadog/smoketest/QuarkusNativeSmokeTest.groovy +++ b/dd-smoke-tests/quarkus-native/src/test/groovy/datadog/smoketest/QuarkusNativeSmokeTest.groovy @@ -55,8 +55,6 @@ abstract class QuarkusNativeSmokeTest extends AbstractServerSmokeTest { }) waitForTraceCount(totalInvocations) == totalInvocations validateLogInjection(resourceName()) == totalInvocations - checkLogPostExit() - !logHasErrors } void doAndValidateRequest(int id) { diff --git a/dd-smoke-tests/quarkus/src/test/groovy/datadog/smoketest/QuarkusSmokeTest.groovy b/dd-smoke-tests/quarkus/src/test/groovy/datadog/smoketest/QuarkusSmokeTest.groovy index ec5198fe05a..c8c31db7d26 100644 --- a/dd-smoke-tests/quarkus/src/test/groovy/datadog/smoketest/QuarkusSmokeTest.groovy +++ b/dd-smoke-tests/quarkus/src/test/groovy/datadog/smoketest/QuarkusSmokeTest.groovy @@ -57,8 +57,6 @@ abstract class QuarkusSmokeTest extends AbstractServerSmokeTest { }) waitForTraceCount(totalInvocations) == totalInvocations validateLogInjection(resourceName()) == totalInvocations - checkLogPostExit() - !logHasErrors } void doAndValidateRequest(int id) { diff --git a/dd-smoke-tests/spring-boot-3.0-native/src/test/groovy/SpringBootNativeInstrumentationTest.groovy b/dd-smoke-tests/spring-boot-3.0-native/src/test/groovy/SpringBootNativeInstrumentationTest.groovy index 3d1aeb427cb..c5b9edeea24 100644 --- a/dd-smoke-tests/spring-boot-3.0-native/src/test/groovy/SpringBootNativeInstrumentationTest.groovy +++ b/dd-smoke-tests/spring-boot-3.0-native/src/test/groovy/SpringBootNativeInstrumentationTest.groovy @@ -60,6 +60,12 @@ class SpringBootNativeInstrumentationTest extends AbstractServerSmokeTest { false } + @Override + boolean isErrorLog(String log) { + // Check that there are no ClassNotFound errors printed from bad reflect-config.json + super.isErrorLog(log) || log.contains("ClassNotFoundException") + } + def "check native instrumentation"() { setup: String url = "http://localhost:${httpPort}/hello" @@ -81,18 +87,6 @@ class SpringBootNativeInstrumentationTest extends AbstractServerSmokeTest { LockSupport.parkNanos(1_000_000) } countJfrs() > 0 - - when: - checkLogPostExit { - // Check that there are no ClassNotFound errors printed from bad reflect-config.json - if (it.contains("ClassNotFoundException")) { - println "Found ClassNotFoundException in log: ${it}" - logHasErrors = true - } - } - - then: - !logHasErrors } int countJfrs() { diff --git a/dd-smoke-tests/spring-boot-rabbit/src/test/groovy/datadog/smoketest/SpringBootRabbitIntegrationTest.groovy b/dd-smoke-tests/spring-boot-rabbit/src/test/groovy/datadog/smoketest/SpringBootRabbitIntegrationTest.groovy index ccad8eb2f6f..85bf8c70208 100644 --- a/dd-smoke-tests/spring-boot-rabbit/src/test/groovy/datadog/smoketest/SpringBootRabbitIntegrationTest.groovy +++ b/dd-smoke-tests/spring-boot-rabbit/src/test/groovy/datadog/smoketest/SpringBootRabbitIntegrationTest.groovy @@ -89,6 +89,14 @@ class SpringBootRabbitIntegrationTest extends AbstractServerSmokeTest { return expected } + @Override + boolean isErrorLog(String log) { + if (log.contains('org.springframework.amqp.rabbit.listener.SimpleMessageListenerContainer - Failed to check/redeclare auto-delete queue(s).')) { + return false + } + return super.isErrorLog(log) + } + def "check message #message roundtrip"() { setup: String url = "http://localhost:${httpPort}/roundtrip/${message}" diff --git a/dd-smoke-tests/springboot/src/test/groovy/datadog/smoketest/IastSpringBootSmokeTest.groovy b/dd-smoke-tests/springboot/src/test/groovy/datadog/smoketest/IastSpringBootSmokeTest.groovy index 5835b4efe28..cbc660a19a0 100644 --- a/dd-smoke-tests/springboot/src/test/groovy/datadog/smoketest/IastSpringBootSmokeTest.groovy +++ b/dd-smoke-tests/springboot/src/test/groovy/datadog/smoketest/IastSpringBootSmokeTest.groovy @@ -40,13 +40,17 @@ class IastSpringBootSmokeTest extends AbstractIastSpringBootTest { then: response.successful - hasVulnerabilityInLogs { - vul -> - vul.type == 'HARDCODED_SECRET' - && vul.location.method == 'hardcodedSecret' - && vul.location.path == 'datadog.smoketest.springboot.controller.HardcodedSecretController' - && vul.location.line == 11 - && vul.evidence.value == 'age-secret-key' + isLogPresent { + String log -> + def vulns = parseVulnerabilitiesLog(log) + vulns.any { + vul -> + vul.type == 'HARDCODED_SECRET' + && vul.location.method == 'hardcodedSecret' + && vul.location.path == 'datadog.smoketest.springboot.controller.HardcodedSecretController' + && vul.location.line == 11 + && vul.evidence.value == 'age-secret-key' + } } } diff --git a/dd-smoke-tests/src/main/groovy/datadog/smoketest/AbstractSmokeTest.groovy b/dd-smoke-tests/src/main/groovy/datadog/smoketest/AbstractSmokeTest.groovy index 17e33ce4aa4..0d1192696f5 100644 --- a/dd-smoke-tests/src/main/groovy/datadog/smoketest/AbstractSmokeTest.groovy +++ b/dd-smoke-tests/src/main/groovy/datadog/smoketest/AbstractSmokeTest.groovy @@ -243,6 +243,7 @@ abstract class AbstractSmokeTest extends ProcessManager { def cleanupSpec() { stopServer() + assertNoErrorLogs() } def startServer() { diff --git a/dd-smoke-tests/src/main/groovy/datadog/smoketest/ProcessManager.groovy b/dd-smoke-tests/src/main/groovy/datadog/smoketest/ProcessManager.groovy index f2993c256f5..e73c8c4ebd9 100644 --- a/dd-smoke-tests/src/main/groovy/datadog/smoketest/ProcessManager.groovy +++ b/dd-smoke-tests/src/main/groovy/datadog/smoketest/ProcessManager.groovy @@ -55,13 +55,6 @@ abstract class ProcessManager extends Specification { @Shared protected Process testedProcess - /** - * Will be initialized after calling {@linkplain AbstractSmokeTest#checkLogPostExit} and hold {@literal true} - * if there are any ERROR or WARN lines in the test application log. - */ - @Shared - def logHasErrors - @Shared private String[] logFilePaths = (0.. "${buildDirectory}/reports/testProcess.${this.getClass().getName()}.${idx}.log" @@ -253,33 +246,63 @@ abstract class ProcessManager extends Specification { } /** - * Check the test application log and set {@linkplain AbstractSmokeTest#logHasErrors} variable - * - * This should only be called after the process exits, otherwise it's not guaranteed that - * reading the log file will yield its final contents. If you want to check whether a particular - * line is emitted during a test, consider using {@link #processTestLogLines(groovy.lang.Closure)} + * Checks if a log line is an error. This method may be overridden by test suites to consider additional messages. + * These will be checked on suite shutdown, or explicitly by calling {@link #assertNoErrorLogs()}. + */ + boolean isErrorLog(String line) { + // FIXME: Flaky profiler exception. See PROF-11068. + if (line.contains('ERROR com.datadog.profiling.controller.ProfilingSystem - Fatal exception in profiling thread, trying to continue')) { + return false + } + + return line.contains("ERROR") || line.contains("ASSERTION FAILED") + || line.contains("Failed to handle exception in instrumentation") + } + + /** + * Asserts that there are no errors printed by the application to the log. + * This should usually be called after the process exits, otherwise it's not guaranteed that reading the log file will + * yield its final contents. Most tests should not need this, since it will be called at the end of every smoke test + * suite. * - * @param checker custom closure to run on each log line + * @param errorFilter Returns true if certain log line must be considered an error. */ - def checkLogPostExit(Closure checker) { - logFilePaths.each { lfp -> - def hasError = false + void assertNoErrorLogs(final Closure errorFilter = this.&isErrorLog) { + final List errorLogs = new ArrayList<>() + forEachLogLine { String line -> + if (errorFilter(line)) { + errorLogs << line + } + } + if (!errorLogs.isEmpty()) { + final StringBuilder sb = new StringBuilder("Test application log contains ${errorLogs.size()} errors:\n") + errorLogs.eachWithIndex { String entry, int i -> + sb.append("${i + 1}: ${entry}\n") + } + assert errorLogs.isEmpty(), sb.toString() + } + } + + void forEachLogLine(Closure checker) { + for (String lfp : logFilePaths) { ProcessManager.eachLine(new File(lfp)) { - if (it.contains("ERROR") || it.contains("ASSERTION FAILED") - || it.contains("Failed to handle exception in instrumentation")) { - println it - hasError = logHasErrors = true - } checker(it) } - if (hasError) { - println "Test application log contains errors. See full run logs in ${lfp}" - } } } - def checkLogPostExit() { - checkLogPostExit {} + /** + * Check if at least one log is present. It checks it since the beginning of the application, and not just during + * the test. If the log is not present, it does not wait for it. See {@link #processTestLogLines(Closure)} for that. + */ + boolean isLogPresent(final Closure checker) { + boolean found = false + forEachLogLine { + if (checker(it)) { + found = true + } + } + return found } /**