From cd1b746d93b0bd00c807efbb15184b2514e2f92e Mon Sep 17 00:00:00 2001 From: "Santiago M. Mola" Date: Thu, 21 Nov 2024 14:38:07 +0100 Subject: [PATCH] Add smoke tests for telemetry (#7955) --- .../SpringBootWebmvcIntegrationTest.groovy | 5 ++ ...SpringBootNativeInstrumentationTest.groovy | 5 ++ .../SpringBootWebmvcIntegrationTest.groovy | 5 ++ .../SpringBootWebmvcIntegrationTest.groovy | 5 ++ .../SpringBootOpenLibertySmokeTest.groovy | 5 ++ ...otOpenLibertySmokeVulnerabilityTest.groovy | 5 ++ .../SpringBootOpenLibertySmokeTest.groovy | 5 ++ ...otOpenLibertySmokeVulnerabilityTest.groovy | 5 ++ .../smoketest/AbstractServerSmokeTest.groovy | 51 ++++++++++++++++ .../smoketest/AbstractSmokeTest.groovy | 60 +++++++++++++++++++ .../groovy/datadog/smoketest/RunLast.groovy | 19 ++++++ .../datadog/smoketest/RunLastExtension.groovy | 13 ++++ 12 files changed, 183 insertions(+) create mode 100644 dd-smoke-tests/src/main/groovy/datadog/smoketest/RunLast.groovy create mode 100644 dd-smoke-tests/src/main/groovy/datadog/smoketest/RunLastExtension.groovy diff --git a/dd-smoke-tests/spring-boot-2.3-webmvc-jetty/src/test/groovy/SpringBootWebmvcIntegrationTest.groovy b/dd-smoke-tests/spring-boot-2.3-webmvc-jetty/src/test/groovy/SpringBootWebmvcIntegrationTest.groovy index 04fe5a66951..297b44bf382 100644 --- a/dd-smoke-tests/spring-boot-2.3-webmvc-jetty/src/test/groovy/SpringBootWebmvcIntegrationTest.groovy +++ b/dd-smoke-tests/spring-boot-2.3-webmvc-jetty/src/test/groovy/SpringBootWebmvcIntegrationTest.groovy @@ -86,4 +86,9 @@ class SpringBootWebmvcIntegrationTest extends AbstractServerSmokeTest { response.code() == 404 waitForTraceCount(1) } + + @Override + List expectedTelemetryDependencies() { + ['org.eclipse.jetty:jetty-client'] + } } diff --git a/dd-smoke-tests/spring-boot-3.0-native/src/test/groovy/SpringBootNativeInstrumentationTest.groovy b/dd-smoke-tests/spring-boot-3.0-native/src/test/groovy/SpringBootNativeInstrumentationTest.groovy index 61d93f8aeb1..3d1aeb427cb 100644 --- a/dd-smoke-tests/spring-boot-3.0-native/src/test/groovy/SpringBootNativeInstrumentationTest.groovy +++ b/dd-smoke-tests/spring-boot-3.0-native/src/test/groovy/SpringBootNativeInstrumentationTest.groovy @@ -55,6 +55,11 @@ class SpringBootNativeInstrumentationTest extends AbstractServerSmokeTest { return ["[servlet.request[spring.handler[WebController.doHello[WebController.sayHello]]]]"] } + @Override + boolean testTelemetry() { + false + } + def "check native instrumentation"() { setup: String url = "http://localhost:${httpPort}/hello" diff --git a/dd-smoke-tests/spring-boot-3.0-webmvc/src/test/groovy/SpringBootWebmvcIntegrationTest.groovy b/dd-smoke-tests/spring-boot-3.0-webmvc/src/test/groovy/SpringBootWebmvcIntegrationTest.groovy index e1be966627a..362a365a00f 100644 --- a/dd-smoke-tests/spring-boot-3.0-webmvc/src/test/groovy/SpringBootWebmvcIntegrationTest.groovy +++ b/dd-smoke-tests/spring-boot-3.0-webmvc/src/test/groovy/SpringBootWebmvcIntegrationTest.groovy @@ -78,4 +78,9 @@ class SpringBootWebmvcIntegrationTest extends AbstractServerSmokeTest { responseBodyStr.contains("banana") waitForTraceCount(1) } + + @Override + List expectedTelemetryDependencies() { + ['spring-core'] + } } diff --git a/dd-smoke-tests/spring-boot-3.3-webmvc/src/test/groovy/SpringBootWebmvcIntegrationTest.groovy b/dd-smoke-tests/spring-boot-3.3-webmvc/src/test/groovy/SpringBootWebmvcIntegrationTest.groovy index e1be966627a..362a365a00f 100644 --- a/dd-smoke-tests/spring-boot-3.3-webmvc/src/test/groovy/SpringBootWebmvcIntegrationTest.groovy +++ b/dd-smoke-tests/spring-boot-3.3-webmvc/src/test/groovy/SpringBootWebmvcIntegrationTest.groovy @@ -78,4 +78,9 @@ class SpringBootWebmvcIntegrationTest extends AbstractServerSmokeTest { responseBodyStr.contains("banana") waitForTraceCount(1) } + + @Override + List expectedTelemetryDependencies() { + ['spring-core'] + } } diff --git a/dd-smoke-tests/springboot-openliberty-20/src/test/groovy/datadog/smoketest/SpringBootOpenLibertySmokeTest.groovy b/dd-smoke-tests/springboot-openliberty-20/src/test/groovy/datadog/smoketest/SpringBootOpenLibertySmokeTest.groovy index d6a761d39f7..600a81d2359 100644 --- a/dd-smoke-tests/springboot-openliberty-20/src/test/groovy/datadog/smoketest/SpringBootOpenLibertySmokeTest.groovy +++ b/dd-smoke-tests/springboot-openliberty-20/src/test/groovy/datadog/smoketest/SpringBootOpenLibertySmokeTest.groovy @@ -61,6 +61,11 @@ class SpringBootOpenLibertySmokeTest extends AbstractServerSmokeTest { ].toSet() } + @Override + boolean testTelemetry() { + false + } + def "Test concurrent requests to Spring Boot running Open Liberty"() { setup: def url = "http://localhost:${httpPort}/connect" diff --git a/dd-smoke-tests/springboot-openliberty-20/src/test/groovy/datadog/smoketest/SpringBootOpenLibertySmokeVulnerabilityTest.groovy b/dd-smoke-tests/springboot-openliberty-20/src/test/groovy/datadog/smoketest/SpringBootOpenLibertySmokeVulnerabilityTest.groovy index b77111f1c74..84c9a4ebff6 100644 --- a/dd-smoke-tests/springboot-openliberty-20/src/test/groovy/datadog/smoketest/SpringBootOpenLibertySmokeVulnerabilityTest.groovy +++ b/dd-smoke-tests/springboot-openliberty-20/src/test/groovy/datadog/smoketest/SpringBootOpenLibertySmokeVulnerabilityTest.groovy @@ -59,6 +59,11 @@ class SpringBootOpenLibertySmokeVulnerabilityTest extends AbstractServerSmokeTes return {} // force traces decoding } + @Override + boolean testTelemetry() { + false + } + private static boolean contains(String s) { System.out.println("Checking span:" + s) return s.contains("MD5") diff --git a/dd-smoke-tests/springboot-openliberty-23/src/test/groovy/datadog/smoketest/SpringBootOpenLibertySmokeTest.groovy b/dd-smoke-tests/springboot-openliberty-23/src/test/groovy/datadog/smoketest/SpringBootOpenLibertySmokeTest.groovy index 78c4d9c415c..a19b1a7989c 100644 --- a/dd-smoke-tests/springboot-openliberty-23/src/test/groovy/datadog/smoketest/SpringBootOpenLibertySmokeTest.groovy +++ b/dd-smoke-tests/springboot-openliberty-23/src/test/groovy/datadog/smoketest/SpringBootOpenLibertySmokeTest.groovy @@ -68,6 +68,11 @@ class SpringBootOpenLibertySmokeTest extends AbstractServerSmokeTest { ].toSet() } + @Override + boolean testTelemetry() { + false + } + def "Test concurrent requests to Spring Boot running Open Liberty"() { setup: def url = "http://localhost:${httpPort}/connect" diff --git a/dd-smoke-tests/springboot-openliberty-23/src/test/groovy/datadog/smoketest/SpringBootOpenLibertySmokeVulnerabilityTest.groovy b/dd-smoke-tests/springboot-openliberty-23/src/test/groovy/datadog/smoketest/SpringBootOpenLibertySmokeVulnerabilityTest.groovy index b77111f1c74..84c9a4ebff6 100644 --- a/dd-smoke-tests/springboot-openliberty-23/src/test/groovy/datadog/smoketest/SpringBootOpenLibertySmokeVulnerabilityTest.groovy +++ b/dd-smoke-tests/springboot-openliberty-23/src/test/groovy/datadog/smoketest/SpringBootOpenLibertySmokeVulnerabilityTest.groovy @@ -59,6 +59,11 @@ class SpringBootOpenLibertySmokeVulnerabilityTest extends AbstractServerSmokeTes return {} // force traces decoding } + @Override + boolean testTelemetry() { + false + } + private static boolean contains(String s) { System.out.println("Checking span:" + s) return s.contains("MD5") diff --git a/dd-smoke-tests/src/main/groovy/datadog/smoketest/AbstractServerSmokeTest.groovy b/dd-smoke-tests/src/main/groovy/datadog/smoketest/AbstractServerSmokeTest.groovy index 13baba60481..a71b32de9cb 100644 --- a/dd-smoke-tests/src/main/groovy/datadog/smoketest/AbstractServerSmokeTest.groovy +++ b/dd-smoke-tests/src/main/groovy/datadog/smoketest/AbstractServerSmokeTest.groovy @@ -5,6 +5,7 @@ import datadog.trace.agent.test.utils.OkHttpUtils import datadog.trace.agent.test.utils.PortUtils import okhttp3.OkHttpClient import spock.lang.Shared +import static org.junit.Assume.assumeTrue import java.util.concurrent.TimeUnit import java.util.concurrent.atomic.AtomicInteger @@ -116,4 +117,54 @@ abstract class AbstractServerSmokeTest extends AbstractSmokeTest { } return remaining } + + @RunLast + void 'receive telemetry app-started'() { + when: + assumeTrue(testTelemetry()) + waitForTelemetryCount(1) + + then: + telemetryMessages.size() >= 1 + Object msg = telemetryMessages.get(0) + msg['request_type'] == 'app-started' + } + + List expectedTelemetryDependencies() { + [] + } + + @RunLast + @SuppressWarnings('UnnecessaryBooleanExpression') + void 'receive telemetry app-dependencies-loaded'() { + when: + assumeTrue(testTelemetry()) + // app-started + 3 message-batch + waitForTelemetryCount(4) + waitForTelemetryFlat { it.get('request_type') == 'app-dependencies-loaded' } + + then: 'received some dependencies' + def dependenciesLoaded = telemetryFlatMessages.findAll { it.get('request_type') == 'app-dependencies-loaded' } + def dependencies = [] + dependenciesLoaded.each { + def payload = it.get('payload') as Map + dependencies.addAll(payload.get('dependencies')) } + dependencies.size() > 0 + + Set dependencyNames = dependencies.collect { + def dependency = it as Map + dependency.get('name') as String + }.toSet() + + and: 'received tracer dependencies' + // Not exhaustive list of tracer dependencies. + Set missingDependencyNames = ['com.github.jnr:jnr-ffi', 'net.bytebuddy:byte-buddy-agent',].toSet() + missingDependencyNames.removeAll(dependencyNames) || true + missingDependencyNames.isEmpty() + + and: 'received application dependencies' + Set missingExtraDependencyNames = expectedTelemetryDependencies().toSet() + missingExtraDependencyNames.removeAll(dependencyNames) || true + missingExtraDependencyNames.isEmpty() + } } diff --git a/dd-smoke-tests/src/main/groovy/datadog/smoketest/AbstractSmokeTest.groovy b/dd-smoke-tests/src/main/groovy/datadog/smoketest/AbstractSmokeTest.groovy index ea79551487d..092c5b84fb9 100644 --- a/dd-smoke-tests/src/main/groovy/datadog/smoketest/AbstractSmokeTest.groovy +++ b/dd-smoke-tests/src/main/groovy/datadog/smoketest/AbstractSmokeTest.groovy @@ -7,6 +7,7 @@ import datadog.trace.test.agent.decoder.Decoder import datadog.trace.test.agent.decoder.DecodedMessage import datadog.trace.test.agent.decoder.DecodedTrace import datadog.trace.util.Strings +import groovy.json.JsonSlurper import java.nio.charset.StandardCharsets import java.util.concurrent.CopyOnWriteArrayList @@ -37,6 +38,15 @@ abstract class AbstractSmokeTest extends ProcessManager { @Shared private Throwable traceDecodingFailure = null + @Shared + protected CopyOnWriteArrayList> telemetryMessages = new CopyOnWriteArrayList() + + @Shared + protected CopyOnWriteArrayList> telemetryFlatMessages = new CopyOnWriteArrayList() + + @Shared + private Throwable telemetryDecodingFailure = null + @Shared protected TestHttpServer.Headers lastTraceRequestHeaders = null @@ -119,6 +129,23 @@ abstract class AbstractSmokeTest extends ProcessManager { response.status(200).send(remoteConfigResponse) } prefix("/telemetry/proxy/api/v2/apmtelemetry") { + try { + byte[] body = request.getBody() + if (body != null) { + Map msg = new JsonSlurper().parseText(new String(body, StandardCharsets.UTF_8)) as Map + telemetryMessages.add(msg) + if (msg.get("request_type") == "message-batch") { + msg.get("payload")?.each { telemetryFlatMessages.add(it as Map) } + } else { + telemetryFlatMessages.add(msg) + } + } + } catch (Throwable t) { + println("=== Failure during telemetry decoding ===") + t.printStackTrace(System.out) + telemetryDecodingFailure = t + throw t + } response.status(202).send() } } @@ -160,6 +187,9 @@ abstract class AbstractSmokeTest extends ProcessManager { if (inferServiceName()) { ret += "-Ddd.service.name=${SERVICE_NAME}" } + if (testTelemetry()) { + ret += "-Ddd.telemetry.heartbeat.interval=5" + } ret as String[] } @@ -172,6 +202,11 @@ abstract class AbstractSmokeTest extends ProcessManager { return !Platform.isJ9() } + + boolean testTelemetry() { + return true + } + def setup() { traceCount.set(0) decodeTraces.clear() @@ -272,6 +307,31 @@ abstract class AbstractSmokeTest extends ProcessManager { } } + void waitForTelemetryCount(final int count) { + def conditions = new PollingConditions(timeout: 30, initialDelay: 0, delay: 1, factor: 1) + waitForTelemetryCount(conditions, count) + } + + void waitForTelemetryCount(final PollingConditions poll, final int count) { + poll.eventually { + telemetryMessages.size() >= count + } + } + + void waitForTelemetryFlat(final Function, Boolean> predicate) { + def conditions = new PollingConditions(timeout: 30, initialDelay: 0, delay: 1, factor: 1) + waitForTelemetryFlat(conditions, predicate) + } + + void waitForTelemetryFlat(final PollingConditions poll, final Function, Boolean> predicate) { + poll.eventually { + if (telemetryDecodingFailure != null) { + throw telemetryDecodingFailure + } + assert telemetryFlatMessages.find { predicate.apply(it) } != null + } + } + List getTraces() { decodeTraces } diff --git a/dd-smoke-tests/src/main/groovy/datadog/smoketest/RunLast.groovy b/dd-smoke-tests/src/main/groovy/datadog/smoketest/RunLast.groovy new file mode 100644 index 00000000000..07c751a4ede --- /dev/null +++ b/dd-smoke-tests/src/main/groovy/datadog/smoketest/RunLast.groovy @@ -0,0 +1,19 @@ +package datadog.smoketest + +import org.spockframework.runtime.extension.ExtensionAnnotation + +import java.lang.annotation.ElementType +import java.lang.annotation.Retention +import java.lang.annotation.RetentionPolicy +import java.lang.annotation.Target + +/** + * Spock test methods annotated with this will be executed last. + * This is useful for tests that need to wait for some test to settle while other tests run (e.g. telemetry), so it is + * more efficient to run them at the end. + */ +@Retention(RetentionPolicy.RUNTIME) +@Target([ElementType.METHOD]) +@ExtensionAnnotation(RunLastExtension) +@interface RunLast { +} diff --git a/dd-smoke-tests/src/main/groovy/datadog/smoketest/RunLastExtension.groovy b/dd-smoke-tests/src/main/groovy/datadog/smoketest/RunLastExtension.groovy new file mode 100644 index 00000000000..40bdd408156 --- /dev/null +++ b/dd-smoke-tests/src/main/groovy/datadog/smoketest/RunLastExtension.groovy @@ -0,0 +1,13 @@ +package datadog.smoketest + +import org.spockframework.runtime.extension.IAnnotationDrivenExtension +import org.spockframework.runtime.model.FeatureInfo + +class RunLastExtension implements IAnnotationDrivenExtension { + @Override + void visitFeatureAnnotations(List annotations, FeatureInfo feature) { + if (!annotations.isEmpty()) { + feature.setExecutionOrder(Integer.MAX_VALUE) + } + } +}