From e605ec22dce8b0a97600a7a51517a17be92f3fde Mon Sep 17 00:00:00 2001 From: "Montague, Brent" Date: Wed, 28 Feb 2024 23:35:58 -0500 Subject: [PATCH 01/30] Add option to not include couchbase internal spans --- .../couchbase_31/client/DatadogRequestTracer.java | 5 +++++ .../couchbase_32/client/DatadogRequestTracer.java | 5 +++++ .../main/java/datadog/trace/api/ConfigDefaults.java | 1 + .../trace/api/config/TraceInstrumentationConfig.java | 2 ++ .../src/main/java/datadog/trace/api/Config.java | 11 +++++++++++ 5 files changed, 24 insertions(+) diff --git a/dd-java-agent/instrumentation/couchbase/couchbase-3.1/src/main/java/datadog/trace/instrumentation/couchbase_31/client/DatadogRequestTracer.java b/dd-java-agent/instrumentation/couchbase/couchbase-3.1/src/main/java/datadog/trace/instrumentation/couchbase_31/client/DatadogRequestTracer.java index f37dc7de3d6..7294f81c8b6 100644 --- a/dd-java-agent/instrumentation/couchbase/couchbase-3.1/src/main/java/datadog/trace/instrumentation/couchbase_31/client/DatadogRequestTracer.java +++ b/dd-java-agent/instrumentation/couchbase/couchbase-3.1/src/main/java/datadog/trace/instrumentation/couchbase_31/client/DatadogRequestTracer.java @@ -5,6 +5,7 @@ import com.couchbase.client.core.Core; import com.couchbase.client.core.cnc.RequestSpan; import com.couchbase.client.core.cnc.RequestTracer; +import datadog.trace.api.Config; import datadog.trace.bootstrap.ContextStore; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; import datadog.trace.bootstrap.instrumentation.api.AgentTracer; @@ -33,11 +34,15 @@ public RequestSpan requestSpan(String requestName, RequestSpan requestParent) { boolean measured = true; Object seedNodes = null; + final Config config = Config.get(); AgentSpan parent = DatadogRequestSpan.unwrap(requestParent); if (null == parent) { parent = tracer.activeSpan(); } if (null != parent && COUCHBASE_CLIENT.equals(parent.getTag(Tags.COMPONENT))) { + if (!config.isCouchbaseInternalEnabled()) { + return requestParent; + } spanName = COUCHBASE_INTERNAL; measured = false; seedNodes = parent.getTag(InstrumentationTags.COUCHBASE_SEED_NODES); diff --git a/dd-java-agent/instrumentation/couchbase/couchbase-3.2/src/main/java/datadog/trace/instrumentation/couchbase_32/client/DatadogRequestTracer.java b/dd-java-agent/instrumentation/couchbase/couchbase-3.2/src/main/java/datadog/trace/instrumentation/couchbase_32/client/DatadogRequestTracer.java index f887c6c4c71..1386bbf58c9 100644 --- a/dd-java-agent/instrumentation/couchbase/couchbase-3.2/src/main/java/datadog/trace/instrumentation/couchbase_32/client/DatadogRequestTracer.java +++ b/dd-java-agent/instrumentation/couchbase/couchbase-3.2/src/main/java/datadog/trace/instrumentation/couchbase_32/client/DatadogRequestTracer.java @@ -6,6 +6,7 @@ import com.couchbase.client.core.Core; import com.couchbase.client.core.cnc.RequestSpan; import com.couchbase.client.core.cnc.RequestTracer; +import datadog.trace.api.Config; import datadog.trace.bootstrap.ContextStore; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; import datadog.trace.bootstrap.instrumentation.api.AgentTracer; @@ -35,11 +36,15 @@ public RequestSpan requestSpan(String requestName, RequestSpan requestParent) { boolean measured = true; Object seedNodes = null; + final Config config = Config.get(); AgentSpan parent = DatadogRequestSpan.unwrap(requestParent); if (null == parent) { parent = tracer.activeSpan(); } if (null != parent && COUCHBASE_CLIENT.equals(parent.getTag(Tags.COMPONENT))) { + if (!config.isCouchbaseInternalEnabled()) { + return requestParent; + } spanName = COUCHBASE_INTERNAL; measured = false; seedNodes = parent.getTag(InstrumentationTags.COUCHBASE_SEED_NODES); diff --git a/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java b/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java index 82793c81585..0c009dfbb7b 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java @@ -214,6 +214,7 @@ public final class ConfigDefaults { static final float DEFAULT_TRACE_FLUSH_INTERVAL = 1; + static final boolean DEFAULT_COUCHBASE_INTERNAL_ENABLED = true; static final boolean DEFAULT_ELASTICSEARCH_BODY_ENABLED = false; static final boolean DEFAULT_ELASTICSEARCH_PARAMS_ENABLED = true; static final boolean DEFAULT_ELASTICSEARCH_BODY_AND_PARAMS_ENABLED = false; diff --git a/dd-trace-api/src/main/java/datadog/trace/api/config/TraceInstrumentationConfig.java b/dd-trace-api/src/main/java/datadog/trace/api/config/TraceInstrumentationConfig.java index b916c7479d3..75ac22e6812 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/config/TraceInstrumentationConfig.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/config/TraceInstrumentationConfig.java @@ -125,6 +125,8 @@ public final class TraceInstrumentationConfig { public static final String RESOLVER_RESET_INTERVAL = "resolver.reset.interval"; public static final String RESOLVER_NAMES_ARE_UNIQUE = "resolver.names.are.unique"; + public static final String COUCHBASE_INTERNAL_ENABLED = "trace.couchbase.internal.enabled"; + public static final String ELASTICSEARCH_BODY_ENABLED = "trace.elasticsearch.body.enabled"; public static final String ELASTICSEARCH_PARAMS_ENABLED = "trace.elasticsearch.params.enabled"; public static final String ELASTICSEARCH_BODY_AND_PARAMS_ENABLED = diff --git a/internal-api/src/main/java/datadog/trace/api/Config.java b/internal-api/src/main/java/datadog/trace/api/Config.java index b393501bffa..8ccfe3e4924 100644 --- a/internal-api/src/main/java/datadog/trace/api/Config.java +++ b/internal-api/src/main/java/datadog/trace/api/Config.java @@ -30,6 +30,7 @@ import static datadog.trace.api.ConfigDefaults.DEFAULT_CIVISIBILITY_SOURCE_DATA_ROOT_CHECK_ENABLED; import static datadog.trace.api.ConfigDefaults.DEFAULT_CLIENT_IP_ENABLED; import static datadog.trace.api.ConfigDefaults.DEFAULT_CLOCK_SYNC_PERIOD; +import static datadog.trace.api.ConfigDefaults.DEFAULT_COUCHBASE_INTERNAL_ENABLED; import static datadog.trace.api.ConfigDefaults.DEFAULT_CWS_ENABLED; import static datadog.trace.api.ConfigDefaults.DEFAULT_CWS_TLS_REFRESH; import static datadog.trace.api.ConfigDefaults.DEFAULT_DATA_STREAMS_BUCKET_DURATION; @@ -328,6 +329,7 @@ import static datadog.trace.api.config.RemoteConfigConfig.REMOTE_CONFIG_TARGETS_KEY_ID; import static datadog.trace.api.config.RemoteConfigConfig.REMOTE_CONFIG_URL; import static datadog.trace.api.config.TraceInstrumentationConfig.AXIS_PROMOTE_RESOURCE_NAME; +import static datadog.trace.api.config.TraceInstrumentationConfig.COUCHBASE_INTERNAL_ENABLED; import static datadog.trace.api.config.TraceInstrumentationConfig.DB_CLIENT_HOST_SPLIT_BY_HOST; import static datadog.trace.api.config.TraceInstrumentationConfig.DB_CLIENT_HOST_SPLIT_BY_INSTANCE; import static datadog.trace.api.config.TraceInstrumentationConfig.DB_CLIENT_HOST_SPLIT_BY_INSTANCE_TYPE_SUFFIX; @@ -894,6 +896,7 @@ static class HostNameHolder { private final boolean longRunningTraceEnabled; private final long longRunningTraceInitialFlushInterval; private final long longRunningTraceFlushInterval; + private final boolean couchbaseInternalEnabled; private final boolean elasticsearchBodyEnabled; private final boolean elasticsearchParamsEnabled; private final boolean elasticsearchBodyAndParamsEnabled; @@ -981,6 +984,8 @@ private Config(final ConfigProvider configProvider, final InstrumenterConfig ins } else { secureRandom = configProvider.getBoolean(SECURE_RANDOM, DEFAULT_SECURE_RANDOM); } + couchbaseInternalEnabled = + configProvider.getBoolean(COUCHBASE_INTERNAL_ENABLED, DEFAULT_COUCHBASE_INTERNAL_ENABLED); elasticsearchBodyEnabled = configProvider.getBoolean(ELASTICSEARCH_BODY_ENABLED, DEFAULT_ELASTICSEARCH_BODY_ENABLED); elasticsearchParamsEnabled = @@ -3287,6 +3292,10 @@ public BitSet getGrpcClientErrorStatuses() { return grpcClientErrorStatuses; } + public boolean isCouchbaseInternalEnabled() { + return couchbaseInternalEnabled; + } + public boolean isElasticsearchBodyEnabled() { return elasticsearchBodyEnabled; } @@ -4361,6 +4370,8 @@ public String toString() { + longRunningTraceInitialFlushInterval + ", longRunningTraceFlushInterval=" + longRunningTraceFlushInterval + + ", couchbaseInternalEnabled=" + + couchbaseInternalEnabled + ", elasticsearchBodyEnabled=" + elasticsearchBodyEnabled + ", elasticsearchParamsEnabled=" From 8be0fe8da2e9021d027621590de6dd4969c851b0 Mon Sep 17 00:00:00 2001 From: Andrea Marziali Date: Fri, 1 Mar 2024 13:01:38 +0100 Subject: [PATCH 02/30] properly mute internal couchbase spans --- .../client/DatadogRequestTracer.java | 8 +- .../test/groovy/CouchbaseClient31Test.groovy | 118 ++++++------------ .../client/DatadogRequestTracer.java | 41 +++--- .../test/groovy/CouchbaseClient32Test.groovy | 68 ++++++---- .../datadog/trace/api/ConfigDefaults.java | 2 +- .../config/TraceInstrumentationConfig.java | 5 +- .../main/java/datadog/trace/api/Config.java | 19 +-- 7 files changed, 118 insertions(+), 143 deletions(-) diff --git a/dd-java-agent/instrumentation/couchbase/couchbase-3.1/src/main/java/datadog/trace/instrumentation/couchbase_31/client/DatadogRequestTracer.java b/dd-java-agent/instrumentation/couchbase/couchbase-3.1/src/main/java/datadog/trace/instrumentation/couchbase_31/client/DatadogRequestTracer.java index 7294f81c8b6..3ef0efdf3b3 100644 --- a/dd-java-agent/instrumentation/couchbase/couchbase-3.1/src/main/java/datadog/trace/instrumentation/couchbase_31/client/DatadogRequestTracer.java +++ b/dd-java-agent/instrumentation/couchbase/couchbase-3.1/src/main/java/datadog/trace/instrumentation/couchbase_31/client/DatadogRequestTracer.java @@ -1,5 +1,6 @@ package datadog.trace.instrumentation.couchbase_31.client; +import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.blackholeSpan; import static datadog.trace.instrumentation.couchbase_31.client.CouchbaseClientDecorator.COUCHBASE_CLIENT; import com.couchbase.client.core.Core; @@ -34,14 +35,15 @@ public RequestSpan requestSpan(String requestName, RequestSpan requestParent) { boolean measured = true; Object seedNodes = null; - final Config config = Config.get(); AgentSpan parent = DatadogRequestSpan.unwrap(requestParent); if (null == parent) { parent = tracer.activeSpan(); } + if (null != parent && COUCHBASE_CLIENT.equals(parent.getTag(Tags.COMPONENT))) { - if (!config.isCouchbaseInternalEnabled()) { - return requestParent; + if (!Config.get().isCouchbaseInternalSpansEnabled()) { + // mute the tracing related to internal spans + return DatadogRequestSpan.wrap(blackholeSpan(), coreContext); } spanName = COUCHBASE_INTERNAL; measured = false; diff --git a/dd-java-agent/instrumentation/couchbase/couchbase-3.1/src/test/groovy/CouchbaseClient31Test.groovy b/dd-java-agent/instrumentation/couchbase/couchbase-3.1/src/test/groovy/CouchbaseClient31Test.groovy index 9dfe354032d..1c76f2a6cba 100644 --- a/dd-java-agent/instrumentation/couchbase/couchbase-3.1/src/test/groovy/CouchbaseClient31Test.groovy +++ b/dd-java-agent/instrumentation/couchbase/couchbase-3.1/src/test/groovy/CouchbaseClient31Test.groovy @@ -1,5 +1,4 @@ import com.couchbase.client.core.env.TimeoutConfig -import com.couchbase.client.core.error.CouchbaseException import com.couchbase.client.core.error.DocumentNotFoundException import com.couchbase.client.core.error.ParsingFailureException import com.couchbase.client.java.Bucket @@ -107,8 +106,9 @@ abstract class CouchbaseClient31Test extends VersionedNamingTestBase { } } - def "check basic error spans"() { + def "check basic error spans with internal spans enabled #internalEnabled"() { setup: + injectSysConfig("trace.couchbase.internal-spans.enabled", "$internalEnabled") def collection = bucket.defaultCollection() Throwable ex = null @@ -122,7 +122,7 @@ abstract class CouchbaseClient31Test extends VersionedNamingTestBase { then: assertTraces(1) { sortSpansByStart() - trace(2) { + trace(internalEnabled ? 2 : 1) { assertCouchbaseCall(it, "cb.get", [ 'db.couchbase.collection': '_default', 'db.couchbase.retries' : { Long }, @@ -131,9 +131,15 @@ abstract class CouchbaseClient31Test extends VersionedNamingTestBase { 'db.name' : BUCKET, 'db.operation' : 'get', ], false, ex) - assertCouchbaseDispatchCall(it, span(0)) + if (internalEnabled) { + assertCouchbaseDispatchCall(it, span(0)) + } } } + where: + internalEnabled | _ + true | _ + false | _ } def "check query spans"() { @@ -218,9 +224,11 @@ abstract class CouchbaseClient31Test extends VersionedNamingTestBase { adhoc << [true, false] } - def "check multiple query spans with parent and adhoc false"() { - def query = 'select count(1) from `test-bucket` where (`something` = "wonderful") limit 1' - def normalizedQuery = 'select count(?) from `test-bucket` where (`something` = "wonderful") limit ?' + def "check multiple query spans with parent and adhoc false and internal spans enabled = #internalEnabled"() { + setup: + injectSysConfig("trace.couchbase.internal-spans.enabled", "$internalEnabled") + def query = "select count(1) from `test-bucket` where (`something` = \"$queryArg\") limit 1" + def normalizedQuery = "select count(?) from `test-bucket` where (`something` = \"$queryArg\") limit ?" int count1 = 0 int count2 = 0 @@ -240,32 +248,40 @@ abstract class CouchbaseClient31Test extends VersionedNamingTestBase { } then: - count1 == 250 - count2 == 250 + count1 == expectedCount + count2 == expectedCount assertTraces(1) { sortSpansByStart() - trace(7) { + trace(internalEnabled ? 7 : 3) { basicSpan(it, 'multiple.parent') assertCouchbaseCall(it, "cb.query", [ 'db.couchbase.retries' : { Long }, 'db.couchbase.service' : 'query', ], normalizedQuery, span(0), false) - assertCouchbaseCall(it, "prepare", [ - 'db.couchbase.retries' : { Long }, - 'db.couchbase.service' : 'query', - ], "PREPARE $normalizedQuery", span(1), true) - assertCouchbaseDispatchCall(it, span(2)) + if (internalEnabled) { + assertCouchbaseCall(it, "prepare", [ + 'db.couchbase.retries': { Long }, + 'db.couchbase.service': 'query', + ], "PREPARE $normalizedQuery", span(1), true) + assertCouchbaseDispatchCall(it, span(2)) + } assertCouchbaseCall(it, "cb.query", [ 'db.couchbase.retries' : { Long }, 'db.couchbase.service' : 'query', ], normalizedQuery, span(0), false) - assertCouchbaseCall(it, "execute", [ - 'db.couchbase.retries' : { Long }, - 'db.couchbase.service' : 'query', - ], normalizedQuery, span(4), true) - assertCouchbaseDispatchCall(it, span(5)) + if (internalEnabled) { + assertCouchbaseCall(it, "execute", [ + 'db.couchbase.retries': { Long }, + 'db.couchbase.service': 'query', + ], normalizedQuery, span(4), true) + assertCouchbaseDispatchCall(it, span(5)) + } } } + where: + internalEnabled | queryArg | expectedCount + true | "wonderful" | 250 + false | "notinternal" | 0 // avoid having the query engine reusing previous prepared query } def "check error query spans with parent"() { @@ -299,68 +315,6 @@ abstract class CouchbaseClient31Test extends VersionedNamingTestBase { } } - def "check multiple error query spans with parent and adhoc false"() { - def query = 'select count(1) from `test-bucket` where (`something` = "wonderful") limeit 1' - def normalizedQuery = 'select count(?) from `test-bucket` where (`something` = "wonderful") limeit ?' - int count1 = 0 - int count2 = 0 - Throwable ex1 = null - Throwable ex2 = null - - when: - runUnderTrace('multiple.parent') { - // This results in a call to AsyncCluster.query(...) - try { - cluster.query(query, QueryOptions.queryOptions().adhoc(false)).each { - it.rowsAsObject().each { - count1 = it.getInt('$1') - } - } - } catch (CouchbaseException expected) { - ex1 = expected - } - try { - cluster.query(query, QueryOptions.queryOptions().adhoc(false)).each { - it.rowsAsObject().each { - count2 = it.getInt('$1') - } - } - } catch (CouchbaseException expected) { - ex2 = expected - } - } - - then: - count1 == 0 - count2 == 0 - ex1 != null - ex2 != null - assertTraces(1) { - sortSpansByStart() - trace(7) { - basicSpan(it, 'multiple.parent') - assertCouchbaseCall(it, "cb.query", [ - 'db.couchbase.retries' : { Long }, - 'db.couchbase.service' : 'query', - ], normalizedQuery, span(0), false, ex1) - assertCouchbaseCall(it, "prepare", [ - 'db.couchbase.retries' : { Long }, - 'db.couchbase.service' : 'query', - ], "PREPARE $normalizedQuery", span(1), true, ex1) - assertCouchbaseDispatchCall(it, span(2)) - assertCouchbaseCall(it, "cb.query", [ - 'db.couchbase.retries' : { Long }, - 'db.couchbase.service' : 'query', - ], normalizedQuery, span(0), false, ex2) - assertCouchbaseCall(it, "prepare", [ - 'db.couchbase.retries' : { Long }, - 'db.couchbase.service' : 'query', - ], "PREPARE $normalizedQuery", span(4), true, ex2) - assertCouchbaseDispatchCall(it, span(5)) - } - } - } - void assertCouchbaseCall(TraceAssert trace, String name, Map extraTags, boolean internal = false, Throwable ex = null) { assertCouchbaseCall(trace, name, extraTags, null, null, internal, ex) } diff --git a/dd-java-agent/instrumentation/couchbase/couchbase-3.2/src/main/java/datadog/trace/instrumentation/couchbase_32/client/DatadogRequestTracer.java b/dd-java-agent/instrumentation/couchbase/couchbase-3.2/src/main/java/datadog/trace/instrumentation/couchbase_32/client/DatadogRequestTracer.java index 1386bbf58c9..f3bc994fa3c 100644 --- a/dd-java-agent/instrumentation/couchbase/couchbase-3.2/src/main/java/datadog/trace/instrumentation/couchbase_32/client/DatadogRequestTracer.java +++ b/dd-java-agent/instrumentation/couchbase/couchbase-3.2/src/main/java/datadog/trace/instrumentation/couchbase_32/client/DatadogRequestTracer.java @@ -1,5 +1,6 @@ package datadog.trace.instrumentation.couchbase_32.client; +import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.blackholeSpan; import static datadog.trace.instrumentation.couchbase_32.client.CouchbaseClientDecorator.COUCHBASE_CLIENT; import static datadog.trace.instrumentation.couchbase_32.client.CouchbaseClientDecorator.OPERATION_NAME; @@ -36,32 +37,36 @@ public RequestSpan requestSpan(String requestName, RequestSpan requestParent) { boolean measured = true; Object seedNodes = null; - final Config config = Config.get(); AgentSpan parent = DatadogRequestSpan.unwrap(requestParent); if (null == parent) { parent = tracer.activeSpan(); } + DatadogRequestSpan requestSpan = null; + if (null != parent && COUCHBASE_CLIENT.equals(parent.getTag(Tags.COMPONENT))) { - if (!config.isCouchbaseInternalEnabled()) { - return requestParent; + if (!Config.get().isCouchbaseInternalSpansEnabled()) { + // mute the tracing related to internal spans + requestSpan = DatadogRequestSpan.wrap(blackholeSpan(), coreContext); + } else { + spanName = COUCHBASE_INTERNAL; + measured = false; + seedNodes = parent.getTag(InstrumentationTags.COUCHBASE_SEED_NODES); } - spanName = COUCHBASE_INTERNAL; - measured = false; - seedNodes = parent.getTag(InstrumentationTags.COUCHBASE_SEED_NODES); - } - - AgentTracer.SpanBuilder builder = tracer.buildSpan(spanName); - if (null != parent) { - builder.asChildOf(parent.context()); } - AgentSpan span = builder.start(); - CouchbaseClientDecorator.DECORATE.afterStart(span); - span.setResourceName(requestName); - span.setMeasured(measured); - if (seedNodes != null) { - span.setTag(InstrumentationTags.COUCHBASE_SEED_NODES, seedNodes); + if (requestSpan == null) { + AgentTracer.SpanBuilder builder = tracer.buildSpan(spanName); + if (null != parent) { + builder.asChildOf(parent.context()); + } + AgentSpan span = builder.start(); + CouchbaseClientDecorator.DECORATE.afterStart(span); + span.setResourceName(requestName); + span.setMeasured(measured); + if (seedNodes != null) { + span.setTag(InstrumentationTags.COUCHBASE_SEED_NODES, seedNodes); + } + requestSpan = DatadogRequestSpan.wrap(span, coreContext); } - DatadogRequestSpan requestSpan = DatadogRequestSpan.wrap(span, coreContext); // When Couchbase converts a query to a prepare statement or execute statement, // it will not finish the original span switch (requestName) { diff --git a/dd-java-agent/instrumentation/couchbase/couchbase-3.2/src/test/groovy/CouchbaseClient32Test.groovy b/dd-java-agent/instrumentation/couchbase/couchbase-3.2/src/test/groovy/CouchbaseClient32Test.groovy index 0d851c6fcb8..2a8cd522888 100644 --- a/dd-java-agent/instrumentation/couchbase/couchbase-3.2/src/test/groovy/CouchbaseClient32Test.groovy +++ b/dd-java-agent/instrumentation/couchbase/couchbase-3.2/src/test/groovy/CouchbaseClient32Test.groovy @@ -112,8 +112,9 @@ abstract class CouchbaseClient32Test extends VersionedNamingTestBase { } } - def "check basic error spans"() { + def "check basic error spans with internal spans enabled #internalEnabled"() { setup: + injectSysConfig("trace.couchbase.internal-spans.enabled", "$internalEnabled") def collection = bucket.defaultCollection() Throwable ex = null @@ -128,7 +129,7 @@ abstract class CouchbaseClient32Test extends VersionedNamingTestBase { ex != null assertTraces(1) { sortSpansByStart() - trace(2) { + trace(internalEnabled ? 2: 1) { assertCouchbaseCall(it, "get", [ 'db.couchbase.collection' : '_default', 'db.couchbase.document_id': { String }, @@ -138,14 +139,18 @@ abstract class CouchbaseClient32Test extends VersionedNamingTestBase { 'db.name' : BUCKET, 'db.operation' : 'get' ], false, ex) - assertCouchbaseDispatchCall(it, span(0), [ - 'db.couchbase.collection' : '_default', - 'db.couchbase.document_id' : { String }, - 'db.couchbase.scope' : '_default', - 'db.name' : BUCKET - ]) + if (internalEnabled) { + assertCouchbaseDispatchCall(it, span(0), [ + 'db.couchbase.collection' : '_default', + 'db.couchbase.document_id' : { String }, + 'db.couchbase.scope' : '_default', + 'db.name' : BUCKET + ]) + } } } + where: + internalEnabled << [true, false] } def "check query spans"() { @@ -228,10 +233,11 @@ abstract class CouchbaseClient32Test extends VersionedNamingTestBase { adhoc << [true, false] } - def "check multiple async query spans with parent and adhoc false"() { + def "check multiple async query spans with parent and adhoc false and internal spans enabled = #internalEnabled"() { setup: - def query = 'select count(1) from `test-bucket` where (`something` = "wonderful") limit 1' - def normalizedQuery = 'select count(?) from `test-bucket` where (`something` = "wonderful") limit ?' + injectSysConfig("trace.couchbase.internal-spans.enabled", "$internalEnabled") + def query = "select count(1) from `test-bucket` where (`something` = \"$queryArg\") limit 1" + def normalizedQuery = "select count(?) from `test-bucket` where (`something` = \"$queryArg\") limit ?" int count1 = 0 int count2 = 0 def extraPrepare = isLatestDepTest @@ -252,38 +258,46 @@ abstract class CouchbaseClient32Test extends VersionedNamingTestBase { } then: - count1 == 250 - count2 == 250 + count1 == expectedCount + count2 == expectedCount assertTraces(1) { - sortSpansByStart() - trace(extraPrepare ? 8 : 7) { + trace(internalEnabled ? (extraPrepare ? 8 : 7) : 3) { + sortSpansByStart() basicSpan(it, 'async.multiple') assertCouchbaseCall(it, normalizedQuery, [ 'db.couchbase.retries' : { Long }, 'db.couchbase.service' : 'query' ], span(0)) - assertCouchbaseCall(it, "PREPARE $normalizedQuery", [ - 'db.couchbase.retries': { Long }, - 'db.couchbase.service': 'query' - ], span(1), true) - assertCouchbaseDispatchCall(it, span(2)) + if (internalEnabled) { + assertCouchbaseCall(it, "PREPARE $normalizedQuery", [ + 'db.couchbase.retries': { Long }, + 'db.couchbase.service': 'query' + ], span(1), true) + assertCouchbaseDispatchCall(it, span(2)) + } assertCouchbaseCall(it, normalizedQuery, [ 'db.couchbase.retries' : { Long }, 'db.couchbase.service' : 'query' ], span(0)) - if (extraPrepare) { - assertCouchbaseCall(it, "PREPARE $normalizedQuery", [ + if (internalEnabled) { + if (extraPrepare) { + assertCouchbaseCall(it, "PREPARE $normalizedQuery", [ + 'db.couchbase.retries': { Long }, + 'db.couchbase.service': 'query' + ], span(4), true) + } + assertCouchbaseCall(it, normalizedQuery, [ 'db.couchbase.retries': { Long }, 'db.couchbase.service': 'query' ], span(4), true) + assertCouchbaseDispatchCall(it, span(extraPrepare ? 6 : 5)) } - assertCouchbaseCall(it, normalizedQuery, [ - 'db.couchbase.retries': { Long }, - 'db.couchbase.service': 'query' - ], span(4), true) - assertCouchbaseDispatchCall(it, span(extraPrepare ? 6 : 5)) } } + where: + internalEnabled | queryArg | expectedCount + true | "wonderful" | 250 + false | "notinternal" | 0 // avoid having the query engine reusing previous prepared query } def "check error query spans with parent"() { diff --git a/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java b/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java index 0c009dfbb7b..92d63abd628 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java @@ -214,7 +214,7 @@ public final class ConfigDefaults { static final float DEFAULT_TRACE_FLUSH_INTERVAL = 1; - static final boolean DEFAULT_COUCHBASE_INTERNAL_ENABLED = true; + static final boolean DEFAULT_COUCHBASE_INTERNAL_SPANS_ENABLED = true; static final boolean DEFAULT_ELASTICSEARCH_BODY_ENABLED = false; static final boolean DEFAULT_ELASTICSEARCH_PARAMS_ENABLED = true; static final boolean DEFAULT_ELASTICSEARCH_BODY_AND_PARAMS_ENABLED = false; diff --git a/dd-trace-api/src/main/java/datadog/trace/api/config/TraceInstrumentationConfig.java b/dd-trace-api/src/main/java/datadog/trace/api/config/TraceInstrumentationConfig.java index 75ac22e6812..1057e520244 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/config/TraceInstrumentationConfig.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/config/TraceInstrumentationConfig.java @@ -124,9 +124,8 @@ public final class TraceInstrumentationConfig { public static final String RESOLVER_USE_URL_CACHES = "resolver.use.url.caches"; public static final String RESOLVER_RESET_INTERVAL = "resolver.reset.interval"; public static final String RESOLVER_NAMES_ARE_UNIQUE = "resolver.names.are.unique"; - - public static final String COUCHBASE_INTERNAL_ENABLED = "trace.couchbase.internal.enabled"; - + public static final String COUCHBASE_INTERNAL_SPANS_ENABLED = + "trace.couchbase.internal-spans.enabled"; public static final String ELASTICSEARCH_BODY_ENABLED = "trace.elasticsearch.body.enabled"; public static final String ELASTICSEARCH_PARAMS_ENABLED = "trace.elasticsearch.params.enabled"; public static final String ELASTICSEARCH_BODY_AND_PARAMS_ENABLED = diff --git a/internal-api/src/main/java/datadog/trace/api/Config.java b/internal-api/src/main/java/datadog/trace/api/Config.java index 8ccfe3e4924..18058fbaeaf 100644 --- a/internal-api/src/main/java/datadog/trace/api/Config.java +++ b/internal-api/src/main/java/datadog/trace/api/Config.java @@ -30,7 +30,7 @@ import static datadog.trace.api.ConfigDefaults.DEFAULT_CIVISIBILITY_SOURCE_DATA_ROOT_CHECK_ENABLED; import static datadog.trace.api.ConfigDefaults.DEFAULT_CLIENT_IP_ENABLED; import static datadog.trace.api.ConfigDefaults.DEFAULT_CLOCK_SYNC_PERIOD; -import static datadog.trace.api.ConfigDefaults.DEFAULT_COUCHBASE_INTERNAL_ENABLED; +import static datadog.trace.api.ConfigDefaults.DEFAULT_COUCHBASE_INTERNAL_SPANS_ENABLED; import static datadog.trace.api.ConfigDefaults.DEFAULT_CWS_ENABLED; import static datadog.trace.api.ConfigDefaults.DEFAULT_CWS_TLS_REFRESH; import static datadog.trace.api.ConfigDefaults.DEFAULT_DATA_STREAMS_BUCKET_DURATION; @@ -329,7 +329,7 @@ import static datadog.trace.api.config.RemoteConfigConfig.REMOTE_CONFIG_TARGETS_KEY_ID; import static datadog.trace.api.config.RemoteConfigConfig.REMOTE_CONFIG_URL; import static datadog.trace.api.config.TraceInstrumentationConfig.AXIS_PROMOTE_RESOURCE_NAME; -import static datadog.trace.api.config.TraceInstrumentationConfig.COUCHBASE_INTERNAL_ENABLED; +import static datadog.trace.api.config.TraceInstrumentationConfig.COUCHBASE_INTERNAL_SPANS_ENABLED; import static datadog.trace.api.config.TraceInstrumentationConfig.DB_CLIENT_HOST_SPLIT_BY_HOST; import static datadog.trace.api.config.TraceInstrumentationConfig.DB_CLIENT_HOST_SPLIT_BY_INSTANCE; import static datadog.trace.api.config.TraceInstrumentationConfig.DB_CLIENT_HOST_SPLIT_BY_INSTANCE_TYPE_SUFFIX; @@ -896,7 +896,7 @@ static class HostNameHolder { private final boolean longRunningTraceEnabled; private final long longRunningTraceInitialFlushInterval; private final long longRunningTraceFlushInterval; - private final boolean couchbaseInternalEnabled; + private final boolean couchbaseInternalSpansEnabled; private final boolean elasticsearchBodyEnabled; private final boolean elasticsearchParamsEnabled; private final boolean elasticsearchBodyAndParamsEnabled; @@ -984,8 +984,9 @@ private Config(final ConfigProvider configProvider, final InstrumenterConfig ins } else { secureRandom = configProvider.getBoolean(SECURE_RANDOM, DEFAULT_SECURE_RANDOM); } - couchbaseInternalEnabled = - configProvider.getBoolean(COUCHBASE_INTERNAL_ENABLED, DEFAULT_COUCHBASE_INTERNAL_ENABLED); + couchbaseInternalSpansEnabled = + configProvider.getBoolean( + COUCHBASE_INTERNAL_SPANS_ENABLED, DEFAULT_COUCHBASE_INTERNAL_SPANS_ENABLED); elasticsearchBodyEnabled = configProvider.getBoolean(ELASTICSEARCH_BODY_ENABLED, DEFAULT_ELASTICSEARCH_BODY_ENABLED); elasticsearchParamsEnabled = @@ -3292,8 +3293,8 @@ public BitSet getGrpcClientErrorStatuses() { return grpcClientErrorStatuses; } - public boolean isCouchbaseInternalEnabled() { - return couchbaseInternalEnabled; + public boolean isCouchbaseInternalSpansEnabled() { + return couchbaseInternalSpansEnabled; } public boolean isElasticsearchBodyEnabled() { @@ -4370,8 +4371,8 @@ public String toString() { + longRunningTraceInitialFlushInterval + ", longRunningTraceFlushInterval=" + longRunningTraceFlushInterval - + ", couchbaseInternalEnabled=" - + couchbaseInternalEnabled + + ", couchbaseInternalSpansEnabled=" + + couchbaseInternalSpansEnabled + ", elasticsearchBodyEnabled=" + elasticsearchBodyEnabled + ", elasticsearchParamsEnabled=" From e82d3f4c7132eb2293ec2ccca4c03116011f4092 Mon Sep 17 00:00:00 2001 From: Paul Laffon Date: Mon, 4 Mar 2024 09:45:01 +0100 Subject: [PATCH 03/30] Add aws.object.key tag for aws-sdk-2 (#6709) Add aws.object.key for aws-sdk-2 to know which key is targeted by S3 get/put requests More information on S3 request by having the key of the targeted object. It also unlocks object-level data lineage for S3 --- .../aws/v2/AwsSdkClientDecorator.java | 16 ++++++++++++++++ .../src/test/groovy/Aws2ClientTest.groovy | 14 ++++++++++++++ .../groovy/LegacyAws2ClientForkedTest.groovy | 14 ++++++++++++++ 3 files changed, 44 insertions(+) diff --git a/dd-java-agent/instrumentation/aws-java-sdk-2.2/src/main/java/datadog/trace/instrumentation/aws/v2/AwsSdkClientDecorator.java b/dd-java-agent/instrumentation/aws-java-sdk-2.2/src/main/java/datadog/trace/instrumentation/aws/v2/AwsSdkClientDecorator.java index bc50763d0ff..b938b4b8fa0 100644 --- a/dd-java-agent/instrumentation/aws-java-sdk-2.2/src/main/java/datadog/trace/instrumentation/aws/v2/AwsSdkClientDecorator.java +++ b/dd-java-agent/instrumentation/aws-java-sdk-2.2/src/main/java/datadog/trace/instrumentation/aws/v2/AwsSdkClientDecorator.java @@ -106,6 +106,7 @@ public AgentSpan onSdkRequest( // S3 request.getValueForField("Bucket", String.class).ifPresent(name -> setBucketName(span, name)); + getRequestKey(request).ifPresent(key -> setObjectKey(span, key)); request .getValueForField("StorageClass", String.class) .ifPresent( @@ -215,6 +216,21 @@ private static void setBucketName(AgentSpan span, String name) { setPeerService(span, InstrumentationTags.AWS_BUCKET_NAME, name); } + private static Optional getRequestKey(SdkRequest request) { + Optional key = Optional.empty(); + try { + key = request.getValueForField("Key", String.class); + } catch (ClassCastException ignored) { + // Key is not always a string, like for dynamodb GetItemRequest + } + + return key; + } + + private static void setObjectKey(AgentSpan span, String key) { + span.setTag(InstrumentationTags.AWS_OBJECT_KEY, key); + } + private static void setQueueName(AgentSpan span, String name) { span.setTag(InstrumentationTags.AWS_QUEUE_NAME, name); span.setTag(InstrumentationTags.QUEUE_NAME, name); diff --git a/dd-java-agent/instrumentation/aws-java-sdk-2.2/src/test/groovy/Aws2ClientTest.groovy b/dd-java-agent/instrumentation/aws-java-sdk-2.2/src/test/groovy/Aws2ClientTest.groovy index b9a6ec826fd..85d82ada722 100644 --- a/dd-java-agent/instrumentation/aws-java-sdk-2.2/src/test/groovy/Aws2ClientTest.groovy +++ b/dd-java-agent/instrumentation/aws-java-sdk-2.2/src/test/groovy/Aws2ClientTest.groovy @@ -17,7 +17,10 @@ import software.amazon.awssdk.http.apache.ApacheHttpClient import software.amazon.awssdk.regions.Region import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient import software.amazon.awssdk.services.dynamodb.DynamoDbClient +import software.amazon.awssdk.services.dynamodb.model.AttributeValue import software.amazon.awssdk.services.dynamodb.model.CreateTableRequest +import software.amazon.awssdk.services.dynamodb.model.GetItemRequest +import software.amazon.awssdk.services.dynamodb.model.UpdateItemRequest import software.amazon.awssdk.services.ec2.Ec2AsyncClient import software.amazon.awssdk.services.ec2.Ec2Client import software.amazon.awssdk.services.kinesis.KinesisClient @@ -140,6 +143,9 @@ abstract class Aws2ClientTest extends VersionedNamingTestBase { if (operation == "PutObject") { "aws.storage.class" "GLACIER" } + if (operation == "PutObject" || operation == "GetObject") { + "aws.object.key" "somekey" + } peerServiceFrom("aws.bucket.name") checkPeerService = true } else if (service == "Sqs" && operation == "CreateQueue") { @@ -181,6 +187,8 @@ abstract class Aws2ClientTest extends VersionedNamingTestBase { "S3" | "GetObject" | "GET" | "/somebucket/somekey" | "UNKNOWN" | S3Client.builder() | { c -> c.getObject(GetObjectRequest.builder().bucket("somebucket").key("somekey").build()) } | "" "S3" | "PutObject" | "PUT" | "/somebucket/somekey" | "UNKNOWN" | S3Client.builder() | { c -> c.putObject(PutObjectRequest.builder().bucket("somebucket").key("somekey").storageClass(StorageClass.GLACIER).build(), RequestBody.fromString("body")) } | "body" "DynamoDb" | "CreateTable" | "POST" | "/" | "UNKNOWN" | DynamoDbClient.builder() | { c -> c.createTable(CreateTableRequest.builder().tableName("sometable").build()) } | "" + "DynamoDb" | "GetItem" | "POST" | "/" | "UNKNOWN" | DynamoDbClient.builder() | { c -> c.getItem(GetItemRequest.builder().tableName("sometable").key(["attribute": AttributeValue.builder().s("somevalue").build()]).build()) } | "" + "DynamoDb" | "UpdateItem" | "POST" | "/" | "UNKNOWN" | DynamoDbClient.builder() | { c -> c.updateItem(UpdateItemRequest.builder().tableName("sometable").key(["attribute": AttributeValue.builder().s("somevalue").build()]).build()) } | "" "Kinesis" | "DeleteStream" | "POST" | "/" | "UNKNOWN" | KinesisClient.builder() | { c -> c.deleteStream(DeleteStreamRequest.builder().streamName("somestream").build()) } | "" "Sqs" | "CreateQueue" | "POST" | "/" | "7a62c49f-347e-4fc4-9331-6e8e7a96aa73" | SqsClient.builder() | { c -> c.createQueue(CreateQueueRequest.builder().queueName("somequeue").build()) } | """ @@ -270,6 +278,9 @@ abstract class Aws2ClientTest extends VersionedNamingTestBase { if (service == "S3") { "aws.bucket.name" "somebucket" "bucketname" "somebucket" + if (operation == "PutObject" || operation == "GetObject") { + "aws.object.key" "somekey" + } peerServiceFrom("aws.bucket.name") checkPeerService = true } else if (service == "Sqs" && operation == "CreateQueue") { @@ -310,6 +321,8 @@ abstract class Aws2ClientTest extends VersionedNamingTestBase { "S3" | "CreateBucket" | "PUT" | "/somebucket" | "UNKNOWN" | S3AsyncClient.builder() | { c -> c.createBucket(CreateBucketRequest.builder().bucket("somebucket").build()) } | "" "S3" | "GetObject" | "GET" | "/somebucket/somekey" | "UNKNOWN" | S3AsyncClient.builder() | { c -> c.getObject(GetObjectRequest.builder().bucket("somebucket").key("somekey").build(), AsyncResponseTransformer.toBytes()) } | "1234567890" "DynamoDb" | "CreateTable" | "POST" | "/" | "UNKNOWN" | DynamoDbAsyncClient.builder() | { c -> c.createTable(CreateTableRequest.builder().tableName("sometable").build()) } | "" + "DynamoDb" | "GetItem" | "POST" | "/" | "UNKNOWN" | DynamoDbAsyncClient.builder() | { c -> c.getItem(GetItemRequest.builder().tableName("sometable").key(["attribute": AttributeValue.builder().s("somevalue").build()]).build()) } | "" + "DynamoDb" | "UpdateItem" | "POST" | "/" | "UNKNOWN" | DynamoDbAsyncClient.builder() | { c -> c.updateItem(UpdateItemRequest.builder().tableName("sometable").key(["attribute": AttributeValue.builder().s("somevalue").build()]).build()) } | "" // Kinesis seems to expect an http2 response which is incompatible with our test server. // "Kinesis" | "DeleteStream" | "java-aws-sdk" | "POST" | "/" | "UNKNOWN" | KinesisAsyncClient.builder() | { c -> c.deleteStream(DeleteStreamRequest.builder().streamName("somestream").build()) } | "" "Sqs" | "CreateQueue" | "POST" | "/" | "7a62c49f-347e-4fc4-9331-6e8e7a96aa73" | SqsAsyncClient.builder() | { c -> c.createQueue(CreateQueueRequest.builder().queueName("somequeue").build()) } | """ @@ -395,6 +408,7 @@ abstract class Aws2ClientTest extends VersionedNamingTestBase { "aws.agent" "java-aws-sdk" "aws.bucket.name" "somebucket" "bucketname" "somebucket" + "aws.object.key" "somekey" errorTags SdkClientException, "Unable to execute HTTP request: Read timed out" peerServiceFrom("aws.bucket.name") defaultTags() diff --git a/dd-java-agent/instrumentation/aws-java-sdk-2.2/src/test/groovy/LegacyAws2ClientForkedTest.groovy b/dd-java-agent/instrumentation/aws-java-sdk-2.2/src/test/groovy/LegacyAws2ClientForkedTest.groovy index a50e2584bcc..b1a09feca00 100644 --- a/dd-java-agent/instrumentation/aws-java-sdk-2.2/src/test/groovy/LegacyAws2ClientForkedTest.groovy +++ b/dd-java-agent/instrumentation/aws-java-sdk-2.2/src/test/groovy/LegacyAws2ClientForkedTest.groovy @@ -14,7 +14,10 @@ import software.amazon.awssdk.http.apache.ApacheHttpClient import software.amazon.awssdk.regions.Region import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient import software.amazon.awssdk.services.dynamodb.DynamoDbClient +import software.amazon.awssdk.services.dynamodb.model.AttributeValue import software.amazon.awssdk.services.dynamodb.model.CreateTableRequest +import software.amazon.awssdk.services.dynamodb.model.GetItemRequest +import software.amazon.awssdk.services.dynamodb.model.UpdateItemRequest import software.amazon.awssdk.services.ec2.Ec2AsyncClient import software.amazon.awssdk.services.ec2.Ec2Client import software.amazon.awssdk.services.kinesis.KinesisClient @@ -130,6 +133,9 @@ class LegacyAws2ClientForkedTest extends AgentTestRunner { if (operation == "PutObject") { "aws.storage.class" "GLACIER" } + if (operation == "PutObject" || operation == "GetObject") { + "aws.object.key" "somekey" + } } else if (service == "Sqs" && operation == "CreateQueue") { "aws.queue.name" "somequeue" "queuename" "somequeue" @@ -177,6 +183,8 @@ class LegacyAws2ClientForkedTest extends AgentTestRunner { "S3" | "GetObject" | "java-aws-sdk" | "GET" | "/somebucket/somekey" | "UNKNOWN" | S3Client.builder() | { c -> c.getObject(GetObjectRequest.builder().bucket("somebucket").key("somekey").build()) } | "" "S3" | "PutObject" | "java-aws-sdk" | "PUT" | "/somebucket/somekey" | "UNKNOWN" | S3Client.builder() | { c -> c.putObject(PutObjectRequest.builder().bucket("somebucket").key("somekey").storageClass(StorageClass.GLACIER).build(), RequestBody.fromString("body")) } | "body" "DynamoDb" | "CreateTable" | "java-aws-sdk" | "POST" | "/" | "UNKNOWN" | DynamoDbClient.builder() | { c -> c.createTable(CreateTableRequest.builder().tableName("sometable").build()) } | "" + "DynamoDb" | "GetItem" | "java-aws-sdk" | "POST" | "/" | "UNKNOWN" | DynamoDbClient.builder() | { c -> c.getItem(GetItemRequest.builder().tableName("sometable").key(["attribute": AttributeValue.builder().s("somevalue").build()]).build()) } | "" + "DynamoDb" | "UpdateItem" | "java-aws-sdk" | "POST" | "/" | "UNKNOWN" | DynamoDbClient.builder() | { c -> c.updateItem(UpdateItemRequest.builder().tableName("sometable").key(["attribute": AttributeValue.builder().s("somevalue").build()]).build()) } | "" "Kinesis" | "DeleteStream" | "java-aws-sdk" | "POST" | "/" | "UNKNOWN" | KinesisClient.builder() | { c -> c.deleteStream(DeleteStreamRequest.builder().streamName("somestream").build()) } | "" "Sqs" | "CreateQueue" | "java-aws-sdk" | "POST" | "/" | "7a62c49f-347e-4fc4-9331-6e8e7a96aa73" | SqsClient.builder() | { c -> c.createQueue(CreateQueueRequest.builder().queueName("somequeue").build()) } | """ @@ -265,6 +273,9 @@ class LegacyAws2ClientForkedTest extends AgentTestRunner { if (service == "S3") { "aws.bucket.name" "somebucket" "bucketname" "somebucket" + if (operation == "PutObject" || operation == "GetObject") { + "aws.object.key" "somekey" + } } else if (service == "Sqs" && operation == "CreateQueue") { "aws.queue.name" "somequeue" "queuename" "somequeue" @@ -315,6 +326,8 @@ class LegacyAws2ClientForkedTest extends AgentTestRunner { "S3" | "CreateBucket" | "java-aws-sdk" | "PUT" | "/somebucket" | "UNKNOWN" | S3AsyncClient.builder() | { c -> c.createBucket(CreateBucketRequest.builder().bucket("somebucket").build()) } | "" "S3" | "GetObject" | "java-aws-sdk" | "GET" | "/somebucket/somekey" | "UNKNOWN" | S3AsyncClient.builder() | { c -> c.getObject(GetObjectRequest.builder().bucket("somebucket").key("somekey").build(), AsyncResponseTransformer.toBytes()) } | "1234567890" "DynamoDb" | "CreateTable" | "java-aws-sdk" | "POST" | "/" | "UNKNOWN" | DynamoDbAsyncClient.builder() | { c -> c.createTable(CreateTableRequest.builder().tableName("sometable").build()) } | "" + "DynamoDb" | "GetItem" | "java-aws-sdk" | "POST" | "/" | "UNKNOWN" | DynamoDbAsyncClient.builder() | { c -> c.getItem(GetItemRequest.builder().tableName("sometable").key(["attribute": AttributeValue.builder().s("somevalue").build()]).build()) } | "" + "DynamoDb" | "UpdateItem" | "java-aws-sdk" | "POST" | "/" | "UNKNOWN" | DynamoDbAsyncClient.builder() | { c -> c.updateItem(UpdateItemRequest.builder().tableName("sometable").key(["attribute": AttributeValue.builder().s("somevalue").build()]).build()) } | "" // Kinesis seems to expect an http2 response which is incompatible with our test server. // "Kinesis" | "DeleteStream" | "java-aws-sdk" | "POST" | "/" | "UNKNOWN" | KinesisAsyncClient.builder() | { c -> c.deleteStream(DeleteStreamRequest.builder().streamName("somestream").build()) } | "" "Sqs" | "CreateQueue" | "java-aws-sdk" | "POST" | "/" | "7a62c49f-347e-4fc4-9331-6e8e7a96aa73" | SqsAsyncClient.builder() | { c -> c.createQueue(CreateQueueRequest.builder().queueName("somequeue").build()) } | """ @@ -400,6 +413,7 @@ class LegacyAws2ClientForkedTest extends AgentTestRunner { "aws.agent" "java-aws-sdk" "aws.bucket.name" "somebucket" "bucketname" "somebucket" + "aws.object.key" "somekey" errorTags SdkClientException, "Unable to execute HTTP request: Read timed out" defaultTags() } From 938c791c4dc6a033d41e928d20050639b640bad4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vandon?= Date: Mon, 4 Mar 2024 12:02:19 +0100 Subject: [PATCH 04/30] Add DSM support to SNS (#6721) --- .../aws-java-sdk-1.11.0/build.gradle | 15 +- .../groovy/AWS1KinesisClientTest.groovy | 0 .../dsmTest/groovy/AWS1SnsClientTest.groovy | 183 ++++++++++ .../aws/v0/AwsSdkClientDecorator.java | 67 +++- .../instrumentation/aws/v0/GetterAccess.java | 6 + .../aws-java-sdk-2.2/build.gradle | 21 +- .../groovy/Aws2KinesisDataStreamsTest.groovy | 63 +++- .../groovy/Aws2SnsDataStreamsTest.groovy | 341 ++++++++++++++++++ .../aws/v2/AwsSdkClientDecorator.java | 59 ++- .../trace/api/naming/v1/CloudNamingV1.java | 2 + 10 files changed, 687 insertions(+), 70 deletions(-) rename dd-java-agent/instrumentation/aws-java-sdk-1.11.0/src/{kinesisDsmTest => dsmTest}/groovy/AWS1KinesisClientTest.groovy (100%) create mode 100644 dd-java-agent/instrumentation/aws-java-sdk-1.11.0/src/dsmTest/groovy/AWS1SnsClientTest.groovy rename dd-java-agent/instrumentation/aws-java-sdk-2.2/src/{kinesisDsmTest => dsmTest}/groovy/Aws2KinesisDataStreamsTest.groovy (91%) create mode 100644 dd-java-agent/instrumentation/aws-java-sdk-2.2/src/dsmTest/groovy/Aws2SnsDataStreamsTest.groovy diff --git a/dd-java-agent/instrumentation/aws-java-sdk-1.11.0/build.gradle b/dd-java-agent/instrumentation/aws-java-sdk-1.11.0/build.gradle index 6625bc50fb0..7e6d421b0d3 100644 --- a/dd-java-agent/instrumentation/aws-java-sdk-1.11.0/build.gradle +++ b/dd-java-agent/instrumentation/aws-java-sdk-1.11.0/build.gradle @@ -28,10 +28,10 @@ addTestSuiteExtendingForDir('test_before_1_11_106ForkedTest', 'test_before_1_11_ addTestSuiteForDir('latestDepTest', 'test') addTestSuiteExtendingForDir('latestDepForkedTest', 'latestDepTest', 'test') -addTestSuite('kinesisDsmTest') -addTestSuiteExtendingForDir('kinesisDsmForkedTest', 'kinesisDsmTest', 'kinesisDsmTest') -addTestSuiteForDir('latestKinesisDsmTest', 'kinesisDsmTest') -addTestSuiteExtendingForDir('latestKinesisDsmForkedTest', 'latestKinesisDsmTest', 'kinesisDsmTest') +addTestSuite('dsmTest') +addTestSuiteExtendingForDir('dsmForkedTest', 'dsmTest', 'dsmTest') +addTestSuiteForDir('latestDsmTest', 'dsmTest') +addTestSuiteExtendingForDir('latestDsmForkedTest', 'latestDsmTest', 'dsmTest') dependencies { compileOnly group: 'com.amazonaws', name: 'aws-java-sdk-core', version: '1.11.0' @@ -88,8 +88,11 @@ dependencies { } } - kinesisDsmTestImplementation group: 'com.amazonaws', name: 'aws-java-sdk-kinesis', version: '1.12.366' - latestKinesisDsmTestImplementation group: 'com.amazonaws', name: 'aws-java-sdk-kinesis', version: '+' + dsmTestImplementation group: 'com.amazonaws', name: 'aws-java-sdk-kinesis', version: '1.12.366' + // no batch publish before v1.12 + dsmTestImplementation group: 'com.amazonaws', name: 'aws-java-sdk-sns', version: '1.12.366' + latestDsmTestImplementation group: 'com.amazonaws', name: 'aws-java-sdk-kinesis', version: '+' + latestDsmTestImplementation group: 'com.amazonaws', name: 'aws-java-sdk-sns', version: '+' latestDepTestImplementation group: 'com.amazonaws', name: 'aws-java-sdk-s3', version: '+' latestDepTestImplementation group: 'com.amazonaws', name: 'aws-java-sdk-rds', version: '+' diff --git a/dd-java-agent/instrumentation/aws-java-sdk-1.11.0/src/kinesisDsmTest/groovy/AWS1KinesisClientTest.groovy b/dd-java-agent/instrumentation/aws-java-sdk-1.11.0/src/dsmTest/groovy/AWS1KinesisClientTest.groovy similarity index 100% rename from dd-java-agent/instrumentation/aws-java-sdk-1.11.0/src/kinesisDsmTest/groovy/AWS1KinesisClientTest.groovy rename to dd-java-agent/instrumentation/aws-java-sdk-1.11.0/src/dsmTest/groovy/AWS1KinesisClientTest.groovy diff --git a/dd-java-agent/instrumentation/aws-java-sdk-1.11.0/src/dsmTest/groovy/AWS1SnsClientTest.groovy b/dd-java-agent/instrumentation/aws-java-sdk-1.11.0/src/dsmTest/groovy/AWS1SnsClientTest.groovy new file mode 100644 index 00000000000..28bd6dd4741 --- /dev/null +++ b/dd-java-agent/instrumentation/aws-java-sdk-1.11.0/src/dsmTest/groovy/AWS1SnsClientTest.groovy @@ -0,0 +1,183 @@ +import com.amazonaws.auth.AWSStaticCredentialsProvider +import com.amazonaws.auth.AnonymousAWSCredentials +import com.amazonaws.client.builder.AwsClientBuilder +import com.amazonaws.services.sns.AmazonSNS +import com.amazonaws.services.sns.AmazonSNSClientBuilder +import com.amazonaws.services.sns.model.PublishBatchRequest +import com.amazonaws.services.sns.model.PublishBatchRequestEntry +import com.amazonaws.services.sns.model.PublishRequest +import datadog.trace.agent.test.naming.VersionedNamingTestBase +import datadog.trace.api.Config +import datadog.trace.api.DDSpanTypes +import datadog.trace.api.DDTags +import datadog.trace.bootstrap.instrumentation.api.Tags +import datadog.trace.core.datastreams.StatsGroup +import spock.lang.AutoCleanup +import spock.lang.Shared +import spock.util.concurrent.PollingConditions + +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicReference + +import static datadog.trace.agent.test.server.http.TestHttpServer.httpServer + +abstract class AWS1SnsClientTest extends VersionedNamingTestBase { + + @Shared + def credentialsProvider = new AWSStaticCredentialsProvider(new AnonymousAWSCredentials()) + @Shared + def responseBody = new AtomicReference() + @AutoCleanup + @Shared + def server = httpServer { + handlers { + all { + response.status(200).send(responseBody.get()) + } + } + } + @Shared + def endpoint = new AwsClientBuilder.EndpointConfiguration("http://localhost:$server.address.port", "us-west-2") + + @Shared + final String topicName = "sometopic" + + @Shared + final String topicArn = "arnprefix:" + topicName + + @Override + protected boolean isDataStreamsEnabled() { + return true + } + + @Override + protected long dataStreamsBucketDuration() { + TimeUnit.MILLISECONDS.toNanos(250) + } + + @Override + String operation() { + null + } + + @Override + String service() { + null + } + + abstract String expectedOperation(String awsService, String awsOperation) + + abstract String expectedService(String awsService, String awsOperation) + + def "send #operation request with mocked response produces #dsmStatCount stat points"() { + setup: + def conditions = new PollingConditions(timeout: 1) + responseBody.set(body) + AmazonSNS client = AmazonSNSClientBuilder.standard() + .withEndpointConfiguration(endpoint) + .withCredentials(credentialsProvider) + .build() + + when: + def response = call.call(client) + + TEST_WRITER.waitForTraces(1) + TEST_DATA_STREAMS_WRITER.waitForGroups(1) + + then: + response != null + + conditions.eventually { + List results = TEST_DATA_STREAMS_WRITER.groups.findAll { it.parentHash == 0 } + assert results.size() >= 1 + def pathwayLatencyCount = 0 + def edgeLatencyCount = 0 + results.each { group -> + pathwayLatencyCount += group.pathwayLatency.count + edgeLatencyCount += group.edgeLatency.count + verifyAll(group) { + edgeTags.containsAll(["direction:" + dsmDirection, "topic:" + topicName, "type:sns"]) + edgeTags.size() == 3 + } + } + verifyAll { + pathwayLatencyCount == dsmStatCount + edgeLatencyCount == dsmStatCount + } + } + + assertTraces(1) { + trace(1) { + span { + serviceName expectedService(service, operation) + operationName expectedOperation(service, operation) + resourceName "$service.$operation" + spanType DDSpanTypes.HTTP_CLIENT + errored false + measured true + parent() + tags { + "$Tags.COMPONENT" "java-aws-sdk" + "$Tags.SPAN_KIND" Tags.SPAN_KIND_CLIENT + "$Tags.HTTP_URL" "$server.address/" + "$Tags.HTTP_METHOD" "$method" + "$Tags.HTTP_STATUS" 200 + "$Tags.PEER_PORT" server.address.port + "$Tags.PEER_HOSTNAME" "localhost" + "aws.service" { it.contains(service) } + "aws_service" { it.contains(service.toLowerCase()) } + "aws.endpoint" "$server.address" + "aws.operation" "${operation}Request" + "aws.agent" "java-aws-sdk" + "aws.topic.name" topicName + "topicname" topicName + "$DDTags.PATHWAY_HASH" { String } + peerServiceFrom("aws.topic.name") + defaultTags() + } + } + } + } + + where: + service | operation | dsmDirection | dsmStatCount | method | path | call | body + "SNS" | "Publish" | "out" | 1 | "POST" | "/" | { AmazonSNS c -> c.publish(new PublishRequest().withTopicArn(topicArn).withMessage("hello")) } | "" + "SNS" | "PublishBatch" | "out" | 2 | "POST" | "/" | { AmazonSNS c -> c.publishBatch(new PublishBatchRequest().withTopicArn(topicArn).withPublishBatchRequestEntries(new PublishBatchRequestEntry().withMessage("hello"), new PublishBatchRequestEntry().withMessage("world"))) } | "" + } +} + +class AWS1SnsClientV0Test extends AWS1SnsClientTest { + + @Override + String expectedOperation(String awsService, String awsOperation) { + "aws.http" + } + + @Override + String expectedService(String awsService, String awsOperation) { + return "sns" + } + + @Override + int version() { + 0 + } +} + +class AWS1SnsClientV1ForkedTest extends AWS1SnsClientTest { + + @Override + String expectedOperation(String awsService, String awsOperation) { + return "aws.${awsService.toLowerCase()}.send" + } + + @Override + String expectedService(String awsService, String awsOperation) { + Config.get().getServiceName() + } + + @Override + int version() { + 1 + } +} diff --git a/dd-java-agent/instrumentation/aws-java-sdk-1.11.0/src/main/java/datadog/trace/instrumentation/aws/v0/AwsSdkClientDecorator.java b/dd-java-agent/instrumentation/aws-java-sdk-1.11.0/src/main/java/datadog/trace/instrumentation/aws/v0/AwsSdkClientDecorator.java index 8715e85e2c1..7e5bb045146 100644 --- a/dd-java-agent/instrumentation/aws-java-sdk-1.11.0/src/main/java/datadog/trace/instrumentation/aws/v0/AwsSdkClientDecorator.java +++ b/dd-java-agent/instrumentation/aws-java-sdk-1.11.0/src/main/java/datadog/trace/instrumentation/aws/v0/AwsSdkClientDecorator.java @@ -52,6 +52,8 @@ public class AwsSdkClientDecorator extends HttpClientDecorator objectType) { getStreamName = findStringGetter(objectType, "getStreamName"); getStreamARN = findStringGetter(objectType, "getStreamARN"); getRecords = findListGetter(objectType, "getRecords"); + getPublishBatchRequestEntries = findListGetter(objectType, "getPublishBatchRequestEntries"); getApproximateArrivalTimestamp = findGetter(objectType, "getApproximateArrivalTimestamp", Date.class); getTableName = findStringGetter(objectType, "getTableName"); @@ -91,6 +93,10 @@ List getRecords(final Object object) { return invokeForList(getRecords, object); } + List getEntries(final Object object) { + return invokeForList(getPublishBatchRequestEntries, object); + } + String getTableName(final Object object) { return invokeForString(getTableName, object); } diff --git a/dd-java-agent/instrumentation/aws-java-sdk-2.2/build.gradle b/dd-java-agent/instrumentation/aws-java-sdk-2.2/build.gradle index c5fe90b2312..e0a2dea22e5 100644 --- a/dd-java-agent/instrumentation/aws-java-sdk-2.2/build.gradle +++ b/dd-java-agent/instrumentation/aws-java-sdk-2.2/build.gradle @@ -14,10 +14,10 @@ addTestSuiteForDir('latestDepTest', 'test') // Broken: at some point S3 moved the bucket name to the hostname resulting in host not found somebucket.localhost on all S3 tests // addTestSuiteExtendingForDir('latestDepForkedTest', 'latestDepTest', 'test') -addTestSuite('kinesisDsmTest') -addTestSuiteExtendingForDir('kinesisDsmForkedTest', 'kinesisDsmTest', 'kinesisDsmTest') -addTestSuiteForDir('latestKinesisDsmTest', 'kinesisDsmTest') -addTestSuiteExtendingForDir('latestKinesisDsmForkedTest', 'latestKinesisDsmTest', 'kinesisDsmTest') +addTestSuite('dsmTest') +addTestSuiteExtendingForDir('dsmForkedTest', 'dsmTest', 'dsmTest') +addTestSuiteForDir('latestDsmTest', 'dsmTest') +addTestSuiteExtendingForDir('latestDsmForkedTest', 'latestDsmTest', 'dsmTest') def fixedSdkVersion = '2.20.33' // 2.20.34 is missing and breaks IDEA import @@ -40,11 +40,14 @@ dependencies { testImplementation group: 'org.eclipse.jetty', name: 'jetty-server', version: '9.3.0.v20150612' testImplementation group: 'org.eclipse.jetty.http2', name: 'http2-server', version: '9.3.0.v20150612' - // First version where dsm traced operations have required StreamARN parameter - kinesisDsmTestImplementation group: 'software.amazon.awssdk', name: 'apache-client', version: '2.18.40' - kinesisDsmTestImplementation group: 'software.amazon.awssdk', name: 'kinesis', version: '2.18.40' - latestKinesisDsmTestImplementation group: 'software.amazon.awssdk', name: 'apache-client', version: '+' - latestKinesisDsmTestImplementation group: 'software.amazon.awssdk', name: 'kinesis', version: '+' + // First version where dsm traced operations have required StreamARN parameter for kinesis + // and publishBatch is available for SNS + dsmTestImplementation group: 'software.amazon.awssdk', name: 'apache-client', version: '2.18.40' + dsmTestImplementation group: 'software.amazon.awssdk', name: 'kinesis', version: '2.18.40' + dsmTestImplementation group: 'software.amazon.awssdk', name: 'sns', version: '2.18.40' + latestDsmTestImplementation group: 'software.amazon.awssdk', name: 'apache-client', version: '+' + latestDsmTestImplementation group: 'software.amazon.awssdk', name: 'kinesis', version: '+' + latestDsmTestImplementation group: 'software.amazon.awssdk', name: 'sns', version: '+' latestDepTestImplementation project(':dd-java-agent:instrumentation:apache-httpclient-4') latestDepTestImplementation project(':dd-java-agent:instrumentation:netty-4.1') diff --git a/dd-java-agent/instrumentation/aws-java-sdk-2.2/src/kinesisDsmTest/groovy/Aws2KinesisDataStreamsTest.groovy b/dd-java-agent/instrumentation/aws-java-sdk-2.2/src/dsmTest/groovy/Aws2KinesisDataStreamsTest.groovy similarity index 91% rename from dd-java-agent/instrumentation/aws-java-sdk-2.2/src/kinesisDsmTest/groovy/Aws2KinesisDataStreamsTest.groovy rename to dd-java-agent/instrumentation/aws-java-sdk-2.2/src/dsmTest/groovy/Aws2KinesisDataStreamsTest.groovy index 342b5befb58..1e649cbb25e 100644 --- a/dd-java-agent/instrumentation/aws-java-sdk-2.2/src/kinesisDsmTest/groovy/Aws2KinesisDataStreamsTest.groovy +++ b/dd-java-agent/instrumentation/aws-java-sdk-2.2/src/dsmTest/groovy/Aws2KinesisDataStreamsTest.groovy @@ -26,6 +26,7 @@ import software.amazon.awssdk.services.kinesis.model.PutRecordsRequest import software.amazon.awssdk.services.kinesis.model.PutRecordsRequestEntry import spock.lang.AutoCleanup import spock.lang.Shared +import spock.util.concurrent.PollingConditions import java.time.Instant import java.util.concurrent.Future @@ -120,6 +121,7 @@ abstract class Aws2KinesisDataStreamsTest extends VersionedNamingTestBase { def "send #operation request with builder #builder.class.getSimpleName() mocked response"() { setup: + def conditions = new PollingConditions(timeout: 1) boolean executed = false def client = builder // tests that our instrumentation doesn't disturb any overridden configuration @@ -144,6 +146,27 @@ abstract class Aws2KinesisDataStreamsTest extends VersionedNamingTestBase { response != null response.class.simpleName.startsWith(operation) || response instanceof ResponseInputStream + and: + conditions.eventually { + List results = TEST_DATA_STREAMS_WRITER.groups.findAll { it.parentHash == 0 } + assert results.size() >= 1 + def pathwayLatencyCount = 0 + def edgeLatencyCount = 0 + results.each { group -> + pathwayLatencyCount += group.pathwayLatency.count + edgeLatencyCount += group.edgeLatency.count + verifyAll(group) { + edgeTags.containsAll(["direction:" + dsmDirection, "topic:arnprefix:stream/somestream", "type:kinesis"]) + edgeTags.size() == 3 + } + } + verifyAll { + pathwayLatencyCount == dsmStatCount + edgeLatencyCount == dsmStatCount + } + } + + and: assertTraces(1) { trace(1) { span { @@ -179,15 +202,6 @@ abstract class Aws2KinesisDataStreamsTest extends VersionedNamingTestBase { } } - and: - StatsGroup first = TEST_DATA_STREAMS_WRITER.groups.find { it.parentHash == 0 } - verifyAll(first) { - edgeTags.containsAll(["direction:" + dsmDirection, "topic:arnprefix:stream/somestream", "type:kinesis"]) - edgeTags.size() == 3 - pathwayLatency.count == dsmStatCount - edgeLatency.count == dsmStatCount - } - cleanup: servedRequestId.set(null) @@ -230,6 +244,7 @@ abstract class Aws2KinesisDataStreamsTest extends VersionedNamingTestBase { def "send #operation async request with builder #builder.class.getSimpleName() mocked response"() { setup: + def conditions = new PollingConditions(timeout: 1) boolean executed = false def client = builder // tests that our instrumentation doesn't disturb any overridden configuration @@ -253,6 +268,27 @@ abstract class Aws2KinesisDataStreamsTest extends VersionedNamingTestBase { executed response != null + and: + conditions.eventually { + List results = TEST_DATA_STREAMS_WRITER.groups.findAll { it.parentHash == 0 } + assert results.size() >= 1 + def pathwayLatencyCount = 0 + def edgeLatencyCount = 0 + results.each { group -> + pathwayLatencyCount += group.pathwayLatency.count + edgeLatencyCount += group.edgeLatency.count + verifyAll(group) { + edgeTags.containsAll(["direction:" + dsmDirection, "topic:arnprefix:stream/somestream", "type:kinesis"]) + edgeTags.size() == 3 + } + } + verifyAll { + pathwayLatencyCount == dsmStatCount + edgeLatencyCount == dsmStatCount + } + } + + and: assertTraces(1) { trace(1) { span { @@ -286,15 +322,6 @@ abstract class Aws2KinesisDataStreamsTest extends VersionedNamingTestBase { } } - and: - StatsGroup first = TEST_DATA_STREAMS_WRITER.groups.find { it.parentHash == 0 } - verifyAll(first) { - edgeTags.containsAll(["direction:" + dsmDirection, "topic:arnprefix:stream/somestream", "type:kinesis"]) - edgeTags.size() == 3 - pathwayLatency.count == dsmStatCount - edgeLatency.count == dsmStatCount - } - cleanup: servedRequestId.set(null) diff --git a/dd-java-agent/instrumentation/aws-java-sdk-2.2/src/dsmTest/groovy/Aws2SnsDataStreamsTest.groovy b/dd-java-agent/instrumentation/aws-java-sdk-2.2/src/dsmTest/groovy/Aws2SnsDataStreamsTest.groovy new file mode 100644 index 00000000000..8ccf140d75d --- /dev/null +++ b/dd-java-agent/instrumentation/aws-java-sdk-2.2/src/dsmTest/groovy/Aws2SnsDataStreamsTest.groovy @@ -0,0 +1,341 @@ +import datadog.trace.agent.test.naming.VersionedNamingTestBase +import datadog.trace.api.Config +import datadog.trace.api.DDSpanTypes +import datadog.trace.api.DDTags +import datadog.trace.bootstrap.instrumentation.api.Tags +import datadog.trace.core.datastreams.StatsGroup +import org.eclipse.jetty.http2.server.HTTP2CServerConnectionFactory +import org.eclipse.jetty.server.HttpConfiguration +import org.eclipse.jetty.server.HttpConnectionFactory +import org.eclipse.jetty.server.Server +import org.eclipse.jetty.server.ServerConnector +import org.eclipse.jetty.server.SslConnectionFactory +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider +import software.amazon.awssdk.core.ResponseInputStream +import software.amazon.awssdk.core.interceptor.Context +import software.amazon.awssdk.core.interceptor.ExecutionAttributes +import software.amazon.awssdk.core.interceptor.ExecutionInterceptor +import software.amazon.awssdk.regions.Region +import software.amazon.awssdk.services.sns.SnsAsyncClient +import software.amazon.awssdk.services.sns.SnsClient +import software.amazon.awssdk.services.sns.model.PublishBatchRequest +import software.amazon.awssdk.services.sns.model.PublishBatchRequestEntry +import software.amazon.awssdk.services.sns.model.PublishRequest +import spock.lang.AutoCleanup +import spock.lang.Shared +import spock.lang.Unroll +import spock.util.concurrent.PollingConditions + +import java.util.concurrent.Future +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicReference + +import static datadog.trace.agent.test.server.http.TestHttpServer.httpServer + +abstract class Aws2SnsDataStreamsTest extends VersionedNamingTestBase { + + private static final StaticCredentialsProvider CREDENTIALS_PROVIDER = StaticCredentialsProvider + .create(AwsBasicCredentials.create("my-access-key", "my-secret-key")) + + @Shared + def responseBody = new AtomicReference() + @Shared + def servedRequestId = new AtomicReference() + + @AutoCleanup + @Shared + def server = httpServer { + customizer { { + Server server -> { + ServerConnector httpConnector = server.getConnectors().find { + !it.connectionFactories.any { + it instanceof SslConnectionFactory + } + } + HttpConfiguration config = (httpConnector.connectionFactories.find { + it instanceof HttpConnectionFactory + } + as HttpConnectionFactory).getHttpConfiguration() + httpConnector.addConnectionFactory(new HTTP2CServerConnectionFactory(config)) + } + } + } + handlers { + all { + response + .status(200) + .addHeader("x-amzn-RequestId", servedRequestId.get()) + .sendWithType("application/x-amz-json-1.1", responseBody.get()) + } + } + } + + @Override + String operation() { + null + } + + @Override + String service() { + null + } + + @Override + protected boolean isDataStreamsEnabled() { + true + } + + @Override + protected long dataStreamsBucketDuration() { + TimeUnit.MILLISECONDS.toNanos(250) + } + + abstract String expectedOperation(String awsService, String awsOperation) + + abstract String expectedService(String awsService, String awsOperation) + + def watch(builder, callback) { + builder.addExecutionInterceptor(new ExecutionInterceptor() { + @Override + void afterExecution(Context.AfterExecution context, ExecutionAttributes executionAttributes) { + callback.call() + } + }) + } + + @Unroll + def "send #operation request with builder #builder.class.getSimpleName() mocked response"() { + setup: + def conditions = new PollingConditions(timeout: 1) + boolean executed = false + def client = builder + // tests that our instrumentation doesn't disturb any overridden configuration + .overrideConfiguration({ watch(it, { executed = true }) }) + .endpointOverride(server.address) + .region(Region.AP_NORTHEAST_1) + .credentialsProvider(CREDENTIALS_PROVIDER) + .build() + responseBody.set(body) + servedRequestId.set(requestId) + when: + def response = call.call(client) + + if (response instanceof Future) { + response = response.get() + } + TEST_WRITER.waitForTraces(1) + TEST_DATA_STREAMS_WRITER.waitForGroups(1) + + + then: + executed + response != null + response.class.simpleName.startsWith(operation) || response instanceof ResponseInputStream + + and: + conditions.eventually { + List results = TEST_DATA_STREAMS_WRITER.groups.findAll { it.parentHash == 0 } + assert results.size() >= 1 + def pathwayLatencyCount = 0 + def edgeLatencyCount = 0 + results.each { group -> + pathwayLatencyCount += group.pathwayLatency.count + edgeLatencyCount += group.edgeLatency.count + verifyAll(group) { + edgeTags.containsAll(["direction:" + dsmDirection, "topic:mytopic", "type:sns"]) + edgeTags.size() == 3 + } + } + verifyAll { + pathwayLatencyCount == dsmStatCount + edgeLatencyCount == dsmStatCount + } + } + + and: + assertTraces(1) { + trace(1) { + span { + serviceName expectedService(service, operation) + operationName expectedOperation(service, operation) + resourceName "$service.$operation" + spanType DDSpanTypes.HTTP_CLIENT + errored false + measured true + parent() + tags { + def checkPeerService = false + "$Tags.COMPONENT" "java-aws-sdk" + "$Tags.SPAN_KIND" Tags.SPAN_KIND_CLIENT + "$Tags.PEER_HOSTNAME" "localhost" + "$Tags.PEER_PORT" server.address.port + "$Tags.HTTP_URL" "${server.address}${path}" + "$Tags.HTTP_METHOD" "$method" + "$Tags.HTTP_STATUS" 200 + "aws.service" "$service" + "aws_service" "$service" + "aws.operation" "${operation}" + "aws.agent" "java-aws-sdk" + "aws.requestId" "$requestId" + "aws.topic.name" "mytopic" + "topicname" "mytopic" + "$DDTags.PATHWAY_HASH" { String } + checkPeerService = true + defaultTags(false, checkPeerService) + } + } + } + } + + cleanup: + servedRequestId.set(null) + + where: + service | operation | dsmDirection | dsmStatCount | method | path | requestId | builder | call | body + "Sns" | "Publish" | "out" | 1 | "POST" | "/" | "7a62c49f-347e-4fc4-9331-6e8e7a96aa73" | SnsClient.builder() | { SnsClient c -> c.publish(PublishRequest.builder().topicArn("arnprefix:mytopic").message("hello").build()) } | """f2edefec-298a-58d7-bcc0-b1bd2077fccb""" + "Sns" | "PublishBatch" | "out" | 2 | "POST" | "/" | "7a62c49f-347e-4fc4-9331-6e8e7a96aa73" | SnsClient.builder() | { SnsClient c -> c.publishBatch(PublishBatchRequest.builder().topicArn("arnprefix:mytopic").publishBatchRequestEntries(PublishBatchRequestEntry.builder().id("1").message("hello").build(), PublishBatchRequestEntry.builder().id("2").message("world").build()).build()) } | """ + + 1 + 4898a3df-db3a-5078-a6a9-fd895f9acb64 + + + 2 + 0967c76c-5cbb-5637-82f8-993ad81bed2b + + """ + } + + def "send #operation async request with builder #builder.class.getSimpleName() mocked response"() { + setup: + def conditions = new PollingConditions(timeout: 1) + boolean executed = false + def client = builder + // tests that our instrumentation doesn't disturb any overridden configuration + .overrideConfiguration({ watch(it, { executed = true }) }) + .endpointOverride(server.address) + .region(Region.AP_NORTHEAST_1) + .credentialsProvider(CREDENTIALS_PROVIDER) + .build() + responseBody.set(body) + servedRequestId.set(requestId) + when: + def response = call.call(client) + + if (response instanceof Future) { + response = response.get() + } + TEST_WRITER.waitForTraces(1) + TEST_DATA_STREAMS_WRITER.waitForGroups(1) + + then: + executed + response != null + + and: + conditions.eventually { + List results = TEST_DATA_STREAMS_WRITER.groups.findAll { it.parentHash == 0 } + assert results.size() >= 1 + def pathwayLatencyCount = 0 + def edgeLatencyCount = 0 + results.each { group -> + pathwayLatencyCount += group.pathwayLatency.count + edgeLatencyCount += group.edgeLatency.count + verifyAll(group) { + edgeTags.containsAll(["direction:" + dsmDirection, "topic:mytopic", "type:sns"]) + edgeTags.size() == 3 + } + } + verifyAll { + pathwayLatencyCount == dsmStatCount + edgeLatencyCount == dsmStatCount + } + } + + and: + assertTraces(1) { + trace(1) { + span { + serviceName expectedService(service, operation) + operationName expectedOperation(service, operation) + resourceName "$service.$operation" + spanType DDSpanTypes.HTTP_CLIENT + errored false + measured true + parent() + tags { + "$Tags.COMPONENT" "java-aws-sdk" + "$Tags.SPAN_KIND" Tags.SPAN_KIND_CLIENT + "$Tags.PEER_HOSTNAME" "localhost" + "$Tags.PEER_PORT" server.address.port + "$Tags.HTTP_URL" "${server.address}${path}" + "$Tags.HTTP_METHOD" "$method" + "$Tags.HTTP_STATUS" 200 + "aws.service" "$service" + "aws_service" "$service" + "aws.operation" "${operation}" + "aws.agent" "java-aws-sdk" + "aws.requestId" "$requestId" + "aws.topic.name" "mytopic" + "topicname" "mytopic" + "$DDTags.PATHWAY_HASH" { String } + defaultTags(false, true) + } + } + } + } + + cleanup: + servedRequestId.set(null) + + where: + service | operation | dsmDirection | dsmStatCount | method | path | requestId | builder | call | body + "Sns" | "Publish" | "out" | 1 | "POST" | "/" | "7a62c49f-347e-4fc4-9331-6e8e7a96aa73" | SnsAsyncClient.builder() | { SnsAsyncClient c -> c.publish(PublishRequest.builder().topicArn("arnprefix:mytopic").message("hello").build()) } | """f2edefec-298a-58d7-bcc0-b1bd2077fccb""" + "Sns" | "PublishBatch" | "out" | 2 | "POST" | "/" | "7a62c49f-347e-4fc4-9331-6e8e7a96aa73" | SnsAsyncClient.builder() | { SnsAsyncClient c -> c.publishBatch(PublishBatchRequest.builder().topicArn("arnprefix:mytopic").publishBatchRequestEntries(PublishBatchRequestEntry.builder().id("1").message("hello").build(), PublishBatchRequestEntry.builder().id("2").message("world").build()).build()) } | """ + + 1 + 4898a3df-db3a-5078-a6a9-fd895f9acb64 + + + 2 + 0967c76c-5cbb-5637-82f8-993ad81bed2b + + """ + } +} + +class Aws2SnsDataStreamsV0Test extends Aws2SnsDataStreamsTest { + + @Override + String expectedOperation(String awsService, String awsOperation) { + "aws.http" + } + + @Override + String expectedService(String awsService, String awsOperation) { + "sns" + } + + @Override + int version() { + 0 + } +} + +class Aws2SnsDataStreamsV1ForkedTest extends Aws2SnsDataStreamsTest { + + @Override + String expectedOperation(String awsService, String awsOperation) { + return "aws.${awsService.toLowerCase()}.send" + } + + @Override + String expectedService(String awsService, String awsOperation) { + Config.get().getServiceName() + } + + @Override + int version() { + 1 + } +} diff --git a/dd-java-agent/instrumentation/aws-java-sdk-2.2/src/main/java/datadog/trace/instrumentation/aws/v2/AwsSdkClientDecorator.java b/dd-java-agent/instrumentation/aws-java-sdk-2.2/src/main/java/datadog/trace/instrumentation/aws/v2/AwsSdkClientDecorator.java index b938b4b8fa0..a2409d84e7a 100644 --- a/dd-java-agent/instrumentation/aws-java-sdk-2.2/src/main/java/datadog/trace/instrumentation/aws/v2/AwsSdkClientDecorator.java +++ b/dd-java-agent/instrumentation/aws-java-sdk-2.2/src/main/java/datadog/trace/instrumentation/aws/v2/AwsSdkClientDecorator.java @@ -74,6 +74,14 @@ public class AwsSdkClientDecorator extends HttpClientDecorator SNS_PUBLISH_OPERATION_NAMES; + + static { + SNS_PUBLISH_OPERATION_NAMES = new HashSet<>(); + SNS_PUBLISH_OPERATION_NAMES.add("Publish"); + SNS_PUBLISH_OPERATION_NAMES.add("PublishBatch"); + } + public static final ExecutionAttribute KINESIS_STREAM_ARN_ATTRIBUTE = InstanceStore.of(ExecutionAttribute.class) .putIfAbsent("KinesisStreamArn", () -> new ExecutionAttribute<>("KinesisStreamArn")); @@ -123,9 +131,12 @@ public AgentSpan onSdkRequest( request.getValueForField("QueueName", String.class).ifPresent(name -> setQueueName(span, name)); // SNS - request - .getValueForField("TopicArn", String.class) - .ifPresent(arn -> setTopicName(span, arn.substring(arn.lastIndexOf(':') + 1))); + Optional snsTopicArn = request.getValueForField("TopicArn", String.class); + if (!snsTopicArn.isPresent()) { + snsTopicArn = request.getValueForField("TargetArn", String.class); + } + Optional snsTopicName = snsTopicArn.map(arn -> arn.substring(arn.lastIndexOf(':') + 1)); + snsTopicName.ifPresent(topic -> setTopicName(span, topic)); // Kinesis request @@ -147,20 +158,34 @@ public AgentSpan onSdkRequest( request.getValueForField("TableName", String.class).ifPresent(name -> setTableName(span, name)); // DSM - if (span.traceConfig().isDataStreamsEnabled() - && kinesisStreamArn.isPresent() - && "kinesis".equalsIgnoreCase(awsServiceName) - && KINESIS_PUT_RECORD_OPERATION_NAMES.contains(awsOperationName)) { - // https://github.com/DataDog/dd-trace-py/blob/864abb6c99e1cb0449904260bac93e8232261f2a/ddtrace/contrib/botocore/patch.py#L368 - List records = - request - .getValueForField("Records", List.class) - .orElse(Collections.singletonList(request)); // For PutRecord use request - - for (Object ignored : records) { - AgentTracer.get() - .getDataStreamsMonitoring() - .setProduceCheckpoint("kinesis", kinesisStreamArn.get(), NoOp.INSTANCE); + if (span.traceConfig().isDataStreamsEnabled()) { + if (kinesisStreamArn.isPresent() + && "kinesis".equalsIgnoreCase(awsServiceName) + && KINESIS_PUT_RECORD_OPERATION_NAMES.contains(awsOperationName)) { + // https://github.com/DataDog/dd-trace-py/blob/864abb6c99e1cb0449904260bac93e8232261f2a/ddtrace/contrib/botocore/patch.py#L368 + List records = + request + .getValueForField("Records", List.class) + .orElse(Collections.singletonList(request)); // For PutRecord use request + + for (Object ignored : records) { + AgentTracer.get() + .getDataStreamsMonitoring() + .setProduceCheckpoint("kinesis", kinesisStreamArn.get(), NoOp.INSTANCE); + } + } else if (snsTopicName.isPresent() + && "sns".equalsIgnoreCase(awsServiceName) + && SNS_PUBLISH_OPERATION_NAMES.contains(awsOperationName)) { + List entries = + request + .getValueForField("PublishBatchRequestEntries", List.class) + .orElse(Collections.singletonList(request)); + + for (Object ignored : entries) { + AgentTracer.get() + .getDataStreamsMonitoring() + .setProduceCheckpoint("sns", snsTopicName.get(), NoOp.INSTANCE); + } } } diff --git a/internal-api/src/main/java/datadog/trace/api/naming/v1/CloudNamingV1.java b/internal-api/src/main/java/datadog/trace/api/naming/v1/CloudNamingV1.java index 7ad12a9ee8e..e1c513bd2a9 100644 --- a/internal-api/src/main/java/datadog/trace/api/naming/v1/CloudNamingV1.java +++ b/internal-api/src/main/java/datadog/trace/api/naming/v1/CloudNamingV1.java @@ -29,6 +29,8 @@ public String operationForRequest( return SpanNaming.instance().namingSchema().messaging().inboundOperation("sqs"); case "Sns.Publish": case "SNS.Publish": + case "Sns.PublishBatch": + case "SNS.PublishBatch": return SpanNaming.instance().namingSchema().messaging().outboundOperation("sns"); default: final String lowercaseService = cloudService.toLowerCase(Locale.ROOT); From 7e58261ef8e2c8493d55b5f556700017b831cd28 Mon Sep 17 00:00:00 2001 From: "Santiago M. Mola" Date: Tue, 5 Mar 2024 12:59:13 +0100 Subject: [PATCH 05/30] Upgrade to AppSec rules v1.11.0 (#6754) --- .../src/main/resources/default_config.json | 94 +++++++++++++------ 1 file changed, 67 insertions(+), 27 deletions(-) diff --git a/dd-java-agent/appsec/src/main/resources/default_config.json b/dd-java-agent/appsec/src/main/resources/default_config.json index d572c003911..7912743b40a 100644 --- a/dd-java-agent/appsec/src/main/resources/default_config.json +++ b/dd-java-agent/appsec/src/main/resources/default_config.json @@ -1,7 +1,7 @@ { "version": "2.2", "metadata": { - "rules_version": "1.10.0" + "rules_version": "1.11.0" }, "rules": [ { @@ -141,7 +141,10 @@ "appscan_fingerprint", "w00tw00t.at.isc.sans.dfind", "w00tw00t.at.blackhats.romanian.anti-sec" - ] + ], + "options": { + "enforce_word_boundary": true + } }, "operator": "phrase_match" } @@ -1778,7 +1781,10 @@ "windows\\win.ini", "default\\ntuser.dat", "/var/run/secrets/kubernetes.io/serviceaccount" - ] + ], + "options": { + "enforce_word_boundary": true + } }, "operator": "phrase_match" } @@ -1895,6 +1901,9 @@ "address": "graphql.server.resolver" } ], + "options": { + "enforce_word_boundary": true + }, "list": [ "${cdpath}", "${dirstack}", @@ -2471,7 +2480,10 @@ "settings.local.php", "local.xml", ".env" - ] + ], + "options": { + "enforce_word_boundary": true + } }, "operator": "phrase_match" } @@ -2567,6 +2579,9 @@ "address": "graphql.server.resolver" } ], + "options": { + "enforce_word_boundary": true + }, "list": [ "$globals", "$_cookie", @@ -2765,7 +2780,10 @@ "wp_safe_remote_post", "wp_safe_remote_request", "zlib_decode" - ] + ], + "options": { + "enforce_word_boundary": true + } }, "operator": "phrase_match" } @@ -2980,9 +2998,6 @@ { "address": "server.request.path_params" }, - { - "address": "grpc.server.request.message" - }, { "address": "graphql.server.all_resolvers" }, @@ -3037,9 +3052,6 @@ { "address": "server.request.path_params" }, - { - "address": "grpc.server.request.message" - }, { "address": "graphql.server.all_resolvers" }, @@ -3271,6 +3283,9 @@ "address": "graphql.server.resolver" } ], + "options": { + "enforce_word_boundary": true + }, "list": [ "document.cookie", "document.write", @@ -3546,9 +3561,6 @@ { "address": "server.request.path_params" }, - { - "address": "grpc.server.request.message" - }, { "address": "graphql.server.all_resolvers" }, @@ -3863,9 +3875,6 @@ { "address": "server.request.path_params" }, - { - "address": "grpc.server.request.message" - }, { "address": "graphql.server.all_resolvers" }, @@ -4454,7 +4463,10 @@ "org.apache.struts2", "org.omg.corba", "java.beans.xmldecode" - ] + ], + "options": { + "enforce_word_boundary": true + } }, "operator": "phrase_match" } @@ -4581,9 +4593,6 @@ { "address": "server.request.path_params" }, - { - "address": "grpc.server.request.message" - }, { "address": "graphql.server.all_resolvers" }, @@ -5342,6 +5351,40 @@ ], "transformers": [] }, + { + "id": "dog-920-001", + "name": "JWT authentication bypass", + "tags": { + "type": "http_protocol_violation", + "category": "attack_attempt", + "cwe": "287", + "capec": "1000/225/115", + "confidence": "0" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.cookies" + }, + { + "address": "server.request.headers.no_cookies", + "key_path": [ + "authorization" + ] + } + ], + "regex": "^(?:Bearer )?ey[A-Za-z0-9+_\\-/]*([QY][UW]x[Hn]Ij([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]ICJ[Ou][Tb][02]5[Fl]|[QY][UW]x[Hn]I([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gOiAi[Tb][km]9[Ou][RZ][Q-Za-f]|[QY][UW]x[Hn]Ij([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gI[km]5[Pv][Tb][km][U-X]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ci([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gO([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]Ai[Tb][km]9[Ou][RZ][Q-Za-f]|[QY][UW]x[Hn]I([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gO([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*ICJ[Ou][Tb][02]5[Fl]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]y([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]A6([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*ICJ[Ou][Tb][02]5[Fl]|[QY][UW]x[Hn]I([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*ID([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*ICJ[Ou][Tb][02]5[Fl]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ciID([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]ICJ[Ou][Tb][02]5[Fl]|[QY][UW]x[Hn]I([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]IDogI[km]5[Pv][Tb][km][U-X]|[QY][UW]x[Hn]I([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]A6ICJ[Ou][Tb][02]5[Fl]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]y([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*ID([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]Ai[Tb][km]9[Ou][RZ][Q-Za-f]|[QY][UW]x[Hn]IiA6ICJ[Ou][Tb][02]5[Fl]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ci([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gO([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gI[km]5[Pv][Tb][km][U-X]|[QY][UW]x[Hn]I([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*ID([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gI[km]5[Pv][Tb][km][U-X]|[QY][UW]x[Hn]I([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]IDoi[Tb][km]9[Ou][RZ][Q-Za-f]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ci([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]A6([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]Ai[Tb][km]9[Ou][RZ][Q-Za-f]|[QY][UW]x[Hn]I([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]A6([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]ICJ[Ou][Tb][02]5[Fl]|[QY][UW]x[Hn]Ij([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*ICJ[Ou][Tb][02]5[Fl]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]y([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gOiAi[Tb][km]9[Ou][RZ][Q-Za-f]|[QY][UW]x[Hn]I([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]ID([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]ICJ[Ou][Tb][02]5[Fl]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ci([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*ID([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]Ai[Tb][km]9[Ou][RZ][Q-Za-f]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ci([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*ID([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]ICJ[Ou][Tb][02]5[Fl]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ciOiAi[Tb][km]9[Ou][RZ][Q-Za-f]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ci([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*IDogI[km]5[Pv][Tb][km][U-X]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]y([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]IDogI[km]5[Pv][Tb][km][U-X]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]yIgO([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]Ai[Tb][km]9[Ou][RZ][Q-Za-f]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ci([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]A6([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*ICJ[Ou][Tb][02]5[Fl]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]y([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]A6([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gI[km]5[Pv][Tb][km][U-X]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ciIDoi[Tb][km]9[Ou][RZ][Q-Za-f]|[QY][UW]x[Hn]I([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*IDogI[km]5[Pv][Tb][km][U-X]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]yIgOiJ[Ou][Tb][02]5[Fl]|[QY][UW]x[Hn]I([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gO([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]ICJ[Ou][Tb][02]5[Fl]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ciO([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gI[km]5[Pv][Tb][km][U-X]|[QY][UW]x[Hn]I([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]ID([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gI[km]5[Pv][Tb][km][U-X]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ci([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]IDoi[Tb][km]9[Ou][RZ][Q-Za-f]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ciID([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*ICJ[Ou][Tb][02]5[Fl]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]y([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gO([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]ICJ[Ou][Tb][02]5[Fl]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ci([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*ID([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gI[km]5[Pv][Tb][km][U-X]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ci([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]IDogI[km]5[Pv][Tb][km][U-X]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]yI6([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*ICJ[Ou][Tb][02]5[Fl]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ci([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]ID([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gI[km]5[Pv][Tb][km][U-X]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]yI6([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]ICJ[Ou][Tb][02]5[Fl]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ci([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gO([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]ICJ[Ou][Tb][02]5[Fl]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]y([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]ID([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]ICJ[Ou][Tb][02]5[Fl]|[QY][UW]x[Hn]IiA6([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*ICJ[Ou][Tb][02]5[Fl]|[QY][UW]x[Hn]I([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gO([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gI[km]5[Pv][Tb][km][U-X]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]y([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]ID([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]Ai[Tb][km]9[Ou][RZ][Q-Za-f]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]y([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*IDogI[km]5[Pv][Tb][km][U-X]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]yIgO([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*ICJ[Ou][Tb][02]5[Fl]|[QY][UW]x[Hn]Ij([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]Ai[Tb][km]9[Ou][RZ][Q-Za-f]|[QY][UW]x[Hn]I([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]A6([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gI[km]5[Pv][Tb][km][U-X]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]y([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]A6ICJ[Ou][Tb][02]5[Fl]|[QY][UW]x[Hn]I([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]ID([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]Ai[Tb][km]9[Ou][RZ][Q-Za-f]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ciOiJ[Ou][Tb][02]5[Fl]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ci([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*IDoi[Tb][km]9[Ou][RZ][Q-Za-f]|[QY][UW]x[Hn]I([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gOiJ[Ou][Tb][02]5[Fl]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ciO([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*ICJ[Ou][Tb][02]5[Fl]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]yIgOiAi[Tb][km]9[Ou][RZ][Q-Za-f]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]y([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]IDoi[Tb][km]9[Ou][RZ][Q-Za-f]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ciID([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gI[km]5[Pv][Tb][km][U-X]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ci([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]A6ICJ[Ou][Tb][02]5[Fl]|[QY][UW]x[Hn]IjogI[km]5[Pv][Tb][km][U-X]|[QY][UW]x[Hn]I([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]ID([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*ICJ[Ou][Tb][02]5[Fl]|[QY][UW]x[Hn]I([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]A6([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]Ai[Tb][km]9[Ou][RZ][Q-Za-f]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]y([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gO([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]Ai[Tb][km]9[Ou][RZ][Q-Za-f]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]y([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*ID([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]ICJ[Ou][Tb][02]5[Fl]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ciO([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]Ai[Tb][km]9[Ou][RZ][Q-Za-f]|[QY][UW]x[Hn]IiA6I[km]5[Pv][Tb][km][U-X]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ci([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]ID([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*ICJ[Ou][Tb][02]5[Fl]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ciID([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]Ai[Tb][km]9[Ou][RZ][Q-Za-f]|[QY][UW]x[Hn]I([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*ID([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]Ai[Tb][km]9[Ou][RZ][Q-Za-f]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]y([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]A6I[km]5[Pv][Tb][km][U-X]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]yI6I[km]5[Pv][Tb][km][U-X]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]yI6ICJ[Ou][Tb][02]5[Fl]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ciIDogI[km]5[Pv][Tb][km][U-X]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]y([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]ID([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*ICJ[Ou][Tb][02]5[Fl]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]y([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gO([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gI[km]5[Pv][Tb][km][U-X]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ci([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*ID([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*ICJ[Ou][Tb][02]5[Fl]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]y([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]ID([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gI[km]5[Pv][Tb][km][U-X]|[QY][UW]x[Hn]IiA6([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]Ai[Tb][km]9[Ou][RZ][Q-Za-f]|[QY][UW]x[Hn]I([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gO([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]Ai[Tb][km]9[Ou][RZ][Q-Za-f]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ci([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gO([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*ICJ[Ou][Tb][02]5[Fl]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ci([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]A6([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]ICJ[Ou][Tb][02]5[Fl]|[QY][UW]x[Hn]IiA6([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]ICJ[Ou][Tb][02]5[Fl]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ci([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]ID([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]ICJ[Ou][Tb][02]5[Fl]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]y([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*IDoi[Tb][km]9[Ou][RZ][Q-Za-f]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ciO([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]ICJ[Ou][Tb][02]5[Fl]|[QY][UW]x[Hn]I([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]A6I[km]5[Pv][Tb][km][U-X]|[QY][UW]x[Hn]IiA6([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gI[km]5[Pv][Tb][km][U-X]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]y([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]A6([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]ICJ[Ou][Tb][02]5[Fl]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ci([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]A6([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gI[km]5[Pv][Tb][km][U-X]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]y([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gO([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*ICJ[Ou][Tb][02]5[Fl]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]yI6([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gI[km]5[Pv][Tb][km][U-X]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]yIgO([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gI[km]5[Pv][Tb][km][U-X]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ci([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gOiJ[Ou][Tb][02]5[Fl]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]y([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*ID([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*ICJ[Ou][Tb][02]5[Fl]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]y([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]A6([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]Ai[Tb][km]9[Ou][RZ][Q-Za-f]|[QY][UW]x[Hn]I([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*ID([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]ICJ[Ou][Tb][02]5[Fl]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ci([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]ID([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]Ai[Tb][km]9[Ou][RZ][Q-Za-f]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]y([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*ID([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gI[km]5[Pv][Tb][km][U-X]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]yIgO([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[\\x2b\\x2f-9A-Za-z]ICJ[Ou][Tb][02]5[Fl]|[QY][UW]x[Hn]I([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]A6([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*ICJ[Ou][Tb][02]5[Fl]|[QY][UW]x[Hn]I([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*IDoi[Tb][km]9[Ou][RZ][Q-Za-f]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ci([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]A6I[km]5[Pv][Tb][km][U-X]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]y([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gOiJ[Ou][Tb][02]5[Fl]|[QY][UW]x[Hn]Ijoi[Tb][km]9[Ou][RZ][Q-Za-f]|[\\x2b\\x2f-9A-Za-z]{2}[159BFJNRVZdhlptx][Bh][Tb][EG]ci([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[048AEIMQUYcgkosw]gOiAi[Tb][km]9[Ou][RZ][Q-Za-f]|[\\x2b\\x2f-9A-Za-z][02EGUWkm]F[Ms][RZ]yI6([048ACEIMQSUYcgikoswy]|[\\x2b\\x2f-9A-Za-z]I)*[CSiy]Ai[Tb][km]9[Ou][RZ][Q-Za-f])[A-Za-z0-9+-/]*\\.[A-Za-z0-9+_\\-/]+\\.(?:[A-Za-z0-9+_\\-/]+)?$", + "options": { + "case_sensitive": true + } + }, + "operator": "match_regex" + } + ], + "transformers": [] + }, { "id": "dog-931-001", "name": "RFI: URL Payload to well known RFI target", @@ -5603,6 +5646,9 @@ { "operator": "phrase_match", "parameters": { + "options": { + "enforce_word_boundary": true + }, "inputs": [ { "address": "server.request.uri.raw" @@ -6606,9 +6652,6 @@ { "address": "server.request.headers.no_cookies" }, - { - "address": "grpc.server.request.message" - }, { "address": "graphql.server.all_resolvers" }, @@ -6654,9 +6697,6 @@ { "address": "server.request.headers.no_cookies" }, - { - "address": "grpc.server.request.message" - }, { "address": "graphql.server.all_resolvers" }, From e9e1afb2b2d47b6f9958c044e70a1173ecc37459 Mon Sep 17 00:00:00 2001 From: Brian Marks Date: Tue, 5 Mar 2024 09:03:03 -0500 Subject: [PATCH 06/30] Add documentation for adding new instrumentations and how instrumentations work (#6771) --- CONTRIBUTING.md | 172 ++----- docs/add_new_instrumentation.md | 374 +++++++++++++++ docs/how_instrumentations_work.md | 735 ++++++++++++++++++++++++++++++ 3 files changed, 1154 insertions(+), 127 deletions(-) create mode 100644 docs/add_new_instrumentation.md create mode 100644 docs/how_instrumentations_work.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 741d49efde9..d9bb8ec0751 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,9 +1,13 @@ # Contributing -Pull requests for bug fixes are welcome, but before submitting new features or changes to current functionality [open an issue](https://github.com/DataDog/dd-trace-java/issues/new) -and discuss your ideas or propose the changes you wish to make. After a resolution is reached a PR can be submitted for review. +Pull requests for bug fixes are welcome, but before submitting new features or changes to current +functionality [open an issue](https://github.com/DataDog/dd-trace-java/issues/new) +and discuss your ideas or propose the changes you wish to make. After a resolution is reached a PR can be submitted for +review. -When opening a pull request, please open it as a [draft](https://github.blog/2019-02-14-introducing-draft-pull-requests/) to not auto assign reviewers before you feel the pull request is in a reviewable state. +When opening a pull request, please open it as +a [draft](https://github.blog/2019-02-14-introducing-draft-pull-requests/) to not auto assign reviewers before you feel +the pull request is in a reviewable state. ## Requirements @@ -22,150 +26,57 @@ To build the full project: MacOS users, remember that `/usr/libexec/java_home` may control which JDK is in your path. -In contrast to the [IntelliJ IDEA setup](#intellij-idea) the default JVM to build and run tests from the command line should be Java 8. +In contrast to the [IntelliJ IDEA set up](#intellij-idea) the default JVM to build and run tests from the command line +should be Java 8. -There is no Oracle JDK v8 for ARM. ARM users might want to use [Azul's Zulu](/Users/albert.cintora/go/src/github.com/DataDog/dd-trace-java/dd-java-agent/instrumentation/build.gradle) builds of Java 8. On MacOS, they can be installed using `brew tap homebrew/cask-versions && brew install --cask zulu8`. [Amazon Corretto](https://aws.amazon.com/corretto/) builds have also been proven to work. +There is no Oracle JDK v8 for ARM. ARM users might want to +use [Azul's Zulu](https://www.azul.com/downloads/?version=java-8-lts&architecture=arm-64-bit&package=jdk#zulu) +builds of Java 8. On macOS, they can be installed +using `brew tap homebrew/cask-versions && brew install --cask zulu8`. [Amazon Corretto](https://aws.amazon.com/corretto/) +builds have also been proven to work. # Building To build the project without running tests run: + ```bash ./gradlew clean assemble ``` To build the entire project with tests (this can take a very long time) run: + ```bash ./gradlew clean build ``` # Adding Instrumentations -All instrumentations are in the directory `/dd-java-agent/instrumentation/$framework?/$framework-$minVersion`, where `$framework` is the framework name, and `$minVersion` is the minimum version of the framework supported by the instrumentation. -In some cases, such as [Hibernate](https://github.com/DataDog/dd-trace-java/tree/master/dd-java-agent/instrumentation/hibernate), there is a submodule containing different version-specific instrumentations, but typically a version-specific module is enough when there is only one instrumentation implemented (e.g. [Akka-HTTP](https://github.com/DataDog/dd-trace-java/tree/master/dd-java-agent/instrumentation/akka-http-10.0)). -When adding an instrumentation to `/dd-java-agent/instrumentation/$framework?/$framework-$minVersion`, an include must be added to [`settings.gradle`](https://github.com/DataDog/dd-trace-java/blob/master/settings.gradle): - -```groovy -include ':dd-java-agent:instrumentation:$framework?:$framework-$minVersion' -``` - -Note that the includes are maintained in alphabetical order. - -An instrumentation consists of the following components: - -* An _Instrumentation_ - * Must implement `Instrumenter` - note that it is recommended to implement `Instrumenter.Default` in every case. - * The instrumentation must be annotated with `@AutoService(Instrumenter.class)` for annotation processing. - * The instrumentation must declare a type matcher by implementing the method `typeMatcher()`, which matches the types the instrumentation will transform. - * The instrumentation must declare every class it needs to load (except for Datadog bootstrap classes, and for the framework itself) in `helperClassNames()`. - It is recommended to keep the number of classes to a minimum both to reduce deployment size and optimise startup time. - * If state must be associated with instances of framework types, a definition of the _context store_ by implementing `contextStore()`. - * The method `transformers()`: this is a map of method matchers to the Bytebuddy advice class which handles matching methods. - For example, if you want to inject an instance of `com.foo.Foo` into a `com.bar.Bar` (or fall back to a weak map backed association if this is impossible) you would return `singletonMap("com.foo.Foo", "com.bar.Bar")`. - It may be tempting to write `Foo.class.getName()`, but this will lead to the class being loaded during bootstrapping, which is usually not safe. - See the section on the [context store](#context-store) for more details. -* A _Decorator_. - * This will typically extend one of decorator [implementations](https://github.com/DataDog/dd-trace-java/tree/master/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator), which provide templates for span enrichment behaviour. - For example, all instrumentations for HTTP server frameworks have decorators which extend [`HttpServerDecorator`](https://github.com/DataDog/dd-trace-java/blob/master/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/HttpServerDecorator.java). - * The name of this class must be included in the instrumentation's helper class names, as it will need to be loaded with the instrumentation. -* _Advice_ - * Snippets of code to be inserted at the entry or exit of a method. - * Associated with the methods they apply to by the instrumentation's `transformers()` method. -* Any more classes required to implement the instrumentation, which must be included in the instrumentation's helper class names. - -## Verifying Instrumentations - -There are four verification strategies, three of which are mandatory. - -### Muzzle directive -A _muzzle directive_ which checks for a range of framework versions that it would be safe to load the instrumentation. -At the top of the instrumentation's gradle file, the following would be added (see [rediscala](https://github.com/DataDog/dd-trace-java/blob/master/dd-java-agent/instrumentation/rediscala-1.8.0/rediscala-1.8.0.gradle)) - ```groovy - muzzle { - pass { - group = "com.github.etaty" - module = "rediscala_2.11" - versions = "[1.5.0,)" - assertInverse = true - } - - pass { - group = "com.github.etaty" - module = "rediscala_2.12" - versions = "[1.8.0,)" - assertInverse = true - } - } - ``` -This means that the instrumentation should be safe with `rediscala_2.11` from version `1.5.0` and all later versions, but should fail (and so will not be loaded), for older versions (see `assertInverse`). -A similar range of versions is specified for `rediscala_2.12`. -When the agent is built, the muzzle plugin will download versions of the framework and check these directives hold. -To run muzzle on your instrumentation, run: - -```groovy - ./gradlew :dd-java-agent:instrumentation:rediscala-1.8.0:muzzle -``` -* ⚠️ Muzzle does _not_ run tests. - It checks that the types and methods used by the instrumentation are present in particular versions of libraries. - It can be subverted with `MethodHandle` and reflection, so muzzle passing is not the end of the story. +See [Adding a New Instrumentation](./docs/add_new_instrumentation.md) for instructions on adding a new instrumentation. -### Instrumentation Tests - -Tests are written in Groovy using the [Spock framework](http://spockframework.org/). -For instrumentations, `AgentTestRunner` must be extended by the test fixture. -For e.g. HTTP server frameworks, there are base tests which enforce consistency between different implementations - see [HttpServerTest](https://github.com/DataDog/dd-trace-java/blob/master/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/base/HttpServerTest.groovy) - -When writing an instrumentation it is much faster to test just the instrumentation rather than build the entire project, for example: - -```bash -./gradlew :dd-java-agent:instrumentation:play-ws-2.1:test -``` - -### Latest Dependency Tests - -Adding a directive to the build file lets us get early warning when breaking changes are released by framework maintainers. -For example, for Play 2.4, based on the following: - -```groovy - latestDepTestCompile group: 'com.typesafe.play', name: 'play-java_2.11', version: '2.5.+' - latestDepTestCompile group: 'com.typesafe.play', name: 'play-java-ws_2.11', version: '2.5.+' - latestDepTestCompile(group: 'com.typesafe.play', name: 'play-test_2.11', version: '2.5.+') { - exclude group: 'org.eclipse.jetty.websocket', module: 'websocket-client' - } -``` - -We download the latest dependency and run tests against it. - -### Smoke tests - -These are tests which run with a real agent jar file set as the `javaagent`. -See [here](https://github.com/DataDog/dd-trace-java/tree/master/dd-smoke-tests). -These are optional and not all frameworks have these, but contributions are very welcome. +See [How Instrumentations Work](./docs/how_instrumentations_work.md) for a deep dive into how instrumentations work. # Automatic code formatting -This project includes a `.editorconfig` file for basic editor settings. This file is supported by most common text editors. +This project includes a `.editorconfig` file for basic editor settings. This file is supported by most common text +editors. We have automatic code formatting enabled in Gradle configuration using [Spotless](https://github.com/diffplug/spotless) [Gradle plugin](https://github.com/diffplug/spotless/tree/master/plugin-gradle). Main goal is to avoid extensive reformatting caused by different IDEs having different opinion about how things should be formatted by establishing single 'point of truth'. -Running +To reformat all the files that need reformatting. ```bash ./gradlew spotlessApply ``` -reformats all the files that need reformatting. - -Running +To run formatting verify task only. ```bash ./gradlew spotlessCheck ``` -runs formatting verify task only. - ## Pre-commit hook There is a pre-commit hook setup to verify formatting before committing. It can be activated with this command: @@ -178,7 +89,8 @@ git config core.hooksPath .githooks Git does not automatically update submodules when switching branches. -Add the following configuration setting or you will need to remember to add `--recurse-submodules` to `git checkout` when switching to old branches. +Add the following configuration setting, or you will need to remember to add `--recurse-submodules` to `git checkout` +when switching to old branches. ```bash git config --local submodule.recurse true @@ -186,45 +98,51 @@ git config --local submodule.recurse true This will keep the submodule in `dd-java-agent/agent-jmxfetch/integrations-core` up to date. - ## Intellij IDEA Compiler settings: -* OpenJDK 11 must be installed to build the entire project. Under `SDKs` it must have the name `11`. +* OpenJDK 11 must be installed to build the entire project. Under `SDKs` it must have the name `11`. * Under `Build, Execution, Deployment > Compiler > Java Compiler` disable `Use '--release' option for cross-compilation` Suggested plugins and settings: * Editor > Code Style > Java/Groovy > Imports - * Class count to use import with '*': `9999` (some number sufficiently large that is unlikely to matter) - * Names count to use static import with '*': `9999` - * With java use the following import layout (groovy should still use the default) to ensure consistency with google-java-format: - ![import layout](https://user-images.githubusercontent.com/734411/43430811-28442636-94ae-11e8-86f1-f270ddcba023.png) + * Class count to use import with '*': `9999` (some number sufficiently large that is unlikely to matter) + * Names count to use static import with '*': `9999` + * With java use the following import layout (groovy should still use the default) to ensure consistency with + google-java-format: + ![import layout](https://user-images.githubusercontent.com/734411/43430811-28442636-94ae-11e8-86f1-f270ddcba023.png) * [Google Java Format](https://plugins.jetbrains.com/plugin/8527-google-java-format) ## Troubleshooting -* When Gradle is building the project, the error `Could not find netty-transport-native-epoll-4.1.43.Final-linux-x86_64.jar` is shown. - * Execute `rm -rf ~/.m2/repository/io/netty/netty-transport*` in a Terminal and re-build again. +* When Gradle is building the project, the + error `Could not find netty-transport-native-epoll-4.1.43.Final-linux-x86_64.jar` is shown. + * Execute `rm -rf ~/.m2/repository/io/netty/netty-transport*` in a Terminal and re-build again. -* IntelliJ 2021.3 complains `Failed to find KotlinGradleProjectData for GradleSourceSetData` https://youtrack.jetbrains.com/issue/KTIJ-20173 - * Switch to `IntelliJ IDEA CE 2021.2.3` +* IntelliJ 2021.3 + complains `Failed to find KotlinGradleProjectData for GradleSourceSetData` https://youtrack.jetbrains.com/issue/KTIJ-20173 + * Switch to `IntelliJ IDEA CE 2021.2.3` * IntelliJ Gradle fails to import the project with `JAVA_11_HOME must be set to build Java 11 code` - * A workaround is to run IntelliJ from terminal with `JAVA_11_HOME` - * In order to verify what's visible from IntelliJ use `Add Configuration` bar and go to `Add New` -> `Gradle` -> `Environmental Variables` + * A workaround is to run IntelliJ from terminal with `JAVA_11_HOME` + * In order to verify what's visible from IntelliJ use `Add Configuration` bar and go + to `Add New` -> `Gradle` -> `Environmental Variables` * Gradle fails with a "too many open files" error. - * You can check the `ulimit` for open files in your current shell by doing `ulimit -n` and raise it by calling `ulimit -n ` + * You can check the `ulimit` for open files in your current shell by doing `ulimit -n` and raise it by + calling `ulimit -n ` ## Running tests on another JVM To run tests on a different JVM than the one used for doing the build, you need two things: -1) An environment variable pointing to the JVM to use on the form `JAVA_[JDKNAME]_HOME`, e.g. `JAVA_ZULU15_HOME`, `JAVA_GRAALVM17_HOME` +1) An environment variable pointing to the JVM to use on the form `JAVA_[JDKNAME]_HOME`, + e.g. `JAVA_ZULU15_HOME`, `JAVA_GRAALVM17_HOME` -2) A command line option to the gradle task on the form `-PtestJvm=[JDKNAME]`, e.g. `-PtestJvm=ZULU15`, `-PtestJvm=GRAALVM17` +2) A command line option to the gradle task on the form `-PtestJvm=[JDKNAME]`, + e.g. `-PtestJvm=ZULU15`, `-PtestJvm=GRAALVM17` Please note that the JDK name needs to end with the JDK version, e.g. `11`, `ZULU15`, `ORACLE8`, `GRAALVM17`, etc. diff --git a/docs/add_new_instrumentation.md b/docs/add_new_instrumentation.md new file mode 100644 index 00000000000..1e213323d00 --- /dev/null +++ b/docs/add_new_instrumentation.md @@ -0,0 +1,374 @@ +# Add a New Instrumentation + +Now we will step through adding a very basic instrumentation to the trace agent. The +existing [google-http-client instrumentation](../dd-java-agent/instrumentation/google-http-client) +will be used as an example. + +## Clone the dd-trace-java repo + +```shell +git clone https://github.com/DataDog/dd-trace-java.git +``` + +## Name your instrumentation + +Follow existing naming conventions for instrumentations. In this case, the instrumentation is +named `google-http-client`. (see [Naming](./how_instrumentations_work.md#naming)) + +## Configuring Gradle + +Add the new instrumentation to [`settings.gradle`](../settings.gradle) +in alpha order with the other instrumentations in this format: + +```groovy +include ':dd-java-agent:instrumentation:$framework?:$framework-$minVersion' +``` + +In this case +we [added](https://github.com/DataDog/dd-trace-java/blob/297b575f0f265c1dc78f9958e7b4b9365c80d1f9/settings.gradle#L209C3-L209C3): + +```groovy +include ':dd-java-agent:instrumentation:google-http-client' +``` + +## Create the Instrumentation class + +1. Choose an appropriate package name for the instrumentation + like `package datadog.trace.instrumentation.googlehttpclient.` (see [Naming](./how_instrumentations_work.md#naming)) +2. Create an appropriate directory structure for your instrumentation which agrees with the package name. ( + see [Files and Directories](./how_instrumentations_work.md#filesdirectories)) +3. Choose an appropriate class name + like `datadog.trace.instrumentation.googlehttpclient.GoogleHttpClientInstrumentation` ( + see [Naming](./how_instrumentations_work.md#naming)) +4. Include the required `@AutoService(Instrumenter.class) `annotation. +5. Choose `Instrumenter.Tracing` as the parent class. +6. Since this instrumentation class will only modify one specific type, it can implement + the `Instrumenter.ForSingleType `interface which provides the `instrumentedType()` method. ( + see [Type Matching](./how_instrumentations_work.md#type-matching)) +7. Pass the instrumentation name to the superclass constructor + +```java + +@AutoService(Instrumenter.class) +public class GoogleHttpClientInstrumentation extends Instrumenter.Tracing implements Instrumenter.ForSingleType { + public GoogleHttpClientInstrumentation() { + super("google-http-client"); + } + // ... +} +``` + +## Match the target class + +In this case we target only one known class to instrument. This is the class which contains the method this +instrumentation should modify. (see [Type Matching](./how_instrumentations_work.md#type-matching)) + +```java + +@Override +public String instrumentedType() { + return "com.google.api.client.http.HttpRequest"; +} +``` + +## Match the target method + +We want to apply advice to +the [`HttpRequest.execute()`](https://github.com/googleapis/google-http-java-client/blob/1acedf75368f11ab03e5f84dd2c58a8a8a662d41/google-http-client/src/main/java/com/google/api/client/http/HttpRequest.java#L849) +method. It has this signature: + +```java +public HttpResponse execute() throws IOException {/* */} +``` + +Target the method using [appropriate Method Matchers](./how_instrumentations_work.md#method-matching) and include the +name String to be used for the Advice class when calling `transformation.applyAdvice()`: + +```java +public void adviceTransformations(AdviceTransformation transformation) { + transformation.applyAdvice( + isMethod() + .and(isPublic()) + .and(named("execute")) + .and(takesArguments(0)), + GoogleHttpClientInstrumentation.class.getName() + "$GoogleHttpClientAdvice" + ); +} +``` + +## Add the HeadersInjectAdapter + +This particular instrumentation uses +a [HeadersInjectAdapter](../dd-java-agent/instrumentation/google-http-client/src/main/java/datadog/trace/instrumentation/googlehttpclient/HeadersInjectAdapter.java) +class to assist with HTTP header injection. This is not required of all instrumentations. ( +See [InjectorAdapters](./how_instrumentations_work.md#injectadapters--custom-getterssetters)). + +```java +public class HeadersInjectAdapter { + @Override + public void set(final HttpRequest carrier, final String key, final + String value) { + carrier.getHeaders().put(key, value); + } +} +``` + +## Create a Decorator class + +1. The class name should end in Decorator. `GoogleHttpClientDecorator `is good. +2. Since this is an HTTP client instrumentation, the class should extend `HttpClientDecorator.` +3. Override the methods as needed to provide behaviors specific to this instrumentation. For + example `getResponseHeader()` and `getRequestHeader()` require functionality specific to the Google `HttpRequest` + and `HttpResponse` classes used when declaring this Decorator class: + 1. `public class GoogleHttpClientDecorator extends HttpClientDecorator {/* */}` + 2. Instrumentations of other HTTP clients would declare Decorators that extend the same HttpClientDecorator but + using their own Request and Response classes instead. +4. Typically, we create one static instance of the Decorator named `DECORATE`. +5. For efficiency, create and retain frequently used CharSequences such as `GOOGLE_HTTP_CLIENT` and `HTTP_REQUEST`, etc. +6. Add methods like `prepareSpan()` that will be called + from [multiple](https://github.com/DataDog/dd-trace-java/blob/5307b46fe3956f0d1f09f84e1dab580af222ddc5/dd-java-agent/instrumentation/google-http-client/src/main/java/datadog/trace/instrumentation/googlehttpclient/GoogleHttpClientInstrumentation.java#L75) + different [places](https://github.com/DataDog/dd-trace-java/blob/5307b46fe3956f0d1f09f84e1dab580af222ddc5/dd-java-agent/instrumentation/google-http-client/src/main/java/datadog/trace/instrumentation/googlehttpclient/GoogleHttpClientInstrumentation.java#L103) + to reduce code duplication. Confining extensive tag manipulation to the Decorators also makes the Advice class easier + to understand and maintain. + +```java +import com.google.api.client.http.HttpRequest; +import com.google.api.client.http.HttpResponse; + +public class GoogleHttpClientDecorator + extends HttpClientDecorator { + private static final Pattern URL_REPLACEMENT = Pattern.compile("%20"); + public static final CharSequence GOOGLE_HTTP_CLIENT = + UTF8BytesString.create("google-http-client"); + public static final GoogleHttpClientDecorator DECORATE = new + GoogleHttpClientDecorator(); + public static final CharSequence HTTP_REQUEST = + UTF8BytesString.create(DECORATE.operationName()); + + @Override + protected String method(final HttpRequest httpRequest) { + return httpRequest.getRequestMethod(); + } + + @Override + protected URI url(final HttpRequest httpRequest) throws URISyntaxException { + final String url = httpRequest.getUrl().build(); + final String fixedUrl = URL_REPLACEMENT.matcher(url).replaceAll("+"); + return URIUtils.safeParse(fixedUrl); + } + + public AgentSpan prepareSpan(AgentSpan span, HttpRequest request) { + DECORATE.afterStart(span); + DECORATE.onRequest(span, request); + propagate().inject(span, request, SETTER); + propagate().injectPathwayContext(span, request, SETTER, + HttpClientDecorator.CLIENT_PATHWAY_EDGE_TAGS); + return span; + } + + @Override + protected int status(final HttpResponse httpResponse) { + return httpResponse.getStatusCode(); + } + + @Override + protected String[] instrumentationNames() { + return new String[]{"google-http-client"}; + } + + @Override + protected CharSequence component() { + return GOOGLE_HTTP_CLIENT; + } + + @Override + protected String getRequestHeader(HttpRequest request, String headerName) { + return request.getHeaders().getFirstHeaderStringValue(headerName); + } + + @Override + protected String getResponseHeader(HttpResponse response, + String headerName) { + return response.getHeaders().getFirstHeaderStringValue(headerName); + } +} +``` + +## Add helper class names + +The `GoogleHttpClientDecorator` and `HeadersInjectAdapter` class names must be included in helper classes defined in the +Instrumentation class, or they will not be available at runtime. `packageName` is used for convenience but helper +classes outside the current package could also be included. + +```java + +@Override +public String[] helperClassNames() { + return new String[]{ + packageName + ".GoogleHttpClientDecorator", + packageName + ".HeadersInjectAdapter" + }; +} +``` + +## Add Advice class + +1. Add a new static class to the Instrumentation class. The name must match what was passed to + the `adviceTransformations()` method earlier, here `GoogleHttpClientAdvice.` +2. Create two static methods named whatever you like. `methodEnter` and `methodExit` are good choices. These **must** + be static. +3. With `methodEnter:` + 1. Annotate the method using `@Advice.OnMethodEnter(suppress = Throwable.class) `( + see [Exceptions in Advice](./how_instrumentations_work.md#exceptions-in-advice)) + 2. Add parameter `@Advice.This HttpRequest request`. It will point to the target `execute()` method’s _this_ + reference which must be of the same `HttpRequest` type. + 3. Add a parameter, `@Advice.Local("inherited") boolean inheritedScope`. This shared local variable will be visible + to both `OnMethodEnter` and `OnMethodExit` methods. + 4. Use `activeScope()` __to __see if an `AgentScope` is already active. If so, return that `AgentScope`, but first + let the exit method know by setting the shared `inheritedScope` boolean. + 5. If an `AgentScope` was not active then start a new span, decorate it, activate it and return it. +4. With `methodExit:` + 1. Annotate the method using `@Advice.OnMethodExit(onThrowable=Throwable.class, suppress=Throwable.class). `( + see [Exceptions in Advice](./how_instrumentations_work.md#exceptions-in-advice)) + 2. Add parameter `@Advice.Enter AgentScope scope. `This is the `AgentScope` object returned earlier + by `methodEnter()`. Note this is not the return value of the target `execute()` method. + 3. Add a parameter, `@Advice.Local("inherited") boolean inheritedScope`. This is the shared local variable created + earlier. + 4. Add a parameter `@Advice.Return final HttpResponse response`. This is the `HttpResponse` returned by the + instrumented target method (in this case `execute()`). Note this is not the same as the return value + of `methodEnter()`.` ` + 5. Add a parameter `@Advice.Thrown final Throwable throwable`. This makes available any exception thrown by the + target `execute()` method. + 6. Use `scope.span() `to obtain the `AgentSpan` and decorate the span as needed. + 7. If the scope was just created (not inherited), close it. + +```java +public static class GoogleHttpClientAdvice { + @Advice.OnMethodEnter(suppress = Throwable.class) + public static AgentScope methodEnter( + @Advice.This HttpRequest request, + @Advice.Local("inherited") boolean inheritedScope + ) { + AgentScope scope = activeScope(); + if (null != scope) { + AgentSpan span = scope.span(); + if (HTTP_REQUEST == span.getOperationName()) { + inheritedScope = true; + return scope; + } + } + return activateSpan(DECORATE.prepareSpan(startSpan(HTTP_REQUEST), + request)); + } + + + @Advice.OnMethodExit(onThrowable = Throwable.class, suppress = Throwable.class) + public static void methodExit( + @Advice.Enter AgentScope scope, + @Advice.Local("inherited") boolean inheritedScope, + @Advice.Return final HttpResponse response, + @Advice.Thrown final Throwable throwable) { + try { + AgentSpan span = scope.span(); + DECORATE.onError(span, throwable); + DECORATE.onResponse(span, response); + DECORATE.beforeFinish(span); + span.finish(); + } finally { + if (!inheritedScope) { + scope.close(); + } + } + } +} +``` + +## Debugging + +Debuggers include helpful features like breakpoints, watches and stepping through code. Unfortunately those features are +not available in Advice code during development of a Java agent. You’ll need to add `println()` statements and rebuild +the tracer JAR to test/debug in a traced client application. `println()` is used instead of log statements because the +logger may not be initialized yet. Debugging should work as usual in helper methods that are called from advice code. + +By default, advice code is inlined into instrumented code. In that case breakpoints can not be set in the advice code. +But when a method is annotated like this: + +`@Advice.OnMethodExit(inline = false)` + +or + +`@Advice.OnMethodEnter(inline = false)` + +the advice bytecode is not copied and the advice is invoked like a common Java method call, making it work like a helper +class. Debugging information is copied from the advice method into the instrumented method and debugging is possible. + +It is not possible to use `inline=false` for all advice code. For example, when modifying argument +values, `@Argument(value = 0, readOnly = false)` is impossible since the advice is now a regular method invocation which +cannot be modified. + +It is important to remove `inline=false` after debugging is finished for performance reasons. + +( +see [inline](https://javadoc.io/static/net.bytebuddy/byte-buddy/1.14.10/net/bytebuddy/asm/Advice.OnMethodExit.html#inline--)) + +## Building + +Configure your environment as discussed +in [CONTRIBUTING.md](../CONTRIBUTING.md). Make sure you have installed +the necessary JDK versions and set all environment variables as described there. + +If you need to clean all results from a previous build: + +```shell +./gradlew -p buildSrc clean +``` + +Build your new tracer jar: + +```shell +./gradlew shadowJar +``` + +You will find the compiled SNAPSHOT jar here for example: + +```shell +./dd-java-agent/build/libs/dd-java-agent-1.25.0-SNAPSHOT.jar +``` + +You can confirm your new integration is included in the jar: + +```shell +java -jar dd-java-agent.jar --list-integrations +``` + +If Gradle is behaving badly you might try: + +``` +./gradlew --stop ; ./gradlew clean assemble +``` + +## Verifying Instrumentations + +There are four verification strategies, three of which are mandatory. + +- [Muzzle directives](./how_instrumentations_work.md#muzzle) +- [Instrumentation Tests](./how_instrumentations_work.md#instrumentation-tests) +- [Latest Dependency Tests](./how_instrumentations_work.md#latest-dependency-tests) +- [Smoke tests](./how_instrumentations_work.md#smoke-tests) + +All integrations must include sufficient test coverage. This HTTP client integration will include +a [standard HTTP test class](../dd-java-agent/instrumentation/google-http-client/src/test/groovy/GoogleHttpClientTest.groovy) +and +an [async HTTP test class](../dd-java-agent/instrumentation/google-http-client/src/test/groovy/GoogleHttpClientAsyncTest.groovy). +Both test classes inherit +from [HttpClientTest](../dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/base/HttpClientTest.groovy) +which provides a testing framework used by many HTTP client integrations. ( +see [Testing](./how_instrumentations_work.md#testing)) + +## Running Tests + +You can run only the tests applicable for this instrumentation: + +```shell +./gradlew :dd-java-agent:instrumentation:google-http-client:test +``` diff --git a/docs/how_instrumentations_work.md b/docs/how_instrumentations_work.md new file mode 100644 index 00000000000..32c44d341cc --- /dev/null +++ b/docs/how_instrumentations_work.md @@ -0,0 +1,735 @@ +# How Instrumentations Work + +## Introduction + +Around 120 integrations consisting of about 200 instrumentations are currently provided with the Datadog Java Trace +Agent. An auto-instrumentation allows compiled Java applications to be instrumented at runtime by a Java agent. This +happens when compiled classes matching rules defined in the instrumentation undergo bytecode manipulation to accomplish +some of what could be done by a developer instrumenting the code manually. Instrumentations are maintained +in `/dd-java-agent/instrumentation/` + +## Files/Directories + +Instrumentations are in the directory: + +`/dd-java-agent/instrumentation/$framework/$framework-$minVersion` + +where `$framework` is the framework name, and `$minVersion` is the minimum version of the framework supported by the +instrumentation. For example: + +``` +$ tree dd-java-agent/instrumentation/couchbase -L 2 +dd-java-agent/instrumentation/couchbase +├── build.gradle +├── couchbase-2.0 +│ ├── build.gradle +│ └── src +├── couchbase-2.6 +│ ├── build.gradle +│ └── src +├── couchbase-3.1 +│ ├── build.gradle +│ └── src +└── couchbase-3.2 + ├── build.gradle + └── src +``` + +In some cases, such +as [Hibernate](../dd-java-agent/instrumentation/hibernate), there is a +submodule containing different version-specific instrumentations, but typically a version-specific module is enough when +there is only one instrumentation implemented ( +e.g. [Akka-HTTP](../dd-java-agent/instrumentation/akka-http-10.0)) + +## Gradle + +Instrumentations included when building the Datadog java trace agent are defined +in [`/settings.gradle`](../settings.gradle) in alphabetical order with +the other instrumentations in this format: + +```groovy +include ':dd-java-agent:instrumentation:$framework?:$framework-$minVersion' +``` + +Dependencies specific to a particular instrumentation are added to the `build.gradle` file in that instrumentation’s +directory. Add necessary dependencies as `compileOnly` so they do not leak into the tracer. + +## Muzzle + +Muzzle directives are applied at build time from the `build.gradle` file. OpenTelemetry provides +some [Muzzle documentation](https://github.com/open-telemetry/opentelemetry-java-instrumentation/blob/main/docs/contributing/muzzle.md). +Muzzle directives check for a range of framework versions that are safe to load the instrumentation. + +See this excerpt as an example from [rediscala](../dd-java-agent/instrumentation/rediscala-1.8.0/build.gradle) + +```groovy +muzzle { + pass { + group = "com.github.etaty" + module = "rediscala_2.11" + versions = "[1.5.0,)" + assertInverse = true + } + + pass { + group = "com.github.etaty" + module = "rediscala_2.12" + versions = "[1.8.0,)" + assertInverse = true + } +} +``` + +This means that the instrumentation should be safe with `rediscala_2.11` from version `1.5.0` and all later versions, +but should fail (and so will not be loaded), for older versions (see `assertInverse`). +A similar range of versions is specified for `rediscala_2.12`. +When the agent is built, the muzzle plugin will download versions of the framework and check these directives hold. +To run muzzle on your instrumentation, run: + +```shell +./gradlew :dd-java-agent:instrumentation:rediscala-1.8.0:muzzle +``` + +* ⚠️ Muzzle does _not_ run tests. + It checks that the types and methods used by the instrumentation are present in particular versions of libraries. + It can be subverted with `MethodHandle` and reflection, so muzzle passing is not the end of the story. + +**TODO: discuss why 'name' is not always included.** + +## Instrumentation classes + +The Instrumentation class is where the Instrumentation begins. It will: + +1. Use Matchers to choose target types (i.e., classes) +2. From only those target types, use Matchers to select the members (i.e., methods) to instrument. +3. Apply instrumentation code from an Advice class to those members. + +Instrumentation classes: + +1. Must be annotated with `@AutoService(Instrumenter.class)` +2. Should extend one of the six abstract TargetSystem `Instrumenter` classes +3. Should implement one of the `Instrumenter` interfaces + +For example: + +```java +import datadog.trace.agent.tooling.Instrumenter; + +@AutoService(Instrumenter.class) +public class RabbitChannelInstrumentation extends Instrumenter.Tracing + implements Instrumenter.ForTypeHierarchy {/* */ +} +``` + +| | | +|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------| +| **TargetSystem** | **Usage** | +| `Instrumenter.`[`Tracing`](https://github.com/DataDog/dd-trace-java/blob/d18bc66fd448d40b2ea1b5a461e24dbcf036cfab/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/Instrumenter.java#L300C18-L300C25) | An Instrumentation class should extend an appropriate provided TargetSystem class when possible. | +| `Instrumenter.`[`Profiling`](https://github.com/DataDog/dd-trace-java/blob/d18bc66fd448d40b2ea1b5a461e24dbcf036cfab/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/Instrumenter.java#L312C22-L312C22) | | +| `Instrumenter.`[`AppSec`](https://github.com/DataDog/dd-trace-java/blob/d18bc66fd448d40b2ea1b5a461e24dbcf036cfab/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/Instrumenter.java#L331C18-L331C18) | | +| `Instrumenter.`[`Iast`](https://github.com/DataDog/dd-trace-java/blob/d18bc66fd448d40b2ea1b5a461e24dbcf036cfab/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/Instrumenter.java#L344) | | +| `Instrumenter.`[`CiVisibility`](https://github.com/DataDog/dd-trace-java/blob/d18bc66fd448d40b2ea1b5a461e24dbcf036cfab/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/Instrumenter.java#L405C18-L405C30) | | +| `Instrumenter.`[`Usm`](https://github.com/DataDog/dd-trace-java/blob/d18bc66fd448d40b2ea1b5a461e24dbcf036cfab/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/Instrumenter.java#L393) | | +| `Instrumenter.`[`Default`](https://github.com/DataDog/dd-trace-java/blob/d18bc66fd448d40b2ea1b5a461e24dbcf036cfab/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/Instrumenter.java) | Avoid extending `Default`. When no other TargetGroup is applicable we generally default to `Tracing`.` ` | + +### Type Matching + +Instrumentation classes should implement an +appropriate [Instrumenter interface](../dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/Instrumenter.java) +that specifies how target types will be selected for instrumentation. + +| | | | +|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Instrumenter Interface** | **Method(s)** | **Usage(Example)** | +| [`ForSingleType`](https://github.com/DataDog/dd-trace-java/blob/297b575f0f265c1dc78f9958e7b4b9365c80d1f9/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/Instrumenter.java#L70) | `String instrumentedType()` | Instruments only a single class name known at compile time.(see [Json2FactoryInstrumentation](https://github.com/DataDog/dd-trace-java/blob/9a28dc3f0333e781b2defc378c9020bf0a44ee9a/dd-java-agent/instrumentation/jackson-core/src/main/java/datadog/trace/instrumentation/jackson/core/Json2FactoryInstrumentation.java#L19)) | +| [`ForKnownTypes`](https://github.com/DataDog/dd-trace-java/blob/297b575f0f265c1dc78f9958e7b4b9365c80d1f9/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/Instrumenter.java#L75) | `String[] knownMatchingTypes()` | Instruments multiple class names known at compile time. | +| [`ForTypeHierarchy`](https://github.com/DataDog/dd-trace-java/blob/297b575f0f265c1dc78f9958e7b4b9365c80d1f9/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/Instrumenter.java#L80) | `String hierarchyMarkerType()``ElementMatcher hierarchyMatcher()` | Composes more complex matchers using chained [HierarchyMatchers](https://github.com/DataDog/dd-trace-java/blob/9a28dc3f0333e781b2defc378c9020bf0a44ee9a/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/bytebuddy/matcher/HierarchyMatchers.java#L18) methods. The `hierarchyMarkerType()` method should return a type name. Classloaders without this type can skip the more expensive `hierarchyMatcher()` method. (see [HttpClientInstrumentation](https://github.com/DataDog/dd-trace-java/blob/9a28dc3f0333e781b2defc378c9020bf0a44ee9a/dd-java-agent/instrumentation/java-http-client/src/main/java/datadog/trace/instrumentation/httpclient/HttpClientInstrumentation.java#L43)) | +| [`ForConfiguredType`](https://github.com/DataDog/dd-trace-java/blob/297b575f0f265c1dc78f9958e7b4b9365c80d1f9/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/Instrumenter.java#L93C13-L93C30) | `Collection configuredMatchingTypes()` | **_Do not implement this interface_**_._Use `ForKnownType` instead. `ForConfiguredType` is only used for last minute additions in the field - such as when a customer has a new JDBC driver that's not in the allowed list and we need to test it and provide a workaround until the next release. | +| [`ForConfiguredTypes`](https://github.com/DataDog/dd-trace-java/blob/297b575f0f265c1dc78f9958e7b4b9365c80d1f9/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/Instrumenter.java#L88) | `String configuredMatchingType();` | **_Do not implement this interface._** __Like `ForConfiguredType,` for multiple classes | + +When matching your instrumentation against target types, +prefer [ForSingleType](https://github.com/DataDog/dd-trace-java/blob/5cab82068b689a46970d9132a142a364548a82fa/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/Instrumenter.java#L68C13-L68C26) +or [ForKnownTypes](https://github.com/DataDog/dd-trace-java/blob/5cab82068b689a46970d9132a142a364548a82fa/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/Instrumenter.java#L73C13-L73C26) +over more +expensive [ForTypeHierarchy](https://github.com/DataDog/dd-trace-java/blob/5cab82068b689a46970d9132a142a364548a82fa/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/Instrumenter.java#L78C13-L78C29) +matching. + +Consider adding an +appropriate [ClassLoaderMatcher](https://github.com/DataDog/dd-trace-java/blob/3e81c006b54f73aae61f88c39b52a7267267075b/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/Instrumenter.java#L253) +so the Instrumentation only activates when that class is loaded. For example: + +```java + +@Override +public ElementMatcher classLoaderMatcher() { + return hasClassNamed("java.net.http.HttpClient"); +} +``` + +The `Instrumenter.ForBootstrap` interface is a hint that this instrumenter works on bootstrap types and there is no +classloader present to interrogate. Use it when instrumenting something from the JDK that will be on the bootstrap +classpath. For +example, [`ShutdownInstrumentation`](https://github.com/DataDog/dd-trace-java/blob/3e81c006b54f73aae61f88c39b52a7267267075b/dd-java-agent/instrumentation/shutdown/src/main/java/datadog/trace/instrumentation/shutdown/ShutdownInstrumentation.java#L18) +or [`UrlInstrumentation`](https://github.com/DataDog/dd-trace-java/blob/3e81c006b54f73aae61f88c39b52a7267267075b/dd-java-agent/instrumentation/http-url-connection/src/main/java/datadog/trace/instrumentation/http_url_connection/UrlInstrumentation.java#L21). + +### Method Matching + +After the type is selected, the type’s target members(e.g., methods) must next be selected using the Instrumentation +class’s `adviceTransformations()` method. +ByteBuddy’s [`ElementMatchers`](https://javadoc.io/doc/net.bytebuddy/byte-buddy/1.4.17/net/bytebuddy/matcher/ElementMatchers.html) +are used to describe the target members to be instrumented. +Datadog’s [`DDElementMatchers`](../dd-java-agent/agent-builder/src/main/java/datadog/trace/agent/tooling/bytebuddy/matcher/DDElementMatchers.java) +class also provides these 10 additional matchers: + +* implementsInterface +* hasInterface +* hasSuperType +* declaresMethod +* extendsClass +* concreteClass +* declaresField +* declaresContextField +* declaresAnnotation +* hasSuperMethod + +Here, any public `execute()` method taking no arguments will have `PreparedStatementAdvice` applied: + +```java + +@Override +public void adviceTransformations(AdviceTransformation transformation) { + transformation.applyAdvice( + nameStartsWith("execute") + .and(takesArguments(0)) + .and(isPublic()), + getClass().getName() + "$PreparedStatementAdvice" + ); +} +``` + +Here, any matching `connect()` method will have `DriverAdvice` applied: + +```java + +@Override +public void adviceTransformations(AdviceTransformation transformation) { + transformation.applyAdvice( + nameStartsWith("connect") + .and(takesArgument(0, String.class)) + .and(takesArgument(1, Properties.class)) + .and(returns(named("java.sql.Connection"))), + getClass().getName() + "$DriverAdvice"); +} +``` + +Be precise in matching to avoid inadvertently instrumenting something unintended in a current or future version of the +target class. Having multiple precise matchers is preferable to one more vague catch-all matcher which leaves some +method characteristics undefined. + +Instrumentation class names should end in _Instrumentation._ + +## Helper Classes + +Classes referenced by Advice that are not provided on the bootclasspath must be defined in Helper Classes otherwise they +will not be loaded at runtime. This included any decorators, extractors/injectors or wrapping classes such as tracing +listeners that extend or implement types provided by the library being instrumented. Also watch out for implicit types +such as anonymous/nested classes because they must be listed alongside the main helper class. + +If an instrumentation is producing no results it may be that a required class is missing. Running muzzle + +```shell +./gradlew muzzle +``` + +can quickly tell you if you missed a required helper class. Messages like this in debug logs also indicate that classes +are missing: + +``` +[MSC service thread 1-3] DEBUG datadog.trace.agent.tooling.muzzle.MuzzleCheck - Muzzled mismatch - instrumentation.names=[jakarta-mdb] instrumentation.class=datadog.trace.instrumentation.jakarta.jms.MDBMessageConsumerInstrumentation instrumentation.target.classloader=ModuleClassLoader for Module "deployment.cmt.war" from Service Module Loader muzzle.mismatch="datadog.trace.instrumentation.jakarta.jms.MessageExtractAdapter:20 Missing class datadog.trace.instrumentation.jakarta.jms.MessageExtractAdapter$1" +``` + +The missing class must be added in the helperClassNames method, for example: + +```java + +@Override +public String[] helperClassNames() { + return new String[]{ + "datadog.trace.instrumentation.jakarta.jms.MessageExtractAdapter", + "datadog.trace.instrumentation.jakarta.jms.JMSDecorator", + "datadog.trace.instrumentation.jakarta.jms.MessageExtractAdapter$1" + }; +} +``` + +## Enums + +Use care when deciding to include enums in your Advice and Decorator classes because each element of the enum will need +to be added to the helper classes individually. For example not just `MyDecorator.MyEnum` but +also `MyDecorator.MyEnum$1, MyDecorator.MyEnum$2`, etc. + +## Decorator Classes + +Decorators contain extra code that will be injected into the instrumented methods. + +These provided Decorator classes sit +in [dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator](../dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator) + +| | | | +|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------:|:-------------------------:|:-----------------------------------------------------------------------------------------------------------------------------------------------------:| +| **Decorator** | **Parent Class** | **Usage(see JavaDoc for more detail)** | +| [`AppSecUserEventDecorator`](https://github.com/DataDog/dd-trace-java/blob/297b575f0f265c1dc78f9958e7b4b9365c80d1f9/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/AppSecUserEventDecorator.java#L13) | `-` | Provides mostly login-related functions to the Spring Security instrumentation. | +| [`AsyncResultDecorator`](https://github.com/DataDog/dd-trace-java/blob/297b575f0f265c1dc78f9958e7b4b9365c80d1f9/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/AsyncResultDecorator.java#L18) | `BaseDecorator` | Handles asynchronous result types, finishing spans only when the async calls are complete. | +| [`BaseDecorator`](https://github.com/DataDog/dd-trace-java/blob/297b575f0f265c1dc78f9958e7b4b9365c80d1f9/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/BaseDecorator.java#L21) | `-` | Provides many convenience methods related to span naming and error handling. New Decorators should extend BaseDecorator or one of its child classes. | +| [`ClientDecorator`](https://github.com/DataDog/dd-trace-java/blob/297b575f0f265c1dc78f9958e7b4b9365c80d1f9/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/ClientDecorator.java#L6) | `BaseDecorator` | Parent of many Client Decorators. Used to set client specific tags, serviceName, etc | +| [`DBTypeProcessingDatabaseClientDecorator`](https://github.com/DataDog/dd-trace-java/blob/297b575f0f265c1dc78f9958e7b4b9365c80d1f9/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/DBTypeProcessingDatabaseClientDecorator.java#L5) | `DatabaseClientDecorator` | Adds automatic `processDatabaseType() `call to `DatabaseClientDecorator.` | +| [`DatabaseClientDecorator`](https://github.com/DataDog/dd-trace-java/blob/297b575f0f265c1dc78f9958e7b4b9365c80d1f9/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/DatabaseClientDecorator.java#L14) | `ClientDecorator` | Provides general db-related methods. | +| [`HttpClientDecorator`](https://github.com/DataDog/dd-trace-java/blob/297b575f0f265c1dc78f9958e7b4b9365c80d1f9/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/HttpClientDecorator.java#L23) | `UriBasedClientDecorator` | Mostly adds span tags to HTTP client requests and responses. | +| [`HttpServerDecorator`](https://github.com/DataDog/dd-trace-java/blob/297b575f0f265c1dc78f9958e7b4b9365c80d1f9/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/HttpServerDecorator.java#L46) | `ServerDecorator` | Adds connection and HTTP response tagging often used for server frameworks. | +| [`MessagingClientDecorator`](https://github.com/DataDog/dd-trace-java/blob/297b575f0f265c1dc78f9958e7b4b9365c80d1f9/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/MessagingClientDecorator.java#L6) | `ClientDecorator` | Adds e2e (end-to-end) duration monitoring. | +| [`OrmClientDecorator`](https://github.com/DataDog/dd-trace-java/blob/297b575f0f265c1dc78f9958e7b4b9365c80d1f9/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/OrmClientDecorator.java#L5) | `DatabaseClientDecorator` | Set the span’s resourceName to the entityName value. | +| [`ServerDecorator`](https://github.com/DataDog/dd-trace-java/blob/297b575f0f265c1dc78f9958e7b4b9365c80d1f9/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/ServerDecorator.java#L7) | `BaseDecorator` | Adding server and language tags to the span. | +| [`UriBasedClientDecorator`](https://github.com/DataDog/dd-trace-java/blob/297b575f0f265c1dc78f9958e7b4b9365c80d1f9/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/UriBasedClientDecorator.java#L9) | `ClientDecorator` | Adds hostname, port and service values from URIs to HttpClient spans. | +| [`UrlConnectionDecorator`](https://github.com/DataDog/dd-trace-java/blob/297b575f0f265c1dc78f9958e7b4b9365c80d1f9/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/UrlConnectionDecorator.java#L18) | `UriBasedClientDecorator` | Sets some tags based on URI and URL values. Also provides some caching. Only used by `UrlInstrumentation`. | + +Instrumentations often include their own Decorators which extend those classes, for example: + +| | | | +|:--------------------------:|:----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------:|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------:| +| **Instrumentation** | **Decorator** | **Parent Class** | +| JDBC | [`DataSourceDecorator`](https://github.com/DataDog/dd-trace-java/blob/master/dd-java-agent/instrumentation/jdbc/src/main/java/datadog/trace/instrumentation/jdbc/DataSourceDecorator.java) | [`BaseDecorator`](https://github.com/DataDog/dd-trace-java/blob/master/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/BaseDecorator.java) | +| RabbitMQ | [`RabbitDecorator`](https://github.com/DataDog/dd-trace-java/blob/297b575f0f265c1dc78f9958e7b4b9365c80d1f9/dd-java-agent/instrumentation/rabbitmq-amqp-2.7/src/main/java/datadog/trace/instrumentation/rabbitmq/amqp/RabbitDecorator.java#L34) | [`MessagingClientDecorator`](https://github.com/DataDog/dd-trace-java/blob/297b575f0f265c1dc78f9958e7b4b9365c80d1f9/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/MessagingClientDecorator.java#L6) | +| All HTTP Server frameworks | various | [`HttpServerDecorator`](https://github.com/DataDog/dd-trace-java/blob/297b575f0f265c1dc78f9958e7b4b9365c80d1f9/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/HttpServerDecorator.java#L46) | + +Decorator class names must be in the instrumentation's helper classes since Decorators need to be loaded with the +instrumentation. + +Decorator class names should end in _Decorator._ + +## Advice Classes + +Byte Buddy injects compiled bytecode at runtime to wrap existing methods, so they communicate with Datadog at entry or +exit. These modifications are referred to as _advice transformation_ or just _advice_. + +Instrumenters register advice transformations by calling + +`AdviceTransformation.applyAdvice(ElementMatcher, String) `and Methods are matched by the +instrumentation's `adviceTransformations()` method. + +The Advice is injected into the type so Advice can only refer to those classes on the bootstrap class-path or helpers +injected into the application class-loader. Advice must not refer to any methods in the instrumentation class or even +other methods in the same advice class because the advice is really only a template of bytecode to be inserted into the +target class. It is only the advice bytecode (plus helpers) that is copied over. The rest of the instrumenter and advice +class is ignored. Do not place code in the Advice constructor because the constructor is never called. + +You can not use methods like `InstrumentationContext.get()`outside of the instrumentation advice because the tracer +currently patches the method stub with the real call at runtime. But you can pass the ContextStore into a +helper/decorator like +in [DatadogMessageListener](https://github.com/DataDog/dd-trace-java/blob/743bacde52ba4369e05631436168bfde9b815c8b/dd-java-agent/instrumentation/jms/src/main/java/datadog/trace/instrumentation/jms/DatadogMessageListener.java). +This could reduce duplication if you re-used the helper. But unlike most applications, some duplication can be the +better choice in the tracer if it simplifies things and reduces overhead. You might end up with very similar code +scattered around, but it will be simple to maintain. Trying to find an abstraction that works well across +instrumentations can take time and may introduce extra indirection. + +Advice classes provide the code to be executed before and/or after a matched method. The classes use a static method +annotated by `@Advice.OnMethodEnter` and/or `@Advice.OnMethodExit` to provide the code. The method name is irrelevant. + +A method that is annotated with `@Advice.OnMethodEnter `can annotate its parameters +with `@Advice.Argument`. `@Advice.Argument` will substitute this parameter with the corresponding argument of the +instrumented method. This allows the `@Advice.OnMethodEnter` code to see and modify the parameters that would be passed +to the target method. + +Alternatively, a parameter can be annotated by `Advice.This` where the `this` reference of the instrumented method is +assigned to the new parameter. This can also be used to assign a new value to the `this` reference of an instrumented +method. + +If no annotation is used on a parameter, it is assigned the n-th parameter of the instrumented method for the n-th +parameter of the advice method. Explicitly specifying which parameter is intended is recommended to be more clear, for +example: + +`@Advice.Argument(0) final HttpUriRequest request` + +All parameters must declare the exact same type as the parameters of the instrumented type or the method's declaring +type for `Advice.This`. If they are marked as read-only, then the parameter type may be a super type of the original. + +A method that is annotated with `Advice.OnMethodExit` can also annotate its parameters with `Advice.Argument` +and `Advice.This`. It can also annotate a parameter with `Advice.Return` to receive the original method's return value. +By reassigning the return value, it can replace the returned value. If an instrumented method does not return a value, +this annotation must not be used. If a method throws an exception, the parameter is set to its default value (0 for +primitive types and to null for reference types). The parameter's type must equal the instrumented method's return type +if it is not set to read-only. If the parameter is read-only it may be a super type of the instrumented method's return +type. + +Advice class names should end in _Advice._ + +## Exceptions in Advice + +Advice methods are typically annotated like + +`@Advice.OnMethodEnter(suppress = Throwable.class)` + +and + +`@Advice.OnMethodExit(onThrowable = Throwable.class, suppress = Throwable.class)` + +Using `suppress = Throwable.class` is considered our default for both unless there is a reason not to. It means the +exception handler is triggered on any exception thrown within the Advice and the Advice method terminates. The opposite +would be either no `suppress` annotation or equivalently `suppress = NoExceptionHandler.class` which would both allow +exceptions in Advice code to surface and is usually undesirable. + +If +the [`Advice.OnMethodEnter`](https://javadoc.io/static/net.bytebuddy/byte-buddy/1.10.2/net/bytebuddy/asm/Advice.OnMethodEnter.html) +method throws an exception, +the [`Advice.OnMethodExit`](https://javadoc.io/static/net.bytebuddy/byte-buddy/1.10.2/net/bytebuddy/asm/Advice.OnMethodExit.html) +method is not invoked. + +The [`Advice.Thrown`](https://javadoc.io/static/net.bytebuddy/byte-buddy/1.10.2/net/bytebuddy/asm/Advice.Thrown.html) +annotation passes any thrown exception from the instrumented method to +the [`Advice.OnMethodExit`](https://javadoc.io/static/net.bytebuddy/byte-buddy/1.10.2/net/bytebuddy/asm/Advice.OnMethodExit.html) +advice +method. [`Advice.Thrown`](https://javadoc.io/static/net.bytebuddy/byte-buddy/1.10.2/net/bytebuddy/asm/Advice.Thrown.html) **** +should annotate at most one parameter on the exit advice. + +If the instrumented method throws an exception, +the [Advice.OnMethodExit](https://javadoc.io/static/net.bytebuddy/byte-buddy/1.10.2/net/bytebuddy/asm/Advice.OnMethodExit.html) +method is still invoked unless +the [Advice.OnMethodExit.onThrowable()](https://javadoc.io/static/net.bytebuddy/byte-buddy/1.10.2/net/bytebuddy/asm/Advice.OnMethodExit.html#onThrowable--) +property is set to false. If this property is set to false, +the [Advice.Thrown](https://javadoc.io/static/net.bytebuddy/byte-buddy/1.10.2/net/bytebuddy/asm/Advice.Thrown.html) +annotation must not be used on any parameter. + +If an instrumented method throws an exception, the return parameter is set to its default of 0 for primitive types or +null for reference types. An exception can be read by annotating an exit +method’s [Throwable](http://docs.oracle.com/javase/1.5.0/docs/api/java/lang/Throwable.html?is-external=true) parameter +with [Advice.Thrown](https://javadoc.io/static/net.bytebuddy/byte-buddy/1.10.2/net/bytebuddy/asm/Advice.Thrown.html) +which is assigned the +thrown [Throwable](http://docs.oracle.com/javase/1.5.0/docs/api/java/lang/Throwable.html?is-external=true) or null if a +method returns normally. This allows exchanging a thrown exception with any checked or unchecked exception. For example, +either the result or the exception will be passed to the helper method here: + +```java + +@Advice.OnMethodExit(onThrowable = Throwable.class, suppress = Throwable.class) +public static void methodExit( + @Advice.Return final Object result, + @Advice.Thrown final Throwable throwable +) { + HelperMethods.doMethodExit(result, throwable); +} +``` + +## InjectAdapters & Custom GETTERs/SETTERs + +Custom Inject Adapter static instances typically named `SETTER` implement the `AgentPropagation.Setter` interface and +are used to normalize setting shared context values such as in HTTP headers. + +Custom inject adapter static instances typically named `GETTER` implement the `AgentPropagation.Getter` interface and +are used to normalize extracting shared context values such as from HTTP headers. + +For example `google-http-client` sets its header values using: + +`com.google.api.client.http.HttpRequest.getHeaders().put(key,value)` + +```java +package datadog.trace.instrumentation.googlehttpclient; + +import com.google.api.client.http.HttpRequest; +import datadog.trace.bootstrap.instrumentation.api.AgentPropagation; + +public class HeadersInjectAdapter implements AgentPropagation.Setter { + public static final HeadersInjectAdapter SETTER = new HeadersInjectAdapter(); + + @Override + public void set(final HttpRequest carrier, final String key, final String value) { + carrier.getHeaders().put(key, value); + } +} +``` + +But notice `apache-http-client5` sets its header values using: + +`org.apache.hc.core5.http.HttpRequest.setHeader(key,value)` + +```java +package datadog.trace.instrumentation.apachehttpclient5; + +import datadog.trace.bootstrap.instrumentation.api.AgentPropagation; +import org.apache.hc.core5.http.HttpRequest; + +public class HttpHeadersInjectAdapter implements AgentPropagation.Setter { + public static final HttpHeadersInjectAdapter SETTER = new HttpHeadersInjectAdapter(); + + @Override + public void set(final HttpRequest carrier, final String key, final String value) { + carrier.setHeader(key, value); + } +} +``` + +These implementation-specific methods are both wrapped in a standard set(...) method by the SETTER. + +## To Wrap or Not To Wrap? + +Typically, an instrumentation will use ByteBuddy to apply new code from an Advice class before and/or after the targeted +code using `@Advice.OnMethodEnter` and `@Advice.OnMethodExit.` + +Alternatively, you can replace the call to the target method with your own code which wraps the original method call. An +example is the JMS Instrumentation which replaces the `MessageListener.onMessage() `method +with `DatadogMessageListener.onMessage(). `The `DatadogMessageListener` +then [calls the original` onMessage() `method](https://github.com/DataDog/dd-trace-java/blob/9a28dc3f0333e781b2defc378c9020bf0a44ee9a/dd-java-agent/instrumentation/jms/src/main/java/datadog/trace/instrumentation/jms/DatadogMessageListener.java#L73). +Note that this style is **_not recommended_** because it can cause datadog packages to appear in stack traces generated +by errors in user code. This has created confusion in the past. + +## Context Stores + +Context stores pass information between instrumented methods, using library objects that both methods have access to. +They can be used to attach data to a request when the request is received, and read that data where the request is +deserialized. Context stores work internally by dynamically adding a field to the “carrier” object by manipulating the +bytecode. Since they manipulate bytecode, context stores can only be created within Advice classes. For example: + +```java +ContextStore store = InstrumentationContext.get( + "com.amazonaws.services.sqs.model.ReceiveMessageResult", "java.lang.String"); +``` + +It’s also possible to pass the types as class objects, but this is only possible for classes that are in the bootstrap +classpath. Basic types like `String` would work and the usual datadog types like `AgentSpan` are OK too, but classes +from the library you are instrumenting are not. + +In the example above, that context store is used to store an arbitrary `String` in a `ReceiveMessageResult` class. It is +used like a Map: + +```java +store.put(response, "my string"); +``` + +and/or + +```java +String stored = store.get(response); // "my string" +``` + +Context stores also need to be pre-declared in the Advice by overriding the `contextStore()` method otherwise, using +them throws exceptions. + +```java + +@Override +public Map contextStore() { + return singletonMap( + "com.amazonaws.services.sqs.model.ReceiveMessageResult", + "java.lang.String" + ); +} +``` + +It is important to understand that even though they look like maps, since the value is stored in the key, you can only +retrieve a value if you use the exact same key object as when it was set. Using a different object that is “`.equals()`” +to the first will yield nothing. + +## CallDepthThreadLocalMap + +In order to avoid activating new spans on recursive calls to the same method +a [CallDepthThreadLocalMap](https://github.com/DataDog/dd-trace-java/blob/9d5c7ea524cfec982176e687a489fc8c2865e445/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/CallDepthThreadLocalMap.java#L11) +is often used to determine if a call is recursive by using a counter. It is incremented with each call to the method +and [decremented](https://github.com/DataDog/dd-trace-java/blob/9d5c7ea524cfec982176e687a489fc8c2865e445/dd-java-agent/instrumentation/vertx-redis-client-3.9/src/main/java/datadog/trace/instrumentation/vertx_redis_client/RedisSendAdvice.java#L82) ( +or [reset](https://github.com/DataDog/dd-trace-java/blob/9d5c7ea524cfec982176e687a489fc8c2865e445/dd-java-agent/instrumentation/java-http-client/src/main/java11/datadog/trace/instrumentation/httpclient/SendAdvice.java#L44)) +when exiting. + +## Span Lifecycle + +In Advice classes, the `@Advice.OnMethodEnter` methods typically start spans and `@Advice.OnMethodExit` methods +typically finish spans. + +Starting the span may be done directly or with helper methods which eventually make a call to one of the +various `AgentTracer.startSpan(...)` methods. + +Finishing the span is normally done by calling `span.finish()` in the exit method; + +The basic span lifecycle in an Advice class looks like: + +1. Start the span + +2. Decorate the span + +3. Activate the span and get the AgentScope + +4. Run the instrumented target method + +5. Close the Agent Scope + +6. Finish the span + +```java + +@Advice.OnMethodEnter(suppress = Throwable.class) +public static AgentScope begin() { + final AgentSpan span = startSpan(/* */); + DECORATE.afterStart(span); + return activateSpan(span); +} + +@Advice.OnMethodExit(suppress = Throwable.class) +public static void end(@Advice.Enter final AgentScope scope) { + AgentSpan span = scope.span(); + DECORATE.beforeFinish(span); + scope.close(); + span.finish(); +} +``` + +For example, +the [`HttpUrlConnectionInstrumentation`](https://github.com/DataDog/dd-trace-java/blob/4d0b113c4c9dc23ef2a44d30952d38d09ff28ff3/dd-java-agent/instrumentation/http-url-connection/src/main/java/datadog/trace/instrumentation/http_url_connection/HttpUrlConnectionInstrumentation.java#L26) +class contains +the [`HttpUrlConnectionAdvice`](https://github.com/DataDog/dd-trace-java/blob/4d0b113c4c9dc23ef2a44d30952d38d09ff28ff3/dd-java-agent/instrumentation/http-url-connection/src/main/java/datadog/trace/instrumentation/http_url_connection/HttpUrlConnectionInstrumentation.java#L66C23-L66C46) +class which calls +the `HttpUrlState.`[`start`](https://github.com/DataDog/dd-trace-java/blob/4d0b113c4c9dc23ef2a44d30952d38d09ff28ff3/dd-java-agent/instrumentation/http-url-connection/src/main/java/datadog/trace/instrumentation/http_url_connection/HttpUrlConnectionInstrumentation.java#L84)`()` +and `HttpUrlState.`[`finishSpan`](https://github.com/DataDog/dd-trace-java/blob/4d0b113c4c9dc23ef2a44d30952d38d09ff28ff3/dd-java-agent/instrumentation/http-url-connection/src/main/java/datadog/trace/instrumentation/http_url_connection/HttpUrlConnectionInstrumentation.java#L113C19-L113C29)`()` +methods. + +## Continuations + +- [`AgentScope.Continuation`](https://github.com/DataDog/dd-trace-java/blob/09ac78ff0b54fbbbee0ab1c89c901d2043fda40b/dd-trace-api/src/main/java/datadog/trace/context/TraceScope.java#L47) + is used to pass context between threads. +- Continuations must be either activated or canceled. +- If a Continuation is activated it returns a TraceScope which must eventually be closed. +- Only after all TraceScopes are closed and any non-activated Continuations are canceled may the Trace finally close. + +Notice +in [`HttpClientRequestTracingHandler`](https://github.com/DataDog/dd-trace-java/blob/3fe1b2d6010e50f61518fa25af3bdeb03ae7712b/dd-java-agent/instrumentation/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/client/HttpClientRequestTracingHandler.java#L56) +how the AgentScope.Continuation is used to obtain the `parentScope` which is +finally [closed](https://github.com/DataDog/dd-trace-java/blob/3fe1b2d6010e50f61518fa25af3bdeb03ae7712b/dd-java-agent/instrumentation/netty-4.1/src/main/java/datadog/trace/instrumentation/netty41/client/HttpClientRequestTracingHandler.java#L111). + +## Naming + +- Instrumentation names use kebab case. For example: `google-http-client` +- Instrumentation module name and package name should be consistent. For example, the + instrumentation `google-http-client `contains the `GoogleHttpClientInstrumentation` class in the + package` datadog.trace.instrumentation.googlehttpclient.` +- As usual, class names should be nouns, in camel case with the first letter of each internal word capitalized. Use + whole words-avoid acronyms and abbreviations (unless the abbreviation is much more widely used than the long form, + such as URL or HTML). +- Advice class names should end in _Advice._ +- Instrumentation class names should end in _Instrumentation._ +- Decorator class names should end in _Decorator._ + +## Tooling + +### ignored\_class\_name.trie + +The +file [ignored\_class\_name.trie](../dd-java-agent/agent-tooling/src/main/resources/datadog/trace/agent/tooling/bytebuddy/matcher/ignored_class_name.trie) +lists classes that are to be globally ignored by matchers because they are unsafe, pointless or expensive to transform. +If you notice an expected class is not being transformed, it may be covered by an entry in this list. + +## GraalVM + +Instrumentations running on GraalVM should avoid using reflection if possible. If reflection must be used the reflection +usage should be added to + +`dd-java-agent/agent-bootstrap/src/main/resources/META-INF/native-image/com.datadoghq/dd-java-agent/reflect-config.json` + +See [GraalVM configuration docs](https://www.graalvm.org/jdk17/reference-manual/native-image/dynamic-features/Reflection/#manual-configuration). + +## Testing + +### Instrumentation Tests + +Tests are written in Groovy using the [Spock framework](http://spockframework.org). For +instrumentations, `AgentTestRunner` must be extended. For example, HTTP server frameworks use base tests which enforce +consistency between different implementations ( +see [HttpServerTest](../dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/base/HttpServerTest.groovy)). +When writing an instrumentation it is much faster to test just the instrumentation rather than build the entire project, +for example: + +```shell +./gradlew :dd-java-agent:instrumentation:play-ws-2.1:test +``` + +Sometimes it is necessary to force Gradle to discard cached test results +and [rerun all tasks](https://docs.gradle.org/current/userguide/command_line_interface.html#sec:rerun_tasks). + +```shell +./gradle test --rerun-tasks +``` + +Running tests that require JDK-21 will require the `JAVA_21_HOME` env var set and can be done like this: + +```shell +./gradlew :dd-java-agent:instrumentation:aerospike-4:allLatestDepTests -PtestJvm=21 +``` + +### Latest Dependency Tests + +Adding a directive to the build file gives early warning when breaking changes are released by framework maintainers. +For example, for Play 2.5, we download the latest dependency and run tests against it: + +```groovy +latestDepTestCompile group: 'com.typesafe.play', name: 'play-java_2.11', version: '2.5.+' + +latestDepTestCompile group: 'com.typesafe.play', name: 'play-java-ws_2.11', version: '2.5.+' + +latestDepTestCompile(group: 'com.typesafe.play', name: 'play-test_2.11', version: '2.5.+') { + exclude group: 'org.eclipse.jetty.websocket', module: 'websocket-client' +} +``` + +Dependency tests can be run like: + +```shell +./gradlew :dd-java-agent:instrumentation:play-ws-2.1:latestDepTest +``` + +### Additional Test Suites + +The +file [dd-trace-java/gradle/test-suites.gradle](../gradle/test-suites.gradle) +contains these macros for adding different test suites to individual instrumentation builds. Notice how `addTestSuite` +and `addTestSuiteForDir` pass values +to [`addTestSuiteExtendingForDir`](https://github.com/DataDog/dd-trace-java/blob/c3ea017590f10941232bbb0f694525bf124d4b49/gradle/test-suites.gradle#L3) +which configures the tests. + +```groovy +ext.addTestSuite = (String testSuiteName) -> { + ext.addTestSuiteForDir(testSuiteName, testSuiteName) +} + +ext.addTestSuiteForDir = (String testSuiteName, String dirName) -> { + ext.addTestSuiteExtendingForDir(testSuiteName, 'test', dirName) +} + +ext.addTestSuiteExtendingForDir = (String testSuiteName, String parentSuiteName, String dirName) -> { /* */ } +``` + +For example: + +```groovy +addTestSuite('latestDepTest') +``` + +Also, the forked test for latestDep is not run by default without declaring something like: + +```groovy +addTestSuiteExtendingForDir('latestDepForkedTest', 'latestDepTest', 'test') +``` + +(also +example [`vertx-web-3.5/build.gradle`](https://github.com/DataDog/dd-trace-java/blob/c3ea017590f10941232bbb0f694525bf124d4b49/dd-java-agent/instrumentation/vertx-web-3.5/build.gradle#L18)`)` + +### Smoke Tests + +In addition to unit tests, [Smoke tests](../dd-smoke-tests) may be +needed. Smoke tests run with a real agent jar file set as the `javaagent`. These are optional and not all frameworks +have them, but contributions are very welcome. + +# Summary + +Integrations have evolved over time. Newer examples of integrations such as Spring and JDBC illustrate current best +practices. + +# Additional Reading + +- Datadog Instrumentations rely heavily on ByteBuddy. You may find the + ByteBuddy [tutorial](https://bytebuddy.net/#/tutorial) useful. +- The [Groovy docs](https://groovy-lang.org/single-page-documentation.html). +- [Spock Framework Reference Documentation](https://spockframework.org/spock/docs/2.3/index.html). From 67ed8bb516499ddb7b266eaba8e977cc11b36829 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Tue, 5 Mar 2024 16:54:35 +0100 Subject: [PATCH 07/30] Bump ddprof-java to 1.1.0 (#6776) --- gradle/dependencies.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index 4116468d550..580f4cd2a86 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -33,7 +33,7 @@ final class CachedData { testcontainers: '1.19.3', jmc : "8.1.0", autoservice : "1.0-rc7", - ddprof : "0.99.0", + ddprof : "1.1.0", asm : "9.6", cafe_crypto : "0.1.0", lz4 : "1.7.1" From 84b49e940f87667c7084515144d989918b674628 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Tue, 5 Mar 2024 17:40:08 +0100 Subject: [PATCH 08/30] Make the native image smoke test check presence of ExecutionSample events (#6775) --- .../spring-boot-3.0-native/build.gradle | 1 + ...SpringBootNativeInstrumentationTest.groovy | 22 ++++++++++++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/dd-smoke-tests/spring-boot-3.0-native/build.gradle b/dd-smoke-tests/spring-boot-3.0-native/build.gradle index ecd02639128..1b153663bb4 100644 --- a/dd-smoke-tests/spring-boot-3.0-native/build.gradle +++ b/dd-smoke-tests/spring-boot-3.0-native/build.gradle @@ -4,6 +4,7 @@ description = 'Spring Boot 3.0 Native Smoke Tests.' dependencies { testImplementation project(':dd-smoke-tests') + testImplementation deps.jmc } def testJvm = gradle.startParameter.projectProperties.getOrDefault('testJvm', '') diff --git a/dd-smoke-tests/spring-boot-3.0-native/src/test/groovy/SpringBootNativeInstrumentationTest.groovy b/dd-smoke-tests/spring-boot-3.0-native/src/test/groovy/SpringBootNativeInstrumentationTest.groovy index e1b622f7e7d..61d93f8aeb1 100644 --- a/dd-smoke-tests/spring-boot-3.0-native/src/test/groovy/SpringBootNativeInstrumentationTest.groovy +++ b/dd-smoke-tests/spring-boot-3.0-native/src/test/groovy/SpringBootNativeInstrumentationTest.groovy @@ -1,14 +1,20 @@ import datadog.smoketest.AbstractServerSmokeTest import okhttp3.Request +import org.openjdk.jmc.common.item.IItemCollection +import org.openjdk.jmc.common.item.ItemFilters +import org.openjdk.jmc.flightrecorder.internal.InvalidJfrFileException import spock.lang.Shared import spock.lang.TempDir +import org.openjdk.jmc.flightrecorder.JfrLoaderToolkit + import java.nio.file.FileVisitResult import java.nio.file.Files import java.nio.file.Path import java.nio.file.SimpleFileVisitor import java.nio.file.attribute.BasicFileAttributes import java.util.concurrent.atomic.AtomicInteger +import java.util.concurrent.locks.LockSupport class SpringBootNativeInstrumentationTest extends AbstractServerSmokeTest { @Shared @@ -57,12 +63,18 @@ class SpringBootNativeInstrumentationTest extends AbstractServerSmokeTest { def response = client.newCall(new Request.Builder().url(url).get().build()).execute() then: + def ts = System.nanoTime() def responseBodyStr = response.body().string() responseBodyStr != null responseBodyStr.contains("Hello world") waitForTraceCount(1) // sanity test for profiler generating JFR files + // the recording is collected after 1 second of execution + // make sure the app has been up and running for at least 1.5 seconds + while (System.nanoTime() - ts < 1_500_000_000L) { + LockSupport.parkNanos(1_000_000) + } countJfrs() > 0 when: @@ -89,7 +101,15 @@ class SpringBootNativeInstrumentationTest extends AbstractServerSmokeTest { @Override FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { if (file.toString().endsWith(".jfr")) { - jfrCount.incrementAndGet() + try { + IItemCollection events = JfrLoaderToolkit.loadEvents(file.toFile()) + if (events.apply(ItemFilters.type("jdk.ExecutionSample")).hasItems()) { + jfrCount.incrementAndGet() + return FileVisitResult.SKIP_SIBLINGS + } + } catch (InvalidJfrFileException ignored) { + // the recording captured at process exit might be incomplete + } } return FileVisitResult.CONTINUE } From 9bbee451cca38a007fa7c14416e54575a214457a Mon Sep 17 00:00:00 2001 From: Andrea Marziali Date: Tue, 5 Mar 2024 22:19:08 +0100 Subject: [PATCH 09/30] improve instrumentation doc (#6777) * improve instrumentation doc * Update docs/how_instrumentations_work.md * Update docs/how_instrumentations_work.md --------- Co-authored-by: Brian Marks --- docs/how_instrumentations_work.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/how_instrumentations_work.md b/docs/how_instrumentations_work.md index 32c44d341cc..19f57e24fdb 100644 --- a/docs/how_instrumentations_work.md +++ b/docs/how_instrumentations_work.md @@ -94,7 +94,10 @@ To run muzzle on your instrumentation, run: It checks that the types and methods used by the instrumentation are present in particular versions of libraries. It can be subverted with `MethodHandle` and reflection, so muzzle passing is not the end of the story. -**TODO: discuss why 'name' is not always included.** +By default, all the muzzle directives are checked against all the instrumentations included in a module. +However, there can be situations in which it’s only needed to check one specific directive on an instrumentation. +At this point the instrumentation should override the method `muzzleDirective()` by returning the name of the directive to execute. + ## Instrumentation classes @@ -527,6 +530,8 @@ It is important to understand that even though they look like maps, since the va retrieve a value if you use the exact same key object as when it was set. Using a different object that is “`.equals()`” to the first will yield nothing. +Since `ContextStore` does not support null keys, null checks must be enforced _before_ using an object as a key. + ## CallDepthThreadLocalMap In order to avoid activating new spans on recursive calls to the same method @@ -536,6 +541,8 @@ and [decremented](https://github.com/DataDog/dd-trace-java/blob/9d5c7ea524cfec98 or [reset](https://github.com/DataDog/dd-trace-java/blob/9d5c7ea524cfec982176e687a489fc8c2865e445/dd-java-agent/instrumentation/java-http-client/src/main/java11/datadog/trace/instrumentation/httpclient/SendAdvice.java#L44)) when exiting. +This only works if the methods are called on the same thread since the counter is a ThreadLocal variable. + ## Span Lifecycle In Advice classes, the `@Advice.OnMethodEnter` methods typically start spans and `@Advice.OnMethodExit` methods From 02d3cb7aac1beda6b2277e64cb78c183e358df85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Gonz=C3=A1lez=20Garc=C3=ADa?= Date: Wed, 6 Mar 2024 07:47:51 +0100 Subject: [PATCH 10/30] Update evidence-redaction-suite.yml and ignore not defined vulnerability type tests (#6774) --- .../model/json/EvidenceRedactionTest.groovy | 18 +- .../redaction/evidence-redaction-suite.yml | 171 ++++++++++++++++++ 2 files changed, 184 insertions(+), 5 deletions(-) diff --git a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/model/json/EvidenceRedactionTest.groovy b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/model/json/EvidenceRedactionTest.groovy index a0387af6b6a..26b5a8408d6 100644 --- a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/model/json/EvidenceRedactionTest.groovy +++ b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/model/json/EvidenceRedactionTest.groovy @@ -11,6 +11,7 @@ import datadog.trace.test.util.DDSpecification import groovy.json.JsonOutput import groovy.json.JsonSlurper import groovy.yaml.YamlSlurper +import org.junit.Assume import org.skyscreamer.jsonassert.JSONAssert import org.skyscreamer.jsonassert.JSONCompareMode import spock.lang.Shared @@ -76,6 +77,7 @@ class EvidenceRedactionTest extends DDSpecification { void 'test #suite'() { given: + Assume.assumeFalse("Ignored test", suite.ignored) final type = suite.type == Type.SOURCES ? Types.newParameterizedType(List, Source) : VulnerabilityBatch final adapter = VulnerabilityEncoding.MOSHI.adapter(type) @@ -133,12 +135,17 @@ class EvidenceRedactionTest extends DDSpecification { suite.input = sourcesParser.fromJson(input) break default: - final batch = new VulnerabilityBatch(vulnerabilities: vulnerabilitiesParser.fromJson(input)) - if (suite.context != null) { - final context = json.parseText(suite.context) as Map - batch.vulnerabilities.evidence.context.each { context.each(it.&put) } + try{ + final batch = new VulnerabilityBatch(vulnerabilities: vulnerabilitiesParser.fromJson(input)) + if (suite.context != null) { + final context = json.parseText(suite.context) as Map + batch.vulnerabilities.evidence.context.each { context.each(it.&put) } + } + suite.input = batch + }catch (Exception ex){ + suite.ignored = true + println "Failed to parse test ${ex.message}" } - suite.input = batch break } return suite @@ -198,6 +205,7 @@ class EvidenceRedactionTest extends DDSpecification { String context Object input String expected + boolean ignored @Override String toString() { diff --git a/dd-java-agent/agent-iast/src/test/resources/redaction/evidence-redaction-suite.yml b/dd-java-agent/agent-iast/src/test/resources/redaction/evidence-redaction-suite.yml index 8fcd41232a8..4c579100d64 100644 --- a/dd-java-agent/agent-iast/src/test/resources/redaction/evidence-redaction-suite.yml +++ b/dd-java-agent/agent-iast/src/test/resources/redaction/evidence-redaction-suite.yml @@ -1958,6 +1958,177 @@ suite: ] } + - type: 'VULNERABILITIES' + description: 'Mongodb json query with sensitive source' + input: > + [ + { + "type": "NOSQL_MONGODB_INJECTION", + "evidence": { + "value": "{\n \"password\": \"1234\"\n}", + "ranges": [ + { "start" : 17, "length" : 4, "source": { "origin": "http.request.parameter", "name": "password", "value": "1234" } } + ] + } + } + ] + expected: > + { + "sources": [ + { "origin": "http.request.parameter", "name": "password", "redacted": true, "pattern": "abcd" } + ], + "vulnerabilities": [ + { + "type": "NOSQL_MONGODB_INJECTION", + "evidence": { + "valueParts": [ + { "value": "{\n \"password\": \"" }, + { "source": 0, "redacted": true, "pattern": "abcd"}, + { "value": "\"\n}" } + ] + } + } + ] + } + + - type: 'VULNERABILITIES' + description: 'Mongodb json query with non sensitive source' + input: > + [ + { + "type": "NOSQL_MONGODB_INJECTION", + "evidence": { + "value": "{\n \"username\": \"user\"\n}", + "ranges": [ + { "start" : 17, "length" : 4, "source": { "origin": "http.request.parameter", "name": "username", "value": "user" } } + ] + } + } + ] + expected: > + { + "sources": [ + { "origin": "http.request.parameter", "name": "username", "redacted": true, "pattern": "abcd" } + ], + "vulnerabilities": [ + { + "type": "NOSQL_MONGODB_INJECTION", + "evidence": { + "valueParts": [ + { "value": "{\n \"username\": \"" }, + { "source": 0, "redacted": true, "pattern": "abcd"}, + { "value": "\"\n}" } + ] + } + } + ] + } + + - type: 'VULNERABILITIES' + description: 'Mongodb json query with partial non sensitive source' + input: > + [ + { + "type": "NOSQL_MONGODB_INJECTION", + "evidence": { + "value": "{\n \"username\": \"user\"\n}", + "ranges": [ + { "start" : 17, "length" : 4, "source": { "origin": "http.request.parameter", "name": "username", "value": "PREFIX_user" } } + ] + } + } + ] + expected: > + { + "sources": [ + { "origin": "http.request.parameter", "name": "username", "redacted": true, "pattern": "abcdefghijk" } + ], + "vulnerabilities": [ + { + "type": "NOSQL_MONGODB_INJECTION", + "evidence": { + "valueParts": [ + { "value": "{\n \"username\": \"" }, + { "source": 0, "redacted": true, "pattern": "hijk"}, + { "value": "\"\n}" } + ] + } + } + ] + } + + - type: 'VULNERABILITIES' + description: 'Mongodb json query with non sensitive source and other fields' + input: > + [ + { + "type": "NOSQL_MONGODB_INJECTION", + "evidence": { + "value": "{\n \"username\": \"user\",\n \"secret\": \"SECRET_VALUE\"\n}", + "ranges": [ + { "start" : 17, "length" : 4, "source": { "origin": "http.request.parameter", "name": "username", "value": "user" } } + ] + } + } + ] + expected: > + { + "sources": [ + { "origin": "http.request.parameter", "name": "username", "redacted": true, "pattern": "abcd" } + ], + "vulnerabilities": [ + { + "type": "NOSQL_MONGODB_INJECTION", + "evidence": { + "valueParts": [ + { "value": "{\n \"username\": \"" }, + { "source": 0, "redacted": true, "pattern": "abcd"}, + { "value": "\",\n \"secret\": \"" }, + { "redacted": true }, + { "value": "\"\n}" } + ] + } + } + ] + } + + - type: 'VULNERABILITIES' + description: 'Mongodb json query with sensitive value in a key' + input: > + [ + { + "type": "NOSQL_MONGODB_INJECTION", + "evidence": { + "value": "{\n \"username\": \"user\",\n \"token_usage\": {\n \"bearer zss8dR9QP81A\": 10\n }\n}", + "ranges": [ + { "start" : 17, "length" : 4, "source": { "origin": "http.request.parameter", "name": "username", "value": "user" } } + ] + } + } + ] + expected: > + { + "sources": [ + { "origin": "http.request.parameter", "name": "username", "redacted": true, "pattern": "abcd" } + ], + "vulnerabilities": [ + { + "type": "NOSQL_MONGODB_INJECTION", + "evidence": { + "valueParts": [ + { "value": "{\n \"username\": \"" }, + { "source": 0, "redacted": true, "pattern": "abcd"}, + { "value": "\",\n \"token_usage\": {\n \"" }, + { "redacted": true }, + { "value": "\": " }, + { "redacted": true }, + { "value": "\n }\n}" } + ] + } + } + ] + } + - type: 'VULNERABILITIES' description: 'Redacted source that needs to be truncated' input: > From 58e35cb1f82e6bc9e8c0e0b84083bd1ee6a3304f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20=C3=81lvarez=20=C3=81lvarez?= Date: Wed, 6 Mar 2024 13:06:58 +0100 Subject: [PATCH 11/30] Add support for kafka bytebuffers in 3.x (#6736) Add support for kafka bytebuffers in 3.x --- .../core/Json1FactoryInstrumentation.java | 34 +- .../Json1FactoryInstrumentationTest.groovy | 35 +- .../core/Json2FactoryInstrumentation.java | 29 +- .../Json2FactoryInstrumentationTest.groovy | 16 + .../kafka-clients-0.11/build.gradle | 19 +- .../iast/KafkaIastDeserializerTest.groovy | 252 +++++++++++++ .../KafkaDeserializerInstrumentation.java | 44 ++- .../kafka_clients/KafkaIastHelper.java | 56 ++- .../kafka_clients/UtilsInstrumentation.java | 85 +++++ .../KafkaIastDeserializerForkedTest.groovy | 8 +- .../UtilsInstrumentationForkedTest.groovy | 64 ++++ .../{kafka => kafka-2}/build.gradle | 2 +- .../smoketest/kafka/KafkaApplication.java | 0 .../kafka/iast/IastConfiguration.java | 332 ++++++++++++++++++ .../smoketest/kafka/iast/IastController.java | 165 +++++++++ .../smoketest/kafka/iast/IastMessage.java | 6 + .../test/groovy/IastKafka2SmokeTest.groovy} | 20 +- .../kafka-3/application/build.gradle | 31 ++ .../kafka-3/application/settings.gradle | 22 ++ .../smoketest/kafka/KafkaApplication.java | 16 + .../kafka/iast/IastConfiguration.java | 332 ++++++++++++++++++ .../smoketest/kafka/iast/IastController.java | 165 +++++++++ .../smoketest/kafka/iast/IastMessage.java | 20 ++ dd-smoke-tests/kafka-3/build.gradle | 53 +++ .../test/groovy/IastKafka3SmokeTest.groovy | 104 ++++++ .../kafka/iast/IastConfiguration.java | 92 ----- .../smoketest/kafka/iast/IastController.java | 87 ----- settings.gradle | 3 +- 28 files changed, 1885 insertions(+), 207 deletions(-) create mode 100644 dd-java-agent/instrumentation/kafka-clients-0.11/src/iastLatestDepTest3/groovy/iast/KafkaIastDeserializerTest.groovy create mode 100644 dd-java-agent/instrumentation/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/UtilsInstrumentation.java create mode 100644 dd-java-agent/instrumentation/kafka-clients-0.11/src/test/groovy/iast/UtilsInstrumentationForkedTest.groovy rename dd-smoke-tests/{kafka => kafka-2}/build.gradle (95%) rename dd-smoke-tests/{kafka => kafka-2}/src/main/java/datadog/smoketest/kafka/KafkaApplication.java (100%) create mode 100644 dd-smoke-tests/kafka-2/src/main/java/datadog/smoketest/kafka/iast/IastConfiguration.java create mode 100644 dd-smoke-tests/kafka-2/src/main/java/datadog/smoketest/kafka/iast/IastController.java rename dd-smoke-tests/{kafka => kafka-2}/src/main/java/datadog/smoketest/kafka/iast/IastMessage.java (68%) rename dd-smoke-tests/{kafka/src/test/groovy/IastKafkaSmokeTest.groovy => kafka-2/src/test/groovy/IastKafka2SmokeTest.groovy} (79%) create mode 100644 dd-smoke-tests/kafka-3/application/build.gradle create mode 100644 dd-smoke-tests/kafka-3/application/settings.gradle create mode 100644 dd-smoke-tests/kafka-3/application/src/main/java/datadog/smoketest/kafka/KafkaApplication.java create mode 100644 dd-smoke-tests/kafka-3/application/src/main/java/datadog/smoketest/kafka/iast/IastConfiguration.java create mode 100644 dd-smoke-tests/kafka-3/application/src/main/java/datadog/smoketest/kafka/iast/IastController.java create mode 100644 dd-smoke-tests/kafka-3/application/src/main/java/datadog/smoketest/kafka/iast/IastMessage.java create mode 100644 dd-smoke-tests/kafka-3/build.gradle create mode 100644 dd-smoke-tests/kafka-3/src/test/groovy/IastKafka3SmokeTest.groovy delete mode 100644 dd-smoke-tests/kafka/src/main/java/datadog/smoketest/kafka/iast/IastConfiguration.java delete mode 100644 dd-smoke-tests/kafka/src/main/java/datadog/smoketest/kafka/iast/IastController.java diff --git a/dd-java-agent/instrumentation/jackson-core/jackson-core-1/src/main/java/datadog/trace/instrumentation/jackson/codehouse/core/Json1FactoryInstrumentation.java b/dd-java-agent/instrumentation/jackson-core/jackson-core-1/src/main/java/datadog/trace/instrumentation/jackson/codehouse/core/Json1FactoryInstrumentation.java index 28671c45e03..69c72fe99b4 100644 --- a/dd-java-agent/instrumentation/jackson-core/jackson-core-1/src/main/java/datadog/trace/instrumentation/jackson/codehouse/core/Json1FactoryInstrumentation.java +++ b/dd-java-agent/instrumentation/jackson-core/jackson-core-1/src/main/java/datadog/trace/instrumentation/jackson/codehouse/core/Json1FactoryInstrumentation.java @@ -1,13 +1,16 @@ package datadog.trace.instrumentation.jackson.codehouse.core; +import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; +import static datadog.trace.api.iast.VulnerabilityMarks.NOT_MARKED; import static net.bytebuddy.matcher.ElementMatchers.isMethod; +import static net.bytebuddy.matcher.ElementMatchers.isPublic; import static net.bytebuddy.matcher.ElementMatchers.takesArguments; import com.google.auto.service.AutoService; import datadog.trace.agent.tooling.Instrumenter; import datadog.trace.agent.tooling.InstrumenterModule; -import datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers; import datadog.trace.api.iast.InstrumentationBridge; +import datadog.trace.api.iast.Propagation; import datadog.trace.api.iast.Sink; import datadog.trace.api.iast.VulnerabilityTypes; import datadog.trace.api.iast.propagation.PropagationModule; @@ -16,7 +19,6 @@ import java.io.Reader; import java.net.URL; import net.bytebuddy.asm.Advice; -import net.bytebuddy.description.method.MethodDescription; @AutoService(Instrumenter.class) public class Json1FactoryInstrumentation extends InstrumenterModule.Iast @@ -29,14 +31,20 @@ public Json1FactoryInstrumentation() { @Override public void methodAdvice(MethodTransformer transformer) { transformer.applyAdvice( - NameMatchers.named("createJsonParser") + named("createJsonParser") .and(isMethod()) .and( takesArguments(String.class) .or(takesArguments(InputStream.class)) .or(takesArguments(Reader.class)) - .or(takesArguments(URL.class))), + .or(takesArguments(URL.class)) + .or(takesArguments(byte[].class))), Json1FactoryInstrumentation.class.getName() + "$InstrumenterAdvice"); + transformer.applyAdvice( + named("createJsonParser") + .and(isMethod()) + .and(isPublic().and(takesArguments(byte[].class, int.class, int.class))), + Json1FactoryInstrumentation.class.getName() + "$Instrumenter2Advice"); } @Override @@ -64,4 +72,22 @@ public static void onExit( } } } + + public static class Instrumenter2Advice { + + @Advice.OnMethodExit(suppress = Throwable.class) + @Propagation + public static void onExit( + @Advice.Argument(0) final byte[] input, + @Advice.Argument(1) final int offset, + @Advice.Argument(2) final int length, + @Advice.Return final Object parser) { + if (input != null || length <= 0) { + final PropagationModule propagation = InstrumentationBridge.PROPAGATION; + if (propagation != null) { + propagation.taintIfTainted(parser, input, offset, length, false, NOT_MARKED); + } + } + } + } } diff --git a/dd-java-agent/instrumentation/jackson-core/jackson-core-1/src/test/groovy/Json1FactoryInstrumentationTest.groovy b/dd-java-agent/instrumentation/jackson-core/jackson-core-1/src/test/groovy/Json1FactoryInstrumentationTest.groovy index 287967222f3..839d12baf30 100644 --- a/dd-java-agent/instrumentation/jackson-core/jackson-core-1/src/test/groovy/Json1FactoryInstrumentationTest.groovy +++ b/dd-java-agent/instrumentation/jackson-core/jackson-core-1/src/test/groovy/Json1FactoryInstrumentationTest.groovy @@ -1,6 +1,7 @@ import datadog.trace.agent.test.AgentTestRunner import datadog.trace.agent.test.server.http.TestHttpServer import datadog.trace.api.iast.InstrumentationBridge +import datadog.trace.api.iast.VulnerabilityMarks import datadog.trace.api.iast.propagation.PropagationModule import datadog.trace.api.iast.sink.SsrfModule import org.codehaus.jackson.JsonFactory @@ -56,7 +57,7 @@ class Json1FactoryInstrumentationTest extends AgentTestRunner { then: result != null 1 * propagationModule.taintIfTainted(_ as JsonParser, is) - 2 * is.read(_,_,_) + 2 * is.read(_, _, _) 0 * _ } @@ -92,4 +93,36 @@ class Json1FactoryInstrumentationTest extends AgentTestRunner { 1 * ssrfModule.onURLConnection(url) 0 * _ } + + + void 'test createParser(byte[])'() { + setup: + final propagationModule = Mock(PropagationModule) + InstrumentationBridge.registerIastModule(propagationModule) + final bytes = '{}'.bytes + + when: + final result = new JsonFactory().createJsonParser(bytes) + + + then: + result != null + 1 * propagationModule.taintIfTainted(_ as JsonParser, bytes) + 0 * _ + } + + void 'test createParser(byte[], int, int)'() { + setup: + final propagationModule = Mock(PropagationModule) + InstrumentationBridge.registerIastModule(propagationModule) + final bytes = '{}'.bytes + + when: + final parser = new JsonFactory().createJsonParser(bytes, 0, 2) + + then: + parser != null + 1 * propagationModule.taintIfTainted(_ as JsonParser, bytes, 0, 2, false, VulnerabilityMarks.NOT_MARKED) + 0 * _ + } } diff --git a/dd-java-agent/instrumentation/jackson-core/src/main/java/datadog/trace/instrumentation/jackson/core/Json2FactoryInstrumentation.java b/dd-java-agent/instrumentation/jackson-core/src/main/java/datadog/trace/instrumentation/jackson/core/Json2FactoryInstrumentation.java index 6d8a62c84fb..35f2f4c0b4b 100644 --- a/dd-java-agent/instrumentation/jackson-core/src/main/java/datadog/trace/instrumentation/jackson/core/Json2FactoryInstrumentation.java +++ b/dd-java-agent/instrumentation/jackson-core/src/main/java/datadog/trace/instrumentation/jackson/core/Json2FactoryInstrumentation.java @@ -1,12 +1,16 @@ package datadog.trace.instrumentation.jackson.core; import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; -import static net.bytebuddy.matcher.ElementMatchers.*; +import static datadog.trace.api.iast.VulnerabilityMarks.NOT_MARKED; +import static net.bytebuddy.matcher.ElementMatchers.isMethod; +import static net.bytebuddy.matcher.ElementMatchers.isPublic; +import static net.bytebuddy.matcher.ElementMatchers.takesArguments; import com.google.auto.service.AutoService; import datadog.trace.agent.tooling.Instrumenter; import datadog.trace.agent.tooling.InstrumenterModule; import datadog.trace.api.iast.InstrumentationBridge; +import datadog.trace.api.iast.Propagation; import datadog.trace.api.iast.Sink; import datadog.trace.api.iast.VulnerabilityTypes; import datadog.trace.api.iast.propagation.PropagationModule; @@ -38,6 +42,11 @@ public void methodAdvice(MethodTransformer transformer) { .or(takesArguments(URL.class)) .or(takesArguments(byte[].class)))), Json2FactoryInstrumentation.class.getName() + "$InstrumenterAdvice"); + transformer.applyAdvice( + named("createParser") + .and(isMethod()) + .and(isPublic().and(takesArguments(byte[].class, int.class, int.class))), + Json2FactoryInstrumentation.class.getName() + "$Instrumenter2Advice"); } @Override @@ -65,4 +74,22 @@ public static void onExit( } } } + + public static class Instrumenter2Advice { + + @Advice.OnMethodExit(suppress = Throwable.class) + @Propagation + public static void onExit( + @Advice.Argument(0) final byte[] input, + @Advice.Argument(1) final int offset, + @Advice.Argument(2) final int length, + @Advice.Return final Object parser) { + if (input != null || length <= 0) { + final PropagationModule propagation = InstrumentationBridge.PROPAGATION; + if (propagation != null) { + propagation.taintIfTainted(parser, input, offset, length, false, NOT_MARKED); + } + } + } + } } diff --git a/dd-java-agent/instrumentation/jackson-core/src/test/groovy/Json2FactoryInstrumentationTest.groovy b/dd-java-agent/instrumentation/jackson-core/src/test/groovy/Json2FactoryInstrumentationTest.groovy index e3edc9eb81e..6f602c1c8e4 100644 --- a/dd-java-agent/instrumentation/jackson-core/src/test/groovy/Json2FactoryInstrumentationTest.groovy +++ b/dd-java-agent/instrumentation/jackson-core/src/test/groovy/Json2FactoryInstrumentationTest.groovy @@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.ObjectMapper import datadog.trace.agent.test.AgentTestRunner import datadog.trace.agent.test.server.http.TestHttpServer import datadog.trace.api.iast.InstrumentationBridge +import datadog.trace.api.iast.VulnerabilityMarks import datadog.trace.api.iast.propagation.PropagationModule import datadog.trace.api.iast.sink.SsrfModule import groovy.transform.CompileDynamic @@ -96,6 +97,21 @@ class Json2FactoryInstrumentationTest extends AgentTestRunner { 0 * _ } + void 'test createParser(byte[], int, int)'() { + setup: + final propagationModule = Mock(PropagationModule) + InstrumentationBridge.registerIastModule(propagationModule) + final bytes = '{}'.bytes + + when: + final parser = new JsonFactory().createParser(bytes, 0, 2) + + then: + parser != null + 1 * propagationModule.taintIfTainted(_ as JsonParser, bytes, 0, 2, false, VulnerabilityMarks.NOT_MARKED) + 0 * _ + } + void 'test createParser(URL)'() { setup: final propagationModule = Mock(PropagationModule) diff --git a/dd-java-agent/instrumentation/kafka-clients-0.11/build.gradle b/dd-java-agent/instrumentation/kafka-clients-0.11/build.gradle index 1eb017a6310..b03714d31ec 100644 --- a/dd-java-agent/instrumentation/kafka-clients-0.11/build.gradle +++ b/dd-java-agent/instrumentation/kafka-clients-0.11/build.gradle @@ -10,6 +10,7 @@ muzzle { apply from: "$rootDir/gradle/java.gradle" addTestSuite('latestDepTest') +addTestSuite('iastLatestDepTest3') dependencies { compileOnly group: 'org.apache.kafka', name: 'kafka-clients', version: '0.11.0.0' @@ -28,7 +29,7 @@ dependencies { testRuntimeOnly project(':dd-java-agent:instrumentation:java-lang') testRuntimeOnly project(':dd-java-agent:instrumentation:java-io') testRuntimeOnly project(':dd-java-agent:instrumentation:jackson-core') - testRuntimeOnly(group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.9.10') + testImplementation(group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.9.10') // Include latest version of kafka itself along with latest version of client libs. // This seems to help with jar compatibility hell. @@ -38,9 +39,25 @@ dependencies { latestDepTestImplementation group: 'org.springframework.kafka', name: 'spring-kafka-test', version: '2.+' latestDepTestImplementation group: 'org.assertj', name: 'assertj-core', version: '3.19.+' latestDepTestImplementation deps.guava + + // Add kafka version 3.x for IAST + iastLatestDepTest3Implementation group: 'org.apache.kafka', name: 'kafka-clients', version: '3.+' + iastLatestDepTest3Implementation group: 'org.springframework.kafka', name: 'spring-kafka', version: '3.+' + iastLatestDepTest3RuntimeOnly project(':dd-java-agent:instrumentation:iast-instrumenter') + iastLatestDepTest3RuntimeOnly project(':dd-java-agent:instrumentation:java-lang') + iastLatestDepTest3RuntimeOnly project(':dd-java-agent:instrumentation:java-io') + iastLatestDepTest3RuntimeOnly project(':dd-java-agent:instrumentation:jackson-core') + iastLatestDepTest3Implementation(group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.15.3') } configurations.testRuntimeClasspath { // spock-core depends on assertj version that is not compatible with kafka-clients resolutionStrategy.force 'org.assertj:assertj-core:2.9.1' } + +iastLatestDepTest3.configure { + javaLauncher = getJavaLauncherFor(17) + jvmArgs = ['--add-opens', 'java.base/java.util=ALL-UNNAMED'] +} + + diff --git a/dd-java-agent/instrumentation/kafka-clients-0.11/src/iastLatestDepTest3/groovy/iast/KafkaIastDeserializerTest.groovy b/dd-java-agent/instrumentation/kafka-clients-0.11/src/iastLatestDepTest3/groovy/iast/KafkaIastDeserializerTest.groovy new file mode 100644 index 00000000000..43ccb4f069d --- /dev/null +++ b/dd-java-agent/instrumentation/kafka-clients-0.11/src/iastLatestDepTest3/groovy/iast/KafkaIastDeserializerTest.groovy @@ -0,0 +1,252 @@ +package iast + +import com.fasterxml.jackson.core.JsonParser +import datadog.trace.agent.test.AgentTestRunner +import datadog.trace.api.iast.InstrumentationBridge +import datadog.trace.api.iast.SourceTypes +import datadog.trace.api.iast.Taintable +import datadog.trace.api.iast.propagation.CodecModule +import datadog.trace.api.iast.propagation.PropagationModule +import org.apache.kafka.common.header.internals.RecordHeaders +import org.apache.kafka.common.serialization.ByteArrayDeserializer +import org.apache.kafka.common.serialization.ByteBufferDeserializer +import org.apache.kafka.common.serialization.Deserializer +import org.apache.kafka.common.serialization.StringDeserializer +import org.springframework.kafka.support.serializer.JsonDeserializer + +import java.nio.ByteBuffer + +import static datadog.trace.api.iast.VulnerabilityMarks.NOT_MARKED + +class KafkaIastDeserializerTest extends AgentTestRunner { + + private static final int BUFF_OFFSET = 10 + + @Override + protected void configurePreAgent() { + injectSysConfig('dd.iast.enabled', 'true') + } + + void 'test string deserializer: #test'() { + given: + final source = test.source + final propagationModule = Mock(PropagationModule) + final codecModule = Mock(CodecModule) + [propagationModule, codecModule].each { InstrumentationBridge.registerIastModule(it) } + + and: + final payload = "Hello World!".bytes + final deserializer = new StringDeserializer() + + when: + deserializer.configure([:], source == SourceTypes.KAFKA_MESSAGE_KEY) + test.method.deserialize(deserializer, "test", payload) + + then: + switch (test.method) { + case Method.DEFAULT: + 1 * propagationModule.taint(payload, source) // taint byte[] + 1 * codecModule.onStringFromBytes(payload, 0, payload.length, _, _ as String) // taint byte[] => string + break + case Method.WITH_HEADERS: + 1 * propagationModule.taint(payload, source) // taint byte[] + 1 * codecModule.onStringFromBytes(payload, 0, payload.length, _, _ as String) // taint byte[] => string + break + case Method.WITH_BYTE_BUFFER: + 1 * propagationModule.taint(_ as ByteBuffer, source, 0, payload.length) // taint ByteBuffer + 1 * propagationModule.taintIfTainted(payload, _ as ByteBuffer, 0, payload.length, false, NOT_MARKED) // taint ByteBuffer => byte[] + 1 * codecModule.onStringFromBytes(payload, 0, payload.length, _, _ as String) // taint byte[] => string + break + case Method.WITH_BYTE_BUFFER_OFFSET: + 1 * propagationModule.taint(_ as ByteBuffer, source, BUFF_OFFSET, payload.length) // taint ByteBuffer + 1 * propagationModule.taintIfTainted(_ as byte[], _ as ByteBuffer, true, NOT_MARKED) // taint ByteBuffer => byte[] + 1 * codecModule.onStringFromBytes(_ as byte[], BUFF_OFFSET, payload.length, _, _ as String) // taint byte[] => string + break + } + 0 * _ + + where: + test << testSuite() + } + + void 'test byte array deserializer: #test'() { + given: + final source = test.source + final propagationModule = Mock(PropagationModule) + InstrumentationBridge.registerIastModule(propagationModule) + + and: + final payload = "Hello World!".bytes + final deserializer = new ByteArrayDeserializer() + + when: + deserializer.configure([:], source == SourceTypes.KAFKA_MESSAGE_KEY) + test.method.deserialize(deserializer, "test", payload) + + then: + switch (test.method) { + case Method.DEFAULT: + 1 * propagationModule.taint(payload, source) // taint byte[] + break + case Method.WITH_HEADERS: + 1 * propagationModule.taint(payload, source) // taint byte[] + break + case Method.WITH_BYTE_BUFFER: + 1 * propagationModule.taint(_ as ByteBuffer, source, 0, payload.length) // taint ByteBuffer + 1 * propagationModule.taintIfTainted(payload, _ as ByteBuffer, 0, payload.length, false, NOT_MARKED) // taint ByteBuffer => byte[] + break + case Method.WITH_BYTE_BUFFER_OFFSET: + 1 * propagationModule.taint(_ as ByteBuffer, source, BUFF_OFFSET, payload.length) // taint ByteBuffer + 1 * propagationModule.taintIfTainted(payload, _ as ByteBuffer, BUFF_OFFSET, payload.length, false, NOT_MARKED) // taint ByteBuffer => byte[] + break + } + 0 * _ + + where: + test << testSuite() + } + + void 'test byte buffer deserializer: #test'() { + given: + final source = test.source + final propagationModule = Mock(PropagationModule) + InstrumentationBridge.registerIastModule(propagationModule) + + and: + final payload = "Hello World!".bytes + final deserializer = new ByteBufferDeserializer() + + when: + deserializer.configure([:], source == SourceTypes.KAFKA_MESSAGE_KEY) + test.method.deserialize(deserializer, "test", payload) + + then: + switch (test.method) { + case Method.DEFAULT: + 1 * propagationModule.taint(payload, source) // taint byte[] + 1 * propagationModule.taintIfTainted(_ as ByteBuffer, payload, true, NOT_MARKED) // taint byte[] => ByteBuffer + break + case Method.WITH_HEADERS: + 1 * propagationModule.taint(payload, source) // taint byte[] + 1 * propagationModule.taintIfTainted(_ as ByteBuffer, payload, true, NOT_MARKED) // taint byte[] => ByteBuffer + break + case Method.WITH_BYTE_BUFFER: + 1 * propagationModule.taint(_ as ByteBuffer, source, 0, payload.length) // taint ByteBuffer + break + case Method.WITH_BYTE_BUFFER_OFFSET: + 1 * propagationModule.taint(_ as ByteBuffer, source, BUFF_OFFSET, payload.length) // taint ByteBuffer + break + } + 0 * _ + + where: + test << testSuite() + } + + void 'test json deserialization: #test'() { + given: + final source = test.source + final propagationModule = Mock(PropagationModule) + InstrumentationBridge.registerIastModule(propagationModule) + + and: + final json = '{ "name": "Mr Bean" }' + final payload = json.bytes + final deserializer = new JsonDeserializer(TestBean) + + when: + deserializer.configure([:], source == SourceTypes.KAFKA_MESSAGE_KEY) + test.method.deserialize(deserializer, 'test', payload) + + then: + switch (test.method) { + case Method.DEFAULT: + 1 * propagationModule.taint(payload, source) // taint byte[] + break + case Method.WITH_HEADERS: + 1 * propagationModule.taint(payload, source) // taint byte[] + break + case Method.WITH_BYTE_BUFFER: + 1 * propagationModule.taint(_ as ByteBuffer, source, 0, payload.length) // taint ByteBuffer + 1 * propagationModule.taintIfTainted(payload, _ as ByteBuffer, 0, payload.length, false, NOT_MARKED) // taint byte[] => ByteBuffer + break + case Method.WITH_BYTE_BUFFER_OFFSET: + 1 * propagationModule.taint(_ as ByteBuffer, source, BUFF_OFFSET, payload.length) // taint ByteBuffer + 1 * propagationModule.taintIfTainted(payload, _ as ByteBuffer, BUFF_OFFSET, payload.length, false, NOT_MARKED) // taint byte[] => ByteBuffer + break + } + // taint JSON + 1 * propagationModule.taintIfTainted(_ as JsonParser, payload) + 1 * propagationModule.findSource(_) >> Stub(Taintable.Source) { + getOrigin() >> source + getValue() >> json + } + 1 * propagationModule.taint(_, 'name', source, 'name', json) + 1 * propagationModule.taint(_, 'Mr Bean', source, 'name', json) + 0 * _ + + where: + test << testSuite() + } + + private static List testSuite() { + return [SourceTypes.KAFKA_MESSAGE_KEY, SourceTypes.KAFKA_MESSAGE_VALUE].collectMany { source -> + return [ + new Suite(source: source, method: Method.DEFAULT), + new Suite(source: source, method: Method.WITH_HEADERS), + new Suite(source: source, method: Method.WITH_BYTE_BUFFER), + new Suite(source: source, method: Method.WITH_BYTE_BUFFER_OFFSET) + ] + } + } + + enum Method { + DEFAULT{ + @Override + T deserialize(Deserializer deserializer, String topic, byte[] payload) { + return deserializer.deserialize(topic, payload) + } + }, + WITH_HEADERS{ + @Override + T deserialize(Deserializer deserializer, String topic, byte[] payload) { + return deserializer.deserialize(topic, new RecordHeaders(), payload) + } + }, + WITH_BYTE_BUFFER{ + @SuppressWarnings('GroovyAssignabilityCheck') + @Override + T deserialize(Deserializer deserializer, String topic, byte[] payload) { + ByteBuffer buffer = ByteBuffer.allocateDirect(payload.length) + buffer.put(payload) + buffer.position(0) + return deserializer.deserialize(topic, new RecordHeaders(), buffer) + } + }, + WITH_BYTE_BUFFER_OFFSET{ + @SuppressWarnings('GroovyAssignabilityCheck') + @Override + T deserialize(Deserializer deserializer, String topic, byte[] payload) { + final byte[] buffer = new byte[payload.length + BUFF_OFFSET] + System.arraycopy(payload, 0, buffer, BUFF_OFFSET, payload.length) + return deserializer.deserialize(topic, new RecordHeaders(), ByteBuffer.wrap(buffer, BUFF_OFFSET, payload.length)) + } + } + + abstract T deserialize(Deserializer deserializer, String topic, byte[] payload) + } + + static class Suite { + byte source + Method method + + @Override + String toString() { + return "${method.name()}: ${SourceTypes.toString(source)}" + } + } + + static class TestBean { + String name + } +} diff --git a/dd-java-agent/instrumentation/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaDeserializerInstrumentation.java b/dd-java-agent/instrumentation/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaDeserializerInstrumentation.java index 65dc65f7090..f8c81679a3b 100644 --- a/dd-java-agent/instrumentation/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaDeserializerInstrumentation.java +++ b/dd-java-agent/instrumentation/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaDeserializerInstrumentation.java @@ -1,6 +1,6 @@ package datadog.trace.instrumentation.kafka_clients; -import static datadog.trace.agent.tooling.bytebuddy.matcher.HierarchyMatchers.implementsInterface; +import static datadog.trace.agent.tooling.bytebuddy.matcher.HierarchyMatchers.hasInterface; import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; import static java.util.Collections.singletonMap; import static net.bytebuddy.matcher.ElementMatchers.takesArgument; @@ -14,6 +14,7 @@ import datadog.trace.api.iast.SourceTypes; import datadog.trace.bootstrap.ContextStore; import datadog.trace.bootstrap.InstrumentationContext; +import java.nio.ByteBuffer; import java.util.Map; import net.bytebuddy.asm.Advice; import net.bytebuddy.description.type.TypeDescription; @@ -28,7 +29,7 @@ public class KafkaDeserializerInstrumentation extends InstrumenterModule.Iast "org.apache.kafka.common.serialization.Deserializer"; /** Ensure same compatibility as the tracer */ - private static final Reference[] MUZZLE_CHECK = { + static final Reference[] MUZZLE_CHECK = { new Reference.Builder("org.apache.kafka.clients.consumer.ConsumerRecord") .withMethod( new String[0], @@ -49,7 +50,7 @@ public String hierarchyMarkerType() { @Override public ElementMatcher hierarchyMatcher() { - return implementsInterface(named(hierarchyMarkerType())); + return hasInterface(named(hierarchyMarkerType())); } @Override @@ -74,9 +75,14 @@ public void methodAdvice(final MethodTransformer transformer) { named("configure").and(takesArguments(Map.class, boolean.class)), baseName + "$ConfigureAdvice"); transformer.applyAdvice( - named("deserialize").and(takesArgument(1, byte[].class)), baseName + "$Deserialize2Advice"); + named("deserialize").and(takesArguments(2)).and(takesArgument(1, byte[].class)), + baseName + "$Deserialize2Advice"); transformer.applyAdvice( - named("deserialize").and(takesArgument(2, byte[].class)), baseName + "$Deserialize3Advice"); + named("deserialize").and(takesArguments(3)).and(takesArgument(2, byte[].class)), + baseName + "$Deserialize3Advice"); + transformer.applyAdvice( + named("deserialize").and(takesArguments(3)).and(takesArgument(2, ByteBuffer.class)), + baseName + "$DeserializeByteBufferAdvice"); } @SuppressWarnings("rawtypes") @@ -101,6 +107,11 @@ public static void deserialize( InstrumentationContext.get(Deserializer.class, Boolean.class); KafkaIastHelper.taint(store, deserializer, data); } + + @Advice.OnMethodExit(suppress = Throwable.class, onThrowable = Throwable.class) + public static void afterDeserialize() { + KafkaIastHelper.afterDeserialize(); + } } @SuppressWarnings("rawtypes") @@ -114,5 +125,28 @@ public static void deserialize( InstrumentationContext.get(Deserializer.class, Boolean.class); KafkaIastHelper.taint(store, deserializer, data); } + + @Advice.OnMethodExit(suppress = Throwable.class, onThrowable = Throwable.class) + public static void afterDeserialize() { + KafkaIastHelper.afterDeserialize(); + } + } + + @SuppressWarnings("rawtypes") + public static class DeserializeByteBufferAdvice { + + @Source(SourceTypes.KAFKA_MESSAGE) + @Advice.OnMethodEnter(suppress = Throwable.class) + public static void deserialize( + @Advice.This final Deserializer deserializer, @Advice.Argument(2) ByteBuffer data) { + final ContextStore store = + InstrumentationContext.get(Deserializer.class, Boolean.class); + KafkaIastHelper.taint(store, deserializer, data); + } + + @Advice.OnMethodExit(suppress = Throwable.class, onThrowable = Throwable.class) + public static void afterDeserialize() { + KafkaIastHelper.afterDeserialize(); + } } } diff --git a/dd-java-agent/instrumentation/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaIastHelper.java b/dd-java-agent/instrumentation/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaIastHelper.java index bf08bf63bb4..2b3c29d3bed 100644 --- a/dd-java-agent/instrumentation/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaIastHelper.java +++ b/dd-java-agent/instrumentation/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaIastHelper.java @@ -5,7 +5,9 @@ import datadog.trace.api.iast.InstrumentationBridge; import datadog.trace.api.iast.propagation.PropagationModule; +import datadog.trace.bootstrap.CallDepthThreadLocalMap; import datadog.trace.bootstrap.ContextStore; +import java.nio.ByteBuffer; import org.apache.kafka.common.serialization.Deserializer; @SuppressWarnings("rawtypes") @@ -23,15 +25,53 @@ public static void configure( public static void taint( final ContextStore store, final Deserializer deserializer, - final byte[] data) { + final Object data) { + if (CallDepthThreadLocalMap.incrementCallDepth(Deserializer.class) > 0) { + return; + } + if (data == null) { + return; + } final PropagationModule module = InstrumentationBridge.PROPAGATION; - if (module != null) { - byte source = KAFKA_MESSAGE_VALUE; - if (store != null) { - final Boolean key = store.get(deserializer); - source = key != null && key ? KAFKA_MESSAGE_KEY : KAFKA_MESSAGE_VALUE; - } - module.taint(data, source); + if (module == null) { + return; + } + final byte source = getSource(store, deserializer); + module.taint(data, source); + } + + public static void taint( + final ContextStore store, + final Deserializer deserializer, + final ByteBuffer data) { + if (CallDepthThreadLocalMap.incrementCallDepth(Deserializer.class) > 0) { + return; + } + if (data == null || data.remaining() == 0) { + return; + } + final PropagationModule module = InstrumentationBridge.PROPAGATION; + if (module == null) { + return; + } + final byte source = getSource(store, deserializer); + int start = data.position(); + if (data.hasArray()) { + start += data.arrayOffset(); + } + module.taint(data, source, start, data.remaining()); + } + + public static void afterDeserialize() { + CallDepthThreadLocalMap.decrementCallDepth(Deserializer.class); + } + + private static byte getSource( + final ContextStore store, final Deserializer deserializer) { + if (store == null) { + return KAFKA_MESSAGE_VALUE; } + final Boolean key = store.get(deserializer); + return key != null && key ? KAFKA_MESSAGE_KEY : KAFKA_MESSAGE_VALUE; } } diff --git a/dd-java-agent/instrumentation/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/UtilsInstrumentation.java b/dd-java-agent/instrumentation/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/UtilsInstrumentation.java new file mode 100644 index 00000000000..fe2fa602f16 --- /dev/null +++ b/dd-java-agent/instrumentation/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/UtilsInstrumentation.java @@ -0,0 +1,85 @@ +package datadog.trace.instrumentation.kafka_clients; + +import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; +import static datadog.trace.api.iast.VulnerabilityMarks.NOT_MARKED; +import static datadog.trace.instrumentation.kafka_clients.KafkaDeserializerInstrumentation.MUZZLE_CHECK; +import static net.bytebuddy.matcher.ElementMatchers.takesArguments; + +import com.google.auto.service.AutoService; +import datadog.trace.agent.tooling.Instrumenter; +import datadog.trace.agent.tooling.InstrumenterModule; +import datadog.trace.agent.tooling.muzzle.Reference; +import datadog.trace.api.iast.InstrumentationBridge; +import datadog.trace.api.iast.Propagation; +import datadog.trace.api.iast.propagation.PropagationModule; +import java.nio.ByteBuffer; +import net.bytebuddy.asm.Advice; + +@AutoService(Instrumenter.class) +public class UtilsInstrumentation extends InstrumenterModule.Iast + implements Instrumenter.ForSingleType { + + public UtilsInstrumentation() { + super("kafka"); + } + + @Override + public Reference[] additionalMuzzleReferences() { + return MUZZLE_CHECK; + } + + @Override + public String instrumentedType() { + return "org.apache.kafka.common.utils.Utils"; + } + + @Override + public void methodAdvice(final MethodTransformer transformer) { + final String baseName = UtilsInstrumentation.class.getName(); + transformer.applyAdvice( + named("toArray").and(takesArguments(ByteBuffer.class, int.class, int.class)), + baseName + "$ToArrayAdvice"); + transformer.applyAdvice( + named("wrapNullable").and(takesArguments(byte[].class)), baseName + "$WrapAdvice"); + } + + public static class ToArrayAdvice { + @Advice.OnMethodExit(suppress = Throwable.class) + @Propagation + public static void toArray( + @Advice.Argument(0) final ByteBuffer buffer, + @Advice.Argument(1) final int offset, + @Advice.Argument(2) final int length, + @Advice.Return final byte[] bytes) { + if (buffer == null || bytes == null || bytes.length == 0) { + return; + } + final PropagationModule propagation = InstrumentationBridge.PROPAGATION; + if (propagation == null) { + return; + } + int start = buffer.position() + offset; + if (buffer.hasArray()) { + start += buffer.arrayOffset(); + } + // create a new range shifted to the result byte array coordinates + propagation.taintIfTainted(bytes, buffer, start, length, false, NOT_MARKED); + } + } + + public static class WrapAdvice { + @Advice.OnMethodExit(suppress = Throwable.class) + @Propagation + public static void wrapNullable( + @Advice.Argument(0) final byte[] bytes, @Advice.Return final ByteBuffer buffer) { + if (buffer == null || bytes == null || bytes.length == 0) { + return; + } + final PropagationModule propagation = InstrumentationBridge.PROPAGATION; + if (propagation == null) { + return; + } + propagation.taintIfTainted(buffer, bytes, true, NOT_MARKED); + } + } +} diff --git a/dd-java-agent/instrumentation/kafka-clients-0.11/src/test/groovy/iast/KafkaIastDeserializerForkedTest.groovy b/dd-java-agent/instrumentation/kafka-clients-0.11/src/test/groovy/iast/KafkaIastDeserializerForkedTest.groovy index e716b376385..15729faf655 100644 --- a/dd-java-agent/instrumentation/kafka-clients-0.11/src/test/groovy/iast/KafkaIastDeserializerForkedTest.groovy +++ b/dd-java-agent/instrumentation/kafka-clients-0.11/src/test/groovy/iast/KafkaIastDeserializerForkedTest.groovy @@ -1,9 +1,11 @@ package iast +import com.fasterxml.jackson.core.JsonParser import datadog.trace.agent.test.AgentTestRunner import datadog.trace.api.iast.InstrumentationBridge import datadog.trace.api.iast.SourceTypes import datadog.trace.api.iast.Taintable.Source +import datadog.trace.api.iast.VulnerabilityMarks import datadog.trace.api.iast.propagation.CodecModule import datadog.trace.api.iast.propagation.PropagationModule import org.apache.kafka.common.serialization.ByteArrayDeserializer @@ -34,7 +36,7 @@ class KafkaIastDeserializerForkedTest extends AgentTestRunner { then: 1 * propagationModule.taint(payload, source) - 1 * codecModule.onStringFromBytes(payload, _, _) + 1 * codecModule.onStringFromBytes(payload, _, _, _, _) 0 * _ where: @@ -81,7 +83,7 @@ class KafkaIastDeserializerForkedTest extends AgentTestRunner { then: 1 * propagationModule.taint(payload, source) - 1 * propagationModule.taintIfTainted(_, payload) + 1 * propagationModule.taintIfTainted(_, payload, true, VulnerabilityMarks.NOT_MARKED) 0 * _ where: @@ -106,7 +108,7 @@ class KafkaIastDeserializerForkedTest extends AgentTestRunner { then: 1 * propagationModule.taint(payload, source) - 1 * propagationModule.taintIfTainted(_, payload) + 1 * propagationModule.taintIfTainted(_ as JsonParser, payload) 1 * propagationModule.findSource(_) >> Stub(Source) { getOrigin() >> source getValue() >> json diff --git a/dd-java-agent/instrumentation/kafka-clients-0.11/src/test/groovy/iast/UtilsInstrumentationForkedTest.groovy b/dd-java-agent/instrumentation/kafka-clients-0.11/src/test/groovy/iast/UtilsInstrumentationForkedTest.groovy new file mode 100644 index 00000000000..d3332fc6105 --- /dev/null +++ b/dd-java-agent/instrumentation/kafka-clients-0.11/src/test/groovy/iast/UtilsInstrumentationForkedTest.groovy @@ -0,0 +1,64 @@ +package iast + +import datadog.trace.agent.test.AgentTestRunner +import datadog.trace.api.iast.InstrumentationBridge +import datadog.trace.api.iast.VulnerabilityMarks +import datadog.trace.api.iast.propagation.PropagationModule +import org.apache.kafka.common.utils.Utils + +import java.nio.ByteBuffer + +class UtilsInstrumentationForkedTest extends AgentTestRunner { + + @Override + protected void configurePreAgent() { + injectSysConfig('dd.iast.enabled', 'true') + } + + void 'test toArray'() { + setup: + final propagationModule = Mock(PropagationModule) + InstrumentationBridge.registerIastModule(propagationModule) + final buffer = args[0] + + when: + final bytes = Utils.&"$method".call(args as Object[]) + + then: + bytes != null + 1 * propagationModule.taintIfTainted(_ as byte[], buffer, offset, length, false, VulnerabilityMarks.NOT_MARKED) + 0 * _ + + where: + method | offset | length | args | _ + 'toNullableArray' | 0 | 12 | [ByteBuffer.wrap('Hello World!'.bytes)] | _ + 'toArray' | 0 | 12 | [ByteBuffer.wrap('Hello World!'.bytes)] | _ + 'toArray' | 0 | 3 | [ByteBuffer.wrap('Hello World!'.bytes), length] | _ + 'toArray' | 0 | 3 | [ByteBuffer.wrap('Hello World!'.bytes), offset, length] | _ + 'toArray' | 0 | 12 | [offHeap('Hello World!'.bytes)] | _ + 'toArray' | 0 | 3 | [offHeap('Hello World!'.bytes), length] | _ + 'toArray' | 0 | 3 | [offHeap('Hello World!'.bytes), offset, length] | _ + } + + void 'test wrapNullable'() { + setup: + final propagationModule = Mock(PropagationModule) + InstrumentationBridge.registerIastModule(propagationModule) + final bytes = 'Hello World!'.bytes + + when: + final buffer = Utils.wrapNullable(bytes) + + then: + buffer != null + 1 * propagationModule.taintIfTainted(_ as ByteBuffer, bytes, true, VulnerabilityMarks.NOT_MARKED) + 0 * _ + } + + private static ByteBuffer offHeap(final byte[] bytes) { + final buffer = ByteBuffer.allocateDirect(bytes.length) + buffer.put(bytes) + buffer.position(0) + return buffer + } +} diff --git a/dd-smoke-tests/kafka/build.gradle b/dd-smoke-tests/kafka-2/build.gradle similarity index 95% rename from dd-smoke-tests/kafka/build.gradle rename to dd-smoke-tests/kafka-2/build.gradle index 0fb51721d13..7afe3d79873 100644 --- a/dd-smoke-tests/kafka/build.gradle +++ b/dd-smoke-tests/kafka-2/build.gradle @@ -5,7 +5,7 @@ plugins { } apply from: "$rootDir/gradle/java.gradle" -description = 'Kafka Smoke Tests.' +description = 'Kafka 2.x Smoke Tests.' dependencies { implementation('org.springframework.boot:spring-boot-starter-web') diff --git a/dd-smoke-tests/kafka/src/main/java/datadog/smoketest/kafka/KafkaApplication.java b/dd-smoke-tests/kafka-2/src/main/java/datadog/smoketest/kafka/KafkaApplication.java similarity index 100% rename from dd-smoke-tests/kafka/src/main/java/datadog/smoketest/kafka/KafkaApplication.java rename to dd-smoke-tests/kafka-2/src/main/java/datadog/smoketest/kafka/KafkaApplication.java diff --git a/dd-smoke-tests/kafka-2/src/main/java/datadog/smoketest/kafka/iast/IastConfiguration.java b/dd-smoke-tests/kafka-2/src/main/java/datadog/smoketest/kafka/iast/IastConfiguration.java new file mode 100644 index 00000000000..f70dbd50b30 --- /dev/null +++ b/dd-smoke-tests/kafka-2/src/main/java/datadog/smoketest/kafka/iast/IastConfiguration.java @@ -0,0 +1,332 @@ +package datadog.smoketest.kafka.iast; + +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.Map; +import org.apache.kafka.clients.admin.NewTopic; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.common.serialization.ByteArrayDeserializer; +import org.apache.kafka.common.serialization.ByteArraySerializer; +import org.apache.kafka.common.serialization.ByteBufferDeserializer; +import org.apache.kafka.common.serialization.ByteBufferSerializer; +import org.apache.kafka.common.serialization.Deserializer; +import org.apache.kafka.common.serialization.Serializer; +import org.apache.kafka.common.serialization.StringDeserializer; +import org.apache.kafka.common.serialization.StringSerializer; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory; +import org.springframework.kafka.config.TopicBuilder; +import org.springframework.kafka.core.DefaultKafkaConsumerFactory; +import org.springframework.kafka.core.DefaultKafkaProducerFactory; +import org.springframework.kafka.core.KafkaAdmin; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.kafka.listener.ConcurrentMessageListenerContainer; +import org.springframework.kafka.requestreply.ReplyingKafkaTemplate; +import org.springframework.kafka.support.serializer.JsonDeserializer; +import org.springframework.kafka.support.serializer.JsonSerializer; + +@Configuration +public class IastConfiguration { + + public static final String STRING_TOPIC = "iast_string"; + + public static final String BYTE_ARRAY_TOPIC = "iast_byteArray"; + + public static final String BYTE_BUFFER_TOPIC = "iast_byteBuffer"; + + public static final String JSON_TOPIC = "iast_json"; + + public static final String REPLY_STRING_TOPIC = "iast_string_reply"; + + public static final String REPLY_BYTE_ARRAY_TOPIC = "iast_byteArray_reply"; + + public static final String REPLY_BYTE_BUFFER_TOPIC = "iast_byteBuffer_reply"; + + public static final String REPLY_JSON_TOPIC = "iast_json_reply"; + + @Value("${spring.kafka.bootstrap-servers}") + private String boostrapServers; + + @Bean + public KafkaAdmin.NewTopics iastTopics() { + return new KafkaAdmin.NewTopics( + newTopic(STRING_TOPIC), + newTopic(BYTE_ARRAY_TOPIC), + newTopic(BYTE_BUFFER_TOPIC), + newTopic(JSON_TOPIC), + newTopic(REPLY_STRING_TOPIC), + newTopic(REPLY_BYTE_ARRAY_TOPIC), + newTopic(REPLY_BYTE_BUFFER_TOPIC), + newTopic(REPLY_JSON_TOPIC)); + } + + @Bean + public DefaultKafkaConsumerFactory iastStringConsumer() { + return consumerFor(STRING_TOPIC, StringDeserializer.class, StringDeserializer.class); + } + + @Bean + public DefaultKafkaConsumerFactory iastByteArrayConsumer() { + return consumerFor(BYTE_ARRAY_TOPIC, ByteArrayDeserializer.class, ByteArrayDeserializer.class); + } + + @Bean + public DefaultKafkaConsumerFactory iastByteBufferConsumer() { + return consumerFor( + BYTE_BUFFER_TOPIC, ByteBufferDeserializer.class, ByteBufferDeserializer.class); + } + + @Bean + public DefaultKafkaConsumerFactory iastJsonConsumer() { + final Class> deserializer = jsonDeserializer(); + return consumerFor(JSON_TOPIC, deserializer, deserializer); + } + + @Bean + public DefaultKafkaConsumerFactory iastReplyStringConsumer() { + return consumerFor(REPLY_STRING_TOPIC, StringDeserializer.class, StringDeserializer.class); + } + + @Bean + public DefaultKafkaConsumerFactory iastReplyByteArrayConsumer() { + return consumerFor( + REPLY_BYTE_ARRAY_TOPIC, ByteArrayDeserializer.class, StringDeserializer.class); + } + + @Bean + public DefaultKafkaConsumerFactory iastReplyByteBufferConsumer() { + return consumerFor( + REPLY_BYTE_BUFFER_TOPIC, ByteBufferDeserializer.class, StringDeserializer.class); + } + + @Bean + public DefaultKafkaConsumerFactory iastReplyJsonConsumer() { + return consumerFor(REPLY_JSON_TOPIC, jsonDeserializer(), StringDeserializer.class); + } + + @Bean + public ConcurrentKafkaListenerContainerFactory iastStringListener() { + final ConcurrentKafkaListenerContainerFactory factory = + new ConcurrentKafkaListenerContainerFactory<>(); + factory.setConsumerFactory(iastStringConsumer()); + factory.setReplyTemplate(iastReplyStringTemplate()); + return factory; + } + + @Bean + public ConcurrentKafkaListenerContainerFactory iastByteArrayListener() { + final ConcurrentKafkaListenerContainerFactory factory = + new ConcurrentKafkaListenerContainerFactory<>(); + factory.setConsumerFactory(iastByteArrayConsumer()); + factory.setReplyTemplate(iastReplyByteArrayTemplate()); + return factory; + } + + @Bean + public ConcurrentKafkaListenerContainerFactory iastByteBufferListener() { + final ConcurrentKafkaListenerContainerFactory factory = + new ConcurrentKafkaListenerContainerFactory<>(); + factory.setConsumerFactory(iastByteBufferConsumer()); + factory.setReplyTemplate(iastReplyByteBufferTemplate()); + return factory; + } + + @Bean + public ConcurrentKafkaListenerContainerFactory iastJsonListener() { + final ConcurrentKafkaListenerContainerFactory factory = + new ConcurrentKafkaListenerContainerFactory<>(); + factory.setConsumerFactory(iastJsonConsumer()); + factory.setReplyTemplate(iastReplyJsonTemplate()); + return factory; + } + + @Bean + public ConcurrentKafkaListenerContainerFactory iastReplyStringListener() { + final ConcurrentKafkaListenerContainerFactory factory = + new ConcurrentKafkaListenerContainerFactory<>(); + factory.setConsumerFactory(iastReplyStringConsumer()); + return factory; + } + + @Bean + public ConcurrentKafkaListenerContainerFactory iastReplyByteArrayListener() { + final ConcurrentKafkaListenerContainerFactory factory = + new ConcurrentKafkaListenerContainerFactory<>(); + factory.setConsumerFactory(iastReplyByteArrayConsumer()); + return factory; + } + + @Bean + public ConcurrentKafkaListenerContainerFactory iastReplyByteBufferListener() { + final ConcurrentKafkaListenerContainerFactory factory = + new ConcurrentKafkaListenerContainerFactory<>(); + factory.setConsumerFactory(iastReplyByteBufferConsumer()); + return factory; + } + + @Bean + public ConcurrentKafkaListenerContainerFactory iastReplyJsonListener() { + final ConcurrentKafkaListenerContainerFactory factory = + new ConcurrentKafkaListenerContainerFactory<>(); + factory.setConsumerFactory(iastReplyJsonConsumer()); + return factory; + } + + @Bean + public ConcurrentMessageListenerContainer iastReplyStringContainer() { + ConcurrentMessageListenerContainer repliesContainer = + iastReplyStringListener().createContainer(REPLY_STRING_TOPIC); + repliesContainer.setAutoStartup(false); + return repliesContainer; + } + + @Bean + public ConcurrentMessageListenerContainer iastReplyByteArrayContainer() { + ConcurrentMessageListenerContainer repliesContainer = + iastReplyByteArrayListener().createContainer(REPLY_BYTE_ARRAY_TOPIC); + repliesContainer.setAutoStartup(false); + return repliesContainer; + } + + @Bean + public ConcurrentMessageListenerContainer iastReplyByteBufferContainer() { + ConcurrentMessageListenerContainer repliesContainer = + iastReplyByteBufferListener().createContainer(REPLY_BYTE_BUFFER_TOPIC); + repliesContainer.setAutoStartup(false); + return repliesContainer; + } + + @Bean + public ConcurrentMessageListenerContainer iastReplyJsonContainer() { + ConcurrentMessageListenerContainer repliesContainer = + iastReplyJsonListener().createContainer(REPLY_JSON_TOPIC); + repliesContainer.setAutoStartup(false); + return repliesContainer; + } + + @Bean + public DefaultKafkaProducerFactory iastStringProducer() { + return producerFor(StringSerializer.class, StringSerializer.class); + } + + @Bean + public DefaultKafkaProducerFactory iastByteArrayProducer() { + return producerFor(ByteArraySerializer.class, ByteArraySerializer.class); + } + + @Bean + public DefaultKafkaProducerFactory iastByteBufferProducer() { + return producerFor(ByteBufferSerializer.class, ByteBufferSerializer.class); + } + + @Bean + public DefaultKafkaProducerFactory iastJsonProducer() { + final Class> serializer = jsonSerializer(); + return producerFor(serializer, serializer); + } + + @Bean + public DefaultKafkaProducerFactory iastReplyStringProducer() { + return producerFor(StringSerializer.class, StringSerializer.class); + } + + @Bean + public DefaultKafkaProducerFactory iastReplyByteArrayProducer() { + return producerFor(ByteArraySerializer.class, StringSerializer.class); + } + + @Bean + public DefaultKafkaProducerFactory iastReplyByteBufferProducer() { + return producerFor(ByteBufferSerializer.class, StringSerializer.class); + } + + @Bean + public DefaultKafkaProducerFactory iastReplyJsonProducer() { + return producerFor(jsonSerializer(), StringSerializer.class); + } + + @Bean + public ReplyingKafkaTemplate iastStringTemplate() { + return new ReplyingKafkaTemplate<>(iastStringProducer(), iastReplyStringContainer()); + } + + @Bean + public ReplyingKafkaTemplate iastByteArrayTemplate() { + return new ReplyingKafkaTemplate<>(iastByteArrayProducer(), iastReplyByteArrayContainer()); + } + + @Bean + public ReplyingKafkaTemplate iastByteBufferTemplate() { + return new ReplyingKafkaTemplate<>(iastByteBufferProducer(), iastReplyByteBufferContainer()); + } + + @Bean + public ReplyingKafkaTemplate iastJsonTemplate() { + return new ReplyingKafkaTemplate<>(iastJsonProducer(), iastReplyJsonContainer()); + } + + @Bean + public KafkaTemplate iastReplyStringTemplate() { + return new KafkaTemplate<>(iastReplyStringProducer()); + } + + @Bean + public KafkaTemplate iastReplyByteArrayTemplate() { + return new KafkaTemplate<>(iastReplyByteArrayProducer()); + } + + @Bean + public KafkaTemplate iastReplyByteBufferTemplate() { + return new KafkaTemplate<>(iastReplyByteBufferProducer()); + } + + @Bean + public KafkaTemplate iastReplyJsonTemplate() { + return new KafkaTemplate<>(iastReplyJsonProducer()); + } + + private DefaultKafkaProducerFactory producerFor( + final Class> keySerializer, + final Class> valueSerializer) { + final Map configProps = new HashMap<>(); + configProps.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, boostrapServers); + configProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, keySerializer); + configProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, valueSerializer); + return new DefaultKafkaProducerFactory<>(configProps); + } + + private DefaultKafkaConsumerFactory consumerFor( + final String topic, + final Class> keyDeserializer, + final Class> valueDeserializer) { + final Map config = new HashMap<>(); + config.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, boostrapServers); + config.put(ConsumerConfig.GROUP_ID_CONFIG, topic); // one group per topic + config.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, keyDeserializer); + config.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, valueDeserializer); + if (JsonDeserializer.class.isAssignableFrom(keyDeserializer) + || JsonDeserializer.class.isAssignableFrom(valueDeserializer)) { + config.put(JsonDeserializer.TRUSTED_PACKAGES, "datadog.*"); + } + return new DefaultKafkaConsumerFactory<>(config); + } + + private NewTopic newTopic(final String name) { + return TopicBuilder.name(name).partitions(1).replicas(1).build(); + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + private Class> jsonDeserializer() { + final Class type = JsonDeserializer.class; + return (Class>) type; + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + private Class> jsonSerializer() { + final Class type = JsonSerializer.class; + return (Class>) type; + } +} diff --git a/dd-smoke-tests/kafka-2/src/main/java/datadog/smoketest/kafka/iast/IastController.java b/dd-smoke-tests/kafka-2/src/main/java/datadog/smoketest/kafka/iast/IastController.java new file mode 100644 index 00000000000..3defbd5863f --- /dev/null +++ b/dd-smoke-tests/kafka-2/src/main/java/datadog/smoketest/kafka/iast/IastController.java @@ -0,0 +1,165 @@ +package datadog.smoketest.kafka.iast; + +import static datadog.smoketest.kafka.iast.IastConfiguration.BYTE_ARRAY_TOPIC; +import static datadog.smoketest.kafka.iast.IastConfiguration.BYTE_BUFFER_TOPIC; +import static datadog.smoketest.kafka.iast.IastConfiguration.JSON_TOPIC; +import static datadog.smoketest.kafka.iast.IastConfiguration.STRING_TOPIC; + +import java.nio.ByteBuffer; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.http.ResponseEntity; +import org.springframework.kafka.annotation.KafkaListener; +import org.springframework.kafka.requestreply.ReplyingKafkaTemplate; +import org.springframework.kafka.requestreply.RequestReplyFuture; +import org.springframework.messaging.handler.annotation.SendTo; +import org.springframework.stereotype.Controller; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestParam; + +@Controller +public class IastController { + + private static final Logger LOGGER = LoggerFactory.getLogger(IastController.class); + + private final ReplyingKafkaTemplate stringTemplate; + private final ReplyingKafkaTemplate byteArrayTemplate; + private final ReplyingKafkaTemplate byteBufferTemplate; + private final ReplyingKafkaTemplate jsonTemplate; + + public IastController( + @Qualifier("iastStringTemplate") + final ReplyingKafkaTemplate stringTemplate, + @Qualifier("iastByteArrayTemplate") + final ReplyingKafkaTemplate byteArrayTemplate, + @Qualifier("iastByteBufferTemplate") + final ReplyingKafkaTemplate byteBufferTemplate, + @Qualifier("iastJsonTemplate") + final ReplyingKafkaTemplate jsonTemplate) { + this.stringTemplate = stringTemplate; + this.byteArrayTemplate = byteArrayTemplate; + this.byteBufferTemplate = byteBufferTemplate; + this.jsonTemplate = jsonTemplate; + } + + @GetMapping("/iast/health") + public ResponseEntity health() { + return sendAndReceive("health", null, stringTemplate, STRING_TOPIC); + } + + @GetMapping("/iast/kafka/string") + public ResponseEntity string(@RequestParam("type") final String type) { + return sendAndReceive(type, stringTemplate, STRING_TOPIC, Function.identity()); + } + + @GetMapping("/iast/kafka/byteArray") + public ResponseEntity byteArray(@RequestParam("type") final String type) { + return sendAndReceive(type, byteArrayTemplate, BYTE_ARRAY_TOPIC, String::getBytes); + } + + @GetMapping("/iast/kafka/byteBuffer") + public ResponseEntity byteBuffer(@RequestParam("type") final String type) { + return sendAndReceive( + type, byteBufferTemplate, BYTE_BUFFER_TOPIC, it -> ByteBuffer.wrap(it.getBytes())); + } + + @GetMapping("/iast/kafka/json") + public ResponseEntity json(@RequestParam("type") final String type) { + return sendAndReceive(type, jsonTemplate, JSON_TOPIC, IastMessage::new); + } + + @KafkaListener(topics = STRING_TOPIC, containerFactory = "iastStringListener") + @SendTo + public String listenString(final ConsumerRecord record) { + String key = record.key(); + String value = record.value(); + return handle(STRING_TOPIC, key, value); + } + + @KafkaListener(topics = BYTE_ARRAY_TOPIC, containerFactory = "iastByteArrayListener") + @SendTo + public String listenByteArray(final ConsumerRecord record) { + byte[] key = record.key(); + byte[] value = record.value(); + return handle(BYTE_ARRAY_TOPIC, new String(key), value == null ? null : new String(value)); + } + + @KafkaListener(topics = BYTE_BUFFER_TOPIC, containerFactory = "iastByteBufferListener") + @SendTo + public String listenByteBuffer(final ConsumerRecord record) { + ByteBuffer key = record.key(); + ByteBuffer value = record.value(); + return handle( + BYTE_BUFFER_TOPIC, + new String(key.array(), key.arrayOffset(), key.limit()), + value == null ? null : new String(value.array(), value.arrayOffset(), value.limit())); + } + + @KafkaListener(topics = JSON_TOPIC, containerFactory = "iastJsonListener") + @SendTo + public String listenJson(final ConsumerRecord record) { + final IastMessage key = record.key(); + final IastMessage value = record.value(); + return handle(JSON_TOPIC, key.getValue(), value == null ? null : value.getValue()); + } + + private ResponseEntity sendAndReceive( + final String type, + final ReplyingKafkaTemplate template, + final String topic, + final Function mapper) { + final boolean isKey = isKey(type); + final String key = isKey ? type : "mock key"; + final String value = !isKey ? type : "mock value"; + LOGGER.info("Sending message to {}: {} {}", topic, key, value); + return sendAndReceive(mapper.apply(key), mapper.apply(value), template, topic); + } + + private ResponseEntity sendAndReceive( + final E key, + final E value, + final ReplyingKafkaTemplate template, + final String topic) { + final ProducerRecord record = new ProducerRecord<>(topic, key, value); + final RequestReplyFuture future = template.sendAndReceive(record); + try { + future.getSendFuture().get(10, TimeUnit.SECONDS); // send ok + final ConsumerRecord reply = future.get(10, TimeUnit.SECONDS); // reply + if (reply == null || !"OK".equals(reply.value())) { + return ResponseEntity.internalServerError() + .body(reply == null ? "REPLY_TIMEOUT" : reply.value()); + } else { + return ResponseEntity.ok("OK"); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private String handle(final String topic, final String key, final String value) { + LOGGER.info("Received message from {}: {} {}", topic, key, value); + if (isKey(key)) { + LOGGER.info("Kafka tainted key: " + key); + return "OK"; + } else if (isValue(value)) { + LOGGER.info("Kafka tainted value: " + value); + return "OK"; + } else if ("health".equals(key)) { + return "OK"; + } + return "NO_OK"; + } + + private boolean isKey(final String key) { + return key != null && key.endsWith("source_key"); + } + + private boolean isValue(final String value) { + return value != null && value.endsWith("source_value"); + } +} diff --git a/dd-smoke-tests/kafka/src/main/java/datadog/smoketest/kafka/iast/IastMessage.java b/dd-smoke-tests/kafka-2/src/main/java/datadog/smoketest/kafka/iast/IastMessage.java similarity index 68% rename from dd-smoke-tests/kafka/src/main/java/datadog/smoketest/kafka/iast/IastMessage.java rename to dd-smoke-tests/kafka-2/src/main/java/datadog/smoketest/kafka/iast/IastMessage.java index 90c8fc29fad..8d9a4edf45f 100644 --- a/dd-smoke-tests/kafka/src/main/java/datadog/smoketest/kafka/iast/IastMessage.java +++ b/dd-smoke-tests/kafka-2/src/main/java/datadog/smoketest/kafka/iast/IastMessage.java @@ -4,6 +4,12 @@ public class IastMessage { private String value; + public IastMessage() {} + + public IastMessage(final String value) { + this.value = value; + } + public String getValue() { return value; } diff --git a/dd-smoke-tests/kafka/src/test/groovy/IastKafkaSmokeTest.groovy b/dd-smoke-tests/kafka-2/src/test/groovy/IastKafka2SmokeTest.groovy similarity index 79% rename from dd-smoke-tests/kafka/src/test/groovy/IastKafkaSmokeTest.groovy rename to dd-smoke-tests/kafka-2/src/test/groovy/IastKafka2SmokeTest.groovy index 23d9da8058c..9c3b9a46685 100644 --- a/dd-smoke-tests/kafka/src/test/groovy/IastKafkaSmokeTest.groovy +++ b/dd-smoke-tests/kafka-2/src/test/groovy/IastKafka2SmokeTest.groovy @@ -1,4 +1,5 @@ import datadog.smoketest.AbstractIastServerSmokeTest +import datadog.trace.agent.test.utils.OkHttpUtils import okhttp3.Request import org.springframework.kafka.test.EmbeddedKafkaBroker import spock.lang.Shared @@ -8,7 +9,7 @@ import static datadog.trace.api.config.IastConfig.IAST_DEBUG_ENABLED import static datadog.trace.api.config.IastConfig.IAST_DETECTION_MODE import static datadog.trace.api.config.IastConfig.IAST_ENABLED -class IastKafkaSmokeTest extends AbstractIastServerSmokeTest { +class IastKafka2SmokeTest extends AbstractIastServerSmokeTest { @Shared EmbeddedKafkaBroker embeddedKafka @@ -48,6 +49,19 @@ class IastKafkaSmokeTest extends AbstractIastServerSmokeTest { return processBuilder } + def setupSpec() { + // ensure everything is working fine + final client = OkHttpUtils.client() + final url = "http://localhost:${httpPort}/iast/health" + for (int attempt : (0..<3)) { + final result = client.newCall(new Request.Builder().url(url).get().build()).execute() + if (result.body().string() == 'OK') { + return + } + } + throw new IllegalStateException('Server not properly initialized') + } + void 'test kafka #endpoint key source'() { setup: final type = "${endpoint}_source_key" @@ -65,7 +79,7 @@ class IastKafkaSmokeTest extends AbstractIastServerSmokeTest { } where: - endpoint << ['json', 'string'] + endpoint << ['json', 'string', 'byteArray', 'byteBuffer'] } void 'test kafka #endpoint value source'() { @@ -85,6 +99,6 @@ class IastKafkaSmokeTest extends AbstractIastServerSmokeTest { } where: - endpoint << ['json', 'string'] + endpoint << ['json', 'string', 'byteArray', 'byteBuffer'] } } diff --git a/dd-smoke-tests/kafka-3/application/build.gradle b/dd-smoke-tests/kafka-3/application/build.gradle new file mode 100644 index 00000000000..25cedb7aca3 --- /dev/null +++ b/dd-smoke-tests/kafka-3/application/build.gradle @@ -0,0 +1,31 @@ +plugins { + id 'java' + id 'org.springframework.boot' version '3.2.2' + id 'io.spring.dependency-management' version '1.1.4' + id 'com.diffplug.spotless' version '6.11.0' +} + +java { + toolchain { + languageVersion.set(JavaLanguageVersion.of(17)) + } +} + +def sharedRootDir = "$rootDir/../../../" +def sharedConfigDirectory = "$sharedRootDir/gradle" +rootProject.ext.sharedConfigDirectory = sharedConfigDirectory + +apply from: "$sharedConfigDirectory/repositories.gradle" +apply from: "$sharedConfigDirectory/spotless.gradle" + +if (hasProperty('appBuildDir')) { + buildDir = property('appBuildDir') +} + +version = "" + +dependencies { + implementation('org.springframework.boot:spring-boot-starter-web') + implementation('org.springframework.boot:spring-boot-starter-actuator') + implementation('org.springframework.kafka:spring-kafka') +} diff --git a/dd-smoke-tests/kafka-3/application/settings.gradle b/dd-smoke-tests/kafka-3/application/settings.gradle new file mode 100644 index 00000000000..c115e2b5db1 --- /dev/null +++ b/dd-smoke-tests/kafka-3/application/settings.gradle @@ -0,0 +1,22 @@ +pluginManagement { + repositories { + mavenLocal() + mavenCentral() + gradlePluginPortal() + } +} + +def isCI = System.getenv("CI") != null + +// Don't pollute the dependency cache with the build cache +if (isCI) { + def sharedRootDir = "$rootDir/../../../" + buildCache { + local { + // This needs to line up with the code in the outer project settings.gradle + directory = "$sharedRootDir/workspace/build-cache" + } + } +} + +rootProject.name='kafka-3-smoketest' diff --git a/dd-smoke-tests/kafka-3/application/src/main/java/datadog/smoketest/kafka/KafkaApplication.java b/dd-smoke-tests/kafka-3/application/src/main/java/datadog/smoketest/kafka/KafkaApplication.java new file mode 100644 index 00000000000..26a84e8ade2 --- /dev/null +++ b/dd-smoke-tests/kafka-3/application/src/main/java/datadog/smoketest/kafka/KafkaApplication.java @@ -0,0 +1,16 @@ +package datadog.smoketest.kafka; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.kafka.annotation.EnableKafka; +import org.springframework.scheduling.annotation.EnableAsync; + +@SpringBootApplication +@EnableKafka +@EnableAsync +public class KafkaApplication { + + public static void main(final String[] args) { + SpringApplication.run(KafkaApplication.class, args); + } +} diff --git a/dd-smoke-tests/kafka-3/application/src/main/java/datadog/smoketest/kafka/iast/IastConfiguration.java b/dd-smoke-tests/kafka-3/application/src/main/java/datadog/smoketest/kafka/iast/IastConfiguration.java new file mode 100644 index 00000000000..f70dbd50b30 --- /dev/null +++ b/dd-smoke-tests/kafka-3/application/src/main/java/datadog/smoketest/kafka/iast/IastConfiguration.java @@ -0,0 +1,332 @@ +package datadog.smoketest.kafka.iast; + +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.Map; +import org.apache.kafka.clients.admin.NewTopic; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.common.serialization.ByteArrayDeserializer; +import org.apache.kafka.common.serialization.ByteArraySerializer; +import org.apache.kafka.common.serialization.ByteBufferDeserializer; +import org.apache.kafka.common.serialization.ByteBufferSerializer; +import org.apache.kafka.common.serialization.Deserializer; +import org.apache.kafka.common.serialization.Serializer; +import org.apache.kafka.common.serialization.StringDeserializer; +import org.apache.kafka.common.serialization.StringSerializer; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory; +import org.springframework.kafka.config.TopicBuilder; +import org.springframework.kafka.core.DefaultKafkaConsumerFactory; +import org.springframework.kafka.core.DefaultKafkaProducerFactory; +import org.springframework.kafka.core.KafkaAdmin; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.kafka.listener.ConcurrentMessageListenerContainer; +import org.springframework.kafka.requestreply.ReplyingKafkaTemplate; +import org.springframework.kafka.support.serializer.JsonDeserializer; +import org.springframework.kafka.support.serializer.JsonSerializer; + +@Configuration +public class IastConfiguration { + + public static final String STRING_TOPIC = "iast_string"; + + public static final String BYTE_ARRAY_TOPIC = "iast_byteArray"; + + public static final String BYTE_BUFFER_TOPIC = "iast_byteBuffer"; + + public static final String JSON_TOPIC = "iast_json"; + + public static final String REPLY_STRING_TOPIC = "iast_string_reply"; + + public static final String REPLY_BYTE_ARRAY_TOPIC = "iast_byteArray_reply"; + + public static final String REPLY_BYTE_BUFFER_TOPIC = "iast_byteBuffer_reply"; + + public static final String REPLY_JSON_TOPIC = "iast_json_reply"; + + @Value("${spring.kafka.bootstrap-servers}") + private String boostrapServers; + + @Bean + public KafkaAdmin.NewTopics iastTopics() { + return new KafkaAdmin.NewTopics( + newTopic(STRING_TOPIC), + newTopic(BYTE_ARRAY_TOPIC), + newTopic(BYTE_BUFFER_TOPIC), + newTopic(JSON_TOPIC), + newTopic(REPLY_STRING_TOPIC), + newTopic(REPLY_BYTE_ARRAY_TOPIC), + newTopic(REPLY_BYTE_BUFFER_TOPIC), + newTopic(REPLY_JSON_TOPIC)); + } + + @Bean + public DefaultKafkaConsumerFactory iastStringConsumer() { + return consumerFor(STRING_TOPIC, StringDeserializer.class, StringDeserializer.class); + } + + @Bean + public DefaultKafkaConsumerFactory iastByteArrayConsumer() { + return consumerFor(BYTE_ARRAY_TOPIC, ByteArrayDeserializer.class, ByteArrayDeserializer.class); + } + + @Bean + public DefaultKafkaConsumerFactory iastByteBufferConsumer() { + return consumerFor( + BYTE_BUFFER_TOPIC, ByteBufferDeserializer.class, ByteBufferDeserializer.class); + } + + @Bean + public DefaultKafkaConsumerFactory iastJsonConsumer() { + final Class> deserializer = jsonDeserializer(); + return consumerFor(JSON_TOPIC, deserializer, deserializer); + } + + @Bean + public DefaultKafkaConsumerFactory iastReplyStringConsumer() { + return consumerFor(REPLY_STRING_TOPIC, StringDeserializer.class, StringDeserializer.class); + } + + @Bean + public DefaultKafkaConsumerFactory iastReplyByteArrayConsumer() { + return consumerFor( + REPLY_BYTE_ARRAY_TOPIC, ByteArrayDeserializer.class, StringDeserializer.class); + } + + @Bean + public DefaultKafkaConsumerFactory iastReplyByteBufferConsumer() { + return consumerFor( + REPLY_BYTE_BUFFER_TOPIC, ByteBufferDeserializer.class, StringDeserializer.class); + } + + @Bean + public DefaultKafkaConsumerFactory iastReplyJsonConsumer() { + return consumerFor(REPLY_JSON_TOPIC, jsonDeserializer(), StringDeserializer.class); + } + + @Bean + public ConcurrentKafkaListenerContainerFactory iastStringListener() { + final ConcurrentKafkaListenerContainerFactory factory = + new ConcurrentKafkaListenerContainerFactory<>(); + factory.setConsumerFactory(iastStringConsumer()); + factory.setReplyTemplate(iastReplyStringTemplate()); + return factory; + } + + @Bean + public ConcurrentKafkaListenerContainerFactory iastByteArrayListener() { + final ConcurrentKafkaListenerContainerFactory factory = + new ConcurrentKafkaListenerContainerFactory<>(); + factory.setConsumerFactory(iastByteArrayConsumer()); + factory.setReplyTemplate(iastReplyByteArrayTemplate()); + return factory; + } + + @Bean + public ConcurrentKafkaListenerContainerFactory iastByteBufferListener() { + final ConcurrentKafkaListenerContainerFactory factory = + new ConcurrentKafkaListenerContainerFactory<>(); + factory.setConsumerFactory(iastByteBufferConsumer()); + factory.setReplyTemplate(iastReplyByteBufferTemplate()); + return factory; + } + + @Bean + public ConcurrentKafkaListenerContainerFactory iastJsonListener() { + final ConcurrentKafkaListenerContainerFactory factory = + new ConcurrentKafkaListenerContainerFactory<>(); + factory.setConsumerFactory(iastJsonConsumer()); + factory.setReplyTemplate(iastReplyJsonTemplate()); + return factory; + } + + @Bean + public ConcurrentKafkaListenerContainerFactory iastReplyStringListener() { + final ConcurrentKafkaListenerContainerFactory factory = + new ConcurrentKafkaListenerContainerFactory<>(); + factory.setConsumerFactory(iastReplyStringConsumer()); + return factory; + } + + @Bean + public ConcurrentKafkaListenerContainerFactory iastReplyByteArrayListener() { + final ConcurrentKafkaListenerContainerFactory factory = + new ConcurrentKafkaListenerContainerFactory<>(); + factory.setConsumerFactory(iastReplyByteArrayConsumer()); + return factory; + } + + @Bean + public ConcurrentKafkaListenerContainerFactory iastReplyByteBufferListener() { + final ConcurrentKafkaListenerContainerFactory factory = + new ConcurrentKafkaListenerContainerFactory<>(); + factory.setConsumerFactory(iastReplyByteBufferConsumer()); + return factory; + } + + @Bean + public ConcurrentKafkaListenerContainerFactory iastReplyJsonListener() { + final ConcurrentKafkaListenerContainerFactory factory = + new ConcurrentKafkaListenerContainerFactory<>(); + factory.setConsumerFactory(iastReplyJsonConsumer()); + return factory; + } + + @Bean + public ConcurrentMessageListenerContainer iastReplyStringContainer() { + ConcurrentMessageListenerContainer repliesContainer = + iastReplyStringListener().createContainer(REPLY_STRING_TOPIC); + repliesContainer.setAutoStartup(false); + return repliesContainer; + } + + @Bean + public ConcurrentMessageListenerContainer iastReplyByteArrayContainer() { + ConcurrentMessageListenerContainer repliesContainer = + iastReplyByteArrayListener().createContainer(REPLY_BYTE_ARRAY_TOPIC); + repliesContainer.setAutoStartup(false); + return repliesContainer; + } + + @Bean + public ConcurrentMessageListenerContainer iastReplyByteBufferContainer() { + ConcurrentMessageListenerContainer repliesContainer = + iastReplyByteBufferListener().createContainer(REPLY_BYTE_BUFFER_TOPIC); + repliesContainer.setAutoStartup(false); + return repliesContainer; + } + + @Bean + public ConcurrentMessageListenerContainer iastReplyJsonContainer() { + ConcurrentMessageListenerContainer repliesContainer = + iastReplyJsonListener().createContainer(REPLY_JSON_TOPIC); + repliesContainer.setAutoStartup(false); + return repliesContainer; + } + + @Bean + public DefaultKafkaProducerFactory iastStringProducer() { + return producerFor(StringSerializer.class, StringSerializer.class); + } + + @Bean + public DefaultKafkaProducerFactory iastByteArrayProducer() { + return producerFor(ByteArraySerializer.class, ByteArraySerializer.class); + } + + @Bean + public DefaultKafkaProducerFactory iastByteBufferProducer() { + return producerFor(ByteBufferSerializer.class, ByteBufferSerializer.class); + } + + @Bean + public DefaultKafkaProducerFactory iastJsonProducer() { + final Class> serializer = jsonSerializer(); + return producerFor(serializer, serializer); + } + + @Bean + public DefaultKafkaProducerFactory iastReplyStringProducer() { + return producerFor(StringSerializer.class, StringSerializer.class); + } + + @Bean + public DefaultKafkaProducerFactory iastReplyByteArrayProducer() { + return producerFor(ByteArraySerializer.class, StringSerializer.class); + } + + @Bean + public DefaultKafkaProducerFactory iastReplyByteBufferProducer() { + return producerFor(ByteBufferSerializer.class, StringSerializer.class); + } + + @Bean + public DefaultKafkaProducerFactory iastReplyJsonProducer() { + return producerFor(jsonSerializer(), StringSerializer.class); + } + + @Bean + public ReplyingKafkaTemplate iastStringTemplate() { + return new ReplyingKafkaTemplate<>(iastStringProducer(), iastReplyStringContainer()); + } + + @Bean + public ReplyingKafkaTemplate iastByteArrayTemplate() { + return new ReplyingKafkaTemplate<>(iastByteArrayProducer(), iastReplyByteArrayContainer()); + } + + @Bean + public ReplyingKafkaTemplate iastByteBufferTemplate() { + return new ReplyingKafkaTemplate<>(iastByteBufferProducer(), iastReplyByteBufferContainer()); + } + + @Bean + public ReplyingKafkaTemplate iastJsonTemplate() { + return new ReplyingKafkaTemplate<>(iastJsonProducer(), iastReplyJsonContainer()); + } + + @Bean + public KafkaTemplate iastReplyStringTemplate() { + return new KafkaTemplate<>(iastReplyStringProducer()); + } + + @Bean + public KafkaTemplate iastReplyByteArrayTemplate() { + return new KafkaTemplate<>(iastReplyByteArrayProducer()); + } + + @Bean + public KafkaTemplate iastReplyByteBufferTemplate() { + return new KafkaTemplate<>(iastReplyByteBufferProducer()); + } + + @Bean + public KafkaTemplate iastReplyJsonTemplate() { + return new KafkaTemplate<>(iastReplyJsonProducer()); + } + + private DefaultKafkaProducerFactory producerFor( + final Class> keySerializer, + final Class> valueSerializer) { + final Map configProps = new HashMap<>(); + configProps.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, boostrapServers); + configProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, keySerializer); + configProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, valueSerializer); + return new DefaultKafkaProducerFactory<>(configProps); + } + + private DefaultKafkaConsumerFactory consumerFor( + final String topic, + final Class> keyDeserializer, + final Class> valueDeserializer) { + final Map config = new HashMap<>(); + config.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, boostrapServers); + config.put(ConsumerConfig.GROUP_ID_CONFIG, topic); // one group per topic + config.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, keyDeserializer); + config.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, valueDeserializer); + if (JsonDeserializer.class.isAssignableFrom(keyDeserializer) + || JsonDeserializer.class.isAssignableFrom(valueDeserializer)) { + config.put(JsonDeserializer.TRUSTED_PACKAGES, "datadog.*"); + } + return new DefaultKafkaConsumerFactory<>(config); + } + + private NewTopic newTopic(final String name) { + return TopicBuilder.name(name).partitions(1).replicas(1).build(); + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + private Class> jsonDeserializer() { + final Class type = JsonDeserializer.class; + return (Class>) type; + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + private Class> jsonSerializer() { + final Class type = JsonSerializer.class; + return (Class>) type; + } +} diff --git a/dd-smoke-tests/kafka-3/application/src/main/java/datadog/smoketest/kafka/iast/IastController.java b/dd-smoke-tests/kafka-3/application/src/main/java/datadog/smoketest/kafka/iast/IastController.java new file mode 100644 index 00000000000..d1638fb7bda --- /dev/null +++ b/dd-smoke-tests/kafka-3/application/src/main/java/datadog/smoketest/kafka/iast/IastController.java @@ -0,0 +1,165 @@ +package datadog.smoketest.kafka.iast; + +import static datadog.smoketest.kafka.iast.IastConfiguration.BYTE_ARRAY_TOPIC; +import static datadog.smoketest.kafka.iast.IastConfiguration.BYTE_BUFFER_TOPIC; +import static datadog.smoketest.kafka.iast.IastConfiguration.JSON_TOPIC; +import static datadog.smoketest.kafka.iast.IastConfiguration.STRING_TOPIC; + +import java.nio.ByteBuffer; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.http.ResponseEntity; +import org.springframework.kafka.annotation.KafkaListener; +import org.springframework.kafka.requestreply.ReplyingKafkaTemplate; +import org.springframework.kafka.requestreply.RequestReplyFuture; +import org.springframework.messaging.handler.annotation.SendTo; +import org.springframework.stereotype.Controller; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestParam; + +@Controller +public class IastController { + + private static final Logger LOGGER = LoggerFactory.getLogger(IastController.class); + + private final ReplyingKafkaTemplate stringTemplate; + private final ReplyingKafkaTemplate byteArrayTemplate; + private final ReplyingKafkaTemplate byteBufferTemplate; + private final ReplyingKafkaTemplate jsonTemplate; + + public IastController( + @Qualifier("iastStringTemplate") + final ReplyingKafkaTemplate stringTemplate, + @Qualifier("iastByteArrayTemplate") + final ReplyingKafkaTemplate byteArrayTemplate, + @Qualifier("iastByteBufferTemplate") + final ReplyingKafkaTemplate byteBufferTemplate, + @Qualifier("iastJsonTemplate") + final ReplyingKafkaTemplate jsonTemplate) { + this.stringTemplate = stringTemplate; + this.byteArrayTemplate = byteArrayTemplate; + this.byteBufferTemplate = byteBufferTemplate; + this.jsonTemplate = jsonTemplate; + } + + @GetMapping("/iast/health") + public ResponseEntity health() { + return sendAndReceive("health", null, stringTemplate, STRING_TOPIC); + } + + @GetMapping("/iast/kafka/string") + public ResponseEntity string(@RequestParam("type") final String type) { + return sendAndReceive(type, stringTemplate, STRING_TOPIC, Function.identity()); + } + + @GetMapping("/iast/kafka/byteArray") + public ResponseEntity byteArray(@RequestParam("type") final String type) { + return sendAndReceive(type, byteArrayTemplate, BYTE_ARRAY_TOPIC, String::getBytes); + } + + @GetMapping("/iast/kafka/byteBuffer") + public ResponseEntity byteBuffer(@RequestParam("type") final String type) { + return sendAndReceive( + type, byteBufferTemplate, BYTE_BUFFER_TOPIC, it -> ByteBuffer.wrap(it.getBytes())); + } + + @GetMapping("/iast/kafka/json") + public ResponseEntity json(@RequestParam("type") final String type) { + return sendAndReceive(type, jsonTemplate, JSON_TOPIC, IastMessage::new); + } + + @KafkaListener(topics = STRING_TOPIC, containerFactory = "iastStringListener") + @SendTo + public String listenString(final ConsumerRecord record) { + String key = record.key(); + String value = record.value(); + return handle(STRING_TOPIC, key, value); + } + + @KafkaListener(topics = BYTE_ARRAY_TOPIC, containerFactory = "iastByteArrayListener") + @SendTo + public String listenByteArray(final ConsumerRecord record) { + byte[] key = record.key(); + byte[] value = record.value(); + return handle(BYTE_ARRAY_TOPIC, new String(key), value == null ? null : new String(value)); + } + + @KafkaListener(topics = BYTE_BUFFER_TOPIC, containerFactory = "iastByteBufferListener") + @SendTo + public String listenByteBuffer(final ConsumerRecord record) { + ByteBuffer key = record.key(); + ByteBuffer value = record.value(); + return handle( + BYTE_BUFFER_TOPIC, + new String(key.array(), key.arrayOffset(), key.limit()), + value == null ? null : new String(value.array(), value.arrayOffset(), value.limit())); + } + + @KafkaListener(topics = JSON_TOPIC, containerFactory = "iastJsonListener") + @SendTo + public String listenJson(final ConsumerRecord record) { + final IastMessage key = record.key(); + final IastMessage value = record.value(); + return handle(JSON_TOPIC, key.getValue(), value == null ? null : value.getValue()); + } + + private ResponseEntity sendAndReceive( + final String type, + final ReplyingKafkaTemplate template, + final String topic, + final Function mapper) { + final boolean isKey = isKey(type); + final String key = isKey ? type : "mock key"; + final String value = !isKey ? type : "mock value"; + LOGGER.info("Sending message to {}: {} {}", topic, key, value); + return sendAndReceive(mapper.apply(key), mapper.apply(value), template, topic); + } + + private ResponseEntity sendAndReceive( + final E key, + final E value, + final ReplyingKafkaTemplate template, + final String topic) { + final ProducerRecord record = new ProducerRecord<>(topic, key, value); + final RequestReplyFuture future = template.sendAndReceive(record); + try { + future.getSendFuture().get(10, TimeUnit.SECONDS); // send ok + final ConsumerRecord reply = future.get(10, TimeUnit.SECONDS); // reply + if (reply == null || !"OK".equals(reply.value())) { + return ResponseEntity.internalServerError() + .body(reply == null ? "REPLY_TIMEOUT" : reply.value()); + } else { + return ResponseEntity.ok("OK"); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private String handle(final String topic, final String key, final String value) { + LOGGER.info("Received message from {}: {} {}", topic, key, value); + if (isKey(key)) { + LOGGER.info("Kafka tainted key: " + key); + return "OK"; + } else if (isValue(value)) { + LOGGER.info("Kafka tainted value: " + value); + return "OK"; + } else if ("health".equals(key)) { + return "OK"; + } + return "NO_OK"; + } + + private boolean isKey(final String key) { + return key != null && key.endsWith("source_key"); + } + + private boolean isValue(final String value) { + return value != null && value.endsWith("source_value"); + } +} diff --git a/dd-smoke-tests/kafka-3/application/src/main/java/datadog/smoketest/kafka/iast/IastMessage.java b/dd-smoke-tests/kafka-3/application/src/main/java/datadog/smoketest/kafka/iast/IastMessage.java new file mode 100644 index 00000000000..8d9a4edf45f --- /dev/null +++ b/dd-smoke-tests/kafka-3/application/src/main/java/datadog/smoketest/kafka/iast/IastMessage.java @@ -0,0 +1,20 @@ +package datadog.smoketest.kafka.iast; + +public class IastMessage { + + private String value; + + public IastMessage() {} + + public IastMessage(final String value) { + this.value = value; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } +} diff --git a/dd-smoke-tests/kafka-3/build.gradle b/dd-smoke-tests/kafka-3/build.gradle new file mode 100644 index 00000000000..7fa33782393 --- /dev/null +++ b/dd-smoke-tests/kafka-3/build.gradle @@ -0,0 +1,53 @@ +ext { + minJavaVersionForTests = JavaVersion.VERSION_17 +} + +apply from: "$rootDir/gradle/java.gradle" +apply plugin: 'java-test-fixtures' +description = 'Kafka 3.x Smoke Tests.' + +dependencies { + testImplementation('org.springframework.kafka:spring-kafka-test:2.9.13') + + testImplementation project(':dd-smoke-tests') + implementation project(':dd-smoke-tests:iast-util') + testImplementation(testFixtures(project(":dd-smoke-tests:iast-util"))) +} + +final appDir = "$projectDir/application" +final appBuildDir = "$buildDir/application" +final isWindows = System.getProperty('os.name').toLowerCase().contains('win') +final gradlewCommand = isWindows ? 'gradlew.bat' : 'gradlew' + +tasks.register('bootJar', Exec) { + workingDir appDir + + final toolchain17 = getJavaLauncherFor(17).get() + environment += ["GRADLE_OPTS": "-Dorg.gradle.jvmargs='-Xmx512M'", "JAVA_HOME": "$toolchain17.metadata.installationPath"] + commandLine "$rootDir/${gradlewCommand}", "bootJar", "--no-daemon", "--max-workers=4", "-PappBuildDir=$appBuildDir" + + outputs.cacheIf { true } + + outputs.dir(appBuildDir) + .withPropertyName("applicationJar") + + inputs.files(fileTree(appDir) { + include '**/*' + exclude '.gradle/**' + }) + .withPropertyName("application") + .withPathSensitivity(PathSensitivity.RELATIVE) + + group('build') +} + +tasks.named('compileTestGroovy').configure { + dependsOn 'bootJar' + outputs.upToDateWhen { + !bootJar.didWork + } +} + +tasks.withType(Test).configureEach { + jvmArgs "-Ddatadog.smoketest.springboot.shadowJar.path=${appBuildDir}/libs/kafka-3-smoketest.jar" +} diff --git a/dd-smoke-tests/kafka-3/src/test/groovy/IastKafka3SmokeTest.groovy b/dd-smoke-tests/kafka-3/src/test/groovy/IastKafka3SmokeTest.groovy new file mode 100644 index 00000000000..dd4f3a75d93 --- /dev/null +++ b/dd-smoke-tests/kafka-3/src/test/groovy/IastKafka3SmokeTest.groovy @@ -0,0 +1,104 @@ +import datadog.smoketest.AbstractIastServerSmokeTest +import datadog.trace.agent.test.utils.OkHttpUtils +import okhttp3.Request +import org.springframework.kafka.test.EmbeddedKafkaBroker +import spock.lang.Shared + +import static datadog.trace.api.config.IastConfig.IAST_CONTEXT_MODE +import static datadog.trace.api.config.IastConfig.IAST_DEBUG_ENABLED +import static datadog.trace.api.config.IastConfig.IAST_DETECTION_MODE +import static datadog.trace.api.config.IastConfig.IAST_ENABLED + +class IastKafka3SmokeTest extends AbstractIastServerSmokeTest { + + @Shared + EmbeddedKafkaBroker embeddedKafka + + @Override + protected void beforeProcessBuilders() { + embeddedKafka = new EmbeddedKafkaBroker(1, true) + embeddedKafka.afterPropertiesSet() + } + + @Override + def cleanupSpec() { + embeddedKafka.destroy() + } + + @Override + ProcessBuilder createProcessBuilder() { + String springBootShadowJar = System.getProperty('datadog.smoketest.springboot.shadowJar.path') + List command = [] + command.add(javaPath()) + command.addAll(defaultJavaProperties) + command.addAll((String[]) [ + withSystemProperty(IAST_ENABLED, true), + withSystemProperty(IAST_DETECTION_MODE, 'FULL'), + withSystemProperty(IAST_CONTEXT_MODE, 'GLOBAL'), + withSystemProperty(IAST_DEBUG_ENABLED, true), + ]) + command.addAll((String[]) [ + '-jar', + springBootShadowJar, + "--server.port=${httpPort}", + "--spring.kafka.bootstrap-servers=${embeddedKafka.getBrokersAsString()}" + ]) + ProcessBuilder processBuilder = new ProcessBuilder(command) + processBuilder.directory(new File(buildDirectory)) + processBuilder.environment().clear() + return processBuilder + } + + def setupSpec() { + // ensure everything is working fine + final client = OkHttpUtils.client() + final url = "http://localhost:${httpPort}/iast/health" + for (int attempt : (0..<3)) { + final result = client.newCall(new Request.Builder().url(url).get().build()).execute() + if (result.body().string() == 'OK') { + return + } + } + throw new IllegalStateException('Server not properly initialized') + } + + void 'test kafka #endpoint key source'() { + setup: + final type = "${endpoint}_source_key" + final url = "http://localhost:${httpPort}/iast/kafka/$endpoint?type=${type}" + + when: + final response = client.newCall(new Request.Builder().url(url).get().build()).execute() + + then: + response.body().string() == 'OK' + hasTainted { tainted -> + tainted.value == "Kafka tainted key: $type" && + tainted.ranges[0].source.origin == 'kafka.message.key' && + tainted.ranges[0].source.value == type + } + + where: + endpoint << ['json', 'string', 'byteArray', 'byteBuffer'] + } + + void 'test kafka #endpoint value source'() { + setup: + final type = "${endpoint}_source_value" + final url = "http://localhost:${httpPort}/iast/kafka/$endpoint?type=${type}" + + when: + final response = client.newCall(new Request.Builder().url(url).get().build()).execute() + + then: + response.body().string() == 'OK' + hasTainted { tainted -> + tainted.value == "Kafka tainted value: $type" && + tainted.ranges[0].source.origin == 'kafka.message.value' && + tainted.ranges[0].source.value == type + } + + where: + endpoint << ['json', 'string', 'byteArray', 'byteBuffer'] + } +} diff --git a/dd-smoke-tests/kafka/src/main/java/datadog/smoketest/kafka/iast/IastConfiguration.java b/dd-smoke-tests/kafka/src/main/java/datadog/smoketest/kafka/iast/IastConfiguration.java deleted file mode 100644 index 0964b417e7d..00000000000 --- a/dd-smoke-tests/kafka/src/main/java/datadog/smoketest/kafka/iast/IastConfiguration.java +++ /dev/null @@ -1,92 +0,0 @@ -package datadog.smoketest.kafka.iast; - -import java.util.HashMap; -import java.util.Map; -import org.apache.kafka.clients.consumer.ConsumerConfig; -import org.apache.kafka.clients.producer.ProducerConfig; -import org.apache.kafka.common.serialization.StringDeserializer; -import org.apache.kafka.common.serialization.StringSerializer; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory; -import org.springframework.kafka.config.TopicBuilder; -import org.springframework.kafka.core.DefaultKafkaConsumerFactory; -import org.springframework.kafka.core.DefaultKafkaProducerFactory; -import org.springframework.kafka.core.KafkaAdmin; -import org.springframework.kafka.core.KafkaTemplate; -import org.springframework.kafka.support.serializer.JsonDeserializer; -import org.springframework.kafka.support.serializer.JsonSerializer; - -@Configuration -public class IastConfiguration { - - public static final String GROUP_ID = "iast"; - - public static final String STRING_TOPIC = "iast_string"; - - public static final String JSON_TOPIC = "iast_json"; - - @Value("${spring.kafka.bootstrap-servers}") - private String boostrapServers; - - @Bean - public KafkaAdmin.NewTopics iastTopics() { - return new KafkaAdmin.NewTopics( - TopicBuilder.name(STRING_TOPIC).partitions(1).replicas(1).compact().build(), - TopicBuilder.name(JSON_TOPIC).partitions(1).replicas(1).compact().build()); - } - - @Bean - public ConcurrentKafkaListenerContainerFactory iastStringListenerFactory() { - final Map config = new HashMap<>(); - config.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, boostrapServers); - config.put(ConsumerConfig.GROUP_ID_CONFIG, GROUP_ID); - config.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); - config.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); - final DefaultKafkaConsumerFactory consumerFactory = - new DefaultKafkaConsumerFactory<>(config); - ConcurrentKafkaListenerContainerFactory factory = - new ConcurrentKafkaListenerContainerFactory<>(); - factory.setConsumerFactory(consumerFactory); - return factory; - } - - @Bean - public ConcurrentKafkaListenerContainerFactory iastJsonListenerFactory() { - final Map config = new HashMap<>(); - config.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, boostrapServers); - config.put(ConsumerConfig.GROUP_ID_CONFIG, GROUP_ID); - config.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); - config.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, JsonDeserializer.class); - config.put(JsonDeserializer.TRUSTED_PACKAGES, "datadog.*"); - final DefaultKafkaConsumerFactory consumerFactory = - new DefaultKafkaConsumerFactory<>(config); - ConcurrentKafkaListenerContainerFactory factory = - new ConcurrentKafkaListenerContainerFactory<>(); - factory.setConsumerFactory(consumerFactory); - return factory; - } - - @Bean - public KafkaTemplate iastStringKafkaTemplate() { - Map configProps = new HashMap<>(); - configProps.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, boostrapServers); - configProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class); - configProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class); - final DefaultKafkaProducerFactory factory = - new DefaultKafkaProducerFactory<>(configProps); - return new KafkaTemplate<>(factory); - } - - @Bean - public KafkaTemplate iastJsonKafkaTemplate() { - Map configProps = new HashMap<>(); - configProps.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, boostrapServers); - configProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class); - configProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, JsonSerializer.class); - final DefaultKafkaProducerFactory factory = - new DefaultKafkaProducerFactory<>(configProps); - return new KafkaTemplate<>(factory); - } -} diff --git a/dd-smoke-tests/kafka/src/main/java/datadog/smoketest/kafka/iast/IastController.java b/dd-smoke-tests/kafka/src/main/java/datadog/smoketest/kafka/iast/IastController.java deleted file mode 100644 index d906761b5dd..00000000000 --- a/dd-smoke-tests/kafka/src/main/java/datadog/smoketest/kafka/iast/IastController.java +++ /dev/null @@ -1,87 +0,0 @@ -package datadog.smoketest.kafka.iast; - -import static datadog.smoketest.kafka.iast.IastConfiguration.GROUP_ID; -import static datadog.smoketest.kafka.iast.IastConfiguration.JSON_TOPIC; -import static datadog.smoketest.kafka.iast.IastConfiguration.STRING_TOPIC; - -import java.util.concurrent.CompletableFuture; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.http.ResponseEntity; -import org.springframework.kafka.annotation.KafkaListener; -import org.springframework.kafka.core.KafkaTemplate; -import org.springframework.kafka.support.SendResult; -import org.springframework.stereotype.Controller; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.RequestParam; - -@Controller -public class IastController { - - private static final Logger LOGGER = LoggerFactory.getLogger(IastController.class); - - private final KafkaTemplate stringTemplate; - private final KafkaTemplate jsonTemplate; - - public IastController( - @Qualifier("iastStringKafkaTemplate") final KafkaTemplate stringTemplate, - @Qualifier("iastJsonKafkaTemplate") final KafkaTemplate jsonTemplate) { - this.stringTemplate = stringTemplate; - this.jsonTemplate = jsonTemplate; - } - - @GetMapping("/iast/kafka/string") - public CompletableFuture> string(@RequestParam("type") final String type) { - return stringTemplate - .send(STRING_TOPIC, type, type) - .completable() - .thenApply(this::handleKafkaResponse); - } - - @GetMapping("/iast/kafka/json") - public CompletableFuture> json(@RequestParam("type") final String type) { - final IastMessage message = new IastMessage(); - message.setValue(type); - return jsonTemplate - .send(JSON_TOPIC, type, message) - .completable() - .thenApply(this::handleKafkaResponse); - } - - @KafkaListener( - groupId = GROUP_ID, - topics = STRING_TOPIC, - containerFactory = "iastStringListenerFactory") - public void listenString(final ConsumerRecord record) { - handle(record.key(), record.value()); - } - - @KafkaListener( - groupId = GROUP_ID, - topics = JSON_TOPIC, - containerFactory = "iastJsonListenerFactory") - public void listenJson(final ConsumerRecord record) { - final IastMessage message = record.value(); - handle(record.key(), message.getValue()); - } - - private void handle(final String key, final String value) { - if (key.endsWith("source_key")) { - LOGGER.info("Kafka tainted key: " + key); - } else if (key.endsWith("source_value")) { - LOGGER.info("Kafka tainted value: " + value); - } else { - throw new IllegalArgumentException("Non valid key " + key); - } - } - - private ResponseEntity handleKafkaResponse(final SendResult result) { - if (result.getRecordMetadata().hasOffset()) { - return ResponseEntity.ok("OK"); - } else { - return ResponseEntity.internalServerError().body("NO_OK"); - } - } -} diff --git a/settings.gradle b/settings.gradle index 018aee8b5b5..9177cda5697 100644 --- a/settings.gradle +++ b/settings.gradle @@ -100,7 +100,8 @@ include ':dd-smoke-tests:jersey' include ':dd-smoke-tests:jersey-2' include ':dd-smoke-tests:jersey-3' include ':dd-smoke-tests:jboss-modules' -include ':dd-smoke-tests:kafka' +include ':dd-smoke-tests:kafka-2' +include ':dd-smoke-tests:kafka-3' include ':dd-smoke-tests:log-injection' include ':dd-smoke-tests:maven' include ':dd-smoke-tests:opentracing' From ea771efa14a84f429790dc1f1736fb1b278fd9e9 Mon Sep 17 00:00:00 2001 From: Nikita Tkachenko <121111529+nikita-tkachenko-datadog@users.noreply.github.com> Date: Wed, 6 Mar 2024 15:41:04 +0100 Subject: [PATCH 12/30] Update Spock ITR instrumentation to skip spec setup methods if all features in a spec are skipped (#6780) --- .../events/TestEventsHandlerImpl.java | 5 + .../junit5/JUnit5SpockItrInstrumentation.java | 40 ++- .../instrumentation/junit5/SpockUtils.java | 11 + .../src/test/groovy/SpockTest.groovy | 38 ++- .../TestParameterizedSetupSpecSpock.groovy | 29 +++ .../example/TestSucceedSetupSpecSpock.groovy | 29 +++ .../coverages.ftl | 1 + .../events.ftl | 240 +++++++++++++++++ .../coverages.ftl | 1 + .../events.ftl | 238 +++++++++++++++++ .../coverages.ftl | 1 + .../events.ftl | 242 ++++++++++++++++++ .../coverages.ftl | 1 + .../test-itr-skipping-spec-setup/events.ftl | 222 ++++++++++++++++ .../events/TestEventsHandler.java | 2 + 15 files changed, 1077 insertions(+), 23 deletions(-) create mode 100644 dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/groovy/org/example/TestParameterizedSetupSpecSpock.groovy create mode 100644 dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/groovy/org/example/TestSucceedSetupSpecSpock.groovy create mode 100644 dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-itr-not-skipping-parameterized-spec-setup/coverages.ftl create mode 100644 dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-itr-not-skipping-parameterized-spec-setup/events.ftl create mode 100644 dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-itr-not-skipping-spec-setup/coverages.ftl create mode 100644 dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-itr-not-skipping-spec-setup/events.ftl create mode 100644 dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-itr-skipping-parameterized-spec-setup/coverages.ftl create mode 100644 dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-itr-skipping-parameterized-spec-setup/events.ftl create mode 100644 dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-itr-skipping-spec-setup/coverages.ftl create mode 100644 dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-itr-skipping-spec-setup/events.ftl diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/events/TestEventsHandlerImpl.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/events/TestEventsHandlerImpl.java index 33d28c2629d..c1e4a67adcc 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/events/TestEventsHandlerImpl.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/events/TestEventsHandlerImpl.java @@ -291,6 +291,11 @@ public boolean skip(TestIdentifier test) { return testModule.skip(test); } + @Override + public boolean isSkippable(TestIdentifier test) { + return testModule.isSkippable(test); + } + @Override @Nonnull public TestRetryPolicy retryPolicy(TestIdentifier test) { diff --git a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/main/java/datadog/trace/instrumentation/junit5/JUnit5SpockItrInstrumentation.java b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/main/java/datadog/trace/instrumentation/junit5/JUnit5SpockItrInstrumentation.java index 1af2114614c..7b4f4f85cf7 100644 --- a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/main/java/datadog/trace/instrumentation/junit5/JUnit5SpockItrInstrumentation.java +++ b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/main/java/datadog/trace/instrumentation/junit5/JUnit5SpockItrInstrumentation.java @@ -13,14 +13,14 @@ import datadog.trace.api.civisibility.config.TestIdentifier; import datadog.trace.bootstrap.CallDepthThreadLocalMap; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; -import java.util.Collection; import java.util.Set; import net.bytebuddy.asm.Advice; import net.bytebuddy.description.type.TypeDescription; import net.bytebuddy.matcher.ElementMatcher; -import org.junit.platform.engine.TestTag; +import org.junit.platform.engine.TestDescriptor; import org.junit.platform.engine.support.hierarchical.Node; import org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService; +import org.spockframework.runtime.SpecNode; import org.spockframework.runtime.SpockNode; @AutoService(Instrumenter.class) @@ -102,16 +102,38 @@ public static void shouldBeSkipped( return; } - Collection tags = SpockUtils.getTags(spockNode); - for (TestTag tag : tags) { - if (InstrumentationBridge.ITR_UNSKIPPABLE_TAG.equals(tag.getName())) { - return; - } + if (SpockUtils.isUnskippable(spockNode)) { + return; } - TestIdentifier test = SpockUtils.toTestIdentifier(spockNode); - if (test != null && TestEventsHandlerHolder.TEST_EVENTS_HANDLER.skip(test)) { + if (spockNode instanceof SpecNode) { + // suite + SpecNode specNode = (SpecNode) spockNode; + Set features = specNode.getChildren(); + for (TestDescriptor feature : features) { + if (feature instanceof SpockNode && SpockUtils.isUnskippable((SpockNode) feature)) { + return; + } + + TestIdentifier featureIdentifier = SpockUtils.toTestIdentifier(feature); + if (featureIdentifier == null + || !TestEventsHandlerHolder.TEST_EVENTS_HANDLER.isSkippable(featureIdentifier)) { + return; + } + } + + // all children are skippable + for (TestDescriptor feature : features) { + TestEventsHandlerHolder.TEST_EVENTS_HANDLER.skip(SpockUtils.toTestIdentifier(feature)); + } skipResult = Node.SkipResult.skip(InstrumentationBridge.ITR_SKIP_REASON); + + } else { + // individual test case + TestIdentifier test = SpockUtils.toTestIdentifier(spockNode); + if (test != null && TestEventsHandlerHolder.TEST_EVENTS_HANDLER.skip(test)) { + skipResult = Node.SkipResult.skip(InstrumentationBridge.ITR_SKIP_REASON); + } } } diff --git a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/main/java/datadog/trace/instrumentation/junit5/SpockUtils.java b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/main/java/datadog/trace/instrumentation/junit5/SpockUtils.java index 056f6aca21a..eaf849edf13 100644 --- a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/main/java/datadog/trace/instrumentation/junit5/SpockUtils.java +++ b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/main/java/datadog/trace/instrumentation/junit5/SpockUtils.java @@ -1,5 +1,6 @@ package datadog.trace.instrumentation.junit5; +import datadog.trace.api.civisibility.InstrumentationBridge; import datadog.trace.api.civisibility.config.TestIdentifier; import java.lang.invoke.MethodHandle; import java.lang.reflect.Method; @@ -64,6 +65,16 @@ public static Collection getTags(SpockNode spockNode) { } } + public static boolean isUnskippable(SpockNode spockNode) { + Collection tags = SpockUtils.getTags(spockNode); + for (TestTag tag : tags) { + if (InstrumentationBridge.ITR_UNSKIPPABLE_TAG.equals(tag.getName())) { + return true; + } + } + return false; + } + public static Method getTestMethod(MethodSource methodSource) { String methodName = methodSource.getMethodName(); if (methodName == null) { diff --git a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/groovy/SpockTest.groovy b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/groovy/SpockTest.groovy index ef9e154e15f..cae43f6aaaa 100644 --- a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/groovy/SpockTest.groovy +++ b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/groovy/SpockTest.groovy @@ -6,8 +6,10 @@ import org.example.TestFailedParameterizedSpock import org.example.TestFailedSpock import org.example.TestFailedThenSucceedParameterizedSpock import org.example.TestFailedThenSucceedSpock +import org.example.TestParameterizedSetupSpecSpock import org.example.TestParameterizedSpock import org.example.TestSucceedAndFailedSpock +import org.example.TestSucceedSetupSpecSpock import org.example.TestSucceedSpock import org.example.TestSucceedSpockSlow import org.example.TestSucceedSpockUnskippable @@ -47,13 +49,21 @@ class SpockTest extends CiVisibilityInstrumentationTest { assertSpansData(testcaseName, expectedTracesCount) where: - testcaseName | tests | expectedTracesCount | skippedTests - "test-itr-skipping" | [TestSucceedSpock] | 2 | [new TestIdentifier("org.example.TestSucceedSpock", "test success", null, null)] - "test-itr-skipping-parameterized" | [TestParameterizedSpock] | 3 | [ + testcaseName | tests | expectedTracesCount | skippedTests + "test-itr-skipping" | [TestSucceedSpock] | 2 | [new TestIdentifier("org.example.TestSucceedSpock", "test success", null, null)] + "test-itr-skipping-parameterized" | [TestParameterizedSpock] | 3 | [ new TestIdentifier("org.example.TestParameterizedSpock", "test add 1 and 2", '{"metadata":{"test_name":"test add 1 and 2"}}', null) ] - "test-itr-unskippable" | [TestSucceedSpockUnskippable] | 2 | [new TestIdentifier("org.example.TestSucceedSpockUnskippable", "test success", null, null)] - "test-itr-unskippable-suite" | [TestSucceedSpockUnskippableSuite] | 2 | [new TestIdentifier("org.example.TestSucceedSpockUnskippableSuite", "test success", null, null)] + "test-itr-unskippable" | [TestSucceedSpockUnskippable] | 2 | [new TestIdentifier("org.example.TestSucceedSpockUnskippable", "test success", null, null)] + "test-itr-unskippable-suite" | [TestSucceedSpockUnskippableSuite] | 2 | [new TestIdentifier("org.example.TestSucceedSpockUnskippableSuite", "test success", null, null)] + "test-itr-skipping-spec-setup" | [TestSucceedSetupSpecSpock] | 2 | [ + new TestIdentifier("org.example.TestSucceedSetupSpecSpock", "test success", null, null), + new TestIdentifier("org.example.TestSucceedSetupSpecSpock", "test another success", null, null) + ] + "test-itr-not-skipping-spec-setup" | [TestSucceedSetupSpecSpock] | 2 | [new TestIdentifier("org.example.TestSucceedSetupSpecSpock", "test success", null, null)] + "test-itr-not-skipping-parameterized-spec-setup" | [TestParameterizedSetupSpecSpock] | 2 | [ + new TestIdentifier("org.example.TestParameterizedSetupSpecSpock", "test add 1 and 2", '{"metadata":{"test_name":"test add 1 and 2"}}', null) + ] } def "test flaky retries #testcaseName"() { @@ -86,18 +96,18 @@ class SpockTest extends CiVisibilityInstrumentationTest { assertSpansData(testcaseName, expectedTracesCount) where: - testcaseName | tests | expectedTracesCount | knownTestsList - "test-efd-known-test" | [TestSucceedSpock] | 2 | [new TestIdentifier("org.example.TestSucceedSpock", "test success", null, null)] - "test-efd-known-parameterized-test" | [TestParameterizedSpock] | 3 | [ + testcaseName | tests | expectedTracesCount | knownTestsList + "test-efd-known-test" | [TestSucceedSpock] | 2 | [new TestIdentifier("org.example.TestSucceedSpock", "test success", null, null)] + "test-efd-known-parameterized-test" | [TestParameterizedSpock] | 3 | [ new TestIdentifier("org.example.TestParameterizedSpock", "test add 1 and 2", null, null), new TestIdentifier("org.example.TestParameterizedSpock", "test add 4 and 4", null, null) ] - "test-efd-new-test" | [TestSucceedSpock] | 4 | [] - "test-efd-new-parameterized-test" | [TestParameterizedSpock] | 7 | [] - "test-efd-known-tests-and-new-test" | [TestParameterizedSpock] | 5 | [new TestIdentifier("org.example.TestParameterizedSpock", "test add 1 and 2", null, null)] - "test-efd-new-slow-test" | [TestSucceedSpockSlow] | 3 | [] // is executed only twice - "test-efd-new-very-slow-test" | [TestSucceedSpockVerySlow] | 2 | [] // is executed only once - "test-efd-faulty-session-threshold" | [TestSucceedAndFailedSpock] | 8 | [] + "test-efd-new-test" | [TestSucceedSpock] | 4 | [] + "test-efd-new-parameterized-test" | [TestParameterizedSpock] | 7 | [] + "test-efd-known-tests-and-new-test" | [TestParameterizedSpock] | 5 | [new TestIdentifier("org.example.TestParameterizedSpock", "test add 1 and 2", null, null)] + "test-efd-new-slow-test" | [TestSucceedSpockSlow] | 3 | [] // is executed only twice + "test-efd-new-very-slow-test" | [TestSucceedSpockVerySlow] | 2 | [] // is executed only once + "test-efd-faulty-session-threshold" | [TestSucceedAndFailedSpock] | 8 | [] } private static void runTests(List> classes) { diff --git a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/groovy/org/example/TestParameterizedSetupSpecSpock.groovy b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/groovy/org/example/TestParameterizedSetupSpecSpock.groovy new file mode 100644 index 00000000000..bd48300107f --- /dev/null +++ b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/groovy/org/example/TestParameterizedSetupSpecSpock.groovy @@ -0,0 +1,29 @@ +package org.example + +import datadog.trace.bootstrap.instrumentation.api.AgentScope +import datadog.trace.bootstrap.instrumentation.api.AgentSpan +import datadog.trace.bootstrap.instrumentation.api.AgentTracer +import datadog.trace.bootstrap.instrumentation.api.ScopeSource +import spock.lang.Specification + +class TestParameterizedSetupSpecSpock extends Specification { + + def setupSpec() { + AgentTracer.TracerAPI agentTracer = AgentTracer.get() + AgentSpan span = agentTracer.buildSpan("spock-manual", "spec-setup").start() + try (AgentScope scope = agentTracer.activateSpan(span, ScopeSource.MANUAL)) { + // manually trace spec setup to check whether ITR skips it or not + } + span.finish() + } + + def "test add #a and #b"() { + expect: + a + b == c + + where: + a | b | c + 1 | 2 | 3 + 4 | 4 | 8 + } +} diff --git a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/groovy/org/example/TestSucceedSetupSpecSpock.groovy b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/groovy/org/example/TestSucceedSetupSpecSpock.groovy new file mode 100644 index 00000000000..56c505e200b --- /dev/null +++ b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/groovy/org/example/TestSucceedSetupSpecSpock.groovy @@ -0,0 +1,29 @@ +package org.example + +import datadog.trace.bootstrap.instrumentation.api.AgentScope +import datadog.trace.bootstrap.instrumentation.api.AgentSpan +import datadog.trace.bootstrap.instrumentation.api.AgentTracer +import datadog.trace.bootstrap.instrumentation.api.ScopeSource +import spock.lang.Specification + +class TestSucceedSetupSpecSpock extends Specification { + + def setupSpec() { + AgentTracer.TracerAPI agentTracer = AgentTracer.get() + AgentSpan span = agentTracer.buildSpan("spock-manual", "spec-setup").start() + try (AgentScope scope = agentTracer.activateSpan(span, ScopeSource.MANUAL)) { + // manually trace spec setup to check whether ITR skips it or not + } + span.finish() + } + + def "test success"() { + expect: + 1 == 1 + } + + def "test another success"() { + expect: + 1 == 1 + } +} diff --git a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-itr-not-skipping-parameterized-spec-setup/coverages.ftl b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-itr-not-skipping-parameterized-spec-setup/coverages.ftl new file mode 100644 index 00000000000..8878e547a79 --- /dev/null +++ b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-itr-not-skipping-parameterized-spec-setup/coverages.ftl @@ -0,0 +1 @@ +[ ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-itr-not-skipping-parameterized-spec-setup/events.ftl b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-itr-not-skipping-parameterized-spec-setup/events.ftl new file mode 100644 index 00000000000..a1b382fa3f5 --- /dev/null +++ b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-itr-not-skipping-parameterized-spec-setup/events.ftl @@ -0,0 +1,240 @@ +[ { + "type" : "test_suite_end", + "version" : 1, + "content" : { + "test_session_id" : ${content_test_session_id}, + "test_module_id" : ${content_test_module_id}, + "test_suite_id" : ${content_test_suite_id}, + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "name" : "junit.test_suite", + "resource" : "org.example.TestParameterizedSetupSpecSpock", + "start" : ${content_start}, + "duration" : ${content_duration}, + "error" : 0, + "metrics" : { }, + "meta" : { + "test.type" : "test", + "os.architecture" : ${content_meta_os_architecture}, + "test.source.file" : "dummy_source_path", + "test.module" : "spock-junit-5", + "test.status" : "pass", + "runtime.name" : ${content_meta_runtime_name}, + "runtime.vendor" : ${content_meta_runtime_vendor}, + "env" : "none", + "os.platform" : ${content_meta_os_platform}, + "dummy_ci_tag" : "dummy_ci_tag_value", + "os.version" : ${content_meta_os_version}, + "library_version" : ${content_meta_library_version}, + "component" : "junit", + "span.kind" : "test_suite_end", + "test.suite" : "org.example.TestParameterizedSetupSpecSpock", + "runtime.version" : ${content_meta_runtime_version}, + "test.framework_version" : ${content_meta_test_framework_version}, + "test.framework" : "spock" + } + } +}, { + "type" : "test", + "version" : 2, + "content" : { + "trace_id" : ${content_trace_id}, + "span_id" : ${content_span_id}, + "parent_id" : ${content_parent_id}, + "test_session_id" : ${content_test_session_id}, + "test_module_id" : ${content_test_module_id}, + "test_suite_id" : ${content_test_suite_id}, + "itr_correlation_id" : "itrCorrelationId", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "name" : "junit.test", + "resource" : "org.example.TestParameterizedSetupSpecSpock.test add 1 and 2", + "start" : ${content_start_2}, + "duration" : ${content_duration_2}, + "error" : 0, + "metrics" : { + "process_id" : ${content_metrics_process_id}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "test.source.end" : 18, + "test.source.start" : 12 + }, + "meta" : { + "os.architecture" : ${content_meta_os_architecture}, + "test.source.file" : "dummy_source_path", + "test.source.method" : "test add #a and #b(Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Object;)V", + "test.module" : "spock-junit-5", + "test.status" : "skip", + "language" : "jvm", + "runtime.name" : ${content_meta_runtime_name}, + "os.platform" : ${content_meta_os_platform}, + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "os.version" : ${content_meta_os_version}, + "library_version" : ${content_meta_library_version}, + "test.name" : "test add 1 and 2", + "span.kind" : "test", + "test.suite" : "org.example.TestParameterizedSetupSpecSpock", + "runtime.version" : ${content_meta_runtime_version}, + "runtime-id" : ${content_meta_runtime_id}, + "test.type" : "test", + "test.skip_reason" : "Skipped by Datadog Intelligent Test Runner", + "runtime.vendor" : ${content_meta_runtime_vendor}, + "env" : "none", + "dummy_ci_tag" : "dummy_ci_tag_value", + "test.parameters" : "{\"metadata\":{\"test_name\":\"test add 1 and 2\"}}", + "component" : "junit", + "_dd.profiling.ctx" : "test", + "test.skipped_by_itr" : "true", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.framework" : "spock" + } + } +}, { + "type" : "test", + "version" : 2, + "content" : { + "trace_id" : ${content_trace_id_2}, + "span_id" : ${content_span_id_2}, + "parent_id" : ${content_parent_id}, + "test_session_id" : ${content_test_session_id}, + "test_module_id" : ${content_test_module_id}, + "test_suite_id" : ${content_test_suite_id}, + "itr_correlation_id" : "itrCorrelationId", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "name" : "junit.test", + "resource" : "org.example.TestParameterizedSetupSpecSpock.test add 4 and 4", + "start" : ${content_start_3}, + "duration" : ${content_duration_3}, + "error" : 0, + "metrics" : { + "process_id" : ${content_metrics_process_id}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "test.source.end" : 18, + "test.source.start" : 12 + }, + "meta" : { + "os.architecture" : ${content_meta_os_architecture}, + "test.source.file" : "dummy_source_path", + "test.source.method" : "test add #a and #b(Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Object;)V", + "test.module" : "spock-junit-5", + "test.status" : "pass", + "language" : "jvm", + "runtime.name" : ${content_meta_runtime_name}, + "os.platform" : ${content_meta_os_platform}, + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "os.version" : ${content_meta_os_version}, + "library_version" : ${content_meta_library_version}, + "test.name" : "test add 4 and 4", + "span.kind" : "test", + "test.suite" : "org.example.TestParameterizedSetupSpecSpock", + "runtime.version" : ${content_meta_runtime_version}, + "runtime-id" : ${content_meta_runtime_id}, + "test.type" : "test", + "runtime.vendor" : ${content_meta_runtime_vendor}, + "env" : "none", + "dummy_ci_tag" : "dummy_ci_tag_value", + "test.parameters" : "{\"metadata\":{\"test_name\":\"test add 4 and 4\"}}", + "component" : "junit", + "_dd.profiling.ctx" : "test", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.framework" : "spock" + } + } +}, { + "type" : "span", + "version" : 1, + "content" : { + "trace_id" : ${content_trace_id_3}, + "span_id" : ${content_span_id_3}, + "parent_id" : ${content_test_suite_id}, + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "name" : "spec_setup", + "resource" : "spec_setup", + "start" : ${content_start_4}, + "duration" : ${content_duration_4}, + "error" : 0, + "metrics" : { }, + "meta" : { + "library_version" : ${content_meta_library_version}, + "env" : "none" + } + } +}, { + "type" : "test_session_end", + "version" : 1, + "content" : { + "test_session_id" : ${content_test_session_id}, + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "name" : "junit.test_session", + "resource" : "spock-junit-5", + "start" : ${content_start_5}, + "duration" : ${content_duration_5}, + "error" : 0, + "metrics" : { + "process_id" : ${content_metrics_process_id}, + "test.itr.tests_skipping.count" : 1, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0 + }, + "meta" : { + "os.architecture" : ${content_meta_os_architecture}, + "test.status" : "pass", + "_dd.ci.itr.tests_skipped" : "true", + "language" : "jvm", + "runtime.name" : ${content_meta_runtime_name}, + "os.platform" : ${content_meta_os_platform}, + "os.version" : ${content_meta_os_version}, + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_session_end", + "runtime.version" : ${content_meta_runtime_version}, + "runtime-id" : ${content_meta_runtime_id}, + "test.itr.tests_skipping.enabled" : "true", + "test.type" : "test", + "runtime.vendor" : ${content_meta_runtime_vendor}, + "env" : "none", + "dummy_ci_tag" : "dummy_ci_tag_value", + "component" : "junit", + "_dd.profiling.ctx" : "test", + "test.itr.tests_skipping.type" : "test", + "test.command" : "spock-junit-5", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.framework" : "spock" + } + } +}, { + "type" : "test_module_end", + "version" : 1, + "content" : { + "test_session_id" : ${content_test_session_id}, + "test_module_id" : ${content_test_module_id}, + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "name" : "junit.test_module", + "resource" : "spock-junit-5", + "start" : ${content_start_6}, + "duration" : ${content_duration_6}, + "error" : 0, + "metrics" : { + "test.itr.tests_skipping.count" : 1 + }, + "meta" : { + "test.type" : "test", + "os.architecture" : ${content_meta_os_architecture}, + "test.module" : "spock-junit-5", + "test.status" : "pass", + "_dd.ci.itr.tests_skipped" : "true", + "runtime.name" : ${content_meta_runtime_name}, + "runtime.vendor" : ${content_meta_runtime_vendor}, + "env" : "none", + "os.platform" : ${content_meta_os_platform}, + "dummy_ci_tag" : "dummy_ci_tag_value", + "os.version" : ${content_meta_os_version}, + "library_version" : ${content_meta_library_version}, + "component" : "junit", + "span.kind" : "test_module_end", + "test.itr.tests_skipping.type" : "test", + "runtime.version" : ${content_meta_runtime_version}, + "test.framework_version" : ${content_meta_test_framework_version}, + "test.framework" : "spock", + "test.itr.tests_skipping.enabled" : "true" + } + } +} ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-itr-not-skipping-spec-setup/coverages.ftl b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-itr-not-skipping-spec-setup/coverages.ftl new file mode 100644 index 00000000000..8878e547a79 --- /dev/null +++ b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-itr-not-skipping-spec-setup/coverages.ftl @@ -0,0 +1 @@ +[ ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-itr-not-skipping-spec-setup/events.ftl b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-itr-not-skipping-spec-setup/events.ftl new file mode 100644 index 00000000000..98d5b008e22 --- /dev/null +++ b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-itr-not-skipping-spec-setup/events.ftl @@ -0,0 +1,238 @@ +[ { + "type" : "test_suite_end", + "version" : 1, + "content" : { + "test_session_id" : ${content_test_session_id}, + "test_module_id" : ${content_test_module_id}, + "test_suite_id" : ${content_test_suite_id}, + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "name" : "junit.test_suite", + "resource" : "org.example.TestSucceedSetupSpecSpock", + "start" : ${content_start}, + "duration" : ${content_duration}, + "error" : 0, + "metrics" : { }, + "meta" : { + "test.type" : "test", + "os.architecture" : ${content_meta_os_architecture}, + "test.source.file" : "dummy_source_path", + "test.module" : "spock-junit-5", + "test.status" : "pass", + "runtime.name" : ${content_meta_runtime_name}, + "runtime.vendor" : ${content_meta_runtime_vendor}, + "env" : "none", + "os.platform" : ${content_meta_os_platform}, + "dummy_ci_tag" : "dummy_ci_tag_value", + "os.version" : ${content_meta_os_version}, + "library_version" : ${content_meta_library_version}, + "component" : "junit", + "span.kind" : "test_suite_end", + "test.suite" : "org.example.TestSucceedSetupSpecSpock", + "runtime.version" : ${content_meta_runtime_version}, + "test.framework_version" : ${content_meta_test_framework_version}, + "test.framework" : "spock" + } + } +}, { + "type" : "test", + "version" : 2, + "content" : { + "trace_id" : ${content_trace_id}, + "span_id" : ${content_span_id}, + "parent_id" : ${content_parent_id}, + "test_session_id" : ${content_test_session_id}, + "test_module_id" : ${content_test_module_id}, + "test_suite_id" : ${content_test_suite_id}, + "itr_correlation_id" : "itrCorrelationId", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "name" : "junit.test", + "resource" : "org.example.TestSucceedSetupSpecSpock.test another success", + "start" : ${content_start_2}, + "duration" : ${content_duration_2}, + "error" : 0, + "metrics" : { + "process_id" : ${content_metrics_process_id}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "test.source.end" : 18, + "test.source.start" : 12 + }, + "meta" : { + "os.architecture" : ${content_meta_os_architecture}, + "test.source.file" : "dummy_source_path", + "test.source.method" : "test another success()V", + "test.module" : "spock-junit-5", + "test.status" : "pass", + "language" : "jvm", + "runtime.name" : ${content_meta_runtime_name}, + "os.platform" : ${content_meta_os_platform}, + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "os.version" : ${content_meta_os_version}, + "library_version" : ${content_meta_library_version}, + "test.name" : "test another success", + "span.kind" : "test", + "test.suite" : "org.example.TestSucceedSetupSpecSpock", + "runtime.version" : ${content_meta_runtime_version}, + "runtime-id" : ${content_meta_runtime_id}, + "test.type" : "test", + "runtime.vendor" : ${content_meta_runtime_vendor}, + "env" : "none", + "dummy_ci_tag" : "dummy_ci_tag_value", + "component" : "junit", + "_dd.profiling.ctx" : "test", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.framework" : "spock" + } + } +}, { + "type" : "test", + "version" : 2, + "content" : { + "trace_id" : ${content_trace_id_2}, + "span_id" : ${content_span_id_2}, + "parent_id" : ${content_parent_id}, + "test_session_id" : ${content_test_session_id}, + "test_module_id" : ${content_test_module_id}, + "test_suite_id" : ${content_test_suite_id}, + "itr_correlation_id" : "itrCorrelationId", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "name" : "junit.test", + "resource" : "org.example.TestSucceedSetupSpecSpock.test success", + "start" : ${content_start_3}, + "duration" : ${content_duration_3}, + "error" : 0, + "metrics" : { + "process_id" : ${content_metrics_process_id}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "test.source.end" : 18, + "test.source.start" : 12 + }, + "meta" : { + "os.architecture" : ${content_meta_os_architecture}, + "test.source.file" : "dummy_source_path", + "test.source.method" : "test success()V", + "test.module" : "spock-junit-5", + "test.status" : "skip", + "language" : "jvm", + "runtime.name" : ${content_meta_runtime_name}, + "os.platform" : ${content_meta_os_platform}, + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "os.version" : ${content_meta_os_version}, + "library_version" : ${content_meta_library_version}, + "test.name" : "test success", + "span.kind" : "test", + "test.suite" : "org.example.TestSucceedSetupSpecSpock", + "runtime.version" : ${content_meta_runtime_version}, + "runtime-id" : ${content_meta_runtime_id}, + "test.type" : "test", + "test.skip_reason" : "Skipped by Datadog Intelligent Test Runner", + "runtime.vendor" : ${content_meta_runtime_vendor}, + "env" : "none", + "dummy_ci_tag" : "dummy_ci_tag_value", + "component" : "junit", + "_dd.profiling.ctx" : "test", + "test.skipped_by_itr" : "true", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.framework" : "spock" + } + } +}, { + "type" : "span", + "version" : 1, + "content" : { + "trace_id" : ${content_trace_id_3}, + "span_id" : ${content_span_id_3}, + "parent_id" : ${content_test_suite_id}, + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "name" : "spec_setup", + "resource" : "spec_setup", + "start" : ${content_start_4}, + "duration" : ${content_duration_4}, + "error" : 0, + "metrics" : { }, + "meta" : { + "library_version" : ${content_meta_library_version}, + "env" : "none" + } + } +}, { + "type" : "test_session_end", + "version" : 1, + "content" : { + "test_session_id" : ${content_test_session_id}, + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "name" : "junit.test_session", + "resource" : "spock-junit-5", + "start" : ${content_start_5}, + "duration" : ${content_duration_5}, + "error" : 0, + "metrics" : { + "process_id" : ${content_metrics_process_id}, + "test.itr.tests_skipping.count" : 1, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0 + }, + "meta" : { + "os.architecture" : ${content_meta_os_architecture}, + "test.status" : "pass", + "_dd.ci.itr.tests_skipped" : "true", + "language" : "jvm", + "runtime.name" : ${content_meta_runtime_name}, + "os.platform" : ${content_meta_os_platform}, + "os.version" : ${content_meta_os_version}, + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_session_end", + "runtime.version" : ${content_meta_runtime_version}, + "runtime-id" : ${content_meta_runtime_id}, + "test.itr.tests_skipping.enabled" : "true", + "test.type" : "test", + "runtime.vendor" : ${content_meta_runtime_vendor}, + "env" : "none", + "dummy_ci_tag" : "dummy_ci_tag_value", + "component" : "junit", + "_dd.profiling.ctx" : "test", + "test.itr.tests_skipping.type" : "test", + "test.command" : "spock-junit-5", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.framework" : "spock" + } + } +}, { + "type" : "test_module_end", + "version" : 1, + "content" : { + "test_session_id" : ${content_test_session_id}, + "test_module_id" : ${content_test_module_id}, + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "name" : "junit.test_module", + "resource" : "spock-junit-5", + "start" : ${content_start_6}, + "duration" : ${content_duration_6}, + "error" : 0, + "metrics" : { + "test.itr.tests_skipping.count" : 1 + }, + "meta" : { + "test.type" : "test", + "os.architecture" : ${content_meta_os_architecture}, + "test.module" : "spock-junit-5", + "test.status" : "pass", + "_dd.ci.itr.tests_skipped" : "true", + "runtime.name" : ${content_meta_runtime_name}, + "runtime.vendor" : ${content_meta_runtime_vendor}, + "env" : "none", + "os.platform" : ${content_meta_os_platform}, + "dummy_ci_tag" : "dummy_ci_tag_value", + "os.version" : ${content_meta_os_version}, + "library_version" : ${content_meta_library_version}, + "component" : "junit", + "span.kind" : "test_module_end", + "test.itr.tests_skipping.type" : "test", + "runtime.version" : ${content_meta_runtime_version}, + "test.framework_version" : ${content_meta_test_framework_version}, + "test.framework" : "spock", + "test.itr.tests_skipping.enabled" : "true" + } + } +} ] diff --git a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-itr-skipping-parameterized-spec-setup/coverages.ftl b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-itr-skipping-parameterized-spec-setup/coverages.ftl new file mode 100644 index 00000000000..8878e547a79 --- /dev/null +++ b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-itr-skipping-parameterized-spec-setup/coverages.ftl @@ -0,0 +1 @@ +[ ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-itr-skipping-parameterized-spec-setup/events.ftl b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-itr-skipping-parameterized-spec-setup/events.ftl new file mode 100644 index 00000000000..62461f62870 --- /dev/null +++ b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-itr-skipping-parameterized-spec-setup/events.ftl @@ -0,0 +1,242 @@ +[ { + "type" : "test_suite_end", + "version" : 1, + "content" : { + "test_session_id" : ${content_test_session_id}, + "test_module_id" : ${content_test_module_id}, + "test_suite_id" : ${content_test_suite_id}, + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "name" : "junit.test_suite", + "resource" : "org.example.TestParameterizedSetupSpecSpock", + "start" : ${content_start}, + "duration" : ${content_duration}, + "error" : 0, + "metrics" : { }, + "meta" : { + "test.type" : "test", + "os.architecture" : ${content_meta_os_architecture}, + "test.source.file" : "dummy_source_path", + "test.module" : "spock-junit-5", + "test.status" : "skip", + "runtime.name" : ${content_meta_runtime_name}, + "runtime.vendor" : ${content_meta_runtime_vendor}, + "env" : "none", + "os.platform" : ${content_meta_os_platform}, + "dummy_ci_tag" : "dummy_ci_tag_value", + "os.version" : ${content_meta_os_version}, + "library_version" : ${content_meta_library_version}, + "component" : "junit", + "span.kind" : "test_suite_end", + "test.suite" : "org.example.TestParameterizedSetupSpecSpock", + "runtime.version" : ${content_meta_runtime_version}, + "test.framework_version" : ${content_meta_test_framework_version}, + "test.framework" : "spock" + } + } +}, { + "type" : "test", + "version" : 2, + "content" : { + "trace_id" : ${content_trace_id}, + "span_id" : ${content_span_id}, + "parent_id" : ${content_parent_id}, + "test_session_id" : ${content_test_session_id}, + "test_module_id" : ${content_test_module_id}, + "test_suite_id" : ${content_test_suite_id}, + "itr_correlation_id" : "itrCorrelationId", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "name" : "junit.test", + "resource" : "org.example.TestParameterizedSetupSpecSpock.test add 1 and 2", + "start" : ${content_start_2}, + "duration" : ${content_duration_2}, + "error" : 0, + "metrics" : { + "process_id" : ${content_metrics_process_id}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "test.source.end" : 18, + "test.source.start" : 12 + }, + "meta" : { + "os.architecture" : ${content_meta_os_architecture}, + "test.source.file" : "dummy_source_path", + "test.source.method" : "test add #a and #b(Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Object;)V", + "test.module" : "spock-junit-5", + "test.status" : "skip", + "language" : "jvm", + "runtime.name" : ${content_meta_runtime_name}, + "os.platform" : ${content_meta_os_platform}, + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "os.version" : ${content_meta_os_version}, + "library_version" : ${content_meta_library_version}, + "test.name" : "test add 1 and 2", + "span.kind" : "test", + "test.suite" : "org.example.TestParameterizedSetupSpecSpock", + "runtime.version" : ${content_meta_runtime_version}, + "runtime-id" : ${content_meta_runtime_id}, + "test.type" : "test", + "test.skip_reason" : "Skipped by Datadog Intelligent Test Runner", + "runtime.vendor" : ${content_meta_runtime_vendor}, + "env" : "none", + "dummy_ci_tag" : "dummy_ci_tag_value", + "test.parameters" : "{\"metadata\":{\"test_name\":\"test add 1 and 2\"}}", + "component" : "junit", + "_dd.profiling.ctx" : "test", + "test.skipped_by_itr" : "true", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.framework" : "spock" + } + } +}, { + "type" : "test", + "version" : 2, + "content" : { + "trace_id" : ${content_trace_id_2}, + "span_id" : ${content_span_id_2}, + "parent_id" : ${content_parent_id}, + "test_session_id" : ${content_test_session_id}, + "test_module_id" : ${content_test_module_id}, + "test_suite_id" : ${content_test_suite_id}, + "itr_correlation_id" : "itrCorrelationId", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "name" : "junit.test", + "resource" : "org.example.TestParameterizedSetupSpecSpock.test add 4 and 4", + "start" : ${content_start_3}, + "duration" : ${content_duration_3}, + "error" : 0, + "metrics" : { + "process_id" : ${content_metrics_process_id}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "test.source.end" : 18, + "test.source.start" : 12 + }, + "meta" : { + "os.architecture" : ${content_meta_os_architecture}, + "test.source.file" : "dummy_source_path", + "test.source.method" : "test add #a and #b(Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Object;)V", + "test.module" : "spock-junit-5", + "test.status" : "skip", + "language" : "jvm", + "runtime.name" : ${content_meta_runtime_name}, + "os.platform" : ${content_meta_os_platform}, + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "os.version" : ${content_meta_os_version}, + "library_version" : ${content_meta_library_version}, + "test.name" : "test add 4 and 4", + "span.kind" : "test", + "test.suite" : "org.example.TestParameterizedSetupSpecSpock", + "runtime.version" : ${content_meta_runtime_version}, + "runtime-id" : ${content_meta_runtime_id}, + "test.type" : "test", + "test.skip_reason" : "Skipped by Datadog Intelligent Test Runner", + "runtime.vendor" : ${content_meta_runtime_vendor}, + "env" : "none", + "dummy_ci_tag" : "dummy_ci_tag_value", + "test.parameters" : "{\"metadata\":{\"test_name\":\"test add 4 and 4\"}}", + "component" : "junit", + "_dd.profiling.ctx" : "test", + "test.skipped_by_itr" : "true", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.framework" : "spock" + } + } +}, { + "type" : "span", + "version" : 1, + "content" : { + "trace_id" : ${content_trace_id_3}, + "span_id" : ${content_span_id_3}, + "parent_id" : ${content_test_suite_id}, + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "name" : "spec_setup", + "resource" : "spec_setup", + "start" : ${content_start_4}, + "duration" : ${content_duration_4}, + "error" : 0, + "metrics" : { }, + "meta" : { + "library_version" : ${content_meta_library_version}, + "env" : "none" + } + } +}, { + "type" : "test_session_end", + "version" : 1, + "content" : { + "test_session_id" : ${content_test_session_id}, + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "name" : "junit.test_session", + "resource" : "spock-junit-5", + "start" : ${content_start_5}, + "duration" : ${content_duration_5}, + "error" : 0, + "metrics" : { + "process_id" : ${content_metrics_process_id}, + "test.itr.tests_skipping.count" : 2, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0 + }, + "meta" : { + "os.architecture" : ${content_meta_os_architecture}, + "test.status" : "skip", + "_dd.ci.itr.tests_skipped" : "true", + "language" : "jvm", + "runtime.name" : ${content_meta_runtime_name}, + "os.platform" : ${content_meta_os_platform}, + "os.version" : ${content_meta_os_version}, + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_session_end", + "runtime.version" : ${content_meta_runtime_version}, + "runtime-id" : ${content_meta_runtime_id}, + "test.itr.tests_skipping.enabled" : "true", + "test.type" : "test", + "runtime.vendor" : ${content_meta_runtime_vendor}, + "env" : "none", + "dummy_ci_tag" : "dummy_ci_tag_value", + "component" : "junit", + "_dd.profiling.ctx" : "test", + "test.itr.tests_skipping.type" : "test", + "test.command" : "spock-junit-5", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.framework" : "spock" + } + } +}, { + "type" : "test_module_end", + "version" : 1, + "content" : { + "test_session_id" : ${content_test_session_id}, + "test_module_id" : ${content_test_module_id}, + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "name" : "junit.test_module", + "resource" : "spock-junit-5", + "start" : ${content_start_6}, + "duration" : ${content_duration_6}, + "error" : 0, + "metrics" : { + "test.itr.tests_skipping.count" : 2 + }, + "meta" : { + "test.type" : "test", + "os.architecture" : ${content_meta_os_architecture}, + "test.module" : "spock-junit-5", + "test.status" : "skip", + "_dd.ci.itr.tests_skipped" : "true", + "runtime.name" : ${content_meta_runtime_name}, + "runtime.vendor" : ${content_meta_runtime_vendor}, + "env" : "none", + "os.platform" : ${content_meta_os_platform}, + "dummy_ci_tag" : "dummy_ci_tag_value", + "os.version" : ${content_meta_os_version}, + "library_version" : ${content_meta_library_version}, + "component" : "junit", + "span.kind" : "test_module_end", + "test.itr.tests_skipping.type" : "test", + "runtime.version" : ${content_meta_runtime_version}, + "test.framework_version" : ${content_meta_test_framework_version}, + "test.framework" : "spock", + "test.itr.tests_skipping.enabled" : "true" + } + } +} ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-itr-skipping-spec-setup/coverages.ftl b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-itr-skipping-spec-setup/coverages.ftl new file mode 100644 index 00000000000..8878e547a79 --- /dev/null +++ b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-itr-skipping-spec-setup/coverages.ftl @@ -0,0 +1 @@ +[ ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-itr-skipping-spec-setup/events.ftl b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-itr-skipping-spec-setup/events.ftl new file mode 100644 index 00000000000..41783104f48 --- /dev/null +++ b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-itr-skipping-spec-setup/events.ftl @@ -0,0 +1,222 @@ +[ { + "type" : "test_suite_end", + "version" : 1, + "content" : { + "test_session_id" : ${content_test_session_id}, + "test_module_id" : ${content_test_module_id}, + "test_suite_id" : ${content_test_suite_id}, + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "name" : "junit.test_suite", + "resource" : "org.example.TestSucceedSetupSpecSpock", + "start" : ${content_start}, + "duration" : ${content_duration}, + "error" : 0, + "metrics" : { }, + "meta" : { + "test.type" : "test", + "os.architecture" : ${content_meta_os_architecture}, + "test.source.file" : "dummy_source_path", + "test.module" : "spock-junit-5", + "test.status" : "skip", + "test.skip_reason" : "Skipped by Datadog Intelligent Test Runner", + "runtime.name" : ${content_meta_runtime_name}, + "runtime.vendor" : ${content_meta_runtime_vendor}, + "env" : "none", + "os.platform" : ${content_meta_os_platform}, + "dummy_ci_tag" : "dummy_ci_tag_value", + "os.version" : ${content_meta_os_version}, + "library_version" : ${content_meta_library_version}, + "component" : "junit", + "span.kind" : "test_suite_end", + "test.suite" : "org.example.TestSucceedSetupSpecSpock", + "runtime.version" : ${content_meta_runtime_version}, + "test.framework_version" : ${content_meta_test_framework_version}, + "test.framework" : "spock" + } + } +}, { + "type" : "test", + "version" : 2, + "content" : { + "trace_id" : ${content_trace_id}, + "span_id" : ${content_span_id}, + "parent_id" : ${content_parent_id}, + "test_session_id" : ${content_test_session_id}, + "test_module_id" : ${content_test_module_id}, + "test_suite_id" : ${content_test_suite_id}, + "itr_correlation_id" : "itrCorrelationId", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "name" : "junit.test", + "resource" : "org.example.TestSucceedSetupSpecSpock.test another success", + "start" : ${content_start_2}, + "duration" : ${content_duration_2}, + "error" : 0, + "metrics" : { + "process_id" : ${content_metrics_process_id}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "test.source.end" : 18, + "test.source.start" : 12 + }, + "meta" : { + "os.architecture" : ${content_meta_os_architecture}, + "test.source.file" : "dummy_source_path", + "test.source.method" : "test another success()V", + "test.module" : "spock-junit-5", + "test.status" : "skip", + "language" : "jvm", + "runtime.name" : ${content_meta_runtime_name}, + "os.platform" : ${content_meta_os_platform}, + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "os.version" : ${content_meta_os_version}, + "library_version" : ${content_meta_library_version}, + "test.name" : "test another success", + "span.kind" : "test", + "test.suite" : "org.example.TestSucceedSetupSpecSpock", + "runtime.version" : ${content_meta_runtime_version}, + "runtime-id" : ${content_meta_runtime_id}, + "test.type" : "test", + "test.skip_reason" : "Skipped by Datadog Intelligent Test Runner", + "runtime.vendor" : ${content_meta_runtime_vendor}, + "env" : "none", + "dummy_ci_tag" : "dummy_ci_tag_value", + "component" : "junit", + "_dd.profiling.ctx" : "test", + "test.skipped_by_itr" : "true", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.framework" : "spock" + } + } +}, { + "type" : "test", + "version" : 2, + "content" : { + "trace_id" : ${content_trace_id_2}, + "span_id" : ${content_span_id_2}, + "parent_id" : ${content_parent_id}, + "test_session_id" : ${content_test_session_id}, + "test_module_id" : ${content_test_module_id}, + "test_suite_id" : ${content_test_suite_id}, + "itr_correlation_id" : "itrCorrelationId", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "name" : "junit.test", + "resource" : "org.example.TestSucceedSetupSpecSpock.test success", + "start" : ${content_start_3}, + "duration" : ${content_duration_3}, + "error" : 0, + "metrics" : { + "process_id" : ${content_metrics_process_id}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "test.source.end" : 18, + "test.source.start" : 12 + }, + "meta" : { + "os.architecture" : ${content_meta_os_architecture}, + "test.source.file" : "dummy_source_path", + "test.source.method" : "test success()V", + "test.module" : "spock-junit-5", + "test.status" : "skip", + "language" : "jvm", + "runtime.name" : ${content_meta_runtime_name}, + "os.platform" : ${content_meta_os_platform}, + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "os.version" : ${content_meta_os_version}, + "library_version" : ${content_meta_library_version}, + "test.name" : "test success", + "span.kind" : "test", + "test.suite" : "org.example.TestSucceedSetupSpecSpock", + "runtime.version" : ${content_meta_runtime_version}, + "runtime-id" : ${content_meta_runtime_id}, + "test.type" : "test", + "test.skip_reason" : "Skipped by Datadog Intelligent Test Runner", + "runtime.vendor" : ${content_meta_runtime_vendor}, + "env" : "none", + "dummy_ci_tag" : "dummy_ci_tag_value", + "component" : "junit", + "_dd.profiling.ctx" : "test", + "test.skipped_by_itr" : "true", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.framework" : "spock" + } + } +}, { + "type" : "test_session_end", + "version" : 1, + "content" : { + "test_session_id" : ${content_test_session_id}, + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "name" : "junit.test_session", + "resource" : "spock-junit-5", + "start" : ${content_start_4}, + "duration" : ${content_duration_4}, + "error" : 0, + "metrics" : { + "process_id" : ${content_metrics_process_id}, + "test.itr.tests_skipping.count" : 2, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0 + }, + "meta" : { + "os.architecture" : ${content_meta_os_architecture}, + "test.status" : "skip", + "_dd.ci.itr.tests_skipped" : "true", + "language" : "jvm", + "runtime.name" : ${content_meta_runtime_name}, + "os.platform" : ${content_meta_os_platform}, + "os.version" : ${content_meta_os_version}, + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_session_end", + "runtime.version" : ${content_meta_runtime_version}, + "runtime-id" : ${content_meta_runtime_id}, + "test.itr.tests_skipping.enabled" : "true", + "test.type" : "test", + "runtime.vendor" : ${content_meta_runtime_vendor}, + "env" : "none", + "dummy_ci_tag" : "dummy_ci_tag_value", + "component" : "junit", + "_dd.profiling.ctx" : "test", + "test.itr.tests_skipping.type" : "test", + "test.command" : "spock-junit-5", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.framework" : "spock" + } + } +}, { + "type" : "test_module_end", + "version" : 1, + "content" : { + "test_session_id" : ${content_test_session_id}, + "test_module_id" : ${content_test_module_id}, + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "name" : "junit.test_module", + "resource" : "spock-junit-5", + "start" : ${content_start_5}, + "duration" : ${content_duration_5}, + "error" : 0, + "metrics" : { + "test.itr.tests_skipping.count" : 2 + }, + "meta" : { + "test.type" : "test", + "os.architecture" : ${content_meta_os_architecture}, + "test.module" : "spock-junit-5", + "test.status" : "skip", + "_dd.ci.itr.tests_skipped" : "true", + "runtime.name" : ${content_meta_runtime_name}, + "runtime.vendor" : ${content_meta_runtime_vendor}, + "env" : "none", + "os.platform" : ${content_meta_os_platform}, + "dummy_ci_tag" : "dummy_ci_tag_value", + "os.version" : ${content_meta_os_version}, + "library_version" : ${content_meta_library_version}, + "component" : "junit", + "span.kind" : "test_module_end", + "test.itr.tests_skipping.type" : "test", + "runtime.version" : ${content_meta_runtime_version}, + "test.framework_version" : ${content_meta_test_framework_version}, + "test.framework" : "spock", + "test.itr.tests_skipping.enabled" : "true" + } + } +} ] \ No newline at end of file diff --git a/internal-api/src/main/java/datadog/trace/api/civisibility/events/TestEventsHandler.java b/internal-api/src/main/java/datadog/trace/api/civisibility/events/TestEventsHandler.java index f2e5c7be2b0..2397112c8cd 100644 --- a/internal-api/src/main/java/datadog/trace/api/civisibility/events/TestEventsHandler.java +++ b/internal-api/src/main/java/datadog/trace/api/civisibility/events/TestEventsHandler.java @@ -85,6 +85,8 @@ void onTestIgnore( boolean skip(TestIdentifier test); + boolean isSkippable(TestIdentifier test); + @Nonnull TestRetryPolicy retryPolicy(TestIdentifier test); From c9082c35bb5fbf30310a45e2ba054f43f6cec017 Mon Sep 17 00:00:00 2001 From: Nikita Tkachenko <121111529+nikita-tkachenko-datadog@users.noreply.github.com> Date: Wed, 6 Mar 2024 15:49:55 +0100 Subject: [PATCH 13/30] Fix source path calculation for classes in default package (#6781) --- .../instrumentation/CoverageInstrumentationFilter.java | 4 ++++ .../trace/civisibility/source/index/PackageResolver.java | 3 +++ .../civisibility/source/index/PackageResolverImpl.java | 6 +++++- .../trace/civisibility/source/index/RepoIndexBuilder.java | 8 ++++++-- .../source/index/PackageResolverImplTest.groovy | 4 ++-- 5 files changed, 20 insertions(+), 5 deletions(-) diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/coverage/instrumentation/CoverageInstrumentationFilter.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/coverage/instrumentation/CoverageInstrumentationFilter.java index 014ab5d941a..7b58ea705f7 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/coverage/instrumentation/CoverageInstrumentationFilter.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/coverage/instrumentation/CoverageInstrumentationFilter.java @@ -13,6 +13,10 @@ public CoverageInstrumentationFilter(String[] includedPackages, String[] exclude @Override public boolean test(String className) { + if (!className.contains("/")) { + // always include classes in default package + return true; + } for (String excludedPackage : excludedPackages) { if (className.startsWith(excludedPackage)) { return false; diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/source/index/PackageResolver.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/source/index/PackageResolver.java index e409f2cba71..82ec4ea641d 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/source/index/PackageResolver.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/source/index/PackageResolver.java @@ -2,7 +2,10 @@ import java.io.IOException; import java.nio.file.Path; +import javax.annotation.Nullable; public interface PackageResolver { + /** @return the package path or null if the file is in the default package */ + @Nullable Path getPackage(Path sourceFile) throws IOException; } diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/source/index/PackageResolverImpl.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/source/index/PackageResolverImpl.java index 1da3595d38a..4fc72a67f7e 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/source/index/PackageResolverImpl.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/source/index/PackageResolverImpl.java @@ -7,6 +7,7 @@ import java.nio.file.Files; import java.nio.file.InvalidPathException; import java.nio.file.Path; +import javax.annotation.Nullable; public class PackageResolverImpl implements PackageResolver { @@ -26,7 +27,10 @@ public PackageResolverImpl(FileSystem fileSystem) { *

It simply looks for a line, that contains the package keyword and extracts the * part that goes after it and until the nearest ; character, then verifies that the * extracted part looks plausible by checking the actual file path. + * + * @return the package path or null if the file is in the default package */ + @Nullable @Override public Path getPackage(Path sourceFile) throws IOException { Path folder = sourceFile.getParent(); @@ -65,6 +69,6 @@ public Path getPackage(Path sourceFile) throws IOException { } // apparently there is no package declaration - class is located in the default package - return fileSystem.getPath(""); + return null; } } diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/source/index/RepoIndexBuilder.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/source/index/RepoIndexBuilder.java index 89c16c18971..beda75a35d4 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/source/index/RepoIndexBuilder.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/source/index/RepoIndexBuilder.java @@ -177,8 +177,12 @@ private Path getSourceRoot(Path file) throws IOException { } else if (!sourceType.isResource()) { indexingStats.sourceFilesVisited++; Path packagePath = packageResolver.getPackage(file); - packageTree.add(packagePath); - return getSourceRoot(file, packagePath); + if (packagePath != null) { + packageTree.add(packagePath); + return getSourceRoot(file, packagePath); + } else { + return file.getParent(); + } } else { indexingStats.resourceFilesVisited++; diff --git a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/source/index/PackageResolverImplTest.groovy b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/source/index/PackageResolverImplTest.groovy index 9dae5c64903..5aa21e5db00 100644 --- a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/source/index/PackageResolverImplTest.groovy +++ b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/source/index/PackageResolverImplTest.groovy @@ -21,11 +21,11 @@ class PackageResolverImplTest extends Specification { def packagePath = packageResolver.getPackage(javaFilePath) then: - packagePath == fileSystem.getPath(expectedPackageName) + packagePath == (expectedPackageName != null ? fileSystem.getPath(expectedPackageName) : null) where: path | contents | expectedPackageName - "/root/src/MyClass.java" | CLASS_IN_DEFAULT_PACKAGE | "" + "/root/src/MyClass.java" | CLASS_IN_DEFAULT_PACKAGE | null "/root/src/foo/bar/MyClass.java" | CLASS_IN_FOO_BAR_PACKAGE | "foo/bar" "/root/src/foo/bar/MyClass.java" | BLANK_LINES_BEFORE_PACKAGE | "foo/bar" "/root/src/foo/bar/MyClass.java" | SPACES_BEFORE_PACKAGE | "foo/bar" From 2a65faed38848f644ee3099cb6f4cad011a61f84 Mon Sep 17 00:00:00 2001 From: Wan Tsui Date: Wed, 6 Mar 2024 11:19:26 -0500 Subject: [PATCH 14/30] Clarify which three are required (#6784) --- docs/add_new_instrumentation.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/add_new_instrumentation.md b/docs/add_new_instrumentation.md index 1e213323d00..49eede54c4e 100644 --- a/docs/add_new_instrumentation.md +++ b/docs/add_new_instrumentation.md @@ -351,10 +351,10 @@ If Gradle is behaving badly you might try: There are four verification strategies, three of which are mandatory. -- [Muzzle directives](./how_instrumentations_work.md#muzzle) -- [Instrumentation Tests](./how_instrumentations_work.md#instrumentation-tests) -- [Latest Dependency Tests](./how_instrumentations_work.md#latest-dependency-tests) -- [Smoke tests](./how_instrumentations_work.md#smoke-tests) +- [Muzzle directives](./how_instrumentations_work.md#muzzle) (Required) +- [Instrumentation Tests](./how_instrumentations_work.md#instrumentation-tests) (Required) +- [Latest Dependency Tests](./how_instrumentations_work.md#latest-dependency-tests) (Required) +- [Smoke tests](./how_instrumentations_work.md#smoke-tests) (Not required) All integrations must include sufficient test coverage. This HTTP client integration will include a [standard HTTP test class](../dd-java-agent/instrumentation/google-http-client/src/test/groovy/GoogleHttpClientTest.groovy) From 30e6c9354aefe74d9a730e70bae5732953a4bb90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vandon?= Date: Wed, 6 Mar 2024 17:20:50 +0100 Subject: [PATCH 15/30] Add host and db name to tags injected in SQL comments (#6778) --- .../jdbc/SQLCommenterBenchmark.java | 13 ++- ...BMCompatibleConnectionInstrumentation.java | 16 ++- .../instrumentation/jdbc/SQLCommenter.java | 54 ++++++++- .../jdbc/StatementInstrumentation.java | 2 + .../src/test/groovy/SQLCommenterTest.groovy | 104 +++++++++--------- 5 files changed, 130 insertions(+), 59 deletions(-) diff --git a/dd-java-agent/instrumentation/jdbc/src/jmh/java/datadog/trace/instrumentation/jdbc/SQLCommenterBenchmark.java b/dd-java-agent/instrumentation/jdbc/src/jmh/java/datadog/trace/instrumentation/jdbc/SQLCommenterBenchmark.java index 7e754e7c7c3..e2a5b195416 100644 --- a/dd-java-agent/instrumentation/jdbc/src/jmh/java/datadog/trace/instrumentation/jdbc/SQLCommenterBenchmark.java +++ b/dd-java-agent/instrumentation/jdbc/src/jmh/java/datadog/trace/instrumentation/jdbc/SQLCommenterBenchmark.java @@ -15,8 +15,9 @@ public class SQLCommenterBenchmark { private static final String traceParent = "00-00000000000000007fffffffffffffff-000000024cb016ea-01"; - private static final String injectionMode = "full"; private static final String dbService = "users-db"; + private static final String hostname = "my-host"; + private static final String dbName = "credit-card-numbers"; private static final String parentService = "parent"; private static final String env = "env"; private static final String version = "version"; @@ -26,6 +27,14 @@ public class SQLCommenterBenchmark { public void testToComment() { StringBuilder stringBuilder = new StringBuilder(); SQLCommenter.toComment( - stringBuilder, injectTrace, parentService, dbService, env, version, traceParent); + stringBuilder, + injectTrace, + parentService, + dbService, + hostname, + dbName, + env, + version, + traceParent); } } diff --git a/dd-java-agent/instrumentation/jdbc/src/main/java/datadog/trace/instrumentation/jdbc/DBMCompatibleConnectionInstrumentation.java b/dd-java-agent/instrumentation/jdbc/src/main/java/datadog/trace/instrumentation/jdbc/DBMCompatibleConnectionInstrumentation.java index 6498ee5b653..6d73762c887 100644 --- a/dd-java-agent/instrumentation/jdbc/src/main/java/datadog/trace/instrumentation/jdbc/DBMCompatibleConnectionInstrumentation.java +++ b/dd-java-agent/instrumentation/jdbc/src/main/java/datadog/trace/instrumentation/jdbc/DBMCompatibleConnectionInstrumentation.java @@ -116,9 +116,21 @@ public static String onEnter( JDBCDecorator.parseDBInfo( connection, InstrumentationContext.get(Connection.class, DBInfo.class)); if (dbInfo.getType().equals("sqlserver")) { - sql = SQLCommenter.append(sql, DECORATE.getDbService(dbInfo), dbInfo.getType()); + sql = + SQLCommenter.append( + sql, + DECORATE.getDbService(dbInfo), + dbInfo.getType(), + dbInfo.getHost(), + dbInfo.getDb()); } else { - sql = SQLCommenter.prepend(sql, DECORATE.getDbService(dbInfo), dbInfo.getType()); + sql = + SQLCommenter.prepend( + sql, + DECORATE.getDbService(dbInfo), + dbInfo.getType(), + dbInfo.getHost(), + dbInfo.getDb()); } return inputSql; } diff --git a/dd-java-agent/instrumentation/jdbc/src/main/java/datadog/trace/instrumentation/jdbc/SQLCommenter.java b/dd-java-agent/instrumentation/jdbc/src/main/java/datadog/trace/instrumentation/jdbc/SQLCommenter.java index 1f75a8dda92..1e939044d4d 100644 --- a/dd-java-agent/instrumentation/jdbc/src/main/java/datadog/trace/instrumentation/jdbc/SQLCommenter.java +++ b/dd-java-agent/instrumentation/jdbc/src/main/java/datadog/trace/instrumentation/jdbc/SQLCommenter.java @@ -13,6 +13,8 @@ public class SQLCommenter { private static final String UTF8 = StandardCharsets.UTF_8.toString(); private static final String PARENT_SERVICE = encode("ddps"); private static final String DATABASE_SERVICE = encode("dddbs"); + private static final String DD_HOSTNAME = encode("ddh"); + private static final String DD_DB_NAME = encode("dddb"); private static final String DD_ENV = encode("dde"); private static final String DD_VERSION = encode("ddpv"); private static final String TRACEPARENT = encode("traceparent"); @@ -24,18 +26,30 @@ public class SQLCommenter { private static final String CLOSE_COMMENT = "*/"; private static final int INITIAL_CAPACITY = computeInitialCapacity(); - public static String append(final String sql, final String dbService, final String dbType) { - return inject(sql, dbService, dbType, null, false, true); + public static String append( + final String sql, + final String dbService, + final String dbType, + final String hostname, + final String dbName) { + return inject(sql, dbService, dbType, hostname, dbName, null, false, true); } - public static String prepend(final String sql, final String dbService, final String dbType) { - return inject(sql, dbService, dbType, null, false, false); + public static String prepend( + final String sql, + final String dbService, + final String dbType, + final String hostname, + final String dbName) { + return inject(sql, dbService, dbType, hostname, dbName, null, false, false); } public static String inject( final String sql, final String dbService, final String dbType, + final String hostname, + final String dbName, final String traceParent, final boolean injectTrace, final boolean appendComment) { @@ -72,12 +86,30 @@ public static String inject( sb.append(SPACE); sb.append(OPEN_COMMENT); commentAdded = - toComment(sb, injectTrace, parentService, dbService, env, version, traceParent); + toComment( + sb, + injectTrace, + parentService, + dbService, + hostname, + dbName, + env, + version, + traceParent); sb.append(CLOSE_COMMENT); } else { sb.append(OPEN_COMMENT); commentAdded = - toComment(sb, injectTrace, parentService, dbService, env, version, traceParent); + toComment( + sb, + injectTrace, + parentService, + dbService, + hostname, + dbName, + env, + version, + traceParent); sb.append(CLOSE_COMMENT); sb.append(SPACE); sb.append(sql); @@ -106,6 +138,10 @@ private static boolean hasDDComment(String sql, final boolean appendComment) { found = true; } else if (hasMatchingSubstring(sql, startIdx, DATABASE_SERVICE)) { found = true; + } else if (hasMatchingSubstring(sql, startIdx, DD_HOSTNAME)) { + found = true; + } else if (hasMatchingSubstring(sql, startIdx, DD_DB_NAME)) { + found = true; } else if (hasMatchingSubstring(sql, startIdx, DD_ENV)) { found = true; } else if (hasMatchingSubstring(sql, startIdx, DD_VERSION)) { @@ -141,12 +177,16 @@ protected static boolean toComment( final boolean injectTrace, final String parentService, final String dbService, + final String hostname, + final String dbName, final String env, final String version, final String traceparent) { int emptySize = sb.length(); append(sb, PARENT_SERVICE, parentService, false); append(sb, DATABASE_SERVICE, dbService, sb.length() > emptySize); + append(sb, DD_HOSTNAME, hostname, sb.length() > emptySize); + append(sb, DD_DB_NAME, dbName, sb.length() > emptySize); append(sb, DD_ENV, env, sb.length() > emptySize); append(sb, DD_VERSION, version, sb.length() > emptySize); if (injectTrace) { @@ -203,6 +243,8 @@ private static int computeInitialCapacity() { int tagKeysLen = PARENT_SERVICE.length() + DATABASE_SERVICE.length() + + DD_HOSTNAME.length() + + DD_DB_NAME.length() + DD_ENV.length() + DD_VERSION.length() + TRACEPARENT.length(); diff --git a/dd-java-agent/instrumentation/jdbc/src/main/java/datadog/trace/instrumentation/jdbc/StatementInstrumentation.java b/dd-java-agent/instrumentation/jdbc/src/main/java/datadog/trace/instrumentation/jdbc/StatementInstrumentation.java index 6c85408caf5..6cf3ded4bc3 100644 --- a/dd-java-agent/instrumentation/jdbc/src/main/java/datadog/trace/instrumentation/jdbc/StatementInstrumentation.java +++ b/dd-java-agent/instrumentation/jdbc/src/main/java/datadog/trace/instrumentation/jdbc/StatementInstrumentation.java @@ -106,6 +106,8 @@ public static AgentScope onEnter( sql, span.getServiceName(), dbInfo.getType(), + dbInfo.getHost(), + dbInfo.getDb(), traceParent, injectTraceContext, appendComment); diff --git a/dd-java-agent/instrumentation/jdbc/src/test/groovy/SQLCommenterTest.groovy b/dd-java-agent/instrumentation/jdbc/src/test/groovy/SQLCommenterTest.groovy index 843f5c8bd07..4d3fc979429 100644 --- a/dd-java-agent/instrumentation/jdbc/src/test/groovy/SQLCommenterTest.groovy +++ b/dd-java-agent/instrumentation/jdbc/src/test/groovy/SQLCommenterTest.groovy @@ -12,12 +12,12 @@ class SQLCommenterTest extends AgentTestRunner { when: String sqlWithComment = "" if (injectTrace) { - sqlWithComment = SQLCommenter.inject(query, dbService, dbType, traceParent, true, appendComment) + sqlWithComment = SQLCommenter.inject(query, dbService, dbType, host, dbName, traceParent, true, appendComment) } else { if (appendComment) { - sqlWithComment = SQLCommenter.append(query, dbService, dbType) + sqlWithComment = SQLCommenter.append(query, dbService, dbType, host, dbName) } else { - sqlWithComment = SQLCommenter.prepend(query, dbService, dbType) + sqlWithComment = SQLCommenter.prepend(query, dbService, dbType, host, dbName) } } @@ -27,51 +27,57 @@ class SQLCommenterTest extends AgentTestRunner { sqlWithComment == expected where: - query | ddService | ddEnv | dbService | dbType | ddVersion | injectTrace | appendComment | traceParent | expected - "SELECT * FROM foo" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "TestVersion" | true | true | "00-00000000000000007fffffffffffffff-000000024cb016ea-00" | "SELECT * FROM foo /*ddps='SqlCommenter',dddbs='my-service',dde='Test',ddpv='TestVersion',traceparent='00-00000000000000007fffffffffffffff-000000024cb016ea-00'*/" - "SELECT * FROM foo" | "SqlCommenter" | "Test" | "my-service" | "postgres" | "TestVersion" | true | true | "00-00000000000000007fffffffffffffff-000000024cb016ea-00" | "SELECT * FROM foo /*ddps='SqlCommenter',dddbs='my-service',dde='Test',ddpv='TestVersion',traceparent='00-00000000000000007fffffffffffffff-000000024cb016ea-00'*/" - "{call dogshelterProc(?, ?)}" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "TestVersion" | true | true | "00-00000000000000007fffffffffffffff-000000024cb016ea-00" | "{call dogshelterProc(?, ?)} /*ddps='SqlCommenter',dddbs='my-service',dde='Test',ddpv='TestVersion',traceparent='00-00000000000000007fffffffffffffff-000000024cb016ea-00'*/" - "{call dogshelterProc(?, ?)}" | "SqlCommenter" | "Test" | "my-service" | "postgres" | "TestVersion" | true | true | "00-00000000000000007fffffffffffffff-000000024cb016ea-00" | "{call dogshelterProc(?, ?)}" - "SELECT * FROM foo" | "" | "Test" | "" | "mysql" | "TestVersion" | true | true | "00-00000000000000007fffffffffffffff-000000024cb016ea-00" | "SELECT * FROM foo /*dde='Test',ddpv='TestVersion',traceparent='00-00000000000000007fffffffffffffff-000000024cb016ea-00'*/" - "SELECT * FROM foo" | "" | "Test" | "my-service" | "mysql" | "TestVersion" | true | true | "00-00000000000000007fffffffffffffff-000000024cb016ea-00" | "SELECT * FROM foo /*dddbs='my-service',dde='Test',ddpv='TestVersion',traceparent='00-00000000000000007fffffffffffffff-000000024cb016ea-00'*/" - "SELECT * FROM foo" | "" | "Test" | "" | "" | "" | true | true | "00-00000000000000007fffffffffffffff-000000024cb016ea-00" | "SELECT * FROM foo /*dde='Test',traceparent='00-00000000000000007fffffffffffffff-000000024cb016ea-00'*/" - "SELECT * FROM foo" | "" | "" | "" | "" | "" | true | true | "00-00000000000000007fffffffffffffff-000000024cb016ea-00" | "SELECT * FROM foo /*traceparent='00-00000000000000007fffffffffffffff-000000024cb016ea-00'*/" - "SELECT * from FOO -- test query" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "TestVersion" | true | true | "00-00000000000000007fffffffffffffff-000000024cb016ea-01" | "SELECT * from FOO -- test query /*ddps='SqlCommenter',dddbs='my-service',dde='Test',ddpv='TestVersion',traceparent='00-00000000000000007fffffffffffffff-000000024cb016ea-01'*/" - "SELECT /* customer-comment */ * FROM foo" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "TestVersion" | true | true | "00-00000000000000007fffffffffffffff-000000024cb016ea-01" | "SELECT /* customer-comment */ * FROM foo /*ddps='SqlCommenter',dddbs='my-service',dde='Test',ddpv='TestVersion',traceparent='00-00000000000000007fffffffffffffff-000000024cb016ea-01'*/" - "SELECT * FROM foo" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "TestVersion" | false | true | null | "SELECT * FROM foo /*ddps='SqlCommenter',dddbs='my-service',dde='Test',ddpv='TestVersion'*/" - "SELECT /* customer-comment */ * FROM foo" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "TestVersion" | false | true | null | "SELECT /* customer-comment */ * FROM foo /*ddps='SqlCommenter',dddbs='my-service',dde='Test',ddpv='TestVersion'*/" - "SELECT * from FOO -- test query" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "TestVersion" | false | true | null | "SELECT * from FOO -- test query /*ddps='SqlCommenter',dddbs='my-service',dde='Test',ddpv='TestVersion'*/" - "" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "TestVersion" | true | true | "00-00000000000000007fffffffffffffff-000000024cb016ea-00" | "" - " " | "SqlCommenter" | "Test" | "my-service" | "mysql" | "TestVersion" | true | true | "00-00000000000000007fffffffffffffff-000000024cb016ea-01" | " /*ddps='SqlCommenter',dddbs='my-service',dde='Test',ddpv='TestVersion',traceparent='00-00000000000000007fffffffffffffff-000000024cb016ea-01'*/" - "" | "SqlCommenter" | "Test" | "postgres" | "mysql" | "TestVersion" | true | true | "00-00000000000000007fffffffffffffff-000000024cb016ea-00" | "" - " " | "SqlCommenter" | "Test" | "postgres" | "mysql" | "TestVersion" | true | true | "00-00000000000000007fffffffffffffff-000000024cb016ea-01" | " /*ddps='SqlCommenter',dddbs='postgres',dde='Test',ddpv='TestVersion',traceparent='00-00000000000000007fffffffffffffff-000000024cb016ea-01'*/" - "SELECT * FROM foo /*dddbs='my-service',dde='Test',ddps='SqlCommenter',ddpv='TestVersion'*/" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "TestVersion" | false | true | null | "SELECT * FROM foo /*dddbs='my-service',dde='Test',ddps='SqlCommenter',ddpv='TestVersion'*/" - "SELECT * FROM foo /*dde='Test',ddps='SqlCommenter',ddpv='TestVersion'*/" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "TestVersion" | false | true | null | "SELECT * FROM foo /*dde='Test',ddps='SqlCommenter',ddpv='TestVersion'*/" - "SELECT * FROM foo /*ddps='SqlCommenter',ddpv='TestVersion'*/" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "TestVersion" | false | true | null | "SELECT * FROM foo /*ddps='SqlCommenter',ddpv='TestVersion'*/" - "SELECT * FROM foo /*ddpv='TestVersion'*/" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "TestVersion" | false | true | null | "SELECT * FROM foo /*ddpv='TestVersion'*/" - "/*ddjk its a customer */ SELECT * FROM foo" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "TestVersion" | false | true | null | "/*ddjk its a customer */ SELECT * FROM foo /*ddps='SqlCommenter',dddbs='my-service',dde='Test',ddpv='TestVersion'*/" - "SELECT * FROM foo /*traceparent='00-00000000000000007fffffffffffffff-000000024cb016ea-01'*/" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "TestVersion" | false | true | null | "SELECT * FROM foo /*traceparent='00-00000000000000007fffffffffffffff-000000024cb016ea-01'*/" - "/*customer-comment*/ SELECT * FROM foo" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "TestVersion" | false | true | null | "/*customer-comment*/ SELECT * FROM foo /*ddps='SqlCommenter',dddbs='my-service',dde='Test',ddpv='TestVersion'*/" - "/*traceparent" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "TestVersion" | false | true | null | "/*traceparent /*ddps='SqlCommenter',dddbs='my-service',dde='Test',ddpv='TestVersion'*/" - "SELECT * FROM foo" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "TestVersion" | true | false | "00-00000000000000007fffffffffffffff-000000024cb016ea-00" | "/*ddps='SqlCommenter',dddbs='my-service',dde='Test',ddpv='TestVersion',traceparent='00-00000000000000007fffffffffffffff-000000024cb016ea-00'*/ SELECT * FROM foo" - "SELECT * FROM foo" | "" | "Test" | "" | "mysql" | "TestVersion" | true | false | "00-00000000000000007fffffffffffffff-000000024cb016ea-00" | "/*dde='Test',ddpv='TestVersion',traceparent='00-00000000000000007fffffffffffffff-000000024cb016ea-00'*/ SELECT * FROM foo" - "SELECT * FROM foo" | "" | "Test" | "my-service" | "mysql" | "TestVersion" | true | false | "00-00000000000000007fffffffffffffff-000000024cb016ea-00" | "/*dddbs='my-service',dde='Test',ddpv='TestVersion',traceparent='00-00000000000000007fffffffffffffff-000000024cb016ea-00'*/ SELECT * FROM foo" - "SELECT * FROM foo" | "" | "Test" | "" | "" | "" | true | false | "00-00000000000000007fffffffffffffff-000000024cb016ea-00" | "/*dde='Test',traceparent='00-00000000000000007fffffffffffffff-000000024cb016ea-00'*/ SELECT * FROM foo" - "SELECT * FROM foo" | "" | "" | "" | "" | "" | true | false | "00-00000000000000007fffffffffffffff-000000024cb016ea-00" | "/*traceparent='00-00000000000000007fffffffffffffff-000000024cb016ea-00'*/ SELECT * FROM foo" - "SELECT * from FOO -- test query" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "TestVersion" | true | false | "00-00000000000000007fffffffffffffff-000000024cb016ea-01" | "/*ddps='SqlCommenter',dddbs='my-service',dde='Test',ddpv='TestVersion',traceparent='00-00000000000000007fffffffffffffff-000000024cb016ea-01'*/ SELECT * from FOO -- test query" - "SELECT /* customer-comment */ * FROM foo" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "TestVersion" | true | false | "00-00000000000000007fffffffffffffff-000000024cb016ea-01" | "/*ddps='SqlCommenter',dddbs='my-service',dde='Test',ddpv='TestVersion',traceparent='00-00000000000000007fffffffffffffff-000000024cb016ea-01'*/ SELECT /* customer-comment */ * FROM foo" - "SELECT * FROM foo" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "TestVersion" | false | false | null | "/*ddps='SqlCommenter',dddbs='my-service',dde='Test',ddpv='TestVersion'*/ SELECT * FROM foo" - "SELECT /* customer-comment */ * FROM foo" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "TestVersion" | false | false | null | "/*ddps='SqlCommenter',dddbs='my-service',dde='Test',ddpv='TestVersion'*/ SELECT /* customer-comment */ * FROM foo" - "SELECT * from FOO -- test query" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "TestVersion" | false | false | null | "/*ddps='SqlCommenter',dddbs='my-service',dde='Test',ddpv='TestVersion'*/ SELECT * from FOO -- test query" - "" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "TestVersion" | true | false | "00-00000000000000007fffffffffffffff-000000024cb016ea-00" | "" - " " | "SqlCommenter" | "Test" | "my-service" | "mysql" | "TestVersion" | true | false | "00-00000000000000007fffffffffffffff-000000024cb016ea-01" | "/*ddps='SqlCommenter',dddbs='my-service',dde='Test',ddpv='TestVersion',traceparent='00-00000000000000007fffffffffffffff-000000024cb016ea-01'*/ " - "/*dddbs='my-service',dde='Test',ddps='SqlCommenter',ddpv='TestVersion'*/ SELECT * FROM foo" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "TestVersion" | false | false | null | "/*dddbs='my-service',dde='Test',ddps='SqlCommenter',ddpv='TestVersion'*/ SELECT * FROM foo" - "/*dde='Test',ddps='SqlCommenter',ddpv='TestVersion'*/ SELECT * FROM foo" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "TestVersion" | false | false | null | "/*dde='Test',ddps='SqlCommenter',ddpv='TestVersion'*/ SELECT * FROM foo" - "/*ddps='SqlCommenter',ddpv='TestVersion'*/ SELECT * FROM foo" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "TestVersion" | false | false | null | "/*ddps='SqlCommenter',ddpv='TestVersion'*/ SELECT * FROM foo" - "/*ddpv='TestVersion'*/ SELECT * FROM foo" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "TestVersion" | false | false | null | "/*ddpv='TestVersion'*/ SELECT * FROM foo" - "/*ddjk its a customer */ SELECT * FROM foo" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "TestVersion" | false | false | null | "/*ddps='SqlCommenter',dddbs='my-service',dde='Test',ddpv='TestVersion'*/ /*ddjk its a customer */ SELECT * FROM foo" - "/*traceparent='00-00000000000000007fffffffffffffff-000000024cb016ea-01'*/ SELECT * FROM foo" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "TestVersion" | false | false | null | "/*traceparent='00-00000000000000007fffffffffffffff-000000024cb016ea-01'*/ SELECT * FROM foo" - "/*customer-comment*/ SELECT * FROM foo" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "TestVersion" | false | false | null | "/*ddps='SqlCommenter',dddbs='my-service',dde='Test',ddpv='TestVersion'*/ /*customer-comment*/ SELECT * FROM foo" - "/*traceparent" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "TestVersion" | false | false | null | "/*ddps='SqlCommenter',dddbs='my-service',dde='Test',ddpv='TestVersion'*/ /*traceparent" + query | ddService | ddEnv | dbService | dbType | host | dbName | ddVersion | injectTrace | appendComment | traceParent | expected + "SELECT * FROM foo" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "h" | "n" | "TestVersion" | true | true | "00-00000000000000007fffffffffffffff-000000024cb016ea-00" | "SELECT * FROM foo /*ddps='SqlCommenter',dddbs='my-service',ddh='h',dddb='n',dde='Test',ddpv='TestVersion',traceparent='00-00000000000000007fffffffffffffff-000000024cb016ea-00'*/" + "SELECT * FROM foo" | "SqlCommenter" | "Test" | "my-service" | "postgres" | "h" | "n" | "TestVersion" | true | true | "00-00000000000000007fffffffffffffff-000000024cb016ea-00" | "SELECT * FROM foo /*ddps='SqlCommenter',dddbs='my-service',ddh='h',dddb='n',dde='Test',ddpv='TestVersion',traceparent='00-00000000000000007fffffffffffffff-000000024cb016ea-00'*/" + "{call dogshelterProc(?, ?)}" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "h" | "n" | "TestVersion" | true | true | "00-00000000000000007fffffffffffffff-000000024cb016ea-00" | "{call dogshelterProc(?, ?)} /*ddps='SqlCommenter',dddbs='my-service',ddh='h',dddb='n',dde='Test',ddpv='TestVersion',traceparent='00-00000000000000007fffffffffffffff-000000024cb016ea-00'*/" + "{call dogshelterProc(?, ?)}" | "SqlCommenter" | "Test" | "my-service" | "postgres" | "h" | "n" | "TestVersion" | true | true | "00-00000000000000007fffffffffffffff-000000024cb016ea-00" | "{call dogshelterProc(?, ?)}" + "SELECT * FROM foo" | "" | "Test" | "" | "mysql" | "h" | "n" | "TestVersion" | true | true | "00-00000000000000007fffffffffffffff-000000024cb016ea-00" | "SELECT * FROM foo /*ddh='h',dddb='n',dde='Test',ddpv='TestVersion',traceparent='00-00000000000000007fffffffffffffff-000000024cb016ea-00'*/" + "SELECT * FROM foo" | "" | "Test" | "my-service" | "mysql" | "h" | "n" | "TestVersion" | true | true | "00-00000000000000007fffffffffffffff-000000024cb016ea-00" | "SELECT * FROM foo /*dddbs='my-service',ddh='h',dddb='n',dde='Test',ddpv='TestVersion',traceparent='00-00000000000000007fffffffffffffff-000000024cb016ea-00'*/" + "SELECT * FROM foo" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "" | "" | "TestVersion" | true | true | "00-00000000000000007fffffffffffffff-000000024cb016ea-00" | "SELECT * FROM foo /*ddps='SqlCommenter',dddbs='my-service',dde='Test',ddpv='TestVersion',traceparent='00-00000000000000007fffffffffffffff-000000024cb016ea-00'*/" + "SELECT * FROM foo" | "" | "Test" | "" | "" | "h" | "n" | "" | true | true | "00-00000000000000007fffffffffffffff-000000024cb016ea-00" | "SELECT * FROM foo /*ddh='h',dddb='n',dde='Test',traceparent='00-00000000000000007fffffffffffffff-000000024cb016ea-00'*/" + "SELECT * FROM foo" | "" | "" | "" | "" | "h" | "n" | "" | true | true | "00-00000000000000007fffffffffffffff-000000024cb016ea-00" | "SELECT * FROM foo /*ddh='h',dddb='n',traceparent='00-00000000000000007fffffffffffffff-000000024cb016ea-00'*/" + "SELECT * FROM foo" | "" | "" | "" | "" | "" | "" | "" | true | true | "00-00000000000000007fffffffffffffff-000000024cb016ea-00" | "SELECT * FROM foo /*traceparent='00-00000000000000007fffffffffffffff-000000024cb016ea-00'*/" + "SELECT * from FOO -- test query" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "h" | "n" | "TestVersion" | true | true | "00-00000000000000007fffffffffffffff-000000024cb016ea-01" | "SELECT * from FOO -- test query /*ddps='SqlCommenter',dddbs='my-service',ddh='h',dddb='n',dde='Test',ddpv='TestVersion',traceparent='00-00000000000000007fffffffffffffff-000000024cb016ea-01'*/" + "SELECT /* customer-comment */ * FROM foo" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "h" | "n" | "TestVersion" | true | true | "00-00000000000000007fffffffffffffff-000000024cb016ea-01" | "SELECT /* customer-comment */ * FROM foo /*ddps='SqlCommenter',dddbs='my-service',ddh='h',dddb='n',dde='Test',ddpv='TestVersion',traceparent='00-00000000000000007fffffffffffffff-000000024cb016ea-01'*/" + "SELECT * FROM foo" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "h" | "n" | "TestVersion" | false | true | null | "SELECT * FROM foo /*ddps='SqlCommenter',dddbs='my-service',ddh='h',dddb='n',dde='Test',ddpv='TestVersion'*/" + "SELECT /* customer-comment */ * FROM foo" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "h" | "n" | "TestVersion" | false | true | null | "SELECT /* customer-comment */ * FROM foo /*ddps='SqlCommenter',dddbs='my-service',ddh='h',dddb='n',dde='Test',ddpv='TestVersion'*/" + "SELECT * from FOO -- test query" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "h" | "n" | "TestVersion" | false | true | null | "SELECT * from FOO -- test query /*ddps='SqlCommenter',dddbs='my-service',ddh='h',dddb='n',dde='Test',ddpv='TestVersion'*/" + "" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "h" | "n" | "TestVersion" | true | true | "00-00000000000000007fffffffffffffff-000000024cb016ea-00" | "" + " " | "SqlCommenter" | "Test" | "my-service" | "mysql" | "h" | "n" | "TestVersion" | true | true | "00-00000000000000007fffffffffffffff-000000024cb016ea-01" | " /*ddps='SqlCommenter',dddbs='my-service',ddh='h',dddb='n',dde='Test',ddpv='TestVersion',traceparent='00-00000000000000007fffffffffffffff-000000024cb016ea-01'*/" + "" | "SqlCommenter" | "Test" | "postgres" | "mysql" | "h" | "n" | "TestVersion" | true | true | "00-00000000000000007fffffffffffffff-000000024cb016ea-00" | "" + " " | "SqlCommenter" | "Test" | "postgres" | "mysql" | "h" | "n" | "TestVersion" | true | true | "00-00000000000000007fffffffffffffff-000000024cb016ea-01" | " /*ddps='SqlCommenter',dddbs='postgres',ddh='h',dddb='n',dde='Test',ddpv='TestVersion',traceparent='00-00000000000000007fffffffffffffff-000000024cb016ea-01'*/" + "SELECT * FROM foo /*dddbs='my-service',ddh='h',dddb='n',dde='Test',ddps='SqlCommenter',ddpv='TestVersion'*/" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "h" | "n" | "TestVersion" | false | true | null | "SELECT * FROM foo /*dddbs='my-service',ddh='h',dddb='n',dde='Test',ddps='SqlCommenter',ddpv='TestVersion'*/" + "SELECT * FROM foo /*ddh='h',dddb='n',dde='Test',ddps='SqlCommenter',ddpv='TestVersion'*/" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "h" | "n" | "TestVersion" | false | true | null | "SELECT * FROM foo /*ddh='h',dddb='n',dde='Test',ddps='SqlCommenter',ddpv='TestVersion'*/" + "SELECT * FROM foo /*dddb='n',dde='Test',ddps='SqlCommenter',ddpv='TestVersion'*/" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "h" | "n" | "TestVersion" | false | true | null | "SELECT * FROM foo /*dddb='n',dde='Test',ddps='SqlCommenter',ddpv='TestVersion'*/" + "SELECT * FROM foo /*dde='Test',ddps='SqlCommenter',ddpv='TestVersion'*/" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "h" | "n" | "TestVersion" | false | true | null | "SELECT * FROM foo /*dde='Test',ddps='SqlCommenter',ddpv='TestVersion'*/" + "SELECT * FROM foo /*ddps='SqlCommenter',ddpv='TestVersion'*/" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "h" | "n" | "TestVersion" | false | true | null | "SELECT * FROM foo /*ddps='SqlCommenter',ddpv='TestVersion'*/" + "SELECT * FROM foo /*ddpv='TestVersion'*/" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "h" | "n" | "TestVersion" | false | true | null | "SELECT * FROM foo /*ddpv='TestVersion'*/" + "/*ddjk its a customer */ SELECT * FROM foo" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "h" | "n" | "TestVersion" | false | true | null | "/*ddjk its a customer */ SELECT * FROM foo /*ddps='SqlCommenter',dddbs='my-service',ddh='h',dddb='n',dde='Test',ddpv='TestVersion'*/" + "SELECT * FROM foo /*traceparent='00-00000000000000007fffffffffffffff-000000024cb016ea-01'*/" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "h" | "n" | "TestVersion" | false | true | null | "SELECT * FROM foo /*traceparent='00-00000000000000007fffffffffffffff-000000024cb016ea-01'*/" + "/*customer-comment*/ SELECT * FROM foo" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "h" | "n" | "TestVersion" | false | true | null | "/*customer-comment*/ SELECT * FROM foo /*ddps='SqlCommenter',dddbs='my-service',ddh='h',dddb='n',dde='Test',ddpv='TestVersion'*/" + "/*traceparent" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "h" | "n" | "TestVersion" | false | true | null | "/*traceparent /*ddps='SqlCommenter',dddbs='my-service',ddh='h',dddb='n',dde='Test',ddpv='TestVersion'*/" + "SELECT * FROM foo" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "h" | "n" | "TestVersion" | true | false | "00-00000000000000007fffffffffffffff-000000024cb016ea-00" | "/*ddps='SqlCommenter',dddbs='my-service',ddh='h',dddb='n',dde='Test',ddpv='TestVersion',traceparent='00-00000000000000007fffffffffffffff-000000024cb016ea-00'*/ SELECT * FROM foo" + "SELECT * FROM foo" | "" | "Test" | "" | "mysql" | "h" | "n" | "TestVersion" | true | false | "00-00000000000000007fffffffffffffff-000000024cb016ea-00" | "/*ddh='h',dddb='n',dde='Test',ddpv='TestVersion',traceparent='00-00000000000000007fffffffffffffff-000000024cb016ea-00'*/ SELECT * FROM foo" + "SELECT * FROM foo" | "" | "Test" | "my-service" | "mysql" | "h" | "n" | "TestVersion" | true | false | "00-00000000000000007fffffffffffffff-000000024cb016ea-00" | "/*dddbs='my-service',ddh='h',dddb='n',dde='Test',ddpv='TestVersion',traceparent='00-00000000000000007fffffffffffffff-000000024cb016ea-00'*/ SELECT * FROM foo" + "SELECT * FROM foo" | "" | "Test" | "" | "" | "h" | "n" | "" | true | false | "00-00000000000000007fffffffffffffff-000000024cb016ea-00" | "/*ddh='h',dddb='n',dde='Test',traceparent='00-00000000000000007fffffffffffffff-000000024cb016ea-00'*/ SELECT * FROM foo" + "SELECT * FROM foo" | "" | "" | "" | "" | "" | "" | "" | true | false | "00-00000000000000007fffffffffffffff-000000024cb016ea-00" | "/*traceparent='00-00000000000000007fffffffffffffff-000000024cb016ea-00'*/ SELECT * FROM foo" + "SELECT * from FOO -- test query" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "h" | "n" | "TestVersion" | true | false | "00-00000000000000007fffffffffffffff-000000024cb016ea-01" | "/*ddps='SqlCommenter',dddbs='my-service',ddh='h',dddb='n',dde='Test',ddpv='TestVersion',traceparent='00-00000000000000007fffffffffffffff-000000024cb016ea-01'*/ SELECT * from FOO -- test query" + "SELECT /* customer-comment */ * FROM foo" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "h" | "n" | "TestVersion" | true | false | "00-00000000000000007fffffffffffffff-000000024cb016ea-01" | "/*ddps='SqlCommenter',dddbs='my-service',ddh='h',dddb='n',dde='Test',ddpv='TestVersion',traceparent='00-00000000000000007fffffffffffffff-000000024cb016ea-01'*/ SELECT /* customer-comment */ * FROM foo" + "SELECT * FROM foo" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "h" | "n" | "TestVersion" | false | false | null | "/*ddps='SqlCommenter',dddbs='my-service',ddh='h',dddb='n',dde='Test',ddpv='TestVersion'*/ SELECT * FROM foo" + "SELECT /* customer-comment */ * FROM foo" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "h" | "n" | "TestVersion" | false | false | null | "/*ddps='SqlCommenter',dddbs='my-service',ddh='h',dddb='n',dde='Test',ddpv='TestVersion'*/ SELECT /* customer-comment */ * FROM foo" + "SELECT * from FOO -- test query" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "h" | "n" | "TestVersion" | false | false | null | "/*ddps='SqlCommenter',dddbs='my-service',ddh='h',dddb='n',dde='Test',ddpv='TestVersion'*/ SELECT * from FOO -- test query" + "" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "h" | "n" | "TestVersion" | true | false | "00-00000000000000007fffffffffffffff-000000024cb016ea-00" | "" + " " | "SqlCommenter" | "Test" | "my-service" | "mysql" | "h" | "n" | "TestVersion" | true | false | "00-00000000000000007fffffffffffffff-000000024cb016ea-01" | "/*ddps='SqlCommenter',dddbs='my-service',ddh='h',dddb='n',dde='Test',ddpv='TestVersion',traceparent='00-00000000000000007fffffffffffffff-000000024cb016ea-01'*/ " + "/*dddbs='my-service',ddh='h',dddb='n',dde='Test',ddps='SqlCommenter',ddpv='TestVersion'*/ SELECT * FROM foo" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "h" | "n" | "TestVersion" | false | false | null | "/*dddbs='my-service',ddh='h',dddb='n',dde='Test',ddps='SqlCommenter',ddpv='TestVersion'*/ SELECT * FROM foo" + "/*ddh='h',dddb='n',dde='Test',ddps='SqlCommenter',ddpv='TestVersion'*/ SELECT * FROM foo" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "h" | "n" | "TestVersion" | false | false | null | "/*ddh='h',dddb='n',dde='Test',ddps='SqlCommenter',ddpv='TestVersion'*/ SELECT * FROM foo" + "/*dddb='n',dde='Test',ddps='SqlCommenter',ddpv='TestVersion'*/ SELECT * FROM foo" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "h" | "n" | "TestVersion" | false | false | null | "/*dddb='n',dde='Test',ddps='SqlCommenter',ddpv='TestVersion'*/ SELECT * FROM foo" + "/*dde='Test',ddps='SqlCommenter',ddpv='TestVersion'*/ SELECT * FROM foo" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "h" | "n" | "TestVersion" | false | false | null | "/*dde='Test',ddps='SqlCommenter',ddpv='TestVersion'*/ SELECT * FROM foo" + "/*ddps='SqlCommenter',ddpv='TestVersion'*/ SELECT * FROM foo" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "h" | "n" | "TestVersion" | false | false | null | "/*ddps='SqlCommenter',ddpv='TestVersion'*/ SELECT * FROM foo" + "/*ddpv='TestVersion'*/ SELECT * FROM foo" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "h" | "n" | "TestVersion" | false | false | null | "/*ddpv='TestVersion'*/ SELECT * FROM foo" + "/*ddjk its a customer */ SELECT * FROM foo" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "h" | "n" | "TestVersion" | false | false | null | "/*ddps='SqlCommenter',dddbs='my-service',ddh='h',dddb='n',dde='Test',ddpv='TestVersion'*/ /*ddjk its a customer */ SELECT * FROM foo" + "/*traceparent='00-00000000000000007fffffffffffffff-000000024cb016ea-01'*/ SELECT * FROM foo" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "h" | "n" | "TestVersion" | false | false | null | "/*traceparent='00-00000000000000007fffffffffffffff-000000024cb016ea-01'*/ SELECT * FROM foo" + "/*customer-comment*/ SELECT * FROM foo" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "h" | "n" | "TestVersion" | false | false | null | "/*ddps='SqlCommenter',dddbs='my-service',ddh='h',dddb='n',dde='Test',ddpv='TestVersion'*/ /*customer-comment*/ SELECT * FROM foo" + "/*traceparent" | "SqlCommenter" | "Test" | "my-service" | "mysql" | "h" | "n" | "TestVersion" | false | false | null | "/*ddps='SqlCommenter',dddbs='my-service',ddh='h',dddb='n',dde='Test',ddpv='TestVersion'*/ /*traceparent" } } From d20df91089973d08976ed56fd91079efad026418 Mon Sep 17 00:00:00 2001 From: Wan Tsui Date: Wed, 6 Mar 2024 11:23:13 -0500 Subject: [PATCH 16/30] put equal sign in httpResourceRemoveTrailingSlash when printing the config (#6571) --- internal-api/src/main/java/datadog/trace/api/Config.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal-api/src/main/java/datadog/trace/api/Config.java b/internal-api/src/main/java/datadog/trace/api/Config.java index 3a78f0bc40b..6c5d6e89404 100644 --- a/internal-api/src/main/java/datadog/trace/api/Config.java +++ b/internal-api/src/main/java/datadog/trace/api/Config.java @@ -4163,7 +4163,7 @@ public String toString() { + httpClientTagQueryString + ", httpClientSplitByDomain=" + httpClientSplitByDomain - + ", httpResourceRemoveTrailingSlash" + + ", httpResourceRemoveTrailingSlash=" + httpResourceRemoveTrailingSlash + ", dbClientSplitByInstance=" + dbClientSplitByInstance From 888704399b0af0b45ac171459e89f7118889fce6 Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Fri, 8 Mar 2024 13:33:34 +0100 Subject: [PATCH 17/30] Enhance 'runtime_version' tag to encode graalvm source (#6788) --- .../java/com/datadog/profiling/uploader/ProfileUploader.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dd-java-agent/agent-profiling/profiling-uploader/src/main/java/com/datadog/profiling/uploader/ProfileUploader.java b/dd-java-agent/agent-profiling/profiling-uploader/src/main/java/com/datadog/profiling/uploader/ProfileUploader.java index 159c16ecb9a..ea57991beba 100644 --- a/dd-java-agent/agent-profiling/profiling-uploader/src/main/java/com/datadog/profiling/uploader/ProfileUploader.java +++ b/dd-java-agent/agent-profiling/profiling-uploader/src/main/java/com/datadog/profiling/uploader/ProfileUploader.java @@ -25,6 +25,7 @@ import datadog.communication.http.OkHttpUtils; import datadog.trace.api.Config; import datadog.trace.api.DDTags; +import datadog.trace.api.Platform; import datadog.trace.api.git.GitInfo; import datadog.trace.api.git.GitInfoProvider; import datadog.trace.api.profiling.RecordingData; @@ -169,6 +170,9 @@ public ProfileUploader(final Config config, final ConfigProvider configProvider) tagsMap.put(Tags.GIT_REPOSITORY_URL, gitInfo.getRepositoryURL()); tagsMap.put(Tags.GIT_COMMIT_SHA, gitInfo.getCommit().getSha()); } + if (Platform.isGraalVM()) { + tagsMap.put(DDTags.RUNTIME_VERSION_TAG, tagsMap.get(DDTags.RUNTIME_VERSION_TAG) + "-graalvm"); + } // Comma separated tags string for V2.4 format Pattern quotes = Pattern.compile("\""); From b17471575716c59f8afde572e8b1192f1b938eec Mon Sep 17 00:00:00 2001 From: ValentinZakharov Date: Fri, 8 Mar 2024 14:04:24 +0100 Subject: [PATCH 18/30] Improved API Security schema computation performance (#6765) * Compute API Security schema extraction at the end of request * Better formatting --- .../datadog/appsec/gateway/GatewayBridge.java | 46 +++++++++++++++---- 1 file changed, 37 insertions(+), 9 deletions(-) diff --git a/dd-java-agent/appsec/src/main/java/com/datadog/appsec/gateway/GatewayBridge.java b/dd-java-agent/appsec/src/main/java/com/datadog/appsec/gateway/GatewayBridge.java index 95972c59ddd..d23087b157e 100644 --- a/dd-java-agent/appsec/src/main/java/com/datadog/appsec/gateway/GatewayBridge.java +++ b/dd-java-agent/appsec/src/main/java/com/datadog/appsec/gateway/GatewayBridge.java @@ -73,6 +73,7 @@ public class GatewayBridge { private volatile DataSubscriberInfo pathParamsSubInfo; private volatile DataSubscriberInfo respDataSubInfo; private volatile DataSubscriberInfo grpcServerRequestMsgSubInfo; + private volatile DataSubscriberInfo requestEndSubInfo; public GatewayBridge( SubscriptionService subscriptionService, @@ -110,6 +111,8 @@ public void init() { return NoopFlow.INSTANCE; } + maybeExtractSchemas(ctx); + // WAF call ctx.closeAdditive(); @@ -558,18 +561,10 @@ private Flow maybePublishResponseData(AppSecRequestContext ctx) { ctx.setRespDataPublished(true); - boolean extractSchema = false; - if (Config.get().isApiSecurityEnabled() && requestSampler != null) { - extractSchema = requestSampler.sampleRequest(); - } - MapDataBundle bundle = MapDataBundle.of( KnownAddresses.RESPONSE_STATUS, String.valueOf(ctx.getResponseStatus()), - KnownAddresses.RESPONSE_HEADERS_NO_COOKIES, ctx.getResponseHeaders(), - // Extract api schema on response stage - KnownAddresses.WAF_CONTEXT_PROCESSOR, - Collections.singletonMap("extract-schema", extractSchema)); + KnownAddresses.RESPONSE_HEADERS_NO_COOKIES, ctx.getResponseHeaders()); while (true) { DataSubscriberInfo subInfo = respDataSubInfo; @@ -588,6 +583,39 @@ private Flow maybePublishResponseData(AppSecRequestContext ctx) { } } + private void maybeExtractSchemas(AppSecRequestContext ctx) { + boolean extractSchema = false; + if (Config.get().isApiSecurityEnabled() && requestSampler != null) { + extractSchema = requestSampler.sampleRequest(); + } + + if (!extractSchema) { + return; + } + + while (true) { + DataSubscriberInfo subInfo = requestEndSubInfo; + if (subInfo == null) { + subInfo = producerService.getDataSubscribers(KnownAddresses.WAF_CONTEXT_PROCESSOR); + requestEndSubInfo = subInfo; + } + if (subInfo == null || subInfo.isEmpty()) { + return; + } + + DataBundle bundle = + new SingletonDataBundle<>( + KnownAddresses.WAF_CONTEXT_PROCESSOR, + Collections.singletonMap("extract-schema", true)); + try { + producerService.publishDataEvent(subInfo, ctx, bundle, false); + return; + } catch (ExpiredSubscriberInfoException e) { + requestEndSubInfo = null; + } + } + } + private static Map> parseQueryStringParams( String queryString, Charset uriEncoding) { if (queryString == null) { From fa3cf4dc0e5cc07208b55482ff92d731c7335472 Mon Sep 17 00:00:00 2001 From: Richard Startin Date: Fri, 8 Mar 2024 13:33:50 +0000 Subject: [PATCH 19/30] Upgrade ddprof to 1.2.0 (#6790) --- gradle/dependencies.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index 580f4cd2a86..1a2192dbd2d 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -33,7 +33,7 @@ final class CachedData { testcontainers: '1.19.3', jmc : "8.1.0", autoservice : "1.0-rc7", - ddprof : "1.1.0", + ddprof : "1.2.0", asm : "9.6", cafe_crypto : "0.1.0", lz4 : "1.7.1" From 0c7e4ea9ca0522ec004a7a38b412858f2ae1a5d9 Mon Sep 17 00:00:00 2001 From: Nikita Tkachenko <121111529+nikita-tkachenko-datadog@users.noreply.github.com> Date: Fri, 8 Mar 2024 15:17:31 +0100 Subject: [PATCH 20/30] Optimize in-progress tests and suites lookup (#6728) --- .../civisibility/CiVisibilitySystem.java | 51 ++++++-- .../events/TestEventsHandlerImpl.java | 109 +++++----------- .../CiVisibilityInstrumentationTest.groovy | 36 +++++- .../junit4/CucumberTracingListener.java | 47 +++---- .../instrumentation/junit4/CucumberUtils.java | 6 + .../junit4/MUnitInstrumentation.java | 1 + .../junit4/MUnitTracingListener.java | 60 +++++---- .../instrumentation/junit4/MUnitUtils.java | 15 +++ .../junit4/JUnit4TracingListener.java | 68 ++++------ .../instrumentation/junit4/JUnit4Utils.java | 7 + .../junit4/TestEventsHandlerHolder.java | 4 +- .../junit5/CucumberTracingListener.java | 88 +++++-------- .../instrumentation/junit5/CucumberUtils.java | 23 +--- .../junit5/JUnit5CucumberInstrumentation.java | 16 +++ .../src/test/groovy/CucumberTest.groovy | 2 +- .../junit5/JUnit5SpockInstrumentation.java | 16 +++ .../junit5/SpockTracingListener.java | 112 +++++++--------- .../instrumentation/junit5/SpockUtils.java | 23 +--- .../src/test/groovy/SpockTest.groovy | 2 +- .../junit5/JUnit5Instrumentation.java | 17 ++- .../junit5/JUnitPlatformUtils.java | 28 ---- .../junit5/TestEventsHandlerHolder.java | 37 +++++- .../junit5/TracingListener.java | 122 +++++++----------- .../src/test/groovy/JUnit5Test.groovy | 2 +- .../karate/KarateTracingHook.java | 41 +++--- .../instrumentation/karate/KarateUtils.java | 10 +- .../karate/TestEventsHandlerHolder.java | 4 +- .../scalatest/DatadogReporter.java | 62 +++++---- .../instrumentation/scalatest/RunContext.java | 6 +- .../testng/TestEventsHandlerHolder.java | 4 +- .../instrumentation/testng/TestNGUtils.java | 7 + .../testng/TracingListener.java | 64 ++++----- .../civisibility/InstrumentationBridge.java | 11 +- .../civisibility/events/TestDescriptor.java | 19 +++ .../events/TestEventsHandler.java | 49 +++---- .../events/TestSuiteDescriptor.java | 11 ++ 36 files changed, 605 insertions(+), 575 deletions(-) diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/CiVisibilitySystem.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/CiVisibilitySystem.java index 0faf602f522..05f50f8a9b0 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/CiVisibilitySystem.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/CiVisibilitySystem.java @@ -4,6 +4,8 @@ import datadog.communication.ddagent.TracerVersion; import datadog.trace.api.Config; import datadog.trace.api.civisibility.CIVisibility; +import datadog.trace.api.civisibility.DDTest; +import datadog.trace.api.civisibility.DDTestSuite; import datadog.trace.api.civisibility.InstrumentationBridge; import datadog.trace.api.civisibility.config.ModuleExecutionSettings; import datadog.trace.api.civisibility.coverage.CoverageBridge; @@ -13,6 +15,7 @@ import datadog.trace.api.civisibility.telemetry.CiVisibilityMetricCollector; import datadog.trace.api.civisibility.telemetry.NoOpMetricCollector; import datadog.trace.api.git.GitInfoProvider; +import datadog.trace.bootstrap.ContextStore; import datadog.trace.civisibility.config.JvmInfo; import datadog.trace.civisibility.coverage.instrumentation.CoverageClassTransformer; import datadog.trace.civisibility.coverage.instrumentation.CoverageInstrumentationFilter; @@ -30,6 +33,7 @@ import datadog.trace.civisibility.events.TestEventsHandlerImpl; import datadog.trace.civisibility.ipc.SignalServer; import datadog.trace.civisibility.telemetry.CiVisibilityMetricCollectorImpl; +import datadog.trace.civisibility.utils.ConcurrentHashMapContextStore; import datadog.trace.civisibility.utils.ProcessHierarchyUtils; import datadog.trace.util.throwable.FatalAgentMisconfigurationError; import java.lang.instrument.Instrumentation; @@ -94,7 +98,7 @@ public static void start(Instrumentation inst, SharedCommunicationObjects sco) { } InstrumentationBridge.registerTestEventsHandlerFactory( - testEventsHandlerFactory(services, repoServices, executionSettings)); + new TestEventsHandlerFactory(services, repoServices, executionSettings)); CoverageBridge.registerCoverageProbeStoreRegistry(services.coverageProbeStoreFactory); } } @@ -138,24 +142,43 @@ public BuildEventsHandler create() { }; } - private static TestEventsHandler.Factory testEventsHandlerFactory( - CiVisibilityServices services, - CiVisibilityRepoServices repoServices, - ModuleExecutionSettings executionSettings) { - TestFrameworkSession.Factory sessionFactory; - if (ProcessHierarchyUtils.isChild()) { - sessionFactory = childTestFrameworkSessionFactory(services, repoServices, executionSettings); - } else { - sessionFactory = - headlessTestFrameworkEssionFactory(services, repoServices, executionSettings); + private static final class TestEventsHandlerFactory implements TestEventsHandler.Factory { + private final CiVisibilityServices services; + private final CiVisibilityRepoServices repoServices; + private final TestFrameworkSession.Factory sessionFactory; + + private TestEventsHandlerFactory( + CiVisibilityServices services, + CiVisibilityRepoServices repoServices, + ModuleExecutionSettings executionSettings) { + this.services = services; + this.repoServices = repoServices; + if (ProcessHierarchyUtils.isChild()) { + sessionFactory = + childTestFrameworkSessionFactory(services, repoServices, executionSettings); + } else { + sessionFactory = + headlessTestFrameworkEssionFactory(services, repoServices, executionSettings); + } } - return (String component) -> { + @Override + public TestEventsHandler create(String component) { + return create( + component, new ConcurrentHashMapContextStore<>(), new ConcurrentHashMapContextStore<>()); + } + + @Override + public TestEventsHandler create( + String component, + ContextStore suiteStore, + ContextStore testStore) { TestFrameworkSession testSession = sessionFactory.startSession(repoServices.moduleName, component, null); TestFrameworkModule testModule = testSession.testModuleStart(repoServices.moduleName, null); - return new TestEventsHandlerImpl(services.metricCollector, testSession, testModule); - }; + return new TestEventsHandlerImpl<>( + services.metricCollector, testSession, testModule, suiteStore, testStore); + } } private static BuildSystemSession.Factory buildSystemSessionFactory( diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/events/TestEventsHandlerImpl.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/events/TestEventsHandlerImpl.java index c1e4a67adcc..53393444e2f 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/events/TestEventsHandlerImpl.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/events/TestEventsHandlerImpl.java @@ -3,11 +3,11 @@ import static datadog.trace.util.Strings.toJson; import datadog.trace.api.DisableTestTrace; +import datadog.trace.api.civisibility.DDTest; +import datadog.trace.api.civisibility.DDTestSuite; import datadog.trace.api.civisibility.InstrumentationBridge; import datadog.trace.api.civisibility.config.TestIdentifier; -import datadog.trace.api.civisibility.events.TestDescriptor; import datadog.trace.api.civisibility.events.TestEventsHandler; -import datadog.trace.api.civisibility.events.TestSuiteDescriptor; import datadog.trace.api.civisibility.retry.TestRetryPolicy; import datadog.trace.api.civisibility.telemetry.CiVisibilityCountMetric; import datadog.trace.api.civisibility.telemetry.CiVisibilityMetricCollector; @@ -19,7 +19,6 @@ import datadog.trace.civisibility.domain.TestFrameworkSession; import datadog.trace.civisibility.domain.TestImpl; import datadog.trace.civisibility.domain.TestSuiteImpl; -import datadog.trace.civisibility.utils.ConcurrentHashMapContextStore; import java.lang.reflect.Method; import java.util.Collection; import java.util.Collections; @@ -29,31 +28,33 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class TestEventsHandlerImpl implements TestEventsHandler { +public class TestEventsHandlerImpl + implements TestEventsHandler { private static final Logger log = LoggerFactory.getLogger(TestEventsHandlerImpl.class); private final CiVisibilityMetricCollector metricCollector; private final TestFrameworkSession testSession; private final TestFrameworkModule testModule; - - private final ContextStore inProgressTestSuites = - new ConcurrentHashMapContextStore<>(); - - private final ContextStore inProgressTests = - new ConcurrentHashMapContextStore<>(); + private final ContextStore inProgressTestSuites; + private final ContextStore inProgressTests; public TestEventsHandlerImpl( CiVisibilityMetricCollector metricCollector, TestFrameworkSession testSession, - TestFrameworkModule testModule) { + TestFrameworkModule testModule, + ContextStore suiteStore, + ContextStore testStore) { this.metricCollector = metricCollector; this.testSession = testSession; this.testModule = testModule; + this.inProgressTestSuites = (ContextStore) suiteStore; + this.inProgressTests = (ContextStore) testStore; } @Override public void onTestSuiteStart( + final SuiteKey descriptor, final String testSuiteName, final @Nullable String testFramework, final @Nullable String testFrameworkVersion, @@ -79,45 +80,34 @@ public void onTestSuiteStart( Tags.TEST_TRAITS, toJson(Collections.singletonMap("category", toJson(categories)), true)); } - TestSuiteDescriptor descriptor = new TestSuiteDescriptor(testSuiteName, testClass); inProgressTestSuites.put(descriptor, testSuite); } @Override - public void onTestSuiteFinish(final String testSuiteName, final @Nullable Class testClass) { - if (skipTrace(testClass)) { + public void onTestSuiteFinish(SuiteKey descriptor) { + if (skipTrace(descriptor.getClass())) { return; } - TestSuiteDescriptor descriptor = new TestSuiteDescriptor(testSuiteName, testClass); TestSuiteImpl testSuite = inProgressTestSuites.remove(descriptor); testSuite.end(null); } @Override - public void onTestSuiteSkip(String testSuiteName, Class testClass, @Nullable String reason) { - TestSuiteDescriptor descriptor = new TestSuiteDescriptor(testSuiteName, testClass); + public void onTestSuiteSkip(SuiteKey descriptor, @Nullable String reason) { TestSuiteImpl testSuite = inProgressTestSuites.get(descriptor); if (testSuite == null) { - log.debug( - "Ignoring skip event, could not find test suite with name {} and class {}", - testSuiteName, - testClass); + log.debug("Ignoring skip event, could not find test suite {}", descriptor); return; } testSuite.setSkipReason(reason); } @Override - public void onTestSuiteFailure( - String testSuiteName, Class testClass, @Nullable Throwable throwable) { - TestSuiteDescriptor descriptor = new TestSuiteDescriptor(testSuiteName, testClass); + public void onTestSuiteFailure(SuiteKey descriptor, @Nullable Throwable throwable) { TestSuiteImpl testSuite = inProgressTestSuites.get(descriptor); if (testSuite == null) { - log.debug( - "Ignoring fail event, could not find test suite with name {} and class {}", - testSuiteName, - testClass); + log.debug("Ignoring fail event, could not find test suite {}", descriptor); return; } testSuite.setErrorInfo(throwable); @@ -125,9 +115,10 @@ public void onTestSuiteFailure( @Override public void onTestStart( + final SuiteKey suiteDescriptor, + final TestKey descriptor, final String testSuiteName, final String testName, - final @Nullable Object testQualifier, final @Nullable String testFramework, final @Nullable String testFrameworkVersion, final @Nullable String testParameters, @@ -140,7 +131,6 @@ public void onTestStart( return; } - TestSuiteDescriptor suiteDescriptor = new TestSuiteDescriptor(testSuiteName, testClass); TestSuiteImpl testSuite = inProgressTestSuites.get(suiteDescriptor); TestImpl test = testSuite.testStart(testName, testMethod, null); @@ -183,71 +173,34 @@ public void onTestStart( test.setTag(Tags.TEST_IS_RETRY, true); } - TestDescriptor descriptor = - new TestDescriptor(testSuiteName, testClass, testName, testParameters, testQualifier); inProgressTests.put(descriptor, test); } @Override - public void onTestSkip( - String testSuiteName, - Class testClass, - String testName, - @Nullable Object testQualifier, - @Nullable String testParameters, - @Nullable String reason) { - TestDescriptor descriptor = - new TestDescriptor(testSuiteName, testClass, testName, testParameters, testQualifier); + public void onTestSkip(TestKey descriptor, @Nullable String reason) { TestImpl test = inProgressTests.get(descriptor); if (test == null) { - log.debug( - "Ignoring skip event, could not find test with name {}, suite name{} and class {}", - testName, - testSuiteName, - testClass); + log.debug("Ignoring skip event, could not find test {}}", descriptor); return; } test.setSkipReason(reason); } @Override - public void onTestFailure( - String testSuiteName, - Class testClass, - String testName, - @Nullable Object testQualifier, - @Nullable String testParameters, - @Nullable Throwable throwable) { - TestDescriptor descriptor = - new TestDescriptor(testSuiteName, testClass, testName, testParameters, testQualifier); + public void onTestFailure(TestKey descriptor, @Nullable Throwable throwable) { TestImpl test = inProgressTests.get(descriptor); if (test == null) { - log.debug( - "Ignoring fail event, could not find test with name {}, suite name{} and class {}", - testName, - testSuiteName, - testClass); + log.debug("Ignoring fail event, could not find test {}", descriptor); return; } test.setErrorInfo(throwable); } @Override - public void onTestFinish( - final String testSuiteName, - final Class testClass, - final String testName, - final @Nullable Object testQualifier, - final @Nullable String testParameters) { - TestDescriptor descriptor = - new TestDescriptor(testSuiteName, testClass, testName, testParameters, testQualifier); + public void onTestFinish(TestKey descriptor) { TestImpl test = inProgressTests.remove(descriptor); if (test == null) { - log.debug( - "Ignoring finish event, could not find test with name {}, suite name{} and class {}", - testName, - testSuiteName, - testClass); + log.debug("Ignoring finish event, could not find test {}", descriptor); return; } test.end(null); @@ -255,9 +208,10 @@ public void onTestFinish( @Override public void onTestIgnore( + final SuiteKey suiteDescriptor, + final TestKey testDescriptor, final String testSuiteName, final String testName, - final @Nullable Object testQualifier, final @Nullable String testFramework, final @Nullable String testFrameworkVersion, final @Nullable String testParameters, @@ -267,9 +221,10 @@ public void onTestIgnore( final @Nullable Method testMethod, final @Nullable String reason) { onTestStart( + suiteDescriptor, + testDescriptor, testSuiteName, testName, - testQualifier, testFramework, testFrameworkVersion, testParameters, @@ -278,8 +233,8 @@ public void onTestIgnore( testMethodName, testMethod, false); - onTestSkip(testSuiteName, testClass, testName, testQualifier, testParameters, reason); - onTestFinish(testSuiteName, testClass, testName, testQualifier, testParameters); + onTestSkip(testDescriptor, reason); + onTestFinish(testDescriptor); } private static boolean skipTrace(final Class testClass) { diff --git a/dd-java-agent/agent-ci-visibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilityInstrumentationTest.groovy b/dd-java-agent/agent-ci-visibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilityInstrumentationTest.groovy index 2d0f0cce499..24b9a99b977 100644 --- a/dd-java-agent/agent-ci-visibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilityInstrumentationTest.groovy +++ b/dd-java-agent/agent-ci-visibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilityInstrumentationTest.groovy @@ -5,13 +5,18 @@ import datadog.communication.serialization.GrowableBuffer import datadog.communication.serialization.msgpack.MsgPackWriter import datadog.trace.agent.test.AgentTestRunner import datadog.trace.api.Config +import datadog.trace.api.civisibility.DDTest +import datadog.trace.api.civisibility.DDTestSuite import datadog.trace.api.civisibility.InstrumentationBridge import datadog.trace.api.civisibility.config.EarlyFlakeDetectionSettings import datadog.trace.api.civisibility.config.ModuleExecutionSettings import datadog.trace.api.civisibility.config.TestIdentifier import datadog.trace.api.civisibility.coverage.CoverageBridge +import datadog.trace.api.civisibility.events.TestEventsHandler +import datadog.trace.api.civisibility.telemetry.CiVisibilityMetricCollector import datadog.trace.api.config.CiVisibilityConfig import datadog.trace.api.config.GeneralConfig +import datadog.trace.bootstrap.ContextStore import datadog.trace.civisibility.codeowners.Codeowners import datadog.trace.civisibility.config.JvmInfo import datadog.trace.civisibility.config.JvmInfoFactoryImpl @@ -32,6 +37,7 @@ import datadog.trace.civisibility.source.MethodLinesResolver import datadog.trace.civisibility.source.SourcePathResolver import datadog.trace.civisibility.source.index.RepoIndexBuilder import datadog.trace.civisibility.telemetry.CiVisibilityMetricCollectorImpl +import datadog.trace.civisibility.utils.ConcurrentHashMapContextStore import datadog.trace.civisibility.writer.ddintake.CiTestCovMapperV2 import datadog.trace.civisibility.writer.ddintake.CiTestCycleMapperV1 import datadog.trace.common.writer.RemoteMapper @@ -43,7 +49,6 @@ import java.nio.ByteBuffer import java.nio.file.Files import java.nio.file.Path import java.nio.file.Paths -import java.util.stream.Collectors abstract class CiVisibilityInstrumentationTest extends AgentTestRunner { @@ -132,12 +137,7 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner { ) } - InstrumentationBridge.registerTestEventsHandlerFactory { - component -> - TestFrameworkSession testSession = testFrameworkSessionFactory.startSession(dummyModule, component, null) - TestFrameworkModule testModule = testSession.testModuleStart(dummyModule, null) - new TestEventsHandlerImpl(metricCollector, testSession, testModule) - } + InstrumentationBridge.registerTestEventsHandlerFactory(new TestEventHandlerFactory(testFrameworkSessionFactory, metricCollector)) BuildSystemSession.Factory buildSystemSessionFactory = (String projectName, Path projectRoot, String startCommand, String component, Long startTime) -> { def ciTags = [(DUMMY_CI_TAG): DUMMY_CI_TAG_VALUE] @@ -172,6 +172,28 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner { CoverageBridge.registerCoverageProbeStoreRegistry(coverageProbeStoreFactory) } + private static final class TestEventHandlerFactory implements TestEventsHandler.Factory { + private final TestFrameworkSession.Factory testFrameworkSessionFactory + private final CiVisibilityMetricCollector metricCollector + + TestEventHandlerFactory(testFrameworkSessionFactory, metricCollector) { + this.testFrameworkSessionFactory = testFrameworkSessionFactory + this.metricCollector = metricCollector + } + + @Override + TestEventsHandler create(String component) { + return create(component, new ConcurrentHashMapContextStore<>(), new ConcurrentHashMapContextStore()) + } + + @Override + TestEventsHandler create(String component, ContextStore suiteStore, ContextStore testStore) { + TestFrameworkSession testSession = testFrameworkSessionFactory.startSession(dummyModule, component, null) + TestFrameworkModule testModule = testSession.testModuleStart(dummyModule, null) + new TestEventsHandlerImpl(metricCollector, testSession, testModule, suiteStore, testStore) + } + } + @Override void setup() { skippableTests.clear() diff --git a/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/main/java/datadog/trace/instrumentation/junit4/CucumberTracingListener.java b/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/main/java/datadog/trace/instrumentation/junit4/CucumberTracingListener.java index fb868b29aca..6d04ecde5cf 100644 --- a/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/main/java/datadog/trace/instrumentation/junit4/CucumberTracingListener.java +++ b/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/main/java/datadog/trace/instrumentation/junit4/CucumberTracingListener.java @@ -1,6 +1,8 @@ package datadog.trace.instrumentation.junit4; import datadog.trace.api.civisibility.coverage.CoverageBridge; +import datadog.trace.api.civisibility.events.TestDescriptor; +import datadog.trace.api.civisibility.events.TestSuiteDescriptor; import datadog.trace.api.civisibility.retry.TestRetryPolicy; import datadog.trace.api.civisibility.telemetry.tag.TestFrameworkInstrumentation; import datadog.trace.bootstrap.ContextStore; @@ -39,8 +41,10 @@ public CucumberTracingListener( @Override public void testSuiteStarted(final Description description) { if (isFeature(description)) { + TestSuiteDescriptor suiteDescriptor = CucumberUtils.toSuiteDescriptor(description); String testSuiteName = CucumberUtils.getTestSuiteNameForFeature(description); TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteStart( + suiteDescriptor, testSuiteName, FRAMEWORK_NAME, FRAMEWORK_VERSION, @@ -54,8 +58,8 @@ public void testSuiteStarted(final Description description) { @Override public void testSuiteFinished(final Description description) { if (isFeature(description)) { - String testSuiteName = CucumberUtils.getTestSuiteNameForFeature(description); - TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteFinish(testSuiteName, null); + TestSuiteDescriptor suiteDescriptor = CucumberUtils.toSuiteDescriptor(description); + TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteFinish(suiteDescriptor); } } @@ -67,9 +71,10 @@ public void testStarted(final Description description) { TestRetryPolicy retryPolicy = retryPolicies.get(description); TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestStart( + new TestSuiteDescriptor(testSuiteName, null), + CucumberUtils.toTestDescriptor(description), testSuiteName, testName, - null, FRAMEWORK_NAME, FRAMEWORK_VERSION, null, @@ -94,10 +99,8 @@ private static void recordFeatureFileCodeCoverage(Description scenarioDescriptio @Override public void testFinished(final Description description) { - String testSuiteName = CucumberUtils.getTestSuiteNameForScenario(description); - String testName = description.getMethodName(); - TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestFinish( - testSuiteName, null, testName, null, null); + TestDescriptor testDescriptor = CucumberUtils.toTestDescriptor(description); + TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestFinish(testDescriptor); } // same callback is executed both for test cases and test suites (for setup/teardown errors) @@ -105,16 +108,13 @@ public void testFinished(final Description description) { public void testFailure(final Failure failure) { Description description = failure.getDescription(); if (isFeature(description)) { - String testSuiteName = CucumberUtils.getTestSuiteNameForFeature(description); + TestSuiteDescriptor suiteDescriptor = CucumberUtils.toSuiteDescriptor(description); Throwable throwable = failure.getException(); - TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteFailure( - testSuiteName, null, throwable); + TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteFailure(suiteDescriptor, throwable); } else { - String testSuiteName = CucumberUtils.getTestSuiteNameForScenario(description); - String testName = description.getMethodName(); + TestDescriptor testDescriptor = CucumberUtils.toTestDescriptor(description); Throwable throwable = failure.getException(); - TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestFailure( - testSuiteName, null, testName, null, null, throwable); + TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestFailure(testDescriptor, throwable); } } @@ -130,13 +130,11 @@ public void testAssumptionFailure(final Failure failure) { Description description = failure.getDescription(); if (isFeature(description)) { - String testSuiteName = CucumberUtils.getTestSuiteNameForFeature(description); - TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteSkip(testSuiteName, null, reason); + TestSuiteDescriptor suiteDescriptor = CucumberUtils.toSuiteDescriptor(description); + TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteSkip(suiteDescriptor, reason); } else { - String testSuiteName = CucumberUtils.getTestSuiteNameForScenario(description); - String testName = description.getMethodName(); - TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSkip( - testSuiteName, null, testName, null, null, reason); + TestDescriptor testDescriptor = CucumberUtils.toTestDescriptor(description); + TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSkip(testDescriptor, reason); } } @@ -146,8 +144,10 @@ public void testIgnored(final Description description) { String reason = ignore != null ? ignore.value() : null; if (isFeature(description)) { + TestSuiteDescriptor suiteDescriptor = CucumberUtils.toSuiteDescriptor(description); String testSuiteName = CucumberUtils.getTestSuiteNameForFeature(description); TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteStart( + suiteDescriptor, testSuiteName, FRAMEWORK_NAME, FRAMEWORK_VERSION, @@ -155,16 +155,17 @@ public void testIgnored(final Description description) { Collections.emptyList(), false, TestFrameworkInstrumentation.CUCUMBER); - TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteSkip(testSuiteName, null, reason); - TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteFinish(testSuiteName, null); + TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteSkip(suiteDescriptor, reason); + TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteFinish(suiteDescriptor); } else { String testSuiteName = CucumberUtils.getTestSuiteNameForScenario(description); String testName = description.getMethodName(); List categories = getCategories(description); TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestIgnore( + new TestSuiteDescriptor(testSuiteName, null), + CucumberUtils.toTestDescriptor(description), testSuiteName, testName, - null, FRAMEWORK_NAME, FRAMEWORK_VERSION, null, diff --git a/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/main/java/datadog/trace/instrumentation/junit4/CucumberUtils.java b/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/main/java/datadog/trace/instrumentation/junit4/CucumberUtils.java index 01b9760fce7..dfa4d92154c 100644 --- a/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/main/java/datadog/trace/instrumentation/junit4/CucumberUtils.java +++ b/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/main/java/datadog/trace/instrumentation/junit4/CucumberUtils.java @@ -3,6 +3,7 @@ import datadog.trace.agent.tooling.muzzle.Reference; import datadog.trace.api.civisibility.config.TestIdentifier; import datadog.trace.api.civisibility.events.TestDescriptor; +import datadog.trace.api.civisibility.events.TestSuiteDescriptor; import datadog.trace.util.MethodHandles; import datadog.trace.util.Strings; import io.cucumber.core.gherkin.Feature; @@ -109,6 +110,11 @@ public static TestDescriptor toTestDescriptor(Description description) { return new TestDescriptor(suite, null, name, null, null); } + public static TestSuiteDescriptor toSuiteDescriptor(Description description) { + String testSuiteName = CucumberUtils.getTestSuiteNameForFeature(description); + return new TestSuiteDescriptor(testSuiteName, null); + } + public static final class MuzzleHelper { public static Reference[] additionalMuzzleReferences() { return new Reference[] { diff --git a/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/main/java/datadog/trace/instrumentation/junit4/MUnitInstrumentation.java b/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/main/java/datadog/trace/instrumentation/junit4/MUnitInstrumentation.java index fdaa51bd1e3..029a8ec81d0 100644 --- a/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/main/java/datadog/trace/instrumentation/junit4/MUnitInstrumentation.java +++ b/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/main/java/datadog/trace/instrumentation/junit4/MUnitInstrumentation.java @@ -35,6 +35,7 @@ public String[] helperClassNames() { packageName + ".TestEventsHandlerHolder", packageName + ".SkippedByItr", packageName + ".JUnit4Utils", + packageName + ".MUnitUtils", packageName + ".TracingListener", packageName + ".MUnitTracingListener", }; diff --git a/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/main/java/datadog/trace/instrumentation/junit4/MUnitTracingListener.java b/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/main/java/datadog/trace/instrumentation/junit4/MUnitTracingListener.java index 7f8d6abc3ee..6c267df687f 100644 --- a/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/main/java/datadog/trace/instrumentation/junit4/MUnitTracingListener.java +++ b/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/main/java/datadog/trace/instrumentation/junit4/MUnitTracingListener.java @@ -1,5 +1,7 @@ package datadog.trace.instrumentation.junit4; +import datadog.trace.api.civisibility.events.TestDescriptor; +import datadog.trace.api.civisibility.events.TestSuiteDescriptor; import datadog.trace.api.civisibility.retry.TestRetryPolicy; import datadog.trace.api.civisibility.telemetry.tag.TestFrameworkInstrumentation; import datadog.trace.bootstrap.ContextStore; @@ -39,10 +41,12 @@ public void testSuiteStarted(final Description description) { return; } + TestSuiteDescriptor suiteDescriptor = MUnitUtils.toSuiteDescriptor(description); Class testClass = description.getTestClass(); String testSuiteName = description.getClassName(); List categories = getCategories(description); TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteStart( + suiteDescriptor, testSuiteName, FRAMEWORK_NAME, FRAMEWORK_VERSION, @@ -60,22 +64,24 @@ public void testSuiteFinished(final Description description) { return; } - Class testClass = description.getTestClass(); - String testSuiteName = description.getClassName(); - TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteFinish(testSuiteName, testClass); + TestSuiteDescriptor suiteDescriptor = MUnitUtils.toSuiteDescriptor(description); + TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteFinish(suiteDescriptor); } @Override public void testStarted(final Description description) { + TestSuiteDescriptor suiteDescriptor = MUnitUtils.toSuiteDescriptor(description); + TestDescriptor testDescriptor = MUnitUtils.toTestDescriptor(description); String testSuiteName = description.getClassName(); Class testClass = description.getTestClass(); String testName = description.getMethodName(); List categories = getCategories(description); TestRetryPolicy retryPolicy = retryPolicies.get(description); TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestStart( + suiteDescriptor, + testDescriptor, testSuiteName, testName, - null, FRAMEWORK_NAME, FRAMEWORK_VERSION, null, @@ -88,11 +94,8 @@ public void testStarted(final Description description) { @Override public void testFinished(final Description description) { - Class testClass = description.getTestClass(); - String testSuiteName = description.getClassName(); - String testName = description.getMethodName(); - TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestFinish( - testSuiteName, testClass, testName, null, null); + TestDescriptor testDescriptor = MUnitUtils.toTestDescriptor(description); + TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestFinish(testDescriptor); } // same callback is executed both for test cases and test suites (for setup/teardown errors) @@ -100,16 +103,14 @@ public void testFinished(final Description description) { public void testFailure(final Failure failure) { Throwable throwable = failure.getException(); Description description = failure.getDescription(); - Class testClass = description.getTestClass(); - String testSuiteName = description.getClassName(); - String testName = description.getMethodName(); + String testName = description.getMethodName(); if (Strings.isNotBlank(testName)) { - TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestFailure( - testSuiteName, testClass, testName, null, null, throwable); + TestDescriptor testDescriptor = MUnitUtils.toTestDescriptor(description); + TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestFailure(testDescriptor, throwable); } else { - TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteFailure( - testSuiteName, testClass, throwable); + TestSuiteDescriptor suiteDescriptor = MUnitUtils.toSuiteDescriptor(description); + TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteFailure(suiteDescriptor, throwable); } } @@ -125,15 +126,15 @@ public void testAssumptionFailure(final Failure failure) { Description description = failure.getDescription(); Class testClass = description.getTestClass(); - String testSuiteName = description.getClassName(); String testName = description.getMethodName(); if (Strings.isNotBlank(testName)) { - TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSkip( - testSuiteName, testClass, testName, null, null, reason); + TestDescriptor testDescriptor = MUnitUtils.toTestDescriptor(description); + TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSkip(testDescriptor, reason); } else if (testClass != null) { - TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteSkip(testSuiteName, testClass, reason); + TestSuiteDescriptor suiteDescriptor = MUnitUtils.toSuiteDescriptor(description); + TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteSkip(suiteDescriptor, reason); for (Description child : description.getChildren()) { testCaseIgnored(child); } @@ -147,14 +148,17 @@ public void testIgnored(final Description description) { String testName = description.getMethodName(); if (Strings.isNotBlank(testName)) { + TestDescriptor testDescriptor = MUnitUtils.toTestDescriptor(description); if (!isTestInProgress()) { // earlier versions of MUnit (e.g. 0.7.28) trigger "testStarted" event for ignored tests, // while newer versions don't + TestSuiteDescriptor suiteDescriptor = MUnitUtils.toSuiteDescriptor(description); List categories = getCategories(description); TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestStart( + suiteDescriptor, + testDescriptor, testSuiteName, testName, - null, FRAMEWORK_NAME, FRAMEWORK_VERSION, null, @@ -164,13 +168,12 @@ public void testIgnored(final Description description) { null, false); } - TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSkip( - testSuiteName, testClass, testName, null, null, null); - TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestFinish( - testSuiteName, testClass, testName, null, null); + TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSkip(testDescriptor, null); + TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestFinish(testDescriptor); } else if (testClass != null) { - TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteSkip(testSuiteName, testClass, null); + TestSuiteDescriptor suiteDescriptor = MUnitUtils.toSuiteDescriptor(description); + TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteSkip(suiteDescriptor, null); for (Description child : description.getChildren()) { testCaseIgnored(child); } @@ -188,14 +191,17 @@ private boolean isTestInProgress() { } private void testCaseIgnored(final Description description) { + TestSuiteDescriptor suiteDescriptor = MUnitUtils.toSuiteDescriptor(description); + TestDescriptor testDescriptor = MUnitUtils.toTestDescriptor(description); String testSuiteName = description.getClassName(); String testName = description.getMethodName(); Class testClass = description.getTestClass(); List categories = getCategories(description); TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestIgnore( + suiteDescriptor, + testDescriptor, testSuiteName, testName, - null, FRAMEWORK_NAME, FRAMEWORK_VERSION, null, diff --git a/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/main/java/datadog/trace/instrumentation/junit4/MUnitUtils.java b/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/main/java/datadog/trace/instrumentation/junit4/MUnitUtils.java index aff81abc695..1f2bad85589 100644 --- a/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/main/java/datadog/trace/instrumentation/junit4/MUnitUtils.java +++ b/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/main/java/datadog/trace/instrumentation/junit4/MUnitUtils.java @@ -1,5 +1,7 @@ package datadog.trace.instrumentation.junit4; +import datadog.trace.api.civisibility.events.TestDescriptor; +import datadog.trace.api.civisibility.events.TestSuiteDescriptor; import datadog.trace.util.MethodHandles; import java.lang.invoke.MethodHandle; import munit.MUnitRunner; @@ -19,4 +21,17 @@ private MUnitUtils() {} public static Description createDescription(MUnitRunner runner, Object test) { return METHOD_HANDLES.invoke(RUNNER_CREATE_TEST_DESCRIPTION, runner, test); } + + public static TestDescriptor toTestDescriptor(Description description) { + String testSuiteName = description.getClassName(); + Class testClass = description.getTestClass(); + String testName = description.getMethodName(); + return new TestDescriptor(testSuiteName, testClass, testName, null, null); + } + + public static TestSuiteDescriptor toSuiteDescriptor(Description description) { + Class testClass = description.getTestClass(); + String testSuiteName = description.getClassName(); + return new TestSuiteDescriptor(testSuiteName, testClass); + } } diff --git a/dd-java-agent/instrumentation/junit-4.10/src/main/java/datadog/trace/instrumentation/junit4/JUnit4TracingListener.java b/dd-java-agent/instrumentation/junit-4.10/src/main/java/datadog/trace/instrumentation/junit4/JUnit4TracingListener.java index 2699db2b6a1..d0798060cd2 100644 --- a/dd-java-agent/instrumentation/junit-4.10/src/main/java/datadog/trace/instrumentation/junit4/JUnit4TracingListener.java +++ b/dd-java-agent/instrumentation/junit-4.10/src/main/java/datadog/trace/instrumentation/junit4/JUnit4TracingListener.java @@ -1,5 +1,7 @@ package datadog.trace.instrumentation.junit4; +import datadog.trace.api.civisibility.events.TestDescriptor; +import datadog.trace.api.civisibility.events.TestSuiteDescriptor; import datadog.trace.api.civisibility.retry.TestRetryPolicy; import datadog.trace.api.civisibility.telemetry.tag.TestFrameworkInstrumentation; import datadog.trace.bootstrap.ContextStore; @@ -29,10 +31,12 @@ public void testSuiteStarted(final Description description) { return; } + TestSuiteDescriptor suiteDescriptor = JUnit4Utils.toSuiteDescriptor(description); Class testClass = description.getTestClass(); String testSuiteName = JUnit4Utils.getSuiteName(testClass, description); List categories = JUnit4Utils.getCategories(testClass, null); TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteStart( + suiteDescriptor, testSuiteName, FRAMEWORK_NAME, FRAMEWORK_VERSION, @@ -50,28 +54,27 @@ public void testSuiteFinished(final Description description) { return; } - Class testClass = description.getTestClass(); - String testSuiteName = JUnit4Utils.getSuiteName(testClass, description); - TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteFinish(testSuiteName, testClass); + TestSuiteDescriptor suiteDescriptor = JUnit4Utils.toSuiteDescriptor(description); + TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteFinish(suiteDescriptor); } @Override public void testStarted(final Description description) { + TestSuiteDescriptor suiteDescriptor = JUnit4Utils.toSuiteDescriptor(description); + TestDescriptor testDescriptor = JUnit4Utils.toTestDescriptor(description); Class testClass = description.getTestClass(); Method testMethod = JUnit4Utils.getTestMethod(description); String testMethodName = testMethod != null ? testMethod.getName() : null; - String testSuiteName = JUnit4Utils.getSuiteName(testClass, description); String testName = JUnit4Utils.getTestName(description, testMethod); - String testParameters = JUnit4Utils.getParameters(description); List categories = JUnit4Utils.getCategories(testClass, testMethod); - TestRetryPolicy retryPolicy = retryPolicies.get(description); TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestStart( + suiteDescriptor, + testDescriptor, testSuiteName, testName, - null, FRAMEWORK_NAME, FRAMEWORK_VERSION, testParameters, @@ -84,32 +87,22 @@ public void testStarted(final Description description) { @Override public void testFinished(final Description description) { - Class testClass = description.getTestClass(); - String testSuiteName = JUnit4Utils.getSuiteName(testClass, description); - Method testMethod = JUnit4Utils.getTestMethod(description); - String testName = JUnit4Utils.getTestName(description, testMethod); - String testParameters = JUnit4Utils.getParameters(description); - TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestFinish( - testSuiteName, testClass, testName, null, testParameters); + TestDescriptor testDescriptor = JUnit4Utils.toTestDescriptor(description); + TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestFinish(testDescriptor); } // same callback is executed both for test cases and test suites (for setup/teardown errors) @Override public void testFailure(final Failure failure) { Description description = failure.getDescription(); - Class testClass = description.getTestClass(); - String testSuiteName = JUnit4Utils.getSuiteName(testClass, description); if (JUnit4Utils.isTestSuiteDescription(description)) { + TestSuiteDescriptor suiteDescriptor = JUnit4Utils.toSuiteDescriptor(description); Throwable throwable = failure.getException(); - TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteFailure( - testSuiteName, testClass, throwable); + TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteFailure(suiteDescriptor, throwable); } else { - Method testMethod = JUnit4Utils.getTestMethod(description); - String testName = JUnit4Utils.getTestName(description, testMethod); - String testParameters = JUnit4Utils.getParameters(description); + TestDescriptor testDescriptor = JUnit4Utils.toTestDescriptor(description); Throwable throwable = failure.getException(); - TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestFailure( - testSuiteName, testClass, testName, null, testParameters, throwable); + TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestFailure(testDescriptor, throwable); } } @@ -124,23 +117,17 @@ public void testAssumptionFailure(final Failure failure) { } Description description = failure.getDescription(); - Class testClass = description.getTestClass(); - String testSuiteName = JUnit4Utils.getSuiteName(testClass, description); - if (JUnit4Utils.isTestSuiteDescription(description)) { - TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteSkip(testSuiteName, testClass, reason); + TestSuiteDescriptor suiteDescriptor = JUnit4Utils.toSuiteDescriptor(description); + TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteSkip(suiteDescriptor, reason); List testMethods = JUnit4Utils.getTestMethods(description.getTestClass()); for (Method testMethod : testMethods) { testIgnored(description, testMethod, reason); } } else { - Method testMethod = JUnit4Utils.getTestMethod(description); - String testName = JUnit4Utils.getTestName(description, testMethod); - String testParameters = JUnit4Utils.getParameters(description); - - TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSkip( - testSuiteName, testClass, testName, null, testParameters, reason); + TestDescriptor testDescriptor = JUnit4Utils.toTestDescriptor(description); + TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSkip(testDescriptor, reason); } } @@ -155,12 +142,13 @@ public void testIgnored(final Description description) { } else if (JUnit4Utils.isTestSuiteDescription(description)) { + TestSuiteDescriptor suiteDescriptor = JUnit4Utils.toSuiteDescriptor(description); Class testClass = description.getTestClass(); String testSuiteName = JUnit4Utils.getSuiteName(testClass, description); - List categories = JUnit4Utils.getCategories(testClass, null); TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteStart( + suiteDescriptor, testSuiteName, FRAMEWORK_NAME, FRAMEWORK_VERSION, @@ -168,31 +156,31 @@ public void testIgnored(final Description description) { categories, false, TestFrameworkInstrumentation.JUNIT4); - TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteSkip(testSuiteName, testClass, reason); + TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteSkip(suiteDescriptor, reason); List testMethods = JUnit4Utils.getTestMethods(testClass); for (Method testMethod : testMethods) { testIgnored(description, testMethod, reason); } - TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteFinish(testSuiteName, testClass); + TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteFinish(suiteDescriptor); } } private void testIgnored(Description description, Method testMethod, String reason) { + TestSuiteDescriptor suiteDescriptor = JUnit4Utils.toSuiteDescriptor(description); + TestDescriptor testDescriptor = JUnit4Utils.toTestDescriptor(description); Class testClass = description.getTestClass(); String testMethodName = testMethod != null ? testMethod.getName() : null; - String testSuiteName = JUnit4Utils.getSuiteName(testClass, description); String testName = JUnit4Utils.getTestName(description, testMethod); - String testParameters = JUnit4Utils.getParameters(description); List categories = JUnit4Utils.getCategories(testClass, testMethod); - TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestIgnore( + suiteDescriptor, + testDescriptor, testSuiteName, testName, - null, FRAMEWORK_NAME, FRAMEWORK_VERSION, testParameters, diff --git a/dd-java-agent/instrumentation/junit-4.10/src/main/java/datadog/trace/instrumentation/junit4/JUnit4Utils.java b/dd-java-agent/instrumentation/junit-4.10/src/main/java/datadog/trace/instrumentation/junit4/JUnit4Utils.java index 9b83fd3370d..08b05727d4f 100644 --- a/dd-java-agent/instrumentation/junit-4.10/src/main/java/datadog/trace/instrumentation/junit4/JUnit4Utils.java +++ b/dd-java-agent/instrumentation/junit-4.10/src/main/java/datadog/trace/instrumentation/junit4/JUnit4Utils.java @@ -2,6 +2,7 @@ import datadog.trace.api.civisibility.config.TestIdentifier; import datadog.trace.api.civisibility.events.TestDescriptor; +import datadog.trace.api.civisibility.events.TestSuiteDescriptor; import datadog.trace.util.MethodHandles; import datadog.trace.util.Strings; import java.lang.annotation.Annotation; @@ -305,4 +306,10 @@ public static TestDescriptor toTestDescriptor(Description description) { String testParameters = JUnit4Utils.getParameters(description); return new TestDescriptor(testSuiteName, testClass, testName, testParameters, null); } + + public static TestSuiteDescriptor toSuiteDescriptor(Description description) { + Class testClass = description.getTestClass(); + String testSuiteName = JUnit4Utils.getSuiteName(testClass, description); + return new TestSuiteDescriptor(testSuiteName, testClass); + } } diff --git a/dd-java-agent/instrumentation/junit-4.10/src/main/java/datadog/trace/instrumentation/junit4/TestEventsHandlerHolder.java b/dd-java-agent/instrumentation/junit-4.10/src/main/java/datadog/trace/instrumentation/junit4/TestEventsHandlerHolder.java index 7653fd0ce86..d3bb72ccd20 100644 --- a/dd-java-agent/instrumentation/junit-4.10/src/main/java/datadog/trace/instrumentation/junit4/TestEventsHandlerHolder.java +++ b/dd-java-agent/instrumentation/junit-4.10/src/main/java/datadog/trace/instrumentation/junit4/TestEventsHandlerHolder.java @@ -1,12 +1,14 @@ package datadog.trace.instrumentation.junit4; import datadog.trace.api.civisibility.InstrumentationBridge; +import datadog.trace.api.civisibility.events.TestDescriptor; import datadog.trace.api.civisibility.events.TestEventsHandler; +import datadog.trace.api.civisibility.events.TestSuiteDescriptor; import datadog.trace.util.AgentThreadFactory; public abstract class TestEventsHandlerHolder { - public static volatile TestEventsHandler TEST_EVENTS_HANDLER; + public static volatile TestEventsHandler TEST_EVENTS_HANDLER; static { start(); diff --git a/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/main/java/datadog/trace/instrumentation/junit5/CucumberTracingListener.java b/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/main/java/datadog/trace/instrumentation/junit5/CucumberTracingListener.java index bde52afdc81..629f4d703cb 100644 --- a/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/main/java/datadog/trace/instrumentation/junit5/CucumberTracingListener.java +++ b/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/main/java/datadog/trace/instrumentation/junit5/CucumberTracingListener.java @@ -36,34 +36,35 @@ public void reportingEntryPublished(TestDescriptor testDescriptor, ReportEntry e } @Override - public void executionStarted(final TestDescriptor testDescriptor) { - if (testDescriptor.isContainer()) { - containerExecutionStarted(testDescriptor); - } else if (testDescriptor.isTest()) { - testCaseExecutionStarted(testDescriptor); + public void executionStarted(final TestDescriptor descriptor) { + if (descriptor.isContainer()) { + containerExecutionStarted(descriptor); + } else if (descriptor.isTest()) { + testCaseExecutionStarted(descriptor); } } @Override public void executionFinished( - TestDescriptor testDescriptor, TestExecutionResult testExecutionResult) { - if (testDescriptor.isContainer()) { - containerExecutionFinished(testDescriptor, testExecutionResult); - } else if (testDescriptor.isTest()) { - testCaseExecutionFinished(testDescriptor, testExecutionResult); + TestDescriptor descriptor, TestExecutionResult testExecutionResult) { + if (descriptor.isContainer()) { + containerExecutionFinished(descriptor, testExecutionResult); + } else if (descriptor.isTest()) { + testCaseExecutionFinished(descriptor, testExecutionResult); } } - private void containerExecutionStarted(final TestDescriptor testDescriptor) { - UniqueId uniqueId = testDescriptor.getUniqueId(); + private void containerExecutionStarted(final TestDescriptor suiteDescriptor) { + UniqueId uniqueId = suiteDescriptor.getUniqueId(); if (!CucumberUtils.isFeature(uniqueId)) { return; } - String testSuiteName = CucumberUtils.getFeatureName(testDescriptor); + String testSuiteName = CucumberUtils.getFeatureName(suiteDescriptor); List tags = - testDescriptor.getTags().stream().map(TestTag::getName).collect(Collectors.toList()); + suiteDescriptor.getTags().stream().map(TestTag::getName).collect(Collectors.toList()); TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteStart( + suiteDescriptor, testSuiteName, testFramework, testFrameworkVersion, @@ -74,30 +75,25 @@ private void containerExecutionStarted(final TestDescriptor testDescriptor) { } private void containerExecutionFinished( - final TestDescriptor testDescriptor, final TestExecutionResult testExecutionResult) { - if (!CucumberUtils.isFeature(testDescriptor.getUniqueId())) { + final TestDescriptor suiteDescriptor, final TestExecutionResult testExecutionResult) { + if (!CucumberUtils.isFeature(suiteDescriptor.getUniqueId())) { return; } - String testSuiteName = CucumberUtils.getFeatureName(testDescriptor); Throwable throwable = testExecutionResult.getThrowable().orElse(null); if (throwable != null) { if (JUnitPlatformUtils.isAssumptionFailure(throwable)) { - String reason = throwable.getMessage(); - TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteSkip(testSuiteName, null, reason); + TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteSkip(suiteDescriptor, reason); - for (TestDescriptor child : testDescriptor.getChildren()) { + for (TestDescriptor child : suiteDescriptor.getChildren()) { executionSkipped(child, reason); } - } else { - TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteFailure( - testSuiteName, null, throwable); + TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteFailure(suiteDescriptor, throwable); } } - - TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteFinish(testSuiteName, null); + TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteFinish(suiteDescriptor); } private void testCaseExecutionStarted(final TestDescriptor testDescriptor) { @@ -109,20 +105,19 @@ private void testCaseExecutionStarted(final TestDescriptor testDescriptor) { private void testResourceExecutionStarted( TestDescriptor testDescriptor, ClasspathResourceSource testSource) { + TestDescriptor suiteDescriptor = CucumberUtils.getFeatureDescriptor(testDescriptor); String classpathResourceName = testSource.getClasspathResourceName(); - Pair names = CucumberUtils.getFeatureAndScenarioNames(testDescriptor, classpathResourceName); String testSuiteName = names.getLeft(); String testName = names.getRight(); - List tags = testDescriptor.getTags().stream().map(TestTag::getName).collect(Collectors.toList()); - TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestStart( + suiteDescriptor, + testDescriptor, testSuiteName, testName, - null, testFramework, testFrameworkVersion, null, @@ -139,60 +134,47 @@ private void testCaseExecutionFinished( final TestDescriptor testDescriptor, final TestExecutionResult testExecutionResult) { TestSource testSource = testDescriptor.getSource().orElse(null); if (testSource instanceof ClasspathResourceSource) { - testResourceExecutionFinished( - testDescriptor, testExecutionResult, (ClasspathResourceSource) testSource); + testResourceExecutionFinished(testDescriptor, testExecutionResult); } } private void testResourceExecutionFinished( - TestDescriptor testDescriptor, - TestExecutionResult testExecutionResult, - ClasspathResourceSource testSource) { - String classpathResourceName = testSource.getClasspathResourceName(); - - Pair names = - CucumberUtils.getFeatureAndScenarioNames(testDescriptor, classpathResourceName); - String testSuiteName = names.getLeft(); - String testName = names.getRight(); - + TestDescriptor testDescriptor, TestExecutionResult testExecutionResult) { Throwable throwable = testExecutionResult.getThrowable().orElse(null); if (throwable != null) { if (JUnitPlatformUtils.isAssumptionFailure(throwable)) { TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSkip( - testSuiteName, null, testName, null, null, throwable.getMessage()); + testDescriptor, throwable.getMessage()); } else { - TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestFailure( - testSuiteName, null, testName, null, null, throwable); + TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestFailure(testDescriptor, throwable); } } - - TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestFinish( - testSuiteName, null, testName, null, null); + TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestFinish(testDescriptor); } @Override - public void executionSkipped(final TestDescriptor testDescriptor, final String reason) { - TestSource testSource = testDescriptor.getSource().orElse(null); + public void executionSkipped(final TestDescriptor descriptor, final String reason) { + TestSource testSource = descriptor.getSource().orElse(null); if (testSource instanceof ClasspathResourceSource) { - testResourceExecutionSkipped(testDescriptor, (ClasspathResourceSource) testSource, reason); + testResourceExecutionSkipped(descriptor, (ClasspathResourceSource) testSource, reason); } } private void testResourceExecutionSkipped( TestDescriptor testDescriptor, ClasspathResourceSource testSource, String reason) { + TestDescriptor suiteDescriptor = CucumberUtils.getFeatureDescriptor(testDescriptor); String classpathResourceName = testSource.getClasspathResourceName(); Pair names = CucumberUtils.getFeatureAndScenarioNames(testDescriptor, classpathResourceName); String testSuiteName = names.getLeft(); String testName = names.getRight(); - List tags = testDescriptor.getTags().stream().map(TestTag::getName).collect(Collectors.toList()); - TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestIgnore( + suiteDescriptor, + testDescriptor, testSuiteName, testName, - null, testFramework, testFrameworkVersion, null, diff --git a/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/main/java/datadog/trace/instrumentation/junit5/CucumberUtils.java b/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/main/java/datadog/trace/instrumentation/junit5/CucumberUtils.java index 194da2d474c..56513d64560 100644 --- a/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/main/java/datadog/trace/instrumentation/junit5/CucumberUtils.java +++ b/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/main/java/datadog/trace/instrumentation/junit5/CucumberUtils.java @@ -96,6 +96,13 @@ public static boolean isFeature(UniqueId uniqueId) { return "feature".equals(lastSegment.getType()); } + public static TestDescriptor getFeatureDescriptor(TestDescriptor testDescriptor) { + while (testDescriptor != null && !isFeature(testDescriptor.getUniqueId())) { + testDescriptor = testDescriptor.getParent().orElse(null); + } + return testDescriptor; + } + public static TestIdentifier toTestIdentifier(TestDescriptor testDescriptor) { TestSource testSource = testDescriptor.getSource().orElse(null); if (testSource instanceof ClasspathResourceSource) { @@ -112,20 +119,4 @@ public static TestIdentifier toTestIdentifier(TestDescriptor testDescriptor) { return null; } } - - private static datadog.trace.api.civisibility.events.TestDescriptor toTestDescriptor( - TestDescriptor testDescriptor) { - TestSource testSource = testDescriptor.getSource().orElse(null); - if (!(testSource instanceof ClasspathResourceSource)) { - return null; - } - ClasspathResourceSource classpathResourceSource = (ClasspathResourceSource) testSource; - String classpathResourceName = classpathResourceSource.getClasspathResourceName(); - Pair names = - CucumberUtils.getFeatureAndScenarioNames(testDescriptor, classpathResourceName); - String testSuiteName = names.getLeft(); - String testName = names.getRight(); - return new datadog.trace.api.civisibility.events.TestDescriptor( - testSuiteName, null, testName, null, null); - } } diff --git a/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/main/java/datadog/trace/instrumentation/junit5/JUnit5CucumberInstrumentation.java b/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/main/java/datadog/trace/instrumentation/junit5/JUnit5CucumberInstrumentation.java index 82a7c566543..3732beeb20d 100644 --- a/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/main/java/datadog/trace/instrumentation/junit5/JUnit5CucumberInstrumentation.java +++ b/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/main/java/datadog/trace/instrumentation/junit5/JUnit5CucumberInstrumentation.java @@ -7,12 +7,17 @@ import com.google.auto.service.AutoService; import datadog.trace.agent.tooling.Instrumenter; import datadog.trace.agent.tooling.InstrumenterModule; +import datadog.trace.bootstrap.ContextStore; +import datadog.trace.bootstrap.InstrumentationContext; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import io.cucumber.junit.platform.engine.CucumberTestEngine; +import java.util.Collections; +import java.util.Map; import net.bytebuddy.asm.Advice; import net.bytebuddy.matcher.ElementMatcher; import org.junit.platform.engine.EngineExecutionListener; import org.junit.platform.engine.ExecutionRequest; +import org.junit.platform.engine.TestDescriptor; import org.junit.platform.engine.TestEngine; import org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService; @@ -46,6 +51,11 @@ public String[] helperClassNames() { }; } + @Override + public Map contextStore() { + return Collections.singletonMap("org.junit.platform.engine.TestDescriptor", "java.lang.Object"); + } + @Override public void methodAdvice(MethodTransformer transformer) { transformer.applyAdvice( @@ -77,6 +87,12 @@ public static void addTracingListener( return; } + ContextStore contextStore = + InstrumentationContext.get(TestDescriptor.class, Object.class); + TestEventsHandlerHolder.setContextStores( + (ContextStore) contextStore, (ContextStore) contextStore); + TestEventsHandlerHolder.start(); + CucumberTracingListener tracingListener = new CucumberTracingListener(testEngine); EngineExecutionListener originalListener = executionRequest.getEngineExecutionListener(); EngineExecutionListener compositeListener = diff --git a/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/groovy/CucumberTest.groovy b/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/groovy/CucumberTest.groovy index b0a986c76d6..cdb6707e47e 100644 --- a/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/groovy/CucumberTest.groovy +++ b/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/groovy/CucumberTest.groovy @@ -103,7 +103,7 @@ class CucumberTest extends CiVisibilityInstrumentationTest { } protected void runFeatures(List classpathFeatures, boolean parallel) { - TestEventsHandlerHolder.start() + TestEventsHandlerHolder.startForcefully() DiscoverySelector[] selectors = new DiscoverySelector[classpathFeatures.size()] for (i in 0.. contextStore() { + return Collections.singletonMap("org.junit.platform.engine.TestDescriptor", "java.lang.Object"); + } + @Override public void methodAdvice(MethodTransformer transformer) { transformer.applyAdvice( @@ -77,6 +87,12 @@ public static void addTracingListener( return; } + ContextStore contextStore = + InstrumentationContext.get(TestDescriptor.class, Object.class); + TestEventsHandlerHolder.setContextStores( + (ContextStore) contextStore, (ContextStore) contextStore); + TestEventsHandlerHolder.start(); + SpockTracingListener tracingListener = new SpockTracingListener(testEngine); EngineExecutionListener originalListener = executionRequest.getEngineExecutionListener(); EngineExecutionListener compositeListener = diff --git a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/main/java/datadog/trace/instrumentation/junit5/SpockTracingListener.java b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/main/java/datadog/trace/instrumentation/junit5/SpockTracingListener.java index 0fc219828d1..db554cb127d 100644 --- a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/main/java/datadog/trace/instrumentation/junit5/SpockTracingListener.java +++ b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/main/java/datadog/trace/instrumentation/junit5/SpockTracingListener.java @@ -35,36 +35,36 @@ public void reportingEntryPublished(TestDescriptor testDescriptor, ReportEntry e } @Override - public void executionStarted(final TestDescriptor testDescriptor) { - if (testDescriptor.isContainer()) { - containerExecutionStarted(testDescriptor); - } else if (testDescriptor.isTest()) { - testCaseExecutionStarted(testDescriptor); + public void executionStarted(final TestDescriptor descriptor) { + if (descriptor.isContainer()) { + containerExecutionStarted(descriptor); + } else if (descriptor.isTest()) { + testCaseExecutionStarted(descriptor); } } @Override public void executionFinished( - TestDescriptor testDescriptor, TestExecutionResult testExecutionResult) { - if (testDescriptor.isContainer()) { - containerExecutionFinished(testDescriptor, testExecutionResult); - } else if (testDescriptor.isTest()) { - testCaseExecutionFinished(testDescriptor, testExecutionResult); + TestDescriptor descriptor, TestExecutionResult testExecutionResult) { + if (descriptor.isContainer()) { + containerExecutionFinished(descriptor, testExecutionResult); + } else if (descriptor.isTest()) { + testCaseExecutionFinished(descriptor, testExecutionResult); } } - private void containerExecutionStarted(final TestDescriptor testDescriptor) { - if (!SpockUtils.isSpec(testDescriptor)) { + private void containerExecutionStarted(final TestDescriptor suiteDescriptor) { + if (!SpockUtils.isSpec(suiteDescriptor)) { return; } - Class testClass = JUnitPlatformUtils.getJavaClass(testDescriptor); + Class testClass = JUnitPlatformUtils.getJavaClass(suiteDescriptor); String testSuiteName = - testClass != null ? testClass.getName() : testDescriptor.getLegacyReportingName(); - + testClass != null ? testClass.getName() : suiteDescriptor.getLegacyReportingName(); List tags = - testDescriptor.getTags().stream().map(TestTag::getName).collect(Collectors.toList()); + suiteDescriptor.getTags().stream().map(TestTag::getName).collect(Collectors.toList()); TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteStart( + suiteDescriptor, testSuiteName, testFramework, testFrameworkVersion, @@ -75,34 +75,28 @@ private void containerExecutionStarted(final TestDescriptor testDescriptor) { } private void containerExecutionFinished( - final TestDescriptor testDescriptor, final TestExecutionResult testExecutionResult) { - if (!SpockUtils.isSpec(testDescriptor)) { + final TestDescriptor suiteDescriptor, final TestExecutionResult testExecutionResult) { + if (!SpockUtils.isSpec(suiteDescriptor)) { return; } - Class testClass = JUnitPlatformUtils.getJavaClass(testDescriptor); - String testSuiteName = - testClass != null ? testClass.getName() : testDescriptor.getLegacyReportingName(); - Throwable throwable = testExecutionResult.getThrowable().orElse(null); if (throwable != null) { if (JUnitPlatformUtils.isAssumptionFailure(throwable)) { String reason = throwable.getMessage(); - TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteSkip( - testSuiteName, testClass, reason); + TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteSkip(suiteDescriptor, reason); - for (TestDescriptor child : testDescriptor.getChildren()) { + for (TestDescriptor child : suiteDescriptor.getChildren()) { executionSkipped(child, reason); } } else { - TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteFailure( - testSuiteName, testClass, throwable); + TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteFailure(suiteDescriptor, throwable); } } - TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteFinish(testSuiteName, testClass); + TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteFinish(suiteDescriptor); } private void testCaseExecutionStarted(final TestDescriptor testDescriptor) { @@ -113,21 +107,20 @@ private void testCaseExecutionStarted(final TestDescriptor testDescriptor) { } private void testMethodExecutionStarted(TestDescriptor testDescriptor, MethodSource testSource) { + TestDescriptor suiteDescriptor = SpockUtils.getSpecDescriptor(testDescriptor); String testSuitName = testSource.getClassName(); String displayName = testDescriptor.getDisplayName(); - String testParameters = JUnitPlatformUtils.getParameters(testSource, displayName); List tags = testDescriptor.getTags().stream().map(TestTag::getName).collect(Collectors.toList()); - Class testClass = testSource.getJavaClass(); Method testMethod = SpockUtils.getTestMethod(testSource); String testMethodName = testSource.getMethodName(); - TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestStart( + suiteDescriptor, + testDescriptor, testSuitName, displayName, - null, testFramework, testFrameworkVersion, testParameters, @@ -142,61 +135,51 @@ private void testCaseExecutionFinished( final TestDescriptor testDescriptor, final TestExecutionResult testExecutionResult) { TestSource testSource = testDescriptor.getSource().orElse(null); if (testSource instanceof MethodSource) { - testMethodExecutionFinished(testDescriptor, testExecutionResult, (MethodSource) testSource); + testMethodExecutionFinished(testDescriptor, testExecutionResult); } } private static void testMethodExecutionFinished( - TestDescriptor testDescriptor, - TestExecutionResult testExecutionResult, - MethodSource testSource) { - String testSuiteName = testSource.getClassName(); - Class testClass = testSource.getJavaClass(); - String displayName = testDescriptor.getDisplayName(); - String testParameters = JUnitPlatformUtils.getParameters(testSource, displayName); - + TestDescriptor testDescriptor, TestExecutionResult testExecutionResult) { Throwable throwable = testExecutionResult.getThrowable().orElse(null); if (throwable != null) { if (JUnitPlatformUtils.isAssumptionFailure(throwable)) { TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSkip( - testSuiteName, testClass, displayName, null, testParameters, throwable.getMessage()); + testDescriptor, throwable.getMessage()); } else { - TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestFailure( - testSuiteName, testClass, displayName, null, testParameters, throwable); + TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestFailure(testDescriptor, throwable); } } - - TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestFinish( - testSuiteName, testClass, displayName, null, testParameters); + TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestFinish(testDescriptor); } @Override - public void executionSkipped(final TestDescriptor testDescriptor, final String reason) { - TestSource testSource = testDescriptor.getSource().orElse(null); - + public void executionSkipped(final TestDescriptor descriptor, final String reason) { + TestSource testSource = descriptor.getSource().orElse(null); if (testSource instanceof ClassSource) { // The annotation @Disabled is kept at type level. - containerExecutionSkipped(testDescriptor, reason); + containerExecutionSkipped(descriptor, reason); } else if (testSource instanceof MethodSource) { // The annotation @Disabled is kept at method level. - testMethodExecutionSkipped(testDescriptor, (MethodSource) testSource, reason); + testMethodExecutionSkipped(descriptor, (MethodSource) testSource, reason); } } - private void containerExecutionSkipped(final TestDescriptor testDescriptor, final String reason) { - if (!SpockUtils.isSpec(testDescriptor)) { + private void containerExecutionSkipped( + final TestDescriptor suiteDescriptor, final String reason) { + if (!SpockUtils.isSpec(suiteDescriptor)) { return; } - Class testClass = JUnitPlatformUtils.getJavaClass(testDescriptor); + Class testClass = JUnitPlatformUtils.getJavaClass(suiteDescriptor); String testSuiteName = - testClass != null ? testClass.getName() : testDescriptor.getLegacyReportingName(); - + testClass != null ? testClass.getName() : suiteDescriptor.getLegacyReportingName(); List tags = - testDescriptor.getTags().stream().map(TestTag::getName).collect(Collectors.toList()); + suiteDescriptor.getTags().stream().map(TestTag::getName).collect(Collectors.toList()); TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteStart( + suiteDescriptor, testSuiteName, testFramework, testFrameworkVersion, @@ -204,32 +187,31 @@ private void containerExecutionSkipped(final TestDescriptor testDescriptor, fina tags, false, TestFrameworkInstrumentation.SPOCK); - TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteSkip(testSuiteName, testClass, reason); + TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteSkip(suiteDescriptor, reason); - for (TestDescriptor child : testDescriptor.getChildren()) { + for (TestDescriptor child : suiteDescriptor.getChildren()) { executionSkipped(child, reason); } - TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteFinish(testSuiteName, testClass); + TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteFinish(suiteDescriptor); } private void testMethodExecutionSkipped( final TestDescriptor testDescriptor, final MethodSource methodSource, final String reason) { + TestDescriptor suiteDescriptor = SpockUtils.getSpecDescriptor(testDescriptor); String testSuiteName = methodSource.getClassName(); String displayName = testDescriptor.getDisplayName(); - String testParameters = JUnitPlatformUtils.getParameters(methodSource, displayName); List tags = testDescriptor.getTags().stream().map(TestTag::getName).collect(Collectors.toList()); - Class testClass = methodSource.getJavaClass(); Method testMethod = SpockUtils.getTestMethod(methodSource); String testMethodName = methodSource.getMethodName(); - TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestIgnore( + suiteDescriptor, + testDescriptor, testSuiteName, displayName, - null, testFramework, testFrameworkVersion, testParameters, diff --git a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/main/java/datadog/trace/instrumentation/junit5/SpockUtils.java b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/main/java/datadog/trace/instrumentation/junit5/SpockUtils.java index eaf849edf13..909eacb5340 100644 --- a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/main/java/datadog/trace/instrumentation/junit5/SpockUtils.java +++ b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/main/java/datadog/trace/instrumentation/junit5/SpockUtils.java @@ -119,26 +119,17 @@ public static TestIdentifier toTestIdentifier(TestDescriptor testDescriptor) { } } - private static datadog.trace.api.civisibility.events.TestDescriptor toTestDescriptor( - TestDescriptor testDescriptor) { - TestSource testSource = testDescriptor.getSource().orElse(null); - if (!(testSource instanceof MethodSource) || !(testDescriptor instanceof SpockNode)) { - return null; - } - - MethodSource methodSource = (MethodSource) testSource; - String testSuiteName = methodSource.getClassName(); - String displayName = testDescriptor.getDisplayName(); - String testParameters = JUnitPlatformUtils.getParameters(methodSource, displayName); - Class testClass = methodSource.getJavaClass(); - return new datadog.trace.api.civisibility.events.TestDescriptor( - testSuiteName, testClass, displayName, testParameters, null); - } - public static boolean isSpec(TestDescriptor testDescriptor) { UniqueId uniqueId = testDescriptor.getUniqueId(); List segments = uniqueId.getSegments(); UniqueId.Segment lastSegment = segments.get(segments.size() - 1); return "spec".equals(lastSegment.getType()); } + + public static TestDescriptor getSpecDescriptor(TestDescriptor testDescriptor) { + while (testDescriptor != null && !isSpec(testDescriptor)) { + testDescriptor = testDescriptor.getParent().orElse(null); + } + return testDescriptor; + } } diff --git a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/groovy/SpockTest.groovy b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/groovy/SpockTest.groovy index cae43f6aaaa..3dcdddd2976 100644 --- a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/groovy/SpockTest.groovy +++ b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/groovy/SpockTest.groovy @@ -111,7 +111,7 @@ class SpockTest extends CiVisibilityInstrumentationTest { } private static void runTests(List> classes) { - TestEventsHandlerHolder.start() + TestEventsHandlerHolder.startForcefully() DiscoverySelector[] selectors = new DiscoverySelector[classes.size()] for (i in 0.. contextStore() { + return Collections.singletonMap("org.junit.platform.engine.TestDescriptor", "java.lang.Object"); + } + @Override public void methodAdvice(MethodTransformer transformer) { transformer.applyAdvice( @@ -57,7 +67,6 @@ public void methodAdvice(MethodTransformer transformer) { } public static class JUnit5Advice { - @SuppressFBWarnings( value = "UC_USELESS_OBJECT", justification = "executionRequest is the argument of the original method") @@ -85,6 +94,12 @@ public static void addTracingListener( return; } + ContextStore contextStore = + InstrumentationContext.get(TestDescriptor.class, Object.class); + TestEventsHandlerHolder.setContextStores( + (ContextStore) contextStore, (ContextStore) contextStore); + TestEventsHandlerHolder.start(); + TracingListener tracingListener = new TracingListener(testEngine); EngineExecutionListener originalListener = executionRequest.getEngineExecutionListener(); EngineExecutionListener compositeListener = diff --git a/dd-java-agent/instrumentation/junit-5.3/src/main/java/datadog/trace/instrumentation/junit5/JUnitPlatformUtils.java b/dd-java-agent/instrumentation/junit-5.3/src/main/java/datadog/trace/instrumentation/junit5/JUnitPlatformUtils.java index fbb193fac5d..e978304a644 100644 --- a/dd-java-agent/instrumentation/junit-5.3/src/main/java/datadog/trace/instrumentation/junit5/JUnitPlatformUtils.java +++ b/dd-java-agent/instrumentation/junit-5.3/src/main/java/datadog/trace/instrumentation/junit5/JUnitPlatformUtils.java @@ -107,34 +107,6 @@ public static TestIdentifier toTestIdentifier(TestDescriptor testDescriptor) { } } - public static datadog.trace.api.civisibility.events.TestDescriptor toTestDescriptor( - TestDescriptor testDescriptor) { - TestSource testSource = testDescriptor.getSource().orElse(null); - if (!(testSource instanceof MethodSource)) { - return null; - } - - MethodSource methodSource = (MethodSource) testSource; - TestDescriptor suiteDescriptor = JUnitPlatformUtils.getSuiteDescriptor(testDescriptor); - - Class testClass; - String testSuiteName; - if (suiteDescriptor != null) { - testClass = JUnitPlatformUtils.getJavaClass(suiteDescriptor); - testSuiteName = - testClass != null ? testClass.getName() : suiteDescriptor.getLegacyReportingName(); - } else { - testClass = JUnitPlatformUtils.getTestClass(methodSource); - testSuiteName = methodSource.getClassName(); - } - - String testName = methodSource.getMethodName(); - String displayName = testDescriptor.getDisplayName(); - String testParameters = JUnitPlatformUtils.getParameters(methodSource, displayName); - return new datadog.trace.api.civisibility.events.TestDescriptor( - testSuiteName, testClass, testName, testParameters, null); - } - public static boolean isAssumptionFailure(Throwable throwable) { switch (throwable.getClass().getName()) { case "org.junit.AssumptionViolatedException": diff --git a/dd-java-agent/instrumentation/junit-5.3/src/main/java/datadog/trace/instrumentation/junit5/TestEventsHandlerHolder.java b/dd-java-agent/instrumentation/junit-5.3/src/main/java/datadog/trace/instrumentation/junit5/TestEventsHandlerHolder.java index 2d466049dc0..13f9f1b188a 100644 --- a/dd-java-agent/instrumentation/junit-5.3/src/main/java/datadog/trace/instrumentation/junit5/TestEventsHandlerHolder.java +++ b/dd-java-agent/instrumentation/junit-5.3/src/main/java/datadog/trace/instrumentation/junit5/TestEventsHandlerHolder.java @@ -1,15 +1,20 @@ package datadog.trace.instrumentation.junit5; +import datadog.trace.api.civisibility.DDTest; +import datadog.trace.api.civisibility.DDTestSuite; import datadog.trace.api.civisibility.InstrumentationBridge; import datadog.trace.api.civisibility.events.TestEventsHandler; +import datadog.trace.bootstrap.ContextStore; import datadog.trace.util.AgentThreadFactory; +import org.junit.platform.engine.TestDescriptor; public abstract class TestEventsHandlerHolder { - public static volatile TestEventsHandler TEST_EVENTS_HANDLER; + public static volatile TestEventsHandler TEST_EVENTS_HANDLER; + private static ContextStore SUITE_STORE; + private static ContextStore TEST_STORE; static { - start(); Runtime.getRuntime() .addShutdownHook( AgentThreadFactory.newAgentThread( @@ -18,11 +23,33 @@ public abstract class TestEventsHandlerHolder { false)); } - public static void start() { - TEST_EVENTS_HANDLER = InstrumentationBridge.createTestEventsHandler("junit"); + public static synchronized void setContextStores( + ContextStore suiteStore, + ContextStore testStore) { + if (SUITE_STORE == null) { + SUITE_STORE = suiteStore; + } + if (TEST_STORE == null) { + TEST_STORE = testStore; + } + } + + public static synchronized void start() { + if (TEST_EVENTS_HANDLER == null) { + TEST_EVENTS_HANDLER = + InstrumentationBridge.createTestEventsHandler("junit", SUITE_STORE, TEST_STORE); + } + } + + // used by instrumentation tests + public static synchronized void startForcefully() { + if (SUITE_STORE != null && TEST_STORE != null) { + TEST_EVENTS_HANDLER = + InstrumentationBridge.createTestEventsHandler("junit", SUITE_STORE, TEST_STORE); + } } - public static void stop() { + public static synchronized void stop() { if (TEST_EVENTS_HANDLER != null) { TEST_EVENTS_HANDLER.close(); TEST_EVENTS_HANDLER = null; diff --git a/dd-java-agent/instrumentation/junit-5.3/src/main/java/datadog/trace/instrumentation/junit5/TracingListener.java b/dd-java-agent/instrumentation/junit-5.3/src/main/java/datadog/trace/instrumentation/junit5/TracingListener.java index 8257d4d5b26..7cb32504963 100644 --- a/dd-java-agent/instrumentation/junit-5.3/src/main/java/datadog/trace/instrumentation/junit5/TracingListener.java +++ b/dd-java-agent/instrumentation/junit-5.3/src/main/java/datadog/trace/instrumentation/junit5/TracingListener.java @@ -36,36 +36,36 @@ public void reportingEntryPublished(TestDescriptor testDescriptor, ReportEntry e } @Override - public void executionStarted(final TestDescriptor testDescriptor) { - if (testDescriptor.isContainer()) { - containerExecutionStarted(testDescriptor); - } else if (testDescriptor.isTest()) { - testCaseExecutionStarted(testDescriptor); + public void executionStarted(final TestDescriptor descriptor) { + if (descriptor.isContainer()) { + containerExecutionStarted(descriptor); + } else if (descriptor.isTest()) { + testCaseExecutionStarted(descriptor); } } @Override public void executionFinished( - TestDescriptor testDescriptor, TestExecutionResult testExecutionResult) { - if (testDescriptor.isContainer()) { - containerExecutionFinished(testDescriptor, testExecutionResult); - } else if (testDescriptor.isTest()) { - testCaseExecutionFinished(testDescriptor, testExecutionResult); + TestDescriptor descriptor, TestExecutionResult testExecutionResult) { + if (descriptor.isContainer()) { + containerExecutionFinished(descriptor, testExecutionResult); + } else if (descriptor.isTest()) { + testCaseExecutionFinished(descriptor, testExecutionResult); } } - private void containerExecutionStarted(final TestDescriptor testDescriptor) { - if (!JUnitPlatformUtils.isSuite(testDescriptor)) { + private void containerExecutionStarted(final TestDescriptor suiteDescriptor) { + if (!JUnitPlatformUtils.isSuite(suiteDescriptor)) { return; } - Class testClass = JUnitPlatformUtils.getJavaClass(testDescriptor); + Class testClass = JUnitPlatformUtils.getJavaClass(suiteDescriptor); String testSuiteName = - testClass != null ? testClass.getName() : testDescriptor.getLegacyReportingName(); - + testClass != null ? testClass.getName() : suiteDescriptor.getLegacyReportingName(); List tags = - testDescriptor.getTags().stream().map(TestTag::getName).collect(Collectors.toList()); + suiteDescriptor.getTags().stream().map(TestTag::getName).collect(Collectors.toList()); TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteStart( + suiteDescriptor, testSuiteName, testFramework, testFrameworkVersion, @@ -76,34 +76,25 @@ private void containerExecutionStarted(final TestDescriptor testDescriptor) { } private void containerExecutionFinished( - final TestDescriptor testDescriptor, final TestExecutionResult testExecutionResult) { - if (!JUnitPlatformUtils.isSuite(testDescriptor)) { + final TestDescriptor suiteDescriptor, final TestExecutionResult testExecutionResult) { + if (!JUnitPlatformUtils.isSuite(suiteDescriptor)) { return; } - Class testClass = JUnitPlatformUtils.getJavaClass(testDescriptor); - String testSuiteName = - testClass != null ? testClass.getName() : testDescriptor.getLegacyReportingName(); - Throwable throwable = testExecutionResult.getThrowable().orElse(null); if (throwable != null) { if (JUnitPlatformUtils.isAssumptionFailure(throwable)) { - String reason = throwable.getMessage(); - TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteSkip( - testSuiteName, testClass, reason); + TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteSkip(suiteDescriptor, reason); - for (TestDescriptor child : testDescriptor.getChildren()) { + for (TestDescriptor child : suiteDescriptor.getChildren()) { executionSkipped(child, reason); } - } else { - TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteFailure( - testSuiteName, testClass, throwable); + TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteFailure(suiteDescriptor, throwable); } } - - TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteFinish(testSuiteName, testClass); + TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteFinish(suiteDescriptor); } private void testCaseExecutionStarted(final TestDescriptor testDescriptor) { @@ -129,18 +120,17 @@ private void testMethodExecutionStarted(TestDescriptor testDescriptor, MethodSou String displayName = testDescriptor.getDisplayName(); String testName = testSource.getMethodName(); - String testParameters = JUnitPlatformUtils.getParameters(testSource, displayName); List tags = testDescriptor.getTags().stream().map(TestTag::getName).collect(Collectors.toList()); - Method testMethod = JUnitPlatformUtils.getTestMethod(testSource); String testMethodName = testSource.getMethodName(); TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestStart( + suiteDescriptor, + testDescriptor, testSuiteName, testName, - null, testFramework, testFrameworkVersion, testParameters, @@ -155,73 +145,52 @@ private void testCaseExecutionFinished( final TestDescriptor testDescriptor, final TestExecutionResult testExecutionResult) { TestSource testSource = testDescriptor.getSource().orElse(null); if (testSource instanceof MethodSource) { - testMethodExecutionFinished(testDescriptor, testExecutionResult, (MethodSource) testSource); + testMethodExecutionFinished(testDescriptor, testExecutionResult); } } private static void testMethodExecutionFinished( - TestDescriptor testDescriptor, - TestExecutionResult testExecutionResult, - MethodSource testSource) { - TestDescriptor suiteDescriptor = JUnitPlatformUtils.getSuiteDescriptor(testDescriptor); - - Class testClass; - String testSuiteName; - if (suiteDescriptor != null) { - testClass = JUnitPlatformUtils.getJavaClass(suiteDescriptor); - testSuiteName = - testClass != null ? testClass.getName() : suiteDescriptor.getLegacyReportingName(); - } else { - testClass = JUnitPlatformUtils.getTestClass(testSource); - testSuiteName = testSource.getClassName(); - } - - String displayName = testDescriptor.getDisplayName(); - String testName = testSource.getMethodName(); - String testParameters = JUnitPlatformUtils.getParameters(testSource, displayName); - + TestDescriptor testDescriptor, TestExecutionResult testExecutionResult) { Throwable throwable = testExecutionResult.getThrowable().orElse(null); if (throwable != null) { if (JUnitPlatformUtils.isAssumptionFailure(throwable)) { TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSkip( - testSuiteName, testClass, testName, null, testParameters, throwable.getMessage()); + testDescriptor, throwable.getMessage()); } else { - TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestFailure( - testSuiteName, testClass, testName, null, testParameters, throwable); + TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestFailure(testDescriptor, throwable); } } - - TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestFinish( - testSuiteName, testClass, testName, null, testParameters); + TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestFinish(testDescriptor); } @Override - public void executionSkipped(final TestDescriptor testDescriptor, final String reason) { - TestSource testSource = testDescriptor.getSource().orElse(null); + public void executionSkipped(final TestDescriptor descriptor, final String reason) { + TestSource testSource = descriptor.getSource().orElse(null); if (testSource instanceof ClassSource) { // The annotation @Disabled is kept at type level. - containerExecutionSkipped(testDescriptor, reason); + containerExecutionSkipped(descriptor, reason); } else if (testSource instanceof MethodSource) { // The annotation @Disabled is kept at method level. - testMethodExecutionSkipped(testDescriptor, (MethodSource) testSource, reason); + testMethodExecutionSkipped(descriptor, (MethodSource) testSource, reason); } } - private void containerExecutionSkipped(final TestDescriptor testDescriptor, final String reason) { - if (!JUnitPlatformUtils.isSuite(testDescriptor)) { + private void containerExecutionSkipped( + final TestDescriptor suiteDescriptor, final String reason) { + if (!JUnitPlatformUtils.isSuite(suiteDescriptor)) { return; } - Class testClass = JUnitPlatformUtils.getJavaClass(testDescriptor); + Class testClass = JUnitPlatformUtils.getJavaClass(suiteDescriptor); String testSuiteName = - testClass != null ? testClass.getName() : testDescriptor.getLegacyReportingName(); - + testClass != null ? testClass.getName() : suiteDescriptor.getLegacyReportingName(); List tags = - testDescriptor.getTags().stream().map(TestTag::getName).collect(Collectors.toList()); + suiteDescriptor.getTags().stream().map(TestTag::getName).collect(Collectors.toList()); TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteStart( + suiteDescriptor, testSuiteName, testFramework, testFrameworkVersion, @@ -229,13 +198,13 @@ private void containerExecutionSkipped(final TestDescriptor testDescriptor, fina tags, false, TestFrameworkInstrumentation.JUNIT5); - TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteSkip(testSuiteName, testClass, reason); + TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteSkip(suiteDescriptor, reason); - for (TestDescriptor child : testDescriptor.getChildren()) { + for (TestDescriptor child : suiteDescriptor.getChildren()) { executionSkipped(child, reason); } - TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteFinish(testSuiteName, testClass); + TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteFinish(suiteDescriptor); } private void testMethodExecutionSkipped( @@ -255,18 +224,17 @@ private void testMethodExecutionSkipped( String displayName = testDescriptor.getDisplayName(); String testName = testSource.getMethodName(); - String testParameters = JUnitPlatformUtils.getParameters(testSource, displayName); List tags = testDescriptor.getTags().stream().map(TestTag::getName).collect(Collectors.toList()); - Method testMethod = JUnitPlatformUtils.getTestMethod(testSource); String testMethodName = testSource.getMethodName(); TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestIgnore( + suiteDescriptor, + testDescriptor, testSuiteName, testName, - null, testFramework, testFrameworkVersion, testParameters, diff --git a/dd-java-agent/instrumentation/junit-5.3/src/test/groovy/JUnit5Test.groovy b/dd-java-agent/instrumentation/junit-5.3/src/test/groovy/JUnit5Test.groovy index dc701acc40a..7bf1b58dd6e 100644 --- a/dd-java-agent/instrumentation/junit-5.3/src/test/groovy/JUnit5Test.groovy +++ b/dd-java-agent/instrumentation/junit-5.3/src/test/groovy/JUnit5Test.groovy @@ -138,7 +138,7 @@ class JUnit5Test extends CiVisibilityInstrumentationTest { } private static void runTests(List> tests) { - TestEventsHandlerHolder.start() + TestEventsHandlerHolder.startForcefully() DiscoverySelector[] selectors = new DiscoverySelector[tests.size()] for (i in 0.. TEST_EVENTS_HANDLER; static { start(); diff --git a/dd-java-agent/instrumentation/scalatest/src/main/java/datadog/trace/instrumentation/scalatest/DatadogReporter.java b/dd-java-agent/instrumentation/scalatest/src/main/java/datadog/trace/instrumentation/scalatest/DatadogReporter.java index fd74f133a6f..bd26a3ed2ae 100644 --- a/dd-java-agent/instrumentation/scalatest/src/main/java/datadog/trace/instrumentation/scalatest/DatadogReporter.java +++ b/dd-java-agent/instrumentation/scalatest/src/main/java/datadog/trace/instrumentation/scalatest/DatadogReporter.java @@ -2,7 +2,9 @@ import datadog.trace.api.civisibility.InstrumentationBridge; import datadog.trace.api.civisibility.config.TestIdentifier; +import datadog.trace.api.civisibility.events.TestDescriptor; import datadog.trace.api.civisibility.events.TestEventsHandler; +import datadog.trace.api.civisibility.events.TestSuiteDescriptor; import datadog.trace.api.civisibility.retry.TestRetryPolicy; import datadog.trace.api.civisibility.telemetry.tag.TestFrameworkInstrumentation; import datadog.trace.instrumentation.scalatest.retry.SuppressedTestFailedException; @@ -84,7 +86,7 @@ private static void stop(Event event) { private static void onSuiteStart(SuiteStarting event) { int runStamp = event.ordinal().runStamp(); RunContext context = RunContext.getOrCreate(runStamp); - TestEventsHandler eventHandler = context.getEventHandler(); + TestEventsHandler eventHandler = context.getEventHandler(); String testSuiteName = event.suiteId(); Class testClass = ScalatestUtils.getClass(event.suiteClassName()); @@ -92,6 +94,7 @@ private static void onSuiteStart(SuiteStarting event) { boolean parallelized = true; eventHandler.onTestSuiteStart( + new TestSuiteDescriptor(testSuiteName, testClass), testSuiteName, TEST_FRAMEWORK, TEST_FRAMEWORK_VERSION, @@ -104,29 +107,29 @@ private static void onSuiteStart(SuiteStarting event) { private static void onSuiteFinish(SuiteCompleted event) { int runStamp = event.ordinal().runStamp(); RunContext context = RunContext.getOrCreate(runStamp); - TestEventsHandler eventHandler = context.getEventHandler(); + TestEventsHandler eventHandler = context.getEventHandler(); String testSuiteName = event.suiteId(); Class testClass = ScalatestUtils.getClass(event.suiteClassName()); - eventHandler.onTestSuiteFinish(testSuiteName, testClass); + eventHandler.onTestSuiteFinish(new TestSuiteDescriptor(testSuiteName, testClass)); } private static void onSuiteAbort(SuiteAborted event) { int runStamp = event.ordinal().runStamp(); RunContext context = RunContext.getOrCreate(runStamp); - TestEventsHandler eventHandler = context.getEventHandler(); + TestEventsHandler eventHandler = context.getEventHandler(); String testSuiteName = event.suiteId(); Class testClass = ScalatestUtils.getClass(event.suiteClassName()); Throwable throwable = event.throwable().getOrElse(null); - eventHandler.onTestSuiteFailure(testSuiteName, testClass, throwable); - eventHandler.onTestSuiteFinish(testSuiteName, testClass); + eventHandler.onTestSuiteFailure(new TestSuiteDescriptor(testSuiteName, testClass), throwable); + eventHandler.onTestSuiteFinish(new TestSuiteDescriptor(testSuiteName, testClass)); } private static void onTestStart(TestStarting event) { int runStamp = event.ordinal().runStamp(); RunContext context = RunContext.getOrCreate(runStamp); - TestEventsHandler eventHandler = context.getEventHandler(); + TestEventsHandler eventHandler = context.getEventHandler(); String testSuiteName = event.suiteId(); String testName = event.testName(); @@ -145,9 +148,10 @@ private static void onTestStart(TestStarting event) { TestRetryPolicy retryPolicy = context.popRetryPolicy(testIdentifier); eventHandler.onTestStart( + new TestSuiteDescriptor(testSuiteName, testClass), + new TestDescriptor(testSuiteName, testClass, testName, testParameters, testQualifier), testSuiteName, testName, - testQualifier, TEST_FRAMEWORK, TEST_FRAMEWORK_VERSION, testParameters, @@ -161,20 +165,22 @@ private static void onTestStart(TestStarting event) { private static void onTestSuccess(TestSucceeded event) { int runStamp = event.ordinal().runStamp(); RunContext context = RunContext.getOrCreate(runStamp); - TestEventsHandler eventHandler = context.getEventHandler(); + TestEventsHandler eventHandler = context.getEventHandler(); String testSuiteName = event.suiteId(); Class testClass = ScalatestUtils.getClass(event.suiteClassName()); String testName = event.testName(); Object testQualifier = null; String testParameters = null; - eventHandler.onTestFinish(testSuiteName, testClass, testName, testQualifier, testParameters); + TestDescriptor testDescriptor = + new TestDescriptor(testSuiteName, testClass, testName, testParameters, testQualifier); + eventHandler.onTestFinish(testDescriptor); } private static void onTestFailure(TestFailed event) { int runStamp = event.ordinal().runStamp(); RunContext context = RunContext.getOrCreate(runStamp); - TestEventsHandler eventHandler = context.getEventHandler(); + TestEventsHandler eventHandler = context.getEventHandler(); String testSuiteName = event.suiteId(); Class testClass = ScalatestUtils.getClass(event.suiteClassName()); @@ -182,15 +188,16 @@ private static void onTestFailure(TestFailed event) { Object testQualifier = null; String testParameters = null; Throwable throwable = event.throwable().getOrElse(null); - eventHandler.onTestFailure( - testSuiteName, testClass, testName, testQualifier, testParameters, throwable); - eventHandler.onTestFinish(testSuiteName, testClass, testName, testQualifier, testParameters); + TestDescriptor testDescriptor = + new TestDescriptor(testSuiteName, testClass, testName, testParameters, testQualifier); + eventHandler.onTestFailure(testDescriptor, throwable); + eventHandler.onTestFinish(testDescriptor); } private static void onTestIgnore(TestIgnored event) { int runStamp = event.ordinal().runStamp(); RunContext context = RunContext.getOrCreate(runStamp); - TestEventsHandler eventHandler = context.getEventHandler(); + TestEventsHandler eventHandler = context.getEventHandler(); String testSuiteName = event.suiteId(); String testName = event.testName(); @@ -210,9 +217,10 @@ private static void onTestIgnore(TestIgnored event) { } eventHandler.onTestIgnore( + new TestSuiteDescriptor(testSuiteName, testClass), + new TestDescriptor(testSuiteName, testClass, testName, testParameters, testQualifier), testSuiteName, testName, - testQualifier, TEST_FRAMEWORK, TEST_FRAMEWORK_VERSION, testParameters, @@ -226,7 +234,7 @@ private static void onTestIgnore(TestIgnored event) { private static void onTestCancel(TestCanceled event) { int runStamp = event.ordinal().runStamp(); RunContext context = RunContext.getOrCreate(runStamp); - TestEventsHandler eventHandler = context.getEventHandler(); + TestEventsHandler eventHandler = context.getEventHandler(); String testSuiteName = event.suiteId(); String testName = event.testName(); @@ -236,21 +244,20 @@ private static void onTestCancel(TestCanceled event) { Throwable throwable = event.throwable().getOrElse(null); String reason = throwable != null ? throwable.getMessage() : null; + TestDescriptor testDescriptor = + new TestDescriptor(testSuiteName, testClass, testName, testParameters, testQualifier); if (throwable instanceof SuppressedTestFailedException) { - eventHandler.onTestFailure( - testSuiteName, testClass, testName, testQualifier, testParameters, throwable.getCause()); + eventHandler.onTestFailure(testDescriptor, throwable.getCause()); } else { - eventHandler.onTestSkip( - testSuiteName, testClass, testName, testQualifier, testParameters, reason); + eventHandler.onTestSkip(testDescriptor, reason); } - - eventHandler.onTestFinish(testSuiteName, testClass, testName, testQualifier, testParameters); + eventHandler.onTestFinish(testDescriptor); } private static void onTestPending(TestPending event) { int runStamp = event.ordinal().runStamp(); RunContext context = RunContext.getOrCreate(runStamp); - TestEventsHandler eventHandler = context.getEventHandler(); + TestEventsHandler eventHandler = context.getEventHandler(); String testSuiteName = event.suiteId(); String testName = event.testName(); @@ -259,8 +266,9 @@ private static void onTestPending(TestPending event) { Class testClass = ScalatestUtils.getClass(event.suiteClassName()); String reason = "pending"; - eventHandler.onTestSkip( - testSuiteName, testClass, testName, testQualifier, testParameters, reason); - eventHandler.onTestFinish(testSuiteName, testClass, testName, testQualifier, testParameters); + TestDescriptor testDescriptor = + new TestDescriptor(testSuiteName, testClass, testName, testParameters, testQualifier); + eventHandler.onTestSkip(testDescriptor, reason); + eventHandler.onTestFinish(testDescriptor); } } diff --git a/dd-java-agent/instrumentation/scalatest/src/main/java/datadog/trace/instrumentation/scalatest/RunContext.java b/dd-java-agent/instrumentation/scalatest/src/main/java/datadog/trace/instrumentation/scalatest/RunContext.java index b65d1e1af7f..0db5359fcc1 100644 --- a/dd-java-agent/instrumentation/scalatest/src/main/java/datadog/trace/instrumentation/scalatest/RunContext.java +++ b/dd-java-agent/instrumentation/scalatest/src/main/java/datadog/trace/instrumentation/scalatest/RunContext.java @@ -2,7 +2,9 @@ import datadog.trace.api.civisibility.InstrumentationBridge; import datadog.trace.api.civisibility.config.TestIdentifier; +import datadog.trace.api.civisibility.events.TestDescriptor; import datadog.trace.api.civisibility.events.TestEventsHandler; +import datadog.trace.api.civisibility.events.TestSuiteDescriptor; import datadog.trace.api.civisibility.retry.TestRetryPolicy; import java.util.ArrayList; import java.util.concurrent.ConcurrentHashMap; @@ -29,7 +31,7 @@ public static void destroy(int runStamp) { } private final int runStamp; - private final TestEventsHandler eventHandler = + private final TestEventsHandler eventHandler = InstrumentationBridge.createTestEventsHandler("scalatest"); private final java.util.Set skippedTests = ConcurrentHashMap.newKeySet(); private final java.util.Set unskippableTests = ConcurrentHashMap.newKeySet(); @@ -44,7 +46,7 @@ public int getRunStamp() { return runStamp; } - public TestEventsHandler getEventHandler() { + public TestEventsHandler getEventHandler() { return eventHandler; } diff --git a/dd-java-agent/instrumentation/testng/src/main/java/datadog/trace/instrumentation/testng/TestEventsHandlerHolder.java b/dd-java-agent/instrumentation/testng/src/main/java/datadog/trace/instrumentation/testng/TestEventsHandlerHolder.java index 74eb806085b..0af046fa417 100644 --- a/dd-java-agent/instrumentation/testng/src/main/java/datadog/trace/instrumentation/testng/TestEventsHandlerHolder.java +++ b/dd-java-agent/instrumentation/testng/src/main/java/datadog/trace/instrumentation/testng/TestEventsHandlerHolder.java @@ -1,12 +1,14 @@ package datadog.trace.instrumentation.testng; import datadog.trace.api.civisibility.InstrumentationBridge; +import datadog.trace.api.civisibility.events.TestDescriptor; import datadog.trace.api.civisibility.events.TestEventsHandler; +import datadog.trace.api.civisibility.events.TestSuiteDescriptor; import datadog.trace.util.AgentThreadFactory; public abstract class TestEventsHandlerHolder { - public static volatile TestEventsHandler TEST_EVENTS_HANDLER; + public static volatile TestEventsHandler TEST_EVENTS_HANDLER; static { start(); diff --git a/dd-java-agent/instrumentation/testng/src/main/java/datadog/trace/instrumentation/testng/TestNGUtils.java b/dd-java-agent/instrumentation/testng/src/main/java/datadog/trace/instrumentation/testng/TestNGUtils.java index f7e422d68a1..da9ea40ff43 100644 --- a/dd-java-agent/instrumentation/testng/src/main/java/datadog/trace/instrumentation/testng/TestNGUtils.java +++ b/dd-java-agent/instrumentation/testng/src/main/java/datadog/trace/instrumentation/testng/TestNGUtils.java @@ -2,6 +2,7 @@ import datadog.trace.api.civisibility.config.TestIdentifier; import datadog.trace.api.civisibility.events.TestDescriptor; +import datadog.trace.api.civisibility.events.TestSuiteDescriptor; import datadog.trace.util.Strings; import java.io.InputStream; import java.lang.invoke.MethodHandle; @@ -255,4 +256,10 @@ public static TestDescriptor toTestDescriptor(ITestResult result) { String parameters = TestNGUtils.getParameters(result); return new TestDescriptor(testSuiteName, testClass, testName, parameters, result); } + + public static TestSuiteDescriptor toSuiteDescriptor(ITestClass testClass) { + String testSuiteName = testClass.getName(); + Class testSuiteClass = testClass.getRealClass(); + return new TestSuiteDescriptor(testSuiteName, testSuiteClass); + } } diff --git a/dd-java-agent/instrumentation/testng/src/main/java/datadog/trace/instrumentation/testng/TracingListener.java b/dd-java-agent/instrumentation/testng/src/main/java/datadog/trace/instrumentation/testng/TracingListener.java index 4c28cd217d6..784a6233eae 100644 --- a/dd-java-agent/instrumentation/testng/src/main/java/datadog/trace/instrumentation/testng/TracingListener.java +++ b/dd-java-agent/instrumentation/testng/src/main/java/datadog/trace/instrumentation/testng/TracingListener.java @@ -1,5 +1,7 @@ package datadog.trace.instrumentation.testng; +import datadog.trace.api.civisibility.events.TestDescriptor; +import datadog.trace.api.civisibility.events.TestSuiteDescriptor; import datadog.trace.api.civisibility.telemetry.tag.TestFrameworkInstrumentation; import datadog.trace.instrumentation.testng.retry.RetryAnalyzer; import java.lang.reflect.Method; @@ -29,10 +31,12 @@ public void onFinish(final ITestContext context) { @Override protected void onBeforeClass(ITestClass testClass, boolean parallelized) { + TestSuiteDescriptor suiteDescriptor = TestNGUtils.toSuiteDescriptor(testClass); String testSuiteName = testClass.getName(); Class testSuiteClass = testClass.getRealClass(); List groups = TestNGUtils.getGroups(testClass); TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteStart( + suiteDescriptor, testSuiteName, FRAMEWORK_NAME, FRAMEWORK_VERSION, @@ -44,9 +48,8 @@ protected void onBeforeClass(ITestClass testClass, boolean parallelized) { @Override protected void onAfterClass(ITestClass testClass) { - String testSuiteName = testClass.getName(); - Class testSuiteClass = testClass.getRealClass(); - TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteFinish(testSuiteName, testSuiteClass); + TestSuiteDescriptor suiteDescriptor = TestNGUtils.toSuiteDescriptor(testClass); + TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteFinish(suiteDescriptor); } @Override @@ -57,10 +60,10 @@ public void onConfigurationSuccess(ITestResult result) { @Override public void onConfigurationFailure(ITestResult result) { // suite setup or suite teardown failed - String testSuiteName = result.getInstanceName(); - Class testClass = TestNGUtils.getTestClass(result); + TestSuiteDescriptor suiteDescriptor = + TestNGUtils.toSuiteDescriptor(result.getMethod().getTestClass()); TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteFailure( - testSuiteName, testClass, result.getThrowable()); + suiteDescriptor, result.getThrowable()); } @Override @@ -70,20 +73,22 @@ public void onConfigurationSkip(ITestResult result) { @Override public void onTestStart(final ITestResult result) { + TestSuiteDescriptor suiteDescriptor = + TestNGUtils.toSuiteDescriptor(result.getMethod().getTestClass()); + TestDescriptor testDescriptor = TestNGUtils.toTestDescriptor(result); String testSuiteName = result.getInstanceName(); String testName = (result.getName() != null) ? result.getName() : result.getMethod().getMethodName(); String testParameters = TestNGUtils.getParameters(result); List groups = TestNGUtils.getGroups(result); - Class testClass = TestNGUtils.getTestClass(result); Method testMethod = TestNGUtils.getTestMethod(result); String testMethodName = testMethod != null ? testMethod.getName() : null; - TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestStart( + suiteDescriptor, + testDescriptor, testSuiteName, testName, - result, FRAMEWORK_NAME, FRAMEWORK_VERSION, testParameters, @@ -105,28 +110,16 @@ private boolean isRetry(final ITestResult result) { @Override public void onTestSuccess(final ITestResult result) { - final String testSuiteName = result.getInstanceName(); - final Class testClass = TestNGUtils.getTestClass(result); - String testName = - (result.getName() != null) ? result.getName() : result.getMethod().getMethodName(); - String testParameters = TestNGUtils.getParameters(result); - TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestFinish( - testSuiteName, testClass, testName, result, testParameters); + TestDescriptor testDescriptor = TestNGUtils.toTestDescriptor(result); + TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestFinish(testDescriptor); } @Override public void onTestFailure(final ITestResult result) { - final String testSuiteName = result.getInstanceName(); - final Class testClass = TestNGUtils.getTestClass(result); - String testName = - (result.getName() != null) ? result.getName() : result.getMethod().getMethodName(); - String testParameters = TestNGUtils.getParameters(result); - - final Throwable throwable = result.getThrowable(); - TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestFailure( - testSuiteName, testClass, testName, result, testParameters, throwable); - TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestFinish( - testSuiteName, testClass, testName, result, testParameters); + TestDescriptor testDescriptor = TestNGUtils.toTestDescriptor(result); + Throwable throwable = result.getThrowable(); + TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestFailure(testDescriptor, throwable); + TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestFinish(testDescriptor); } @Override @@ -136,28 +129,19 @@ public void onTestFailedButWithinSuccessPercentage(final ITestResult result) { @Override public void onTestSkipped(final ITestResult result) { - final String testSuiteName = result.getInstanceName(); - final Class testClass = TestNGUtils.getTestClass(result); - String testName = - (result.getName() != null) ? result.getName() : result.getMethod().getMethodName(); - String testParameters = TestNGUtils.getParameters(result); - + TestDescriptor testDescriptor = TestNGUtils.toTestDescriptor(result); Throwable throwable = result.getThrowable(); if (TestNGUtils.wasRetried(result)) { // TestNG reports tests retried with IRetryAnalyzer as skipped, // this is done to avoid failing the build when retrying tests. // We want to report such tests as failed to Datadog, // to provide more accurate data (and to enable flakiness detection) - TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestFailure( - testSuiteName, testClass, testName, result, testParameters, throwable); + TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestFailure(testDescriptor, throwable); } else { // Typically the way of skipping a TestNG test is throwing a SkipException String reason = throwable != null ? throwable.getMessage() : null; - TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSkip( - testSuiteName, testClass, testName, result, testParameters, reason); + TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSkip(testDescriptor, reason); } - - TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestFinish( - testSuiteName, testClass, testName, result, testParameters); + TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestFinish(testDescriptor); } } diff --git a/internal-api/src/main/java/datadog/trace/api/civisibility/InstrumentationBridge.java b/internal-api/src/main/java/datadog/trace/api/civisibility/InstrumentationBridge.java index 39dfc23096c..7ab0726b84c 100644 --- a/internal-api/src/main/java/datadog/trace/api/civisibility/InstrumentationBridge.java +++ b/internal-api/src/main/java/datadog/trace/api/civisibility/InstrumentationBridge.java @@ -4,6 +4,7 @@ import datadog.trace.api.civisibility.events.TestEventsHandler; import datadog.trace.api.civisibility.telemetry.CiVisibilityMetricCollector; import datadog.trace.api.civisibility.telemetry.NoOpMetricCollector; +import datadog.trace.bootstrap.ContextStore; public abstract class InstrumentationBridge { @@ -20,10 +21,18 @@ public static void registerTestEventsHandlerFactory( TEST_EVENTS_HANDLER_FACTORY = testEventsHandlerFactory; } - public static TestEventsHandler createTestEventsHandler(String component) { + public static TestEventsHandler createTestEventsHandler( + String component) { return TEST_EVENTS_HANDLER_FACTORY.create(component); } + public static TestEventsHandler createTestEventsHandler( + String component, + ContextStore suiteStore, + ContextStore testStore) { + return TEST_EVENTS_HANDLER_FACTORY.create(component, suiteStore, testStore); + } + public static void registerBuildEventsHandlerFactory( BuildEventsHandler.Factory buildEventsHandlerFactory) { BUILD_EVENTS_HANDLER_FACTORY = buildEventsHandlerFactory; diff --git a/internal-api/src/main/java/datadog/trace/api/civisibility/events/TestDescriptor.java b/internal-api/src/main/java/datadog/trace/api/civisibility/events/TestDescriptor.java index 68b6274abd5..1b7c970df07 100644 --- a/internal-api/src/main/java/datadog/trace/api/civisibility/events/TestDescriptor.java +++ b/internal-api/src/main/java/datadog/trace/api/civisibility/events/TestDescriptor.java @@ -48,4 +48,23 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(testSuiteName, testClass, testName, testParameters, testQualifier); } + + @Override + public String toString() { + return "TestDescriptor{" + + "testSuiteName='" + + testSuiteName + + '\'' + + ", testClass=" + + testClass + + ", testName='" + + testName + + '\'' + + ", testParameters='" + + testParameters + + '\'' + + ", testQualifier=" + + testQualifier + + '}'; + } } diff --git a/internal-api/src/main/java/datadog/trace/api/civisibility/events/TestEventsHandler.java b/internal-api/src/main/java/datadog/trace/api/civisibility/events/TestEventsHandler.java index 2397112c8cd..6b73b74a3ca 100644 --- a/internal-api/src/main/java/datadog/trace/api/civisibility/events/TestEventsHandler.java +++ b/internal-api/src/main/java/datadog/trace/api/civisibility/events/TestEventsHandler.java @@ -1,15 +1,18 @@ package datadog.trace.api.civisibility.events; +import datadog.trace.api.civisibility.DDTest; +import datadog.trace.api.civisibility.DDTestSuite; import datadog.trace.api.civisibility.config.TestIdentifier; import datadog.trace.api.civisibility.retry.TestRetryPolicy; import datadog.trace.api.civisibility.telemetry.tag.TestFrameworkInstrumentation; +import datadog.trace.bootstrap.ContextStore; import java.io.Closeable; import java.lang.reflect.Method; import java.util.Collection; import javax.annotation.Nonnull; import javax.annotation.Nullable; -public interface TestEventsHandler extends Closeable { +public interface TestEventsHandler extends Closeable { /** * @param testFramework Name of the testing framework that executes the suite. @@ -17,9 +20,9 @@ public interface TestEventsHandler extends Closeable { * framework, because one instrumentation can support multiple frameworks. For example, there * are many testing frameworks based on JUnit 5. For some of those frameworks we have * dedicated instrumentations, while others are handled with "generic" JUnit 5 instrumentation - * . */ void onTestSuiteStart( + SuiteKey descriptor, String testSuiteName, @Nullable String testFramework, @Nullable String testFrameworkVersion, @@ -28,16 +31,17 @@ void onTestSuiteStart( boolean parallelized, TestFrameworkInstrumentation instrumentation); - void onTestSuiteFinish(String testSuiteName, @Nullable Class testClass); + void onTestSuiteSkip(SuiteKey descriptor, @Nullable String reason); - void onTestSuiteSkip(String testSuiteName, Class testClass, @Nullable String reason); + void onTestSuiteFailure(SuiteKey descriptor, @Nullable Throwable throwable); - void onTestSuiteFailure(String testSuiteName, Class testClass, @Nullable Throwable throwable); + void onTestSuiteFinish(SuiteKey descriptor); void onTestStart( + SuiteKey suiteDescriptor, + TestKey descriptor, String testSuiteName, String testName, - @Nullable Object testQualifier, @Nullable String testFramework, @Nullable String testFrameworkVersion, @Nullable String testParameters, @@ -47,33 +51,17 @@ void onTestStart( @Nullable Method testMethod, boolean isRetry); - void onTestSkip( - String testSuiteName, - Class testClass, - String testName, - @Nullable Object testQualifier, - @Nullable String testParameters, - @Nullable String reason); + void onTestSkip(TestKey descriptor, @Nullable String reason); - void onTestFailure( - String testSuiteName, - Class testClass, - String testName, - @Nullable Object testQualifier, - @Nullable String testParameters, - @Nullable Throwable throwable); + void onTestFailure(TestKey descriptor, @Nullable Throwable throwable); - void onTestFinish( - String testSuiteName, - Class testClass, - String testName, - @Nullable Object testQualifier, - @Nullable String testParameters); + void onTestFinish(TestKey descriptor); void onTestIgnore( + SuiteKey suiteDescriptor, + TestKey testDescriptor, String testSuiteName, String testName, - @Nullable Object testQualifier, @Nullable String testFramework, @Nullable String testFrameworkVersion, @Nullable String testParameters, @@ -94,6 +82,11 @@ void onTestIgnore( void close(); interface Factory { - TestEventsHandler create(String component); + TestEventsHandler create(String component); + + TestEventsHandler create( + String component, + ContextStore suiteStore, + ContextStore testStore); } } diff --git a/internal-api/src/main/java/datadog/trace/api/civisibility/events/TestSuiteDescriptor.java b/internal-api/src/main/java/datadog/trace/api/civisibility/events/TestSuiteDescriptor.java index 1b9d0ebb017..f972f9e4c87 100644 --- a/internal-api/src/main/java/datadog/trace/api/civisibility/events/TestSuiteDescriptor.java +++ b/internal-api/src/main/java/datadog/trace/api/civisibility/events/TestSuiteDescriptor.java @@ -28,4 +28,15 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(testSuiteName, testClass); } + + @Override + public String toString() { + return "TestSuiteDescriptor{" + + "testSuiteName='" + + testSuiteName + + '\'' + + ", testClass=" + + testClass + + '}'; + } } From e9c489fd8cac5ff142baf503eb8d38bf832d6895 Mon Sep 17 00:00:00 2001 From: Richard Startin Date: Fri, 8 Mar 2024 14:47:58 +0000 Subject: [PATCH 21/30] JFR based timer (#6703) --- .../controller/jfr/SimpleJFRAccess.java | 10 ++++ .../controller/jfr/JPMSJFRAccess.java | 51 ++++++++++++++++--- .../profiling/controller/jfr/JFRAccess.java | 3 +- .../profiling/ddprof/DatadogProfiler.java | 43 +++++++++++++--- .../profiling/ddprof/QueueTimeTracker.java | 24 +++------ .../datadog/profiling/utils/Timestamper.java | 48 +++++++++++++++++ .../profiling/agent/ProfilingAgent.java | 2 + 7 files changed, 149 insertions(+), 32 deletions(-) create mode 100644 dd-java-agent/agent-profiling/profiling-utils/src/main/java/com/datadog/profiling/utils/Timestamper.java diff --git a/dd-java-agent/agent-profiling/profiling-controller-jfr/implementation/src/main/java/com/datadog/profiling/controller/jfr/SimpleJFRAccess.java b/dd-java-agent/agent-profiling/profiling-controller-jfr/implementation/src/main/java/com/datadog/profiling/controller/jfr/SimpleJFRAccess.java index 6b19b05544f..95ea50d41af 100644 --- a/dd-java-agent/agent-profiling/profiling-controller-jfr/implementation/src/main/java/com/datadog/profiling/controller/jfr/SimpleJFRAccess.java +++ b/dd-java-agent/agent-profiling/profiling-controller-jfr/implementation/src/main/java/com/datadog/profiling/controller/jfr/SimpleJFRAccess.java @@ -42,4 +42,14 @@ public boolean setBaseLocation(String location) { } return true; } + + @Override + public long timestamp() { + return JVM.counterTime(); + } + + @Override + public double toNanosConversionFactor() { + return JVM.getJVM().getTimeConversionFactor(); + } } diff --git a/dd-java-agent/agent-profiling/profiling-controller-jfr/implementation/src/main/java11/com/datadog/profiling/controller/jfr/JPMSJFRAccess.java b/dd-java-agent/agent-profiling/profiling-controller-jfr/implementation/src/main/java11/com/datadog/profiling/controller/jfr/JPMSJFRAccess.java index 52ee3707fef..03690412043 100644 --- a/dd-java-agent/agent-profiling/profiling-controller-jfr/implementation/src/main/java11/com/datadog/profiling/controller/jfr/JPMSJFRAccess.java +++ b/dd-java-agent/agent-profiling/profiling-controller-jfr/implementation/src/main/java11/com/datadog/profiling/controller/jfr/JPMSJFRAccess.java @@ -39,9 +39,13 @@ public JFRAccess create(Instrumentation inst) { private final Class repositoryClass; private final Class safePathClass; + // TODO consider refactoring to make these private static final private final MethodHandle setStackDepthMH; private final MethodHandle setRepositoryBaseMH; + private final MethodHandle counterTimeMH; + private final MethodHandle getTimeConversionFactorMH; + public JPMSJFRAccess(Instrumentation inst) throws Exception { patchModuleAccess(inst); @@ -49,18 +53,29 @@ public JPMSJFRAccess(Instrumentation inst) throws Exception { repositoryClass = JFRAccess.class.getClassLoader().loadClass("jdk.jfr.internal.Repository"); safePathClass = JFRAccess.class.getClassLoader().loadClass("jdk.jfr.internal.SecuritySupport$SafePath"); - setStackDepthMH = setStackDepthMethodHandle(); + Object jvm = getJvm(); + setStackDepthMH = getJvmMethodHandle(jvm, "setStackDepth", int.class); setRepositoryBaseMH = setRepositoryBaseMethodHandle(); + counterTimeMH = getJvmMethodHandle(jvm, "counterTime"); + getTimeConversionFactorMH = getJvmMethodHandle(jvm, "getTimeConversionFactor"); } - private MethodHandle setStackDepthMethodHandle() - throws NoSuchMethodException, IllegalAccessException, InvocationTargetException { - Method m = jvmClass.getMethod("setStackDepth", int.class); + private Object getJvm() + throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { + return jvmClass.getMethod("getJVM").invoke(null); + } + + private MethodHandle getJvmMethodHandle(Object jvm, String method, Class... args) + throws NoSuchMethodException, IllegalAccessException { + Method m = jvmClass.getMethod(method, args); m.setAccessible(true); - MethodHandle mh = MethodHandles.publicLookup().unreflect(m); - if (!Modifier.isStatic(m.getModifiers())) { - // instance method - need to call JVM.getJVM() and bind the instance - Object jvm = jvmClass.getMethod("getJVM").invoke(null); + return unreflectAndBind(m, jvm); + } + + private static MethodHandle unreflectAndBind(Method method, Object jvm) + throws IllegalAccessException { + MethodHandle mh = MethodHandles.publicLookup().unreflect(method); + if (!Modifier.isStatic(method.getModifiers())) { mh = mh.bindTo(jvm); } return mh; @@ -119,4 +134,24 @@ public boolean setBaseLocation(String location) { } return false; } + + @Override + public long timestamp() { + try { + return (long) counterTimeMH.invokeExact(); + } catch (Throwable t) { + log.debug("Unable to get TSC from JFR", t); + } + return super.timestamp(); + } + + @Override + public double toNanosConversionFactor() { + try { + return (double) getTimeConversionFactorMH.invokeExact(); + } catch (Throwable t) { + log.debug("Unable to get time conversion factor from JFR", t); + } + return super.toNanosConversionFactor(); + } } diff --git a/dd-java-agent/agent-profiling/profiling-controller-jfr/src/main/java/com/datadog/profiling/controller/jfr/JFRAccess.java b/dd-java-agent/agent-profiling/profiling-controller-jfr/src/main/java/com/datadog/profiling/controller/jfr/JFRAccess.java index 1b417cf3c94..92f51a009ef 100644 --- a/dd-java-agent/agent-profiling/profiling-controller-jfr/src/main/java/com/datadog/profiling/controller/jfr/JFRAccess.java +++ b/dd-java-agent/agent-profiling/profiling-controller-jfr/src/main/java/com/datadog/profiling/controller/jfr/JFRAccess.java @@ -1,5 +1,6 @@ package com.datadog.profiling.controller.jfr; +import com.datadog.profiling.utils.Timestamper; import java.lang.instrument.Instrumentation; import java.util.ServiceLoader; import javax.annotation.Nullable; @@ -11,7 +12,7 @@ * Provides access to the JFR internal API. For Java 9 and newer, the JFR access requires * instrumentation in order to patch the module access. */ -public abstract class JFRAccess { +public abstract class JFRAccess implements Timestamper { private static final Logger log = LoggerFactory.getLogger(JFRAccess.class); /** No-op JFR access implementation. */ diff --git a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfiler.java b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfiler.java index d9271359279..7e676095aab 100644 --- a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfiler.java +++ b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfiler.java @@ -29,10 +29,12 @@ import com.datadog.profiling.controller.OngoingRecording; import com.datadog.profiling.utils.ProfilingMode; +import com.datadog.profiling.utils.Timestamper; import com.datadoghq.profiler.ContextSetter; import com.datadoghq.profiler.JavaProfiler; import datadog.trace.api.profiling.RecordingData; import datadog.trace.bootstrap.config.provider.ConfigProvider; +import datadog.trace.bootstrap.instrumentation.api.TaskWrapper; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -95,7 +97,7 @@ public static DatadogProfiler newInstance(ConfigProvider configProvider) { private final List orderedContextAttributes; - private final long queueTimeThreshold; + private final double queueTimeThresholdNanos; private DatadogProfiler(ConfigProvider configProvider) { this(configProvider, getContextAttributes(configProvider)); @@ -132,10 +134,11 @@ private DatadogProfiler(ConfigProvider configProvider) { orderedContextAttributes.add(RESOURCE); } this.contextSetter = new ContextSetter(profiler, orderedContextAttributes); - this.queueTimeThreshold = - configProvider.getLong( - PROFILING_QUEUEING_TIME_THRESHOLD_MILLIS, - PROFILING_QUEUEING_TIME_THRESHOLD_MILLIS_DEFAULT); + this.queueTimeThresholdNanos = + 1_000_000D + * configProvider.getLong( + PROFILING_QUEUEING_TIME_THRESHOLD_MILLIS, + PROFILING_QUEUEING_TIME_THRESHOLD_MILLIS_DEFAULT); } void addThread() { @@ -374,6 +377,34 @@ public void recordSetting(String name, String value, String unit) { } public QueueTimeTracker newQueueTimeTracker() { - return new QueueTimeTracker(profiler, queueTimeThreshold); + return new QueueTimeTracker(this, timestamper().timestamp()); + } + + void recordQueueTimeEvent(long startTicks, Object task, Class scheduler, Thread origin) { + if (profiler != null) { + Timestamper timestamper = timestamper(); + long endTicks = timestamper.timestamp(); + double durationNanos = timestamper.toNanosConversionFactor() * (endTicks - startTicks); + if (durationNanos >= queueTimeThresholdNanos) { + // note: because this type traversal can update secondary_super_cache (see JDK-8180450) + // we avoid doing this unless we are absolutely certain we will record the event + Class taskType = TaskWrapper.getUnwrappedType(task); + if (taskType != null) { + profiler.recordQueueTime(startTicks, endTicks, taskType, scheduler, origin); + } + } + } + } + + private Timestamper timestamper() { + // FIXME intended to be injectable, but still a singleton for currently pragmatic reasons. + // We need a way to make the Controller responsible for creating the context integration + // for the tracer, which also allows the context integration to be constant in the tracer, + // as well as allowing for the various late initialisation needs for JFR on certain JDK + // versions + // note that this access does not risk using the default version so long as queue time is + // guarded + // by checking if JFR is ready (this currently happens in QueueTimeHelper, so this is safe) + return Timestamper.timestamper(); } } diff --git a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/QueueTimeTracker.java b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/QueueTimeTracker.java index cf7ac164347..c4915ba1326 100644 --- a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/QueueTimeTracker.java +++ b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/QueueTimeTracker.java @@ -1,25 +1,23 @@ package com.datadog.profiling.ddprof; -import com.datadoghq.profiler.JavaProfiler; import datadog.trace.api.profiling.QueueTiming; -import datadog.trace.bootstrap.instrumentation.api.TaskWrapper; import java.lang.ref.WeakReference; public class QueueTimeTracker implements QueueTiming { - private final JavaProfiler profiler; + private final DatadogProfiler profiler; private final Thread origin; - private final long threshold; private final long startTicks; private WeakReference weakTask; + // FIXME this can be eliminated by altering the instrumentation + // since it is known when the item is polled from the queue private Class scheduler; - public QueueTimeTracker(JavaProfiler profiler, long threshold) { + public QueueTimeTracker(DatadogProfiler profiler, long startTicks) { this.profiler = profiler; this.origin = Thread.currentThread(); - this.threshold = threshold; // TODO get this from JFR if available instead of making a JNI call - this.startTicks = profiler.getCurrentTicks(); + this.startTicks = startTicks; } @Override @@ -37,16 +35,8 @@ public void close() { assert weakTask != null && scheduler != null; Object task = this.weakTask.get(); if (task != null) { - // potentially avoidable JNI call - long endTicks = profiler.getCurrentTicks(); - if (profiler.isThresholdExceeded(threshold, startTicks, endTicks)) { - // note: because this type traversal can update secondary_super_cache (see JDK-8180450) - // we avoid doing this unless we are absolutely certain we will record the event - Class taskType = TaskWrapper.getUnwrappedType(task); - if (taskType != null) { - profiler.recordQueueTime(startTicks, endTicks, taskType, scheduler, origin); - } - } + // indirection reduces shallow size of the tracker instance + profiler.recordQueueTimeEvent(startTicks, task, scheduler, origin); } } } diff --git a/dd-java-agent/agent-profiling/profiling-utils/src/main/java/com/datadog/profiling/utils/Timestamper.java b/dd-java-agent/agent-profiling/profiling-utils/src/main/java/com/datadog/profiling/utils/Timestamper.java new file mode 100644 index 00000000000..24da6779d5d --- /dev/null +++ b/dd-java-agent/agent-profiling/profiling-utils/src/main/java/com/datadog/profiling/utils/Timestamper.java @@ -0,0 +1,48 @@ +package com.datadog.profiling.utils; + +import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; + +public interface Timestamper { + + Timestamper DEFAULT = new Timestamper() {}; + + default long timestamp() { + return System.nanoTime(); + } + + default double toNanosConversionFactor() { + return 1D; + } + + final class Registration { + volatile Timestamper pending = Timestamper.DEFAULT; + private static final AtomicReferenceFieldUpdater UPDATER = + AtomicReferenceFieldUpdater.newUpdater(Registration.class, Timestamper.class, "pending"); + + private static final Registration INSTANCE = new Registration(); + } + + /** + * One shot chance to override the timestamper, which allows delayed initialisation of the + * timestamp source. + * + * @return whether override was successful + */ + static boolean override(Timestamper timestamper) { + return Registration.UPDATER.compareAndSet( + Registration.INSTANCE, Timestamper.DEFAULT, timestamper); + } + + final class Singleton { + // + static final Timestamper TIMESTAMPER = Registration.INSTANCE.pending; + } + + /** + * Gets the registered timestamper (e.g. using JFR) if one has been registered, otherwise uses the + * default timer. + */ + static Timestamper timestamper() { + return Singleton.TIMESTAMPER; + } +} diff --git a/dd-java-agent/agent-profiling/src/main/java/com/datadog/profiling/agent/ProfilingAgent.java b/dd-java-agent/agent-profiling/src/main/java/com/datadog/profiling/agent/ProfilingAgent.java index 20e607adf74..ef04453b990 100644 --- a/dd-java-agent/agent-profiling/src/main/java/com/datadog/profiling/agent/ProfilingAgent.java +++ b/dd-java-agent/agent-profiling/src/main/java/com/datadog/profiling/agent/ProfilingAgent.java @@ -11,6 +11,7 @@ import com.datadog.profiling.controller.UnsupportedEnvironmentException; import com.datadog.profiling.controller.jfr.JFRAccess; import com.datadog.profiling.uploader.ProfileUploader; +import com.datadog.profiling.utils.Timestamper; import datadog.trace.api.Config; import datadog.trace.api.Platform; import datadog.trace.api.config.ProfilingConfig; @@ -117,6 +118,7 @@ public static synchronized void run( try { JFRAccess.setup(inst); + Timestamper.override(JFRAccess.instance()); ControllerContext context = new ControllerContext(); final Controller controller = CompositeController.build(configProvider, context); From df7e2a1abb9594edd1c103bede934bbf2afa86f2 Mon Sep 17 00:00:00 2001 From: Andrew Munn Date: Mon, 11 Mar 2024 05:22:42 -0400 Subject: [PATCH 22/30] Support JDK-21 virtual thread executor (#6789) add JAVA_21_HOME to workflows and create TaskRunnerInstrumentation --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/trivy-analysis.yml | 2 +- .../java-concurrent/build.gradle | 10 +++ .../groovy/VirtualThreadTest.groovy | 79 +++++++++++++++++++ .../latestDepTest/java/JavaAsyncChild.java | 59 ++++++++++++++ .../concurrent/TaskRunnerInstrumentation.java | 61 ++++++++++++++ gradle.properties | 2 +- lib-injection/build_java_agent.sh | 4 +- 8 files changed, 214 insertions(+), 5 deletions(-) create mode 100644 dd-java-agent/instrumentation/java-concurrent/src/latestDepTest/groovy/VirtualThreadTest.groovy create mode 100644 dd-java-agent/instrumentation/java-concurrent/src/latestDepTest/java/JavaAsyncChild.java create mode 100644 dd-java-agent/instrumentation/java-concurrent/src/main/java/datadog/trace/instrumentation/java/concurrent/TaskRunnerInstrumentation.java diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index fa97090e5a8..dcfc2df8f60 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -33,7 +33,7 @@ jobs: # queries: ./path/to/local/query, your-org/your-repo/queries@main - name: Build dd-trace-java for creating the CodeQL database - run: JAVA_HOME=$JAVA_HOME_8_X64 JAVA_8_HOME=$JAVA_HOME_8_X64 JAVA_11_HOME=$JAVA_HOME_11_X64 JAVA_17_HOME=$JAVA_HOME_17_X64 ./gradlew clean :dd-java-agent:shadowJar --build-cache --parallel --stacktrace --no-daemon --max-workers=8 + run: JAVA_HOME=$JAVA_HOME_8_X64 JAVA_8_HOME=$JAVA_HOME_8_X64 JAVA_11_HOME=$JAVA_HOME_11_X64 JAVA_17_HOME=$JAVA_HOME_17_X64 JAVA_21_HOME=$JAVA_HOME_21_X64 ./gradlew clean :dd-java-agent:shadowJar --build-cache --parallel --stacktrace --no-daemon --max-workers=8 - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@1a927e9307bc11970b2c679922ebc4d03a5bd980 # 1.0.31 diff --git a/.github/workflows/trivy-analysis.yml b/.github/workflows/trivy-analysis.yml index f7bc4d80fb2..c14ccd9b607 100644 --- a/.github/workflows/trivy-analysis.yml +++ b/.github/workflows/trivy-analysis.yml @@ -31,7 +31,7 @@ jobs: - name: Build and publish artifacts locally run: | - GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx2G -Xms2G'" JAVA_HOME=$JAVA_HOME_8_X64 JAVA_8_HOME=$JAVA_HOME_8_X64 JAVA_11_HOME=$JAVA_HOME_11_X64 JAVA_17_HOME=$JAVA_HOME_17_X64 ./gradlew clean publishToMavenLocal --build-cache --parallel --stacktrace --no-daemon --max-workers=4 + GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx2G -Xms2G'" JAVA_HOME=$JAVA_HOME_8_X64 JAVA_8_HOME=$JAVA_HOME_8_X64 JAVA_11_HOME=$JAVA_HOME_11_X64 JAVA_17_HOME=$JAVA_HOME_17_X64 JAVA_21_HOME=$JAVA_HOME_21_X64 ./gradlew clean publishToMavenLocal --build-cache --parallel --stacktrace --no-daemon --max-workers=4 - name: Copy published artifacts run: | diff --git a/dd-java-agent/instrumentation/java-concurrent/build.gradle b/dd-java-agent/instrumentation/java-concurrent/build.gradle index 9160ba47137..802c60ab1f0 100644 --- a/dd-java-agent/instrumentation/java-concurrent/build.gradle +++ b/dd-java-agent/instrumentation/java-concurrent/build.gradle @@ -1,3 +1,7 @@ +ext { + latestDepTestMinJavaVersionForTests = JavaVersion.VERSION_21 +} + muzzle { pass { coreJdk() @@ -6,6 +10,11 @@ muzzle { apply from: "$rootDir/gradle/java.gradle" +addTestSuite('latestDepTest') + +compileLatestDepTestGroovy.configure { + javaLauncher = getJavaLauncherFor(21) +} dependencies { testImplementation project(':dd-java-agent:instrumentation:trace-annotation') @@ -13,4 +22,5 @@ dependencies { testImplementation 'org.apache.tomcat.embed:tomcat-embed-core:7.0.0' testImplementation deps.guava testImplementation group: 'io.netty', name: 'netty-all', version: '4.1.9.Final' + latestDepTestImplementation group: 'io.netty', name: 'netty-all', version: '4.+' } diff --git a/dd-java-agent/instrumentation/java-concurrent/src/latestDepTest/groovy/VirtualThreadTest.groovy b/dd-java-agent/instrumentation/java-concurrent/src/latestDepTest/groovy/VirtualThreadTest.groovy new file mode 100644 index 00000000000..15ba703c93e --- /dev/null +++ b/dd-java-agent/instrumentation/java-concurrent/src/latestDepTest/groovy/VirtualThreadTest.groovy @@ -0,0 +1,79 @@ +import datadog.trace.agent.test.AgentTestRunner +import datadog.trace.api.Trace +import datadog.trace.core.DDSpan +import spock.lang.Shared + +import java.util.concurrent.Callable +import java.util.concurrent.ExecutorCompletionService +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit + +import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activeScope + +class VirtualThreadTest extends AgentTestRunner { + + @Shared + def executeRunnable = { e, c -> e.execute((Runnable) c) } + @Shared + def submitRunnable = { e, c -> e.submit((Runnable) c) } + @Shared + def submitCallable = { e, c -> e.submit((Callable) c) } + @Shared + def submitRunnableExecutorCompletionService = { ecs, c -> ecs.submit((Runnable) c, null) } + @Shared + def invokeAll = { e, c -> e.invokeAll([(Callable) c]) } + @Shared + def invokeAllTimeout = { e, c -> e.invokeAll([(Callable) c], 10, TimeUnit.SECONDS) } + @Shared + def invokeAny = { e, c -> e.invokeAny([(Callable) c]) } + @Shared + def invokeAnyTimeout = { e, c -> e.invokeAny([(Callable) c], 10, TimeUnit.SECONDS) } + + def "virtualThreadPool #name"() { + setup: + def pool = poolImpl + def m = method + + new Runnable() { + @Override + @Trace(operationName = "parent") + void run() { + activeScope().setAsyncPropagation(true) + // this child will have a span + m(pool, new JavaAsyncChild()) + // this child won't + m(pool, new JavaAsyncChild(false, false)) + blockUntilChildSpansFinished(1) + } + }.run() + + TEST_WRITER.waitForTraces(1) + List trace = TEST_WRITER.get(0) + + expect: + TEST_WRITER.size() == 1 + trace.size() == 2 + trace.get(0).operationName == "parent" + trace.get(1).operationName == "asyncChild" + trace.get(1).parentId == trace.get(0).spanId + + cleanup: + if (pool?.hasProperty("shutdown")) { + pool?.shutdown() + } + + where: + // spotless:off + name | method | poolImpl + "execute Runnable" | executeRunnable | Executors.newVirtualThreadPerTaskExecutor() + "submit Runnable" | submitRunnable | Executors.newVirtualThreadPerTaskExecutor() + "submit Callable" | submitCallable | Executors.newVirtualThreadPerTaskExecutor() + "submit Runnable ECS" | submitRunnableExecutorCompletionService | new ExecutorCompletionService<>(Executors.newVirtualThreadPerTaskExecutor()) + "submit Callable ECS" | submitCallable | new ExecutorCompletionService<>(Executors.newVirtualThreadPerTaskExecutor()) + "invokeAll" | invokeAll | Executors.newVirtualThreadPerTaskExecutor() + "invokeAll with timeout" | invokeAllTimeout | Executors.newVirtualThreadPerTaskExecutor() + "invokeAny" | invokeAny | Executors.newVirtualThreadPerTaskExecutor() + "invokeAny with timeout" | invokeAnyTimeout | Executors.newVirtualThreadPerTaskExecutor() + // spotless:on + } +} diff --git a/dd-java-agent/instrumentation/java-concurrent/src/latestDepTest/java/JavaAsyncChild.java b/dd-java-agent/instrumentation/java-concurrent/src/latestDepTest/java/JavaAsyncChild.java new file mode 100644 index 00000000000..67266dcb122 --- /dev/null +++ b/dd-java-agent/instrumentation/java-concurrent/src/latestDepTest/java/JavaAsyncChild.java @@ -0,0 +1,59 @@ +import datadog.trace.api.Trace; +import java.util.concurrent.Callable; +import java.util.concurrent.ForkJoinTask; +import java.util.concurrent.atomic.AtomicBoolean; + +public class JavaAsyncChild extends ForkJoinTask implements Runnable, Callable { + private final AtomicBoolean blockThread; + private final boolean doTraceableWork; + + public JavaAsyncChild() { + this(true, false); + } + + @Override + public Object getRawResult() { + return null; + } + + @Override + protected void setRawResult(final Object value) {} + + @Override + protected boolean exec() { + runImpl(); + return true; + } + + public JavaAsyncChild(final boolean doTraceableWork, final boolean blockThread) { + this.doTraceableWork = doTraceableWork; + this.blockThread = new AtomicBoolean(blockThread); + } + + public void unblock() { + blockThread.set(false); + } + + @Override + public void run() { + runImpl(); + } + + @Override + public Object call() throws Exception { + runImpl(); + return null; + } + + private void runImpl() { + while (blockThread.get()) { + // busy-wait to block thread + } + if (doTraceableWork) { + asyncChild(); + } + } + + @Trace(operationName = "asyncChild") + private void asyncChild() {} +} diff --git a/dd-java-agent/instrumentation/java-concurrent/src/main/java/datadog/trace/instrumentation/java/concurrent/TaskRunnerInstrumentation.java b/dd-java-agent/instrumentation/java-concurrent/src/main/java/datadog/trace/instrumentation/java/concurrent/TaskRunnerInstrumentation.java new file mode 100644 index 00000000000..fc312be3b81 --- /dev/null +++ b/dd-java-agent/instrumentation/java-concurrent/src/main/java/datadog/trace/instrumentation/java/concurrent/TaskRunnerInstrumentation.java @@ -0,0 +1,61 @@ +package datadog.trace.instrumentation.java.concurrent; + +import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; +import static datadog.trace.bootstrap.instrumentation.java.concurrent.AdviceUtils.capture; +import static datadog.trace.bootstrap.instrumentation.java.concurrent.AdviceUtils.endTaskScope; +import static datadog.trace.bootstrap.instrumentation.java.concurrent.AdviceUtils.startTaskScope; +import static java.util.Collections.singletonMap; +import static net.bytebuddy.matcher.ElementMatchers.isConstructor; +import static net.bytebuddy.matcher.ElementMatchers.isMethod; + +import com.google.auto.service.AutoService; +import datadog.trace.agent.tooling.Instrumenter; +import datadog.trace.agent.tooling.InstrumenterModule; +import datadog.trace.bootstrap.InstrumentationContext; +import datadog.trace.bootstrap.instrumentation.api.AgentScope; +import datadog.trace.bootstrap.instrumentation.java.concurrent.State; +import java.util.Map; +import net.bytebuddy.asm.Advice; + +@AutoService(Instrumenter.class) +public final class TaskRunnerInstrumentation extends InstrumenterModule.Tracing + implements Instrumenter.ForBootstrap, Instrumenter.ForSingleType { + public TaskRunnerInstrumentation() { + super("java_concurrent", "task-runner"); + } + + @Override + public String instrumentedType() { + return "java.util.concurrent.ThreadPerTaskExecutor$TaskRunner"; + } + + @Override + public Map contextStore() { + return singletonMap("java.lang.Runnable", State.class.getName()); + } + + @Override + public void methodAdvice(MethodTransformer transformer) { + transformer.applyAdvice(isConstructor(), getClass().getName() + "$Construct"); + transformer.applyAdvice(isMethod().and(named("run")), getClass().getName() + "$Run"); + } + + public static final class Construct { + @Advice.OnMethodExit + public static void captureScope(@Advice.This Runnable task) { + capture(InstrumentationContext.get(Runnable.class, State.class), task, true); + } + } + + public static final class Run { + @Advice.OnMethodEnter + public static AgentScope activate(@Advice.This Runnable task) { + return startTaskScope(InstrumentationContext.get(Runnable.class, State.class), task); + } + + @Advice.OnMethodExit(onThrowable = Throwable.class) + public static void close(@Advice.Enter AgentScope scope) { + endTaskScope(scope); + } + } +} diff --git a/gradle.properties b/gradle.properties index 6e9b22c03cc..8f4c908580f 100644 --- a/gradle.properties +++ b/gradle.properties @@ -4,4 +4,4 @@ org.gradle.jvmargs=-XX:MaxMetaspaceSize=1g org.gradle.java.installations.auto-detect=false org.gradle.java.installations.auto-download=false # 8 and 11 is needed to build -org.gradle.java.installations.fromEnv=JAVA_8_HOME,JAVA_11_HOME,JAVA_17_HOME +org.gradle.java.installations.fromEnv=JAVA_8_HOME,JAVA_11_HOME,JAVA_17_HOME,JAVA_21_HOME diff --git a/lib-injection/build_java_agent.sh b/lib-injection/build_java_agent.sh index 91939af4da2..1ae4be26608 100755 --- a/lib-injection/build_java_agent.sh +++ b/lib-injection/build_java_agent.sh @@ -1,8 +1,8 @@ #!/bin/sh -JAVA_HOME=$JAVA_HOME_8_X64 JAVA_8_HOME=$JAVA_HOME_8_X64 JAVA_11_HOME=$JAVA_HOME_11_X64 JAVA_17_HOME=$JAVA_HOME_17_X64 ./gradlew dd-java-agent:build dd-java-agent:shadowJar --build-cache --parallel --no-daemon --max-workers=8 +JAVA_HOME=$JAVA_HOME_8_X64 JAVA_8_HOME=$JAVA_HOME_8_X64 JAVA_11_HOME=$JAVA_HOME_11_X64 JAVA_17_HOME=$JAVA_HOME_17_X64 JAVA_21_HOME=$JAVA_HOME_21_X64 ./gradlew dd-java-agent:build dd-java-agent:shadowJar --build-cache --parallel --no-daemon --max-workers=8 cp workspace/dd-java-agent/build/libs/dd-java-agent-*.jar lib-injection/ rm lib-injection/*-sources.jar lib-injection/*-javadoc.jar mv lib-injection/*.jar lib-injection/dd-java-agent.jar echo "Java tracer copied to lib-injection folder" -ls lib-injection/ \ No newline at end of file +ls lib-injection/ From aee5f6f8c9be22baec07aeb1fefa3b9ff86f61ef Mon Sep 17 00:00:00 2001 From: "Santiago M. Mola" Date: Mon, 11 Mar 2024 11:18:52 +0100 Subject: [PATCH 23/30] Fix logged exception for dependency URIs representing directories (#6792) --- .../dependency/DependencyResolver.java | 66 +++++++++++-------- .../telemetry/dependency/JarReader.java | 17 ++++- .../DependencyResolverSpecification.groovy | 40 +++++++++++ 3 files changed, 91 insertions(+), 32 deletions(-) diff --git a/telemetry/src/main/java/datadog/telemetry/dependency/DependencyResolver.java b/telemetry/src/main/java/datadog/telemetry/dependency/DependencyResolver.java index 51cfc9726ec..8e992a36d71 100644 --- a/telemetry/src/main/java/datadog/telemetry/dependency/DependencyResolver.java +++ b/telemetry/src/main/java/datadog/telemetry/dependency/DependencyResolver.java @@ -2,6 +2,7 @@ import java.io.File; import java.io.FileInputStream; +import java.io.IOException; import java.io.InputStream; import java.net.URI; import java.util.Collections; @@ -12,41 +13,48 @@ public class DependencyResolver { private static final Logger log = LoggerFactory.getLogger(DependencyResolver.class); - private static final String JAR_SUFFIX = ".jar"; public static List resolve(URI uri) { - final String scheme = uri.getScheme(); try { - JarReader.Extracted metadata = null; - String path = null; - if ("file".equals(scheme)) { - File f; - if (uri.isOpaque()) { - f = new File(uri.getSchemeSpecificPart()); - } else { - f = new File(uri); - } - path = f.getAbsolutePath(); - metadata = JarReader.readJarFile(path); - } else if ("jar".equals(scheme) && uri.getSchemeSpecificPart().startsWith("file:")) { - path = uri.getSchemeSpecificPart().substring("file:".length()); - metadata = JarReader.readNestedJarFile(path); - } else { - log.debug("unsupported dependency type: {}", uri); - return Collections.emptyList(); - } - final List dependencies = - Dependency.fromMavenPom(metadata.jarName, metadata.pomProperties); - if (!dependencies.isEmpty()) { - return dependencies; - } - try (final InputStream is = new FileInputStream(path)) { - return Collections.singletonList( - Dependency.guessFallbackNoPom(metadata.manifest, metadata.jarName, is)); - } + return internalResolve(uri); } catch (Throwable t) { log.debug("Failed to determine dependency for uri {}", uri, t); } return Collections.emptyList(); } + + static List internalResolve(final URI uri) throws IOException { + final String scheme = uri.getScheme(); + JarReader.Extracted metadata; + String path; + if ("file".equals(scheme)) { + File f; + if (uri.isOpaque()) { + f = new File(uri.getSchemeSpecificPart()); + } else { + f = new File(uri); + } + path = f.getAbsolutePath(); + metadata = JarReader.readJarFile(path); + } else if ("jar".equals(scheme) && uri.getSchemeSpecificPart().startsWith("file:")) { + path = uri.getSchemeSpecificPart().substring("file:".length()); + metadata = JarReader.readNestedJarFile(path); + } else { + log.debug("unsupported dependency type: {}", uri); + return Collections.emptyList(); + } + if (metadata.isDirectory) { + log.debug("Extracting dependencies from directories is not supported: {}", uri); + return Collections.emptyList(); + } + final List dependencies = + Dependency.fromMavenPom(metadata.jarName, metadata.pomProperties); + if (!dependencies.isEmpty()) { + return dependencies; + } + try (final InputStream is = new FileInputStream(path)) { + return Collections.singletonList( + Dependency.guessFallbackNoPom(metadata.manifest, metadata.jarName, is)); + } + } } diff --git a/telemetry/src/main/java/datadog/telemetry/dependency/JarReader.java b/telemetry/src/main/java/datadog/telemetry/dependency/JarReader.java index 4d0616199b2..85382f27b2f 100644 --- a/telemetry/src/main/java/datadog/telemetry/dependency/JarReader.java +++ b/telemetry/src/main/java/datadog/telemetry/dependency/JarReader.java @@ -19,18 +19,25 @@ static class Extracted { final String jarName; final Map pomProperties; final Attributes manifest; + final boolean isDirectory; public Extracted( final String jarName, final Map pomProperties, - final Attributes manifest) { + final Attributes manifest, + final boolean isDirectory) { this.jarName = jarName; this.pomProperties = pomProperties; this.manifest = manifest; + this.isDirectory = isDirectory; } } public static Extracted readJarFile(String jarPath) throws IOException { + final File jarFile = new File(jarPath); + if (jarFile.isDirectory()) { + return new Extracted(jarFile.getName(), new HashMap<>(), new Attributes(), true); + } try (final JarFile jar = new JarFile(jarPath, false /* no verify */)) { final Map pomProperties = new HashMap<>(); final Enumeration entries = jar.entries(); @@ -47,7 +54,7 @@ public static Extracted readJarFile(String jarPath) throws IOException { final Manifest manifest = jar.getManifest(); final Attributes attributes = (manifest == null) ? new Attributes() : manifest.getMainAttributes(); - return new Extracted(new File(jar.getName()).getName(), pomProperties, attributes); + return new Extracted(new File(jar.getName()).getName(), pomProperties, attributes, false); } } @@ -66,6 +73,10 @@ public static Extracted readNestedJarFile(final String jarPath) throws IOExcepti if (entry == null) { throw new NoSuchFileException("Nested jar not found: " + jarPath); } + if (entry.isDirectory()) { + return new Extracted( + new File(innerJarPath).getName(), new HashMap<>(), new Attributes(), true); + } try (final InputStream is = outerJar.getInputStream(entry); final JarInputStream innerJar = new JarInputStream(is, false /* no verify */)) { final Map pomProperties = new HashMap<>(); @@ -80,7 +91,7 @@ public static Extracted readNestedJarFile(final String jarPath) throws IOExcepti final Manifest manifest = innerJar.getManifest(); final Attributes attributes = (manifest == null) ? new Attributes() : manifest.getMainAttributes(); - return new Extracted(new File(innerJarPath).getName(), pomProperties, attributes); + return new Extracted(new File(innerJarPath).getName(), pomProperties, attributes, false); } } } diff --git a/telemetry/src/test/groovy/datadog/telemetry/dependency/DependencyResolverSpecification.groovy b/telemetry/src/test/groovy/datadog/telemetry/dependency/DependencyResolverSpecification.groovy index 9038f090105..a47b8ebb9e8 100644 --- a/telemetry/src/test/groovy/datadog/telemetry/dependency/DependencyResolverSpecification.groovy +++ b/telemetry/src/test/groovy/datadog/telemetry/dependency/DependencyResolverSpecification.groovy @@ -192,6 +192,46 @@ class DependencyResolverSpecification extends DepSpecification { hash: '6438819DAB9C9AC18D8A6922C8A923C2ADAEA85D') } + void 'attempt to extract dependencies from directory'() { + given: + File dir = new File(testDir, 'dir') + dir.mkdirs() + + when: + def deps = DependencyResolver.resolve(dir.toURI()) + + then: + deps.isEmpty() + + when: 'resolve without catching exceptions' + deps = DependencyResolver.internalResolve(dir.toURI()) + + then: 'it does not throw' + deps.isEmpty() + } + + void 'attempt to extract dependencies from directory within jar'() { + given: + File file = new File(testDir, "app.jar") + ZipOutputStream out = new ZipOutputStream(new FileOutputStream(file)) + ZipEntry e = new ZipEntry("classes/") + out.putNextEntry(e) + out.closeEntry() + out.close() + + when: + def deps = DependencyResolver.resolve(new URI('jar:file:' + file.getAbsolutePath() + "!/classes!/")) + + then: + deps.isEmpty() + + when: 'resolve without catching exceptions' + deps = DependencyResolver.internalResolve(new URI('jar:file:' + file.getAbsolutePath() + "!/classes!/")) + + then: 'it does not throw' + deps.isEmpty() + } + private static void knownJarCheck(Map opts) { File jarFile = getJar(opts['jarName']) List deps = DependencyResolver.resolve(jarFile.toURI()) From cab4638496cadcf048f01451db88bb913f182a2d Mon Sep 17 00:00:00 2001 From: "Santiago M. Mola" Date: Mon, 11 Mar 2024 11:57:03 +0100 Subject: [PATCH 24/30] Fix nested JAR resolver when URI has no trailing slash (#6794) --- .../dependency/DependencyResolver.java | 3 +- .../telemetry/dependency/JarReader.java | 74 ++++++++++++++++--- .../DependencyResolverSpecification.groovy | 42 +++++++++++ 3 files changed, 108 insertions(+), 11 deletions(-) diff --git a/telemetry/src/main/java/datadog/telemetry/dependency/DependencyResolver.java b/telemetry/src/main/java/datadog/telemetry/dependency/DependencyResolver.java index 8e992a36d71..c6ce8e71254 100644 --- a/telemetry/src/main/java/datadog/telemetry/dependency/DependencyResolver.java +++ b/telemetry/src/main/java/datadog/telemetry/dependency/DependencyResolver.java @@ -1,7 +1,6 @@ package datadog.telemetry.dependency; import java.io.File; -import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.net.URI; @@ -52,7 +51,7 @@ static List internalResolve(final URI uri) throws IOException { if (!dependencies.isEmpty()) { return dependencies; } - try (final InputStream is = new FileInputStream(path)) { + try (final InputStream is = metadata.inputStreamSupplier.get()) { return Collections.singletonList( Dependency.guessFallbackNoPom(metadata.manifest, metadata.jarName, is)); } diff --git a/telemetry/src/main/java/datadog/telemetry/dependency/JarReader.java b/telemetry/src/main/java/datadog/telemetry/dependency/JarReader.java index 85382f27b2f..653f85e8ee3 100644 --- a/telemetry/src/main/java/datadog/telemetry/dependency/JarReader.java +++ b/telemetry/src/main/java/datadog/telemetry/dependency/JarReader.java @@ -1,6 +1,7 @@ package datadog.telemetry.dependency; import java.io.File; +import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.nio.file.NoSuchFileException; @@ -20,23 +21,30 @@ static class Extracted { final Map pomProperties; final Attributes manifest; final boolean isDirectory; + final InputStreamSupplier inputStreamSupplier; public Extracted( final String jarName, final Map pomProperties, final Attributes manifest, - final boolean isDirectory) { + final boolean isDirectory, + final InputStreamSupplier inputStreamSupplier) { this.jarName = jarName; this.pomProperties = pomProperties; this.manifest = manifest; this.isDirectory = isDirectory; + this.inputStreamSupplier = inputStreamSupplier; + } + + public interface InputStreamSupplier { + InputStream get() throws IOException; } } public static Extracted readJarFile(String jarPath) throws IOException { final File jarFile = new File(jarPath); if (jarFile.isDirectory()) { - return new Extracted(jarFile.getName(), new HashMap<>(), new Attributes(), true); + return new Extracted(jarFile.getName(), new HashMap<>(), new Attributes(), true, () -> null); } try (final JarFile jar = new JarFile(jarPath, false /* no verify */)) { final Map pomProperties = new HashMap<>(); @@ -54,7 +62,12 @@ public static Extracted readJarFile(String jarPath) throws IOException { final Manifest manifest = jar.getManifest(); final Attributes attributes = (manifest == null) ? new Attributes() : manifest.getMainAttributes(); - return new Extracted(new File(jar.getName()).getName(), pomProperties, attributes, false); + return new Extracted( + new File(jar.getName()).getName(), + pomProperties, + attributes, + false, + () -> new FileInputStream(jarPath)); } } @@ -64,10 +77,7 @@ public static Extracted readNestedJarFile(final String jarPath) throws IOExcepti throw new IllegalArgumentException("Invalid nested jar path: " + jarPath); } final String outerJarPath = jarPath.substring(0, sepIdx); - String innerJarPath = jarPath.substring(sepIdx + 2); - if (innerJarPath.endsWith("!/")) { - innerJarPath = innerJarPath.substring(0, innerJarPath.length() - 2); - } + final String innerJarPath = getInnerJarPath(jarPath); try (final JarFile outerJar = new JarFile(outerJarPath, false /* no verify */)) { final ZipEntry entry = outerJar.getEntry(innerJarPath); if (entry == null) { @@ -75,7 +85,7 @@ public static Extracted readNestedJarFile(final String jarPath) throws IOExcepti } if (entry.isDirectory()) { return new Extracted( - new File(innerJarPath).getName(), new HashMap<>(), new Attributes(), true); + new File(innerJarPath).getName(), new HashMap<>(), new Attributes(), true, () -> null); } try (final InputStream is = outerJar.getInputStream(entry); final JarInputStream innerJar = new JarInputStream(is, false /* no verify */)) { @@ -91,8 +101,54 @@ public static Extracted readNestedJarFile(final String jarPath) throws IOExcepti final Manifest manifest = innerJar.getManifest(); final Attributes attributes = (manifest == null) ? new Attributes() : manifest.getMainAttributes(); - return new Extracted(new File(innerJarPath).getName(), pomProperties, attributes, false); + return new Extracted( + new File(innerJarPath).getName(), + pomProperties, + attributes, + false, + () -> new NestedJarInputStream(outerJarPath, innerJarPath)); } } } + + private static String getInnerJarPath(final String jarPath) { + final int sepIdx = jarPath.indexOf("!/"); + if (sepIdx == -1) { + throw new IllegalArgumentException("Invalid nested jar path: " + jarPath); + } + String innerJarPath = jarPath.substring(sepIdx + 2); + final int innerSepIdx = innerJarPath.indexOf('!'); + if (innerSepIdx != -1) { + innerJarPath = innerJarPath.substring(0, innerSepIdx); + } + return innerJarPath; + } + + static class NestedJarInputStream extends InputStream implements AutoCloseable { + private final JarFile outerJar; + private final InputStream innerInputStream; + + public NestedJarInputStream(final String outerPath, final String innerPath) throws IOException { + super(); + this.outerJar = new JarFile(outerPath, false /* no verify */); + final ZipEntry entry = outerJar.getEntry(innerPath); + this.innerInputStream = outerJar.getInputStream(entry); + } + + @Override + public int read() throws IOException { + return this.innerInputStream.read(); + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + return this.innerInputStream.read(b, off, len); + } + + @Override + public void close() throws IOException { + this.innerInputStream.close(); + this.outerJar.close(); + } + } } diff --git a/telemetry/src/test/groovy/datadog/telemetry/dependency/DependencyResolverSpecification.groovy b/telemetry/src/test/groovy/datadog/telemetry/dependency/DependencyResolverSpecification.groovy index a47b8ebb9e8..76c37047d89 100644 --- a/telemetry/src/test/groovy/datadog/telemetry/dependency/DependencyResolverSpecification.groovy +++ b/telemetry/src/test/groovy/datadog/telemetry/dependency/DependencyResolverSpecification.groovy @@ -144,6 +144,48 @@ class DependencyResolverSpecification extends DepSpecification { dep.source == 'opentracing-util-0.33.0.jar' } + void 'spring boot dependency without trailing slash'() throws IOException { + when: + String zipPath = Classloader.classLoader.getResource('datadog/telemetry/dependencies/spring-boot-app.jar').path + URI uri = new URI("jar:file:$zipPath!/BOOT-INF/lib/opentracing-util-0.33.0.jar!") + + Dependency dep = DependencyResolver.resolve(uri).get(0) + + then: + dep != null + dep.name == 'io.opentracing:opentracing-util' + dep.version == '0.33.0' + dep.hash == null + dep.source == 'opentracing-util-0.33.0.jar' + } + + void 'spring boot dependency without maven metadata'() throws IOException { + given: + def innerJarData = new ByteArrayOutputStream() + ZipOutputStream out = new ZipOutputStream(innerJarData) + ZipEntry e = new ZipEntry("META-INF/MANIFEST.MF") + out.putNextEntry(e) + out.closeEntry() + out.close() + + File file = new File(testDir, "app.jar") + out = new ZipOutputStream(new FileOutputStream(file)) + e = new ZipEntry("BOOT-INF/lib/lib-1.0.jar") + out.putNextEntry(e) + out.write(innerJarData.toByteArray()) + out.closeEntry() + out.close() + + when: + URI uri = new URI("jar:file:" + file.getAbsolutePath() + "!/BOOT-INF/lib/lib-1.0.jar!/") + List deps = DependencyResolver.resolve(uri) + + then: + deps.size() == 1 + deps[0].source == "lib-1.0.jar" + deps[0].hash != null + } + void 'fat jar with multiple pom.properties'() throws IOException { when: URI uri = Classloader.classLoader.getResource('datadog/telemetry/dependencies/budgetapp.jar').toURI() From 40a62c54c0ff385c9059ca9c72c5408d4d5f4111 Mon Sep 17 00:00:00 2001 From: "Santiago M. Mola" Date: Mon, 11 Mar 2024 12:01:45 +0100 Subject: [PATCH 25/30] Minify AppSec rules (#6773) --- dd-java-agent/appsec/build.gradle | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/dd-java-agent/appsec/build.gradle b/dd-java-agent/appsec/build.gradle index ef2d6ec4b13..00abce5cc68 100644 --- a/dd-java-agent/appsec/build.gradle +++ b/dd-java-agent/appsec/build.gradle @@ -1,3 +1,6 @@ +import groovy.json.JsonOutput +import groovy.json.JsonSlurper + plugins { id "com.github.johnrengelman.shadow" id "me.champeau.jmh" @@ -36,6 +39,14 @@ jar { archiveClassifier = 'unbundled' } +processResources { + doLast { + fileTree(dir: outputs.files.asPath, includes: ['**/*.json']).each { + it.text = JsonOutput.toJson(new JsonSlurper().parse(it)) + } + } +} + jmh { jmhVersion = '1.32' duplicateClassesStrategy = DuplicatesStrategy.EXCLUDE From aeee9f097b4137a2648e87b84d68be27ce441c51 Mon Sep 17 00:00:00 2001 From: Zarir Hamza Date: Mon, 11 Mar 2024 07:24:59 -0400 Subject: [PATCH 26/30] Fixes muzzle reporting to be more accurate for telemetry (#6787) * remove aop from muzzle reporting --- buildSrc/src/main/groovy/MuzzlePlugin.groovy | 4 +++- dd-java-agent/instrumentation/spring-data-1.8/build.gradle | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/buildSrc/src/main/groovy/MuzzlePlugin.groovy b/buildSrc/src/main/groovy/MuzzlePlugin.groovy index 51f92feaa4f..c28e6022a0a 100644 --- a/buildSrc/src/main/groovy/MuzzlePlugin.groovy +++ b/buildSrc/src/main/groovy/MuzzlePlugin.groovy @@ -247,6 +247,7 @@ class MuzzlePlugin implements Plugin { ] as BiFunction) } } + dumpVersionsToCsv(project, map) } @@ -255,7 +256,7 @@ class MuzzlePlugin implements Plugin { final RepositorySystem system = newRepositorySystem() final RepositorySystemSession session = newRepositorySystemSession(system) def versions = new TreeMap() - project.muzzle.directives.findAll { !((MuzzleDirective) it).isCoreJdk() }.each { + project.muzzle.directives.findAll { !((MuzzleDirective) it).isCoreJdk() && !((MuzzleDirective) it).isSkipFromReport() }.each { def range = resolveVersionRange(it as MuzzleDirective, system, session) def cp = project.sourceSets.main.runtimeClasspath def cl = new URLClassLoader(cp*.toURI()*.toURL() as URL[], null as ClassLoader) @@ -552,6 +553,7 @@ class MuzzleDirective { List excludedDependencies = new ArrayList<>() boolean assertPass boolean assertInverse = false + boolean skipFromReport = false boolean coreJdk = false String javaVersion diff --git a/dd-java-agent/instrumentation/spring-data-1.8/build.gradle b/dd-java-agent/instrumentation/spring-data-1.8/build.gradle index f4b0fe90748..b25340b5199 100644 --- a/dd-java-agent/instrumentation/spring-data-1.8/build.gradle +++ b/dd-java-agent/instrumentation/spring-data-1.8/build.gradle @@ -14,12 +14,14 @@ muzzle { extraDependency "org.springframework:spring-aop:1.2" assertInverse = true } + pass { group = 'org.springframework' module = 'spring-aop' versions = "[1.2,6)" extraDependency "org.springframework.data:spring-data-commons:1.8.0.RELEASE" assertInverse = true + skipFromReport = true } } From e8da14ea490f30651fcdc5f4526246e79a3543a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20=C3=81lvarez=20=C3=81lvarez?= Date: Mon, 11 Mar 2024 12:26:26 +0100 Subject: [PATCH 27/30] Fix string format when escaped patterns are used (#6795) --- .../iast/propagation/StringModuleImpl.java | 54 +++++++++++++++---- .../iast/propagation/StringModuleTest.groovy | 4 ++ 2 files changed, 48 insertions(+), 10 deletions(-) diff --git a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/propagation/StringModuleImpl.java b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/propagation/StringModuleImpl.java index 37bed151dd1..db1f1ad865d 100644 --- a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/propagation/StringModuleImpl.java +++ b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/propagation/StringModuleImpl.java @@ -5,6 +5,7 @@ import static com.datadog.iast.taint.Ranges.mergeRanges; import static com.datadog.iast.taint.Tainteds.canBeTainted; import static com.datadog.iast.taint.Tainteds.getTainted; +import static datadog.trace.api.telemetry.LogCollector.SEND_TELEMETRY; import com.datadog.iast.model.Range; import com.datadog.iast.taint.Ranges; @@ -20,8 +21,12 @@ import java.util.Iterator; import java.util.LinkedList; import java.util.Locale; +import java.util.Map; +import java.util.function.Function; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -31,6 +36,10 @@ public class StringModuleImpl implements StringModule { private static final Pattern FORMAT_PATTERN = Pattern.compile("%(?\\d+\\$)?([-#+ 0,(\\<]*)?(\\d+)?(\\.\\d+)?([tT])?([a-zA-Z%])"); + /** Escaped format patterns * */ + private static final Map ESCAPED_PATTERNS = + Stream.of("%%", "%n").collect(Collectors.toMap(Function.identity(), String::format)); + private static final Ranged END = Ranged.build(Integer.MAX_VALUE, 0); private static final int NULL_STR_LENGTH = "null".length(); @@ -442,21 +451,32 @@ public void onStringFormat( final String placeholder = matcher.group(); final Object parameter; final String formattedValue; - final String index = matcher.group("index"); - if (index != null) { - // indexes are 1-based - final int parsedIndex = Integer.parseInt(index.substring(0, index.length() - 1)) - 1; - // remove the index before the formatting without increment the current state - parameter = parameters[parsedIndex]; - formattedValue = String.format(locale, placeholder.replace(index, ""), parameter); + final TaintedObject taintedObject; + final String escaped = ESCAPED_PATTERNS.get(placeholder); + if (escaped != null) { + parameter = placeholder; + formattedValue = escaped; + taintedObject = null; } else { - parameter = parameters[paramIndex++]; - formattedValue = String.format(locale, placeholder, parameter); + final String index = matcher.group("index"); + if (index != null) { + // indexes are 1-based + final int parsedIndex = Integer.parseInt(index.substring(0, index.length() - 1)) - 1; + // remove the index before the formatting without increment the current state + parameter = parameters[parsedIndex]; + formattedValue = String.format(locale, placeholder.replace(index, ""), parameter); + } else { + if (!checkParameterBounds(format, parameters, paramIndex)) { + return; // return without tainting the string in case of error + } + parameter = parameters[paramIndex++]; + formattedValue = String.format(locale, placeholder, parameter); + } + taintedObject = to.get(parameter); } final Ranged placeholderPos = Ranged.build(matcher.start(), placeholder.length()); final Range placeholderRange = addFormatTaintedRanges(placeholderPos, offset, formatRanges, finalRanges); - final TaintedObject taintedObject = to.get(parameter); final Range[] paramRanges = taintedObject == null ? null : taintedObject.getRanges(); final int shift = placeholderPos.getStart() + offset; addParameterTaintedRanges( @@ -473,6 +493,20 @@ public void onStringFormat( } } + private static boolean checkParameterBounds( + final String format, final Object[] parameters, int paramIndex) { + if (paramIndex < parameters.length) { + return true; + } + LOG.debug( + SEND_TELEMETRY, + "Error handling string format pattern {} with args {} at index {}", + format, + parameters.length, + paramIndex); + return false; + } + @Override public void onStringFormat( @Nonnull final Iterable literals, diff --git a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/propagation/StringModuleTest.groovy b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/propagation/StringModuleTest.groovy index 79f3643d4f4..e8d7ece64e0 100644 --- a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/propagation/StringModuleTest.groovy +++ b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/propagation/StringModuleTest.groovy @@ -915,6 +915,10 @@ class StringModuleTest extends IastModuleImplTestBase { 'Hello ==>%s<==' | ['World!'] | 'Hello ==>World!<==' // tainted placeholder [non tainted parameter] 'He==>llo %s!<==' | ['World'] | 'He==>llo <====>World<====>!<==' // tainted placeholder (2) [non tainted parameter] 'He==>llo %s!<==' | ['W==>or<==ld'] | 'He==>llo <==W==>or<==ld==>!<==' // tainted placeholder (3) [mixing with tainted parameter] + 'Hello %n %n %s!%n' | ['W==>or<==ld'] | 'Hello \n \n W==>or<==ld!\n' // \n character + 'Hello %% %% %s!%%' | ['W==>or<==ld'] | 'Hello % % W==>or<==ld!%' // % character + '==>Hello %n %s!<==' | ['World'] | '==>Hello <====>\n<====> <====>World<====>!<==' // \n character in tainted format (each placeholder generates a separate range) + '==>Hello %% %s!<==' | ['World'] | '==>Hello <====>%<====> <====>World<====>!<==' // % character in tainted format (each placeholder generates a separate range) } void 'onStringFormat literals: #literals args: #argsTainted'() { From 4fa5f84da9a47cddfc23ef283488fea08dd85f1a Mon Sep 17 00:00:00 2001 From: Stuart McCulloch Date: Mon, 11 Mar 2024 19:32:55 +0000 Subject: [PATCH 28/30] This special code-path was used by some old tests, but is no longer required. (#6796) --- .../tooling/AbstractTransformerBuilder.java | 4 --- .../tooling/CombiningTransformerBuilder.java | 26 ------------------- .../tooling/LegacyTransformerBuilder.java | 12 --------- 3 files changed, 42 deletions(-) diff --git a/dd-java-agent/agent-builder/src/main/java/datadog/trace/agent/tooling/AbstractTransformerBuilder.java b/dd-java-agent/agent-builder/src/main/java/datadog/trace/agent/tooling/AbstractTransformerBuilder.java index 33bee1dbcc3..5d397cb7c1f 100644 --- a/dd-java-agent/agent-builder/src/main/java/datadog/trace/agent/tooling/AbstractTransformerBuilder.java +++ b/dd-java-agent/agent-builder/src/main/java/datadog/trace/agent/tooling/AbstractTransformerBuilder.java @@ -42,8 +42,6 @@ public final void applyInstrumentation(Instrumenter instrumenter) { buildInstrumentation(module, member); } } - } else if (instrumenter instanceof Instrumenter.ForSingleType) { - buildSingleAdvice((Instrumenter.ForSingleType) instrumenter); // for testing purposes } else { throw new IllegalArgumentException("Unexpected Instrumenter type"); } @@ -53,8 +51,6 @@ public final void applyInstrumentation(Instrumenter instrumenter) { protected abstract void buildInstrumentation(InstrumenterModule module, Instrumenter member); - protected abstract void buildSingleAdvice(Instrumenter.ForSingleType instrumenter); - protected static final class VisitingTransformer implements AgentBuilder.Transformer { private final AsmVisitorWrapper visitor; diff --git a/dd-java-agent/agent-builder/src/main/java/datadog/trace/agent/tooling/CombiningTransformerBuilder.java b/dd-java-agent/agent-builder/src/main/java/datadog/trace/agent/tooling/CombiningTransformerBuilder.java index 3b56a775ec0..da3bf09e93a 100644 --- a/dd-java-agent/agent-builder/src/main/java/datadog/trace/agent/tooling/CombiningTransformerBuilder.java +++ b/dd-java-agent/agent-builder/src/main/java/datadog/trace/agent/tooling/CombiningTransformerBuilder.java @@ -2,9 +2,7 @@ import static datadog.trace.agent.tooling.bytebuddy.DDTransformers.defaultTransformers; import static datadog.trace.agent.tooling.bytebuddy.matcher.ClassLoaderMatchers.ANY_CLASS_LOADER; -import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.namedOneOf; -import static net.bytebuddy.matcher.ElementMatchers.isSynthetic; import static net.bytebuddy.matcher.ElementMatchers.not; import datadog.trace.agent.tooling.Instrumenter.WithPostProcessor; @@ -145,30 +143,6 @@ private void buildInstrumentationAdvice(InstrumenterModule module, Instrumenter advice.clear(); } - @Override - protected void buildSingleAdvice(Instrumenter.ForSingleType instrumenter) { - - // this is a test instrumenter which needs a dynamic id - int id = nextSupplementaryId++; - if (transformers.length <= id) { - transformers = Arrays.copyOf(transformers, id + 1); - } - - // can't use known-types index because it doesn't include test instrumenters - matchers.add(new MatchRecorder.ForType(id, named(instrumenter.instrumentedType()))); - - ignoredMethods = isSynthetic(); - if (instrumenter instanceof Instrumenter.HasTypeAdvice) { - ((Instrumenter.HasTypeAdvice) instrumenter).typeAdvice(this); - } - if (instrumenter instanceof Instrumenter.HasMethodAdvice) { - ((Instrumenter.HasMethodAdvice) instrumenter).methodAdvice(this); - } - transformers[id] = new AdviceStack(advice); - - advice.clear(); - } - @Override public void applyAdvice(Instrumenter.TransformingAdvice typeAdvice) { advice.add(typeAdvice::transform); diff --git a/dd-java-agent/agent-builder/src/main/java/datadog/trace/agent/tooling/LegacyTransformerBuilder.java b/dd-java-agent/agent-builder/src/main/java/datadog/trace/agent/tooling/LegacyTransformerBuilder.java index db342872108..26f36cc7e6c 100644 --- a/dd-java-agent/agent-builder/src/main/java/datadog/trace/agent/tooling/LegacyTransformerBuilder.java +++ b/dd-java-agent/agent-builder/src/main/java/datadog/trace/agent/tooling/LegacyTransformerBuilder.java @@ -3,7 +3,6 @@ import static datadog.trace.agent.tooling.bytebuddy.DDTransformers.defaultTransformers; import static datadog.trace.agent.tooling.bytebuddy.matcher.ClassLoaderMatchers.ANY_CLASS_LOADER; import static datadog.trace.agent.tooling.bytebuddy.matcher.ClassLoaderMatchers.hasClassNamed; -import static net.bytebuddy.matcher.ElementMatchers.isSynthetic; import static net.bytebuddy.matcher.ElementMatchers.none; import static net.bytebuddy.matcher.ElementMatchers.not; @@ -157,17 +156,6 @@ private AgentBuilder.RawMatcher typeMatcher(InstrumenterModule module, Instrumen + module.getClass().getName()); } - @Override - protected void buildSingleAdvice(Instrumenter.ForSingleType instrumenter) { - AgentBuilder.RawMatcher matcher = new SingleTypeMatcher(instrumenter.instrumentedType()); - - ignoreMatcher = isSynthetic(); - adviceBuilder = - agentBuilder.type(matcher).and(NOT_DECORATOR_MATCHER).transform(defaultTransformers()); - - agentBuilder = registerAdvice((Instrumenter) instrumenter); - } - @Override public void applyAdvice(Instrumenter.TransformingAdvice typeAdvice) { adviceBuilder = adviceBuilder.transform(typeAdvice::transform); From 39767bb1e5e04d544219b0db9208b34af013387e Mon Sep 17 00:00:00 2001 From: Stuart McCulloch Date: Mon, 11 Mar 2024 22:09:13 +0000 Subject: [PATCH 29/30] Remove legacy agent installer (#6797) --- .../tooling/AbstractTransformerBuilder.java | 136 ------------- .../trace/agent/tooling/AgentInstaller.java | 8 +- .../tooling/CombiningTransformerBuilder.java | 124 +++++++++++- .../tooling/LegacyTransformerBuilder.java | 187 ------------------ .../config/TraceInstrumentationConfig.java | 1 - .../datadog/trace/api/InstrumenterConfig.java | 10 - 6 files changed, 120 insertions(+), 346 deletions(-) delete mode 100644 dd-java-agent/agent-builder/src/main/java/datadog/trace/agent/tooling/AbstractTransformerBuilder.java delete mode 100644 dd-java-agent/agent-builder/src/main/java/datadog/trace/agent/tooling/LegacyTransformerBuilder.java diff --git a/dd-java-agent/agent-builder/src/main/java/datadog/trace/agent/tooling/AbstractTransformerBuilder.java b/dd-java-agent/agent-builder/src/main/java/datadog/trace/agent/tooling/AbstractTransformerBuilder.java deleted file mode 100644 index 5d397cb7c1f..00000000000 --- a/dd-java-agent/agent-builder/src/main/java/datadog/trace/agent/tooling/AbstractTransformerBuilder.java +++ /dev/null @@ -1,136 +0,0 @@ -package datadog.trace.agent.tooling; - -import static datadog.trace.agent.tooling.bytebuddy.matcher.ClassLoaderMatchers.ANY_CLASS_LOADER; -import static datadog.trace.agent.tooling.bytebuddy.matcher.ClassLoaderMatchers.hasClassNamed; -import static datadog.trace.agent.tooling.bytebuddy.matcher.ClassLoaderMatchers.hasClassNamedOneOf; -import static datadog.trace.agent.tooling.bytebuddy.matcher.HierarchyMatchers.declaresAnnotation; -import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.namedOneOf; -import static net.bytebuddy.matcher.ElementMatchers.not; - -import java.lang.instrument.ClassFileTransformer; -import java.lang.instrument.Instrumentation; -import java.security.ProtectionDomain; -import java.util.HashMap; -import java.util.Map; -import net.bytebuddy.agent.builder.AgentBuilder; -import net.bytebuddy.asm.AsmVisitorWrapper; -import net.bytebuddy.description.type.TypeDescription; -import net.bytebuddy.dynamic.DynamicType; -import net.bytebuddy.matcher.ElementMatcher; -import net.bytebuddy.utility.JavaModule; - -abstract class AbstractTransformerBuilder - implements Instrumenter.TypeTransformer, Instrumenter.MethodTransformer { - - // Added here instead of byte-buddy's ignores because it's relatively - // expensive. https://github.com/DataDog/dd-trace-java/pull/1045 - protected static final ElementMatcher.Junction NOT_DECORATOR_MATCHER = - not( - declaresAnnotation( - namedOneOf("javax.decorator.Decorator", "jakarta.decorator.Decorator"))); - - /** Associates context stores with the class-loader matchers to activate them. */ - private final Map, ElementMatcher> contextStoreInjection = - new HashMap<>(); - - public final void applyInstrumentation(Instrumenter instrumenter) { - if (instrumenter instanceof InstrumenterModule) { - InstrumenterModule module = (InstrumenterModule) instrumenter; - if (module.isEnabled()) { - InstrumenterState.registerInstrumentation(module); - for (Instrumenter member : module.typeInstrumentations()) { - buildInstrumentation(module, member); - } - } - } else { - throw new IllegalArgumentException("Unexpected Instrumenter type"); - } - } - - public abstract ClassFileTransformer installOn(Instrumentation instrumentation); - - protected abstract void buildInstrumentation(InstrumenterModule module, Instrumenter member); - - protected static final class VisitingTransformer implements AgentBuilder.Transformer { - private final AsmVisitorWrapper visitor; - - VisitingTransformer(AsmVisitorWrapper visitor) { - this.visitor = visitor; - } - - @Override - public DynamicType.Builder transform( - DynamicType.Builder builder, - TypeDescription typeDescription, - ClassLoader classLoader, - JavaModule module, - ProtectionDomain pd) { - return builder.visit(visitor); - } - } - - protected static final class HelperTransformer extends HelperInjector - implements AgentBuilder.Transformer { - HelperTransformer(String requestingName, String... helperClassNames) { - super(requestingName, helperClassNames); - } - } - - protected static ElementMatcher requireBoth( - ElementMatcher lhs, ElementMatcher rhs) { - if (ANY_CLASS_LOADER == lhs) { - return rhs; - } else if (ANY_CLASS_LOADER == rhs) { - return lhs; - } else { - return new ElementMatcher.Junction.Conjunction<>(lhs, rhs); - } - } - - /** Tracks which class-loader matchers are associated with each store request. */ - protected final void registerContextStoreInjection( - InstrumenterModule module, Instrumenter member, Map contextStore) { - ElementMatcher activation; - - if (member instanceof Instrumenter.ForBootstrap) { - activation = ANY_CLASS_LOADER; - } else if (member instanceof Instrumenter.ForTypeHierarchy) { - String hierarchyHint = ((Instrumenter.ForTypeHierarchy) member).hierarchyMarkerType(); - activation = null != hierarchyHint ? hasClassNamed(hierarchyHint) : ANY_CLASS_LOADER; - } else if (member instanceof Instrumenter.ForSingleType) { - activation = hasClassNamed(((Instrumenter.ForSingleType) member).instrumentedType()); - } else if (member instanceof Instrumenter.ForKnownTypes) { - activation = hasClassNamedOneOf(((Instrumenter.ForKnownTypes) member).knownMatchingTypes()); - } else { - activation = ANY_CLASS_LOADER; - } - - activation = requireBoth(activation, module.classLoaderMatcher()); - - for (Map.Entry storeEntry : contextStore.entrySet()) { - ElementMatcher oldActivation = contextStoreInjection.get(storeEntry); - // optimization: treat 'any' as if there wasn't an old matcher - if (null == oldActivation || ANY_CLASS_LOADER == activation) { - contextStoreInjection.put(storeEntry, activation); - } else if (ANY_CLASS_LOADER != oldActivation) { - // store can be activated by either the old OR new matcher - contextStoreInjection.put( - storeEntry, new ElementMatcher.Junction.Disjunction<>(oldActivation, activation)); - } - } - } - - /** Counts the number of distinct context store injections registered with this builder. */ - protected final int contextStoreCount() { - return contextStoreInjection.size(); - } - - /** Applies each context store injection, guarded by the associated class-loader matcher. */ - protected final void applyContextStoreInjection() { - contextStoreInjection.forEach(this::applyContextStoreInjection); - } - - /** Arranges for a context value field to be injected into types extending the context key. */ - protected abstract void applyContextStoreInjection( - Map.Entry contextStore, ElementMatcher activation); -} diff --git a/dd-java-agent/agent-builder/src/main/java/datadog/trace/agent/tooling/AgentInstaller.java b/dd-java-agent/agent-builder/src/main/java/datadog/trace/agent/tooling/AgentInstaller.java index f7b50d50628..7dcfbe3279b 100644 --- a/dd-java-agent/agent-builder/src/main/java/datadog/trace/agent/tooling/AgentInstaller.java +++ b/dd-java-agent/agent-builder/src/main/java/datadog/trace/agent/tooling/AgentInstaller.java @@ -189,12 +189,8 @@ public static ClassFileTransformer installBytebuddyAgent( } } - AbstractTransformerBuilder transformerBuilder; - if (InstrumenterConfig.get().isLegacyInstallerEnabled()) { - transformerBuilder = new LegacyTransformerBuilder(agentBuilder); - } else { - transformerBuilder = new CombiningTransformerBuilder(agentBuilder, maxInstrumentationId); - } + CombiningTransformerBuilder transformerBuilder = + new CombiningTransformerBuilder(agentBuilder, maxInstrumentationId); int installedCount = 0; for (Instrumenter instrumenter : instrumenters) { diff --git a/dd-java-agent/agent-builder/src/main/java/datadog/trace/agent/tooling/CombiningTransformerBuilder.java b/dd-java-agent/agent-builder/src/main/java/datadog/trace/agent/tooling/CombiningTransformerBuilder.java index da3bf09e93a..961f13cc8fe 100644 --- a/dd-java-agent/agent-builder/src/main/java/datadog/trace/agent/tooling/CombiningTransformerBuilder.java +++ b/dd-java-agent/agent-builder/src/main/java/datadog/trace/agent/tooling/CombiningTransformerBuilder.java @@ -2,6 +2,9 @@ import static datadog.trace.agent.tooling.bytebuddy.DDTransformers.defaultTransformers; import static datadog.trace.agent.tooling.bytebuddy.matcher.ClassLoaderMatchers.ANY_CLASS_LOADER; +import static datadog.trace.agent.tooling.bytebuddy.matcher.ClassLoaderMatchers.hasClassNamed; +import static datadog.trace.agent.tooling.bytebuddy.matcher.ClassLoaderMatchers.hasClassNamedOneOf; +import static datadog.trace.agent.tooling.bytebuddy.matcher.HierarchyMatchers.declaresAnnotation; import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.namedOneOf; import static net.bytebuddy.matcher.ElementMatchers.not; @@ -14,19 +17,38 @@ import datadog.trace.api.InstrumenterConfig; import java.lang.instrument.ClassFileTransformer; import java.lang.instrument.Instrumentation; +import java.security.ProtectionDomain; import java.util.ArrayList; import java.util.Arrays; import java.util.BitSet; import java.util.Collection; +import java.util.HashMap; import java.util.List; import java.util.Map; import net.bytebuddy.agent.builder.AgentBuilder; import net.bytebuddy.asm.Advice; +import net.bytebuddy.asm.AsmVisitorWrapper; import net.bytebuddy.description.method.MethodDescription; +import net.bytebuddy.description.type.TypeDescription; +import net.bytebuddy.dynamic.DynamicType; import net.bytebuddy.matcher.ElementMatcher; +import net.bytebuddy.utility.JavaModule; /** Builds multiple instrumentations into a single combining-matcher and splitting-transformer. */ -public final class CombiningTransformerBuilder extends AbstractTransformerBuilder { +public final class CombiningTransformerBuilder + implements Instrumenter.TypeTransformer, Instrumenter.MethodTransformer { + + // Added here instead of byte-buddy's ignores because it's relatively + // expensive. https://github.com/DataDog/dd-trace-java/pull/1045 + private static final ElementMatcher.Junction NOT_DECORATOR_MATCHER = + not( + declaresAnnotation( + namedOneOf("javax.decorator.Decorator", "jakarta.decorator.Decorator"))); + + /** Associates context stores with the class-loader matchers to activate them. */ + private final Map, ElementMatcher> contextStoreInjection = + new HashMap<>(); + private final AgentBuilder agentBuilder; private final List matchers = new ArrayList<>(); @@ -52,8 +74,21 @@ public CombiningTransformerBuilder(AgentBuilder agentBuilder, int maxInstrumenta this.nextSupplementaryId = maxInstrumentationId + 1; } - @Override - protected void buildInstrumentation(InstrumenterModule module, Instrumenter member) { + public void applyInstrumentation(Instrumenter instrumenter) { + if (instrumenter instanceof InstrumenterModule) { + InstrumenterModule module = (InstrumenterModule) instrumenter; + if (module.isEnabled()) { + InstrumenterState.registerInstrumentation(module); + for (Instrumenter member : module.typeInstrumentations()) { + buildInstrumentation(module, member); + } + } + } else { + throw new IllegalArgumentException("Unexpected Instrumenter type"); + } + } + + private void buildInstrumentation(InstrumenterModule module, Instrumenter member) { int id = module.instrumentationId(); if (module != member) { @@ -161,8 +196,51 @@ public void applyAdvice(ElementMatcher matcher, Strin .advice(not(ignoredMethods).and(matcher), className)); } - @Override - protected void applyContextStoreInjection( + /** Counts the number of distinct context store injections registered with this builder. */ + private int contextStoreCount() { + return contextStoreInjection.size(); + } + + /** Applies each context store injection, guarded by the associated class-loader matcher. */ + private void applyContextStoreInjection() { + contextStoreInjection.forEach(this::applyContextStoreInjection); + } + + /** Tracks which class-loader matchers are associated with each store request. */ + private void registerContextStoreInjection( + InstrumenterModule module, Instrumenter member, Map contextStore) { + ElementMatcher activation; + + if (member instanceof Instrumenter.ForBootstrap) { + activation = ANY_CLASS_LOADER; + } else if (member instanceof Instrumenter.ForTypeHierarchy) { + String hierarchyHint = ((Instrumenter.ForTypeHierarchy) member).hierarchyMarkerType(); + activation = null != hierarchyHint ? hasClassNamed(hierarchyHint) : ANY_CLASS_LOADER; + } else if (member instanceof Instrumenter.ForSingleType) { + activation = hasClassNamed(((Instrumenter.ForSingleType) member).instrumentedType()); + } else if (member instanceof Instrumenter.ForKnownTypes) { + activation = hasClassNamedOneOf(((Instrumenter.ForKnownTypes) member).knownMatchingTypes()); + } else { + activation = ANY_CLASS_LOADER; + } + + activation = requireBoth(activation, module.classLoaderMatcher()); + + for (Map.Entry storeEntry : contextStore.entrySet()) { + ElementMatcher oldActivation = contextStoreInjection.get(storeEntry); + // optimization: treat 'any' as if there wasn't an old matcher + if (null == oldActivation || ANY_CLASS_LOADER == activation) { + contextStoreInjection.put(storeEntry, activation); + } else if (ANY_CLASS_LOADER != oldActivation) { + // store can be activated by either the old OR new matcher + contextStoreInjection.put( + storeEntry, new ElementMatcher.Junction.Disjunction<>(oldActivation, activation)); + } + } + } + + /** Arranges for a context value field to be injected into types extending the context key. */ + private void applyContextStoreInjection( Map.Entry contextStore, ElementMatcher activation) { String keyClassName = contextStore.getKey(); String contextClassName = contextStore.getValue(); @@ -178,7 +256,6 @@ protected void applyContextStoreInjection( transformers[id] = new AdviceStack(new VisitingTransformer(contextAdvice)); } - @Override public ClassFileTransformer installOn(Instrumentation instrumentation) { if (InstrumenterConfig.get().isRuntimeContextFieldInjection()) { // expand so we have enough space for a context injecting transformer for each store @@ -193,4 +270,39 @@ public ClassFileTransformer installOn(Instrumentation instrumentation) { .transform(new SplittingTransformer(transformers)) .installOn(instrumentation); } + + static final class VisitingTransformer implements AgentBuilder.Transformer { + private final AsmVisitorWrapper visitor; + + VisitingTransformer(AsmVisitorWrapper visitor) { + this.visitor = visitor; + } + + @Override + public DynamicType.Builder transform( + DynamicType.Builder builder, + TypeDescription typeDescription, + ClassLoader classLoader, + JavaModule module, + ProtectionDomain pd) { + return builder.visit(visitor); + } + } + + static final class HelperTransformer extends HelperInjector implements AgentBuilder.Transformer { + HelperTransformer(String requestingName, String... helperClassNames) { + super(requestingName, helperClassNames); + } + } + + static ElementMatcher requireBoth( + ElementMatcher lhs, ElementMatcher rhs) { + if (ANY_CLASS_LOADER == lhs) { + return rhs; + } else if (ANY_CLASS_LOADER == rhs) { + return lhs; + } else { + return new ElementMatcher.Junction.Conjunction<>(lhs, rhs); + } + } } diff --git a/dd-java-agent/agent-builder/src/main/java/datadog/trace/agent/tooling/LegacyTransformerBuilder.java b/dd-java-agent/agent-builder/src/main/java/datadog/trace/agent/tooling/LegacyTransformerBuilder.java deleted file mode 100644 index 26f36cc7e6c..00000000000 --- a/dd-java-agent/agent-builder/src/main/java/datadog/trace/agent/tooling/LegacyTransformerBuilder.java +++ /dev/null @@ -1,187 +0,0 @@ -package datadog.trace.agent.tooling; - -import static datadog.trace.agent.tooling.bytebuddy.DDTransformers.defaultTransformers; -import static datadog.trace.agent.tooling.bytebuddy.matcher.ClassLoaderMatchers.ANY_CLASS_LOADER; -import static datadog.trace.agent.tooling.bytebuddy.matcher.ClassLoaderMatchers.hasClassNamed; -import static net.bytebuddy.matcher.ElementMatchers.none; -import static net.bytebuddy.matcher.ElementMatchers.not; - -import datadog.trace.agent.tooling.bytebuddy.ExceptionHandlers; -import datadog.trace.agent.tooling.bytebuddy.matcher.FailSafeRawMatcher; -import datadog.trace.agent.tooling.bytebuddy.matcher.InjectContextFieldMatcher; -import datadog.trace.agent.tooling.bytebuddy.matcher.KnownTypesMatcher; -import datadog.trace.agent.tooling.bytebuddy.matcher.MuzzleMatcher; -import datadog.trace.agent.tooling.bytebuddy.matcher.SingleTypeMatcher; -import datadog.trace.agent.tooling.context.FieldBackedContextInjector; -import datadog.trace.agent.tooling.context.FieldBackedContextRequestRewriter; -import datadog.trace.api.InstrumenterConfig; -import java.lang.instrument.ClassFileTransformer; -import java.lang.instrument.Instrumentation; -import java.util.Collection; -import java.util.Map; -import net.bytebuddy.agent.builder.AgentBuilder; -import net.bytebuddy.description.method.MethodDescription; -import net.bytebuddy.description.type.TypeDescription; -import net.bytebuddy.matcher.ElementMatcher; - -public final class LegacyTransformerBuilder extends AbstractTransformerBuilder { - - private AgentBuilder agentBuilder; - private ElementMatcher ignoreMatcher; - private AgentBuilder.Identified.Extendable adviceBuilder; - - LegacyTransformerBuilder(AgentBuilder agentBuilder) { - this.agentBuilder = agentBuilder; - } - - @Override - public ClassFileTransformer installOn(Instrumentation instrumentation) { - if (InstrumenterConfig.get().isRuntimeContextFieldInjection()) { - applyContextStoreInjection(); - } - - return agentBuilder.installOn(instrumentation); - } - - @Override - protected void buildInstrumentation(InstrumenterModule module, Instrumenter member) { - - ignoreMatcher = module.methodIgnoreMatcher(); - adviceBuilder = - agentBuilder - .type(typeMatcher(module, member)) - .and(NOT_DECORATOR_MATCHER) - .and(new MuzzleMatcher(module)) - .transform(defaultTransformers()); - - String[] helperClassNames = module.helperClassNames(); - if (module.injectHelperDependencies()) { - helperClassNames = HelperScanner.withClassDependencies(helperClassNames); - } - if (helperClassNames.length > 0) { - adviceBuilder = - adviceBuilder.transform( - new HelperTransformer(module.getClass().getSimpleName(), helperClassNames)); - } - - Map contextStore = module.contextStore(); - if (!contextStore.isEmpty()) { - // rewrite context store access to call FieldBackedContextStores with assigned store-id - adviceBuilder = - adviceBuilder.transform( - new VisitingTransformer( - new FieldBackedContextRequestRewriter(contextStore, module.name()))); - - registerContextStoreInjection(module, member, contextStore); - } - - agentBuilder = registerAdvice(member); - } - - private AgentBuilder registerAdvice(Instrumenter instrumenter) { - if (instrumenter instanceof Instrumenter.HasTypeAdvice) { - ((Instrumenter.HasTypeAdvice) instrumenter).typeAdvice(this); - } - if (instrumenter instanceof Instrumenter.HasMethodAdvice) { - ((Instrumenter.HasMethodAdvice) instrumenter).methodAdvice(this); - } - return adviceBuilder; - } - - private AgentBuilder.RawMatcher typeMatcher(InstrumenterModule module, Instrumenter member) { - ElementMatcher typeMatcher; - String hierarchyHint = null; - - if (member instanceof Instrumenter.ForSingleType) { - String name = ((Instrumenter.ForSingleType) member).instrumentedType(); - typeMatcher = new SingleTypeMatcher(name); - } else if (member instanceof Instrumenter.ForKnownTypes) { - String[] names = ((Instrumenter.ForKnownTypes) member).knownMatchingTypes(); - typeMatcher = new KnownTypesMatcher(names); - } else if (member instanceof Instrumenter.ForTypeHierarchy) { - typeMatcher = ((Instrumenter.ForTypeHierarchy) member).hierarchyMatcher(); - hierarchyHint = ((Instrumenter.ForTypeHierarchy) member).hierarchyMarkerType(); - } else if (member instanceof Instrumenter.ForConfiguredTypes) { - typeMatcher = none(); // handle below, just like when it's combined with other matchers - } else if (member instanceof Instrumenter.ForCallSite) { - typeMatcher = ((Instrumenter.ForCallSite) member).callerType(); - } else { - return AgentBuilder.RawMatcher.Trivial.NON_MATCHING; - } - - if (member instanceof Instrumenter.CanShortcutTypeMatching - && !((Instrumenter.CanShortcutTypeMatching) member).onlyMatchKnownTypes()) { - // not taking shortcuts, so include wider hierarchical matching - typeMatcher = - new ElementMatcher.Junction.Disjunction( - typeMatcher, ((Instrumenter.ForTypeHierarchy) member).hierarchyMatcher()); - hierarchyHint = ((Instrumenter.ForTypeHierarchy) member).hierarchyMarkerType(); - } - - if (member instanceof Instrumenter.ForConfiguredTypes) { - Collection names = - ((Instrumenter.ForConfiguredTypes) member).configuredMatchingTypes(); - // only add this optional matcher when it's been configured - if (null != names && !names.isEmpty()) { - typeMatcher = - new ElementMatcher.Junction.Disjunction(typeMatcher, new KnownTypesMatcher(names)); - } - } - - if (member instanceof Instrumenter.WithTypeStructure) { - // only perform structure matching after we've matched the type - typeMatcher = - new ElementMatcher.Junction.Conjunction( - typeMatcher, ((Instrumenter.WithTypeStructure) member).structureMatcher()); - } - - ElementMatcher classLoaderMatcher = module.classLoaderMatcher(); - - if (null != hierarchyHint) { - // use hint to limit expensive type matching to class-loaders with marker type - classLoaderMatcher = requireBoth(hasClassNamed(hierarchyHint), classLoaderMatcher); - } - - if (ANY_CLASS_LOADER == classLoaderMatcher && typeMatcher instanceof AgentBuilder.RawMatcher) { - // optimization when using raw (named) type matcher with no classloader filtering - return (AgentBuilder.RawMatcher) typeMatcher; - } - - return new FailSafeRawMatcher( - typeMatcher, - classLoaderMatcher, - "Instrumentation matcher unexpected exception - instrumentation.names=" - + module.names() - + " instrumentation.class=" - + module.getClass().getName()); - } - - @Override - public void applyAdvice(Instrumenter.TransformingAdvice typeAdvice) { - adviceBuilder = adviceBuilder.transform(typeAdvice::transform); - } - - @Override - public void applyAdvice(ElementMatcher matcher, String className) { - adviceBuilder = - adviceBuilder.transform( - new AgentBuilder.Transformer.ForAdvice() - .include(Utils.getBootstrapProxy(), Utils.getAgentClassLoader()) - .withExceptionHandler(ExceptionHandlers.defaultExceptionHandler()) - .advice(not(ignoreMatcher).and(matcher), className)); - } - - @Override - protected void applyContextStoreInjection( - Map.Entry contextStore, ElementMatcher activation) { - String keyClassName = contextStore.getKey(); - String contextClassName = contextStore.getValue(); - agentBuilder = - agentBuilder - .type(new InjectContextFieldMatcher(keyClassName, contextClassName, activation)) - .and(NOT_DECORATOR_MATCHER) - .transform( - new VisitingTransformer( - new FieldBackedContextInjector(keyClassName, contextClassName))); - } -} diff --git a/dd-trace-api/src/main/java/datadog/trace/api/config/TraceInstrumentationConfig.java b/dd-trace-api/src/main/java/datadog/trace/api/config/TraceInstrumentationConfig.java index ee64514f908..7369c5d46d3 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/config/TraceInstrumentationConfig.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/config/TraceInstrumentationConfig.java @@ -12,7 +12,6 @@ public final class TraceInstrumentationConfig { public static final String TRACE_ENABLED = "trace.enabled"; public static final String TRACE_OTEL_ENABLED = "trace.otel.enabled"; public static final String INTEGRATIONS_ENABLED = "integrations.enabled"; - public static final String LEGACY_INSTALLER_ENABLED = "legacy.installer.enabled"; public static final String INTEGRATION_SYNAPSE_LEGACY_OPERATION_NAME = "integration.synapse.legacy-operation-name"; diff --git a/internal-api/src/main/java/datadog/trace/api/InstrumenterConfig.java b/internal-api/src/main/java/datadog/trace/api/InstrumenterConfig.java index e1bb041ddf2..62901a32f27 100644 --- a/internal-api/src/main/java/datadog/trace/api/InstrumenterConfig.java +++ b/internal-api/src/main/java/datadog/trace/api/InstrumenterConfig.java @@ -32,7 +32,6 @@ import static datadog.trace.api.config.TraceInstrumentationConfig.JAX_RS_ADDITIONAL_ANNOTATIONS; import static datadog.trace.api.config.TraceInstrumentationConfig.JDBC_CONNECTION_CLASS_NAME; import static datadog.trace.api.config.TraceInstrumentationConfig.JDBC_PREPARED_STATEMENT_CLASS_NAME; -import static datadog.trace.api.config.TraceInstrumentationConfig.LEGACY_INSTALLER_ENABLED; import static datadog.trace.api.config.TraceInstrumentationConfig.MEASURE_METHODS; import static datadog.trace.api.config.TraceInstrumentationConfig.RESOLVER_CACHE_CONFIG; import static datadog.trace.api.config.TraceInstrumentationConfig.RESOLVER_CACHE_DIR; @@ -134,8 +133,6 @@ public class InstrumenterConfig { private final boolean internalExitOnFailure; - private final boolean legacyInstallerEnabled; - private final Collection additionalJaxRsAnnotations; private InstrumenterConfig() { @@ -229,7 +226,6 @@ private InstrumenterConfig() { configProvider.getString(MEASURE_METHODS, DEFAULT_MEASURE_METHODS)); internalExitOnFailure = configProvider.getBoolean(INTERNAL_EXIT_ON_FAILURE, false); - legacyInstallerEnabled = configProvider.getBoolean(LEGACY_INSTALLER_ENABLED, false); this.additionalJaxRsAnnotations = tryMakeImmutableSet(configProvider.getList(JAX_RS_ADDITIONAL_ANNOTATIONS)); } @@ -423,10 +419,6 @@ public boolean isInternalExitOnFailure() { return internalExitOnFailure; } - public boolean isLegacyInstallerEnabled() { - return legacyInstallerEnabled; - } - public boolean isLegacyInstrumentationEnabled( final boolean defaultEnabled, final String... integrationNames) { return configProvider.isEnabled( @@ -523,8 +515,6 @@ public String toString() { + '\'' + ", internalExitOnFailure=" + internalExitOnFailure - + ", legacyInstallerEnabled=" - + legacyInstallerEnabled + ", additionalJaxRsAnnotations=" + additionalJaxRsAnnotations + '}'; From 0d49c12f5398d2205b8e0d4b18b70fd26ff35c22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Gonz=C3=A1lez=20Garc=C3=ADa?= Date: Tue, 12 Mar 2024 08:45:08 +0100 Subject: [PATCH 30/30] update MAX_SIZE_EXCEEDED value (#6779) --- .../iast/model/json/TruncatedVulnerabilitiesAdapter.java | 2 +- .../iast/model/json/VulnerabilityEncodingTest.groovy | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/model/json/TruncatedVulnerabilitiesAdapter.java b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/model/json/TruncatedVulnerabilitiesAdapter.java index ffe32b726bc..f4a72d77602 100644 --- a/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/model/json/TruncatedVulnerabilitiesAdapter.java +++ b/dd-java-agent/agent-iast/src/main/java/com/datadog/iast/model/json/TruncatedVulnerabilitiesAdapter.java @@ -14,7 +14,7 @@ class TruncatedVulnerabilitiesAdapter extends FormattingAdapter { - private static final String MAX_SIZE_EXCEEDED = "MAX SIZE EXCEEDED"; + private static final String MAX_SIZE_EXCEEDED = "MAX_SIZE_EXCEEDED"; private final JsonAdapter vulnerabilityAdapter; diff --git a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/model/json/VulnerabilityEncodingTest.groovy b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/model/json/VulnerabilityEncodingTest.groovy index 76c81d5696e..1f404357475 100644 --- a/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/model/json/VulnerabilityEncodingTest.groovy +++ b/dd-java-agent/agent-iast/src/test/groovy/com/datadog/iast/model/json/VulnerabilityEncodingTest.groovy @@ -415,7 +415,7 @@ class VulnerabilityEncodingTest extends DDSpecification { { "type": "WEAK_HASH", "evidence": { - "value": "MAX SIZE EXCEEDED" + "value": "MAX_SIZE_EXCEEDED" }, "hash":1042880134, "location": { @@ -454,7 +454,7 @@ class VulnerabilityEncodingTest extends DDSpecification { "vulnerabilities": [ { "evidence": { - "value": "MAX SIZE EXCEEDED" + "value": "MAX_SIZE_EXCEEDED" }, "hash": 1042880134, "location": { @@ -467,7 +467,7 @@ class VulnerabilityEncodingTest extends DDSpecification { }, { "evidence": { - "value": "MAX SIZE EXCEEDED" + "value": "MAX_SIZE_EXCEEDED" }, "hash": 1042880134, "location": { @@ -543,7 +543,7 @@ class VulnerabilityEncodingTest extends DDSpecification { } private static int countGenericEvidenceOccurrences(final String input){ - Pattern pattern = Pattern.compile("\"evidence\":\\{\"value\":\"MAX SIZE EXCEEDED\"}") + Pattern pattern = Pattern.compile("\"evidence\":\\{\"value\":\"MAX_SIZE_EXCEEDED\"}") Matcher matcher = pattern.matcher(input) int count = 0 while (matcher.find()){