From 0b031046b31ee3369279417fdf7e37f29a8bd192 Mon Sep 17 00:00:00 2001 From: Gary Huang Date: Tue, 7 Jan 2025 10:36:10 -0500 Subject: [PATCH] Add LLMObs configuration (#8076) * add LLM obs configs * refactor & centralize api key checks * rename LLM_OBS consts to be more aligned with other languages * undo jmx fetch changes --- .../trace/agent/tooling/AgentInstaller.java | 3 + .../agent/tooling/InstrumenterModule.java | 3 +- .../datadog/trace/api/ConfigDefaults.java | 3 + .../trace/api/config/LlmObsConfig.java | 16 +++ .../main/java/datadog/trace/api/Config.java | 65 +++++++-- .../datadog/trace/api/InstrumenterConfig.java | 9 ++ .../datadog/trace/api/ConfigTest.groovy | 124 ++++++++++++++++++ 7 files changed, 212 insertions(+), 11 deletions(-) create mode 100644 dd-trace-api/src/main/java/datadog/trace/api/config/LlmObsConfig.java diff --git a/dd-java-agent/agent-builder/src/main/java/datadog/trace/agent/tooling/AgentInstaller.java b/dd-java-agent/agent-builder/src/main/java/datadog/trace/agent/tooling/AgentInstaller.java index 6fef809fef7..e56fda40497 100644 --- a/dd-java-agent/agent-builder/src/main/java/datadog/trace/agent/tooling/AgentInstaller.java +++ b/dd-java-agent/agent-builder/src/main/java/datadog/trace/agent/tooling/AgentInstaller.java @@ -311,6 +311,9 @@ public static Set getEnabledSystems() { if (cfg.isUsmEnabled()) { enabledSystems.add(InstrumenterModule.TargetSystem.USM); } + if (cfg.isLlmObsEnabled()) { + enabledSystems.add(InstrumenterModule.TargetSystem.LLMOBS); + } return enabledSystems; } diff --git a/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/InstrumenterModule.java b/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/InstrumenterModule.java index 0f4fa35dd92..42ea6b7207f 100644 --- a/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/InstrumenterModule.java +++ b/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/InstrumenterModule.java @@ -49,7 +49,8 @@ public enum TargetSystem { APPSEC, IAST, CIVISIBILITY, - USM + USM, + LLMOBS, } private static final Logger log = LoggerFactory.getLogger(InstrumenterModule.class); diff --git a/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java b/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java index ceb390acfe9..fe360b81dd8 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java @@ -137,6 +137,9 @@ public final class ConfigDefaults { static final boolean DEFAULT_IAST_STACK_TRACE_ENABLED = true; + static final boolean DEFAULT_LLM_OBS_ENABLED = false; + static final boolean DEFAULT_LLM_OBS_AGENTLESS_ENABLED = false; + static final boolean DEFAULT_USM_ENABLED = false; static final boolean DEFAULT_CIVISIBILITY_ENABLED = false; diff --git a/dd-trace-api/src/main/java/datadog/trace/api/config/LlmObsConfig.java b/dd-trace-api/src/main/java/datadog/trace/api/config/LlmObsConfig.java new file mode 100644 index 00000000000..111f8ebc7af --- /dev/null +++ b/dd-trace-api/src/main/java/datadog/trace/api/config/LlmObsConfig.java @@ -0,0 +1,16 @@ +package datadog.trace.api.config; + +/** + * Constant with names of configuration options for LLM Observability. (EXPERIMENTAL AND SUBJECT TO + * CHANGE) + */ +public final class LlmObsConfig { + + public static final String LLMOBS_ENABLED = "llmobs.enabled"; + + public static final String LLMOBS_ML_APP = "llmobs.ml.app"; + + public static final String LLMOBS_AGENTLESS_ENABLED = "llmobs.agentless.enabled"; + + private LlmObsConfig() {} +} diff --git a/internal-api/src/main/java/datadog/trace/api/Config.java b/internal-api/src/main/java/datadog/trace/api/Config.java index 60a64f9e3d9..02b78b68031 100644 --- a/internal-api/src/main/java/datadog/trace/api/Config.java +++ b/internal-api/src/main/java/datadog/trace/api/Config.java @@ -13,6 +13,7 @@ import static datadog.trace.api.config.GeneralConfig.SERVICE_NAME; import static datadog.trace.api.config.IastConfig.*; import static datadog.trace.api.config.JmxFetchConfig.*; +import static datadog.trace.api.config.LlmObsConfig.*; import static datadog.trace.api.config.ProfilingConfig.*; import static datadog.trace.api.config.RemoteConfigConfig.*; import static datadog.trace.api.config.TraceInstrumentationConfig.*; @@ -309,6 +310,9 @@ public static String getHostName() { private final boolean iastExperimentalPropagationEnabled; private final String iastSecurityControlsConfiguration; + private final boolean llmObsAgentlessEnabled; + private final String llmObsMlApp; + private final boolean ciVisibilityTraceSanitationEnabled; private final boolean ciVisibilityAgentlessEnabled; private final String ciVisibilityAgentlessUrl; @@ -1336,6 +1340,10 @@ PROFILING_DATADOG_PROFILER_ENABLED, isDatadogProfilerSafeInCurrentEnvironment()) iastSecurityControlsConfiguration = configProvider.getString(IAST_SECURITY_CONTROLS_CONFIGURATION, null); + llmObsAgentlessEnabled = + configProvider.getBoolean(LLMOBS_AGENTLESS_ENABLED, DEFAULT_LLM_OBS_AGENTLESS_ENABLED); + llmObsMlApp = configProvider.getString(LLMOBS_ML_APP); + ciVisibilityTraceSanitationEnabled = configProvider.getBoolean(CIVISIBILITY_TRACE_SANITATION_ENABLED, true); @@ -1766,21 +1774,46 @@ PROFILING_DATADOG_PROFILER_ENABLED, isDatadogProfilerSafeInCurrentEnvironment()) this.traceFlushIntervalSeconds = configProvider.getFloat( TracerConfig.TRACE_FLUSH_INTERVAL, ConfigDefaults.DEFAULT_TRACE_FLUSH_INTERVAL); - if (profilingAgentless && apiKey == null) { - log.warn( - "Agentless profiling activated but no api key provided. Profile uploading will likely fail"); - } this.tracePostProcessingTimeout = configProvider.getLong( TRACE_POST_PROCESSING_TIMEOUT, ConfigDefaults.DEFAULT_TRACE_POST_PROCESSING_TIMEOUT); - if (isCiVisibilityEnabled() - && ciVisibilityAgentlessEnabled - && (apiKey == null || apiKey.isEmpty())) { - throw new FatalAgentMisconfigurationError( - "Attempt to start in Agentless mode without API key. " - + "Please ensure that either an API key is configured, or the tracer is set up to work with the Agent"); + if (isLlmObsEnabled()) { + log.debug("Attempting to enable LLM Observability"); + if (llmObsMlApp == null || llmObsMlApp.isEmpty()) { + throw new IllegalArgumentException( + "Attempt to enable LLM Observability without ML app defined." + + "Please ensure that the name of the ML app is provided through properties or env variable"); + } + + log.debug( + "LLM Observability enabled for ML app {}, agentless mode {}", + llmObsMlApp, + llmObsAgentlessEnabled); + } + + // if API key is not provided, check if any products are using agentless mode and require it + if (apiKey == null || apiKey.isEmpty()) { + // CI Visibility + if (isCiVisibilityEnabled() && ciVisibilityAgentlessEnabled) { + throw new FatalAgentMisconfigurationError( + "Attempt to start in CI Visibility in Agentless mode without API key. " + + "Please ensure that either an API key is configured, or the tracer is set up to work with the Agent"); + } + + // Profiling + if (profilingAgentless) { + log.warn( + "Agentless profiling activated but no api key provided. Profile uploading will likely fail"); + } + + // LLM Observability + if (isLlmObsEnabled() && llmObsAgentlessEnabled) { + throw new FatalAgentMisconfigurationError( + "Attempt to start LLM Observability in Agentless mode without API key. " + + "Please ensure that either an API key is configured, or the tracer is set up to work with the Agent"); + } } this.telemetryDebugRequestsEnabled = @@ -2640,6 +2673,18 @@ public String getIastSecurityControlsConfiguration() { return iastSecurityControlsConfiguration; } + public boolean isLlmObsEnabled() { + return instrumenterConfig.isLlmObsEnabled(); + } + + public boolean isLlmObsAgentlessEnabled() { + return llmObsAgentlessEnabled; + } + + public String getLlmObsMlApp() { + return llmObsMlApp; + } + public boolean isCiVisibilityEnabled() { return instrumenterConfig.isCiVisibilityEnabled(); } diff --git a/internal-api/src/main/java/datadog/trace/api/InstrumenterConfig.java b/internal-api/src/main/java/datadog/trace/api/InstrumenterConfig.java index 8585ddcf54f..2732e02746b 100644 --- a/internal-api/src/main/java/datadog/trace/api/InstrumenterConfig.java +++ b/internal-api/src/main/java/datadog/trace/api/InstrumenterConfig.java @@ -5,6 +5,7 @@ import static datadog.trace.api.ConfigDefaults.DEFAULT_CODE_ORIGIN_FOR_SPANS_ENABLED; import static datadog.trace.api.ConfigDefaults.DEFAULT_IAST_ENABLED; import static datadog.trace.api.ConfigDefaults.DEFAULT_INTEGRATIONS_ENABLED; +import static datadog.trace.api.ConfigDefaults.DEFAULT_LLM_OBS_ENABLED; import static datadog.trace.api.ConfigDefaults.DEFAULT_MEASURE_METHODS; import static datadog.trace.api.ConfigDefaults.DEFAULT_RESOLVER_RESET_INTERVAL; import static datadog.trace.api.ConfigDefaults.DEFAULT_RUNTIME_CONTEXT_FIELD_INJECTION; @@ -26,6 +27,7 @@ import static datadog.trace.api.config.GeneralConfig.TRACE_TRIAGE; import static datadog.trace.api.config.GeneralConfig.TRIAGE_REPORT_TRIGGER; import static datadog.trace.api.config.IastConfig.IAST_ENABLED; +import static datadog.trace.api.config.LlmObsConfig.LLMOBS_ENABLED; import static datadog.trace.api.config.ProfilingConfig.PROFILING_DIRECT_ALLOCATION_ENABLED; import static datadog.trace.api.config.ProfilingConfig.PROFILING_DIRECT_ALLOCATION_ENABLED_DEFAULT; import static datadog.trace.api.config.ProfilingConfig.PROFILING_ENABLED; @@ -111,6 +113,7 @@ public class InstrumenterConfig { private final boolean iastFullyDisabled; private final boolean usmEnabled; private final boolean telemetryEnabled; + private final boolean llmObsEnabled; private final String traceExtensionsPath; @@ -199,6 +202,7 @@ private InstrumenterConfig() { iastFullyDisabled = iastEnabled != null && !iastEnabled; usmEnabled = configProvider.getBoolean(USM_ENABLED, DEFAULT_USM_ENABLED); telemetryEnabled = configProvider.getBoolean(TELEMETRY_ENABLED, DEFAULT_TELEMETRY_ENABLED); + llmObsEnabled = configProvider.getBoolean(LLMOBS_ENABLED, DEFAULT_LLM_OBS_ENABLED); } else { // disable these features in native-image ciVisibilityEnabled = false; @@ -207,6 +211,7 @@ private InstrumenterConfig() { iastFullyDisabled = true; telemetryEnabled = false; usmEnabled = false; + llmObsEnabled = false; } traceExtensionsPath = configProvider.getString(TRACE_EXTENSIONS_PATH); @@ -355,6 +360,10 @@ public boolean isIastFullyDisabled() { return iastFullyDisabled; } + public boolean isLlmObsEnabled() { + return llmObsEnabled; + } + public boolean isUsmEnabled() { return usmEnabled; } diff --git a/internal-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy b/internal-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy index 8abf3daf450..ae07c2d5cd6 100644 --- a/internal-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy +++ b/internal-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy @@ -63,6 +63,9 @@ import static datadog.trace.api.config.JmxFetchConfig.JMX_FETCH_REFRESH_BEANS_PE import static datadog.trace.api.config.JmxFetchConfig.JMX_FETCH_STATSD_HOST import static datadog.trace.api.config.JmxFetchConfig.JMX_FETCH_STATSD_PORT import static datadog.trace.api.config.JmxFetchConfig.JMX_TAGS +import static datadog.trace.api.config.LlmObsConfig.LLMOBS_AGENTLESS_ENABLED +import static datadog.trace.api.config.LlmObsConfig.LLMOBS_ML_APP +import static datadog.trace.api.config.LlmObsConfig.LLMOBS_ENABLED import static datadog.trace.api.config.ProfilingConfig.PROFILING_AGENTLESS import static datadog.trace.api.config.ProfilingConfig.PROFILING_API_KEY_FILE_OLD import static datadog.trace.api.config.ProfilingConfig.PROFILING_API_KEY_FILE_VERY_OLD @@ -163,6 +166,9 @@ class ConfigTest extends DDSpecification { private static final DD_PROFILING_TAGS_ENV = "DD_PROFILING_TAGS" private static final DD_PROFILING_PROXY_PASSWORD_ENV = "DD_PROFILING_PROXY_PASSWORD" private static final DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH = "DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH" + private static final DD_LLMOBS_ENABLED_ENV = "DD_LLMOBS_ENABLED" + private static final DD_LLMOBS_ML_APP_ENV = "DD_LLMOBS_ML_APP" + private static final DD_LLMOBS_AGENTLESS_ENABLED_ENV = "DD_LLMOBS_AGENTLESS_ENABLED" def "specify overrides via properties"() { setup: @@ -2208,6 +2214,124 @@ class ConfigTest extends DDSpecification { !hostname.trim().isEmpty() } + def "config instantiation should fail if llm obs is enabled via sys prop and ml app is not set"() { + setup: + Properties properties = new Properties() + properties.setProperty(LLMOBS_ENABLED, "true") + + when: + new Config(ConfigProvider.withPropertiesOverride(properties)) + + then: + thrown IllegalArgumentException + } + + def "config instantiation should fail if llm obs is enabled via env var and ml app is not set"() { + setup: + environmentVariables.set(DD_LLMOBS_ENABLED_ENV, "true") + + when: + new Config() + + then: + thrown IllegalArgumentException + } + + + def "config instantiation should NOT fail if llm obs is enabled (agentless disabled) via sys prop and ml app is set"() { + setup: + Properties properties = new Properties() + properties.setProperty(LLMOBS_ENABLED, "true") + properties.setProperty(LLMOBS_AGENTLESS_ENABLED, "false") + properties.setProperty(LLMOBS_ML_APP, "test-ml-app") + + when: + def config = new Config(ConfigProvider.withPropertiesOverride(properties)) + + then: + noExceptionThrown() + config.isLlmObsEnabled() + !config.isLlmObsAgentlessEnabled() + config.llmObsMlApp == "test-ml-app" + } + + def "config instantiation should NOT fail if llm obs is enabled (agentless disabled) via env var and ml app is set"() { + setup: + environmentVariables.set(DD_LLMOBS_ENABLED_ENV, "true") + environmentVariables.set(DD_LLMOBS_ML_APP_ENV, "test-ml-app") + + when: + def config = new Config() + + then: + noExceptionThrown() + config.isLlmObsEnabled() + !config.isLlmObsAgentlessEnabled() + config.llmObsMlApp == "test-ml-app" + } + + def "config instantiation should fail if llm obs is in agentless mode via sys prop and API key is not set"() { + setup: + Properties properties = new Properties() + properties.setProperty(LLMOBS_ENABLED, "true") + properties.setProperty(LLMOBS_AGENTLESS_ENABLED, "true") + properties.setProperty(LLMOBS_ML_APP, "test-ml-app") + + when: + new Config(ConfigProvider.withPropertiesOverride(properties)) + + then: + thrown FatalAgentMisconfigurationError + } + + def "config instantiation should fail if llm obs is in agentless mode via env var and API key is not set"() { + setup: + environmentVariables.set(DD_LLMOBS_ENABLED_ENV, "true") + environmentVariables.set(DD_LLMOBS_ML_APP_ENV, "a") + environmentVariables.set(DD_LLMOBS_AGENTLESS_ENABLED_ENV, "true") + + when: + new Config() + + then: + thrown FatalAgentMisconfigurationError + } + + def "config instantiation should NOT fail if llm obs is enabled (agentless enabled) and API key & ml app are set via sys prop"() { + setup: + Properties properties = new Properties() + properties.setProperty(LLMOBS_ENABLED, "true") + properties.setProperty(LLMOBS_AGENTLESS_ENABLED, "true") + properties.setProperty(LLMOBS_ML_APP, "test-ml-app") + properties.setProperty(API_KEY, "123456789") + + when: + def config = new Config(ConfigProvider.withPropertiesOverride(properties)) + + then: + noExceptionThrown() + config.isLlmObsEnabled() + config.isLlmObsAgentlessEnabled() + config.llmObsMlApp == "test-ml-app" + } + + def "config instantiation should NOT fail if llm obs is enabled (agentless enabled) and API key & ml app are set via env var"() { + setup: + environmentVariables.set(DD_LLMOBS_ENABLED_ENV, "true") + environmentVariables.set(DD_LLMOBS_ML_APP_ENV, "a") + environmentVariables.set(DD_LLMOBS_AGENTLESS_ENABLED_ENV, "true") + environmentVariables.set(DD_API_KEY_ENV, "8663294466") + + when: + def config = new Config() + + then: + noExceptionThrown() + config.isLlmObsEnabled() + config.isLlmObsAgentlessEnabled() + config.llmObsMlApp == "a" + } + def "config instantiation should fail if CI visibility agentless mode is enabled and API key is not set"() { setup: Properties properties = new Properties()