Metering API Implementation

TRACING.md

@@ -158,6 +158,6 @@ explicitly opening a scope via the `Tracer`.
 [otel]: https://opentelemetry.io/
 [thread-context]: ./server/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java
 [w3c]: https://www.w3.org/TR/trace-context/
-[tracing]: ./server/src/main/java/org/elasticsearch/tracing
+[tracing]: ./server/src/main/java/org/elasticsearch/telemetry
 [agent-config]: https://www.elastic.co/guide/en/apm/agent/java/master/configuration.html
 [agent]: https://www.elastic.co/guide/en/apm/agent/java/current/index.html

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/APMJvmOptions.java

@@ -50,10 +50,12 @@ class APMJvmOptions {
         // by the agent. Don't disable writing to a log file, as the agent will then
         // require extra Security Manager permissions when it tries to do something
         // else, and it's just painful.
-        "log_file", "_AGENT_HOME_/../../logs/apm.log",
-
+        "log_file", "/Users/przemyslawgomulka/workspace/pgomulka/apm.log",
+        "log_level", "debug",
         // ES does not use auto-instrumentation.
-        "instrument", "false"
+        "instrument", "false",
+        "experimental", "true",
+        "enable_experimental_instrumentations", "true"
         );
 
     /**
@@ -82,7 +84,7 @@ class APMJvmOptions {
         // is doing, leave this value alone.
         "log_level", "error",
         "application_packages", "org.elasticsearch,org.apache.lucene",
-        "metrics_interval", "120s",
+        "metrics_interval", "5s",
         "breakdown_metrics", "false",
         "central_config", "false"
         );
@@ -316,9 +318,7 @@ static Path findAgentJar(String installDir) throws IOException, UserException {
         }
 
         try (var apmStream = Files.list(apmModule)) {
-            final List<Path> paths = apmStream.filter(
-                path -> path.getFileName().toString().matches("elastic-apm-agent-\\d+\\.\\d+\\.\\d+\\.jar")
-            ).toList();
+            final List<Path> paths = apmStream.filter(path -> path.getFileName().toString().matches("elastic-apm-agent-.*.jar")).toList();
 
             if (paths.size() > 1) {
                 throw new UserException(

gradle/verification-metadata.xml

@@ -74,6 +74,11 @@
             <sha256 value="3be90268557494f88fe252caf6358d4107aa9349852b8704b1e0ff4831cb74d6" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="co.elastic.apm" name="elastic-apm-agent" version="1.37.0">
+         <artifact name="elastic-apm-agent-1.37.0.jar">
+            <sha256 value="10496390d1dcc7c558ae63e115beff9477157e3d219fce686ef5ca59c744dfea" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="co.elastic.logging" name="ecs-logging-core" version="1.2.0">
          <artifact name="ecs-logging-core-1.2.0.jar">
             <sha256 value="0d6318af29848ea588b1c244834e3e762315c68de2bcfc9f1adf3f0633d22d37" origin="Generated by Gradle"/>

modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java

@@ -131,7 +131,7 @@
 import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.synonyms.SynonymsManagementAPIService;
-import org.elasticsearch.telemetry.tracing.Tracer;
+import org.elasticsearch.telemetry.TelemetryProvider;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.watcher.ResourceWatcherService;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
@@ -168,7 +168,7 @@ public Collection<Object> createComponents(
         NamedWriteableRegistry namedWriteableRegistry,
         IndexNameExpressionResolver expressionResolver,
         Supplier<RepositoriesService> repositoriesServiceSupplier,
-        Tracer tracer,
+        TelemetryProvider telemetryProvider,
         AllocationService allocationService,
         IndicesService indicesService
     ) {

modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/PredicateTokenScriptFilterTests.java

@@ -28,7 +28,7 @@
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.telemetry.tracing.Tracer;
+import org.elasticsearch.telemetry.TelemetryProvider;
 import org.elasticsearch.test.ESTokenStreamTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -68,7 +68,22 @@ public <FactoryType> FactoryType compile(Script script, ScriptContext<FactoryTyp
         };
         Client client = new MockClient(Settings.EMPTY, null);
         CommonAnalysisPlugin plugin = new CommonAnalysisPlugin();
-        plugin.createComponents(client, null, null, null, scriptService, null, null, null, null, null, null, Tracer.NOOP, null, null);
+        plugin.createComponents(
+            client,
+            null,
+            null,
+            null,
+            scriptService,
+            null,
+            null,
+            null,
+            null,
+            null,
+            null,
+            TelemetryProvider.NOOP,
+            null,
+            null
+        );
         AnalysisModule module = new AnalysisModule(
             TestEnvironment.newEnvironment(settings),
             Collections.singletonList(plugin),

modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/ScriptedConditionTokenFilterTests.java

@@ -28,7 +28,7 @@
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.telemetry.tracing.Tracer;
+import org.elasticsearch.telemetry.TelemetryProvider;
 import org.elasticsearch.test.ESTokenStreamTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -68,7 +68,22 @@ public <FactoryType> FactoryType compile(Script script, ScriptContext<FactoryTyp
         };
         Client client = new MockClient(Settings.EMPTY, null);
         CommonAnalysisPlugin plugin = new CommonAnalysisPlugin();
-        plugin.createComponents(client, null, null, null, scriptService, null, null, null, null, null, null, Tracer.NOOP, null, null);
+        plugin.createComponents(
+            client,
+            null,
+            null,
+            null,
+            scriptService,
+            null,
+            null,
+            null,
+            null,
+            null,
+            null,
+            TelemetryProvider.NOOP,
+            null,
+            null
+        );
         AnalysisModule module = new AnalysisModule(
             TestEnvironment.newEnvironment(settings),
             Collections.singletonList(plugin),

modules/apm/build.gradle

@@ -14,11 +14,17 @@ esplugin {
 
 def otelVersion = '1.17.0'
 
+repositories {
+  maven {
+    name "sonatype-nexus-snapshots"
+    url "https://oss.sonatype.org/content/repositories/snapshots"
+  }
+}
 dependencies {
   implementation "io.opentelemetry:opentelemetry-api:${otelVersion}"
   implementation "io.opentelemetry:opentelemetry-context:${otelVersion}"
   implementation "io.opentelemetry:opentelemetry-semconv:${otelVersion}-alpha"
-  runtimeOnly    "co.elastic.apm:elastic-apm-agent:1.36.0"
+  implementation    "co.elastic.apm:elastic-apm-agent:1.42.1-SNAPSHOT"
 }
 
 tasks.named("dependencyLicenses").configure {

modules/apm/src/main/java/org/elasticsearch/telemetry/apm/APM.java

@@ -21,12 +21,12 @@
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.plugins.NetworkPlugin;
 import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.plugins.TracerPlugin;
+import org.elasticsearch.plugins.TelemetryPlugin;
 import org.elasticsearch.repositories.RepositoriesService;
 import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.telemetry.TelemetryProvider;
 import org.elasticsearch.telemetry.apm.settings.APMAgentSettings;
 import org.elasticsearch.telemetry.apm.tracing.APMTracer;
-import org.elasticsearch.telemetry.tracing.Tracer;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.watcher.ResourceWatcherService;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
@@ -55,19 +55,19 @@
  * be passed via system properties to the Java agent, which periodically checks for changes
  * and applies the new settings values, provided those settings can be dynamically updated.
  */
-public class APM extends Plugin implements NetworkPlugin, TracerPlugin {
-    private final SetOnce<APMTracer> tracer = new SetOnce<>();
+public class APM extends Plugin implements NetworkPlugin, TelemetryPlugin {
+    private final SetOnce<APMTelemetryProvider> telemetryProvider = new SetOnce<>();
     private final Settings settings;
 
     public APM(Settings settings) {
         this.settings = settings;
     }
 
     @Override
-    public Tracer getTracer(Settings settings) {
-        final APMTracer apmTracer = new APMTracer(settings);
-        tracer.set(apmTracer);
-        return apmTracer;
+    public TelemetryProvider getTelemetryProvider(Settings settings) {
+        final APMTelemetryProvider apmTelemetryProvider = new APMTelemetryProvider(settings);
+        telemetryProvider.set(apmTelemetryProvider);
+        return apmTelemetryProvider;
     }
 
     @Override
@@ -83,20 +83,22 @@ public Collection<Object> createComponents(
         NamedWriteableRegistry namedWriteableRegistry,
         IndexNameExpressionResolver indexNameExpressionResolver,
         Supplier<RepositoriesService> repositoriesServiceSupplier,
-        Tracer unused,
+        TelemetryProvider unused,
         AllocationService allocationService,
         IndicesService indicesService
     ) {
-        final APMTracer apmTracer = tracer.get();
+        final APMTracer apmTracer = telemetryProvider.get().getTracer();
 
         apmTracer.setClusterName(clusterService.getClusterName().value());
         apmTracer.setNodeName(clusterService.getNodeName());
 
         final APMAgentSettings apmAgentSettings = new APMAgentSettings();
         apmAgentSettings.syncAgentSystemProperties(settings);
-        apmAgentSettings.addClusterSettingsListeners(clusterService, apmTracer);
+        apmAgentSettings.addClusterSettingsListeners(clusterService, telemetryProvider.get());
 
-        return List.of(apmTracer);
+        final APMMetric apmMetric = telemetryProvider.get().getMetric();
+
+        return List.of(apmTracer, apmMetric);
     }
 
     @Override