PHOENIX-5215 opentelemetry changes replacing htrace

bin/phoenix_utils.py

@@ -78,19 +78,21 @@ def findClasspath(command_name):
 
 def setPath():
     PHOENIX_CLIENT_EMBEDDED_JAR_PATTERN = "phoenix-client-embedded-hbase-*[!s].jar"
-    PHOENIX_TRACESERVER_JAR_PATTERN = "phoenix-tracing-webapp-*-runnable.jar"
+    PHOENIX_TRACESERVER_JAR_PATTERN = "phoenix-tracing-webapp-*[!s].jar"
     PHOENIX_TESTS_JAR_PATTERN = "phoenix-core-*-tests*.jar"
     PHOENIX_PHERF_JAR_PATTERN = "phoenix-pherf-*[!s].jar"
     SLF4J_BACKEND_JAR_PATTERN = "log4j-slf4j*.jar"
     LOGGING_JAR_PATTERN = "log4j-core*.jar"
     LOGGING_JAR_PATTERN2 = "log4j-api*.jar"
     LOGGING_JAR_PATTERN3 = "log4j-1.2-api*.jar"
     SQLLINE_WITH_DEPS_PATTERN = "sqlline-*-jar-with-dependencies.jar"
+    OPENTELEMTRY_JAVAAGENT_JAR_PATTERN = "opentelemetry-javaagent-*[!s]-all.jar"
 
 
     OVERRIDE_SLF4J_BACKEND = "OVERRIDE_SLF4J_BACKEND_JAR_LOCATION"
     OVERRIDE_LOGGING = "OVERRIDE_LOGGING_JAR_LOCATION"
     OVERRIDE_SQLLINE = "OVERRIDE_SQLLINE_JAR_LOCATION"
+    OVERRIDE_OPENTELEMETRY_JAVAAGENT = "OVERRIDE_OPENTELEMETRY_JAVAAGENT_LOCATION"
 
     # Backward support old env variable PHOENIX_LIB_DIR replaced by PHOENIX_CLASS_PATH
     global phoenix_class_path
@@ -161,7 +163,7 @@ def setPath():
     global phoenix_traceserver_jar
     phoenix_traceserver_jar = find(PHOENIX_TRACESERVER_JAR_PATTERN, os.path.join(current_dir, "..", "phoenix-tracing-webapp", "target", "*"))
     if phoenix_traceserver_jar == "":
-        phoenix_traceserver_jar = findFileInPathWithoutRecursion(PHOENIX_TRACESERVER_JAR_PATTERN, os.path.join(current_dir, "..", "lib"))
+        phoenix_traceserver_jar = findFileInPathWithoutRecursion(PHOENIX_TRACESERVER_JAR_PATTERN, os.path.join(current_dir, "..", "trace"))
     if phoenix_traceserver_jar == "":
         phoenix_traceserver_jar = findFileInPathWithoutRecursion(PHOENIX_TRACESERVER_JAR_PATTERN, os.path.join(current_dir, ".."))
 
@@ -189,6 +191,14 @@ def setPath():
         logging_jar += ":"+findFileInPathWithoutRecursion(LOGGING_JAR_PATTERN2, os.path.join(current_dir, "..","lib"))
         logging_jar += ":"+findFileInPathWithoutRecursion(LOGGING_JAR_PATTERN3, os.path.join(current_dir, "..","lib"))
 
+    global phoenix_tracing_jar
+    phoenix_tracing_jar = phoenix_traceserver_jar
+
+    global opentelemetry_javaagent_jar
+    opentelemetry_javaagent_jar = os.environ.get(OVERRIDE_OPENTELEMETRY_JAVAAGENT)
+    if opentelemetry_javaagent_jar is None or opentelemetry_javaagent_jar == "":
+        opentelemetry_javaagent_jar = findFileInPathWithoutRecursion(OPENTELEMTRY_JAVAAGENT_JAR_PATTERN, os.path.join(current_dir, "..","trace"))
+
     return ""
 
 def shell_quote(args):
@@ -224,3 +234,6 @@ def common_sqlline_args(parser):
     print("sqlline_with_deps_jar:", sqlline_with_deps_jar)
     print("slf4j_backend_jar:", slf4j_backend_jar)
     print("logging_jar:", logging_jar)
+    print("phoenix_tracing_jar:", phoenix_tracing_jar)
+    print("phoenix_traceserver_jar:", phoenix_traceserver_jar)
+    print("opentelemetry_javaagent_jar", opentelemetry_javaagent_jar)

bin/sqlline.py

@@ -120,11 +120,14 @@ def kill_child():
     phoenix_utils.slf4j_backend_jar + os.pathsep + \
     phoenix_utils.logging_jar + os.pathsep + \
     phoenix_utils.phoenix_client_embedded_jar + \
+    # os.pathsep + phoenix_utils.phoenix_tracing_jar + \
+    #'" -Dotel.traces.exporter=zipkin -Dotel.metrics.exporter=none -Dotel.resource.attributes=service.name=phoenix' + \
+    #'-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=9900' + \
+    #' -javaagent:' + phoenix_utils.opentelemetry_javaagent_jar + \
     '" -Dlog4j2.configurationFile=file:' + os.path.join(phoenix_utils.current_dir, "log4j2.properties") + \
     disable_jna + \
     " sqlline.SqlLine -d org.apache.phoenix.jdbc.PhoenixDriver" + \
     " -u jdbc:phoenix:" + phoenix_utils.shell_quote([zookeeper]) + \
     " -n none -p none --color=" + colorSetting + " --fastConnect=" + tryDecode(args.fastconnect) + \
     " --verbose=" + tryDecode(args.verbose) + " --incremental=false --isolation=TRANSACTION_READ_COMMITTED " + sqlfile
-
 os.execl("/bin/sh", "/bin/sh", "-c", java_cmd)

bin/traceserver.py

@@ -123,7 +123,6 @@
     phoenix_utils.phoenix_traceserver_jar + os.pathsep + phoenix_utils.slf4j_backend_jar + os.pathsep + \
     phoenix_utils.logging_jar + os.pathsep + \
     phoenix_utils.phoenix_client_embedded_jar + os.pathsep + phoenix_utils.phoenix_queryserver_jar + \
-
     " -Dproc_phoenixtraceserver" + \
     " -Dlog4j2.configurationFile=file:" + os.path.join(phoenix_utils.current_dir, "log4j2.properties") + \
     " -Dpsql.root.logger=%(root_logger)s" + \
@@ -133,6 +132,9 @@
     " org.apache.phoenix.tracingwebapp.http.Main " + args
 
 if command == 'start':
+    print("java_cmd:", java_cmd)
+    cmd = java_cmd % {'java': java, 'root_logger': 'INFO,DRFA', 'log_dir': phoenix_log_dir, 'log_file': phoenix_log_file}
+    print("cmd:", cmd)
     if not daemon_supported:
         sys.stderr.write("daemon mode not supported on this platform{}".format(os.linesep))
         sys.exit(-1)

phoenix-assembly/src/build/components/all-common-jars.xml

@@ -47,5 +47,13 @@
         <include>phoenix-pherf.jar</include>
       </includes>
     </fileSet>
+    <fileSet>
+      <directory>${project.basedir}/../phoenix-tracing-webapp/target</directory>
+      <outputDirectory>/trace</outputDirectory>
+      <includes>
+        <include>phoenix-tracing-webapp-${project.version}.jar</include>
+        <include>phoenix-tracing-webapp.jar</include>
+      </includes>
+    </fileSet>
   </fileSets>
 </component>

phoenix-assembly/src/build/package-to-tar-all.xml

@@ -31,6 +31,16 @@
   <includeBaseDirectory>true</includeBaseDirectory>
 
   <!-- Components that we don't want in jars that are used with other libraries, but we want for a standalone client -->
+  <dependencySets>
+  <dependencySet>
+    <unpack>false</unpack>
+    <outputDirectory>/trace</outputDirectory>
+    <includes>
+      <include>io.opentelemetry.javaagent:opentelemetry-javaagent</include>
+    </includes>
+  </dependencySet>
+  </dependencySets>
+
   <componentDescriptors>
     <componentDescriptor>src/build/components/all-common-jars.xml</componentDescriptor>
     <componentDescriptor>src/build/components/all-common-files.xml</componentDescriptor>

phoenix-client-parent/phoenix-client-embedded/pom.xml

@@ -54,6 +54,7 @@
                 <excludes>
                   <exclude>org.apache.phoenix:phoenix-client</exclude>
                   <exclude>xom:xom</exclude>
+                  <exclude>io.opentelemetry.*</exclude>
                 </excludes>
               </artifactSet>
             </configuration>

phoenix-client-parent/pom.xml

@@ -193,6 +193,7 @@
                   <exclude>io/skip/checksum/errors</exclude>
                   <exclude>io/sort/*</exclude>
                   <exclude>io/serializations</exclude>
+                  <exclude>io/opentelemetry/**</exclude>
                 </excludes>
               </relocation>
               <!-- JSRs that haven't made it to inclusion in J2SE -->

phoenix-core/pom.xml

@@ -457,6 +457,16 @@
       <type>test-jar</type>
     </dependency>
 
+    <!-- Tracing Dependencies -->
+    <dependency>
+      <groupId>io.opentelemetry</groupId>
+      <artifactId>opentelemetry-api</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>io.opentelemetry</groupId>
+      <artifactId>opentelemetry-context</artifactId>
+    </dependency>
+
     <!-- Other dependencies -->
     <dependency>
       <groupId>org.antlr</groupId>
@@ -502,10 +512,6 @@
       <groupId>com.google.protobuf</groupId>
       <artifactId>protobuf-java</artifactId>
     </dependency>
-    <dependency>
-      <groupId>org.apache.htrace</groupId>
-      <artifactId>htrace-core</artifactId>
-    </dependency>
     <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-api</artifactId>
@@ -619,5 +625,24 @@
       <artifactId>hamcrest-core</artifactId>
       <scope>test</scope>
     </dependency>
+    <!-- Tracing test dependencies -->
+    <dependency>
+      <groupId>io.opentelemetry</groupId>
+      <artifactId>opentelemetry-sdk</artifactId>
+      <version>${opentelemetry.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>io.opentelemetry</groupId>
+      <artifactId>opentelemetry-sdk-trace</artifactId>
+      <version>${opentelemetry.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>io.opentelemetry</groupId>
+      <artifactId>opentelemetry-sdk-testing</artifactId>
+      <version>${opentelemetry.version}</version>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 </project>

phoenix-core/src/it/java/org/apache/phoenix/trace/BaseTracingTestIT.java

@@ -29,13 +29,10 @@
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 
-import org.apache.htrace.Span;
-import org.apache.htrace.Trace;
-import org.apache.htrace.impl.MilliSpan;
+import io.opentelemetry.api.trace.Span;
 import org.apache.phoenix.end2end.ParallelStatsDisabledIT;
 import org.apache.phoenix.jdbc.DelegateConnection;
 import org.apache.phoenix.trace.util.Tracing;
-import org.apache.phoenix.trace.util.Tracing.Frequency;
 import org.apache.phoenix.util.PhoenixRuntime;
 import org.apache.phoenix.util.PropertiesUtil;
 import org.junit.After;
@@ -56,22 +53,18 @@ public abstract class BaseTracingTestIT extends ParallelStatsDisabledIT {
     protected int defaultTracingThreadPoolForTest = 1;
     protected int defaultTracingBatchSizeForTest = 1;
     protected String tracingTableName;
-    protected TraceSpanReceiver traceSpanReceiver = null;
     protected TestTraceWriter testTraceWriter = null;
 
     @Before
     public void setup() {
         tracingTableName = "TRACING_" + generateUniqueName();
-        traceSpanReceiver = new TraceSpanReceiver();
-        Trace.addReceiver(traceSpanReceiver);
         testTraceWriter =
                 new TestTraceWriter(tracingTableName, defaultTracingThreadPoolForTest,
                         defaultTracingBatchSizeForTest);
     }
 
     @After
     public void cleanUp() {
-        Trace.removeReceiver(traceSpanReceiver);
         if (testTraceWriter != null) testTraceWriter.stop();
     }
 
@@ -81,7 +74,7 @@ public static Connection getConnectionWithoutTracing() throws SQLException {
     }
 
     public static Connection getConnectionWithoutTracing(Properties props) throws SQLException {
-        Connection conn = getConnectionWithTracingFrequency(props, Frequency.NEVER);
+        Connection conn = getConnectionWithTracingFrequency(props);
         return conn;
     }
 
@@ -98,26 +91,21 @@ public static Connection getTracingConnection(Map<String, String> customAnnotati
         if (tenantId != null) {
             props.put(PhoenixRuntime.TENANT_ID_ATTRIB, tenantId);
         }
-        return getConnectionWithTracingFrequency(props, Tracing.Frequency.ALWAYS);
+        return getConnectionWithTracingFrequency(props);
     }
 
-    public static Connection getConnectionWithTracingFrequency(Properties props,
-            Tracing.Frequency frequency) throws SQLException {
-        Tracing.setSampling(props, frequency);
+    public static Connection getConnectionWithTracingFrequency(Properties props) throws SQLException {
         return DriverManager.getConnection(getUrl(), props);
     }
 
     protected Span createNewSpan(long traceid, long parentid, long spanid, String description,
             long startTime, long endTime, String processid, String... tags) {
 
-        Span span =
-                new MilliSpan.Builder().description(description).traceId(traceid)
-                        .parents(new long[] { parentid }).spanId(spanid).processId(processid)
-                        .begin(startTime).end(endTime).build();
+        Span span = TraceUtil.getGlobalTracer().spanBuilder(description).startSpan();
 
         int tagCount = 0;
         for (String annotation : tags) {
-            span.addKVAnnotation((Integer.toString(tagCount++)).getBytes(), annotation.getBytes());
+            span.addEvent((Integer.toString(tagCount++)) + " " + annotation);
         }
         return span;
     }
@@ -159,11 +147,6 @@ protected Connection getConnection(String tableName) {
             }
         }
 
-        @Override
-        protected TraceSpanReceiver getTraceSpanReceiver() {
-            return traceSpanReceiver;
-        }
-
         public void stop() {
             if (executor == null) return;
             try {

phoenix-core/src/it/java/org/apache/phoenix/trace/PhoenixTableMetricsWriterIT.java

@@ -26,12 +26,19 @@
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 
-import org.apache.htrace.Span;
-import org.apache.htrace.Tracer;
+import io.opentelemetry.api.GlobalOpenTelemetry;
+import io.opentelemetry.api.OpenTelemetry;
+import io.opentelemetry.sdk.OpenTelemetrySdk;
+import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter;
+import io.opentelemetry.sdk.trace.SdkTracerProvider;
+import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor;
+import io.opentelemetry.api.trace.Span;
 import org.apache.phoenix.end2end.ParallelStatsDisabledTest;
 import org.apache.phoenix.query.QueryServicesOptions;
 import org.apache.phoenix.trace.TraceReader.SpanInfo;
 import org.apache.phoenix.trace.TraceReader.TraceHolder;
+import org.junit.After;
+import org.junit.Before;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
@@ -41,6 +48,20 @@
 @Category(ParallelStatsDisabledTest.class)
 public class PhoenixTableMetricsWriterIT extends BaseTracingTestIT {
 
+    private final InMemorySpanExporter testExporter = InMemorySpanExporter.create();
+    private OpenTelemetry openTelemetry = null;
+
+    @Before
+    public void setup() {
+        GlobalOpenTelemetry.resetForTest();
+        SdkTracerProvider sdkTracerProvider = SdkTracerProvider.builder()
+            .addSpanProcessor(SimpleSpanProcessor.create(testExporter))
+            .build();
+        openTelemetry = OpenTelemetrySdk.builder()
+            .setTracerProvider(sdkTracerProvider)
+            .buildAndRegisterGlobal();
+    }
+
     /**
      * IT should create the target table if it hasn't been created yet, but not fail if the table
      * has already been created
@@ -87,9 +108,6 @@ public void writeMetrics() throws Exception {
         Span span = createNewSpan(traceid, parentid, spanid, description, startTime, endTime,
             processid, annotation);
 
-        Tracer.getInstance().deliver(span);
-        assertTrue("Span never committed to table", latch.await(30, TimeUnit.SECONDS));
-
         // make sure we only get expected stat entry (matcing the trace id), otherwise we could the
         // stats for the update as well
         TraceReader reader = new TraceReader(conn, tracingTableName);
@@ -111,4 +129,12 @@ public void writeMetrics() throws Exception {
         assertEquals("Wrong number of annotations", 1, spanInfo.annotationCount);
     }
 
+    @After
+    public void tearDown(){
+        if(testExporter != null){
+            testExporter.close();
+        }
+        GlobalOpenTelemetry.resetForTest();
+    }
+
 }