Start adding log correlation for Stackdriver Logging.

all/build.gradle

@@ -17,6 +17,7 @@ def subprojects = [
         project(':opencensus-contrib-grpc-util'),
         project(':opencensus-contrib-grpc-metrics'),
         project(':opencensus-contrib-http-util'),
+        project(':opencensus-contrib-log-correlation-stackdriver'),
         project(':opencensus-contrib-monitored-resource-util'),
         project(':opencensus-contrib-zpages'),
         project(':opencensus-exporter-trace-logging'),
@@ -37,6 +38,7 @@ def subprojects_javadoc = [
         project(':opencensus-contrib-grpc-util'),
         project(':opencensus-contrib-grpc-metrics'),
         project(':opencensus-contrib-http-util'),
+        project(':opencensus-contrib-log-correlation-stackdriver'),
         project(':opencensus-contrib-monitored-resource-util'),
         project(':opencensus-contrib-zpages'),
         project(':opencensus-exporter-trace-logging'),

build.gradle

@@ -180,6 +180,7 @@ subprojects {
                 errorprone: "com.google.errorprone:error_prone_annotations:${errorProneVersion}",
                 findbugs_annotations: "com.google.code.findbugs:annotations:${findBugsVersion}",
                 google_auth: "com.google.auth:google-auth-library-credentials:${googleAuthVersion}",
+                google_cloud_logging: "com.google.cloud:google-cloud-logging:1.33.0",
                 google_cloud_trace: "com.google.cloud:google-cloud-trace:${googleCloudVersion}",
                 zipkin_reporter: "io.zipkin.reporter2:zipkin-reporter:${zipkinReporterVersion}",
                 zipkin_urlconnection: "io.zipkin.reporter2:zipkin-sender-urlconnection:${zipkinReporterVersion}",
@@ -373,6 +374,8 @@ subprojects {
                  'opencensus-contrib-grpc-metrics',
                  'opencensus-contrib-grpc-util',
                  'opencensus-contrib-http-util',
+                 // TODO(sebright): Release 'opencensus-contrib-log-correlation-stackdriver' after
+                 // testing it with Logback and java.util.logging.
                  'opencensus-contrib-monitored-resource-util',
                  'opencensus-contrib-zpages',
                  'opencensus-exporter-stats-prometheus',

buildscripts/import-control.xml

@@ -93,6 +93,10 @@ General guidelines on imports:
       <allow pkg="io.opencensus.tags"/>
       <allow pkg="io.opencensus.trace"/>
     </subpackage>
+    <subpackage name="logcorrelation.stackdriver">
+      <allow pkg="com.google.cloud"/>
+      <allow pkg="io.opencensus.trace"/>
+    </subpackage>
     <subpackage name="zpages">
       <allow pkg="com.sun.net.httpserver"/>
       <allow pkg="io.opencensus.contrib.grpc.metrics"/>

contrib/log_correlation/stackdriver/README.md

@@ -0,0 +1,10 @@
+# OpenCensus Stackdriver Log Correlation
+
+This subproject is currently experimental. It provides a Stackdriver Logging
+[LoggingEnhancer](http://googlecloudplatform.github.io/google-cloud-java/google-cloud-clients/apidocs/com/google/cloud/logging/LoggingEnhancer.html)
+that automatically adds tracing data to log entries. The LoggingEnhancer adds the trace ID, which
+allows Stackdriver to display log entries associated with each trace or filter logs based on trace
+ID. It currently also adds the span ID and sampling decision.
+
+TODO(sebright): Add a demo to https://github.com/census-ecosystem/opencensus-experiments and link to
+it.

contrib/log_correlation/stackdriver/build.gradle

@@ -0,0 +1,10 @@
+description = 'OpenCensus Stackdriver Log Correlation'
+
+apply plugin: 'java'
+
+dependencies {
+    compile project(':opencensus-api'),
+            libraries.google_cloud_logging
+
+    signature "org.codehaus.mojo.signature:java16:+@signature"
+}

contrib/log_correlation/stackdriver/src/main/java/io/opencensus/contrib/logcorrelation/stackdriver/OpenCensusTraceLoggingEnhancer.java

@@ -0,0 +1,64 @@
+/*
+ * Copyright 2018, OpenCensus Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.opencensus.contrib.logcorrelation.stackdriver;
+
+import com.google.cloud.ServiceOptions;
+import com.google.cloud.logging.LogEntry;
+import com.google.cloud.logging.LoggingEnhancer;
+import io.opencensus.common.ExperimentalApi;
+import io.opencensus.trace.SpanContext;
+import io.opencensus.trace.TraceId;
+import io.opencensus.trace.Tracer;
+import io.opencensus.trace.Tracing;
+
+/**
+ * Stackdriver {@link LoggingEnhancer} that adds OpenCensus tracing data to log entries.
+ *
+ * <p>This feature is currently experimental.
+ */
+
+// TODO(sebright): Allow configuring the LoggingEnhancer with Java properties. One property should
+// control the types of spans to correlate, with these three choices:
+// - off
+// - only add tracing data for sampled spans
+// - add tracing data for all spans
+@ExperimentalApi
+public final class OpenCensusTraceLoggingEnhancer implements LoggingEnhancer {
+  private static final String SAMPLED_KEY = "sampled";
+
+  private static final Tracer tracer = Tracing.getTracer();
+
+  /** Constructor to be called by Stackdriver logging. */
+  public OpenCensusTraceLoggingEnhancer() {}
+
+  @Override
+  public void enhanceLogEntry(LogEntry.Builder builder) {
+    SpanContext span = tracer.getCurrentSpan().getContext();
+    builder.setTrace(formatTraceId(span.getTraceId()));
+    builder.setSpanId(span.getSpanId().toLowerBase16());
+
+    // TODO(sebright): Find the correct way to add the sampling decision.
+    builder.addLabel(SAMPLED_KEY, Boolean.toString(span.getTraceOptions().isSampled()));
+  }
+
+  private static String formatTraceId(TraceId traceId) {
+    // TODO(sebright): Cache the project ID.
+    // TODO(sebright): Add a way to override the project ID in the logging configuration.
+    String projectId = ServiceOptions.getDefaultProjectId();
+    return "projects/" + projectId + "/traces/" + traceId.toLowerBase16();
+  }
+}

contrib/log_correlation/stackdriver/src/test/java/io/opencensus/contrib/logcorrelation/stackdriver/OpenCensusTraceLoggingEnhancerTest.java

@@ -0,0 +1,133 @@
+/*
+ * Copyright 2018, OpenCensus Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.opencensus.contrib.logcorrelation.stackdriver;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import com.google.cloud.logging.LogEntry;
+import io.opencensus.common.Scope;
+import io.opencensus.trace.Annotation;
+import io.opencensus.trace.AttributeValue;
+import io.opencensus.trace.BlankSpan;
+import io.opencensus.trace.EndSpanOptions;
+import io.opencensus.trace.Link;
+import io.opencensus.trace.Span;
+import io.opencensus.trace.SpanContext;
+import io.opencensus.trace.SpanId;
+import io.opencensus.trace.TraceId;
+import io.opencensus.trace.TraceOptions;
+import io.opencensus.trace.Tracer;
+import io.opencensus.trace.Tracing;
+import java.util.EnumSet;
+import java.util.Map;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Test for {@link OpenCensusTraceLoggingEnhancer}. */
+// TODO(sebright): Find a way to test that OpenCensusTraceLoggingEnhancer is called from Stackdriver
+// logging. See
+// https://github.com/GoogleCloudPlatform/google-cloud-java/blob/master/TESTING.md#testing-code-that-uses-logging.
+@RunWith(JUnit4.class)
+public class OpenCensusTraceLoggingEnhancerTest {
+  private static final String GOOGLE_CLOUD_PROJECT = "GOOGLE_CLOUD_PROJECT";
+
+  private static Tracer tracer = Tracing.getTracer();
+
+  @Test
+  public void enhanceLogEntry_Sampled() {
+    String projectId = "my-test-project-1";
+    String traceId = "4c9874d0b41224cce77ff74ee10f5ee6";
+    String spanId = "592ae363e92cb3dd";
+    boolean isSampled = true;
+    testLoggingEnhancer(projectId, traceId, spanId, isSampled);
+  }
+
+  @Test
+  public void enhanceLogEntry_NotSampled() {
+    String projectId = "my-test-project-2";
+    String traceId = "7f4703d9bb02f4f2e67fb840103cdd34";
+    String spanId = "2d7d95a555557434";
+    boolean isSampled = false;
+    testLoggingEnhancer(projectId, traceId, spanId, isSampled);
+  }
+
+  private static void testLoggingEnhancer(
+      String projectId, String traceId, String spanId, boolean isSampled) {
+    System.setProperty(GOOGLE_CLOUD_PROJECT, projectId);
+    try {
+      Scope scope =
+          tracer.withSpan(
+              new TestSpan(
+                  SpanContext.create(
+                      TraceId.fromLowerBase16(traceId),
+                      SpanId.fromLowerBase16(spanId),
+                      TraceOptions.builder().setIsSampled(isSampled).build())));
+      try {
+        LogEntry.Builder builder = LogEntry.newBuilder(null);
+        new OpenCensusTraceLoggingEnhancer().enhanceLogEntry(builder);
+        LogEntry logEntry = builder.build();
+        assertThat(logEntry.getLabels().get("sampled")).isEqualTo(isSampled ? "true" : "false");
+        assertThat(logEntry.getTrace()).isEqualTo("projects/" + projectId + "/traces/" + traceId);
+        assertThat(logEntry.getSpanId()).isEqualTo(spanId);
+      } finally {
+        scope.close();
+      }
+    } finally {
+      System.clearProperty(GOOGLE_CLOUD_PROJECT);
+    }
+  }
+
+  @Test
+  public void enhanceLogEntry_BlankSpan() {
+    System.setProperty(GOOGLE_CLOUD_PROJECT, "my-test-project-3");
+    try {
+      Scope scope = tracer.withSpan(BlankSpan.INSTANCE);
+      try {
+        LogEntry.Builder builder = LogEntry.newBuilder(null);
+        new OpenCensusTraceLoggingEnhancer().enhanceLogEntry(builder);
+        LogEntry logEntry = builder.build();
+        assertThat(logEntry.getLabels().get("sampled")).isEqualTo("false");
+        assertThat(logEntry.getTrace())
+            .isEqualTo("projects/my-test-project-3/traces/00000000000000000000000000000000");
+        assertThat(logEntry.getSpanId()).isEqualTo("0000000000000000");
+      } finally {
+        scope.close();
+      }
+    } finally {
+      System.clearProperty(GOOGLE_CLOUD_PROJECT);
+    }
+  }
+
+  private static final class TestSpan extends Span {
+    TestSpan(SpanContext context) {
+      super(context, EnumSet.of(Options.RECORD_EVENTS));
+    }
+
+    @Override
+    public void end(EndSpanOptions options) {}
+
+    @Override
+    public void addLink(Link link) {}
+
+    @Override
+    public void addAnnotation(Annotation annotation) {}
+
+    @Override
+    public void addAnnotation(String description, Map<String, AttributeValue> attributes) {}
+  }
+}

settings.gradle

@@ -18,6 +18,7 @@ include ":opencensus-contrib-appengine-standard-util"
 include ":opencensus-contrib-grpc-metrics"
 include ":opencensus-contrib-grpc-util"
 include ":opencensus-contrib-http-util"
+include ":opencensus-contrib-log-correlation-stackdriver"
 include ":opencensus-contrib-monitored-resource-util"
 // TODO(songya): uncomment this once classes were added to Metrics library.
 //include ":opencensus-metrics"
@@ -33,6 +34,8 @@ project(':opencensus-contrib-appengine-standard-util').projectDir =
 project(':opencensus-contrib-grpc-metrics').projectDir = "$rootDir/contrib/grpc_metrics" as File
 project(':opencensus-contrib-grpc-util').projectDir = "$rootDir/contrib/grpc_util" as File
 project(':opencensus-contrib-http-util').projectDir = "$rootDir/contrib/http_util" as File
+project(':opencensus-contrib-log-correlation-stackdriver').projectDir =
+        "$rootDir/contrib/log_correlation/stackdriver" as File
 project(':opencensus-contrib-monitored-resource-util').projectDir =
         "$rootDir/contrib/monitored_resource_util" as File
 project(':opencensus-exporter-trace-instana').projectDir =