Adds AutocompleteTags and corresponding REST api (openzipkin#2332)

* Adds AutocompleteTags and corresponding REST api This adds an optional storage interface `AutocompleteTags` to the `StorageComponent` and corresponding endpoints to support the UI. /api/v2/autocompleteKeys and /api/v2/autocompleteKeys?key=http.host The server accepts a parameter `zipkin.storage.autocompleteKeys` which acts as a whitelist as storing values for every tag would be expensive. * Info when not implemented and backfill tests
abesto · Sep 10, 2019 · 7804665 · 7804665
1 parent faf2816
commit 7804665
Show file tree

Hide file tree

Showing 11 changed files with 308 additions and 22 deletions.
diff --git a/zipkin-server/README.md b/zipkin-server/README.md
@@ -129,6 +129,7 @@ Defaults to true
 * `QUERY_LOOKBACK`: How many milliseconds queries can look back from endTs; Defaults to 24 hours (two daily buckets: one for today and one for yesterday)
 * `STORAGE_TYPE`: SpanStore implementation: one of `mem`, `mysql`, `cassandra`, `elasticsearch`
 * `COLLECTOR_SAMPLE_RATE`: Percentage of traces to retain, defaults to always sample (1.0).
+* `AUTOCOMPLETE_KEYS`: list of span tag keys which will be returned by the `/api/v2/autocompleteTags` endpoint
 
 ### Cassandra Storage
 Zipkin's [Cassandra storage component](../zipkin-storage/cassandra)

diff --git a/zipkin-server/src/main/java/zipkin2/server/internal/ZipkinQueryApiV2.java b/zipkin-server/src/main/java/zipkin2/server/internal/ZipkinQueryApiV2.java
@@ -23,6 +23,7 @@
 import org.springframework.http.HttpStatus;
 import org.springframework.http.ResponseEntity;
 import org.springframework.web.bind.annotation.ExceptionHandler;
+import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RequestMethod;
@@ -52,19 +53,22 @@ public class ZipkinQueryApiV2 {
   final long defaultLookback;
   /** The Cache-Control max-age (seconds) for /api/v2/services and /api/v2/spans */
   final int namesMaxAge;
+  final List<String> autocompleteKeys;
 
   volatile int serviceCount; // used as a threshold to start returning cache-control headers
 
   ZipkinQueryApiV2(
-      StorageComponent storage,
-      @Value("${zipkin.storage.type:mem}") String storageType,
-      @Value("${zipkin.query.lookback:86400000}") long defaultLookback, // 1 day in millis
-      @Value("${zipkin.query.names-max-age:300}") int namesMaxAge // 5 minutes
-      ) {
+    StorageComponent storage,
+    @Value("${zipkin.storage.type:mem}") String storageType,
+    @Value("${zipkin.query.lookback:86400000}") long defaultLookback, // 1 day in millis
+    @Value("${zipkin.query.names-max-age:300}") int namesMaxAge, // 5 minutes
+    @Value("${zipkin.storage.autocomplete-keys:}") List<String> autocompleteKeys
+  ) {
     this.storage = storage;
     this.storageType = storageType;
     this.defaultLookback = defaultLookback;
     this.namesMaxAge = namesMaxAge;
+    this.autocompleteKeys = autocompleteKeys;
   }
 
   @RequestMapping(
@@ -89,7 +93,7 @@ public ResponseEntity<List<String>> getServiceNames() throws IOException {
 
   @RequestMapping(value = "/spans", method = RequestMethod.GET)
   public ResponseEntity<List<String>> getSpanNames(
-      @RequestParam(value = "serviceName", required = true) String serviceName) throws IOException {
+      @RequestParam(value = "serviceName") String serviceName) throws IOException {
     return maybeCacheNames(storage.spanStore().getSpanNames(serviceName).execute());
   }
 
@@ -130,6 +134,32 @@ public String getTrace(@PathVariable String traceIdHex, WebRequest request) thro
     return new String(SpanBytesEncoder.JSON_V2.encodeList(trace), UTF_8);
   }
 
+  @GetMapping(value = "/autocompleteKeys", produces = APPLICATION_JSON_VALUE)
+  public ResponseEntity<List<String>> getAutocompleteKeys() {
+    return ResponseEntity.ok()
+      .cacheControl(CacheControl.maxAge(namesMaxAge, TimeUnit.SECONDS).mustRevalidate())
+      .body(autocompleteKeys);
+  }
+
+  @GetMapping(value = "/autocompleteValues", produces = APPLICATION_JSON_VALUE)
+  public ResponseEntity<List<String>> getAutocompleteValues(@RequestParam String key)
+    throws IOException {
+    return maybeCacheAutocompleteValues(storage.autocompleteTags().getValues(key).execute());
+  }
+
+  /**
+   * We cache tag values to minimize the number of requests made to the storage backend. The tag
+   * values doesn't change frequently and cache expires in 5 minutes
+   *
+   */
+  ResponseEntity<List<String>> maybeCacheAutocompleteValues(List<String> values) {
+    ResponseEntity.BodyBuilder response = ResponseEntity.ok();
+    if (values.size() > 3) {
+      response.cacheControl(CacheControl.maxAge(namesMaxAge, TimeUnit.SECONDS).mustRevalidate());
+    }
+    return response.body(values);
+  }
+
   @ExceptionHandler(TraceNotFoundException.class)
   @ResponseStatus(HttpStatus.NOT_FOUND)
   public void notFound() {}

diff --git a/zipkin-server/src/main/java/zipkin2/server/internal/ZipkinServerConfiguration.java b/zipkin-server/src/main/java/zipkin2/server/internal/ZipkinServerConfiguration.java
@@ -16,6 +16,7 @@
 import brave.Tracing;
 import io.micrometer.core.instrument.MeterRegistry;
 import io.micrometer.core.instrument.config.MeterFilter;
+import java.util.List;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.beans.factory.annotation.Value;
@@ -147,11 +148,13 @@ static class InMemoryConfiguration {
     StorageComponent storage(
         @Value("${zipkin.storage.strict-trace-id:true}") boolean strictTraceId,
         @Value("${zipkin.storage.search-enabled:true}") boolean searchEnabled,
-        @Value("${zipkin.storage.mem.max-spans:500000}") int maxSpans) {
+        @Value("${zipkin.storage.mem.max-spans:500000}") int maxSpans,
+        @Value("${zipkin.storage.autocomplete-keys:}") List<String> autocompleteKeys) {
       return InMemoryStorage.newBuilder()
           .strictTraceId(strictTraceId)
           .searchEnabled(searchEnabled)
           .maxSpanCount(maxSpans)
+          .autocompleteKeys(autocompleteKeys)
           .build();
     }
   }

diff --git a/zipkin-server/src/main/java/zipkin2/server/internal/brave/TracingStorageComponent.java b/zipkin-server/src/main/java/zipkin2/server/internal/brave/TracingStorageComponent.java
@@ -20,6 +20,7 @@
 import zipkin2.Call;
 import zipkin2.DependencyLink;
 import zipkin2.Span;
+import zipkin2.storage.AutocompleteTags;
 import zipkin2.storage.QueryRequest;
 import zipkin2.storage.SpanConsumer;
 import zipkin2.storage.SpanStore;
@@ -35,13 +36,15 @@ public TracingStorageComponent(Tracing tracing, StorageComponent delegate) {
     this.delegate = delegate;
   }
 
-  @Override
-  public SpanStore spanStore() {
+  @Override public SpanStore spanStore() {
     return new TracingSpanStore(tracing, delegate.spanStore());
   }
 
-  @Override
-  public SpanConsumer spanConsumer() {
+  @Override public AutocompleteTags autocompleteTags() {
+    return new TracingAutocompleteTags(tracing, delegate.autocompleteTags());
+  }
+
+  @Override public SpanConsumer spanConsumer() {
     // prevents accidental write amplification
     return delegate.spanConsumer();
   }
@@ -52,8 +55,8 @@ public void close() throws IOException {
   }
 
   static final class TracingSpanStore implements SpanStore {
-    private final Tracer tracer;
-    private final SpanStore delegate;
+    final Tracer tracer;
+    final SpanStore delegate;
 
     TracingSpanStore(Tracing tracing, SpanStore delegate) {
       this.tracer = tracing.tracer();
@@ -83,7 +86,25 @@ public Call<List<String>> getSpanNames(String serviceName) {
     @Override
     public Call<List<DependencyLink>> getDependencies(long endTs, long lookback) {
       return new TracedCall<>(
-          tracer, delegate.getDependencies(endTs, lookback), "get-dependencies");
+        tracer, delegate.getDependencies(endTs, lookback), "get-dependencies");
+    }
+  }
+
+  static final class TracingAutocompleteTags implements AutocompleteTags {
+    final Tracer tracer;
+    final AutocompleteTags delegate;
+
+    TracingAutocompleteTags(Tracing tracing, AutocompleteTags delegate) {
+      this.tracer = tracing.tracer();
+      this.delegate = delegate;
+    }
+
+    @Override public Call<List<String>> getKeys() {
+      return new TracedCall<>(tracer, delegate.getKeys(), "get-keys");
+    }
+
+    @Override public Call<List<String>> getValues(String key) {
+      return new TracedCall<>(tracer, delegate.getValues(key), "get-values");
     }
   }
 }
diff --git a/zipkin-server/src/main/resources/zipkin-server-shared.yml b/zipkin-server/src/main/resources/zipkin-server-shared.yml
@@ -46,6 +46,7 @@ zipkin:
   storage:
     strict-trace-id: ${STRICT_TRACE_ID:true}
     search-enabled: ${SEARCH_ENABLED:true}
+    autocomplete-keys: ${AUTOCOMPLETE_KEYS:}
     type: ${STORAGE_TYPE:mem}
     mem:
       # Maximum number of spans to keep in memory.  When exceeded, oldest traces (and their spans) will be purged.

diff --git a/zipkin/src/main/java/zipkin2/storage/AutocompleteTags.java b/zipkin/src/main/java/zipkin2/storage/AutocompleteTags.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2015-2018 The OpenZipkin Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package zipkin2.storage;
+
+import java.util.List;
+import zipkin2.Call;
+
+/**
+ * Provides autocomplete functionality by providing values for a given tag key, usually derived from
+ * {@link SpanConsumer}.
+ */
+public interface AutocompleteTags {
+
+  /**
+   * Retrieves the list of tag getKeys whose values may be returned by {@link #getValues(String)}.
+   *
+   * @see StorageComponent.Builder#autocompleteKeys(List)
+   */
+  Call<List<String>> getKeys();
+
+  /**
+   * Retrieves the list of values, if the input is configured for autocompletion. If a key is not
+   * configured, or there are no values available, an empty result will be returned.
+   *
+   * @throws IllegalArgumentException if the input is empty.
+   * @see StorageComponent.Builder#autocompleteKeys(List)
+   */
+  Call<List<String>> getValues(String key);
+}
diff --git a/zipkin/src/main/java/zipkin2/storage/InMemoryStorage.java b/zipkin/src/main/java/zipkin2/storage/InMemoryStorage.java
@@ -63,7 +63,8 @@
  *    foo --> ( GET, POST )
  * }</pre>
  */
-public final class InMemoryStorage extends StorageComponent implements SpanStore, SpanConsumer {
+public final class InMemoryStorage extends StorageComponent implements SpanStore, SpanConsumer,
+  AutocompleteTags {
 
   public static Builder newBuilder() {
     return new Builder();
@@ -72,6 +73,7 @@ public static Builder newBuilder() {
   public static final class Builder extends StorageComponent.Builder {
     boolean strictTraceId = true, searchEnabled = true;
     int maxSpanCount = 500000;
+    List<String> autocompleteKeys = Collections.emptyList();
 
     /** {@inheritDoc} */
     @Override
@@ -86,6 +88,12 @@ public Builder searchEnabled(boolean searchEnabled) {
       return this;
     }
 
+    @Override public Builder autocompleteKeys(List<String> autocompleteKeys) {
+      if (autocompleteKeys == null) throw new NullPointerException("autocompleteKeys == null");
+      this.autocompleteKeys = autocompleteKeys;
+      return this;
+    }
+
     /** Eldest traces are removed to ensure spans in memory don't exceed this value */
     public Builder maxSpanCount(int maxSpanCount) {
       if (maxSpanCount <= 0) throw new IllegalArgumentException("maxSpanCount <= 0");
@@ -112,17 +120,17 @@ Collection<Span> valueContainer() {
       }
     };
 
-  /** This supports span lookup by {@link Span#traceId lower 64-bits of the trace ID} */
+  /** This supports span lookup by {@link Span#traceId() lower 64-bits of the trace ID} */
   private final SortedMultimap<String, TraceIdTimestamp> traceIdToTraceIdTimeStamps =
     new SortedMultimap<String, TraceIdTimestamp>(STRING_COMPARATOR) {
       @Override
       Collection<TraceIdTimestamp> valueContainer() {
         return new LinkedHashSet<>();
       }
     };
-  /** This is an index of {@link Span#traceId} by {@link Endpoint#serviceName() service name} */
+  /** This is an index of {@link Span#traceId()} by {@link Endpoint#serviceName() service name} */
   private final ServiceNameToTraceIds serviceToTraceIds = new ServiceNameToTraceIds();
-  /** This is an index of {@link Span#name} by {@link Endpoint#serviceName() service name} */
+  /** This is an index of {@link Span#name()} by {@link Endpoint#serviceName() service name} */
   private final SortedMultimap<String, String> serviceToSpanNames =
     new SortedMultimap<String, String>(STRING_COMPARATOR) {
       @Override
@@ -131,14 +139,26 @@ Collection<String> valueContainer() {
       }
     };
 
+  private final SortedMultimap<String, String> autocompleteTags =
+    new SortedMultimap<String, String>(STRING_COMPARATOR) {
+      @Override
+      Collection<String> valueContainer() {
+        return new LinkedHashSet<>();
+      }
+    };
+
   final boolean strictTraceId, searchEnabled;
   final int maxSpanCount;
+  final Call<List<String>> autocompleteKeysCall;
+  final Set<String> autocompleteKeys;
   volatile int acceptedSpanCount;
 
   InMemoryStorage(Builder builder) {
     this.strictTraceId = builder.strictTraceId;
     this.searchEnabled = builder.searchEnabled;
     this.maxSpanCount = builder.maxSpanCount;
+    this.autocompleteKeysCall = Call.create(builder.autocompleteKeys);
+    this.autocompleteKeys = new LinkedHashSet<>(builder.autocompleteKeys);
   }
 
   public int acceptedSpanCount() {
@@ -151,6 +171,7 @@ public synchronized void clear() {
     spansByTraceIdTimeStamp.clear();
     serviceToTraceIds.clear();
     serviceToSpanNames.clear();
+    autocompleteTags.clear();
   }
 
   @Override
@@ -176,6 +197,11 @@ public synchronized Call<Void> accept(List<Span> spans) {
         serviceToTraceIds.put(span.remoteServiceName(), lowTraceId);
         if (spanName != null) serviceToSpanNames.put(span.remoteServiceName(), spanName);
       }
+      for (Map.Entry<String, String> tag : span.tags().entrySet()) {
+        if (autocompleteKeys.contains(tag.getKey())) {
+          autocompleteTags.put(tag.getKey(), tag.getValue());
+        }
+      }
     }
     return Call.create(null /* Void == null */);
   }
@@ -331,6 +357,18 @@ public synchronized Call<List<DependencyLink>> getDependencies(long endTs, long
     return getTracesCall.map(LinkDependencies.INSTANCE);
   }
 
+  @Override public Call<List<String>> getKeys() {
+    if (!searchEnabled) return Call.emptyList();
+    return autocompleteKeysCall.clone();
+  }
+
+  @Override public Call<List<String>> getValues(String key) {
+    if (key == null) throw new NullPointerException("key == null");
+    if (key.isEmpty()) throw new IllegalArgumentException("key was empty");
+    if (!searchEnabled) return Call.emptyList();
+    return Call.create(new ArrayList<>(autocompleteTags.get(key)));
+  }
+
   enum LinkDependencies implements Call.Mapper<List<List<Span>>, List<DependencyLink>> {
     INSTANCE;
 
@@ -471,6 +509,10 @@ public InMemoryStorage spanStore() {
     return this;
   }
 
+  @Override public InMemoryStorage autocompleteTags() {
+    return this;
+  }
+
   @Override
   public SpanConsumer spanConsumer() {
     return this;