Merge branch 'develop' into 'master'

Release 0.34.0-SNAPSHOT See merge request hercules/hercules!205
vostok · Jan 16, 2020 · 7d33787 · 7d33787
2 parents ad9bf19 + ecd858c
commit 7d33787
Show file tree

Hide file tree

Showing 66 changed files with 512 additions and 139 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -1,5 +1,7 @@
 FROM openjdk:8-jre-alpine
 
+RUN apk add --no-cache jattach --repository http://dl-cdn.alpinelinux.org/alpine/edge/community/
+
 ARG SERVICENAME="unknown"
 ARG VERSION="unknown"
 ARG WORKDIR="/usr/lib/hercules/${SERVICENAME}"

diff --git a/hercules-application/pom.xml b/hercules-application/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <artifactId>hercules</artifactId>
         <groupId>ru.kontur.vostok.hercules</groupId>
-        <version>0.33.1-SNAPSHOT</version>
+        <version>0.34.0-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

diff --git a/hercules-application/src/main/java/ru/kontur/vostok/hercules/application/Version.java b/hercules-application/src/main/java/ru/kontur/vostok/hercules/application/Version.java
@@ -14,7 +14,7 @@ public final class Version {
     private static final Version INSTANCE;
 
     static {
-        Properties properties = PropertiesLoader.load("resource://git.properties");
+        Properties properties = PropertiesLoader.load("resource://git.properties", false);
 
         final String version = properties.getProperty("git.build.version", "unknown");
         final String commitId = properties.getProperty("git.commit.id", "unknown");

diff --git a/hercules-auth/pom.xml b/hercules-auth/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <artifactId>hercules</artifactId>
         <groupId>ru.kontur.vostok.hercules</groupId>
-        <version>0.33.1-SNAPSHOT</version>
+        <version>0.34.0-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

diff --git a/hercules-cassandra-sink/pom.xml b/hercules-cassandra-sink/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <artifactId>hercules</artifactId>
         <groupId>ru.kontur.vostok.hercules</groupId>
-        <version>0.33.1-SNAPSHOT</version>
+        <version>0.34.0-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

diff --git a/hercules-cassandra-util/pom.xml b/hercules-cassandra-util/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <artifactId>hercules</artifactId>
         <groupId>ru.kontur.vostok.hercules</groupId>
-        <version>0.33.1-SNAPSHOT</version>
+        <version>0.34.0-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

diff --git a/hercules-client/pom.xml b/hercules-client/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <artifactId>hercules</artifactId>
         <groupId>ru.kontur.vostok.hercules</groupId>
-        <version>0.33.1-SNAPSHOT</version>
+        <version>0.34.0-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

diff --git a/hercules-configuration/pom.xml b/hercules-configuration/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <artifactId>hercules</artifactId>
         <groupId>ru.kontur.vostok.hercules</groupId>
-        <version>0.33.1-SNAPSHOT</version>
+        <version>0.34.0-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

diff --git a/hercules-curator/pom.xml b/hercules-curator/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <artifactId>hercules</artifactId>
         <groupId>ru.kontur.vostok.hercules</groupId>
-        <version>0.33.1-SNAPSHOT</version>
+        <version>0.34.0-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

diff --git a/hercules-elastic-sink/pom.xml b/hercules-elastic-sink/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <artifactId>hercules</artifactId>
         <groupId>ru.kontur.vostok.hercules</groupId>
-        <version>0.33.1-SNAPSHOT</version>
+        <version>0.34.0-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

diff --git a/hercules-gate-client/pom.xml b/hercules-gate-client/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <artifactId>hercules</artifactId>
         <groupId>ru.kontur.vostok.hercules</groupId>
-        <version>0.33.1-SNAPSHOT</version>
+        <version>0.34.0-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 
@@ -14,10 +14,6 @@
             <groupId>ru.kontur.vostok.hercules</groupId>
             <artifactId>hercules-protocol</artifactId>
         </dependency>
-        <dependency>
-            <groupId>ru.kontur.vostok.hercules</groupId>
-            <artifactId>hercules-configuration</artifactId>
-        </dependency>
 
         <dependency>
             <groupId>org.apache.httpcomponents</groupId>

diff --git a/hercules-gate-client/src/main/java/ru/kontur/vostok/hercules/gate/client/EventPublisher.java b/hercules-gate-client/src/main/java/ru/kontur/vostok/hercules/gate/client/EventPublisher.java
@@ -2,7 +2,6 @@
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import ru.kontur.vostok.hercules.configuration.Scopes;
 import ru.kontur.vostok.hercules.gate.client.exception.BadRequestException;
 import ru.kontur.vostok.hercules.gate.client.exception.UnavailableClusterException;
 import ru.kontur.vostok.hercules.gate.client.util.EventWriterUtil;
@@ -52,7 +51,7 @@ public EventPublisher(Properties properties,
         final int threads = PropertiesUtil.get(Props.THREAD_COUNT, properties).get();
         final String[] urls = PropertiesUtil.get(Props.URLS, properties).get();
         final String apiKey = PropertiesUtil.get(Props.API_KEY, properties).get();
-        final Properties gateClientProperties = PropertiesUtil.ofScope(properties, Scopes.GATE_CLIENT);
+        final Properties gateClientProperties = PropertiesUtil.ofScope(properties, "gate.client");
 
         this.urls = urls;
         this.apiKey = apiKey;

diff --git a/hercules-gate/pom.xml b/hercules-gate/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <artifactId>hercules</artifactId>
         <groupId>ru.kontur.vostok.hercules</groupId>
-        <version>0.33.1-SNAPSHOT</version>
+        <version>0.34.0-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

diff --git a/hercules-gate/src/main/java/ru/kontur/vostok/hercules/gate/SendRequestProcessor.java b/hercules-gate/src/main/java/ru/kontur/vostok/hercules/gate/SendRequestProcessor.java
@@ -2,6 +2,7 @@
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.slf4j.MDC;
 import ru.kontur.vostok.hercules.gate.validation.EventValidator;
 import ru.kontur.vostok.hercules.health.Meter;
 import ru.kontur.vostok.hercules.health.MetricsCollector;
@@ -14,6 +15,7 @@
 import ru.kontur.vostok.hercules.protocol.decoder.exceptions.InvalidDataException;
 import ru.kontur.vostok.hercules.throttling.RequestProcessor;
 import ru.kontur.vostok.hercules.throttling.ThrottleCallback;
+import ru.kontur.vostok.hercules.util.text.StringUtil;
 
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -42,22 +44,27 @@ public void processAsync(HttpServerRequest request, SendContext context, Throttl
             request.readBodyAsync(
                     (r, bytes) -> request.dispatchAsync(
                             () -> {
-                                ReaderIterator<Event> reader;
                                 try {
-                                    reader = new ReaderIterator<>(new Decoder(bytes), EventReader.readTags(context.getTags()));
-                                } catch (RuntimeException | InvalidDataException exception) {
-                                    request.complete(HttpStatusCodes.BAD_REQUEST);
-                                    callback.call();
-                                    LOGGER.error("Cannot create ReaderIterator", exception);
-                                    return;
-                                }
-                                if (reader.getTotal() == 0) {
-                                    request.complete(HttpStatusCodes.OK);
-                                    callback.call();
-                                    return;
-                                }
+                                    initMDC(request, context);
+                                    ReaderIterator<Event> reader;
+                                    try {
+                                        reader = new ReaderIterator<>(new Decoder(bytes), EventReader.readTags(context.getTags()));
+                                    } catch (RuntimeException | InvalidDataException ex) {
+                                        request.complete(HttpStatusCodes.BAD_REQUEST);
+                                        callback.call();
+                                        LOGGER.error("Cannot create ReaderIterator", ex);
+                                        return;
+                                    }
+                                    if (reader.getTotal() == 0) {
+                                        request.complete(HttpStatusCodes.OK);
+                                        callback.call();
+                                        return;
+                                    }
 
-                                send(request, reader, context, callback);
+                                    send(request, reader, context, callback);
+                                } finally {
+                                    cleanMDC();
+                                }
                             }),
                     (r, e) -> {
                         try {
@@ -68,13 +75,14 @@ public void processAsync(HttpServerRequest request, SendContext context, Throttl
                         }
                     });
         } catch (Throwable throwable) {
+            // Should never happened
             callback.call();
             LOGGER.error("Error on request body read full bytes", throwable);
             throw throwable;
         }
     }
 
-    public void send(HttpServerRequest request, ReaderIterator<Event> reader, SendContext context, ThrottleCallback callback) {
+    private void send(HttpServerRequest request, ReaderIterator<Event> reader, SendContext context, ThrottleCallback callback) {
         AtomicInteger pendingEvents = new AtomicInteger(reader.getTotal());
         AtomicBoolean processed = new AtomicBoolean(false);
         while (reader.hasNext()) {
@@ -90,8 +98,8 @@ public void send(HttpServerRequest request, ReaderIterator<Event> reader, SendCo
                     }
                     return;
                 }
-            } catch (Exception e) {
-                LOGGER.error("Exception on validation event", e);
+            } catch (Exception ex) {
+                LOGGER.error("Exception on validation event", ex);
                 //TODO: Metrics are coming!
                 if (processed.compareAndSet(false, true)) {
                     request.complete(HttpStatusCodes.BAD_REQUEST);
@@ -139,4 +147,23 @@ public void send(HttpServerRequest request, ReaderIterator<Event> reader, SendCo
             request.complete(HttpStatusCodes.OK);
         }
     }
+
+    private void initMDC(HttpServerRequest request, SendContext context) {
+        MDC.put("topic",context.getTopic());
+        MDC.put("apiKey", getProtectedApiKey(request));
+    }
+
+    private void cleanMDC() {
+        MDC.remove("topic");
+        MDC.remove("apiKey");
+    }
+
+    private String getProtectedApiKey(HttpServerRequest request) {
+        String apiKey = request.getHeader("apiKey");
+        int pos = apiKey.lastIndexOf('_') + 1;
+        if (pos > 0) {
+            return StringUtil.mask(apiKey, '*', pos);
+        }
+        return StringUtil.mask(apiKey, '*', apiKey.length() / 2);
+    }
 }
diff --git a/hercules-gate/src/main/java/ru/kontur/vostok/hercules/gate/validation/EventValidator.java b/hercules-gate/src/main/java/ru/kontur/vostok/hercules/gate/validation/EventValidator.java
@@ -1,17 +1,23 @@
 package ru.kontur.vostok.hercules.gate.validation;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import ru.kontur.vostok.hercules.protocol.Event;
 
 /**
  * @author Gregory Koshelev
  */
 public class EventValidator {
+    private static final Logger LOGGER = LoggerFactory.getLogger(EventValidator.class);
+
     public boolean validate(Event event) {
         if (event.getVersion() != 1) { // Gate supports Event of version 1 only
+            LOGGER.warn("Event version != 1");
             return false;
         }
 
         if (event.getTimestamp() < 0) { // Event timestamp should be non-negative
+            LOGGER.warn("Event timestamp < 0");
             return false;
         }
 

diff --git a/hercules-gate/src/main/resources/logback.xml b/hercules-gate/src/main/resources/logback.xml
@@ -2,13 +2,13 @@
 
     <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
         <encoder>
-            <pattern>%d{"yyyy-MM-dd'T'HH:mm:ss.SSSXXX", UTC} [%thread] %-5level %logger{36} - %msg%n</pattern>
+            <pattern>%d{"yyyy-MM-dd'T'HH:mm:ss.SSSXXX", UTC} [%thread] %-5level %logger{36} CTX: [topic=%X{topic} apiKey=%X{apiKey}] - %msg%n</pattern>
         </encoder>
     </appender>
 
     <appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
         <encoder>
-            <pattern>%d{"yyyy-MM-dd'T'HH:mm:ss.SSSXXX", UTC} [%thread] %-5level %logger{36} - %msg%n</pattern>
+            <pattern>%d{"yyyy-MM-dd'T'HH:mm:ss.SSSXXX", UTC} [%thread] %-5level %logger{36} CTX: [topic=%X{topic} apiKey=%X{apiKey}] - %msg%n</pattern>
         </encoder>
         <file>hercules_gate.log</file>
         <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">

diff --git a/hercules-graphite-sink/pom.xml b/hercules-graphite-sink/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <artifactId>hercules</artifactId>
         <groupId>ru.kontur.vostok.hercules</groupId>
-        <version>0.33.1-SNAPSHOT</version>
+        <version>0.34.0-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

diff --git a/hercules-health/pom.xml b/hercules-health/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <artifactId>hercules</artifactId>
         <groupId>ru.kontur.vostok.hercules</groupId>
-        <version>0.33.1-SNAPSHOT</version>
+        <version>0.34.0-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

diff --git a/hercules-http/pom.xml b/hercules-http/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <artifactId>hercules</artifactId>
         <groupId>ru.kontur.vostok.hercules</groupId>
-        <version>0.33.1-SNAPSHOT</version>
+        <version>0.34.0-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

diff --git a/hercules-http/src/main/java/ru/kontur/vostok/hercules/http/ContentEncodings.java b/hercules-http/src/main/java/ru/kontur/vostok/hercules/http/ContentEncodings.java
@@ -0,0 +1,9 @@
+package ru.kontur.vostok.hercules.http;
+
+/**
+ * @author Gregory Koshelev
+ */
+public class ContentEncodings {
+    public static final String GZIP = "gzip";
+    public static final String LZ4 = "lz4";
+}
diff --git a/hercules-http/src/main/java/ru/kontur/vostok/hercules/http/HttpHeaders.java b/hercules-http/src/main/java/ru/kontur/vostok/hercules/http/HttpHeaders.java
@@ -10,6 +10,17 @@ public final class HttpHeaders {
     public static final String CONTENT_TYPE = "Content-Type";
     public static final String TRANSFER_ENCODING = "Transfer-Encoding";
 
+    /* Non-Standard HTTP headers */
+
+    /**
+     * Non-Standard HTTP header contains original (non-compressed) content length
+     * when {@link #CONTENT_ENCODING Content-Encoding} is used.
+     * <p>
+     * Thus, {@link #CONTENT_LENGTH Content-Length} refers to compressed content length,
+     * but {@link #ORIGINAL_CONTENT_LENGTH} refers to uncompressed content length.
+     */
+    public static final String ORIGINAL_CONTENT_LENGTH = "Original-Content-Length";
+
     private HttpHeaders() {
         /* static class */
     }

diff --git a/hercules-http/src/main/java/ru/kontur/vostok/hercules/http/HttpServerRequest.java b/hercules-http/src/main/java/ru/kontur/vostok/hercules/http/HttpServerRequest.java
@@ -26,15 +26,23 @@ public interface HttpServerRequest {
     /**
      * Get header value of the request. If header is used for multiple times, then return first value.
      *
-     * @param name of the header
+     * @param name header name
      * @return header value or <code>null</code> if it doesn't exist
      */
     String getHeader(String name);
 
+    /**
+     * Get header values of the request. If there are no header values in the request, then return empty array.
+     *
+     * @param name header name
+     * @return header values
+     */
+    String[] getHeaders(String name);
+
     /**
      * Get query parameter value of the request. If parameter is used for multiple times, then return first value.
      *
-     * @param name of the query parameter
+     * @param name query parameter name
      * @return parameter value or <code>null</code> if it doesn't exist
      */
     String getQueryParameter(String name);
@@ -49,7 +57,7 @@ public interface HttpServerRequest {
      * getPathParameter("book") returns "thehitchhikersguidetothegalaxy" and
      * getPathParameter("page") returns "42"
      *
-     * @param name of the path parameter
+     * @param name path parameter name
      * @return parameter value or <code>null</code> if it doesn't exist
      */
     String getPathParameter(String name);
@@ -64,7 +72,7 @@ public interface HttpServerRequest {
     /**
      * Get all query parameter values of the request. If parameter doesn't present, then return empty array.
      *
-     * @param name of the query parameter
+     * @param name query parameter name
      * @return parameter values
      */
     String[] getQueryParameterValues(String name);
@@ -116,7 +124,7 @@ default void readBodyAsync(ReadBodyCallback callback) {
     /**
      * Complete request processing with specified status code.
      *
-     * @param code of response
+     * @param code response status code
      */
     default void complete(int code) {
         getResponse().setStatusCode(code);