[xserver] TLS support added to xserver, aggregator server, and aggregator client

scripts/docker-integration-tests/aggregator_tls/client.crt

@@ -0,0 +1,20 @@
+-----BEGIN CERTIFICATE-----
+MIIDPzCCAicCFFJVDW2T6lamcRZYStsBCQTKicKwMA0GCSqGSIb3DQEBCwUAMFox
+CzAJBgNVBAYTAk5MMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl
+cm5ldCBXaWRnaXRzIFB0eSBMdGQxEzARBgNVBAMMClRlc3RSb290Q0EwHhcNMjQw
+NDE1MTQ0NjEyWhcNMzQwNDEzMTQ0NjEyWjBeMQswCQYDVQQGEwJOTDETMBEGA1UE
+CAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRk
+MRcwFQYDVQQDDA5UZXN0Q2xpZW50Q2VydDCCASIwDQYJKoZIhvcNAQEBBQADggEP
+ADCCAQoCggEBAMXgv5tgZD5iBW3LfTC27jPHTLlsAw8ZlBFLmd7potdS3Hy512Xa
+kKOaKRKH/0Yla4BKGNh0ZP+xege15CpTFqwtzDPCRkaRvfSxRW19MukeergRDqqI
+tEwURHQ5gfub7XB9e5wuNmW2GGs/xgiWsxxm6UMidXkhA5a2//OmDSm2JdcoxJbk
+1m1OUzXUuLKLzCIRAbJFFIM1RpvjWSonxpJ2lsNCVQ+LdItxE9mAI+Y+PaDonyzg
+ehfAqkl+wxGRevaDFFJr1defxmnJhaYv40AcEpXOWeetZFlspEtrPF7b/3GifRdx
+Mlwq27wn3mk4wF1wa12kRNKiz+HHjZrB7UECAwEAATANBgkqhkiG9w0BAQsFAAOC
+AQEAKblZ+C4Pdv5grKiCQcfFt53SbIys8uavxmzJNn9nq1QVgBmR/hX8yG8ULeh5
+kKrccwe2WXCBWsIj9cvGCzR+MPMvuIXd3d58yLybmIUBDfJSBC0v5TaNWP2ZdJDG
+fH38DfQTeE9XcjpRtCGYtYwReBptSlNiYQlvwkukTNz3mV+0BpvQu/9/R0BNCJAF
+9VeLgUcG7T7HC+foCeU7h83Y0xH0OALe33ntcrMPHUclXqImCiC7dl8pb3H7o/xJ
+jS3KZxfv6ioxpirvpEbX/EeF+H1HQZUZ8y3+J3K37jMgzNamd/xsaJxEfCq4SE+n
+HmgeXgqoNjrM6J0jYAfx1MjbIA==
+-----END CERTIFICATE-----

scripts/docker-integration-tests/aggregator_tls/client.key

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEAxeC/m2BkPmIFbct9MLbuM8dMuWwDDxmUEUuZ3umi11LcfLnX
+ZdqQo5opEof/RiVrgEoY2HRk/7F6B7XkKlMWrC3MM8JGRpG99LFFbX0y6R56uBEO
+qoi0TBREdDmB+5vtcH17nC42ZbYYaz/GCJazHGbpQyJ1eSEDlrb/86YNKbYl1yjE
+luTWbU5TNdS4sovMIhEBskUUgzVGm+NZKifGknaWw0JVD4t0i3ET2YAj5j49oOif
+LOB6F8CqSX7DEZF69oMUUmvV15/GacmFpi/jQBwSlc5Z561kWWykS2s8Xtv/caJ9
+F3EyXCrbvCfeaTjAXXBrXaRE0qLP4ceNmsHtQQIDAQABAoIBAAtyP8MuJT5SjzvV
+rI0317mZCsAjFl42PZFujR0O6MOJ4IU6ftI+fWVpUnzm7wZQvdIy9xL2UK1Vx9hQ
+Vj14hvQ4xfosf8IvRgy0gG6f8mT3xWOGYRHOTJemCHuso+85CtgZ+h+DsNPbX7g8
+fSkcBopbDZ07jg4OsdVzCoU+kr5Z1/VDe0O0rTDzGVP686tN9I+DmDYv2HlBUaSZ
+SyNZrLJFziqFx4ATTIE3aTLd29pzTl09WiCcZSxhlS+ROnN2iW2xDLNgY9SdUo2a
+3r8N+xhYDNQz2paxlsv2x/tCRJvrQDoX7S5QZw4P1pVr6wo6xC+BU2UlpVHifo4M
+egyDbAECgYEA5Y9+JDOSQF2cvIV4xTMgVnkLig/ngYesS44xz8HYTRR5s1WW6oCX
+3j+OosZBbJpXeBlTK31G9Z92+Om2/Cj2uDvff7EJdhZ5sAdMtUy5X7p9rGTEyi5F
+ecZDBfmwHH0lFdwpNtlky1uoaUsVd4IS28WMHoHYDGW241IW8xOlmeECgYEA3Ksb
+KR7JCbHx21wh5oPVn5BLpdag7D+M1jBFtjPrzLDNhXlsV9zkjn7xETvTXWU2vNlh
+OkoRu4VFEAnsgtxh+gT0lo9ZG9IWi2qP99vPpB3AFYuujBu3GA02/iOo5pri/6R0
+ZNEEreAaaOcT2z6K9HG0KMQ6QpYgXSlzJW8sf2ECgYEAvn2fMA03bIAB4xJi0EkH
+qZoScEOYWQ0rdRsOzJbPlc7K2nzImdmRrFRTWVFo0uUUdk2VjX4Mlx/3ir/uHzsi
+2GieowhWkI4/9kloZv2+yegoBxkrj5ZsAov57AhxEoLqdkRWUvR8xp9Nleo/awcd
+/Q7loh8fF9KDvAjPkHAaOCECgYA98bZNI7wxgYcwGbvWdrmX8iyaIBbKWsiRM7nN
+/OM7cYIv7rbwLyzlp1LKkK2zsP7donP9pd82caHCb9a5oV3LjmqOfSz5d08m0cIa
+RNUT79oE8lIMOJd8I/GFA8OdAGuqcaLOzjHvEVK4ke1sBTGCjwyQyQzFtljdbg5J
+utyV4QKBgHkSBKwrgaump08MJFfK9uFo5sr5jy692XYWMlBVhc+piPwUsoztjI1O
+8cvie0mMj6oofSoSyD4FUg9kFUCRMPC0kga/zlHyL1Y/uILX4XUFMl4P/gMbC7kw
+0d6BdEsp5SU8Onc9eV/EW+q2Mz7/t5ZHkUrdDQYGgkIBqRZcXlka
+-----END RSA PRIVATE KEY-----

scripts/docker-integration-tests/aggregator_tls/docker-compose.yml

@@ -0,0 +1,61 @@
+version: "3.5"
+services:
+  dbnode01:
+    expose:
+      - "9000-9004"
+      - "2379-2380"
+      - "7201"
+    ports:
+      - "0.0.0.0:9000-9004:9000-9004"
+      - "0.0.0.0:2379-2380:2379-2380"
+      - "0.0.0.0:7201:7201"
+    networks:
+      - backend
+    image: "m3dbnode_integration:${REVISION}"
+  m3coordinator01:
+    expose:
+      - "7202"
+      - "7203"
+      - "7204"
+    ports:
+      - "0.0.0.0:7202:7202"
+      - "0.0.0.0:7203:7203"
+      - "0.0.0.0:7204:7204"
+    networks:
+      - backend
+    image: "m3coordinator_integration:${REVISION}"
+    volumes:
+      - "./m3coordinator.yml:/etc/m3coordinator/m3coordinator.yml"
+      - "./client.crt:/tmp/client.crt"
+      - "./client.key:/tmp/client.key"
+      - "./rootCA.crt:/tmp/rootCA.crt"
+  m3aggregator01:
+    expose:
+      - "6001"
+      - "6000"
+    ports:
+      - "127.0.0.1:6001:6001"
+      - "127.0.0.1:6000:6000"
+    networks:
+      - backend
+    environment:
+      - M3AGGREGATOR_HOST_ID=m3aggregator01
+    image: "m3aggregator_integration:${REVISION}"
+    volumes:
+      - "./m3aggregator.yml:/etc/m3aggregator/m3aggregator.yml"
+      - "./server.crt:/tmp/server.crt"
+      - "./server.key:/tmp/server.key"
+      - "./rootCA.crt:/tmp/rootCA.crt"
+  m3aggregator02:
+    networks:
+      - backend
+    environment:
+      - M3AGGREGATOR_HOST_ID=m3aggregator02
+    image: "m3aggregator_integration:${REVISION}"
+    volumes:
+      - "./m3aggregator.yml:/etc/m3aggregator/m3aggregator.yml"
+      - "./server.crt:/tmp/server.crt"
+      - "./server.key:/tmp/server.key"
+      - "./rootCA.crt:/tmp/rootCA.crt"
+networks:
+  backend:

scripts/docker-integration-tests/aggregator_tls/m3aggregator.yml

@@ -0,0 +1,262 @@
+logging:
+  level: info
+
+metrics:
+  scope:
+    prefix: m3aggregator
+  prometheus:
+    onError: none
+    handlerPath: /metrics
+  sanitization: prometheus
+  samplingRate: 1.0
+  extended: none
+
+http:
+  listenAddress: 0.0.0.0:6001
+  readTimeout: 60s
+  writeTimeout: 60s
+
+rawtcp:
+  listenAddress: 0.0.0.0:6000
+  keepAliveEnabled: true
+  keepAlivePeriod: 1m
+  tls:
+    mode: enforced
+    mTLSEnabled: true
+    certFile: /tmp/server.crt
+    keyFile: /tmp/server.key
+    clientCAFile: /tmp/rootCA.crt
+  retry:
+    initialBackoff: 5ms
+    backoffFactor: 2.0
+    maxBackoff: 1s
+    forever: true
+    jitter: true
+  readBufferSize: 65536
+  protobufIterator:
+    initBufferSize: 1440
+    maxMessageSize: 50000000  # max message size is 50MB
+    bytesPool:
+      buckets:
+        - count: 1024
+          capacity: 2048
+        - count: 512
+          capacity: 4096
+        - count: 256
+          capacity: 8192
+        - count: 128
+          capacity: 16384
+        - count: 64
+          capacity: 32768
+        - count: 32
+          capacity: 65536
+      watermark:
+        low: 0.001
+        high: 0.002
+
+kvClient:
+  etcd:
+    env: override_test_env
+    zone: embedded
+    service: m3aggregator
+    cacheDir: /var/lib/m3kv
+    etcdClusters:
+      - zone: embedded
+        endpoints:
+          - dbnode01:2379
+
+runtimeOptions:
+  kvConfig:
+    environment: override_test_env
+    zone: embedded
+  writeValuesPerMetricLimitPerSecondKey: write-values-per-metric-limit-per-second
+  writeValuesPerMetricLimitPerSecond: 0
+  writeNewMetricLimitClusterPerSecondKey: write-new-metric-limit-cluster-per-second
+  writeNewMetricLimitClusterPerSecond: 0
+  writeNewMetricNoLimitWarmupDuration: 0
+
+aggregator:
+  hostID:
+    resolver: environment
+    envVarName: M3AGGREGATOR_HOST_ID
+  instanceID:
+    type: host_id
+  metricPrefix: ""
+  counterPrefix: ""
+  timerPrefix: ""
+  gaugePrefix: ""
+  aggregationTypes:
+    counterTransformFnType: empty
+    timerTransformFnType: suffix
+    gaugeTransformFnType: empty
+    aggregationTypesPool:
+      size: 1024
+    quantilesPool:
+      buckets:
+        - count: 256
+          capacity: 4
+        - count: 128
+          capacity: 8
+  stream:
+    eps: 0.001
+    capacity: 32
+    streamPool:
+      size: 4096
+    samplePool:
+      size: 4096
+    floatsPool:
+      buckets:
+        - count: 4096
+          capacity: 16
+        - count: 2048
+          capacity: 32
+        - count: 1024
+          capacity: 64
+  client:
+    placementKV:
+      namespace: /placement
+      zone: embedded
+      environment: override_test_env
+    placementWatcher:
+      key: m3aggregator
+      initWatchTimeout: 15s
+    hashType: murmur32
+    shardCutoffLingerDuration: 1m
+    encoder:
+      initBufferSize: 100
+      maxMessageSize: 50000000
+      bytesPool:
+        buckets:
+          - capacity: 16
+            count: 10
+          - capacity: 32
+            count: 20
+        watermark:
+          low: 0.001
+          high: 0.01
+    maxTimerBatchSize: 140
+    queueSize: 1000
+    queueDropType: oldest
+    connection:
+      connectionTimeout: 1s
+      connectionKeepAlive: true
+      writeTimeout: 1s
+      initReconnectThreshold: 2
+      maxReconnectThreshold: 5000
+      reconnectThresholdMultiplier: 2
+      maxReconnectDuration: 1m
+  placementManager:
+    kvConfig:
+      namespace: /placement
+      environment: override_test_env
+      zone: embedded
+    placementWatcher:
+      key: m3aggregator
+      initWatchTimeout: 10s
+  hashType: murmur32
+  bufferDurationBeforeShardCutover: 10m
+  bufferDurationAfterShardCutoff: 10m
+  resignTimeout: 1m
+  flushTimesManager:
+    kvConfig:
+      environment: override_test_env
+      zone: embedded
+    flushTimesKeyFmt: shardset/%d/flush
+    flushTimesPersistRetrier:
+      initialBackoff: 100ms
+      backoffFactor: 2.0
+      maxBackoff: 2s
+      maxRetries: 3
+  electionManager:
+    election:
+      leaderTimeout: 10s
+      resignTimeout: 10s
+      ttlSeconds: 10
+    serviceID:
+      name: m3aggregator
+      environment: override_test_env
+      zone: embedded
+    electionKeyFmt: shardset/%d/lock
+    campaignRetrier:
+      initialBackoff: 100ms
+      backoffFactor: 2.0
+      maxBackoff: 2s
+      forever: true
+      jitter: true
+    changeRetrier:
+      initialBackoff: 100ms
+      backoffFactor: 2.0
+      maxBackoff: 5s
+      forever: true
+      jitter: true
+    resignRetrier:
+      initialBackoff: 100ms
+      backoffFactor: 2.0
+      maxBackoff: 5s
+      forever: true
+      jitter: true
+    campaignStateCheckInterval: 1s
+    shardCutoffCheckOffset: 30s
+  flushManager:
+    checkEvery: 1s
+    jitterEnabled: true
+    maxJitters:
+      - flushInterval: 5s
+        maxJitterPercent: 1.0
+      - flushInterval: 10s
+        maxJitterPercent: 0.5
+      - flushInterval: 1m
+        maxJitterPercent: 0.5
+      - flushInterval: 10m
+        maxJitterPercent: 0.5
+      - flushInterval: 1h
+        maxJitterPercent: 0.25
+    numWorkersPerCPU: 0.5 
+    maxBufferSize: 5m
+    forcedFlushWindowSize: 10s
+  flush:
+    handlers:
+      - dynamicBackend:
+          name: m3msg
+          hashType: murmur32
+          producer:
+            buffer:
+              maxBufferSize: 1000000000 # max buffer before m3msg start dropping data.
+            writer:
+              topicName: aggregated_metrics
+              topicServiceOverride:
+                zone: embedded
+                environment: override_test_env
+              messageRetry:
+                initialBackoff: 1m
+                maxBackoff: 2m
+              messageQueueNewWritesScanInterval: 1s
+              ackErrorRetry:
+                initialBackoff: 2s
+                maxBackoff: 10s
+              connection:
+                dialTimeout: 5s
+                writeTimeout: 5s
+                retry:
+                  initialBackoff: 1s
+                  maxBackoff: 10s
+                flushInterval: 1s
+                writeBufferSize: 16384
+                readBufferSize: 256
+  forwarding:
+    maxSingleDelay: 5s
+  entryTTL: 6h
+  entryCheckInterval: 10m
+  maxTimerBatchSizePerWrite: 140
+  defaultStoragePolicies:
+    - 10s:2d
+  maxNumCachedSourceSets: 2
+  discardNaNAggregatedValues: true
+  entryPool:
+    size: 4096
+  counterElemPool:
+    size: 4096
+  timerElemPool:
+    size: 4096
+  gaugeElemPool:
+    size: 4096