M3DB client asynchronous replication (#1859)

* Parse multiple services from dynamic client config * WIP added client opts support for multiple clusters * Returned to Options + MultiClusterOptions * Added replication logic to replicatedSession * Proxy Open method to sessions * Removed accidental glide.yaml * Move async flag outside etcd configuration * Simplified new client logic * Added PooledWorkerPool todo * Instrumented replicated session * Removed obselete param from etcd config * Updated obselete etcd test * Fixes from rebase * Made replicated session more testable * Added replicated session tests * Reinstated proto schema constructor code * Added support for multiple/async static configs * Updated import to src/x * Pass correct Options to coordinator client * Updated server config to use the syncronous cluster only * MultiClusterOptions => ReplicatedOptions * trying really hard to put a square peg in a round hole * Adapt query server logic to match replicated config * Updated m3sch/query servers to use replicated options * Updated repair tests * Removed loading proto schema from KVStore in config * missed a line * Removed unused var * regen mocks * fix integration compilation issues * fix configuration test * fix more compilation issues * fix more compilation issues * fix runtime interface * fix more compilation issue * fix more compilation errors * rename file + add copyright notice * fix broken test * fix more tests * add prints and handle error * fix docker integration tests * Improved environment config tests * Moved Validate call to environment config.Configure method * Grouped common methods in ReplicationOptions * Renamed test * Updated mocks * prints * remove prints * Merged ReplicatedOptions in Options * Updated occurences of options in server code * Fixed broken integration test * Further integration test fixes * Missed one * Removed replicationoptions from test * Improved environment config validation tests * Update environment config to reflect intended config schema * removed redundant code * Log error when closing async session * Validate in SyncCluster * Skip chan output unless chan is allocated * Avoid allocation in replicate * Regen'd mocks * Comment formatting * Nits
m3db · Sep 13, 2019 · 885491a · 885491a
1 parent 5898e6a
commit 885491a
Show file tree

Hide file tree

Showing 30 changed files with 1,638 additions and 231 deletions.
diff --git a/glide.lock b/glide.lock
diff --git a/scripts/docker-integration-tests/multi_cluster_write/docker-compose.yml b/scripts/docker-integration-tests/multi_cluster_write/docker-compose.yml
@@ -0,0 +1,88 @@
+version: "3.5"
+services:
+  cluster_a_dbnode01:
+    expose:
+      - "9000-9004"
+      - "2379-2380"
+    ports:
+      - "0.0.0.0:9012:9002"
+      - "0.0.0.0:9013:9003"
+    networks:
+      - backend
+    image: "m3dbnode_integration:${REVISION}"
+    environment:
+      - M3DB_HOST_ID=cluster_a_m3db_local_1
+    volumes:
+      - "./m3dbnode-cluster-a.yml:/etc/m3dbnode/m3dbnode.yml"
+  cluster_a_dbnode02:
+    expose:
+      - "9000-9004"
+      - "2379-2380"
+    ports:
+      - "0.0.0.0:9022:9002"
+      - "0.0.0.0:9023:9003"
+    networks:
+      - backend
+    image: "m3dbnode_integration:${REVISION}"
+    environment:
+      - M3DB_HOST_ID=cluster_a_m3db_local_2
+    volumes:
+      - "./m3dbnode-cluster-a.yml:/etc/m3dbnode/m3dbnode.yml"
+  cluster_a_coordinator01:
+    expose:
+      - "7201"
+      - "7203"
+      - "7204"
+    ports:
+      - "0.0.0.0:7201:7201"
+      - "0.0.0.0:7203:7203"
+      - "0.0.0.0:7204:7204"
+    networks:
+      - backend
+    image: "m3coordinator_integration:${REVISION}"
+    volumes:
+      - "./m3coordinator-cluster-a.yml/:/etc/m3coordinator/m3coordinator.yml"
+  cluster_b_dbnode01:
+    expose:
+      - "9000-9004"
+      - "2379-2380"
+    ports:
+      - "0.0.0.0:9112:9002"
+      - "0.0.0.0:9113:9003"
+    networks:
+      - backend
+    image: "m3dbnode_integration:${REVISION}"
+    environment:
+      - M3DB_HOST_ID=cluster_b_m3db_local_1
+    volumes:
+      - "./m3dbnode-cluster-b.yml:/etc/m3dbnode/m3dbnode.yml"
+  cluster_b_dbnode02:
+    expose:
+      - "9000-9004"
+      - "2379-2380"
+    ports:
+      - "0.0.0.0:9122:9002"
+      - "0.0.0.0:9123:9003"
+    networks:
+      - backend
+    image: "m3dbnode_integration:${REVISION}"
+    environment:
+      - M3DB_HOST_ID=cluster_b_m3db_local_2
+    volumes:
+      - "./m3dbnode-cluster-b.yml:/etc/m3dbnode/m3dbnode.yml"
+  cluster_b_coordinator01:
+    expose:
+      - "7201"
+      - "7203"
+      - "7204"
+    ports:
+      - "0.0.0.0:17201:7201"
+      - "0.0.0.0:17203:7203"
+      - "0.0.0.0:17204:7204"
+    networks:
+      - backend
+    image: "m3coordinator_integration:${REVISION}"
+    volumes:
+      - "./m3coordinator-cluster-b.yml:/etc/m3coordinator/m3coordinator.yml"
+networks:
+  backend:
diff --git a/scripts/docker-integration-tests/multi_cluster_write/m3coordinator-cluster-a.yml b/scripts/docker-integration-tests/multi_cluster_write/m3coordinator-cluster-a.yml
@@ -0,0 +1,46 @@
+listenAddress:
+  type: "config"
+  value: "0.0.0.0:7201"
+
+logging:
+  level: info
+
+metrics:
+  scope:
+    prefix: "coordinator"
+  prometheus:
+    handlerPath: /metrics
+    listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved
+  sanitization: prometheus
+  samplingRate: 1.0
+  extended: none
+
+limits:
+  perQuery:
+    maxFetchedSeries: 100
+
+clusters:
+  - namespaces:
+      - namespace: agg
+        type: aggregated
+        retention: 10h
+        resolution: 15s
+      - namespace: unagg
+        type: unaggregated
+        retention: 10h
+    client:
+      config:
+        service:
+          env: default_env
+          zone: embedded
+          service: m3db
+          cacheDir: /var/lib/m3kv
+          etcdClusters:
+            - zone: embedded
+              endpoints:
+                - cluster_a_dbnode01:2379
+      writeConsistencyLevel: majority
+      readConsistencyLevel: unstrict_majority
+
+tagOptions:
+  idScheme: quoted
diff --git a/scripts/docker-integration-tests/multi_cluster_write/m3coordinator-cluster-b.yml b/scripts/docker-integration-tests/multi_cluster_write/m3coordinator-cluster-b.yml
@@ -0,0 +1,46 @@
+listenAddress:
+  type: "config"
+  value: "0.0.0.0:7201"
+
+logging:
+  level: info
+
+metrics:
+  scope:
+    prefix: "coordinator"
+  prometheus:
+    handlerPath: /metrics
+    listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved
+  sanitization: prometheus
+  samplingRate: 1.0
+  extended: none
+
+limits:
+  perQuery:
+    maxFetchedSeries: 100
+
+clusters:
+  - namespaces:
+      - namespace: agg
+        type: aggregated
+        retention: 10h
+        resolution: 15s
+      - namespace: unagg
+        type: unaggregated
+        retention: 10h
+    client:
+      config:
+        service:
+          env: default_env
+          zone: embedded
+          service: m3db
+          cacheDir: /var/lib/m3kv
+          etcdClusters:
+            - zone: embedded
+              endpoints:
+                - cluster_b_dbnode01:2379
+      writeConsistencyLevel: majority
+      readConsistencyLevel: unstrict_majority
+
+tagOptions:
+  idScheme: quoted
diff --git a/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-a.yml b/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-a.yml
@@ -0,0 +1,100 @@
+db:
+  logging:
+    level: info
+
+  tracing:
+    backend: jaeger
+    jaeger:
+      reporter:
+        localAgentHostPort: jaeger:6831
+      sampler:
+        type: const
+        param: 1
+
+  metrics:
+    prometheus:
+      handlerPath: /metrics
+    sanitization: prometheus
+    samplingRate: 1.0
+    extended: detailed
+
+  listenAddress: 0.0.0.0:9000
+  clusterListenAddress: 0.0.0.0:9001
+  httpNodeListenAddress: 0.0.0.0:9002
+  httpClusterListenAddress: 0.0.0.0:9003
+  debugListenAddress: 0.0.0.0:9004
+
+  hostID:
+    resolver: environment
+    envVarName: M3DB_HOST_ID
+
+  client:
+    writeConsistencyLevel: majority
+    readConsistencyLevel: unstrict_majority
+    config:
+      services:
+        - service:
+            env: default_env
+            zone: embedded
+            service: m3db
+            cacheDir: /var/lib/m3kv
+            etcdClusters:
+              - zone: embedded
+                endpoints:
+                    - cluster_a_dbnode01:2379
+        - service:
+            env: default_env
+            zone: embedded
+            service: m3db
+            cacheDir: /var/lib/m3kv
+            etcdClusters:
+                - zone: embedded
+                  endpoints:
+                      - cluster_b_dbnode01:2379
+          async: true
+
+  gcPercentage: 100
+
+  writeNewSeriesAsync: true
+  writeNewSeriesLimitPerSecond: 1048576
+  writeNewSeriesBackoffDuration: 2ms
+
+  bootstrap:
+    # Intentionally disable peers bootstrapper to ensure it doesn't interfere with test.
+    bootstrappers:
+        - filesystem
+        - commitlog
+        - uninitialized_topology
+    commitlog:
+      returnUnfulfilledForCorruptCommitLogFiles: false
+
+  cache:
+    series:
+      policy: lru
+    postingsList:
+      size: 262144
+
+  commitlog:
+    flushMaxBytes: 524288
+    flushEvery: 1s
+    queue:
+      calculationType: fixed
+      size: 2097152
+
+  fs:
+    filePathPrefix: /var/lib/m3db
+
+  config:
+    service:
+      env: default_env
+      zone: embedded
+      service: m3db
+      cacheDir: /var/lib/m3kv
+      etcdClusters:
+        - zone: embedded
+          endpoints:
+            - cluster_a_dbnode01:2379
+    seedNodes:
+      initialCluster:
+        - hostID: cluster_a_m3db_local_1
+          endpoint: http://cluster_a_dbnode01:2380