Merge pull request #2 from facebookresearch/master

Update from FB

narwhal/benchmark/README.md

@@ -224,15 +224,15 @@ $ fab plot
 This command creates a latency graph, a throughput graph, and a robustness graph in a folder called `plots` (which is automatically created if it doesn't already exists). You can adjust the plot parameters to filter which curves to add to the plot:
 ```python
 plot_params = {
-    'nodes': [10, 20],
-    'tx_size': 512,
     'faults': [0],
-    'max_latency': [2_000, 5_000]
+    'nodes': [10, 20, 50],
+    'workers': [1],
+    'collocate': True,
+    'tx_size': 512,
+    'max_latency': [3_500, 4_500]
 }
 ```
 
 The first graph ('latency') plots the latency versus the throughput. It shows that the latency is low until a fairly neat threshold after which it drastically increases. Determining this threshold is crucial to understand the limits of the system. 
 
 Another challenge is comparing apples-to-apples between different deployments of the system. The challenge here is again that latency and throughput are interdependent, as a result a throughput/number of nodes chart could be tricky to produce fairly. The way to do it is to define a maximum latency and measure the throughput at this point instead of simply pushing every system to its peak throughput (where latency is meaningless). The second graph ('tps') plots the maximum achievable throughput under a maximum latency for different numbers of nodes.
-
-The last graph ('robustness') plots the throughput versus the input rate (and provides no information on latency). This graph is a bit redundant given the other two but clearly shows the threshold where the system saturates (and the throughput may decrease). This threshold is crucial to determine how to configure a rate-limiter to block excess transactions from entering the system.

narwhal/benchmark/benchmark/aggregate.py

@@ -108,7 +108,6 @@ def print(self):
             self._print_latency(),
             self._print_tps(scalability=False),
             self._print_tps(scalability=True),
-            self._print_robustness()
         ]
         for name, records in results:
             for setup, values in records.items():
@@ -181,14 +180,3 @@ def _print_tps(self, scalability):
 
         [v.sort(key=lambda x: x[0]) for v in organized.values()]
         return 'tps', organized
-
-    def _print_robustness(self):
-        records = deepcopy(self.records)
-        organized = defaultdict(list)
-        for setup, result in records.items():
-            rate = setup.rate
-            setup.rate = 'x'
-            organized[setup] += [(rate, result)]
-
-        [v.sort(key=lambda x: x[0]) for v in organized.values()]
-        return 'robustness', organized

narwhal/benchmark/benchmark/config.py

@@ -1,6 +1,6 @@
 # Copyright(C) Facebook, Inc. and its affiliates.
 from json import dump, load
-from collections import defaultdict, OrderedDict
+from collections import OrderedDict
 
 
 class ConfigError(Exception):

narwhal/benchmark/aws/instance.py → narwhal/benchmark/benchmark/instance.py

@@ -5,7 +5,7 @@
 from time import sleep
 
 from benchmark.utils import Print, BenchError, progress_bar
-from aws.settings import Settings, SettingsError
+from benchmark.settings import Settings, SettingsError
 
 
 class AWSError(Exception):

narwhal/benchmark/benchmark/logs.py

@@ -96,7 +96,7 @@ def _parse_clients(self, log):
 
     def _parse_primaries(self, log):
         if search(r'(?:panicked|Error)', log) is not None:
-            raise ParseError('Node(s) panicked')
+            raise ParseError('Primary(s) panicked')
 
         tmp = findall(r'\[(.*Z) .* Created B\d+\([^ ]+\) -> ([^ ]+=)', log)
         tmp = [(d, self._to_posix(t)) for t, d in tmp]

narwhal/benchmark/benchmark/plot.py

@@ -148,16 +148,6 @@ def plot_latency(cls, files, scalability):
         ploter = cls(files)
         ploter._plot(x_label, y_label, ploter._latency, z_axis, 'latency')
 
-    @classmethod
-    def plot_robustness(cls, files, scalability):
-        assert isinstance(files, list)
-        assert all(isinstance(x, str) for x in files)
-        z_axis = cls.workers if scalability else cls.nodes
-        x_label = 'Input rate (tx/s)'
-        y_label = ['Throughput (tx/s)', 'Throughput (MB/s)']
-        ploter = cls(files)
-        ploter._plot(x_label, y_label, ploter._tps, z_axis, 'robustness')
-
     @classmethod
     def plot_tps(cls, files, scalability):
         assert isinstance(files, list)
@@ -180,7 +170,7 @@ def plot(cls, params_dict):
 
         # Make the latency, tps, and robustness graphs.
         iterator = params.workers if params.scalability() else params.nodes
-        latency_files, robustness_files, tps_files = [], [], []
+        latency_files, tps_files = [], []
         for f in params.faults:
             for x in iterator:
                 latency_files += glob(
@@ -194,17 +184,6 @@ def plot(cls, params_dict):
                         params.tx_size,
                     )
                 )
-                robustness_files += glob(
-                    PathMaker.agg_file(
-                        'robustness',
-                        f,
-                        x if not params.scalability() else params.nodes[0],
-                        x if params.scalability() else params.workers[0],
-                        params.collocate,
-                        'x',
-                        params.tx_size,
-                    )
-                )
 
             for l in params.max_latency:
                 tps_files += glob(
@@ -222,4 +201,3 @@ def plot(cls, params_dict):
 
         cls.plot_latency(latency_files, params.scalability())
         cls.plot_tps(tps_files, params.scalability())
-        cls.plot_robustness(latency_files, params.scalability())

narwhal/benchmark/aws/remote.py → narwhal/benchmark/benchmark/remote.py

@@ -1,6 +1,5 @@
 # Copyright(C) Facebook, Inc. and its affiliates.
 from collections import OrderedDict
-from os import error
 from fabric import Connection, ThreadingGroup as Group
 from fabric.exceptions import GroupException
 from paramiko import RSAKey
@@ -15,7 +14,7 @@
 from benchmark.utils import BenchError, Print, PathMaker, progress_bar
 from benchmark.commands import CommandMaker
 from benchmark.logs import LogParser, ParseError
-from aws.instance import InstanceManager
+from benchmark.instance import InstanceManager
 
 
 class FabricError(Exception):
@@ -153,7 +152,7 @@ def _update(self, hosts, collocate):
             'source $HOME/.cargo/env',
             f'(cd {self.settings.repo_name}/node && {CommandMaker.compile()})',
             CommandMaker.alias_binaries(
-                f'./{self.settings.repo_name}/rust/target/release/'
+                f'./{self.settings.repo_name}/target/release/'
             )
         ]
         g = Group(*ips, user='ubuntu', connect_kwargs=self.connect)

narwhal/benchmark/data/README.md

@@ -0,0 +1,52 @@
+# Experimental Data
+
+This folder contains some raw data and plots obtained running a geo-replicated benchmark on AWS as explained in the [benchmark's readme file](https://github.com/facebookresearch/narwhal/tree/master/benchmark#readme). The results are taken running the code tagged as [v0.1.1](https://github.com/facebookresearch/narwhal/tree/v0.1.1).
+
+### Filename format
+The filename format of raw data is the following:
+```
+bench-FAULTS-NODES-WORKERS-COLLOCATE-INPUT_RATE-TX_SIZE.txt
+```
+where:
+- `FAULTS`: The number of faulty (dead) nodes.
+- `NODES`: The number of nodes in the testbed.
+- `WORKERS`: The number of workers per node.
+- `COLLOCATE`: Whether the primary and its worker are collocated on the same machine.
+- `INPUT_RATE`: The total rate at which clients submit transactions to the system.
+- `TX_SIZE`: The size of each transactions (in bytes).
+
+For instance, a file called `bench-0-50-1-True-100000-512.txt` indicates it contains results of a benchmark run with 50 nodes, 1 worker per node collocated on the same machine as the primary, 100K input rate, a transaction size of 512B, and 0 faulty nodes.
+
+### Experimental step
+The content of our [settings.json](https://github.com/facebookresearch/narwhal/blob/master/benchmark/settings.json) file looks as follows:
+```json
+{
+    "key": {
+        "name": "aws",
+        "path": "/absolute/key/path"
+    },
+    "port": 5000,
+    "repo": {
+        "name": "narwhal",
+        "url": "https://github.com/facebookresearch/narwhal",
+        "branch": "master"
+    },
+    "instances": {
+        "type": "m5d.8xlarge",
+        "regions": ["us-east-1", "eu-north-1", "ap-southeast-2", "us-west-1", "ap-northeast-1"]
+    }
+}
+```
+We set the following `node_params` in our [fabfile](https://github.com/facebookresearch/narwhal/blob/master/benchmark/fabfile.py):
+```python
+node_params = {
+    'header_size': 1_000,  # bytes
+    'max_header_delay': 200,  # ms
+    'gc_depth': 50,  # rounds
+    'sync_retry_delay': 10_000,  # ms
+    'sync_retry_nodes': 3,  # number of nodes
+    'batch_size': 500_000,  # bytes
+    'max_batch_delay': 200  # ms
+}
+```
+

narwhal/benchmark/data/tusk/bench-0-10-1-True-10000-512.txt

@@ -0,0 +1,60 @@
+
+-----------------------------------------
+ SUMMARY:
+-----------------------------------------
+ + CONFIG:
+ Faults: 0 node(s)
+ Committee size: 10 node(s)
+ Worker(s) per node: 1 worker(s)
+ Collocate primary and workers: True
+ Input rate: 10,000 tx/s
+ Transaction size: 512 B
+ Execution time: 301 s
+
+ Header size: 1,000 B
+ Max header delay: 200 ms
+ GC depth: 50 round(s)
+ Sync retry delay: 10,000 ms
+ Sync retry nodes: 3 node(s)
+ batch size: 500,000 B
+ Max batch delay: 200 ms
+
+ + RESULTS:
+ Consensus TPS: 9,522 tx/s
+ Consensus BPS: 4,875,199 B/s
+ Consensus latency: 2,376 ms
+
+ End-to-end TPS: 9,501 tx/s
+ End-to-end BPS: 4,864,434 B/s
+ End-to-end latency: 2,910 ms
+-----------------------------------------
+
+-----------------------------------------
+ SUMMARY:
+-----------------------------------------
+ + CONFIG:
+ Faults: 0 node(s)
+ Committee size: 10 node(s)
+ Worker(s) per node: 1 worker(s)
+ Collocate primary and workers: True
+ Input rate: 10,000 tx/s
+ Transaction size: 512 B
+ Execution time: 301 s
+
+ Header size: 1,000 B
+ Max header delay: 200 ms
+ GC depth: 50 round(s)
+ Sync retry delay: 10,000 ms
+ Sync retry nodes: 3 node(s)
+ batch size: 500,000 B
+ Max batch delay: 200 ms
+
+ + RESULTS:
+ Consensus TPS: 9,514 tx/s
+ Consensus BPS: 4,871,264 B/s
+ Consensus latency: 2,371 ms
+
+ End-to-end TPS: 9,490 tx/s
+ End-to-end BPS: 4,859,081 B/s
+ End-to-end latency: 2,910 ms
+-----------------------------------------

narwhal/benchmark/data/tusk/bench-0-10-1-True-100000-512.txt

@@ -0,0 +1,60 @@
+
+-----------------------------------------
+ SUMMARY:
+-----------------------------------------
+ + CONFIG:
+ Faults: 0 node(s)
+ Committee size: 10 node(s)
+ Worker(s) per node: 1 worker(s)
+ Collocate primary and workers: True
+ Input rate: 100,000 tx/s
+ Transaction size: 512 B
+ Execution time: 301 s
+
+ Header size: 1,000 B
+ Max header delay: 200 ms
+ GC depth: 50 round(s)
+ Sync retry delay: 10,000 ms
+ Sync retry nodes: 3 node(s)
+ batch size: 500,000 B
+ Max batch delay: 200 ms
+
+ + RESULTS:
+ Consensus TPS: 93,666 tx/s
+ Consensus BPS: 47,956,782 B/s
+ Consensus latency: 2,423 ms
+
+ End-to-end TPS: 93,227 tx/s
+ End-to-end BPS: 47,732,331 B/s
+ End-to-end latency: 3,034 ms
+-----------------------------------------
+
+-----------------------------------------
+ SUMMARY:
+-----------------------------------------
+ + CONFIG:
+ Faults: 0 node(s)
+ Committee size: 10 node(s)
+ Worker(s) per node: 1 worker(s)
+ Collocate primary and workers: True
+ Input rate: 100,000 tx/s
+ Transaction size: 512 B
+ Execution time: 301 s
+
+ Header size: 1,000 B
+ Max header delay: 200 ms
+ GC depth: 50 round(s)
+ Sync retry delay: 10,000 ms
+ Sync retry nodes: 3 node(s)
+ batch size: 500,000 B
+ Max batch delay: 200 ms
+
+ + RESULTS:
+ Consensus TPS: 93,146 tx/s
+ Consensus BPS: 47,690,875 B/s
+ Consensus latency: 2,384 ms
+
+ End-to-end TPS: 92,758 tx/s
+ End-to-end BPS: 47,491,983 B/s
+ End-to-end latency: 3,000 ms
+-----------------------------------------

narwhal/benchmark/data/tusk/bench-0-10-1-True-110000-512.txt

@@ -0,0 +1,60 @@
+
+-----------------------------------------
+ SUMMARY:
+-----------------------------------------
+ + CONFIG:
+ Faults: 0 node(s)
+ Committee size: 10 node(s)
+ Worker(s) per node: 1 worker(s)
+ Collocate primary and workers: True
+ Input rate: 110,000 tx/s
+ Transaction size: 512 B
+ Execution time: 301 s
+
+ Header size: 1,000 B
+ Max header delay: 200 ms
+ GC depth: 50 round(s)
+ Sync retry delay: 10,000 ms
+ Sync retry nodes: 3 node(s)
+ batch size: 500,000 B
+ Max batch delay: 200 ms
+
+ + RESULTS:
+ Consensus TPS: 100,446 tx/s
+ Consensus BPS: 51,428,456 B/s
+ Consensus latency: 2,374 ms
+
+ End-to-end TPS: 100,064 tx/s
+ End-to-end BPS: 51,232,980 B/s
+ End-to-end latency: 2,974 ms
+-----------------------------------------
+
+-----------------------------------------
+ SUMMARY:
+-----------------------------------------
+ + CONFIG:
+ Faults: 0 node(s)
+ Committee size: 10 node(s)
+ Worker(s) per node: 1 worker(s)
+ Collocate primary and workers: True
+ Input rate: 110,000 tx/s
+ Transaction size: 512 B
+ Execution time: 301 s
+
+ Header size: 1,000 B
+ Max header delay: 200 ms
+ GC depth: 50 round(s)
+ Sync retry delay: 10,000 ms
+ Sync retry nodes: 3 node(s)
+ batch size: 500,000 B
+ Max batch delay: 200 ms
+
+ + RESULTS:
+ Consensus TPS: 99,983 tx/s
+ Consensus BPS: 51,191,107 B/s
+ Consensus latency: 2,372 ms
+
+ End-to-end TPS: 99,584 tx/s
+ End-to-end BPS: 50,986,971 B/s
+ End-to-end latency: 2,971 ms
+-----------------------------------------