Refactor: Benchmarking tool

tests/riot.go

@@ -288,6 +288,22 @@ func stats(c *Config, t Task, message string) {
 	}
 }
 
+func benchmarkAdd(numFollowers, numReqests, readConcurrency, writeConcurrency, offset int) {
+	fmt.Println("\nStarting benchmark run...")
+
+	client := &http.Client{}
+
+	c := NewDefaultConfig()
+	c.req.client = client
+	c.numRequests = numReqests
+	c.maxGoRoutines = writeConcurrency
+	c.startVersion = offset
+	c.req.expectedStatusCode = 201
+	c.req.endpoint += "/events"
+	stats(c, addSampleEvents, "Add")
+
+}
+
 func benchmarkMembership(numFollowers, numReqests, readConcurrency, writeConcurrency int) {
 	fmt.Println("\nStarting benchmark run...")
 	var queryWg sync.WaitGroup
@@ -447,41 +463,56 @@ func benchmarkIncremental(numFollowers, numReqests, readConcurrency, writeConcur
 }
 
 var (
+	wantAdd          bool
+	wantIncremental  bool
 	wantMembership   bool
 	offload          bool
 	charts           bool
 	incrementalDelta int
+	offset           int
 	numRequests      int
 	readConcurrency  int
 	writeConcurrency int
 )
 
 func init() {
 	const (
+		// Default values
+		defaultWantAdd          = false
+		defaultWantIncremental  = false
 		defaultWantMembership   = false
+		defaultOffset           = 0
 		defaultIncrementalDelta = 1000
 		defaultNumRequests      = 100000
-		usage                   = "Benchmark MembershipProof"
-		usageDelta              = "Specify delta for the IncrementalProof"
-		usageNumRequests        = "Number of requests for the attack"
-		usageReadConcurrency    = "Set read concurrency value"
-		usageWriteConcurrency   = "Set write concurrency value"
-		usageOffload            = "Perform reads only on %50 of the cluster size (With cluster size 2 reads will be perofmed only on follower1)"
-		usageCharts             = "Create charts while executing the benchmarks. Output: graph-$testname.png"
+
+		// Usage
+		usage                 = "Benchmark MembershipProof"
+		usageDelta            = "Specify delta for the IncrementalProof"
+		usageNumRequests      = "Number of requests for the attack"
+		usageReadConcurrency  = "Set read concurrency value"
+		usageWriteConcurrency = "Set write concurrency value"
+		usageOffload          = "Perform reads only on %50 of the cluster size (With cluster size 2 reads will be perofmed only on follower1)"
+		usageCharts           = "Create charts while executing the benchmarks. Output: graph-$testname.png"
+		usageOffset           = "The starting version from which we start the load"
+		usageWantAdd          = "Execute add benchmark"
+		usageWantIncremental  = "Execute Incremental benchmark"
 	)
 
 	// Create a default config to use as default values in flags
 	config := NewDefaultConfig()
 
+	flag.BoolVar(&wantAdd, "add", defaultWantAdd, usageWantAdd)
 	flag.BoolVar(&wantMembership, "membership", defaultWantMembership, usage)
 	flag.BoolVar(&wantMembership, "m", defaultWantMembership, usage+" (shorthand)")
+	flag.BoolVar(&wantIncremental, "incremental", defaultWantIncremental, usageWantIncremental)
 	flag.BoolVar(&offload, "offload", false, usageOffload)
 	flag.BoolVar(&charts, "charts", false, usageCharts)
 	flag.IntVar(&incrementalDelta, "delta", defaultIncrementalDelta, usageDelta)
 	flag.IntVar(&incrementalDelta, "d", defaultIncrementalDelta, usageDelta+" (shorthand)")
 	flag.IntVar(&numRequests, "n", defaultNumRequests, usageNumRequests)
 	flag.IntVar(&readConcurrency, "r", config.maxGoRoutines, usageReadConcurrency)
 	flag.IntVar(&writeConcurrency, "w", config.maxGoRoutines, usageWriteConcurrency)
+	flag.IntVar(&offset, "offset", defaultOffset, usageOffset)
 
 }
 
@@ -533,11 +564,18 @@ func main() {
 		fmt.Printf("Offload: %v | %d\n", offload, n)
 	}
 
+	if wantAdd {
+		fmt.Print("Benchmark ADD")
+		benchmarkAdd(n, numRequests, readConcurrency, writeConcurrency, offset)
+	}
+
 	if wantMembership {
-		fmt.Println("Benchmark MEMBERSHIP")
+		fmt.Print("Benchmark MEMBERSHIP")
 		benchmarkMembership(n, numRequests, readConcurrency, writeConcurrency)
-	} else {
-		fmt.Println("Benchmark INCREMENTAL")
+	}
+
+	if wantIncremental {
+		fmt.Print("Benchmark INCREMENTAL")
 		benchmarkIncremental(n, numRequests, readConcurrency, writeConcurrency)
 	}
 }

tests/start_profiler

@@ -0,0 +1,49 @@
+#!/usr/bin/env sh
+
+# Copyright 2018 Banco Bilbao Vizcaya Argentaria, S.A.
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+echo "export PROFILING_RATE_HZ=30 (in seconds) env variable to set the default profiling delay. Default profiling rate is 30. Output: results/cpu|mem.pb.gz"
+
+function check_profiling { 
+    while ps ux | grep [p]prof > /dev/null
+    do
+        echo "Profiler running"
+        sleep $PROFILING_RATE_HZ
+    done
+    echo "Profiler stopped"
+}
+
+function profiler_cmd {
+    ELAPSED=0
+    while curl -s -X POST http://localhost:8080/health-check 2>&1 > /dev/null ; do
+        (BALLOON_VERSION=$(curl -s http://localhost:6060/debug/vars | awk -F '"version": ' '/balloon_stats/ {print $2}' | tr -d '},')
+        # Avoid empty version beacause preload process is not already started... 
+        : ${BALLOON_VERSION:=0}
+        go tool pprof -proto -sample_index=alloc_objects -output results/$(date +%s)-${BALLOON_VERSION}-mem-alloc-objects.pb.gz http://localhost:6060/debug/pprof/heap
+        go tool pprof -proto -sample_index=alloc_space -output results/$(date +%s)-${BALLOON_VERSION}-mem-alloc-space-top.pb.gz http://localhost:6060/debug/pprof/heap
+        go tool pprof -proto -sample_index=inuse_objects -output results/$(date +%s)-${BALLOON_VERSION}-mem-inuse-objects.pb.gz http://localhost:6060/debug/pprof/heap
+        go tool pprof -proto -sample_index=inuse_space -output results/$(date +%s)-${BALLOON_VERSION}-mem-inuse-space-top.pb.gz http://localhost:6060/debug/pprof/heap
+        go tool pprof -proto -seconds 30 -output results/$(date +%s)-${BALLOON_VERSION}-cpu-profile.pb.gz http://localhost:6060) 2>&1 > /dev/null
+        sleep $PROFILING_RATE_HZ
+    done
+}
+
+: ${PROFILING_RATE_HZ:=30}
+mkdir -p results
+echo "PROFILING_RATE_HZ=${PROFILING_RATE_HZ}s"
+profiler_cmd
+check_profiling
+
+

tests/start_server

@@ -15,46 +15,25 @@
 # limitations under the License.
 
 echo "export CLUSTER_SIZE=N [2|4] env variable to execute the benchmark in replica mode. N number of folowers"
-echo "export PROFILING=true env variable to enable go profiling. Default profiling rate is 30. Output: results/cpu|mem.pb.gz"
-echo "export PROFILING_RATE=30 (in seconds) env variable to set the default profiling delay"
+echo "export PROFILING=true env variable to enable go profiling"
 
 echo Create id_ed25519 key
 ssh-keygen -t ed25519 -N '' -f /var/tmp/id_ed25519
 
-function profiler_cmd {
-    ELAPSED=0
-    while curl -s -X POST http://localhost:8080/health-check 2>&1 > /dev/null ; do
-        (BALLOON_VERSION=$(curl -s http://localhost:6060/debug/vars | awk -F '"version": ' '/balloon_stats/ {print $2}' | tr -d '},')
-        # Avoid empty version because preload process is not already started...
-        : ${BALLOON_VERSION:=0}
-        go tool pprof -proto -sample_index=alloc_objects -output results/$(date +%s)-${BALLOON_VERSION}-mem-alloc-objects.pb.gz http://localhost:6060/debug/pprof/heap
-        go tool pprof -proto -sample_index=alloc_space -output results/$(date +%s)-${BALLOON_VERSION}-mem-alloc-space-top.pb.gz http://localhost:6060/debug/pprof/heap
-        go tool pprof -proto -sample_index=inuse_objects -output results/$(date +%s)-${BALLOON_VERSION}-mem-inuse-objects.pb.gz http://localhost:6060/debug/pprof/heap
-        go tool pprof -proto -sample_index=inuse_space -output results/$(date +%s)-${BALLOON_VERSION}-mem-inuse-space-top.pb.gz http://localhost:6060/debug/pprof/heap
-        go tool pprof -proto -seconds 30 -output results/$(date +%s)-${BALLOON_VERSION}-cpu-profile.pb.gz http://localhost:6060) 2>&1 > /dev/null
-        sleep $PROFILING_RATE
-    done
-}
-
 if [ ! -z "$PROFILING" ]
 then
     echo "PROFILING=enabled"
-    : ${PROFILING_RATE:=30}
     mkdir -p results
     PROFILING=--profiling
-    echo "PROFILING_RATE=${PROFILING_RATE}s"
-    PROFILER_CMD="profiler_cmd"
 else
     echo PROFILING=disabled
-    PROFILER_CMD="true"
 fi
 
 if [ -z "$CLUSTER_SIZE" ]
 then
     echo Starting single server...
     go run ../main.go start -k pepe -p $(mktemp -d /var/tmp/demo.XXX) --raftpath $(mktemp -d /var/tmp/demo.XXX) -y /var/tmp/id_ed25519 -l error --http-addr :8080 --raft-addr :9000 --mgmt-addr :8090 $PROFILING &
     sleep 10
-    $PROFILER_CMD &
     echo done.
 else
     echo Starting cluster mode...
@@ -65,6 +44,5 @@ else
         go run ../main.go start -k pepe -p $(mktemp -d /var/tmp/demo.XXX) --raftpath $(mktemp -d /var/tmp/demo.XXX) -y /var/tmp/id_ed25519 -l error --http-addr :808$i --join-addr :8090 --raft-addr :900$i --mgmt-addr :809$i --node-id node$i &
     done
     sleep 10
-    $PROFILER_CMD &
     echo done.
 fi

tests/stop_server

@@ -14,20 +14,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-function check_profiling { 
-    while ps ux | grep [p]prof > /dev/null
-    do
-        echo "Waiting profiler task"
-        sleep 1
-    done
-}
-
 echo Cleanup...
 if [ $(uname) == "Darwin" ]; then
-    check_profiling
     killall main || exit 0
 else
-    check_profiling
     fuser -k -n tcp 8080 8081 8082 8083 8084 || true
 fi
 sleep 5

tests/stress

@@ -16,6 +16,7 @@
 
 
 ./start_server
+./start_profiler
 
 go run riot.go "$@"