Added end to end test for mem ballast (#80)

This commit adds an end to end test to verify performance gain acheived
with memory ballast

It mainly checks the CPU usage and does not test against memory usage.
I'll be submitting another PR that will specifically test memory
behaviour and ensure the ballast does not actually consume memory.

`TestNoBackend10kSPS` test was very reliably consuming >20 CPU on my
machine so I increased the accepted values to 30 for this test.

go.mod

@@ -6,24 +6,20 @@ require (
 	contrib.go.opencensus.io/exporter/prometheus v0.1.0
 	contrib.go.opencensus.io/exporter/zipkin v0.1.1
 	contrib.go.opencensus.io/resource v0.1.1
-	github.com/VividCortex/gohistogram v1.0.0 // indirect
 	github.com/apache/thrift v0.0.0-20161221203622-b2a4d4ae21c7
-	github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b // indirect
 	github.com/census-instrumentation/opencensus-proto v0.2.0
 	github.com/go-kit/kit v0.8.0
-	github.com/gogo/googleapis v1.2.0 // indirect
 	github.com/golang/protobuf v1.3.1
 	github.com/google/go-cmp v0.3.0
 	github.com/gorilla/mux v1.6.2
 	github.com/grpc-ecosystem/grpc-gateway v1.9.0
-	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/jaegertracing/jaeger v1.9.0
 	github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024
 	github.com/omnition/scribe-go v0.0.0-20190131012523-9e3c68f31124
+	github.com/open-telemetry/opentelemetry-service/testbed v0.0.0-20190702215910-6b130d62dbaa // indirect
 	github.com/openzipkin/zipkin-go v0.1.6
 	github.com/orijtech/prometheus-go-metrics-exporter v0.0.3-0.20190313163149-b321c5297f60
 	github.com/pkg/errors v0.8.0
-	github.com/prashantv/protectmem v0.0.0-20171002184600-e20412882b3a // indirect
 	github.com/prometheus/client_golang v0.9.3
 	github.com/prometheus/common v0.4.0
 	github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084
@@ -33,7 +29,6 @@ require (
 	github.com/spf13/cast v1.3.0
 	github.com/spf13/cobra v0.0.3
 	github.com/spf13/viper v1.4.0
-	github.com/streadway/quantile v0.0.0-20150917103942-b0c588724d25 // indirect
 	github.com/stretchr/testify v1.3.0
 	github.com/uber-go/atomic v1.4.0 // indirect
 	github.com/uber/jaeger-lib v2.0.0+incompatible

testbed/go.mod

@@ -4,9 +4,9 @@ go 1.12
 
 require (
 	contrib.go.opencensus.io/exporter/jaeger v0.1.1-0.20190430175949-e8b55949d948
+	github.com/gogo/googleapis v1.2.0 // indirect
 	github.com/open-telemetry/opentelemetry-service v0.0.0-20190625135304-4bd705a25a35
 	github.com/shirou/gopsutil v2.18.12+incompatible
-	github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 // indirect
 	github.com/spf13/viper v1.4.0
 	github.com/stretchr/testify v1.3.0
 	go.opencensus.io v0.22.0

testbed/go.sum

@@ -185,6 +185,7 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/open-telemetry/opentelemetry-service/testbed v0.0.0-20190702215910-6b130d62dbaa/go.mod h1:55KQ0SPaonTFJxENtpPuZkIV853aovZ6bUtZYB9ROj0=
 github.com/opentracing-contrib/go-stdlib v0.0.0-20170113013457-1de4cc2120e7/go.mod h1:PLldrQSroqzH70Xl+1DQcGnefIbqsKR7UDaiux3zV+w=
 github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74=
 github.com/opentracing/opentracing-go v1.0.1 h1:IYN/cK5AaULfeMAlgFZSIBLSpsZ5MRHDy1fKBEqqJfQ=

testbed/testbed/test_case.go

@@ -139,7 +139,7 @@ func (tc *TestCase) StartAgent(args ...string) {
 		name:         "Agent",
 		logFilePath:  logFileName,
 		cmd:          testBedConfig.Agent,
-		cmdArgs:      args, 
+		cmdArgs:      args,
 		resourceSpec: &tc.resourceSpec,
 	})
 
@@ -189,7 +189,7 @@ func (tc *TestCase) StopBackend() {
 	tc.MockBackend.Stop()
 }
 
-// AgentMemoryInfo returns raw memory info struct about the agent 
+// AgentMemoryInfo returns raw memory info struct about the agent
 // as returned by github.com/shirou/gopsutil/process
 func (tc *TestCase) AgentMemoryInfo() (uint32, uint32, error) {
 	stat, err := tc.agentProc.processMon.MemoryInfo()
@@ -297,7 +297,7 @@ func (tc *TestCase) WaitForN(cond func() bool, duration time.Duration, errMsg ..
 
 // WaitFor is like WaitForN but with a fixed duration of 10 seconds
 func (tc *TestCase) WaitFor(cond func() bool, errMsg ...interface{}) bool {
-	return tc.WaitForN(cond, time.Second * 10, errMsg...)
+	return tc.WaitForN(cond, time.Second*10, errMsg...)
 }
 
 func (tc *TestCase) indicateError(err error) {

testbed/tests/perf_test.go

@@ -100,55 +100,25 @@ func TestNoBackend10kSPS(t *testing.T) {
 	tc.Sleep(10 * time.Second)
 }
 
-func Test1000SPSWithAttributes(t *testing.T) {
-
-	tests := []struct {
-		attrCount    int
-		attrSizeByte int
-		expectedCPU  uint32
-	}{
-		// No attributes.
-		{
-			attrCount:    0,
-			attrSizeByte: 0,
-			expectedCPU:  20,
-		},
-
-		// We generate 10 attributes each with average key length of 100 bytes and
-		// average value length of 50 bytes so total size of attributes values is
-		// 15000 bytes.
-		{
-			attrCount:    100,
-			attrSizeByte: 50,
-			expectedCPU:  120,
-		},
-
-		// Approx 10 KiB attributes.
-		{
-			attrCount:    10,
-			attrSizeByte: 1000,
-			expectedCPU:  100,
-		},
-
-		// Approx 100 KiB attributes.
-		{
-			attrCount:    20,
-			attrSizeByte: 5000,
-			expectedCPU:  250,
-		},
-	}
+type testCase struct {
+	attrCount    int
+	attrSizeByte int
+	expectedMaxCPU  uint32
+	expectedMaxRAM uint32
+}
 
+func test1000SPSWithAttributes(t *testing.T, args []string, tests []testCase) {
 	for _, test := range tests {
 		t.Run(fmt.Sprintf("%d*%dbytes", test.attrCount, test.attrSizeByte), func(t *testing.T) {
 
 			tc := testbed.NewTestCase(t)
 			defer tc.Stop()
 
-			tc.SetExpectedMaxCPU(test.expectedCPU)
-			tc.SetExpectedMaxRAM(100)
+			tc.SetExpectedMaxCPU(test.expectedMaxCPU)
+			tc.SetExpectedMaxRAM(test.expectedMaxRAM)
 
 			tc.StartBackend(testbed.BackendOC)
-			tc.StartAgent()
+			tc.StartAgent(args...)
 
 			options := testbed.LoadOptions{SpansPerSecond: 1000}
 			options.Attributes = make(map[string]interface{})
@@ -172,3 +142,73 @@ func Test1000SPSWithAttributes(t *testing.T) {
 		})
 	}
 }
+
+
+func Test1000SPSWithAttributes(t *testing.T) {
+	test1000SPSWithAttributes(t, []string{}, []testCase{
+		// No attributes.
+		{
+			attrCount:    0,
+			attrSizeByte: 0,
+			expectedMaxCPU:  30,
+			expectedMaxRAM:  100,
+		},
+
+		// We generate 10 attributes each with average key length of 100 bytes and
+		// average value length of 50 bytes so total size of attributes values is
+		// 15000 bytes.
+		{
+			attrCount:    100,
+			attrSizeByte: 50,
+			expectedMaxCPU:  120,
+			expectedMaxRAM:  100,
+		},
+
+		// Approx 10 KiB attributes.
+		{
+			attrCount:    10,
+			attrSizeByte: 1000,
+			expectedMaxCPU:  100,
+			expectedMaxRAM:  100,
+		},
+
+		// Approx 100 KiB attributes.
+		{
+			attrCount:    20,
+			attrSizeByte: 5000,
+			expectedMaxCPU:  250,
+			expectedMaxRAM:  100,
+		},
+	})
+}
+
+func TestBallast1000SPSWithAttributes(t *testing.T) {
+	args := []string{"--mem-ballast-size-mib", "1000"}
+	test1000SPSWithAttributes(t, args, []testCase{
+		// No attributes.
+		{
+			attrCount:    0,
+			attrSizeByte: 0,
+			expectedMaxCPU:  30,
+			expectedMaxRAM:  2000,
+		},
+		{
+			attrCount:    100,
+			attrSizeByte: 50,
+			expectedMaxCPU:  80,
+			expectedMaxRAM:  2000,
+		},
+		{
+			attrCount:    10,
+			attrSizeByte: 1000,
+			expectedMaxCPU:  80,
+			expectedMaxRAM:  2000,
+		},
+		{
+			attrCount:    20,
+			attrSizeByte: 5000,
+			expectedMaxCPU:  120,
+			expectedMaxRAM:  2000,
+		},
+	})
+}

testbed/tests/results/BASELINE.md

@@ -1,14 +1,18 @@
 # Test Results
-Started: Tue, 25 Jun 2019 10:31:56 -0400
+Started: Wed, 03 Jul 2019 05:42:19 +0530
 
-Test|Result|Duration|CPU Avg%|CPU Max%|RAM Avg MiB|RAM Max MiB|Sent Spans|Received Spans
-----|------|-------:|-------:|-------:|----------:|----------:|---------:|-------------:
-TestIdleMode|Passed|10s|0.3|1.0|25|34|0|0
-Test10kSPS|Passed|17s|91.4|95.7|35|43|149610|149610
-TestNoBackend10kSPS|Passed|10s|59.1|59.3|29|40|99910|0
-Test1000SPSWithAttributes/0*0bytes|Passed|12s|18.7|19.3|30|41|10000|10000
-Test1000SPSWithAttributes/100*50bytes|Passed|12s|70.2|71.0|32|45|10000|10000
-Test1000SPSWithAttributes/10*1000bytes|Passed|12s|68.0|74.3|32|44|10000|10000
-Test1000SPSWithAttributes/20*5000bytes|Passed|12s|158.2|173.7|39|55|10000|10000
+Test                                    |Result|Duration|CPU Avg%|CPU Max%|RAM Avg MiB|RAM Max MiB|Sent Spans|Received Spans
+----------------------------------------|------|-------:|-------:|-------:|----------:|----------:|---------:|-------------:
+TestIdleMode                            |PASS  |     10s|     0.3|     1.0|         25|         34|         0|             0
+Test10kSPS                              |PASS  |     16s|    72.6|    92.0|         34|         41|    144520|        144520
+TestNoBackend10kSPS                     |PASS  |     10s|    75.6|    78.3|         29|         40|     97460|             0
+Test1000SPSWithAttributes/0*0bytes      |PASS  |     11s|    22.3|    24.0|         31|         42|     10000|         10000
+Test1000SPSWithAttributes/100*50bytes   |PASS  |     11s|    92.3|   106.3|         33|         46|      9990|          9990
+Test1000SPSWithAttributes/10*1000bytes  |PASS  |     11s|    86.2|    93.3|         34|         46|     10000|         10000
+Test1000SPSWithAttributes/20*5000bytes  |PASS  |     11s|   126.3|   130.0|         47|         64|      9990|          9990
+TestBallast1000SPSWithAttributes/0*0bytes|PASS  |     11s|    20.4|    21.7|         71|        109|     10000|         10000
+TestBallast1000SPSWithAttributes/100*50bytes|PASS  |     11s|    64.4|    68.0|        492|        845|      9950|          9950
+TestBallast1000SPSWithAttributes/10*1000bytes|PASS  |     11s|    55.1|    59.7|        350|        575|      9990|          9990
+TestBallast1000SPSWithAttributes/20*5000bytes|PASS  |     11s|    67.8|    69.3|        786|       1081|      9950|          9950
 
-Total duration: 87s
+Total duration: 127s