Skip to content

Commit

Permalink
Enable E2E test running on Travis CI
Browse files Browse the repository at this point in the history 
- We want Travis build to fail if E2E tests fail.
- Renamed "perf-test" makefile target to "e2e-test" (since these
  are not just perf tests but also other types of end-to-end tests).
- Fix TestGeneratorAndBackend to make sure it waits properly for
  the generator to produce results.

Issue: open-telemetry#68
  • Loading branch information
Tigran Najaryan committed Jun 28, 2019
1 parent f28c6b8 commit dfe1515
Show file tree
Hide file tree
Showing 9 changed files with 113 additions and 38 deletions.
8 changes: 5 additions & 3 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,16 +35,18 @@ all-srcs:
.PHONY: fmt-vet-lint-test
fmt-vet-lint-test: fmt vet lint test

.PHONY: perf-test
perf-test: otelsvc
.PHONY: e2e-test
e2e-test: otelsvc
$(MAKE) -C testbed runtests

.PHONY: test
test:
$(GOTEST) $(GOTEST_OPT) $(ALL_PKGS)

.PHONY: travis-ci
travis-ci: fmt vet lint test-with-cover
travis-ci: fmt vet lint test-with-cover otelsvc
$(MAKE) -C testbed install-tools
$(MAKE) -C testbed runtests

.PHONY: test-with-cover
test-with-cover:
Expand Down
18 changes: 8 additions & 10 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ require (
github.com/golang/protobuf v1.3.1
github.com/google/go-cmp v0.3.0
github.com/gorilla/mux v1.6.2
github.com/grpc-ecosystem/grpc-gateway v1.8.5
github.com/grpc-ecosystem/grpc-gateway v1.9.0
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/jaegertracing/jaeger v1.9.0
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024
Expand All @@ -27,25 +27,23 @@ require (
github.com/orijtech/prometheus-go-metrics-exporter v0.0.3-0.20190313163149-b321c5297f60
github.com/pkg/errors v0.8.0
github.com/prashantv/protectmem v0.0.0-20171002184600-e20412882b3a // indirect
github.com/prometheus/client_golang v0.9.2
github.com/prometheus/common v0.0.0-20181126121408-4724e9255275
github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1
github.com/prometheus/client_golang v0.9.3
github.com/prometheus/common v0.4.0
github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084
github.com/prometheus/prometheus v0.0.0-20190131111325-62e591f928dd
github.com/rs/cors v1.6.0
github.com/soheilhy/cmux v0.1.4
github.com/spf13/cast v1.2.0
github.com/spf13/cast v1.3.0
github.com/spf13/cobra v0.0.3
github.com/spf13/viper v1.2.1
github.com/spf13/viper v1.4.0
github.com/streadway/quantile v0.0.0-20150917103942-b0c588724d25 // indirect
github.com/stretchr/testify v1.3.0
github.com/uber-go/atomic v1.3.2 // indirect
github.com/uber-go/atomic v1.4.0 // indirect
github.com/uber/jaeger-client-go v2.16.0+incompatible // indirect
github.com/uber/jaeger-lib v2.0.0+incompatible
github.com/uber/tchannel-go v1.10.0
go.opencensus.io v0.22.0
go.uber.org/atomic v1.3.2 // indirect
go.uber.org/multierr v1.1.0 // indirect
go.uber.org/zap v1.9.1
go.uber.org/zap v1.10.0
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3
golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a // indirect
google.golang.org/api v0.5.0
Expand Down
76 changes: 60 additions & 16 deletions go.sum
View file

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions testbed/go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -190,8 +190,6 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW
github.com/oklog/oklog v0.0.0-20170918173356-f857583a70c3/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs=
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
github.com/olekukonko/tablewriter v0.0.0-20180912035003-be2c049b30cc/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
github.com/omnition/opencensus-proto v0.2.2-gogo3-unary h1:N/Eb32LTEBVpUV2tvxTNyi6bznvpfn5SBaxBAGQgI2Q=
github.com/omnition/opencensus-proto v0.2.2-gogo3-unary/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/omnition/scribe-go v0.0.0-20190131012523-9e3c68f31124/go.mod h1:GnPmaNTr3pdt/V0JmVNVgDq+JEMb/oXxNlsG+pN6gg4=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
Expand Down Expand Up @@ -320,6 +318,7 @@ golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3 h1:XQyxROzUlZH+WIQwySDgnISgOivlhjIEwaQaJEJrrN0=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
Expand Down Expand Up @@ -368,6 +367,7 @@ golang.org/x/tools v0.0.0-20181023010539-40a48ad93fbe/go.mod h1:n7NCudcB/nEzxVGm
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190312170243-e65039ee4138 h1:H3uGjxCR/6Ds0Mjgyp7LMK81+LvmbvWWEnJhzk1Pi9E=
golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
google.golang.org/api v0.0.0-20180506000402-20530fd5d65a/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
google.golang.org/api v0.4.0 h1:KKgc1aqhV8wDPbDzlDtpvyjZFY3vjz85FP7p4wcQUyI=
Expand Down
1 change: 1 addition & 0 deletions testbed/testbed/.gitignore
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
mockbackend.log
4 changes: 4 additions & 0 deletions testbed/testbed/load_generator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,10 @@ func (lg *LoadGenerator) GetStats() string {
return fmt.Sprintf("Sent:%5d spans", atomic.LoadUint64(&lg.SpansSent))
}

func (lg *LoadGenerator) GetSpansSent() uint64 {
return atomic.LoadUint64(&lg.SpansSent)
}

func (lg *LoadGenerator) generate() {
// Indicate that generation is done at the end
defer lg.stopWait.Done()
Expand Down
37 changes: 31 additions & 6 deletions testbed/testbed/mock_backend_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,17 +24,42 @@ func TestGeneratorAndBackend(t *testing.T) {

assert.EqualValues(t, 0, lg.SpansSent)

// Generate for about 10ms at 1000 SPS
// Generate at 1000 SPS
lg.Start(LoadOptions{SpansPerSecond: 1000})

time.Sleep(time.Millisecond * 10)
// Wait until at least 50 spans are sent
WaitFor(t, func() bool { return lg.GetSpansSent() > 50 }, "SpansSent > 50")

lg.Stop()

// Presumably should have generated something. If not then the testbed is very slow
// so we will consider it a failure.
assert.True(t, lg.SpansSent > 0)

// The backend should receive everything generated.
assert.Equal(t, lg.SpansSent, mb.SpansReceived())
}

// WaitFor the specific condition for up to 5 seconds. Records a test error
// if condition does not become true.
func WaitFor(t *testing.T, cond func() bool, errMsg ...interface{}) bool {
startTime := time.Now()

// Start with 5 ms waiting interval between condition re-evaluation.
waitInterval := time.Millisecond * 5

for {
time.Sleep(waitInterval)

// Increase waiting interval exponentially up to 500 ms.
if waitInterval < time.Millisecond*500 {
waitInterval = waitInterval * 2
}

if cond() {
return true
}

if time.Since(startTime) > time.Second*5 {
// Waited too long
t.Error("Time out waiting for", errMsg)
return false
}
}
}
1 change: 1 addition & 0 deletions testbed/tests/.gitignore
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
results/*
2 changes: 1 addition & 1 deletion testbed/tests/perf_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,7 @@ func Test1000SPSWithAttributes(t *testing.T) {
defer tc.Stop()

tc.SetExpectedMaxCPU(test.expectedCPU)
tc.SetExpectedMaxRAM(70)
tc.SetExpectedMaxRAM(100)

tc.StartBackend(testbed.BackendOC)
tc.StartAgent()
Expand Down

0 comments on commit dfe1515

Please sign in to comment.