Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Benchmarks, fix tests and perfomance issues, and packing layers
Browse files Browse the repository at this point in the history
Urban Ishimwe committed Jul 25, 2020
2 parents 7165384 + 2f81fba commit fae93a5
Showing 47 changed files with 661 additions and 498 deletions.
4 changes: 4 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
*.swp
*.gor
*.rpm
*.dep
*.pkg

*.out

@@ -21,6 +24,7 @@ gor

.DS_Store

goreplay
corpus
crashers
suppressions
4 changes: 2 additions & 2 deletions Dockerfile.dev
Original file line number Diff line number Diff line change
@@ -3,8 +3,8 @@ FROM golang:1.14
RUN apt-get update && apt-get install ruby vim-common -y

RUN apt-get install flex bison -y
RUN wget http://www.tcpdump.org/release/libpcap-1.8.1.tar.gz && tar xzf libpcap-1.8.1.tar.gz && cd libpcap-1.8.1 && ./configure && make install
RUN apt-get install default-jre default-jdk -y
RUN wget http://www.tcpdump.org/release/libpcap-1.9.1.tar.gz && tar xzf libpcap-1.9.1.tar.gz && cd libpcap-1.9.1 && ./configure && make install

RUN go get github.com/google/gopacket
RUN go get -u golang.org/x/lint/golint

32 changes: 29 additions & 3 deletions Makefile
Original file line number Diff line number Diff line change
@@ -12,19 +12,45 @@ LDFLAGS = -ldflags "-X main.VERSION=$(VERSION)$(PREFIX) -extldflags \"-static\"
MAC_LDFLAGS = -ldflags "-X main.VERSION=$(VERSION)$(PREFIX) -X main.DEMO=$(DEMO)"
FADDR = ":8000"

FPMCOMMON= \
--name goreplay \
--description "GoReplay is an open-source network monitoring tool which can record your live traffic, and use it for shadowing, load testing, monitoring and detailed analysis." \
-v $(VERSION) \
--vendor "Leonid Bugaev" \
-m "<[email protected]>" \
--url "https://goreplay.org" \
-s dir \
-C /tmp/gor-build \

release: release-x64 release-mac

release-bin:
docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=amd64 -i gor go build -o gor -tags netgo $(LDFLAGS)

release-x64:
docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=amd64 -i $(CONTAINER) go build -o gor -tags netgo $(LDFLAGS) && tar -czf gor_$(VERSION)$(PREFIX)_x64.tar.gz gor && rm gor
docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=amd64 -i $(CONTAINER) go build -o gor -tags netgo $(LDFLAGS)
tar -czf gor_$(VERSION)$(PREFIX)_x64.tar.gz gor
mkdir -p /tmp/gor-build
mv ./gor /tmp/gor-build/gor
cd /tmp/gor-build
fpm $(FPMCOMMON) -a amd64 -t deb ./=/usr/local/bin
fpm $(FPMCOMMON) -a amd64 -t rpm ./=/usr/local/bin
rm -rf /tmp/gor-build

release-x86:
docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=386 -i $(CONTAINER) go build -o gor -tags netgo $(LDFLAGS) && tar -czf gor_$(VERSION)$(PREFIX)_x86.tar.gz gor && rm gor
docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=386 -i $(CONTAINER) go build -o gor -tags netgo $(LDFLAGS)
tar -czf gor_$(VERSION)$(PREFIX)_x86.tar.gz gor
rm gor

release-mac:
go build -o gor $(MAC_LDFLAGS) && tar -czf gor_$(VERSION)$(PREFIX)_mac.tar.gz gor && rm gor
go build -o gor $(MAC_LDFLAGS)
tar -czf gor_$(VERSION)$(PREFIX)_mac.tar.gz gor
mkdir -p /tmp/gor-build
mv ./gor /tmp/gor-build/gor
cd /tmp/gor-build
fpm $(FPMCOMMON) -a amd64 -t osxpkg ./=/usr/local/bin
rm -rf /tmp/gor-build


install:
go install $(MAC_LDFLAGS)
14 changes: 7 additions & 7 deletions capture/capture.go
Original file line number Diff line number Diff line change
@@ -26,13 +26,13 @@ type Handler func(gopacket.Packet)
// PcapOptions options that can be set on a pcap capture handle,
// these options take effect on inactive pcap handles
type PcapOptions struct {
Promiscuous bool
Monitor bool
Snaplen bool
BufferTimeout time.Duration
TimestampType string
BufferSize size.Size
BPFFilter string // custom bpf filter
Promiscuous bool `json:"input-raw-promisc"`
Monitor bool `json:"input-raw-monitor"`
Snaplen bool `json:"input-raw-override-snaplen"`
BufferTimeout time.Duration `json:"input-raw-buffer-timeout"`
TimestampType string `json:"input-raw-timestamp-type"`
BufferSize size.Size `json:"input-raw-buffer-size"`
BPFFilter string `json:"input-raw-bpf-filter"`
}

// NetInterface represents network interface
6 changes: 3 additions & 3 deletions elasticsearch.go
Original file line number Diff line number Diff line change
@@ -69,7 +69,7 @@ func parseURI(URI string) (err error, index string) {
return
}

// check URL validity by extracting host and undex values.
// check URL validity by extracting host and index values.
host := parsedUrl.Host
urlPathParts := strings.Split(parsedUrl.Path, "/")
index = urlPathParts[len(urlPathParts)-1]
@@ -101,7 +101,7 @@ func (p *ESPlugin) Init(URI string) {

go p.ErrorHandler()

Debug(0, "Initialized Elasticsearch Plugin")
Debug(1, "Initialized Elasticsearch Plugin")
return
}

@@ -113,7 +113,7 @@ func (p *ESPlugin) IndexerShutdown() {
func (p *ESPlugin) ErrorHandler() {
for {
errBuf := <-p.indexor.ErrorChannel
Debug(1, errBuf.Err)
Debug(1, "[ELASTICSEARCH]", errBuf.Err)
}
}

16 changes: 8 additions & 8 deletions elasticsearch_test.go
Original file line number Diff line number Diff line change
@@ -42,7 +42,7 @@ func TestElasticConnectionBuildFailWithoutScheme(t *testing.T) {
assertExpectedError(err, t)
}

// Argument scheme://host:port
// Argument scheme://Host:port
// i.e : http://localhost:9200
// Fail : explicit index is required
func TestElasticConnectionBuildFailWithoutIndex(t *testing.T) {
@@ -55,7 +55,7 @@ func TestElasticConnectionBuildFailWithoutIndex(t *testing.T) {
assertExpectedError(err, t)
}

// Argument scheme://host/index_name
// Argument scheme://Host/index_name
// i.e : http://localhost/gor
func TestElasticConnectionBuildFailWithoutPort(t *testing.T) {
uri := "http://localhost/" + expectedIndex
@@ -67,7 +67,7 @@ func TestElasticConnectionBuildFailWithoutPort(t *testing.T) {
assertExpectedGorIndex(index, t)
}

// Argument scheme://host:port/index_name
// Argument scheme://Host:port/index_name
// i.e : http://localhost:9200/gor
func TestElasticLocalConnectionBuild(t *testing.T) {
uri := "http://localhost:9200/" + expectedIndex
@@ -79,7 +79,7 @@ func TestElasticLocalConnectionBuild(t *testing.T) {
assertExpectedGorIndex(index, t)
}

// Argument scheme://host:port/index_name
// Argument scheme://Host:port/index_name
// i.e : http://localhost.local:9200/gor or https://localhost.local:9200/gor
func TestElasticSimpleLocalWithSchemeConnectionBuild(t *testing.T) {
uri := "http://localhost.local:9200/" + expectedIndex
@@ -91,7 +91,7 @@ func TestElasticSimpleLocalWithSchemeConnectionBuild(t *testing.T) {
assertExpectedGorIndex(index, t)
}

// Argument scheme://host:port/index_name
// Argument scheme://Host:port/index_name
// i.e : http://localhost.local:9200/gor or https://localhost.local:9200/gor
func TestElasticSimpleLocalWithHTTPSConnectionBuild(t *testing.T) {
uri := "https://localhost.local:9200/" + expectedIndex
@@ -103,7 +103,7 @@ func TestElasticSimpleLocalWithHTTPSConnectionBuild(t *testing.T) {
assertExpectedGorIndex(index, t)
}

// Argument scheme://host:port/index_name
// Argument scheme://Host:port/index_name
// i.e : localhost.local:9200/pathtoElastic/gor
func TestElasticLongPathConnectionBuild(t *testing.T) {
uri := "http://localhost.local:9200/pathtoElastic/" + expectedIndex
@@ -115,7 +115,7 @@ func TestElasticLongPathConnectionBuild(t *testing.T) {
assertExpectedGorIndex(index, t)
}

// Argument scheme://host:userinfo@port/index_name
// Argument scheme://Host:userinfo@port/index_name
// i.e : http://user:[email protected]:9200/gor
func TestElasticBasicAuthConnectionBuild(t *testing.T) {
uri := "http://user:[email protected]:9200/" + expectedIndex
@@ -127,7 +127,7 @@ func TestElasticBasicAuthConnectionBuild(t *testing.T) {
assertExpectedGorIndex(index, t)
}

// Argument scheme://host:port/path/index_name
// Argument scheme://Host:port/path/index_name
// i.e : http://localhost.local:9200/path/gor or https://localhost.local:9200/path/gor
func TestElasticComplexPathConnectionBuild(t *testing.T) {
uri := "http://localhost.local:9200/path/" + expectedIndex
8 changes: 4 additions & 4 deletions emitter.go
Original file line number Diff line number Diff line change
@@ -116,7 +116,7 @@ func (e *emitter) Close() {
func CopyMulty(stop chan int, src io.Reader, writers ...io.Writer) error {
buf := make([]byte, Settings.copyBufferSize)
wIndex := 0
modifier := NewHTTPModifier(&Settings.modifierConfig)
modifier := NewHTTPModifier(&Settings.ModifierConfig)
filteredRequests := make(map[string]time.Time)
filteredRequestsLastCleanTime := time.Now()

@@ -176,15 +176,15 @@ func CopyMulty(stop chan int, src io.Reader, writers ...io.Writer) error {
}
}

if Settings.prettifyHTTP {
if Settings.PrettifyHTTP {
payload = prettifyHTTP(payload)
if len(payload) == 0 {
continue
}
}

if Settings.splitOutput {
if Settings.recognizeTCPSessions {
if Settings.SplitOutput {
if Settings.RecognizeTCPSessions {
if !PRO {
log.Fatal("Detailed TCP sessions work only with PRO license")
}
30 changes: 15 additions & 15 deletions emitter_test.go
Original file line number Diff line number Diff line change
@@ -32,7 +32,7 @@ func TestEmitter(t *testing.T) {
plugins.All = append(plugins.All, input, output)

emitter := NewEmitter(quit)
go emitter.Start(plugins, Settings.middleware)
go emitter.Start(plugins, Settings.Middleware)

for i := 0; i < 1000; i++ {
wg.Add(1)
@@ -61,7 +61,7 @@ func TestEmitterFiltered(t *testing.T) {
plugins.All = append(plugins.All, input, output)

methods := HTTPMethods{[]byte("GET")}
Settings.modifierConfig = HTTPModifierConfig{methods: methods}
Settings.ModifierConfig = HTTPModifierConfig{Methods: methods}

emitter := &emitter{quit: quit}
go emitter.Start(plugins, "")
@@ -91,7 +91,7 @@ func TestEmitterFiltered(t *testing.T) {
wg.Wait()
emitter.Close()

Settings.modifierConfig = HTTPModifierConfig{}
Settings.ModifierConfig = HTTPModifierConfig{}
}

func TestEmitterSplitRoundRobin(t *testing.T) {
@@ -117,10 +117,10 @@ func TestEmitterSplitRoundRobin(t *testing.T) {
Outputs: []io.Writer{output1, output2},
}

Settings.splitOutput = true
Settings.SplitOutput = true

emitter := NewEmitter(quit)
go emitter.Start(plugins, Settings.middleware)
go emitter.Start(plugins, Settings.Middleware)

for i := 0; i < 1000; i++ {
wg.Add(1)
@@ -135,7 +135,7 @@ func TestEmitterSplitRoundRobin(t *testing.T) {
t.Errorf("Round robin should split traffic equally: %d vs %d", counter1, counter2)
}

Settings.splitOutput = false
Settings.SplitOutput = false
}

func TestEmitterRoundRobin(t *testing.T) {
@@ -162,10 +162,10 @@ func TestEmitterRoundRobin(t *testing.T) {
}
plugins.All = append(plugins.All, input, output1, output2)

Settings.splitOutput = true
Settings.SplitOutput = true

emitter := NewEmitter(quit)
go emitter.Start(plugins, Settings.middleware)
go emitter.Start(plugins, Settings.Middleware)

for i := 0; i < 1000; i++ {
wg.Add(1)
@@ -179,7 +179,7 @@ func TestEmitterRoundRobin(t *testing.T) {
t.Errorf("Round robin should split traffic equally: %d vs %d", counter1, counter2)
}

Settings.splitOutput = false
Settings.SplitOutput = false
}

func TestEmitterSplitSession(t *testing.T) {
@@ -212,11 +212,11 @@ func TestEmitterSplitSession(t *testing.T) {
Outputs: []io.Writer{output1, output2},
}

Settings.splitOutput = true
Settings.recognizeTCPSessions = true
Settings.SplitOutput = true
Settings.RecognizeTCPSessions = true

emitter := NewEmitter(quit)
go emitter.Start(plugins, Settings.middleware)
go emitter.Start(plugins, Settings.Middleware)

for i := 0; i < 200; i++ {
// Keep session but randomize
@@ -235,8 +235,8 @@ func TestEmitterSplitSession(t *testing.T) {
t.Errorf("Round robin should split traffic equally: %d vs %d", counter1, counter2)
}

Settings.splitOutput = false
Settings.recognizeTCPSessions = false
Settings.SplitOutput = false
Settings.RecognizeTCPSessions = false
emitter.Close()
}

@@ -257,7 +257,7 @@ func BenchmarkEmitter(b *testing.B) {
plugins.All = append(plugins.All, input, output)

emitter := NewEmitter(quit)
go emitter.Start(plugins, Settings.middleware)
go emitter.Start(plugins, Settings.Middleware)

b.ResetTimer()

5 changes: 2 additions & 3 deletions go.mod
Original file line number Diff line number Diff line change
@@ -5,7 +5,7 @@ go 1.14
require (
github.com/Shopify/sarama v1.26.4
github.com/araddon/gou v0.0.0-20190110011759-c797efecbb61 // indirect
github.com/aws/aws-sdk-go v1.32.7
github.com/aws/aws-sdk-go v1.33.2
github.com/bitly/go-hostpool v0.1.0 // indirect
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect
github.com/google/gopacket v1.1.17
@@ -14,6 +14,5 @@ require (
github.com/pierrec/lz4 v2.5.2+incompatible // indirect
github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 // indirect
github.com/smartystreets/goconvey v1.6.4 // indirect
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 // indirect
golang.org/x/net v0.0.0-20200602114024-627f9648deb9 // indirect
golang.org/x/net v0.0.0-20200707034311-ab3426394381 // indirect
)
8 changes: 4 additions & 4 deletions go.sum
Original file line number Diff line number Diff line change
@@ -4,8 +4,8 @@ github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWso
github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
github.com/araddon/gou v0.0.0-20190110011759-c797efecbb61 h1:Xz25cuW4REGC5W5UtpMU3QItMIImag615HiQcRbxqKQ=
github.com/araddon/gou v0.0.0-20190110011759-c797efecbb61/go.mod h1:ikc1XA58M+Rx7SEbf0bLJCfBkwayZ8T5jBo5FXK8Uz8=
github.com/aws/aws-sdk-go v1.32.7 h1:H4VgdCSF1cHw0VD8zGc98T1bGdACoLkh/vK2L6wgOUU=
github.com/aws/aws-sdk-go v1.32.7/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
github.com/aws/aws-sdk-go v1.33.2 h1:8TVrnPnSD7I+AmDp66xBUvS3K0J+jH09YXdrkJ34ey0=
github.com/aws/aws-sdk-go v1.33.2/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
github.com/bitly/go-hostpool v0.1.0 h1:XKmsF6k5el6xHG3WPJ8U0Ku/ye7njX7W81Ng7O2ioR0=
github.com/bitly/go-hostpool v0.1.0/go.mod h1:4gOCgp6+NZnVqlKyZ/iBZFTAJKembaVENUpMkpg42fw=
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY=
@@ -82,8 +82,8 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200602114024-627f9648deb9 h1:pNX+40auqi2JqRfOP1akLGtYcn15TUbkhwuCO3foqqM=
golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU=
golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
Loading

0 comments on commit fae93a5

Please sign in to comment.