Libpcap support and vairous bugfixes

Merge #260

.gitignore

@@ -13,3 +13,5 @@
 .idea
 *.iml
 gor
+
+*.mprof

.travis.yml

@@ -1,3 +1,6 @@
 language: go
 go: 1.6
-script: sudo -E bash -c "source /etc/profile && eval '$(gimme 1.6)' && export GOPATH=$HOME/gopath:$GOPATH && go get && GORACE='halt_on_error=1' go test ./... -v -timeout 60s -race"
+script: sudo -E bash -c "source /etc/profile && eval '$(gimme 1.6)' && export GOPATH=$HOME/gopath:$GOPATH && go get && GORACE='halt_on_error=1' go test ./... -v -timeout 120s -race"
+
+before_install:
+  - sudo apt-get install libpcap-dev -y

Dockerfile

@@ -13,10 +13,12 @@ RUN apt-get install oracle-java8-installer -y
 RUN wget http://apache-mirror.rbc.ru/pub/apache//commons/io/binaries/commons-io-2.4-bin.tar.gz -P /tmp
 RUN tar xzf /tmp/commons-io-2.4-bin.tar.gz -C /tmp
 
+RUN apt-get install libpcap-dev -y
+RUN go get github.com/google/gopacket
+RUN go get -u github.com/golang/lint/golint
+
 WORKDIR /go/src/github.com/buger/gor/
 ADD . /go/src/github.com/buger/gor/
 
 RUN javac -cp /tmp/commons-io-2.4/commons-io-2.4.jar ./examples/middleware/echo.java
-
-RUN go get -u github.com/golang/lint/golint
 RUN go get

LICENSE.txt

@@ -1,9 +1,7 @@
-Copyright (c) Contributed Systems LLC
-
-Sidekiq is an Open Source project licensed under the terms of
+Gor is an Open Source project licensed under the terms of
 the LGPLv3 license.  Please see <http://www.gnu.org/licenses/lgpl-3.0.html>
 for license text.
 
 Gor Pro has a commercial-friendly license allowing private forks
-and modifications of Gor.  Please see http://gortool.com/pro/ for
+and modifications of Gor.  Please see http://gortool.com/#pro for
 more detail.  You can find the commercial license terms in COMM-LICENSE.

Makefile

@@ -1,6 +1,7 @@
 SOURCE = emitter.go gor.go gor_stat.go input_dummy.go input_file.go input_raw.go input_tcp.go limiter.go output_dummy.go output_file.go input_http.go output_http.go output_tcp.go plugins.go settings.go test_input.go elasticsearch.go http_modifier.go http_modifier_settings.go http_client.go middleware.go protocol.go
 SOURCE_PATH = /go/src/github.com/buger/gor/
 RUN = docker run -v `pwd`:$(SOURCE_PATH) -p 0.0.0.0:8000:8000 -t -i gor
+BENCHMARK = BenchmarkRAWInput
 
 release: release-x86 release-x64
 
@@ -24,7 +25,10 @@ race:
 	$(RUN) go test ./... $(ARGS) -v -race -timeout 15s
 
 test:
-	$(RUN) go test ./... -timeout 10s $(ARGS) -v
+	$(RUN) go test ./. -timeout 30s $(ARGS) -v
+
+test_all:
+	$(RUN) go test ./... -timeout 30s $(ARGS) -v
 
 testone:
 	$(RUN) go test ./... -timeout 4s -run $(TEST) $(ARGS) -v
@@ -40,7 +44,12 @@ vet:
 	$(RUN) go vet
 
 bench:
-	$(RUN) go test -v -run NOT_EXISTING -bench HTTP
+	$(RUN) go test -v -run NOT_EXISTING -bench $(BENCHMARK) -benchtime 5s
+
+profile_test:
+	$(RUN) go test $(LDFLAGS) -run NOT_EXISTING -test.benchmem -bench $(BENCHMARK) ./. $(ARGS) -benchtime 5s -memprofile mem.mprof -v
+	$(RUN) go test $(LDFLAGS) -run NOT_EXISTING -test.benchmem -bench $(BENCHMARK) ./. $(ARGS) -benchtime 5s -cpuprofile cpu.out -v
+	$(RUN) go test $(LDFLAGS) -run NOT_EXISTING -test.benchmem -bench $(BENCHMARK) ./. $(ARGS) -c
 
 # Used mainly for debugging, because docker container do not have access to parent machine ports
 run:

README.md

@@ -41,15 +41,6 @@ It's recommended to use separate server for replaying traffic, but if you have e
 sudo gor --input-raw :80 --output-http "http://staging.com"
 ```
 
-### Guarantee of replay and HTTP input
-Due to how traffic interception works, there is chance of missing requests. If you want guarantee that requests will be replayed you can use http input, but it will require changes in your app as well. 
-
-```
-sudo gor --input-http :28019 --output-http "http://staging.com"
-```
-
-Then in your application you should send copy (e.g. like reverse proxy) all incoming requests to Gor http input. 
-
 ## Configuration
 
 ### Forward to multiple addresses
@@ -282,6 +273,13 @@ gor --input-raw :80 --output-http "http://user:pass@staging .com"
 
 Note: This will overwrite any Authorization headers in the original request.
 
+### Traffic interception engine
+By default Gor use `libpcap` for intercepting traffic. If you have any troubles with it, you may try alternative engine: `raw_socket`.
+
+```
+sudo gor --input-raw :80 --input-raw-engine "libpcap" --output-http "http://staging.com"
+```
+
 
 ## Stats 
 

circle.yml

@@ -0,0 +1,7 @@
+dependencies:
+  pre:
+    - sudo apt-get install libpcap-dev -y
+
+test:
+  override:
+    - sudo bash -l -c "export GOPATH='/home/ubuntu/.go_workspace:/usr/local/go_workspace:/home/ubuntu/.go_project' && GORACE='halt_on_error=1' /usr/local/go/bin/go test ./... -v -timeout 120s -race"

examples/middleware/token_modifier.go

@@ -45,7 +45,7 @@ func main() {
 		buf := make([]byte, len(encoded)/2)
 		hex.Decode(buf, encoded)
 
-		go process(buf)
+		process(buf)
 	}
 }
 

http_client.go

@@ -8,9 +8,9 @@ import (
 	"net"
 	"net/url"
 	"runtime/debug"
+	"strconv"
 	"strings"
 	"time"
-	"strconv"
 )
 
 var defaultPorts = map[string]string{

input_raw.go

@@ -14,15 +14,23 @@ type RAWInput struct {
 	address  string
 	expire   time.Duration
 	quit     chan bool
+	engine   int
 	listener *raw.Listener
 }
 
+// Available engines for intercepting traffic
+const (
+	EngineRawSocket = 1 << iota
+	EnginePcap
+)
+
 // NewRAWInput constructor for RAWInput. Accepts address with port as argument.
-func NewRAWInput(address string, expire time.Duration) (i *RAWInput) {
+func NewRAWInput(address string, engine int, expire time.Duration) (i *RAWInput) {
 	i = new(RAWInput)
 	i.data = make(chan *raw.TCPMessage)
 	i.address = address
 	i.expire = expire
+	i.engine = engine
 	i.quit = make(chan bool)
 
 	go i.listen(address)
@@ -59,7 +67,7 @@ func (i *RAWInput) listen(address string) {
 		log.Fatal("input-raw: error while parsing address", err)
 	}
 
-	i.listener = raw.NewListener(host, port, i.expire)
+	i.listener = raw.NewListener(host, port, i.engine, i.expire)
 
 	for {
 		select {
@@ -76,7 +84,7 @@ func (i *RAWInput) listen(address string) {
 }
 
 func (i *RAWInput) String() string {
-	return "RAW Socket input: " + i.address
+	return "Intercepting traffic from: " + i.address
 }
 
 func (i *RAWInput) Close() {

input_raw_test.go

@@ -5,17 +5,18 @@ import (
 	"io"
 	"io/ioutil"
 	"log"
+	"math/rand"
 	"net/http"
 	"net/http/httptest"
 	"net/http/httputil"
 	"os"
 	"os/exec"
+	"strconv"
 	"strings"
 	"sync"
 	"sync/atomic"
 	"testing"
 	"time"
-	"math/rand"
 )
 
 const testRawExpire = time.Millisecond * 200
@@ -30,7 +31,7 @@ func TestRAWInput(t *testing.T) {
 
 	var respCounter, reqCounter int64
 
-	input := NewRAWInput(originAddr, testRawExpire)
+	input := NewRAWInput(originAddr, EnginePcap, testRawExpire)
 	defer input.Close()
 
 	output := NewTestOutput(func(data []byte) {
@@ -52,14 +53,14 @@ func TestRAWInput(t *testing.T) {
 
 	client := NewHTTPClient(origin.URL, &HTTPClientConfig{})
 
-	time.Sleep(time.Millisecond)
-
 	go Start(quit)
+	time.Sleep(100 * time.Millisecond)
 
 	for i := 0; i < 100; i++ {
 		// request + response
 		wg.Add(2)
 		client.Get("/")
+		time.Sleep(2 * time.Millisecond)
 	}
 
 	wg.Wait()
@@ -70,7 +71,7 @@ func TestInputRAW100Expect(t *testing.T) {
 	wg := new(sync.WaitGroup)
 	quit := make(chan int)
 
-	fileContent, _ := ioutil.ReadFile("LICENSE.txt")
+	fileContent, _ := ioutil.ReadFile("COMM-LICENSE")
 
 	// Origing and Replay server initialization
 	origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -82,7 +83,7 @@ func TestInputRAW100Expect(t *testing.T) {
 
 	originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
 
-	input := NewRAWInput(originAddr, time.Second)
+	input := NewRAWInput(originAddr, EnginePcap, time.Second)
 	defer input.Close()
 
 	// We will use it to get content of raw HTTP request
@@ -117,10 +118,11 @@ func TestInputRAW100Expect(t *testing.T) {
 	Plugins.Outputs = []io.Writer{testOutput, httpOutput}
 
 	go Start(quit)
+	time.Sleep(100 * time.Millisecond)
 
 	// Origin + Response/Request Test Output + Request Http Output
 	wg.Add(4)
-	curl := exec.Command("curl", "http://"+originAddr, "--data-binary", "@LICENSE.txt")
+	curl := exec.Command("curl", "http://"+originAddr, "--data-binary", "@COMM-LICENSE")
 	err := curl.Run()
 	if err != nil {
 		log.Fatal(err)
@@ -145,7 +147,7 @@ func TestInputRAWChunkedEncoding(t *testing.T) {
 	}))
 
 	originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
-	input := NewRAWInput(originAddr, time.Second)
+	input := NewRAWInput(originAddr, EnginePcap, time.Second)
 	defer input.Close()
 
 	replay := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -167,10 +169,11 @@ func TestInputRAWChunkedEncoding(t *testing.T) {
 	Plugins.Outputs = []io.Writer{httpOutput}
 
 	go Start(quit)
+	time.Sleep(100 * time.Millisecond)
 
 	wg.Add(2)
 
-	curl := exec.Command("curl", "http://"+originAddr, "--header", "Transfer-Encoding: chunked", "--data-binary", "@README.md")
+	curl := exec.Command("curl", "http://"+originAddr, "--header", "Transfer-Encoding: chunked", "--header", "Expect:", "--data-binary", "@README.md")
 	err := curl.Run()
 	if err != nil {
 		log.Fatal(err)
@@ -188,9 +191,10 @@ func TestInputRAWLargePayload(t *testing.T) {
 	}
 	wg := new(sync.WaitGroup)
 	quit := make(chan int)
+	sizeKb := 100
 
 	// Generate 100kb file
-	dd := exec.Command("dd", "if=/dev/urandom", "of=/tmp/large", "bs=1KB", "count=100")
+	dd := exec.Command("dd", "if=/dev/urandom", "of=/tmp/large", "bs=1KB", "count="+strconv.Itoa(sizeKb))
 	err := dd.Run()
 	if err != nil {
 		log.Fatal("dd error:", err)
@@ -200,25 +204,26 @@ func TestInputRAWLargePayload(t *testing.T) {
 		defer req.Body.Close()
 		body, _ := ioutil.ReadAll(req.Body)
 
-		if len(body) != 100*1000 {
+		if len(body) != sizeKb*1000 {
 			t.Error("File size should be 1mb:", len(body))
 		}
 
 		wg.Done()
 	}))
 	originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
 
-	input := NewRAWInput(originAddr, time.Second)
+	input := NewRAWInput(originAddr, EnginePcap, testRawExpire)
 	defer input.Close()
 
 	replay := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
-		req.Body = http.MaxBytesReader(w, req.Body, 1*1024*1024)
-		buf := make([]byte, 1*1024*1024)
-		n, _ := req.Body.Read(buf)
-		body := buf[0:n]
+		body, _ := ioutil.ReadAll(req.Body)
+		// // req.Body = http.MaxBytesReader(w, req.Body, 1*1024*1024)
+		// // buf := make([]byte, 1*1024*1024)
+		// n, _ := req.Body.Read(buf)
+		// body := buf[0:n]
 
-		if len(body) != 100*1000 {
-			t.Error("File size should be 100000 bytes:", len(body))
+		if len(body) != sizeKb*1000 {
+			t.Errorf("File size should be %d bytes: %d", sizeKb*1000, len(body))
 		}
 
 		wg.Done()
@@ -232,8 +237,10 @@ func TestInputRAWLargePayload(t *testing.T) {
 
 	go Start(quit)
 
+	time.Sleep(100 * time.Millisecond)
+
 	wg.Add(2)
-	curl := exec.Command("curl", "http://"+originAddr, "--header", "Transfer-Encoding: chunked", "--data-binary", "@/tmp/large")
+	curl := exec.Command("curl", "http://"+originAddr, "--header", "Transfer-Encoding: chunked", "--header", "Expect:", "--data-binary", "@/tmp/large")
 	err = curl.Run()
 	if err != nil {
 		log.Fatal("curl error:", err)
@@ -252,7 +259,7 @@ func BenchmarkRAWInput(b *testing.B) {
 
 	var respCounter, reqCounter int64
 
-	input := NewRAWInput(originAddr, testRawExpire)
+	input := NewRAWInput(originAddr, EnginePcap, testRawExpire)
 	defer input.Close()
 
 	output := NewTestOutput(func(data []byte) {
@@ -278,9 +285,9 @@ func BenchmarkRAWInput(b *testing.B) {
 	for i := 0; i < b.N; i++ {
 		wg := new(sync.WaitGroup)
 		wg.Add(10 * 100)
-		emitted += 10*100
+		emitted += 10 * 100
 		for w := 0; w < 100; w++ {
-			go func(){
+			go func() {
 				client := NewHTTPClient(origin.URL, &HTTPClientConfig{})
 				for i := 0; i < 10; i++ {
 					if rand.Int63n(2) == 0 {
@@ -296,7 +303,7 @@ func BenchmarkRAWInput(b *testing.B) {
 		wg.Wait()
 	}
 
-	time.Sleep(201 * time.Millisecond)
+	time.Sleep(400 * time.Millisecond)
 	log.Println("Emitted ", emitted, ", Captured ", reqCounter, "requests and ", respCounter, " responses")
 
 	close(quit)