Add --output-http-compatibility-mode

Add support for using default Go HTTP client

http_client.go

@@ -9,6 +9,7 @@ import (
 	"log"
 	"net"
 	"net/http"
+	"net/http/httputil"
 	"net/url"
 	"runtime/debug"
 	"strconv"
@@ -41,6 +42,7 @@ type HTTPClientConfig struct {
 	ConnectionTimeout  time.Duration
 	Timeout            time.Duration
 	ResponseBufferSize int
+	CompatibilityMode  bool
 }
 
 type HTTPClient struct {
@@ -53,6 +55,7 @@ type HTTPClient struct {
 	proxyAuth      string
 	respBuf        []byte
 	config         *HTTPClientConfig
+	goClient       *http.Client
 	redirectsCount int
 }
 
@@ -80,6 +83,13 @@ func NewHTTPClient(baseURL string, config *HTTPClientConfig) *HTTPClient {
 	client.respBuf = make([]byte, config.ResponseBufferSize)
 	client.config = config
 
+	if config.CompatibilityMode {
+		client.goClient = &http.Client{
+			// #TODO
+			// CheckRedirect: redirectPolicyFunc,
+		}
+	}
+
 	if u.User != nil {
 		client.auth = "Basic " + base64.StdEncoding.EncodeToString([]byte(u.User.String()))
 	}
@@ -194,6 +204,35 @@ func (c *HTTPClient) isAlive(readBytes *int) bool {
 	return true
 }
 
+func (c *HTTPClient) SendGoClient(data []byte) ([]byte, error) {
+	var req *http.Request
+	var resp *http.Response
+	var err error
+
+	req, err = http.ReadRequest(bufio.NewReader(bytes.NewBuffer(data)))
+	if err != nil {
+		return nil, err
+	}
+
+	if !c.config.OriginalHost {
+		req.Host = c.host
+	}
+
+	if c.auth != "" {
+		req.Header.Add("Authorization", c.auth)
+	}
+
+	req.URL, _ = url.ParseRequestURI(c.scheme + "://" + c.host + req.RequestURI)
+	req.RequestURI = ""
+
+	resp, err = c.goClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+
+	return httputil.DumpResponse(resp, true)
+}
+
 func (c *HTTPClient) Send(data []byte) (response []byte, err error) {
 	// Don't exit on panic
 	defer func() {
@@ -208,6 +247,10 @@ func (c *HTTPClient) Send(data []byte) (response []byte, err error) {
 		}
 	}()
 
+	if c.config.CompatibilityMode {
+		return c.SendGoClient(data)
+	}
+
 	var readBytes int
 	if c.conn == nil || !c.isAlive(&readBytes) {
 		Debug("[HTTPClient] Connecting:", c.baseURL)

http_client_test.go

@@ -87,6 +87,25 @@ func TestHTTPClientSend(t *testing.T) {
 	client.Send(payload("POST"))
 
 	wg.Wait()
+
+	client = NewHTTPClient(server.URL, &HTTPClientConfig{Debug: true, CompatibilityMode: true})
+
+	wg.Add(4)
+
+	if _, err := client.Send(payload("POST")); err != nil {
+		t.Fatal(err.Error())
+	}
+	if _, err := client.Send(payload("GET")); err != nil {
+		t.Fatal(err.Error())
+	}
+	if _, err := client.Send(payload("POST_CHUNKED")); err != nil {
+		t.Fatal(err.Error())
+	}
+	if _, err := client.Send(payload("POST")); err != nil {
+		t.Fatal(err.Error())
+	}
+
+	wg.Wait()
 }
 
 func TestHTTPClientResonseByClose(t *testing.T) {

output_file.go

@@ -247,8 +247,8 @@ func (o *FileOutput) flush() {
 		if stat, err := o.file.Stat(); err == nil {
 			o.chunkSize = int(stat.Size())
 		} else {
-            log.Println("Error accessing file sats", err)
-        }
+			log.Println("Error accessing file sats", err)
+		}
 	}
 }
 

output_http.go

@@ -25,16 +25,18 @@ type HTTPOutputConfig struct {
 	stats      bool
 	workersMin int
 	workersMax int
-	statsMs int
-	workers int
-	queueLen int
+	statsMs    int
+	workers    int
+	queueLen   int
 
 	elasticSearch string
 
 	Timeout      time.Duration
 	OriginalHost bool
 	BufferSize   int
 
+	CompatibilityMode bool
+
 	Debug bool
 
 	TrackResponses bool
@@ -113,6 +115,7 @@ func (o *HTTPOutput) startWorker() {
 		OriginalHost:       o.config.OriginalHost,
 		Timeout:            o.config.Timeout,
 		ResponseBufferSize: o.config.BufferSize,
+		CompatibilityMode:  o.config.CompatibilityMode,
 	})
 
 	deathCount := 0

settings.go

@@ -50,16 +50,16 @@ type AppSettings struct {
 	outputFile       MultiOption
 	outputFileConfig FileOutputConfig
 
-	inputRAW              MultiOption
-	inputRAWEngine        string
-	inputRAWTrackResponse bool
-	inputRAWRealIPHeader  string
-	inputRAWExpire        time.Duration
-	inputRAWBpfFilter     string
-	inputRAWTimestampType string
-	copyBufferSize        int
-	inputRAWImmediateMode bool
-	inputRawBufferSize    int
+	inputRAW                MultiOption
+	inputRAWEngine          string
+	inputRAWTrackResponse   bool
+	inputRAWRealIPHeader    string
+	inputRAWExpire          time.Duration
+	inputRAWBpfFilter       string
+	inputRAWTimestampType   string
+	copyBufferSize          int
+	inputRAWImmediateMode   bool
+	inputRawBufferSize      int
 	inputRAWOverrideSnapLen bool
 
 	middleware string
@@ -153,8 +153,9 @@ func init() {
 
 	flag.Var(&Settings.outputHTTP, "output-http", "Forwards incoming requests to given http address.\n\t# Redirect all incoming requests to staging.com address \n\tgor --input-raw :80 --output-http http://staging.com")
 	flag.IntVar(&Settings.outputHTTPConfig.BufferSize, "output-http-response-buffer", 0, "HTTP response buffer size, all data after this size will be discarded.")
+	flag.BoolVar(&Settings.outputHTTPConfig.CompatibilityMode, "output-http-compatibility-mode", false, "Use standard Go client, instead of built-in implementation. Can be slower, but more compatible.")
 
-  flag.IntVar(&Settings.outputHTTPConfig.workersMin, "output-http-workers-min", 0, "Gor uses dynamic worker scaling. Enter a number to set a minimum number of workers. default = 1.")
+	flag.IntVar(&Settings.outputHTTPConfig.workersMin, "output-http-workers-min", 0, "Gor uses dynamic worker scaling. Enter a number to set a minimum number of workers. default = 1.")
 	flag.IntVar(&Settings.outputHTTPConfig.workersMax, "output-http-workers", 0, "Gor uses dynamic worker scaling. Enter a number to set a maximum number of workers. default = 0 = unlimited.")
 	flag.IntVar(&Settings.outputHTTPConfig.queueLen, "output-http-queue-len", 1000, "Number of requests that can be queued for output, if all workers are busy. default = 1000")