diff --git a/http_client.go b/http_client.go index 876a3082..a7be48e6 100644 --- a/http_client.go +++ b/http_client.go @@ -9,6 +9,7 @@ import ( "log" "net" "net/http" + "net/http/httputil" "net/url" "runtime/debug" "strconv" @@ -41,6 +42,7 @@ type HTTPClientConfig struct { ConnectionTimeout time.Duration Timeout time.Duration ResponseBufferSize int + CompatibilityMode bool } type HTTPClient struct { @@ -53,6 +55,7 @@ type HTTPClient struct { proxyAuth string respBuf []byte config *HTTPClientConfig + goClient *http.Client redirectsCount int } @@ -80,6 +83,13 @@ func NewHTTPClient(baseURL string, config *HTTPClientConfig) *HTTPClient { client.respBuf = make([]byte, config.ResponseBufferSize) client.config = config + if config.CompatibilityMode { + client.goClient = &http.Client{ + // #TODO + // CheckRedirect: redirectPolicyFunc, + } + } + if u.User != nil { client.auth = "Basic " + base64.StdEncoding.EncodeToString([]byte(u.User.String())) } @@ -194,6 +204,35 @@ func (c *HTTPClient) isAlive(readBytes *int) bool { return true } +func (c *HTTPClient) SendGoClient(data []byte) ([]byte, error) { + var req *http.Request + var resp *http.Response + var err error + + req, err = http.ReadRequest(bufio.NewReader(bytes.NewBuffer(data))) + if err != nil { + return nil, err + } + + if !c.config.OriginalHost { + req.Host = c.host + } + + if c.auth != "" { + req.Header.Add("Authorization", c.auth) + } + + req.URL, _ = url.ParseRequestURI(c.scheme + "://" + c.host + req.RequestURI) + req.RequestURI = "" + + resp, err = c.goClient.Do(req) + if err != nil { + return nil, err + } + + return httputil.DumpResponse(resp, true) +} + func (c *HTTPClient) Send(data []byte) (response []byte, err error) { // Don't exit on panic defer func() { @@ -208,6 +247,10 @@ func (c *HTTPClient) Send(data []byte) (response []byte, err error) { } }() + if c.config.CompatibilityMode { + return c.SendGoClient(data) + } + var readBytes int if c.conn == nil || !c.isAlive(&readBytes) { Debug("[HTTPClient] Connecting:", c.baseURL) diff --git a/http_client_test.go b/http_client_test.go index 963c8167..f29701f3 100644 --- a/http_client_test.go +++ b/http_client_test.go @@ -87,6 +87,25 @@ func TestHTTPClientSend(t *testing.T) { client.Send(payload("POST")) wg.Wait() + + client = NewHTTPClient(server.URL, &HTTPClientConfig{Debug: true, CompatibilityMode: true}) + + wg.Add(4) + + if _, err := client.Send(payload("POST")); err != nil { + t.Fatal(err.Error()) + } + if _, err := client.Send(payload("GET")); err != nil { + t.Fatal(err.Error()) + } + if _, err := client.Send(payload("POST_CHUNKED")); err != nil { + t.Fatal(err.Error()) + } + if _, err := client.Send(payload("POST")); err != nil { + t.Fatal(err.Error()) + } + + wg.Wait() } func TestHTTPClientResonseByClose(t *testing.T) { diff --git a/output_file.go b/output_file.go index 6f026d22..de97186c 100644 --- a/output_file.go +++ b/output_file.go @@ -247,8 +247,8 @@ func (o *FileOutput) flush() { if stat, err := o.file.Stat(); err == nil { o.chunkSize = int(stat.Size()) } else { - log.Println("Error accessing file sats", err) - } + log.Println("Error accessing file sats", err) + } } } diff --git a/output_http.go b/output_http.go index 937c8150..b8f06dfb 100644 --- a/output_http.go +++ b/output_http.go @@ -25,9 +25,9 @@ type HTTPOutputConfig struct { stats bool workersMin int workersMax int - statsMs int - workers int - queueLen int + statsMs int + workers int + queueLen int elasticSearch string @@ -35,6 +35,8 @@ type HTTPOutputConfig struct { OriginalHost bool BufferSize int + CompatibilityMode bool + Debug bool TrackResponses bool @@ -113,6 +115,7 @@ func (o *HTTPOutput) startWorker() { OriginalHost: o.config.OriginalHost, Timeout: o.config.Timeout, ResponseBufferSize: o.config.BufferSize, + CompatibilityMode: o.config.CompatibilityMode, }) deathCount := 0 diff --git a/settings.go b/settings.go index 5eb2596a..f4865858 100644 --- a/settings.go +++ b/settings.go @@ -50,16 +50,16 @@ type AppSettings struct { outputFile MultiOption outputFileConfig FileOutputConfig - inputRAW MultiOption - inputRAWEngine string - inputRAWTrackResponse bool - inputRAWRealIPHeader string - inputRAWExpire time.Duration - inputRAWBpfFilter string - inputRAWTimestampType string - copyBufferSize int - inputRAWImmediateMode bool - inputRawBufferSize int + inputRAW MultiOption + inputRAWEngine string + inputRAWTrackResponse bool + inputRAWRealIPHeader string + inputRAWExpire time.Duration + inputRAWBpfFilter string + inputRAWTimestampType string + copyBufferSize int + inputRAWImmediateMode bool + inputRawBufferSize int inputRAWOverrideSnapLen bool middleware string @@ -153,8 +153,9 @@ func init() { flag.Var(&Settings.outputHTTP, "output-http", "Forwards incoming requests to given http address.\n\t# Redirect all incoming requests to staging.com address \n\tgor --input-raw :80 --output-http http://staging.com") flag.IntVar(&Settings.outputHTTPConfig.BufferSize, "output-http-response-buffer", 0, "HTTP response buffer size, all data after this size will be discarded.") + flag.BoolVar(&Settings.outputHTTPConfig.CompatibilityMode, "output-http-compatibility-mode", false, "Use standard Go client, instead of built-in implementation. Can be slower, but more compatible.") - flag.IntVar(&Settings.outputHTTPConfig.workersMin, "output-http-workers-min", 0, "Gor uses dynamic worker scaling. Enter a number to set a minimum number of workers. default = 1.") + flag.IntVar(&Settings.outputHTTPConfig.workersMin, "output-http-workers-min", 0, "Gor uses dynamic worker scaling. Enter a number to set a minimum number of workers. default = 1.") flag.IntVar(&Settings.outputHTTPConfig.workersMax, "output-http-workers", 0, "Gor uses dynamic worker scaling. Enter a number to set a maximum number of workers. default = 0 = unlimited.") flag.IntVar(&Settings.outputHTTPConfig.queueLen, "output-http-queue-len", 1000, "Number of requests that can be queued for output, if all workers are busy. default = 1000")