-
Notifications
You must be signed in to change notification settings - Fork 1.7k
/
http.go
224 lines (198 loc) · 7.43 KB
/
http.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
package common
import (
"crypto/tls"
"crypto/x509"
"io"
"net"
"net/http"
"strings"
"time"
"github.com/hashicorp/go-retryablehttp"
)
var caCerts = []string{
// CN = ISRG Root X1
// TODO: Expires Monday, June 4, 2035 at 4:04:38 AM Pacific
`
-----BEGIN CERTIFICATE-----
MIIFazCCA1OgAwIBAgIRAIIQz7DSQONZRGPgu2OCiwAwDQYJKoZIhvcNAQELBQAw
TzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2Vh
cmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwHhcNMTUwNjA0MTEwNDM4
WhcNMzUwNjA0MTEwNDM4WjBPMQswCQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJu
ZXQgU2VjdXJpdHkgUmVzZWFyY2ggR3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBY
MTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAK3oJHP0FDfzm54rVygc
h77ct984kIxuPOZXoHj3dcKi/vVqbvYATyjb3miGbESTtrFj/RQSa78f0uoxmyF+
0TM8ukj13Xnfs7j/EvEhmkvBioZxaUpmZmyPfjxwv60pIgbz5MDmgK7iS4+3mX6U
A5/TR5d8mUgjU+g4rk8Kb4Mu0UlXjIB0ttov0DiNewNwIRt18jA8+o+u3dpjq+sW
T8KOEUt+zwvo/7V3LvSye0rgTBIlDHCNAymg4VMk7BPZ7hm/ELNKjD+Jo2FR3qyH
B5T0Y3HsLuJvW5iB4YlcNHlsdu87kGJ55tukmi8mxdAQ4Q7e2RCOFvu396j3x+UC
B5iPNgiV5+I3lg02dZ77DnKxHZu8A/lJBdiB3QW0KtZB6awBdpUKD9jf1b0SHzUv
KBds0pjBqAlkd25HN7rOrFleaJ1/ctaJxQZBKT5ZPt0m9STJEadao0xAH0ahmbWn
OlFuhjuefXKnEgV4We0+UXgVCwOPjdAvBbI+e0ocS3MFEvzG6uBQE3xDk3SzynTn
jh8BCNAw1FtxNrQHusEwMFxIt4I7mKZ9YIqioymCzLq9gwQbooMDQaHWBfEbwrbw
qHyGO0aoSCqI3Haadr8faqU9GY/rOPNk3sgrDQoo//fb4hVC1CLQJ13hef4Y53CI
rU7m2Ys6xt0nUW7/vGT1M0NPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNV
HRMBAf8EBTADAQH/MB0GA1UdDgQWBBR5tFnme7bl5AFzgAiIyBpY9umbbjANBgkq
hkiG9w0BAQsFAAOCAgEAVR9YqbyyqFDQDLHYGmkgJykIrGF1XIpu+ILlaS/V9lZL
ubhzEFnTIZd+50xx+7LSYK05qAvqFyFWhfFQDlnrzuBZ6brJFe+GnY+EgPbk6ZGQ
3BebYhtF8GaV0nxvwuo77x/Py9auJ/GpsMiu/X1+mvoiBOv/2X/qkSsisRcOj/KK
NFtY2PwByVS5uCbMiogziUwthDyC3+6WVwW6LLv3xLfHTjuCvjHIInNzktHCgKQ5
ORAzI4JMPJ+GslWYHb4phowim57iaztXOoJwTdwJx4nLCgdNbOhdjsnvzqvHu7Ur
TkXWStAmzOVyyghqpZXjFaH3pO3JLF+l+/+sKAIuvtd7u+Nxe5AW0wdeRlN8NwdC
jNPElpzVmbUq4JUagEiuTDkHzsxHpFKVK7q4+63SM1N95R1NbdWhscdCb+ZAJzVc
oyi3B43njTOQ5yOf+1CceWxG1bQVs5ZufpsMljq4Ui0/1lvh+wjChP4kqKOJ2qxq
4RgqsahDYVvTH9w7jXbyLeiNdd8XM2w9U/t7y0Ff/9yi0GE44Za4rF2LN9d11TPA
mRGunUHBcnWEvgJBQl9nJEiU0Zsnvgc/ubhPgXRR4Xq37Z0j4r7g1SgEEzwxA57d
emyPxgcYxn/eR44/KJ4EBs+lVDR3veyJm+kXQ99b21/+jh5Xos1AnX5iItreGCc=
-----END CERTIFICATE-----
`,
// CN = ISRG Root X2
// TODO: Expires September 17, 2040 at 9:00:00 AM Pacific Daylight Time
`
-----BEGIN CERTIFICATE-----
MIICGzCCAaGgAwIBAgIQQdKd0XLq7qeAwSxs6S+HUjAKBggqhkjOPQQDAzBPMQsw
CQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJuZXQgU2VjdXJpdHkgUmVzZWFyY2gg
R3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBYMjAeFw0yMDA5MDQwMDAwMDBaFw00
MDA5MTcxNjAwMDBaME8xCzAJBgNVBAYTAlVTMSkwJwYDVQQKEyBJbnRlcm5ldCBT
ZWN1cml0eSBSZXNlYXJjaCBHcm91cDEVMBMGA1UEAxMMSVNSRyBSb290IFgyMHYw
EAYHKoZIzj0CAQYFK4EEACIDYgAEzZvVn4CDCuwJSvMWSj5cz3es3mcFDR0HttwW
+1qLFNvicWDEukWVEYmO6gbf9yoWHKS5xcUy4APgHoIYOIvXRdgKam7mAHf7AlF9
ItgKbppbd9/w+kHsOdx1ymgHDB/qo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0T
AQH/BAUwAwEB/zAdBgNVHQ4EFgQUfEKWrt5LSDv6kviejM9ti6lyN5UwCgYIKoZI
zj0EAwMDaAAwZQIwe3lORlCEwkSHRhtFcP9Ymd70/aTSVaYgLXTWNLxBo1BfASdW
tL4ndQavEi51mI38AjEAi/V3bNTIZargCyzuFJ0nN6T5U6VR5CmD1/iQMVtCnwr1
/q4AaOeMSQ+2b1tbFfLn
-----END CERTIFICATE-----
`,
}
func PinnedCertPool() *x509.CertPool {
trustedCerts := x509.NewCertPool()
for _, cert := range caCerts {
trustedCerts.AppendCertsFromPEM([]byte(strings.TrimSpace(cert)))
}
return trustedCerts
}
type FakeTransport struct {
CreateResponse func(req *http.Request) (*http.Response, error)
}
func (t FakeTransport) RoundTrip(req *http.Request) (*http.Response, error) {
return t.CreateResponse(req)
}
type CustomTransport struct {
T http.RoundTripper
}
func (t *CustomTransport) RoundTrip(req *http.Request) (*http.Response, error) {
req.Header.Add("User-Agent", "TruffleHog")
return t.T.RoundTrip(req)
}
func NewCustomTransport(T http.RoundTripper) *CustomTransport {
if T == nil {
T = http.DefaultTransport
}
return &CustomTransport{T}
}
func ConstantResponseHttpClient(statusCode int, body string) *http.Client {
return &http.Client{
Timeout: DefaultResponseTimeout,
Transport: FakeTransport{
CreateResponse: func(req *http.Request) (*http.Response, error) {
return &http.Response{
Request: req,
Body: io.NopCloser(strings.NewReader(body)),
StatusCode: statusCode,
}, nil
},
},
}
}
// ClientOption configures how we set up the client.
type ClientOption func(*retryablehttp.Client)
// WithCheckRetry allows setting a custom CheckRetry policy.
func WithCheckRetry(cr retryablehttp.CheckRetry) ClientOption {
return func(c *retryablehttp.Client) { c.CheckRetry = cr }
}
// WithBackoff allows setting a custom backoff policy.
func WithBackoff(b retryablehttp.Backoff) ClientOption {
return func(c *retryablehttp.Client) { c.Backoff = b }
}
// WithTimeout allows setting a custom timeout.
func WithTimeout(timeout time.Duration) ClientOption {
return func(c *retryablehttp.Client) { c.HTTPClient.Timeout = timeout }
}
// WithMaxRetries allows setting a custom maximum number of retries.
func WithMaxRetries(retries int) ClientOption {
return func(c *retryablehttp.Client) { c.RetryMax = retries }
}
// WithRetryWaitMin allows setting a custom minimum retry wait.
func WithRetryWaitMin(wait time.Duration) ClientOption {
return func(c *retryablehttp.Client) { c.RetryWaitMin = wait }
}
// WithRetryWaitMax allows setting a custom maximum retry wait.
func WithRetryWaitMax(wait time.Duration) ClientOption {
return func(c *retryablehttp.Client) { c.RetryWaitMax = wait }
}
func PinnedRetryableHttpClient() *http.Client {
httpClient := retryablehttp.NewClient()
httpClient.Logger = nil
httpClient.HTTPClient.Transport = NewCustomTransport(&http.Transport{
TLSClientConfig: &tls.Config{
RootCAs: PinnedCertPool(),
},
Proxy: http.ProxyFromEnvironment,
DialContext: (&net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
}).DialContext,
ForceAttemptHTTP2: true,
MaxIdleConns: 100,
IdleConnTimeout: 90 * time.Second,
TLSHandshakeTimeout: 10 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
})
return httpClient.StandardClient()
}
func RetryableHTTPClient(opts ...ClientOption) *http.Client {
httpClient := retryablehttp.NewClient()
httpClient.RetryMax = 3
httpClient.Logger = nil
httpClient.HTTPClient.Transport = NewCustomTransport(nil)
for _, opt := range opts {
opt(httpClient)
}
return httpClient.StandardClient()
}
func RetryableHTTPClientTimeout(timeOutSeconds int64, opts ...ClientOption) *http.Client {
httpClient := retryablehttp.NewClient()
httpClient.RetryMax = 3
httpClient.Logger = nil
httpClient.HTTPClient.Timeout = time.Duration(timeOutSeconds) * time.Second
httpClient.HTTPClient.Transport = NewCustomTransport(nil)
for _, opt := range opts {
opt(httpClient)
}
return httpClient.StandardClient()
}
const DefaultResponseTimeout = 5 * time.Second
var saneTransport = &http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: (&net.Dialer{
Timeout: 2 * time.Second,
KeepAlive: 5 * time.Second,
}).DialContext,
MaxIdleConns: 5,
IdleConnTimeout: 5 * time.Second,
TLSHandshakeTimeout: 3 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
}
func SaneHttpClient() *http.Client {
httpClient := &http.Client{}
httpClient.Timeout = DefaultResponseTimeout
httpClient.Transport = NewCustomTransport(saneTransport)
return httpClient
}
// SaneHttpClientTimeOut adds a custom timeout for some scanners
func SaneHttpClientTimeOut(timeout time.Duration) *http.Client {
httpClient := &http.Client{}
httpClient.Timeout = timeout
httpClient.Transport = NewCustomTransport(nil)
return httpClient
}