From 6a399fd16ddddc6beb067977ef72f81507b5120a Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Thu, 5 Jan 2023 09:29:49 +0100 Subject: [PATCH] feat: implement better integration tests (#34) There's a need to write better integration tests for this repository. In particular: 1. we need to have confidence that the JA3 signature produced when using a uTLS client is different than the default one (i.e., can we be confident that it's possible to replace TLS?) 2. we need to have robust tests that do not depend onto external services but only use localhost, so they don't break often. Part of https://github.com/ooni/probe/issues/2273 --- .github/workflows/go.yml | 8 +- example/example-proxy/go.mod | 16 -- example/example-proxy/go.sum | 14 - example/example-proxy/main.go | 22 +- example/example-proxy/tls.go | 62 ----- example/example-utls-with-dial/.gitignore | 2 +- example/example-utls-with-dial/go.mod | 16 -- example/example-utls-with-dial/go.sum | 12 - example/example-utls-with-dial/http.go | 62 ----- example/example-utls-with-dial/main.go | 17 ++ example/example-utls-with-dial/main_test.go | 41 ++- example/example-utls-with-dial/tls.go | 49 ---- example/example-utls/go.mod | 16 -- example/example-utls/go.sum | 12 - example/example-utls/http.go | 25 -- example/example-utls/main.go | 12 +- example/example-utls/tls.go | 62 ----- example/go.mod | 23 ++ example/go.sum | 122 +++++++++ example/internal/ja3x/conn.go | 49 ++++ example/internal/ja3x/doc.go | 2 + example/internal/ja3x/integration_test.go | 160 +++++++++++ example/internal/ja3x/server.go | 242 +++++++++++++++++ example/internal/ja3x/server_test.go | 90 +++++++ example/internal/ja3x/utils.go | 16 ++ example/internal/ja3x/utils_test.go | 16 ++ example/internal/runtimex/runtimex.go | 36 +++ example/internal/runtimex/runtimex_test.go | 93 +++++++ example/internal/utlsx/utlsx.go | 172 ++++++++++++ example/internal/utlsx/utlsx_test.go | 284 ++++++++++++++++++++ 30 files changed, 1382 insertions(+), 371 deletions(-) delete mode 100644 example/example-proxy/go.mod delete mode 100644 example/example-proxy/go.sum delete mode 100644 example/example-proxy/tls.go delete mode 100644 example/example-utls-with-dial/go.mod delete mode 100644 example/example-utls-with-dial/go.sum delete mode 100644 example/example-utls-with-dial/http.go delete mode 100644 example/example-utls-with-dial/tls.go delete mode 100644 example/example-utls/go.mod delete mode 100644 example/example-utls/go.sum delete mode 100644 example/example-utls/http.go delete mode 100644 example/example-utls/tls.go create mode 100644 example/go.mod create mode 100644 example/go.sum create mode 100644 example/internal/ja3x/conn.go create mode 100644 example/internal/ja3x/doc.go create mode 100644 example/internal/ja3x/integration_test.go create mode 100644 example/internal/ja3x/server.go create mode 100644 example/internal/ja3x/server_test.go create mode 100644 example/internal/ja3x/utils.go create mode 100644 example/internal/ja3x/utils_test.go create mode 100644 example/internal/runtimex/runtimex.go create mode 100644 example/internal/runtimex/runtimex_test.go create mode 100644 example/internal/utlsx/utlsx.go create mode 100644 example/internal/utlsx/utlsx_test.go diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index b77672b5..63678c6f 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -22,4 +22,10 @@ jobs: run: go build -v ./... - name: Test - run: go test -v -race ./... + run: go test -cover -race ./... + + - name: Install dependencies + run: sudo apt-get install libpcap-dev + + - name: Test Examples + run: cd example && go test -cover -race ./... diff --git a/example/example-proxy/go.mod b/example/example-proxy/go.mod deleted file mode 100644 index c5cb4fea..00000000 --- a/example/example-proxy/go.mod +++ /dev/null @@ -1,16 +0,0 @@ -module github.com/ooni/oohttp/example/example-proxy - -go 1.18 - -require ( - github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 - github.com/ooni/oohttp v0.0.0-20220521113303-fb27ebcf5f1e - github.com/refraction-networking/utls v1.1.0 -) - -require ( - golang.org/x/crypto v0.0.0-20220518034528-6f7dac969898 // indirect - golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2 // indirect - golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect - golang.org/x/text v0.3.7 // indirect -) diff --git a/example/example-proxy/go.sum b/example/example-proxy/go.sum deleted file mode 100644 index c547d88f..00000000 --- a/example/example-proxy/go.sum +++ /dev/null @@ -1,14 +0,0 @@ -github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= -github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= -github.com/ooni/oohttp v0.0.0-20220521113303-fb27ebcf5f1e h1:hM6+SmEh6aCzXZDIHTwA0UeyjXNy7EfK1NpE3zr3WBo= -github.com/ooni/oohttp v0.0.0-20220521113303-fb27ebcf5f1e/go.mod h1:p2VVLbs+BXBIgTHITV9Vw8Rv6G1u66JUWP/8KCgDGNo= -github.com/refraction-networking/utls v1.1.0 h1:dKXJwSqni/t5csYJ+aQcEgqB7AMWYi6EUc9u3bEmjX0= -github.com/refraction-networking/utls v1.1.0/go.mod h1:tz9gX959MEFfFN5whTIocCLUG57WiILqtdVxI8c6Wj0= -golang.org/x/crypto v0.0.0-20220518034528-6f7dac969898 h1:SLP7Q4Di66FONjDJbCYrCRrh97focO6sLogHO7/g8F0= -golang.org/x/crypto v0.0.0-20220518034528-6f7dac969898/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2 h1:NWy5+hlRbC7HK+PmcXVUmW1IMyFce7to56IUvhUFm7Y= -golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= diff --git a/example/example-proxy/main.go b/example/example-proxy/main.go index 56944d80..75462486 100644 --- a/example/example-proxy/main.go +++ b/example/example-proxy/main.go @@ -9,10 +9,10 @@ import ( "net" "net/http" "net/url" - "time" "github.com/armon/go-socks5" oohttp "github.com/ooni/oohttp" + "github.com/ooni/oohttp/example/internal/utlsx" ) // startProxyServer starts a SOCKS5 proxy server at the given endpoint. @@ -36,19 +36,13 @@ func startProxyServer(endpoint string, ch chan<- interface{}) { // when communicating with a remote TLS endpoint through the given proxy. func useProxy(URL, proxy string, tlsClientFactory func(conn net.Conn, config *tls.Config) oohttp.TLSConn) { - w := &oohttp.StdlibTransport{ - Transport: &oohttp.Transport{ - Proxy: func(*oohttp.Request) (*url.URL, error) { - return &url.URL{Scheme: "socks5", Host: proxy}, nil - }, - ForceAttemptHTTP2: true, - MaxIdleConns: 100, - IdleConnTimeout: 90 * time.Second, - TLSHandshakeTimeout: 10 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - TLSClientFactory: tlsClientFactory, + w := utlsx.NewOOHTTPTransport( + func(*oohttp.Request) (*url.URL, error) { + return &url.URL{Scheme: "socks5", Host: proxy}, nil }, - } + tlsClientFactory, + nil, // we're using tlsClientFactory to wrap dialed connections + ) clnt := &http.Client{Transport: w} resp, err := clnt.Get(URL) if err != nil { @@ -73,7 +67,7 @@ func main() { flag.Parse() var ffun func(conn net.Conn, config *tls.Config) oohttp.TLSConn if *utls { - ffun = utlsFactory + ffun = (&utlsx.FactoryWithParrot{}).NewUTLSConn } ch := make(chan interface{}) go startProxyServer(*proxy, ch) diff --git a/example/example-proxy/tls.go b/example/example-proxy/tls.go deleted file mode 100644 index 755e0ad6..00000000 --- a/example/example-proxy/tls.go +++ /dev/null @@ -1,62 +0,0 @@ -package main - -import ( - "context" - "crypto/tls" - "net" - - oohttp "github.com/ooni/oohttp" - utls "github.com/refraction-networking/utls" -) - -// utlsConnAdapter adapts utls.UConn to the oohttp.TLSConn interface. -type utlsConnAdapter struct { - *utls.UConn -} - -var _ oohttp.TLSConn = &utlsConnAdapter{} - -// ConnectionState implements TLSConn's ConnectionState. -func (c *utlsConnAdapter) ConnectionState() tls.ConnectionState { - ustate := c.UConn.ConnectionState() - return tls.ConnectionState{ - Version: ustate.Version, - HandshakeComplete: ustate.HandshakeComplete, - DidResume: ustate.DidResume, - CipherSuite: ustate.CipherSuite, - NegotiatedProtocol: ustate.NegotiatedProtocol, - NegotiatedProtocolIsMutual: ustate.NegotiatedProtocolIsMutual, - ServerName: ustate.ServerName, - PeerCertificates: ustate.PeerCertificates, - VerifiedChains: ustate.VerifiedChains, - SignedCertificateTimestamps: ustate.SignedCertificateTimestamps, - OCSPResponse: ustate.OCSPResponse, - TLSUnique: ustate.TLSUnique, - } -} - -// HandshakeContext implements TLSConn's HandshakeContext. -func (c *utlsConnAdapter) HandshakeContext(ctx context.Context) error { - errch := make(chan error, 1) - go func() { - errch <- c.UConn.Handshake() - }() - select { - case err := <-errch: - return err - case <-ctx.Done(): - return ctx.Err() - } -} - -// utlsFactory creates a new uTLS connection. -func utlsFactory(conn net.Conn, config *tls.Config) oohttp.TLSConn { - uConfig := &utls.Config{ - RootCAs: config.RootCAs, - NextProtos: config.NextProtos, - ServerName: config.ServerName, - InsecureSkipVerify: config.InsecureSkipVerify, - DynamicRecordSizingDisabled: config.DynamicRecordSizingDisabled, - } - return &utlsConnAdapter{utls.UClient(conn, uConfig, utls.HelloFirefox_55)} -} diff --git a/example/example-utls-with-dial/.gitignore b/example/example-utls-with-dial/.gitignore index 19a41141..b7c87cc8 100644 --- a/example/example-utls-with-dial/.gitignore +++ b/example/example-utls-with-dial/.gitignore @@ -1 +1 @@ -/example-utls +/example-utls-with-dial diff --git a/example/example-utls-with-dial/go.mod b/example/example-utls-with-dial/go.mod deleted file mode 100644 index f93add2a..00000000 --- a/example/example-utls-with-dial/go.mod +++ /dev/null @@ -1,16 +0,0 @@ -module github.com/ooni/oohttp/example/example-utls-with-dial - -go 1.18 - -require github.com/ooni/oohttp v0.0.0-20220521113303-fb27ebcf5f1e - -require ( - github.com/refraction-networking/utls v1.1.0 - golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2 // indirect - golang.org/x/text v0.3.7 // indirect -) - -require ( - golang.org/x/crypto v0.0.0-20220518034528-6f7dac969898 // indirect - golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect -) diff --git a/example/example-utls-with-dial/go.sum b/example/example-utls-with-dial/go.sum deleted file mode 100644 index e6ecaec5..00000000 --- a/example/example-utls-with-dial/go.sum +++ /dev/null @@ -1,12 +0,0 @@ -github.com/ooni/oohttp v0.0.0-20220521113303-fb27ebcf5f1e h1:hM6+SmEh6aCzXZDIHTwA0UeyjXNy7EfK1NpE3zr3WBo= -github.com/ooni/oohttp v0.0.0-20220521113303-fb27ebcf5f1e/go.mod h1:p2VVLbs+BXBIgTHITV9Vw8Rv6G1u66JUWP/8KCgDGNo= -github.com/refraction-networking/utls v1.1.0 h1:dKXJwSqni/t5csYJ+aQcEgqB7AMWYi6EUc9u3bEmjX0= -github.com/refraction-networking/utls v1.1.0/go.mod h1:tz9gX959MEFfFN5whTIocCLUG57WiILqtdVxI8c6Wj0= -golang.org/x/crypto v0.0.0-20220518034528-6f7dac969898 h1:SLP7Q4Di66FONjDJbCYrCRrh97focO6sLogHO7/g8F0= -golang.org/x/crypto v0.0.0-20220518034528-6f7dac969898/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2 h1:NWy5+hlRbC7HK+PmcXVUmW1IMyFce7to56IUvhUFm7Y= -golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= diff --git a/example/example-utls-with-dial/http.go b/example/example-utls-with-dial/http.go deleted file mode 100644 index e908d578..00000000 --- a/example/example-utls-with-dial/http.go +++ /dev/null @@ -1,62 +0,0 @@ -package main - -import ( - "context" - "log" - "net" - "net/http" - "time" - - oohttp "github.com/ooni/oohttp" - utls "github.com/refraction-networking/utls" -) - -// defaultDialer is the default Dialer. -var defaultDialer = &net.Dialer{ - Timeout: 30 * time.Second, - KeepAlive: 30 * time.Second, -} - -// dialTLSContext dials a TLS connection using refraction-networking/utls -// and returns a TLSConn compatible net.Conn. -func dialTLSContext(ctx context.Context, network string, addr string) (net.Conn, error) { - conn, err := defaultDialer.DialContext(ctx, network, addr) - if err != nil { - return nil, err - } - sni, _, err := net.SplitHostPort(addr) - if err != nil { - return nil, err - } - uconfig := &utls.Config{ServerName: sni} - // Implementation note: using Firefox 55 ClientHello because that - // avoids a bunch of issues, e.g., Brotli encrypted x509 certs. - uconn := utls.UClient(conn, uconfig, utls.HelloFirefox_55) - uadapter := &utlsConnAdapter{uconn} - if err := uadapter.HandshakeContext(ctx); err != nil { - conn.Close() - return nil, err - } - proto := uadapter.ConnectionState().NegotiatedProtocol - log.Printf("negotiated protocol: %s", proto) - return uadapter, nil -} - -// newTransport returns a new http.Transport using the provided tls dialer. -func newTransport(f func(ctx context.Context, network, address string) (net.Conn, error)) http.RoundTripper { - return &oohttp.StdlibTransport{ - Transport: &oohttp.Transport{ - Proxy: oohttp.ProxyFromEnvironment, - DialContext: defaultDialer.DialContext, - DialTLSContext: f, - ForceAttemptHTTP2: true, - MaxIdleConns: 100, - IdleConnTimeout: 90 * time.Second, - TLSHandshakeTimeout: 10 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - }, - } -} - -// defaultTransport is the default http.RoundTripper. -var defaultTransport = newTransport(dialTLSContext) diff --git a/example/example-utls-with-dial/main.go b/example/example-utls-with-dial/main.go index 40cbb489..3ae65be7 100644 --- a/example/example-utls-with-dial/main.go +++ b/example/example-utls-with-dial/main.go @@ -1,13 +1,30 @@ package main import ( + "context" "flag" "fmt" "io" "log" + "net" "net/http" + + oohttp "github.com/ooni/oohttp" + "github.com/ooni/oohttp/example/internal/utlsx" ) +// newTransport returns a new http.Transport using the provided tls dialer. +func newTransport(f func(ctx context.Context, network, address string) (net.Conn, error)) http.RoundTripper { + return utlsx.NewOOHTTPTransport( + oohttp.ProxyFromEnvironment, + nil, // we're using f to directly create TLS connections + f, + ) +} + +// defaultTransport is the default http.RoundTripper. +var defaultTransport = newTransport((&utlsx.TLSDialer{}).DialTLSContext) + // newClient creates a new http.Client using the given transport. func newClient(txp http.RoundTripper) *http.Client { return &http.Client{Transport: txp} diff --git a/example/example-utls-with-dial/main_test.go b/example/example-utls-with-dial/main_test.go index a4e1a7fd..9f4f3d4a 100644 --- a/example/example-utls-with-dial/main_test.go +++ b/example/example-utls-with-dial/main_test.go @@ -2,24 +2,33 @@ package main import ( "context" + "crypto/tls" "io" "log" "net" + "net/http" + "net/http/httptest" "sync" "testing" oohttp "github.com/ooni/oohttp" + "github.com/ooni/oohttp/example/internal/ja3x" + "github.com/ooni/oohttp/example/internal/utlsx" ) // tlsDialerRecorder performs TLS dials and records the ALPN. type tlsDialerRecorder struct { - alpn map[string]int - mu sync.Mutex + alpn map[string]int + config *tls.Config + mu sync.Mutex } // do is like dialTLSContext but also records the ALPN. func (d *tlsDialerRecorder) do(ctx context.Context, network string, addr string) (net.Conn, error) { - conn, err := dialTLSContext(ctx, network, addr) + child := &utlsx.TLSDialer{ + Config: d.config, + } + conn, err := child.DialTLSContext(ctx, network, addr) if err != nil { return nil, err } @@ -35,9 +44,15 @@ func (d *tlsDialerRecorder) do(ctx context.Context, network string, addr string) } func TestWorkAsIntendedWithH2(t *testing.T) { - d := &tlsDialerRecorder{} + srvr := ja3x.NewServer("h2") + defer srvr.Close() + d := &tlsDialerRecorder{ + alpn: map[string]int{}, + config: srvr.ClientConfig(), + mu: sync.Mutex{}, + } clnt := newClient(newTransport(d.do)) - resp, err := clnt.Get("https://www.facebook.com") + resp, err := clnt.Get(srvr.URL()) if err != nil { log.Fatal(err) } @@ -53,9 +68,15 @@ func TestWorkAsIntendedWithH2(t *testing.T) { } func TestWorkAsIntendedWithHTTP11(t *testing.T) { - d := &tlsDialerRecorder{} + srvr := ja3x.NewServer("http/1.1") + defer srvr.Close() + d := &tlsDialerRecorder{ + alpn: map[string]int{}, + config: srvr.ClientConfig(), + mu: sync.Mutex{}, + } clnt := newClient(newTransport(d.do)) - resp, err := clnt.Get("https://nexa.polito.it") + resp, err := clnt.Get(srvr.URL()) if err != nil { log.Fatal(err) } @@ -71,7 +92,11 @@ func TestWorkAsIntendedWithHTTP11(t *testing.T) { } func TestWorkAsIntendedWithHTTP(t *testing.T) { - resp, err := defaultClient.Get("http://example.com") + srvr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("0xdeadbeef")) + })) + defer srvr.Close() + resp, err := defaultClient.Get(srvr.URL) if err != nil { log.Fatal(err) } diff --git a/example/example-utls-with-dial/tls.go b/example/example-utls-with-dial/tls.go deleted file mode 100644 index 7835ec20..00000000 --- a/example/example-utls-with-dial/tls.go +++ /dev/null @@ -1,49 +0,0 @@ -package main - -import ( - "context" - "crypto/tls" - - oohttp "github.com/ooni/oohttp" - utls "github.com/refraction-networking/utls" -) - -// utlsConnAdapter adapts utls.UConn to the oohttp.TLSConn interface. -type utlsConnAdapter struct { - *utls.UConn -} - -var _ oohttp.TLSConn = &utlsConnAdapter{} - -// ConnectionState implements TLSConn's ConnectionState. -func (c *utlsConnAdapter) ConnectionState() tls.ConnectionState { - ustate := c.UConn.ConnectionState() - return tls.ConnectionState{ - Version: ustate.Version, - HandshakeComplete: ustate.HandshakeComplete, - DidResume: ustate.DidResume, - CipherSuite: ustate.CipherSuite, - NegotiatedProtocol: ustate.NegotiatedProtocol, - NegotiatedProtocolIsMutual: ustate.NegotiatedProtocolIsMutual, - ServerName: ustate.ServerName, - PeerCertificates: ustate.PeerCertificates, - VerifiedChains: ustate.VerifiedChains, - SignedCertificateTimestamps: ustate.SignedCertificateTimestamps, - OCSPResponse: ustate.OCSPResponse, - TLSUnique: ustate.TLSUnique, - } -} - -// HandshakeContext implements TLSConn's HandshakeContext. -func (c *utlsConnAdapter) HandshakeContext(ctx context.Context) error { - errch := make(chan error, 1) - go func() { - errch <- c.UConn.Handshake() - }() - select { - case err := <-errch: - return err - case <-ctx.Done(): - return ctx.Err() - } -} diff --git a/example/example-utls/go.mod b/example/example-utls/go.mod deleted file mode 100644 index 26923abb..00000000 --- a/example/example-utls/go.mod +++ /dev/null @@ -1,16 +0,0 @@ -module github.com/ooni/oohttp/example/example-utls - -go 1.18 - -require github.com/ooni/oohttp v0.0.0-20220521113303-fb27ebcf5f1e - -require ( - github.com/refraction-networking/utls v1.1.0 - golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2 // indirect - golang.org/x/text v0.3.7 // indirect -) - -require ( - golang.org/x/crypto v0.0.0-20220518034528-6f7dac969898 // indirect - golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect -) diff --git a/example/example-utls/go.sum b/example/example-utls/go.sum deleted file mode 100644 index e6ecaec5..00000000 --- a/example/example-utls/go.sum +++ /dev/null @@ -1,12 +0,0 @@ -github.com/ooni/oohttp v0.0.0-20220521113303-fb27ebcf5f1e h1:hM6+SmEh6aCzXZDIHTwA0UeyjXNy7EfK1NpE3zr3WBo= -github.com/ooni/oohttp v0.0.0-20220521113303-fb27ebcf5f1e/go.mod h1:p2VVLbs+BXBIgTHITV9Vw8Rv6G1u66JUWP/8KCgDGNo= -github.com/refraction-networking/utls v1.1.0 h1:dKXJwSqni/t5csYJ+aQcEgqB7AMWYi6EUc9u3bEmjX0= -github.com/refraction-networking/utls v1.1.0/go.mod h1:tz9gX959MEFfFN5whTIocCLUG57WiILqtdVxI8c6Wj0= -golang.org/x/crypto v0.0.0-20220518034528-6f7dac969898 h1:SLP7Q4Di66FONjDJbCYrCRrh97focO6sLogHO7/g8F0= -golang.org/x/crypto v0.0.0-20220518034528-6f7dac969898/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2 h1:NWy5+hlRbC7HK+PmcXVUmW1IMyFce7to56IUvhUFm7Y= -golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= diff --git a/example/example-utls/http.go b/example/example-utls/http.go deleted file mode 100644 index 9815a040..00000000 --- a/example/example-utls/http.go +++ /dev/null @@ -1,25 +0,0 @@ -package main - -import ( - "crypto/tls" - "net" - "net/http" - "time" - - oohttp "github.com/ooni/oohttp" -) - -// newTransport returns a new http.Transport using the provided tls dialer. -func newTransport(f func(conn net.Conn, config *tls.Config) oohttp.TLSConn) http.RoundTripper { - return &oohttp.StdlibTransport{ - Transport: &oohttp.Transport{ - Proxy: oohttp.ProxyFromEnvironment, - ForceAttemptHTTP2: true, - MaxIdleConns: 100, - IdleConnTimeout: 90 * time.Second, - TLSHandshakeTimeout: 10 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - TLSClientFactory: f, - }, - } -} diff --git a/example/example-utls/main.go b/example/example-utls/main.go index 1b94eb54..da74c18d 100644 --- a/example/example-utls/main.go +++ b/example/example-utls/main.go @@ -10,8 +10,18 @@ import ( "net/http" oohttp "github.com/ooni/oohttp" + "github.com/ooni/oohttp/example/internal/utlsx" ) +// newTransport returns a new http.Transport using the provided tls dialer. +func newTransport(f func(conn net.Conn, config *tls.Config) oohttp.TLSConn) http.RoundTripper { + return utlsx.NewOOHTTPTransport( + oohttp.ProxyFromEnvironment, + f, + nil, // we're using f to wrap dialed connections + ) +} + // newClient creates a new http.Client using the given transport. func newClient(txp http.RoundTripper) *http.Client { return &http.Client{Transport: txp} @@ -26,7 +36,7 @@ func main() { flag.Parse() var ffun func(conn net.Conn, config *tls.Config) oohttp.TLSConn if *utls { - ffun = utlsFactory + ffun = (&utlsx.FactoryWithParrot{}).NewUTLSConn } resp, err := newClient(newTransport(ffun)).Get(*url) if err != nil { diff --git a/example/example-utls/tls.go b/example/example-utls/tls.go deleted file mode 100644 index 755e0ad6..00000000 --- a/example/example-utls/tls.go +++ /dev/null @@ -1,62 +0,0 @@ -package main - -import ( - "context" - "crypto/tls" - "net" - - oohttp "github.com/ooni/oohttp" - utls "github.com/refraction-networking/utls" -) - -// utlsConnAdapter adapts utls.UConn to the oohttp.TLSConn interface. -type utlsConnAdapter struct { - *utls.UConn -} - -var _ oohttp.TLSConn = &utlsConnAdapter{} - -// ConnectionState implements TLSConn's ConnectionState. -func (c *utlsConnAdapter) ConnectionState() tls.ConnectionState { - ustate := c.UConn.ConnectionState() - return tls.ConnectionState{ - Version: ustate.Version, - HandshakeComplete: ustate.HandshakeComplete, - DidResume: ustate.DidResume, - CipherSuite: ustate.CipherSuite, - NegotiatedProtocol: ustate.NegotiatedProtocol, - NegotiatedProtocolIsMutual: ustate.NegotiatedProtocolIsMutual, - ServerName: ustate.ServerName, - PeerCertificates: ustate.PeerCertificates, - VerifiedChains: ustate.VerifiedChains, - SignedCertificateTimestamps: ustate.SignedCertificateTimestamps, - OCSPResponse: ustate.OCSPResponse, - TLSUnique: ustate.TLSUnique, - } -} - -// HandshakeContext implements TLSConn's HandshakeContext. -func (c *utlsConnAdapter) HandshakeContext(ctx context.Context) error { - errch := make(chan error, 1) - go func() { - errch <- c.UConn.Handshake() - }() - select { - case err := <-errch: - return err - case <-ctx.Done(): - return ctx.Err() - } -} - -// utlsFactory creates a new uTLS connection. -func utlsFactory(conn net.Conn, config *tls.Config) oohttp.TLSConn { - uConfig := &utls.Config{ - RootCAs: config.RootCAs, - NextProtos: config.NextProtos, - ServerName: config.ServerName, - InsecureSkipVerify: config.InsecureSkipVerify, - DynamicRecordSizingDisabled: config.DynamicRecordSizingDisabled, - } - return &utlsConnAdapter{utls.UClient(conn, uConfig, utls.HelloFirefox_55)} -} diff --git a/example/go.mod b/example/go.mod new file mode 100644 index 00000000..4b1d9bf1 --- /dev/null +++ b/example/go.mod @@ -0,0 +1,23 @@ +module github.com/ooni/oohttp/example + +go 1.18 + +require ( + github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 + github.com/dreadl0ck/ja3 v1.0.4 + github.com/dreadl0ck/tlsx v1.0.1-google-gopacket + github.com/google/go-cmp v0.5.9 + github.com/google/martian/v3 v3.3.2 + github.com/ooni/oohttp v0.2.3 + github.com/refraction-networking/utls v1.2.0 + golang.org/x/net v0.4.0 +) + +require ( + github.com/andybalholm/brotli v1.0.4 // indirect + github.com/google/gopacket v1.1.18 // indirect + github.com/klauspost/compress v1.15.14 // indirect + golang.org/x/crypto v0.4.0 // indirect + golang.org/x/sys v0.3.0 // indirect + golang.org/x/text v0.5.0 // indirect +) diff --git a/example/go.sum b/example/go.sum new file mode 100644 index 00000000..374fec70 --- /dev/null +++ b/example/go.sum @@ -0,0 +1,122 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= +github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dreadl0ck/ja3 v1.0.4 h1:/2wao59Ezn8xBWxn8CVq8eRcPZHbhoTdX6fmg7tQtnw= +github.com/dreadl0ck/ja3 v1.0.4/go.mod h1:jATodgf1qBzTGieskRF2O1DXEwDgzEdqQjVcMMrCNpI= +github.com/dreadl0ck/tlsx v1.0.1-google-gopacket h1:/P3y+CGRiCQbW0nZU2jWkEwKfXLkpEgHNhbbqlnrTTM= +github.com/dreadl0ck/tlsx v1.0.1-google-gopacket/go.mod h1:amAb73WEEgPHWniMfwro6UpN6St3e5ypgq2tXM89IOo= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gopacket v1.1.17/go.mod h1:UdDNZ1OO62aGYVnPhxT1U6aI7ukYtA/kB8vaU0diBUM= +github.com/google/gopacket v1.1.18 h1:lum7VRA9kdlvBi7/v2p7/zcbkduHaCH/SVVyurs7OpY= +github.com/google/gopacket v1.1.18/go.mod h1:UdDNZ1OO62aGYVnPhxT1U6aI7ukYtA/kB8vaU0diBUM= +github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= +github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/klauspost/compress v1.15.14 h1:i7WCKDToww0wA+9qrUZ1xOjp218vfFo3nTU6UHp+gOc= +github.com/klauspost/compress v1.15.14/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= +github.com/ooni/oohttp v0.2.3 h1:D3B1HQ6YebRnBWsvWWUgEtTBpcsuNitsirYtase0rdg= +github.com/ooni/oohttp v0.2.3/go.mod h1:XRJVx6aCswQrE7Fp3j4d8SjHNh1rPEdhpTyPu3zkyBw= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/refraction-networking/utls v1.2.0 h1:U5f8wkij2NVinfLuJdFP3gCMwIHs+EzvhxmYdXgiapo= +github.com/refraction-networking/utls v1.2.0/go.mod h1:NPq+cVqzH7D1BeOkmOcb5O/8iVewAsiVt2x1/eO0hgQ= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.4.0 h1:UVQgzMY87xqpKNgb+kDsll2Igd33HszWHFLmpaRMq/8= +golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.4.0 h1:Q5QPcMlvfxFTAPV0+07Xz/MpK9NTXu2VDUuy0FeMfaU= +golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM= +golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.37.0 h1:uSZWeQJX5j11bIQ4AJoj+McDBo29cY1MCoC1wO3ts+c= +google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/example/internal/ja3x/conn.go b/example/internal/ja3x/conn.go new file mode 100644 index 00000000..ba4839c7 --- /dev/null +++ b/example/internal/ja3x/conn.go @@ -0,0 +1,49 @@ +package ja3x + +import ( + "net" + "sync" +) + +// newConnWrapper creates a new [connWrapper]. +func newConnWrapper(conn net.Conn) *connWrapper { + return &connWrapper{ + Conn: conn, + hello: []byte{}, + mu: sync.Mutex{}, + } +} + +// connWrapper is a wrapper that extracts the raw TLS client hello. +type connWrapper struct { + // Conn is the underlying conn. + net.Conn + + // hello contains the client hello. + hello []byte + + // mu provides mutual exclusion. + mu sync.Mutex +} + +// Hello returns a copy of the bytes inside the client hello. +func (cw *connWrapper) Hello() (out []byte) { + defer cw.mu.Unlock() + cw.mu.Lock() + out = append(out, cw.hello...) + return +} + +// Read implements net.Conn. +func (cw *connWrapper) Read(data []byte) (int, error) { + count, err := cw.Conn.Read(data) + if err != nil { + return 0, err + } + defer cw.mu.Unlock() + cw.mu.Lock() + if len(cw.hello) <= 0 { + cw.hello = append(cw.hello, data[:count]...) // makes a copy + } + return count, nil +} diff --git a/example/internal/ja3x/doc.go b/example/internal/ja3x/doc.go new file mode 100644 index 00000000..146f0cdf --- /dev/null +++ b/example/internal/ja3x/doc.go @@ -0,0 +1,2 @@ +// Package ja3x contains code to produce ja3 signatures. +package ja3x diff --git a/example/internal/ja3x/integration_test.go b/example/internal/ja3x/integration_test.go new file mode 100644 index 00000000..38f045ed --- /dev/null +++ b/example/internal/ja3x/integration_test.go @@ -0,0 +1,160 @@ +package ja3x + +import ( + "crypto/tls" + "errors" + "io" + "net/http" + "net/url" + "strings" + "testing" + + "golang.org/x/net/http2" +) + +func TestServerWithSpecificALPN(t *testing.T) { + srvr := NewServer("antani") + defer srvr.Close() + tlsConfig := srvr.ClientConfig() + tlsConfig.NextProtos = []string{"antani"} + conn, err := tls.Dial("tcp", srvr.Endpoint(), tlsConfig) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + if p := conn.ConnectionState().NegotiatedProtocol; p != "antani" { + t.Fatal("unexpected negotiated protocol", p) + } +} + +func TestServerWithHTTP1ClientAndTLS(t *testing.T) { + srvr := NewServer() + defer srvr.Close() + tlsConfig := srvr.ClientConfig() + tlsConfig.NextProtos = []string{"http/1.1"} + txp := &http.Transport{ + TLSClientConfig: tlsConfig, + ForceAttemptHTTP2: false, // explicitly say no to HTTP2 + } + req, err := http.NewRequest("GET", srvr.URL(), nil) + if err != nil { + t.Fatal(err) + } + resp, err := txp.RoundTrip(req) + if err != nil { + t.Fatal(err) + } + if resp.StatusCode != 200 { + t.Fatal("unexpected status code", resp.StatusCode) + } + defer resp.Body.Close() + if value := resp.Header.Get("X-ALPN"); value != "http/1.1" { + t.Fatal("unexpected X-ALPN header value", value) + } + respbody, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + if len(respbody) != 32 { + t.Fatal("not a valid JA3 response length") + } +} + +func TestServerWithHTTP2ClientAndTLS(t *testing.T) { + srvr := NewServer() + defer srvr.Close() + tlsConfig := srvr.ClientConfig() + tlsConfig.NextProtos = []string{"h2"} + txp := &http2.Transport{ + TLSClientConfig: tlsConfig, + } + req, err := http.NewRequest("GET", srvr.URL(), nil) + if err != nil { + t.Fatal(err) + } + resp, err := txp.RoundTrip(req) + if err != nil { + t.Fatal(err) + } + if resp.StatusCode != 200 { + t.Fatal("unexpected status code", resp.StatusCode) + } + defer resp.Body.Close() + if value := resp.Header.Get("X-ALPN"); value != "h2" { + t.Fatal("unexpected X-ALPN header value", value) + } + respbody, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + if len(respbody) != 32 { + t.Fatal("not a valid JA3 response length") + } +} + +func TestServerWithHTTP1ClientAndTCP(t *testing.T) { + srvr := NewServer() + defer srvr.Close() + parsed, err := url.Parse(srvr.URL()) + if err != nil { + t.Fatal(err) + } + parsed.Scheme = "http" // forces using the cleartext HTTP protocol + txp := &http.Transport{} + req, err := http.NewRequest("GET", parsed.String(), nil) + if err != nil { + t.Fatal(err) + } + resp, err := txp.RoundTrip(req) + if !errors.Is(err, io.EOF) { + t.Fatal("unexpected error", err) + } + if resp != nil { + t.Fatal("expected nil response") + } +} + +func TestServerWithEmptyALPN(t *testing.T) { + srvr := NewServer() + defer srvr.Close() + tlsConfig := srvr.ClientConfig() + tlsConfig.NextProtos = []string{} // explicitly empty so it will fail + txp := &http.Transport{ + TLSClientConfig: tlsConfig, + ForceAttemptHTTP2: false, // explicitly say no to HTTP2 + } + req, err := http.NewRequest("GET", srvr.URL(), nil) + if err != nil { + t.Fatal(err) + } + resp, err := txp.RoundTrip(req) + // the former error happens on macOS and the latter in github actions + if !errors.Is(err, io.EOF) && !(err != nil && err.Error() == "http: server closed idle connection") { + t.Fatal("unexpected error", err) + } + if resp != nil { + t.Fatal("expected nil response") + } +} + +func TestServerWithInvalidALPN(t *testing.T) { + srvr := NewServer() + defer srvr.Close() + tlsConfig := srvr.ClientConfig() + tlsConfig.NextProtos = []string{"antani"} // explicitly invalid so it will fail + txp := &http.Transport{ + TLSClientConfig: tlsConfig, + ForceAttemptHTTP2: false, // explicitly say no to HTTP2 + } + req, err := http.NewRequest("GET", srvr.URL(), nil) + if err != nil { + t.Fatal(err) + } + resp, err := txp.RoundTrip(req) + if err == nil || !strings.HasSuffix(err.Error(), "tls: no application protocol") { + t.Fatal("unexpected error", err) + } + if resp != nil { + t.Fatal("expected nil response") + } +} diff --git a/example/internal/ja3x/server.go b/example/internal/ja3x/server.go new file mode 100644 index 00000000..f035f212 --- /dev/null +++ b/example/internal/ja3x/server.go @@ -0,0 +1,242 @@ +package ja3x + +import ( + "bufio" + "context" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "errors" + "io" + "log" + "net" + "net/http" + "net/url" + "strings" + "sync" + "time" + + "github.com/google/martian/v3/mitm" + "github.com/ooni/oohttp/example/internal/runtimex" + "golang.org/x/net/http2" +) + +// serverTLSConfigMITM creates a MITM TLS configuration. +func serverTLSConfigMITM() (*x509.Certificate, *rsa.PrivateKey, *mitm.Config) { + cert, privkey, err := mitm.NewAuthority("ja3x", "OONI", time.Hour) + runtimex.PanicOnError(err, "mitm.NewAuthority failed") + config, err := mitm.NewConfig(cert, privkey) + runtimex.PanicOnError(err, "mitm.NewConfig failed") + return cert, privkey, config +} + +// serverConfigALPN configures the ALPN or a default ALPN. +func serverConfigALPN(alpn []string) []string { + if len(alpn) > 0 { + return alpn + } + return []string{"h2", "http/1.1"} +} + +// NewServer creates a new [Server] instance. +func NewServer(alpn ...string) *Server { + listener, err := net.Listen("tcp", "127.0.0.1:0") + runtimex.PanicOnError(err, "net.Listen failed") + cert, privkey, config := serverTLSConfigMITM() + ctx, cancel := context.WithCancel(context.Background()) + endpoint := listener.Addr().String() + server := &Server{ + alpn: serverConfigALPN(alpn), + cancel: cancel, + cert: cert, + config: config, + done: make(chan bool), + endpoint: endpoint, + listener: listener, + privkey: privkey, + once: sync.Once{}, + } + go server.mainloop(ctx) + return server +} + +// Server is an HTTP server that returns JA3 information. This server +// only supports HTTP/1.1 and HTTP/2. +// +// This server will use its own internal CA. You should use [Server.ClientConfig] +// to obtain a suitable [tls.Config] or [Server.CertPool] to obtain a suitable +// [x509.CertPool]. If you use [Server.CertPool] keep in mind that you also need +// to specify a ServerName in the [tls.Config] for an handshake to complete +// successfully. By default [Server.Config] uses "example.com" as the server +// name but you can override that. +// +// Please, use the [NewServer] factory to create a new instance: the +// zero-value [Server] is invalid and will panic if used. +type Server struct { + alpn []string + cancel context.CancelFunc + cert *x509.Certificate + config *mitm.Config + done chan bool + endpoint string + listener net.Listener + once sync.Once + privkey *rsa.PrivateKey +} + +// Endpoint returns the server's TCP endpoint. +func (p *Server) Endpoint() string { + return p.endpoint +} + +// CertPool returns the internal CA as a cert pool. +func (p *Server) CertPool() *x509.CertPool { + pool := x509.NewCertPool() + pool.AddCert(p.cert) + return pool +} + +// ClientConfig returns a [tls.Config] where the ServerName +// is set to "example.com" and the RootCAs is set to the value returned by +// the [Server.CertPool] method. +func (p *Server) ClientConfig() *tls.Config { + return &tls.Config{ + ServerName: "example.com", + RootCAs: p.CertPool(), + } +} + +// Close terminates a running [Server] instance. +func (p *Server) Close() (err error) { + p.once.Do(func() { + p.cancel() + err = p.listener.Close() + <-p.done + }) + return err +} + +// URL returns the server's URL, which is guaranteed to contain +// an explicit port even in the very unlikely case in which +// the randomly selected port is 443/tcp. +func (p *Server) URL() string { + u := &url.URL{ + Scheme: "https", + Host: p.endpoint, + Path: "/", + } + return u.String() +} + +// mainloop is the main loop of a [Server]. +func (p *Server) mainloop(ctx context.Context) { + defer close(p.done) + for { + canContinue, _ := p.serveConn(ctx) + if !canContinue { + break + } + } +} + +// serveConn accepts and services a client connection +func (p *Server) serveConn(ctx context.Context) (bool, error) { + conn, err := p.listener.Accept() + if err != nil { + return !errors.Is(err, net.ErrClosed), err + } + done := make(chan bool) + go func() { + // make sure we close the connection both in case of + // normal shutdown and when the ctx is done + defer conn.Close() + select { + case <-done: + return + case <-ctx.Done(): + return + } + }() + err = p.setupTLSAndHandle(conn) + close(done) + return true, err // we can continue running +} + +// errInvalidALPN indicates we were passed an invalid ALPN. +var errInvalidALPN = errors.New("ja3x: invalid ALPN") + +// setupTLSAndHandle setups a TLS connection and handles the request. +func (p *Server) setupTLSAndHandle(tcpConn net.Conn) error { + cw := newConnWrapper(tcpConn) + tlsConn := tls.Server(cw, &tls.Config{ + GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) { + sni, alpn := info.ServerName, info.SupportedProtos + log.Printf("ja3x: got connection request for %s %s", sni, alpn) + return p.config.TLSForHost(sni).GetCertificate(info) + }, + NextProtos: p.alpn, + }) + defer tlsConn.Close() + if err := tlsConn.Handshake(); err != nil { + log.Printf("ja3x: unexpected TLS handshake failure: %v", err) + return err + } + hello := cw.Hello() + return serverHandleCommon(tlsConn, hello) +} + +func serverHandleCommon(tlsConn *tls.Conn, hello []byte) error { + digest, err := clientHelloDigest(hello) + if err != nil { + log.Printf("ja3x: cannot parse client hello: %v", err) + return err + } + switch alpn := tlsConn.ConnectionState().NegotiatedProtocol; alpn { + case "http/1.1": + return serverHandleV1(tlsConn, digest) + case "h2": + return serverHandleV2(tlsConn, digest) + default: + log.Printf("ja3x: got invalid ALPN: '%s'", alpn) + return errInvalidALPN + } +} + +// serverHandleV1 handles an HTTP/1.1 client request. +func serverHandleV1(conn io.ReadWriter, digest string) error { + rbio := bufio.NewReader(conn) + req, err := http.ReadRequest(rbio) + if err != nil { + log.Printf("ja3x: cannot read request: %v", err) + return err + } + // TODO(bassosimone): read the request body when it becomes useful to do so + resp := &http.Response{ + StatusCode: 200, + ProtoMajor: 1, + ProtoMinor: 1, + Header: map[string][]string{ + "X-ALPN": {"http/1.1"}, // allows to easily verify we're using http/1.1 + }, + Body: io.NopCloser(strings.NewReader(digest)), + Request: req, + } + if err := resp.Write(conn); err != nil { + log.Printf("ja3x: cannot write response: %s", err) + return err + } + return nil +} + +// serverHandleV2 handles an HTTP/2 client request. +func serverHandleV2(conn net.Conn, digest string) error { + options := &http2.ServeConnOpts{ + Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Add("X-ALPN", "h2") // allows to easily verify we're using HTTP/2 + w.Write([]byte(digest)) + }), + } + srv := &http2.Server{} + srv.ServeConn(conn, options) + return nil +} diff --git a/example/internal/ja3x/server_test.go b/example/internal/ja3x/server_test.go new file mode 100644 index 00000000..5bc8054b --- /dev/null +++ b/example/internal/ja3x/server_test.go @@ -0,0 +1,90 @@ +package ja3x + +import ( + "errors" + "io" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" +) + +func Test_serverHandleCommon(t *testing.T) { + t.Run("with unparseable ClientHello", func(t *testing.T) { + clientHello := make([]byte, 155) + err := serverHandleCommon(nil, clientHello) + if err == nil || err.Error() != "handshake is of wrong type, or not a handshake message" { + t.Fatal("unexpected error", err) + } + }) +} + +// serverHandleV1MockedConn is a mocked conn for testing [serverHandleV1] +type serverHandleV1MockedConn struct { + // read is called when reading + read func(data []byte) (int, error) + + // write is called when writing + write func(data []byte) (int, error) +} + +var _ io.ReadWriter = &serverHandleV1MockedConn{} + +// Read implements io.ReadWriter +func (c *serverHandleV1MockedConn) Read(p []byte) (n int, err error) { + return c.read(p) +} + +// Write implements io.ReadWriter +func (c *serverHandleV1MockedConn) Write(p []byte) (n int, err error) { + return c.write(p) +} + +func Test_serverHandleV1(t *testing.T) { + t.Run("cannot read request", func(t *testing.T) { + expected := errors.New("mocked error") + conn := &serverHandleV1MockedConn{ + read: func(data []byte) (int, error) { + return 0, expected + }, + } + err := serverHandleV1(conn, "deadbeef") + if !errors.Is(err, expected) { + t.Fatal("unexpected error", err) + } + }) + + t.Run("cannot write response", func(t *testing.T) { + expected := errors.New("mocked error") + reader := strings.NewReader("GET / HTTP/1.1\r\n\r\n") + conn := &serverHandleV1MockedConn{ + read: reader.Read, + write: func(data []byte) (int, error) { + return 0, expected + }, + } + err := serverHandleV1(conn, "deadbeef") + if !errors.Is(err, expected) { + t.Fatal("unexpected error", err) + } + }) +} + +func TestNewServer(t *testing.T) { + t.Run("with no ALPN arguments", func(t *testing.T) { + srvr := NewServer() + defer srvr.Close() + if diff := cmp.Diff(srvr.alpn, []string{"h2", "http/1.1"}); diff != "" { + t.Fatal(diff) + } + }) + + t.Run("with specific ALPN", func(t *testing.T) { + expected := []string{"h2"} + srvr := NewServer("h2") + defer srvr.Close() + if diff := cmp.Diff(srvr.alpn, expected); diff != "" { + t.Fatal(diff) + } + }) +} diff --git a/example/internal/ja3x/utils.go b/example/internal/ja3x/utils.go new file mode 100644 index 00000000..0b225c21 --- /dev/null +++ b/example/internal/ja3x/utils.go @@ -0,0 +1,16 @@ +package ja3x + +import ( + "github.com/dreadl0ck/ja3" + "github.com/dreadl0ck/tlsx" +) + +// clientHelloDigest generates a ja3 digest from the bytes +// of a given TLS Client Hello message. +func clientHelloDigest(raw []byte) (string, error) { + chb := &tlsx.ClientHelloBasic{} + if err := chb.Unmarshal(raw); err != nil { + return "", err + } + return ja3.BareToDigestHex(ja3.Bare(chb)), nil +} diff --git a/example/internal/ja3x/utils_test.go b/example/internal/ja3x/utils_test.go new file mode 100644 index 00000000..9310a07b --- /dev/null +++ b/example/internal/ja3x/utils_test.go @@ -0,0 +1,16 @@ +package ja3x + +import "testing" + +func Test_clientHelloDigest(t *testing.T) { + t.Run("unmarshal fails for invalid ClientHello", func(t *testing.T) { + data := make([]byte, 128) + out, err := clientHelloDigest(data) + if err == nil || err.Error() != "handshake is of wrong type, or not a handshake message" { + t.Fatal("unexpected error", err) + } + if out != "" { + t.Fatal("expected empty string") + } + }) +} diff --git a/example/internal/runtimex/runtimex.go b/example/internal/runtimex/runtimex.go new file mode 100644 index 00000000..82056948 --- /dev/null +++ b/example/internal/runtimex/runtimex.go @@ -0,0 +1,36 @@ +// Package runtimex contains runtime extensions. This package is inspired to +// https://pkg.go.dev/github.com/m-lab/go/rtx, except that it's simpler. +package runtimex + +import ( + "errors" + "fmt" +) + +// PanicOnError calls panic() if err is not nil. The type passed +// to panic is an error type wrapping the original error. +func PanicOnError(err error, message string) { + if err != nil { + panic(fmt.Errorf("%s: %w", message, err)) + } +} + +// Assert calls panic if assertion is false. The type passed to +// panic is an error constructed using errors.New(message). +func Assert(assertion bool, message string) { + if !assertion { + panic(errors.New(message)) + } +} + +// PanicIfTrue calls panic if assertion is true. The type passed to +// panic is an error constructed using errors.New(message). +func PanicIfTrue(assertion bool, message string) { + Assert(!assertion, message) +} + +// PanicIfNil calls panic if the given interface is nil. The type passed to +// panic is an error constructed using errors.New(message). +func PanicIfNil(v any, message string) { + PanicIfTrue(v == nil, message) +} diff --git a/example/internal/runtimex/runtimex_test.go b/example/internal/runtimex/runtimex_test.go new file mode 100644 index 00000000..a656a661 --- /dev/null +++ b/example/internal/runtimex/runtimex_test.go @@ -0,0 +1,93 @@ +package runtimex + +import ( + "errors" + "testing" +) + +func TestPanicOnError(t *testing.T) { + badfunc := func(in error) (out error) { + defer func() { + out = recover().(error) + }() + PanicOnError(in, "we expect this assertion to fail") + return + } + + t.Run("error is nil", func(t *testing.T) { + PanicOnError(nil, "this assertion should not fail") + }) + + t.Run("error is not nil", func(t *testing.T) { + expected := errors.New("mocked error") + if !errors.Is(badfunc(expected), expected) { + t.Fatal("not the error we expected") + } + }) +} + +func TestAssert(t *testing.T) { + badfunc := func(in bool, message string) (out error) { + defer func() { + out = recover().(error) + }() + Assert(in, message) + return + } + + t.Run("assertion is true", func(t *testing.T) { + Assert(true, "this assertion should not fail") + }) + + t.Run("assertion is false", func(t *testing.T) { + message := "mocked error" + err := badfunc(false, message) + if err == nil || err.Error() != message { + t.Fatal("not the error we expected", err) + } + }) +} + +func TestPanicIfTrue(t *testing.T) { + badfunc := func(in bool, message string) (out error) { + defer func() { + out = recover().(error) + }() + PanicIfTrue(in, message) + return + } + + t.Run("assertion is false", func(t *testing.T) { + PanicIfTrue(false, "this assertion should not fail") + }) + + t.Run("assertion is true", func(t *testing.T) { + message := "mocked error" + err := badfunc(true, message) + if err == nil || err.Error() != message { + t.Fatal("not the error we expected", err) + } + }) +} + +func TestPanicIfNil(t *testing.T) { + badfunc := func(in interface{}, message string) (out error) { + defer func() { + out = recover().(error) + }() + PanicIfNil(in, message) + return + } + + t.Run("value is not nil", func(t *testing.T) { + PanicIfNil(false, "this assertion should not fail") + }) + + t.Run("value is nil", func(t *testing.T) { + message := "mocked error" + err := badfunc(nil, message) + if err == nil || err.Error() != message { + t.Fatal("not the error we expected", err) + } + }) +} diff --git a/example/internal/utlsx/utlsx.go b/example/internal/utlsx/utlsx.go new file mode 100644 index 00000000..0b091097 --- /dev/null +++ b/example/internal/utlsx/utlsx.go @@ -0,0 +1,172 @@ +// Package utlsx contains UTLS extensions. +package utlsx + +import ( + "context" + "crypto/tls" + "log" + "net" + "net/url" + "time" + + oohttp "github.com/ooni/oohttp" + "github.com/ooni/oohttp/example/internal/runtimex" + utls "github.com/refraction-networking/utls" +) + +// connAdapter adapts utls.UConn to the oohttp.TLSConn interface. +type connAdapter struct { + *utls.UConn +} + +var _ oohttp.TLSConn = &connAdapter{} + +// ConnectionState implements oohttp.TLSConn. +func (c *connAdapter) ConnectionState() tls.ConnectionState { + cs := c.UConn.ConnectionState() + return tls.ConnectionState{ + Version: cs.Version, + HandshakeComplete: cs.HandshakeComplete, + DidResume: cs.DidResume, + CipherSuite: cs.CipherSuite, + NegotiatedProtocol: cs.NegotiatedProtocol, + NegotiatedProtocolIsMutual: cs.NegotiatedProtocolIsMutual, + ServerName: cs.ServerName, + PeerCertificates: cs.PeerCertificates, + VerifiedChains: cs.VerifiedChains, + SignedCertificateTimestamps: cs.SignedCertificateTimestamps, + OCSPResponse: cs.OCSPResponse, + TLSUnique: cs.TLSUnique, + } +} + +// HandshakeContext implements oohttp.TLSConn. +func (c *connAdapter) HandshakeContext(ctx context.Context) error { + errch := make(chan error, 1) + go func() { + errch <- c.UConn.Handshake() + }() + select { + case err := <-errch: + return err + case <-ctx.Done(): + return ctx.Err() + } +} + +// DefaultClientHelloID is the default [utls.ClientHelloID]. +var DefaultClientHelloID = &utls.HelloFirefox_55 + +// ConnFactory is a factory for creating UTLS connections. +type ConnFactory interface { + // NewUTLSConn creates a new UTLS connection. The conn and config arguments MUST + // NOT be nil. We will honour the following fields of config: + // + // - RootCAs + // + // - NextProtos + // + // - ServerName + // + // - InsecureSkipVerify + // + // - DynamicRecordSizingDisabled + // + // However, some of these fields values MAY be ignored depending on the parrot we use. + NewUTLSConn(conn net.Conn, config *tls.Config) oohttp.TLSConn +} + +// FactoryWithParrot implements ConnFactory. +type FactoryWithParrot struct { + // Parrot is the OPTIONAL parrot. + Parrot *utls.ClientHelloID +} + +// NewUTLSConn implements ConnFactory. +func (f *FactoryWithParrot) NewUTLSConn(conn net.Conn, config *tls.Config) oohttp.TLSConn { + parrot := f.Parrot + if parrot == nil { + parrot = DefaultClientHelloID + } + uConfig := &utls.Config{ + RootCAs: config.RootCAs, + NextProtos: config.NextProtos, + ServerName: config.ServerName, + InsecureSkipVerify: config.InsecureSkipVerify, + DynamicRecordSizingDisabled: config.DynamicRecordSizingDisabled, + } + return &connAdapter{utls.UClient(conn, uConfig, *parrot)} +} + +// DefaultNetDialer is the default [net.Dialer]. +var DefaultNetDialer = &net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, +} + +// TLSDialer is a dialer that uses UTLS +type TLSDialer struct { + // Config is the OPTIONAL Config. In case it's not nil, we will + // pass this config to [Factory] rather than a default one. + Config *tls.Config + + // Parrot is the OPTIONAL parrot to use. + Parrot *utls.ClientHelloID + + // beforeHandshakeFunc is a function called before the + // TLS handshake, which is only useful for testing. + beforeHandshakeFunc func() +} + +// DialTLSContext dials a TLS connection using UTLS. +func (d *TLSDialer) DialTLSContext(ctx context.Context, network string, addr string) (net.Conn, error) { + conn, err := DefaultNetDialer.DialContext(ctx, network, addr) + if err != nil { + return nil, err + } + sni, _, err := net.SplitHostPort(addr) + runtimex.PanicOnError(err, "net.SplitHostPort failed") // cannot fail after successful dial + config := &tls.Config{ServerName: sni} + if d.Config != nil { + config = d.Config // as documented + } + if d.beforeHandshakeFunc != nil { + d.beforeHandshakeFunc() // useful for testing + } + uadapter := (&FactoryWithParrot{d.Parrot}).NewUTLSConn(conn, config) + if err := uadapter.HandshakeContext(ctx); err != nil { + conn.Close() + return nil, err + } + proto := uadapter.ConnectionState().NegotiatedProtocol + log.Printf("negotiated protocol: %s", proto) + return uadapter, nil +} + +// TLSFactoryFunc is the type of the [ConnFactory.NewUTLSConn] func. +type TLSFactoryFunc func(conn net.Conn, config *tls.Config) oohttp.TLSConn + +// ProxyFunc is the the type of the [http.ProxyFromEnvironment] func. +type ProxyFunc func(*oohttp.Request) (*url.URL, error) + +// TLSDialFunc is the type of the [TLSDialer.DialTLSContext] func. +type TLSDialFunc func(ctx context.Context, network string, addr string) (net.Conn, error) + +// NewOOHTTPTransport creates a new OOHTTP transport using the given funcs. Each +// function MAY be nil. In such a case, we'll use a reasonable default. +func NewOOHTTPTransport( + proxy ProxyFunc, tlsFactory TLSFactoryFunc, tlsDialer TLSDialFunc) *oohttp.StdlibTransport { + return &oohttp.StdlibTransport{ + Transport: &oohttp.Transport{ + Proxy: proxy, + DialContext: DefaultNetDialer.DialContext, + DialTLSContext: tlsDialer, + ForceAttemptHTTP2: true, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + TLSClientFactory: tlsFactory, + }, + } +} diff --git a/example/internal/utlsx/utlsx_test.go b/example/internal/utlsx/utlsx_test.go new file mode 100644 index 00000000..a441e78a --- /dev/null +++ b/example/internal/utlsx/utlsx_test.go @@ -0,0 +1,284 @@ +package utlsx + +import ( + "context" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/url" + "strings" + "testing" + + oohttp "github.com/ooni/oohttp" + "github.com/ooni/oohttp/example/internal/ja3x" + utls "github.com/refraction-networking/utls" +) + +func TestTLSDialerWorkingAsIntended(t *testing.T) { + srvr := ja3x.NewServer("h2", "http/1.1") + defer srvr.Close() + dialer := &TLSDialer{ + Config: srvr.ClientConfig(), + Parrot: &utls.HelloFirefox_105, + beforeHandshakeFunc: nil, + } + ctx := context.Background() + conn, err := dialer.DialTLSContext(ctx, "tcp", srvr.Endpoint()) + if err != nil { + t.Fatal(err) + } + conn.Close() +} + +func TestTLSDialerWithHandshakeError(t *testing.T) { + srvr := ja3x.NewServer("h2", "http/1.1") + defer srvr.Close() + ctx, cancel := context.WithCancel(context.Background()) + dialer := &TLSDialer{ + Config: srvr.ClientConfig(), + Parrot: &utls.HelloFirefox_105, + beforeHandshakeFunc: func() { + cancel() // make sure the handshake fails + }, + } + conn, err := dialer.DialTLSContext(ctx, "tcp", srvr.Endpoint()) + if !errors.Is(err, context.Canceled) { + t.Fatal("unexpected err", err) + } + if conn != nil { + t.Fatal("expected nil conn") + } +} + +func TestTLSDialerWithDialError(t *testing.T) { + srvr := ja3x.NewServer("h2", "http/1.1") + defer srvr.Close() + ctx, cancel := context.WithCancel(context.Background()) + cancel() // cause immediate failure + dialer := &TLSDialer{ + Config: srvr.ClientConfig(), + Parrot: &utls.HelloFirefox_105, + beforeHandshakeFunc: nil, + } + conn, err := dialer.DialTLSContext(ctx, "tcp", srvr.Endpoint()) + if err == nil || !strings.HasSuffix(err.Error(), "operation was canceled") { + t.Fatal("unexpected err", err) + } + if conn != nil { + t.Fatal("expected nil conn") + } +} + +func TestNewOOHTTPTransportWithCustomProxy(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer listener.Close() + acceptch := make(chan error) + go func() { + conn, err := listener.Accept() + if err != nil { + acceptch <- err + return + } + conn.Close() + acceptch <- nil + }() + proxyURL := &url.URL{ + Scheme: "http", + Host: listener.Addr().String(), + } + txp := NewOOHTTPTransport( + func(r *oohttp.Request) (*url.URL, error) { + return proxyURL, nil + }, + nil, // use default + nil, // use default + ) + req, err := http.NewRequest("GET", "https://google.com/", nil) + if err != nil { + t.Fatal(err) + } + resp, err := txp.RoundTrip(req) + if !errors.Is(err, io.ErrUnexpectedEOF) { + t.Fatal("unexpected err", err) + } + if resp != nil { + t.Fatal("expected nil resp") + } + if err := <-acceptch; err != nil { + t.Fatal("accept failed") + } +} + +const ( + // defaultJA3 is the default JA3 + defaultJA3 = "0ffee3ba8e615ad22535e7f771690a28" + + // customJA3 is the custom JA3 + customJA3 = "579ccef312d18482fc42e2b822ca2430" +) + +func TestJA3SignaturesDifference(t *testing.T) { + if defaultJA3 == customJA3 { + t.Fatal("the two JA3 signatures must be different") + } +} + +func TestNewOOHTTPTransportWithTLSFactoryFunc(t *testing.T) { + // doit is the common function implementing all of this test's test cases. + doit := func(parrot *utls.ClientHelloID, expectedJA3 string, alpn ...string) error { + srvr := ja3x.NewServer(alpn...) + defer srvr.Close() + factory := &FactoryWithParrot{ + Parrot: parrot, // may or may not be nil + } + txp := NewOOHTTPTransport( + nil, // default proxy function + factory.NewUTLSConn, + nil, // default TLS dialing + ) + txp.Transport.TLSClientConfig = srvr.ClientConfig() + txp.Transport.TLSClientConfig.NextProtos = alpn + req, err := http.NewRequest("GET", srvr.URL(), nil) + if err != nil { + t.Fatal(err) + } + resp, err := txp.RoundTrip(req) + if err != nil { + return err + } + defer resp.Body.Close() + data, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + switch { + case len(alpn) <= 0: + return errors.New("expected at least one ALPN") + case alpn[0] == "h2" && resp.Header.Get("X-ALPN") != "h2": + return fmt.Errorf("expected ALPN h2 but got %s", resp.Header.Get("X-ALPN")) + case alpn[0] == "http/1.1" && resp.Header.Get("X-ALPN") != "http/1.1": + return fmt.Errorf("expected ALPN http/1.1 but got %s", resp.Header.Get("X-ALPN")) + default: + // all seems good in ALPN terms + } + if got := string(data); got != expectedJA3 { + return fmt.Errorf("expected JA3 %s but got %s", expectedJA3, got) + } + return nil + } + + t.Run("with http/1.1", func(t *testing.T) { + t.Run("with default parrot", func(t *testing.T) { + err := doit(nil, defaultJA3, "http/1.1") + if err != nil { + t.Fatal(err) + } + }) + + t.Run("with custom parrot", func(t *testing.T) { + err := doit(&utls.HelloFirefox_105, customJA3, "http/1.1") + if err != nil { + t.Fatal(err) + } + }) + }) + + t.Run("with h2 and http/1.1", func(t *testing.T) { + t.Run("with default parrot", func(t *testing.T) { + err := doit(nil, defaultJA3, "h2", "http/1.1") + if err != nil { + t.Fatal(err) + } + }) + + t.Run("with custom parrot", func(t *testing.T) { + err := doit(&utls.HelloFirefox_105, customJA3, "h2", "http/1.1") + if err != nil { + t.Fatal(err) + } + }) + }) +} + +func TestNewOOHTTPTransportWithTLSDialerFunc(t *testing.T) { + // doit is the common function implementing all of this test's test cases. + doit := func(parrot *utls.ClientHelloID, expectedJA3 string, alpn ...string) error { + srvr := ja3x.NewServer(alpn...) + defer srvr.Close() + dialer := &TLSDialer{ + Config: srvr.ClientConfig(), + Parrot: parrot, + beforeHandshakeFunc: nil, + } + dialer.Config.NextProtos = alpn + txp := NewOOHTTPTransport( + nil, // default proxy function + nil, // no factory + dialer.DialTLSContext, // but use a custom dialer + ) + req, err := http.NewRequest("GET", srvr.URL(), nil) + if err != nil { + t.Fatal(err) + } + resp, err := txp.RoundTrip(req) + if err != nil { + return err + } + defer resp.Body.Close() + data, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + switch { + case len(alpn) <= 0: + return errors.New("expected at least one ALPN") + case alpn[0] == "h2" && resp.Header.Get("X-ALPN") != "h2": + return fmt.Errorf("expected ALPN h2 but got %s", resp.Header.Get("X-ALPN")) + case alpn[0] == "http/1.1" && resp.Header.Get("X-ALPN") != "http/1.1": + return fmt.Errorf("expected ALPN http/1.1 but got %s", resp.Header.Get("X-ALPN")) + default: + // all seems good in ALPN terms + } + if got := string(data); got != expectedJA3 { + return fmt.Errorf("expected JA3 %s but got %s", expectedJA3, got) + } + return nil + } + + t.Run("with http/1.1", func(t *testing.T) { + t.Run("with default parrot", func(t *testing.T) { + err := doit(nil, defaultJA3, "http/1.1") + if err != nil { + t.Fatal(err) + } + }) + + t.Run("with custom parrot", func(t *testing.T) { + err := doit(&utls.HelloFirefox_105, customJA3, "http/1.1") + if err != nil { + t.Fatal(err) + } + }) + }) + + t.Run("with h2 and http/1.1", func(t *testing.T) { + t.Run("with default parrot", func(t *testing.T) { + err := doit(nil, defaultJA3, "h2", "http/1.1") + if err != nil { + t.Fatal(err) + } + }) + + t.Run("with custom parrot", func(t *testing.T) { + err := doit(&utls.HelloFirefox_105, customJA3, "h2", "http/1.1") + if err != nil { + t.Fatal(err) + } + }) + }) +}