feat: implement better integration tests (#34)

There's a need to write better integration tests for this repository.

In particular:

1. we need to have confidence that the JA3 signature produced when using a uTLS client is different than the default one (i.e., can we be confident that it's possible to replace TLS?)

2. we need to have robust tests that do not depend onto external services but only use localhost, so they don't break often.

Part of ooni/probe#2273

.github/workflows/go.yml

@@ -22,4 +22,10 @@ jobs:
       run: go build -v ./...
 
     - name: Test
-      run: go test -v -race ./...
+      run: go test -cover -race ./...
+
+    - name: Install dependencies
+      run: sudo apt-get install libpcap-dev
+
+    - name: Test Examples
+      run: cd example && go test -cover -race ./...

example/example-proxy/main.go

@@ -9,10 +9,10 @@ import (
 	"net"
 	"net/http"
 	"net/url"
-	"time"
 
 	"github.com/armon/go-socks5"
 	oohttp "github.com/ooni/oohttp"
+	"github.com/ooni/oohttp/example/internal/utlsx"
 )
 
 // startProxyServer starts a SOCKS5 proxy server at the given endpoint.
@@ -36,19 +36,13 @@ func startProxyServer(endpoint string, ch chan<- interface{}) {
 // when communicating with a remote TLS endpoint through the given proxy.
 func useProxy(URL, proxy string,
 	tlsClientFactory func(conn net.Conn, config *tls.Config) oohttp.TLSConn) {
-	w := &oohttp.StdlibTransport{
-		Transport: &oohttp.Transport{
-			Proxy: func(*oohttp.Request) (*url.URL, error) {
-				return &url.URL{Scheme: "socks5", Host: proxy}, nil
-			},
-			ForceAttemptHTTP2:     true,
-			MaxIdleConns:          100,
-			IdleConnTimeout:       90 * time.Second,
-			TLSHandshakeTimeout:   10 * time.Second,
-			ExpectContinueTimeout: 1 * time.Second,
-			TLSClientFactory:      tlsClientFactory,
+	w := utlsx.NewOOHTTPTransport(
+		func(*oohttp.Request) (*url.URL, error) {
+			return &url.URL{Scheme: "socks5", Host: proxy}, nil
 		},
-	}
+		tlsClientFactory,
+		nil, // we're using tlsClientFactory to wrap dialed connections
+	)
 	clnt := &http.Client{Transport: w}
 	resp, err := clnt.Get(URL)
 	if err != nil {
@@ -73,7 +67,7 @@ func main() {
 	flag.Parse()
 	var ffun func(conn net.Conn, config *tls.Config) oohttp.TLSConn
 	if *utls {
-		ffun = utlsFactory
+		ffun = (&utlsx.FactoryWithParrot{}).NewUTLSConn
 	}
 	ch := make(chan interface{})
 	go startProxyServer(*proxy, ch)

example/example-utls-with-dial/.gitignore

@@ -1 +1 @@
-/example-utls
+/example-utls-with-dial

example/example-utls-with-dial/main.go

@@ -1,13 +1,30 @@
 package main
 
 import (
+	"context"
 	"flag"
 	"fmt"
 	"io"
 	"log"
+	"net"
 	"net/http"
+
+	oohttp "github.com/ooni/oohttp"
+	"github.com/ooni/oohttp/example/internal/utlsx"
 )
 
+// newTransport returns a new http.Transport using the provided tls dialer.
+func newTransport(f func(ctx context.Context, network, address string) (net.Conn, error)) http.RoundTripper {
+	return utlsx.NewOOHTTPTransport(
+		oohttp.ProxyFromEnvironment,
+		nil, // we're using f to directly create TLS connections
+		f,
+	)
+}
+
+// defaultTransport is the default http.RoundTripper.
+var defaultTransport = newTransport((&utlsx.TLSDialer{}).DialTLSContext)
+
 // newClient creates a new http.Client using the given transport.
 func newClient(txp http.RoundTripper) *http.Client {
 	return &http.Client{Transport: txp}

example/example-utls-with-dial/main_test.go

@@ -2,24 +2,33 @@ package main
 
 import (
 	"context"
+	"crypto/tls"
 	"io"
 	"log"
 	"net"
+	"net/http"
+	"net/http/httptest"
 	"sync"
 	"testing"
 
 	oohttp "github.com/ooni/oohttp"
+	"github.com/ooni/oohttp/example/internal/ja3x"
+	"github.com/ooni/oohttp/example/internal/utlsx"
 )
 
 // tlsDialerRecorder performs TLS dials and records the ALPN.
 type tlsDialerRecorder struct {
-	alpn map[string]int
-	mu   sync.Mutex
+	alpn   map[string]int
+	config *tls.Config
+	mu     sync.Mutex
 }
 
 // do is like dialTLSContext but also records the ALPN.
 func (d *tlsDialerRecorder) do(ctx context.Context, network string, addr string) (net.Conn, error) {
-	conn, err := dialTLSContext(ctx, network, addr)
+	child := &utlsx.TLSDialer{
+		Config: d.config,
+	}
+	conn, err := child.DialTLSContext(ctx, network, addr)
 	if err != nil {
 		return nil, err
 	}
@@ -35,9 +44,15 @@ func (d *tlsDialerRecorder) do(ctx context.Context, network string, addr string)
 }
 
 func TestWorkAsIntendedWithH2(t *testing.T) {
-	d := &tlsDialerRecorder{}
+	srvr := ja3x.NewServer("h2")
+	defer srvr.Close()
+	d := &tlsDialerRecorder{
+		alpn:   map[string]int{},
+		config: srvr.ClientConfig(),
+		mu:     sync.Mutex{},
+	}
 	clnt := newClient(newTransport(d.do))
-	resp, err := clnt.Get("https://www.facebook.com")
+	resp, err := clnt.Get(srvr.URL())
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -53,9 +68,15 @@ func TestWorkAsIntendedWithH2(t *testing.T) {
 }
 
 func TestWorkAsIntendedWithHTTP11(t *testing.T) {
-	d := &tlsDialerRecorder{}
+	srvr := ja3x.NewServer("http/1.1")
+	defer srvr.Close()
+	d := &tlsDialerRecorder{
+		alpn:   map[string]int{},
+		config: srvr.ClientConfig(),
+		mu:     sync.Mutex{},
+	}
 	clnt := newClient(newTransport(d.do))
-	resp, err := clnt.Get("https://nexa.polito.it")
+	resp, err := clnt.Get(srvr.URL())
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -71,7 +92,11 @@ func TestWorkAsIntendedWithHTTP11(t *testing.T) {
 }
 
 func TestWorkAsIntendedWithHTTP(t *testing.T) {
-	resp, err := defaultClient.Get("http://example.com")
+	srvr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Write([]byte("0xdeadbeef"))
+	}))
+	defer srvr.Close()
+	resp, err := defaultClient.Get(srvr.URL)
 	if err != nil {
 		log.Fatal(err)
 	}