Nervous system: nervous resolver that handles bogons

cmd/httpclient/main.go

@@ -30,6 +30,7 @@ import (
 	"net/http"
 	"net/url"
 	"os"
+	"time"
 
 	"github.com/apex/log"
 	"github.com/apex/log/handlers/cli"
@@ -39,6 +40,7 @@ import (
 	"github.com/ooni/netx/handlers/logger"
 	"github.com/ooni/netx/httpx"
 	"github.com/ooni/netx/model"
+	"github.com/ooni/netx/x/nervousresolver"
 )
 
 var (
@@ -62,7 +64,7 @@ func mainfunc() (err error) {
 	}()
 	log.SetLevel(log.DebugLevel)
 	log.SetHandler(cli.Default)
-	client := httpx.NewClient(makehandler())
+	client := httpx.NewClient(handlers.NoHandler)
 	if *common.FlagHelp {
 		flag.CommandLine.SetOutput(os.Stdout)
 		fmt.Printf("Usage: httpclient -dns-server <URL> -sni <string> -url <url>\n")
@@ -106,8 +108,17 @@ func fetch(client *http.Client, url string) (err error) {
 			// JUST KNOW WE ARRIVED HERE
 		}
 	}()
-	resp, err := client.Get(url)
-	rtx.PanicOnError(err, "client.Get failed")
+	req, err := http.NewRequest("GET", url, nil)
+	rtx.PanicOnError(err, "http.NewRequest failed")
+	root := &model.MeasurementRoot{
+		Beginning:  time.Now(),
+		Handler:    makehandler(),
+		LookupHost: nervousresolver.Default.LookupHost,
+	}
+	ctx := model.WithMeasurementRoot(req.Context(), root)
+	req = req.WithContext(ctx)
+	resp, err := client.Do(req)
+	rtx.PanicOnError(err, "client.Do failed")
 	defer resp.Body.Close()
 	_, err = ioutil.ReadAll(resp.Body)
 	rtx.PanicOnError(err, "ioutil.ReadAll failed")

handlers/logger/logger.go

@@ -202,6 +202,17 @@ func (h *Handler) OnMeasurement(m model.Measurement) {
 			"transactionID": m.HTTPResponseDone.TransactionID,
 		}).Debug("http: got whole body")
 	}
+
+	// Extensions
+	if m.Extension != nil {
+		h.logger.WithFields(log.Fields{
+			"elapsed":       m.Extension.DurationSinceBeginning,
+			"key":           m.Extension.Key,
+			"severity":      m.Extension.Severity,
+			"transactionID": m.Extension.TransactionID,
+			"value":         fmt.Sprintf("%+v", m.Extension.Value),
+		}).Debug("extension:")
+	}
 }
 
 func reformat(s string) string {

handlers/logger/logger_test.go

@@ -7,6 +7,7 @@ import (
 	"github.com/apex/log"
 	"github.com/apex/log/handlers/discard"
 	"github.com/ooni/netx/httpx"
+	"github.com/ooni/netx/model"
 )
 
 func TestIntegration(t *testing.T) {
@@ -27,3 +28,10 @@ func TestIntegration(t *testing.T) {
 	}
 	client.HTTPClient.CloseIdleConnections()
 }
+
+func TestExtension(t *testing.T) {
+	logger := NewHandler(log.Log)
+	logger.OnMeasurement(model.Measurement{
+		Extension: &model.ExtensionEvent{},
+	})
+}

model/model.go

@@ -72,6 +72,14 @@ type Measurement struct {
 	// something you are supposed to do, so you should be fine.
 	HTTPResponseBodyPart *HTTPResponseBodyPartEvent `json:",omitempty"`
 	HTTPResponseDone     *HTTPResponseDoneEvent     `json:",omitempty"`
+
+	// Extension events.
+	//
+	// The purpose of these events is to give us some flexibility to
+	// experiment with message formats before blessing something as
+	// part of the official API of the library. The intent however is
+	// to avoid keeping something as an extension for a long time.
+	Extension *ExtensionEvent `json:",omitempty"`
 }
 
 // ErrWrapper is our error wrapper for Go errors. The key objective of
@@ -200,6 +208,28 @@ type DNSReplyEvent struct {
 	Msg *dns.Msg `json:"-"`
 }
 
+// ExtensionEvent is emitted by a netx extension.
+type ExtensionEvent struct {
+	// DurationSinceBeginning is the number of nanoseconds since
+	// the time configured as the "zero" time.
+	DurationSinceBeginning time.Duration
+
+	// Key is the unique identifier of the event. A good rule of
+	// thumb is to use `${packageName}.${messageType}`.
+	Key string
+
+	// Severity of the emitted message ("WARN", "INFO", "DEBUG")
+	Severity string
+
+	// TransactionID is the identifier of this transaction, provided
+	// that we have an active one, otherwise is zero.
+	TransactionID int64
+
+	// Value is the extension dependent message. This message
+	// has the only requirement of being JSON serializable.
+	Value interface{}
+}
+
 // HTTPRoundTripStartEvent is emitted when the HTTP transport
 // starts the HTTP "round trip". That is, when the transport
 // receives from the HTTP client a request to sent. The round

x/nervousresolver/bogon/bogon.go

@@ -0,0 +1,53 @@
+// Package bogon contains code to determine if an IP is private/bogon. The
+// code was adapted from https://stackoverflow.com/a/50825191/4354461.
+//
+// See https://badpackets.net/hunting-for-bogons-and-the-isps-that-announce-them/
+// from which I have drawn the full list of private/bogons.
+package bogon
+
+import (
+	"net"
+
+	"github.com/m-lab/go/rtx"
+)
+
+var privateIPBlocks []*net.IPNet
+
+func init() {
+	for _, cidr := range []string{
+		"0.0.0.0/8",      // "This" network (however, Linux...)
+		"10.0.0.0/8",     // RFC1918
+		"100.64.0.0/10",  // Carrier grade NAT
+		"127.0.0.0/8",    // IPv4 loopback
+		"169.254.0.0/16", // RFC3927 link-local
+		"172.16.0.0/12",  // RFC1918
+		"192.168.0.0/16", // RFC1918
+		"224.0.0.0/4",    // Multicast
+		"::1/128",        // IPv6 loopback
+		"fe80::/10",      // IPv6 link-local
+		"fc00::/7",       // IPv6 unique local addr
+	} {
+		_, block, err := net.ParseCIDR(cidr)
+		rtx.PanicOnError(err, "net.ParseCIDR failed")
+		privateIPBlocks = append(privateIPBlocks, block)
+	}
+}
+
+func isPrivate(ip net.IP) bool {
+	if ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() {
+		return true
+	}
+	for _, block := range privateIPBlocks {
+		if block.Contains(ip) {
+			return true
+		}
+	}
+	return false
+}
+
+// Check returns whether if an IP address is bogon. Passing to this
+// function a non-IP address causes it to return bogon.
+func Check(address string) bool {
+	ip := net.ParseIP(address)
+	return ip == nil || isPrivate(ip)
+}

x/nervousresolver/bogon/bogon_test.go

@@ -0,0 +1,18 @@
+package bogon
+
+import "testing"
+
+func TestIntegration(t *testing.T) {
+	if Check("antani") != true {
+		t.Fatal("unexpected result")
+	}
+	if Check("127.0.0.1") != true {
+		t.Fatal("unexpected result")
+	}
+	if Check("1.1.1.1") != false {
+		t.Fatal("unexpected result")
+	}
+	if Check("10.0.1.1") != true {
+		t.Fatal("unexpected result")
+	}
+}

x/nervousresolver/nervousresolver.go

@@ -0,0 +1,130 @@
+// Package nervousresolver contains OONI's nervous resolver
+// that reacts to errors and performs actions.
+//
+// This package is still experimental. See LookupHost docs for
+// an overview of what we're doing here.
+package nervousresolver
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"sync/atomic"
+	"time"
+
+	"github.com/m-lab/go/rtx"
+	"github.com/ooni/netx/handlers"
+	"github.com/ooni/netx/internal"
+	"github.com/ooni/netx/internal/transactionid"
+	"github.com/ooni/netx/model"
+	"github.com/ooni/netx/x/nervousresolver/bogon"
+)
+
+// Resolver is OONI's nervous resolver.
+type Resolver struct {
+	bogonsCount int64
+	primary     model.DNSResolver
+	secondary   model.DNSResolver
+}
+
+// New creates a new OONI nervous resolver instance.
+func New(primary, secondary model.DNSResolver) *Resolver {
+	return &Resolver{
+		primary:   primary,
+		secondary: secondary,
+	}
+}
+
+// LookupAddr returns the name of the provided IP address
+func (c *Resolver) LookupAddr(ctx context.Context, addr string) ([]string, error) {
+	return c.primary.LookupAddr(ctx, addr)
+}
+
+// LookupCNAME returns the canonical name of a host
+func (c *Resolver) LookupCNAME(ctx context.Context, host string) (cname string, err error) {
+	return c.primary.LookupCNAME(ctx, host)
+}
+
+type bogonLookup struct {
+	Addresses []string
+	Comment   string
+	Hostname  string
+}
+
+// LookupHost returns the IP addresses of a host.
+//
+// This code in particular checks whether the first DNS reply is
+// reasonable and, if not, it will query a secondary resolver.
+//
+// The general idea here is that the first resolver is hopefully
+// getaddrinfo and the secondary resolver is DoH/DoT.
+//
+// The code in here is an initial, experimental implementation of a
+// design document on which we're working with Vinicius Fortuna,
+// Jigsaw, aimed at significantly improving OONI measurements quality.
+//
+// See https://docs.google.com/document/d/1jcidvZGxBlucyLivAtvwrCidkIgb3yV6bhQIH_jf21M/edit?ts=5db300e4#
+//
+// TODO(bassosimone): integrate more ideas from the design doc.
+func (c *Resolver) LookupHost(ctx context.Context, hostname string) ([]string, error) {
+	addrs, err := c.primary.LookupHost(ctx, hostname)
+	if err == nil {
+		for _, addr := range addrs {
+			if bogon.Check(addr) == false {
+				// If we see at least a non bogon IP address, let's continue
+				// since it's gonna be interesting :^).
+				return addrs, err
+			}
+		}
+	}
+	atomic.AddInt64(&c.bogonsCount, 1)
+	root := model.ContextMeasurementRootOrDefault(ctx)
+	value := bogonLookup{
+		Addresses: addrs,
+		Comment:   "detected bogon DNS reply, using another resolver",
+		Hostname:  hostname,
+	}
+	// TODO(bassosimone): because this is a PoC, I'm using the
+	// extension event model. I believe there should be a specific
+	// first class event emitted when we see a bogon, tho.
+	root.Handler.OnMeasurement(model.Measurement{
+		Extension: &model.ExtensionEvent{
+			DurationSinceBeginning: time.Now().Sub(root.Beginning),
+			Key:                    fmt.Sprintf("%T", value),
+			Severity:               "WARN",
+			TransactionID:          transactionid.ContextTransactionID(ctx),
+			Value:                  value,
+		},
+	})
+	return c.secondary.LookupHost(ctx, hostname)
+}
+
+// LookupMX returns the MX records of a specific name
+func (c *Resolver) LookupMX(ctx context.Context, name string) ([]*net.MX, error) {
+	return c.primary.LookupMX(ctx, name)
+}
+
+// LookupNS returns the NS records of a specific name
+func (c *Resolver) LookupNS(ctx context.Context, name string) ([]*net.NS, error) {
+	return c.primary.LookupNS(ctx, name)
+}
+
+// Default is the default nervous resolver
+var Default *Resolver
+
+func init() {
+	system, err := internal.NewResolver(
+		time.Time{}, handlers.NoHandler,
+		"system", "",
+	)
+	rtx.PanicOnError(err, "internal.NewResolver #1 failed")
+	// TODO(bassosimone): because this is a PoC, I'm using for
+	// now the address of Cloudflare. We should probably configure
+	// this when integrating in probe-engine.
+	overhttps, err := internal.NewResolver(
+		time.Time{}, handlers.NoHandler,
+		"doh", "https://cloudflare-dns.com/dns-query",
+	)
+	rtx.PanicOnError(err, "internal.NewResolver #2 failed")
+	Default = New(system, overhttps)
+}