diff --git a/.github/workflows/gofmt.yml b/.github/workflows/gofmt.yml index 399bc95dd..e0d41aec9 100644 --- a/.github/workflows/gofmt.yml +++ b/.github/workflows/gofmt.yml @@ -18,7 +18,7 @@ jobs: - uses: actions/setup-go@v5 with: - go-version-file: 'go.mod' + go-version: '1.22' check-latest: true - name: Install goimports diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b5b8cedb9..a199f1d4d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -14,7 +14,7 @@ jobs: - uses: actions/setup-go@v5 with: - go-version-file: 'go.mod' + go-version: '1.22' check-latest: true - name: Build @@ -37,7 +37,7 @@ jobs: - uses: actions/setup-go@v5 with: - go-version-file: 'go.mod' + go-version: '1.22' check-latest: true - name: Build @@ -70,7 +70,7 @@ jobs: - uses: actions/setup-go@v5 with: - go-version-file: 'go.mod' + go-version: '1.22' check-latest: true - name: Import certificates diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index f02b1ba2c..54833bdc7 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -22,7 +22,7 @@ jobs: - uses: actions/setup-go@v5 with: - go-version-file: 'go.mod' + go-version: '1.22' check-latest: true - name: build diff --git a/.github/workflows/smoke/smoke-relay.sh b/.github/workflows/smoke/smoke-relay.sh index 8926091ee..9c113e185 100755 --- a/.github/workflows/smoke/smoke-relay.sh +++ b/.github/workflows/smoke/smoke-relay.sh @@ -76,7 +76,7 @@ docker exec host4 sh -c 'kill 1' docker exec host3 sh -c 'kill 1' docker exec host2 sh -c 'kill 1' docker exec lighthouse1 sh -c 'kill 1' -sleep 1 +sleep 5 if [ "$(jobs -r)" ] then diff --git a/.github/workflows/smoke/smoke.sh b/.github/workflows/smoke/smoke.sh index 3177255bd..6d04027aa 100755 --- a/.github/workflows/smoke/smoke.sh +++ b/.github/workflows/smoke/smoke.sh @@ -129,7 +129,7 @@ docker exec host4 sh -c 'kill 1' docker exec host3 sh -c 'kill 1' docker exec host2 sh -c 'kill 1' docker exec lighthouse1 sh -c 'kill 1' -sleep 1 +sleep 5 if [ "$(jobs -r)" ] then diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 34fe5f325..d71262aef 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -22,7 +22,7 @@ jobs: - uses: actions/setup-go@v5 with: - go-version-file: 'go.mod' + go-version: '1.22' check-latest: true - name: Build @@ -55,7 +55,7 @@ jobs: - uses: actions/setup-go@v5 with: - go-version-file: 'go.mod' + go-version: '1.22' check-latest: true - name: Build @@ -79,7 +79,7 @@ jobs: - uses: actions/setup-go@v5 with: - go-version-file: 'go.mod' + go-version: '1.22' check-latest: true - name: Build nebula diff --git a/LOGGING.md b/LOGGING.md index bd2fdefc8..e2508c83c 100644 --- a/LOGGING.md +++ b/LOGGING.md @@ -33,6 +33,5 @@ l.WithError(err). WithField("vpnIp", IntIp(hostinfo.hostId)). WithField("udpAddr", addr). WithField("handshake", m{"stage": 1, "style": "ix"}). - WithField("cert", remoteCert). Info("Invalid certificate from host") ``` \ No newline at end of file diff --git a/Makefile b/Makefile index 6fa69ce48..fa6713199 100644 --- a/Makefile +++ b/Makefile @@ -1,22 +1,14 @@ -GOMINVERSION = 1.20 NEBULA_CMD_PATH = "./cmd/nebula" -GO111MODULE = on -export GO111MODULE CGO_ENABLED = 0 export CGO_ENABLED # Set up OS specific bits ifeq ($(OS),Windows_NT) - #TODO: we should be able to ditch awk as well - GOVERSION := $(shell go version | awk "{print substr($$3, 3)}") - GOISMIN := $(shell IF "$(GOVERSION)" GEQ "$(GOMINVERSION)" ECHO 1) NEBULA_CMD_SUFFIX = .exe NULL_FILE = nul # RIO on windows does pointer stuff that makes go vet angry VET_FLAGS = -unsafeptr=false else - GOVERSION := $(shell go version | awk '{print substr($$3, 3)}') - GOISMIN := $(shell expr "$(GOVERSION)" ">=" "$(GOMINVERSION)") NEBULA_CMD_SUFFIX = NULL_FILE = /dev/null endif diff --git a/cmd/nebula-cert/ca.go b/cmd/nebula-cert/ca.go index 69df4ab05..4e5d51d0c 100644 --- a/cmd/nebula-cert/ca.go +++ b/cmd/nebula-cert/ca.go @@ -180,9 +180,15 @@ func ca(args []string, out io.Writer, errOut io.Writer, pr PasswordReader) error if err != nil { return fmt.Errorf("error while generating ecdsa keys: %s", err) } - // ref: https://github.com/golang/go/blob/go1.19/src/crypto/x509/sec1.go#L60 - rawPriv = key.D.FillBytes(make([]byte, 32)) - pub = elliptic.Marshal(elliptic.P256(), key.X, key.Y) + + // ecdh.PrivateKey lets us get at the encoded bytes, even though + // we aren't using ECDH here. + eKey, err := key.ECDH() + if err != nil { + return fmt.Errorf("error while converting ecdsa key: %s", err) + } + rawPriv = eKey.Bytes() + pub = eKey.PublicKey().Bytes() } nc := cert.NebulaCertificate{ diff --git a/connection_manager.go b/connection_manager.go index f5dd5942e..0b277b5c1 100644 --- a/connection_manager.go +++ b/connection_manager.go @@ -457,7 +457,7 @@ func (n *connectionManager) sendPunch(hostinfo *HostInfo) { } if n.punchy.GetTargetEverything() { - hostinfo.remotes.ForEach(n.hostMap.preferredRanges, func(addr *udp.Addr, preferred bool) { + hostinfo.remotes.ForEach(n.hostMap.GetPreferredRanges(), func(addr *udp.Addr, preferred bool) { n.metricsTxPunchy.Inc(1) n.intf.outside.WriteTo([]byte{1}, addr) }) diff --git a/connection_manager_test.go b/connection_manager_test.go index a2607a2b7..f50bcf862 100644 --- a/connection_manager_test.go +++ b/connection_manager_test.go @@ -43,7 +43,9 @@ func Test_NewConnectionManagerTest(t *testing.T) { preferredRanges := []*net.IPNet{localrange} // Very incomplete mock objects - hostMap := NewHostMap(l, vpncidr, preferredRanges) + hostMap := newHostMap(l, vpncidr) + hostMap.preferredRanges.Store(&preferredRanges) + cs := &CertState{ RawCertificate: []byte{}, PrivateKey: []byte{}, @@ -123,7 +125,9 @@ func Test_NewConnectionManagerTest2(t *testing.T) { preferredRanges := []*net.IPNet{localrange} // Very incomplete mock objects - hostMap := NewHostMap(l, vpncidr, preferredRanges) + hostMap := newHostMap(l, vpncidr) + hostMap.preferredRanges.Store(&preferredRanges) + cs := &CertState{ RawCertificate: []byte{}, PrivateKey: []byte{}, @@ -210,7 +214,8 @@ func Test_NewConnectionManagerTest_DisconnectInvalid(t *testing.T) { _, vpncidr, _ := net.ParseCIDR("172.1.1.1/24") _, localrange, _ := net.ParseCIDR("10.1.1.1/24") preferredRanges := []*net.IPNet{localrange} - hostMap := NewHostMap(l, vpncidr, preferredRanges) + hostMap := newHostMap(l, vpncidr) + hostMap.preferredRanges.Store(&preferredRanges) // Generate keys for CA and peer's cert. pubCA, privCA, _ := ed25519.GenerateKey(rand.Reader) diff --git a/control.go b/control.go index 1e27b0fa4..c227b207b 100644 --- a/control.go +++ b/control.go @@ -145,7 +145,7 @@ func (c *Control) GetHostInfoByVpnIp(vpnIp iputil.VpnIp, pending bool) *ControlH return nil } - ch := copyHostInfo(h, c.f.hostMap.preferredRanges) + ch := copyHostInfo(h, c.f.hostMap.GetPreferredRanges()) return &ch } @@ -157,7 +157,7 @@ func (c *Control) SetRemoteForTunnel(vpnIp iputil.VpnIp, addr udp.Addr) *Control } hostInfo.SetRemote(addr.Copy()) - ch := copyHostInfo(hostInfo, c.f.hostMap.preferredRanges) + ch := copyHostInfo(hostInfo, c.f.hostMap.GetPreferredRanges()) return &ch } diff --git a/control_test.go b/control_test.go index 847332b4d..c64a3a4b7 100644 --- a/control_test.go +++ b/control_test.go @@ -18,7 +18,9 @@ func TestControl_GetHostInfoByVpnIp(t *testing.T) { l := test.NewLogger() // Special care must be taken to re-use all objects provided to the hostmap and certificate in the expectedInfo object // To properly ensure we are not exposing core memory to the caller - hm := NewHostMap(l, &net.IPNet{}, make([]*net.IPNet, 0)) + hm := newHostMap(l, &net.IPNet{}) + hm.preferredRanges.Store(&[]*net.IPNet{}) + remote1 := udp.NewAddr(net.ParseIP("0.0.0.100"), 4444) remote2 := udp.NewAddr(net.ParseIP("1:2:3:4:5:6:7:8"), 4444) ipNet := net.IPNet{ diff --git a/dist/arch/nebula.service b/dist/arch/nebula.service deleted file mode 100644 index 831c71a53..000000000 --- a/dist/arch/nebula.service +++ /dev/null @@ -1,15 +0,0 @@ -[Unit] -Description=Nebula overlay networking tool -Wants=basic.target network-online.target nss-lookup.target time-sync.target -After=basic.target network.target network-online.target - -[Service] -Type=notify -NotifyAccess=main -SyslogIdentifier=nebula -ExecReload=/bin/kill -HUP $MAINPID -ExecStart=/usr/bin/nebula -config /etc/nebula/config.yml -Restart=always - -[Install] -WantedBy=multi-user.target diff --git a/dist/fedora/nebula.service b/dist/fedora/nebula.service deleted file mode 100644 index 0f947ead4..000000000 --- a/dist/fedora/nebula.service +++ /dev/null @@ -1,16 +0,0 @@ -[Unit] -Description=Nebula overlay networking tool -Wants=basic.target network-online.target nss-lookup.target time-sync.target -After=basic.target network.target network-online.target -Before=sshd.service - -[Service] -Type=notify -NotifyAccess=main -SyslogIdentifier=nebula -ExecReload=/bin/kill -HUP $MAINPID -ExecStart=/usr/bin/nebula -config /etc/nebula/config.yml -Restart=always - -[Install] -WantedBy=multi-user.target diff --git a/dns_server.go b/dns_server.go index 3109b4cf7..4e7bb83af 100644 --- a/dns_server.go +++ b/dns_server.go @@ -56,7 +56,7 @@ func (d *dnsRecords) QueryCert(data string) string { return "" } cert := q.Details - c := fmt.Sprintf("\"Name: %s\" \"Ips: %s\" \"Subnets %s\" \"Groups %s\" \"NotBefore %s\" \"NotAFter %s\" \"PublicKey %x\" \"IsCA %t\" \"Issuer %s\"", cert.Name, cert.Ips, cert.Subnets, cert.Groups, cert.NotBefore, cert.NotAfter, cert.PublicKey, cert.IsCA, cert.Issuer) + c := fmt.Sprintf("\"Name: %s\" \"Ips: %s\" \"Subnets %s\" \"Groups %s\" \"NotBefore %s\" \"NotAfter %s\" \"PublicKey %x\" \"IsCA %t\" \"Issuer %s\"", cert.Name, cert.Ips, cert.Subnets, cert.Groups, cert.NotBefore, cert.NotAfter, cert.PublicKey, cert.IsCA, cert.Issuer) return c } @@ -96,6 +96,10 @@ func parseQuery(l *logrus.Logger, m *dns.Msg, w dns.ResponseWriter) { } } } + + if len(m.Answer) == 0 { + m.Rcode = dns.RcodeNameError + } } func handleDnsRequest(l *logrus.Logger, w dns.ResponseWriter, r *dns.Msg) { @@ -129,7 +133,12 @@ func dnsMain(l *logrus.Logger, hostMap *HostMap, c *config.C) func() { } func getDnsServerAddr(c *config.C) string { - return c.GetString("lighthouse.dns.host", "") + ":" + strconv.Itoa(c.GetInt("lighthouse.dns.port", 53)) + dnsHost := strings.TrimSpace(c.GetString("lighthouse.dns.host", "")) + // Old guidance was to provide the literal `[::]` in `lighthouse.dns.host` but that won't resolve. + if dnsHost == "[::]" { + dnsHost = "::" + } + return net.JoinHostPort(dnsHost, strconv.Itoa(c.GetInt("lighthouse.dns.port", 53))) } func startDns(l *logrus.Logger, c *config.C) { diff --git a/dns_server_test.go b/dns_server_test.go index 830dc8a84..69f6ae84f 100644 --- a/dns_server_test.go +++ b/dns_server_test.go @@ -4,6 +4,8 @@ import ( "testing" "github.com/miekg/dns" + "github.com/slackhq/nebula/config" + "github.com/stretchr/testify/assert" ) func TestParsequery(t *testing.T) { @@ -17,3 +19,40 @@ func TestParsequery(t *testing.T) { //parseQuery(m) } + +func Test_getDnsServerAddr(t *testing.T) { + c := config.NewC(nil) + + c.Settings["lighthouse"] = map[interface{}]interface{}{ + "dns": map[interface{}]interface{}{ + "host": "0.0.0.0", + "port": "1", + }, + } + assert.Equal(t, "0.0.0.0:1", getDnsServerAddr(c)) + + c.Settings["lighthouse"] = map[interface{}]interface{}{ + "dns": map[interface{}]interface{}{ + "host": "::", + "port": "1", + }, + } + assert.Equal(t, "[::]:1", getDnsServerAddr(c)) + + c.Settings["lighthouse"] = map[interface{}]interface{}{ + "dns": map[interface{}]interface{}{ + "host": "[::]", + "port": "1", + }, + } + assert.Equal(t, "[::]:1", getDnsServerAddr(c)) + + // Make sure whitespace doesn't mess us up + c.Settings["lighthouse"] = map[interface{}]interface{}{ + "dns": map[interface{}]interface{}{ + "host": "[::] ", + "port": "1", + }, + } + assert.Equal(t, "[::]:1", getDnsServerAddr(c)) +} diff --git a/examples/config.yml b/examples/config.yml index ff5b40315..9064c2300 100644 --- a/examples/config.yml +++ b/examples/config.yml @@ -244,7 +244,10 @@ tun: # TODO # Configure logging level logging: - # panic, fatal, error, warning, info, or debug. Default is info + # panic, fatal, error, warning, info, or debug. Default is info and is reloadable. + #NOTE: Debug mode can log remotely controlled/untrusted data which can quickly fill a disk in some + # scenarios. Debug logging is also CPU intensive and will decrease performance overall. + # Only enable debug logging while actively investigating an issue. level: info # json or text formats currently available. Default is text format: text diff --git a/examples/service_scripts/nebula.open-rc b/examples/service_scripts/nebula.open-rc new file mode 100644 index 000000000..2beca66c9 --- /dev/null +++ b/examples/service_scripts/nebula.open-rc @@ -0,0 +1,35 @@ +#!/sbin/openrc-run +# +# nebula service for open-rc systems + +extra_commands="checkconfig" + +: ${NEBULA_CONFDIR:=${RC_PREFIX%/}/etc/nebula} +: ${NEBULA_CONFIG:=${NEBULA_CONFDIR}/config.yml} +: ${NEBULA_BINARY:=${NEBULA_BINARY}${RC_PREFIX%/}/usr/local/sbin/nebula} + +command="${NEBULA_BINARY}" +command_args="${NEBULA_OPTS} -config ${NEBULA_CONFIG}" + +supervisor="supervise-daemon" + +description="A scalable overlay networking tool with a focus on performance, simplicity and security" + +required_dirs="${NEBULA_CONFDIR}" +required_files="${NEBULA_CONFIG}" + +checkconfig() { + "${command}" -test ${command_args} || return 1 +} + +start_pre() { + if [ "${RC_CMD}" != "restart" ] ; then + checkconfig || return $? + fi +} + +stop_pre() { + if [ "${RC_CMD}" = "restart" ] ; then + checkconfig || return $? + fi +} diff --git a/firewall.go b/firewall.go index cf2bc5282..3e760feb3 100644 --- a/firewall.go +++ b/firewall.go @@ -2,7 +2,6 @@ package nebula import ( "crypto/sha256" - "encoding/binary" "encoding/hex" "errors" "fmt" @@ -22,17 +21,12 @@ import ( "github.com/slackhq/nebula/firewall" ) -const tcpACK = 0x10 -const tcpFIN = 0x01 - type FirewallInterface interface { AddRule(incoming bool, proto uint8, startPort int32, endPort int32, groups []string, host string, ip *net.IPNet, localIp *net.IPNet, caName string, caSha string) error } type conn struct { Expires time.Time // Time when this conntrack entry will expire - Sent time.Time // If tcp rtt tracking is enabled this will be when Seq was last set - Seq uint32 // If tcp rtt tracking is enabled this will be the seq we are looking for an ack // record why the original connection passed the firewall, so we can re-validate // after ruleset changes. Note, rulesVersion is a uint16 so that these two @@ -66,8 +60,6 @@ type Firewall struct { rulesVersion uint16 defaultLocalCIDRAny bool - trackTCPRTT bool - metricTCPRTT metrics.Histogram incomingMetrics firewallMetrics outgoingMetrics firewallMetrics @@ -183,7 +175,6 @@ func NewFirewall(l *logrus.Logger, tcpTimeout, UDPTimeout, defaultTimeout time.D hasSubnets: len(c.Details.Subnets) > 0, l: l, - metricTCPRTT: metrics.GetOrRegisterHistogram("network.tcp.rtt", nil, metrics.NewExpDecaySample(1028, 0.015)), incomingMetrics: firewallMetrics{ droppedLocalIP: metrics.GetOrRegisterCounter("firewall.incoming.dropped.local_ip", nil), droppedRemoteIP: metrics.GetOrRegisterCounter("firewall.incoming.dropped.remote_ip", nil), @@ -422,9 +413,9 @@ var ErrNoMatchingRule = errors.New("no matching rule in firewall table") // Drop returns an error if the packet should be dropped, explaining why. It // returns nil if the packet should not be dropped. -func (f *Firewall) Drop(packet []byte, fp firewall.Packet, incoming bool, h *HostInfo, caPool *cert.NebulaCAPool, localCache firewall.ConntrackCache) error { +func (f *Firewall) Drop(fp firewall.Packet, incoming bool, h *HostInfo, caPool *cert.NebulaCAPool, localCache firewall.ConntrackCache) error { // Check if we spoke to this tuple, if we did then allow this packet - if f.inConns(packet, fp, incoming, h, caPool, localCache) { + if f.inConns(fp, h, caPool, localCache) { return nil } @@ -462,7 +453,7 @@ func (f *Firewall) Drop(packet []byte, fp firewall.Packet, incoming bool, h *Hos } // We always want to conntrack since it is a faster operation - f.addConn(packet, fp, incoming) + f.addConn(fp, incoming) return nil } @@ -491,7 +482,7 @@ func (f *Firewall) EmitStats() { metrics.GetOrRegisterGauge("firewall.rules.hash", nil).Update(int64(f.GetRuleHashFNV())) } -func (f *Firewall) inConns(packet []byte, fp firewall.Packet, incoming bool, h *HostInfo, caPool *cert.NebulaCAPool, localCache firewall.ConntrackCache) bool { +func (f *Firewall) inConns(fp firewall.Packet, h *HostInfo, caPool *cert.NebulaCAPool, localCache firewall.ConntrackCache) bool { if localCache != nil { if _, ok := localCache[fp]; ok { return true @@ -551,11 +542,6 @@ func (f *Firewall) inConns(packet []byte, fp firewall.Packet, incoming bool, h * switch fp.Protocol { case firewall.ProtoTCP: c.Expires = time.Now().Add(f.TCPTimeout) - if incoming { - f.checkTCPRTT(c, packet) - } else { - setTCPRTTTracking(c, packet) - } case firewall.ProtoUDP: c.Expires = time.Now().Add(f.UDPTimeout) default: @@ -571,16 +557,13 @@ func (f *Firewall) inConns(packet []byte, fp firewall.Packet, incoming bool, h * return true } -func (f *Firewall) addConn(packet []byte, fp firewall.Packet, incoming bool) { +func (f *Firewall) addConn(fp firewall.Packet, incoming bool) { var timeout time.Duration c := &conn{} switch fp.Protocol { case firewall.ProtoTCP: timeout = f.TCPTimeout - if !incoming { - setTCPRTTTracking(c, packet) - } case firewall.ProtoUDP: timeout = f.UDPTimeout default: @@ -876,13 +859,15 @@ func (fr *FirewallRule) match(p firewall.Packet, c *cert.NebulaCertificate) bool } func (flc *firewallLocalCIDR) addRule(f *Firewall, localIp *net.IPNet) error { - if localIp == nil || (localIp != nil && localIp.Contains(net.IPv4(0, 0, 0, 0))) { + if localIp == nil { if !f.hasSubnets || f.defaultLocalCIDRAny { flc.Any = true return nil } localIp = f.assignedCIDR + } else if localIp.Contains(net.IPv4(0, 0, 0, 0)) { + flc.Any = true } flc.LocalCIDR.AddCIDR(localIp, struct{}{}) @@ -1015,42 +1000,3 @@ func parsePort(s string) (startPort, endPort int32, err error) { return } - -// TODO: write tests for these -func setTCPRTTTracking(c *conn, p []byte) { - if c.Seq != 0 { - return - } - - ihl := int(p[0]&0x0f) << 2 - - // Don't track FIN packets - if p[ihl+13]&tcpFIN != 0 { - return - } - - c.Seq = binary.BigEndian.Uint32(p[ihl+4 : ihl+8]) - c.Sent = time.Now() -} - -func (f *Firewall) checkTCPRTT(c *conn, p []byte) bool { - if c.Seq == 0 { - return false - } - - ihl := int(p[0]&0x0f) << 2 - if p[ihl+13]&tcpACK == 0 { - return false - } - - // Deal with wrap around, signed int cuts the ack window in half - // 0 is a bad ack, no data acknowledged - // positive number is a bad ack, ack is over half the window away - if int32(c.Seq-binary.BigEndian.Uint32(p[ihl+8:ihl+12])) >= 0 { - return false - } - - f.metricTCPRTT.Update(time.Since(c.Sent).Nanoseconds()) - c.Seq = 0 - return true -} diff --git a/firewall_test.go b/firewall_test.go index 7d65cb501..b5beff61e 100644 --- a/firewall_test.go +++ b/firewall_test.go @@ -2,14 +2,12 @@ package nebula import ( "bytes" - "encoding/binary" "errors" "math" "net" "testing" "time" - "github.com/rcrowley/go-metrics" "github.com/slackhq/nebula/cert" "github.com/slackhq/nebula/config" "github.com/slackhq/nebula/firewall" @@ -163,44 +161,44 @@ func TestFirewall_Drop(t *testing.T) { cp := cert.NewCAPool() // Drop outbound - assert.Equal(t, fw.Drop([]byte{}, p, false, &h, cp, nil), ErrNoMatchingRule) + assert.Equal(t, fw.Drop(p, false, &h, cp, nil), ErrNoMatchingRule) // Allow inbound resetConntrack(fw) - assert.NoError(t, fw.Drop([]byte{}, p, true, &h, cp, nil)) + assert.NoError(t, fw.Drop(p, true, &h, cp, nil)) // Allow outbound because conntrack - assert.NoError(t, fw.Drop([]byte{}, p, false, &h, cp, nil)) + assert.NoError(t, fw.Drop(p, false, &h, cp, nil)) // test remote mismatch oldRemote := p.RemoteIP p.RemoteIP = iputil.Ip2VpnIp(net.IPv4(1, 2, 3, 10)) - assert.Equal(t, fw.Drop([]byte{}, p, false, &h, cp, nil), ErrInvalidRemoteIP) + assert.Equal(t, fw.Drop(p, false, &h, cp, nil), ErrInvalidRemoteIP) p.RemoteIP = oldRemote // ensure signer doesn't get in the way of group checks fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c) assert.Nil(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", nil, nil, "", "signer-shasum")) assert.Nil(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", nil, nil, "", "signer-shasum-bad")) - assert.Equal(t, fw.Drop([]byte{}, p, true, &h, cp, nil), ErrNoMatchingRule) + assert.Equal(t, fw.Drop(p, true, &h, cp, nil), ErrNoMatchingRule) // test caSha doesn't drop on match fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c) assert.Nil(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", nil, nil, "", "signer-shasum-bad")) assert.Nil(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", nil, nil, "", "signer-shasum")) - assert.NoError(t, fw.Drop([]byte{}, p, true, &h, cp, nil)) + assert.NoError(t, fw.Drop(p, true, &h, cp, nil)) // ensure ca name doesn't get in the way of group checks cp.CAs["signer-shasum"] = &cert.NebulaCertificate{Details: cert.NebulaCertificateDetails{Name: "ca-good"}} fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c) assert.Nil(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", nil, nil, "ca-good", "")) assert.Nil(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", nil, nil, "ca-good-bad", "")) - assert.Equal(t, fw.Drop([]byte{}, p, true, &h, cp, nil), ErrNoMatchingRule) + assert.Equal(t, fw.Drop(p, true, &h, cp, nil), ErrNoMatchingRule) // test caName doesn't drop on match cp.CAs["signer-shasum"] = &cert.NebulaCertificate{Details: cert.NebulaCertificateDetails{Name: "ca-good"}} fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c) assert.Nil(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", nil, nil, "ca-good-bad", "")) assert.Nil(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", nil, nil, "ca-good", "")) - assert.NoError(t, fw.Drop([]byte{}, p, true, &h, cp, nil)) + assert.NoError(t, fw.Drop(p, true, &h, cp, nil)) } func BenchmarkFirewallTable_match(b *testing.B) { @@ -412,10 +410,10 @@ func TestFirewall_Drop2(t *testing.T) { cp := cert.NewCAPool() // h1/c1 lacks the proper groups - assert.Error(t, fw.Drop([]byte{}, p, true, &h1, cp, nil), ErrNoMatchingRule) + assert.Error(t, fw.Drop(p, true, &h1, cp, nil), ErrNoMatchingRule) // c has the proper groups resetConntrack(fw) - assert.NoError(t, fw.Drop([]byte{}, p, true, &h, cp, nil)) + assert.NoError(t, fw.Drop(p, true, &h, cp, nil)) } func TestFirewall_Drop3(t *testing.T) { @@ -495,13 +493,13 @@ func TestFirewall_Drop3(t *testing.T) { cp := cert.NewCAPool() // c1 should pass because host match - assert.NoError(t, fw.Drop([]byte{}, p, true, &h1, cp, nil)) + assert.NoError(t, fw.Drop(p, true, &h1, cp, nil)) // c2 should pass because ca sha match resetConntrack(fw) - assert.NoError(t, fw.Drop([]byte{}, p, true, &h2, cp, nil)) + assert.NoError(t, fw.Drop(p, true, &h2, cp, nil)) // c3 should fail because no match resetConntrack(fw) - assert.Equal(t, fw.Drop([]byte{}, p, true, &h3, cp, nil), ErrNoMatchingRule) + assert.Equal(t, fw.Drop(p, true, &h3, cp, nil), ErrNoMatchingRule) } func TestFirewall_DropConntrackReload(t *testing.T) { @@ -545,12 +543,12 @@ func TestFirewall_DropConntrackReload(t *testing.T) { cp := cert.NewCAPool() // Drop outbound - assert.Equal(t, fw.Drop([]byte{}, p, false, &h, cp, nil), ErrNoMatchingRule) + assert.Equal(t, fw.Drop(p, false, &h, cp, nil), ErrNoMatchingRule) // Allow inbound resetConntrack(fw) - assert.NoError(t, fw.Drop([]byte{}, p, true, &h, cp, nil)) + assert.NoError(t, fw.Drop(p, true, &h, cp, nil)) // Allow outbound because conntrack - assert.NoError(t, fw.Drop([]byte{}, p, false, &h, cp, nil)) + assert.NoError(t, fw.Drop(p, false, &h, cp, nil)) oldFw := fw fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c) @@ -559,7 +557,7 @@ func TestFirewall_DropConntrackReload(t *testing.T) { fw.rulesVersion = oldFw.rulesVersion + 1 // Allow outbound because conntrack and new rules allow port 10 - assert.NoError(t, fw.Drop([]byte{}, p, false, &h, cp, nil)) + assert.NoError(t, fw.Drop(p, false, &h, cp, nil)) oldFw = fw fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c) @@ -568,7 +566,7 @@ func TestFirewall_DropConntrackReload(t *testing.T) { fw.rulesVersion = oldFw.rulesVersion + 1 // Drop outbound because conntrack doesn't match new ruleset - assert.Equal(t, fw.Drop([]byte{}, p, false, &h, cp, nil), ErrNoMatchingRule) + assert.Equal(t, fw.Drop(p, false, &h, cp, nil), ErrNoMatchingRule) } func BenchmarkLookup(b *testing.B) { @@ -830,97 +828,6 @@ func TestAddFirewallRulesFromConfig(t *testing.T) { assert.EqualError(t, AddFirewallRulesFromConfig(l, true, conf, mf), "firewall.inbound rule #0; `test error`") } -func TestTCPRTTTracking(t *testing.T) { - b := make([]byte, 200) - - // Max ip IHL (60 bytes) and tcp IHL (60 bytes) - b[0] = 15 - b[60+12] = 15 << 4 - f := Firewall{ - metricTCPRTT: metrics.GetOrRegisterHistogram("nope", nil, metrics.NewExpDecaySample(1028, 0.015)), - } - - // Set SEQ to 1 - binary.BigEndian.PutUint32(b[60+4:60+8], 1) - - c := &conn{} - setTCPRTTTracking(c, b) - assert.Equal(t, uint32(1), c.Seq) - - // Bad ack - no ack flag - binary.BigEndian.PutUint32(b[60+8:60+12], 80) - assert.False(t, f.checkTCPRTT(c, b)) - - // Bad ack, number is too low - binary.BigEndian.PutUint32(b[60+8:60+12], 0) - b[60+13] = uint8(0x10) - assert.False(t, f.checkTCPRTT(c, b)) - - // Good ack - binary.BigEndian.PutUint32(b[60+8:60+12], 80) - assert.True(t, f.checkTCPRTT(c, b)) - assert.Equal(t, uint32(0), c.Seq) - - // Set SEQ to 1 - binary.BigEndian.PutUint32(b[60+4:60+8], 1) - c = &conn{} - setTCPRTTTracking(c, b) - assert.Equal(t, uint32(1), c.Seq) - - // Good acks - binary.BigEndian.PutUint32(b[60+8:60+12], 81) - assert.True(t, f.checkTCPRTT(c, b)) - assert.Equal(t, uint32(0), c.Seq) - - // Set SEQ to max uint32 - 20 - binary.BigEndian.PutUint32(b[60+4:60+8], ^uint32(0)-20) - c = &conn{} - setTCPRTTTracking(c, b) - assert.Equal(t, ^uint32(0)-20, c.Seq) - - // Good acks - binary.BigEndian.PutUint32(b[60+8:60+12], 81) - assert.True(t, f.checkTCPRTT(c, b)) - assert.Equal(t, uint32(0), c.Seq) - - // Set SEQ to max uint32 / 2 - binary.BigEndian.PutUint32(b[60+4:60+8], ^uint32(0)/2) - c = &conn{} - setTCPRTTTracking(c, b) - assert.Equal(t, ^uint32(0)/2, c.Seq) - - // Below - binary.BigEndian.PutUint32(b[60+8:60+12], ^uint32(0)/2-1) - assert.False(t, f.checkTCPRTT(c, b)) - assert.Equal(t, ^uint32(0)/2, c.Seq) - - // Halfway below - binary.BigEndian.PutUint32(b[60+8:60+12], uint32(0)) - assert.False(t, f.checkTCPRTT(c, b)) - assert.Equal(t, ^uint32(0)/2, c.Seq) - - // Halfway above is ok - binary.BigEndian.PutUint32(b[60+8:60+12], ^uint32(0)) - assert.True(t, f.checkTCPRTT(c, b)) - assert.Equal(t, uint32(0), c.Seq) - - // Set SEQ to max uint32 - binary.BigEndian.PutUint32(b[60+4:60+8], ^uint32(0)) - c = &conn{} - setTCPRTTTracking(c, b) - assert.Equal(t, ^uint32(0), c.Seq) - - // Halfway + 1 above - binary.BigEndian.PutUint32(b[60+8:60+12], ^uint32(0)/2+1) - assert.False(t, f.checkTCPRTT(c, b)) - assert.Equal(t, ^uint32(0), c.Seq) - - // Halfway above - binary.BigEndian.PutUint32(b[60+8:60+12], ^uint32(0)/2) - assert.True(t, f.checkTCPRTT(c, b)) - assert.Equal(t, uint32(0), c.Seq) -} - func TestFirewall_convertRule(t *testing.T) { l := test.NewLogger() ob := &bytes.Buffer{} diff --git a/go.mod b/go.mod index abc1134eb..4676616ff 100644 --- a/go.mod +++ b/go.mod @@ -1,52 +1,54 @@ module github.com/slackhq/nebula -go 1.20 +go 1.22.0 + +toolchain go1.22.2 require ( dario.cat/mergo v1.0.0 github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be github.com/armon/go-radix v1.0.0 github.com/cyberdelia/go-metrics-graphite v0.0.0-20161219230853-39f87cc3b432 - github.com/flynn/noise v1.0.1 + github.com/flynn/noise v1.1.0 github.com/gogo/protobuf v1.3.2 github.com/google/gopacket v1.1.19 github.com/kardianos/service v1.2.2 - github.com/miekg/dns v1.1.58 + github.com/miekg/dns v1.1.59 github.com/nbrownus/go-metrics-prometheus v0.0.0-20210712211119-974a6260965f github.com/prometheus/client_golang v1.18.0 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 github.com/sirupsen/logrus v1.9.3 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8 - github.com/stretchr/testify v1.8.4 + github.com/stretchr/testify v1.9.0 github.com/vishvananda/netlink v1.2.1-beta.2 - golang.org/x/crypto v0.18.0 - golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53 - golang.org/x/net v0.20.0 - golang.org/x/sync v0.6.0 - golang.org/x/sys v0.16.0 - golang.org/x/term v0.16.0 + golang.org/x/crypto v0.22.0 + golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 + golang.org/x/net v0.24.0 + golang.org/x/sync v0.7.0 + golang.org/x/sys v0.19.0 + golang.org/x/term v0.19.0 golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b golang.zx2c4.com/wireguard/windows v0.5.3 - google.golang.org/protobuf v1.32.0 + google.golang.org/protobuf v1.33.0 gopkg.in/yaml.v2 v2.4.0 - gvisor.dev/gvisor v0.0.0-20230504175454-7b0a1988a28f + gvisor.dev/gvisor v0.0.0-20240423190808-9d7a357edefe ) require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/google/btree v1.0.1 // indirect + github.com/google/btree v1.1.2 // indirect github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.45.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect github.com/vishvananda/netns v0.0.4 // indirect - golang.org/x/mod v0.14.0 // indirect - golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect - golang.org/x/tools v0.17.0 // indirect + golang.org/x/mod v0.16.0 // indirect + golang.org/x/time v0.5.0 // indirect + golang.org/x/tools v0.19.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 6226e1508..773a56137 100644 --- a/go.sum +++ b/go.sum @@ -22,8 +22,8 @@ github.com/cyberdelia/go-metrics-graphite v0.0.0-20161219230853-39f87cc3b432/go. github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/flynn/noise v1.0.1 h1:vPp/jdQLXC6ppsXSj/pM3W1BIJ5FEHE2TulSJBpb43Y= -github.com/flynn/noise v1.0.1/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag= +github.com/flynn/noise v1.1.0 h1:KjPQoQCEFdZDiP03phOvGi11+SVVhBG2wOWAorLsstg= +github.com/flynn/noise v1.1.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= @@ -44,14 +44,15 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= -github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= +github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= +github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8= github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= @@ -71,14 +72,15 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= -github.com/miekg/dns v1.1.58 h1:ca2Hdkz+cDg/7eNF6V56jjzuZ4aCAE+DbVkILdQWG/4= -github.com/miekg/dns v1.1.58/go.mod h1:Ypv+3b/KadlvW9vJfXOTf300O4UqaHFzFCuHz+rPkBY= +github.com/miekg/dns v1.1.59 h1:C9EXc/UToRwKLhK5wKU/I4QVsBUc8kE6MkHBkeypWZs= +github.com/miekg/dns v1.1.59/go.mod h1:nZpewl5p6IvctfgrckopVx2OlSEHPRO/U4SYkRklrEk= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= @@ -117,6 +119,7 @@ github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3c github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= @@ -132,8 +135,8 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/vishvananda/netlink v1.2.1-beta.2 h1:Llsql0lnQEbHj0I1OuKyp8otXp0r3q0mPkuhwHfStVs= github.com/vishvananda/netlink v1.2.1-beta.2/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= @@ -146,16 +149,16 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= -golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= -golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= -golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53 h1:5llv2sWeaMSnA3w2kS57ouQQ4pudlXrR0dCgw51QK9o= -golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= +golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= +golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= +golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 h1:Di6/M8l0O2lCLc6VVRWhgCiApHV8MnQurBnFSHsQtNY= +golang.org/x/exp v0.0.0-20230725093048-515e97ebf090/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= -golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= +golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -166,8 +169,8 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= -golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= +golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= +golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -175,8 +178,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -194,23 +197,23 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= -golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= +golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE= -golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= +golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q= +golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 h1:vVKdlvoWBphwdxWKrFZEuM0kGgGLxUOYcY4U/2Vjg44= -golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc= -golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= +golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw= +golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -229,8 +232,8 @@ google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miE google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= -google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -246,5 +249,5 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gvisor.dev/gvisor v0.0.0-20230504175454-7b0a1988a28f h1:8GE2MRjGiFmfpon8dekPI08jEuNMQzSffVHgdupcO4E= -gvisor.dev/gvisor v0.0.0-20230504175454-7b0a1988a28f/go.mod h1:pzr6sy8gDLfVmDAg8OYrlKvGEHw5C3PGTiBXBTCx76Q= +gvisor.dev/gvisor v0.0.0-20240423190808-9d7a357edefe h1:fre4i6mv4iBuz5lCMOzHD1rH1ljqHWSICFmZRbbgp3g= +gvisor.dev/gvisor v0.0.0-20240423190808-9d7a357edefe/go.mod h1:sxc3Uvk/vHcd3tj7/DHVBoR5wvWT/MmRq2pj7HRJnwU= diff --git a/handshake_ix.go b/handshake_ix.go index 1905c004d..8727b16f1 100644 --- a/handshake_ix.go +++ b/handshake_ix.go @@ -90,9 +90,14 @@ func ixHandshakeStage1(f *Interface, addr *udp.Addr, via *ViaSender, packet []by remoteCert, err := RecombineCertAndValidate(ci.H, hs.Details.Cert, f.pki.GetCAPool()) if err != nil { - f.l.WithError(err).WithField("udpAddr", addr). - WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).WithField("cert", remoteCert). - Info("Invalid certificate from host") + e := f.l.WithError(err).WithField("udpAddr", addr). + WithField("handshake", m{"stage": 1, "style": "ix_psk0"}) + + if f.l.Level > logrus.DebugLevel { + e = e.WithField("cert", remoteCert) + } + + e.Info("Invalid certificate from host") return } vpnIp := iputil.Ip2VpnIp(remoteCert.Details.Ips[0].IP) @@ -372,9 +377,14 @@ func ixHandshakeStage2(f *Interface, addr *udp.Addr, via *ViaSender, hh *Handsha remoteCert, err := RecombineCertAndValidate(ci.H, hs.Details.Cert, f.pki.GetCAPool()) if err != nil { - f.l.WithError(err).WithField("vpnIp", hostinfo.vpnIp).WithField("udpAddr", addr). - WithField("cert", remoteCert).WithField("handshake", m{"stage": 2, "style": "ix_psk0"}). - Error("Invalid certificate from host") + e := f.l.WithError(err).WithField("vpnIp", hostinfo.vpnIp).WithField("udpAddr", addr). + WithField("handshake", m{"stage": 2, "style": "ix_psk0"}) + + if f.l.Level > logrus.DebugLevel { + e = e.WithField("cert", remoteCert) + } + + e.Error("Invalid certificate from host") // The handshake state machine is complete, if things break now there is no chance to recover. Tear down and start again return true @@ -406,7 +416,7 @@ func ixHandshakeStage2(f *Interface, addr *udp.Addr, via *ViaSender, hh *Handsha hostinfo.remotes = f.lightHouse.QueryCache(vpnIp) f.l.WithField("blockedUdpAddrs", newHH.hostinfo.remotes.CopyBlockedRemotes()).WithField("vpnIp", vpnIp). - WithField("remotes", newHH.hostinfo.remotes.CopyAddrs(f.hostMap.preferredRanges)). + WithField("remotes", newHH.hostinfo.remotes.CopyAddrs(f.hostMap.GetPreferredRanges())). Info("Blocked addresses for handshakes") // Swap the packet store to benefit the original intended recipient diff --git a/handshake_manager.go b/handshake_manager.go index b568cc88d..640227a7e 100644 --- a/handshake_manager.go +++ b/handshake_manager.go @@ -181,7 +181,7 @@ func (hm *HandshakeManager) handleOutbound(vpnIp iputil.VpnIp, lighthouseTrigger hostinfo := hh.hostinfo // If we are out of time, clean up if hh.counter >= hm.config.retries { - hh.hostinfo.logger(hm.l).WithField("udpAddrs", hh.hostinfo.remotes.CopyAddrs(hm.mainHostMap.preferredRanges)). + hh.hostinfo.logger(hm.l).WithField("udpAddrs", hh.hostinfo.remotes.CopyAddrs(hm.mainHostMap.GetPreferredRanges())). WithField("initiatorIndex", hh.hostinfo.localIndexId). WithField("remoteIndex", hh.hostinfo.remoteIndexId). WithField("handshake", m{"stage": 1, "style": "ix_psk0"}). @@ -211,7 +211,7 @@ func (hm *HandshakeManager) handleOutbound(vpnIp iputil.VpnIp, lighthouseTrigger hostinfo.remotes = hm.lightHouse.QueryCache(vpnIp) } - remotes := hostinfo.remotes.CopyAddrs(hm.mainHostMap.preferredRanges) + remotes := hostinfo.remotes.CopyAddrs(hm.mainHostMap.GetPreferredRanges()) remotesHaveChanged := !udp.AddrSlice(remotes).Equal(hh.lastRemotes) // We only care about a lighthouse trigger if we have new remotes to send to. @@ -235,7 +235,7 @@ func (hm *HandshakeManager) handleOutbound(vpnIp iputil.VpnIp, lighthouseTrigger // Send the handshake to all known ips, stage 2 takes care of assigning the hostinfo.remote based on the first to reply var sentTo []*udp.Addr - hostinfo.remotes.ForEach(hm.mainHostMap.preferredRanges, func(addr *udp.Addr, _ bool) { + hostinfo.remotes.ForEach(hm.mainHostMap.GetPreferredRanges(), func(addr *udp.Addr, _ bool) { hm.messageMetrics.Tx(header.Handshake, header.MessageSubType(hostinfo.HandshakePacket[0][1]), 1) err := hm.outside.WriteTo(hostinfo.HandshakePacket[0], addr) if err != nil { @@ -362,7 +362,7 @@ func (hm *HandshakeManager) GetOrHandshake(vpnIp iputil.VpnIp, cacheCb func(*Han hm.mainHostMap.RUnlock() // Do not attempt promotion if you are a lighthouse if !hm.lightHouse.amLighthouse { - h.TryPromoteBest(hm.mainHostMap.preferredRanges, hm.f) + h.TryPromoteBest(hm.mainHostMap.GetPreferredRanges(), hm.f) } return h, true } @@ -374,13 +374,13 @@ func (hm *HandshakeManager) GetOrHandshake(vpnIp iputil.VpnIp, cacheCb func(*Han // StartHandshake will ensure a handshake is currently being attempted for the provided vpn ip func (hm *HandshakeManager) StartHandshake(vpnIp iputil.VpnIp, cacheCb func(*HandshakeHostInfo)) *HostInfo { hm.Lock() - defer hm.Unlock() if hh, ok := hm.vpnIps[vpnIp]; ok { // We are already trying to handshake with this vpn ip if cacheCb != nil { cacheCb(hh) } + hm.Unlock() return hh.hostinfo } @@ -421,6 +421,7 @@ func (hm *HandshakeManager) StartHandshake(vpnIp iputil.VpnIp, cacheCb func(*Han } } + hm.Unlock() hm.lightHouse.QueryServer(vpnIp) return hostinfo } @@ -599,7 +600,7 @@ func (hm *HandshakeManager) queryIndex(index uint32) *HandshakeHostInfo { } func (c *HandshakeManager) GetPreferredRanges() []*net.IPNet { - return c.mainHostMap.preferredRanges + return c.mainHostMap.GetPreferredRanges() } func (c *HandshakeManager) ForEachVpnIp(f controlEach) { diff --git a/handshake_manager_test.go b/handshake_manager_test.go index 303aa5054..9a6335757 100644 --- a/handshake_manager_test.go +++ b/handshake_manager_test.go @@ -19,7 +19,9 @@ func Test_NewHandshakeManagerVpnIp(t *testing.T) { _, localrange, _ := net.ParseCIDR("10.1.1.1/24") ip := iputil.Ip2VpnIp(net.ParseIP("172.1.1.2")) preferredRanges := []*net.IPNet{localrange} - mainHM := NewHostMap(l, vpncidr, preferredRanges) + mainHM := newHostMap(l, vpncidr) + mainHM.preferredRanges.Store(&preferredRanges) + lh := newTestLighthouse() cs := &CertState{ diff --git a/hostmap.go b/hostmap.go index a5adeb996..589a12463 100644 --- a/hostmap.go +++ b/hostmap.go @@ -11,6 +11,7 @@ import ( "github.com/sirupsen/logrus" "github.com/slackhq/nebula/cert" "github.com/slackhq/nebula/cidr" + "github.com/slackhq/nebula/config" "github.com/slackhq/nebula/header" "github.com/slackhq/nebula/iputil" "github.com/slackhq/nebula/udp" @@ -57,9 +58,8 @@ type HostMap struct { Relays map[uint32]*HostInfo // Maps a Relay IDX to a Relay HostInfo object RemoteIndexes map[uint32]*HostInfo Hosts map[iputil.VpnIp]*HostInfo - preferredRanges []*net.IPNet + preferredRanges atomic.Pointer[[]*net.IPNet] vpnCIDR *net.IPNet - metricsEnabled bool l *logrus.Logger } @@ -254,21 +254,53 @@ type cachedPacketMetrics struct { dropped metrics.Counter } -func NewHostMap(l *logrus.Logger, vpnCIDR *net.IPNet, preferredRanges []*net.IPNet) *HostMap { - h := map[iputil.VpnIp]*HostInfo{} - i := map[uint32]*HostInfo{} - r := map[uint32]*HostInfo{} - relays := map[uint32]*HostInfo{} - m := HostMap{ - Indexes: i, - Relays: relays, - RemoteIndexes: r, - Hosts: h, - preferredRanges: preferredRanges, - vpnCIDR: vpnCIDR, - l: l, +func NewHostMapFromConfig(l *logrus.Logger, vpnCIDR *net.IPNet, c *config.C) *HostMap { + hm := newHostMap(l, vpnCIDR) + + hm.reload(c, true) + c.RegisterReloadCallback(func(c *config.C) { + hm.reload(c, false) + }) + + l.WithField("network", hm.vpnCIDR.String()). + WithField("preferredRanges", hm.GetPreferredRanges()). + Info("Main HostMap created") + + return hm +} + +func newHostMap(l *logrus.Logger, vpnCIDR *net.IPNet) *HostMap { + return &HostMap{ + Indexes: map[uint32]*HostInfo{}, + Relays: map[uint32]*HostInfo{}, + RemoteIndexes: map[uint32]*HostInfo{}, + Hosts: map[iputil.VpnIp]*HostInfo{}, + vpnCIDR: vpnCIDR, + l: l, + } +} + +func (hm *HostMap) reload(c *config.C, initial bool) { + if initial || c.HasChanged("preferred_ranges") { + var preferredRanges []*net.IPNet + rawPreferredRanges := c.GetStringSlice("preferred_ranges", []string{}) + + for _, rawPreferredRange := range rawPreferredRanges { + _, preferredRange, err := net.ParseCIDR(rawPreferredRange) + + if err != nil { + hm.l.WithError(err).WithField("range", rawPreferredRanges).Warn("Failed to parse preferred ranges, ignoring") + continue + } + + preferredRanges = append(preferredRanges, preferredRange) + } + + oldRanges := hm.preferredRanges.Swap(&preferredRanges) + if !initial { + hm.l.WithField("oldPreferredRanges", *oldRanges).WithField("newPreferredRanges", preferredRanges).Info("preferred_ranges changed") + } } - return &m } // EmitStats reports host, index, and relay counts to the stats collection system @@ -457,7 +489,7 @@ func (hm *HostMap) queryVpnIp(vpnIp iputil.VpnIp, promoteIfce *Interface) *HostI hm.RUnlock() // Do not attempt promotion if you are a lighthouse if promoteIfce != nil && !promoteIfce.lightHouse.amLighthouse { - h.TryPromoteBest(hm.preferredRanges, promoteIfce) + h.TryPromoteBest(hm.GetPreferredRanges(), promoteIfce) } return h @@ -504,7 +536,8 @@ func (hm *HostMap) unlockedAddHostInfo(hostinfo *HostInfo, f *Interface) { } func (hm *HostMap) GetPreferredRanges() []*net.IPNet { - return hm.preferredRanges + //NOTE: if preferredRanges is ever not stored before a load this will fail to dereference a nil pointer + return *hm.preferredRanges.Load() } func (hm *HostMap) ForEachVpnIp(f controlEach) { @@ -596,7 +629,7 @@ func (i *HostInfo) SetRemoteIfPreferred(hm *HostMap, newRemote *udp.Addr) bool { // NOTE: We do this loop here instead of calling `isPreferred` in // remote_list.go so that we only have to loop over preferredRanges once. newIsPreferred := false - for _, l := range hm.preferredRanges { + for _, l := range hm.GetPreferredRanges() { // return early if we are already on a preferred remote if l.Contains(currentRemote.IP) { return false diff --git a/hostmap_test.go b/hostmap_test.go index c1c0dcead..8311cef0b 100644 --- a/hostmap_test.go +++ b/hostmap_test.go @@ -4,19 +4,19 @@ import ( "net" "testing" + "github.com/slackhq/nebula/config" "github.com/slackhq/nebula/test" "github.com/stretchr/testify/assert" ) func TestHostMap_MakePrimary(t *testing.T) { l := test.NewLogger() - hm := NewHostMap( + hm := newHostMap( l, &net.IPNet{ IP: net.IP{10, 0, 0, 1}, Mask: net.IPMask{255, 255, 255, 0}, }, - []*net.IPNet{}, ) f := &Interface{} @@ -91,13 +91,12 @@ func TestHostMap_MakePrimary(t *testing.T) { func TestHostMap_DeleteHostInfo(t *testing.T) { l := test.NewLogger() - hm := NewHostMap( + hm := newHostMap( l, &net.IPNet{ IP: net.IP{10, 0, 0, 1}, Mask: net.IPMask{255, 255, 255, 0}, }, - []*net.IPNet{}, ) f := &Interface{} @@ -205,3 +204,33 @@ func TestHostMap_DeleteHostInfo(t *testing.T) { prim = hm.QueryVpnIp(1) assert.Nil(t, prim) } + +func TestHostMap_reload(t *testing.T) { + l := test.NewLogger() + c := config.NewC(l) + + hm := NewHostMapFromConfig( + l, + &net.IPNet{ + IP: net.IP{10, 0, 0, 1}, + Mask: net.IPMask{255, 255, 255, 0}, + }, + c, + ) + + toS := func(ipn []*net.IPNet) []string { + var s []string + for _, n := range ipn { + s = append(s, n.String()) + } + return s + } + + assert.Empty(t, hm.GetPreferredRanges()) + + c.ReloadConfigString("preferred_ranges: [1.1.1.0/24, 10.1.1.0/24]") + assert.EqualValues(t, []string{"1.1.1.0/24", "10.1.1.0/24"}, toS(hm.GetPreferredRanges())) + + c.ReloadConfigString("preferred_ranges: [1.1.1.1/32]") + assert.EqualValues(t, []string{"1.1.1.1/32"}, toS(hm.GetPreferredRanges())) +} diff --git a/inside.go b/inside.go index 62309628a..079e4dd2f 100644 --- a/inside.go +++ b/inside.go @@ -62,7 +62,7 @@ func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet return } - dropReason := f.firewall.Drop(packet, *fwPacket, false, hostinfo, f.pki.GetCAPool(), localCache) + dropReason := f.firewall.Drop(*fwPacket, false, hostinfo, f.pki.GetCAPool(), localCache) if dropReason == nil { f.sendNoMetrics(header.Message, 0, hostinfo.ConnectionState, hostinfo, nil, packet, nb, out, q) @@ -142,7 +142,7 @@ func (f *Interface) sendMessageNow(t header.MessageType, st header.MessageSubTyp } // check if packet is in outbound fw rules - dropReason := f.firewall.Drop(p, *fp, false, hostinfo, f.pki.GetCAPool(), nil) + dropReason := f.firewall.Drop(*fp, false, hostinfo, f.pki.GetCAPool(), nil) if dropReason != nil { if f.l.Level >= logrus.DebugLevel { f.l.WithField("fwPacket", fp). diff --git a/main.go b/main.go index 8c94e80db..7a0a0cff3 100644 --- a/main.go +++ b/main.go @@ -183,52 +183,7 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg } } - // Set up my internal host map - var preferredRanges []*net.IPNet - rawPreferredRanges := c.GetStringSlice("preferred_ranges", []string{}) - // First, check if 'preferred_ranges' is set and fallback to 'local_range' - if len(rawPreferredRanges) > 0 { - for _, rawPreferredRange := range rawPreferredRanges { - _, preferredRange, err := net.ParseCIDR(rawPreferredRange) - if err != nil { - return nil, util.ContextualizeIfNeeded("Failed to parse preferred ranges", err) - } - preferredRanges = append(preferredRanges, preferredRange) - } - } - - // local_range was superseded by preferred_ranges. If it is still present, - // merge the local_range setting into preferred_ranges. We will probably - // deprecate local_range and remove in the future. - rawLocalRange := c.GetString("local_range", "") - if rawLocalRange != "" { - _, localRange, err := net.ParseCIDR(rawLocalRange) - if err != nil { - return nil, util.ContextualizeIfNeeded("Failed to parse local_range", err) - } - - // Check if the entry for local_range was already specified in - // preferred_ranges. Don't put it into the slice twice if so. - var found bool - for _, r := range preferredRanges { - if r.String() == localRange.String() { - found = true - break - } - } - if !found { - preferredRanges = append(preferredRanges, localRange) - } - } - - hostMap := NewHostMap(l, tunCidr, preferredRanges) - hostMap.metricsEnabled = c.GetBool("stats.message_metrics", false) - - l. - WithField("network", hostMap.vpnCIDR.String()). - WithField("preferredRanges", hostMap.preferredRanges). - Info("Main HostMap created") - + hostMap := NewHostMapFromConfig(l, tunCidr, c) punchy := NewPunchyFromConfig(l, c) lightHouse, err := NewLightHouseFromConfig(ctx, l, c, tunCidr, udpConns[0], punchy) if err != nil { diff --git a/outside.go b/outside.go index 29189110c..818e2ae4b 100644 --- a/outside.go +++ b/outside.go @@ -404,7 +404,7 @@ func (f *Interface) decryptToTun(hostinfo *HostInfo, messageCounter uint64, out return false } - dropReason := f.firewall.Drop(out, *fwPacket, true, hostinfo, f.pki.GetCAPool(), localCache) + dropReason := f.firewall.Drop(*fwPacket, true, hostinfo, f.pki.GetCAPool(), localCache) if dropReason != nil { // NOTE: We give `packet` as the `out` here since we already decrypted from it and we don't need it anymore // This gives us a buffer to build the reject packet in diff --git a/overlay/route.go b/overlay/route.go index 793c8fd06..64c624c7e 100644 --- a/overlay/route.go +++ b/overlay/route.go @@ -1,6 +1,7 @@ package overlay import ( + "bytes" "fmt" "math" "net" @@ -21,6 +22,35 @@ type Route struct { Install bool } +// Equal determines if a route that could be installed in the system route table is equal to another +// Via is ignored since that is only consumed within nebula itself +func (r Route) Equal(t Route) bool { + if !r.Cidr.IP.Equal(t.Cidr.IP) { + return false + } + if !bytes.Equal(r.Cidr.Mask, t.Cidr.Mask) { + return false + } + if r.Metric != t.Metric { + return false + } + if r.MTU != t.MTU { + return false + } + if r.Install != t.Install { + return false + } + return true +} + +func (r Route) String() string { + s := r.Cidr.String() + if r.Metric != 0 { + s += fmt.Sprintf(" metric: %v", r.Metric) + } + return s +} + func makeRouteTree(l *logrus.Logger, routes []Route, allowMTU bool) (*cidr.Tree4[iputil.VpnIp], error) { routeTree := cidr.NewTree4[iputil.VpnIp]() for _, r := range routes { diff --git a/overlay/tun.go b/overlay/tun.go index ca1a64a33..cedd7fe76 100644 --- a/overlay/tun.go +++ b/overlay/tun.go @@ -10,60 +10,63 @@ import ( const DefaultMTU = 1300 +// TODO: We may be able to remove routines type DeviceFactory func(c *config.C, l *logrus.Logger, tunCidr *net.IPNet, routines int) (Device, error) func NewDeviceFromConfig(c *config.C, l *logrus.Logger, tunCidr *net.IPNet, routines int) (Device, error) { - routes, err := parseRoutes(c, tunCidr) - if err != nil { - return nil, util.NewContextualError("Could not parse tun.routes", nil, err) - } - - unsafeRoutes, err := parseUnsafeRoutes(c, tunCidr) - if err != nil { - return nil, util.NewContextualError("Could not parse tun.unsafe_routes", nil, err) - } - routes = append(routes, unsafeRoutes...) - switch { case c.GetBool("tun.disabled", false): tun := newDisabledTun(tunCidr, c.GetInt("tun.tx_queue", 500), c.GetBool("stats.message_metrics", false), l) return tun, nil default: - return newTun( - l, - c.GetString("tun.dev", ""), - tunCidr, - c.GetInt("tun.mtu", DefaultMTU), - routes, - c.GetInt("tun.tx_queue", 500), - routines > 1, - c.GetBool("tun.use_system_route_table", false), - ) + return newTun(c, l, tunCidr, routines > 1) } } func NewFdDeviceFromConfig(fd *int) DeviceFactory { return func(c *config.C, l *logrus.Logger, tunCidr *net.IPNet, routines int) (Device, error) { - routes, err := parseRoutes(c, tunCidr) - if err != nil { - return nil, util.NewContextualError("Could not parse tun.routes", nil, err) - } + return newTunFromFd(c, l, *fd, tunCidr) + } +} + +func getAllRoutesFromConfig(c *config.C, cidr *net.IPNet, initial bool) (bool, []Route, error) { + if !initial && !c.HasChanged("tun.routes") && !c.HasChanged("tun.unsafe_routes") { + return false, nil, nil + } - unsafeRoutes, err := parseUnsafeRoutes(c, tunCidr) - if err != nil { - return nil, util.NewContextualError("Could not parse tun.unsafe_routes", nil, err) + routes, err := parseRoutes(c, cidr) + if err != nil { + return true, nil, util.NewContextualError("Could not parse tun.routes", nil, err) + } + + unsafeRoutes, err := parseUnsafeRoutes(c, cidr) + if err != nil { + return true, nil, util.NewContextualError("Could not parse tun.unsafe_routes", nil, err) + } + + routes = append(routes, unsafeRoutes...) + return true, routes, nil +} + +// findRemovedRoutes will return all routes that are not present in the newRoutes list and would affect the system route table. +// Via is not used to evaluate since it does not affect the system route table. +func findRemovedRoutes(newRoutes, oldRoutes []Route) []Route { + var removed []Route + has := func(entry Route) bool { + for _, check := range newRoutes { + if check.Equal(entry) { + return true + } } - routes = append(routes, unsafeRoutes...) - return newTunFromFd( - l, - *fd, - tunCidr, - c.GetInt("tun.mtu", DefaultMTU), - routes, - c.GetInt("tun.tx_queue", 500), - c.GetBool("tun.use_system_route_table", false), - ) + return false + } + for _, oldEntry := range oldRoutes { + if !has(oldEntry) { + removed = append(removed, oldEntry) + } } + + return removed } diff --git a/overlay/tun_android.go b/overlay/tun_android.go index c5c52db25..c15827fe6 100644 --- a/overlay/tun_android.go +++ b/overlay/tun_android.go @@ -8,45 +8,57 @@ import ( "io" "net" "os" + "sync/atomic" "github.com/sirupsen/logrus" "github.com/slackhq/nebula/cidr" + "github.com/slackhq/nebula/config" "github.com/slackhq/nebula/iputil" + "github.com/slackhq/nebula/util" ) type tun struct { io.ReadWriteCloser fd int cidr *net.IPNet - routeTree *cidr.Tree4[iputil.VpnIp] + Routes atomic.Pointer[[]Route] + routeTree atomic.Pointer[cidr.Tree4[iputil.VpnIp]] l *logrus.Logger } -func newTunFromFd(l *logrus.Logger, deviceFd int, cidr *net.IPNet, _ int, routes []Route, _ int, _ bool) (*tun, error) { - routeTree, err := makeRouteTree(l, routes, false) - if err != nil { - return nil, err - } - +func newTunFromFd(c *config.C, l *logrus.Logger, deviceFd int, cidr *net.IPNet) (*tun, error) { // XXX Android returns an fd in non-blocking mode which is necessary for shutdown to work properly. // Be sure not to call file.Fd() as it will set the fd to blocking mode. file := os.NewFile(uintptr(deviceFd), "/dev/net/tun") - return &tun{ + t := &tun{ ReadWriteCloser: file, fd: deviceFd, cidr: cidr, l: l, - routeTree: routeTree, - }, nil + } + + err := t.reload(c, true) + if err != nil { + return nil, err + } + + c.RegisterReloadCallback(func(c *config.C) { + err := t.reload(c, false) + if err != nil { + util.LogWithContextIfNeeded("failed to reload tun device", err, t.l) + } + }) + + return t, nil } -func newTun(_ *logrus.Logger, _ string, _ *net.IPNet, _ int, _ []Route, _ int, _ bool, _ bool) (*tun, error) { +func newTun(_ *config.C, _ *logrus.Logger, _ *net.IPNet, _ bool) (*tun, error) { return nil, fmt.Errorf("newTun not supported in Android") } func (t *tun) RouteFor(ip iputil.VpnIp) iputil.VpnIp { - _, r := t.routeTree.MostSpecificContains(ip) + _, r := t.routeTree.Load().MostSpecificContains(ip) return r } @@ -54,6 +66,27 @@ func (t tun) Activate() error { return nil } +func (t *tun) reload(c *config.C, initial bool) error { + change, routes, err := getAllRoutesFromConfig(c, t.cidr, initial) + if err != nil { + return err + } + + if !initial && !change { + return nil + } + + routeTree, err := makeRouteTree(t.l, routes, false) + if err != nil { + return err + } + + // Teach nebula how to handle the routes + t.Routes.Store(&routes) + t.routeTree.Store(routeTree) + return nil +} + func (t *tun) Cidr() *net.IPNet { return t.cidr } diff --git a/overlay/tun_darwin.go b/overlay/tun_darwin.go index caec580f5..1c6382827 100644 --- a/overlay/tun_darwin.go +++ b/overlay/tun_darwin.go @@ -9,12 +9,15 @@ import ( "io" "net" "os" + "sync/atomic" "syscall" "unsafe" "github.com/sirupsen/logrus" "github.com/slackhq/nebula/cidr" + "github.com/slackhq/nebula/config" "github.com/slackhq/nebula/iputil" + "github.com/slackhq/nebula/util" netroute "golang.org/x/net/route" "golang.org/x/sys/unix" ) @@ -24,8 +27,9 @@ type tun struct { Device string cidr *net.IPNet DefaultMTU int - Routes []Route - routeTree *cidr.Tree4[iputil.VpnIp] + Routes atomic.Pointer[[]Route] + routeTree atomic.Pointer[cidr.Tree4[iputil.VpnIp]] + linkAddr *netroute.LinkAddr l *logrus.Logger // cache out buffer since we need to prepend 4 bytes for tun metadata @@ -69,12 +73,8 @@ type ifreqMTU struct { pad [8]byte } -func newTun(l *logrus.Logger, name string, cidr *net.IPNet, defaultMTU int, routes []Route, _ int, _ bool, _ bool) (*tun, error) { - routeTree, err := makeRouteTree(l, routes, false) - if err != nil { - return nil, err - } - +func newTun(c *config.C, l *logrus.Logger, cidr *net.IPNet, _ bool) (*tun, error) { + name := c.GetString("tun.dev", "") ifIndex := -1 if name != "" && name != "utun" { _, err := fmt.Sscanf(name, "utun%d", &ifIndex) @@ -142,17 +142,27 @@ func newTun(l *logrus.Logger, name string, cidr *net.IPNet, defaultMTU int, rout file := os.NewFile(uintptr(fd), "") - tun := &tun{ + t := &tun{ ReadWriteCloser: file, Device: name, cidr: cidr, - DefaultMTU: defaultMTU, - Routes: routes, - routeTree: routeTree, + DefaultMTU: c.GetInt("tun.mtu", DefaultMTU), l: l, } - return tun, nil + err = t.reload(c, true) + if err != nil { + return nil, err + } + + c.RegisterReloadCallback(func(c *config.C) { + err := t.reload(c, false) + if err != nil { + util.LogWithContextIfNeeded("failed to reload tun device", err, t.l) + } + }) + + return t, nil } func (t *tun) deviceBytes() (o [16]byte) { @@ -162,7 +172,7 @@ func (t *tun) deviceBytes() (o [16]byte) { return } -func newTunFromFd(_ *logrus.Logger, _ int, _ *net.IPNet, _ int, _ []Route, _ int, _ bool) (*tun, error) { +func newTunFromFd(_ *config.C, _ *logrus.Logger, _ int, _ *net.IPNet) (*tun, error) { return nil, fmt.Errorf("newTunFromFd not supported in Darwin") } @@ -260,6 +270,7 @@ func (t *tun) Activate() error { if linkAddr == nil { return fmt.Errorf("unable to discover link_addr for tun interface") } + t.linkAddr = linkAddr copy(routeAddr.IP[:], addr[:]) copy(maskAddr.IP[:], mask[:]) @@ -278,33 +289,48 @@ func (t *tun) Activate() error { } // Unsafe path routes - for _, r := range t.Routes { - if r.Via == nil || !r.Install { - // We don't allow route MTUs so only install routes with a via - continue - } + return t.addRoutes(false) +} - copy(routeAddr.IP[:], r.Cidr.IP.To4()) - copy(maskAddr.IP[:], net.IP(r.Cidr.Mask).To4()) +func (t *tun) reload(c *config.C, initial bool) error { + change, routes, err := getAllRoutesFromConfig(c, t.cidr, initial) + if err != nil { + return err + } + + if !initial && !change { + return nil + } - err = addRoute(routeSock, routeAddr, maskAddr, linkAddr) + routeTree, err := makeRouteTree(t.l, routes, false) + if err != nil { + return err + } + + // Teach nebula how to handle the routes before establishing them in the system table + oldRoutes := t.Routes.Swap(&routes) + t.routeTree.Store(routeTree) + + if !initial { + // Remove first, if the system removes a wanted route hopefully it will be re-added next + err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes)) if err != nil { - if errors.Is(err, unix.EEXIST) { - t.l.WithField("route", r.Cidr). - Warnf("unable to add unsafe_route, identical route already exists") - } else { - return err - } + util.LogWithContextIfNeeded("Failed to remove routes", err, t.l) } - // TODO how to set metric + // Ensure any routes we actually want are installed + err = t.addRoutes(true) + if err != nil { + // Catch any stray logs + util.LogWithContextIfNeeded("Failed to add routes", err, t.l) + } } return nil } func (t *tun) RouteFor(ip iputil.VpnIp) iputil.VpnIp { - ok, r := t.routeTree.MostSpecificContains(ip) + ok, r := t.routeTree.Load().MostSpecificContains(ip) if ok { return r } @@ -340,6 +366,88 @@ func getLinkAddr(name string) (*netroute.LinkAddr, error) { return nil, nil } +func (t *tun) addRoutes(logErrors bool) error { + routeSock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC) + if err != nil { + return fmt.Errorf("unable to create AF_ROUTE socket: %v", err) + } + + defer func() { + unix.Shutdown(routeSock, unix.SHUT_RDWR) + err := unix.Close(routeSock) + if err != nil { + t.l.WithError(err).Error("failed to close AF_ROUTE socket") + } + }() + + routeAddr := &netroute.Inet4Addr{} + maskAddr := &netroute.Inet4Addr{} + routes := *t.Routes.Load() + for _, r := range routes { + if r.Via == nil || !r.Install { + // We don't allow route MTUs so only install routes with a via + continue + } + + copy(routeAddr.IP[:], r.Cidr.IP.To4()) + copy(maskAddr.IP[:], net.IP(r.Cidr.Mask).To4()) + + err := addRoute(routeSock, routeAddr, maskAddr, t.linkAddr) + if err != nil { + if errors.Is(err, unix.EEXIST) { + t.l.WithField("route", r.Cidr). + Warnf("unable to add unsafe_route, identical route already exists") + } else { + retErr := util.NewContextualError("Failed to add route", map[string]interface{}{"route": r}, err) + if logErrors { + retErr.Log(t.l) + } else { + return retErr + } + } + } else { + t.l.WithField("route", r).Info("Added route") + } + } + + return nil +} + +func (t *tun) removeRoutes(routes []Route) error { + routeSock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC) + if err != nil { + return fmt.Errorf("unable to create AF_ROUTE socket: %v", err) + } + + defer func() { + unix.Shutdown(routeSock, unix.SHUT_RDWR) + err := unix.Close(routeSock) + if err != nil { + t.l.WithError(err).Error("failed to close AF_ROUTE socket") + } + }() + + routeAddr := &netroute.Inet4Addr{} + maskAddr := &netroute.Inet4Addr{} + + for _, r := range routes { + if !r.Install { + continue + } + + copy(routeAddr.IP[:], r.Cidr.IP.To4()) + copy(maskAddr.IP[:], net.IP(r.Cidr.Mask).To4()) + + err := delRoute(routeSock, routeAddr, maskAddr, t.linkAddr) + if err != nil { + t.l.WithError(err).WithField("route", r).Error("Failed to remove route") + } else { + t.l.WithField("route", r).Info("Removed route") + } + } + return nil +} + func addRoute(sock int, addr, mask *netroute.Inet4Addr, link *netroute.LinkAddr) error { r := netroute.RouteMessage{ Version: unix.RTM_VERSION, @@ -365,6 +473,30 @@ func addRoute(sock int, addr, mask *netroute.Inet4Addr, link *netroute.LinkAddr) return nil } +func delRoute(sock int, addr, mask *netroute.Inet4Addr, link *netroute.LinkAddr) error { + r := netroute.RouteMessage{ + Version: unix.RTM_VERSION, + Type: unix.RTM_DELETE, + Seq: 1, + Addrs: []netroute.Addr{ + unix.RTAX_DST: addr, + unix.RTAX_GATEWAY: link, + unix.RTAX_NETMASK: mask, + }, + } + + data, err := r.Marshal() + if err != nil { + return fmt.Errorf("failed to create route.RouteMessage: %w", err) + } + _, err = unix.Write(sock, data[:]) + if err != nil { + return fmt.Errorf("failed to write route.RouteMessage to socket: %w", err) + } + + return nil +} + func (t *tun) Read(to []byte) (int, error) { buf := make([]byte, len(to)+4) diff --git a/overlay/tun_freebsd.go b/overlay/tun_freebsd.go index 338b8f61f..3b1b80f1a 100644 --- a/overlay/tun_freebsd.go +++ b/overlay/tun_freebsd.go @@ -13,12 +13,15 @@ import ( "os" "os/exec" "strconv" + "sync/atomic" "syscall" "unsafe" "github.com/sirupsen/logrus" "github.com/slackhq/nebula/cidr" + "github.com/slackhq/nebula/config" "github.com/slackhq/nebula/iputil" + "github.com/slackhq/nebula/util" ) const ( @@ -47,8 +50,8 @@ type tun struct { Device string cidr *net.IPNet MTU int - Routes []Route - routeTree *cidr.Tree4[iputil.VpnIp] + Routes atomic.Pointer[[]Route] + routeTree atomic.Pointer[cidr.Tree4[iputil.VpnIp]] l *logrus.Logger io.ReadWriteCloser @@ -76,14 +79,15 @@ func (t *tun) Close() error { return nil } -func newTunFromFd(_ *logrus.Logger, _ int, _ *net.IPNet, _ int, _ []Route, _ int, _ bool) (*tun, error) { +func newTunFromFd(_ *config.C, _ *logrus.Logger, _ int, _ *net.IPNet) (*tun, error) { return nil, fmt.Errorf("newTunFromFd not supported in FreeBSD") } -func newTun(l *logrus.Logger, deviceName string, cidr *net.IPNet, defaultMTU int, routes []Route, _ int, _ bool, _ bool) (*tun, error) { +func newTun(c *config.C, l *logrus.Logger, cidr *net.IPNet, _ bool) (*tun, error) { // Try to open existing tun device var file *os.File var err error + deviceName := c.GetString("tun.dev", "") if deviceName != "" { file, err = os.OpenFile("/dev/"+deviceName, os.O_RDWR, 0) } @@ -144,47 +148,85 @@ func newTun(l *logrus.Logger, deviceName string, cidr *net.IPNet, defaultMTU int ioctl(fd, syscall.SIOCSIFNAME, uintptr(unsafe.Pointer(&ifrr))) } - routeTree, err := makeRouteTree(l, routes, false) - if err != nil { - return nil, err - } - - return &tun{ + t := &tun{ ReadWriteCloser: file, Device: deviceName, cidr: cidr, - MTU: defaultMTU, - Routes: routes, - routeTree: routeTree, + MTU: c.GetInt("tun.mtu", DefaultMTU), l: l, - }, nil + } + + err = t.reload(c, true) + if err != nil { + return nil, err + } + + c.RegisterReloadCallback(func(c *config.C) { + err := t.reload(c, false) + if err != nil { + util.LogWithContextIfNeeded("failed to reload tun device", err, t.l) + } + }) + + return t, nil } func (t *tun) Activate() error { var err error // TODO use syscalls instead of exec.Command - t.l.Debug("command: ifconfig", t.Device, t.cidr.String(), t.cidr.IP.String()) - if err = exec.Command("/sbin/ifconfig", t.Device, t.cidr.String(), t.cidr.IP.String()).Run(); err != nil { + cmd := exec.Command("/sbin/ifconfig", t.Device, t.cidr.String(), t.cidr.IP.String()) + t.l.Debug("command: ", cmd.String()) + if err = cmd.Run(); err != nil { return fmt.Errorf("failed to run 'ifconfig': %s", err) } - t.l.Debug("command: route", "-n", "add", "-net", t.cidr.String(), "-interface", t.Device) - if err = exec.Command("/sbin/route", "-n", "add", "-net", t.cidr.String(), "-interface", t.Device).Run(); err != nil { + + cmd = exec.Command("/sbin/route", "-n", "add", "-net", t.cidr.String(), "-interface", t.Device) + t.l.Debug("command: ", cmd.String()) + if err = cmd.Run(); err != nil { return fmt.Errorf("failed to run 'route add': %s", err) } - t.l.Debug("command: ifconfig", t.Device, "mtu", strconv.Itoa(t.MTU)) - if err = exec.Command("/sbin/ifconfig", t.Device, "mtu", strconv.Itoa(t.MTU)).Run(); err != nil { + + cmd = exec.Command("/sbin/ifconfig", t.Device, "mtu", strconv.Itoa(t.MTU)) + t.l.Debug("command: ", cmd.String()) + if err = cmd.Run(); err != nil { return fmt.Errorf("failed to run 'ifconfig': %s", err) } + // Unsafe path routes - for _, r := range t.Routes { - if r.Via == nil || !r.Install { - // We don't allow route MTUs so only install routes with a via - continue + return t.addRoutes(false) +} + +func (t *tun) reload(c *config.C, initial bool) error { + change, routes, err := getAllRoutesFromConfig(c, t.cidr, initial) + if err != nil { + return err + } + + if !initial && !change { + return nil + } + + routeTree, err := makeRouteTree(t.l, routes, false) + if err != nil { + return err + } + + // Teach nebula how to handle the routes before establishing them in the system table + oldRoutes := t.Routes.Swap(&routes) + t.routeTree.Store(routeTree) + + if !initial { + // Remove first, if the system removes a wanted route hopefully it will be re-added next + err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes)) + if err != nil { + util.LogWithContextIfNeeded("Failed to remove routes", err, t.l) } - t.l.Debug("command: route", "-n", "add", "-net", r.Cidr.String(), "-interface", t.Device) - if err = exec.Command("/sbin/route", "-n", "add", "-net", r.Cidr.String(), "-interface", t.Device).Run(); err != nil { - return fmt.Errorf("failed to run 'route add' for unsafe_route %s: %s", r.Cidr.String(), err) + // Ensure any routes we actually want are installed + err = t.addRoutes(true) + if err != nil { + // Catch any stray logs + util.LogWithContextIfNeeded("Failed to add routes", err, t.l) } } @@ -192,7 +234,7 @@ func (t *tun) Activate() error { } func (t *tun) RouteFor(ip iputil.VpnIp) iputil.VpnIp { - _, r := t.routeTree.MostSpecificContains(ip) + _, r := t.routeTree.Load().MostSpecificContains(ip) return r } @@ -208,6 +250,46 @@ func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) { return nil, fmt.Errorf("TODO: multiqueue not implemented for freebsd") } +func (t *tun) addRoutes(logErrors bool) error { + routes := *t.Routes.Load() + for _, r := range routes { + if r.Via == nil || !r.Install { + // We don't allow route MTUs so only install routes with a via + continue + } + + cmd := exec.Command("/sbin/route", "-n", "add", "-net", r.Cidr.String(), "-interface", t.Device) + t.l.Debug("command: ", cmd.String()) + if err := cmd.Run(); err != nil { + retErr := util.NewContextualError("failed to run 'route add' for unsafe_route", map[string]interface{}{"route": r}, err) + if logErrors { + retErr.Log(t.l) + } else { + return retErr + } + } + } + + return nil +} + +func (t *tun) removeRoutes(routes []Route) error { + for _, r := range routes { + if !r.Install { + continue + } + + cmd := exec.Command("/sbin/route", "-n", "delete", "-net", r.Cidr.String(), "-interface", t.Device) + t.l.Debug("command: ", cmd.String()) + if err := cmd.Run(); err != nil { + t.l.WithError(err).WithField("route", r).Error("Failed to remove route") + } else { + t.l.WithField("route", r).Info("Removed route") + } + } + return nil +} + func (t *tun) deviceBytes() (o [16]byte) { for i, c := range t.Device { o[i] = byte(c) diff --git a/overlay/tun_ios.go b/overlay/tun_ios.go index ce65b33cd..ba15d665e 100644 --- a/overlay/tun_ios.go +++ b/overlay/tun_ios.go @@ -10,43 +10,78 @@ import ( "net" "os" "sync" + "sync/atomic" "syscall" "github.com/sirupsen/logrus" "github.com/slackhq/nebula/cidr" + "github.com/slackhq/nebula/config" "github.com/slackhq/nebula/iputil" + "github.com/slackhq/nebula/util" ) type tun struct { io.ReadWriteCloser cidr *net.IPNet - routeTree *cidr.Tree4[iputil.VpnIp] + Routes atomic.Pointer[[]Route] + routeTree atomic.Pointer[cidr.Tree4[iputil.VpnIp]] + l *logrus.Logger } -func newTun(_ *logrus.Logger, _ string, _ *net.IPNet, _ int, _ []Route, _ int, _ bool, _ bool) (*tun, error) { +func newTun(_ *config.C, _ *logrus.Logger, _ *net.IPNet, _ bool) (*tun, error) { return nil, fmt.Errorf("newTun not supported in iOS") } -func newTunFromFd(l *logrus.Logger, deviceFd int, cidr *net.IPNet, _ int, routes []Route, _ int, _ bool) (*tun, error) { - routeTree, err := makeRouteTree(l, routes, false) +func newTunFromFd(c *config.C, l *logrus.Logger, deviceFd int, cidr *net.IPNet) (*tun, error) { + file := os.NewFile(uintptr(deviceFd), "/dev/tun") + t := &tun{ + cidr: cidr, + ReadWriteCloser: &tunReadCloser{f: file}, + l: l, + } + + err := t.reload(c, true) if err != nil { return nil, err } - file := os.NewFile(uintptr(deviceFd), "/dev/tun") - return &tun{ - cidr: cidr, - ReadWriteCloser: &tunReadCloser{f: file}, - routeTree: routeTree, - }, nil + c.RegisterReloadCallback(func(c *config.C) { + err := t.reload(c, false) + if err != nil { + util.LogWithContextIfNeeded("failed to reload tun device", err, t.l) + } + }) + + return t, nil } func (t *tun) Activate() error { return nil } +func (t *tun) reload(c *config.C, initial bool) error { + change, routes, err := getAllRoutesFromConfig(c, t.cidr, initial) + if err != nil { + return err + } + + if !initial && !change { + return nil + } + + routeTree, err := makeRouteTree(t.l, routes, false) + if err != nil { + return err + } + + // Teach nebula how to handle the routes + t.Routes.Store(&routes) + t.routeTree.Store(routeTree) + return nil +} + func (t *tun) RouteFor(ip iputil.VpnIp) iputil.VpnIp { - _, r := t.routeTree.MostSpecificContains(ip) + _, r := t.routeTree.Load().MostSpecificContains(ip) return r } diff --git a/overlay/tun_linux.go b/overlay/tun_linux.go index a576bf344..1f6580edb 100644 --- a/overlay/tun_linux.go +++ b/overlay/tun_linux.go @@ -15,21 +15,25 @@ import ( "github.com/sirupsen/logrus" "github.com/slackhq/nebula/cidr" + "github.com/slackhq/nebula/config" "github.com/slackhq/nebula/iputil" + "github.com/slackhq/nebula/util" "github.com/vishvananda/netlink" "golang.org/x/sys/unix" ) type tun struct { io.ReadWriteCloser - fd int - Device string - cidr *net.IPNet - MaxMTU int - DefaultMTU int - TXQueueLen int - - Routes []Route + fd int + Device string + cidr *net.IPNet + MaxMTU int + DefaultMTU int + TXQueueLen int + deviceIndex int + ioctlFd uintptr + + Routes atomic.Pointer[[]Route] routeTree atomic.Pointer[cidr.Tree4[iputil.VpnIp]] routeChan chan struct{} useSystemRoutes bool @@ -61,30 +65,20 @@ type ifreqQLEN struct { pad [8]byte } -func newTunFromFd(l *logrus.Logger, deviceFd int, cidr *net.IPNet, defaultMTU int, routes []Route, txQueueLen int, useSystemRoutes bool) (*tun, error) { - routeTree, err := makeRouteTree(l, routes, true) +func newTunFromFd(c *config.C, l *logrus.Logger, deviceFd int, cidr *net.IPNet) (*tun, error) { + file := os.NewFile(uintptr(deviceFd), "/dev/net/tun") + + t, err := newTunGeneric(c, l, file, cidr) if err != nil { return nil, err } - file := os.NewFile(uintptr(deviceFd), "/dev/net/tun") + t.Device = "tun0" - t := &tun{ - ReadWriteCloser: file, - fd: int(file.Fd()), - Device: "tun0", - cidr: cidr, - DefaultMTU: defaultMTU, - TXQueueLen: txQueueLen, - Routes: routes, - useSystemRoutes: useSystemRoutes, - l: l, - } - t.routeTree.Store(routeTree) return t, nil } -func newTun(l *logrus.Logger, deviceName string, cidr *net.IPNet, defaultMTU int, routes []Route, txQueueLen int, multiqueue bool, useSystemRoutes bool) (*tun, error) { +func newTun(c *config.C, l *logrus.Logger, cidr *net.IPNet, multiqueue bool) (*tun, error) { fd, err := unix.Open("/dev/net/tun", os.O_RDWR, 0) if err != nil { return nil, err @@ -95,46 +89,113 @@ func newTun(l *logrus.Logger, deviceName string, cidr *net.IPNet, defaultMTU int if multiqueue { req.Flags |= unix.IFF_MULTI_QUEUE } - copy(req.Name[:], deviceName) + copy(req.Name[:], c.GetString("tun.dev", "")) if err = ioctl(uintptr(fd), uintptr(unix.TUNSETIFF), uintptr(unsafe.Pointer(&req))); err != nil { return nil, err } name := strings.Trim(string(req.Name[:]), "\x00") file := os.NewFile(uintptr(fd), "/dev/net/tun") - - maxMTU := defaultMTU - for _, r := range routes { - if r.MTU == 0 { - r.MTU = defaultMTU - } - - if r.MTU > maxMTU { - maxMTU = r.MTU - } - } - - routeTree, err := makeRouteTree(l, routes, true) + t, err := newTunGeneric(c, l, file, cidr) if err != nil { return nil, err } + t.Device = name + + return t, nil +} + +func newTunGeneric(c *config.C, l *logrus.Logger, file *os.File, cidr *net.IPNet) (*tun, error) { t := &tun{ ReadWriteCloser: file, fd: int(file.Fd()), - Device: name, cidr: cidr, - MaxMTU: maxMTU, - DefaultMTU: defaultMTU, - TXQueueLen: txQueueLen, - Routes: routes, - useSystemRoutes: useSystemRoutes, + TXQueueLen: c.GetInt("tun.tx_queue", 500), + useSystemRoutes: c.GetBool("tun.use_system_route_table", false), l: l, } - t.routeTree.Store(routeTree) + + err := t.reload(c, true) + if err != nil { + return nil, err + } + + c.RegisterReloadCallback(func(c *config.C) { + err := t.reload(c, false) + if err != nil { + util.LogWithContextIfNeeded("failed to reload tun device", err, t.l) + } + }) + return t, nil } +func (t *tun) reload(c *config.C, initial bool) error { + routeChange, routes, err := getAllRoutesFromConfig(c, t.cidr, initial) + if err != nil { + return err + } + + if !initial && !routeChange && !c.HasChanged("tun.mtu") { + return nil + } + + routeTree, err := makeRouteTree(t.l, routes, true) + if err != nil { + return err + } + + oldDefaultMTU := t.DefaultMTU + oldMaxMTU := t.MaxMTU + newDefaultMTU := c.GetInt("tun.mtu", DefaultMTU) + newMaxMTU := newDefaultMTU + for i, r := range routes { + if r.MTU == 0 { + routes[i].MTU = newDefaultMTU + } + + if r.MTU > t.MaxMTU { + newMaxMTU = r.MTU + } + } + + t.MaxMTU = newMaxMTU + t.DefaultMTU = newDefaultMTU + + // Teach nebula how to handle the routes before establishing them in the system table + oldRoutes := t.Routes.Swap(&routes) + t.routeTree.Store(routeTree) + + if !initial { + if oldMaxMTU != newMaxMTU { + t.setMTU() + t.l.Infof("Set max MTU to %v was %v", t.MaxMTU, oldMaxMTU) + } + + if oldDefaultMTU != newDefaultMTU { + err := t.setDefaultRoute() + if err != nil { + t.l.Warn(err) + } else { + t.l.Infof("Set default MTU to %v was %v", t.DefaultMTU, oldDefaultMTU) + } + } + + // Remove first, if the system removes a wanted route hopefully it will be re-added next + t.removeRoutes(findRemovedRoutes(routes, *oldRoutes)) + + // Ensure any routes we actually want are installed + err = t.addRoutes(true) + if err != nil { + // This should never be called since addRoutes should log its own errors in a reload condition + util.LogWithContextIfNeeded("Failed to refresh routes", err, t.l) + } + } + + return nil +} + func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) { fd, err := unix.Open("/dev/net/tun", os.O_RDWR, 0) if err != nil { @@ -208,7 +269,7 @@ func (t *tun) Activate() error { if err != nil { return err } - fd := uintptr(s) + t.ioctlFd = uintptr(s) ifra := ifreqAddr{ Name: devName, @@ -219,52 +280,76 @@ func (t *tun) Activate() error { } // Set the device ip address - if err = ioctl(fd, unix.SIOCSIFADDR, uintptr(unsafe.Pointer(&ifra))); err != nil { + if err = ioctl(t.ioctlFd, unix.SIOCSIFADDR, uintptr(unsafe.Pointer(&ifra))); err != nil { return fmt.Errorf("failed to set tun address: %s", err) } // Set the device network ifra.Addr.Addr = mask - if err = ioctl(fd, unix.SIOCSIFNETMASK, uintptr(unsafe.Pointer(&ifra))); err != nil { + if err = ioctl(t.ioctlFd, unix.SIOCSIFNETMASK, uintptr(unsafe.Pointer(&ifra))); err != nil { return fmt.Errorf("failed to set tun netmask: %s", err) } // Set the device name ifrf := ifReq{Name: devName} - if err = ioctl(fd, unix.SIOCGIFFLAGS, uintptr(unsafe.Pointer(&ifrf))); err != nil { + if err = ioctl(t.ioctlFd, unix.SIOCGIFFLAGS, uintptr(unsafe.Pointer(&ifrf))); err != nil { return fmt.Errorf("failed to set tun device name: %s", err) } - // Set the MTU on the device - ifm := ifreqMTU{Name: devName, MTU: int32(t.MaxMTU)} - if err = ioctl(fd, unix.SIOCSIFMTU, uintptr(unsafe.Pointer(&ifm))); err != nil { - // This is currently a non fatal condition because the route table must have the MTU set appropriately as well - t.l.WithError(err).Error("Failed to set tun mtu") - } + // Setup our default MTU + t.setMTU() // Set the transmit queue length ifrq := ifreqQLEN{Name: devName, Value: int32(t.TXQueueLen)} - if err = ioctl(fd, unix.SIOCSIFTXQLEN, uintptr(unsafe.Pointer(&ifrq))); err != nil { + if err = ioctl(t.ioctlFd, unix.SIOCSIFTXQLEN, uintptr(unsafe.Pointer(&ifrq))); err != nil { // If we can't set the queue length nebula will still work but it may lead to packet loss t.l.WithError(err).Error("Failed to set tun tx queue length") } // Bring up the interface ifrf.Flags = ifrf.Flags | unix.IFF_UP - if err = ioctl(fd, unix.SIOCSIFFLAGS, uintptr(unsafe.Pointer(&ifrf))); err != nil { + if err = ioctl(t.ioctlFd, unix.SIOCSIFFLAGS, uintptr(unsafe.Pointer(&ifrf))); err != nil { return fmt.Errorf("failed to bring the tun device up: %s", err) } - // Set the routes link, err := netlink.LinkByName(t.Device) if err != nil { return fmt.Errorf("failed to get tun device link: %s", err) } + t.deviceIndex = link.Attrs().Index + + if err = t.setDefaultRoute(); err != nil { + return err + } + + // Set the routes + if err = t.addRoutes(false); err != nil { + return err + } + // Run the interface + ifrf.Flags = ifrf.Flags | unix.IFF_UP | unix.IFF_RUNNING + if err = ioctl(t.ioctlFd, unix.SIOCSIFFLAGS, uintptr(unsafe.Pointer(&ifrf))); err != nil { + return fmt.Errorf("failed to run tun device: %s", err) + } + + return nil +} + +func (t *tun) setMTU() { + // Set the MTU on the device + ifm := ifreqMTU{Name: t.deviceBytes(), MTU: int32(t.MaxMTU)} + if err := ioctl(t.ioctlFd, unix.SIOCSIFMTU, uintptr(unsafe.Pointer(&ifm))); err != nil { + // This is currently a non fatal condition because the route table must have the MTU set appropriately as well + t.l.WithError(err).Error("Failed to set tun mtu") + } +} + +func (t *tun) setDefaultRoute() error { // Default route dr := &net.IPNet{IP: t.cidr.IP.Mask(t.cidr.Mask), Mask: t.cidr.Mask} nr := netlink.Route{ - LinkIndex: link.Attrs().Index, + LinkIndex: t.deviceIndex, Dst: dr, MTU: t.DefaultMTU, AdvMSS: t.advMSS(Route{}), @@ -274,19 +359,24 @@ func (t *tun) Activate() error { Table: unix.RT_TABLE_MAIN, Type: unix.RTN_UNICAST, } - err = netlink.RouteReplace(&nr) + err := netlink.RouteReplace(&nr) if err != nil { return fmt.Errorf("failed to set mtu %v on the default route %v; %v", t.DefaultMTU, dr, err) } + return nil +} + +func (t *tun) addRoutes(logErrors bool) error { // Path routes - for _, r := range t.Routes { + routes := *t.Routes.Load() + for _, r := range routes { if !r.Install { continue } nr := netlink.Route{ - LinkIndex: link.Attrs().Index, + LinkIndex: t.deviceIndex, Dst: r.Cidr, MTU: r.MTU, AdvMSS: t.advMSS(r), @@ -297,21 +387,49 @@ func (t *tun) Activate() error { nr.Priority = r.Metric } - err = netlink.RouteAdd(&nr) + err := netlink.RouteReplace(&nr) if err != nil { - return fmt.Errorf("failed to set mtu %v on route %v; %v", r.MTU, r.Cidr, err) + retErr := util.NewContextualError("Failed to add route", map[string]interface{}{"route": r}, err) + if logErrors { + retErr.Log(t.l) + } else { + return retErr + } + } else { + t.l.WithField("route", r).Info("Added route") } } - // Run the interface - ifrf.Flags = ifrf.Flags | unix.IFF_UP | unix.IFF_RUNNING - if err = ioctl(fd, unix.SIOCSIFFLAGS, uintptr(unsafe.Pointer(&ifrf))); err != nil { - return fmt.Errorf("failed to run tun device: %s", err) - } - return nil } +func (t *tun) removeRoutes(routes []Route) { + for _, r := range routes { + if !r.Install { + continue + } + + nr := netlink.Route{ + LinkIndex: t.deviceIndex, + Dst: r.Cidr, + MTU: r.MTU, + AdvMSS: t.advMSS(r), + Scope: unix.RT_SCOPE_LINK, + } + + if r.Metric > 0 { + nr.Priority = r.Metric + } + + err := netlink.RouteDel(&nr) + if err != nil { + t.l.WithError(err).WithField("route", r).Error("Failed to remove route") + } else { + t.l.WithField("route", r).Info("Removed route") + } + } +} + func (t *tun) Cidr() *net.IPNet { return t.cidr } @@ -410,5 +528,9 @@ func (t *tun) Close() error { t.ReadWriteCloser.Close() } + if t.ioctlFd > 0 { + os.NewFile(t.ioctlFd, "ioctlFd").Close() + } + return nil } diff --git a/overlay/tun_netbsd.go b/overlay/tun_netbsd.go index b1135fe7d..cc0216fe9 100644 --- a/overlay/tun_netbsd.go +++ b/overlay/tun_netbsd.go @@ -11,12 +11,15 @@ import ( "os/exec" "regexp" "strconv" + "sync/atomic" "syscall" "unsafe" "github.com/sirupsen/logrus" "github.com/slackhq/nebula/cidr" + "github.com/slackhq/nebula/config" "github.com/slackhq/nebula/iputil" + "github.com/slackhq/nebula/util" ) type ifreqDestroy struct { @@ -28,8 +31,8 @@ type tun struct { Device string cidr *net.IPNet MTU int - Routes []Route - routeTree *cidr.Tree4[iputil.VpnIp] + Routes atomic.Pointer[[]Route] + routeTree atomic.Pointer[cidr.Tree4[iputil.VpnIp]] l *logrus.Logger io.ReadWriteCloser @@ -56,43 +59,50 @@ func (t *tun) Close() error { return nil } -func newTunFromFd(_ *logrus.Logger, _ int, _ *net.IPNet, _ int, _ []Route, _ int, _ bool) (*tun, error) { +func newTunFromFd(_ *config.C, _ *logrus.Logger, _ int, _ *net.IPNet) (*tun, error) { return nil, fmt.Errorf("newTunFromFd not supported in NetBSD") } var deviceNameRE = regexp.MustCompile(`^tun[0-9]+$`) -func newTun(l *logrus.Logger, deviceName string, cidr *net.IPNet, defaultMTU int, routes []Route, _ int, _ bool, _ bool) (*tun, error) { +func newTun(c *config.C, l *logrus.Logger, cidr *net.IPNet, _ bool) (*tun, error) { // Try to open tun device var file *os.File var err error + deviceName := c.GetString("tun.dev", "") if deviceName == "" { return nil, fmt.Errorf("a device name in the format of /dev/tunN must be specified") } if !deviceNameRE.MatchString(deviceName) { return nil, fmt.Errorf("a device name in the format of /dev/tunN must be specified") } - file, err = os.OpenFile("/dev/"+deviceName, os.O_RDWR, 0) + file, err = os.OpenFile("/dev/"+deviceName, os.O_RDWR, 0) if err != nil { return nil, err } - routeTree, err := makeRouteTree(l, routes, false) + t := &tun{ + ReadWriteCloser: file, + Device: deviceName, + cidr: cidr, + MTU: c.GetInt("tun.mtu", DefaultMTU), + l: l, + } + err = t.reload(c, true) if err != nil { return nil, err } - return &tun{ - ReadWriteCloser: file, - Device: deviceName, - cidr: cidr, - MTU: defaultMTU, - Routes: routes, - routeTree: routeTree, - l: l, - }, nil + c.RegisterReloadCallback(func(c *config.C) { + err := t.reload(c, false) + if err != nil { + util.LogWithContextIfNeeded("failed to reload tun device", err, t.l) + } + }) + + return t, nil } func (t *tun) Activate() error { @@ -116,17 +126,42 @@ func (t *tun) Activate() error { if err = cmd.Run(); err != nil { return fmt.Errorf("failed to run 'ifconfig': %s", err) } + // Unsafe path routes - for _, r := range t.Routes { - if r.Via == nil || !r.Install { - // We don't allow route MTUs so only install routes with a via - continue + return t.addRoutes(false) +} + +func (t *tun) reload(c *config.C, initial bool) error { + change, routes, err := getAllRoutesFromConfig(c, t.cidr, initial) + if err != nil { + return err + } + + if !initial && !change { + return nil + } + + routeTree, err := makeRouteTree(t.l, routes, false) + if err != nil { + return err + } + + // Teach nebula how to handle the routes before establishing them in the system table + oldRoutes := t.Routes.Swap(&routes) + t.routeTree.Store(routeTree) + + if !initial { + // Remove first, if the system removes a wanted route hopefully it will be re-added next + err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes)) + if err != nil { + util.LogWithContextIfNeeded("Failed to remove routes", err, t.l) } - cmd = exec.Command("/sbin/route", "-n", "add", "-net", r.Cidr.String(), t.cidr.IP.String()) - t.l.Debug("command: ", cmd.String()) - if err = cmd.Run(); err != nil { - return fmt.Errorf("failed to run 'route add' for unsafe_route %s: %s", r.Cidr.String(), err) + // Ensure any routes we actually want are installed + err = t.addRoutes(true) + if err != nil { + // Catch any stray logs + util.LogWithContextIfNeeded("Failed to add routes", err, t.l) } } @@ -134,7 +169,7 @@ func (t *tun) Activate() error { } func (t *tun) RouteFor(ip iputil.VpnIp) iputil.VpnIp { - _, r := t.routeTree.MostSpecificContains(ip) + _, r := t.routeTree.Load().MostSpecificContains(ip) return r } @@ -150,6 +185,46 @@ func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) { return nil, fmt.Errorf("TODO: multiqueue not implemented for netbsd") } +func (t *tun) addRoutes(logErrors bool) error { + routes := *t.Routes.Load() + for _, r := range routes { + if r.Via == nil || !r.Install { + // We don't allow route MTUs so only install routes with a via + continue + } + + cmd := exec.Command("/sbin/route", "-n", "add", "-net", r.Cidr.String(), t.cidr.IP.String()) + t.l.Debug("command: ", cmd.String()) + if err := cmd.Run(); err != nil { + retErr := util.NewContextualError("failed to run 'route add' for unsafe_route", map[string]interface{}{"route": r}, err) + if logErrors { + retErr.Log(t.l) + } else { + return retErr + } + } + } + + return nil +} + +func (t *tun) removeRoutes(routes []Route) error { + for _, r := range routes { + if !r.Install { + continue + } + + cmd := exec.Command("/sbin/route", "-n", "delete", "-net", r.Cidr.String(), t.cidr.IP.String()) + t.l.Debug("command: ", cmd.String()) + if err := cmd.Run(); err != nil { + t.l.WithError(err).WithField("route", r).Error("Failed to remove route") + } else { + t.l.WithField("route", r).Info("Removed route") + } + } + return nil +} + func (t *tun) deviceBytes() (o [16]byte) { for i, c := range t.Device { o[i] = byte(c) diff --git a/overlay/tun_openbsd.go b/overlay/tun_openbsd.go index 45c06dcf4..53f57b137 100644 --- a/overlay/tun_openbsd.go +++ b/overlay/tun_openbsd.go @@ -11,19 +11,22 @@ import ( "os/exec" "regexp" "strconv" + "sync/atomic" "syscall" "github.com/sirupsen/logrus" "github.com/slackhq/nebula/cidr" + "github.com/slackhq/nebula/config" "github.com/slackhq/nebula/iputil" + "github.com/slackhq/nebula/util" ) type tun struct { Device string cidr *net.IPNet MTU int - Routes []Route - routeTree *cidr.Tree4[iputil.VpnIp] + Routes atomic.Pointer[[]Route] + routeTree atomic.Pointer[cidr.Tree4[iputil.VpnIp]] l *logrus.Logger io.ReadWriteCloser @@ -40,13 +43,14 @@ func (t *tun) Close() error { return nil } -func newTunFromFd(_ *logrus.Logger, _ int, _ *net.IPNet, _ int, _ []Route, _ int, _ bool) (*tun, error) { +func newTunFromFd(_ *config.C, _ *logrus.Logger, _ int, _ *net.IPNet) (*tun, error) { return nil, fmt.Errorf("newTunFromFd not supported in OpenBSD") } var deviceNameRE = regexp.MustCompile(`^tun[0-9]+$`) -func newTun(l *logrus.Logger, deviceName string, cidr *net.IPNet, defaultMTU int, routes []Route, _ int, _ bool, _ bool) (*tun, error) { +func newTun(c *config.C, l *logrus.Logger, cidr *net.IPNet, _ bool) (*tun, error) { + deviceName := c.GetString("tun.dev", "") if deviceName == "" { return nil, fmt.Errorf("a device name in the format of tunN must be specified") } @@ -60,20 +64,64 @@ func newTun(l *logrus.Logger, deviceName string, cidr *net.IPNet, defaultMTU int return nil, err } - routeTree, err := makeRouteTree(l, routes, false) - if err != nil { - return nil, err - } - - return &tun{ + t := &tun{ ReadWriteCloser: file, Device: deviceName, cidr: cidr, - MTU: defaultMTU, - Routes: routes, - routeTree: routeTree, + MTU: c.GetInt("tun.mtu", DefaultMTU), l: l, - }, nil + } + + err = t.reload(c, true) + if err != nil { + return nil, err + } + + c.RegisterReloadCallback(func(c *config.C) { + err := t.reload(c, false) + if err != nil { + util.LogWithContextIfNeeded("failed to reload tun device", err, t.l) + } + }) + + return t, nil +} + +func (t *tun) reload(c *config.C, initial bool) error { + change, routes, err := getAllRoutesFromConfig(c, t.cidr, initial) + if err != nil { + return err + } + + if !initial && !change { + return nil + } + + routeTree, err := makeRouteTree(t.l, routes, false) + if err != nil { + return err + } + + // Teach nebula how to handle the routes before establishing them in the system table + oldRoutes := t.Routes.Swap(&routes) + t.routeTree.Store(routeTree) + + if !initial { + // Remove first, if the system removes a wanted route hopefully it will be re-added next + err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes)) + if err != nil { + util.LogWithContextIfNeeded("Failed to remove routes", err, t.l) + } + + // Ensure any routes we actually want are installed + err = t.addRoutes(true) + if err != nil { + // Catch any stray logs + util.LogWithContextIfNeeded("Failed to add routes", err, t.l) + } + } + + return nil } func (t *tun) Activate() error { @@ -98,25 +146,52 @@ func (t *tun) Activate() error { } // Unsafe path routes - for _, r := range t.Routes { + return t.addRoutes(false) +} + +func (t *tun) RouteFor(ip iputil.VpnIp) iputil.VpnIp { + _, r := t.routeTree.Load().MostSpecificContains(ip) + return r +} + +func (t *tun) addRoutes(logErrors bool) error { + routes := *t.Routes.Load() + for _, r := range routes { if r.Via == nil || !r.Install { // We don't allow route MTUs so only install routes with a via continue } - cmd = exec.Command("/sbin/route", "-n", "add", "-inet", r.Cidr.String(), t.cidr.IP.String()) + cmd := exec.Command("/sbin/route", "-n", "add", "-inet", r.Cidr.String(), t.cidr.IP.String()) t.l.Debug("command: ", cmd.String()) - if err = cmd.Run(); err != nil { - return fmt.Errorf("failed to run 'route add' for unsafe_route %s: %s", r.Cidr.String(), err) + if err := cmd.Run(); err != nil { + retErr := util.NewContextualError("failed to run 'route add' for unsafe_route", map[string]interface{}{"route": r}, err) + if logErrors { + retErr.Log(t.l) + } else { + return retErr + } } } return nil } -func (t *tun) RouteFor(ip iputil.VpnIp) iputil.VpnIp { - _, r := t.routeTree.MostSpecificContains(ip) - return r +func (t *tun) removeRoutes(routes []Route) error { + for _, r := range routes { + if !r.Install { + continue + } + + cmd := exec.Command("/sbin/route", "-n", "delete", "-inet", r.Cidr.String(), t.cidr.IP.String()) + t.l.Debug("command: ", cmd.String()) + if err := cmd.Run(); err != nil { + t.l.WithError(err).WithField("route", r).Error("Failed to remove route") + } else { + t.l.WithField("route", r).Info("Removed route") + } + } + return nil } func (t *tun) Cidr() *net.IPNet { diff --git a/overlay/tun_tester.go b/overlay/tun_tester.go index 964315a05..383398322 100644 --- a/overlay/tun_tester.go +++ b/overlay/tun_tester.go @@ -12,6 +12,7 @@ import ( "github.com/sirupsen/logrus" "github.com/slackhq/nebula/cidr" + "github.com/slackhq/nebula/config" "github.com/slackhq/nebula/iputil" ) @@ -27,14 +28,18 @@ type TestTun struct { TxPackets chan []byte // Packets transmitted outside by nebula } -func newTun(l *logrus.Logger, deviceName string, cidr *net.IPNet, _ int, routes []Route, _ int, _ bool, _ bool) (*TestTun, error) { +func newTun(c *config.C, l *logrus.Logger, cidr *net.IPNet, _ bool) (*TestTun, error) { + _, routes, err := getAllRoutesFromConfig(c, cidr, true) + if err != nil { + return nil, err + } routeTree, err := makeRouteTree(l, routes, false) if err != nil { return nil, err } return &TestTun{ - Device: deviceName, + Device: c.GetString("tun.dev", ""), cidr: cidr, Routes: routes, routeTree: routeTree, @@ -44,7 +49,7 @@ func newTun(l *logrus.Logger, deviceName string, cidr *net.IPNet, _ int, routes }, nil } -func newTunFromFd(_ *logrus.Logger, _ int, _ *net.IPNet, _ int, _ []Route, _ int, _ bool) (*TestTun, error) { +func newTunFromFd(_ *config.C, _ *logrus.Logger, _ int, _ *net.IPNet) (*TestTun, error) { return nil, fmt.Errorf("newTunFromFd not supported") } diff --git a/overlay/tun_water_windows.go b/overlay/tun_water_windows.go index e27cff20d..a1acd2b25 100644 --- a/overlay/tun_water_windows.go +++ b/overlay/tun_water_windows.go @@ -6,10 +6,13 @@ import ( "net" "os/exec" "strconv" + "sync/atomic" "github.com/sirupsen/logrus" "github.com/slackhq/nebula/cidr" + "github.com/slackhq/nebula/config" "github.com/slackhq/nebula/iputil" + "github.com/slackhq/nebula/util" "github.com/songgao/water" ) @@ -17,25 +20,34 @@ type waterTun struct { Device string cidr *net.IPNet MTU int - Routes []Route - routeTree *cidr.Tree4[iputil.VpnIp] - + Routes atomic.Pointer[[]Route] + routeTree atomic.Pointer[cidr.Tree4[iputil.VpnIp]] + l *logrus.Logger + f *net.Interface *water.Interface } -func newWaterTun(l *logrus.Logger, cidr *net.IPNet, defaultMTU int, routes []Route) (*waterTun, error) { - routeTree, err := makeRouteTree(l, routes, false) +func newWaterTun(c *config.C, l *logrus.Logger, cidr *net.IPNet, _ bool) (*waterTun, error) { + // NOTE: You cannot set the deviceName under Windows, so you must check tun.Device after calling .Activate() + t := &waterTun{ + cidr: cidr, + MTU: c.GetInt("tun.mtu", DefaultMTU), + l: l, + } + + err := t.reload(c, true) if err != nil { return nil, err } - // NOTE: You cannot set the deviceName under Windows, so you must check tun.Device after calling .Activate() - return &waterTun{ - cidr: cidr, - MTU: defaultMTU, - Routes: routes, - routeTree: routeTree, - }, nil + c.RegisterReloadCallback(func(c *config.C) { + err := t.reload(c, false) + if err != nil { + util.LogWithContextIfNeeded("failed to reload tun device", err, t.l) + } + }) + + return t, nil } func (t *waterTun) Activate() error { @@ -74,30 +86,104 @@ func (t *waterTun) Activate() error { return fmt.Errorf("failed to run 'netsh' to set MTU: %s", err) } - iface, err := net.InterfaceByName(t.Device) + t.f, err = net.InterfaceByName(t.Device) if err != nil { return fmt.Errorf("failed to find interface named %s: %v", t.Device, err) } - for _, r := range t.Routes { + err = t.addRoutes(false) + if err != nil { + return err + } + + return nil +} + +func (t *waterTun) reload(c *config.C, initial bool) error { + change, routes, err := getAllRoutesFromConfig(c, t.cidr, initial) + if err != nil { + return err + } + + if !initial && !change { + return nil + } + + routeTree, err := makeRouteTree(t.l, routes, false) + if err != nil { + return err + } + + // Teach nebula how to handle the routes before establishing them in the system table + oldRoutes := t.Routes.Swap(&routes) + t.routeTree.Store(routeTree) + + if !initial { + // Remove first, if the system removes a wanted route hopefully it will be re-added next + t.removeRoutes(findRemovedRoutes(routes, *oldRoutes)) + + // Ensure any routes we actually want are installed + err = t.addRoutes(true) + if err != nil { + // Catch any stray logs + util.LogWithContextIfNeeded("Failed to set routes", err, t.l) + } else { + for _, r := range findRemovedRoutes(routes, *oldRoutes) { + t.l.WithField("route", r).Info("Removed route") + } + } + } + + return nil +} + +func (t *waterTun) addRoutes(logErrors bool) error { + // Path routes + routes := *t.Routes.Load() + for _, r := range routes { if r.Via == nil || !r.Install { // We don't allow route MTUs so only install routes with a via continue } - err = exec.Command( - "C:\\Windows\\System32\\route.exe", "add", r.Cidr.String(), r.Via.String(), "IF", strconv.Itoa(iface.Index), "METRIC", strconv.Itoa(r.Metric), + err := exec.Command( + "C:\\Windows\\System32\\route.exe", "add", r.Cidr.String(), r.Via.String(), "IF", strconv.Itoa(t.f.Index), "METRIC", strconv.Itoa(r.Metric), ).Run() + if err != nil { - return fmt.Errorf("failed to add the unsafe_route %s: %v", r.Cidr.String(), err) + retErr := util.NewContextualError("Failed to add route", map[string]interface{}{"route": r}, err) + if logErrors { + retErr.Log(t.l) + } else { + return retErr + } + } else { + t.l.WithField("route", r).Info("Added route") } } return nil } +func (t *waterTun) removeRoutes(routes []Route) { + for _, r := range routes { + if !r.Install { + continue + } + + err := exec.Command( + "C:\\Windows\\System32\\route.exe", "delete", r.Cidr.String(), r.Via.String(), "IF", strconv.Itoa(t.f.Index), "METRIC", strconv.Itoa(r.Metric), + ).Run() + if err != nil { + t.l.WithError(err).WithField("route", r).Error("Failed to remove route") + } else { + t.l.WithField("route", r).Info("Removed route") + } + } +} + func (t *waterTun) RouteFor(ip iputil.VpnIp) iputil.VpnIp { - _, r := t.routeTree.MostSpecificContains(ip) + _, r := t.routeTree.Load().MostSpecificContains(ip) return r } diff --git a/overlay/tun_windows.go b/overlay/tun_windows.go index 57d90cbdf..f85ee9cee 100644 --- a/overlay/tun_windows.go +++ b/overlay/tun_windows.go @@ -12,13 +12,14 @@ import ( "syscall" "github.com/sirupsen/logrus" + "github.com/slackhq/nebula/config" ) -func newTunFromFd(_ *logrus.Logger, _ int, _ *net.IPNet, _ int, _ []Route, _ int, _ bool) (Device, error) { +func newTunFromFd(_ *config.C, _ *logrus.Logger, _ int, _ *net.IPNet) (Device, error) { return nil, fmt.Errorf("newTunFromFd not supported in Windows") } -func newTun(l *logrus.Logger, deviceName string, cidr *net.IPNet, defaultMTU int, routes []Route, _ int, _ bool, _ bool) (Device, error) { +func newTun(c *config.C, l *logrus.Logger, cidr *net.IPNet, multiqueue bool) (Device, error) { useWintun := true if err := checkWinTunExists(); err != nil { l.WithError(err).Warn("Check Wintun driver failed, fallback to wintap driver") @@ -26,14 +27,14 @@ func newTun(l *logrus.Logger, deviceName string, cidr *net.IPNet, defaultMTU int } if useWintun { - device, err := newWinTun(l, deviceName, cidr, defaultMTU, routes) + device, err := newWinTun(c, l, cidr, multiqueue) if err != nil { return nil, fmt.Errorf("create Wintun interface failed, %w", err) } return device, nil } - device, err := newWaterTun(l, cidr, defaultMTU, routes) + device, err := newWaterTun(c, l, cidr, multiqueue) if err != nil { return nil, fmt.Errorf("create wintap driver failed, %w", err) } diff --git a/overlay/tun_wintun_windows.go b/overlay/tun_wintun_windows.go index 9647024f7..197e3a717 100644 --- a/overlay/tun_wintun_windows.go +++ b/overlay/tun_wintun_windows.go @@ -6,11 +6,14 @@ import ( "io" "net" "net/netip" + "sync/atomic" "unsafe" "github.com/sirupsen/logrus" "github.com/slackhq/nebula/cidr" + "github.com/slackhq/nebula/config" "github.com/slackhq/nebula/iputil" + "github.com/slackhq/nebula/util" "github.com/slackhq/nebula/wintun" "golang.org/x/sys/windows" "golang.zx2c4.com/wireguard/windows/tunnel/winipcfg" @@ -23,8 +26,9 @@ type winTun struct { cidr *net.IPNet prefix netip.Prefix MTU int - Routes []Route - routeTree *cidr.Tree4[iputil.VpnIp] + Routes atomic.Pointer[[]Route] + routeTree atomic.Pointer[cidr.Tree4[iputil.VpnIp]] + l *logrus.Logger tun *wintun.NativeTun } @@ -48,83 +52,148 @@ func generateGUIDByDeviceName(name string) (*windows.GUID, error) { return (*windows.GUID)(unsafe.Pointer(&sum[0])), nil } -func newWinTun(l *logrus.Logger, deviceName string, cidr *net.IPNet, defaultMTU int, routes []Route) (*winTun, error) { +func newWinTun(c *config.C, l *logrus.Logger, cidr *net.IPNet, _ bool) (*winTun, error) { + deviceName := c.GetString("tun.dev", "") guid, err := generateGUIDByDeviceName(deviceName) if err != nil { return nil, fmt.Errorf("generate GUID failed: %w", err) } + prefix, err := iputil.ToNetIpPrefix(*cidr) + if err != nil { + return nil, err + } + + t := &winTun{ + Device: deviceName, + cidr: cidr, + prefix: prefix, + MTU: c.GetInt("tun.mtu", DefaultMTU), + l: l, + } + + err = t.reload(c, true) + if err != nil { + return nil, err + } + var tunDevice wintun.Device - tunDevice, err = wintun.CreateTUNWithRequestedGUID(deviceName, guid, defaultMTU) + tunDevice, err = wintun.CreateTUNWithRequestedGUID(deviceName, guid, t.MTU) if err != nil { // Windows 10 has an issue with unclean shutdowns not fully cleaning up the wintun device. // Trying a second time resolves the issue. l.WithError(err).Debug("Failed to create wintun device, retrying") - tunDevice, err = wintun.CreateTUNWithRequestedGUID(deviceName, guid, defaultMTU) + tunDevice, err = wintun.CreateTUNWithRequestedGUID(deviceName, guid, t.MTU) if err != nil { return nil, fmt.Errorf("create TUN device failed: %w", err) } } + t.tun = tunDevice.(*wintun.NativeTun) + + c.RegisterReloadCallback(func(c *config.C) { + err := t.reload(c, false) + if err != nil { + util.LogWithContextIfNeeded("failed to reload tun device", err, t.l) + } + }) + + return t, nil +} - routeTree, err := makeRouteTree(l, routes, false) +func (t *winTun) reload(c *config.C, initial bool) error { + change, routes, err := getAllRoutesFromConfig(c, t.cidr, initial) if err != nil { - return nil, err + return err } - prefix, err := iputil.ToNetIpPrefix(*cidr) + if !initial && !change { + return nil + } + + routeTree, err := makeRouteTree(t.l, routes, false) if err != nil { - return nil, err + return err } - return &winTun{ - Device: deviceName, - cidr: cidr, - prefix: prefix, - MTU: defaultMTU, - Routes: routes, - routeTree: routeTree, + // Teach nebula how to handle the routes before establishing them in the system table + oldRoutes := t.Routes.Swap(&routes) + t.routeTree.Store(routeTree) - tun: tunDevice.(*wintun.NativeTun), - }, nil + if !initial { + // Remove first, if the system removes a wanted route hopefully it will be re-added next + err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes)) + if err != nil { + util.LogWithContextIfNeeded("Failed to remove routes", err, t.l) + } + + // Ensure any routes we actually want are installed + err = t.addRoutes(true) + if err != nil { + // Catch any stray logs + util.LogWithContextIfNeeded("Failed to add routes", err, t.l) + } + } + + return nil } func (t *winTun) Activate() error { luid := winipcfg.LUID(t.tun.LUID()) - if err := luid.SetIPAddresses([]netip.Prefix{t.prefix}); err != nil { + err := luid.SetIPAddresses([]netip.Prefix{t.prefix}) + if err != nil { return fmt.Errorf("failed to set address: %w", err) } + err = t.addRoutes(false) + if err != nil { + return err + } + + return nil +} + +func (t *winTun) addRoutes(logErrors bool) error { + luid := winipcfg.LUID(t.tun.LUID()) + routes := *t.Routes.Load() foundDefault4 := false - routes := make([]*winipcfg.RouteData, 0, len(t.Routes)+1) - for _, r := range t.Routes { + for _, r := range routes { if r.Via == nil || !r.Install { // We don't allow route MTUs so only install routes with a via continue } - if !foundDefault4 { - if ones, bits := r.Cidr.Mask.Size(); ones == 0 && bits != 0 { - foundDefault4 = true - } - } - prefix, err := iputil.ToNetIpPrefix(*r.Cidr) if err != nil { - return err + retErr := util.NewContextualError("Failed to parse cidr to netip prefix, ignoring route", map[string]interface{}{"route": r}, err) + if logErrors { + retErr.Log(t.l) + continue + } else { + return retErr + } } // Add our unsafe route - routes = append(routes, &winipcfg.RouteData{ - Destination: prefix, - NextHop: r.Via.ToNetIpAddr(), - Metric: uint32(r.Metric), - }) - } + err = luid.AddRoute(prefix, r.Via.ToNetIpAddr(), uint32(r.Metric)) + if err != nil { + retErr := util.NewContextualError("Failed to add route", map[string]interface{}{"route": r}, err) + if logErrors { + retErr.Log(t.l) + continue + } else { + return retErr + } + } else { + t.l.WithField("route", r).Info("Added route") + } - if err := luid.AddRoutes(routes); err != nil { - return fmt.Errorf("failed to add routes: %w", err) + if !foundDefault4 { + if ones, bits := r.Cidr.Mask.Size(); ones == 0 && bits != 0 { + foundDefault4 = true + } + } } ipif, err := luid.IPInterface(windows.AF_INET) @@ -141,12 +210,35 @@ func (t *winTun) Activate() error { if err := ipif.Set(); err != nil { return fmt.Errorf("failed to set ip interface: %w", err) } + return nil +} + +func (t *winTun) removeRoutes(routes []Route) error { + luid := winipcfg.LUID(t.tun.LUID()) + + for _, r := range routes { + if !r.Install { + continue + } + prefix, err := iputil.ToNetIpPrefix(*r.Cidr) + if err != nil { + t.l.WithError(err).WithField("route", r).Info("Failed to convert cidr to netip prefix") + continue + } + + err = luid.DeleteRoute(prefix, r.Via.ToNetIpAddr()) + if err != nil { + t.l.WithError(err).WithField("route", r).Error("Failed to remove route") + } else { + t.l.WithField("route", r).Info("Removed route") + } + } return nil } func (t *winTun) RouteFor(ip iputil.VpnIp) iputil.VpnIp { - _, r := t.routeTree.MostSpecificContains(ip) + _, r := t.routeTree.Load().MostSpecificContains(ip) return r } diff --git a/service/service.go b/service/service.go index 66ce8649a..6816be673 100644 --- a/service/service.go +++ b/service/service.go @@ -17,7 +17,7 @@ import ( "github.com/slackhq/nebula/config" "github.com/slackhq/nebula/overlay" "golang.org/x/sync/errgroup" - "gvisor.dev/gvisor/pkg/bufferv2" + "gvisor.dev/gvisor/pkg/buffer" "gvisor.dev/gvisor/pkg/tcpip" "gvisor.dev/gvisor/pkg/tcpip/adapters/gonet" "gvisor.dev/gvisor/pkg/tcpip/header" @@ -81,7 +81,7 @@ func New(config *config.C) (*Service, error) { if tcpipProblem := s.ipstack.CreateNIC(nicID, linkEP); tcpipProblem != nil { return nil, fmt.Errorf("could not create netstack NIC: %v", tcpipProblem) } - ipv4Subnet, _ := tcpip.NewSubnet(tcpip.Address(strings.Repeat("\x00", 4)), tcpip.AddressMask(strings.Repeat("\x00", 4))) + ipv4Subnet, _ := tcpip.NewSubnet(tcpip.AddrFrom4([4]byte{0x00, 0x00, 0x00, 0x00}), tcpip.MaskFrom(strings.Repeat("\x00", 4))) s.ipstack.SetRouteTable([]tcpip.Route{ { Destination: ipv4Subnet, @@ -91,7 +91,7 @@ func New(config *config.C) (*Service, error) { ipNet := device.Cidr() pa := tcpip.ProtocolAddress{ - AddressWithPrefix: tcpip.Address(ipNet.IP).WithPrefix(), + AddressWithPrefix: tcpip.AddrFromSlice(ipNet.IP).WithPrefix(), Protocol: ipv4.ProtocolNumber, } if err := s.ipstack.AddProtocolAddress(nicID, pa, stack.AddressProperties{ @@ -124,7 +124,7 @@ func New(config *config.C) (*Service, error) { return err } packetBuf := stack.NewPacketBuffer(stack.PacketBufferOptions{ - Payload: bufferv2.MakeWithData(bytes.Clone(buf[:n])), + Payload: buffer.MakeWithData(bytes.Clone(buf[:n])), }) linkEP.InjectInbound(header.IPv4ProtocolNumber, packetBuf) @@ -136,7 +136,7 @@ func New(config *config.C) (*Service, error) { eg.Go(func() error { for { packet := linkEP.ReadContext(ctx) - if packet.IsNil() { + if packet == nil { if err := ctx.Err(); err != nil { return err } @@ -166,7 +166,7 @@ func (s *Service) DialContext(ctx context.Context, network, address string) (net fullAddr := tcpip.FullAddress{ NIC: nicID, - Addr: tcpip.Address(addr.IP), + Addr: tcpip.AddrFromSlice(addr.IP), Port: uint16(addr.Port), } diff --git a/ssh.go b/ssh.go index e99205ca2..0f4f65194 100644 --- a/ssh.go +++ b/ssh.go @@ -51,6 +51,11 @@ type sshCreateTunnelFlags struct { Address string } +type sshDeviceInfoFlags struct { + Json bool + Pretty bool +} + func wireSSHReload(l *logrus.Logger, ssh *sshd.SSHServer, c *config.C) { c.RegisterReloadCallback(func(c *config.C) { if c.GetBool("sshd.enabled", false) { @@ -231,7 +236,7 @@ func attachCommands(l *logrus.Logger, c *config.C, ssh *sshd.SSHServer, f *Inter ssh.RegisterCommand(&sshd.Command{ Name: "start-cpu-profile", - ShortDescription: "Starts a cpu profile and write output to the provided file", + ShortDescription: "Starts a cpu profile and write output to the provided file, ex: `cpu-profile.pb.gz`", Callback: sshStartCpuProfile, }) @@ -246,7 +251,7 @@ func attachCommands(l *logrus.Logger, c *config.C, ssh *sshd.SSHServer, f *Inter ssh.RegisterCommand(&sshd.Command{ Name: "save-heap-profile", - ShortDescription: "Saves a heap profile to the provided path", + ShortDescription: "Saves a heap profile to the provided path, ex: `heap-profile.pb.gz`", Callback: sshGetHeapProfile, }) @@ -258,7 +263,7 @@ func attachCommands(l *logrus.Logger, c *config.C, ssh *sshd.SSHServer, f *Inter ssh.RegisterCommand(&sshd.Command{ Name: "save-mutex-profile", - ShortDescription: "Saves a mutex profile to the provided path", + ShortDescription: "Saves a mutex profile to the provided path, ex: `mutex-profile.pb.gz`", Callback: sshGetMutexProfile, }) @@ -286,6 +291,21 @@ func attachCommands(l *logrus.Logger, c *config.C, ssh *sshd.SSHServer, f *Inter }, }) + ssh.RegisterCommand(&sshd.Command{ + Name: "device-info", + ShortDescription: "Prints information about the network device.", + Flags: func() (*flag.FlagSet, interface{}) { + fl := flag.NewFlagSet("", flag.ContinueOnError) + s := sshDeviceInfoFlags{} + fl.BoolVar(&s.Json, "json", false, "outputs as json with more information") + fl.BoolVar(&s.Pretty, "pretty", false, "pretty prints json, assumes -json") + return fl, &s + }, + Callback: func(fs interface{}, a []string, w sshd.StringWriter) error { + return sshDeviceInfo(f, fs, w) + }, + }) + ssh.RegisterCommand(&sshd.Command{ Name: "print-cert", ShortDescription: "Prints the current certificate being used or the certificate for the provided vpn ip", @@ -939,7 +959,34 @@ func sshPrintTunnel(ifce *Interface, fs interface{}, a []string, w sshd.StringWr enc.SetIndent("", " ") } - return enc.Encode(copyHostInfo(hostInfo, ifce.hostMap.preferredRanges)) + return enc.Encode(copyHostInfo(hostInfo, ifce.hostMap.GetPreferredRanges())) +} + +func sshDeviceInfo(ifce *Interface, fs interface{}, w sshd.StringWriter) error { + + data := struct { + Name string `json:"name"` + Cidr string `json:"cidr"` + }{ + Name: ifce.inside.Name(), + Cidr: ifce.inside.Cidr().String(), + } + + flags, ok := fs.(*sshDeviceInfoFlags) + if !ok { + return fmt.Errorf("internal error: expected flags to be sshDeviceInfoFlags but was %+v", fs) + } + + if flags.Json || flags.Pretty { + js := json.NewEncoder(w.GetWriter()) + if flags.Pretty { + js.SetIndent("", " ") + } + + return js.Encode(data) + } else { + return w.WriteLine(fmt.Sprintf("name=%v cidr=%v", data.Name, data.Cidr)) + } } func sshReload(c *config.C, w sshd.StringWriter) error {