Skip to content

Commit

Permalink
Check for fwmark and ip rule support and unify env vars
Browse files Browse the repository at this point in the history
  • Loading branch information
lixmal committed Jan 20, 2025
1 parent 9f4db0a commit c6b72c8
Show file tree
Hide file tree
Showing 8 changed files with 147 additions and 53 deletions.
2 changes: 1 addition & 1 deletion client/iface/configurer/usp.go
Original file line number Diff line number Diff line change
Expand Up @@ -362,7 +362,7 @@ func toWgUserspaceString(wgCfg wgtypes.Config) string {
}

func getFwmark() int {
if runtime.GOOS == "linux" && !nbnet.CustomRoutingDisabled() {
if nbnet.AdvancedRouting() {
return nbnet.NetbirdFwmark
}
return 0
Expand Down
3 changes: 3 additions & 0 deletions client/internal/connect.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ import (
relayClient "github.com/netbirdio/netbird/relay/client"
signal "github.com/netbirdio/netbird/signal/client"
"github.com/netbirdio/netbird/util"
nbnet "github.com/netbirdio/netbird/util/net"
"github.com/netbirdio/netbird/version"
)

Expand Down Expand Up @@ -114,6 +115,8 @@ func (c *ConnectClient) run(mobileDependency MobileDependency, probes *ProbeHold

log.Infof("starting NetBird client version %s on %s/%s", version.NetbirdVersion(), runtime.GOOS, runtime.GOARCH)

nbnet.Init()

backOff := &backoff.ExponentialBackOff{
InitialInterval: time.Second,
RandomizationFactor: 1,
Expand Down
29 changes: 5 additions & 24 deletions client/internal/routemanager/systemops/systemops_linux.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,20 +53,6 @@ type ruleParams struct {
description string
}

// isLegacy determines whether to use the legacy routing setup
func isLegacy() bool {
return os.Getenv("NB_USE_LEGACY_ROUTING") == "true" || nbnet.CustomRoutingDisabled() || nbnet.SkipSocketMark()
}

// setIsLegacy sets the legacy routing setup
func setIsLegacy(b bool) {
if b {
os.Setenv("NB_USE_LEGACY_ROUTING", "true")
} else {
os.Unsetenv("NB_USE_LEGACY_ROUTING")
}
}

func getSetupRules() []ruleParams {
return []ruleParams{
{100, -1, syscall.RT_TABLE_MAIN, netlink.FAMILY_V4, false, 0, "rule with suppress prefixlen v4"},
Expand All @@ -87,7 +73,7 @@ func getSetupRules() []ruleParams {
// This table is where a default route or other specific routes received from the management server are configured,
// enabling VPN connectivity.
func (r *SysOps) SetupRouting(initAddresses []net.IP, stateManager *statemanager.Manager) (_ nbnet.AddHookFunc, _ nbnet.RemoveHookFunc, err error) {
if isLegacy() {
if !nbnet.AdvancedRouting() {
log.Infof("Using legacy routing setup")
return r.setupRefCounter(initAddresses, stateManager)
}
Expand All @@ -103,11 +89,6 @@ func (r *SysOps) SetupRouting(initAddresses []net.IP, stateManager *statemanager
rules := getSetupRules()
for _, rule := range rules {
if err := addRule(rule); err != nil {
if errors.Is(err, syscall.EOPNOTSUPP) {
log.Warnf("Rule operations are not supported, falling back to the legacy routing setup")
setIsLegacy(true)
return r.setupRefCounter(initAddresses, stateManager)
}
return nil, nil, fmt.Errorf("%s: %w", rule.description, err)
}
}
Expand All @@ -130,7 +111,7 @@ func (r *SysOps) SetupRouting(initAddresses []net.IP, stateManager *statemanager
// It systematically removes the three rules and any associated routing table entries to ensure a clean state.
// The function uses error aggregation to report any errors encountered during the cleanup process.
func (r *SysOps) CleanupRouting(stateManager *statemanager.Manager) error {
if isLegacy() {
if !nbnet.AdvancedRouting() {
return r.cleanupRefCounter(stateManager)
}

Expand Down Expand Up @@ -168,7 +149,7 @@ func (r *SysOps) removeFromRouteTable(prefix netip.Prefix, nexthop Nexthop) erro
}

func (r *SysOps) AddVPNRoute(prefix netip.Prefix, intf *net.Interface) error {
if isLegacy() {
if !nbnet.AdvancedRouting() {
return r.genericAddVPNRoute(prefix, intf)
}

Expand All @@ -191,7 +172,7 @@ func (r *SysOps) AddVPNRoute(prefix netip.Prefix, intf *net.Interface) error {
}

func (r *SysOps) RemoveVPNRoute(prefix netip.Prefix, intf *net.Interface) error {
if isLegacy() {
if !nbnet.AdvancedRouting() {
return r.genericRemoveVPNRoute(prefix, intf)
}

Expand Down Expand Up @@ -504,7 +485,7 @@ func getAddressFamily(prefix netip.Prefix) int {
}

func hasSeparateRouting() ([]netip.Prefix, error) {
if isLegacy() {
if !nbnet.AdvancedRouting() {
return GetRoutesFromTable()
}
return nil, ErrRoutingIsSeparate
Expand Down
6 changes: 3 additions & 3 deletions util/grpc/dialer.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,14 @@ import (
"context"
"crypto/tls"
"fmt"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"net"
"os/user"
"runtime"
"time"

"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"

"github.com/cenkalti/backoff/v4"
log "github.com/sirupsen/logrus"
"google.golang.org/grpc"
Expand All @@ -37,7 +38,6 @@ func WithCustomDialer() grpc.DialOption {
}
}

log.Debug("Using nbnet.NewDialer()")
conn, err := nbnet.NewDialer().DialContext(ctx, "tcp", addr)
if err != nil {
log.Errorf("Failed to dial: %s", err)
Expand Down
21 changes: 13 additions & 8 deletions util/net/env.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package net

import (
"os"
"strconv"

log "github.com/sirupsen/logrus"

Expand All @@ -10,20 +11,24 @@ import (

const (
envDisableCustomRouting = "NB_DISABLE_CUSTOM_ROUTING"
envSkipSocketMark = "NB_SKIP_SOCKET_MARK"
)

// CustomRoutingDisabled returns true if custom routing is disabled.
// This will fall back to the operation mode before the exit node functionality was implemented.
// In particular exclusion routes won't be set up and all dialers and listeners will use net.Dial and net.Listen, respectively.
func CustomRoutingDisabled() bool {
if netstack.IsEnabled() {
return true
}
return os.Getenv(envDisableCustomRouting) == "true"
}

func SkipSocketMark() bool {
if skipSocketMark := os.Getenv(envSkipSocketMark); skipSocketMark == "true" {
log.Infof("%s is set to true, skipping SO_MARK", envSkipSocketMark)
return true
var customRoutingDisabled bool
if val := os.Getenv(envDisableCustomRouting); val != "" {
var err error
customRoutingDisabled, err = strconv.ParseBool(val)
if err != nil {
log.Warnf("failed to parse %s: %v", envDisableCustomRouting, err)
}
}
return false

return customRoutingDisabled
}
11 changes: 11 additions & 0 deletions util/net/env_generic.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
//go:build !linux || android

package net

func Init() {
}

func AdvancedRouting() bool {
// non-linux currently doesn't support advanced routing
return false
}
108 changes: 108 additions & 0 deletions util/net/env_linux.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
//go:build linux && !android

package net

import (
"errors"
"os"
"strconv"
"syscall"

log "github.com/sirupsen/logrus"
"github.com/vishvananda/netlink"

"github.com/netbirdio/netbird/client/iface/netstack"
)

const (
// these have the same effect, skip socket env supported for backward compatibility
envSkipSocketMark = "NB_SKIP_SOCKET_MARK"
envUseLegacyRouting = "NB_USE_LEGACY_ROUTING"
)

var advancedRoutingSupported bool

func Init() {
advancedRoutingSupported = checkAdvancedRoutingSupport()
}

func AdvancedRouting() bool {
return advancedRoutingSupported
}

func checkAdvancedRoutingSupport() bool {
var err error

var legacyRouting bool
if val := os.Getenv("NB_USE_LEGACY_ROUTING"); val != "" {
legacyRouting, err = strconv.ParseBool(val)
if err != nil {
log.Warnf("failed to parse %s: %v", envUseLegacyRouting, err)
}
}

var skipSocketMark bool
if val := os.Getenv(envSkipSocketMark); val != "" {
skipSocketMark, err = strconv.ParseBool(val)
if err != nil {
log.Warnf("failed to parse %s: %v", envSkipSocketMark, err)
}
}

// requested to disable advanced routing
if legacyRouting || skipSocketMark ||
// envCustomRoutingDisabled disables the custom dialers.
// There is no point in using advanced routing without those, as they set up fwmarks on the sockets.
CustomRoutingDisabled() ||
// netstack mode doesn't need routing at all
netstack.IsEnabled() {

log.Info("advanced routing has been requested to be disabled")
return false
}

if !CheckFwmarkSupport() || !CheckRuleOperationsSupport() {
log.Warn("system doesn't support required routing features, falling back to legacy routing")
return false
}

return true
}

func CheckFwmarkSupport() bool {
fd, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_DGRAM, 0)
if err != nil {
log.Warnf("failed to create test socket: %v", err)
return false
}
defer syscall.Close(fd)

err = syscall.SetsockoptInt(fd, syscall.SOL_SOCKET, syscall.SO_MARK, 1)
if err != nil {
log.Warnf("fwmark is not supported: %v", err)
return false
}
return true
}

func CheckRuleOperationsSupport() bool {
rule := netlink.NewRule()
// low precedence, semi-random
rule.Priority = 32321
rule.Table = syscall.RT_TABLE_MAIN
rule.Family = netlink.FAMILY_V4

if err := netlink.RuleAdd(rule); err != nil {
if errors.Is(err, syscall.EOPNOTSUPP) {
log.Warn("IP rule operations are not supported")
return false
}
log.Warnf("failed to test rule support: %v", err)
return false
}

if err := netlink.RuleDel(rule); err != nil {
log.Warnf("failed to delete test rule: %v", err)
}
return true
}
20 changes: 3 additions & 17 deletions util/net/net_linux.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,11 @@ package net
import (
"fmt"
"syscall"

log "github.com/sirupsen/logrus"
)

// SetSocketMark sets the SO_MARK option on the given socket connection
func SetSocketMark(conn syscall.Conn) error {
if isSocketMarkDisabled() {
if !AdvancedRouting() {
return nil
}

Expand All @@ -25,7 +23,7 @@ func SetSocketMark(conn syscall.Conn) error {

// SetSocketOpt sets the SO_MARK option on the given file descriptor
func SetSocketOpt(fd int) error {
if isSocketMarkDisabled() {
if !AdvancedRouting() {
return nil
}

Expand All @@ -36,7 +34,7 @@ func setRawSocketMark(conn syscall.RawConn) error {
var setErr error

err := conn.Control(func(fd uintptr) {
if isSocketMarkDisabled() {
if !AdvancedRouting() {
return
}
setErr = setSocketOptInt(int(fd))
Expand All @@ -55,15 +53,3 @@ func setRawSocketMark(conn syscall.RawConn) error {
func setSocketOptInt(fd int) error {
return syscall.SetsockoptInt(fd, syscall.SOL_SOCKET, syscall.SO_MARK, NetbirdFwmark)
}

func isSocketMarkDisabled() bool {
if CustomRoutingDisabled() {
log.Infof("Custom routing is disabled, skipping SO_MARK")
return true
}

if SkipSocketMark() {
return true
}
return false
}

0 comments on commit c6b72c8

Please sign in to comment.