Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add support for late binding to IP addresses using go-sockaddr/template #2399

Merged
merged 5 commits into from
May 12, 2017
Merged

Add support for late binding to IP addresses using go-sockaddr/template #2399

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
a33af1c
Add support for late binding to IP addresses using go-sockaddr/template
multani Feb 26, 2017
02af50f
Parse template before splitting host/port
multani Mar 13, 2017
a61b70b
Default to private IP advertise address in non-dev mode
multani Mar 13, 2017
5fb0b30
Allow to advertise 127.0.0.1 in non-dev mode if explicitly configured
multani Mar 13, 2017
544d9e9
Update doc to explain support for go-sockaddr/template format
multani Mar 22, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
117 changes: 65 additions & 52 deletions command/agent/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package agent

import (
"encoding/base64"
"errors"
"fmt"
"io"
"net"
Expand All @@ -13,6 +14,8 @@ import (
"strings"
"time"

"github.com/hashicorp/go-sockaddr/template"

client "github.com/hashicorp/nomad/client/config"
"github.com/hashicorp/nomad/helper"
"github.com/hashicorp/nomad/nomad"
Expand Down Expand Up @@ -692,30 +695,53 @@ func (c *Config) Merge(b *Config) *Config {
// normalizeAddrs normalizes Addresses and AdvertiseAddrs to always be
// initialized and have sane defaults.
func (c *Config) normalizeAddrs() error {
c.Addresses.HTTP = normalizeBind(c.Addresses.HTTP, c.BindAddr)
c.Addresses.RPC = normalizeBind(c.Addresses.RPC, c.BindAddr)
c.Addresses.Serf = normalizeBind(c.Addresses.Serf, c.BindAddr)
if c.BindAddr != "" {
ipStr, err := parseSingleIPTemplate(c.BindAddr)
if err != nil {
return fmt.Errorf("Bind address resolution failed: %v", err)
}
c.BindAddr = ipStr
}

addr, err := normalizeBind(c.Addresses.HTTP, c.BindAddr)
if err != nil {
return fmt.Errorf("Failed to parse HTTP address: %v", err)
}
c.Addresses.HTTP = addr

addr, err = normalizeBind(c.Addresses.RPC, c.BindAddr)
if err != nil {
return fmt.Errorf("Failed to parse RPC address: %v", err)
}
c.Addresses.RPC = addr

addr, err = normalizeBind(c.Addresses.Serf, c.BindAddr)
if err != nil {
return fmt.Errorf("Failed to parse Serf address: %v", err)
}
c.Addresses.Serf = addr

c.normalizedAddrs = &Addresses{
HTTP: net.JoinHostPort(c.Addresses.HTTP, strconv.Itoa(c.Ports.HTTP)),
RPC: net.JoinHostPort(c.Addresses.RPC, strconv.Itoa(c.Ports.RPC)),
Serf: net.JoinHostPort(c.Addresses.Serf, strconv.Itoa(c.Ports.Serf)),
}

addr, err := normalizeAdvertise(c.AdvertiseAddrs.HTTP, c.Addresses.HTTP, c.Ports.HTTP, c.DevMode)
addr, err = normalizeAdvertise(c.AdvertiseAddrs.HTTP, c.Addresses.HTTP, c.Ports.HTTP)
if err != nil {
return fmt.Errorf("Failed to parse HTTP advertise address: %v", err)
}
c.AdvertiseAddrs.HTTP = addr

addr, err = normalizeAdvertise(c.AdvertiseAddrs.RPC, c.Addresses.RPC, c.Ports.RPC, c.DevMode)
addr, err = normalizeAdvertise(c.AdvertiseAddrs.RPC, c.Addresses.RPC, c.Ports.RPC)
if err != nil {
return fmt.Errorf("Failed to parse RPC advertise address: %v", err)
}
c.AdvertiseAddrs.RPC = addr

// Skip serf if server is disabled
if c.Server != nil && c.Server.Enabled {
addr, err = normalizeAdvertise(c.AdvertiseAddrs.Serf, c.Addresses.Serf, c.Ports.Serf, c.DevMode)
addr, err = normalizeAdvertise(c.AdvertiseAddrs.Serf, c.Addresses.Serf, c.Ports.Serf)
if err != nil {
return fmt.Errorf("Failed to parse Serf advertise address: %v", err)
}
Expand All @@ -725,14 +751,34 @@ func (c *Config) normalizeAddrs() error {
return nil
}

// parseSingleIPTemplate is used as a helper function to parse out a single IP
// address from a config parameter.
func parseSingleIPTemplate(ipTmpl string) (string, error) {
out, err := template.Parse(ipTmpl)
if err != nil {
return "", fmt.Errorf("Unable to parse address template %q: %v", ipTmpl, err)
}

ips := strings.Split(out, " ")
switch len(ips) {
case 0:
return "", errors.New("No addresses found, please configure one.")
case 1:
return ips[0], nil
default:
return "", fmt.Errorf("Multiple addresses found (%q), please configure one.", out)
}
}

// normalizeBind returns a normalized bind address.
//
// If addr is set it is used, if not the default bind address is used.
func normalizeBind(addr, bind string) string {
func normalizeBind(addr, bind string) (string, error) {
if addr == "" {
return bind
return bind, nil
} else {
return parseSingleIPTemplate(addr)
}
return addr
}

// normalizeAdvertise returns a normalized advertise address.
Expand All @@ -747,61 +793,28 @@ func normalizeBind(addr, bind string) string {
// is resolved and returned with the port.
//
// Loopback is only considered a valid advertise address in dev mode.
func normalizeAdvertise(addr string, bind string, defport int, dev bool) (string, error) {
func normalizeAdvertise(addr string, bind string, defport int) (string, error) {
if addr != "" {
// Default to using manually configured address
_, _, err := net.SplitHostPort(addr)
host, port, err := net.SplitHostPort(addr)
if err != nil {
if !isMissingPort(err) {
return "", fmt.Errorf("Error parsing advertise address %q: %v", addr, err)
}

// missing port, append the default
return net.JoinHostPort(addr, strconv.Itoa(defport)), nil
host = addr
port = strconv.Itoa(defport)
}
return addr, nil
}

// Fallback to bind address first, and then try resolving the local hostname
ips, err := net.LookupIP(bind)
if err != nil {
return "", fmt.Errorf("Error resolving bind address %q: %v", bind, err)
}

// Return the first unicast address
for _, ip := range ips {
if ip.IsLinkLocalUnicast() || ip.IsGlobalUnicast() {
return net.JoinHostPort(ip.String(), strconv.Itoa(defport)), nil
}
if ip.IsLoopback() && dev {
// loopback is fine for dev mode
return net.JoinHostPort(ip.String(), strconv.Itoa(defport)), nil
ipStr, err := parseSingleIPTemplate(host)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think you need to parse the template before attempting to SplitHostPort as SplitHostPort may fail on valid templates.

if err != nil {
return "", fmt.Errorf("Error parsing advertise address template: %v", err)
}
}

// As a last resort resolve the hostname and use it if it's not
// localhost (as localhost is never a sensible default)
host, err := os.Hostname()
if err != nil {
return "", fmt.Errorf("Unable to get hostname to set advertise address: %v", err)
}

ips, err = net.LookupIP(host)
if err != nil {
return "", fmt.Errorf("Error resolving hostname %q for advertise address: %v", host, err)
return net.JoinHostPort(ipStr, port), nil
}

// Return the first unicast address
for _, ip := range ips {
if ip.IsLinkLocalUnicast() || ip.IsGlobalUnicast() {
return net.JoinHostPort(ip.String(), strconv.Itoa(defport)), nil
}
if ip.IsLoopback() && dev {
// loopback is fine for dev mode
return net.JoinHostPort(ip.String(), strconv.Itoa(defport)), nil
}
}
return "", fmt.Errorf("No valid advertise addresses, please set `advertise` manually")
// Fallback to bind address, as it has been resolved before.
return net.JoinHostPort(bind, strconv.Itoa(defport)), nil
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There are 2 subtle behaviors I think your code is missing:

  1. Advertising localhost is disallowed unless:
  • Explicitly configured per advertise addr (basically use any explicitly configured addresses without asking questions)
  • DevMode == true
  1. Default to advertising the IP the hostname resolves as.
  • Sadly outside of Google Compute Engine few systems are properly configured to be able to resolve their hostname.

Since 2. doesn't work well in practice I'd be open to changing it to GetPrivateIP, but we'll need to update docs and mark this as a backward incompatible change in CHANGELOG.md

This is out of scope for what you're trying to do, but I also think we'll move bind to default to GetPrivateIP+127.0.0.1 in the future. Sadly we don't support multiple bind addresses at the moment, so it's going to be a larger effort.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

FWIW, Consul will be likely making a change to its defaults in time for its 0.8 release to use GetPrivateIP as its default bind address. And yes, listening on multiple interfaces is something we all want to do (though I'd argue it should be a UNIX socket in /tmp vs 127.0.0.1 that way filesystem permissions could be applied if necessary, and you can figure out the UID of who is connecting to you via a unix socket, but not via loopback - but more on that when we get closer to multiple listeners).

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@schmichael OK, so I changed the code slightly and re-added bits from the previous version, except:

  • I removed the os.Hostname() and related net.LookupIP(host) as per the problems you exposed in 2. and I'm looking up the private IP address using GetPrivateIP instead.
  • before that, I'm trying to use the bind address. I still need to do the ip.IsLinkLocalUnicast() || ip.IsGlobalUnicast() dance since the bind address could be something like localhost which needs to be parsed in order to use IsLoopback(). I thought I could remove this part and profit from what go-sockaddr/template offers for this instead, but I'm not sure how I could prevent 127.0.0.1 to be advertised without resolving the bind address value, in which case I need to call the functions above to do the right thing...

}

// isMissingPort returns true if an error is a "missing port" error from
Expand Down
116 changes: 116 additions & 0 deletions command/agent/config_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package agent

import (
"fmt"
"io/ioutil"
"net"
"os"
Expand Down Expand Up @@ -520,6 +521,121 @@ func TestConfig_Listener(t *testing.T) {
}
}

func TestConfig_normalizeAddrs(t *testing.T) {
c := &Config{
BindAddr: "127.0.0.1",
Ports: &Ports{
HTTP: 4646,
RPC: 4647,
Serf: 4648,
},
Addresses: &Addresses{},
AdvertiseAddrs: &AdvertiseAddrs{},
DevMode: true,
}

if err := c.normalizeAddrs(); err != nil {
t.Fatalf("unable to normalize addresses: %s", err)
}

if c.BindAddr != "127.0.0.1" {
t.Fatalf("expected BindAddr 127.0.0.1, got %s", c.BindAddr)
}

if c.normalizedAddrs.HTTP != "127.0.0.1:4646" {
t.Fatalf("expected HTTP address 127.0.0.1:4646, got %s", c.normalizedAddrs.HTTP)
}

if c.normalizedAddrs.RPC != "127.0.0.1:4647" {
t.Fatalf("expected RPC address 127.0.0.1:4647, got %s", c.normalizedAddrs.RPC)
}

if c.normalizedAddrs.Serf != "127.0.0.1:4648" {
t.Fatalf("expected Serf address 127.0.0.1:4648, got %s", c.normalizedAddrs.Serf)
}

if c.AdvertiseAddrs.HTTP != "127.0.0.1:4646" {
t.Fatalf("expected HTTP advertise address 127.0.0.1:4646, got %s", c.AdvertiseAddrs.HTTP)
}

if c.AdvertiseAddrs.RPC != "127.0.0.1:4647" {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, we don't want this to happen unless AdvertiseAddrs.RPC is explicitly set to localhost.

The reason is advertising localhost any time other than for a local test node can do Very Bad Things. The worst case scenario is having a server agent advertise localhost to the cluster - even briefly! Nodes will spam their own localhost trying to contact that other node that advertised localhost. CPU and network usage will be extremely high, but things might still be able to limp along depending on other factors. So basically you end up with a crippled cluster in a difficult to diagnose way.

We could add better heuristics for advertising localhost... like only disallow it if boostrap_expect>1. But then things just get even more magical and complicated. I'd prefer to just force everyone not in dev mode to advertise a real address! :)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok, but the configuration explicitly enabled dev mode in this case, so ... I guess it's OK to advertise 127.0.0.1 here?
I'm going to add another test to specifically check that 127.0.0.1 is NOT advertised except if dev mode is enabled or if it has been explicitly configured to do so, but could I make this particular test above clearer?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

re: dev mode, yes, that sounds good and is a sane approach.

t.Fatalf("expected RPC advertise address 127.0.0.1:4647, got %s", c.AdvertiseAddrs.RPC)
}

// Client mode, no Serf address defined
if c.AdvertiseAddrs.Serf != "" {
t.Fatalf("expected unset Serf advertise address, got %s", c.AdvertiseAddrs.Serf)
}

c = &Config{
BindAddr: "169.254.1.5",
Ports: &Ports{
HTTP: 4646,
RPC: 4647,
Serf: 4648,
},
Addresses: &Addresses{
HTTP: "169.254.1.10",
},
AdvertiseAddrs: &AdvertiseAddrs{
RPC: "169.254.1.40",
},
Server: &ServerConfig{
Enabled: true,
},
}

if err := c.normalizeAddrs(); err != nil {
t.Fatalf("unable to normalize addresses: %s", err)
}

if c.BindAddr != "169.254.1.5" {
t.Fatalf("expected BindAddr 169.254.1.5, got %s", c.BindAddr)
}

if c.AdvertiseAddrs.HTTP != "169.254.1.10:4646" {
t.Fatalf("expected HTTP advertise address 169.254.1.10:4646, got %s", c.AdvertiseAddrs.HTTP)
}

if c.AdvertiseAddrs.RPC != "169.254.1.40:4647" {
t.Fatalf("expected RPC advertise address 169.254.1.40:4647, got %s", c.AdvertiseAddrs.RPC)
}

if c.AdvertiseAddrs.Serf != "169.254.1.5:4648" {
t.Fatalf("expected Serf advertise address 169.254.1.5:4648, got %s", c.AdvertiseAddrs.Serf)
}

c = &Config{
BindAddr: "{{ GetPrivateIP }}",
Ports: &Ports{
HTTP: 4646,
RPC: 4647,
Serf: 4648,
},
Addresses: &Addresses{},
AdvertiseAddrs: &AdvertiseAddrs{},
Server: &ServerConfig{
Enabled: true,
},
}

if err := c.normalizeAddrs(); err != nil {
t.Fatalf("unable to normalize addresses: %s", err)
}

if c.AdvertiseAddrs.HTTP != fmt.Sprintf("%s:4646", c.BindAddr) {
t.Fatalf("expected HTTP advertise address %s:4646, got %s", c.BindAddr, c.AdvertiseAddrs.HTTP)
}

if c.AdvertiseAddrs.RPC != fmt.Sprintf("%s:4647", c.BindAddr) {
t.Fatalf("expected RPC advertise address %s:4647, got %s", c.BindAddr, c.AdvertiseAddrs.RPC)
}

if c.AdvertiseAddrs.Serf != fmt.Sprintf("%s:4648", c.BindAddr) {
t.Fatalf("expected Serf advertise address %s:4648, got %s", c.BindAddr, c.AdvertiseAddrs.Serf)
}
}

func TestResources_ParseReserved(t *testing.T) {
cases := []struct {
Input string
Expand Down
2 changes: 2 additions & 0 deletions vendor/github.com/hashicorp/go-sockaddr/template/Makefile
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 6 additions & 0 deletions vendor/github.com/hashicorp/go-sockaddr/template/README.md
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading