Skip to content
This repository has been archived by the owner on Jul 11, 2023. It is now read-only.

Commit

Permalink
injector: make iptable rules idempotent (#4373)
Browse files Browse the repository at this point in the history 
Makes the iptable rules idempotent by using
`iptables-restore --noflush`. With this change,
the init-container can be executed multiple times
without any side effects. The `--noflush` option
ensures the exising rules do not get flushed
during `iptables-restore`, which would otherwise
compromise the security.

Also simplifies the init container test to avoid
redundancy.

Resolves #4307

Signed-off-by: Shashank Ram <[email protected]>
  • Loading branch information
@shashankram
shashankram authored Nov 24, 2021
1 parent 332b52b commit fe85f60
Show file tree
Hide file tree
Showing 4 changed files with 102 additions and 213 deletions.
5 changes: 1 addition & 4 deletions pkg/injector/init_container.go
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
package injector

import (
"strings"

corev1 "k8s.io/api/core/v1"
"k8s.io/utils/pointer"

Expand All @@ -11,8 +9,7 @@ import (

func getInitContainerSpec(containerName string, cfg configurator.Configurator, outboundIPRangeExclusionList []string, outboundPortExclusionList []int,
inboundPortExclusionList []int, enablePrivilegedInitContainer bool) corev1.Container {
iptablesInitCommandsList := generateIptablesCommands(outboundIPRangeExclusionList, outboundPortExclusionList, inboundPortExclusionList)
iptablesInitCommand := strings.Join(iptablesInitCommandsList, " && ")
iptablesInitCommand := generateIptablesCommands(outboundIPRangeExclusionList, outboundPortExclusionList, inboundPortExclusionList)

return corev1.Container{
Name: containerName,
Expand Down
159 changes: 23 additions & 136 deletions pkg/injector/init_container_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ var _ = Describe("Test functions creating Envoy bootstrap configuration", func()
)

privilegedFalse := false
privilegedTrue := true
runAsNonRootFalse := false
runAsUserID := int64(0)

Expand All @@ -36,141 +35,29 @@ var _ = Describe("Test functions creating Envoy bootstrap configuration", func()
Command: []string{"/bin/sh"},
Args: []string{
"-c",
"iptables -t nat -N PROXY_INBOUND && iptables -t nat -N PROXY_IN_REDIRECT && iptables -t nat -N PROXY_OUTPUT && iptables -t nat -N PROXY_REDIRECT && iptables -t nat -A PROXY_REDIRECT -p tcp -j REDIRECT --to-port 15001 && iptables -t nat -A PROXY_REDIRECT -p tcp --dport 15000 -j ACCEPT && iptables -t nat -A OUTPUT -p tcp -j PROXY_OUTPUT && iptables -t nat -A PROXY_OUTPUT -m owner --uid-owner 1500 -j RETURN && iptables -t nat -A PROXY_OUTPUT -d 127.0.0.1/32 -j RETURN && iptables -t nat -A PROXY_OUTPUT -j PROXY_REDIRECT && iptables -t nat -A PROXY_IN_REDIRECT -p tcp -j REDIRECT --to-port 15003 && iptables -t nat -A PREROUTING -p tcp -j PROXY_INBOUND && iptables -t nat -A PROXY_INBOUND -p tcp --dport 15010 -j RETURN && iptables -t nat -A PROXY_INBOUND -p tcp --dport 15901 -j RETURN && iptables -t nat -A PROXY_INBOUND -p tcp --dport 15902 -j RETURN && iptables -t nat -A PROXY_INBOUND -p tcp --dport 15903 -j RETURN && iptables -t nat -A PROXY_INBOUND -p tcp -j PROXY_IN_REDIRECT",
},
WorkingDir: "",
Resources: corev1.ResourceRequirements{},
SecurityContext: &corev1.SecurityContext{
Capabilities: &corev1.Capabilities{
Add: []corev1.Capability{
"NET_ADMIN",
},
},
Privileged: &privilegedFalse,
RunAsNonRoot: &runAsNonRootFalse,
RunAsUser: &runAsUserID,
},
Stdin: false,
StdinOnce: false,
TTY: false,
}

Expect(actual).To(Equal(expected))
})

It("Creates init container with outbound exclusion list", func() {
mockConfigurator.EXPECT().GetInitContainerImage().Return(containerImage).Times(1)
outboundIPRangeExclusionList := []string{"1.1.1.1/32", "10.0.0.10/24"}
privileged := privilegedFalse
actual := getInitContainerSpec(containerName, mockConfigurator, outboundIPRangeExclusionList, nil, nil, privileged)

expected := corev1.Container{
Name: "-container-name-",
Image: "-init-container-image-",
Command: []string{"/bin/sh"},
Args: []string{
"-c",
"iptables -t nat -N PROXY_INBOUND && iptables -t nat -N PROXY_IN_REDIRECT && iptables -t nat -N PROXY_OUTPUT && iptables -t nat -N PROXY_REDIRECT && iptables -t nat -A PROXY_REDIRECT -p tcp -j REDIRECT --to-port 15001 && iptables -t nat -A PROXY_REDIRECT -p tcp --dport 15000 -j ACCEPT && iptables -t nat -A OUTPUT -p tcp -j PROXY_OUTPUT && iptables -t nat -A PROXY_OUTPUT -m owner --uid-owner 1500 -j RETURN && iptables -t nat -A PROXY_OUTPUT -d 127.0.0.1/32 -j RETURN && iptables -t nat -A PROXY_OUTPUT -j PROXY_REDIRECT && iptables -t nat -A PROXY_IN_REDIRECT -p tcp -j REDIRECT --to-port 15003 && iptables -t nat -A PREROUTING -p tcp -j PROXY_INBOUND && iptables -t nat -A PROXY_INBOUND -p tcp --dport 15010 -j RETURN && iptables -t nat -A PROXY_INBOUND -p tcp --dport 15901 -j RETURN && iptables -t nat -A PROXY_INBOUND -p tcp --dport 15902 -j RETURN && iptables -t nat -A PROXY_INBOUND -p tcp --dport 15903 -j RETURN && iptables -t nat -A PROXY_INBOUND -p tcp -j PROXY_IN_REDIRECT && iptables -t nat -I PROXY_OUTPUT -d 1.1.1.1/32 -j RETURN && iptables -t nat -I PROXY_OUTPUT -d 10.0.0.10/24 -j RETURN",
},
WorkingDir: "",
Resources: corev1.ResourceRequirements{},
SecurityContext: &corev1.SecurityContext{
Capabilities: &corev1.Capabilities{
Add: []corev1.Capability{
"NET_ADMIN",
},
},
Privileged: &privilegedFalse,
RunAsNonRoot: &runAsNonRootFalse,
RunAsUser: &runAsUserID,
},
Stdin: false,
StdinOnce: false,
TTY: false,
}

Expect(actual).To(Equal(expected))
})

It("Creates init container with privileged true", func() {
mockConfigurator.EXPECT().GetInitContainerImage().Return(containerImage).Times(1)
privileged := privilegedTrue
actual := getInitContainerSpec(containerName, mockConfigurator, nil, nil, nil, privileged)

expected := corev1.Container{
Name: "-container-name-",
Image: "-init-container-image-",
Command: []string{"/bin/sh"},
Args: []string{
"-c",
"iptables -t nat -N PROXY_INBOUND && iptables -t nat -N PROXY_IN_REDIRECT && iptables -t nat -N PROXY_OUTPUT && iptables -t nat -N PROXY_REDIRECT && iptables -t nat -A PROXY_REDIRECT -p tcp -j REDIRECT --to-port 15001 && iptables -t nat -A PROXY_REDIRECT -p tcp --dport 15000 -j ACCEPT && iptables -t nat -A OUTPUT -p tcp -j PROXY_OUTPUT && iptables -t nat -A PROXY_OUTPUT -m owner --uid-owner 1500 -j RETURN && iptables -t nat -A PROXY_OUTPUT -d 127.0.0.1/32 -j RETURN && iptables -t nat -A PROXY_OUTPUT -j PROXY_REDIRECT && iptables -t nat -A PROXY_IN_REDIRECT -p tcp -j REDIRECT --to-port 15003 && iptables -t nat -A PREROUTING -p tcp -j PROXY_INBOUND && iptables -t nat -A PROXY_INBOUND -p tcp --dport 15010 -j RETURN && iptables -t nat -A PROXY_INBOUND -p tcp --dport 15901 -j RETURN && iptables -t nat -A PROXY_INBOUND -p tcp --dport 15902 -j RETURN && iptables -t nat -A PROXY_INBOUND -p tcp --dport 15903 -j RETURN && iptables -t nat -A PROXY_INBOUND -p tcp -j PROXY_IN_REDIRECT",
},
WorkingDir: "",
Resources: corev1.ResourceRequirements{},
SecurityContext: &corev1.SecurityContext{
Capabilities: &corev1.Capabilities{
Add: []corev1.Capability{
"NET_ADMIN",
},
},
Privileged: &privilegedTrue,
RunAsNonRoot: &runAsNonRootFalse,
RunAsUser: &runAsUserID,
},
Stdin: false,
StdinOnce: false,
TTY: false,
}

Expect(actual).To(Equal(expected))
})

It("Creates init container without outbound port exclusion list", func() {
mockConfigurator.EXPECT().GetInitContainerImage().Return(containerImage).Times(1)
privileged := privilegedFalse
actual := getInitContainerSpec(containerName, mockConfigurator, nil, nil, nil, privileged)

expected := corev1.Container{
Name: "-container-name-",
Image: "-init-container-image-",
Command: []string{"/bin/sh"},
Args: []string{
"-c",
"iptables -t nat -N PROXY_INBOUND && iptables -t nat -N PROXY_IN_REDIRECT && iptables -t nat -N PROXY_OUTPUT && iptables -t nat -N PROXY_REDIRECT && iptables -t nat -A PROXY_REDIRECT -p tcp -j REDIRECT --to-port 15001 && iptables -t nat -A PROXY_REDIRECT -p tcp --dport 15000 -j ACCEPT && iptables -t nat -A OUTPUT -p tcp -j PROXY_OUTPUT && iptables -t nat -A PROXY_OUTPUT -m owner --uid-owner 1500 -j RETURN && iptables -t nat -A PROXY_OUTPUT -d 127.0.0.1/32 -j RETURN && iptables -t nat -A PROXY_OUTPUT -j PROXY_REDIRECT && iptables -t nat -A PROXY_IN_REDIRECT -p tcp -j REDIRECT --to-port 15003 && iptables -t nat -A PREROUTING -p tcp -j PROXY_INBOUND && iptables -t nat -A PROXY_INBOUND -p tcp --dport 15010 -j RETURN && iptables -t nat -A PROXY_INBOUND -p tcp --dport 15901 -j RETURN && iptables -t nat -A PROXY_INBOUND -p tcp --dport 15902 -j RETURN && iptables -t nat -A PROXY_INBOUND -p tcp --dport 15903 -j RETURN && iptables -t nat -A PROXY_INBOUND -p tcp -j PROXY_IN_REDIRECT",
},
WorkingDir: "",
Resources: corev1.ResourceRequirements{},
SecurityContext: &corev1.SecurityContext{
Capabilities: &corev1.Capabilities{
Add: []corev1.Capability{
"NET_ADMIN",
},
},
Privileged: &privilegedFalse,
RunAsNonRoot: &runAsNonRootFalse,
RunAsUser: &runAsUserID,
},
Stdin: false,
StdinOnce: false,
TTY: false,
}

Expect(actual).To(Equal(expected))
})

It("init container with outbound port exclusion list", func() {
mockConfigurator.EXPECT().GetInitContainerImage().Return(containerImage).Times(1)
outboundPortExclusionList := []int{6060, 7070}
privileged := privilegedFalse
actual := getInitContainerSpec(containerName, mockConfigurator, nil, outboundPortExclusionList, nil, privileged)

expected := corev1.Container{
Name: "-container-name-",
Image: "-init-container-image-",
Command: []string{"/bin/sh"},
Args: []string{
"-c",
"iptables -t nat -N PROXY_INBOUND && iptables -t nat -N PROXY_IN_REDIRECT && iptables -t nat -N PROXY_OUTPUT && iptables -t nat -N PROXY_REDIRECT && iptables -t nat -A PROXY_REDIRECT -p tcp -j REDIRECT --to-port 15001 && iptables -t nat -A PROXY_REDIRECT -p tcp --dport 15000 -j ACCEPT && iptables -t nat -A OUTPUT -p tcp -j PROXY_OUTPUT && iptables -t nat -A PROXY_OUTPUT -m owner --uid-owner 1500 -j RETURN && iptables -t nat -A PROXY_OUTPUT -d 127.0.0.1/32 -j RETURN && iptables -t nat -A PROXY_OUTPUT -j PROXY_REDIRECT && iptables -t nat -A PROXY_IN_REDIRECT -p tcp -j REDIRECT --to-port 15003 && iptables -t nat -A PREROUTING -p tcp -j PROXY_INBOUND && iptables -t nat -A PROXY_INBOUND -p tcp --dport 15010 -j RETURN && iptables -t nat -A PROXY_INBOUND -p tcp --dport 15901 -j RETURN && iptables -t nat -A PROXY_INBOUND -p tcp --dport 15902 -j RETURN && iptables -t nat -A PROXY_INBOUND -p tcp --dport 15903 -j RETURN && iptables -t nat -A PROXY_INBOUND -p tcp -j PROXY_IN_REDIRECT && iptables -t nat -I PROXY_OUTPUT -p tcp --match multiport --dports 6060,7070 -j RETURN",
`iptables-restore --noflush <<EOF
# OSM sidecar interception rules
*nat
:PROXY_INBOUND - [0:0]
:PROXY_IN_REDIRECT - [0:0]
:PROXY_OUTPUT - [0:0]
:PROXY_REDIRECT - [0:0]
-A PROXY_IN_REDIRECT -p tcp -j REDIRECT --to-port 15003
-A PREROUTING -p tcp -j PROXY_INBOUND
-A PROXY_INBOUND -p tcp --dport 15010 -j RETURN
-A PROXY_INBOUND -p tcp --dport 15901 -j RETURN
-A PROXY_INBOUND -p tcp --dport 15902 -j RETURN
-A PROXY_INBOUND -p tcp --dport 15903 -j RETURN
-A PROXY_INBOUND -p tcp -j PROXY_IN_REDIRECT
-A PROXY_REDIRECT -p tcp -j REDIRECT --to-port 15001
-A PROXY_REDIRECT -p tcp --dport 15000 -j ACCEPT
-A OUTPUT -p tcp -j PROXY_OUTPUT
-A PROXY_OUTPUT -m owner --uid-owner 1500 -j RETURN
-A PROXY_OUTPUT -d 127.0.0.1/32 -j RETURN
-A PROXY_OUTPUT -j PROXY_REDIRECT
COMMIT
EOF
`,
},
WorkingDir: "",
Resources: corev1.ResourceRequirements{},
Expand Down
99 changes: 50 additions & 49 deletions pkg/injector/iptables.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,83 +8,84 @@ import (
"github.com/openservicemesh/osm/pkg/constants"
)

// iptablesRedirectionChains is the list of iptables chains created for traffic redirection via the proxy sidecar
var iptablesRedirectionChains = []string{
// Chain to intercept inbound traffic
"iptables -t nat -N PROXY_INBOUND",

// Chain to redirect inbound traffic to the proxy
"iptables -t nat -N PROXY_IN_REDIRECT",

// Chain to intercept outbound traffic
"iptables -t nat -N PROXY_OUTPUT",

// Chain to redirect outbound traffic to the proxy
"iptables -t nat -N PROXY_REDIRECT",
}

// iptablesOutboundStaticRules is the list of iptables rules related to outbound traffic interception and redirection
var iptablesOutboundStaticRules = []string{
// Redirects outbound TCP traffic hitting PROXY_REDIRECT chain to Envoy's outbound listener port
fmt.Sprintf("iptables -t nat -A PROXY_REDIRECT -p tcp -j REDIRECT --to-port %d", constants.EnvoyOutboundListenerPort),
fmt.Sprintf("-A PROXY_REDIRECT -p tcp -j REDIRECT --to-port %d", constants.EnvoyOutboundListenerPort),

// Traffic to the Proxy Admin port flows to the Proxy -- not redirected
fmt.Sprintf("iptables -t nat -A PROXY_REDIRECT -p tcp --dport %d -j ACCEPT", constants.EnvoyAdminPort),
fmt.Sprintf("-A PROXY_REDIRECT -p tcp --dport %d -j ACCEPT", constants.EnvoyAdminPort),

// For outbound TCP traffic jump from OUTPUT chain to PROXY_OUTPUT chain
"iptables -t nat -A OUTPUT -p tcp -j PROXY_OUTPUT",
"-A OUTPUT -p tcp -j PROXY_OUTPUT",

// Don't redirect Envoy traffic back to itself, return it to the next chain for processing
fmt.Sprintf("iptables -t nat -A PROXY_OUTPUT -m owner --uid-owner %d -j RETURN", constants.EnvoyUID),
fmt.Sprintf("-A PROXY_OUTPUT -m owner --uid-owner %d -j RETURN", constants.EnvoyUID),

// Skip localhost traffic, doesn't need to be routed via the proxy
"iptables -t nat -A PROXY_OUTPUT -d 127.0.0.1/32 -j RETURN",
"-A PROXY_OUTPUT -d 127.0.0.1/32 -j RETURN",

// Redirect remaining outbound traffic to Envoy
"iptables -t nat -A PROXY_OUTPUT -j PROXY_REDIRECT",
"-A PROXY_OUTPUT -j PROXY_REDIRECT",
}

// iptablesInboundStaticRules is the list of iptables rules related to inbound traffic interception and redirection
var iptablesInboundStaticRules = []string{
// Redirects inbound TCP traffic hitting the PROXY_IN_REDIRECT chain to Envoy's inbound listener port
fmt.Sprintf("iptables -t nat -A PROXY_IN_REDIRECT -p tcp -j REDIRECT --to-port %d", constants.EnvoyInboundListenerPort),
fmt.Sprintf("-A PROXY_IN_REDIRECT -p tcp -j REDIRECT --to-port %d", constants.EnvoyInboundListenerPort),

// For inbound traffic jump from PREROUTING chain to PROXY_INBOUND chain
"iptables -t nat -A PREROUTING -p tcp -j PROXY_INBOUND",
"-A PREROUTING -p tcp -j PROXY_INBOUND",

// Skip metrics query traffic being directed to Envoy's inbound prometheus listener port
fmt.Sprintf("iptables -t nat -A PROXY_INBOUND -p tcp --dport %d -j RETURN", constants.EnvoyPrometheusInboundListenerPort),
fmt.Sprintf("-A PROXY_INBOUND -p tcp --dport %d -j RETURN", constants.EnvoyPrometheusInboundListenerPort),

// Skip inbound health probes; These ports will be explicitly handled by listeners configured on the
// Envoy proxy IF any health probes have been configured in the Pod Spec.
// TODO(draychev): Do not add these if no health probes have been defined (https://github.com/openservicemesh/osm/issues/2243)
fmt.Sprintf("iptables -t nat -A PROXY_INBOUND -p tcp --dport %d -j RETURN", livenessProbePort),
fmt.Sprintf("iptables -t nat -A PROXY_INBOUND -p tcp --dport %d -j RETURN", readinessProbePort),
fmt.Sprintf("iptables -t nat -A PROXY_INBOUND -p tcp --dport %d -j RETURN", startupProbePort),
fmt.Sprintf("-A PROXY_INBOUND -p tcp --dport %d -j RETURN", livenessProbePort),
fmt.Sprintf("-A PROXY_INBOUND -p tcp --dport %d -j RETURN", readinessProbePort),
fmt.Sprintf("-A PROXY_INBOUND -p tcp --dport %d -j RETURN", startupProbePort),

// Redirect remaining inbound traffic to Envoy
"iptables -t nat -A PROXY_INBOUND -p tcp -j PROXY_IN_REDIRECT",
"-A PROXY_INBOUND -p tcp -j PROXY_IN_REDIRECT",
}

// generateIptablesCommands generates a list of iptables commands to set up sidecar interception and redirection
func generateIptablesCommands(outboundIPRangeExclusionList []string, outboundPortExclusionList []int, inboundPortExclusionList []int) []string {
var cmd []string
func generateIptablesCommands(outboundIPRangeExclusionList []string, outboundPortExclusionList []int, inboundPortExclusionList []int) string {
var rules strings.Builder

// 1. Create redirection chains
cmd = append(cmd, iptablesRedirectionChains...)
fmt.Fprintln(&rules, `# OSM sidecar interception rules
*nat
:PROXY_INBOUND - [0:0]
:PROXY_IN_REDIRECT - [0:0]
:PROXY_OUTPUT - [0:0]
:PROXY_REDIRECT - [0:0]`)
var cmds []string

// 2. Create outbound rules
cmd = append(cmd, iptablesOutboundStaticRules...)
// 1. Create inbound rules
cmds = append(cmds, iptablesInboundStaticRules...)

// 3. Create inbound rules
cmd = append(cmd, iptablesInboundStaticRules...)
// 2. Create dynamic inbound ports exclusion rules
if len(inboundPortExclusionList) > 0 {
var portExclusionListStr []string
for _, port := range inboundPortExclusionList {
portExclusionListStr = append(portExclusionListStr, strconv.Itoa(port))
}
inboundPortsToExclude := strings.Join(portExclusionListStr, ",")
rule := fmt.Sprintf("-I PROXY_INBOUND -p tcp --match multiport --dports %s -j RETURN", inboundPortsToExclude)
cmds = append(cmds, rule)
}

// 3. Create outbound rules
cmds = append(cmds, iptablesOutboundStaticRules...)

// 4. Create dynamic outbound ip ranges exclusion rules
for _, cidr := range outboundIPRangeExclusionList {
// *Note: it is important to use the insert option '-I' instead of the append option '-A' to ensure the exclusion
// rules take precedence over the static redirection rules. Iptables rules are evaluated in order.
rule := fmt.Sprintf("iptables -t nat -I PROXY_OUTPUT -d %s -j RETURN", cidr)
cmd = append(cmd, rule)
rule := fmt.Sprintf("-I PROXY_OUTPUT -d %s -j RETURN", cidr)
cmds = append(cmds, rule)
}

// 5. Create dynamic outbound ports exclusion rules
Expand All @@ -94,20 +95,20 @@ func generateIptablesCommands(outboundIPRangeExclusionList []string, outboundPor
portExclusionListStr = append(portExclusionListStr, strconv.Itoa(port))
}
outboundPortsToExclude := strings.Join(portExclusionListStr, ",")
rule := fmt.Sprintf("iptables -t nat -I PROXY_OUTPUT -p tcp --match multiport --dports %s -j RETURN", outboundPortsToExclude)
cmd = append(cmd, rule)
rule := fmt.Sprintf("-I PROXY_OUTPUT -p tcp --match multiport --dports %s -j RETURN", outboundPortsToExclude)
cmds = append(cmds, rule)
}

// 6. Create dynamic inbound ports exclusion rules
if len(inboundPortExclusionList) > 0 {
var portExclusionListStr []string
for _, port := range inboundPortExclusionList {
portExclusionListStr = append(portExclusionListStr, strconv.Itoa(port))
}
inboundPortsToExclude := strings.Join(portExclusionListStr, ",")
rule := fmt.Sprintf("iptables -t nat -I PROXY_INBOUND -p tcp --match multiport --dports %s -j RETURN", inboundPortsToExclude)
cmd = append(cmd, rule)
for _, rule := range cmds {
fmt.Fprintln(&rules, rule)
}

fmt.Fprint(&rules, "COMMIT")

cmd := fmt.Sprintf(`iptables-restore --noflush <<EOF
%s
EOF
`, rules.String())

return cmd
}
Loading

0 comments on commit fe85f60

Please sign in to comment.