Skip to content

Commit

Permalink
Evaluate Opportunistic IPsec tunnels for node startup
Browse files Browse the repository at this point in the history 
Try with passthrough option.

Signed-off-by: Periyasamy Palanisamy <[email protected]>
  • Loading branch information
@pperiyasamy
pperiyasamy committed Jan 8, 2025
1 parent d1e9d2c commit bb48347
Showing 1 changed file with 91 additions and 167 deletions.
258 changes: 91 additions & 167 deletions bindata/network/ovn-kubernetes/common/ipsec-host.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -224,119 +224,6 @@ spec:
done
echo "ovnkube-node has configured node."
if ! pgrep pluto; then
echo "pluto is not running, enable the service and/or check system logs"
exit 2
fi
# The ovs-monitor-ipsec doesn't set authby, so when it calls ipsec auto --start
# the default ones defined at Libreswan's compile time will be used. On restart,
# Libreswan will use authby from libreswan.config. If libreswan.config is
# incompatible with the Libreswan's compiled-in defaults, then we'll have an
# authentication problem. But OTOH, ovs-monitor-ipsec does set ike and esp algorithms,
# so those may be incompatible with libreswan.config as well. Hence commenting out the
# "include" from libreswan.conf to avoid such conflicts.
defaultcpinclude="include \/etc\/crypto-policies\/back-ends\/libreswan.config"
if ! grep -q "# ${defaultcpinclude}" /etc/ipsec.conf; then
sed -i "/${defaultcpinclude}/s/^/# /" /etc/ipsec.conf
# since pluto is on the host, we need to restart it after changing connection
# parameters.
chroot /proc/1/root ipsec restart
counter=0
until [ -r /run/pluto/pluto.ctl ]; do
counter=$((counter+1))
sleep 1
if [ $counter -gt 300 ];
then
echo "ipsec has not started after $counter seconds"
exit 1
fi
done
echo "ipsec service is restarted"
fi
# Workaround for https://github.com/libreswan/libreswan/issues/373
ulimit -n 1024
/usr/libexec/ipsec/addconn --config /etc/ipsec.conf --checkconfig
# Check kernel modules
/usr/libexec/ipsec/_stackmanager start
# Check nss database status
/usr/sbin/ipsec --checknss
# Start ovs-monitor-ipsec which will monitor for changes in the ovs
# tunnelling configuration (for example addition of a node) and configures
# libreswan appropriately.
# We are running this in the foreground so that the container will be restarted when ovs-monitor-ipsec fails.
/usr/libexec/platform-python /usr/share/openvswitch/scripts/ovs-monitor-ipsec \
--pidfile=/var/run/openvswitch/ovs-monitor-ipsec.pid --ike-daemon=libreswan --no-restart-ike-daemon \
--ipsec-conf /etc/ipsec.d/openshift.conf --ipsec-d /var/lib/ipsec/nss \
--log-file --monitor unix:/var/run/openvswitch/db.sock
lifecycle:
preStop:
exec:
command:
- /bin/bash
- -c
- |
#!/bin/bash
set -exuo pipefail
# In order to maintain traffic flows during container restart, we
# need to ensure that xfrm state and policies are not flushed.
# Don't allow ovs monitor to cleanup persistent state
kill "$(cat /var/run/openvswitch/ovs-monitor-ipsec.pid 2>/dev/null)" 2>/dev/null || true
env:
- name: K8S_NODE
valueFrom:
fieldRef:
fieldPath: spec.nodeName
securityContext:
privileged: true
volumeMounts:
# To check that network setup is complete
- mountPath: /etc/cni/net.d
name: host-cni-netd
- mountPath: /var/run
name: host-var-run
- mountPath: /var/log/openvswitch/
name: host-var-log-ovs
- mountPath: /etc/openvswitch
name: etc-openvswitch
- mountPath: /var/lib
name: host-var-lib
- mountPath: /etc
name: host-etc
- mountPath: /usr/sbin/ipsec
name: ipsec-bin
- mountPath: /usr/libexec/ipsec
name: ipsec-lib
resources:
requests:
cpu: 10m
memory: 100Mi
terminationMessagePolicy: FallbackToLogsOnError
livenessProbe:
exec:
command:
- /bin/bash
- -c
- |
#!/bin/bash
if [[ $(ipsec whack --trafficstatus | wc -l) -eq 0 ]]; then
echo "no ipsec traffic configured"
exit 10
fi
initialDelaySeconds: 15
periodSeconds: 60
- name: ovn-ipsec-cleanup
image: "{{.OvnImage}}"
command:
- /bin/bash
- -c
- |
#!/bin/bash
{{ if .NETWORK_NODE_IDENTITY_ENABLE }}
# When NETWORK_NODE_IDENTITY_ENABLE is true, use the per-node certificate to create a kubeconfig
# that will be used to talk to the API
Expand Down Expand Up @@ -380,18 +267,7 @@ spec:
export KUBECONFIG=/var/run/ovnkube-kubeconfig
{{ end }}

# It is safe to flush xfrm states and policies and delete openshift.conf
# file when east-west ipsec is disabled. This fixes a race condition when
# ovs-monitor-ipsec is not fast enough to notice ipsec config change and
# delete entries before it's being killed.
# Since it's cleaning up all xfrm states and policies, it may cause slight
# interruption until ipsec is restarted in case of external ipsec config.
# We must do this before killing ovs-monitor-ipsec script, otherwise
# preStop hook doesn't get a chance to run it because ovn-ipsec container
# is abruptly terminated.
# When east-west ipsec is not disabled, then do not flush xfrm states and
# policies in order to maintain traffic flows during container restart.
ipsecflush() {
function cleanup() {
if [ "$(kubectl get networks.operator.openshift.io cluster -ojsonpath='{.spec.defaultNetwork.ovnKubernetesConfig.ipsecConfig.mode}')" != "Full" ] && \
[ "$(kubectl get networks.operator.openshift.io cluster -ojsonpath='{.spec.defaultNetwork.ovnKubernetesConfig.ipsecConfig}')" != "{}" ]; then
ip x s flush
Expand All @@ -400,70 +276,118 @@ spec:
# since pluto is on the host, we need to restart it after the flush
chroot /proc/1/root ipsec restart
fi
}

# Function to handle SIGTERM
cleanup() {
echo "received SIGTERM, flushing ipsec config"
# Wait upto 15 seconds for ovs-monitor-ipsec process to terminate before
# cleaning up ipsec entries.
counter=0
while kill -0 "$(cat /var/run/openvswitch/ovs-monitor-ipsec.pid 2>/dev/null)"; do
counter=$((counter+1))
sleep 1
if [ $counter -gt 15 ];
then
echo "ovs-monitor-ipsec has not terminated after $counter seconds"
break
fi
done
ipsecflush
exit 0
}

# Trap SIGTERM and call cleanup function
trap cleanup SIGTERM

counter=0
until [ -r /var/run/openvswitch/ovs-monitor-ipsec.pid ]; do
counter=$((counter+1))
sleep 1
if [ $counter -gt 300 ];
then
echo "ovs-monitor-ipsec has not started after $counter seconds"
exit 1
fi
done
echo "ovs-monitor-ipsec is started"
ulimit -n 1024
leftid=$(openssl x509 -noout -subject -nameopt RFC2253 -in /etc/openvswitch/keys/ipsec-cert.pem | grep -Eo "CN=[0-9a-z\-]+" | sed 's/CN=/@/')
leftcert=$(openssl x509 -noout -subject -nameopt RFC2253 -in /etc/openvswitch/keys/ipsec-cert.pem | grep -Eo "CN=[0-9a-z\-]+" | sed 's/CN=/ovs_certkey_/')

cat > /etc/ipsec.d/openshift.conf << EOF
config setup
uniqueids=yes

conn %default
keyingtries=%forever
type=transport
auto=route
ike=aes_gcm256-sha2_256
esp=aes_gcm256
ikev2=insist

conn ovn-opportunistic-in
left=%defaultroute # Local interface (this can be %defaultroute or specific IP)
right=%opportunisticgroup
leftid=$leftid # Local identity
leftrsasigkey=%cert # RSA signature (certificate-based authentication)
rightid=%fromcert # Use identity from the peer's certificate
leftcert="$leftcert"
rightca=%same
leftprotoport=udp/6081 # Match traffic from port 6081 (Geneve)
rightprotoport=udp # Match any UDP port on the peer side
narrowing=yes
negotiationshunt=drop
failureshunt=passthrough

conn ovn-opportunistic-out
left=%defaultroute # Local interface
right=%opportunisticgroup
leftid=$leftid # Local identity
leftrsasigkey=%cert # RSA signature (certificate-based authentication)
rightid=%fromcert # Use identity from the peer's certificate
leftcert="$leftcert"
rightca=%same
leftprotoport=udp
rightprotoport=udp/6081
narrowing=yes
negotiationshunt=drop
failureshunt=passthrough
EOF

# Monitor the ovs-monitor-ipsec process.
while kill -0 "$(cat /var/run/openvswitch/ovs-monitor-ipsec.pid 2>/dev/null)"; do
sleep 1
done
certutil -A -a -i /etc/openvswitch/keys/ipsec-cacert.pem -d /var/lib/ipsec/nss -n ovs_cert_cacert -t CT,,
openssl pkcs12 -export -in /etc/openvswitch/keys/ipsec-cert.pem -inkey /etc/openvswitch/keys/ipsec-privkey.pem -out /tmp/blah.p12 -name $leftcert -passout pass:
pk12util -i /tmp/blah.p12 -d /var/lib/ipsec/nss -W ''

# Once the ovs-monitor-ipsec process terminates, execute the cleanup command.
echo "ovs-monitor-ipsec is terminated, flushing ipsec config"
ipsecflush
# The ovs-monitor-ipsec doesn't set authby, so when it calls ipsec auto --start
# the default ones defined at Libreswan's compile time will be used. On restart,
# Libreswan will use authby from libreswan.config. If libreswan.config is
# incompatible with the Libreswan's compiled-in defaults, then we'll have an
# authentication problem. But OTOH, ovs-monitor-ipsec does set ike and esp algorithms,
# so those may be incompatible with libreswan.config as well. Hence commenting out the
# "include" from libreswan.conf to avoid such conflicts.
defaultcpinclude="include \/etc\/crypto-policies\/back-ends\/libreswan.config"
if ! grep -q "# ${defaultcpinclude}" /etc/ipsec.conf; then
sed -i "/${defaultcpinclude}/s/^/# /" /etc/ipsec.conf
fi

# Continue running until SIGTERM is received (or exit naturally)
while true; do
sleep 1
done
/usr/libexec/ipsec/addconn --config /etc/ipsec.conf --checkconfig
# Check kernel modules
/usr/libexec/ipsec/_stackmanager start
# Check nss database status
/usr/sbin/ipsec --checknss

# since pluto is on the host, we need to stop it for spinning up in foreground.
chroot /proc/1/root ipsec stop
/usr/libexec/ipsec/pluto --leak-detective --config /etc/ipsec.conf --nofork --stderrlog
env:
- name: K8S_NODE
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: K8S_NODE_IP
valueFrom:
fieldRef:
fieldPath: status.hostIP
securityContext:
privileged: true
volumeMounts:
{{ if .NETWORK_NODE_IDENTITY_ENABLE }}
- mountPath: /etc/ovn/
name: etc-ovn
{{ end }}
# To check that network setup is complete
- mountPath: /etc/cni/net.d
name: host-cni-netd
- mountPath: /var/run
name: host-var-run
- mountPath: /var/log/openvswitch/
name: host-var-log-ovs
- mountPath: /etc/openvswitch
name: etc-openvswitch
- mountPath: /var/lib
name: host-var-lib
- mountPath: /etc
name: host-etc
- mountPath: /usr/sbin/ipsec
name: ipsec-bin
- mountPath: /usr/libexec/ipsec
name: ipsec-lib
resources:
requests:
cpu: 10m
memory: 50Mi
memory: 100Mi
terminationMessagePolicy: FallbackToLogsOnError
nodeSelector:
kubernetes.io/os: "linux"
Expand Down

0 comments on commit bb48347

Please sign in to comment.