diff --git a/bindata/network/ovn-kubernetes/common/ipsec-host.yaml b/bindata/network/ovn-kubernetes/common/ipsec-host.yaml index 4d9b882f86..d315d872c3 100644 --- a/bindata/network/ovn-kubernetes/common/ipsec-host.yaml +++ b/bindata/network/ovn-kubernetes/common/ipsec-host.yaml @@ -224,119 +224,6 @@ spec: done echo "ovnkube-node has configured node." - if ! pgrep pluto; then - echo "pluto is not running, enable the service and/or check system logs" - exit 2 - fi - - # The ovs-monitor-ipsec doesn't set authby, so when it calls ipsec auto --start - # the default ones defined at Libreswan's compile time will be used. On restart, - # Libreswan will use authby from libreswan.config. If libreswan.config is - # incompatible with the Libreswan's compiled-in defaults, then we'll have an - # authentication problem. But OTOH, ovs-monitor-ipsec does set ike and esp algorithms, - # so those may be incompatible with libreswan.config as well. Hence commenting out the - # "include" from libreswan.conf to avoid such conflicts. - defaultcpinclude="include \/etc\/crypto-policies\/back-ends\/libreswan.config" - if ! grep -q "# ${defaultcpinclude}" /etc/ipsec.conf; then - sed -i "/${defaultcpinclude}/s/^/# /" /etc/ipsec.conf - # since pluto is on the host, we need to restart it after changing connection - # parameters. - chroot /proc/1/root ipsec restart - - counter=0 - until [ -r /run/pluto/pluto.ctl ]; do - counter=$((counter+1)) - sleep 1 - if [ $counter -gt 300 ]; - then - echo "ipsec has not started after $counter seconds" - exit 1 - fi - done - echo "ipsec service is restarted" - fi - - # Workaround for https://github.com/libreswan/libreswan/issues/373 - ulimit -n 1024 - - /usr/libexec/ipsec/addconn --config /etc/ipsec.conf --checkconfig - # Check kernel modules - /usr/libexec/ipsec/_stackmanager start - # Check nss database status - /usr/sbin/ipsec --checknss - - # Start ovs-monitor-ipsec which will monitor for changes in the ovs - # tunnelling configuration (for example addition of a node) and configures - # libreswan appropriately. - # We are running this in the foreground so that the container will be restarted when ovs-monitor-ipsec fails. - /usr/libexec/platform-python /usr/share/openvswitch/scripts/ovs-monitor-ipsec \ - --pidfile=/var/run/openvswitch/ovs-monitor-ipsec.pid --ike-daemon=libreswan --no-restart-ike-daemon \ - --ipsec-conf /etc/ipsec.d/openshift.conf --ipsec-d /var/lib/ipsec/nss \ - --log-file --monitor unix:/var/run/openvswitch/db.sock - lifecycle: - preStop: - exec: - command: - - /bin/bash - - -c - - | - #!/bin/bash - set -exuo pipefail - # In order to maintain traffic flows during container restart, we - # need to ensure that xfrm state and policies are not flushed. - - # Don't allow ovs monitor to cleanup persistent state - kill "$(cat /var/run/openvswitch/ovs-monitor-ipsec.pid 2>/dev/null)" 2>/dev/null || true - env: - - name: K8S_NODE - valueFrom: - fieldRef: - fieldPath: spec.nodeName - securityContext: - privileged: true - volumeMounts: - # To check that network setup is complete - - mountPath: /etc/cni/net.d - name: host-cni-netd - - mountPath: /var/run - name: host-var-run - - mountPath: /var/log/openvswitch/ - name: host-var-log-ovs - - mountPath: /etc/openvswitch - name: etc-openvswitch - - mountPath: /var/lib - name: host-var-lib - - mountPath: /etc - name: host-etc - - mountPath: /usr/sbin/ipsec - name: ipsec-bin - - mountPath: /usr/libexec/ipsec - name: ipsec-lib - resources: - requests: - cpu: 10m - memory: 100Mi - terminationMessagePolicy: FallbackToLogsOnError - livenessProbe: - exec: - command: - - /bin/bash - - -c - - | - #!/bin/bash - if [[ $(ipsec whack --trafficstatus | wc -l) -eq 0 ]]; then - echo "no ipsec traffic configured" - exit 10 - fi - initialDelaySeconds: 15 - periodSeconds: 60 - - name: ovn-ipsec-cleanup - image: "{{.OvnImage}}" - command: - - /bin/bash - - -c - - | - #!/bin/bash {{ if .NETWORK_NODE_IDENTITY_ENABLE }} # When NETWORK_NODE_IDENTITY_ENABLE is true, use the per-node certificate to create a kubeconfig # that will be used to talk to the API @@ -380,18 +267,7 @@ spec: export KUBECONFIG=/var/run/ovnkube-kubeconfig {{ end }} - # It is safe to flush xfrm states and policies and delete openshift.conf - # file when east-west ipsec is disabled. This fixes a race condition when - # ovs-monitor-ipsec is not fast enough to notice ipsec config change and - # delete entries before it's being killed. - # Since it's cleaning up all xfrm states and policies, it may cause slight - # interruption until ipsec is restarted in case of external ipsec config. - # We must do this before killing ovs-monitor-ipsec script, otherwise - # preStop hook doesn't get a chance to run it because ovn-ipsec container - # is abruptly terminated. - # When east-west ipsec is not disabled, then do not flush xfrm states and - # policies in order to maintain traffic flows during container restart. - ipsecflush() { + function cleanup() { if [ "$(kubectl get networks.operator.openshift.io cluster -ojsonpath='{.spec.defaultNetwork.ovnKubernetesConfig.ipsecConfig.mode}')" != "Full" ] && \ [ "$(kubectl get networks.operator.openshift.io cluster -ojsonpath='{.spec.defaultNetwork.ovnKubernetesConfig.ipsecConfig}')" != "{}" ]; then ip x s flush @@ -400,55 +276,90 @@ spec: # since pluto is on the host, we need to restart it after the flush chroot /proc/1/root ipsec restart fi - } - - # Function to handle SIGTERM - cleanup() { - echo "received SIGTERM, flushing ipsec config" - # Wait upto 15 seconds for ovs-monitor-ipsec process to terminate before - # cleaning up ipsec entries. - counter=0 - while kill -0 "$(cat /var/run/openvswitch/ovs-monitor-ipsec.pid 2>/dev/null)"; do - counter=$((counter+1)) - sleep 1 - if [ $counter -gt 15 ]; - then - echo "ovs-monitor-ipsec has not terminated after $counter seconds" - break - fi - done - ipsecflush exit 0 } - # Trap SIGTERM and call cleanup function trap cleanup SIGTERM - counter=0 - until [ -r /var/run/openvswitch/ovs-monitor-ipsec.pid ]; do - counter=$((counter+1)) - sleep 1 - if [ $counter -gt 300 ]; - then - echo "ovs-monitor-ipsec has not started after $counter seconds" - exit 1 - fi - done - echo "ovs-monitor-ipsec is started" + ulimit -n 1024 + leftid=$(openssl x509 -noout -subject -nameopt RFC2253 -in /etc/openvswitch/keys/ipsec-cert.pem | grep -Eo "CN=[0-9a-z\-]+" | sed 's/CN=/@/') + leftcert=$(openssl x509 -noout -subject -nameopt RFC2253 -in /etc/openvswitch/keys/ipsec-cert.pem | grep -Eo "CN=[0-9a-z\-]+" | sed 's/CN=/ovs_certkey_/') + + cat > /etc/ipsec.d/openshift.conf << EOF + config setup + uniqueids=yes + + conn %default + keyingtries=%forever + type=transport + auto=route + ike=aes_gcm256-sha2_256 + esp=aes_gcm256 + ikev2=insist + + conn ovn-opportunistic-in + left=%defaultroute # Local interface (this can be %defaultroute or specific IP) + right=%opportunisticgroup + leftid=$leftid # Local identity + leftrsasigkey=%cert # RSA signature (certificate-based authentication) + rightid=%fromcert # Use identity from the peer's certificate + leftcert="$leftcert" + rightca=%same + leftprotoport=udp/6081 # Match traffic from port 6081 (Geneve) + rightprotoport=udp # Match any UDP port on the peer side + narrowing=yes + negotiationshunt=drop + failureshunt=passthrough + + conn ovn-opportunistic-out + left=%defaultroute # Local interface + right=%opportunisticgroup + leftid=$leftid # Local identity + leftrsasigkey=%cert # RSA signature (certificate-based authentication) + rightid=%fromcert # Use identity from the peer's certificate + leftcert="$leftcert" + rightca=%same + leftprotoport=udp + rightprotoport=udp/6081 + narrowing=yes + negotiationshunt=drop + failureshunt=passthrough + EOF - # Monitor the ovs-monitor-ipsec process. - while kill -0 "$(cat /var/run/openvswitch/ovs-monitor-ipsec.pid 2>/dev/null)"; do - sleep 1 - done + certutil -A -a -i /etc/openvswitch/keys/ipsec-cacert.pem -d /var/lib/ipsec/nss -n ovs_cert_cacert -t CT,, + openssl pkcs12 -export -in /etc/openvswitch/keys/ipsec-cert.pem -inkey /etc/openvswitch/keys/ipsec-privkey.pem -out /tmp/blah.p12 -name $leftcert -passout pass: + pk12util -i /tmp/blah.p12 -d /var/lib/ipsec/nss -W '' - # Once the ovs-monitor-ipsec process terminates, execute the cleanup command. - echo "ovs-monitor-ipsec is terminated, flushing ipsec config" - ipsecflush + # The ovs-monitor-ipsec doesn't set authby, so when it calls ipsec auto --start + # the default ones defined at Libreswan's compile time will be used. On restart, + # Libreswan will use authby from libreswan.config. If libreswan.config is + # incompatible with the Libreswan's compiled-in defaults, then we'll have an + # authentication problem. But OTOH, ovs-monitor-ipsec does set ike and esp algorithms, + # so those may be incompatible with libreswan.config as well. Hence commenting out the + # "include" from libreswan.conf to avoid such conflicts. + defaultcpinclude="include \/etc\/crypto-policies\/back-ends\/libreswan.config" + if ! grep -q "# ${defaultcpinclude}" /etc/ipsec.conf; then + sed -i "/${defaultcpinclude}/s/^/# /" /etc/ipsec.conf + fi - # Continue running until SIGTERM is received (or exit naturally) - while true; do - sleep 1 - done + /usr/libexec/ipsec/addconn --config /etc/ipsec.conf --checkconfig + # Check kernel modules + /usr/libexec/ipsec/_stackmanager start + # Check nss database status + /usr/sbin/ipsec --checknss + + # since pluto is on the host, we need to stop it for spinning up in foreground. + chroot /proc/1/root ipsec stop + /usr/libexec/ipsec/pluto --leak-detective --config /etc/ipsec.conf --nofork --stderrlog + env: + - name: K8S_NODE + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: K8S_NODE_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP securityContext: privileged: true volumeMounts: @@ -456,14 +367,27 @@ spec: - mountPath: /etc/ovn/ name: etc-ovn {{ end }} + # To check that network setup is complete + - mountPath: /etc/cni/net.d + name: host-cni-netd - mountPath: /var/run name: host-var-run + - mountPath: /var/log/openvswitch/ + name: host-var-log-ovs + - mountPath: /etc/openvswitch + name: etc-openvswitch + - mountPath: /var/lib + name: host-var-lib - mountPath: /etc name: host-etc + - mountPath: /usr/sbin/ipsec + name: ipsec-bin + - mountPath: /usr/libexec/ipsec + name: ipsec-lib resources: requests: cpu: 10m - memory: 50Mi + memory: 100Mi terminationMessagePolicy: FallbackToLogsOnError nodeSelector: kubernetes.io/os: "linux"