netlink add ecmp route

The default route will be added for each connected network, so when you would try to use two networks it would fail because the default route was already added. While we could ignore eexist error this is not what we want. If we only have one route and disconnect the network with the default route we will loose internet connectivity. To best way is to have each network create the default route. This can be done by not setting the NLM_F_EXCL flag for the netlink request. Also see this CNI bridge plugin PR which does the same there: containernetworking/plugins#615 Signed-off-by: Paul Holzinger <[email protected]>
Luap99 · Nov 4, 2021 · 026c134 · 026c134
1 parent 1c47ca3
commit 026c134
Show file tree

Hide file tree

Showing 2 changed files with 126 additions and 37 deletions.
diff --git a/src/network/core_utils.rs b/src/network/core_utils.rs
@@ -1,13 +1,20 @@
 use futures::stream::TryStreamExt;
+use futures::StreamExt;
 use ipnetwork::IpNetwork;
 use ipnetwork::Ipv4Network;
 use ipnetwork::Ipv6Network;
 use libc;
 use rtnetlink;
+use rtnetlink::packet::constants::*;
 use rtnetlink::packet::rtnl::link::nlas::Nla;
+use rtnetlink::packet::NetlinkPayload;
+use rtnetlink::packet::RouteMessage;
 use std::fmt::Write;
 use std::fs::File;
 use std::io::Error;
+use std::net::IpAddr;
+use std::net::Ipv4Addr;
+use std::net::Ipv6Addr;
 use std::os::unix::prelude::*;
 
 pub struct CoreUtils {
@@ -130,45 +137,67 @@ impl CoreUtils {
     async fn add_route_v4(
         handle: &rtnetlink::Handle,
         dest: &Ipv4Network,
-        gateway: &Ipv4Network,
+        gateway: &Ipv4Addr,
     ) -> Result<(), std::io::Error> {
         let route = handle.route();
-        match route
+        let msg = route
             .add()
             .v4()
             .destination_prefix(dest.ip(), dest.prefix())
-            .gateway(gateway.ip())
-            .execute()
-            .await
-        {
-            Ok(_) => Ok(()),
-            Err(err) => Err(std::io::Error::new(
-                std::io::ErrorKind::Other,
-                format!("failed to add route: {}", err),
-            )),
-        }
+            .gateway(*gateway)
+            .message_mut()
+            .to_owned();
+
+        CoreUtils::execute_route_msg(handle, msg).await
     }
 
     async fn add_route_v6(
         handle: &rtnetlink::Handle,
         dest: &Ipv6Network,
-        gateway: &Ipv6Network,
+        gateway: &Ipv6Addr,
     ) -> Result<(), std::io::Error> {
         let route = handle.route();
-        match route
+        let msg = route
             .add()
             .v6()
             .destination_prefix(dest.ip(), dest.prefix())
-            .gateway(gateway.ip())
-            .execute()
-            .await
-        {
-            Ok(_) => Ok(()),
-            Err(err) => Err(std::io::Error::new(
-                std::io::ErrorKind::Other,
-                format!("failed to add route: {}", err),
-            )),
+            .gateway(*gateway)
+            .message_mut()
+            .to_owned();
+
+        CoreUtils::execute_route_msg(handle, msg).await
+    }
+
+    async fn execute_route_msg(
+        handle: &rtnetlink::Handle,
+        msg: RouteMessage,
+    ) -> Result<(), std::io::Error> {
+        // Note: we do not use .execute because we have to overwrite the request flags
+        // by default NLM_F_EXCL is set and this throws an error if we try to create multiple default routes
+        // We need to create a default route for each network because we need to keep the internet connectivity
+        // after a podman disconnect via the other network.
+        let mut req =
+            rtnetlink::packet::NetlinkMessage::from(rtnetlink::packet::RtnlMessage::NewRoute(msg));
+        req.header.flags = NLM_F_REQUEST | NLM_F_ACK | NLM_F_CREATE;
+
+        let mut response = match handle.clone().request(req) {
+            Ok(res) => res,
+            Err(err) => {
+                return Err(std::io::Error::new(
+                    std::io::ErrorKind::Other,
+                    format!("failed to add route: {}", err),
+                ));
+            }
+        };
+        while let Some(message) = response.next().await {
+            if let NetlinkPayload::Error(err) = message.payload {
+                return Err(std::io::Error::new(
+                    std::io::ErrorKind::Other,
+                    format!("failed to add route: {}", err),
+                ));
+            }
         }
+        Ok(())
     }
 
     #[tokio::main]
@@ -546,7 +575,7 @@ impl CoreUtils {
         for gw_ip_add in gw_ip_addrs {
             match gw_ip_add.to_string().parse() {
                 Ok(gateway) => match gateway {
-                    IpNetwork::V4(gateway) => match "0.0.0.0/0".to_string().parse() {
+                    IpAddr::V4(gateway) => match Ipv4Network::new(Ipv4Addr::new(0, 0, 0, 0), 0) {
                         Ok(dest) => {
                             if let Err(err) =
                                 CoreUtils::add_route_v4(&handle, &dest, &gateway).await
@@ -561,21 +590,23 @@ impl CoreUtils {
                             ))
                         }
                     },
-                    IpNetwork::V6(gateway) => match "::/0".to_string().parse() {
-                        Ok(dest) => {
-                            if let Err(err) =
-                                CoreUtils::add_route_v6(&handle, &dest, &gateway).await
-                            {
-                                return Err(err);
+                    IpAddr::V6(gateway) => {
+                        match Ipv6Network::new(Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 0), 0) {
+                            Ok(dest) => {
+                                if let Err(err) =
+                                    CoreUtils::add_route_v6(&handle, &dest, &gateway).await
+                                {
+                                    return Err(err);
+                                }
+                            }
+                            Err(err) => {
+                                return Err(std::io::Error::new(
+                                    std::io::ErrorKind::Other,
+                                    format!("failed to parse address ::/0: {}", err),
+                                ))
                             }
                         }
-                        Err(err) => {
-                            return Err(std::io::Error::new(
-                                std::io::ErrorKind::Other,
-                                format!("failed to parse address ::/0: {}", err),
-                            ))
-                        }
-                    },
+                    }
                 },
                 Err(err) => {
                     return Err(std::io::Error::new(

diff --git a/src/test/config/twoNetworks.json b/src/test/config/twoNetworks.json
@@ -0,0 +1,58 @@
+{
+    "container_id": "2d0fe608dc86d7ba65dcec724a335b618328f08daa62afd2ee7fa9d41f74e5a9",
+    "container_name": "",
+    "networks": {
+        "podman1": {
+            "static_ips": [
+                "10.0.0.2"
+            ],
+            "interface_name": "eth0"
+        },
+        "podman2": {
+            "static_ips": [
+                "10.1.0.2"
+            ],
+            "interface_name": "eth1"
+        }
+    },
+    "network_info": {
+        "podman1": {
+            "name": "podman1",
+            "id": "4937f73ac0df011d4f2848d5f83f5c20b707e71a8d98789bbe80d8f64a815e79",
+            "driver": "bridge",
+            "network_interface": "podman1",
+            "created": "2021-11-04T19:08:39.124321192+01:00",
+            "subnets": [
+                {
+                    "subnet": "10.0.0.0/24",
+                    "gateway": "10.0.0.1"
+                }
+            ],
+            "ipv6_enabled": false,
+            "internal": false,
+            "dns_enabled": false,
+            "ipam_options": {
+                "driver": "host-local"
+            }
+        },
+        "podman2": {
+            "name": "podman2",
+            "id": "488a7d9be4fa72a5b80811bd847aac1d99d1a09060739b4e08687949c957cda8",
+            "driver": "bridge",
+            "network_interface": "podman2",
+            "created": "2021-11-04T19:08:39.124800596+01:00",
+            "subnets": [
+                {
+                    "subnet": "10.1.0.0/24",
+                    "gateway": "10.1.0.1"
+                }
+            ],
+            "ipv6_enabled": false,
+            "internal": false,
+            "dns_enabled": false,
+            "ipam_options": {
+                "driver": "host-local"
+            }
+        }
+    }
+}