Skip to content

Commit

Permalink
Merge pull request #1245 from flyingcircusio/PL-133360-fix-ipv6-autoc…
Browse files Browse the repository at this point in the history
…onfig-again

[24.11] Fix IPv6 autoconfiguration
  • Loading branch information
sysvinit authored Jan 23, 2025
2 parents 8d1b948 + a552dc7 commit f944419
Show file tree
Hide file tree
Showing 7 changed files with 286 additions and 24 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
<!--
A new changelog entry.
Delete placeholder items that do not apply. Empty sections will be removed
automatically during release.
Leave the XX.XX as is: this is a placeholder and will be automatically filled
correctly during the release and helps when backporting over multiple platform
branches.
-->

### Impact



### NixOS XX.XX platform

- platform: ensure that IPv6 autoconfiguration is correctly disabled
on both physical and virtual hosts. (PL-133360)
1 change: 1 addition & 0 deletions nixos/infrastructure/flyingcircus-physical.nix
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ mkIf (cfg.infrastructureModule == "flyingcircus-physical") (lib.mkMerge [
hardware.cpu.amd.updateMicrocode = true;
hardware.cpu.intel.updateMicrocode = true;
flyingcircus.raid.enable = true;
flyingcircus.networking.physicalHostNetworking = true;

boot = {
initrd.availableKernelModules = [
Expand Down
2 changes: 1 addition & 1 deletion nixos/lib/network.nix
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ let
vxlanInterfaces = lib.filterAttrs (name: value: value.policy or null == "vxlan") encInterfaces;
vxlanCount = length (attrNames vxlanInterfaces);
in
if config.flyingcircus.infrastructureModule != "flyingcircus-physical"
if !config.flyingcircus.networking.physicalHostNetworking
then foldConds encInterfaces
[
{
Expand Down
47 changes: 26 additions & 21 deletions nixos/platform/network.nix
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,11 @@ in
description = "Names of ethernet devices to monitor.";
default = [];
};
flyingcircus.networking.physicalHostNetworking = lib.mkOption {
type = lib.types.bool;
description = "Use a network configuration profile suitable for physical hosts";
default = false;
};
};

config = lib.mkMerge [
Expand Down Expand Up @@ -224,7 +229,7 @@ in
wireguard.enable = true;

firewall.trustedInterfaces =
lib.optionals (!isNull fclib.underlay && cfg.infrastructureModule == "flyingcircus-physical")
lib.optionals (!isNull fclib.underlay && cfg.networking.physicalHostNetworking)
(map (l: l.link) fclib.underlay.links or []);

firewall.extraCommands = ''
Expand Down Expand Up @@ -265,7 +270,7 @@ in

};

flyingcircus.services.telegraf.inputs = lib.optionalAttrs (cfg.infrastructureModule == "flyingcircus-physical") {
flyingcircus.services.telegraf.inputs = lib.optionalAttrs (cfg.networking.physicalHostNetworking) {
exec = [{
commands = [ "${pkgs.fc.telegraf-routes-summary}/bin/telegraf-routes-summary" ];
timeout = "10s";
Expand Down Expand Up @@ -448,7 +453,7 @@ in
# TODO: it'd be preferrable to manage this on a by-interface base
# and distinguish whether an interface is physical.
# Can this be done based on `config.flyingcircus.enc.parameters.interfaces.fe.policy`?
${lib.optionalString (config.flyingcircus.infrastructureModule == "flyingcircus-physical") ''
${lib.optionalString (config.flyingcircus.networking.physicalHostNetworking) ''
echo "Disabling flow control"
ethtool -A ${iface.link} autoneg off rx off tx off || true
''}
Expand Down Expand Up @@ -509,14 +514,14 @@ in
};
})) ethernetLinks) ++


(let
unitName = link: "network-disable-ipv6-autoconfig-${link}";
unitTemplate = link: rec {
unitTemplate = link: fixAddrGen: rec {
description = "Disable IPv6 autoconfig for link ${link}";
wantedBy = [ "network-addresses-${link}.service" ];
before = wantedBy;
requires = [ "${link}-netdev.service" ];
bindsTo = requires;
after = requires;
path = [ pkgs.procps fclib.relaxedIp ];
stopIfChanged = false;
Expand All @@ -527,12 +532,14 @@ in
sysctl net.ipv6.conf.${link}.temp_valid_lft=0
sysctl net.ipv6.conf.${link}.temp_prefered_lft=0
# If an interface has previously been managed by dhcpcd this sysctl might be
# set to a non-zero value, which disables automatic generation of link-local
# addresses. This can leave the interface without a link-local address when
# dhcpcd deletes addresses from the interface when it exits. Resetting this
# to 0 restores the default kernel behaviour.
sysctl net.ipv6.conf.${link}.addr_gen_mode=0
${lib.optionalString fixAddrGen ''
# If an interface has previously been managed by dhcpcd this sysctl might be
# set to a non-zero value, which disables automatic generation of link-local
# addresses. This can leave the interface without a link-local address when
# dhcpcd deletes addresses from the interface when it exits. Resetting this
# to 0 restores the default kernel behaviour.
sysctl net.ipv6.conf.${link}.addr_gen_mode=0
''}
for oldtmp in $(ip -6 address show dev ${link} dynamic scope global | grep inet6 | cut -d ' ' -f6); do
ip addr del $oldtmp dev ${link}
Expand All @@ -543,15 +550,13 @@ in
RemainAfterExit = true;
};
};

virtualLinkUnit = link: ((unitTemplate link) // {
bindsTo = [ "${link}-netdev.service" ];
after = [ "${link}-netdev.service" ];
});
in
(map (link:
lib.nameValuePair (unitName link) (virtualLinkUnit link))
virtualLinks)
lib.nameValuePair (unitName link) (unitTemplate link false))
virtualLinks) ++
(map (link:
lib.nameValuePair (unitName link.link) (unitTemplate link.link true))
ethernetLinks)
) ++

(lib.optionals (!isNull fclib.underlay)
Expand Down Expand Up @@ -900,10 +905,10 @@ in
# as a reasonable size and I'd suggest generalizing this number to all machines.
"net.netfilter.nf_conntrack_max" = 262144;
}
(lib.mkIf (cfg.infrastructureModule != "flyingcircus-physical") {
(lib.mkIf (!cfg.networking.physicalHostNetworking) {
"net.core.rmem_max" = 8388608;
})
(lib.mkIf (cfg.infrastructureModule == "flyingcircus-physical") {
(lib.mkIf (cfg.networking.physicalHostNetworking) {
"vm.min_free_kbytes" = "513690";

"net.core.netdev_max_backlog" = 300000;
Expand Down Expand Up @@ -944,7 +949,7 @@ in
};

}
(lib.mkIf (config.flyingcircus.infrastructureModule == "flyingcircus") {
(lib.mkIf (cfg.infrastructureModule == "flyingcircus") {
# This check is here to identify abysmal but otherwise subtle network speed
# issues *in VMs* as we have seen in PL-132971. If downloading a 1MiB test
# file takes longer than 5-10 seconds, something is very much off.
Expand Down
1 change: 1 addition & 0 deletions tests/default.nix
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ in {
collect-garbage = callTest ./collect-garbage.nix {};
gitlab = callTest ./gitlab.nix {};
haproxy = callTest ./haproxy.nix {};
ipv6-autoconfig = callSubTests ./ipv6-autoconfig.nix {};
java = callTest ./java.nix {};
journal = callTest ./journal.nix {};
journalbeat = callTest ./journalbeat.nix {};
Expand Down
114 changes: 114 additions & 0 deletions tests/ipv6-autoconfig.nix
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
import ./make-test-python.nix ({ pkgs, testlib, ... }:
let
assertZeroSysctl = pkgs.writeScriptBin "assert_zero_sysctl" ''
set -eu
sysctl="$1"
value="$(${pkgs.procps}/bin/sysctl -n -b "$sysctl")"
[ "$value" == "0" ]
'';

makePhysicalHost = { id, links }: { lib, config, ... }:
let
testNodeId = config.virtualisation.test.nodeNumber;
in {
imports = [
(testlib.fcConfig {
inherit id;
net.mgm = true;
net.ul = true;
extraEncParameters = {
inherit id;
interfaces.fe.policy = "vxlan";
interfaces.srv.policy = "vxlan";
interfaces.ul = {
policy = "underlay";
nics = map (link: {
mac = "52:54:00:12:${lib.toLower (lib.toHexString link)}:0${toString testNodeId}";
external_label = "phys/${toString link}";
}) links;
};
};
})
];

# use the hardware networking config profile
flyingcircus.networking.physicalHostNetworking = true;
# extra underlay network links
virtualisation.vlans = links;

services.fail2ban.enable = false;

environment.systemPackages = [ assertZeroSysctl ];
};
in {
name = "ipv6-autoconfig";
testCases = {
virtual = {
name = "virtual";
nodes.machine = { ... }: {
imports = [
(testlib.fcConfig {})
];

environment.systemPackages = [ assertZeroSysctl ];
};
testScript = ''
sysctls = [
"accept_ra",
"autoconf",
"temp_valid_lft",
"temp_prefered_lft",
"addr_gen_mode",
]
machine.wait_for_unit("multi-user.target")
with subtest("testing ipv6 autoconf configuration on ethsrv"):
for sysctl in sysctls:
machine.succeed(f"assert_zero_sysctl net.ipv6.conf.ethsrv.{sysctl}")
with subtest("testing ipv6 autoconf configuration on ethfe"):
for sysctl in sysctls:
machine.succeed(f"assert_zero_sysctl net.ipv6.conf.ethfe.{sysctl}")
'';
};
hardware = {
name = "hardware";
nodes = {
machine = makePhysicalHost { id = 1; links = [ 253 254 ]; };
switch1 = testlib.mockVxlanSwitch { id = 2; links = [ 253 ]; };
switch2 = testlib.mockVxlanSwitch { id = 2; links = [ 254 ]; };
};
testScript = ''
sysctls = [
"accept_ra",
"autoconf",
"temp_valid_lft",
"temp_prefered_lft",
]
hw_sysctls = sysctls.copy()
hw_sysctls.append("addr_gen_mode")
start_all()
for vm in [machine, switch1, switch2]:
vm.wait_for_unit("multi-user.target")
virt_links = ["brsrv", "brfe", "vxsrv", "vxfe"];
phys_links = ["ethmgm", "ul-phys-253", "ul-phys-254"];
with subtest("testing physical links"):
for link in phys_links:
with subtest(f"testing ipv6 autoconf configuration on {link}"):
for sysctl in hw_sysctls:
machine.succeed(f"assert_zero_sysctl net.ipv6.conf.{link}.{sysctl}")
with subtest("testing virtual links"):
for link in virt_links:
with subtest(f"testing ipv6 autoconf configuration on {link}"):
for sysctl in sysctls:
machine.succeed(f"assert_zero_sysctl net.ipv6.conf.{link}.{sysctl}")
'';
};
};
})
Loading

0 comments on commit f944419

Please sign in to comment.