NAS-130981 / 25.04 / Allow specifying custom address pools for docker…

… service (by sonicaj) (#14410)

* Add migration to add address pools to db

(cherry picked from commit 6e05c40)

* Use address pool specified in docker config

(cherry picked from commit 7060a0c)

* Add a validation util to verify used subnets

(cherry picked from commit d5054b2)

* Use validation util when updating docker bits

(cherry picked from commit 4685561)

* Fix bug in validation utils

(cherry picked from commit 7763552)

* Redeploy apps if address pool has changed

(cherry picked from commit 793ef17)

* Make sure that at least 1 addr pool is specified

(cherry picked from commit 629a0b5)

* Bump down revision id of migration

(cherry picked from commit ec67484)

* Make sure docker service is restarted when address pools are changed

(cherry picked from commit 52919c6)

* Add a unit test for validating address pools

(cherry picked from commit 1c41b42)

* Bug fix for starting docker service

(cherry picked from commit 17b939c)

* Fix subnet prefix usage

(cherry picked from commit 2434e80)

* Add merge migration

---------

Co-authored-by: Waqar Ahmed <[email protected]>

src/middlewared/middlewared/alembic/versions/24.10/2024-09-03_20-33_docker_addr_pool.py

@@ -0,0 +1,31 @@
+"""
+Add address_pools column to services_docker
+
+Revision ID: 98c1ebde0079
+Revises: d24d6760fda4
+Create Date: 2024-09-03 20:33:47.996994+00:00
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+revision = '98c1ebde0079'
+down_revision = 'd24d6760fda4'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    with op.batch_alter_table('services_docker', schema=None) as batch_op:
+        batch_op.add_column(
+            sa.Column(
+                'address_pools',
+                sa.TEXT(),
+                nullable=False,
+                server_default='[{"base": "172.30.0.0/16", "size": 27}, {"base": "172.31.0.0/16", "size": 27}]'
+            )
+        )
+
+
+def downgrade():
+    pass

src/middlewared/middlewared/alembic/versions/25.04/2024-09-04_00-35_merge.py

@@ -0,0 +1,19 @@
+"""Merge
+
+Revision ID: 9f51d0be7b07
+Revises: 991d17a5b3a2, 98c1ebde0079
+Create Date: 2024-09-04 00:35:59.547731+00:00
+"""
+
+revision = '9f51d0be7b07'
+down_revision = ('991d17a5b3a2', '98c1ebde0079')
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    pass
+
+
+def downgrade():
+    pass

src/middlewared/middlewared/etc_files/docker/daemon.json.py

@@ -22,16 +22,7 @@ def render(service, middleware):
         'exec-opts': ['native.cgroupdriver=cgroupfs'],
         'iptables': True,
         'storage-driver': 'overlay2',
-        'default-address-pools': [
-            {
-                'base': '172.30.0.0/16',
-                'size': 27
-            },
-            {
-                'base': '172.31.0.0/16',
-                'size': 27
-            },
-        ],
+        'default-address-pools': config['address_pools'],
     }
     isolated = middleware.call_sync('system.advanced.config')['isolated_gpu_pci_ids']
     for gpu in filter(lambda x: x not in isolated, get_nvidia_gpus()):

src/middlewared/middlewared/plugins/docker/update.py

@@ -1,11 +1,13 @@
 import middlewared.sqlalchemy as sa
 
-from middlewared.schema import accepts, Bool, Dict, Int, Patch, Str, ValidationErrors
+from middlewared.schema import accepts, Bool, Dict, Int, IPAddr, List, Patch, Str, ValidationErrors
 from middlewared.service import CallError, ConfigService, job, private, returns
 from middlewared.utils.zfs import query_imported_fast_impl
+from middlewared.validators import Range
 
 from .state_utils import Status
 from .utils import applications_ds_name
+from .validation_utils import validate_address_pools
 
 
 class DockerModel(sa.Model):
@@ -15,6 +17,10 @@ class DockerModel(sa.Model):
     pool = sa.Column(sa.String(255), default=None, nullable=True)
     enable_image_updates = sa.Column(sa.Boolean(), default=True)
     nvidia = sa.Column(sa.Boolean(), default=False)
+    address_pools = sa.Column(sa.JSON(list), default=[
+        {'base': '172.30.0.0/16', 'size': 27},
+        {'base': '172.31.0.0/16', 'size': 27}
+    ])
 
 
 class DockerService(ConfigService):
@@ -32,6 +38,13 @@ class Config:
         Str('dataset', required=True),
         Str('pool', required=True, null=True),
         Bool('nvidia', required=True),
+        List('address_pools', items=[
+             Dict(
+                 'address_pool',
+                 IPAddr('base', cidr=True),
+                 Int('size', validators=[Range(min_=1, max_=32)])
+             )
+        ]),
         update=True,
     )
 
@@ -64,8 +77,13 @@ async def do_update(self, job, data):
 
         verrors.check()
 
+        if config['address_pools'] != old_config['address_pools']:
+            validate_address_pools(
+                await self.middleware.call('interface.ip_in_use', {'static': True}), config['address_pools']
+            )
+
         if old_config != config:
-            if config['pool'] != old_config['pool']:
+            if any(config[k] != old_config[k] for k in ('pool', 'address_pools')):
                 job.set_progress(20, 'Stopping Docker service')
                 try:
                     await self.middleware.call('service.stop', 'docker')
@@ -79,18 +97,29 @@ async def do_update(self, job, data):
             if config['pool'] != old_config['pool']:
                 job.set_progress(60, 'Applying requested configuration')
                 await self.middleware.call('docker.setup.status_change')
+            elif config['pool'] and config['address_pools'] != old_config['address_pools']:
+                job.set_progress(60, 'Starting docker')
+                await self.middleware.call('service.start', 'docker')
 
             if not old_config['nvidia'] and config['nvidia']:
                 await (
                     await self.middleware.call(
                         'nvidia.install',
                         job_on_progress_cb=lambda encoded: job.set_progress(
-                            80 + int(encoded['progress']['percent'] * 0.2),
+                            70 + int(encoded['progress']['percent'] * 0.2),
                             encoded['progress']['description'],
                         )
                     )
                 ).wait(raise_error=True)
 
+            if config['pool'] and config['address_pools'] != old_config['address_pools']:
+                job.set_progress(95, 'Initiating redeployment of applications to apply new address pools changes')
+                await self.middleware.call(
+                    'core.bulk', 'app.redeploy', [
+                        [app['name']] for app in await self.middleware.call('app.query', [['state', '!=', 'STOPPED']])
+                    ]
+                )
+
         job.set_progress(100, 'Requested configuration applied')
         return await self.config()
 

src/middlewared/middlewared/plugins/docker/validation_utils.py

@@ -0,0 +1,44 @@
+import ipaddress
+
+from middlewared.schema import ValidationErrors
+
+
+def validate_address_pools(system_ips: list[dict], user_specified_networks: list[dict]):
+    verrors = ValidationErrors()
+    if not user_specified_networks:
+        verrors.add('docker_update.address_pools', 'At least one address pool must be specified')
+    verrors.check()
+
+    network_cidrs = set([
+        ipaddress.ip_network(f'{ip["address"]}/{ip["netmask"]}', False)
+        for ip in system_ips
+    ])
+    seen_networks = set()
+    for index, user_network in enumerate(user_specified_networks):
+        base_network = ipaddress.ip_network(user_network['base'], False)
+
+        # Validate subnet size vs. base network
+        if base_network.prefixlen > user_network['size']:
+            verrors.add(
+                f'docker_update.address_pools.{index}.base',
+                f'Base network {user_network["base"]} cannot be smaller than '
+                f'the specified subnet size {user_network["size"]}'
+            )
+
+        # Validate no overlaps with system networks
+        if any(base_network.overlaps(system_network) for system_network in network_cidrs):
+            verrors.add(
+                f'docker_update.address_pools.{index}.base',
+                f'Base network {user_network["base"]} overlaps with an existing system network'
+            )
+
+        # Validate no duplicate networks
+        if base_network in seen_networks:
+            verrors.add(
+                f'docker_update.address_pools.{index}.base',
+                f'Base network {user_network["base"]} is a duplicate of another specified network'
+            )
+
+        seen_networks.add(base_network)
+
+    verrors.check()

src/middlewared/middlewared/pytest/unit/plugins/docker/test_docker_address_pools_validtaion.py

@@ -0,0 +1,44 @@
+import pytest
+
+from middlewared.plugins.docker.validation_utils import validate_address_pools
+from middlewared.service_exception import ValidationErrors
+
+
+IP_IN_USE = [
+  {
+    'type': 'INET',
+    'address': '172.20.0.33',
+    'netmask': 16,
+    'broadcast': '172.20.0.63'
+  }
+]
+
+
+@pytest.mark.parametrize('user_specified_networks,error_msg', (
+    (
+        [],
+        'At least one address pool must be specified'),
+    (
+        [{'base': '172.20.2.0/24', 'size': 27}],
+        'Base network 172.20.2.0/24 overlaps with an existing system network'),
+    (
+        [{'base': '172.21.2.0/16', 'size': 10}],
+        'Base network 172.21.2.0/16 cannot be smaller than the specified subnet size 10'),
+    (
+        [{'base': '172.21.2.0/16', 'size': 27}, {'base': '172.21.2.0/16', 'size': 27}],
+        'Base network 172.21.2.0/16 is a duplicate of another specified network'
+    ),
+    (
+        [{'base': '172.21.0.0/16', 'size': 27}, {'base': '172.22.0.0/16', 'size': 27}],
+        ''
+    ),
+))
+@pytest.mark.asyncio
+async def test_address_pools_validation(user_specified_networks, error_msg):
+    if error_msg:
+        with pytest.raises(ValidationErrors) as ve:
+            validate_address_pools(IP_IN_USE, user_specified_networks)
+
+        assert ve.value.errors[0].errmsg == error_msg
+    else:
+        assert validate_address_pools(IP_IN_USE, user_specified_networks) is None