[test] fix potential backbone name conflict when running cert suite (#…

…10596)

### Background

#10550 introduced a new
way to support multiple backbone nework in otbr docker test. Though it
works good while running a single test, a bug exists when running
cert-suite, which runs a batch of tests in parallel.

cert-suite allocates the name of the backbone interfaces dynamically
by setting env PORT_OFFSET for each test, so there is potentially
conflict exists if we hard code the `backbone_network` name in
TOPOLOGY. This PR is targeting to fix this potential naming conflict.

### Fix

We fix it by assigning a number for `backbone_network_id` in each BR
definition in TOPOLOGY, instead of setting a fixed `backbone network`
name. The final backbone network name is decided by both `PORT_OFFSET`
env and the number of `backbone_network` (in
`backbone{PORT_OFFSET}.{backbone_network}` format)

For example, if `PORT_OFFSET` is 0 and `backbone_network_id` is 1,
then backbone network name will be `backbone0.1`. For the tests that
only use one backbone network and the `backbone_network_id` is not
given, the backbone network name is by default
`backbone{PORT_OFFSET}.0`.

### New test case format
```
class NewTestCase(thread_cert.TestCase):
    ...
    BR = 1
    ...
    TOPOLOGY = {
        BR: {
            ...
            'is_otbr': True,
            'backbone_network_id': <backbone-id>,
            ...
        }
        ...
    }
    ...
```

`<backbone-id>` is any integer from 0, for each BR inside a single
test, if `<backbone-id>` is different, the BR use different backbone
interfaces; the same `<backbone-id>` inside a single test case means
the same backbone network interface.

`'backbone_network_id': <backbone-id>` is optional for single backbone
test cases, when it's not given while defining a otbr node, the
backbone is default as `backbone{PORT_OFFSET}.0`.

For developers, if you are defining a new test which has multiple
backbone interfaces, please ensure `backbone_network_id` is explicitly
defined in each BR, otherwize an error is reported.

tests/scripts/thread-cert/border_router/test_multi_ail.py

@@ -42,7 +42,7 @@ class TwoBorderRoutersOnTwoInfrastructures(thread_cert.TestCase):
 
     Topology:
 
-    -------(backbone0)-------- | ---------(backbone1)-------
+    -------(backbone0.0)-------- | ---------(backbone0.1)-------
               |                             |
            BR1 (Leader)  ..............  BR2 (Router)
 
@@ -55,14 +55,14 @@ class TwoBorderRoutersOnTwoInfrastructures(thread_cert.TestCase):
     TOPOLOGY = {
         BR1: {
             'name': 'BR_1',
-            'backbone_network': 'backbone0',
+            'backbone_network_id': 0,
             'allowlist': [BR2],
             'is_otbr': True,
             'version': '1.3',
         },
         BR2: {
             'name': 'BR_2',
-            'backbone_network': 'backbone1',
+            'backbone_network_id': 1,
             'allowlist': [BR1],
             'is_otbr': True,
             'version': '1.3',

tests/scripts/thread-cert/config.py

@@ -64,11 +64,11 @@
 DOMAIN_PREFIX_ALTER = 'fd00:7d04:7d04:7d04::/64'
 
 PORT_OFFSET = int(os.getenv('PORT_OFFSET', '0'))
-BACKBONE_IPV6_ADDR_START_BASE = 0x9100
-BACKBONE_IPV6_ADDR_START = BACKBONE_IPV6_ADDR_START_BASE + PORT_OFFSET
-BACKBONE_PREFIX = f'{BACKBONE_IPV6_ADDR_START:04x}::/64'
-BACKBONE_PREFIX_REGEX_PATTERN = f'^{BACKBONE_IPV6_ADDR_START:04x}:'
+BACKBONE_IPV6_ADDR_START = f'{0x9100 + PORT_OFFSET:04x}'
+BACKBONE_PREFIX = f'{BACKBONE_IPV6_ADDR_START}::/64'
+BACKBONE_PREFIX_REGEX_PATTERN = f'^{BACKBONE_IPV6_ADDR_START}:'
 BACKBONE_DOCKER_NETWORK_NAME = f'backbone{PORT_OFFSET}'
+BACKBONE_DOCKER_NETWORK_DEFAULT_ID = 0
 BACKBONE_IFNAME = 'eth0'
 THREAD_IFNAME = 'wpan0'
 

tests/scripts/thread-cert/node.py

@@ -66,6 +66,8 @@ class OtbrDocker:
 
     def __init__(self, nodeid: int, backbone_network: str, **kwargs):
         self.verbose = int(float(os.getenv('VERBOSE', 0)))
+
+        assert backbone_network is not None
         self.backbone_network = backbone_network
         try:
             self._docker_name = config.OTBR_DOCKER_NAME_PREFIX + str(nodeid)

tests/scripts/thread-cert/run_cert_suite.py

@@ -71,7 +71,7 @@ def run_cert(job_id: int, port_offset: int, script: str, run_directory: str):
         env['PYTHONPATH'] = os.path.dirname(os.path.abspath(__file__))
 
         try:
-            print(f'Running {test_name}')
+            print(f'Running PORT_OFFSET={port_offset} {test_name}')
             with open(logfile, 'wt') as output:
                 abs_script = os.path.abspath(script)
                 subprocess.check_call(abs_script,

tests/scripts/thread-cert/thread_cert.py

@@ -111,6 +111,10 @@ def __init__(self, *args, **kwargs):
         self._do_packet_verification = PACKET_VERIFICATION and hasattr(self, 'verify') \
                                        and self.PACKET_VERIFICATION == PACKET_VERIFICATION
 
+        # store all the backbone network names that are used in the test case,
+        # it keeps empty when there's no backbone traffic in the test (no otbr or host nodes)
+        self._backbone_network_names = []
+
     def skipTest(self, reason: Any) -> None:
         self._testSkipped = True
         super(TestCase, self).skipTest(reason)
@@ -153,7 +157,11 @@ def _setUp(self):
             params = self._parse_params(params)
             initial_topology[i] = params
 
+            backbone_network_name = self._construct_backbone_network_name(params.get('backbone_network_id')) \
+                                    if self._has_backbone_traffic() else None
+
             logging.info("Creating node %d: %r", i, params)
+            logging.info("Backbone network: %s", backbone_network_name)
 
             if params['is_otbr']:
                 nodeclass = OtbrNode
@@ -168,7 +176,7 @@ def _setUp(self):
                              name=params.get('name'),
                              version=params['version'],
                              is_bbr=params['is_bbr'],
-                             backbone_network=params['backbone_network'])
+                             backbone_network=backbone_network_name)
             if 'boot_delay' in params:
                 self.simulator.go(params['boot_delay'])
 
@@ -465,6 +473,18 @@ def _collect_test_info_after_setup(self):
                 ethaddr = node.get_ether_mac()
                 test_info['ethaddrs'][i] = EthAddr(ethaddr).format_octets()
 
+    def _construct_backbone_network_name(self, backbone_network_id) -> str:
+        """
+        Construct the name of the backbone network based on the given backbone network id from TOPOLOGY. If the
+        backbone_network_id is not defined in TOPOLOGY, use the default backbone network id.
+        """
+        id = backbone_network_id if backbone_network_id is not None else config.BACKBONE_DOCKER_NETWORK_DEFAULT_ID
+        backbone_name = f'{config.BACKBONE_DOCKER_NETWORK_NAME}.{id}'
+
+        assert backbone_name in self._backbone_network_names
+
+        return backbone_name
+
     def _output_test_info(self):
         """
         Output test info to json file after tearDown
@@ -500,9 +520,6 @@ def _parse_params(self, params: Optional[dict]) -> dict:
             assert params.get('version', '') == '', params
             params['version'] = ''
 
-        # set default backbone network for bbr/otbr/host if not specified
-        params.setdefault('backbone_network', config.BACKBONE_DOCKER_NETWORK_NAME)
-
         # use 1.3 node for 1.2 tests
         if params.get('version') == '1.2':
             params['version'] = '1.3'
@@ -527,35 +544,45 @@ def _has_backbone_traffic(self):
 
     def _prepare_backbone_network(self):
         """
-        Prepares one or multiple backbone network(s).
+        Creates one or more backbone networks (Docker bridge networks) based on the TOPOLOGY definition.
 
-        This method creates the backbone network based on the `backbone` values defined in the TOPOLOGY in each test.
-        If no backbone values are defined, it sets the default backbone network as BACKBONE_DOCKER_NETWORK_NAME
-        from the config module.
+        * If `backbone_network_id` is defined in the TOPOLOGY:
+            * Network name:   `backbone{PORT_OFFSET}.{backbone_network_id}`      (e.g., "backbone0.0", "backbone0.1")
+            * Network prefix: `backbone{PORT_OFFSET}:{backbone_network_id}::/64` (e.g., "9100:0::/64", "9100:1::/64")
+
+        * If `backbone_network_id` is undefined:
+            * Network name:   `backbone{PORT_OFFSET}.0`    (e.g., "backbone0.0")
+            * Network prefix: `backbone{PORT_OFFSET}::/64` (e.g., "9100::/64")
         """
-        # Use backbone_set to store all the backbone values in TOPOLOGY
-        backbone_set = set()
+        # Create backbone_set to store all the backbone_ids by parsing TOPOLOGY.
+        backbone_id_set = set()
         for node in self.TOPOLOGY:
-            backbone = self.TOPOLOGY[node].get('backbone_network')
-            if backbone:
-                backbone_set.add(backbone)
+            id = self.TOPOLOGY[node].get('backbone_network_id')
+            if id is not None:
+                backbone_id_set.add(id)
 
-        # Set default backbone network if backbone_set is empty
-        if not backbone_set:
-            backbone_set.add(config.BACKBONE_DOCKER_NETWORK_NAME)
+        # Add default backbone network id if backbone_set is empty
+        if not backbone_id_set:
+            backbone_id_set.add(config.BACKBONE_DOCKER_NETWORK_DEFAULT_ID)
 
         # Iterate over the backbone_set and create backbone network(s)
-        for offset, backbone in enumerate(backbone_set, start=PORT_OFFSET):
-            backbone_prefix = f'{config.BACKBONE_IPV6_ADDR_START_BASE + offset:04x}::/64'
+        for id in backbone_id_set:
+            backbone = f'{config.BACKBONE_DOCKER_NETWORK_NAME}.{id}'
+            backbone_prefix = f'{config.BACKBONE_IPV6_ADDR_START}:{id}::/64'
+            self._backbone_network_names.append(backbone)
             self.assure_run_ok(
                 f'docker network create --driver bridge --ipv6 --subnet {backbone_prefix} -o "com.docker.network.bridge.name"="{backbone}" {backbone} || true',
                 shell=True)
 
     def _remove_backbone_network(self):
-        network_name = config.BACKBONE_DOCKER_NETWORK_NAME
-        self.assure_run_ok(f'docker network rm {network_name}', shell=True)
+        for network_name in self._backbone_network_names:
+            self.assure_run_ok(f'docker network rm {network_name}', shell=True)
 
     def _start_backbone_sniffer(self):
+        assert self._backbone_network_names, 'Internal Error: self._backbone_network_names is empty'
+        # TODO: support sniffer on multiple backbone networks
+        sniffer_interface = self._backbone_network_names[0]
+
         # don't know why but I have to create the empty bbr.pcap first, otherwise tshark won't work
         # self.assure_run_ok("truncate --size 0 bbr.pcap && chmod 664 bbr.pcap", shell=True)
         pcap_file = self._get_backbone_pcap_filename()
@@ -565,12 +592,13 @@ def _start_backbone_sniffer(self):
             pass
 
         dumpcap = pvutils.which_dumpcap()
-        self._dumpcap_proc = subprocess.Popen([dumpcap, '-i', config.BACKBONE_DOCKER_NETWORK_NAME, '-w', pcap_file],
+        self._dumpcap_proc = subprocess.Popen([dumpcap, '-i', sniffer_interface, '-w', pcap_file],
                                               stdout=sys.stdout,
                                               stderr=sys.stderr)
         time.sleep(0.2)
         assert self._dumpcap_proc.poll() is None, 'tshark terminated unexpectedly'
-        logging.info('Backbone sniffer launched successfully: pid=%s', self._dumpcap_proc.pid)
+        logging.info('Backbone sniffer launched successfully on interface %s, pid=%s', sniffer_interface,
+                     self._dumpcap_proc.pid)
         os.chmod(pcap_file, stat.S_IWUSR | stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH)
 
     def _get_backbone_pcap_filename(self):