use less space on qvm template install (issue 8876)

.pylintrc

@@ -192,6 +192,6 @@ max-public-methods=100
 
 # Exceptions that will emit a warning when being caught. Defaults to
 # "Exception"
-overgeneral-exceptions=Exception,EnvironmentError
+overgeneral-exceptions=builtins.Exception,builtins.EnvironmentError
 
 # vim: ft=conf

qubesadmin/base.py

@@ -459,8 +459,7 @@ def __contains__(self, item):
 
     def __iter__(self):
         self.refresh_cache()
-        for key in self._names_list:
-            yield key
+        yield from self._names_list
 
     def keys(self):
         '''Get list of names.'''

qubesadmin/tests/tools/qvm_template.py

@@ -1,11 +1,9 @@
 import re
 from unittest import mock, skipUnless
 import argparse
-import asyncio
 import datetime
 import io
 import os
-import pathlib
 import subprocess
 import tempfile
 from shutil import which
@@ -196,13 +194,15 @@ def test_004_verify_rpm_badname(self, mock_proc, mock_call, mock_ts):
         mock_ts.assert_called_once()
         self.assertAllCalled()
 
+    @mock.patch('os.path.exists')
     @mock.patch('subprocess.Popen')
-    def test_010_extract_rpm_success(self, mock_popen):
+    def test_010_extract_rpm_success(self, mock_popen, mock_path_exists):
         mock_popen.return_value.__enter__.return_value = mock_popen.return_value
         pipe = mock.Mock()
         mock_popen.return_value.stdout = pipe
         mock_popen.return_value.wait.return_value = 0
         mock_popen.return_value.returncode = 0
+        mock_path_exists.return_value = True
         with tempfile.NamedTemporaryFile() as fd, \
                 tempfile.TemporaryDirectory() as dir:
             path = fd.name
@@ -220,45 +220,101 @@ def test_010_extract_rpm_success(self, mock_popen):
                     'xz',
                     '-C',
                     dirpath,
-                    './var/lib/qubes/vm-templates/test-vm/'
+                    './var/lib/qubes/vm-templates/test-vm/',
+                    '--exclude=root.img.part.?[!0]',
+                    '--exclude=root.img.part.[!0]0',
                 ], stdin=pipe, stdout=subprocess.DEVNULL),
             mock.call().__enter__(),
             mock.call().__exit__(None, None, None),
             mock.call().__exit__(None, None, None),
-        ])
-        self.assertAllCalled()
-
-    @mock.patch('subprocess.Popen')
-    def test_011_extract_rpm_fail(self, mock_popen):
-        mock_popen.return_value.__enter__.return_value = mock_popen.return_value
-        pipe = mock.Mock()
-        mock_popen.return_value.stdout = pipe
-        mock_popen.return_value.returncode = 1
-        with tempfile.NamedTemporaryFile() as fd, \
-                tempfile.TemporaryDirectory() as dir:
-            path = fd.name
-            dirpath = dir
-            ret = qubesadmin.tools.qvm_template.extract_rpm(
-                'test-vm', path, dirpath)
-        self.assertEqual(ret, False)
-        self.assertEqual(mock_popen.mock_calls, [
-            mock.call(['rpm2archive', '-'],
-                stdin=mock.ANY,
-                stdout=subprocess.PIPE),
-            mock.call().__enter__(),
             mock.call([
-                    'tar',
-                    'xz',
-                    '-C',
-                    dirpath,
-                    './var/lib/qubes/vm-templates/test-vm/'
-                ], stdin=pipe, stdout=subprocess.DEVNULL),
+                'truncate',
+                '--size=512',
+                dirpath + '//var/lib/qubes/vm-templates/test-vm/root.img.part.00'
+            ]),
             mock.call().__enter__(),
             mock.call().__exit__(None, None, None),
+            mock.call([
+                'ln',
+                '-s',
+                path,
+                dirpath + '//var/lib/qubes/vm-templates/test-vm/template.rpm'
+            ]),
+            mock.call().__enter__(),
             mock.call().__exit__(None, None, None),
         ])
         self.assertAllCalled()
 
+    @mock.patch('os.path.exists')
+    @mock.patch('subprocess.Popen')
+    def test_011_extract_rpm_fail(self, mock_popen, mock_path_exists):
+        for failing_call in range(1, 5):
+            mock_popen.reset_mock()
+            with self.subTest(failing_call=failing_call):
+                pipe = mock.Mock()
+
+                def side_effect(_, **__):
+                    side_effect.call_count += 1
+                    o = mock_popen.return_value
+                    o.__enter__.return_value = o
+                    o.stdout = pipe
+                    o.returncode = (
+                        1 if side_effect.call_count >= failing_call else
+                        0
+                    )
+                    return o
+
+                side_effect.call_count = 0
+
+                mock_popen.side_effect = side_effect
+                mock_path_exists.return_value = True
+
+                with tempfile.NamedTemporaryFile() as fd, \
+                        tempfile.TemporaryDirectory() as tmpdir:
+                    path = fd.name
+                    dirpath = tmpdir
+                    ret = qubesadmin.tools.qvm_template.extract_rpm(
+                        'test-vm', path, dirpath)
+                self.assertEqual(ret, False)
+                self.assertEqual(mock_popen.mock_calls, [
+                    mock.call(['rpm2archive', '-'],
+                        stdin=mock.ANY,
+                        stdout=subprocess.PIPE),
+                    mock.call().__enter__(),
+                    mock.call([
+                            'tar',
+                            'xz',
+                            '-C',
+                            dirpath,
+                            './var/lib/qubes/vm-templates/test-vm/',
+                            '--exclude=root.img.part.?[!0]',
+                            '--exclude=root.img.part.[!0]0',
+                        ], stdin=pipe, stdout=subprocess.DEVNULL),
+                    mock.call().__enter__(),
+                    mock.call().__exit__(None, None, None),
+                    mock.call().__exit__(None, None, None),
+                ] + ([] if failing_call < 3 else [
+                    mock.call([
+                        'truncate',
+                        '--size=512',
+                        dirpath
+                        + '//var/lib/qubes/vm-templates/test-vm/root.img.part.00'
+                    ]),
+                    mock.call().__enter__(),
+                    mock.call().__exit__(None, None, None),
+                ]) + ([] if failing_call < 4 else [
+                    mock.call([
+                        'ln',
+                        '-s',
+                        path,
+                        dirpath
+                        + '//var/lib/qubes/vm-templates/test-vm/template.rpm'
+                    ]),
+                    mock.call().__enter__(),
+                    mock.call().__exit__(None, None, None),
+                ]))
+                self.assertAllCalled()
+
     @mock.patch('qubesadmin.tools.qvm_template.get_keys_for_repos')
     def test_090_install_lock(self, mock_get_keys):
         class SuccessError(Exception):

qubesadmin/tests/tools/qvm_template_postprocess.py

@@ -25,6 +25,7 @@
 from unittest import mock
 import qubesadmin.tests
 import qubesadmin.tools.qvm_template_postprocess
+from qubesadmin.exc import QubesException
 
 
 class QubesLocalMock(qubesadmin.tests.QubesTest):
@@ -67,6 +68,37 @@ def test_000_import_root_img_raw(self):
             vm, self.source_dir.name)
         self.assertAllCalled()
 
+    def test_001_import_root_img_tar_pre_mar_2024(self):
+        root_img = os.path.join(self.source_dir.name, 'root.img')
+        volume_data = b'volume data' * 1000
+        with open(root_img, 'wb') as f:
+            f.write(volume_data)
+
+        subprocess.check_call(['tar', 'cf', 'root.img.tar', 'root.img'],
+            cwd=self.source_dir.name)
+        subprocess.check_call(['split', '-d', '-b', '1024', 'root.img.tar',
+            'root.img.part.'], cwd=self.source_dir.name)
+        os.unlink(root_img)
+
+        self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \
+            b'0\0test-vm class=TemplateVM state=Halted\n'
+        self.app.expected_calls[('test-vm', 'admin.vm.volume.List', None,
+                None)] = \
+            b'0\0root\nprivate\nvolatile\nkernel\n'
+
+        self.app.expected_calls[(
+            'test-vm', 'admin.vm.volume.ImportWithSize', 'root',
+            str(len(volume_data)).encode() + b'\n' + volume_data)] = b'0\0'
+        vm = self.app.domains['test-vm']
+        try:
+            qubesadmin.tools.qvm_template_postprocess.import_root_img(
+                vm, self.source_dir.name)
+        except QubesException as e:
+            assert str(e).startswith(
+                'template.rpm symlink not found for multi-part image')
+        else:
+            assert False
+
     def test_001_import_root_img_tar(self):
         root_img = os.path.join(self.source_dir.name, 'root.img')
         volume_data = b'volume data' * 1000
@@ -79,6 +111,40 @@ def test_001_import_root_img_tar(self):
             'root.img.part.'], cwd=self.source_dir.name)
         os.unlink(root_img)
 
+        spec = os.path.join(self.source_dir.name, 'template.spec')
+        with open(spec, 'w') as f:
+            f.writelines((
+                '%define _rpmdir %{expand:%%(pwd)}/build\n',
+                'Name: test\n',
+                'Summary: testing\n',
+                'License: none\n',
+                'Version: 6.6.6\n',
+                'Release: 44\n',
+
+                '%description\n',
+                'test\n',
+
+                '%prep\n',
+                'mkdir -p $RPM_BUILD_ROOT\n',
+                'mv %{expand:%%(pwd)}/root.img.part.* $RPM_BUILD_ROOT\n',
+                'dd',
+                ' if=$RPM_BUILD_ROOT/root.img.part.00',
+                ' count=1',
+                ' of=%{expand:%%(pwd)}/root.img.part.00\n',
+                'ln -s',
+                ' %{expand:%%(pwd)}/build/i386/test-6.6.6-44.i386.rpm',
+                ' %{expand:%%(pwd)}/template.rpm\n',
+
+                '%files\n',
+                '/root.img.part.*\n',
+            ))
+        subprocess.check_call([
+                'rpmbuild', '-bb', '--rmspec', '--target', 'i386-redhat-linux',
+                '--clean', '-D', f'_topdir {self.source_dir.name}', spec
+            ],
+            cwd=self.source_dir.name,
+            stdout=subprocess.DEVNULL)
+
         self.app.expected_calls[('dom0', 'admin.vm.List', None, None)] = \
             b'0\0test-vm class=TemplateVM state=Halted\n'
         self.app.expected_calls[('test-vm', 'admin.vm.volume.List', None,

qubesadmin/tools/qvm_template.py

@@ -58,6 +58,7 @@
 UNVERIFIED_SUFFIX = '.unverified'
 LOCK_FILE = '/var/tmp/qvm-template.lck'
 DATE_FMT = '%Y-%m-%d %H:%M:%S'
+TAR_HEADER_BYTES = 512
 
 UPDATEVM = str('global UpdateVM')
 
@@ -733,6 +734,9 @@ def verify_rpm(path: str, key: str, *, nogpgcheck: bool = False,
 
 def extract_rpm(name: str, path: str, target: str) -> bool:
     """Extract a template RPM package.
+       If the package contains root.img file split across multiple parts,
+       only the first 512 bytes of the 00 part is retained (tar header) and
+       a symlink to the rpm file is created in target directory.
 
     :param name: Name of the template
     :param path: Location of the RPM package
@@ -745,11 +749,32 @@ def extract_rpm(name: str, path: str, target: str) -> bool:
                 stdin=pkg_f,
                 stdout=subprocess.PIPE) as rpm2archive:
             # `-D` is GNUism
-            with subprocess.Popen(
-                    ['tar', 'xz', '-C', target, f'.{PATH_PREFIX}/{name}/'],
-                    stdin=rpm2archive.stdout, stdout=subprocess.DEVNULL) as tar:
+            with subprocess.Popen([
+                'tar', 'xz', '-C', target, f'.{PATH_PREFIX}/{name}/',
+                '--exclude=root.img.part.?[!0]',
+                '--exclude=root.img.part.[!0]0'
+            ], stdin=rpm2archive.stdout, stdout=subprocess.DEVNULL) as tar:
                 pass
-    return rpm2archive.returncode == 0 and tar.returncode == 0
+    if rpm2archive.returncode != 0 or tar.returncode != 0:
+        return False
+
+    part_00_path = f'{target}/{PATH_PREFIX}/{name}/root.img.part.00'
+    if os.path.exists(part_00_path):
+        # retain minimal data needed to interrogate root.img size
+        with subprocess.Popen([
+            'truncate', f'--size={TAR_HEADER_BYTES}', part_00_path
+        ]) as truncate:
+            pass
+        if truncate.returncode != 0:
+            return False
+        # and create rpm file symlink
+        with subprocess.Popen([
+            'ln', '-s', path, f'{target}/{PATH_PREFIX}/{name}/template.rpm'
+        ]) as symlink:
+            pass
+        if symlink.returncode != 0:
+            return False
+    return True
 
 
 def filter_version(

qubesadmin/tools/qvm_template_postprocess.py

@@ -97,19 +97,37 @@ def import_root_img(vm, source_dir):
 
     root_path = os.path.join(source_dir, 'root.img')
     if os.path.exists(root_path + '.part.00'):
-        input_files = glob.glob(root_path + '.part.*')
-        with subprocess.Popen(['cat'] + sorted(input_files),
-                stdout=subprocess.PIPE) as cat:
-            with subprocess.Popen(['tar', 'xSOf', '-'],
-                    stdin=cat.stdout,
-                    stdout=subprocess.PIPE) as tar:
-                cat.stdout.close()
-                vm.volumes['root'].import_data_with_size(
-                    stream=tar.stdout, size=root_size)
-        if tar.returncode != 0:
+        rpm_symlink = os.path.join(source_dir, 'template.rpm')
+        if not os.path.exists(rpm_symlink) or not os.path.islink(rpm_symlink):
             raise qubesadmin.exc.QubesException(
-                'root.img extraction failed')
-        if cat.returncode != 0:
+                'template.rpm symlink not found for multi-part image, ' +
+                'using up-to-date `qvm-template install ...` should help')
+        with open(rpm_symlink, 'rb') as pkg_f:
+            # note: part files assumed to be in proper order, which is OK
+            #    (generated using an RPM spec file with a glob pattern
+            #    POSIX-required to sort matching files + tar preserves order)
+            with subprocess.Popen(
+                ['rpm2archive', '-'],
+                stdin=pkg_f,
+                stdout=subprocess.PIPE
+            ) as rpm2archive:
+                with subprocess.Popen(
+                    ['tar', 'xzSOf', '-', '--wildcards', '*/root.img.part.*'],
+                    stdin=rpm2archive.stdout,
+                    stdout=subprocess.PIPE
+                ) as tar_parts:
+                    with subprocess.Popen(['tar', 'xSOf', '-'],
+                            stdin=tar_parts.stdout,
+                            stdout=subprocess.PIPE) as tar_root_img:
+                        rpm2archive.stdout.close()
+                        tar_parts.stdout.close()
+                        vm.volumes['root'].import_data_with_size(
+                            stream=tar_root_img.stdout, size=root_size)
+        if (
+            rpm2archive.returncode != 0 or
+            tar_parts.returncode != 0 or
+            tar_root_img.returncode != 0
+        ):
             raise qubesadmin.exc.QubesException(
                 'root.img extraction failed')
     elif os.path.exists(root_path + '.tar'):
@@ -272,7 +290,7 @@ async def post_install(args):
         vm = app.domains[args.name]
         if app.qubesd_connection_type == 'socket' and \
                 args.dir == '/var/lib/qubes/vm-templates/' + args.name:
-            # VM exists and use use the same directory as target vm - on
+            # VM exists and uses the same directory as target vm - on
             # final cleanup remove only some files, not the whole directory
             local_reinstall = True
     except KeyError: