Merge branch 'QubesOS:main' into main

dom0-updates/qfile-dom0-unpacker.c

@@ -14,7 +14,7 @@
 #include <libqubes-rpc-filecopy.h>
 
 #define DEFAULT_MAX_UPDATES_BYTES (4LL<<30)
-#define DEFAULT_MAX_UPDATES_FILES 2048
+#define DEFAULT_MAX_UPDATES_FILES 4096
 
 #define min(a,b) ((a) < (b) ? (a) : (b))
 #define max(a,b) ((a) > (b) ? (a) : (b))

rpm_spec/core-dom0-linux.spec.in

@@ -116,6 +116,10 @@ install -m 0644 qubes-rpc-policy/90-default-linux.policy \
 
 install -d $RPM_BUILD_ROOT/var/lib/qubes/updates
 
+# PipeWire workaround
+install -d -- "$RPM_BUILD_ROOT/usr/share/pipewire/pipewire.conf.d/"
+install -m 0644 -- system-config/10_pipewire-high-latency.conf "$RPM_BUILD_ROOT/usr/share/pipewire/pipewire.conf.d/"
+
 # Qrexec services
 mkdir -p $RPM_BUILD_ROOT/usr/lib/qubes/qubes-rpc
 cp qubes-rpc/* $RPM_BUILD_ROOT/usr/lib/qubes/qubes-rpc/
@@ -302,6 +306,7 @@ chmod -x /etc/grub.d/10_linux
 /etc/sudoers.d/qubes
 /etc/polkit-1/rules.d/00-qubes-allow-all.rules
 /etc/security/limits.d/99-qubes.conf
+/usr/share/pipewire/pipewire.conf.d/10_pipewire-high-latency.conf
 %_udevrulesdir/00-qubes-ignore-devices.rules
 %_udevrulesdir/12-qubes-ignore-lvm-devices.rules
 %_udevrulesdir/11-qubes-ignore-zvol-devices.rules

system-config/10_pipewire-high-latency.conf

@@ -0,0 +1,6 @@
+# In dom0, PipeWire doesn't realize it is running in a VM.
+# Therefore, it chooses low quantum values and xruns due
+# to Xen descheduling dom0.
+context.properties = {
+    default.clock.min-quantum = 1024
+}

version

@@ -1 +1 @@
-4.2.16
+4.2.20

vmupdate/agent/source/apt/apt_api.py

@@ -20,14 +20,12 @@
 # USA.
 
 import os
-import logging
 from pathlib import Path
 
 import apt
 import apt.progress.base
 import apt_pkg
 
-from source.common.stream_redirector import StreamRedirector
 from source.common.process_result import ProcessResult
 from source.common.progress_reporter import ProgressReporter, Progress
 
@@ -38,7 +36,8 @@ class APT(APTCLI):
     def __init__(self, log_handler, log_level):
         super().__init__(log_handler, log_level)
         self.apt_cache = apt.Cache()
-        update = FetchProgress(weight=4, log=self.log)  # 4% of total time
+        update = FetchProgress(
+            weight=4, log=self.log, refresh=True)  # 4% of total time
         fetch = FetchProgress(weight=48, log=self.log)  # 48% of total time
         upgrade = UpgradeProgress(weight=48, log=self.log)  # 48% of total time
         self.progress = ProgressReporter(update, fetch, upgrade)
@@ -55,18 +54,17 @@ def refresh(self, hard_fail: bool) -> ProcessResult:
         """
         result = ProcessResult()
         try:
-            with StreamRedirector(result):
-                self.log.debug("Refreshing available packages...")
-                success = self.apt_cache.update(
-                    self.progress.update_progress,
-                    pulse_interval=1000  # microseconds
-                )
-                self.apt_cache.open()
-                if success:
-                    self.log.debug("Cache refresh successful.")
-                else:
-                    self.log.warning("Cache refresh failed.")
-                    result += ProcessResult(1)
+            self.log.debug("Refreshing available packages...")
+            success = self.apt_cache.update(
+                self.progress.update_progress,
+                pulse_interval=1000  # microseconds
+            )
+            self.apt_cache.open()
+            if success:
+                self.log.debug("Cache refresh successful.")
+            else:
+                self.log.warning("Cache refresh failed.")
+                result += ProcessResult(1)
         except Exception as exc:
             self.log.error(
                 "An error occurred while refreshing packages: %s", str(exc))
@@ -78,21 +76,19 @@ def upgrade_internal(self, remove_obsolete: bool) -> ProcessResult:
         """
         Use `apt` package to upgrade and track progress.
         """
-
         result = ProcessResult()
         try:
             self.log.debug("Performing package upgrade...")
             self.apt_cache.upgrade(dist_upgrade=remove_obsolete)
             Path(os.path.join(
                 apt_pkg.config.find_dir("Dir::Cache::Archives"), "partial")
             ).mkdir(parents=True, exist_ok=True)
-            with StreamRedirector(result):
-                self.log.debug("Committing upgrade...")
-                self.apt_cache.commit(
-                    self.progress.fetch_progress,
-                    self.progress.upgrade_progress
-                )
-                self.log.debug("Package upgrade successful.")
+            self.log.debug("Committing upgrade...")
+            self.apt_cache.commit(
+                self.progress.fetch_progress,
+                self.progress.upgrade_progress
+            )
+            self.log.debug("Package upgrade successful.")
         except Exception as exc:
             self.log.error(
                 "An error occurred while upgrading packages: %s", str(exc))
@@ -102,15 +98,16 @@ def upgrade_internal(self, remove_obsolete: bool) -> ProcessResult:
 
 
 class FetchProgress(apt.progress.base.AcquireProgress, Progress):
-    def __init__(self, weight: int, log):
+    def __init__(self, weight: int, log, refresh: bool = False):
         Progress.__init__(self, weight, log)
+        self.action = "refresh" if refresh else "fetch"
 
     def fail(self, item):
         """
         Write an error message to the fake stderr.
         """
-        self.log.info("Fetch failed.")
-        print(f"Fail to fetch {item.shortdesc}: "
+        self.log.info(f"{self.action.capitalize()} failed.")
+        print(f"Fail to {self.action} {item.shortdesc}: "
               f"{item.description} from {item.uri}",
               flush=True, file=self._stderr)
 
@@ -127,13 +124,15 @@ def pulse(self, _owner):
 
     def start(self):
         """Invoked when the Acquire process starts running."""
-        self.log.info("Fetch started.")
+        self.log.info(f"{self.action.capitalize()} started.")
+        print(f"{self.action.capitalize()}ing packages.", flush=True)
         super().start()
         self.notify_callback(0)
 
     def stop(self):
         """Invoked when the Acquire process stops running."""
-        self.log.info("Fetch ended.")
+        self.log.info(f"{self.action.capitalize()} ended.")
+        print(f"{self.action.capitalize()}ed.", flush=True)
         super().stop()
         self.notify_callback(100)
 
@@ -157,9 +156,11 @@ def error(self, pkg, errormsg):
               flush=True, file=self._stderr)
 
     def start_update(self):
+        print("Updating packages.", flush=True)
         super().start_update()
         self.notify_callback(0)
 
     def finish_update(self):
+        print("Updated.", flush=True)
         super().finish_update()
         self.notify_callback(100)

vmupdate/agent/source/apt/apt_cli.py

@@ -45,6 +45,10 @@ def refresh(self, hard_fail: bool) -> ProcessResult:
         """
         cmd = [self.package_manager, "-q", "update"]
         result = self.run_cmd(cmd)
+        # 'apt-get update' reports error with exit code 100, but updater as a
+        # whole reserves it for "no updates"
+        if result.code == 100:
+            result.code = 1
         result.error_from_messages()
         return result
 
@@ -60,7 +64,7 @@ def get_packages(self):
         ]
         # EXAMPLE OUTPUT:
         # install ok installed qubes-core-agent 4.1.35-1+deb11u1
-        result = self.run_cmd(cmd)
+        result = self.run_cmd(cmd, realtime=False)
 
         packages = {}
         for line in result.out.splitlines():

vmupdate/agent/source/common/package_manager.py

@@ -56,7 +56,7 @@ def upgrade(
         result = self._upgrade(
             refresh, hard_fail, remove_obsolete, self.requirements)
         self._log_output("agent", result)
-        if print_streams:
+        if print_streams and not result.posted:
             if result.out:
                 print(result.out, flush=True)
             if result.err:
@@ -70,11 +70,12 @@ def _upgrade(
             remove_obsolete: bool,
             requirements: Optional[Dict[str, str]] = None
     ) -> ProcessResult:
-        result = ProcessResult()
+        result = ProcessResult(realtime=True)
 
         curr_pkg = self.get_packages()
 
         if requirements:
+            print("Install requirements", flush=True)
             result_install = self.install_requirements(requirements, curr_pkg)
             result += result_install
             if result.code != 0:
@@ -85,6 +86,7 @@ def _upgrade(
                     return result
 
         if refresh:
+            print("Refreshing package info", flush=True)
             result_refresh = self.refresh(hard_fail)
             result += result_refresh
             if result.code != 0:
@@ -101,7 +103,7 @@ def _upgrade(
 
         changes = PackageManager.compare_packages(old=curr_pkg, new=new_pkg)
         summary = self._print_changes(changes)
-        result.out += "\n" + summary
+        result += summary
 
         if not result.code and not (changes["installed"] or changes["updated"]):
             result.code = 100  # Nothing to upgrade
@@ -132,7 +134,7 @@ def install_requirements(
         if requirements is None:
             requirements = {}
 
-        result = ProcessResult()
+        result = ProcessResult(realtime=True)
         to_install = []  # install latest (ignore version)
         to_upgrade = {}
         for pkg, version in requirements.items():
@@ -162,18 +164,25 @@ def install_requirements(
 
         return result
 
-    def run_cmd(self, command: List[str]) -> ProcessResult:
+    def run_cmd(
+            self, command: List[str], realtime: bool = True) -> ProcessResult:
         """
         Run command and wait.
 
         :param command: command to execute
+        :param realtime: write directly to stdout/stderr
         """
         self.log.debug("run command: %s", " ".join(command))
-        with subprocess.Popen(command,
-                              stdin=subprocess.PIPE,
-                              stdout=subprocess.PIPE,
-                              stderr=subprocess.PIPE) as proc:
-            result = ProcessResult.process_communicate(proc)
+        if realtime:
+            with subprocess.Popen(command, stdin=subprocess.PIPE) as proc:
+                result = ProcessResult.process_communicate(proc)
+                result.posted = True
+        else:
+            with subprocess.Popen(command,
+                                  stdin=subprocess.PIPE,
+                                  stdout=subprocess.PIPE,
+                                  stderr=subprocess.PIPE) as proc:
+                result = ProcessResult.process_communicate(proc)
         self.log.debug("command exit code: %i", result.code)
 
         return result
@@ -194,31 +203,33 @@ def compare_packages(old, new):
                 "removed": {pkg: old[pkg] for pkg in old if pkg not in new}}
 
     def _print_changes(self, changes):
-        result = ""
-        result += self._print_to_string("Installed packages:")
+        result = ProcessResult()
+        result.out += self._print_to_string("Installed packages:")
         if changes["installed"]:
             for pkg in changes["installed"]:
-                result += self._print_to_string(pkg, changes["installed"][pkg])
+                result.out += self._print_to_string(
+                    pkg, changes["installed"][pkg])
         else:
-            result += self._print_to_string("None")
+            result.out += self._print_to_string("None")
 
-        result += self._print_to_string("Updated packages:")
+        result.out += self._print_to_string("Updated packages:")
         if changes["updated"]:
             for pkg in changes["updated"]:
-                result += self._print_to_string(
+                result.out += self._print_to_string(
                     pkg,
                     str(changes["updated"][pkg]["old"])[2:-2]
                     + " -> " +
                     str(changes["updated"][pkg]["new"])[2:-2])
         else:
-            result += self._print_to_string("None")
+            result.out += self._print_to_string("None")
 
-        result += self._print_to_string("Removed packages:")
+        result.out += self._print_to_string("Removed packages:")
         if changes["removed"]:
             for pkg in changes["removed"]:
-                result += self._print_to_string(pkg, changes["removed"][pkg])
+                result.out += self._print_to_string(
+                    pkg, changes["removed"][pkg])
         else:
-            result += self._print_to_string("None")
+            result.out += self._print_to_string("None")
         return result
 
     @staticmethod