qvm-template: fallback to other mirrors

If download from a selected mirror failed to start at all (missing file,
connection error etc), fallback to another mirror. This requires getting
all the mirror URLs from dnf, and the default dnf's 'download' plugin
does not expose this information. Use a modified plugin for this
purpose.

debian/qubes-core-agent-dom0-updates.install

@@ -2,4 +2,4 @@ etc/qubes-rpc/qubes.TemplateSearch
 etc/qubes-rpc/qubes.TemplateDownload
 usr/lib/qubes/qvm-template-repo-query
 usr/lib/qubes/qubes-download-dom0-updates.sh
-usr/lib/qubes/dnf-plugins/download.py
+usr/lib/qubes/dnf-plugins/downloadurl.py

package-managers/Makefile

@@ -42,10 +42,8 @@ install:
 		upgrades-installed-check \
 		upgrades-status-notify
 	install -d -m 2775 $(DESTDIR)$(QUBESSTATEDIR)/dom0-updates
-ifneq (,$(filter $(DIST),buster bullseye bookworm bionic focal))
-	install -D -m 0644 dnf-plugin-download.py \
-		$(DESTDIR)$(QUBESLIBDIR)/dnf-plugins/download.py
-endif
+	install -D -m 0644 dnf-plugin-downloadurl.py \
+		$(DESTDIR)$(QUBESLIBDIR)/dnf-plugins/downloadurl.py
 
 install-apt:
 	install -d $(DESTDIR)$(APTCONFDIR)/sources.list.d

package-managers/dnf-plugin-download.py → package-managers/dnf-plugin-downloadurl.py

@@ -1,4 +1,4 @@
-# download.py, supplies the 'download' command.
+# download.py, supplies the 'downloadurl' command.
 #
 # Copyright (C) 2013-2015  Red Hat, Inc.
 #
@@ -41,8 +41,8 @@
 @dnf.plugin.register_command
 class DownloadCommand(dnf.cli.Command):
 
-    aliases = ['download']
-    summary = _('Download package to current directory')
+    aliases = ['downloadurl']
+    summary = _('Download package to current directory or print URLs')
 
     def __init__(self, cli):
         super(DownloadCommand, self).__init__(cli)
@@ -70,13 +70,21 @@ def set_argparser(parser):
         parser.add_argument('--url', '--urls', action='store_true', dest='url',
                             help=_('print list of urls where the rpms '
                                    'can be downloaded instead of downloading'))
+        parser.add_argument('--all-mirrors', action='store_true', dest='all_mirrors',
+                            help=_('with --url, print all mirrors of a single package'))
         parser.add_argument('--urlprotocols', action='append',
                             choices=['http', 'https', 'rsync', 'ftp'],
                             default=[],
                             help=_('when running with --url, '
                                    'limit to specific protocols'))
 
     def configure(self):
+        if self.opts.all_mirrors and len(self.opts.packages) > 1:
+            raise dnf.exceptions.Error(
+                    "Only one package can be given with --all-mirrors")
+        if self.opts.all_mirrors and self.opts.resolve:
+            raise dnf.exceptions.Error(
+                    "--all-mirrors cannot be used with --resolve")
         # setup sack and populate it with enabled repos
         demands = self.cli.demands
         demands.sack_activation = True
@@ -113,19 +121,38 @@ def run(self):
             if self.opts.debugsource:
                 pkgs.extend(self._get_pkg_objs_debugsource(self.opts.packages))
 
+        def schemes_filter(url_list):
+            for url in url_list:
+                if schemes:
+                    s = urllib.parse.urlparse(url)[0]
+                    if s in schemes:
+                        return os.path.join(url, location.lstrip('/'))
+                else:
+                    return os.path.join(url, location.lstrip('/'))
+            return None
+
         # If user asked for just urls then print them and we're done
         if self.opts.url:
             for pkg in pkgs:
                 # command line repo packages do not have .remote_location
                 if pkg.repoid != hawkey.CMDLINE_REPO_NAME:
-                    url = pkg.remote_location(schemes=self.opts.urlprotocols)
-                    if url:
-                        print(url)
+                    if self.opts.all_mirrors:
+                        schemas = self.opts.urlprotocols
+                        # pylint: disable=protected-access
+                        for mirror in pkgs.repo._repo.getMirrors():
+                            if schemas:
+                                if urllib.parse.urlparse(mirror)[0] not in schemas:
+                                    continue
+                            print(os.path.join(url, pkg.location.lstrip('/')))
                     else:
-                        msg = _("Failed to get mirror for package: %s") % pkg.name
-                        if self.base.conf.strict:
-                            raise dnf.exceptions.Error(msg)
-                        logger.warning(msg)
+                        url = pkg.remote_location(schemes=self.opts.urlprotocols)
+                        if url:
+                            print(url)
+                        else:
+                            msg = _("Failed to get mirror for package: %s") % pkg.name
+                            if self.base.conf.strict:
+                                raise dnf.exceptions.Error(msg)
+                            logger.warning(msg)
             return
         else:
             self._do_downloads(pkgs)  # download rpms

qubes-rpc/qvm-template-repo-query

@@ -29,11 +29,9 @@ cat > "$repodir/template.repo"
 OPTS+=("--setopt=reposdir=${repodir}")
 OPTS+=("--quiet")
 
-if [ -d '/usr/lib/qubes/dnf-plugins' ]; then
-    # use vendored 'download' dnf-plugin when available, to not require
-    # dnf-plugins-core package (not available in Debian yet)
-    OPTS+=("--setopt=pluginpath=/usr/lib/qubes/dnf-plugins")
-fi
+# use vendored 'downloadurl' dnf-plugin (fork of 'download' plugin), to print
+# all mirrors
+OPTS+=("--setopt=pluginpath=/usr/lib/qubes/dnf-plugins")
 
 if ! command -v dnf >/dev/null; then
     echo "ERROR: dnf command is missing, please use newer template for your UpdateVM to download templates." >&2
@@ -60,8 +58,7 @@ elif [ "$1" = "download" ]; then
     # downloaded - retry from the same one. If download failed and nothing was
     # downloaded, go to the next one. The intention is to retry on interrupted
     # connection, but skip mirrors that are not synchronized yet.
-    # FIXME: 'dnf download --url' prints only one URL, not all the mirrors
-    urls="$(dnf download "${OPTS[@]}" --url "$SPEC" | shuf)"
+    urls="$(dnf downloadurl "${OPTS[@]}" --url --all-mirrors "$SPEC" | shuf)"
     readarray -t urls <<<"$urls"
     downloaded=0
     status_file="$repodir/download-status.tmp"

rpm_spec/core-agent.spec.in

@@ -812,6 +812,7 @@ rm -f %{name}-%{version}
 %dir %attr(0755,user,user) /var/lib/qubes/dom0-updates
 /usr/lib/qubes/qvm-template-repo-query
 /usr/lib/qubes/qubes-download-dom0-updates.sh
+/usr/lib/qubes/dnf-plugins/downloadurl.py
 
 %files networking
 %config(noreplace) /etc/qubes-rpc/qubes.UpdatesProxy