Merge remote-tracking branch 'origin/pr/393'

* origin/pr/393:
  qvm-template: fallback to other mirrors

debian/qubes-core-agent-dom0-updates.install

@@ -2,4 +2,4 @@ etc/qubes-rpc/qubes.TemplateSearch
 etc/qubes-rpc/qubes.TemplateDownload
 usr/lib/qubes/qvm-template-repo-query
 usr/lib/qubes/qubes-download-dom0-updates.sh
-usr/lib/qubes/dnf-plugins/download.py
+usr/lib/qubes/dnf-plugins/downloadurl.py

package-managers/Makefile

@@ -42,10 +42,8 @@ install:
 		upgrades-installed-check \
 		upgrades-status-notify
 	install -d -m 2775 $(DESTDIR)$(QUBESSTATEDIR)/dom0-updates
-ifneq (,$(filter $(DIST),buster bullseye bookworm bionic focal))
-	install -D -m 0644 dnf-plugin-download.py \
-		$(DESTDIR)$(QUBESLIBDIR)/dnf-plugins/download.py
-endif
+	install -D -m 0644 dnf-plugin-downloadurl.py \
+		$(DESTDIR)$(QUBESLIBDIR)/dnf-plugins/downloadurl.py
 
 install-apt:
 	install -d $(DESTDIR)$(APTCONFDIR)/sources.list.d

package-managers/dnf-plugin-download.py → package-managers/dnf-plugin-downloadurl.py

@@ -1,4 +1,4 @@
-# download.py, supplies the 'download' command.
+# download.py, supplies the 'downloadurl' command.
 #
 # Copyright (C) 2013-2015  Red Hat, Inc.
 #
@@ -41,8 +41,8 @@
 @dnf.plugin.register_command
 class DownloadCommand(dnf.cli.Command):
 
-    aliases = ['download']
-    summary = _('Download package to current directory')
+    aliases = ['downloadurl']
+    summary = _('Download package to current directory or print URLs')
 
     def __init__(self, cli):
         super(DownloadCommand, self).__init__(cli)
@@ -70,13 +70,21 @@ def set_argparser(parser):
         parser.add_argument('--url', '--urls', action='store_true', dest='url',
                             help=_('print list of urls where the rpms '
                                    'can be downloaded instead of downloading'))
+        parser.add_argument('--all-mirrors', action='store_true', dest='all_mirrors',
+                            help=_('with --url, print all mirrors of a single package'))
         parser.add_argument('--urlprotocols', action='append',
                             choices=['http', 'https', 'rsync', 'ftp'],
                             default=[],
                             help=_('when running with --url, '
                                    'limit to specific protocols'))
 
     def configure(self):
+        if self.opts.all_mirrors and len(self.opts.packages) > 1:
+            raise dnf.exceptions.Error(
+                    "Only one package can be given with --all-mirrors")
+        if self.opts.all_mirrors and self.opts.resolve:
+            raise dnf.exceptions.Error(
+                    "--all-mirrors cannot be used with --resolve")
         # setup sack and populate it with enabled repos
         demands = self.cli.demands
         demands.sack_activation = True
@@ -113,19 +121,38 @@ def run(self):
             if self.opts.debugsource:
                 pkgs.extend(self._get_pkg_objs_debugsource(self.opts.packages))
 
+        def schemes_filter(url_list):
+            for url in url_list:
+                if schemes:
+                    s = urllib.parse.urlparse(url)[0]
+                    if s in schemes:
+                        return os.path.join(url, location.lstrip('/'))
+                else:
+                    return os.path.join(url, location.lstrip('/'))
+            return None
+
         # If user asked for just urls then print them and we're done
         if self.opts.url:
             for pkg in pkgs:
                 # command line repo packages do not have .remote_location
                 if pkg.repoid != hawkey.CMDLINE_REPO_NAME:
-                    url = pkg.remote_location(schemes=self.opts.urlprotocols)
-                    if url:
-                        print(url)
+                    if self.opts.all_mirrors:
+                        schemas = self.opts.urlprotocols
+                        # pylint: disable=protected-access
+                        for mirror in pkgs.repo._repo.getMirrors():
+                            if schemas:
+                                if urllib.parse.urlparse(mirror)[0] not in schemas:
+                                    continue
+                            print(os.path.join(url, pkg.location.lstrip('/')))
                     else:
-                        msg = _("Failed to get mirror for package: %s") % pkg.name
-                        if self.base.conf.strict:
-                            raise dnf.exceptions.Error(msg)
-                        logger.warning(msg)
+                        url = pkg.remote_location(schemes=self.opts.urlprotocols)
+                        if url:
+                            print(url)
+                        else:
+                            msg = _("Failed to get mirror for package: %s") % pkg.name
+                            if self.base.conf.strict:
+                                raise dnf.exceptions.Error(msg)
+                            logger.warning(msg)
             return
         else:
             self._do_downloads(pkgs)  # download rpms

qubes-rpc/qvm-template-repo-query

@@ -29,11 +29,9 @@ cat > "$repodir/template.repo"
 OPTS+=("--setopt=reposdir=${repodir}")
 OPTS+=("--quiet")
 
-if [ -d '/usr/lib/qubes/dnf-plugins' ]; then
-    # use vendored 'download' dnf-plugin when available, to not require
-    # dnf-plugins-core package (not available in Debian yet)
-    OPTS+=("--setopt=pluginpath=/usr/lib/qubes/dnf-plugins")
-fi
+# use vendored 'downloadurl' dnf-plugin (fork of 'download' plugin), to print
+# all mirrors
+OPTS+=("--setopt=pluginpath=/usr/lib/qubes/dnf-plugins")
 
 if ! command -v dnf >/dev/null; then
     echo "ERROR: dnf command is missing, please use newer template for your UpdateVM to download templates." >&2
@@ -60,8 +58,7 @@ elif [ "$1" = "download" ]; then
     # downloaded - retry from the same one. If download failed and nothing was
     # downloaded, go to the next one. The intention is to retry on interrupted
     # connection, but skip mirrors that are not synchronized yet.
-    # FIXME: 'dnf download --url' prints only one URL, not all the mirrors
-    urls="$(dnf download "${OPTS[@]}" --url "$SPEC" | shuf)"
+    urls="$(dnf downloadurl "${OPTS[@]}" --url --all-mirrors "$SPEC" | shuf)"
     readarray -t urls <<<"$urls"
     downloaded=0
     status_file="$repodir/download-status.tmp"

rpm_spec/core-agent.spec.in

@@ -812,6 +812,7 @@ rm -f %{name}-%{version}
 %dir %attr(0755,user,user) /var/lib/qubes/dom0-updates
 /usr/lib/qubes/qvm-template-repo-query
 /usr/lib/qubes/qubes-download-dom0-updates.sh
+/usr/lib/qubes/dnf-plugins/downloadurl.py
 
 %files networking
 %config(noreplace) /etc/qubes-rpc/qubes.UpdatesProxy