jigdo: Use separate rpmostree/jigdo/... refs

The way we import packages in jigdo mode is different from package layering; we
may only import a subset of files for example. In general, we need to treat
jigdo differently.

Related: coreos#1197

Closes: coreos#1238
Approved by: jlebon

src/daemon/rpmostree-sysroot-core.c

@@ -115,6 +115,7 @@ generate_baselayer_refs (OstreeSysroot            *sysroot,
  */
 static gboolean
 add_package_refs_to_set (RpmOstreeRefSack *rsack,
+                         gboolean          is_jigdo,
                          GHashTable *referenced_pkgs,
                          GCancellable *cancellable,
                          GError **error)
@@ -133,7 +134,8 @@ add_package_refs_to_set (RpmOstreeRefSack *rsack,
       for (guint i = 0; i < pkglist->len; i++)
         {
           DnfPackage *pkg = pkglist->pdata[i];
-          g_autofree char *pkgref = rpmostree_get_cache_branch_pkg (pkg);
+          g_autofree char *pkgref =
+            is_jigdo ? rpmostree_get_jigdo_branch_pkg (pkg) : rpmostree_get_cache_branch_pkg (pkg);
           g_hash_table_add (referenced_pkgs, g_steal_pointer (&pkgref));
         }
     }
@@ -184,9 +186,11 @@ clean_pkgcache_orphans (OstreeSysroot            *sysroot,
   for (guint i = 0; i < deployments->len; i++)
     {
       OstreeDeployment *deployment = deployments->pdata[i];
-
+      const char *current_checksum = ostree_deployment_get_csum (deployment);
       gboolean is_layered;
-      if (!rpmostree_deployment_get_layered_info (repo, deployment, &is_layered, NULL, NULL,
+      g_autofree char *base_commit = NULL;
+      if (!rpmostree_deployment_get_layered_info (repo, deployment, &is_layered,
+                                                  &base_commit, NULL,
                                                   NULL, NULL, error))
         return FALSE;
 
@@ -197,8 +201,12 @@ clean_pkgcache_orphans (OstreeSysroot            *sysroot,
       RpmOstreeRefspecType refspectype;
       rpmostree_origin_classify_refspec (origin, &refspectype, NULL);
 
-      /* In rojig mode, we need to also reference all packages */
-      if (is_layered || refspectype == RPMOSTREE_REFSPEC_TYPE_ROJIG)
+      /* Hold a ref to layered packages; actually right now this injects refs
+       * for *all* packages since we don't have an API to query out which
+       * packages are layered. But it's harmless to have nonexistent refs in the
+       * set.
+       */
+      if (is_layered)
         {
           g_autofree char *deployment_dirpath =
             ostree_sysroot_get_deployment_dirpath (sysroot, deployment);
@@ -212,7 +220,21 @@ clean_pkgcache_orphans (OstreeSysroot            *sysroot,
           if (rsack == NULL)
             return FALSE;
 
-          if (!add_package_refs_to_set (rsack, referenced_pkgs, cancellable, error))
+          if (!add_package_refs_to_set (rsack, FALSE, referenced_pkgs, cancellable, error))
+            return FALSE;
+        }
+      /* In rojig mode, we need to also reference packages from the base; this
+       * is a different refspec format.
+       */
+      if (refspectype == RPMOSTREE_REFSPEC_TYPE_ROJIG)
+        {
+          const char *actual_base_commit = base_commit ?: current_checksum;
+          g_autoptr(RpmOstreeRefSack) base_rsack =
+            rpmostree_get_base_refsack_for_commit (repo, actual_base_commit, cancellable, error);
+          if (base_rsack == NULL)
+            return FALSE;
+
+          if (!add_package_refs_to_set (base_rsack, TRUE, referenced_pkgs, cancellable, error))
             return FALSE;
         }
 
@@ -229,13 +251,29 @@ clean_pkgcache_orphans (OstreeSysroot            *sysroot,
         }
     }
 
-  g_autoptr(GHashTable) current_refs = NULL;
-  if (!ostree_repo_list_refs_ext (repo, "rpmostree/pkg", &current_refs,
+  guint n_freed = 0;
+  /* Loop over layered refs */
+  g_autoptr(GHashTable) pkg_refs = NULL;
+  if (!ostree_repo_list_refs_ext (repo, "rpmostree/pkg", &pkg_refs,
                                   OSTREE_REPO_LIST_REFS_EXT_NONE, cancellable, error))
     return FALSE;
+  GLNX_HASH_TABLE_FOREACH (pkg_refs, const char*, ref)
+    {
+      if (g_hash_table_contains (referenced_pkgs, ref))
+        continue;
 
-  guint n_freed = 0;
-  GLNX_HASH_TABLE_FOREACH (current_refs, const char*, ref)
+      if (!ostree_repo_set_ref_immediate (repo, NULL, ref, NULL,
+                                          cancellable, error))
+        return FALSE;
+      n_freed++;
+    }
+
+  /* Loop over jigdo refs */
+  g_autoptr(GHashTable) jigdo_refs = NULL;
+  if (!ostree_repo_list_refs_ext (repo, "rpmostree/jigdo", &jigdo_refs,
+                                  OSTREE_REPO_LIST_REFS_EXT_NONE, cancellable, error))
+    return FALSE;
+  GLNX_HASH_TABLE_FOREACH (jigdo_refs, const char*, ref)
     {
       if (g_hash_table_contains (referenced_pkgs, ref))
         continue;

src/libpriv/rpmostree-core.c

@@ -1209,10 +1209,12 @@ append_quoted (GString *r, const char *value)
 }
 
 /* Return the ostree cache branch for a nevra */
-char *
-rpmostree_get_cache_branch_for_n_evr_a (const char *name, const char *evr, const char *arch)
+static char *
+get_branch_for_n_evr_a (const char *type, const char *name, const char *evr, const char *arch)
 {
-  GString *r = g_string_new ("rpmostree/pkg/");
+  GString *r = g_string_new ("rpmostree/");
+  g_string_append (r, type);
+  g_string_append_c (r, '/');
   append_quoted (r, name);
   g_string_append_c (r, '/');
   /* Work around the fact that libdnf and librpm have different handling
@@ -1230,6 +1232,12 @@ rpmostree_get_cache_branch_for_n_evr_a (const char *name, const char *evr, const
   return g_string_free (r, FALSE);
 }
 
+char *
+rpmostree_get_cache_branch_for_n_evr_a (const char *name, const char *evr, const char *arch)
+{
+  return get_branch_for_n_evr_a ("pkg", name, evr, arch);
+}
+
 /* Return the ostree cache branch from a Header */
 char *
 rpmostree_get_cache_branch_header (Header hdr)
@@ -1240,6 +1248,15 @@ rpmostree_get_cache_branch_header (Header hdr)
   return rpmostree_get_cache_branch_for_n_evr_a (name, evr, arch);
 }
 
+char *
+rpmostree_get_jigdo_branch_header (Header hdr)
+{
+  g_autofree char *name = headerGetAsString (hdr, RPMTAG_NAME);
+  g_autofree char *evr = headerGetAsString (hdr, RPMTAG_EVR);
+  g_autofree char *arch = headerGetAsString (hdr, RPMTAG_ARCH);
+  return get_branch_for_n_evr_a ("jigdo", name, evr, arch);
+}
+
 /* Return the ostree cache branch from a libdnf Package */
 char *
 rpmostree_get_cache_branch_pkg (DnfPackage *pkg)
@@ -1249,6 +1266,14 @@ rpmostree_get_cache_branch_pkg (DnfPackage *pkg)
                                                  dnf_package_get_arch (pkg));
 }
 
+char *
+rpmostree_get_jigdo_branch_pkg (DnfPackage *pkg)
+{
+  return get_branch_for_n_evr_a ("jigdo", dnf_package_get_name (pkg),
+                                 dnf_package_get_evr (pkg),
+                                 dnf_package_get_arch (pkg));
+}
+
 static gboolean
 commit_has_matching_sepolicy (GVariant       *commit,
                               OstreeSePolicy *sepolicy,
@@ -1348,7 +1373,8 @@ find_pkg_in_ostree (RpmOstreeContext *self,
   if (repo == NULL)
     return TRUE; /* Note early return */
 
-  g_autofree char *cachebranch = rpmostree_get_cache_branch_pkg (pkg);
+  g_autofree char *cachebranch = self->jigdo_spec ?
+      rpmostree_get_jigdo_branch_pkg (pkg) : rpmostree_get_cache_branch_pkg (pkg);
   g_autofree char *cached_rev = NULL;
   if (!ostree_repo_resolve_rev (repo, cachebranch, TRUE,
                                 &cached_rev, error))

src/libpriv/rpmostree-core.h

@@ -122,7 +122,9 @@ gboolean rpmostree_context_get_state_sha512 (RpmOstreeContext *self,
 
 char * rpmostree_get_cache_branch_for_n_evr_a (const char *name, const char *evr, const char *arch);
 char *rpmostree_get_cache_branch_header (Header hdr);
+char *rpmostree_get_jigdo_branch_header (Header hdr);
 char *rpmostree_get_cache_branch_pkg (DnfPackage *pkg);
+char *rpmostree_get_jigdo_branch_pkg (DnfPackage *pkg);
 
 gboolean
 rpmostree_find_cache_branch_by_nevra (OstreeRepo    *pkgcache,

src/libpriv/rpmostree-importer.c

@@ -327,7 +327,12 @@ const char *
 rpmostree_importer_get_ostree_branch (RpmOstreeImporter *self)
 {
   if (!self->ostree_branch)
-    self->ostree_branch = rpmostree_get_cache_branch_header (self->hdr);
+    {
+      if (self->jigdo_mode)
+        self->ostree_branch = rpmostree_get_jigdo_branch_header (self->hdr);
+      else
+        self->ostree_branch = rpmostree_get_cache_branch_header (self->hdr);
+    }
 
   return self->ostree_branch;
 }

src/libpriv/rpmostree-rpm-util.c

@@ -766,6 +766,7 @@ rpmrev_free (struct RpmRevisionData *ptr)
 static gboolean
 checkout_only_rpmdb (OstreeRepo       *repo,
                      const char       *ref,
+                     const char       *rpmdb,
                      GLnxTmpDir       *tmpdir,
                      GCancellable     *cancellable,
                      GError          **error)
@@ -781,7 +782,8 @@ checkout_only_rpmdb (OstreeRepo       *repo,
   /* Check out the database (via copy) */
   OstreeRepoCheckoutAtOptions checkout_options = { 0, };
   checkout_options.mode = OSTREE_REPO_CHECKOUT_MODE_USER;
-  checkout_options.subpath = RPMOSTREE_RPMDB_LOCATION;
+  const char *subpath = glnx_strjoina ("/", rpmdb);
+  checkout_options.subpath = subpath;
   if (!ostree_repo_checkout_at (repo, &checkout_options, tmpdir->fd,
                                 RPMOSTREE_RPMDB_LOCATION, commit,
                                 cancellable, error))
@@ -845,7 +847,37 @@ rpmostree_get_refsack_for_commit (OstreeRepo                *repo,
   if (!glnx_mkdtemp ("rpmostree-dbquery-XXXXXX", 0700, &tmpdir, error))
     return NULL;
 
-  if (!checkout_only_rpmdb (repo, ref, &tmpdir, cancellable, error))
+  if (!checkout_only_rpmdb (repo, ref, RPMOSTREE_RPMDB_LOCATION,
+                            &tmpdir, cancellable, error))
+    return NULL;
+
+  g_autoptr(DnfSack) hsack = NULL; /* NB: refsack adds a ref to it */
+  if (!get_sack_for_root (tmpdir.fd, ".", &hsack, error))
+    return NULL;
+
+  /* Ownership of tmpdir is transferred */
+  return rpmostree_refsack_new (hsack, &tmpdir);
+}
+
+/* Return a sack for the "base" rpmdb without any layering/overrides/etc.
+ * involved.
+ */
+RpmOstreeRefSack *
+rpmostree_get_base_refsack_for_commit (OstreeRepo                *repo,
+                                       const char                *ref,
+                                       GCancellable              *cancellable,
+                                       GError                   **error)
+{
+  g_auto(GLnxTmpDir) tmpdir = { 0, };
+  if (!glnx_mkdtemp ("rpmostree-dbquery-XXXXXX", 0700, &tmpdir, error))
+    return NULL;
+
+  /* This is a bit of a hack; we checkout the "base" dbpath as /usr/share/rpm in
+   * a temporary root. Fixing this would require patching through new APIs into
+   * libdnf → libsolv to teach it about a way to find a user-specified dbpath.
+   */
+  if (!checkout_only_rpmdb (repo, ref, RPMOSTREE_BASE_RPMDB,
+                            &tmpdir, cancellable, error))
     return NULL;
 
   g_autoptr(DnfSack) hsack = NULL; /* NB: refsack adds a ref to it */
@@ -885,7 +917,8 @@ rpmostree_get_refts_for_commit (OstreeRepo                *repo,
   if (!glnx_mkdtemp ("rpmostree-dbquery-XXXXXX", 0700, &tmpdir, error))
     return FALSE;
 
-  if (!checkout_only_rpmdb (repo, ref, &tmpdir, cancellable, error))
+  if (!checkout_only_rpmdb (repo, ref, RPMOSTREE_RPMDB_LOCATION,
+                            &tmpdir, cancellable, error))
     return FALSE;
 
   /* Ownership of tmpdir is transferred */

src/libpriv/rpmostree-rpm-util.h

@@ -107,6 +107,12 @@ rpmostree_get_refsack_for_root (int              dfd,
                                 const char      *path,
                                 GError         **error);
 
+RpmOstreeRefSack *
+rpmostree_get_base_refsack_for_commit (OstreeRepo                *repo,
+                                       const char                *ref,
+                                       GCancellable              *cancellable,
+                                       GError                   **error);
+
 gboolean
 rpmostree_get_refts_for_commit (OstreeRepo                *repo,
                                 const char                *ref,

tests/compose-tests/test-jigdo-e2e.sh

@@ -54,7 +54,7 @@ assert_file_has_content jigdo2commit-out.txt '[1-9][0-9]* packages to import'
 ostree --repo=jigdo-unpack-repo rev-parse ${rev}
 ostree --repo=jigdo-unpack-repo fsck
 ostree --repo=jigdo-unpack-repo refs > jigdo-refs.txt
-assert_file_has_content jigdo-refs.txt 'rpmostree/pkg/test-pkg/1.0-1.x86__64'
+assert_file_has_content jigdo-refs.txt 'rpmostree/jigdo/test-pkg/1.0-1.x86__64'
 
 echo "ok jigdo ♲📦 fresh assembly"
 
@@ -96,7 +96,7 @@ ostree --repo=jigdo-unpack-repo fsck
 ostree --repo=jigdo-unpack-repo refs > jigdo-refs.txt
 # We should have both refs; GC will be handled by the sysroot upgrader
 # via deployments, same way it is for pkg layering.
-assert_file_has_content jigdo-refs.txt 'rpmostree/pkg/test-pkg/1.0-1.x86__64'
-assert_file_has_content jigdo-refs.txt 'rpmostree/pkg/test-pkg/1.1-1.x86__64'
+assert_file_has_content jigdo-refs.txt 'rpmostree/jigdo/test-pkg/1.0-1.x86__64'
+assert_file_has_content jigdo-refs.txt 'rpmostree/jigdo/test-pkg/1.1-1.x86__64'
 
 echo "ok jigdo ♲📦 update!"

tests/vmcheck/test-jigdo-client.sh

@@ -40,5 +40,5 @@ assert_file_has_content_literal err.txt 'rojig:// refspec requires --experimenta
 vm_rpmostree rebase --experimental rojig://fahc:fedora-atomic-host
 vm_assert_status_jq '.deployments[0].origin|startswith("rojig://fahc:fedora-atomic-host")'
 vm_cmd ostree refs > refs.txt
-assert_file_has_content refs.txt '^rpmostree/pkg/kernel-core/'
+assert_file_has_content refs.txt '^rpmostree/jigdo/kernel-core/'
 echo "ok jigdo client"