Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

core: handle shared files and multilib #1227

Closed
wants to merge 5 commits into from
Closed

core: handle shared files and multilib #1227

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
6005c2b
core: move utility function higher up
jlebon Feb 7, 2018
01a1e02
libpriv/importer: factor out pathname translator
jlebon Feb 7, 2018
a6926e0
core: handle shared files and multilib
jlebon Feb 7, 2018
b64afe5
fixup! core: handle shared files and multilib
jlebon Feb 8, 2018
81621c8
fixup! libpriv/importer: factor out pathname translator
jlebon Feb 8, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
254 changes: 211 additions & 43 deletions src/libpriv/rpmostree-core.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2378,12 +2378,197 @@ rpmostree_context_consume_package (RpmOstreeContext *self,
return TRUE;
}

/* Builds a mapping from filename to rpm color */
static void
add_te_files_to_ht (rpmte te,
GHashTable *ht)
{
#ifdef BUILDOPT_HAVE_RPMFILES /* use rpmfiles API if possible, rpmteFI is deprecated */
g_auto(rpmfiles) files = rpmteFiles (te);
g_auto(rpmfi) fi = rpmfilesIter (files, RPMFI_ITER_FWD);
#else
rpmfi fi = rpmteFI (te); /* rpmfi owned by rpmte */
#endif

while (rpmfiNext (fi) >= 0)
{
const char *fn = rpmfiFN (fi);
rpm_color_t color = rpmfiFColor (fi);
g_hash_table_insert (ht, g_strdup (fn), GUINT_TO_POINTER (color));
}
}

static char*
canonicalize_rpmfi_path (const char *path)
{
/* this is a bit awkward; we relativize for the translation, but then make it absolute
* again to match libostree */
path += strspn (path, "/");
g_autofree char *translated =
rpmostree_translate_path_for_ostree (path) ?: g_strdup (path);
return g_build_filename ("/", translated, NULL);
}

/* Convert e.g. lib/foo/bar → usr/lib/foo/bar */
static char *
canonicalize_non_usrmove_path (RpmOstreeContext *self,
const char *path)
{
const char *slash = strchr (path, '/');
if (!slash)
return NULL;
const char *prefix = strndupa (path, slash - path);
const char *link = g_hash_table_lookup (self->rootfs_usrlinks, prefix);
if (!link)
return NULL;
return g_build_filename (link, slash + 1, NULL);
}

/* This is a lighter version of calculations that librpm calls "file disposition".
* Essentially, we determine which file removals/installations should be skipped. The librpm
* functions and APIs for these are unfortunately private since they're just run as part of
* rpmtsRun(). XXX: see if we can make the rpmfs APIs public, but we'd still need to support
* RHEL/CentOS anyway. */
static gboolean
handle_file_dispositions (RpmOstreeContext *self,
int tmprootfs_dfd,
rpmts ts,
GHashTable **out_files_skip_add,
GHashTable **out_files_skip_delete,
GCancellable *cancellable,
GError **error)
{
g_autoptr(GHashTable) pkgs_deleted = g_hash_table_new (g_direct_hash, g_direct_equal);

/* note these entries are *not* canonicalized for ostree conventions */
g_autoptr(GHashTable) files_deleted =
g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL);
g_autoptr(GHashTable) files_added =
g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL);

const guint n_rpmts_elements = (guint)rpmtsNElements (ts);
for (guint i = 0; i < n_rpmts_elements; i++)
{
rpmte te = rpmtsElement (ts, i);
rpmElementType type = rpmteType (te);
if (type == TR_REMOVED)
g_hash_table_add (pkgs_deleted, GUINT_TO_POINTER (rpmteDBInstance (te)));
add_te_files_to_ht (te, type == TR_REMOVED ? files_deleted : files_added);
}

/* this we *do* canonicalize since we'll be comparing against ostree paths */
g_autoptr(GHashTable) files_skip_add =
g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL);
/* this one we *don't* canonicalize since we'll be comparing against rpmfi paths */
g_autoptr(GHashTable) files_skip_delete =
g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL);

/* we deal with color similarly to librpm (compare with skipInstallFiles()) */
rpm_color_t ts_color = rpmtsColor (ts);
rpm_color_t ts_prefcolor = rpmtsPrefColor (ts);

/* ignore colored files not in our rainbow */
GLNX_HASH_TABLE_FOREACH_IT (files_added, it, const char*, fn, gpointer, colorp)
{
rpm_color_t color = GPOINTER_TO_UINT (colorp);
if (color && ts_color && !(ts_color & color))
g_hash_table_add (files_skip_add, canonicalize_rpmfi_path (fn));
}

g_auto(rpmdbMatchIterator) it = rpmtsInitIterator (ts, RPMDBI_PACKAGES, NULL, 0);
Header h;
while ((h = rpmdbNextIterator (it)) != NULL)
{
/* is this pkg to be deleted? */
guint off = rpmdbGetIteratorOffset (it);
if (g_hash_table_contains (pkgs_deleted, GUINT_TO_POINTER (off)))
continue;

/* try to only load what we need: filenames and colors */
rpmfiFlags flags = RPMFI_FLAGS_QUERY;
#ifdef BUILDOPT_HAVE_RPMFILES /* we're using this more as a "if not on CentOS" switch */
/* this is even more restrictive that QUERY */
flags = RPMFI_FLAGS_ONLY_FILENAMES;
#endif
flags &= ~RPMFI_NOFILECOLORS;

g_auto(rpmfi) fi = rpmfiNew (ts, h, RPMTAG_BASENAMES, flags);

fi = rpmfiInit (fi, 0);
while (rpmfiNext (fi) >= 0)
{
const char *fn = rpmfiFN (fi);
g_assert (fn != NULL);

/* check if one of the pkgs to delete wants to delete our file */
if (g_hash_table_contains (files_deleted, fn))
g_hash_table_add (files_skip_delete, g_strdup (fn));

rpm_color_t color = (rpmfiFColor (fi) & ts_color);

/* let's make the safe assumption that the color mess is only an issue for /usr */
const char *fn_rel = fn + strspn (fn, "/");

/* be sure we've canonicalized usr/ */
g_autofree char *fn_rel_owned = canonicalize_non_usrmove_path (self, fn_rel);
if (fn_rel_owned)
fn_rel = fn_rel_owned;

if (!g_str_has_prefix (fn_rel, "usr/"))
continue;

/* check if one of the pkgs to install wants to overwrite our file */
rpm_color_t other_color =
GPOINTER_TO_UINT (g_hash_table_lookup (files_added, fn));
other_color &= ts_color;

/* see handleColorConflict() */
if (color && other_color && (color != other_color))
{
/* do we already have the preferred color installed? */
if (color & ts_prefcolor)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just for my own understanding...could this be == instead of &?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, you're correct AFAIU. They're both bitfields, but ts_color contains all the colors we're compatible with, while ts_prefcolor contains only a single color from that set. And files can only one specific color, so == would work equally well here.

g_hash_table_add (files_skip_add, canonicalize_rpmfi_path (fn));
else if (other_color & ts_prefcolor)
{
/* the new pkg is bringing our favourite color, give way now so we let
* checkout silently write into it */
if (!glnx_shutil_rm_rf_at (tmprootfs_dfd, fn_rel, cancellable, error))
return FALSE;
}
}
}
}

*out_files_skip_add = g_steal_pointer (&files_skip_add);
*out_files_skip_delete = g_steal_pointer (&files_skip_delete);
return TRUE;
}

static OstreeRepoCheckoutFilterResult
checkout_filter (OstreeRepo *self,
const char *path,
struct stat *st_buf,
gpointer user_data)
{
GHashTable *files_skip = user_data;
if (g_hash_table_contains (files_skip, path))
return OSTREE_REPO_CHECKOUT_FILTER_SKIP;
/* Hack for nsswitch.conf: the glibc.i686 copy is identical to the one in glibc.x86_64,
* but because we modify it at treecompose time, UNION_IDENTICAL wouldn't save us here. A
* better heuristic here might be to skip all /etc files which have a different digest
* than the one registered in the rpmdb */
if (g_str_equal (path, "/usr/etc/nsswitch.conf"))
return OSTREE_REPO_CHECKOUT_FILTER_SKIP;
return OSTREE_REPO_CHECKOUT_FILTER_ALLOW;
}

static gboolean
checkout_package (OstreeRepo *repo,
int dfd,
const char *path,
OstreeRepoDevInoCache *devino_cache,
const char *pkg_commit,
GHashTable *files_skip,
OstreeRepoCheckoutOverwriteMode ovwmode,
GCancellable *cancellable,
GError **error)
Expand All @@ -2400,6 +2585,12 @@ checkout_package (OstreeRepo *repo,
/* Always want hardlinks */
opts.no_copy_fallback = TRUE;

if (files_skip && g_hash_table_size (files_skip) > 0)
{
opts.filter = checkout_filter;
opts.filter_user_data = files_skip;
}

return ostree_repo_checkout_at (repo, &opts, dfd, path,
pkg_commit, cancellable, error);
}
Expand All @@ -2411,6 +2602,7 @@ checkout_package_into_root (RpmOstreeContext *self,
const char *path,
OstreeRepoDevInoCache *devino_cache,
const char *pkg_commit,
GHashTable *files_skip,
OstreeRepoCheckoutOverwriteMode ovwmode,
GCancellable *cancellable,
GError **error)
Expand All @@ -2432,7 +2624,7 @@ checkout_package_into_root (RpmOstreeContext *self,
}

if (!checkout_package (pkgcache_repo, dfd, path,
devino_cache, pkg_commit, ovwmode,
devino_cache, pkg_commit, files_skip, ovwmode,
cancellable, error))
return glnx_prefix_error (error, "Checkout %s", dnf_package_get_nevra (pkg));

Expand Down Expand Up @@ -2497,39 +2689,25 @@ build_rootfs_usrlinks (RpmOstreeContext *self,
return TRUE;
}

/* Convert e.g. lib/foo/bar → usr/lib/foo/bar */
static char *
canonicalize_non_usrmove_path (RpmOstreeContext *self,
const char *path)
{
const char *slash = strchr (path, '/');
if (!slash)
return NULL;
const char *prefix = strndupa (path, slash - path);
const char *link = g_hash_table_lookup (self->rootfs_usrlinks, prefix);
if (!link)
return NULL;
return g_build_filename (link, slash + 1, NULL);
}

static gboolean
delete_package_from_root (RpmOstreeContext *self,
rpmte pkg,
int rootfs_dfd,
GHashTable *files_skip,
GCancellable *cancellable,
GError **error)
{
#ifdef BUILDOPT_HAVE_RPMFILES /* use rpmfiles API if possible, rpmteFI is deprecated */
g_auto(rpmfiles) files = rpmteFiles (pkg);
/* NB: new librpm uses RPMFI_ITER_BACK here to empty out dirs before deleting them using
* unlink/rmdir. Older rpm doesn't support this API, so rather than doing some fancy
* compat, we just use shutil_rm_rf anyway since we now skip over shared dirs/files. */
g_auto(rpmfi) fi = rpmfilesIter (files, RPMFI_ITER_FWD);
#else
rpmfi fi = rpmteFI (pkg); /* rpmfi owned by rpmte */
#endif

g_autoptr(GPtrArray) deleted_dirs = g_ptr_array_new_with_free_func (g_free);

int i;
while ((i = rpmfiNext (fi)) >= 0)
while (rpmfiNext (fi) >= 0)
{
/* see also apply_rpmfi_overrides() for a commented version of the loop */
const char *fn = rpmfiFN (fi);
Expand All @@ -2540,6 +2718,9 @@ delete_package_from_root (RpmOstreeContext *self,
S_ISDIR (mode)))
continue;

if (g_hash_table_contains (files_skip, fn))
continue;

g_assert (fn != NULL);
fn += strspn (fn, "/");
g_assert (fn[0]);
Expand All @@ -2560,20 +2741,8 @@ delete_package_from_root (RpmOstreeContext *self,
if (!g_str_has_prefix (fn, "usr/"))
continue;

/* avoiding the stat syscall is worth a bit of userspace computation */
if (rpmostree_str_has_prefix_in_ptrarray (fn, deleted_dirs))
continue;
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I dropped this optimization here since we now do things like librpm does: we traverse the file list backwards, which means directories are emptied out before we delete them.


if (!glnx_fstatat_allow_noent (rootfs_dfd, fn, NULL, AT_SYMLINK_NOFOLLOW, error))
return FALSE;
if (errno == ENOENT)
continue; /* a job well done */

if (!glnx_shutil_rm_rf_at (rootfs_dfd, fn, cancellable, error))
return FALSE;

if (S_ISDIR (mode))
g_ptr_array_add (deleted_dirs, g_strconcat (fn, "/", NULL));
}

return TRUE;
Expand Down Expand Up @@ -2648,7 +2817,7 @@ relabel_in_thread_impl (RpmOstreeContext *self,
g_autoptr(OstreeRepoDevInoCache) cache = ostree_repo_devino_cache_new ();

if (!checkout_package (repo, tmpdir_dfd, pkg_dirname, cache,
commit_csum, OSTREE_REPO_CHECKOUT_OVERWRITE_NONE,
commit_csum, NULL, OSTREE_REPO_CHECKOUT_OVERWRITE_NONE,
cancellable, error))
return FALSE;

Expand Down Expand Up @@ -3479,21 +3648,19 @@ rpmostree_context_assemble (RpmOstreeContext *self,
if (!checkout_package_into_root (self, filesystem_package,
tmprootfs_dfd, ".", self->devino_cache,
g_hash_table_lookup (pkg_to_ostree_commit,
filesystem_package),
filesystem_package), NULL,
OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_IDENTICAL,
cancellable, error))
return FALSE;
n_rpmts_done++;
rpmostree_output_progress_n_items ("Building filesystem", n_rpmts_done, n_rpmts_elements);
}

/* We're completely disregarding how rpm normally does upgrades/downgrade here. rpm
* usually calculates the fate of each file and then installs the new files first, then
* removes the obsoleted files. So the TR_ADDED element for the new pkg comes *before* the
* TR_REMOVED element for the old pkg. This makes sense when operating on the live system.
* Though that's not the case here, so let's just take the easy way out and first nuke
* TR_REMOVED pkgs, then checkout the TR_ADDED ones. This will probably start to matter
* though when we start handling e.g. file triggers. */
g_autoptr(GHashTable) files_skip_add = NULL;
g_autoptr(GHashTable) files_skip_delete = NULL;
if (!handle_file_dispositions (self, tmprootfs_dfd, ordering_ts, &files_skip_add,
&files_skip_delete, cancellable, error))
return FALSE;

for (guint i = 0; i < n_rpmts_elements; i++)
{
Expand All @@ -3504,7 +3671,8 @@ rpmostree_context_assemble (RpmOstreeContext *self,
continue;
g_assert_cmpint (type, ==, TR_REMOVED);

if (!delete_package_from_root (self, te, tmprootfs_dfd, cancellable, error))
if (!delete_package_from_root (self, te, tmprootfs_dfd, files_skip_delete,
cancellable, error))
return FALSE;
n_rpmts_done++;
rpmostree_output_progress_n_items ("Building filesystem", n_rpmts_done, n_rpmts_elements);
Expand Down Expand Up @@ -3533,7 +3701,7 @@ rpmostree_context_assemble (RpmOstreeContext *self,

if (!checkout_package_into_root (self, pkg, tmprootfs_dfd, ".", self->devino_cache,
g_hash_table_lookup (pkg_to_ostree_commit, pkg),
ovwmode, cancellable, error))
files_skip_add, ovwmode, cancellable, error))
return FALSE;
n_rpmts_done++;
rpmostree_output_progress_n_items ("Building filesystem", n_rpmts_done, n_rpmts_elements);
Expand Down
15 changes: 1 addition & 14 deletions src/libpriv/rpmostree-importer.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -585,8 +585,6 @@ path_is_ostree_compliant (const char *path)
g_str_equal (path, "lib64") || g_str_has_prefix (path, "lib64/"));
}

#define VAR_SELINUX_TARGETED_PATH "var/lib/selinux/targeted/"

static OstreeRepoCommitFilterResult
compose_filter_cb (OstreeRepo *repo,
const char *path,
Expand Down Expand Up @@ -762,18 +760,7 @@ handle_translate_pathname (OstreeRepo *repo,
const char *path,
gpointer user_data)
{
if (g_str_has_prefix (path, "etc/"))
return g_strconcat ("usr/", path, NULL);
else if (g_str_has_prefix (path, "boot/"))
return g_strconcat ("usr/lib/ostree-boot/", path + strlen ("boot/"), NULL);
/* Special hack for https://bugzilla.redhat.com/show_bug.cgi?id=1290659
* See also commit 4a86bdd19665700fa308461510c9decd63e31a03
* and rpmostree_postprocess_selinux_policy_store_location().
*/
else if (g_str_has_prefix (path, VAR_SELINUX_TARGETED_PATH))
return g_strconcat ("usr/etc/selinux/targeted/", path + strlen (VAR_SELINUX_TARGETED_PATH), NULL);

return NULL;
return rpmostree_translate_path_for_ostree (path);
}

static gboolean
Expand Down
Loading