From 94948e3522b56b2e80a7cae636aef06c1b372fc8 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 1 Mar 2017 22:42:07 -0500 Subject: [PATCH] checkout: Support a "pure addition" mode I plan to use this for `rpm-ostree livefs`. https://github.com/projectatomic/rpm-ostree/issues/639 Closes: #714 Approved by: jlebon --- man/ostree-checkout.xml | 8 +++ src/libostree/ostree-repo-checkout.c | 74 ++++++++++++++++++---------- src/libostree/ostree-repo.h | 6 ++- src/ostree/ot-builtin-checkout.c | 17 +++++-- tests/basic-test.sh | 20 +++++++- 5 files changed, 94 insertions(+), 31 deletions(-) diff --git a/man/ostree-checkout.xml b/man/ostree-checkout.xml index 67d6469eff..c8585878da 100644 --- a/man/ostree-checkout.xml +++ b/man/ostree-checkout.xml @@ -89,6 +89,14 @@ Boston, MA 02111-1307, USA. + + + + + Keep existing directories and files. + + + diff --git a/src/libostree/ostree-repo-checkout.c b/src/libostree/ostree-repo-checkout.c index 50bc7030db..53409529f3 100644 --- a/src/libostree/ostree-repo-checkout.c +++ b/src/libostree/ostree-repo-checkout.c @@ -201,8 +201,16 @@ checkout_file_from_input_at (OstreeRepo *self, while (G_UNLIKELY (res == -1 && errno == EINTR)); if (res == -1) { - glnx_set_error_from_errno (error); - goto out; + if (errno == EEXIST && options->overwrite_mode == OSTREE_REPO_CHECKOUT_OVERWRITE_ADD_FILES) + { + ret = TRUE; + goto out; + } + else + { + glnx_set_error_from_errno (error); + goto out; + } } if (options->mode != OSTREE_REPO_CHECKOUT_MODE_USER) @@ -240,6 +248,11 @@ checkout_file_from_input_at (OstreeRepo *self, while (G_UNLIKELY (fd == -1 && errno == EINTR)); if (fd == -1) { + if (errno == EEXIST && options->overwrite_mode == OSTREE_REPO_CHECKOUT_OVERWRITE_ADD_FILES) + { + ret = TRUE; + goto out; + } glnx_set_error_from_errno (error); goto out; } @@ -332,6 +345,12 @@ checkout_file_unioning_from_input_at (OstreeRepo *repo, return ret; } +typedef enum { + HARDLINK_RESULT_NOT_SUPPORTED, + HARDLINK_RESULT_SKIP_EXISTED, + HARDLINK_RESULT_LINKED +} HardlinkResult; + static gboolean checkout_file_hardlink (OstreeRepo *self, OstreeRepoCheckoutAtOptions *options, @@ -339,28 +358,32 @@ checkout_file_hardlink (OstreeRepo *self, int destination_dfd, const char *destination_name, gboolean allow_noent, - gboolean *out_was_supported, + HardlinkResult *out_result, GCancellable *cancellable, GError **error) { - gboolean ret = FALSE; - gboolean ret_was_supported = FALSE; + HardlinkResult ret_result = HARDLINK_RESULT_NOT_SUPPORTED; int srcfd = (self->mode == OSTREE_REPO_MODE_BARE || self->mode == OSTREE_REPO_MODE_BARE_USER) ? self->objects_dir_fd : self->uncompressed_objects_dir_fd; again: - if (linkat (srcfd, loose_path, destination_dfd, destination_name, 0) != -1) - ret_was_supported = TRUE; + if (linkat (srcfd, loose_path, destination_dfd, destination_name, 0) == 0) + ret_result = HARDLINK_RESULT_LINKED; else if (!options->no_copy_fallback && (errno == EMLINK || errno == EXDEV || errno == EPERM)) { /* EMLINK, EXDEV and EPERM shouldn't be fatal; we just can't do the * optimization of hardlinking instead of copying. */ - ret_was_supported = FALSE; } else if (allow_noent && errno == ENOENT) { - ret_was_supported = FALSE; + } + else if (errno == EEXIST && options->overwrite_mode == OSTREE_REPO_CHECKOUT_OVERWRITE_ADD_FILES) + { + /* In this mode, we keep existing content. Distinguish this case though to + * avoid inserting into the devino cache. + */ + ret_result = HARDLINK_RESULT_SKIP_EXISTED; } else if (errno == EEXIST && options->overwrite_mode == OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_FILES) { @@ -370,7 +393,7 @@ checkout_file_hardlink (OstreeRepo *self, * the same file, then rename() does nothing, and returns a * success status." * - * So we can't make this atomic. + * So we can't make this atomic. */ (void) unlinkat (destination_dfd, destination_name, 0); goto again; @@ -379,14 +402,12 @@ checkout_file_hardlink (OstreeRepo *self, { g_prefix_error (error, "Hardlinking %s to %s: ", loose_path, destination_name); glnx_set_error_from_errno (error); - goto out; + return FALSE; } - ret = TRUE; - if (out_was_supported) - *out_was_supported = ret_was_supported; - out: - return ret; + if (out_result) + *out_result = ret_result; + return TRUE; } static gboolean @@ -439,7 +460,7 @@ checkout_one_file_at (OstreeRepo *repo, } else { - gboolean did_hardlink = FALSE; + HardlinkResult hardlink_res = HARDLINK_RESULT_NOT_SUPPORTED; /* Try to do a hardlink first, if it's a regular file. This also * traverses all parent repos. */ @@ -469,11 +490,11 @@ checkout_one_file_at (OstreeRepo *repo, options, loose_path_buf, destination_dfd, destination_name, - TRUE, &did_hardlink, + TRUE, &hardlink_res, cancellable, error)) goto out; - if (did_hardlink && options->devino_to_csum_cache) + if (hardlink_res == HARDLINK_RESULT_LINKED && options->devino_to_csum_cache) { struct stat stbuf; OstreeDevIno *key; @@ -492,13 +513,13 @@ checkout_one_file_at (OstreeRepo *repo, g_hash_table_add ((GHashTable*)options->devino_to_csum_cache, key); } - if (did_hardlink) + if (hardlink_res != HARDLINK_RESULT_NOT_SUPPORTED) break; } current_repo = current_repo->parent_repo; } - need_copy = !did_hardlink; + need_copy = (hardlink_res == HARDLINK_RESULT_NOT_SUPPORTED); } can_cache = (options->enable_uncompressed_cache @@ -514,7 +535,7 @@ checkout_one_file_at (OstreeRepo *repo, && repo->mode == OSTREE_REPO_MODE_ARCHIVE_Z2 && options->mode == OSTREE_REPO_CHECKOUT_MODE_USER) { - gboolean did_hardlink; + HardlinkResult hardlink_res = HARDLINK_RESULT_NOT_SUPPORTED; if (!ostree_repo_load_file (repo, checksum, &input, NULL, NULL, cancellable, error)) @@ -560,19 +581,20 @@ checkout_one_file_at (OstreeRepo *repo, if (!checkout_file_hardlink (repo, options, loose_path_buf, destination_dfd, destination_name, - FALSE, &did_hardlink, + FALSE, &hardlink_res, cancellable, error)) { g_prefix_error (error, "Using new cached uncompressed hardlink of %s to %s: ", checksum, destination_name); goto out; } - need_copy = !did_hardlink; + need_copy = (hardlink_res == HARDLINK_RESULT_NOT_SUPPORTED); } /* Fall back to copy if we couldn't hardlink */ if (need_copy) { + g_assert (!options->no_copy_fallback); if (!ostree_repo_load_file (repo, checksum, &input, NULL, &xattrs, cancellable, error)) goto out; @@ -655,7 +677,9 @@ checkout_tree_at (OstreeRepo *self, while (G_UNLIKELY (res == -1 && errno == EINTR)); if (res == -1) { - if (errno == EEXIST && options->overwrite_mode == OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_FILES) + if (errno == EEXIST && + (options->overwrite_mode == OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_FILES + || options->overwrite_mode == OSTREE_REPO_CHECKOUT_OVERWRITE_ADD_FILES)) did_exist = TRUE; else { diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h index 648bd12909..34685cc6c2 100644 --- a/src/libostree/ostree-repo.h +++ b/src/libostree/ostree-repo.h @@ -722,11 +722,13 @@ typedef enum { /** * OstreeRepoCheckoutOverwriteMode: * @OSTREE_REPO_CHECKOUT_OVERWRITE_NONE: No special options - * @OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_FILES: When layering checkouts, overwrite earlier files, but keep earlier directories + * @OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_FILES: When layering checkouts, unlink() and replace existing files, but do not modify existing directories + * @OSTREE_REPO_CHECKOUT_OVERWRITE_ADD_FILES: Only add new files/directories */ typedef enum { OSTREE_REPO_CHECKOUT_OVERWRITE_NONE = 0, - OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_FILES = 1 + OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_FILES = 1, + OSTREE_REPO_CHECKOUT_OVERWRITE_ADD_FILES = 2, /* Since: 2017.3 */ } OstreeRepoCheckoutOverwriteMode; _OSTREE_PUBLIC diff --git a/src/ostree/ot-builtin-checkout.c b/src/ostree/ot-builtin-checkout.c index 95172f8b1c..74e27cfb60 100644 --- a/src/ostree/ot-builtin-checkout.c +++ b/src/ostree/ot-builtin-checkout.c @@ -36,6 +36,7 @@ static gboolean opt_allow_noent; static gboolean opt_disable_cache; static char *opt_subpath; static gboolean opt_union; +static gboolean opt_union_add; static gboolean opt_whiteouts; static gboolean opt_from_stdin; static char *opt_from_file; @@ -63,6 +64,7 @@ static GOptionEntry options[] = { { "disable-cache", 0, 0, G_OPTION_ARG_NONE, &opt_disable_cache, "Do not update or use the internal repository uncompressed object cache", NULL }, { "subpath", 0, 0, G_OPTION_ARG_STRING, &opt_subpath, "Checkout sub-directory PATH", "PATH" }, { "union", 0, 0, G_OPTION_ARG_NONE, &opt_union, "Keep existing directories, overwrite existing files", NULL }, + { "union-add", 0, 0, G_OPTION_ARG_NONE, &opt_union_add, "Keep existing files/directories, only add new", NULL }, { "whiteouts", 0, 0, G_OPTION_ARG_NONE, &opt_whiteouts, "Process 'whiteout' (Docker style) entries", NULL }, { "allow-noent", 0, 0, G_OPTION_ARG_NONE, &opt_allow_noent, "Do nothing if specified path does not exist", NULL }, { "from-stdin", 0, 0, G_OPTION_ARG_NONE, &opt_from_stdin, "Process many checkouts from standard input", NULL }, @@ -87,14 +89,23 @@ process_one_checkout (OstreeRepo *repo, * `ostree_repo_checkout_at` until such time as we have a more * convenient infrastructure for testing C APIs with data. */ - if (opt_disable_cache || opt_whiteouts || opt_require_hardlinks) + if (opt_disable_cache || opt_whiteouts || opt_require_hardlinks || opt_union_add) { OstreeRepoCheckoutAtOptions options = { 0, }; - + if (opt_user_mode) options.mode = OSTREE_REPO_CHECKOUT_MODE_USER; - if (opt_union) + /* Can't union these */ + if (opt_union && opt_union_add) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Cannot specify both --union and --union-add"); + goto out; + } + else if (opt_union) options.overwrite_mode = OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_FILES; + else if (opt_union_add) + options.overwrite_mode = OSTREE_REPO_CHECKOUT_OVERWRITE_ADD_FILES; if (opt_whiteouts) options.process_whiteouts = TRUE; if (subpath) diff --git a/tests/basic-test.sh b/tests/basic-test.sh index 2698279631..045e4217bf 100644 --- a/tests/basic-test.sh +++ b/tests/basic-test.sh @@ -19,7 +19,7 @@ set -euo pipefail -echo "1..62" +echo "1..63" $CMD_PREFIX ostree --version > version.yaml python -c 'import yaml; yaml.safe_load(open("version.yaml"))' @@ -279,6 +279,24 @@ cd checkout-test2-union assert_file_has_content ./yet/another/tree/green "leaf" echo "ok checkout union 1" +cd ${test_tmpdir} +$OSTREE commit -b test-union-add --tree=ref=test2 +$OSTREE checkout test-union-add checkout-test-union-add +echo 'file for union add testing' > checkout-test-union-add/union-add-test +echo 'another file for union add testing' > checkout-test-union-add/union-add-test2 +$OSTREE commit -b test-union-add --tree=dir=checkout-test-union-add +rm checkout-test-union-add -rf +# Check out previous +$OSTREE checkout test-union-add^ checkout-test-union-add +assert_not_has_file checkout-test-union-add/union-add-test +assert_not_has_file checkout-test-union-add/union-add-test2 +# Now create a file we don't want overwritten +echo 'existing file for union add' > checkout-test-union-add/union-add-test +$OSTREE checkout --union-add test-union-add checkout-test-union-add +assert_file_has_content checkout-test-union-add/union-add-test 'existing file for union add' +assert_file_has_content checkout-test-union-add/union-add-test2 'another file for union add testing' +echo "ok checkout union add" + cd ${test_tmpdir} rm -rf shadow-repo mkdir shadow-repo