From b0b7c7fbff3cb5e67a5f8dfbbb1386f3ea579a47 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 1 Sep 2017 16:24:31 -0400 Subject: [PATCH 1/5] tests/prune: Factor out a helper for counting commits in the repo We had lots of duplicates; prep for adding more tests. --- tests/test-prune.sh | 51 ++++++++++++++++++++------------------------- 1 file changed, 23 insertions(+), 28 deletions(-) diff --git a/tests/test-prune.sh b/tests/test-prune.sh index afcfbb5264..4246f75667 100755 --- a/tests/test-prune.sh +++ b/tests/test-prune.sh @@ -44,16 +44,23 @@ done ${CMD_PREFIX} ostree --repo=repo pull --depth=-1 origin test +assert_repo_has_n_commits() { + repo=$1 + count=$2 + + find ${repo}/objects -name '*.commit' | wc -l > commitcount + assert_file_has_content commitcount '^'${count}'$' + rm -f commitcount +} + ${CMD_PREFIX} ostree prune --repo=repo --refs-only --depth=2 -v -find repo | grep \.commit$ | wc -l > commitcount -assert_file_has_content commitcount "^3$" +assert_repo_has_n_commits repo 3 find repo/objects -name '*.tombstone-commit' | wc -l > tombstonecommitcount assert_file_has_content tombstonecommitcount "^0$" $OSTREE fsck ${CMD_PREFIX} ostree prune --repo=repo --refs-only --depth=1 -v -find repo | grep \.commit$ | wc -l > commitcount -assert_file_has_content commitcount "^2$" +assert_repo_has_n_commits repo 2 find repo/objects -name '*.tombstone-commit' | wc -l > tombstonecommitcount assert_file_has_content tombstonecommitcount "^0$" @@ -70,8 +77,7 @@ find repo/objects -name '*.tombstone-commit' | wc -l > tombstonecommitcount assert_file_has_content tombstonecommitcount "^0$" ${CMD_PREFIX} ostree prune --repo=repo --refs-only --depth=0 -v -find repo/objects -name '*.commit' | wc -l > commitcount -assert_file_has_content commitcount "^1$" +assert_repo_has_n_commits repo 1 find repo/objects -name '*.tombstone-commit' | wc -l > tombstonecommitcount assert_not_file_has_content tombstonecommitcount "^0$" $OSTREE fsck @@ -88,28 +94,22 @@ assert_file_has_content tombstonecommitcount "^1$" $OSTREE fsck ${CMD_PREFIX} ostree prune --repo=repo --refs-only --depth=0 -v -find repo/objects -name '*.commit' | wc -l > commitcount -assert_file_has_content commitcount "^1$" +assert_repo_has_n_commits repo 1 ${CMD_PREFIX} ostree --repo=repo commit --branch=test -m test -s test tree --timestamp="2005-10-29 12:43:29 +0000" ${CMD_PREFIX} ostree --repo=repo commit --branch=test -m test -s test tree --timestamp="2010-10-29 12:43:29 +0000" -find repo/objects -name '*.commit' | wc -l > commitcount -assert_file_has_content commitcount "^3$" +assert_repo_has_n_commits repo 3 ${CMD_PREFIX} ostree --repo=repo prune --keep-younger-than="2015-10-29 12:43:29 +0000" -find repo/objects -name '*.commit' | wc -l > commitcount -assert_file_has_content commitcount "^2$" +assert_repo_has_n_commits repo 2 $OSTREE fsck ${CMD_PREFIX} ostree prune --repo=repo --refs-only --depth=0 -v -find repo/objects -name '*.commit' | wc -l > commitcount -assert_file_has_content commitcount "^2$" +assert_repo_has_n_commits repo 2 ${CMD_PREFIX} ostree --repo=repo commit --branch=test -m test -s test tree --timestamp="October 25 1985" ${CMD_PREFIX} ostree --repo=repo commit --branch=test -m test -s test tree --timestamp="October 21 2015" -find repo/objects -name '*.commit' | wc -l > commitcount -assert_file_has_content commitcount "^4$" +assert_repo_has_n_commits repo 4 ${CMD_PREFIX} ostree --repo=repo prune --keep-younger-than="1 week ago" -find repo/objects -name '*.commit' | wc -l > commitcount -assert_file_has_content commitcount "^2$" +assert_repo_has_n_commits repo 2 ${CMD_PREFIX} ostree --repo=repo commit --branch=oldcommit tree --timestamp="2005-10-29 12:43:29 +0000" oldcommit_rev=$($OSTREE --repo=repo rev-parse oldcommit) @@ -200,8 +200,7 @@ for x in $(seq 3); do ${CMD_PREFIX} ostree --repo=datetest-snapshot-repo commit --branch=stable -m test -s "new stable build $x" tree ${CMD_PREFIX} ostree --repo=datetest-snapshot-repo commit --branch=dev -m test -s "new dev build $x" tree done -find datetest-snapshot-repo/objects -name '*.commit' | wc -l > commitcount -assert_file_has_content commitcount "^16$" +assert_repo_has_n_commits datetest-snapshot-repo 16 # Snapshot the above reinitialize_datesnap_repo() { @@ -219,18 +218,15 @@ if ${CMD_PREFIX} ostree --repo=repo prune --keep-younger-than="1 week ago" --ret fi assert_file_has_content err.txt 'Invalid depth BACON' ${CMD_PREFIX} ostree --repo=repo prune --keep-younger-than="1 week ago" --retain-branch-depth=stable=-1 -find repo/objects -name '*.commit' | wc -l > commitcount -assert_file_has_content commitcount "^11$" +assert_repo_has_n_commits repo 11 # Double check our backup is unchanged -find datetest-snapshot-repo/objects -name '*.commit' | wc -l > commitcount -assert_file_has_content commitcount "^16$" +assert_repo_has_n_commits datetest-snapshot-repo 16 $OSTREE fsck # Again but this time only retain 6 (5+1) commits on stable. This should drop # out 8 - 6 = 2 commits (so the 11 above minus 2 = 9) ${CMD_PREFIX} ostree --repo=repo prune --keep-younger-than="1 week ago" --retain-branch-depth=stable=5 -find repo/objects -name '*.commit' | wc -l > commitcount -assert_file_has_content commitcount "^9$" +assert_repo_has_n_commits repo 9 $OSTREE fsck echo "ok retain branch depth and keep-younger-than" @@ -238,8 +234,7 @@ echo "ok retain branch depth and keep-younger-than" # so 8 stable + 1 dev = 9 reinitialize_datesnap_repo ${CMD_PREFIX} ostree --repo=repo prune --depth=0 --retain-branch-depth=stable=-1 -find repo/objects -name '*.commit' | wc -l > commitcount -assert_file_has_content commitcount "^9$" +assert_repo_has_n_commits repo 9 $OSTREE fsck echo "ok retain branch depth (alone)" From 65258d3a193a6bbb3fddcd8294f9bb4c0a4c3708 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 1 Sep 2017 15:55:17 -0400 Subject: [PATCH 2/5] bin/prune: Add --only-branch In 5c940987e768523ef1411b65bcaad09fba6befef / #646 we added `--retain-branch-depth`; this adds a symmetric `--only-branch` for the case where a repo owner just wants to prune a specific branch. The implementation here is pretty straightforward; we just walk all refs and inject the equivalent of `--retain-branch-depth=$ref=-1` if they're *not* in `--only-branch`. Closes: https://github.com/ostreedev/ostree/issues/1115 --- src/ostree/ot-builtin-prune.c | 41 ++++++++++++++++++++++++++++++++++- tests/test-prune.sh | 40 +++++++++++++++++++++++++++++++++- 2 files changed, 79 insertions(+), 2 deletions(-) diff --git a/src/ostree/ot-builtin-prune.c b/src/ostree/ot-builtin-prune.c index 2f66334ec2..35d92acb51 100644 --- a/src/ostree/ot-builtin-prune.c +++ b/src/ostree/ot-builtin-prune.c @@ -35,6 +35,7 @@ static gboolean opt_refs_only; static char *opt_delete_commit; static char *opt_keep_younger_than; static char **opt_retain_branch_depth; +static char **opt_only_branches; /* ATTENTION: * Please remember to update the bash-completion script (bash/ostree) and @@ -49,6 +50,7 @@ static GOptionEntry options[] = { { "keep-younger-than", 0, 0, G_OPTION_ARG_STRING, &opt_keep_younger_than, "Prune all commits older than the specified date", "DATE" }, { "static-deltas-only", 0, 0, G_OPTION_ARG_NONE, &opt_static_deltas_only, "Change the behavior of delete-commit and keep-younger-than to prune only static deltas" }, { "retain-branch-depth", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_retain_branch_depth, "Additionally retain BRANCH=DEPTH commits", "BRANCH=DEPTH" }, + { "only-branch", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_only_branches, "Only prune BRANCH (may be specified multiple times)", "BRANCH" }, { NULL } }; @@ -185,7 +187,7 @@ ostree_builtin_prune (int argc, char **argv, GCancellable *cancellable, GError * gint n_objects_total; gint n_objects_pruned; guint64 objsize_total; - if (!(opt_retain_branch_depth || opt_keep_younger_than)) + if (!(opt_retain_branch_depth || opt_keep_younger_than || opt_only_branches)) { if (!ostree_repo_prune (repo, pruneflags, opt_depth, &n_objects_total, &n_objects_pruned, &objsize_total, @@ -210,6 +212,7 @@ ostree_builtin_prune (int argc, char **argv, GCancellable *cancellable, GError * return glnx_throw (error, "Could not parse '%s'", opt_keep_younger_than); } + /* Process --retain-branch-depth */ for (char **iter = opt_retain_branch_depth; iter && *iter; iter++) { /* bd should look like BRANCH=DEPTH where DEPTH is an int */ @@ -239,6 +242,41 @@ ostree_builtin_prune (int argc, char **argv, GCancellable *cancellable, GError * cancellable, error)) return FALSE; + /* Process --only-branch. Note this combines with --retain-branch-depth; one + * could do e.g.: + * * --only-branch exampleos/x86_64/foo + * * --retain-branch-depth exampleos/x86_64/foo=0 + * to prune exampleos/x86_64/foo to just the latest commit. + */ + if (opt_only_branches) + { + /* Turn --only-branch into a set */ + g_autoptr(GHashTable) only_branches_set = g_hash_table_new (g_str_hash, g_str_equal); + for (char **iter = opt_only_branches; iter && *iter; iter++) + { + const char *ref = *iter; + g_autofree char *commit = NULL; + /* Ensure the specified branch exists */ + if (!ostree_repo_resolve_rev (repo, ref, FALSE, &commit, error)) + return FALSE; + g_hash_table_add (only_branches_set, *iter); + } + + /* Iterate over all refs, add equivalent of --retain-branch-depth=$ref=-1 + * if the ref isn't in --only-branch set. + */ + g_hash_table_iter_init (&hash_iter, all_refs); + while (g_hash_table_iter_next (&hash_iter, &key, &value)) + { + const char *ref = key; + if (!g_hash_table_contains (only_branches_set, ref)) + g_hash_table_insert (retain_branch_depth, g_strdup (ref), GINT_TO_POINTER ((int)-1)); + } + } + + /* Traverse each ref, and gather all objects pointed to by it up to a + * specific depth (if configured). + */ g_hash_table_iter_init (&hash_iter, all_refs); while (g_hash_table_iter_next (&hash_iter, &key, &value)) { @@ -275,6 +313,7 @@ ostree_builtin_prune (int argc, char **argv, GCancellable *cancellable, GError * return FALSE; } + /* We've gathered the reachable set; start the prune ✀ */ { OstreeRepoPruneOptions opts = { pruneflags, reachable }; if (!ostree_repo_prune_from_reachable (repo, &opts, &n_objects_total, diff --git a/tests/test-prune.sh b/tests/test-prune.sh index 4246f75667..eaa317266a 100755 --- a/tests/test-prune.sh +++ b/tests/test-prune.sh @@ -25,7 +25,7 @@ skip_without_user_xattrs setup_fake_remote_repo1 "archive" -echo '1..5' +echo '1..10' cd ${test_tmpdir} mkdir repo @@ -238,3 +238,41 @@ assert_repo_has_n_commits repo 9 $OSTREE fsck echo "ok retain branch depth (alone)" + +# Test --only-branch with --depth=0; this should be exactly identical to the +# above with a result of 9. +reinitialize_datesnap_repo +${CMD_PREFIX} ostree --repo=repo prune --only-branch=dev --depth=0 +assert_repo_has_n_commits repo 9 +$OSTREE fsck +echo "ok --only-branch --depth=0" + +# Test --only-branch with --depth=1; should just add 1 to the above, for 10. +reinitialize_datesnap_repo +${CMD_PREFIX} ostree --repo=repo prune --only-branch=dev --depth=1 +assert_repo_has_n_commits repo 10 +echo "ok --only-branch --depth=1" + +# Test --only-branch with all branches +reinitialize_datesnap_repo +${CMD_PREFIX} ostree --repo=repo prune --only-branch=dev --only-branch=stable --depth=0 +assert_repo_has_n_commits repo 2 +reinitialize_datesnap_repo +${CMD_PREFIX} ostree --repo=repo prune --only-branch=dev --only-branch=stable --depth=1 +assert_repo_has_n_commits repo 4 +echo "ok --only-branch (all) --depth=1" + +# Test --only-branch with --keep-younger-than; this should be identical to the test +# above for --retain-branch-depth=stable=-1 +reinitialize_datesnap_repo +${CMD_PREFIX} ostree --repo=repo prune --only-branch=stable --keep-younger-than="1 week ago" +assert_repo_has_n_commits repo 11 +echo "ok --only-branch --keep-younger-than" + +# Test --only-branch with a nonexistent ref +reinitialize_datesnap_repo +if ${CMD_PREFIX} ostree --repo=repo prune --only-branch=BACON 2>err.txt; then + fatal "we pruned BACON?" +fi +assert_file_has_content err.txt "Refspec.*BACON.*not found" +echo "ok --only-branch=BACON" From f962e2ccacbfd8fe4f099490b5cbc4f6c1391314 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 12 Sep 2017 16:40:17 -0400 Subject: [PATCH 3/5] fixup! tests/prune: Factor out a helper for counting commits in the repo --- tests/test-prune.sh | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/test-prune.sh b/tests/test-prune.sh index eaa317266a..c10061a2ec 100755 --- a/tests/test-prune.sh +++ b/tests/test-prune.sh @@ -47,10 +47,7 @@ ${CMD_PREFIX} ostree --repo=repo pull --depth=-1 origin test assert_repo_has_n_commits() { repo=$1 count=$2 - - find ${repo}/objects -name '*.commit' | wc -l > commitcount - assert_file_has_content commitcount '^'${count}'$' - rm -f commitcount + assert_streq "$(find ${repo}/objects -name '*.commit' | wc -l)" "${count}" } ${CMD_PREFIX} ostree prune --repo=repo --refs-only --depth=2 -v From 8ec80eba50ab5f1e841c0b027e6c824f77143d3e Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 12 Sep 2017 16:40:20 -0400 Subject: [PATCH 4/5] fixup! bin/prune: Add --only-branch --- src/ostree/ot-builtin-prune.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/ostree/ot-builtin-prune.c b/src/ostree/ot-builtin-prune.c index 35d92acb51..0e1e81dabc 100644 --- a/src/ostree/ot-builtin-prune.c +++ b/src/ostree/ot-builtin-prune.c @@ -255,20 +255,19 @@ ostree_builtin_prune (int argc, char **argv, GCancellable *cancellable, GError * for (char **iter = opt_only_branches; iter && *iter; iter++) { const char *ref = *iter; - g_autofree char *commit = NULL; /* Ensure the specified branch exists */ - if (!ostree_repo_resolve_rev (repo, ref, FALSE, &commit, error)) + if (!ostree_repo_resolve_rev (repo, ref, FALSE, NULL, error)) return FALSE; - g_hash_table_add (only_branches_set, *iter); + if (g_hash_table_contains (retain_branch_depth, ref)) + return glnx_throw (error, "--retain-branch-depth conflicts with --only-branch"); + g_hash_table_add (only_branches_set, (char*)ref); } /* Iterate over all refs, add equivalent of --retain-branch-depth=$ref=-1 * if the ref isn't in --only-branch set. */ - g_hash_table_iter_init (&hash_iter, all_refs); - while (g_hash_table_iter_next (&hash_iter, &key, &value)) + GLNX_HASH_TABLE_FOREACH (all_refs, const char *, ref) { - const char *ref = key; if (!g_hash_table_contains (only_branches_set, ref)) g_hash_table_insert (retain_branch_depth, g_strdup (ref), GINT_TO_POINTER ((int)-1)); } From 03aeae74dfef19f03c29df1217f546472f6b3084 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 15 Sep 2017 16:57:32 -0400 Subject: [PATCH 5/5] fixup! bin/prune: Add --only-branch --- src/ostree/ot-builtin-prune.c | 17 +++++++++++------ tests/test-prune.sh | 15 ++++++++++++++- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/src/ostree/ot-builtin-prune.c b/src/ostree/ot-builtin-prune.c index 0e1e81dabc..6120e0a924 100644 --- a/src/ostree/ot-builtin-prune.c +++ b/src/ostree/ot-builtin-prune.c @@ -245,8 +245,11 @@ ostree_builtin_prune (int argc, char **argv, GCancellable *cancellable, GError * /* Process --only-branch. Note this combines with --retain-branch-depth; one * could do e.g.: * * --only-branch exampleos/x86_64/foo + * * --only-branch exampleos/x86_64/bar * * --retain-branch-depth exampleos/x86_64/foo=0 - * to prune exampleos/x86_64/foo to just the latest commit. + * * --depth 5 + * to prune exampleos/x86_64/foo to just the latest commit, and + * exampleos/x86_64/bar to a depth of 5. */ if (opt_only_branches) { @@ -258,18 +261,20 @@ ostree_builtin_prune (int argc, char **argv, GCancellable *cancellable, GError * /* Ensure the specified branch exists */ if (!ostree_repo_resolve_rev (repo, ref, FALSE, NULL, error)) return FALSE; - if (g_hash_table_contains (retain_branch_depth, ref)) - return glnx_throw (error, "--retain-branch-depth conflicts with --only-branch"); g_hash_table_add (only_branches_set, (char*)ref); } /* Iterate over all refs, add equivalent of --retain-branch-depth=$ref=-1 - * if the ref isn't in --only-branch set. + * if the ref isn't in --only-branch set and there wasn't already a + * --retain-branch-depth specified for it. */ GLNX_HASH_TABLE_FOREACH (all_refs, const char *, ref) { - if (!g_hash_table_contains (only_branches_set, ref)) - g_hash_table_insert (retain_branch_depth, g_strdup (ref), GINT_TO_POINTER ((int)-1)); + if (!g_hash_table_contains (only_branches_set, ref) && + !g_hash_table_contains (retain_branch_depth, ref)) + { + g_hash_table_insert (retain_branch_depth, g_strdup (ref), GINT_TO_POINTER ((int)-1)); + } } } diff --git a/tests/test-prune.sh b/tests/test-prune.sh index c10061a2ec..07548822a9 100755 --- a/tests/test-prune.sh +++ b/tests/test-prune.sh @@ -25,7 +25,7 @@ skip_without_user_xattrs setup_fake_remote_repo1 "archive" -echo '1..10' +echo '1..12' cd ${test_tmpdir} mkdir repo @@ -259,6 +259,19 @@ ${CMD_PREFIX} ostree --repo=repo prune --only-branch=dev --only-branch=stable -- assert_repo_has_n_commits repo 4 echo "ok --only-branch (all) --depth=1" +# Test --only-branch and --retain-branch-depth overlap +reinitialize_datesnap_repo +${CMD_PREFIX} ostree --repo=repo prune --only-branch=dev --only-branch=stable --depth=0 \ + --retain-branch-depth=stable=2 +assert_repo_has_n_commits repo 4 +echo "ok --only-branch and --retain-branch-depth overlap" + +# Test --only-branch and --retain-branch-depth together +reinitialize_datesnap_repo +${CMD_PREFIX} ostree --repo=repo prune --only-branch=dev --depth=0 --retain-branch-depth=stable=2 +assert_repo_has_n_commits repo 4 +echo "ok --only-branch and --retain-branch-depth together" + # Test --only-branch with --keep-younger-than; this should be identical to the test # above for --retain-branch-depth=stable=-1 reinitialize_datesnap_repo