commit: Add _CONSUME modifier flag

For many cases of commit, we can actually optimize things by simply "adopting"
the object rather than writing a new copy. For example, in rpm-ostree package
layering.

We can only make that optimization though if we take ownership of the file. This
commit hence adds an API where a caller tells us to do so. For now, that just
means we `unlink()` the files/dirs as we go, but we can now later add the
"adopt" optimization.

Closes: #1255
Approved by: jlebon

bash/ostree

@@ -773,6 +773,7 @@ _ostree_commit() {
         --link-checkout-speedup
         --no-xattrs
         --orphan
+        --consume
         --skip-if-unchanged
         --table-output
         --tar-autocreate-parents

man/ostree-commit.xml

@@ -167,6 +167,17 @@ Boston, MA 02111-1307, USA.
                 </para></listitem>
             </varlistentry>
 
+            <varlistentry>
+                <term><option>--consume</option></term>
+
+                <listitem><para>
+                  When committing from a local directory (i.e. not an archive or --tree=ref),
+                  assume ownership of the content.  This may simply involve deleting it,
+                  but if possible, the content may simply be <literal>rename()</literal>ed
+                  into the repository rather than creating a new copy.
+                </para></listitem>
+            </varlistentry>
+
             <varlistentry>
                 <term><option>--statoverride</option>="PATH"</term>
 

src/libostree/ostree-repo-commit.c

@@ -2589,9 +2589,22 @@ write_directory_content_to_mtree_internal (OstreeRepo                  *self,
     _ostree_repo_commit_modifier_apply (self, modifier, child_relpath, child_info, &modified_info);
   const gboolean child_info_was_modified = !_ostree_gfileinfo_equal (child_info, modified_info);
 
+  /* We currently only honor the CONSUME flag in the dfd_iter case to avoid even
+   * more complexity in this function, and it'd mostly only be useful when
+   * operating on local filesystems anyways.
+   */
+  const gboolean delete_after_commit = dfd_iter && modifier &&
+    (modifier->flags & OSTREE_REPO_COMMIT_MODIFIER_FLAGS_CONSUME);
+
   if (filter_result != OSTREE_REPO_COMMIT_FILTER_ALLOW)
     {
       g_ptr_array_remove_index (path, path->len - 1);
+      if (delete_after_commit)
+        {
+          g_assert (dfd_iter);
+          if (!glnx_shutil_rm_rf_at (dfd_iter->fd, name, cancellable, error))
+            return FALSE;
+        }
       /* Note: early return */
       return TRUE;
     }
@@ -2635,6 +2648,12 @@ write_directory_content_to_mtree_internal (OstreeRepo                  *self,
                                                  modifier, path,
                                                  cancellable, error))
             return FALSE;
+
+          if (delete_after_commit)
+            {
+              if (!glnx_unlinkat (dfd_iter->fd, name, AT_REMOVEDIR, error))
+                return FALSE;
+            }
         }
     }
   else if (repo_dir)
@@ -2711,6 +2730,12 @@ write_directory_content_to_mtree_internal (OstreeRepo                  *self,
                                                  error))
             return FALSE;
         }
+
+      if (delete_after_commit)
+        {
+          if (!glnx_unlinkat (dfd_iter->fd, name, 0, error))
+            return FALSE;
+        }
     }
 
   g_ptr_array_remove_index (path, path->len - 1);
@@ -2991,6 +3016,17 @@ ostree_repo_write_dfd_to_mtree (OstreeRepo                *self,
                                          cancellable, error))
     return FALSE;
 
+  /* And now finally remove the toplevel; see also the handling for this flag in
+   * the write_dfd_iter_to_mtree_internal() function.
+   */
+  const gboolean delete_after_commit = modifier &&
+    (modifier->flags & OSTREE_REPO_COMMIT_MODIFIER_FLAGS_CONSUME);
+  if (delete_after_commit)
+    {
+      if (!glnx_unlinkat (dfd, path, AT_REMOVEDIR, error))
+        return FALSE;
+    }
+
   return TRUE;
 }
 

src/libostree/ostree-repo.h

@@ -630,13 +630,15 @@ typedef OstreeRepoCommitFilterResult (*OstreeRepoCommitFilter) (OstreeRepo    *r
  * @OSTREE_REPO_COMMIT_MODIFIER_FLAGS_GENERATE_SIZES: Generate size information.
  * @OSTREE_REPO_COMMIT_MODIFIER_FLAGS_CANONICAL_PERMISSIONS: Canonicalize permissions for bare-user-only mode.
  * @OSTREE_REPO_COMMIT_MODIFIER_FLAGS_ERROR_ON_UNLABELED: Emit an error if configured SELinux policy does not provide a label
+ * @OSTREE_REPO_COMMIT_MODIFIER_FLAGS_CONSUME: Delete added files/directories after commit; Since: 2017.13
  */
 typedef enum {
   OSTREE_REPO_COMMIT_MODIFIER_FLAGS_NONE = 0,
   OSTREE_REPO_COMMIT_MODIFIER_FLAGS_SKIP_XATTRS = (1 << 0),
   OSTREE_REPO_COMMIT_MODIFIER_FLAGS_GENERATE_SIZES = (1 << 1),
   OSTREE_REPO_COMMIT_MODIFIER_FLAGS_CANONICAL_PERMISSIONS = (1 << 2),
   OSTREE_REPO_COMMIT_MODIFIER_FLAGS_ERROR_ON_UNLABELED = (1 << 3),
+  OSTREE_REPO_COMMIT_MODIFIER_FLAGS_CONSUME = (1 << 4),
 } OstreeRepoCommitModifierFlags;
 
 /**

src/ostree/ot-builtin-commit.c

@@ -50,6 +50,7 @@ static char *opt_tar_pathname_filter;
 static gboolean opt_no_xattrs;
 static char *opt_selinux_policy;
 static gboolean opt_canonical_permissions;
+static gboolean opt_consume;
 static char **opt_trees;
 static gint opt_owner_uid = -1;
 static gint opt_owner_gid = -1;
@@ -102,6 +103,7 @@ static GOptionEntry options[] = {
   { "skip-if-unchanged", 0, 0, G_OPTION_ARG_NONE, &opt_skip_if_unchanged, "If the contents are unchanged from previous commit, do nothing", NULL },
   { "statoverride", 0, 0, G_OPTION_ARG_FILENAME, &opt_statoverride_file, "File containing list of modifications to make to permissions", "PATH" },
   { "skip-list", 0, 0, G_OPTION_ARG_FILENAME, &opt_skiplist_file, "File containing list of files to skip", "PATH" },
+  { "consume", 0, 0, G_OPTION_ARG_NONE, &opt_consume, "Consume (delete) content after commit (for local directories)", NULL },
   { "table-output", 0, 0, G_OPTION_ARG_NONE, &opt_table_output, "Output more information in a KEY: VALUE format", NULL },
   { "gpg-sign", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_key_ids, "GPG Key ID to sign the commit with", "KEY-ID"},
   { "gpg-homedir", 0, 0, G_OPTION_ARG_FILENAME, &opt_gpg_homedir, "GPG Homedir to use when looking for keyrings", "HOMEDIR"},
@@ -476,6 +478,8 @@ ostree_builtin_commit (int argc, char **argv, GCancellable *cancellable, GError
 
   if (opt_no_xattrs)
     flags |= OSTREE_REPO_COMMIT_MODIFIER_FLAGS_SKIP_XATTRS;
+  if (opt_consume)
+    flags |= OSTREE_REPO_COMMIT_MODIFIER_FLAGS_CONSUME;
   if (opt_canonical_permissions)
     flags |= OSTREE_REPO_COMMIT_MODIFIER_FLAGS_CANONICAL_PERMISSIONS;
   if (opt_generate_sizes)

tests/basic-test.sh

@@ -19,7 +19,7 @@
 
 set -euo pipefail
 
-echo "1..$((73 + ${extra_basic_tests:-0}))"
+echo "1..$((74 + ${extra_basic_tests:-0}))"
 
 CHECKOUT_U_ARG=""
 CHECKOUT_H_ARGS="-H"
@@ -177,6 +177,16 @@ assert_file_has_content yet/another/tree/green 'leaf'
 assert_file_has_content four '4'
 echo "ok cwd contents"
 
+cd ${test_tmpdir}
+rm checkout-test2-l -rf
+$OSTREE checkout ${CHECKOUT_H_ARGS} test2 $test_tmpdir/checkout-test2-l
+date > $test_tmpdir/checkout-test2-l/newdatefile.txt
+$OSTREE commit --link-checkout-speedup --consume -b test2 --tree=dir=$test_tmpdir/checkout-test2-l
+assert_not_has_dir $test_tmpdir/checkout-test2-l
+# Some of the later tests are sensitive to state
+$OSTREE reset test2 test2^
+echo "ok consume (nom nom nom)"
+
 cd ${test_tmpdir}
 $OSTREE commit ${COMMIT_ARGS} -b test2-no-parent -s '' $test_tmpdir/checkout-test2-4
 assert_streq $($OSTREE log test2-no-parent |grep '^commit' | wc -l) "1"