From 50e9259f010c7f75f60859aebbcb84a5633ba920 Mon Sep 17 00:00:00 2001
From: Colin Walters <walters@verbum.org>
Date: Thu, 22 Dec 2016 17:57:04 -0500
Subject: [PATCH 01/13] fetcher: Hoist core "mirrored request" API to public

This is in preparation for the libcurl port. We're basically making public what
we had internally. The next step here is to create `ostree-fetcher-util.[ch]`
that only operates in terms of this lower level API.

Also drop the `_mirrored` from the function name since it's
the default now.
---
 src/libostree/ostree-fetcher.c   | 121 ++++++++++++++-----------------
 src/libostree/ostree-fetcher.h   |  31 +++++---
 src/libostree/ostree-repo-pull.c |  37 +++++-----
 3 files changed, 90 insertions(+), 99 deletions(-)

diff --git a/src/libostree/ostree-fetcher.c b/src/libostree/ostree-fetcher.c
index 6394216685..c69bbefad7 100644
--- a/src/libostree/ostree-fetcher.c
+++ b/src/libostree/ostree-fetcher.c
@@ -1179,17 +1179,16 @@ on_request_sent (GObject        *object,
   g_object_unref (task);
 }
 
-static void
-ostree_fetcher_mirrored_request_internal (OstreeFetcher         *self,
-                                          GPtrArray             *mirrorlist,
-                                          const char            *filename,
-                                          gboolean               is_stream,
-                                          guint64                max_size,
-                                          int                    priority,
-                                          GCancellable          *cancellable,
-                                          GAsyncReadyCallback    callback,
-                                          gpointer               user_data,
-                                          gpointer               source_tag)
+void
+_ostree_fetcher_request_async (OstreeFetcher         *self,
+                               GPtrArray             *mirrorlist,
+                               const char            *filename,
+                               OstreeFetcherRequestFlags flags,
+                               guint64                max_size,
+                               int                    priority,
+                               GCancellable          *cancellable,
+                               GAsyncReadyCallback    callback,
+                               gpointer               user_data)
 {
   g_autoptr(GTask) task = NULL;
   OstreeFetcherPendingURI *pending;
@@ -1205,10 +1204,10 @@ ostree_fetcher_mirrored_request_internal (OstreeFetcher         *self,
   pending->mirrorlist = g_ptr_array_ref (mirrorlist);
   pending->filename = g_strdup (filename);
   pending->max_size = max_size;
-  pending->is_stream = is_stream;
+  pending->is_stream = (flags & OSTREE_FETCHER_REQUEST_FLAG_ENABLE_PARTIAL) == 0;
 
   task = g_task_new (self, cancellable, callback, user_data);
-  g_task_set_source_tag (task, source_tag);
+  g_task_set_source_tag (task, _ostree_fetcher_request_async);
   g_task_set_task_data (task, pending, (GDestroyNotify) pending_uri_unref);
 
   /* We'll use the GTask priority for our own priority queue. */
@@ -1220,60 +1219,45 @@ ostree_fetcher_mirrored_request_internal (OstreeFetcher         *self,
                            (GDestroyNotify) g_object_unref);
 }
 
-void
-_ostree_fetcher_mirrored_request_with_partial_async (OstreeFetcher         *self,
-                                                     GPtrArray             *mirrorlist,
-                                                     const char            *filename,
-                                                     guint64                max_size,
-                                                     int                    priority,
-                                                     GCancellable          *cancellable,
-                                                     GAsyncReadyCallback    callback,
-                                                     gpointer               user_data)
-{
-  ostree_fetcher_mirrored_request_internal (self, mirrorlist, filename, FALSE,
-                                            max_size, priority, cancellable,
-                                            callback, user_data,
-                                            _ostree_fetcher_mirrored_request_with_partial_async);
-}
-
-char *
-_ostree_fetcher_mirrored_request_with_partial_finish (OstreeFetcher         *self,
-                                                      GAsyncResult          *result,
-                                                      GError               **error)
+gboolean
+_ostree_fetcher_request_finish (OstreeFetcher         *self,
+                                GAsyncResult          *result,
+                                char                 **out_filename,
+                                GInputStream         **out_stream,
+                                GError               **error)
 {
-  g_return_val_if_fail (g_task_is_valid (result, self), NULL);
-  g_return_val_if_fail (g_async_result_is_tagged (result,
-                        _ostree_fetcher_mirrored_request_with_partial_async), NULL);
-
-  return g_task_propagate_pointer (G_TASK (result), error);
-}
+  GTask *task;
+  OstreeFetcherPendingURI *pending;
+  gpointer ret;
+
+  g_return_val_if_fail (g_task_is_valid (result, self), FALSE);
+  g_return_val_if_fail (g_async_result_is_tagged (result, _ostree_fetcher_request_async), FALSE);
+
+  /* Special dance to implement
+    enum FetchResult {
+       Filename(String path),
+       Membuf(uint8[])
+    } in Rust terms
+  */
+  task = (GTask*)result;
+  pending = g_task_get_task_data (task);
 
-static void
-ostree_fetcher_stream_mirrored_uri_async (OstreeFetcher         *self,
-                                          GPtrArray             *mirrorlist,
-                                          const char            *filename,
-                                          guint64                max_size,
-                                          int                    priority,
-                                          GCancellable          *cancellable,
-                                          GAsyncReadyCallback    callback,
-                                          gpointer               user_data)
-{
-  ostree_fetcher_mirrored_request_internal (self, mirrorlist, filename, TRUE,
-                                            max_size, priority, cancellable,
-                                            callback, user_data,
-                                            ostree_fetcher_stream_mirrored_uri_async);
-}
+  ret = g_task_propagate_pointer (task, error);
+  if (!ret)
+    return FALSE;
 
-static GInputStream *
-ostree_fetcher_stream_mirrored_uri_finish (OstreeFetcher         *self,
-                                           GAsyncResult          *result,
-                                           GError               **error)
-{
-  g_return_val_if_fail (g_task_is_valid (result, self), NULL);
-  g_return_val_if_fail (g_async_result_is_tagged (result,
-                        ostree_fetcher_stream_mirrored_uri_async), NULL);
+  if (pending->is_stream)
+    {
+      g_assert (out_stream);
+      *out_stream = ret;
+    }
+  else
+    {
+      g_assert (out_filename);
+      *out_filename = ret;
+    }
 
-  return g_task_propagate_pointer (G_TASK (result), error);
+  return TRUE;
 }
 
 guint64
@@ -1322,8 +1306,9 @@ fetch_uri_sync_on_complete (GObject        *object,
 {
   FetchUriSyncData *data = user_data;
 
-  data->result_stream = ostree_fetcher_stream_mirrored_uri_finish ((OstreeFetcher*)object,
-                                                                    result, data->error);
+  (void)_ostree_fetcher_request_finish ((OstreeFetcher*)object,
+                                        result, NULL, &data->result_stream,
+                                        data->error);
   data->done = TRUE;
 }
 
@@ -1356,9 +1341,9 @@ _ostree_fetcher_mirrored_request_to_membuf (OstreeFetcher  *fetcher,
   data.done = FALSE;
   data.error = error;
 
-  ostree_fetcher_stream_mirrored_uri_async (fetcher, mirrorlist, filename, max_size,
-                                   OSTREE_FETCHER_DEFAULT_PRIORITY, cancellable,
-                                   fetch_uri_sync_on_complete, &data);
+  _ostree_fetcher_request_async (fetcher, mirrorlist, filename, 0, max_size,
+                                 OSTREE_FETCHER_DEFAULT_PRIORITY, cancellable,
+                                 fetch_uri_sync_on_complete, &data);
   while (!data.done)
     g_main_context_iteration (mainctx, TRUE);
 
diff --git a/src/libostree/ostree-fetcher.h b/src/libostree/ostree-fetcher.h
index 8e282e24e3..124a02bff4 100644
--- a/src/libostree/ostree-fetcher.h
+++ b/src/libostree/ostree-fetcher.h
@@ -103,18 +103,25 @@ void _ostree_fetcher_set_extra_headers (OstreeFetcher *self,
 
 guint64 _ostree_fetcher_bytes_transferred (OstreeFetcher       *self);
 
-void _ostree_fetcher_mirrored_request_with_partial_async (OstreeFetcher         *self,
-                                                          GPtrArray             *mirrorlist,
-                                                          const char            *filename,
-                                                          guint64                max_size,
-                                                          int                    priority,
-                                                          GCancellable          *cancellable,
-                                                          GAsyncReadyCallback    callback,
-                                                          gpointer               user_data);
-
-char *_ostree_fetcher_mirrored_request_with_partial_finish (OstreeFetcher *self,
-                                                            GAsyncResult  *result,
-                                                            GError       **error);
+typedef enum {
+  OSTREE_FETCHER_REQUEST_FLAG_ENABLE_PARTIAL = (1 << 0)
+} OstreeFetcherRequestFlags;
+
+void _ostree_fetcher_request_async (OstreeFetcher         *self,
+                                    GPtrArray             *mirrorlist,
+                                    const char            *filename,
+                                    OstreeFetcherRequestFlags flags,
+                                    guint64                max_size,
+                                    int                    priority,
+                                    GCancellable          *cancellable,
+                                    GAsyncReadyCallback    callback,
+                                    gpointer               user_data);
+
+gboolean _ostree_fetcher_request_finish (OstreeFetcher *self,
+                                         GAsyncResult  *result,
+                                         char         **out_filename,
+                                         GInputStream **out_stream,
+                                         GError       **error);
 
 gboolean _ostree_fetcher_mirrored_request_to_membuf (OstreeFetcher *fetcher,
                                                      GPtrArray     *mirrorlist,
diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c
index 9c99dc4fbd..32ad5fc06c 100644
--- a/src/libostree/ostree-repo-pull.c
+++ b/src/libostree/ostree-repo-pull.c
@@ -707,8 +707,7 @@ content_fetch_on_complete (GObject        *object,
   OstreeObjectType objtype;
   gboolean free_fetch_data = TRUE;
 
-  temp_path = _ostree_fetcher_mirrored_request_with_partial_finish (fetcher, result, error);
-  if (!temp_path)
+  if (!_ostree_fetcher_request_finish (fetcher, result, &temp_path, NULL, error))
     goto out;
 
   ostree_object_name_deserialize (fetch_data->object, &checksum, &objtype);
@@ -841,8 +840,7 @@ meta_fetch_on_complete (GObject           *object,
   g_debug ("fetch of %s%s complete", checksum_obj,
            fetch_data->is_detached_meta ? " (detached)" : "");
 
-  temp_path = _ostree_fetcher_mirrored_request_with_partial_finish (fetcher, result, error);
-  if (!temp_path)
+  if (!_ostree_fetcher_request_finish (fetcher, result, &temp_path, NULL, error))
     {
       if (g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
         {
@@ -982,8 +980,7 @@ static_deltapart_fetch_on_complete (GObject           *object,
 
   g_debug ("fetch static delta part %s complete", fetch_data->expected_checksum);
 
-  temp_path = _ostree_fetcher_mirrored_request_with_partial_finish (fetcher, result, error);
-  if (!temp_path)
+  if (!_ostree_fetcher_request_finish (fetcher, result, &temp_path, NULL, error))
     goto out;
 
   fd = openat (_ostree_fetcher_get_dfd (fetcher), temp_path, O_RDONLY | O_CLOEXEC);
@@ -1382,12 +1379,13 @@ enqueue_one_object_request (OtPullData        *pull_data,
   else
     expected_max_size = 0;
 
-  _ostree_fetcher_mirrored_request_with_partial_async (pull_data->fetcher, mirrorlist,
-                                                       obj_subpath, expected_max_size,
-                                                       is_meta ? OSTREE_REPO_PULL_METADATA_PRIORITY
-                                                               : OSTREE_REPO_PULL_CONTENT_PRIORITY,
-                                                       pull_data->cancellable,
-                                                       is_meta ? meta_fetch_on_complete : content_fetch_on_complete, fetch_data);
+  _ostree_fetcher_request_async (pull_data->fetcher, mirrorlist,
+                                 obj_subpath, OSTREE_FETCHER_REQUEST_FLAG_ENABLE_PARTIAL,
+                                 expected_max_size,
+                                 is_meta ? OSTREE_REPO_PULL_METADATA_PRIORITY
+                                 : OSTREE_REPO_PULL_CONTENT_PRIORITY,
+                                 pull_data->cancellable,
+                                 is_meta ? meta_fetch_on_complete : content_fetch_on_complete, fetch_data);
 }
 
 static gboolean
@@ -1735,13 +1733,14 @@ process_one_static_delta (OtPullData   *pull_data,
         }
       else
         {
-          _ostree_fetcher_mirrored_request_with_partial_async (pull_data->fetcher,
-                                                               pull_data->content_mirrorlist,
-                                                               deltapart_path, size,
-                                                               OSTREE_FETCHER_DEFAULT_PRIORITY,
-                                                               pull_data->cancellable,
-                                                               static_deltapart_fetch_on_complete,
-                                                               fetch_data);
+          _ostree_fetcher_request_async (pull_data->fetcher,
+                                         pull_data->content_mirrorlist,
+                                         deltapart_path, OSTREE_FETCHER_REQUEST_FLAG_ENABLE_PARTIAL,
+                                         size,
+                                         OSTREE_FETCHER_DEFAULT_PRIORITY,
+                                         pull_data->cancellable,
+                                         static_deltapart_fetch_on_complete,
+                                         fetch_data);
           pull_data->n_outstanding_deltapart_fetches++;
         }
     }

From c59b09e2f86c0c49abf237a5884f53f938827851 Mon Sep 17 00:00:00 2001
From: Colin Walters <walters@verbum.org>
Date: Thu, 22 Dec 2016 20:34:07 -0500
Subject: [PATCH 02/13] fetcher: Move high level functions into "fetcher-util"

Conceptually these now lay on top of the core API, and don't reference libsoup.
This is preparation for libcurl porting, but it's also just generally better.
---
 Makefile-libostree.am               |   2 +
 src/libostree/ostree-fetcher-util.c | 140 ++++++++++++++++++++++++++++
 src/libostree/ostree-fetcher-util.h |  49 ++++++++++
 src/libostree/ostree-fetcher.c      | 113 ----------------------
 src/libostree/ostree-fetcher.h      |  18 ----
 src/libostree/ostree-metalink.c     |   1 +
 src/libostree/ostree-repo-pull.c    |   1 +
 7 files changed, 193 insertions(+), 131 deletions(-)
 create mode 100644 src/libostree/ostree-fetcher-util.c
 create mode 100644 src/libostree/ostree-fetcher-util.h

diff --git a/Makefile-libostree.am b/Makefile-libostree.am
index a7d7b11b1e..88210c75ce 100644
--- a/Makefile-libostree.am
+++ b/Makefile-libostree.am
@@ -161,6 +161,8 @@ if USE_LIBSOUP
 libostree_1_la_SOURCES += \
 	src/libostree/ostree-fetcher.h \
 	src/libostree/ostree-fetcher.c \
+	src/libostree/ostree-fetcher-util.h \
+	src/libostree/ostree-fetcher-util.c \
 	src/libostree/ostree-metalink.h \
 	src/libostree/ostree-metalink.c \
 	$(NULL)
diff --git a/src/libostree/ostree-fetcher-util.c b/src/libostree/ostree-fetcher-util.c
new file mode 100644
index 0000000000..bc97674d9e
--- /dev/null
+++ b/src/libostree/ostree-fetcher-util.c
@@ -0,0 +1,140 @@
+/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*-
+ *
+ * Copyright (C) 2017 Colin Walters <walters@verbum.org>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 02111-1307, USA.
+ */
+
+#include "config.h"
+
+#include <gio/gfiledescriptorbased.h>
+#include <gio/gunixoutputstream.h>
+
+#include "ostree-fetcher-util.h"
+#include "otutil.h"
+
+typedef struct
+{
+  GInputStream   *result_stream;
+  gboolean         done;
+  GError         **error;
+}
+  FetchUriSyncData;
+
+static void
+fetch_uri_sync_on_complete (GObject        *object,
+                            GAsyncResult   *result,
+                            gpointer        user_data)
+{
+  FetchUriSyncData *data = user_data;
+
+  (void)_ostree_fetcher_request_finish ((OstreeFetcher*)object,
+                                        result, NULL, &data->result_stream,
+                                        data->error);
+  data->done = TRUE;
+}
+
+gboolean
+_ostree_fetcher_mirrored_request_to_membuf (OstreeFetcher  *fetcher,
+                                            GPtrArray     *mirrorlist,
+                                            const char     *filename,
+                                            gboolean        add_nul,
+                                            gboolean        allow_noent,
+                                            GBytes         **out_contents,
+                                            guint64        max_size,
+                                            GCancellable   *cancellable,
+                                            GError         **error)
+{
+  gboolean ret = FALSE;
+  const guint8 nulchar = 0;
+  g_autoptr(GMemoryOutputStream) buf = NULL;
+  g_autoptr(GMainContext) mainctx = NULL;
+  FetchUriSyncData data;
+  g_assert (error != NULL);
+
+  data.result_stream = NULL;
+
+  if (g_cancellable_set_error_if_cancelled (cancellable, error))
+    return FALSE;
+
+  mainctx = g_main_context_new ();
+  g_main_context_push_thread_default (mainctx);
+
+  data.done = FALSE;
+  data.error = error;
+
+  _ostree_fetcher_request_async (fetcher, mirrorlist, filename, 0, max_size,
+                                 OSTREE_FETCHER_DEFAULT_PRIORITY, cancellable,
+                                 fetch_uri_sync_on_complete, &data);
+  while (!data.done)
+    g_main_context_iteration (mainctx, TRUE);
+
+  if (!data.result_stream)
+    {
+      if (allow_noent)
+        {
+          if (g_error_matches (*error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
+            {
+              g_clear_error (error);
+              ret = TRUE;
+              *out_contents = NULL;
+            }
+        }
+      goto out;
+    }
+
+  buf = (GMemoryOutputStream*)g_memory_output_stream_new (NULL, 0, g_realloc, g_free);
+  if (g_output_stream_splice ((GOutputStream*)buf, data.result_stream,
+                              G_OUTPUT_STREAM_SPLICE_CLOSE_SOURCE,
+                              cancellable, error) < 0)
+    goto out;
+
+  if (add_nul)
+    {
+      if (!g_output_stream_write ((GOutputStream*)buf, &nulchar, 1, cancellable, error))
+        goto out;
+    }
+
+  if (!g_output_stream_close ((GOutputStream*)buf, cancellable, error))
+    goto out;
+
+  ret = TRUE;
+  *out_contents = g_memory_output_stream_steal_as_bytes (buf);
+ out:
+  if (mainctx)
+    g_main_context_pop_thread_default (mainctx);
+  g_clear_object (&(data.result_stream));
+  return ret;
+}
+
+/* Helper for callers who just want to fetch single one-off URIs */
+gboolean
+_ostree_fetcher_request_uri_to_membuf (OstreeFetcher  *fetcher,
+                                       OstreeFetcherURI *uri,
+                                       gboolean        add_nul,
+                                       gboolean        allow_noent,
+                                       GBytes         **out_contents,
+                                       guint64        max_size,
+                                       GCancellable   *cancellable,
+                                       GError         **error)
+{
+  g_autoptr(GPtrArray) mirrorlist = g_ptr_array_new ();
+  g_ptr_array_add (mirrorlist, uri); /* no transfer */
+  return _ostree_fetcher_mirrored_request_to_membuf (fetcher, mirrorlist, NULL,
+                                                     add_nul, allow_noent,
+                                                     out_contents, max_size,
+                                                     cancellable, error);
+}
diff --git a/src/libostree/ostree-fetcher-util.h b/src/libostree/ostree-fetcher-util.h
new file mode 100644
index 0000000000..0f25dc306e
--- /dev/null
+++ b/src/libostree/ostree-fetcher-util.h
@@ -0,0 +1,49 @@
+/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*-
+ *
+ * Copyright (C) 2016 Colin Walters <walters@verbum.org>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 02111-1307, USA.
+ */
+
+#pragma once
+
+#ifndef __GI_SCANNER__
+
+#include "ostree-fetcher.h"
+
+G_BEGIN_DECLS
+
+gboolean _ostree_fetcher_mirrored_request_to_membuf (OstreeFetcher *fetcher,
+                                                     GPtrArray     *mirrorlist,
+                                                     const char    *filename,
+                                                     gboolean       add_nul,
+                                                     gboolean       allow_noent,
+                                                     GBytes         **out_contents,
+                                                     guint64        max_size,
+                                                     GCancellable   *cancellable,
+                                                     GError         **error);
+
+gboolean _ostree_fetcher_request_uri_to_membuf (OstreeFetcher *fetcher,
+                                                OstreeFetcherURI *uri,
+                                                gboolean       add_nul,
+                                                gboolean       allow_noent,
+                                                GBytes         **out_contents,
+                                                guint64        max_size,
+                                                GCancellable   *cancellable,
+                                                GError         **error);
+G_END_DECLS
+
+#endif
diff --git a/src/libostree/ostree-fetcher.c b/src/libostree/ostree-fetcher.c
index c69bbefad7..7917f729de 100644
--- a/src/libostree/ostree-fetcher.c
+++ b/src/libostree/ostree-fetcher.c
@@ -1291,119 +1291,6 @@ _ostree_fetcher_bytes_transferred (OstreeFetcher       *self)
   return ret;
 }
 
-typedef struct
-{
-  GInputStream   *result_stream;
-  gboolean         done;
-  GError         **error;
-}
-FetchUriSyncData;
-
-static void
-fetch_uri_sync_on_complete (GObject        *object,
-                            GAsyncResult   *result,
-                            gpointer        user_data)
-{
-  FetchUriSyncData *data = user_data;
-
-  (void)_ostree_fetcher_request_finish ((OstreeFetcher*)object,
-                                        result, NULL, &data->result_stream,
-                                        data->error);
-  data->done = TRUE;
-}
-
-gboolean
-_ostree_fetcher_mirrored_request_to_membuf (OstreeFetcher  *fetcher,
-                                            GPtrArray     *mirrorlist,
-                                            const char     *filename,
-                                            gboolean        add_nul,
-                                            gboolean        allow_noent,
-                                            GBytes         **out_contents,
-                                            guint64        max_size,
-                                            GCancellable   *cancellable,
-                                            GError         **error)
-{
-  gboolean ret = FALSE;
-  const guint8 nulchar = 0;
-  g_autoptr(GMemoryOutputStream) buf = NULL;
-  g_autoptr(GMainContext) mainctx = NULL;
-  FetchUriSyncData data;
-  g_assert (error != NULL);
-
-  data.result_stream = NULL;
-
-  if (g_cancellable_set_error_if_cancelled (cancellable, error))
-    return FALSE;
-
-  mainctx = g_main_context_new ();
-  g_main_context_push_thread_default (mainctx);
-
-  data.done = FALSE;
-  data.error = error;
-
-  _ostree_fetcher_request_async (fetcher, mirrorlist, filename, 0, max_size,
-                                 OSTREE_FETCHER_DEFAULT_PRIORITY, cancellable,
-                                 fetch_uri_sync_on_complete, &data);
-  while (!data.done)
-    g_main_context_iteration (mainctx, TRUE);
-
-  if (!data.result_stream)
-    {
-      if (allow_noent)
-        {
-          if (g_error_matches (*error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
-            {
-              g_clear_error (error);
-              ret = TRUE;
-              *out_contents = NULL;
-            }
-        }
-      goto out;
-    }
-
-  buf = (GMemoryOutputStream*)g_memory_output_stream_new (NULL, 0, g_realloc, g_free);
-  if (g_output_stream_splice ((GOutputStream*)buf, data.result_stream,
-                              G_OUTPUT_STREAM_SPLICE_CLOSE_SOURCE,
-                              cancellable, error) < 0)
-    goto out;
-
-  if (add_nul)
-    {
-      if (!g_output_stream_write ((GOutputStream*)buf, &nulchar, 1, cancellable, error))
-        goto out;
-    }
-
-  if (!g_output_stream_close ((GOutputStream*)buf, cancellable, error))
-    goto out;
-
-  ret = TRUE;
-  *out_contents = g_memory_output_stream_steal_as_bytes (buf);
- out:
-  if (mainctx)
-    g_main_context_pop_thread_default (mainctx);
-  g_clear_object (&(data.result_stream));
-  return ret;
-}
-
-/* Helper for callers who just want to fetch single one-off URIs */
-gboolean
-_ostree_fetcher_request_uri_to_membuf (OstreeFetcher  *fetcher,
-                                       OstreeFetcherURI *uri,
-                                       gboolean        add_nul,
-                                       gboolean        allow_noent,
-                                       GBytes         **out_contents,
-                                       guint64        max_size,
-                                       GCancellable   *cancellable,
-                                       GError         **error)
-{
-  g_autoptr(GPtrArray) mirrorlist = g_ptr_array_new ();
-  g_ptr_array_add (mirrorlist, uri); /* no transfer */
-  return _ostree_fetcher_mirrored_request_to_membuf (fetcher, mirrorlist, NULL,
-                                                     add_nul, allow_noent,
-                                                     out_contents, max_size,
-                                                     cancellable, error);
-}
-
 void
 _ostree_fetcher_uri_free (OstreeFetcherURI *uri)
 {
diff --git a/src/libostree/ostree-fetcher.h b/src/libostree/ostree-fetcher.h
index 124a02bff4..10ec51dc28 100644
--- a/src/libostree/ostree-fetcher.h
+++ b/src/libostree/ostree-fetcher.h
@@ -123,24 +123,6 @@ gboolean _ostree_fetcher_request_finish (OstreeFetcher *self,
                                          GInputStream **out_stream,
                                          GError       **error);
 
-gboolean _ostree_fetcher_mirrored_request_to_membuf (OstreeFetcher *fetcher,
-                                                     GPtrArray     *mirrorlist,
-                                                     const char    *filename,
-                                                     gboolean       add_nul,
-                                                     gboolean       allow_noent,
-                                                     GBytes         **out_contents,
-                                                     guint64        max_size,
-                                                     GCancellable   *cancellable,
-                                                     GError         **error);
-
-gboolean _ostree_fetcher_request_uri_to_membuf (OstreeFetcher *fetcher,
-                                                OstreeFetcherURI *uri,
-                                                gboolean       add_nul,
-                                                gboolean       allow_noent,
-                                                GBytes         **out_contents,
-                                                guint64        max_size,
-                                                GCancellable   *cancellable,
-                                                GError         **error);
 G_END_DECLS
 
 #endif
diff --git a/src/libostree/ostree-metalink.c b/src/libostree/ostree-metalink.c
index ee52e51be0..6afae59e77 100644
--- a/src/libostree/ostree-metalink.c
+++ b/src/libostree/ostree-metalink.c
@@ -21,6 +21,7 @@
 #include "config.h"
 
 #include "ostree-metalink.h"
+#include "ostree-fetcher-util.h"
 #include <gio/gfiledescriptorbased.h>
 
 #include "otutil.h"
diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c
index 32ad5fc06c..100d930486 100644
--- a/src/libostree/ostree-repo-pull.c
+++ b/src/libostree/ostree-repo-pull.c
@@ -32,6 +32,7 @@
 #include "ostree-repo-private.h"
 #include "ostree-repo-static-delta-private.h"
 #include "ostree-metalink.h"
+#include "ostree-fetcher-util.h"
 #include "ot-fs-utils.h"
 
 #include <gio/gunixinputstream.h>

From 8dc0db34f2e99a2e4e4438d265b7a83b1defa134 Mon Sep 17 00:00:00 2001
From: Colin Walters <walters@verbum.org>
Date: Wed, 28 Dec 2016 14:43:28 -0500
Subject: [PATCH 03/13] fetcher: Split lowlevel API into file/membuf variants

The previous commit introduced a single low level API - however,
we can do things in a more optimal way for the curl backend if
we drop the "streaming API" variant.  Currently, we only use
it to synchronously splice into a memory buffer...which is pretty
silly when we could just do that in the backend.

The only tweak here is that we have an "add NUL character" flag that is
(possibly) needed when fetching into a membuf.

The code here ends up being better I think, since we avoid the double return
value for the `_finish()` invocation, and now most of the fetcher code (in the
soup case) writes to a `GOutputStream` consistently.

This will again make things easier for a curl backend.
---
 src/libostree/ostree-fetcher-util.c |  39 ++----
 src/libostree/ostree-fetcher.c      | 210 ++++++++++++++++++----------
 src/libostree/ostree-fetcher.h      |  46 +++---
 src/libostree/ostree-repo-pull.c    |  34 +++--
 4 files changed, 195 insertions(+), 134 deletions(-)

diff --git a/src/libostree/ostree-fetcher-util.c b/src/libostree/ostree-fetcher-util.c
index bc97674d9e..fd9dfea123 100644
--- a/src/libostree/ostree-fetcher-util.c
+++ b/src/libostree/ostree-fetcher-util.c
@@ -28,7 +28,7 @@
 
 typedef struct
 {
-  GInputStream   *result_stream;
+  GBytes          *result_buf;
   gboolean         done;
   GError         **error;
 }
@@ -41,9 +41,9 @@ fetch_uri_sync_on_complete (GObject        *object,
 {
   FetchUriSyncData *data = user_data;
 
-  (void)_ostree_fetcher_request_finish ((OstreeFetcher*)object,
-                                        result, NULL, &data->result_stream,
-                                        data->error);
+  (void)_ostree_fetcher_request_to_membuf_finish ((OstreeFetcher*)object,
+                                                  result, &data->result_buf,
+                                                  data->error);
   data->done = TRUE;
 }
 
@@ -60,12 +60,11 @@ _ostree_fetcher_mirrored_request_to_membuf (OstreeFetcher  *fetcher,
 {
   gboolean ret = FALSE;
   const guint8 nulchar = 0;
-  g_autoptr(GMemoryOutputStream) buf = NULL;
   g_autoptr(GMainContext) mainctx = NULL;
   FetchUriSyncData data;
   g_assert (error != NULL);
 
-  data.result_stream = NULL;
+  memset (&data, 0, sizeof (data));
 
   if (g_cancellable_set_error_if_cancelled (cancellable, error))
     return FALSE;
@@ -76,13 +75,14 @@ _ostree_fetcher_mirrored_request_to_membuf (OstreeFetcher  *fetcher,
   data.done = FALSE;
   data.error = error;
 
-  _ostree_fetcher_request_async (fetcher, mirrorlist, filename, 0, max_size,
-                                 OSTREE_FETCHER_DEFAULT_PRIORITY, cancellable,
-                                 fetch_uri_sync_on_complete, &data);
+  _ostree_fetcher_request_to_membuf (fetcher, mirrorlist, filename,
+                                     add_nul ? OSTREE_FETCHER_REQUEST_NUL_TERMINATION : 0,
+                                     max_size, OSTREE_FETCHER_DEFAULT_PRIORITY,
+                                     cancellable, fetch_uri_sync_on_complete, &data);
   while (!data.done)
     g_main_context_iteration (mainctx, TRUE);
 
-  if (!data.result_stream)
+  if (!data.result_buf)
     {
       if (allow_noent)
         {
@@ -96,27 +96,12 @@ _ostree_fetcher_mirrored_request_to_membuf (OstreeFetcher  *fetcher,
       goto out;
     }
 
-  buf = (GMemoryOutputStream*)g_memory_output_stream_new (NULL, 0, g_realloc, g_free);
-  if (g_output_stream_splice ((GOutputStream*)buf, data.result_stream,
-                              G_OUTPUT_STREAM_SPLICE_CLOSE_SOURCE,
-                              cancellable, error) < 0)
-    goto out;
-
-  if (add_nul)
-    {
-      if (!g_output_stream_write ((GOutputStream*)buf, &nulchar, 1, cancellable, error))
-        goto out;
-    }
-
-  if (!g_output_stream_close ((GOutputStream*)buf, cancellable, error))
-    goto out;
-
   ret = TRUE;
-  *out_contents = g_memory_output_stream_steal_as_bytes (buf);
+  *out_contents = g_steal_pointer (&data.result_buf);
  out:
   if (mainctx)
     g_main_context_pop_thread_default (mainctx);
-  g_clear_object (&(data.result_stream));
+  g_clear_pointer (&data.result_buf, (GDestroyNotify)g_bytes_unref);
   return ret;
 }
 
diff --git a/src/libostree/ostree-fetcher.c b/src/libostree/ostree-fetcher.c
index 7917f729de..ea75758172 100644
--- a/src/libostree/ostree-fetcher.c
+++ b/src/libostree/ostree-fetcher.c
@@ -22,6 +22,7 @@
 
 #include "config.h"
 
+#include <gio/gio.h>
 #include <gio/gfiledescriptorbased.h>
 #include <gio/gunixoutputstream.h>
 #define LIBSOUP_USE_UNSTABLE_REQUEST_API
@@ -90,7 +91,8 @@ typedef struct {
 
   SoupRequest *request;
 
-  gboolean is_stream;
+  gboolean is_membuf;
+  OstreeFetcherRequestFlags flags;
   GInputStream *request_body;
   char *out_tmpfile;
   GOutputStream *out_stream;
@@ -468,7 +470,7 @@ session_thread_request_uri (ThreadClosure *thread_closure,
         soup_message_headers_append (msg->request_headers, key, value);
     }
 
-  if (pending->is_stream)
+  if (pending->is_membuf)
     {
       soup_request_send_async (pending->request,
                                cancellable,
@@ -855,6 +857,16 @@ finish_stream (OstreeFetcherPendingURI *pending,
    */
   if (pending->out_stream)
     {
+      if ((pending->flags & OSTREE_FETCHER_REQUEST_NUL_TERMINATION) > 0)
+        {
+          const guint8 nulchar = 0;
+          gsize bytes_written;
+
+          if (!g_output_stream_write_all (pending->out_stream, &nulchar, 1, &bytes_written,
+                                          cancellable, error))
+            goto out;
+        }
+
       if (!g_output_stream_close (pending->out_stream, cancellable, error))
         goto out;
 
@@ -864,30 +876,37 @@ finish_stream (OstreeFetcherPendingURI *pending,
       g_mutex_unlock (&pending->thread_closure->output_stream_set_lock);
     }
 
-  pending->state = OSTREE_FETCHER_STATE_COMPLETE;
-  if (fstatat (pending->thread_closure->tmpdir_dfd,
-               pending->out_tmpfile,
-               &stbuf, AT_SYMLINK_NOFOLLOW) != 0)
+  if (!pending->is_membuf)
     {
-      glnx_set_error_from_errno (error);
-      goto out;
+      if (fstatat (pending->thread_closure->tmpdir_dfd,
+                   pending->out_tmpfile,
+                   &stbuf, AT_SYMLINK_NOFOLLOW) != 0)
+        {
+          glnx_set_error_from_errno (error);
+          goto out;
+        }
     }
 
+  pending->state = OSTREE_FETCHER_STATE_COMPLETE;
+
   /* Now that we've finished downloading, continue with other queued
    * requests.
    */
   session_thread_process_pending_queue (pending->thread_closure);
 
-  if (stbuf.st_size < pending->content_length)
+  if (!pending->is_membuf)
     {
-      g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, "Download incomplete");
-      goto out;
-    }
-  else
-    {
-      g_mutex_lock (&pending->thread_closure->output_stream_set_lock);
-      pending->thread_closure->total_downloaded += stbuf.st_size;
-      g_mutex_unlock (&pending->thread_closure->output_stream_set_lock);
+      if (stbuf.st_size < pending->content_length)
+        {
+          g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, "Download incomplete");
+          goto out;
+        }
+      else
+        {
+          g_mutex_lock (&pending->thread_closure->output_stream_set_lock);
+          pending->thread_closure->total_downloaded += stbuf.st_size;
+          g_mutex_unlock (&pending->thread_closure->output_stream_set_lock);
+        }
     }
 
   ret = TRUE;
@@ -973,9 +992,18 @@ on_stream_read (GObject        *object,
     {
       if (!finish_stream (pending, cancellable, &local_error))
         goto out;
-      g_task_return_pointer (task,
-                             g_strdup (pending->out_tmpfile),
-                             (GDestroyNotify) g_free);
+      if (pending->is_membuf)
+        {
+          g_task_return_pointer (task,
+                                 g_memory_output_stream_steal_as_bytes ((GMemoryOutputStream*)pending->out_stream),
+                                 (GDestroyNotify) g_bytes_unref);
+        }
+      else
+        {
+          g_task_return_pointer (task,
+                                 g_strdup (pending->out_tmpfile),
+                                 (GDestroyNotify) g_free);
+        }
       remove_pending_rerun_queue (pending);
     }
   else
@@ -1045,23 +1073,15 @@ on_request_sent (GObject        *object,
   if (SOUP_IS_REQUEST_HTTP (object))
     {
       msg = soup_request_http_get_message ((SoupRequestHTTP*) object);
-      if (msg->status_code == SOUP_STATUS_REQUESTED_RANGE_NOT_SATISFIABLE)
+      if (!pending->is_membuf &&
+          msg->status_code == SOUP_STATUS_REQUESTED_RANGE_NOT_SATISFIABLE)
         {
           // We already have the whole file, so just use it.
           pending->state = OSTREE_FETCHER_STATE_COMPLETE;
           (void) g_input_stream_close (pending->request_body, NULL, NULL);
-          if (pending->is_stream)
-            {
-              g_task_return_pointer (task,
-                                     g_object_ref (pending->request_body),
-                                     (GDestroyNotify) g_object_unref);
-            }
-          else
-            {
-              g_task_return_pointer (task,
-                                     g_strdup (pending->out_tmpfile),
-                                     (GDestroyNotify) g_free);
-            }
+          g_task_return_pointer (task,
+                                 g_strdup (pending->out_tmpfile),
+                                 (GDestroyNotify) g_free);
           remove_pending_rerun_queue (pending);
           goto out;
         }
@@ -1126,7 +1146,7 @@ on_request_sent (GObject        *object,
   
   pending->content_length = soup_request_get_content_length (pending->request);
 
-  if (!pending->is_stream)
+  if (!pending->is_membuf)
     {
       int oflags = O_CREAT | O_WRONLY | O_CLOEXEC;
       int fd;
@@ -1147,26 +1167,23 @@ on_request_sent (GObject        *object,
           goto out;
         }
       pending->out_stream = g_unix_output_stream_new (fd, TRUE);
-
-      g_mutex_lock (&pending->thread_closure->output_stream_set_lock);
-      g_hash_table_add (pending->thread_closure->output_stream_set,
-                        g_object_ref (pending->out_stream));
-      g_mutex_unlock (&pending->thread_closure->output_stream_set_lock);
-
-      g_input_stream_read_bytes_async (pending->request_body,
-                                       8192, G_PRIORITY_DEFAULT,
-                                       cancellable,
-                                       on_stream_read,
-                                       g_object_ref (task));
     }
   else
     {
-      g_task_return_pointer (task,
-                             g_object_ref (pending->request_body),
-                             (GDestroyNotify) g_object_unref);
-      remove_pending_rerun_queue (pending);
+      pending->out_stream = g_memory_output_stream_new_resizable ();
     }
-  
+
+  g_mutex_lock (&pending->thread_closure->output_stream_set_lock);
+  g_hash_table_add (pending->thread_closure->output_stream_set,
+                    g_object_ref (pending->out_stream));
+  g_mutex_unlock (&pending->thread_closure->output_stream_set_lock);
+
+  g_input_stream_read_bytes_async (pending->request_body,
+                                   8192, G_PRIORITY_DEFAULT,
+                                   cancellable,
+                                   on_stream_read,
+                                   g_object_ref (task));
+
  out:
   if (local_error)
     {
@@ -1179,11 +1196,12 @@ on_request_sent (GObject        *object,
   g_object_unref (task);
 }
 
-void
+static void
 _ostree_fetcher_request_async (OstreeFetcher         *self,
                                GPtrArray             *mirrorlist,
                                const char            *filename,
                                OstreeFetcherRequestFlags flags,
+                               gboolean               is_membuf,
                                guint64                max_size,
                                int                    priority,
                                GCancellable          *cancellable,
@@ -1203,8 +1221,9 @@ _ostree_fetcher_request_async (OstreeFetcher         *self,
   pending->thread_closure = thread_closure_ref (self->thread_closure);
   pending->mirrorlist = g_ptr_array_ref (mirrorlist);
   pending->filename = g_strdup (filename);
+  pending->flags = flags;
   pending->max_size = max_size;
-  pending->is_stream = (flags & OSTREE_FETCHER_REQUEST_FLAG_ENABLE_PARTIAL) == 0;
+  pending->is_membuf = is_membuf;
 
   task = g_task_new (self, cancellable, callback, user_data);
   g_task_set_source_tag (task, _ostree_fetcher_request_async);
@@ -1219,12 +1238,26 @@ _ostree_fetcher_request_async (OstreeFetcher         *self,
                            (GDestroyNotify) g_object_unref);
 }
 
+void
+_ostree_fetcher_request_to_tmpfile (OstreeFetcher         *self,
+                                    GPtrArray             *mirrorlist,
+                                    const char            *filename,
+                                    guint64                max_size,
+                                    int                    priority,
+                                    GCancellable          *cancellable,
+                                    GAsyncReadyCallback    callback,
+                                    gpointer               user_data)
+{
+  _ostree_fetcher_request_async (self, mirrorlist, filename, 0, FALSE,
+                                 max_size, priority, cancellable,
+                                 callback, user_data);
+}
+
 gboolean
-_ostree_fetcher_request_finish (OstreeFetcher         *self,
-                                GAsyncResult          *result,
-                                char                 **out_filename,
-                                GInputStream         **out_stream,
-                                GError               **error)
+_ostree_fetcher_request_to_tmpfile_finish (OstreeFetcher *self,
+                                           GAsyncResult  *result,
+                                           char         **out_filename,
+                                           GError       **error)
 {
   GTask *task;
   OstreeFetcherPendingURI *pending;
@@ -1233,12 +1266,6 @@ _ostree_fetcher_request_finish (OstreeFetcher         *self,
   g_return_val_if_fail (g_task_is_valid (result, self), FALSE);
   g_return_val_if_fail (g_async_result_is_tagged (result, _ostree_fetcher_request_async), FALSE);
 
-  /* Special dance to implement
-    enum FetchResult {
-       Filename(String path),
-       Membuf(uint8[])
-    } in Rust terms
-  */
   task = (GTask*)result;
   pending = g_task_get_task_data (task);
 
@@ -1246,20 +1273,57 @@ _ostree_fetcher_request_finish (OstreeFetcher         *self,
   if (!ret)
     return FALSE;
 
-  if (pending->is_stream)
-    {
-      g_assert (out_stream);
-      *out_stream = ret;
-    }
-  else
-    {
-      g_assert (out_filename);
-      *out_filename = ret;
-    }
+  g_assert (!pending->is_membuf);
+  g_assert (out_filename);
+  *out_filename = ret;
+
+  return TRUE;
+}
+
+void
+_ostree_fetcher_request_to_membuf (OstreeFetcher         *self,
+                                   GPtrArray             *mirrorlist,
+                                   const char            *filename,
+                                   OstreeFetcherRequestFlags flags,
+                                   guint64                max_size,
+                                   int                    priority,
+                                   GCancellable          *cancellable,
+                                   GAsyncReadyCallback    callback,
+                                   gpointer               user_data)
+{
+  _ostree_fetcher_request_async (self, mirrorlist, filename, flags, TRUE,
+                                 max_size, priority, cancellable,
+                                 callback, user_data);
+}
+
+gboolean
+_ostree_fetcher_request_to_membuf_finish (OstreeFetcher *self,
+                                          GAsyncResult  *result,
+                                          GBytes       **out_filename,
+                                          GError       **error)
+{
+  GTask *task;
+  OstreeFetcherPendingURI *pending;
+  gpointer ret;
+
+  g_return_val_if_fail (g_task_is_valid (result, self), FALSE);
+  g_return_val_if_fail (g_async_result_is_tagged (result, _ostree_fetcher_request_async), FALSE);
+
+  task = (GTask*)result;
+  pending = g_task_get_task_data (task);
+
+  ret = g_task_propagate_pointer (task, error);
+  if (!ret)
+    return FALSE;
+
+  g_assert (pending->is_membuf);
+  g_assert (out_filename);
+  *out_filename = ret;
 
   return TRUE;
 }
 
+
 guint64
 _ostree_fetcher_bytes_transferred (OstreeFetcher       *self)
 {
diff --git a/src/libostree/ostree-fetcher.h b/src/libostree/ostree-fetcher.h
index 10ec51dc28..da39623166 100644
--- a/src/libostree/ostree-fetcher.h
+++ b/src/libostree/ostree-fetcher.h
@@ -103,25 +103,39 @@ void _ostree_fetcher_set_extra_headers (OstreeFetcher *self,
 
 guint64 _ostree_fetcher_bytes_transferred (OstreeFetcher       *self);
 
+void _ostree_fetcher_request_to_tmpfile (OstreeFetcher         *self,
+                                         GPtrArray             *mirrorlist,
+                                         const char            *filename,
+                                         guint64                max_size,
+                                         int                    priority,
+                                         GCancellable          *cancellable,
+                                         GAsyncReadyCallback    callback,
+                                         gpointer               user_data);
+
+gboolean _ostree_fetcher_request_to_tmpfile_finish (OstreeFetcher *self,
+                                                    GAsyncResult  *result,
+                                                    char         **out_filename,
+                                                    GError       **error);
+
 typedef enum {
-  OSTREE_FETCHER_REQUEST_FLAG_ENABLE_PARTIAL = (1 << 0)
+  OSTREE_FETCHER_REQUEST_NUL_TERMINATION = (1 << 0)
 } OstreeFetcherRequestFlags;
 
-void _ostree_fetcher_request_async (OstreeFetcher         *self,
-                                    GPtrArray             *mirrorlist,
-                                    const char            *filename,
-                                    OstreeFetcherRequestFlags flags,
-                                    guint64                max_size,
-                                    int                    priority,
-                                    GCancellable          *cancellable,
-                                    GAsyncReadyCallback    callback,
-                                    gpointer               user_data);
-
-gboolean _ostree_fetcher_request_finish (OstreeFetcher *self,
-                                         GAsyncResult  *result,
-                                         char         **out_filename,
-                                         GInputStream **out_stream,
-                                         GError       **error);
+void _ostree_fetcher_request_to_membuf (OstreeFetcher         *self,
+                                        GPtrArray             *mirrorlist,
+                                        const char            *filename,
+                                        OstreeFetcherRequestFlags flags,
+                                        guint64                max_size,
+                                        int                    priority,
+                                        GCancellable          *cancellable,
+                                        GAsyncReadyCallback    callback,
+                                        gpointer               user_data);
+
+gboolean _ostree_fetcher_request_to_membuf_finish (OstreeFetcher *self,
+                                                   GAsyncResult  *result,
+                                                   GBytes       **out_filename,
+                                                   GError       **error);
+
 
 G_END_DECLS
 
diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c
index 100d930486..92efd59255 100644
--- a/src/libostree/ostree-repo-pull.c
+++ b/src/libostree/ostree-repo-pull.c
@@ -708,7 +708,7 @@ content_fetch_on_complete (GObject        *object,
   OstreeObjectType objtype;
   gboolean free_fetch_data = TRUE;
 
-  if (!_ostree_fetcher_request_finish (fetcher, result, &temp_path, NULL, error))
+  if (!_ostree_fetcher_request_to_tmpfile_finish (fetcher, result, &temp_path, error))
     goto out;
 
   ostree_object_name_deserialize (fetch_data->object, &checksum, &objtype);
@@ -841,7 +841,7 @@ meta_fetch_on_complete (GObject           *object,
   g_debug ("fetch of %s%s complete", checksum_obj,
            fetch_data->is_detached_meta ? " (detached)" : "");
 
-  if (!_ostree_fetcher_request_finish (fetcher, result, &temp_path, NULL, error))
+  if (!_ostree_fetcher_request_to_tmpfile_finish (fetcher, result, &temp_path, error))
     {
       if (g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
         {
@@ -981,7 +981,7 @@ static_deltapart_fetch_on_complete (GObject           *object,
 
   g_debug ("fetch static delta part %s complete", fetch_data->expected_checksum);
 
-  if (!_ostree_fetcher_request_finish (fetcher, result, &temp_path, NULL, error))
+  if (!_ostree_fetcher_request_to_tmpfile_finish (fetcher, result, &temp_path, error))
     goto out;
 
   fd = openat (_ostree_fetcher_get_dfd (fetcher), temp_path, O_RDONLY | O_CLOEXEC);
@@ -1380,13 +1380,12 @@ enqueue_one_object_request (OtPullData        *pull_data,
   else
     expected_max_size = 0;
 
-  _ostree_fetcher_request_async (pull_data->fetcher, mirrorlist,
-                                 obj_subpath, OSTREE_FETCHER_REQUEST_FLAG_ENABLE_PARTIAL,
-                                 expected_max_size,
-                                 is_meta ? OSTREE_REPO_PULL_METADATA_PRIORITY
-                                 : OSTREE_REPO_PULL_CONTENT_PRIORITY,
-                                 pull_data->cancellable,
-                                 is_meta ? meta_fetch_on_complete : content_fetch_on_complete, fetch_data);
+  _ostree_fetcher_request_to_tmpfile (pull_data->fetcher, mirrorlist,
+                                      obj_subpath, expected_max_size,
+                                      is_meta ? OSTREE_REPO_PULL_METADATA_PRIORITY
+                                      : OSTREE_REPO_PULL_CONTENT_PRIORITY,
+                                      pull_data->cancellable,
+                                      is_meta ? meta_fetch_on_complete : content_fetch_on_complete, fetch_data);
 }
 
 static gboolean
@@ -1734,14 +1733,13 @@ process_one_static_delta (OtPullData   *pull_data,
         }
       else
         {
-          _ostree_fetcher_request_async (pull_data->fetcher,
-                                         pull_data->content_mirrorlist,
-                                         deltapart_path, OSTREE_FETCHER_REQUEST_FLAG_ENABLE_PARTIAL,
-                                         size,
-                                         OSTREE_FETCHER_DEFAULT_PRIORITY,
-                                         pull_data->cancellable,
-                                         static_deltapart_fetch_on_complete,
-                                         fetch_data);
+          _ostree_fetcher_request_to_tmpfile (pull_data->fetcher,
+                                              pull_data->content_mirrorlist,
+                                              deltapart_path, size,
+                                              OSTREE_FETCHER_DEFAULT_PRIORITY,
+                                              pull_data->cancellable,
+                                              static_deltapart_fetch_on_complete,
+                                              fetch_data);
           pull_data->n_outstanding_deltapart_fetches++;
         }
     }

From 6bde81dac2d4df712ea0c674b96979ec2d298e8f Mon Sep 17 00:00:00 2001
From: Colin Walters <walters@verbum.org>
Date: Thu, 29 Dec 2016 15:57:53 -0500
Subject: [PATCH 04/13] build-sys: Minor makefile tweaks

I'm introducing a new binary in a later patch, and it makes
sense to move more things to be common into the common section.

Also I noticed we were missing an inclusion of common `$(AM_LDFLAGS)`, though
AFAIK this doesn't break anything right now.
---
 Makefile-decls.am  |  1 +
 Makefile-ostree.am | 12 +++++++-----
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/Makefile-decls.am b/Makefile-decls.am
index e41586d178..06018594bf 100644
--- a/Makefile-decls.am
+++ b/Makefile-decls.am
@@ -30,6 +30,7 @@ sbin_PROGRAMS =
 bin_SCRIPTS =
 lib_LTLIBRARIES =
 libexec_PROGRAMS =
+pkglibexec_PROGRAMS =
 pkglibexec_SCRIPTS =
 noinst_LTLIBRARIES =
 noinst_PROGRAMS =
diff --git a/Makefile-ostree.am b/Makefile-ostree.am
index e6b1eabe5a..d46fc22528 100644
--- a/Makefile-ostree.am
+++ b/Makefile-ostree.am
@@ -102,12 +102,14 @@ src/ostree/parse-datetime.c: src/ostree/parse-datetime.y Makefile
 EXTRA_DIST += src/ostree/parse-datetime.y
 CLEANFILES += src/ostree/parse-datetime.c
 
-ostree_bin_shared_cflags = $(AM_CFLAGS) -I$(srcdir)/src/libotutil -I$(srcdir)/src/libostree -I$(srcdir)/src/ostree \
-	$(NULL)
-ostree_bin_shared_ldadd = libglnx.la libbsdiff.la libotutil.la libostree-kernel-args.la libostree-1.la
+ostree_bin_shared_cflags = $(AM_CFLAGS) -I$(srcdir)/src/libotutil -I$(srcdir)/src/libostree \
+	-I$(srcdir)/src/ostree -I$(srcdir)/libglnx $(OT_INTERNAL_GIO_UNIX_CFLAGS)
+ostree_bin_shared_ldadd = $(AM_LDFLAGS) libglnx.la libotutil.la libostree-1.la \
+	$(OT_INTERNAL_GIO_UNIX_LIBS)
+
+ostree_CFLAGS = $(ostree_bin_shared_cflags)
+ostree_LDADD = $(ostree_bin_shared_ldadd) libbsdiff.la libostree-kernel-args.la $(LIBSYSTEMD_LIBS)
 
-ostree_CFLAGS = $(ostree_bin_shared_cflags) $(OT_INTERNAL_GIO_UNIX_CFLAGS) -I$(srcdir)/libglnx
-ostree_LDADD = $(ostree_bin_shared_ldadd) $(OT_INTERNAL_GIO_UNIX_LIBS) $(LIBSYSTEMD_LIBS)
 
 if USE_LIBSOUP
 ostree_SOURCES += \

From aff5ed1fe8abd9601cb52af0d3e55e959511c5a2 Mon Sep 17 00:00:00 2001
From: Colin Walters <walters@verbum.org>
Date: Fri, 30 Dec 2016 14:18:34 -0500
Subject: [PATCH 05/13] Split trivial-httpd into separate binary

Working on the libcurl backend, I hit the issue that the trivial-httpd program
depends on libsoup. I briefly considered having two versions, but libcurl is
client only, and moreover trivial-httpd is no longer trivial - it has various
features which are used by the test suite extensively.

Hence, what we'll do is build it as a separate binary which links to libsoup,
and use it during the tests. We *also* currently still provide `ostree
trivial-httpd` since some things use it like `rpm-ostree-toolbox` and the
Cockpit tests.

After those are ported to use some other webserver, I plan to add a build-time
option to drop it.
---
 Makefile-ostree.am                    |  13 +-
 Makefile-tests.am                     |   1 +
 src/ostree/ostree-trivial-httpd.c     | 683 ++++++++++++++++++++++++++
 src/ostree/ot-builtin-trivial-httpd.c | 631 +-----------------------
 tests/libtest.sh                      |  10 +-
 tests/test-commit-sign.sh             |   2 +-
 tests/test-pull-contenturl.sh         |   3 +-
 tests/test-pull-metalink.sh           |   2 +-
 tests/test-pull-mirrorlist.sh         |   2 +-
 tests/test-pull-override-url.sh       |   2 +-
 10 files changed, 714 insertions(+), 635 deletions(-)
 create mode 100644 src/ostree/ostree-trivial-httpd.c

diff --git a/Makefile-ostree.am b/Makefile-ostree.am
index d46fc22528..05fec15520 100644
--- a/Makefile-ostree.am
+++ b/Makefile-ostree.am
@@ -103,7 +103,8 @@ EXTRA_DIST += src/ostree/parse-datetime.y
 CLEANFILES += src/ostree/parse-datetime.c
 
 ostree_bin_shared_cflags = $(AM_CFLAGS) -I$(srcdir)/src/libotutil -I$(srcdir)/src/libostree \
-	-I$(srcdir)/src/ostree -I$(srcdir)/libglnx $(OT_INTERNAL_GIO_UNIX_CFLAGS)
+	-I$(srcdir)/src/ostree -I$(srcdir)/libglnx $(OT_INTERNAL_GIO_UNIX_CFLAGS) \
+	-DPKGLIBEXECDIR=\"$(pkglibexecdir)\"
 ostree_bin_shared_ldadd = $(AM_LDFLAGS) libglnx.la libotutil.la libostree-1.la \
 	$(OT_INTERNAL_GIO_UNIX_LIBS)
 
@@ -112,12 +113,14 @@ ostree_LDADD = $(ostree_bin_shared_ldadd) libbsdiff.la libostree-kernel-args.la
 
 
 if USE_LIBSOUP
-ostree_SOURCES += \
-	src/ostree/ot-builtin-pull.c \
-	src/ostree/ot-builtin-trivial-httpd.c \
-	$(NULL)
+ostree_SOURCES += src/ostree/ot-builtin-pull.c src/ostree/ot-builtin-trivial-httpd.c
 ostree_CFLAGS += $(OT_INTERNAL_SOUP_CFLAGS)
 ostree_LDADD += $(OT_INTERNAL_SOUP_LIBS)
+
+pkglibexec_PROGRAMS += ostree-trivial-httpd
+ostree_trivial_httpd_SOURCES = src/ostree/ostree-trivial-httpd.c
+ostree_trivial_httpd_CFLAGS = $(ostree_bin_shared_cflags) $(OT_INTERNAL_SOUP_CFLAGS)
+ostree_trivial_httpd_LDADD = $(ostree_bin_shared_ldadd) $(OT_INTERNAL_SOUP_LIBS)
 endif
 
 if USE_LIBARCHIVE
diff --git a/Makefile-tests.am b/Makefile-tests.am
index 1f9cad48b5..63ceea1eb1 100644
--- a/Makefile-tests.am
+++ b/Makefile-tests.am
@@ -28,6 +28,7 @@ EXTRA_DIST += \
 # include the builddir in $PATH so we find our just-built ostree
 # binary.
 TESTS_ENVIRONMENT += OT_TESTS_DEBUG=1 \
+	OSTREE_UNINSTALLED=$(abs_top_builddir) \
 	G_DEBUG=fatal-warnings \
 	GI_TYPELIB_PATH=$$(cd $(top_builddir) && pwd)$${GI_TYPELIB_PATH:+:$$GI_TYPELIB_PATH} \
 	LD_LIBRARY_PATH=$$(cd $(top_builddir)/.libs && pwd)$${LD_LIBRARY_PATH:+:$${LD_LIBRARY_PATH}} \
diff --git a/src/ostree/ostree-trivial-httpd.c b/src/ostree/ostree-trivial-httpd.c
new file mode 100644
index 0000000000..ef297af6a2
--- /dev/null
+++ b/src/ostree/ostree-trivial-httpd.c
@@ -0,0 +1,683 @@
+/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*-
+ *
+ * Copyright (C) 2011,2013 Colin Walters <walters@verbum.org>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 02111-1307, USA.
+ */
+
+#include "config.h"
+
+#include <libsoup/soup.h>
+
+#include <gio/gunixoutputstream.h>
+
+#include "ot-main.h"
+#include "ot-builtins.h"
+#include "ostree.h"
+#include "otutil.h"
+
+#include <locale.h>
+#include <sys/socket.h>
+#include <sys/prctl.h>
+#include <signal.h>
+
+static char *opt_port_file = NULL;
+static char *opt_log = NULL;
+static gboolean opt_daemonize;
+static gboolean opt_autoexit;
+static gboolean opt_force_ranges;
+static int opt_random_500s_percentage;
+/* We have a strong upper bound for any unlikely
+ * cases involving repeated random 500s. */
+static int opt_random_500s_max = 100;
+static gint opt_port = 0;
+static gchar **opt_expected_cookies;
+static gchar **opt_expected_headers;
+
+static guint emitted_random_500s_count = 0;
+
+typedef struct {
+  int root_dfd;
+  gboolean running;
+  GOutputStream *log;
+} OtTrivialHttpd;
+
+static GOptionEntry options[] = {
+  { "daemonize", 'd', 0, G_OPTION_ARG_NONE, &opt_daemonize, "Fork into background when ready", NULL },
+  { "autoexit", 0, 0, G_OPTION_ARG_NONE, &opt_autoexit, "Automatically exit when directory is deleted", NULL },
+  { "port", 'P', 0, G_OPTION_ARG_INT, &opt_port, "Use the specified TCP port", NULL },
+  { "port-file", 'p', 0, G_OPTION_ARG_FILENAME, &opt_port_file, "Write port number to PATH (- for standard output)", "PATH" },
+  { "force-range-requests", 0, 0, G_OPTION_ARG_NONE, &opt_force_ranges, "Force range requests by only serving half of files", NULL },
+  { "random-500s", 0, 0, G_OPTION_ARG_INT, &opt_random_500s_percentage, "Generate random HTTP 500 errors approximately for PERCENTAGE requests", "PERCENTAGE" },
+  { "random-500s-max", 0, 0, G_OPTION_ARG_INT, &opt_random_500s_max, "Limit HTTP 500 errors to MAX (default 100)", "MAX" },
+  { "log-file", 0, 0, G_OPTION_ARG_FILENAME, &opt_log, "Put logs here", "PATH" },
+  { "expected-cookies", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_expected_cookies, "Expect given cookies in the http request", "KEY=VALUE" },
+  { "expected-header", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_expected_headers, "Expect given headers in the http request", "KEY=VALUE" },
+  { NULL }
+};
+
+static void
+httpd_log (OtTrivialHttpd *httpd, const gchar *format, ...) __attribute__ ((format(printf, 2, 3)));
+
+static void
+httpd_log (OtTrivialHttpd *httpd, const gchar *format, ...)
+{
+  g_autoptr(GString) str = NULL;
+  va_list args;
+  gsize written;
+
+  if (!httpd->log)
+    return;
+
+  {
+    g_autoptr(GDateTime) now = g_date_time_new_now_local ();
+    g_autofree char *timestamp = g_date_time_format (now, "%F %T");
+    str = g_string_new (timestamp);
+    g_string_append_printf (str, ".%06d - ", g_date_time_get_microsecond (now));
+  }
+
+  va_start (args, format);
+  g_string_append_vprintf (str, format, args);
+  va_end (args);
+
+  g_output_stream_write_all (httpd->log, str->str, str->len, &written, NULL, NULL);
+}
+
+static int
+compare_strings (gconstpointer a, gconstpointer b)
+{
+  const char **sa = (const char **)a;
+  const char **sb = (const char **)b;
+
+  return strcmp (*sa, *sb);
+}
+
+static GString *
+get_directory_listing (int dfd,
+                       const char *path)
+{
+  g_autoptr(GPtrArray) entries = g_ptr_array_new_with_free_func (g_free);
+  g_auto(GLnxDirFdIterator) dfd_iter = { 0, };
+  g_autoptr(GError) local_error = NULL;
+  GError **error = &local_error;
+  guint i;
+  char *escaped;
+  GString *listing;
+
+  listing = g_string_new ("<html>\r\n");
+
+  if (!glnx_dirfd_iterator_init_at (dfd, path, FALSE, &dfd_iter, error))
+    goto out;
+
+  while (TRUE)
+    {
+      struct dirent *dent;
+
+      if (!glnx_dirfd_iterator_next_dent (&dfd_iter, &dent, NULL, error))
+        goto out;
+
+      if (dent == NULL)
+        break;
+
+      escaped = g_markup_escape_text (dent->d_name, -1);
+      g_ptr_array_add (entries, escaped);
+    }
+
+  g_ptr_array_sort (entries, (GCompareFunc)compare_strings);
+
+  escaped = g_markup_escape_text (strchr (path, '/'), -1);
+  g_string_append_printf (listing, "<head><title>Index of %s</title></head>\r\n", escaped);
+  g_string_append_printf (listing, "<body><h1>Index of %s</h1>\r\n<p>\r\n", escaped);
+  g_free (escaped);
+  for (i = 0; i < entries->len; i++)
+    {
+      g_string_append_printf (listing, "<a href=\"%s\">%s</a><br>\r\n",
+                              (char *)entries->pdata[i], 
+                              (char *)entries->pdata[i]);
+      g_free (g_steal_pointer (&entries->pdata[i]));
+    }
+  g_string_append (listing, "</body>\r\n</html>\r\n");
+ out:
+  if (local_error)
+    g_printerr ("%s\n", local_error->message);
+  return listing;
+}
+
+/* Only allow reading files that have o+r, and for directories, o+x.
+ * This makes this server relatively safe to use on multiuser
+ * machines.
+ */
+static gboolean
+is_safe_to_access (struct stat *stbuf)
+{
+  /* Only regular files or directores */
+  if (!(S_ISREG (stbuf->st_mode) || S_ISDIR (stbuf->st_mode)))
+    return FALSE;
+  /* Must be o+r */
+  if (!(stbuf->st_mode & S_IROTH))
+    return FALSE;
+  /* For directories, must be o+x */
+  if (S_ISDIR (stbuf->st_mode) && !(stbuf->st_mode & S_IXOTH))
+    return FALSE;
+  return TRUE;
+}
+
+static void
+close_socket (SoupMessage *msg, gpointer user_data)
+{
+  SoupSocket *sock = user_data;
+  int sockfd;
+
+  /* Actually calling soup_socket_disconnect() here would cause
+   * us to leak memory, so just shutdown the socket instead.
+   */
+  sockfd = soup_socket_get_fd (sock);
+#ifdef G_OS_WIN32
+  shutdown (sockfd, SD_SEND);
+#else
+  shutdown (sockfd, SHUT_WR);
+#endif
+}
+
+static void
+do_get (OtTrivialHttpd    *self,
+        SoupServer        *server,
+        SoupMessage       *msg,
+        const char        *path,
+        SoupClientContext *context)
+{
+  char *slash;
+  int ret;
+  struct stat stbuf;
+
+  httpd_log (self, "serving %s\n", path);
+
+  if (opt_expected_cookies)
+    {
+      GSList *cookies = soup_cookies_from_request (msg);
+      GSList *l;
+      int i;
+
+      for (i = 0 ; opt_expected_cookies[i] != NULL; i++)
+        {
+          gboolean found = FALSE;
+          gchar *k = opt_expected_cookies[i];
+          gchar *v = strchr (k, '=') + 1;
+
+          for (l = cookies;  l != NULL ; l = g_slist_next (l))
+            {
+              SoupCookie *c = l->data;
+
+              if (!strncmp (k, soup_cookie_get_name (c), v - k - 1) &&
+                  !strcmp (v, soup_cookie_get_value (c)))
+                {
+                  found = TRUE;
+                  break;
+                }
+            }
+
+          if (!found)
+            {
+              httpd_log (self, "Expected cookie not found %s\n", k);
+              soup_message_set_status (msg, SOUP_STATUS_FORBIDDEN);
+              soup_cookies_free (cookies);
+              goto out;
+            }
+        }
+      soup_cookies_free (cookies);
+    }
+
+  if (opt_expected_headers)
+    {
+      for (int i = 0 ; opt_expected_headers[i] != NULL; i++)
+        {
+          const gchar *kv = opt_expected_headers[i];
+          const gchar *eq = strchr (kv, '=');
+
+          g_assert (eq);
+
+          {
+            g_autofree char *k = g_strndup (kv, eq - kv);
+            const gchar *expected_v = eq + 1;
+            const gchar *found_v = soup_message_headers_get_one (msg->request_headers, k);
+
+            if (!found_v)
+              {
+                httpd_log (self, "Expected header not found %s\n", k);
+                soup_message_set_status (msg, SOUP_STATUS_FORBIDDEN);
+                goto out;
+              }
+            if (strcmp (found_v, expected_v) != 0)
+              {
+                httpd_log (self, "Expected header %s: %s but found %s\n", k, expected_v, found_v);
+                soup_message_set_status (msg, SOUP_STATUS_FORBIDDEN);
+                goto out;
+              }
+          }
+        }
+    }
+
+  if (strstr (path, "../") != NULL)
+    {
+      soup_message_set_status (msg, SOUP_STATUS_FORBIDDEN);
+      goto out;
+    }
+
+  if (opt_random_500s_percentage > 0 &&
+      emitted_random_500s_count < opt_random_500s_max &&
+      g_random_int_range (0, 100) < opt_random_500s_percentage)
+    {
+      emitted_random_500s_count++;
+      soup_message_set_status (msg, SOUP_STATUS_INTERNAL_SERVER_ERROR);
+      goto out;
+    }
+
+  while (path[0] == '/')
+    path++;
+
+  do
+    ret = fstatat (self->root_dfd, path, &stbuf, 0);
+  while (ret == -1 && errno == EINTR);
+  if (ret == -1)
+    {
+      if (errno == EPERM)
+        soup_message_set_status (msg, SOUP_STATUS_FORBIDDEN);
+      else if (errno == ENOENT)
+        soup_message_set_status (msg, SOUP_STATUS_NOT_FOUND);
+      else
+        soup_message_set_status (msg, SOUP_STATUS_INTERNAL_SERVER_ERROR);
+      goto out;
+    }
+
+  if (!is_safe_to_access (&stbuf))
+    {
+      soup_message_set_status (msg, SOUP_STATUS_FORBIDDEN);
+      goto out;
+    }
+
+  if (S_ISDIR (stbuf.st_mode))
+    {
+      slash = strrchr (path, '/');
+      if (!slash || slash[1])
+        {
+          g_autofree char *redir_uri = NULL;
+
+          redir_uri = g_strdup_printf ("%s/", soup_message_get_uri (msg)->path);
+          soup_message_set_redirect (msg, SOUP_STATUS_MOVED_PERMANENTLY,
+                                     redir_uri);
+        }
+      else
+        {
+          g_autofree char *index_realpath = g_strconcat (path, "/index.html", NULL);
+          if (fstatat (self->root_dfd, index_realpath, &stbuf, 0) != -1)
+            {
+              g_autofree char *index_path = g_strconcat (path, "/index.html", NULL);
+              do_get (self, server, msg, index_path, context);
+            }
+          else
+            {
+              GString *listing = get_directory_listing (self->root_dfd, path);
+              soup_message_set_response (msg, "text/html",
+                                         SOUP_MEMORY_TAKE,
+                                         listing->str, listing->len);
+              soup_message_set_status (msg, SOUP_STATUS_OK);
+              g_string_free (listing, FALSE);
+            }
+        }
+    }
+  else 
+    {
+      if (!S_ISREG (stbuf.st_mode))
+        {
+          soup_message_set_status (msg, SOUP_STATUS_FORBIDDEN);
+          goto out;
+        }
+      
+      if (msg->method == SOUP_METHOD_GET)
+        {
+          glnx_fd_close int fd = -1;
+          g_autoptr(GMappedFile) mapping = NULL;
+          gsize buffer_length, file_size;
+          SoupRange *ranges;
+          int ranges_length;
+          gboolean have_ranges;
+
+          fd = openat (self->root_dfd, path, O_RDONLY | O_CLOEXEC);
+          if (fd < 0)
+            {
+              soup_message_set_status (msg, SOUP_STATUS_INTERNAL_SERVER_ERROR);
+              goto out;
+            }
+
+          mapping = g_mapped_file_new_from_fd (fd, FALSE, NULL);
+          if (!mapping)
+            {
+              soup_message_set_status (msg, SOUP_STATUS_INTERNAL_SERVER_ERROR);
+              goto out;
+            }
+          (void) close (fd); fd = -1;
+
+          file_size = g_mapped_file_get_length (mapping);
+          have_ranges = soup_message_headers_get_ranges(msg->request_headers, file_size, &ranges, &ranges_length);
+          if (opt_force_ranges && !have_ranges && g_strrstr (path, "/objects") != NULL)
+            {
+              SoupSocket *sock;
+              buffer_length = file_size/2;
+              soup_message_headers_set_content_length (msg->response_headers, file_size);
+              soup_message_headers_append (msg->response_headers,
+                                           "Connection", "close");
+
+              /* soup-message-io will wait for us to add
+               * another chunk after the first, to fill out
+               * the declared Content-Length. Instead, we
+               * forcibly close the socket at that point.
+               */
+              sock = soup_client_context_get_socket (context);
+              g_signal_connect (msg, "wrote-chunk", G_CALLBACK (close_socket), sock);
+            }
+          else
+            buffer_length = file_size;
+
+          if (have_ranges)
+            {
+              if (ranges_length > 0 && ranges[0].start >= file_size)
+                {
+                  soup_message_set_status (msg, SOUP_STATUS_REQUESTED_RANGE_NOT_SATISFIABLE);
+                  soup_message_headers_free_ranges (msg->request_headers, ranges);
+                  goto out;
+                }
+              soup_message_headers_free_ranges (msg->request_headers, ranges);
+            }
+          if (buffer_length > 0)
+            {
+              SoupBuffer *buffer;
+
+              buffer = soup_buffer_new_with_owner (g_mapped_file_get_contents (mapping),
+                                                   buffer_length,
+                                                   g_mapped_file_ref (mapping),
+                                                   (GDestroyNotify)g_mapped_file_unref);
+              soup_message_body_append_buffer (msg->response_body, buffer);
+              soup_buffer_free (buffer);
+            }
+        }
+      else /* msg->method == SOUP_METHOD_HEAD */
+        {
+          g_autofree char *length = NULL;
+
+          /* We could just use the same code for both GET and
+           * HEAD (soup-message-server-io.c will fix things up).
+           * But we'll optimize and avoid the extra I/O.
+           */
+          length = g_strdup_printf ("%lu", (gulong)stbuf.st_size);
+          soup_message_headers_append (msg->response_headers,
+                                       "Content-Length", length);
+        }
+      soup_message_set_status (msg, SOUP_STATUS_OK);
+    }
+ out:
+  {
+    guint status = 0;
+    g_autofree gchar *reason = NULL;
+
+    g_object_get (msg,
+                  "status-code", &status,
+                  "reason-phrase", &reason,
+                  NULL);
+    httpd_log (self, "  status: %s (%u)\n", reason, status);
+  }
+  return;
+}
+
+static void
+httpd_callback (SoupServer *server, SoupMessage *msg,
+                const char *path, GHashTable *query,
+                SoupClientContext *context, gpointer data)
+{
+  OtTrivialHttpd *self = data;
+
+  if (msg->method == SOUP_METHOD_GET || msg->method == SOUP_METHOD_HEAD)
+    do_get (self, server, msg, path, context);
+  else
+    soup_message_set_status (msg, SOUP_STATUS_NOT_IMPLEMENTED);
+}
+
+static void
+on_dir_changed (GFileMonitor  *mon,
+		GFile *file,
+		GFile *other,
+		GFileMonitorEvent  event,
+		gpointer user_data)
+{
+  OtTrivialHttpd *self = user_data;
+
+  if (event == G_FILE_MONITOR_EVENT_DELETED)
+    {
+      self->running = FALSE;
+      g_main_context_wakeup (NULL);
+    }
+}
+
+static gboolean
+run (int argc, char **argv, GCancellable *cancellable, GError **error)
+{
+  gboolean ret = FALSE;
+  g_autoptr(GOptionContext) context = NULL;
+  const char *dirpath;
+  OtTrivialHttpd appstruct = { 0, };
+  OtTrivialHttpd *app = &appstruct;
+  glnx_unref_object SoupServer *server = NULL;
+  g_autoptr(GFileMonitor) dirmon = NULL;
+
+  context = g_option_context_new ("[DIR] - Simple webserver");
+  g_option_context_add_main_entries (context, options, NULL);
+
+  app->root_dfd = -1;
+
+  if (!g_option_context_parse (context, &argc, &argv, error))
+    goto out;
+
+  if (argc > 1)
+    dirpath = argv[1];
+  else
+    dirpath = ".";
+
+  if (!glnx_opendirat (AT_FDCWD, dirpath, TRUE, &app->root_dfd, error))
+    goto out;
+
+  if (!(opt_random_500s_percentage >= 0 && opt_random_500s_percentage <= 99))
+    {
+      g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                   "Invalid --random-500s=%u", opt_random_500s_percentage);
+      goto out;
+    }
+
+  if (opt_log)
+    {
+      GOutputStream *stream = NULL;
+
+      if (g_strcmp0 (opt_log, "-") == 0)
+        {
+          if (opt_daemonize)
+            {
+              ot_util_usage_error (context, "Cannot use --log-file=- and --daemonize at the same time", error);
+              goto out;
+            }
+          stream = G_OUTPUT_STREAM (g_unix_output_stream_new (STDOUT_FILENO, FALSE));
+        }
+      else
+        {
+          g_autoptr(GFile) log_file;
+          GFileOutputStream* log_stream;
+
+          log_file = g_file_new_for_path (opt_log);
+          log_stream = g_file_create (log_file,
+                                      G_FILE_CREATE_PRIVATE,
+                                      cancellable,
+                                      error);
+          if (!log_stream)
+            goto out;
+          stream = G_OUTPUT_STREAM (log_stream);
+        }
+
+      app->log = stream;
+    }
+
+#if SOUP_CHECK_VERSION(2, 48, 0)
+  server = soup_server_new (SOUP_SERVER_SERVER_HEADER, "ostree-httpd ", NULL);
+  if (!soup_server_listen_all (server, opt_port, 0, error))
+    goto out;
+#else
+  server = soup_server_new (SOUP_SERVER_PORT, opt_port,
+                            SOUP_SERVER_SERVER_HEADER, "ostree-httpd ",
+                            NULL);
+#endif
+
+  soup_server_add_handler (server, NULL, httpd_callback, app, NULL);
+  if (opt_port_file)
+    {
+      g_autofree char *portstr = NULL;
+#if SOUP_CHECK_VERSION(2, 48, 0)
+      GSList *listeners = soup_server_get_listeners (server);
+      g_autoptr(GSocket) listener = NULL;
+      g_autoptr(GSocketAddress) addr = NULL;
+      
+      g_assert (listeners);
+      listener = g_object_ref (listeners->data);
+      g_slist_free (listeners);
+      listeners = NULL;
+      addr = g_socket_get_local_address (listener, error);
+      if (!addr)
+        goto out;
+
+      g_assert (G_IS_INET_SOCKET_ADDRESS (addr));
+      
+      portstr = g_strdup_printf ("%u\n", g_inet_socket_address_get_port ((GInetSocketAddress*)addr));
+#else
+      portstr = g_strdup_printf ("%u\n", soup_server_get_port (server));
+#endif
+
+      if (g_strcmp0 ("-", opt_port_file) == 0)
+        {
+          fputs (portstr, stdout); // not g_print - this must go to stdout, not a handler
+          fflush (stdout);
+        }
+      else if (!g_file_set_contents (opt_port_file, portstr, strlen (portstr), error))
+        goto out;
+    }
+#if !SOUP_CHECK_VERSION(2, 48, 0)
+  soup_server_run_async (server);
+#endif
+  
+  if (opt_daemonize)
+    {
+      pid_t pid = fork();
+      if (pid == -1)
+        {
+          int errsv = errno;
+          g_set_error_literal (error, G_IO_ERROR, g_io_error_from_errno (errsv),
+                               g_strerror (errsv));
+          goto out;
+        }
+      else if (pid > 0)
+        {
+          ret = TRUE;
+          goto out;
+        }
+      /* Child, continue */
+      /* Daemonising: close stdout/stderr so $() et al work on us */
+      fclose (stdout);
+      fclose (stdin);
+    }
+  else
+    {
+      /* Since we're used for testing purposes, let's just do this by
+       * default.  This ensures we exit when our parent does.
+       */
+      if (prctl (PR_SET_PDEATHSIG, SIGTERM) != 0)
+        {
+          if (errno != ENOSYS)
+            {
+              glnx_set_error_from_errno (error);
+              goto out;
+            }
+        }
+    }
+
+  app->running = TRUE;
+  if (opt_autoexit)
+    {
+      gboolean is_symlink = FALSE;
+      g_autoptr(GFile) root = NULL;
+      g_autoptr(GFileInfo) info = NULL;
+
+      root = g_file_new_for_path (dirpath);
+      info = g_file_query_info (root,
+                                G_FILE_ATTRIBUTE_STANDARD_IS_SYMLINK,
+                                G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
+                                cancellable, error);
+      if (!info)
+        goto out;
+
+      is_symlink = g_file_info_get_is_symlink (info);
+
+      if (is_symlink)
+        dirmon = g_file_monitor_file (root, 0, cancellable, error);
+      else
+        dirmon = g_file_monitor_directory (root, 0, cancellable, error);
+
+      if (!dirmon)
+        goto out;
+      g_signal_connect (dirmon, "changed", G_CALLBACK (on_dir_changed), app);
+    }
+  httpd_log (app, "serving at root %s\n", dirpath);
+  while (app->running)
+    g_main_context_iteration (NULL, TRUE);
+
+  ret = TRUE;
+ out:
+  if (app->root_dfd != -1)
+    (void) close (app->root_dfd);
+  g_clear_object (&app->log);
+  return ret;
+}
+
+int
+main (int    argc,
+      char **argv)
+{
+  g_autoptr(GError) error = NULL;
+  g_autoptr(GCancellable) cancellable = NULL;
+
+  setlocale (LC_ALL, "");
+
+  g_set_prgname (argv[0]);
+
+  if (!run (argc, argv, cancellable, &error))
+    {
+      int is_tty = isatty (1);
+      const char *prefix = "";
+      const char *suffix = "";
+      if (is_tty)
+        {
+          prefix = "\x1b[31m\x1b[1m"; /* red, bold */
+          suffix = "\x1b[22m\x1b[0m"; /* bold off, color reset */
+        }
+      g_printerr ("%serror: %s%s\n", prefix, suffix, error->message);
+      return 1;
+    }
+
+  return 0;
+}
diff --git a/src/ostree/ot-builtin-trivial-httpd.c b/src/ostree/ot-builtin-trivial-httpd.c
index 0a55385895..a411e0cc94 100644
--- a/src/ostree/ot-builtin-trivial-httpd.c
+++ b/src/ostree/ot-builtin-trivial-httpd.c
@@ -1,6 +1,6 @@
 /* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*-
  *
- * Copyright (C) 2011,2013 Colin Walters <walters@verbum.org>
+ * Copyright (C) 2016 Colin Walters <walters@verbum.org>
  *
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
@@ -20,634 +20,21 @@
 
 #include "config.h"
 
-#include <libsoup/soup.h>
-
-#include <gio/gunixoutputstream.h>
-
 #include "ot-main.h"
 #include "ot-builtins.h"
 #include "ostree.h"
 #include "otutil.h"
 
-#include <sys/socket.h>
-#include <sys/prctl.h>
-#include <signal.h>
-
-static char *opt_port_file = NULL;
-static char *opt_log = NULL;
-static gboolean opt_daemonize;
-static gboolean opt_autoexit;
-static gboolean opt_force_ranges;
-static int opt_random_500s_percentage;
-/* We have a strong upper bound for any unlikely
- * cases involving repeated random 500s. */
-static int opt_random_500s_max = 100;
-static gint opt_port = 0;
-static gchar **opt_expected_cookies;
-static gchar **opt_expected_headers;
-
-static guint emitted_random_500s_count = 0;
-
-typedef struct {
-  int root_dfd;
-  gboolean running;
-  GOutputStream *log;
-} OtTrivialHttpd;
-
-static GOptionEntry options[] = {
-  { "daemonize", 'd', 0, G_OPTION_ARG_NONE, &opt_daemonize, "Fork into background when ready", NULL },
-  { "autoexit", 0, 0, G_OPTION_ARG_NONE, &opt_autoexit, "Automatically exit when directory is deleted", NULL },
-  { "port", 'P', 0, G_OPTION_ARG_INT, &opt_port, "Use the specified TCP port", NULL },
-  { "port-file", 'p', 0, G_OPTION_ARG_FILENAME, &opt_port_file, "Write port number to PATH (- for standard output)", "PATH" },
-  { "force-range-requests", 0, 0, G_OPTION_ARG_NONE, &opt_force_ranges, "Force range requests by only serving half of files", NULL },
-  { "random-500s", 0, 0, G_OPTION_ARG_INT, &opt_random_500s_percentage, "Generate random HTTP 500 errors approximately for PERCENTAGE requests", "PERCENTAGE" },
-  { "random-500s-max", 0, 0, G_OPTION_ARG_INT, &opt_random_500s_max, "Limit HTTP 500 errors to MAX (default 100)", "MAX" },
-  { "log-file", 0, 0, G_OPTION_ARG_FILENAME, &opt_log, "Put logs here", "PATH" },
-  { "expected-cookies", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_expected_cookies, "Expect given cookies in the http request", "KEY=VALUE" },
-  { "expected-header", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_expected_headers, "Expect given headers in the http request", "KEY=VALUE" },
-  { NULL }
-};
-
-static void
-httpd_log (OtTrivialHttpd *httpd, const gchar *format, ...) __attribute__ ((format(printf, 2, 3)));
-
-static void
-httpd_log (OtTrivialHttpd *httpd, const gchar *format, ...)
-{
-  g_autoptr(GString) str = NULL;
-  va_list args;
-  gsize written;
-
-  if (!httpd->log)
-    return;
-
-  {
-    g_autoptr(GDateTime) now = g_date_time_new_now_local ();
-    g_autofree char *timestamp = g_date_time_format (now, "%F %T");
-    str = g_string_new (timestamp);
-    g_string_append_printf (str, ".%06d - ", g_date_time_get_microsecond (now));
-  }
-
-  va_start (args, format);
-  g_string_append_vprintf (str, format, args);
-  va_end (args);
-
-  g_output_stream_write_all (httpd->log, str->str, str->len, &written, NULL, NULL);
-}
-
-static int
-compare_strings (gconstpointer a, gconstpointer b)
-{
-  const char **sa = (const char **)a;
-  const char **sb = (const char **)b;
-
-  return strcmp (*sa, *sb);
-}
-
-static GString *
-get_directory_listing (int dfd,
-                       const char *path)
-{
-  g_autoptr(GPtrArray) entries = g_ptr_array_new_with_free_func (g_free);
-  g_auto(GLnxDirFdIterator) dfd_iter = { 0, };
-  g_autoptr(GError) local_error = NULL;
-  GError **error = &local_error;
-  guint i;
-  char *escaped;
-  GString *listing;
-
-  listing = g_string_new ("<html>\r\n");
-
-  if (!glnx_dirfd_iterator_init_at (dfd, path, FALSE, &dfd_iter, error))
-    goto out;
-
-  while (TRUE)
-    {
-      struct dirent *dent;
-
-      if (!glnx_dirfd_iterator_next_dent (&dfd_iter, &dent, NULL, error))
-        goto out;
-
-      if (dent == NULL)
-        break;
-
-      escaped = g_markup_escape_text (dent->d_name, -1);
-      g_ptr_array_add (entries, escaped);
-    }
-
-  g_ptr_array_sort (entries, (GCompareFunc)compare_strings);
-
-  escaped = g_markup_escape_text (strchr (path, '/'), -1);
-  g_string_append_printf (listing, "<head><title>Index of %s</title></head>\r\n", escaped);
-  g_string_append_printf (listing, "<body><h1>Index of %s</h1>\r\n<p>\r\n", escaped);
-  g_free (escaped);
-  for (i = 0; i < entries->len; i++)
-    {
-      g_string_append_printf (listing, "<a href=\"%s\">%s</a><br>\r\n",
-                              (char *)entries->pdata[i], 
-                              (char *)entries->pdata[i]);
-      g_free (g_steal_pointer (&entries->pdata[i]));
-    }
-  g_string_append (listing, "</body>\r\n</html>\r\n");
- out:
-  if (local_error)
-    g_printerr ("%s\n", local_error->message);
-  return listing;
-}
-
-/* Only allow reading files that have o+r, and for directories, o+x.
- * This makes this server relatively safe to use on multiuser
- * machines.
- */
-static gboolean
-is_safe_to_access (struct stat *stbuf)
-{
-  /* Only regular files or directores */
-  if (!(S_ISREG (stbuf->st_mode) || S_ISDIR (stbuf->st_mode)))
-    return FALSE;
-  /* Must be o+r */
-  if (!(stbuf->st_mode & S_IROTH))
-    return FALSE;
-  /* For directories, must be o+x */
-  if (S_ISDIR (stbuf->st_mode) && !(stbuf->st_mode & S_IXOTH))
-    return FALSE;
-  return TRUE;
-}
-
-static void
-close_socket (SoupMessage *msg, gpointer user_data)
-{
-  SoupSocket *sock = user_data;
-  int sockfd;
-
-  /* Actually calling soup_socket_disconnect() here would cause
-   * us to leak memory, so just shutdown the socket instead.
-   */
-  sockfd = soup_socket_get_fd (sock);
-#ifdef G_OS_WIN32
-  shutdown (sockfd, SD_SEND);
-#else
-  shutdown (sockfd, SHUT_WR);
-#endif
-}
-
-static void
-do_get (OtTrivialHttpd    *self,
-        SoupServer        *server,
-        SoupMessage       *msg,
-        const char        *path,
-        SoupClientContext *context)
-{
-  char *slash;
-  int ret;
-  struct stat stbuf;
-
-  httpd_log (self, "serving %s\n", path);
-
-  if (opt_expected_cookies)
-    {
-      GSList *cookies = soup_cookies_from_request (msg);
-      GSList *l;
-      int i;
-
-      for (i = 0 ; opt_expected_cookies[i] != NULL; i++)
-        {
-          gboolean found = FALSE;
-          gchar *k = opt_expected_cookies[i];
-          gchar *v = strchr (k, '=') + 1;
-
-          for (l = cookies;  l != NULL ; l = g_slist_next (l))
-            {
-              SoupCookie *c = l->data;
-
-              if (!strncmp (k, soup_cookie_get_name (c), v - k - 1) &&
-                  !strcmp (v, soup_cookie_get_value (c)))
-                {
-                  found = TRUE;
-                  break;
-                }
-            }
-
-          if (!found)
-            {
-              httpd_log (self, "Expected cookie not found %s\n", k);
-              soup_message_set_status (msg, SOUP_STATUS_FORBIDDEN);
-              soup_cookies_free (cookies);
-              goto out;
-            }
-        }
-      soup_cookies_free (cookies);
-    }
-
-  if (opt_expected_headers)
-    {
-      for (int i = 0 ; opt_expected_headers[i] != NULL; i++)
-        {
-          const gchar *kv = opt_expected_headers[i];
-          const gchar *eq = strchr (kv, '=');
-
-          g_assert (eq);
-
-          {
-            g_autofree char *k = g_strndup (kv, eq - kv);
-            const gchar *expected_v = eq + 1;
-            const gchar *found_v = soup_message_headers_get_one (msg->request_headers, k);
-
-            if (!found_v)
-              {
-                httpd_log (self, "Expected header not found %s\n", k);
-                soup_message_set_status (msg, SOUP_STATUS_FORBIDDEN);
-                goto out;
-              }
-            if (strcmp (found_v, expected_v) != 0)
-              {
-                httpd_log (self, "Expected header %s: %s but found %s\n", k, expected_v, found_v);
-                soup_message_set_status (msg, SOUP_STATUS_FORBIDDEN);
-                goto out;
-              }
-          }
-        }
-    }
-
-  if (strstr (path, "../") != NULL)
-    {
-      soup_message_set_status (msg, SOUP_STATUS_FORBIDDEN);
-      goto out;
-    }
-
-  if (opt_random_500s_percentage > 0 &&
-      emitted_random_500s_count < opt_random_500s_max &&
-      g_random_int_range (0, 100) < opt_random_500s_percentage)
-    {
-      emitted_random_500s_count++;
-      soup_message_set_status (msg, SOUP_STATUS_INTERNAL_SERVER_ERROR);
-      goto out;
-    }
-
-  while (path[0] == '/')
-    path++;
-
-  do
-    ret = fstatat (self->root_dfd, path, &stbuf, 0);
-  while (ret == -1 && errno == EINTR);
-  if (ret == -1)
-    {
-      if (errno == EPERM)
-        soup_message_set_status (msg, SOUP_STATUS_FORBIDDEN);
-      else if (errno == ENOENT)
-        soup_message_set_status (msg, SOUP_STATUS_NOT_FOUND);
-      else
-        soup_message_set_status (msg, SOUP_STATUS_INTERNAL_SERVER_ERROR);
-      goto out;
-    }
-
-  if (!is_safe_to_access (&stbuf))
-    {
-      soup_message_set_status (msg, SOUP_STATUS_FORBIDDEN);
-      goto out;
-    }
-
-  if (S_ISDIR (stbuf.st_mode))
-    {
-      slash = strrchr (path, '/');
-      if (!slash || slash[1])
-        {
-          g_autofree char *redir_uri = NULL;
-
-          redir_uri = g_strdup_printf ("%s/", soup_message_get_uri (msg)->path);
-          soup_message_set_redirect (msg, SOUP_STATUS_MOVED_PERMANENTLY,
-                                     redir_uri);
-        }
-      else
-        {
-          g_autofree char *index_realpath = g_strconcat (path, "/index.html", NULL);
-          if (fstatat (self->root_dfd, index_realpath, &stbuf, 0) != -1)
-            {
-              g_autofree char *index_path = g_strconcat (path, "/index.html", NULL);
-              do_get (self, server, msg, index_path, context);
-            }
-          else
-            {
-              GString *listing = get_directory_listing (self->root_dfd, path);
-              soup_message_set_response (msg, "text/html",
-                                         SOUP_MEMORY_TAKE,
-                                         listing->str, listing->len);
-              soup_message_set_status (msg, SOUP_STATUS_OK);
-              g_string_free (listing, FALSE);
-            }
-        }
-    }
-  else 
-    {
-      if (!S_ISREG (stbuf.st_mode))
-        {
-          soup_message_set_status (msg, SOUP_STATUS_FORBIDDEN);
-          goto out;
-        }
-      
-      if (msg->method == SOUP_METHOD_GET)
-        {
-          glnx_fd_close int fd = -1;
-          g_autoptr(GMappedFile) mapping = NULL;
-          gsize buffer_length, file_size;
-          SoupRange *ranges;
-          int ranges_length;
-          gboolean have_ranges;
-
-          fd = openat (self->root_dfd, path, O_RDONLY | O_CLOEXEC);
-          if (fd < 0)
-            {
-              soup_message_set_status (msg, SOUP_STATUS_INTERNAL_SERVER_ERROR);
-              goto out;
-            }
-
-          mapping = g_mapped_file_new_from_fd (fd, FALSE, NULL);
-          if (!mapping)
-            {
-              soup_message_set_status (msg, SOUP_STATUS_INTERNAL_SERVER_ERROR);
-              goto out;
-            }
-          (void) close (fd); fd = -1;
-
-          file_size = g_mapped_file_get_length (mapping);
-          have_ranges = soup_message_headers_get_ranges(msg->request_headers, file_size, &ranges, &ranges_length);
-          if (opt_force_ranges && !have_ranges && g_strrstr (path, "/objects") != NULL)
-            {
-              SoupSocket *sock;
-              buffer_length = file_size/2;
-              soup_message_headers_set_content_length (msg->response_headers, file_size);
-              soup_message_headers_append (msg->response_headers,
-                                           "Connection", "close");
-
-              /* soup-message-io will wait for us to add
-               * another chunk after the first, to fill out
-               * the declared Content-Length. Instead, we
-               * forcibly close the socket at that point.
-               */
-              sock = soup_client_context_get_socket (context);
-              g_signal_connect (msg, "wrote-chunk", G_CALLBACK (close_socket), sock);
-            }
-          else
-            buffer_length = file_size;
-
-          if (have_ranges)
-            {
-              if (ranges_length > 0 && ranges[0].start >= file_size)
-                {
-                  soup_message_set_status (msg, SOUP_STATUS_REQUESTED_RANGE_NOT_SATISFIABLE);
-                  soup_message_headers_free_ranges (msg->request_headers, ranges);
-                  goto out;
-                }
-              soup_message_headers_free_ranges (msg->request_headers, ranges);
-            }
-          if (buffer_length > 0)
-            {
-              SoupBuffer *buffer;
-
-              buffer = soup_buffer_new_with_owner (g_mapped_file_get_contents (mapping),
-                                                   buffer_length,
-                                                   g_mapped_file_ref (mapping),
-                                                   (GDestroyNotify)g_mapped_file_unref);
-              soup_message_body_append_buffer (msg->response_body, buffer);
-              soup_buffer_free (buffer);
-            }
-        }
-      else /* msg->method == SOUP_METHOD_HEAD */
-        {
-          g_autofree char *length = NULL;
-
-          /* We could just use the same code for both GET and
-           * HEAD (soup-message-server-io.c will fix things up).
-           * But we'll optimize and avoid the extra I/O.
-           */
-          length = g_strdup_printf ("%lu", (gulong)stbuf.st_size);
-          soup_message_headers_append (msg->response_headers,
-                                       "Content-Length", length);
-        }
-      soup_message_set_status (msg, SOUP_STATUS_OK);
-    }
- out:
-  {
-    guint status = 0;
-    g_autofree gchar *reason = NULL;
-
-    g_object_get (msg,
-                  "status-code", &status,
-                  "reason-phrase", &reason,
-                  NULL);
-    httpd_log (self, "  status: %s (%u)\n", reason, status);
-  }
-  return;
-}
-
-static void
-httpd_callback (SoupServer *server, SoupMessage *msg,
-                const char *path, GHashTable *query,
-                SoupClientContext *context, gpointer data)
-{
-  OtTrivialHttpd *self = data;
-
-  if (msg->method == SOUP_METHOD_GET || msg->method == SOUP_METHOD_HEAD)
-    do_get (self, server, msg, path, context);
-  else
-    soup_message_set_status (msg, SOUP_STATUS_NOT_IMPLEMENTED);
-}
-
-static void
-on_dir_changed (GFileMonitor  *mon,
-		GFile *file,
-		GFile *other,
-		GFileMonitorEvent  event,
-		gpointer user_data)
-{
-  OtTrivialHttpd *self = user_data;
-
-  if (event == G_FILE_MONITOR_EVENT_DELETED)
-    {
-      self->running = FALSE;
-      g_main_context_wakeup (NULL);
-    }
-}
-
 gboolean
 ostree_builtin_trivial_httpd (int argc, char **argv, GCancellable *cancellable, GError **error)
 {
-  gboolean ret = FALSE;
-  g_autoptr(GOptionContext) context = NULL;
-  const char *dirpath;
-  OtTrivialHttpd appstruct = { 0, };
-  OtTrivialHttpd *app = &appstruct;
-  glnx_unref_object SoupServer *server = NULL;
-  g_autoptr(GFileMonitor) dirmon = NULL;
-
-  context = g_option_context_new ("[DIR] - Simple webserver");
-
-  app->root_dfd = -1;
-
-  if (!ostree_option_context_parse (context, options, &argc, &argv, OSTREE_BUILTIN_FLAG_NO_REPO, NULL, cancellable, error))
-    goto out;
-
-  if (argc > 1)
-    dirpath = argv[1];
-  else
-    dirpath = ".";
-
-  if (!glnx_opendirat (AT_FDCWD, dirpath, TRUE, &app->root_dfd, error))
-    goto out;
-
-  if (!(opt_random_500s_percentage >= 0 && opt_random_500s_percentage <= 99))
-    {
-      g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
-                   "Invalid --random-500s=%u", opt_random_500s_percentage);
-      goto out;
-    }
-
-  if (opt_log)
-    {
-      GOutputStream *stream = NULL;
-
-      if (g_strcmp0 (opt_log, "-") == 0)
-        {
-          if (opt_daemonize)
-            {
-              ot_util_usage_error (context, "Cannot use --log-file=- and --daemonize at the same time", error);
-              goto out;
-            }
-          stream = G_OUTPUT_STREAM (g_unix_output_stream_new (STDOUT_FILENO, FALSE));
-        }
-      else
-        {
-          g_autoptr(GFile) log_file;
-          GFileOutputStream* log_stream;
-
-          log_file = g_file_new_for_path (opt_log);
-          log_stream = g_file_create (log_file,
-                                      G_FILE_CREATE_PRIVATE,
-                                      cancellable,
-                                      error);
-          if (!log_stream)
-            goto out;
-          stream = G_OUTPUT_STREAM (log_stream);
-        }
-
-      app->log = stream;
-    }
-
-#if SOUP_CHECK_VERSION(2, 48, 0)
-  server = soup_server_new (SOUP_SERVER_SERVER_HEADER, "ostree-httpd ", NULL);
-  if (!soup_server_listen_all (server, opt_port, 0, error))
-    goto out;
-#else
-  server = soup_server_new (SOUP_SERVER_PORT, opt_port,
-                            SOUP_SERVER_SERVER_HEADER, "ostree-httpd ",
-                            NULL);
-#endif
-
-  soup_server_add_handler (server, NULL, httpd_callback, app, NULL);
-  if (opt_port_file)
-    {
-      g_autofree char *portstr = NULL;
-#if SOUP_CHECK_VERSION(2, 48, 0)
-      GSList *listeners = soup_server_get_listeners (server);
-      g_autoptr(GSocket) listener = NULL;
-      g_autoptr(GSocketAddress) addr = NULL;
-      
-      g_assert (listeners);
-      listener = g_object_ref (listeners->data);
-      g_slist_free (listeners);
-      listeners = NULL;
-      addr = g_socket_get_local_address (listener, error);
-      if (!addr)
-        goto out;
-
-      g_assert (G_IS_INET_SOCKET_ADDRESS (addr));
-      
-      portstr = g_strdup_printf ("%u\n", g_inet_socket_address_get_port ((GInetSocketAddress*)addr));
-#else
-      portstr = g_strdup_printf ("%u\n", soup_server_get_port (server));
-#endif
-
-      if (g_strcmp0 ("-", opt_port_file) == 0)
-        {
-          fputs (portstr, stdout); // not g_print - this must go to stdout, not a handler
-          fflush (stdout);
-        }
-      else if (!g_file_set_contents (opt_port_file, portstr, strlen (portstr), error))
-        goto out;
-    }
-#if !SOUP_CHECK_VERSION(2, 48, 0)
-  soup_server_run_async (server);
-#endif
-  
-  if (opt_daemonize)
-    {
-      pid_t pid = fork();
-      if (pid == -1)
-        {
-          int errsv = errno;
-          g_set_error_literal (error, G_IO_ERROR, g_io_error_from_errno (errsv),
-                               g_strerror (errsv));
-          goto out;
-        }
-      else if (pid > 0)
-        {
-          ret = TRUE;
-          goto out;
-        }
-      /* Child, continue */
-      /* Daemonising: close stdout/stderr so $() et al work on us */
-      fclose (stdout);
-      fclose (stdin);
-    }
-  else
-    {
-      /* Since we're used for testing purposes, let's just do this by
-       * default.  This ensures we exit when our parent does.
-       */
-      if (prctl (PR_SET_PDEATHSIG, SIGTERM) != 0)
-        {
-          if (errno != ENOSYS)
-            {
-              glnx_set_error_from_errno (error);
-              goto out;
-            }
-        }
-    }
-
-  app->running = TRUE;
-  if (opt_autoexit)
-    {
-      gboolean is_symlink = FALSE;
-      g_autoptr(GFile) root = NULL;
-      g_autoptr(GFileInfo) info = NULL;
-
-      root = g_file_new_for_path (dirpath);
-      info = g_file_query_info (root,
-                                G_FILE_ATTRIBUTE_STANDARD_IS_SYMLINK,
-                                G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
-                                cancellable, error);
-      if (!info)
-        goto out;
-
-      is_symlink = g_file_info_get_is_symlink (info);
-
-      if (is_symlink)
-        dirmon = g_file_monitor_file (root, 0, cancellable, error);
-      else
-        dirmon = g_file_monitor_directory (root, 0, cancellable, error);
-
-      if (!dirmon)
-        goto out;
-      g_signal_connect (dirmon, "changed", G_CALLBACK (on_dir_changed), app);
-    }
-  httpd_log (app, "serving at root %s\n", dirpath);
-  while (app->running)
-    g_main_context_iteration (NULL, TRUE);
+  g_autoptr(GPtrArray) new_argv = g_ptr_array_new ();
 
-  ret = TRUE;
- out:
-  if (app->root_dfd != -1)
-    (void) close (app->root_dfd);
-  g_clear_object (&app->log);
-  return ret;
+  g_ptr_array_add (new_argv, PKGLIBEXECDIR "/ostree-trivial-httpd");
+  for (int i = 0; i < argc; i++)
+    g_ptr_array_add (new_argv, argv[i]);
+  execvp (new_argv->pdata[0], (char**)new_argv->pdata);
+  /* Fall through on error */
+  glnx_set_error_from_errno (error);
+  return FALSE;
 }
diff --git a/tests/libtest.sh b/tests/libtest.sh
index c0bf8d0dba..ab624e7297 100755
--- a/tests/libtest.sh
+++ b/tests/libtest.sh
@@ -82,6 +82,12 @@ chmod -R u+w "${test_tmpdir}"
 export TEST_GPG_KEYHOME=${test_tmpdir}/gpghome
 export OSTREE_GPG_HOME=${test_tmpdir}/gpghome/trusted
 
+if test -n "${OSTREE_UNINSTALLED:-}"; then
+    OSTREE_HTTPD=${OSTREE_UNINSTALLED}/ostree-trivial-httpd
+else
+    OSTREE_HTTPD="${CMD_PREFIX} ostree trivial-httpd"
+fi
+
 if test -n "${OT_TESTS_DEBUG:-}"; then
     set -x
 fi
@@ -257,7 +263,7 @@ setup_fake_remote_repo1() {
     mkdir ${test_tmpdir}/httpd
     cd httpd
     ln -s ${test_tmpdir}/ostree-srv ostree
-    ${CMD_PREFIX} ostree trivial-httpd --autoexit --daemonize -p ${test_tmpdir}/httpd-port $args
+    ${OSTREE_HTTPD} --autoexit --daemonize -p ${test_tmpdir}/httpd-port $args
     port=$(cat ${test_tmpdir}/httpd-port)
     echo "http://127.0.0.1:${port}" > ${test_tmpdir}/httpd-address
     cd ${oldpwd} 
@@ -379,7 +385,7 @@ EOF
     mkdir ${test_tmpdir}/httpd
     cd httpd
     ln -s ${test_tmpdir} ostree
-    ${CMD_PREFIX} ostree trivial-httpd --autoexit --daemonize -p ${test_tmpdir}/httpd-port
+    ${OSTREE_HTTPD} --autoexit --daemonize -p ${test_tmpdir}/httpd-port
     port=$(cat ${test_tmpdir}/httpd-port)
     echo "http://127.0.0.1:${port}" > ${test_tmpdir}/httpd-address
     cd ${oldpwd} 
diff --git a/tests/test-commit-sign.sh b/tests/test-commit-sign.sh
index 60265c1a99..f963b1045b 100755
--- a/tests/test-commit-sign.sh
+++ b/tests/test-commit-sign.sh
@@ -53,7 +53,7 @@ cd ${test_tmpdir}
 mkdir ${test_tmpdir}/httpd
 cd httpd
 ln -s ${test_tmpdir}/ostree-srv ostree
-${CMD_PREFIX} ostree trivial-httpd --autoexit --daemonize -P 18081 -p ${test_tmpdir}/httpd-port
+${OSTREE_HTTPD} --autoexit --daemonize -P 18081 -p ${test_tmpdir}/httpd-port
 port=$(cat ${test_tmpdir}/httpd-port)
 assert_streq $port 18081
 echo "http://127.0.0.1:${port}" > ${test_tmpdir}/httpd-address
diff --git a/tests/test-pull-contenturl.sh b/tests/test-pull-contenturl.sh
index 16dcbe4fc4..d74d619b3f 100755
--- a/tests/test-pull-contenturl.sh
+++ b/tests/test-pull-contenturl.sh
@@ -51,8 +51,7 @@ fi
 find ${test_tmpdir}/ostree-srv/gnomerepo/objects \
   ! -name '*.commitmeta' -type f | xargs rm
 
-${CMD_PREFIX} ostree trivial-httpd --autoexit --daemonize \
-  -p ${test_tmpdir}/httpd-content-port
+${OSTREE_HTTPD} --autoexit --daemonize -p ${test_tmpdir}/httpd-content-port
 content_port=$(cat ${test_tmpdir}/httpd-content-port)
 echo "http://127.0.0.1:${content_port}" > ${test_tmpdir}/httpd-content-address
 
diff --git a/tests/test-pull-metalink.sh b/tests/test-pull-metalink.sh
index 2a1a73e776..07d619dfe2 100755
--- a/tests/test-pull-metalink.sh
+++ b/tests/test-pull-metalink.sh
@@ -29,7 +29,7 @@ echo '1..9'
 cd ${test_tmpdir}
 mkdir metalink-data
 cd metalink-data
-${CMD_PREFIX} ostree trivial-httpd --autoexit --daemonize -p ${test_tmpdir}/metalink-httpd-port
+${OSTREE_HTTPD} --autoexit --daemonize -p ${test_tmpdir}/metalink-httpd-port
 metalink_port=$(cat ${test_tmpdir}/metalink-httpd-port)
 echo "http://127.0.0.1:${metalink_port}" > ${test_tmpdir}/metalink-httpd-address
 
diff --git a/tests/test-pull-mirrorlist.sh b/tests/test-pull-mirrorlist.sh
index 454014ca78..13f40e7a6b 100755
--- a/tests/test-pull-mirrorlist.sh
+++ b/tests/test-pull-mirrorlist.sh
@@ -33,7 +33,7 @@ setup_mirror () {
   cd $name
   cp -a ${test_tmpdir}/ostree-srv ostree
 
-  ${CMD_PREFIX} ostree trivial-httpd --autoexit --daemonize \
+  ${OSTREE_HTTPD} --autoexit --daemonize \
     -p ${test_tmpdir}/${name}-port
   port=$(cat ${test_tmpdir}/${name}-port)
   echo "http://127.0.0.1:${port}" > ${test_tmpdir}/${name}-address
diff --git a/tests/test-pull-override-url.sh b/tests/test-pull-override-url.sh
index d81b345494..928013a574 100755
--- a/tests/test-pull-override-url.sh
+++ b/tests/test-pull-override-url.sh
@@ -50,7 +50,7 @@ mkdir mirror-httpd
 cd mirror-httpd
 ln -s ${test_tmpdir}/mirror-srv ostree
 mirror_log="${test_tmpdir}/mirror_log"
-${CMD_PREFIX} ostree trivial-httpd --log-file=${mirror_log} --autoexit --daemonize -p ${test_tmpdir}/mirror-httpd-port
+${OSTREE_HTTPD} --log-file=${mirror_log} --autoexit --daemonize -p ${test_tmpdir}/mirror-httpd-port
 port=$(cat ${test_tmpdir}/mirror-httpd-port)
 echo "http://127.0.0.1:${port}" > ${test_tmpdir}/mirror-httpd-address
 

From f31a541f15240eacc8e087df8fce6b4b41327898 Mon Sep 17 00:00:00 2001
From: Colin Walters <walters@verbum.org>
Date: Fri, 30 Dec 2016 14:38:16 -0500
Subject: [PATCH 06/13] fixup! fetcher: Split lowlevel API into file/membuf
 variants

---
 src/libostree/ostree-fetcher-util.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/libostree/ostree-fetcher-util.c b/src/libostree/ostree-fetcher-util.c
index fd9dfea123..b8af972ad1 100644
--- a/src/libostree/ostree-fetcher-util.c
+++ b/src/libostree/ostree-fetcher-util.c
@@ -59,7 +59,6 @@ _ostree_fetcher_mirrored_request_to_membuf (OstreeFetcher  *fetcher,
                                             GError         **error)
 {
   gboolean ret = FALSE;
-  const guint8 nulchar = 0;
   g_autoptr(GMainContext) mainctx = NULL;
   FetchUriSyncData data;
   g_assert (error != NULL);

From 95fc303994ccd6eb2b01068f78237387a449b614 Mon Sep 17 00:00:00 2001
From: Colin Walters <walters@verbum.org>
Date: Tue, 3 Jan 2017 10:45:58 -0500
Subject: [PATCH 07/13] pull: Rework delta superblock fetches to be async

For the pending libcurl port, the backend is a bit more
sensitive to the main context setup.  The delta superblock
fetch here is a synchronous request in the section that's
supposed to be async.

Now, libsoup definitely supports mixing sync and async requests, but it wasn't
hard to help the libcurl port here by making this one async. Now fetchers are
either sync or async.
---
 src/libostree/ostree-repo-pull.c | 213 +++++++++++++++++--------------
 1 file changed, 116 insertions(+), 97 deletions(-)

diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c
index 92efd59255..971ecabffd 100644
--- a/src/libostree/ostree-repo-pull.c
+++ b/src/libostree/ostree-repo-pull.c
@@ -67,6 +67,7 @@ typedef struct {
   gint          n_scanned_metadata;
 
   gboolean          gpg_verify;
+  gboolean          require_static_deltas;
   gboolean          gpg_verify_summary;
   gboolean          has_tombstone_commits;
 
@@ -1416,75 +1417,6 @@ load_remote_repo_config (OtPullData    *pull_data,
   return ret;
 }
 
-static gboolean
-request_static_delta_superblock_sync (OtPullData  *pull_data,
-                                      const char  *from_revision,
-                                      const char  *to_revision,
-                                      GVariant   **out_delta_superblock,
-                                      GCancellable *cancellable,
-                                      GError     **error)
-{
-  gboolean ret = FALSE;
-  g_autoptr(GVariant) ret_delta_superblock = NULL;
-  g_autofree char *delta_name =
-    _ostree_get_relative_static_delta_superblock_path (from_revision, to_revision);
-  g_autoptr(GBytes) delta_superblock_data = NULL;
-
-  if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher,
-                                                   pull_data->content_mirrorlist,
-                                                   delta_name, FALSE, TRUE,
-                                                   &delta_superblock_data,
-                                                   OSTREE_MAX_METADATA_SIZE,
-                                                   pull_data->cancellable, error))
-    goto out;
-
-  if (delta_superblock_data)
-    {
-      {
-        g_autofree gchar *delta = NULL;
-        g_autofree guchar *ret_csum = NULL;
-        guchar *summary_csum;
-        g_autoptr (GInputStream) summary_is = NULL;
-
-        summary_is = g_memory_input_stream_new_from_data (g_bytes_get_data (delta_superblock_data, NULL),
-                                                          g_bytes_get_size (delta_superblock_data),
-                                                          NULL);
-
-        if (!ot_gio_checksum_stream (summary_is, &ret_csum, cancellable, error))
-          goto out;
-
-        delta = g_strconcat (from_revision ? from_revision : "", from_revision ? "-" : "", to_revision, NULL);
-        summary_csum = g_hash_table_lookup (pull_data->summary_deltas_checksums, delta);
-
-        /* At this point we've GPG verified the data, so in theory
-         * could trust that they provided the right data, but let's
-         * make this a hard error.
-         */
-        if (pull_data->gpg_verify_summary && !summary_csum)
-          {
-            g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
-                         "GPG verification enabled, but no summary signatures found (use gpg-verify-summary=false in remote config to disable)");
-            goto out;
-          }
-
-        if (summary_csum && memcmp (summary_csum, ret_csum, 32))
-          {
-            g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, "Invalid checksum for static delta %s", delta);
-            goto out;
-          }
-      }
-
-      ret_delta_superblock = g_variant_ref_sink (g_variant_new_from_bytes ((GVariantType*)OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT,
-                                                                           delta_superblock_data, FALSE));
-    }
-  
-  ret = TRUE;
-  if (out_delta_superblock)
-    *out_delta_superblock = g_steal_pointer (&ret_delta_superblock);
- out:
-  return ret;
-}
-
 static gboolean
 process_one_static_delta_fallback (OtPullData   *pull_data,
                                    gboolean      delta_byteswap,
@@ -1749,6 +1681,100 @@ process_one_static_delta (OtPullData   *pull_data,
   return ret;
 }
 
+typedef struct {
+  OtPullData *pull_data;
+  char *from_revision;
+  char *to_revision;
+} FetchDeltaSuperData;
+
+static void
+on_superblock_fetched (GObject   *src,
+                       GAsyncResult *res,
+                       gpointer      data)
+
+{
+  FetchDeltaSuperData *fdata = data;
+  OtPullData *pull_data = fdata->pull_data;
+  GError *local_error = NULL;
+  GError **error = &local_error;
+  g_autoptr(GBytes) delta_superblock_data = NULL;
+  const char *from_revision = fdata->from_revision;
+  const char *to_revision = fdata->to_revision;
+
+  if (!_ostree_fetcher_request_to_membuf_finish ((OstreeFetcher*)src,
+                                                 res,
+                                                 &delta_superblock_data,
+                                                 error))
+    {
+      if (!g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
+        goto out;
+
+      g_clear_error (&local_error);
+
+      if (pull_data->require_static_deltas)
+        {
+          g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                       "Static deltas required, but none found for %s to %s",
+                       from_revision, to_revision);
+          goto out;
+        }
+
+      queue_scan_one_metadata_object (pull_data, to_revision, OSTREE_OBJECT_TYPE_COMMIT, NULL, 0);
+    }
+  else
+    {
+      g_autofree gchar *delta = NULL;
+      g_autofree guchar *ret_csum = NULL;
+      guchar *summary_csum;
+      g_autoptr (GInputStream) summary_is = NULL;
+      g_autoptr(GVariant) delta_superblock = NULL;
+
+      summary_is = g_memory_input_stream_new_from_data (g_bytes_get_data (delta_superblock_data, NULL),
+                                                        g_bytes_get_size (delta_superblock_data),
+                                                        NULL);
+
+      if (!ot_gio_checksum_stream (summary_is, &ret_csum, pull_data->cancellable, error))
+        goto out;
+
+      delta = g_strconcat (from_revision ? from_revision : "", from_revision ? "-" : "", to_revision, NULL);
+      summary_csum = g_hash_table_lookup (pull_data->summary_deltas_checksums, delta);
+
+      /* At this point we've GPG verified the data, so in theory
+       * could trust that they provided the right data, but let's
+       * make this a hard error.
+       */
+      if (pull_data->gpg_verify_summary && !summary_csum)
+        {
+          g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                       "GPG verification enabled, but no summary signatures found (use gpg-verify-summary=false in remote config to disable)");
+          goto out;
+        }
+
+      if (summary_csum && memcmp (summary_csum, ret_csum, 32))
+        {
+          g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, "Invalid checksum for static delta %s", delta);
+          goto out;
+        }
+
+      delta_superblock = g_variant_ref_sink (g_variant_new_from_bytes ((GVariantType*)OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT,
+                                                                       delta_superblock_data, FALSE));
+
+      g_ptr_array_add (pull_data->static_delta_superblocks, g_variant_ref (delta_superblock));
+      if (!process_one_static_delta (pull_data, from_revision, to_revision, delta_superblock,
+                                     pull_data->cancellable, error))
+        goto out;
+    }
+
+ out:
+  g_free (fdata->from_revision);
+  g_free (fdata->to_revision);
+  g_free (fdata);
+  g_assert (pull_data->n_outstanding_metadata_fetches > 0);
+  pull_data->n_outstanding_metadata_fetches--;
+  pull_data->n_fetched_metadata++;
+  check_outstanding_requests_handle_error (pull_data, local_error);
+}
+
 static gboolean
 validate_variant_is_csum (GVariant       *csum,
                           GError        **error)
@@ -2344,7 +2370,6 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
   g_autofree char **override_commit_ids = NULL;
   GSource *update_timeout = NULL;
   gboolean disable_static_deltas = FALSE;
-  gboolean require_static_deltas = FALSE;
   gboolean opt_gpg_verify_set = FALSE;
   gboolean opt_gpg_verify_summary_set = FALSE;
   const char *url_override = NULL;
@@ -2368,7 +2393,7 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
         g_variant_lookup (options, "gpg-verify-summary", "b", &pull_data->gpg_verify_summary);
       (void) g_variant_lookup (options, "depth", "i", &pull_data->maxdepth);
       (void) g_variant_lookup (options, "disable-static-deltas", "b", &disable_static_deltas);
-      (void) g_variant_lookup (options, "require-static-deltas", "b", &require_static_deltas);
+      (void) g_variant_lookup (options, "require-static-deltas", "b", &pull_data->require_static_deltas);
       (void) g_variant_lookup (options, "override-commit-ids", "^a&s", &override_commit_ids);
       (void) g_variant_lookup (options, "dry-run", "b", &pull_data->dry_run);
       (void) g_variant_lookup (options, "override-url", "&s", &url_override);
@@ -2386,11 +2411,11 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
   for (i = 0; dirs_to_pull != NULL && dirs_to_pull[i] != NULL; i++)
     g_return_val_if_fail (dirs_to_pull[i][0] == '/', FALSE);
 
-  g_return_val_if_fail (!(disable_static_deltas && require_static_deltas), FALSE);
+  g_return_val_if_fail (!(disable_static_deltas && pull_data->require_static_deltas), FALSE);
   /* We only do dry runs with static deltas, because we don't really have any
    * in-advance information for bare fetches.
    */
-  g_return_val_if_fail (!pull_data->dry_run || require_static_deltas, FALSE);
+  g_return_val_if_fail (!pull_data->dry_run || pull_data->require_static_deltas, FALSE);
 
   pull_data->is_mirror = (flags & OSTREE_REPO_PULL_FLAGS_MIRROR) > 0;
   pull_data->is_commit_only = (flags & OSTREE_REPO_PULL_FLAGS_COMMIT_ONLY) > 0;
@@ -2655,7 +2680,7 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
   /* For local pulls, default to disabling static deltas so that the
    * exact object files are copied.
    */
-  if (pull_data->remote_repo_local && !require_static_deltas)
+  if (pull_data->remote_repo_local && !pull_data->require_static_deltas)
     disable_static_deltas = TRUE;
 
   pull_data->static_delta_superblocks = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref);
@@ -2710,7 +2735,7 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
         goto out;
       }
 
-    if (!bytes_summary && require_static_deltas)
+    if (!bytes_summary && pull_data->require_static_deltas)
       {
         g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
                      "Fetch configured to require static deltas, but no summary found");
@@ -2948,31 +2973,25 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
       if (!(disable_static_deltas || mirroring_into_archive || pull_data->is_commit_only) &&
           (from_revision == NULL || g_strcmp0 (from_revision, to_revision) != 0))
         {
-          if (!request_static_delta_superblock_sync (pull_data, from_revision, to_revision,
-                                                     &delta_superblock, cancellable, error))
-            goto out;
-        }
-          
-      if (!delta_superblock)
-        {
-          if (require_static_deltas)
-            {
-              g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
-                           "Static deltas required, but none found for %s to %s",
-                           from_revision, to_revision);
-              goto out;
-            }
-          g_debug ("no delta superblock for %s-%s", from_revision ? from_revision : "empty", to_revision);
-          queue_scan_one_metadata_object (pull_data, to_revision, OSTREE_OBJECT_TYPE_COMMIT, NULL, 0);
+          g_autofree char *delta_name =
+            _ostree_get_relative_static_delta_superblock_path (from_revision, to_revision);
+          FetchDeltaSuperData *fdata = g_new0(FetchDeltaSuperData, 1);
+          fdata->pull_data = pull_data;
+          fdata->from_revision = g_strdup (from_revision);
+          fdata->to_revision = g_strdup (to_revision);
+
+          _ostree_fetcher_request_to_membuf (pull_data->fetcher,
+                                             pull_data->content_mirrorlist,
+                                             delta_name, 0,
+                                             OSTREE_MAX_METADATA_SIZE,
+                                             0, pull_data->cancellable,
+                                             on_superblock_fetched, fdata);
+          pull_data->n_outstanding_metadata_fetches++;
+          pull_data->n_requested_metadata++;
         }
       else
         {
-          g_debug ("processing delta superblock for %s-%s", from_revision ? from_revision : "empty", to_revision);
-          g_ptr_array_add (pull_data->static_delta_superblocks, g_variant_ref (delta_superblock));
-          if (!process_one_static_delta (pull_data, from_revision, to_revision,
-                                         delta_superblock,
-                                         cancellable, error))
-            goto out;
+          queue_scan_one_metadata_object (pull_data, to_revision, OSTREE_OBJECT_TYPE_COMMIT, NULL, 0);
         }
     }
 

From 3f78647f7a6a16e86e56d05fb70cc95acfffdb51 Mon Sep 17 00:00:00 2001
From: Colin Walters <walters@verbum.org>
Date: Tue, 3 Jan 2017 11:13:35 -0500
Subject: [PATCH 08/13] fixup! pull: Rework delta superblock fetches to be
 async

---
 src/libostree/ostree-repo-pull.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c
index 971ecabffd..fb3e4fa352 100644
--- a/src/libostree/ostree-repo-pull.c
+++ b/src/libostree/ostree-repo-pull.c
@@ -179,6 +179,10 @@ update_progress (gpointer user_data)
   if (! pull_data->progress)
     return FALSE;
 
+  /* In dry run, we only emit progress once metadata is done */
+  if (pull_data->dry_run && pull_data->n_outstanding_metadata_fetches > 0)
+    return TRUE;
+
   outstanding_writes = pull_data->n_outstanding_content_write_requests +
     pull_data->n_outstanding_metadata_write_requests +
     pull_data->n_outstanding_deltapart_write_requests;

From c42f55213c1ac25cadcb20cde5abba7184dfeec3 Mon Sep 17 00:00:00 2001
From: Colin Walters <walters@verbum.org>
Date: Tue, 3 Jan 2017 11:17:55 -0500
Subject: [PATCH 09/13] fixup! pull: Rework delta superblock fetches to be
 async

---
 src/libostree/ostree-repo-pull.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c
index fb3e4fa352..54a3a92240 100644
--- a/src/libostree/ostree-repo-pull.c
+++ b/src/libostree/ostree-repo-pull.c
@@ -2968,7 +2968,6 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
       g_autofree char *from_revision = NULL;
       const char *ref = key;
       const char *to_revision = value;
-      g_autoptr(GVariant) delta_superblock = NULL;
 
       if (!ostree_repo_resolve_rev (pull_data->repo, ref, TRUE,
                                     &from_revision, error))

From 1287f9de7c98762523182252ba42ec2e8a348231 Mon Sep 17 00:00:00 2001
From: Colin Walters <walters@verbum.org>
Date: Tue, 3 Jan 2017 12:11:50 -0500
Subject: [PATCH 10/13] fixup! Split trivial-httpd into separate binary

---
 tests/libtest.sh | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/tests/libtest.sh b/tests/libtest.sh
index ab624e7297..137d9534f2 100755
--- a/tests/libtest.sh
+++ b/tests/libtest.sh
@@ -82,12 +82,6 @@ chmod -R u+w "${test_tmpdir}"
 export TEST_GPG_KEYHOME=${test_tmpdir}/gpghome
 export OSTREE_GPG_HOME=${test_tmpdir}/gpghome/trusted
 
-if test -n "${OSTREE_UNINSTALLED:-}"; then
-    OSTREE_HTTPD=${OSTREE_UNINSTALLED}/ostree-trivial-httpd
-else
-    OSTREE_HTTPD="${CMD_PREFIX} ostree trivial-httpd"
-fi
-
 if test -n "${OT_TESTS_DEBUG:-}"; then
     set -x
 fi
@@ -108,6 +102,12 @@ else
     fi
 fi
 
+if test -n "${OSTREE_UNINSTALLED:-}"; then
+    OSTREE_HTTPD=${OSTREE_UNINSTALLED}/ostree-trivial-httpd
+else
+    OSTREE_HTTPD="${CMD_PREFIX} ostree trivial-httpd"
+fi
+
 assert_streq () {
     test "$1" = "$2" || (echo 1>&2 "$1 != $2"; exit 1)
 }

From 0cdd8deefac99644654149de5879046dd27c6662 Mon Sep 17 00:00:00 2001
From: Colin Walters <walters@verbum.org>
Date: Tue, 3 Jan 2017 12:42:55 -0500
Subject: [PATCH 11/13] fixup! Split trivial-httpd into separate binary

---
 src/ostree/ot-builtin-trivial-httpd.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/ostree/ot-builtin-trivial-httpd.c b/src/ostree/ot-builtin-trivial-httpd.c
index a411e0cc94..206970c56d 100644
--- a/src/ostree/ot-builtin-trivial-httpd.c
+++ b/src/ostree/ot-builtin-trivial-httpd.c
@@ -31,8 +31,9 @@ ostree_builtin_trivial_httpd (int argc, char **argv, GCancellable *cancellable,
   g_autoptr(GPtrArray) new_argv = g_ptr_array_new ();
 
   g_ptr_array_add (new_argv, PKGLIBEXECDIR "/ostree-trivial-httpd");
-  for (int i = 0; i < argc; i++)
+  for (int i = 1; i < argc; i++)
     g_ptr_array_add (new_argv, argv[i]);
+  g_ptr_array_add (new_argv, NULL);
   execvp (new_argv->pdata[0], (char**)new_argv->pdata);
   /* Fall through on error */
   glnx_set_error_from_errno (error);

From 78609ca4b37b55db72b647562a2086dd82a0a93d Mon Sep 17 00:00:00 2001
From: Colin Walters <walters@verbum.org>
Date: Wed, 4 Jan 2017 11:13:03 -0500
Subject: [PATCH 12/13] fixup! fetcher: Split lowlevel API into file/membuf
 variants

---
 src/libostree/ostree-fetcher.c | 2 +-
 src/libostree/ostree-fetcher.h | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libostree/ostree-fetcher.c b/src/libostree/ostree-fetcher.c
index ea75758172..a7e8feaf4e 100644
--- a/src/libostree/ostree-fetcher.c
+++ b/src/libostree/ostree-fetcher.c
@@ -1299,7 +1299,7 @@ _ostree_fetcher_request_to_membuf (OstreeFetcher         *self,
 gboolean
 _ostree_fetcher_request_to_membuf_finish (OstreeFetcher *self,
                                           GAsyncResult  *result,
-                                          GBytes       **out_filename,
+                                          GBytes       **out_buf,
                                           GError       **error)
 {
   GTask *task;
diff --git a/src/libostree/ostree-fetcher.h b/src/libostree/ostree-fetcher.h
index da39623166..a57907e429 100644
--- a/src/libostree/ostree-fetcher.h
+++ b/src/libostree/ostree-fetcher.h
@@ -133,7 +133,7 @@ void _ostree_fetcher_request_to_membuf (OstreeFetcher         *self,
 
 gboolean _ostree_fetcher_request_to_membuf_finish (OstreeFetcher *self,
                                                    GAsyncResult  *result,
-                                                   GBytes       **out_filename,
+                                                   GBytes       **out_buf,
                                                    GError       **error);
 
 

From ffcc2509fb7cd3b8337bbc2de5bb5e894cb42cfd Mon Sep 17 00:00:00 2001
From: Colin Walters <walters@verbum.org>
Date: Wed, 4 Jan 2017 11:18:23 -0500
Subject: [PATCH 13/13] fixup! fetcher: Split lowlevel API into file/membuf
 variants

---
 src/libostree/ostree-fetcher.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libostree/ostree-fetcher.c b/src/libostree/ostree-fetcher.c
index a7e8feaf4e..cee33186cb 100644
--- a/src/libostree/ostree-fetcher.c
+++ b/src/libostree/ostree-fetcher.c
@@ -1317,8 +1317,8 @@ _ostree_fetcher_request_to_membuf_finish (OstreeFetcher *self,
     return FALSE;
 
   g_assert (pending->is_membuf);
-  g_assert (out_filename);
-  *out_filename = ret;
+  g_assert (out_buf);
+  *out_buf = ret;
 
   return TRUE;
 }