From 40a1d32067aeed836858da87c1f03ea1dbb3515e Mon Sep 17 00:00:00 2001
From: Jonathan Lebon <jlebon@redhat.com>
Date: Wed, 8 Feb 2017 22:06:05 -0500
Subject: [PATCH] fetcher queue: also throttle on outstanding writes

When fetching over a fast enough connection, we can be receiving files
faster than we write them. This can then lead to EMFILE when we have
enough files open. This was made very easy to notice with the upcoming
libcurl backend, which makes use of pipelining.

Closes: #675
Approved by: cgwalters
---
 src/libostree/ostree-repo-private.h |  8 ++++++++
 src/libostree/ostree-repo-pull.c    | 21 ++++++++++++++++-----
 2 files changed, 24 insertions(+), 5 deletions(-)

diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h
index 73e0244670..f1e00f27b8 100644
--- a/src/libostree/ostree-repo-private.h
+++ b/src/libostree/ostree-repo-private.h
@@ -35,6 +35,14 @@ G_BEGIN_DECLS
 #define _OSTREE_MAX_OUTSTANDING_FETCHER_REQUESTS 8
 #define _OSTREE_MAX_OUTSTANDING_DELTAPART_REQUESTS 2
 
+/* In most cases, writing to disk should be much faster than
+ * fetching from the network, so we shouldn't actually hit
+ * this. But if using pipelining and e.g. pulling over LAN
+ * (or writing to slow media), we can have a runaway
+ * situation towards EMFILE.
+ * */
+#define _OSTREE_MAX_OUTSTANDING_WRITE_REQUESTS 16
+
 typedef enum {
   OSTREE_REPO_TEST_ERROR_PRE_COMMIT = (1 << 0)
 } OstreeRepoTestErrorFlags;
diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c
index 295973ec3c..0e5128d9ee 100644
--- a/src/libostree/ostree-repo-pull.c
+++ b/src/libostree/ostree-repo-pull.c
@@ -345,15 +345,26 @@ check_outstanding_requests_handle_error (OtPullData          *pull_data,
 
 /* We have a total-request limit, as well has a hardcoded max of 2 for delta
  * parts. The logic for the delta one is that processing them is expensive, and
- * doing multiple simultaneously could risk space/memory on smaller devices.
+ * doing multiple simultaneously could risk space/memory on smaller devices. We
+ * also throttle on outstanding writes in case fetches are faster.
  */
 static gboolean
 fetcher_queue_is_full (OtPullData *pull_data)
 {
-  return (pull_data->n_outstanding_metadata_fetches +
-          pull_data->n_outstanding_content_fetches +
-          pull_data->n_outstanding_deltapart_fetches) == _OSTREE_MAX_OUTSTANDING_FETCHER_REQUESTS ||
-    pull_data->n_outstanding_deltapart_fetches == _OSTREE_MAX_OUTSTANDING_DELTAPART_REQUESTS;
+  const gboolean fetch_full =
+      ((pull_data->n_outstanding_metadata_fetches +
+        pull_data->n_outstanding_content_fetches +
+        pull_data->n_outstanding_deltapart_fetches) ==
+         _OSTREE_MAX_OUTSTANDING_FETCHER_REQUESTS);
+  const gboolean deltas_full =
+      (pull_data->n_outstanding_deltapart_fetches ==
+        _OSTREE_MAX_OUTSTANDING_DELTAPART_REQUESTS);
+  const gboolean writes_full =
+      ((pull_data->n_outstanding_metadata_write_requests +
+        pull_data->n_outstanding_content_write_requests +
+        pull_data->n_outstanding_deltapart_write_requests) >=
+         _OSTREE_MAX_OUTSTANDING_WRITE_REQUESTS);
+  return fetch_full || deltas_full || writes_full;
 }
 
 static gboolean