[merged] Libcurl prep

src/libostree/ostree-fetcher.c

@@ -52,6 +52,7 @@ typedef struct {
   SoupSession *session;  /* not referenced */
   GMainContext *main_context;
   volatile gint running;
+  GError *initialization_error; /* Any failure to load the db */
 
   int tmpdir_dfd;
   char *tmpdir_name;
@@ -357,12 +358,14 @@ static void
 session_thread_set_tls_interaction_cb (ThreadClosure *thread_closure,
                                        gpointer data)
 {
-  GTlsCertificate *cert = data;
+  const char *cert_and_key_path = data; /* str\0str\0 in one malloc buf */
+  const char *cert_path = cert_and_key_path;
+  const char *key_path = cert_and_key_path + strlen (cert_and_key_path) + 1;
   glnx_unref_object OstreeTlsCertInteraction *interaction = NULL;
 
   /* The GTlsInteraction instance must be created in the
    * session thread so it uses the correct GMainContext. */
-  interaction = _ostree_tls_cert_interaction_new (cert);
+  interaction = _ostree_tls_cert_interaction_new (cert_path, key_path);
 
   g_object_set (thread_closure->session,
                 SOUP_SESSION_TLS_INTERACTION,
@@ -374,13 +377,19 @@ static void
 session_thread_set_tls_database_cb (ThreadClosure *thread_closure,
                                     gpointer data)
 {
-  GTlsDatabase *database = data;
+  const char *db_path = data;
 
-  if (database != NULL)
+  if (db_path != NULL)
     {
-      g_object_set (thread_closure->session,
-                    SOUP_SESSION_TLS_DATABASE,
-                    database, NULL);
+      glnx_unref_object GTlsDatabase *tlsdb = NULL;
+
+      g_clear_error (&thread_closure->initialization_error);
+      tlsdb = g_tls_file_database_new (db_path, &thread_closure->initialization_error);
+
+      if (tlsdb)
+        g_object_set (thread_closure->session,
+                      SOUP_SESSION_TLS_DATABASE,
+                      tlsdb, NULL);
     }
   else
     {
@@ -452,6 +461,13 @@ session_thread_request_uri (ThreadClosure *thread_closure,
   pending = g_task_get_task_data (task);
   cancellable = g_task_get_cancellable (task);
 
+  /* If we caught an error in init, re-throw it for every request */
+  if (thread_closure->initialization_error)
+    {
+      g_task_return_error (task, g_error_copy (thread_closure->initialization_error));
+      return;
+    }
+
   create_pending_soup_request (pending, &local_error);
   if (local_error != NULL)
     {
@@ -797,41 +813,39 @@ _ostree_fetcher_set_cookie_jar (OstreeFetcher *self,
 
 void
 _ostree_fetcher_set_client_cert (OstreeFetcher   *self,
-                                 GTlsCertificate *cert)
+                                 const char      *cert_path,
+                                 const char      *key_path)
 {
+  g_autoptr(GString) buf = NULL;
   g_return_if_fail (OSTREE_IS_FETCHER (self));
-  g_return_if_fail (G_IS_TLS_CERTIFICATE (cert));
+
+  if (cert_path)
+    {
+      buf = g_string_new (cert_path);
+      g_string_append_c (buf, '\0');
+      g_string_append (buf, key_path);
+    }
 
 #ifdef HAVE_LIBSOUP_CLIENT_CERTS
   session_thread_idle_add (self->thread_closure,
                            session_thread_set_tls_interaction_cb,
-                           g_object_ref (cert),
-                           (GDestroyNotify) g_object_unref);
+                           g_string_free (g_steal_pointer (&buf), FALSE),
+                           (GDestroyNotify) g_free);
 #else
   g_warning ("This version of OSTree is compiled without client side certificate support");
 #endif
 }
 
 void
 _ostree_fetcher_set_tls_database (OstreeFetcher *self,
-                                  GTlsDatabase  *db)
+                                  const char    *tlsdb_path)
 {
   g_return_if_fail (OSTREE_IS_FETCHER (self));
-  g_return_if_fail (db == NULL || G_IS_TLS_DATABASE (db));
 
-  if (db != NULL)
-    {
-      session_thread_idle_add (self->thread_closure,
-                               session_thread_set_tls_database_cb,
-                               g_object_ref (db),
-                               (GDestroyNotify) g_object_unref);
-    }
-  else
-    {
-      session_thread_idle_add (self->thread_closure,
-                               session_thread_set_tls_database_cb,
-                               NULL, (GDestroyNotify) NULL);
-    }
+  session_thread_idle_add (self->thread_closure,
+                           session_thread_set_tls_database_cb,
+                           g_strdup (tlsdb_path),
+                           (GDestroyNotify) g_free);
 }
 
 void

src/libostree/ostree-fetcher.h

@@ -93,10 +93,11 @@ void _ostree_fetcher_set_proxy (OstreeFetcher *fetcher,
                                 const char    *proxy);
 
 void _ostree_fetcher_set_client_cert (OstreeFetcher *fetcher,
-                                     GTlsCertificate *cert);
+                                      const char     *cert_path,
+                                      const char     *key_path);
 
 void _ostree_fetcher_set_tls_database (OstreeFetcher *self,
-                                       GTlsDatabase *db);
+                                       const char    *tlsdb_path);
 
 void _ostree_fetcher_set_extra_headers (OstreeFetcher *self,
                                         GVariant      *extra_headers);

src/libostree/ostree-repo-pull.c

@@ -80,7 +80,7 @@ typedef struct {
   GHashTable       *commit_to_depth; /* Maps commit checksum maximum depth */
   GHashTable       *scanned_metadata; /* Maps object name to itself */
   GHashTable       *requested_metadata; /* Maps object name to itself */
-  GHashTable       *requested_content; /* Maps object name to itself */
+  GHashTable       *requested_content; /* Maps checksum to itself */
   guint             n_outstanding_metadata_fetches;
   guint             n_outstanding_metadata_write_requests;
   guint             n_outstanding_content_fetches;
@@ -1267,7 +1267,7 @@ scan_one_metadata_object_c (OtPullData         *pull_data,
   if (g_hash_table_lookup (pull_data->scanned_metadata, object))
     return TRUE;
 
-  is_requested = g_hash_table_lookup (pull_data->requested_metadata, tmp_checksum) != NULL;
+  is_requested = g_hash_table_lookup (pull_data->requested_metadata, object) != NULL;
   if (!ostree_repo_has_object (pull_data->repo, objtype, tmp_checksum, &is_stored,
                                cancellable, error))
     goto out;
@@ -1292,10 +1292,9 @@ scan_one_metadata_object_c (OtPullData         *pull_data,
 
   if (!is_stored && !is_requested)
     {
-      char *duped_checksum = g_strdup (tmp_checksum);
       gboolean do_fetch_detached;
 
-      g_hash_table_add (pull_data->requested_metadata, duped_checksum);
+      g_hash_table_add (pull_data->requested_metadata, g_variant_ref (object));
 
       do_fetch_detached = (objtype == OSTREE_OBJECT_TYPE_COMMIT);
       enqueue_one_object_request (pull_data, tmp_checksum, objtype, path, do_fetch_detached, FALSE);
@@ -1467,10 +1466,11 @@ process_one_static_delta_fallback (OtPullData   *pull_data,
     { 
       if (OSTREE_OBJECT_TYPE_IS_META (objtype))
         {
-          if (!g_hash_table_lookup (pull_data->requested_metadata, checksum))
+          g_autoptr(GVariant) objname = ostree_object_name_serialize (checksum, objtype);
+          if (!g_hash_table_lookup (pull_data->requested_metadata, objname))
             {
               gboolean do_fetch_detached;
-              g_hash_table_add (pull_data->requested_metadata, checksum);
+              g_hash_table_add (pull_data->requested_metadata, g_variant_ref (objname));
 
               do_fetch_detached = (objtype == OSTREE_OBJECT_TYPE_COMMIT);
               enqueue_one_object_request (pull_data, checksum, objtype, NULL, do_fetch_detached, FALSE);
@@ -1951,17 +1951,7 @@ _ostree_repo_remote_new_fetcher (OstreeRepo  *self,
       }
     else if (tls_client_cert_path != NULL)
       {
-        g_autoptr(GTlsCertificate) client_cert = NULL;
-
-        g_assert (tls_client_key_path != NULL);
-
-        client_cert = g_tls_certificate_new_from_files (tls_client_cert_path,
-                                                        tls_client_key_path,
-                                                        error);
-        if (client_cert == NULL)
-          goto out;
-
-        _ostree_fetcher_set_client_cert (fetcher, client_cert);
+        _ostree_fetcher_set_client_cert (fetcher, tls_client_cert_path, tls_client_key_path);
       }
   }
 
@@ -1975,13 +1965,7 @@ _ostree_repo_remote_new_fetcher (OstreeRepo  *self,
 
     if (tls_ca_path != NULL)
       {
-        g_autoptr(GTlsDatabase) db = NULL;
-
-        db = g_tls_file_database_new (tls_ca_path, error);
-        if (db == NULL)
-          goto out;
-
-        _ostree_fetcher_set_tls_database (fetcher, db);
+        _ostree_fetcher_set_tls_database (fetcher, tls_ca_path);
       }
   }
 
@@ -2451,8 +2435,8 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
                                                        (GDestroyNotify)g_variant_unref, NULL);
   pull_data->requested_content = g_hash_table_new_full (g_str_hash, g_str_equal,
                                                         (GDestroyNotify)g_free, NULL);
-  pull_data->requested_metadata = g_hash_table_new_full (g_str_hash, g_str_equal,
-                                                         (GDestroyNotify)g_free, NULL);
+  pull_data->requested_metadata = g_hash_table_new_full (ostree_hash_object_name, g_variant_equal,
+                                                         (GDestroyNotify)g_variant_unref, NULL);
   if (dir_to_pull != NULL || dirs_to_pull != NULL)
     {
       pull_data->dirs = g_ptr_array_new_with_free_func (g_free);

src/libostree/ostree-tls-cert-interaction.c

@@ -24,6 +24,8 @@ struct _OstreeTlsCertInteraction
 {
   GTlsInteraction parent_instance;
 
+  char *cert_path;
+  char *key_path;
   GTlsCertificate *cert;
 };
 
@@ -44,6 +46,14 @@ request_certificate (GTlsInteraction              *interaction,
                      GError                      **error)
 {
   OstreeTlsCertInteraction *self = (OstreeTlsCertInteraction*)interaction;
+
+  if (!self->cert)
+    {
+      self->cert = g_tls_certificate_new_from_files (self->cert_path, self->key_path, error);
+      if (!self->cert)
+        return G_TLS_INTERACTION_FAILED;
+    }
+
   g_tls_connection_set_certificate (connection, self->cert);
   return G_TLS_INTERACTION_HANDLED;
 }
@@ -61,9 +71,11 @@ _ostree_tls_cert_interaction_class_init (OstreeTlsCertInteractionClass *klass)
 }
 
 OstreeTlsCertInteraction *
-_ostree_tls_cert_interaction_new (GTlsCertificate *cert)
+_ostree_tls_cert_interaction_new (const char *cert_path,
+                                  const char *key_path)
 {
   OstreeTlsCertInteraction *self = g_object_new (OSTREE_TYPE_TLS_CERT_INTERACTION, NULL);
-  self->cert = g_object_ref (cert);
+  self->cert_path = g_strdup (cert_path);
+  self->key_path = g_strdup (key_path);
   return self;
 }

src/libostree/ostree-tls-cert-interaction.h

@@ -34,6 +34,7 @@ typedef struct _OstreeTlsCertInteractionClass   OstreeTlsCertInteractionClass;
 
 GType                       _ostree_tls_cert_interaction_get_type    (void) G_GNUC_CONST;
 
-OstreeTlsCertInteraction *  _ostree_tls_cert_interaction_new         (GTlsCertificate *cert);
+OstreeTlsCertInteraction *  _ostree_tls_cert_interaction_new         (const char *cert_path,
+                                                                      const char *key_path);
 
 G_END_DECLS

tests/libtest.sh

@@ -262,7 +262,7 @@ setup_fake_remote_repo1() {
     mkdir ${test_tmpdir}/httpd
     cd httpd
     ln -s ${test_tmpdir}/ostree-srv ostree
-    ${OSTREE_HTTPD} --autoexit --daemonize -p ${test_tmpdir}/httpd-port $args
+    ${OSTREE_HTTPD} --autoexit --log-file $(pwd)/httpd.log --daemonize -p ${test_tmpdir}/httpd-port $args
     port=$(cat ${test_tmpdir}/httpd-port)
     echo "http://127.0.0.1:${port}" > ${test_tmpdir}/httpd-address
     cd ${oldpwd} 

tests/test-pull-c.c

@@ -49,6 +49,8 @@ test_data_init (TestData *td)
   if (!g_file_get_contents ("httpd-address", &http_address, NULL, error))
     goto out;
 
+  g_strstrip (http_address);
+
   repo_url = g_strconcat (http_address, "/ostree/gnomerepo", NULL);
 
   { g_autoptr(GVariantBuilder) builder = g_variant_builder_new (G_VARIANT_TYPE ("a{sv}"));

tests/test-pull-repeated.sh

@@ -34,7 +34,7 @@ for x in $(seq 200); do
 	echo "Success on iteration ${x}"
 	break;
     fi
-    assert_file_has_content err.txt "500.*Internal Server Error"
+    assert_file_has_content err.txt "\(500.*Internal Server Error\)\|\(HTTP 500\)"
 done
 
 ${CMD_PREFIX} ostree --repo=repo fsck

tests/test-pull-resume.sh

@@ -42,7 +42,7 @@ do
   if ${CMD_PREFIX} ostree --repo=repo pull origin main 2>err.log; then
     break
   fi
-  assert_file_has_content err.log 'error:.*Download incomplete'
+  assert_file_has_content err.log 'error:.*\(Download incomplete\)\|\(Transferred a partial file\)'
 done
 if ${CMD_PREFIX} ostree --repo=repo fsck; then
     echo "ok, pull succeeded!"