From ea7f0ba2d5b6163ec4d94804ff7084bd5fbcd8d0 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 1 Dec 2017 14:49:20 -0500 Subject: [PATCH 1/3] importer: Rework API Now that the importer *only* imports into OSTree repos, let's clean up the API so that the `OstreeRepo` and `OstreeSePolicy` are passed as constructor args. Also rework things so there's only one constructor API that steals the fd. This is prep for adding another async import API. --- src/daemon/rpmostreed-transaction-types.c | 37 +++++++--- src/libpriv/rpmostree-core.c | 8 +-- src/libpriv/rpmostree-importer.c | 83 ++++++++--------------- src/libpriv/rpmostree-importer.h | 19 ++---- tests/check/test-utils.c | 7 +- 5 files changed, 72 insertions(+), 82 deletions(-) diff --git a/src/daemon/rpmostreed-transaction-types.c b/src/daemon/rpmostreed-transaction-types.c index 7f19d55159..28a1f4ee3e 100644 --- a/src/daemon/rpmostreed-transaction-types.c +++ b/src/daemon/rpmostreed-transaction-types.c @@ -468,7 +468,7 @@ deploy_transaction_finalize (GObject *object) static gboolean import_local_rpm (OstreeRepo *repo, - int fd, + int *fd, char **sha256_nevra, GCancellable *cancellable, GError **error) @@ -482,12 +482,11 @@ import_local_rpm (OstreeRepo *repo, if (policy == NULL) return FALSE; - g_autoptr(RpmOstreeImporter) unpacker = rpmostree_importer_new_fd (fd, NULL, 0, error); + g_autoptr(RpmOstreeImporter) unpacker = rpmostree_importer_new_take_fd (fd, repo, NULL, 0, policy, error); if (unpacker == NULL) return FALSE; - if (!rpmostree_importer_run (unpacker, repo, policy, - NULL, cancellable, error)) + if (!rpmostree_importer_run (unpacker, NULL, cancellable, error)) return FALSE; g_autofree char *nevra = rpmostree_importer_get_nevra (unpacker); @@ -497,6 +496,25 @@ import_local_rpm (OstreeRepo *repo, return TRUE; } +static void +ptr_close_fd (gpointer fdp) +{ + int fd = GPOINTER_TO_INT (fdp); + glnx_close_fd (&fd); +} + +/* GUnixFDList doesn't allow stealing individual members */ +static GPtrArray * +unixfdlist_to_ptrarray (GUnixFDList *fdl) +{ + gint len; + gint *fds = g_unix_fd_list_steal_fds (fdl, &len); + GPtrArray *ret = g_ptr_array_new_with_free_func ((GDestroyNotify)ptr_close_fd); + for (int i = 0; i < len; i++) + g_ptr_array_add (ret, GINT_TO_POINTER (fds[i])); + return ret; +} + static gboolean import_many_local_rpms (OstreeRepo *repo, GUnixFDList *fdl, @@ -516,12 +534,15 @@ import_many_local_rpms (OstreeRepo *repo, g_autoptr(GPtrArray) pkgs = g_ptr_array_new_with_free_func (g_free); - gint nfds = 0; - const gint *fds = g_unix_fd_list_peek_fds (fdl, &nfds); - for (guint i = 0; i < nfds; i++) + g_autoptr(GPtrArray) fds = unixfdlist_to_ptrarray (fdl); + for (guint i = 0; i < fds->len; i++) { + /* Steal fd from the ptrarray */ + glnx_autofd int fd = GPOINTER_TO_INT (fds->pdata[i]); + fds->pdata[i] = GINT_TO_POINTER (-1); g_autofree char *sha256_nevra = NULL; - if (!import_local_rpm (repo, fds[i], &sha256_nevra, cancellable, error)) + /* Transfer fd to import */ + if (!import_local_rpm (repo, &fd, &sha256_nevra, cancellable, error)) return FALSE; g_ptr_array_add (pkgs, g_steal_pointer (&sha256_nevra)); diff --git a/src/libpriv/rpmostree-core.c b/src/libpriv/rpmostree-core.c index 560c32e770..f9e6d537c7 100644 --- a/src/libpriv/rpmostree-core.c +++ b/src/libpriv/rpmostree-core.c @@ -2055,7 +2055,9 @@ import_one_package (RpmOstreeContext *self, } /* TODO - tweak the unpacker flags for containers */ - g_autoptr(RpmOstreeImporter) unpacker = rpmostree_importer_new_fd (fd, pkg, flags, error); + OstreeRepo *ostreerepo = get_pkgcache_repo (self); + g_autoptr(RpmOstreeImporter) unpacker = + rpmostree_importer_new_take_fd (&fd, ostreerepo, pkg, flags, sepolicy, error); if (!unpacker) return FALSE; @@ -2065,10 +2067,8 @@ import_one_package (RpmOstreeContext *self, rpmostree_importer_set_jigdo_mode (unpacker, jigdo_xattr_table, jigdo_xattrs); } - OstreeRepo *ostreerepo = get_pkgcache_repo (self); g_autofree char *ostree_commit = NULL; - if (!rpmostree_importer_run (unpacker, ostreerepo, sepolicy, - &ostree_commit, cancellable, error)) + if (!rpmostree_importer_run (unpacker, &ostree_commit, cancellable, error)) return glnx_prefix_error (error, "Unpacking %s", dnf_package_get_nevra (pkg)); diff --git a/src/libpriv/rpmostree-importer.c b/src/libpriv/rpmostree-importer.c index d5735d3095..d69a700432 100644 --- a/src/libpriv/rpmostree-importer.c +++ b/src/libpriv/rpmostree-importer.c @@ -53,9 +53,10 @@ typedef GObjectClass RpmOstreeImporterClass; struct RpmOstreeImporter { GObject parent_instance; + OstreeRepo *repo; + OstreeSePolicy *sepolicy; struct archive *archive; int fd; - gboolean owns_fd; Header hdr; rpmfi fi; off_t cpio_offset; @@ -87,10 +88,11 @@ rpmostree_importer_finalize (GObject *object) archive_read_free (self->archive); if (self->fi) (void) rpmfiFree (self->fi); - if (self->owns_fd) - glnx_close_fd (&self->fd); + glnx_close_fd (&self->fd); g_string_free (self->tmpfiles_d, TRUE); g_free (self->ostree_branch); + g_clear_object (&self->repo); + g_clear_object (&self->sepolicy); g_clear_pointer (&self->rpmfi_overrides, (GDestroyNotify)g_hash_table_unref); g_clear_pointer (&self->doc_files, (GDestroyNotify)g_hash_table_unref); @@ -115,6 +117,7 @@ rpmostree_importer_class_init (RpmOstreeImporterClass *klass) static void rpmostree_importer_init (RpmOstreeImporter *self) { + self->fd = -1; self->rpmfi_overrides = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL); self->tmpfiles_d = g_string_new (""); @@ -226,10 +229,12 @@ build_rpmfi_overrides (RpmOstreeImporter *self) } /* - * rpmostree_importer_new_fd: + * rpmostree_importer_new_take_fd: * @fd: Fd + * @repo: repo * @pkg: (optional): Package reference, used for metadata * @flags: flags + * @sepolicy: (optional): SELinux policy * @error: error * * Create a new unpacker instance. The @pkg argument, if @@ -237,10 +242,12 @@ build_rpmfi_overrides (RpmOstreeImporter *self) * origin repo will be added to the final commit. */ RpmOstreeImporter * -rpmostree_importer_new_fd (int fd, - DnfPackage *pkg, - RpmOstreeImporterFlags flags, - GError **error) +rpmostree_importer_new_take_fd (int *fd, + OstreeRepo *repo, + DnfPackage *pkg, + RpmOstreeImporterFlags flags, + OstreeSePolicy *sepolicy, + GError **error) { RpmOstreeImporter *ret = NULL; g_auto(Header) hdr = NULL; @@ -248,15 +255,17 @@ rpmostree_importer_new_fd (int fd, struct archive *archive; gsize cpio_offset; - archive = rpmostree_unpack_rpm2cpio (fd, error); + archive = rpmostree_unpack_rpm2cpio (*fd, error); if (archive == NULL) goto out; - if (!rpmostree_importer_read_metainfo (fd, &hdr, &cpio_offset, &fi, error)) + if (!rpmostree_importer_read_metainfo (*fd, &hdr, &cpio_offset, &fi, error)) goto out; ret = g_object_new (RPMOSTREE_TYPE_IMPORTER, NULL); - ret->fd = fd; + ret->fd = glnx_steal_fd (fd); + ret->repo = g_object_ref (repo); + ret->sepolicy = sepolicy ? g_object_ref (sepolicy) : NULL; ret->fi = g_steal_pointer (&fi); ret->archive = g_steal_pointer (&archive); ret->flags = flags; @@ -278,38 +287,6 @@ rpmostree_importer_new_fd (int fd, return ret; } -/* - * rpmostree_importer_new_at: - * @dfd: Fd - * @path: Path - * @pkg: (optional): Package reference, used for metadata - * @flags: flags - * @error: error - * - * Create a new unpacker instance. The @pkg argument, if - * specified, will be inspected and metadata such as the - * origin repo will be added to the final commit. - */ -RpmOstreeImporter * -rpmostree_importer_new_at (int dfd, const char *path, - DnfPackage *pkg, - RpmOstreeImporterFlags flags, - GError **error) -{ - glnx_autofd int fd = -1; - if (!glnx_openat_rdonly (dfd, path, TRUE, &fd, error)) - return FALSE; - - RpmOstreeImporter *ret = rpmostree_importer_new_fd (fd, pkg, flags, error); - if (ret == NULL) - return NULL; - - ret->owns_fd = TRUE; - fd = -1; - - return g_steal_pointer (&ret); -} - void rpmostree_importer_set_jigdo_mode (RpmOstreeImporter *self, GVariant *xattr_table, @@ -410,7 +387,6 @@ repo_metadata_for_package (DnfRepo *repo) static gboolean build_metadata_variant (RpmOstreeImporter *self, - OstreeSePolicy *sepolicy, GVariant **out_variant, GCancellable *cancellable, GError **error) @@ -460,10 +436,10 @@ build_metadata_variant (RpmOstreeImporter *self, /* The current sepolicy that was used to label the unpacked files is important * to record. It will help us during future overlays to determine whether the * files should be relabeled. */ - if (sepolicy) + if (self->sepolicy) g_variant_builder_add (&metadata_builder, "{sv}", "rpmostree.sepolicy", g_variant_new_string - (ostree_sepolicy_get_csum (sepolicy))); + (ostree_sepolicy_get_csum (self->sepolicy))); /* let's be nice to our future selves just in case */ g_variant_builder_add (&metadata_builder, "{sv}", "rpmostree.unpack_version", @@ -788,12 +764,11 @@ handle_translate_pathname (OstreeRepo *repo, static gboolean import_rpm_to_repo (RpmOstreeImporter *self, - OstreeRepo *repo, - OstreeSePolicy *sepolicy, char **out_csum, GCancellable *cancellable, GError **error) { + OstreeRepo *repo = self->repo; /* Passed to the commit modifier */ GError *cb_error = NULL; cb_data fdata = { self, &cb_error }; @@ -821,12 +796,12 @@ import_rpm_to_repo (RpmOstreeImporter *self, { ostree_repo_commit_modifier_set_xattr_callback (modifier, jigdo_xattr_cb, NULL, self); - g_assert (sepolicy == NULL); + g_assert (self->sepolicy == NULL); } else { ostree_repo_commit_modifier_set_xattr_callback (modifier, xattr_cb, NULL, self); - ostree_repo_commit_modifier_set_sepolicy (modifier, sepolicy); + ostree_repo_commit_modifier_set_sepolicy (modifier, self->sepolicy); } OstreeRepoImportArchiveOptions opts = { 0 }; @@ -883,7 +858,7 @@ import_rpm_to_repo (RpmOstreeImporter *self, return FALSE; g_autoptr(GVariant) metadata = NULL; - if (!build_metadata_variant (self, sepolicy, &metadata, cancellable, error)) + if (!build_metadata_variant (self, &metadata, cancellable, error)) return FALSE; g_variant_ref_sink (metadata); @@ -903,18 +878,16 @@ import_rpm_to_repo (RpmOstreeImporter *self, gboolean rpmostree_importer_run (RpmOstreeImporter *self, - OstreeRepo *repo, - OstreeSePolicy *sepolicy, char **out_csum, GCancellable *cancellable, GError **error) { g_autofree char *csum = NULL; - if (!import_rpm_to_repo (self, repo, sepolicy, &csum, cancellable, error)) + if (!import_rpm_to_repo (self, &csum, cancellable, error)) return FALSE; const char *branch = rpmostree_importer_get_ostree_branch (self); - ostree_repo_transaction_set_ref (repo, NULL, branch, csum); + ostree_repo_transaction_set_ref (self->repo, NULL, branch, csum); if (out_csum) *out_csum = g_steal_pointer (&csum); diff --git a/src/libpriv/rpmostree-importer.h b/src/libpriv/rpmostree-importer.h index b94a4adc6f..65af301dfb 100644 --- a/src/libpriv/rpmostree-importer.h +++ b/src/libpriv/rpmostree-importer.h @@ -47,17 +47,12 @@ typedef enum { } RpmOstreeImporterFlags; RpmOstreeImporter* -rpmostree_importer_new_fd (int fd, - DnfPackage *pkg, - RpmOstreeImporterFlags flags, - GError **error); - -RpmOstreeImporter* -rpmostree_importer_new_at (int dfd, - const char *path, - DnfPackage *pkg, /* for metadata */ - RpmOstreeImporterFlags flags, - GError **error); +rpmostree_importer_new_take_fd (int *fd, + OstreeRepo *repo, + DnfPackage *pkg, + RpmOstreeImporterFlags flags, + OstreeSePolicy *sepolicy, + GError **error); void rpmostree_importer_set_jigdo_mode (RpmOstreeImporter *self, GVariant *xattr_table, @@ -75,8 +70,6 @@ rpmostree_importer_get_ostree_branch (RpmOstreeImporter *unpacker); gboolean rpmostree_importer_run (RpmOstreeImporter *unpacker, - OstreeRepo *repo, - OstreeSePolicy *sepolicy, char **out_commit, GCancellable *cancellable, GError **error); diff --git a/tests/check/test-utils.c b/tests/check/test-utils.c index bdbf7e42e8..8b8164d669 100644 --- a/tests/check/test-utils.c +++ b/tests/check/test-utils.c @@ -142,11 +142,14 @@ test_variant_to_nevra(void) g_autoptr(RpmOstreeImporter) importer = NULL; g_autofree char *foo_rpm = g_strdup_printf ("yumrepo/packages/%s/%s.rpm", arch, nevra); - importer = rpmostree_importer_new_at (AT_FDCWD, foo_rpm, NULL, 0, &error); + glnx_autofd int foo_fd = -1; + glnx_openat_rdonly (AT_FDCWD, foo_rpm, TRUE, &foo_fd, &error); + g_assert_no_error (error); + importer = rpmostree_importer_new_take_fd (&foo_fd, repo, NULL, 0, NULL, &error); g_assert_no_error (error); g_assert (importer); - ret = rpmostree_importer_run (importer, repo, NULL, NULL, NULL, &error); + ret = rpmostree_importer_run (importer, NULL, NULL, &error); g_assert_no_error (error); g_assert (ret); From 2b979f65c759fa5e0bb6d0f5d6c4cb769c0b3b5a Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 1 Dec 2017 16:27:07 -0500 Subject: [PATCH 2/3] core: Implement parallel importing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Depends: https://github.com/ostreedev/ostree/pull/1358 For jigdo ♲📦 in order to get true image speed like libostree has we need to interleave and parallelize downloading and importing. The messy part about this is having sync API do the "invoke and wait on various async tasks" pattern. It's the same thing in `ostree_repo_pull_with_options()`. Importing is pretty dramatically faster with this, I can only imagine the speed win if we actually interleaved with downloads. However doing that requires libdnf/librepo work. --- src/libpriv/rpmostree-core.c | 136 ++++++++++++++++++------------- src/libpriv/rpmostree-importer.c | 35 ++++++++ src/libpriv/rpmostree-importer.h | 11 +++ 3 files changed, 124 insertions(+), 58 deletions(-) diff --git a/src/libpriv/rpmostree-core.c b/src/libpriv/rpmostree-core.c index f9e6d537c7..10f364030c 100644 --- a/src/libpriv/rpmostree-core.c +++ b/src/libpriv/rpmostree-core.c @@ -252,8 +252,13 @@ struct _RpmOstreeContext { OstreeSePolicy *sepolicy; char *passwd_dir; + gboolean async_running; + gboolean async_error_set; + GError **async_error; + DnfState *async_dnfstate; GPtrArray *pkgs_to_download; GPtrArray *pkgs_to_import; + guint n_async_pkgs_imported; GPtrArray *pkgs_to_relabel; GHashTable *pkgs_to_remove; /* pkgname --> gv_nevra */ @@ -1374,6 +1379,7 @@ sort_packages (RpmOstreeContext *self, self->pkgs_to_download = g_ptr_array_new_with_free_func ((GDestroyNotify)g_object_unref); g_assert (!self->pkgs_to_import); self->pkgs_to_import = g_ptr_array_new_with_free_func ((GDestroyNotify)g_object_unref); + self->n_async_pkgs_imported = 0; g_assert (!self->pkgs_to_relabel); self->pkgs_to_relabel = g_ptr_array_new_with_free_func ((GDestroyNotify)g_object_unref); @@ -1821,6 +1827,7 @@ rpmostree_context_set_packages (RpmOstreeContext *self, { g_clear_pointer (&self->pkgs_to_download, (GDestroyNotify)g_ptr_array_unref); g_clear_pointer (&self->pkgs_to_import, (GDestroyNotify)g_ptr_array_unref); + self->n_async_pkgs_imported = 0; g_clear_pointer (&self->pkgs_to_relabel, (GDestroyNotify)g_ptr_array_unref); return sort_packages (self, packages, cancellable, error); } @@ -2023,64 +2030,35 @@ rpmostree_context_download (RpmOstreeContext *self, return TRUE; } -static gboolean -import_one_package (RpmOstreeContext *self, - DnfContext *dnfctx, - DnfPackage *pkg, - OstreeSePolicy *sepolicy, - GVariant *jigdo_xattr_table, - GVariant *jigdo_xattrs, - GCancellable *cancellable, - GError **error) +static inline void +dnf_state_assert_done (DnfState *dnfstate) { - glnx_fd_close int fd = -1; - if (!rpmostree_context_consume_package (self, pkg, &fd, error)) - return FALSE; - - /* Only set SKIP_EXTRANEOUS for packages we know need it, so that - * people doing custom composes don't have files silently discarded. - * (This will also likely need to be configurable). - */ - const char *pkg_name = dnf_package_get_name (pkg); - - int flags = 0; - if (g_str_equal (pkg_name, "filesystem") || - g_str_equal (pkg_name, "rootfiles")) - flags |= RPMOSTREE_IMPORTER_FLAGS_SKIP_EXTRANEOUS; - - { gboolean docs; - g_assert (g_variant_dict_lookup (self->spec->dict, "documentation", "b", &docs)); - if (!docs) - flags |= RPMOSTREE_IMPORTER_FLAGS_NODOCS; - } - - /* TODO - tweak the unpacker flags for containers */ - OstreeRepo *ostreerepo = get_pkgcache_repo (self); - g_autoptr(RpmOstreeImporter) unpacker = - rpmostree_importer_new_take_fd (&fd, ostreerepo, pkg, flags, sepolicy, error); - if (!unpacker) - return FALSE; - - if (jigdo_xattrs) - { - g_assert (!sepolicy); - rpmostree_importer_set_jigdo_mode (unpacker, jigdo_xattr_table, jigdo_xattrs); - } - - g_autofree char *ostree_commit = NULL; - if (!rpmostree_importer_run (unpacker, &ostree_commit, cancellable, error)) - return glnx_prefix_error (error, "Unpacking %s", - dnf_package_get_nevra (pkg)); - - return TRUE; + g_assert (dnf_state_done (dnfstate, NULL)); } -static inline void -dnf_state_assert_done (DnfState *hifstate) +static void +on_async_import_done (GObject *obj, + GAsyncResult *res, + gpointer user_data) { - gboolean r; - r = dnf_state_done (hifstate, NULL); - g_assert (r); + RpmOstreeImporter *importer = (RpmOstreeImporter*) obj; + RpmOstreeContext *self = user_data; + g_autofree char *rev = + rpmostree_importer_run_async_finish (importer, res, + self->async_error_set ? NULL : self->async_error); + if (!rev) + { + self->async_running = FALSE; + self->async_error_set = TRUE; + } + else + { + g_assert_cmpint (self->n_async_pkgs_imported, <, self->pkgs_to_import->len); + self->n_async_pkgs_imported++; + dnf_state_assert_done (self->async_dnfstate); + if (self->n_async_pkgs_imported == self->pkgs_to_import->len) + self->async_running = FALSE; + } } gboolean @@ -2114,6 +2092,9 @@ rpmostree_context_import_jigdo (RpmOstreeContext *self, G_CALLBACK (on_hifstate_percentage_changed), "Importing:"); + self->async_dnfstate = hifstate; + self->async_running = TRUE; + for (guint i = 0; i < self->pkgs_to_import->len; i++) { DnfPackage *pkg = self->pkgs_to_import->pdata[i]; @@ -2124,13 +2105,52 @@ rpmostree_context_import_jigdo (RpmOstreeContext *self, if (!jigdo_xattrs) g_error ("Failed to find jigdo xattrs for %s", dnf_package_get_nevra (pkg)); } - if (!import_one_package (self, dnfctx, pkg, self->sepolicy, - jigdo_xattr_table, jigdo_xattrs, - cancellable, error)) + + glnx_fd_close int fd = -1; + if (!rpmostree_context_consume_package (self, pkg, &fd, error)) return FALSE; - dnf_state_assert_done (hifstate); + + /* Only set SKIP_EXTRANEOUS for packages we know need it, so that + * people doing custom composes don't have files silently discarded. + * (This will also likely need to be configurable). + */ + const char *pkg_name = dnf_package_get_name (pkg); + + int flags = 0; + if (g_str_equal (pkg_name, "filesystem") || + g_str_equal (pkg_name, "rootfiles")) + flags |= RPMOSTREE_IMPORTER_FLAGS_SKIP_EXTRANEOUS; + + { gboolean docs; + g_assert (g_variant_dict_lookup (self->spec->dict, "documentation", "b", &docs)); + if (!docs) + flags |= RPMOSTREE_IMPORTER_FLAGS_NODOCS; + } + + /* TODO - tweak the unpacker flags for containers */ + OstreeRepo *ostreerepo = get_pkgcache_repo (self); + g_autoptr(RpmOstreeImporter) unpacker = + rpmostree_importer_new_take_fd (&fd, ostreerepo, pkg, flags, + self->sepolicy, error); + if (!unpacker) + return FALSE; + + if (jigdo_xattrs) + { + g_assert (!self->sepolicy); + rpmostree_importer_set_jigdo_mode (unpacker, jigdo_xattr_table, jigdo_xattrs); + } + + rpmostree_importer_run_async (unpacker, cancellable, on_async_import_done, self); } + /* Wait for all of the imports to complete */ + GMainContext *mainctx = g_main_context_get_thread_default (); + while (self->async_running) + g_main_context_iteration (mainctx, TRUE); + if (self->async_error) + return FALSE; + g_signal_handler_disconnect (hifstate, progress_sigid); rpmostree_output_percent_progress_end (); } diff --git a/src/libpriv/rpmostree-importer.c b/src/libpriv/rpmostree-importer.c index d69a700432..56b9a420bd 100644 --- a/src/libpriv/rpmostree-importer.c +++ b/src/libpriv/rpmostree-importer.c @@ -894,6 +894,41 @@ rpmostree_importer_run (RpmOstreeImporter *self, return TRUE; } +static void +import_in_thread (GTask *task, + gpointer source, + gpointer task_data, + GCancellable *cancellable) +{ + GError *local_error = NULL; + RpmOstreeImporter *self = source; + g_autofree char *rev = NULL; + + if (!rpmostree_importer_run (self, &rev, cancellable, &local_error)) + g_task_return_error (task, local_error); + else + g_task_return_pointer (task, g_steal_pointer (&rev), g_free); +} + +void +rpmostree_importer_run_async (RpmOstreeImporter *self, + GCancellable *cancellable, + GAsyncReadyCallback callback, + gpointer user_data) +{ + g_autoptr(GTask) task = g_task_new (self, cancellable, callback, user_data); + g_task_run_in_thread (task, import_in_thread); +} + +char * +rpmostree_importer_run_async_finish (RpmOstreeImporter *self, + GAsyncResult *result, + GError **error) +{ + g_return_val_if_fail (g_task_is_valid (result, self), FALSE); + return g_task_propagate_pointer ((GTask*)result, error); +} + char * rpmostree_importer_get_nevra (RpmOstreeImporter *self) { diff --git a/src/libpriv/rpmostree-importer.h b/src/libpriv/rpmostree-importer.h index 65af301dfb..66f281f4a7 100644 --- a/src/libpriv/rpmostree-importer.h +++ b/src/libpriv/rpmostree-importer.h @@ -74,6 +74,17 @@ rpmostree_importer_run (RpmOstreeImporter *unpacker, GCancellable *cancellable, GError **error); +void +rpmostree_importer_run_async (RpmOstreeImporter *unpacker, + GCancellable *cancellable, + GAsyncReadyCallback callback, + gpointer user_data); + +char * +rpmostree_importer_run_async_finish (RpmOstreeImporter *self, + GAsyncResult *res, + GError **error); + char * rpmostree_importer_get_nevra (RpmOstreeImporter *self); From d7a3f65999bcde95a06b12ae833e28d73cef58a7 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 5 Dec 2017 12:40:49 -0500 Subject: [PATCH 3/3] fixup! core: Implement parallel importing --- src/libpriv/rpmostree-core.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/libpriv/rpmostree-core.c b/src/libpriv/rpmostree-core.c index 10f364030c..3d23a4fdef 100644 --- a/src/libpriv/rpmostree-core.c +++ b/src/libpriv/rpmostree-core.c @@ -253,8 +253,7 @@ struct _RpmOstreeContext { char *passwd_dir; gboolean async_running; - gboolean async_error_set; - GError **async_error; + GError *async_error; DnfState *async_dnfstate; GPtrArray *pkgs_to_download; GPtrArray *pkgs_to_import; @@ -2045,11 +2044,11 @@ on_async_import_done (GObject *obj, RpmOstreeContext *self = user_data; g_autofree char *rev = rpmostree_importer_run_async_finish (importer, res, - self->async_error_set ? NULL : self->async_error); + self->async_error ? NULL : &self->async_error); if (!rev) { self->async_running = FALSE; - self->async_error_set = TRUE; + g_assert (self->async_error != NULL); } else { @@ -2146,10 +2145,14 @@ rpmostree_context_import_jigdo (RpmOstreeContext *self, /* Wait for all of the imports to complete */ GMainContext *mainctx = g_main_context_get_thread_default (); + self->async_error = NULL; while (self->async_running) g_main_context_iteration (mainctx, TRUE); if (self->async_error) - return FALSE; + { + g_propagate_error (error, g_steal_pointer (&self->async_error)); + return FALSE; + } g_signal_handler_disconnect (hifstate, progress_sigid); rpmostree_output_percent_progress_end ();