From aec1f6c0ccc3c98ec1630e64e22e82f62e580d97 Mon Sep 17 00:00:00 2001 From: Philip Withnall Date: Mon, 27 Mar 2017 20:45:16 +0100 Subject: [PATCH] libostree: Add support for finding remote URIs by ref name WIP work to add an OstreeRepoFinder interface and implementations which find remote URIs by ref names, which we consider to be globally unique. This is an API preview. It compiles, but has not been tested beyond that, and the implementation is unfinished in various places, and unpolished everywhere else. It is intended as a strawman for the proposed API. The new API is used in a new ostree_repo_find_updates() function, which resolves a list of ref names to update into a set of remote URIs to pull them from, which can be treated as mirrors. A major feature this adds is the ability to automatically pull from other machines on the local network (as long as they are running an OSTree server and appropriate Avahi DNS-SD adverts, support for which will be added separately). Similarly, it adds the ability to pull from removable file systems without needing further configuration. This bumps the GLib dependency to 2.50, and adds an optional dependency on Avahi. Signed-off-by: Philip Withnall --- Makefile-libostree.am | 20 + Makefile-tests.am | 27 +- Makefile.am | 2 +- apidoc/ostree-sections.txt | 43 + configure.ac | 27 +- src/libostree/libostree.sym | 19 + src/libostree/ostree-bloom-private.h | 78 ++ src/libostree/ostree-bloom.c | 502 +++++++++ src/libostree/ostree-fetcher.h | 2 + .../ostree-repo-finder-avahi-private.h | 39 + src/libostree/ostree-repo-finder-avahi.c | 998 ++++++++++++++++++ src/libostree/ostree-repo-finder-avahi.h | 50 + src/libostree/ostree-repo-finder-config.c | 295 ++++++ src/libostree/ostree-repo-finder-config.h | 43 + src/libostree/ostree-repo-finder-mount.c | 474 +++++++++ src/libostree/ostree-repo-finder-mount.h | 43 + src/libostree/ostree-repo-finder.c | 337 ++++++ src/libostree/ostree-repo-finder.h | 116 ++ src/libostree/ostree-repo-pull.c | 603 ++++++++++- src/libostree/ostree-repo.h | 17 + tests/.gitignore | 7 + tests/test-bloom.c | 148 +++ tests/test-mock-gio.c | 291 +++++ tests/test-mock-gio.h | 68 ++ tests/test-repo-finder-avahi.c | 226 ++++ tests/test-repo-finder-config.c | 271 +++++ tests/test-repo-finder-mount.c | 371 +++++++ 27 files changed, 5112 insertions(+), 5 deletions(-) create mode 100644 src/libostree/ostree-bloom-private.h create mode 100644 src/libostree/ostree-bloom.c create mode 100644 src/libostree/ostree-repo-finder-avahi-private.h create mode 100644 src/libostree/ostree-repo-finder-avahi.c create mode 100644 src/libostree/ostree-repo-finder-avahi.h create mode 100644 src/libostree/ostree-repo-finder-config.c create mode 100644 src/libostree/ostree-repo-finder-config.h create mode 100644 src/libostree/ostree-repo-finder-mount.c create mode 100644 src/libostree/ostree-repo-finder-mount.h create mode 100644 src/libostree/ostree-repo-finder.c create mode 100644 src/libostree/ostree-repo-finder.h create mode 100644 tests/test-bloom.c create mode 100644 tests/test-mock-gio.c create mode 100644 tests/test-mock-gio.h create mode 100644 tests/test-repo-finder-avahi.c create mode 100644 tests/test-repo-finder-config.c create mode 100644 tests/test-repo-finder-mount.c diff --git a/Makefile-libostree.am b/Makefile-libostree.am index 28126487da..f679234a7e 100644 --- a/Makefile-libostree.am +++ b/Makefile-libostree.am @@ -71,6 +71,8 @@ CLEANFILES += $(BUILT_SOURCES) libostree_1_la_SOURCES = \ src/libostree/ostree-async-progress.c \ + src/libostree/ostree-bloom.c \ + src/libostree/ostree-bloom-private.h \ src/libostree/ostree-cmdprivate.h \ src/libostree/ostree-cmdprivate.c \ src/libostree/ostree-core-private.h \ @@ -106,6 +108,12 @@ libostree_1_la_SOURCES = \ src/libostree/ostree-repo-file.c \ src/libostree/ostree-repo-file-enumerator.c \ src/libostree/ostree-repo-file-enumerator.h \ + src/libostree/ostree-repo-finder.c \ + src/libostree/ostree-repo-finder.h \ + src/libostree/ostree-repo-finder-config.c \ + src/libostree/ostree-repo-finder-config.h \ + src/libostree/ostree-repo-finder-mount.c \ + src/libostree/ostree-repo-finder-mount.h \ src/libostree/ostree-sepolicy.c \ src/libostree/ostree-sysroot-private.h \ src/libostree/ostree-sysroot.c \ @@ -145,6 +153,13 @@ libostree_1_la_SOURCES += \ src/libostree/ostree-tls-cert-interaction.h \ $(NULL) endif +if USE_AVAHI +libostree_1_la_SOURCES += \ + src/libostree/ostree-repo-finder-avahi.c \ + src/libostree/ostree-repo-finder-avahi.h \ + src/libostree/ostree-repo-finder-avahi-private.h \ + $(NULL) +endif libostree_1_la_CFLAGS = $(AM_CFLAGS) -I$(srcdir)/bsdiff -I$(srcdir)/libglnx -I$(srcdir)/src/libotutil -I$(srcdir)/src/libostree -I$(builddir)/src/libostree \ $(OT_INTERNAL_GIO_UNIX_CFLAGS) $(OT_INTERNAL_GPGME_CFLAGS) $(OT_DEP_LZMA_CFLAGS) $(OT_DEP_ZLIB_CFLAGS) $(OT_DEP_OPENSSL_CFLAGS) \ @@ -162,6 +177,11 @@ libostree_1_la_CFLAGS += $(OT_DEP_LIBARCHIVE_CFLAGS) libostree_1_la_LIBADD += $(OT_DEP_LIBARCHIVE_LIBS) endif +if USE_AVAHI +libostree_1_la_CFLAGS += $(OT_DEP_AVAHI_CFLAGS) +libostree_1_la_LIBADD += $(OT_DEP_AVAHI_LIBS) +endif + if BUILDOPT_LIBSYSTEMD libostree_1_la_CFLAGS += $(LIBSYSTEMD_CFLAGS) libostree_1_la_LIBADD += $(LIBSYSTEMD_LIBS) diff --git a/Makefile-tests.am b/Makefile-tests.am index fd755ee15a..1932f5b9b5 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -179,7 +179,12 @@ endif test_programs = tests/test-varint tests/test-ot-unix-utils tests/test-bsdiff tests/test-mutable-tree \ tests/test-keyfile-utils tests/test-ot-opt-utils tests/test-ot-tool-util \ tests/test-gpg-verify-result tests/test-checksum tests/test-lzma tests/test-rollsum \ - tests/test-basic-c tests/test-sysroot-c tests/test-pull-c + tests/test-basic-c tests/test-sysroot-c tests/test-pull-c tests/test-bloom \ + tests/test-repo-finder-config tests/test-repo-finder-mount + +if USE_AVAHI +test_programs += tests/test-repo-finder-avahi +endif # An interactive tool noinst_PROGRAMS += tests/test-rollsum-cli @@ -192,7 +197,7 @@ common_tests_cflags = $(ostree_bin_shared_cflags) $(OT_INTERNAL_GIO_UNIX_CFLAGS) common_tests_ldadd = $(ostree_bin_shared_ldadd) $(OT_INTERNAL_GIO_UNIX_LIBS) noinst_LTLIBRARIES += libostreetest.la -libostreetest_la_SOURCES = tests/libostreetest.c +libostreetest_la_SOURCES = tests/libostreetest.c tests/test-mock-gio.c tests/test-mock-gio.h libostreetest_la_CFLAGS = $(common_tests_cflags) -I $(srcdir)/tests libostreetest_la_LIBADD = $(common_tests_ldadd) @@ -207,6 +212,24 @@ tests_test_rollsum_SOURCES = src/libostree/ostree-rollsum.c tests/test-rollsum.c tests_test_rollsum_CFLAGS = $(TESTS_CFLAGS) $(OT_DEP_ZLIB_CFLAGS) tests_test_rollsum_LDADD = $(bupsplitpath) $(TESTS_LDADD) $(OT_DEP_ZLIB_LIBS) +tests_test_bloom_SOURCES = src/libostree/ostree-bloom.c tests/test-bloom.c +tests_test_bloom_CFLAGS = $(TESTS_CFLAGS) +tests_test_bloom_LDADD = $(TESTS_LDADD) + +if USE_AVAHI +tests_test_repo_finder_avahi_SOURCES = src/libostree/ostree-bloom.c src/libostree/ostree-repo-finder-avahi.c tests/test-repo-finder-avahi.c +tests_test_repo_finder_avahi_CFLAGS = $(TESTS_CFLAGS) $(OT_INTERNAL_SOUP_CFLAGS) +tests_test_repo_finder_avahi_LDADD = $(TESTS_LDADD) $(OT_INTERNAL_SOUP_LIBS) +endif + +tests_test_repo_finder_config_SOURCES = tests/test-repo-finder-config.c +tests_test_repo_finder_config_CFLAGS = $(TESTS_CFLAGS) +tests_test_repo_finder_config_LDADD = $(TESTS_LDADD) + +tests_test_repo_finder_mount_SOURCES = tests/test-repo-finder-mount.c +tests_test_repo_finder_mount_CFLAGS = $(TESTS_CFLAGS) +tests_test_repo_finder_mount_LDADD = $(TESTS_LDADD) + tests_test_mutable_tree_CFLAGS = $(TESTS_CFLAGS) tests_test_mutable_tree_LDADD = $(TESTS_LDADD) diff --git a/Makefile.am b/Makefile.am index cc0e76f5fe..f2aee5d73d 100644 --- a/Makefile.am +++ b/Makefile.am @@ -29,7 +29,7 @@ AM_CPPFLAGS += -DDATADIR='"$(datadir)"' -DLIBEXECDIR='"$(libexecdir)"' \ -DOSTREE_COMPILATION \ -DG_LOG_DOMAIN=\"OSTree\" \ -DOSTREE_GITREV='"$(OSTREE_GITREV)"' \ - -DGLIB_VERSION_MIN_REQUIRED=GLIB_VERSION_2_40 -DGLIB_VERSION_MAX_ALLOWED=GLIB_VERSION_2_40 \ + -DGLIB_VERSION_MIN_REQUIRED=GLIB_VERSION_2_50 -DGLIB_VERSION_MAX_ALLOWED=GLIB_VERSION_2_50 \ -DSOUP_VERSION_MIN_REQUIRED=SOUP_VERSION_2_40 -DSOUP_VERSION_MAX_ALLOWED=SOUP_VERSION_2_48 AM_CFLAGS += -std=gnu99 $(WARN_CFLAGS) AM_DISTCHECK_CONFIGURE_FLAGS += \ diff --git a/apidoc/ostree-sections.txt b/apidoc/ostree-sections.txt index 027f25c5c0..f47c8ccbad 100644 --- a/apidoc/ostree-sections.txt +++ b/apidoc/ostree-sections.txt @@ -374,9 +374,11 @@ ostree_repo_prune ostree_repo_prune_static_deltas ostree_repo_prune_from_reachable OstreeRepoPullFlags +ostree_repo_find_remotes ostree_repo_pull ostree_repo_pull_one_dir ostree_repo_pull_with_options +ostree_repo_pull_from_remotes ostree_repo_pull_default_console_progress_changed ostree_repo_sign_commit ostree_repo_append_gpg_signature @@ -422,6 +424,47 @@ OstreeRepoFileClass ostree_repo_file_get_type +
+ostree-repo-finder +OstreeRepoFinder +ostree_repo_finder_resolve_async +ostree_repo_finder_resolve_finish +ostree_repo_finder_resolve_all_async +ostree_repo_finder_resolve_all_finish +OstreeRepoFinderResult +ostree_repo_finder_result_new +ostree_repo_finder_result_free +ostree_repo_finder_result_compare + +ostree_repo_finder_get_type +
+ +
+ostree-repo-finder-avahi +OstreeRepoFinderAvahi +ostree_repo_finder_avahi_new +ostree_repo_finder_avahi_start +ostree_repo_finder_avahi_stop + +ostree_repo_finder_avahi_get_type +
+ +
+ostree-repo-finder-config +OstreeRepoFinderConfig +ostree_repo_finder_config_new + +ostree_repo_finder_config_get_type +
+ +
+ostree-repo-finder-mount +OstreeRepoFinderMount +ostree_repo_finder_mount_new + +ostree_repo_finder_mount_get_type +
+
ostree-sepolicy OstreeSePolicy diff --git a/configure.ac b/configure.ac index d43536e382..899d9813b9 100644 --- a/configure.ac +++ b/configure.ac @@ -75,7 +75,7 @@ AM_PATH_GLIB_2_0(,,AC_MSG_ERROR([GLib not found])) dnl When bumping the gio-unix-2.0 dependency (or glib-2.0 in general), dnl remember to bump GLIB_VERSION_MIN_REQUIRED and dnl GLIB_VERSION_MAX_ALLOWED in Makefile.am -GIO_DEPENDENCY="gio-unix-2.0 >= 2.40.0" +GIO_DEPENDENCY="gio-unix-2.0 >= 2.50.0" PKG_CHECK_MODULES(OT_DEP_GIO_UNIX, $GIO_DEPENDENCY) dnl 5.1.0 is an arbitrary version here @@ -315,6 +315,31 @@ if test x$with_openssl != xno; then OSTREE_FEATURES="$OSTREE_FEATURES openssl"; AM_CONDITIONAL(USE_OPENSSL, test $with_openssl != no) dnl end openssl +dnl Avahi dependency for finding repos +AVAHI_DEPENDENCY="avahi-client >= 0.6.31 avahi-glib >= 0.6.31" + +AC_ARG_WITH(avahi, + AS_HELP_STRING([--without-avahi], [Do not use Avahi]), + :, with_avahi=maybe) + +AS_IF([ test x$with_avahi != xno ], [ + AC_MSG_CHECKING([for $AVAHI_DEPENDENCY]) + PKG_CHECK_EXISTS($AVAHI_DEPENDENCY, have_avahi=yes, have_avahi=no) + AC_MSG_RESULT([$have_avahi]) + AS_IF([ test x$have_avahi = xno && test x$with_avahi != xmaybe ], [ + AC_MSG_ERROR([Avahi is enabled but could not be found]) + ]) + AS_IF([ test x$have_avahi = xyes], [ + AC_DEFINE([HAVE_AVAHI], 1, [Define if we have avahi-client.pc and avahi-glib.pc]) + PKG_CHECK_MODULES(OT_DEP_AVAHI, $AVAHI_DEPENDENCY) + with_avahi=yes + ], [ + with_avahi=no + ]) +], [ with_avahi=no ]) +if test x$with_avahi != xno; then OSTREE_FEATURES="$OSTREE_FEATURES avahi"; fi +AM_CONDITIONAL(USE_AVAHI, test $with_avahi != no) + dnl This is what is in RHEL7.2 right now, picking it arbitrarily LIBMOUNT_DEPENDENCY="mount >= 2.23.0" diff --git a/src/libostree/libostree.sym b/src/libostree/libostree.sym index 649c6f1fc7..153e72ca3b 100644 --- a/src/libostree/libostree.sym +++ b/src/libostree/libostree.sym @@ -390,6 +390,25 @@ LIBOSTREE_2017.4 { global: ostree_check_version; ostree_diff_dirs_with_options; + ostree_repo_find_remotes; + /* TODO: What if we compile without Avahi? */ + ostree_repo_finder_avahi_get_type; + ostree_repo_finder_avahi_new; + ostree_repo_finder_avahi_start; + ostree_repo_finder_avahi_stop; + ostree_repo_finder_config_get_type; + ostree_repo_finder_config_new; + ostree_repo_finder_get_type; + ostree_repo_finder_mount_get_type; + ostree_repo_finder_mount_new; + ostree_repo_finder_resolve_async; + ostree_repo_finder_resolve_all_async; + ostree_repo_finder_resolve_all_finish; + ostree_repo_finder_resolve_finish; + ostree_repo_finder_result_compare; + ostree_repo_finder_result_free; + ostree_repo_finder_result_new; + ostree_repo_pull_from_remotes; } LIBOSTREE_2017.3; /* Stub section for the stable release *after* this development one; don't diff --git a/src/libostree/ostree-bloom-private.h b/src/libostree/ostree-bloom-private.h new file mode 100644 index 0000000000..4758e7a2cc --- /dev/null +++ b/src/libostree/ostree-bloom-private.h @@ -0,0 +1,78 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright © 2017 Endless Mobile, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + * Authors: + * - Philip Withnall + */ + +#pragma once + +#include +#include +#include + +G_BEGIN_DECLS + +/* TODO: Docs */ +typedef struct _OstreeBloom OstreeBloom; + +/* TODO: Docs */ +typedef guint64 (*OstreeBloomHashFunc) (gconstpointer element, + guint8 k); + +#define OSTREE_TYPE_BLOOM (ostree_bloom_get_type ()) + +G_GNUC_INTERNAL +GType ostree_bloom_get_type (void); + +G_GNUC_INTERNAL +OstreeBloom *ostree_bloom_new (gsize n_bytes, + guint8 k, + OstreeBloomHashFunc hash); + +G_GNUC_INTERNAL +OstreeBloom *ostree_bloom_new_from_bytes (GBytes *bytes, + guint8 k, + OstreeBloomHashFunc hash); + +G_GNUC_INTERNAL +OstreeBloom *ostree_bloom_ref (OstreeBloom *bloom); +G_GNUC_INTERNAL +void ostree_bloom_unref (OstreeBloom *bloom); + +G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeBloom, ostree_bloom_unref) + +G_GNUC_INTERNAL +gboolean ostree_bloom_maybe_contains (OstreeBloom *bloom, + gconstpointer element); + +G_GNUC_INTERNAL +GBytes *ostree_bloom_seal (OstreeBloom *bloom); + +G_GNUC_INTERNAL +void ostree_bloom_add_element (OstreeBloom *bloom, + gconstpointer element); + +G_GNUC_INTERNAL +guint64 ostree_str_bloom_hash (gconstpointer element, + guint8 k); + +/* TODO: Building functions */ + +G_END_DECLS diff --git a/src/libostree/ostree-bloom.c b/src/libostree/ostree-bloom.c new file mode 100644 index 0000000000..889bdac918 --- /dev/null +++ b/src/libostree/ostree-bloom.c @@ -0,0 +1,502 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright © 2017 Endless Mobile, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + * Authors: + * - Philip Withnall + */ + +#include "config.h" + +#include +#include +#include +#include +#include +#include + +#include "ostree-bloom-private.h" + +/** + * SECTION:bloom + * + * TODO + * + * TODO: Discuss m, k, universal hash functions, mutability + * + * Reference: + * - https://en.wikipedia.org/wiki/Bloom_filter + * - https://llimllib.github.io/bloomfilter-tutorial/ + * + * Since: 2017.4 + */ + +struct _OstreeBloom +{ + guint ref_count; + gsize n_bytes; + gboolean is_mutable; + union + { + guint8 *mutable_bytes; /* owned; mutually exclusive */ + GBytes *immutable_bytes; /* owned; mutually exclusive */ + }; + guint8 k; + OstreeBloomHashFunc hash_func; +}; + +G_DEFINE_BOXED_TYPE (OstreeBloom, ostree_bloom, ostree_bloom_ref, ostree_bloom_unref) + +/** + * ostree_bloom_new: + * @n_bytes: size to make the bloom filter, in bytes + * @k: number of hash functions to use + * @hash: universal hash function to use + * + * Create a new mutable #OstreeBloom filter, with all its bits initialised to + * zero. Set elements in the filter using ostree_bloom_add_element(), and seal + * it to return an immutable #GBytes using ostree_bloom_seal(). + * + * To load an #OstreeBloom from an existing #GBytes, use + * ostree_bloom_new_from_bytes(). + * + * Note that @n_bytes is in bytes, so is 8 times smaller than the parameter `m` + * which is used when describing bloom filters academically. + * + * Returns: (transfer full): a new mutable bloom filter + * + * Since: 2017.4 + */ +OstreeBloom * +ostree_bloom_new (gsize n_bytes, + guint8 k, + OstreeBloomHashFunc hash) +{ + g_autoptr(OstreeBloom) bloom = NULL; + + g_return_val_if_fail (n_bytes > 0, NULL); + g_return_val_if_fail (k > 0, NULL); + g_return_val_if_fail (hash != NULL, NULL); + + bloom = g_new0 (OstreeBloom, 1); + bloom->ref_count = 1; + + bloom->is_mutable = TRUE; + bloom->mutable_bytes = g_malloc0 (n_bytes); + bloom->n_bytes = n_bytes; + bloom->k = k; + bloom->hash_func = hash; + + return g_steal_pointer (&bloom); +} + +/** + * ostree_bloom_new_from_bytes: + * @bytes: array of bytes containing the filter data + * @k: number of hash functions to use + * @hash: universal hash function to use + * + * Load an immutable #OstreeBloom filter from the given @bytes. Check whether + * elements are probably set in the filter using ostree_bloom_maybe_contains(). + * + * To create a new mutable #OstreeBloom, use ostree_bloom_new(). + * + * Note that all the bits in @bytes are loaded, so the parameter `m` for the + * filter (as commonly used in academic literature) is always a multiple of 8. + * + * Returns: (transfer full): a new immutable bloom filter + * + * Since: 2017.4 + */ +OstreeBloom * +ostree_bloom_new_from_bytes (GBytes *bytes, + guint8 k, + OstreeBloomHashFunc hash) +{ + g_autoptr(OstreeBloom) bloom = NULL; + + g_return_val_if_fail (bytes != NULL, NULL); + g_return_val_if_fail (g_bytes_get_size (bytes) > 0, NULL); + g_return_val_if_fail (k > 0, NULL); + g_return_val_if_fail (hash != NULL, NULL); + + bloom = g_new0 (OstreeBloom, 1); + bloom->ref_count = 1; + + bloom->is_mutable = FALSE; + bloom->immutable_bytes = g_bytes_ref (bytes); + bloom->n_bytes = g_bytes_get_size (bytes); + bloom->k = k; + bloom->hash_func = hash; + + return g_steal_pointer (&bloom); +} + +/** + * ostree_bloom_ref: + * @bloom: an #OstreeBloom + * + * Increase the reference count of @bloom. + * + * Returns: (transfer full): @bloom + * Since: 2017.4 + */ +OstreeBloom * +ostree_bloom_ref (OstreeBloom *bloom) +{ + g_return_val_if_fail (bloom != NULL, NULL); + g_return_val_if_fail (bloom->ref_count >= 1, NULL); + g_return_val_if_fail (bloom->ref_count == G_MAXUINT - 1, NULL); + + bloom->ref_count++; + + return bloom; +} + +/** + * ostree_bloom_unref: + * @bloom: (transfer full): an #OstreeBloom + * + * Decrement the reference count of @bloom. If it reaches zero, the filter + * is destroyed. + * + * Since: 2017.4 + */ +void +ostree_bloom_unref (OstreeBloom *bloom) +{ + g_return_if_fail (bloom != NULL); + g_return_if_fail (bloom->ref_count >= 1); + + bloom->ref_count--; + + if (bloom->ref_count == 0) + { + if (bloom->is_mutable) + g_clear_pointer (&bloom->mutable_bytes, g_free); + else + g_clear_pointer (&bloom->immutable_bytes, g_bytes_unref); + bloom->n_bytes = 0; + g_free (bloom); + } +} + +static inline gboolean +ostree_bloom_get_bit (OstreeBloom *bloom, + gsize idx) +{ + const guint8 *bytes; + + if (bloom->is_mutable) + bytes = bloom->mutable_bytes; + else + bytes = g_bytes_get_data (bloom->immutable_bytes, NULL); + + g_assert (idx / 8 < bloom->n_bytes); + return (bytes[idx / 8] & (1 << (idx % 8))); +} + +static inline void +ostree_bloom_set_bit (OstreeBloom *bloom, + gsize idx) +{ + g_assert (bloom->is_mutable); + g_assert (idx / 8 < bloom->n_bytes); + bloom->mutable_bytes[idx / 8] |= (1 << (idx % 8)); +} + +/** + * ostree_bloom_maybe_contains: + * @bloom: an #OstreeBloom + * @element: (nullable): element to check for membership + * + * TODO; nullability depends on hash function + * + * Returns: %TRUE if @element is potentially in @bloom; %FALSE if it definitely + * isn’t + * Since: 2017.4 + */ +gboolean +ostree_bloom_maybe_contains (OstreeBloom *bloom, + gconstpointer element) +{ + guint8 i; + + g_return_val_if_fail (bloom != NULL, TRUE); + g_return_val_if_fail (bloom->ref_count >= 1, TRUE); + + for (i = 0; i < bloom->k; i++) + { + gsize idx; + + idx = bloom->hash_func (element, i); + + if (!ostree_bloom_get_bit (bloom, idx % (bloom->n_bytes * 8))) + return FALSE; /* definitely not in the set */ + } + + return TRUE; /* possibly in the set */ +} + +/** + * ostree_bloom_seal: + * @bloom: an #OstreeBloom + * + * TODO; OK to call multiple times + * + * Returns: (transfer full): a #GBytes containing the immutable filter data + * Since: 2017.4 + */ +GBytes * +ostree_bloom_seal (OstreeBloom *bloom) +{ + g_return_val_if_fail (bloom != NULL, NULL); + g_return_val_if_fail (bloom->ref_count >= 1, NULL); + + if (bloom->is_mutable) + { + bloom->is_mutable = FALSE; + bloom->immutable_bytes = g_bytes_new_take (g_steal_pointer (&bloom->mutable_bytes), bloom->n_bytes); + } + + return g_bytes_ref (bloom->immutable_bytes); +} + +/** + * ostree_bloom_add_element: + * @bloom: an #OstreeBloom + * @element: (nullable): element to add to the filter + * + * TODO; nullability depends on hash function + * + * Since: 2017.4 + */ +void +ostree_bloom_add_element (OstreeBloom *bloom, + gconstpointer element) +{ + guint8 i; + + g_return_if_fail (bloom != NULL); + g_return_if_fail (bloom->ref_count >= 1); + g_return_if_fail (bloom->is_mutable); + + for (i = 0; i < bloom->k; i++) + { + gsize idx = bloom->hash_func (element, i); + ostree_bloom_set_bit (bloom, idx % (bloom->n_bytes * 8)); + } +} + +/* SipHash code adapted from https://github.com/veorq/SipHash/blob/master/siphash.c */ + +/* + SipHash reference C implementation + Copyright (c) 2012-2016 Jean-Philippe Aumasson + + Copyright (c) 2012-2014 Daniel J. Bernstein + To the extent possible under law, the author(s) have dedicated all copyright + and related and neighboring rights to this software to the public domain + worldwide. This software is distributed without any warranty. + You should have received a copy of the CC0 Public Domain Dedication along + with + this software. If not, see + . + */ + +/* default: SipHash-2-4 */ +#define cROUNDS 2 +#define dROUNDS 4 + +#define ROTL(x, b) (uint64_t)(((x) << (b)) | ((x) >> (64 - (b)))) + +#define U32TO8_LE(p, v) \ + (p)[0] = (uint8_t)((v)); \ + (p)[1] = (uint8_t)((v) >> 8); \ + (p)[2] = (uint8_t)((v) >> 16); \ + (p)[3] = (uint8_t)((v) >> 24); + +#define U64TO8_LE(p, v) \ + U32TO8_LE((p), (uint32_t)((v))); \ + U32TO8_LE((p) + 4, (uint32_t)((v) >> 32)); + +#define U8TO64_LE(p) \ + (((uint64_t)((p)[0])) | ((uint64_t)((p)[1]) << 8) | \ + ((uint64_t)((p)[2]) << 16) | ((uint64_t)((p)[3]) << 24) | \ + ((uint64_t)((p)[4]) << 32) | ((uint64_t)((p)[5]) << 40) | \ + ((uint64_t)((p)[6]) << 48) | ((uint64_t)((p)[7]) << 56)) + +#define SIPROUND \ + do { \ + v0 += v1; \ + v1 = ROTL(v1, 13); \ + v1 ^= v0; \ + v0 = ROTL(v0, 32); \ + v2 += v3; \ + v3 = ROTL(v3, 16); \ + v3 ^= v2; \ + v0 += v3; \ + v3 = ROTL(v3, 21); \ + v3 ^= v0; \ + v2 += v1; \ + v1 = ROTL(v1, 17); \ + v1 ^= v2; \ + v2 = ROTL(v2, 32); \ + } while (0) + +#ifdef DEBUG +#define TRACE \ + do { \ + printf("(%3d) v0 %08x %08x\n", (int)inlen, (uint32_t)(v0 >> 32), \ + (uint32_t)v0); \ + printf("(%3d) v1 %08x %08x\n", (int)inlen, (uint32_t)(v1 >> 32), \ + (uint32_t)v1); \ + printf("(%3d) v2 %08x %08x\n", (int)inlen, (uint32_t)(v2 >> 32), \ + (uint32_t)v2); \ + printf("(%3d) v3 %08x %08x\n", (int)inlen, (uint32_t)(v3 >> 32), \ + (uint32_t)v3); \ + } while (0) +#else +#define TRACE +#endif + +static int siphash(const uint8_t *in, const size_t inlen, const uint8_t *k, + uint8_t *out, const size_t outlen) { + + assert((outlen == 8) || (outlen == 16)); + uint64_t v0 = 0x736f6d6570736575ULL; + uint64_t v1 = 0x646f72616e646f6dULL; + uint64_t v2 = 0x6c7967656e657261ULL; + uint64_t v3 = 0x7465646279746573ULL; + uint64_t k0 = U8TO64_LE(k); + uint64_t k1 = U8TO64_LE(k + 8); + uint64_t m; + int i; + const uint8_t *end = in + inlen - (inlen % sizeof(uint64_t)); + const int left = inlen & 7; + uint64_t b = ((uint64_t)inlen) << 56; + v3 ^= k1; + v2 ^= k0; + v1 ^= k1; + v0 ^= k0; + + if (outlen == 16) + v1 ^= 0xee; + + for (; in != end; in += 8) { + m = U8TO64_LE(in); + v3 ^= m; + + TRACE; + for (i = 0; i < cROUNDS; ++i) + SIPROUND; + + v0 ^= m; + } + + switch (left) { + case 7: + b |= ((uint64_t)in[6]) << 48; + case 6: + b |= ((uint64_t)in[5]) << 40; + case 5: + b |= ((uint64_t)in[4]) << 32; + case 4: + b |= ((uint64_t)in[3]) << 24; + case 3: + b |= ((uint64_t)in[2]) << 16; + case 2: + b |= ((uint64_t)in[1]) << 8; + case 1: + b |= ((uint64_t)in[0]); + break; + case 0: + break; + } + + v3 ^= b; + + TRACE; + for (i = 0; i < cROUNDS; ++i) + SIPROUND; + + v0 ^= b; + + if (outlen == 16) + v2 ^= 0xee; + else + v2 ^= 0xff; + + TRACE; + for (i = 0; i < dROUNDS; ++i) + SIPROUND; + + b = v0 ^ v1 ^ v2 ^ v3; + U64TO8_LE(out, b); + + if (outlen == 8) + return 0; + + v1 ^= 0xdd; + + TRACE; + for (i = 0; i < dROUNDS; ++i) + SIPROUND; + + b = v0 ^ v1 ^ v2 ^ v3; + U64TO8_LE(out + 8, b); + + return 0; +} + +/** + * ostree_str_bloom_hash: + * @element: element to calculate the hash for + * @k: hash function index + * + * TODO: k is in 0..k-1 from the bloom filter; output range is unrestricted; + * type of @element depends on the hash function; %NULL not supported + * + * Reference: + * - https://www.131002.net/siphash/ + * + * Returns: TODO + * Since: 2017.4 + */ +guint64 +ostree_str_bloom_hash (gconstpointer element, + guint8 k) +{ + const gchar *str = element; + gsize str_len; + union + { + guint64 u64; + guint8 u8[8]; + } out_le; + guint8 k_array[16]; + gsize i; + + str_len = strlen (str); + for (i = 0; i < G_N_ELEMENTS (k_array); i++) + k_array[i] = k; + + siphash ((const guint8 *) str, str_len, k_array, out_le.u8, sizeof (out_le)); + + return le64toh (out_le.u64); +} diff --git a/src/libostree/ostree-fetcher.h b/src/libostree/ostree-fetcher.h index 78b29faeea..28c18847ac 100644 --- a/src/libostree/ostree-fetcher.h +++ b/src/libostree/ostree-fetcher.h @@ -46,6 +46,8 @@ struct OstreeFetcherClass GObjectClass parent_class; }; +G_DEFINE_AUTOPTR_CLEANUP_FUNC(OstreeFetcher, g_object_unref) + typedef enum { OSTREE_FETCHER_FLAGS_NONE = 0, OSTREE_FETCHER_FLAGS_TLS_PERMISSIVE = (1 << 0) diff --git a/src/libostree/ostree-repo-finder-avahi-private.h b/src/libostree/ostree-repo-finder-avahi-private.h new file mode 100644 index 0000000000..e95a39b1dc --- /dev/null +++ b/src/libostree/ostree-repo-finder-avahi-private.h @@ -0,0 +1,39 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright © 2017 Endless Mobile, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + * Authors: + * - Philip Withnall + */ + +#pragma once + +#include +#include +#include +#include + +G_BEGIN_DECLS + +GHashTable *_ostree_txt_records_parse (AvahiStringList *txt); + +GVariant *_ostree_txt_records_lookup_variant (GHashTable *attributes, + const gchar *key, + const GVariantType *value_type); + +G_END_DECLS diff --git a/src/libostree/ostree-repo-finder-avahi.c b/src/libostree/ostree-repo-finder-avahi.c new file mode 100644 index 0000000000..04f1bfcd49 --- /dev/null +++ b/src/libostree/ostree-repo-finder-avahi.c @@ -0,0 +1,998 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright © 2016 Kinvolk GmbH + * Copyright © 2017 Endless Mobile, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + * Authors: + * - Krzesimir Nowak + * - Philip Withnall + */ + +#include "config.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ostree-bloom-private.h" +#include "ostree-repo-finder.h" +#include "ostree-repo-finder-avahi.h" +#include "ostree-repo-finder-avahi-private.h" + +/* TODO: Section documentation */ + +/* TODO: Submit these upstream */ +G_DEFINE_AUTOPTR_CLEANUP_FUNC (AvahiClient, avahi_client_free) +G_DEFINE_AUTOPTR_CLEANUP_FUNC (AvahiServiceBrowser, avahi_service_browser_free) +G_DEFINE_AUTOPTR_CLEANUP_FUNC (AvahiServiceResolver, avahi_service_resolver_free) + +/* FIXME: Register this with IANA? https://tools.ietf.org/html/rfc6335#section-5.2 */ +const gchar * const OSTREE_AVAHI_SERVICE_TYPE = "_repo._ostree._tcp"; + +static const gchar * +ostree_avahi_client_state_to_string (AvahiClientState state) +{ + switch (state) + { + case AVAHI_CLIENT_S_REGISTERING: + return "registering"; + case AVAHI_CLIENT_S_RUNNING: + return "running"; + case AVAHI_CLIENT_S_COLLISION: + return "collision"; + case AVAHI_CLIENT_CONNECTING: + return "connecting"; + case AVAHI_CLIENT_FAILURE: + return "failure"; + default: + return "unknown"; + } +} + +static const gchar * +ostree_avahi_resolver_event_to_string (AvahiResolverEvent event) +{ + switch (event) + { + case AVAHI_RESOLVER_FOUND: + return "found"; + case AVAHI_RESOLVER_FAILURE: + return "failure"; + default: + return "unknown"; + } +} + +static const gchar * +ostree_avahi_browser_event_to_string (AvahiBrowserEvent event) +{ + switch (event) + { + case AVAHI_BROWSER_NEW: + return "new"; + case AVAHI_BROWSER_REMOVE: + return "remove"; + case AVAHI_BROWSER_CACHE_EXHAUSTED: + return "cache-exhausted"; + case AVAHI_BROWSER_ALL_FOR_NOW: + return "all-for-now"; + case AVAHI_BROWSER_FAILURE: + return "failure"; + default: + return "unknown"; + } +} + +/* TODO: Docs */ +typedef struct +{ + gchar *name; + gchar *domain; + gchar *address; + guint16 port; + AvahiStringList *txt; +} OstreeAvahiService; + +static void +ostree_avahi_service_free (OstreeAvahiService *service) +{ + g_free (service->name); + g_free (service->domain); + g_free (service->address); + avahi_string_list_free (service->txt); + g_free (service); +} + +G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeAvahiService, ostree_avahi_service_free) + +/* Convert an AvahiAddress to a string which is suitable for use in URIs (for + * example). Take into account the scope ID, if the address is IPv6 and a + * link-local address. + * (See https://en.wikipedia.org/wiki/IPv6_address#Link-local_addresses_and_zone_indices and + * https://github.com/lathiat/avahi/issues/110.) */ +static gchar * +address_to_string (const AvahiAddress *address, + AvahiIfIndex interface) +{ + char address_string[AVAHI_ADDRESS_STR_MAX]; + + avahi_address_snprint (address_string, sizeof (address_string), address); + + switch (address->proto) + { + case AVAHI_PROTO_INET6: + if (IN6_IS_ADDR_LINKLOCAL (address->data.data) || + IN6_IS_ADDR_LOOPBACK (address->data.data)) + return g_strdup_printf ("%s%%%d", address_string, interface); + /* else fall through */ + case AVAHI_PROTO_INET: + case AVAHI_PROTO_UNSPEC: + default: + return g_strdup (address_string); + } +} + +static OstreeAvahiService * +ostree_avahi_service_new (const gchar *name, + const gchar *domain, + const AvahiAddress *address, + AvahiIfIndex interface, + guint16 port, + AvahiStringList *txt) +{ + g_autoptr(OstreeAvahiService) service = NULL; + + g_return_val_if_fail (name != NULL, NULL); + g_return_val_if_fail (domain != NULL, NULL); + g_return_val_if_fail (address != NULL, NULL); + g_return_val_if_fail (port > 0, NULL); + + service = g_new0 (OstreeAvahiService, 1); + + service->name = g_strdup (name); + service->domain = g_strdup (domain); + service->address = address_to_string (address, interface); + service->port = port; + service->txt = avahi_string_list_copy (txt); + + return g_steal_pointer (&service); +} + +/* Reference: RFC 6763, §6. */ +static gboolean +parse_txt_record (const guint8 *txt, + gsize txt_len, + const gchar **key, + gsize *key_len, + const guint8 **value, + gsize *value_len) +{ + gsize i; + + g_return_val_if_fail (key != NULL, FALSE); + g_return_val_if_fail (key_len != NULL, FALSE); + g_return_val_if_fail (value != NULL, FALSE); + g_return_val_if_fail (value_len != NULL, FALSE); + + /* RFC 6763, §6.1. */ + if (txt_len > 8900) + return FALSE; + + *key = (const gchar *) txt; + *key_len = 0; + *value = NULL; + *value_len = 0; + + for (i = 0; i < txt_len; i++) + { + if (txt[i] >= 0x20 && txt[i] <= 0x7e && txt[i] != '=') + { + /* Key character. */ + *key_len = *key_len + 1; + continue; + } + else if (*key_len > 0 && txt[i] == '=') + { + /* Separator. */ + *value = txt + (i + 1); + *value_len = txt_len - (i + 1); + return TRUE; + } + else + { + return FALSE; + } + } + + /* The entire TXT record is the key; there is no ‘=’ or value. */ + *value = NULL; + *value_len = 0; + + return (*key_len > 0); +} + +/* TODO: Docs. Return value is only valid as long as @txt is. Reference: RFC 6763, §6. */ +GHashTable * +_ostree_txt_records_parse (AvahiStringList *txt) +{ + AvahiStringList *l; + g_autoptr(GHashTable) out = NULL; + + out = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, (GDestroyNotify) g_bytes_unref); + + for (l = txt; l != NULL; l = avahi_string_list_get_next (l)) + { + const guint8 *txt; + gsize txt_len; + const gchar *key; + const guint8 *value; + gsize key_len, value_len; + g_autofree gchar *key_allocated = NULL; + g_autoptr(GBytes) value_allocated = NULL; + + txt = avahi_string_list_get_text (l); + txt_len = avahi_string_list_get_size (l); + + if (!parse_txt_record (txt, txt_len, &key, &key_len, &value, &value_len)) + { + g_debug ("Ignoring invalid TXT record of length %" G_GSIZE_FORMAT, + txt_len); + continue; + } + + key_allocated = g_ascii_strdown (key, key_len); + + if (g_hash_table_lookup_extended (out, key_allocated, NULL, NULL)) + { + g_debug ("Ignoring duplicate TXT record ‘%s’", key_allocated); + continue; + } + + /* Distinguish between the case where the entire record is the key + * (value == NULL) and the case where the record is the key + ‘=’ and the + * value is empty (value != NULL && value_len == 0). */ + if (value != NULL) + value_allocated = g_bytes_new_static (value, value_len); + + g_hash_table_insert (out, g_steal_pointer (&key_allocated), g_steal_pointer (&value_allocated)); + } + + return g_steal_pointer (&out); +} + +/* TODO: Maybe make it a valid key check? */ +static gboolean +str_is_lowercase (const gchar *str) +{ + gsize i; + + for (i = 0; str[i] != '\0'; i++) + { + if (!g_ascii_islower (str[i])) + return FALSE; + } + + return TRUE; +} + +/* TODO: docs */ +GVariant * +_ostree_txt_records_lookup_variant (GHashTable *attributes, + const gchar *key, + const GVariantType *value_type) +{ + GBytes *value; + g_autoptr(GVariant) variant = NULL; + + g_return_val_if_fail (attributes != NULL, NULL); + g_return_val_if_fail (str_is_lowercase (key), NULL); + g_return_val_if_fail (value_type != NULL, NULL); + + value = g_hash_table_lookup (attributes, key); + + if (value == NULL) + { + g_debug ("TXT attribute ‘%s’ not found.", key); + return NULL; + } + + variant = g_variant_new_from_bytes (value_type, value, FALSE); + + if (!g_variant_is_normal_form (variant)) + { + g_debug ("TXT attribute ‘%s’ value is not in normal form. Ignoring.", key); + return NULL; + } + + return g_steal_pointer (&variant); +} + +/* TODO: docs */ +static GPtrArray * +bloom_refs_intersection (GVariant *bloom_encoded, + const gchar * const *refs) +{ + g_autoptr(OstreeBloom) bloom = NULL; + g_autoptr(GVariant) bloom_variant = NULL; + guint8 k, hash_id; + OstreeBloomHashFunc hash_func; + const guint8 *bloom_bytes; + gsize n_bloom_bytes; + g_autoptr(GBytes) bytes = NULL; + gsize i; + g_autoptr(GPtrArray) possible_refs = NULL; + + g_variant_get (bloom_encoded, "yy@ay", &k, &hash_id, &bloom_variant); + + if (k == 0) + return NULL; + + switch (hash_id) + { + case 1: + hash_func = ostree_str_bloom_hash; + break; + default: + return NULL; + } + + bloom_bytes = g_variant_get_fixed_array (bloom_variant, &n_bloom_bytes, sizeof (guint8)); + bytes = g_bytes_new_static (bloom_bytes, n_bloom_bytes); + bloom = ostree_bloom_new_from_bytes (bytes, k, hash_func); + + possible_refs = g_ptr_array_new_with_free_func (NULL); + + for (i = 0; refs[i] != NULL; i++) + { + const gchar *ref = refs[i]; + + if (ostree_bloom_maybe_contains (bloom, ref)) + g_ptr_array_add (possible_refs, (gpointer) ref); + } + + return g_steal_pointer (&possible_refs); +} + +/* TODO: docs + +v=1 +rb=refs bloom filter +st=summary timestamp +*/ +static OstreeRepoFinderResult * +ostree_avahi_service_build_repo_finder_result (OstreeAvahiService *self, + gint priority, + const gchar * const *refs) +{ + g_autoptr(GHashTable) attributes = NULL; + g_autoptr(GVariant) version = NULL; + g_autoptr(GVariant) bloom = NULL; + g_autoptr(GVariant) summary_timestamp = NULL; + g_autoptr(GVariant) repo_index = NULL; + g_autofree gchar *repo_path = NULL; + g_autoptr(GPtrArray) possible_refs = NULL; + SoupURI *_uri = NULL; + g_autofree gchar *uri = NULL; + + g_return_val_if_fail (self != NULL, NULL); + g_return_val_if_fail (refs != NULL, NULL); + + attributes = _ostree_txt_records_parse (self->txt); + + /* Check the record version. */ + version = _ostree_txt_records_lookup_variant (attributes, "v", G_VARIANT_TYPE_BYTE); + + if (g_variant_get_byte (version) != 1) + { + g_debug ("Unknown v=%02x attribute provided in TXT record. Ignoring.", + g_variant_get_byte (version)); + return NULL; + } + + /* Refs bloom filter? */ + bloom = _ostree_txt_records_lookup_variant (attributes, "rb", G_VARIANT_TYPE ("yyay")); + + if (bloom != NULL) + { + possible_refs = bloom_refs_intersection (bloom, refs); + + if (possible_refs->len == 0) + { + g_debug ("TXT record definitely has no matching refs. Ignoring."); + return NULL; + } + } + + /* Summary timestamp. */ + summary_timestamp = _ostree_txt_records_lookup_variant (attributes, "st", G_VARIANT_TYPE_UINT64); + + /* Repository index. */ + repo_index = _ostree_txt_records_lookup_variant (attributes, "ri", G_VARIANT_TYPE_UINT32); + repo_path = (repo_index != NULL) ? g_strdup_printf ("/%u/", g_variant_get_uint32 (repo_index)) : g_strdup ("/"); + + /* Build the URI for the repository. */ + _uri = soup_uri_new (NULL); + soup_uri_set_scheme (_uri, "http"); + soup_uri_set_host (_uri, self->address); + soup_uri_set_port (_uri, self->port); + soup_uri_set_path (_uri, repo_path); + uri = soup_uri_to_string (_uri, FALSE); + soup_uri_free (_uri); + + return ostree_repo_finder_result_new (uri, priority, + (possible_refs != NULL) ? (const gchar * const *) possible_refs->pdata : NULL, + (summary_timestamp != NULL) ? g_variant_get_uint64 (summary_timestamp) : 0); +} + +static void ostree_repo_finder_avahi_iface_init (OstreeRepoFinderInterface *iface); + +struct _OstreeRepoFinderAvahi +{ + GObject parent_instance; + + /* All elements of this structure must only be accessed from @avahi_context + * after construction. */ + + GPtrArray *resolve_tasks; /* (element-type (owned) GTask) */ + + AvahiGLibPoll *poll; + AvahiClient *client; + AvahiServiceBrowser *browser; + + AvahiClientState client_state; + gboolean browser_failed; + gboolean browser_all_for_now; + + GCancellable *avahi_cancellable; + GMainContext *avahi_context; + + /* Map of service name (typically human readable) to a #GPtrArray of the + * #AvahiServiceResolver instances we have running against that name. We + * could end up with more than one resolver if the same name is advertised to + * us over multiple interfaces or protocols (for example, IPv4 and IPv6). + * Resolve all of them just in case one doesn’t work. */ + GHashTable *resolvers; /* (element-type (owned) utf8 (owned) GPtrArray (element-type (owned) AvahiServiceResolver)) */ + + /* Array of #OstreeAvahiService instances representing all the services which + * we currently think are valid. */ + GPtrArray *found_services; /* (element-type (owned OstreeAvahiService) */ +}; + +G_DEFINE_TYPE_WITH_CODE (OstreeRepoFinderAvahi, ostree_repo_finder_avahi, G_TYPE_OBJECT, + G_IMPLEMENT_INTERFACE (OSTREE_TYPE_REPO_FINDER, ostree_repo_finder_avahi_iface_init)) + +static void +fail_all_pending_tasks (OstreeRepoFinderAvahi *self, + GQuark domain, + gint code, + const gchar *format, + ...) G_GNUC_PRINTF(4, 5); + +/* TODO: Docs */ +/* Executed in @self->avahi_context. */ +static void +fail_all_pending_tasks (OstreeRepoFinderAvahi *self, + GQuark domain, + gint code, + const gchar *format, + ...) +{ + gsize i; + va_list args; + g_autoptr(GError) error = NULL; + + g_assert (g_main_context_is_owner (self->avahi_context)); + + va_start (args, format); + error = g_error_new_valist (domain, code, format, args); + va_end (args); + + for (i = 0; i < self->resolve_tasks->len; i++) + { + GTask *task = G_TASK (g_ptr_array_index (self->resolve_tasks, i)); + g_task_return_error (task, g_error_copy (error)); + } + + g_ptr_array_set_size (self->resolve_tasks, 0); +} + +/* TODO: docs */ +/* Executed in @self->avahi_context. */ +static void +complete_all_pending_tasks (OstreeRepoFinderAvahi *self) +{ + gsize i; + const gint priority = 60; /* arbitrarily chosen */ + + g_assert (g_main_context_is_owner (self->avahi_context)); + + for (i = 0; i < self->resolve_tasks->len; i++) + { + g_autoptr(GPtrArray) results = NULL; + GTask *task; + const gchar * const *refs; + gsize j; + + task = G_TASK (g_ptr_array_index (self->resolve_tasks, i)); + refs = g_task_get_task_data (task); + results = g_ptr_array_new_with_free_func ((GDestroyNotify) ostree_repo_finder_result_free); + + for (j = 0; j < self->found_services->len; j++) + { + OstreeAvahiService *service = g_ptr_array_index (self->found_services, j); + g_autoptr(OstreeRepoFinderResult) result = NULL; + + result = ostree_avahi_service_build_repo_finder_result (service, priority, refs); + g_ptr_array_add (results, g_steal_pointer (&result)); + } + + g_task_return_pointer (task, g_steal_pointer (&results), (GDestroyNotify) g_ptr_array_unref); + } + + g_ptr_array_set_size (self->resolve_tasks, 0); +} + +/* Executed in @self->avahi_context. */ +static void +maybe_complete_all_pending_tasks (OstreeRepoFinderAvahi *self) +{ + g_assert (g_main_context_is_owner (self->avahi_context)); + + if (self->client_state == AVAHI_CLIENT_FAILURE) + fail_all_pending_tasks (self, G_IO_ERROR, G_IO_ERROR_FAILED, + "Avahi client error: %s", + avahi_strerror (avahi_client_errno (self->client))); + else if (self->browser_failed) + fail_all_pending_tasks (self, G_IO_ERROR, G_IO_ERROR_FAILED, + "Avahi browser error: %s", + avahi_strerror (avahi_client_errno (self->client))); + else if (g_cancellable_is_cancelled (self->avahi_cancellable)) + fail_all_pending_tasks (self, G_IO_ERROR, G_IO_ERROR_CANCELLED, + "Avahi service resolution cancelled."); + else if (self->browser_all_for_now && + g_hash_table_size (self->resolvers) == 0) + complete_all_pending_tasks (self); +} + +/* Executed in @self->avahi_context. */ +static void +client_cb (AvahiClient *client, + AvahiClientState state, + void *finder_ptr) +{ + OstreeRepoFinderAvahi *self = OSTREE_REPO_FINDER_AVAHI (finder_ptr); + + g_assert (g_main_context_is_owner (self->avahi_context)); + + g_debug ("%s: Entered state ‘%s’.", + G_STRFUNC, ostree_avahi_client_state_to_string (state)); + + /* We only care about entering and leaving %AVAHI_CLIENT_FAILURE. */ + self->client_state = state; + maybe_complete_all_pending_tasks (self); +} + +/* Executed in @self->avahi_context. */ +static void +resolve_cb (AvahiServiceResolver *resolver, + AvahiIfIndex interface, + AvahiProtocol protocol, + AvahiResolverEvent event, + const char *name, + const char *type, + const char *domain, + const char *host_name, + const AvahiAddress *address, + uint16_t port, + AvahiStringList *txt, + AvahiLookupResultFlags flags, + void *finder_ptr) +{ + OstreeRepoFinderAvahi *self = OSTREE_REPO_FINDER_AVAHI (finder_ptr); + g_autoptr(OstreeAvahiService) service = NULL; + GPtrArray *resolvers; + + g_assert (g_main_context_is_owner (self->avahi_context)); + + g_debug ("%s: Resolve event ‘%s’ for name ‘%s’.", + G_STRFUNC, ostree_avahi_resolver_event_to_string (event), name); + + /* Track the resolvers active for this @name. There may be several, + * as @name might appear to us over several interfaces or protocols. Most + * commonly this happens when both hosts are connected via IPv4 and IPv6. */ + resolvers = g_hash_table_lookup (self->resolvers, name); + + if (resolvers == NULL || resolvers->len == 0) + { + /* maybe it was removed in the meantime */ + g_hash_table_remove (self->resolvers, name); + return; + } + else if (resolvers->len == 1) + { + g_hash_table_remove (self->resolvers, name); + } + else + { + g_ptr_array_remove_fast (resolvers, resolver); + } + + /* Was resolution successful? */ + switch (event) + { + case AVAHI_RESOLVER_FOUND: + service = ostree_avahi_service_new (name, domain, address, interface, + port, txt); + g_ptr_array_add (self->found_services, g_steal_pointer (&service)); + break; + case AVAHI_RESOLVER_FAILURE: + default: + g_warning ("Failed to resolve service ‘%s’: %s", name, + avahi_strerror (avahi_client_errno (self->client))); + break; + } + + maybe_complete_all_pending_tasks (self); +} + +/* TODO: overall documentation about the split between client, browser, resolver; + * and between the background processing and resolve_async() requests. */ +/* Executed in @self->avahi_context. */ +static void +browse_new (OstreeRepoFinderAvahi *self, + AvahiIfIndex interface, + AvahiProtocol protocol, + const gchar *name, + const gchar *type, + const gchar *domain) +{ + g_autoptr(AvahiServiceResolver) resolver = NULL; + GPtrArray *resolvers; /* (element-type AvahiServiceResolver) */ + + g_assert (g_main_context_is_owner (self->avahi_context)); + + resolver = avahi_service_resolver_new (self->client, + interface, + protocol, + name, + type, + domain, + AVAHI_PROTO_UNSPEC, + 0, + resolve_cb, + self); + if (resolver == NULL) + { + g_warning ("Failed to resolve service ‘%s’: %s", name, + avahi_strerror (avahi_client_errno (self->client))); + return; + } + + g_debug ("Found name service %s on the network; type: %s, domain: %s, " + "protocol: %u, interface: %u", name, type, domain, protocol, + interface); + + /* Start a resolver for this (interface, protocol, name, type, domain) + * combination. */ + resolvers = g_hash_table_lookup (self->resolvers, name); + if (resolvers == NULL) + { + resolvers = g_ptr_array_new_with_free_func ((GDestroyNotify) avahi_service_resolver_free); + g_hash_table_insert (self->resolvers, g_strdup (name), resolvers); + } + + g_ptr_array_add (resolvers, g_steal_pointer (&resolver)); +} + +/* Executed in @self->avahi_context. Caller must call maybe_complete_all_pending_tasks(). */ +static void +browse_remove (OstreeRepoFinderAvahi *self, + const char *name) +{ + gsize i; + gboolean removed = FALSE; + + g_assert (g_main_context_is_owner (self->avahi_context)); + + g_hash_table_remove (self->resolvers, name); + + for (i = 0; i < self->found_services->len; i += (removed ? 0 : 1)) + { + OstreeAvahiService *service = g_ptr_array_index (self->found_services, i); + + removed = FALSE; + + if (g_strcmp0 (service->name, name) == 0) + { + g_ptr_array_remove_index_fast (self->found_services, i); + removed = TRUE; + continue; + } + } +} + +/* Executed in @self->avahi_context. */ +static void +browse_cb (AvahiServiceBrowser *browser, + AvahiIfIndex interface, + AvahiProtocol protocol, + AvahiBrowserEvent event, + const char *name, + const char *type, + const char *domain, + AvahiLookupResultFlags flags, + void *finder_ptr) +{ + OstreeRepoFinderAvahi *self = OSTREE_REPO_FINDER_AVAHI (finder_ptr); + + g_assert (g_main_context_is_owner (self->avahi_context)); + + g_debug ("%s: Browse event ‘%s’ for name ‘%s’.", + G_STRFUNC, ostree_avahi_browser_event_to_string (event), name); + + self->browser_failed = FALSE; + + switch (event) + { + case AVAHI_BROWSER_NEW: + browse_new (self, interface, protocol, name, type, domain); + break; + + case AVAHI_BROWSER_REMOVE: + browse_remove (self, name); + break; + + case AVAHI_BROWSER_CACHE_EXHAUSTED: + /* don’t care about this. */ + break; + + case AVAHI_BROWSER_ALL_FOR_NOW: + self->browser_all_for_now = TRUE; + break; + + case AVAHI_BROWSER_FAILURE: + self->browser_failed = TRUE; + break; + + default: + g_assert_not_reached (); + } + + /* Check all the tasks for any event, since the @browser_failed state + * may have changed. */ + maybe_complete_all_pending_tasks (self);} + +static gboolean add_resolve_task_cb (gpointer user_data); + +/* TODO: Docs, make sure it’s testable */ +static void +ostree_repo_finder_avahi_resolve_async (OstreeRepoFinder *finder, + const gchar * const *refs, + GCancellable *cancellable, + GAsyncReadyCallback callback, + gpointer user_data) +{ + OstreeRepoFinderAvahi *self = OSTREE_REPO_FINDER_AVAHI (finder); + g_autoptr(GTask) task = NULL; + + task = g_task_new (self, cancellable, callback, user_data); + g_task_set_source_tag (task, ostree_repo_finder_avahi_resolve_async); + g_task_set_task_data (task, g_strdupv ((gchar **) refs), (GDestroyNotify) g_strfreev); + + /* Move @task to the @avahi_context where it can be processed. */ + g_main_context_invoke (self->avahi_context, add_resolve_task_cb, g_steal_pointer (&task)); +} + +/* Executed in @self->avahi_context. */ +static gboolean +add_resolve_task_cb (gpointer user_data) +{ + g_autoptr(GTask) task = G_TASK (user_data); + OstreeRepoFinderAvahi *self = g_task_get_source_object (task); + + g_assert (g_main_context_is_owner (self->avahi_context)); + + /* Track the task and check to see if the browser and resolvers are in a + * quiescent state suitable for returning a result immediately. */ + g_ptr_array_add (self->resolve_tasks, g_object_ref (task)); + maybe_complete_all_pending_tasks (self); + + return G_SOURCE_REMOVE; +} + +static GPtrArray * +ostree_repo_finder_avahi_resolve_finish (OstreeRepoFinder *finder, + GAsyncResult *result, + GError **error) +{ + g_return_val_if_fail (g_task_is_valid (result, finder), NULL); + return g_task_propagate_pointer (G_TASK (result), error); +} + +static void +ostree_repo_finder_avahi_dispose (GObject *obj) +{ + OstreeRepoFinderAvahi *self = OSTREE_REPO_FINDER_AVAHI (obj); + + ostree_repo_finder_avahi_stop (self); + + g_assert (self->resolve_tasks == NULL || self->resolve_tasks->len == 0); + + g_clear_pointer (&self->resolve_tasks, g_ptr_array_unref); + g_clear_pointer (&self->browser, avahi_service_browser_free); + g_clear_pointer (&self->client, avahi_client_free); + g_clear_pointer (&self->poll, avahi_glib_poll_free); + g_clear_pointer (&self->avahi_context, g_main_context_unref); + g_clear_pointer (&self->found_services, g_ptr_array_unref); + g_clear_pointer (&self->resolvers, g_hash_table_unref); + + /* Chain up. */ + G_OBJECT_CLASS (ostree_repo_finder_avahi_parent_class)->dispose (obj); +} + +static void +ostree_repo_finder_avahi_class_init (OstreeRepoFinderAvahiClass *klass) +{ + GObjectClass *object_class = G_OBJECT_CLASS (klass); + + object_class->dispose = ostree_repo_finder_avahi_dispose; +} + +static void +ostree_repo_finder_avahi_iface_init (OstreeRepoFinderInterface *iface) +{ + iface->resolve_async = ostree_repo_finder_avahi_resolve_async; + iface->resolve_finish = ostree_repo_finder_avahi_resolve_finish; +} + +static void +ostree_repo_finder_avahi_init (OstreeRepoFinderAvahi *self) +{ + self->resolve_tasks = g_ptr_array_new_with_free_func ((GDestroyNotify) g_object_unref); + self->avahi_cancellable = g_cancellable_new (); + self->client_state = AVAHI_CLIENT_S_REGISTERING; + self->resolvers = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, (GDestroyNotify) g_ptr_array_unref); + self->found_services = g_ptr_array_new_with_free_func ((GDestroyNotify) ostree_avahi_service_free); +} + +/** + * ostree_repo-finder_avahi_new: + * @context: (transfer none) (nullable): a #GMainContext for processing Avahi + * events in, or %NULL to use the current thread-default + * + * TODO + * + * The calling code is responsible for ensuring that @context is iterated while + * the #OstreeRepoFinderAvahi is running (after ostree_repo_finder_avahi_start() + * is called). This may be done from any thread. + * + * If @context is %NULL, the current thread-default #GMainContext is used. + * + * Returns: (transfer full): a new #OstreeRepoFinderAvahi + * Since: 2017.4 + */ +OstreeRepoFinderAvahi * +ostree_repo_finder_avahi_new (GMainContext *context) +{ + g_autoptr(OstreeRepoFinderAvahi) finder = NULL; + + finder = g_object_new (OSTREE_TYPE_REPO_FINDER_AVAHI, NULL); + + /* TODO: Make this a property */ + if (context != NULL) + finder->avahi_context = g_main_context_ref (context); + else + finder->avahi_context = g_main_context_ref_thread_default (); + + /* Avahi setup. Note: Technically the allocator is per-process state which we + * shouldn’t set here, but it’s probably fine. It’s unlikely that code which + * is using libostree is going to use an allocator which is not GLib, and + * *also* use Avahi API itself. */ + avahi_set_allocator (avahi_glib_allocator ()); + finder->poll = avahi_glib_poll_new (finder->avahi_context, G_PRIORITY_DEFAULT); + + return g_steal_pointer (&finder); +} + +/* TODO: Docs */ +void +ostree_repo_finder_avahi_start (OstreeRepoFinderAvahi *self, + GError **error) +{ + g_autoptr(AvahiClient) client = NULL; + g_autoptr(AvahiServiceBrowser) browser = NULL; + int failure = 0; + + g_return_if_fail (OSTREE_IS_REPO_FINDER_AVAHI (self)); + g_return_if_fail (error == NULL || *error == NULL); + g_return_if_fail (g_cancellable_is_cancelled (self->avahi_cancellable)); + + g_assert (self->client == NULL); + + client = avahi_client_new (avahi_glib_poll_get (self->poll), + AVAHI_CLIENT_NO_FAIL, + client_cb, self, &failure); + + if (client == NULL) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Failed to create finder client: %s", + avahi_strerror (failure)); + return; + } + + /* Query for the OSTree DNS-SD service on the local network. */ + browser = avahi_service_browser_new (client, + AVAHI_IF_UNSPEC, + AVAHI_PROTO_UNSPEC, + OSTREE_AVAHI_SERVICE_TYPE, + NULL, + 0, + browse_cb, + self); + + if (browser == NULL) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Failed to create service browser: %s", + avahi_strerror (avahi_client_errno (client))); + return; + } + + /* Success. */ + self->client = g_steal_pointer (&client); + self->browser = g_steal_pointer (&browser); +} + +static gboolean stop_cb (gpointer user_data); + +/* TODO: Docs. Can’t be started again after it’s been stopped. Any in-progress queries will be cancelled. */ +void +ostree_repo_finder_avahi_stop (OstreeRepoFinderAvahi *self) +{ + g_return_if_fail (OSTREE_IS_REPO_FINDER_AVAHI (self)); + + g_main_context_invoke (self->avahi_context, stop_cb, g_object_ref (self)); +} + +static gboolean +stop_cb (gpointer user_data) +{ + g_autoptr(OstreeRepoFinderAvahi) self = OSTREE_REPO_FINDER_AVAHI (user_data); + + g_cancellable_cancel (self->avahi_cancellable); + maybe_complete_all_pending_tasks (self); + + g_clear_pointer (&self->browser, avahi_service_browser_free); + g_clear_pointer (&self->client, avahi_client_free); + g_hash_table_remove_all (self->resolvers); + + return G_SOURCE_REMOVE; +} diff --git a/src/libostree/ostree-repo-finder-avahi.h b/src/libostree/ostree-repo-finder-avahi.h new file mode 100644 index 0000000000..b1f9109a1d --- /dev/null +++ b/src/libostree/ostree-repo-finder-avahi.h @@ -0,0 +1,50 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright © 2017 Endless Mobile, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + * Authors: + * - Philip Withnall + */ + +#pragma once + +#include +#include +#include + +#include "ostree-repo-finder.h" +#include "ostree-types.h" + +G_BEGIN_DECLS + +#define OSTREE_TYPE_REPO_FINDER_AVAHI (ostree_repo_finder_avahi_get_type ()) + +_OSTREE_PUBLIC +G_DECLARE_FINAL_TYPE (OstreeRepoFinderAvahi, ostree_repo_finder_avahi, OSTREE, REPO_FINDER_AVAHI, GObject) + +_OSTREE_PUBLIC +OstreeRepoFinderAvahi *ostree_repo_finder_avahi_new (GMainContext *context); + +_OSTREE_PUBLIC +void ostree_repo_finder_avahi_start (OstreeRepoFinderAvahi *self, + GError **error); + +_OSTREE_PUBLIC +void ostree_repo_finder_avahi_stop (OstreeRepoFinderAvahi *self); + +G_END_DECLS diff --git a/src/libostree/ostree-repo-finder-config.c b/src/libostree/ostree-repo-finder-config.c new file mode 100644 index 0000000000..61b24fedea --- /dev/null +++ b/src/libostree/ostree-repo-finder-config.c @@ -0,0 +1,295 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright © 2017 Endless Mobile, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + * Authors: + * - Philip Withnall + */ + +#include "config.h" + +#include +#include +#include + +#include "ostree-repo-finder.h" +#include "ostree-repo-finder-config.h" + +/* TODO: Section documentation: + * - config directory structure + * - ref names are not escaped in path construction + * - config file format + */ + +static void ostree_repo_finder_config_iface_init (OstreeRepoFinderInterface *iface); + +struct _OstreeRepoFinderConfig +{ + GObject parent_instance; + + GFile *refs_directory; /* owned */ +}; + +G_DEFINE_TYPE_WITH_CODE (OstreeRepoFinderConfig, ostree_repo_finder_config, G_TYPE_OBJECT, + G_IMPLEMENT_INTERFACE (OSTREE_TYPE_REPO_FINDER, ostree_repo_finder_config_iface_init)) + +static void +ostree_repo_finder_config_resolve_async (OstreeRepoFinder *finder, + const gchar * const *refs, + GCancellable *cancellable, + GAsyncReadyCallback callback, + gpointer user_data) +{ + OstreeRepoFinderConfig *self = OSTREE_REPO_FINDER_CONFIG (finder); + g_autoptr(GTask) task = NULL; + g_autoptr(GPtrArray) results = NULL; + const gint priority = 100; /* arbitrarily chosen; lower than the others */ + gsize i; + g_autoptr(GHashTable) repo_uri_to_refs = NULL; /* (element-type utf8 GPtrArray) */ + GPtrArray *supported_refs; /* (element-type utf8) */ + GHashTableIter iter; + const gchar *repo_uri; + + task = g_task_new (finder, cancellable, callback, user_data); + g_task_set_source_tag (task, ostree_repo_finder_config_resolve_async); + results = g_ptr_array_new_with_free_func ((GDestroyNotify) ostree_repo_finder_result_free); + repo_uri_to_refs = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, (GDestroyNotify) g_ptr_array_unref); + + for (i = 0; refs[i] != NULL; i++) + { + g_autoptr(GKeyFile) key_file = NULL; + g_autoptr(GError) local_error = NULL; + g_autofree gchar *config_file_name = NULL; + g_autoptr(GFile) config_file = NULL; + g_autofree gchar *config_file_path = NULL; + g_autofree gchar *resolved_repo_uri = NULL; + + /* Load the ref’s configuration. */ + config_file_name = g_strconcat (refs[i], ".conf", NULL); + config_file = g_file_get_child (self->refs_directory, config_file_name); + config_file_path = g_file_get_path (config_file); + + key_file = g_key_file_new (); + g_key_file_load_from_file (key_file, config_file_path, G_KEY_FILE_NONE, &local_error); + + if (g_error_matches (local_error, G_FILE_ERROR, G_FILE_ERROR_NOENT)) + { + /* Config file does not exist. */ + g_debug ("Ignoring ref ‘%s’ as configuration file ‘%s’ does not exist.", + refs[i], config_file_path); + continue; + } + else if (local_error != NULL) + { + /* Error parsing the config file. */ + g_debug ("Ignoring ref ‘%s’ due to error parsing configuration file ‘%s’: %s", + refs[i], config_file_path, local_error->message); + g_clear_error (&local_error); + continue; + } + + /* Grab options from the config file. */ + resolved_repo_uri = g_key_file_get_string (key_file, "Remote", "url", &local_error); + + if (local_error != NULL && + !g_error_matches (local_error, G_KEY_FILE_ERROR, G_KEY_FILE_ERROR_KEY_NOT_FOUND) && + !g_error_matches (local_error, G_KEY_FILE_ERROR, G_KEY_FILE_ERROR_GROUP_NOT_FOUND)) + { + /* Error parsing the key value. */ + g_debug ("Ignoring ref ‘%s’ due to error reading from configuration file ‘%s’: %s", + refs[i], config_file_path, local_error->message); + g_clear_error (&local_error); + continue; + } + + g_assert (resolved_repo_uri != NULL); + + /* Configuration file /etc/ostree/refs.d/$refs[i].conf exists and gives + * a valid URI in its url= key. Add it to the results, keyed by + * the canonicalised repository URI to deduplicate the results. */ + g_debug ("Resolved ref ‘%s’ to repo URI ‘%s’.", + refs[i], resolved_repo_uri); + + supported_refs = g_hash_table_lookup (repo_uri_to_refs, resolved_repo_uri); + + if (supported_refs == NULL) + { + supported_refs = g_ptr_array_new_with_free_func (NULL); + g_hash_table_insert (repo_uri_to_refs, g_steal_pointer (&resolved_repo_uri), g_ptr_array_ref (supported_refs)); + } + + g_ptr_array_add (supported_refs, (gpointer) refs[i]); + } + + /* Aggregate the results. */ + g_hash_table_iter_init (&iter, repo_uri_to_refs); + + while (g_hash_table_iter_next (&iter, (gpointer *) &repo_uri, (gpointer *) &supported_refs)) + { + g_ptr_array_add (supported_refs, NULL); /* NULL terminator */ + + /* We don’t know what last-modified timestamp the remote has without + * making expensive HTTP queries, so leave that information blank. We + * assume that the configuration which says these @supported_refs are in + * the repository is correct; the code in ostree_repo_find_remotes() will + * check that. */ + g_ptr_array_add (results, ostree_repo_finder_result_new (repo_uri, priority, (const gchar * const *) supported_refs->pdata, 0)); + } + + g_task_return_pointer (task, g_steal_pointer (&results), (GDestroyNotify) g_ptr_array_unref); +} + +static GPtrArray * +ostree_repo_finder_config_resolve_finish (OstreeRepoFinder *finder, + GAsyncResult *result, + GError **error) +{ + g_return_val_if_fail (g_task_is_valid (result, finder), NULL); + return g_task_propagate_pointer (G_TASK (result), error); +} + +static void +ostree_repo_finder_config_init (OstreeRepoFinderConfig *self) +{ + /* Nothing to see here. */ +} + +static void +ostree_repo_finder_config_constructed (GObject *object) +{ + OstreeRepoFinderConfig *self = OSTREE_REPO_FINDER_CONFIG (object); + + G_OBJECT_CLASS (ostree_repo_finder_config_parent_class)->constructed (object); + + if (self->refs_directory == NULL) + self->refs_directory = g_file_new_for_path (SHORTENED_SYSCONFDIR "/ostree/refs.d"); +} + +typedef enum +{ + PROP_REFS_DIRECTORY = 1, +} OstreeRepoFinderConfigProperty; + +static void +ostree_repo_finder_config_get_property (GObject *object, + guint property_id, + GValue *value, + GParamSpec *pspec) +{ + OstreeRepoFinderConfig *self = OSTREE_REPO_FINDER_CONFIG (object); + + switch ((OstreeRepoFinderConfigProperty) property_id) + { + case PROP_REFS_DIRECTORY: + g_value_set_object (value, self->refs_directory); + break; + default: + g_assert_not_reached (); + } +} + +static void +ostree_repo_finder_config_set_property (GObject *object, + guint property_id, + const GValue *value, + GParamSpec *pspec) +{ + OstreeRepoFinderConfig *self = OSTREE_REPO_FINDER_CONFIG (object); + + switch ((OstreeRepoFinderConfigProperty) property_id) + { + case PROP_REFS_DIRECTORY: + /* Construct-only. */ + g_assert (self->refs_directory == NULL); + self->refs_directory = g_value_dup_object (value); + break; + default: + g_assert_not_reached (); + } +} + +static void +ostree_repo_finder_config_dispose (GObject *object) +{ + OstreeRepoFinderConfig *self = OSTREE_REPO_FINDER_CONFIG (object); + + g_clear_object (&self->refs_directory); + + G_OBJECT_CLASS (ostree_repo_finder_config_parent_class)->dispose (object); +} + +static void +ostree_repo_finder_config_class_init (OstreeRepoFinderConfigClass *klass) +{ + GObjectClass *object_class = G_OBJECT_CLASS (klass); + + object_class->get_property = ostree_repo_finder_config_get_property; + object_class->set_property = ostree_repo_finder_config_set_property; + object_class->constructed = ostree_repo_finder_config_constructed; + object_class->dispose = ostree_repo_finder_config_dispose; + + /** + * OstreeRepoFinderConfig:refs-directory: + * + * Directory containing configuration files for refs. Each ref’s configuration + * file is named after the ref, plus a `.conf` suffix. + * + * The default is `/etc/ostree/refs.d`. + * + * Since: 2017.4 + */ + g_object_class_install_property (object_class, PROP_REFS_DIRECTORY, + g_param_spec_object ("refs-directory", + "Refs Directory", + "Directory containing " + "configuration files " + "for refs.", + G_TYPE_FILE, + G_PARAM_CONSTRUCT_ONLY | + G_PARAM_READWRITE | + G_PARAM_STATIC_STRINGS)); +} + +static void +ostree_repo_finder_config_iface_init (OstreeRepoFinderInterface *iface) +{ + iface->resolve_async = ostree_repo_finder_config_resolve_async; + iface->resolve_finish = ostree_repo_finder_config_resolve_finish; +} + +/** + * ostree_repo_finder_config_new: + * @refs_directory: (nullable) (transfer none): refs directory to use, or %NULL to use + * the system default + * + * Create a new #OstreeRepoFinderConfig, loading configuration files from the + * given @refs_directory .If @refs_directory is %NULL, the system default will + * be used. + * + * Returns: (transfer full): a new #OstreeRepoFinderConfig + * Since: 2017.4 + */ +OstreeRepoFinderConfig * +ostree_repo_finder_config_new (GFile *refs_directory) +{ + g_return_val_if_fail (refs_directory == NULL || G_IS_FILE (refs_directory), NULL); + + return g_object_new (OSTREE_TYPE_REPO_FINDER_CONFIG, + "refs-directory", refs_directory, + NULL); +} diff --git a/src/libostree/ostree-repo-finder-config.h b/src/libostree/ostree-repo-finder-config.h new file mode 100644 index 0000000000..cc3eb0ef4f --- /dev/null +++ b/src/libostree/ostree-repo-finder-config.h @@ -0,0 +1,43 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright © 2017 Endless Mobile, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + * Authors: + * - Philip Withnall + */ + +#pragma once + +#include +#include +#include + +#include "ostree-repo-finder.h" +#include "ostree-types.h" + +G_BEGIN_DECLS + +#define OSTREE_TYPE_REPO_FINDER_CONFIG (ostree_repo_finder_config_get_type ()) + +_OSTREE_PUBLIC +G_DECLARE_FINAL_TYPE (OstreeRepoFinderConfig, ostree_repo_finder_config, OSTREE, REPO_FINDER_CONFIG, GObject) + +_OSTREE_PUBLIC +OstreeRepoFinderConfig *ostree_repo_finder_config_new (GFile *refs_directory); + +G_END_DECLS diff --git a/src/libostree/ostree-repo-finder-mount.c b/src/libostree/ostree-repo-finder-mount.c new file mode 100644 index 0000000000..e715d83f4e --- /dev/null +++ b/src/libostree/ostree-repo-finder-mount.c @@ -0,0 +1,474 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright © 2017 Endless Mobile, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + * Authors: + * - Philip Withnall + */ + +#include "config.h" + +#include +#include +#include + +#include "ostree-repo-finder.h" +#include "ostree-repo-finder-mount.h" + +/* TODO: Section documentation: + * - directory structure + * - use of symlinks + * - security properties + * - ref names are not escaped in path construction + */ + +typedef GList/**/ ObjectList; + +static void +object_list_free (ObjectList *list) +{ + g_list_free_full (list, g_object_unref); +} + +G_DEFINE_AUTOPTR_CLEANUP_FUNC (ObjectList, object_list_free) + +static void ostree_repo_finder_mount_iface_init (OstreeRepoFinderInterface *iface); + +struct _OstreeRepoFinderMount +{ + GObject parent_instance; + + GVolumeMonitor *monitor; /* owned */ +}; + +G_DEFINE_TYPE_WITH_CODE (OstreeRepoFinderMount, ostree_repo_finder_mount, G_TYPE_OBJECT, + G_IMPLEMENT_INTERFACE (OSTREE_TYPE_REPO_FINDER, ostree_repo_finder_mount_iface_init)) + +/* Get the child of @parent which is named @child_basename, repeatedly resolving + * it to its target if it’s a symlink. This does not resolve symlinks in + * @parent’s path. If more than 15 symlinks are chained, + * %G_IO_ERROR_TOO_MANY_LINKS will be returned. If @parent/@child_basename is + * not a symlink, it will be returned immediately. */ +static GFile * +expand_symlink (GFile *parent, + const gchar *child_basename, + GCancellable *cancellable, + GError **error) +{ + g_autoptr(GFile) resolved_file = NULL; + g_autofree gchar *parent_path = NULL; + guint i; + const guint n_attempts = 15; /* arbitrarily chosen */ + + resolved_file = g_file_get_child (parent, child_basename); + + for (i = 0; i < n_attempts; i++) + { + g_autoptr(GError) local_error = NULL; + g_autoptr(GFileInfo) file_info = NULL; + + file_info = g_file_query_info (resolved_file, + G_FILE_ATTRIBUTE_STANDARD_TYPE "," + G_FILE_ATTRIBUTE_STANDARD_SYMLINK_TARGET, + G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, + cancellable, + &local_error); + + if (local_error != NULL) + { + g_propagate_error (error, g_steal_pointer (&local_error)); + return NULL; + } + + if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_SYMBOLIC_LINK) + { + g_set_object (&resolved_file, + g_file_resolve_relative_path (parent, + g_file_info_get_symlink_target (file_info))); + continue; + } + else + { + return g_steal_pointer (&resolved_file); + } + } + + /* Reached the retry limit. */ + parent_path = g_file_get_path (parent); + g_set_error (error, G_IO_ERROR, G_IO_ERROR_TOO_MANY_LINKS, + "Too many symlink links when resolving ‘%s/%s’.", + parent_path, child_basename); + + return NULL; +} + +/* Resolve all symlinks in @file’s path, returning a normalised, canonical path + * containing no symlinks. This returns the same errors as expand_symlink(). */ +static GFile * +expand_all_symlinks (GFile *file, + GCancellable *cancellable, + GError **error) +{ + g_autoptr(GFile) parent = NULL; + + parent = g_file_get_parent (file); + + if (parent != NULL) + { + g_autoptr(GFile) parent_expanded = NULL; + g_autofree gchar *basename = NULL; + + parent_expanded = expand_all_symlinks (parent, cancellable, error); + if (parent_expanded == NULL) + return NULL; + + basename = g_file_get_basename (file); + return expand_symlink (parent, basename, cancellable, error); + } + else + { + /* Root directory. */ + return g_object_ref (file); + } +} + +static void +ostree_repo_finder_mount_resolve_async (OstreeRepoFinder *finder, + const gchar * const *refs, + GCancellable *cancellable, + GAsyncReadyCallback callback, + gpointer user_data) +{ + OstreeRepoFinderMount *self = OSTREE_REPO_FINDER_MOUNT (finder); + g_autoptr(GTask) task = NULL; + g_autoptr(ObjectList) volumes = NULL; + g_autoptr(GPtrArray) results = NULL; + GList *l; + const gint priority = 50; /* arbitrarily chosen */ + + task = g_task_new (finder, cancellable, callback, user_data); + g_task_set_source_tag (task, ostree_repo_finder_mount_resolve_async); + + volumes = g_volume_monitor_get_volumes (self->monitor); + results = g_ptr_array_new_with_free_func ((GDestroyNotify) ostree_repo_finder_result_free); + + for (l = volumes; l != NULL; l = l->next) + { + GVolume *volume = G_VOLUME (l->data); + g_autoptr(GDrive) drive = NULL; + g_autoptr(GMount) mount = NULL; + g_autofree gchar *volume_name = NULL; + g_autoptr(GFile) mount_root = NULL; + g_autoptr(GFile) ostree_dir = NULL; + g_autoptr(GFile) repos_dir = NULL; + g_autoptr(GFile) repo_dir = NULL; + g_autoptr(GFile) expanded_mount_root = NULL; + gsize i; + g_autoptr(GHashTable) repo_uri_to_refs = NULL; /* (element-type utf8 GPtrArray) */ + GPtrArray *supported_refs; /* (element-type utf8) */ + GHashTableIter iter; + const gchar *repo_uri; + g_autoptr(GError) local_error = NULL; + + drive = g_volume_get_drive (volume); + mount = g_volume_get_mount (volume); + volume_name = g_volume_get_name (volume); + + /* Check the drive’s general properties. */ + if (drive == NULL || mount == NULL) + { + g_debug ("Ignoring volume ‘%s’ due to NULL drive or mount.", + volume_name); + continue; + } + + if (!g_drive_is_removable (drive)) + { + g_debug ("Ignoring volume ‘%s’ as drive is not removable.", + volume_name); + continue; + } + + /* Check if it contains a .ostree/repos directory. */ + mount_root = g_mount_get_root (mount); + ostree_dir = g_file_get_child (mount_root, ".ostree"); + repos_dir = g_file_get_child (ostree_dir, "repos"); + + if (!g_file_query_exists (repos_dir, cancellable)) + { + g_autofree gchar *repos_dir_path = g_file_get_path (repos_dir); + g_debug ("Ignoring volume ‘%s’ as ‘%s’ directory doesn’t exist.", + volume_name, repos_dir_path); + continue; + } + + /* Expand all symlinks in the mount root so we can later check whether + * the resolved symlink targets for individual refs are descendents of it. */ + expanded_mount_root = expand_all_symlinks (mount_root, cancellable, &local_error); + + if (local_error != NULL) + { + g_autofree gchar *mount_root_path = g_file_get_path (mount_root); + g_debug ("Ignoring volume ‘%s’ as querying info of ‘%s’ failed: %s", + volume_name, mount_root_path, local_error->message); + g_clear_error (&local_error); + continue; + } + + /* Check whether a subdirectory exists for any of the @refs we’re looking + * for. If so, and it’s a symbolic link, dereference it so multiple links + * to the same repository (containing multiple refs) are coalesced. + * Otherwise, include it as a result by itself. */ + repo_uri_to_refs = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, (GDestroyNotify) g_ptr_array_unref); + + for (i = 0; refs[i] != NULL; i++) + { + g_autoptr(GFileInfo) file_info = NULL; + g_autofree gchar *repo_dir_path = NULL; + g_autofree gchar *resolved_repo_uri = NULL; + g_autoptr(GFile) repo_target = NULL; + g_autofree gchar *relative_path = NULL; + + repo_dir = g_file_get_child (repos_dir, refs[i]); + repo_dir_path = g_file_get_path (repo_dir); + + file_info = g_file_query_info (repo_dir, + G_FILE_ATTRIBUTE_STANDARD_TYPE, + G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, + cancellable, + &local_error); + + if (local_error != NULL) + { + g_debug ("Ignoring ref ‘%s’ on volume ‘%s’ as querying info of ‘%s’ failed: %s", + refs[i], volume_name, repo_dir_path, local_error->message); + g_clear_error (&local_error); + continue; + } + + switch (g_file_info_get_file_type (file_info)) + { + case G_FILE_TYPE_DIRECTORY: + repo_target = g_object_ref (repo_dir); + break; + case G_FILE_TYPE_SYMBOLIC_LINK: + repo_target = expand_all_symlinks (repo_dir, cancellable, &local_error); + break; + case G_FILE_TYPE_UNKNOWN: + case G_FILE_TYPE_REGULAR: + case G_FILE_TYPE_SPECIAL: + case G_FILE_TYPE_SHORTCUT: + case G_FILE_TYPE_MOUNTABLE: + default: + /* Nothing to do. */ + g_debug ("Ignoring ref ‘%s’ on volume ‘%s’ as ‘%s’ is of type %u, not a directory or symlink.", + refs[i], volume_name, repo_dir_path, g_file_info_get_file_type (file_info)); + continue; + } + + if (local_error != NULL) + { + g_debug ("Ignoring ref ‘%s’ on volume ‘%s’: %s", + refs[i], volume_name, local_error->message); + g_clear_error (&local_error); + continue; + } + + /* Check the resolved repository path is below the mount point. Do not + * allow ref symlinks to point somewhere outside of the mounted + * volume. */ + relative_path = g_file_get_relative_path (expanded_mount_root, repo_target); + + if (relative_path == NULL) + { + g_debug ("Ignoring ref ‘%s’ on volume ‘%s’ as it’s on a different file system from the mount.", + refs[i], volume_name); + continue; + } + + /* There is a valid repo at (or pointed to by) + * $mount_root/.ostree/repos/$refs[i]. Add it to the results, keyed by + * the canonicalised repository URI to deduplicate the results. */ + resolved_repo_uri = g_file_get_uri (repo_target); + g_debug ("Resolved ref ‘%s’ on volume ‘%s’ to repo URI ‘%s’.", + refs[i], volume_name, resolved_repo_uri); + + supported_refs = g_hash_table_lookup (repo_uri_to_refs, resolved_repo_uri); + + if (supported_refs == NULL) + { + supported_refs = g_ptr_array_new_with_free_func (NULL); + g_hash_table_insert (repo_uri_to_refs, g_steal_pointer (&resolved_repo_uri), g_ptr_array_ref (supported_refs)); + } + + g_ptr_array_add (supported_refs, (gpointer) refs[i]); + } + + /* Aggregate the results. */ + g_hash_table_iter_init (&iter, repo_uri_to_refs); + + while (g_hash_table_iter_next (&iter, (gpointer *) &repo_uri, (gpointer *) &supported_refs)) + { + g_ptr_array_add (supported_refs, NULL); /* NULL terminator */ + + /* Set the timestamp in the #OstreeRepoFinderResult to 0 because + * the code in ostree_repo_pull_from_remotes() will be able to check + * it just as quickly as we can here; so don’t duplicate the code. */ + g_ptr_array_add (results, ostree_repo_finder_result_new (repo_uri, priority, (const gchar * const *) supported_refs->pdata, 0)); + } + } + + g_task_return_pointer (task, g_steal_pointer (&results), (GDestroyNotify) g_ptr_array_unref); +} + +static GPtrArray * +ostree_repo_finder_mount_resolve_finish (OstreeRepoFinder *self, + GAsyncResult *result, + GError **error) +{ + g_return_val_if_fail (g_task_is_valid (result, self), NULL); + return g_task_propagate_pointer (G_TASK (result), error); +} + +static void +ostree_repo_finder_mount_init (OstreeRepoFinderMount *self) +{ + /* Nothing to see here. */ +} + +static void +ostree_repo_finder_mount_constructed (GObject *object) +{ + OstreeRepoFinderMount *self = OSTREE_REPO_FINDER_MOUNT (object); + + G_OBJECT_CLASS (ostree_repo_finder_mount_parent_class)->constructed (object); + + if (self->monitor == NULL) + self->monitor = g_volume_monitor_get (); +} + +typedef enum +{ + PROP_MONITOR = 1, +} OstreeRepoFinderMountProperty; + +static void +ostree_repo_finder_mount_get_property (GObject *object, + guint property_id, + GValue *value, + GParamSpec *pspec) +{ + OstreeRepoFinderMount *self = OSTREE_REPO_FINDER_MOUNT (object); + + switch ((OstreeRepoFinderMountProperty) property_id) + { + case PROP_MONITOR: + g_value_set_object (value, self->monitor); + break; + default: + g_assert_not_reached (); + } +} + +static void +ostree_repo_finder_mount_set_property (GObject *object, + guint property_id, + const GValue *value, + GParamSpec *pspec) +{ + OstreeRepoFinderMount *self = OSTREE_REPO_FINDER_MOUNT (object); + + switch ((OstreeRepoFinderMountProperty) property_id) + { + case PROP_MONITOR: + /* Construct-only. */ + g_assert (self->monitor == NULL); + self->monitor = g_value_dup_object (value); + break; + default: + g_assert_not_reached (); + } +} + +static void +ostree_repo_finder_mount_dispose (GObject *object) +{ + OstreeRepoFinderMount *self = OSTREE_REPO_FINDER_MOUNT (object); + + g_clear_object (&self->monitor); + + G_OBJECT_CLASS (ostree_repo_finder_mount_parent_class)->dispose (object); +} + +static void +ostree_repo_finder_mount_class_init (OstreeRepoFinderMountClass *klass) +{ + GObjectClass *object_class = G_OBJECT_CLASS (klass); + + object_class->get_property = ostree_repo_finder_mount_get_property; + object_class->set_property = ostree_repo_finder_mount_set_property; + object_class->constructed = ostree_repo_finder_mount_constructed; + object_class->dispose = ostree_repo_finder_mount_dispose; + + /** + * OstreeRepoFinderMount:monitor: + * + * Volume monitor to use to look up mounted volumes when queried. + * + * Since: 2017.4 + */ + g_object_class_install_property (object_class, PROP_MONITOR, + g_param_spec_object ("monitor", + "Volume Monitor", + "Volume monitor to use " + "to look up mounted " + "volumes when queried.", + G_TYPE_VOLUME_MONITOR, + G_PARAM_CONSTRUCT_ONLY | + G_PARAM_READWRITE | + G_PARAM_STATIC_STRINGS)); +} + +static void +ostree_repo_finder_mount_iface_init (OstreeRepoFinderInterface *iface) +{ + iface->resolve_async = ostree_repo_finder_mount_resolve_async; + iface->resolve_finish = ostree_repo_finder_mount_resolve_finish; +} + +/** + * ostree_repo_finder_mount_new: + * @monitor: (nullable) (transfer none): volume monitor to use, or %NULL to use + * the system default + * + * Create a new #OstreeRepoFinderMount, using the given @monitor to look up + * volumes. If @monitor is %NULL, the monitor from g_volume_monitor_get() will + * be used. + * + * Returns: (transfer full): a new #OstreeRepoFinderMount + * Since: 2017.4 + */ +OstreeRepoFinderMount * +ostree_repo_finder_mount_new (GVolumeMonitor *monitor) +{ + g_return_val_if_fail (monitor == NULL || G_IS_VOLUME_MONITOR (monitor), NULL); + + return g_object_new (OSTREE_TYPE_REPO_FINDER_MOUNT, + "monitor", monitor, + NULL); +} diff --git a/src/libostree/ostree-repo-finder-mount.h b/src/libostree/ostree-repo-finder-mount.h new file mode 100644 index 0000000000..24f2a56efe --- /dev/null +++ b/src/libostree/ostree-repo-finder-mount.h @@ -0,0 +1,43 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright © 2017 Endless Mobile, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + * Authors: + * - Philip Withnall + */ + +#pragma once + +#include +#include +#include + +#include "ostree-repo-finder.h" +#include "ostree-types.h" + +G_BEGIN_DECLS + +#define OSTREE_TYPE_REPO_FINDER_MOUNT (ostree_repo_finder_mount_get_type ()) + +_OSTREE_PUBLIC +G_DECLARE_FINAL_TYPE (OstreeRepoFinderMount, ostree_repo_finder_mount, OSTREE, REPO_FINDER_MOUNT, GObject) + +_OSTREE_PUBLIC +OstreeRepoFinderMount *ostree_repo_finder_mount_new (GVolumeMonitor *monitor); + +G_END_DECLS diff --git a/src/libostree/ostree-repo-finder.c b/src/libostree/ostree-repo-finder.c new file mode 100644 index 0000000000..0145d09cfe --- /dev/null +++ b/src/libostree/ostree-repo-finder.c @@ -0,0 +1,337 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright © 2017 Endless Mobile, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + * Authors: + * - Philip Withnall + */ + +#include "config.h" + +#include +#include +#include + +#include "ostree-repo-finder.h" +#ifdef HAVE_AVAHI +#include "ostree-repo-finder-avahi.h" +#endif /* HAVE_AVAHI */ +#include "ostree-repo-finder-config.h" +#include "ostree-repo-finder-mount.h" + +static void ostree_repo_finder_default_init (OstreeRepoFinderInterface *iface); + +G_DEFINE_INTERFACE (OstreeRepoFinder, ostree_repo_finder, G_TYPE_OBJECT) + +static void +ostree_repo_finder_default_init (OstreeRepoFinderInterface *iface) +{ + /* Nothing to see here. */ +} + +/* Validate the given string is potentially a ref name. */ +static gboolean +is_valid_ref_name (const gchar *ref_name) +{ + return (ref_name != NULL && *ref_name != '\0' && g_str_is_ascii (ref_name)); +} + +/* TODO: docs */ +static gboolean +is_valid_ref_array (const gchar * const *refs) +{ + gsize i; + + if (refs == NULL || *refs == NULL) + return FALSE; + + for (i = 0; refs[i] != NULL; i++) + { + if (!is_valid_ref_name (refs[i])) + return FALSE; + } + + return TRUE; +} + +static void resolve_cb (GObject *obj, + GAsyncResult *result, + gpointer user_data); + +/* TODO: Docs */ +void +ostree_repo_finder_resolve_async (OstreeRepoFinder *self, + const gchar * const *refs, + GCancellable *cancellable, + GAsyncReadyCallback callback, + gpointer user_data) +{ + OstreeRepoFinderInterface *iface; + g_autoptr(GTask) task = NULL; + + g_return_if_fail (OSTREE_IS_REPO_FINDER (self)); + g_return_if_fail (is_valid_ref_array (refs)); + g_return_if_fail (cancellable == NULL || G_IS_CANCELLABLE (cancellable)); + + task = g_task_new (self, cancellable, callback, user_data); + g_task_set_source_tag (task, ostree_repo_finder_resolve_async); + + iface = OSTREE_REPO_FINDER_GET_IFACE (self); + g_assert (iface->resolve_async != NULL); + g_assert (iface->resolve_finish != NULL); + + iface->resolve_async (self, refs, cancellable, resolve_cb, g_steal_pointer (&task)); +} + +static void +resolve_cb (GObject *obj, + GAsyncResult *result, + gpointer user_data) +{ + OstreeRepoFinder *self; + OstreeRepoFinderInterface *iface; + g_autoptr(GTask) task = NULL; + g_autoptr(GPtrArray) results = NULL; + g_autoptr(GError) local_error = NULL; + + self = OSTREE_REPO_FINDER (obj); + iface = OSTREE_REPO_FINDER_GET_IFACE (self); + task = G_TASK (user_data); + results = iface->resolve_finish (self, result, &local_error); + + g_assert ((local_error == NULL) != (results == NULL)); + + if (local_error != NULL) + g_task_return_error (task, g_steal_pointer (&local_error)); + else + g_task_return_pointer (task, g_steal_pointer (&results), (GDestroyNotify) g_ptr_array_unref); +} + +/* TODO: Docs */ +GPtrArray * +ostree_repo_finder_resolve_finish (OstreeRepoFinder *self, + GAsyncResult *result, + GError **error) +{ + g_return_val_if_fail (OSTREE_IS_REPO_FINDER (self), NULL); + g_return_val_if_fail (g_task_is_valid (result, self), NULL); + g_return_val_if_fail (error == NULL || *error == NULL, NULL); + + return g_task_propagate_pointer (G_TASK (result), error); +} + +static gint +sort_results_cb (gconstpointer a, + gconstpointer b) +{ + const OstreeRepoFinderResult *result_a = *((const OstreeRepoFinderResult **) a); + const OstreeRepoFinderResult *result_b = *((const OstreeRepoFinderResult **) b); + + return ostree_repo_finder_result_compare (result_a, result_b); +} + +typedef struct +{ + gsize n_finders_pending; + GPtrArray *results; +} ResolveAllData; + +static void +resolve_all_data_free (ResolveAllData *data) +{ + g_assert (data->n_finders_pending == 0); + g_ptr_array_unref (data->results); + g_free (data); +} + +G_DEFINE_AUTOPTR_CLEANUP_FUNC (ResolveAllData, resolve_all_data_free) + +static void resolve_all_cb (GObject *obj, + GAsyncResult *result, + gpointer user_data); +static void resolve_all_finished_one (GTask *task); + +/* TODO: Docs */ +void +ostree_repo_finder_resolve_all_async (OstreeRepoFinder * const *finders, + const gchar * const *refs, + GCancellable *cancellable, + GAsyncReadyCallback callback, + gpointer user_data) +{ + g_autoptr(GTask) task = NULL; + g_autoptr(ResolveAllData) data = NULL; + gsize i; + + g_return_if_fail (is_valid_ref_array (refs)); + g_return_if_fail (cancellable == NULL || G_IS_CANCELLABLE (cancellable)); + + task = g_task_new (NULL, cancellable, callback, user_data); + g_task_set_source_tag (task, ostree_repo_finder_resolve_all_async); + + data = g_new0 (ResolveAllData, 1); + data->n_finders_pending = 1; /* while setting up the loop */ + data->results = g_ptr_array_new_with_free_func ((GDestroyNotify) ostree_repo_finder_result_free); + g_task_set_task_data (task, data, (GDestroyNotify) resolve_all_data_free); + + /* Start all the asynchronous queries in parallel. */ + for (i = 0; finders[i] != NULL; i++) + { + OstreeRepoFinder *finder = OSTREE_REPO_FINDER (finders[i]); + OstreeRepoFinderInterface *iface; + + iface = OSTREE_REPO_FINDER_GET_IFACE (finder); + g_assert (iface->resolve_async != NULL); + iface->resolve_async (finder, refs, cancellable, resolve_all_cb, g_object_ref (task)); + data->n_finders_pending++; + } + + resolve_all_finished_one (task); + data = NULL; /* passed to the GTask above */ +} + +static void +array_concatenate (GPtrArray *array, + GPtrArray *to_concatenate) +{ + gsize i; + + for (i = 0; i < to_concatenate->len; i++) + g_ptr_array_add (array, g_steal_pointer (&g_ptr_array_index (to_concatenate, i))); +} + +static void +resolve_all_cb (GObject *obj, + GAsyncResult *result, + gpointer user_data) +{ + OstreeRepoFinder *finder; + OstreeRepoFinderInterface *iface; + g_autoptr(GTask) task = NULL; + g_autoptr(GPtrArray) results = NULL; + g_autoptr(GError) local_error = NULL; + ResolveAllData *data; + + finder = OSTREE_REPO_FINDER (obj); + iface = OSTREE_REPO_FINDER_GET_IFACE (finder); + task = G_TASK (user_data); + data = g_task_get_task_data (task); + results = iface->resolve_finish (finder, result, &local_error); + + g_assert (local_error == NULL || results == NULL || results->len == 0); + + if (local_error != NULL) + g_warning ("Error resolving key ID to repository URI using %s: %s", + g_type_name (G_TYPE_FROM_INSTANCE (finder)), local_error->message); + else + array_concatenate (data->results, results); + + resolve_all_finished_one (task); +} + +static void +resolve_all_finished_one (GTask *task) +{ + ResolveAllData *data; + + data = g_task_get_task_data (task); + + data->n_finders_pending--; + + if (data->n_finders_pending == 0) + { + g_ptr_array_sort (data->results, sort_results_cb); + g_task_return_pointer (task, g_steal_pointer (&data->results), (GDestroyNotify) g_ptr_array_unref); + } +} + +/* TODO: Docs */ +GPtrArray * +ostree_repo_finder_resolve_all_finish (GAsyncResult *result, + GError **error) +{ + g_return_val_if_fail (g_task_is_valid (result, NULL), NULL); + g_return_val_if_fail (error == NULL || *error == NULL, NULL); + + return g_task_propagate_pointer (G_TASK (result), error); +} + +static gboolean +is_valid_uri (const gchar *uri) +{ + g_autoptr(GSocketConnectable) connectable = NULL; + + connectable = g_network_address_parse_uri (uri, 1, NULL); + + return (connectable != NULL); +} + +/* TODO: Docs */ +OstreeRepoFinderResult * +ostree_repo_finder_result_new (const gchar *uri, + gint priority, + const gchar * const *refs, + guint64 summary_last_modified) +{ + g_autoptr(OstreeRepoFinderResult) result = NULL; + + g_return_val_if_fail (is_valid_uri (uri), NULL); + g_return_val_if_fail (is_valid_ref_array (refs), NULL); + + result = g_new0 (OstreeRepoFinderResult, 1); + result->uri = g_strdup (uri); + result->priority = priority; + result->refs = g_strdupv ((gchar **) refs); + result->summary_last_modified = summary_last_modified; + + return g_steal_pointer (&result); +} + +/* TODO: Docs */ +gint +ostree_repo_finder_result_compare (const OstreeRepoFinderResult *a, + const OstreeRepoFinderResult *b) +{ + guint a_n_refs, b_n_refs; + + if (a->priority != b->priority) + return a->priority - b->priority; + + if (a->summary_last_modified != 0 && b->summary_last_modified != 0 && + a->summary_last_modified != b->summary_last_modified) + return a->summary_last_modified - b->summary_last_modified; + + a_n_refs = g_strv_length (a->refs); + b_n_refs = g_strv_length (b->refs); + + if (a_n_refs != b_n_refs) + return (gint) a_n_refs - (gint) b_n_refs; + + return g_strcmp0 (a->uri, b->uri); +} + +/* TODO: Docs */ +void +ostree_repo_finder_result_free (OstreeRepoFinderResult *result) +{ + g_return_if_fail (result != NULL); + + g_strfreev (result->refs); + g_free (result->uri); + g_free (result); +} diff --git a/src/libostree/ostree-repo-finder.h b/src/libostree/ostree-repo-finder.h new file mode 100644 index 0000000000..2de0d2668d --- /dev/null +++ b/src/libostree/ostree-repo-finder.h @@ -0,0 +1,116 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright © 2017 Endless Mobile, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + * Authors: + * - Philip Withnall + */ + +#pragma once + +#include +#include +#include + +#include "ostree-types.h" + +G_BEGIN_DECLS + +#define OSTREE_TYPE_REPO_FINDER (ostree_repo_finder_get_type ()) + +_OSTREE_PUBLIC +G_DECLARE_INTERFACE (OstreeRepoFinder, ostree_repo_finder, OSTREE, REPO_FINDER, GObject) + +struct _OstreeRepoFinderInterface +{ + GTypeInterface g_iface; + + void (*resolve_async) (OstreeRepoFinder *self, + const gchar * const *refs, + GCancellable *cancellable, + GAsyncReadyCallback callback, + gpointer user_data); + GPtrArray *(*resolve_finish) (OstreeRepoFinder *self, + GAsyncResult *result, + GError **error); +}; + +_OSTREE_PUBLIC +void ostree_repo_finder_resolve_async (OstreeRepoFinder *self, + const gchar * const *refs, + GCancellable *cancellable, + GAsyncReadyCallback callback, + gpointer user_data); +_OSTREE_PUBLIC +GPtrArray *ostree_repo_finder_resolve_finish (OstreeRepoFinder *self, + GAsyncResult *result, + GError **error); + +_OSTREE_PUBLIC +void ostree_repo_finder_resolve_all_async (OstreeRepoFinder * const *finders, + const gchar * const *refs, + GCancellable *cancellable, + GAsyncReadyCallback callback, + gpointer user_data); +_OSTREE_PUBLIC +GPtrArray *ostree_repo_finder_resolve_all_finish (GAsyncResult *result, + GError **error); + +/** + * OstreeRepoFinderResult: + * @uri: TODO + * @priority: TODO + * + * TODO: basically a structure which says ‘you can download these refs from + * this URI, and they will be up to date’, along with some relevant metadata. + * + * Since: TODO + */ +typedef struct +{ + /* TODO: Should I extend this to also support mirrors and metalink? Maybe add a GVariant *metadata too? + * For example, should a result be able to force GPG verification? */ + /* TODO: would it support torrenting? */ + gchar *uri; + gint priority; + gchar **refs; /* refs which are in this remote, unique, in ascending lexicographic order TODO enforce this*/ + guint64 summary_last_modified; + + /*< private >*/ + gpointer padding[4]; +} OstreeRepoFinderResult; + + +/* TODO: Make OstreeRepoFinderResult introspectable. */ + +_OSTREE_PUBLIC +OstreeRepoFinderResult *ostree_repo_finder_result_new (const gchar *uri, + gint priority, + const gchar * const *refs, + guint64 summary_last_modified); + +_OSTREE_PUBLIC +gint ostree_repo_finder_result_compare (const OstreeRepoFinderResult *a, + const OstreeRepoFinderResult *b); + +_OSTREE_PUBLIC +void ostree_repo_finder_result_free (OstreeRepoFinderResult *result); + +G_DEFINE_AUTOPTR_CLEANUP_FUNC (OstreeRepoFinderResult, ostree_repo_finder_result_free) + +G_END_DECLS diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index ec085a81fe..fcec857b30 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -1,6 +1,7 @@ /* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- * * Copyright (C) 2011,2012,2013 Colin Walters + * Copyright © 2017 Endless Mobile, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -17,7 +18,9 @@ * Free Software Foundation, Inc., 59 Temple Place - Suite 330, * Boston, MA 02111-1307, USA. * - * Author: Colin Walters + * Authors: + * - Colin Walters + * - Philip Withnall */ #include "config.h" @@ -33,6 +36,12 @@ #include "ostree-repo-static-delta-private.h" #include "ostree-metalink.h" #include "ostree-fetcher-util.h" +#include "ostree-repo-finder.h" +#include "ostree-repo-finder-config.h" +#include "ostree-repo-finder-mount.h" +#ifdef HAVE_AVAHI +#include "ostree-repo-finder-avahi.h" +#endif /* HAVE_AVAHI */ #include "ot-fs-utils.h" #include @@ -3465,6 +3474,598 @@ ostree_repo_pull_with_options (OstreeRepo *self, return ret; } +static void +find_remotes_cb (GObject *obj, + GAsyncResult *result, + gpointer user_data) +{ + GAsyncResult **result_out = user_data; + *result_out = g_object_ref (result); +} + +/* TODO: Docs */ +typedef struct +{ + gchar *checksum; /* always set */ + guint64 commit_size; /* always set */ + guint64 timestamp; /* 0 for unknown */ + GVariant *additional_metadata; + GArray *refs; /* indexes to refs which point to this commit on at least one remote */ +} CommitMetadata; + +static void +commit_metadata_free (CommitMetadata *info) +{ + g_clear_pointer (&info->refs, g_array_unref); + g_free (info->checksum); + g_clear_pointer (&info->additional_metadata, g_variant_unref); + g_free (info); +} + +G_DEFINE_AUTOPTR_CLEANUP_FUNC (CommitMetadata, commit_metadata_free) + +static CommitMetadata * +commit_metadata_new (const gchar *checksum, + guint64 commit_size, + guint64 timestamp, + GVariant *additional_metadata) +{ + g_autoptr(CommitMetadata) info = NULL; + + info = g_new0 (CommitMetadata, 1); + info->checksum = g_strdup (checksum); + info->commit_size = commit_size; + info->timestamp = timestamp; + info->additional_metadata = (additional_metadata != NULL) ? g_variant_ref (additional_metadata) : NULL; + info->refs = g_array_new (FALSE, FALSE, sizeof (gsize)); + + return g_steal_pointer (&info); +} + +typedef struct +{ + gsize width; /* pointers */ + gsize height; /* pointers */ + gconstpointer pointers[]; /* n_pointers = width * height */ +} PointerTable; + +static void +pointer_table_free (PointerTable *table) +{ + g_free (table); +} + +G_DEFINE_AUTOPTR_CLEANUP_FUNC (PointerTable, pointer_table_free) + +/* Both dimensions in pointers. */ +static PointerTable * +pointer_table_new (gsize width, + gsize height) +{ + g_autoptr(PointerTable) table = NULL; + + g_return_val_if_fail (width > 0, NULL); + g_return_val_if_fail (height > 0, NULL); + /* TODO: overflow checks */ + + table = g_malloc0 (sizeof (PointerTable) + sizeof (gconstpointer) * width * height); + table->width = width; + table->height = height; + + return g_steal_pointer (&table); +} + +static gconstpointer +pointer_table_get (const PointerTable *table, + gsize x, + gsize y) +{ + g_return_val_if_fail (table != NULL, FALSE); + g_return_val_if_fail (x < table->width, FALSE); + g_return_val_if_fail (y < table->height, FALSE); + + return table->pointers[table->width * y + x]; +} + +static void +pointer_table_set (PointerTable *table, + gsize x, + gsize y, + gconstpointer value) +{ + g_return_if_fail (table != NULL); + g_return_if_fail (x < table->width); + g_return_if_fail (y < table->height); + + table->pointers[table->width * y + x] = value; +} + +static gboolean +strv_find (const gchar * const *haystack, + const gchar *needle, + gsize *out_index) +{ + gsize i; + + for (i = 0; haystack[i] != NULL; i++) + { + if (g_strcmp0 (haystack[i], needle) == 0) + { + *out_index = i; + return TRUE; + } + } + + return FALSE; +} + +/* TODO: Add another standard summary attribute for the expected summary file TTL */ +#define OSTREE_SUMMARY_LAST_MODIFIED "ostree.summary.last-modified" +#define OSTREE_COMMIT_TIMESTAMP "ostree.commit.timestamp" + +/* TODO: Docs */ +OstreeRepoFinderResult ** +ostree_repo_find_remotes (OstreeRepo *self, + const gchar * const *refs, + GVariant *options, + OstreeAsyncProgress *progress, + GCancellable *cancellable, + GError **error) +{ + /* TODO: Support disabling certain repo finders. */ + + g_autoptr(GMainContext) context = NULL; + g_autoptr(GAsyncResult) result = NULL; + g_autoptr(GPtrArray) results = NULL; /* (element-type OstreeRepoFinderResult) */ + gsize i; + GHashTableIter iter; + CommitMetadata *commit_metadata; + g_autoptr(PointerTable) refs_and_remotes_table = NULL; /* (element-type commit-checksum) */ + g_autoptr(GHashTable) commit_metadatas = NULL; /* (element-type commit-checksum CommitMetadata) */ + g_autoptr(OstreeFetcher) fetcher = NULL; + const gchar **ref_to_latest_commit; /* indexed as @refs; (element-type commit-checksum) */ + gsize n_refs; + const gchar *checksum; + g_autoptr(OstreeRepoFinder) finder_config = NULL; + g_autoptr(OstreeRepoFinder) finder_mount = NULL; + g_autoptr(OstreeRepoFinder) finder_avahi = NULL; + OstreeRepoFinder *finders[4] = { NULL, }; + + g_return_val_if_fail (refs != NULL && *refs != NULL, NULL); + /* TODO: More preconditions */ + + /* TODO: Go back through the mailing list thread and check all the optimisations + * we discussed have been implemented. */ + /* TODO: We currently do nothing with @progress. */ + /* TODO: think about security here: how can we secure this all? */ + + /* Synchronously resolve all possible remotes for the given refs. */ + context = g_main_context_new (); + + /* TODO: work out a sensible way of centralising this while keeping it testable */ + + finder_config = OSTREE_REPO_FINDER (ostree_repo_finder_config_new (NULL)); + finder_mount = OSTREE_REPO_FINDER (ostree_repo_finder_mount_new (NULL)); +#ifdef HAVE_AVAHI + finder_avahi = OSTREE_REPO_FINDER (ostree_repo_finder_avahi_new (context)); +#endif /* HAVE_AVAHI */ + + finders[0] = finder_config; + finders[1] = finder_mount; + finders[2] = finder_avahi; + + result = NULL; + ostree_repo_finder_resolve_all_async (finders, refs, cancellable, + find_remotes_cb, &result); + + while (result == NULL) + g_main_context_iteration (context, TRUE); + + results = ostree_repo_finder_resolve_all_finish (result, error); + + /* TODO: do we distinguish an empty result set from error? */ + if (results == NULL || results->len == 0) + return NULL; + + commit_metadatas = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, (GDestroyNotify) commit_metadata_free); + + /* X dimension is an index into @refs. Y dimension is an index into @results. + * Each cell stores the commit checksum which that ref resolves to on that + * remote, or %NULL if the remote doesn’t have that ref. */ + n_refs = g_strv_length ((gchar **) refs); + refs_and_remotes_table = pointer_table_new (n_refs, results->len); + + /* Fetch and validate the summary file for each result. */ + /* FIXME: All these downloads could be parallelised. */ + for (i = 0; i < results->len; i++) + { + OstreeRepoFinderResult *result = g_ptr_array_index (results, i); + g_autoptr(GBytes) summary_bytes = NULL, summary_sig_bytes = NULL; + g_autoptr(GVariant) summary_v = NULL; + g_autoptr(GError) local_error = NULL; + g_autoptr(GVariant) options = NULL; + const gchar *ref_name; + guint64 summary_last_modified; + g_autoptr(GVariant) summary_refs = NULL; + gsize n, j; + g_autoptr(GVariant) additional_metadata_v = NULL; + + /* TODO: any other relevant options? */ + options = g_variant_new_parsed ("{ 'override-url': <%s> }", + result->uri); + + /* TODO: double-check this does validate the signature */ + /* TODO: can we optimise this out in some cases? */ + /* TODO: Does this use If-Modified-Since with the cache, if signed? */ + ostree_repo_remote_fetch_summary_with_options (self, + "", /* no remote */ + options, + &summary_bytes, + &summary_sig_bytes, + cancellable, + &local_error); + + if (g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_CANCELLED)) + { + g_propagate_error (error, g_steal_pointer (&local_error)); + return FALSE; + } + else if (local_error != NULL) + { + g_debug ("%s: Failed to download summary for result ‘%s’. Ignoring.", + G_STRFUNC, result->uri); + g_clear_pointer (&g_ptr_array_index (results, i), (GDestroyNotify) ostree_repo_finder_result_free); + continue; + } + + /* Check the metadata in the summary file, especially whether it contains + * all the @refs we are interested in. */ + summary_v = g_variant_new_from_bytes (OSTREE_SUMMARY_GVARIANT_FORMAT, + summary_bytes, FALSE); + summary_refs = g_variant_get_child_value (summary_v, 0); + n = g_variant_n_children (summary_refs); + for (j = 0; j < n; j++) + { + const guchar *csum_bytes; + g_autoptr(GVariant) ref_v = NULL, csum_v = NULL, commit_metadata_v = NULL; + guint64 commit_size, commit_timestamp; + gchar tmp_checksum[OSTREE_SHA256_STRING_LEN + 1]; + gsize ref_index; + + /* Check the ref name. */ + ref_v = g_variant_get_child_value (summary_refs, i); + g_variant_get_child (ref_v, 0, "&s", &ref_name); + + /* TODO: Are the allowed and recommended formats for naming refs documented + * somewhere? */ + if (!ostree_validate_rev (ref_name, &local_error)) + { + g_debug ("%s: Summary for result ‘%s’ contained invalid ref name ‘%s’: %s", + G_STRFUNC, result->uri, ref_name, local_error->message); + g_clear_error (&local_error); + break; + } + + /* Check the commit checksum. */ + g_variant_get_child (ref_v, 1, "(t@ay@a{sv})", &commit_size, &csum_v, &commit_metadata_v); + + csum_bytes = ostree_checksum_bytes_peek_validate (csum_v, &local_error); + if (csum_bytes == NULL) + { + g_debug ("%s: Summary for result ‘%s’ contained invalid ref checksum: %s", + G_STRFUNC, result->uri, local_error->message); + g_clear_error (&local_error); + break; + } + + ostree_checksum_inplace_from_bytes (csum_bytes, tmp_checksum); + + /* Is this a ref we care about? */ + if (!strv_find (refs, ref_name, &ref_index)) + continue; + + /* TODO: We probably also want to load the commit from the local repo + * too, if it exists, since that will be the most common case of all + * the peers advertising the commit we’re already on. */ + + /* Check the additional metadata. */ + /* TODO: ensure this is put into generated summary files */ + if (!g_variant_lookup (commit_metadata_v, OSTREE_COMMIT_TIMESTAMP, "t", &commit_timestamp)) + commit_timestamp = 0; /* unknown */ + + /* Check and store the commit metadata. */ + commit_metadata = g_hash_table_lookup (commit_metadatas, tmp_checksum); + + if (commit_metadata == NULL) + { + commit_metadata = commit_metadata_new (tmp_checksum, commit_size, commit_timestamp, NULL); + g_hash_table_insert (commit_metadatas, g_strdup (tmp_checksum), + commit_metadata /* transfer */); + } + + /* Update the metadata if possible. */ + if (commit_metadata->timestamp == 0) + { + commit_metadata->timestamp = commit_timestamp; + } + else if (commit_timestamp != 0 && commit_metadata->timestamp != commit_timestamp) + { + g_debug ("%s: Summary for result ‘%s’ contained commit timestamp %" G_GUINT64_FORMAT " which did not match existing timestamp %" G_GUINT64_FORMAT ". Ignoring.", + G_STRFUNC, result->uri, commit_timestamp, commit_metadata->timestamp); + break; + } + + pointer_table_set (refs_and_remotes_table, ref_index, i, commit_metadata->checksum); + g_array_append_val (commit_metadata->refs, ref_index); + } + + /* Error in the refs loop? */ + if (j != n) + { + g_clear_pointer (&g_ptr_array_index (results, i), (GDestroyNotify) ostree_repo_finder_result_free); + continue; + } + + /* Check the summary’s additional metadata. */ + additional_metadata_v = g_variant_get_child_value (summary_v, 1); + + /* TODO: ensure this is put into generated summary files */ + if (!g_variant_lookup (additional_metadata_v, OSTREE_SUMMARY_LAST_MODIFIED, "t", &summary_last_modified)) + summary_last_modified = 0; + + /* Update the stored result data. Clear the @refs array, since it’s been + * moved to @refs_and_remotes_table and is now potentially out of date. */ + g_clear_pointer (&result->refs, g_strfreev); + result->summary_last_modified = summary_last_modified; + } + + /* Fill in any gaps in the metadata for the most recent commits by pulling + * the commit metadata from the remotes. */ + /* TODO: construct a fetcher properly, paying attention to all the options */ + fetcher = _ostree_fetcher_new (self->tmp_dir_fd /* TODO */, "", 0); + g_hash_table_iter_init (&iter, commit_metadatas); + + while (g_hash_table_iter_next (&iter, (gpointer *) &checksum, (gpointer *) &commit_metadata)) + { + char buf[_OSTREE_LOOSE_PATH_MAX]; + g_autofree gchar *commit_filename = NULL; + g_autoptr(GPtrArray) mirrorlist = NULL; + g_autoptr(GBytes) commit_bytes = NULL; + g_autoptr(GVariant) commit_v = NULL; + guint64 commit_timestamp; + + /* Already complete? */ + if (commit_metadata->timestamp != 0) + continue; + + _ostree_loose_path (buf, commit_metadata->checksum, OSTREE_OBJECT_TYPE_COMMIT, OSTREE_REPO_MODE_ARCHIVE_Z2); + commit_filename = g_build_filename ("objects", buf, NULL); + + /* Build a @mirrorlist for this ref from the remotes whose summary files + * contain it. TODO: Sort the mirrorlist so the fastest-to-access remotes + * are first. */ + mirrorlist = g_ptr_array_new_with_free_func (NULL); + + for (i = 0; i < commit_metadata->refs->len; i++) + { + gsize ref_index = g_array_index (commit_metadata->refs, gsize, i); + gsize j; + + for (j = 0; j < results->len; j++) + { + if (pointer_table_get (refs_and_remotes_table, ref_index, i) == commit_metadata->checksum) + { + OstreeRepoFinderResult *result = g_ptr_array_index (results, i); + g_ptr_array_add (mirrorlist, result->uri); + } + } + } + + /* TODO: ensure this checks the commit signature; or that the checksum matches the downloaded content */ + if (!_ostree_fetcher_mirrored_request_to_membuf (fetcher, + mirrorlist, + commit_filename, + FALSE, /* don’t add trailing nul */ + FALSE, /* return an error on ENOENT */ + &commit_bytes, + 0, /* no maximum size */ + cancellable, + error)) + return NULL; + + /* Parse the commit metadata. */ + commit_v = g_variant_new_from_bytes (OSTREE_COMMIT_GVARIANT_FORMAT, + commit_bytes, FALSE); + g_variant_get_child (commit_v, 5, "t", &commit_timestamp); + + /* Update the #CommitMetadata. */ + commit_metadata->timestamp = commit_timestamp; + } + + /* TODO: The data structures round here are a complete mess and need a fresh eye. */ + /* Find the latest commit for each ref. */ + ref_to_latest_commit = g_new0 (const gchar *, n_refs); + + for (i = 0; i < n_refs; i++) + { + gsize j; + const gchar *latest_checksum = NULL; + const CommitMetadata *latest_commit_metadata = NULL; + + for (j = 0; j < results->len; j++) + { + const CommitMetadata *candidate_commit_metadata; + const gchar *candidate_checksum; + + candidate_checksum = pointer_table_get (refs_and_remotes_table, i, j); + + if (candidate_checksum == NULL) + continue; + + candidate_commit_metadata = g_hash_table_lookup (commit_metadatas, candidate_checksum); + g_assert (candidate_commit_metadata != NULL); + + if (latest_commit_metadata == NULL || + candidate_commit_metadata->timestamp > latest_commit_metadata->timestamp) + { + latest_checksum = candidate_checksum; + latest_commit_metadata = candidate_commit_metadata; + } + } + + ref_to_latest_commit[i] = latest_checksum; + } + + /* Recombine @commit_metadatas and @results to include the list of refs each + * remote contains in #OstreeRepoFinderResult.refs. */ + for (i = 0; i < results->len; i++) + { + OstreeRepoFinderResult *result = g_ptr_array_index (results, i); + g_autoptr(GPtrArray) validated_refs = NULL; + gsize j; + + validated_refs = g_ptr_array_new_with_free_func (g_free); + + for (j = 0; refs[j] != NULL; j++) + { + const gchar *latest_commit_for_ref = ref_to_latest_commit[j]; + + if (pointer_table_get (refs_and_remotes_table, j, i) == latest_commit_for_ref) + g_ptr_array_add (validated_refs, g_strdup (refs[j])); + } + + result->refs = (gchar **) g_ptr_array_free (g_steal_pointer (&validated_refs), FALSE); + } + + /* Ensure the updated results are still in priority order. */ + g_ptr_array_sort (results, (GCompareFunc) ostree_repo_finder_result_compare); + + return g_steal_pointer (&results); +} + +/** + * ostree_repo_pull_from_remotes: + * @self: an #OstreeRepo + * @results: (array zero-terminated=1): %NULL-terminated array of remotes to + * pull from, including the refs to pull from each + * @options: TODO + * @progress: TODO + * @cancellable: (nullable): a #GCancellable, or %NULL + * @error: return location for a #GError, or %NULL + * + * Pull refs from multiple remotes which have been found using + * ostree_repo_find_remotes(). + * TODO: More documentation + * + * Returns: %TRUE on success, %FALSE otherwise + * Since: 2017.4 + */ +gboolean +ostree_repo_pull_from_remotes (OstreeRepo *self, + const OstreeRepoFinderResult * const *results, + GVariant *options, + OstreeAsyncProgress *progress, + GCancellable *cancellable, + GError **error) +{ + g_autoptr(GHashTable) refs_pulled = NULL; /* (element-type utf8 utf8) */ + g_autoptr(GHashTable) refs_not_pulled = NULL; /* (element-type utf8 utf8) */ + gsize i, j; + + /* Keep track of the set of refs we’ve pulled already. */ + refs_pulled = g_hash_table_new_full (g_str_hash, g_str_equal, NULL, NULL); + refs_not_pulled = g_hash_table_new_full (g_str_hash, g_str_equal, NULL, NULL); + + /* FIXME: Rework this code to pull in parallel where possible. At the moment + * we expect the (i == 0) iteration will do all the work (all the refs) and + * subsequent iterations are only there in case of error. */ + for (i = 0; results[i] != NULL; i++) + { + const OstreeRepoFinderResult *result = results[i]; + g_autoptr(GError) local_error = NULL; + g_autoptr(GVariant) local_options = NULL; + g_autoptr(GPtrArray) refs_to_pull = NULL; + + refs_to_pull = g_ptr_array_new_with_free_func (NULL); + + for (j = 0; result->refs[j] != NULL; j++) + if (!g_hash_table_contains (refs_pulled, result->refs[j])) + g_ptr_array_add (refs_to_pull, result->refs[j]); + + if (refs_to_pull->len == 0) + { + g_debug ("Ignoring remote ‘%s’ as it has no relevent refs.", result->uri); + continue; + } + + /* NULL terminator. */ + g_ptr_array_add (refs_to_pull, NULL); + + /* TODO: any other relevant options here? GPG verification? */ + local_options = g_variant_new_parsed ("{" + "'override-url': <%s>," + "'refs': <%as>," + "}", + result->uri, + refs_to_pull->pdata); + + /* TODO: Progress will yo-yo here. */ + ostree_repo_pull_with_options (self, result->uri, local_options, progress, + cancellable, &local_error); + + if (g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_CANCELLED)) + { + g_propagate_error (error, g_steal_pointer (&local_error)); + return FALSE; + } + else if (local_error != NULL) + { + g_debug ("Failed to pull refs from ‘%s’: %s", result->uri, local_error->message); + + for (j = 0; refs_to_pull->pdata[j] != NULL; j++) + g_hash_table_add (refs_not_pulled, refs_to_pull->pdata[j]); + + continue; + } + else + { + g_debug ("Pulled refs from ‘%s’.", result->uri); + + for (j = 0; refs_to_pull->pdata[j] != NULL; j++) + { + g_hash_table_add (refs_pulled, refs_to_pull->pdata[j]); + g_hash_table_remove (refs_not_pulled, refs_to_pull->pdata[j]); + } + } + } + + /* Any refs left un-downloaded? If so, we’ve failed. */ + if (g_hash_table_size (refs_not_pulled) > 0) + { + g_autoptr(GString) refs_string = NULL; + GHashTableIter iter; + const gchar *ref; + + refs_string = g_string_new (""); + g_hash_table_iter_init (&iter, refs_not_pulled); + + while (g_hash_table_iter_next (&iter, (gpointer *) &ref, NULL)) + { + if (refs_string->len != 0) + g_string_append (refs_string, ", "); + + g_string_append (refs_string, ref); + } + + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Failed to pull some refs from the remotes: %s", + refs_string->str); + return FALSE; + } + + return TRUE; +} + /** * ostree_repo_remote_fetch_summary_with_options: * @self: Self diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h index 34685cc6c2..79257ad1fb 100644 --- a/src/libostree/ostree-repo.h +++ b/src/libostree/ostree-repo.h @@ -25,6 +25,7 @@ #include "ostree-core.h" #include "ostree-types.h" #include "ostree-async-progress.h" +#include "ostree-repo-finder.h" #include "ostree-sepolicy.h" #include "ostree-gpg-verify-result.h" @@ -1040,6 +1041,22 @@ gboolean ostree_repo_pull_with_options (OstreeRepo *self, GCancellable *cancellable, GError **error); +_OSTREE_PUBLIC +OstreeRepoFinderResult **ostree_repo_find_remotes (OstreeRepo *self, + const gchar * const *refs, + GVariant *options, + OstreeAsyncProgress *progress, + GCancellable *cancellable, + GError **error); + +_OSTREE_PUBLIC +gboolean ostree_repo_pull_from_remotes (OstreeRepo *self, + const OstreeRepoFinderResult * const *results, + GVariant *options, + OstreeAsyncProgress *progress, + GCancellable *cancellable, + GError **error); + _OSTREE_PUBLIC void ostree_repo_pull_default_console_progress_changed (OstreeAsyncProgress *progress, gpointer user_data); diff --git a/tests/.gitignore b/tests/.gitignore index afa0b23388..9bec67a36d 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -1,8 +1,12 @@ !Makefile +*.log +*.test +*.trs ostree-http-server run-apache tmpdir-lifecycle test-rollsum +test-bloom test-bsdiff test-checksum test-gpg-verify-result @@ -11,4 +15,7 @@ test-mutable-tree test-ot-opt-utils test-ot-tool-util test-ot-unix-utils +test-repo-finder-avahi +test-repo-finder-config +test-repo-finder-mount test-rollsum-cli diff --git a/tests/test-bloom.c b/tests/test-bloom.c new file mode 100644 index 0000000000..561bdd618d --- /dev/null +++ b/tests/test-bloom.c @@ -0,0 +1,148 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright © 2017 Endless Mobile, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + * Authors: + * - Philip Withnall + */ + +#include "config.h" + +#include +#include + +#include "ostree-bloom-private.h" + +/* Test the two different constructors work at a basic level. */ +static void +test_bloom_init (void) +{ + g_autoptr(OstreeBloom) bloom = NULL; + g_autoptr(GBytes) bytes = NULL; + + bloom = ostree_bloom_new (1, 1, ostree_str_bloom_hash); + g_clear_pointer (&bloom, ostree_bloom_unref); + + bytes = g_bytes_new_take (g_malloc0 (4), 4); + bloom = ostree_bloom_new_from_bytes (bytes, 1, ostree_str_bloom_hash); + g_clear_pointer (&bloom, ostree_bloom_unref); +} + +/* Test that building a bloom filter, marshalling it through GBytes, and loading + * it again, gives the same element membership. */ +static void +test_bloom_construction (void) +{ + g_autoptr(OstreeBloom) bloom = NULL; + g_autoptr(OstreeBloom) immutable_bloom = NULL; + g_autoptr(GBytes) bytes = NULL; + gsize i; + const gchar *members[] = + { + "hello", "there", "these", "are", "test", "strings" + }; + const gchar *non_members[] = + { + "not", "an", "element" + }; + const gsize n_bytes = 256; + const guint8 k = 8; + const OstreeBloomHashFunc hash = ostree_str_bloom_hash; + + /* Build a bloom filter. */ + bloom = ostree_bloom_new (n_bytes, k, hash); + + for (i = 0; i < G_N_ELEMENTS (members); i++) + ostree_bloom_add_element (bloom, members[i]); + + bytes = ostree_bloom_seal (bloom); + + /* Read it back from the GBytes. */ + immutable_bloom = ostree_bloom_new_from_bytes (bytes, k, hash); + + for (i = 0; i < G_N_ELEMENTS (members); i++) + g_assert_true (ostree_bloom_maybe_contains (bloom, members[i])); + + /* This should never fail in future, as we guarantee the hash function will + * never change. But given the definition of a bloom filter, it would also + * be valid for these calls to return %TRUE. */ + for (i = 0; i < G_N_ELEMENTS (non_members); i++) + g_assert_false (ostree_bloom_maybe_contains (bloom, non_members[i])); +} + +/* Test that an empty bloom filter definitely contains no elements. */ +static void +test_bloom_empty (void) +{ + g_autoptr(OstreeBloom) bloom = NULL; + const gsize n_bytes = 256; + const guint8 k = 8; + const OstreeBloomHashFunc hash = ostree_str_bloom_hash; + + /* Build an empty bloom filter. */ + bloom = ostree_bloom_new (n_bytes, k, hash); + + g_assert_false (ostree_bloom_maybe_contains (bloom, "hello")); + g_assert_false (ostree_bloom_maybe_contains (bloom, "there")); +} + +/* Build a bloom filter, and check the membership of the members as they are + * added. */ +static void +test_bloom_membership_during_construction (void) +{ + g_autoptr(OstreeBloom) bloom = NULL; + gsize i, j; + const gchar *members[] = + { + "hello", "there", "these", "are", "test", "strings" + }; + const gsize n_bytes = 256; + const guint8 k = 8; + const OstreeBloomHashFunc hash = ostree_str_bloom_hash; + + /* These membership checks should never fail in future, as we guarantee + * the hash function will never change. But given the definition of a bloom + * filter, it would also be valid for these checks to fail. */ + bloom = ostree_bloom_new (n_bytes, k, hash); + + for (i = 0; i < G_N_ELEMENTS (members); i++) + { + ostree_bloom_add_element (bloom, members[i]); + + for (j = 0; j < G_N_ELEMENTS (members); j++) + { + if (j <= i) + g_assert_true (ostree_bloom_maybe_contains (bloom, members[j])); + else + g_assert_false (ostree_bloom_maybe_contains (bloom, members[j])); + } + } +} + +int main (int argc, char **argv) +{ + g_test_init (&argc, &argv, NULL); + + g_test_add_func ("/bloom/init", test_bloom_init); + g_test_add_func ("/bloom/construction", test_bloom_construction); + g_test_add_func ("/bloom/empty", test_bloom_empty); + g_test_add_func ("/bloom/membership-during-construction", test_bloom_membership_during_construction); + + return g_test_run(); +} diff --git a/tests/test-mock-gio.c b/tests/test-mock-gio.c new file mode 100644 index 0000000000..ab02805163 --- /dev/null +++ b/tests/test-mock-gio.c @@ -0,0 +1,291 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright © 2017 Endless Mobile, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + * Authors: + * - Philip Withnall + */ + +#include "config.h" + +#include +#include +#include + +#include "test-mock-gio.h" + +/* TODO: Section docs; class docs; blog about this */ + +/* Mock volume monitor class. This returns a static set of data to the caller, + * which it was initialised with. */ +struct _OstreeMockVolumeMonitor +{ + GVolumeMonitor parent_instance; + + GList *volumes; /* (element-type OstreeMockVolume) */ +}; + +G_DEFINE_TYPE (OstreeMockVolumeMonitor, ostree_mock_volume_monitor, G_TYPE_VOLUME_MONITOR) + +static GList * +ostree_mock_volume_monitor_get_volumes (GVolumeMonitor *monitor) +{ + OstreeMockVolumeMonitor *self = OSTREE_MOCK_VOLUME_MONITOR (monitor); + return g_list_copy_deep (self->volumes, (GCopyFunc) g_object_ref, NULL); +} + +static void +ostree_mock_volume_monitor_init (OstreeMockVolumeMonitor *self) +{ + /* Nothing to see here. */ +} + +static void +ostree_mock_volume_monitor_dispose (GObject *object) +{ + OstreeMockVolumeMonitor *self = OSTREE_MOCK_VOLUME_MONITOR (object); + + g_list_free_full (self->volumes, g_object_unref); + self->volumes = NULL; + + G_OBJECT_CLASS (ostree_mock_volume_monitor_parent_class)->dispose (object); +} + +static void +ostree_mock_volume_monitor_class_init (OstreeMockVolumeMonitorClass *klass) +{ + GObjectClass *object_class = G_OBJECT_CLASS (klass); + GVolumeMonitorClass *monitor_class = G_VOLUME_MONITOR_CLASS (klass); + + object_class->dispose = ostree_mock_volume_monitor_dispose; + + monitor_class->get_volumes = ostree_mock_volume_monitor_get_volumes; +} + +/* TODO: Docs */ +GVolumeMonitor * +ostree_mock_volume_monitor_new (GList *volumes) +{ + g_autoptr(OstreeMockVolumeMonitor) monitor = NULL; + + monitor = g_object_new (OSTREE_TYPE_MOCK_VOLUME_MONITOR, NULL); + monitor->volumes = g_list_copy_deep (volumes, (GCopyFunc) g_object_ref, NULL); + + return g_steal_pointer (&monitor); +} + +/* Mock volume class. This returns a static set of data to the caller, which it + * was initialised with. */ +struct _OstreeMockVolume +{ + GObject parent_instance; + + gchar *name; + GDrive *drive; /* (owned) (nullable) */ + GMount *mount; /* (owned) (nullable) */ +}; + +static void ostree_mock_volume_iface_init (GVolumeIface *iface); + +G_DEFINE_TYPE_WITH_CODE (OstreeMockVolume, ostree_mock_volume, G_TYPE_OBJECT, + G_IMPLEMENT_INTERFACE (G_TYPE_VOLUME, ostree_mock_volume_iface_init)) + +static gchar * +ostree_mock_volume_get_name (GVolume *volume) +{ + OstreeMockVolume *self = OSTREE_MOCK_VOLUME (volume); + return g_strdup (self->name); +} + +static GDrive * +ostree_mock_volume_get_drive (GVolume *volume) +{ + OstreeMockVolume *self = OSTREE_MOCK_VOLUME (volume); + return (self->drive != NULL) ? g_object_ref (self->drive) : NULL; +} + +static GMount * +ostree_mock_volume_get_mount (GVolume *volume) +{ + OstreeMockVolume *self = OSTREE_MOCK_VOLUME (volume); + return (self->mount != NULL) ? g_object_ref (self->mount) : NULL; +} + +static void +ostree_mock_volume_init (OstreeMockVolume *self) +{ + /* Nothing to see here. */ +} + +static void +ostree_mock_volume_dispose (GObject *object) +{ + OstreeMockVolume *self = OSTREE_MOCK_VOLUME (object); + + g_clear_pointer (&self->name, g_free); + g_clear_object (&self->drive); + g_clear_object (&self->mount); + + G_OBJECT_CLASS (ostree_mock_volume_parent_class)->dispose (object); +} + +static void +ostree_mock_volume_class_init (OstreeMockVolumeClass *klass) +{ + GObjectClass *object_class = G_OBJECT_CLASS (klass); + + object_class->dispose = ostree_mock_volume_dispose; +} + +static void +ostree_mock_volume_iface_init (GVolumeIface *iface) +{ + iface->get_name = ostree_mock_volume_get_name; + iface->get_drive = ostree_mock_volume_get_drive; + iface->get_mount = ostree_mock_volume_get_mount; +} + +/* TODO: Docs */ +OstreeMockVolume * +ostree_mock_volume_new (const gchar *name, + GDrive *drive, + GMount *mount) +{ + g_autoptr(OstreeMockVolume) volume = NULL; + + volume = g_object_new (OSTREE_TYPE_MOCK_VOLUME, NULL); + volume->name = g_strdup (name); + volume->drive = (drive != NULL) ? g_object_ref (drive) : NULL; + volume->mount = (mount != NULL) ? g_object_ref (mount) : NULL; + + return g_steal_pointer (&volume); +} + +/* Mock drive class. This returns a static set of data to the caller, which it + * was initialised with. */ +struct _OstreeMockDrive +{ + GObject parent_instance; + + gboolean is_removable; +}; + +static void ostree_mock_drive_iface_init (GDriveIface *iface); + +G_DEFINE_TYPE_WITH_CODE (OstreeMockDrive, ostree_mock_drive, G_TYPE_OBJECT, + G_IMPLEMENT_INTERFACE (G_TYPE_DRIVE, ostree_mock_drive_iface_init)) + +static gboolean +ostree_mock_drive_is_removable (GDrive *drive) +{ + OstreeMockDrive *self = OSTREE_MOCK_DRIVE (drive); + return self->is_removable; +} + +static void +ostree_mock_drive_init (OstreeMockDrive *self) +{ + /* Nothing to see here. */ +} + +static void +ostree_mock_drive_class_init (OstreeMockDriveClass *klass) +{ + /* Nothing to see here. */ +} + +static void +ostree_mock_drive_iface_init (GDriveIface *iface) +{ + iface->is_removable = ostree_mock_drive_is_removable; +} + +/* TODO: Docs */ +OstreeMockDrive * +ostree_mock_drive_new (gboolean is_removable) +{ + g_autoptr(OstreeMockDrive) drive = NULL; + + drive = g_object_new (OSTREE_TYPE_MOCK_DRIVE, NULL); + drive->is_removable = is_removable; + + return g_steal_pointer (&drive); +} + +/* Mock mount class. This returns a static set of data to the caller, which it + * was initialised with. */ +struct _OstreeMockMount +{ + GObject parent_instance; + + GFile *root; /* (owned) */ +}; + +static void ostree_mock_mount_iface_init (GMountIface *iface); + +G_DEFINE_TYPE_WITH_CODE (OstreeMockMount, ostree_mock_mount, G_TYPE_OBJECT, + G_IMPLEMENT_INTERFACE (G_TYPE_MOUNT, ostree_mock_mount_iface_init)) + +static GFile * +ostree_mock_mount_get_root (GMount *mount) +{ + OstreeMockMount *self = OSTREE_MOCK_MOUNT (mount); + return g_object_ref (self->root); +} + +static void +ostree_mock_mount_init (OstreeMockMount *self) +{ + /* Nothing to see here. */ +} + +static void +ostree_mock_mount_dispose (GObject *object) +{ + OstreeMockMount *self = OSTREE_MOCK_MOUNT (object); + + g_clear_object (&self->root); + + G_OBJECT_CLASS (ostree_mock_mount_parent_class)->dispose (object); +} + +static void +ostree_mock_mount_class_init (OstreeMockMountClass *klass) +{ + GObjectClass *object_class = G_OBJECT_CLASS (klass); + + object_class->dispose = ostree_mock_mount_dispose; +} + +static void +ostree_mock_mount_iface_init (GMountIface *iface) +{ + iface->get_root = ostree_mock_mount_get_root; +} + +/* TODO: Docs */ +OstreeMockMount * +ostree_mock_mount_new (GFile *root) +{ + g_autoptr(OstreeMockMount) mount = NULL; + + mount = g_object_new (OSTREE_TYPE_MOCK_MOUNT, NULL); + mount->root = g_object_ref (root); + + return g_steal_pointer (&mount); +} diff --git a/tests/test-mock-gio.h b/tests/test-mock-gio.h new file mode 100644 index 0000000000..1932516195 --- /dev/null +++ b/tests/test-mock-gio.h @@ -0,0 +1,68 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright © 2017 Endless Mobile, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + * Authors: + * - Philip Withnall + */ + +#pragma once + +#include +#include +#include + +#include "ostree-types.h" + +G_BEGIN_DECLS + +#define OSTREE_TYPE_MOCK_VOLUME_MONITOR (ostree_mock_volume_monitor_get_type ()) + +G_GNUC_INTERNAL +G_DECLARE_FINAL_TYPE (OstreeMockVolumeMonitor, ostree_mock_volume_monitor, OSTREE, MOCK_VOLUME_MONITOR, GVolumeMonitor) + +G_GNUC_INTERNAL +GVolumeMonitor *ostree_mock_volume_monitor_new (GList *volumes); + +#define OSTREE_TYPE_MOCK_VOLUME (ostree_mock_volume_get_type ()) + +G_GNUC_INTERNAL +G_DECLARE_FINAL_TYPE (OstreeMockVolume, ostree_mock_volume, OSTREE, MOCK_VOLUME, GObject) + +G_GNUC_INTERNAL +OstreeMockVolume *ostree_mock_volume_new (const gchar *name, + GDrive *drive, + GMount *mount); + +#define OSTREE_TYPE_MOCK_DRIVE (ostree_mock_drive_get_type ()) + +G_GNUC_INTERNAL +G_DECLARE_FINAL_TYPE (OstreeMockDrive, ostree_mock_drive, OSTREE, MOCK_DRIVE, GObject) + +G_GNUC_INTERNAL +OstreeMockDrive *ostree_mock_drive_new (gboolean is_removable); + +#define OSTREE_TYPE_MOCK_MOUNT (ostree_mock_mount_get_type ()) + +G_GNUC_INTERNAL +G_DECLARE_FINAL_TYPE (OstreeMockMount, ostree_mock_mount, OSTREE, MOCK_MOUNT, GObject) + +G_GNUC_INTERNAL +OstreeMockMount *ostree_mock_mount_new (GFile *root); + +G_END_DECLS diff --git a/tests/test-repo-finder-avahi.c b/tests/test-repo-finder-avahi.c new file mode 100644 index 0000000000..ac2bbb820f --- /dev/null +++ b/tests/test-repo-finder-avahi.c @@ -0,0 +1,226 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright © 2017 Endless Mobile, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + * Authors: + * - Philip Withnall + */ + +#include "config.h" + +#include +#include +#include +#include +#include + +#include "ostree-repo-finder.h" +#include "ostree-repo-finder-avahi.h" +#include "ostree-repo-finder-avahi-private.h" + +/* TODO: Upstream this */ +G_DEFINE_AUTOPTR_CLEANUP_FUNC (AvahiStringList, avahi_string_list_free) + +/* Test the object constructor works at a basic level. */ +static void +test_repo_finder_avahi_init (void) +{ + g_autoptr(OstreeRepoFinderAvahi) finder = NULL; + g_autoptr(GMainContext) context = NULL; + + /* Default #GVolumeMonitor. */ + finder = ostree_repo_finder_avahi_new (NULL); + g_clear_object (&finder); + + /* Explicit main context. */ + context = g_main_context_new (); + finder = ostree_repo_finder_avahi_new (context); + g_clear_object (&finder); +} + +/* Test parsing valid and invalid TXT records. */ +static void +test_repo_finder_avahi_txt_records_parse (void) +{ + struct + { + const guint8 *txt; + gsize txt_len; + const gchar *expected_key; /* (nullable) to indicate parse failure */ + const guint8 *expected_value; /* (nullable) to allow for valueless keys */ + gsize expected_value_len; + } + vectors[] = + { + { (const guint8 *) "", 0, NULL, NULL, 0 }, + { (const guint8 *) "\x00", 1, NULL, NULL, 0 }, + { (const guint8 *) "\xff", 1, NULL, NULL, 0 }, + { (const guint8 *) "k\x00", 2, NULL, NULL, 0 }, + { (const guint8 *) "k\xff", 2, NULL, NULL, 0 }, + { (const guint8 *) "=", 1, NULL, NULL, 0 }, + { (const guint8 *) "=value", 6, NULL, NULL, 0 }, + { (const guint8 *) "k=v", 3, "k", (const guint8 *) "v", 1 }, + { (const guint8 *) "key=value", 9, "key", (const guint8 *) "value", 5 }, + { (const guint8 *) "k=v=", 4, "k", (const guint8 *) "v=", 2 }, + { (const guint8 *) "k=", 2, "k", (const guint8 *) "", 0 }, + { (const guint8 *) "k", 1, "k", NULL, 0 }, + { (const guint8 *) "k==", 3, "k", (const guint8 *) "=", 1 }, + { (const guint8 *) "k=\x00\x01\x02", 5, "k", (const guint8 *) "\x00\x01\x02", 3 }, + }; + gsize i; + + for (i = 0; i < G_N_ELEMENTS (vectors); i++) + { + g_autoptr(AvahiStringList) string_list = NULL; + g_autoptr(GHashTable) attributes = NULL; + + g_test_message ("Vector %" G_GSIZE_FORMAT, i); + + string_list = avahi_string_list_add_arbitrary (NULL, vectors[i].txt, vectors[i].txt_len); + + attributes = _ostree_txt_records_parse (string_list); + + if (vectors[i].expected_key != NULL) + { + GBytes *value; + g_autoptr(GBytes) expected_value = NULL; + + g_assert_true (g_hash_table_lookup_extended (attributes, + vectors[i].expected_key, + NULL, + (gpointer *) &value)); + g_assert_cmpuint (g_hash_table_size (attributes), ==, 1); + + if (vectors[i].expected_value != NULL) + { + g_assert_nonnull (value); + expected_value = g_bytes_new_static (vectors[i].expected_value, vectors[i].expected_value_len); + g_assert_true (g_bytes_equal (value, expected_value)); + } + else + { + g_assert_null (value); + } + } + else + { + g_assert_cmpuint (g_hash_table_size (attributes), ==, 0); + } + } +} + +/* Test that the first value for a set of duplicate records is returned. + * See RFC 6763, §6.4. */ +static void +test_repo_finder_avahi_txt_records_duplicates (void) +{ + g_autoptr(AvahiStringList) string_list = NULL; + g_autoptr(GHashTable) attributes = NULL; + GBytes *value; + g_autoptr(GBytes) expected_value = NULL; + + /* Reverse the list before using it, as they are built in reverse order. + * (See the #AvahiStringList documentation.) */ + string_list = avahi_string_list_new ("k=value1", "k=value2", "k=value3", NULL); + string_list = avahi_string_list_reverse (string_list); + attributes = _ostree_txt_records_parse (string_list); + + g_assert_cmpuint (g_hash_table_size (attributes), ==, 1); + value = g_hash_table_lookup (attributes, "k"); + g_assert_nonnull (value); + + expected_value = g_bytes_new_static ("value1", strlen ("value1")); + g_assert_true (g_bytes_equal (value, expected_value)); +} + +/* Test that keys are parsed and looked up case insensitively. + * See RFC 6763, §6.4. */ +static void +test_repo_finder_avahi_txt_records_case_sensitivity (void) +{ + g_autoptr(AvahiStringList) string_list = NULL; + g_autoptr(GHashTable) attributes = NULL; + GBytes *value1, *value2; + g_autoptr(GBytes) expected_value1 = NULL, expected_value2 = NULL; + + /* Reverse the list before using it, as they are built in reverse order. + * (See the #AvahiStringList documentation.) */ + string_list = avahi_string_list_new ("k=value1", + "K=value2", + "KeY2=v", + NULL); + string_list = avahi_string_list_reverse (string_list); + attributes = _ostree_txt_records_parse (string_list); + + g_assert_cmpuint (g_hash_table_size (attributes), ==, 2); + + value1 = g_hash_table_lookup (attributes, "k"); + g_assert_nonnull (value1); + expected_value1 = g_bytes_new_static ("value1", strlen ("value1")); + g_assert_true (g_bytes_equal (value1, expected_value1)); + + g_assert_null (g_hash_table_lookup (attributes, "K")); + + value2 = g_hash_table_lookup (attributes, "key2"); + g_assert_nonnull (value2); + expected_value2 = g_bytes_new_static ("v", 1); + g_assert_true (g_bytes_equal (value2, expected_value2)); + + g_assert_null (g_hash_table_lookup (attributes, "KeY2")); +} + +/* Test that keys which have an empty value can be distinguished from those + * which have no value. See RFC 6763, §6.4. */ +static void +test_repo_finder_avahi_txt_records_empty_and_missing (void) +{ + g_autoptr(AvahiStringList) string_list = NULL; + g_autoptr(GHashTable) attributes = NULL; + GBytes *value1, *value2; + g_autoptr(GBytes) expected_value1 = NULL; + + string_list = avahi_string_list_new ("empty=", + "missing", + NULL); + attributes = _ostree_txt_records_parse (string_list); + + g_assert_cmpuint (g_hash_table_size (attributes), ==, 2); + + value1 = g_hash_table_lookup (attributes, "empty"); + g_assert_nonnull (value1); + expected_value1 = g_bytes_new_static ("", 0); + g_assert_true (g_bytes_equal (value1, expected_value1)); + + g_assert_true (g_hash_table_lookup_extended (attributes, "missing", NULL, (gpointer *) &value2)); + g_assert_null (value2); +} + +int main (int argc, char **argv) +{ + setlocale (LC_ALL, ""); + g_test_init (&argc, &argv, NULL); + + g_test_add_func ("/repo-finder-avahi/init", test_repo_finder_avahi_init); + g_test_add_func ("/repo-finder-avahi/txt-records/parse", test_repo_finder_avahi_txt_records_parse); + g_test_add_func ("/repo-finder-avahi/txt-records/duplicates", test_repo_finder_avahi_txt_records_duplicates); + g_test_add_func ("/repo-finder-avahi/txt-records/case-sensitivity", test_repo_finder_avahi_txt_records_case_sensitivity); + g_test_add_func ("/repo-finder-avahi/txt-records/empty-and-missing", test_repo_finder_avahi_txt_records_empty_and_missing); + /* TODO: Add tests for Avahi functionality. */ + + return g_test_run(); +} diff --git a/tests/test-repo-finder-config.c b/tests/test-repo-finder-config.c new file mode 100644 index 0000000000..2e0a194e19 --- /dev/null +++ b/tests/test-repo-finder-config.c @@ -0,0 +1,271 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright © 2017 Endless Mobile, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + * Authors: + * - Philip Withnall + */ + +#include "config.h" + +#include +#include +#include +#include +#include + +#include "ostree-repo-finder.h" +#include "ostree-repo-finder-config.h" + +/* Test fixture. Creates a temporary directory. */ +typedef struct +{ + GFile *refs_directory; +} Fixture; + +static void +setup (Fixture *fixture, + gconstpointer test_data) +{ + g_autofree gchar *tmp_dir_path = NULL; + g_autoptr(GError) error = NULL; + + tmp_dir_path = g_dir_make_tmp ("test-repo-finder-config-XXXXXX", &error); + g_assert_no_error (error); + + fixture->refs_directory = g_file_new_for_path (tmp_dir_path); +} + +/* Recursively delete @file and its children. @file may be a file or a directory. */ +static gboolean +rm_rf (GFile *file, + GCancellable *cancellable, + GError **error) +{ + g_autoptr(GFileEnumerator) enumerator = NULL; + + enumerator = g_file_enumerate_children (file, G_FILE_ATTRIBUTE_STANDARD_NAME, + G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, + cancellable, NULL); + + while (enumerator != NULL) + { + GFile *child; + + if (!g_file_enumerator_iterate (enumerator, NULL, &child, cancellable, error)) + return FALSE; + if (child == NULL) + break; + if (!rm_rf (child, cancellable, error)) + return FALSE; + } + + return g_file_delete (file, cancellable, error); +} + +static void +teardown (Fixture *fixture, + gconstpointer test_data) +{ + /* Recursively remove the temporary directory. */ + rm_rf (fixture->refs_directory, NULL, NULL); +} + +/* Test the object constructor works at a basic level. */ +static void +test_repo_finder_config_init (void) +{ + g_autoptr(OstreeRepoFinderConfig) finder = NULL; + g_autoptr(GFile) dir = NULL; + + /* Default #GVolumeMonitor. */ + finder = ostree_repo_finder_config_new (NULL); + g_clear_object (&finder); + + /* Explicit directory. */ + dir = g_file_new_for_path ("blah"); + finder = ostree_repo_finder_config_new (dir); + g_clear_object (&finder); +} + +static void +result_cb (GObject *source_object, + GAsyncResult *result, + gpointer user_data) +{ + GAsyncResult **result_out = user_data; + *result_out = g_object_ref (result); +} + +/* Test that no remotes are found if there are no config files in the refs + * directory. */ +static void +test_repo_finder_config_no_configs (Fixture *fixture, + gconstpointer test_data) +{ + g_autoptr(OstreeRepoFinderConfig) finder = NULL; + g_autoptr(GMainContext) context = NULL; + g_autoptr(GAsyncResult) result = NULL; + g_autoptr(GPtrArray) results = NULL; /* (element-type OstreeRepoFinderResult) */ + g_autoptr(GError) error = NULL; + const gchar * const refs[] = + { + "exampleos/x86_64/standard", + "exampleos/x86_64/buildmaster/standard", + NULL + }; + + context = g_main_context_new (); + g_main_context_push_thread_default (context); + + finder = ostree_repo_finder_config_new (fixture->refs_directory); + + ostree_repo_finder_resolve_async (OSTREE_REPO_FINDER (finder), refs, + NULL, result_cb, &result); + + while (result == NULL) + g_main_context_iteration (context, TRUE); + + results = ostree_repo_finder_resolve_finish (OSTREE_REPO_FINDER (finder), + result, &error); + g_assert_no_error (error); + g_assert_nonnull (results); + g_assert_cmpuint (results->len, ==, 0); + + g_main_context_pop_thread_default (context); +} + +/* Create a config file named $ref_name.conf in the given @refs_directory, + * containing @repo_uri as the configuration. If @repo_uri is %NULL, the config + * file will contain invalid content (to trigger parser failure). */ +static void +assert_create_ref_config (GFile *refs_directory, + const gchar *ref_name, + const gchar *repo_uri) +{ + g_autofree gchar *ref_file_name = NULL; + g_autoptr(GFile) ref_file = NULL; + g_autoptr(GFile) ref_parent_dir = NULL; + g_autofree gchar *config_file_contents = NULL; + g_autoptr(GError) error = NULL; + + /* The @ref_parent_dir is not necessarily @refs_directory, since @ref_name + * may contain slashes. */ + ref_file_name = g_strconcat (ref_name, ".conf", NULL); + ref_file = g_file_get_child (refs_directory, ref_file_name); + ref_parent_dir = g_file_get_parent (ref_file); + g_file_make_directory_with_parents (ref_parent_dir, NULL, &error); + if (g_error_matches (error, G_IO_ERROR, G_IO_ERROR_EXISTS)) + g_clear_error (&error); + g_assert_no_error (error); + + if (repo_uri != NULL) + config_file_contents = g_strdup_printf ("[Remote]\nurl=%s\n", repo_uri); + else + config_file_contents = g_strdup ("an invalid config file"); + + g_file_replace_contents (ref_file, config_file_contents, + strlen (config_file_contents), NULL, FALSE, + G_FILE_CREATE_NONE, NULL, NULL, &error); + g_assert_no_error (error); +} + +/* Test resolving the refs against a collection of config files, which contain + * valid, invalid or duplicate repo information. */ +static void +test_repo_finder_config_mixed_configs (Fixture *fixture, + gconstpointer test_data) +{ + g_autoptr(OstreeRepoFinderConfig) finder = NULL; + g_autoptr(GMainContext) context = NULL; + g_autoptr(GAsyncResult) result = NULL; + g_autoptr(GPtrArray) results = NULL; /* (element-type OstreeRepoFinderResult) */ + g_autoptr(GError) error = NULL; + gsize i; + const gchar * const refs[] = + { + "exampleos/x86_64/ref0", + "exampleos/x86_64/ref1", + "exampleos/x86_64/ref2", + "exampleos/x86_64/ref3", + NULL + }; + + context = g_main_context_new (); + g_main_context_push_thread_default (context); + + /* Put together various ref configuration files. */ + assert_create_ref_config (fixture->refs_directory, refs[0], "http://ref0"); + assert_create_ref_config (fixture->refs_directory, refs[1], "http://ref1"); + assert_create_ref_config (fixture->refs_directory, refs[2], "http://ref0"); + assert_create_ref_config (fixture->refs_directory, refs[3], NULL); + + finder = ostree_repo_finder_config_new (fixture->refs_directory); + + /* Resolve the refs. */ + ostree_repo_finder_resolve_async (OSTREE_REPO_FINDER (finder), refs, + NULL, result_cb, &result); + + while (result == NULL) + g_main_context_iteration (context, TRUE); + + results = ostree_repo_finder_resolve_finish (OSTREE_REPO_FINDER (finder), + result, &error); + g_assert_no_error (error); + g_assert_nonnull (results); + g_assert_cmpuint (results->len, ==, 2); + + /* Check that the results are correct: the invalid refs should have been + * ignored, and the valid results canonicalised and deduplicated. */ + for (i = 0; i < results->len; i++) + { + const OstreeRepoFinderResult *result = g_ptr_array_index (results, i); + + if (g_strcmp0 (result->uri, "http://ref0") == 0) + { + g_assert_cmpuint (g_strv_length (result->refs), ==, 2); + g_assert_true (g_strv_contains ((const gchar * const *) result->refs, refs[0])); + g_assert_true (g_strv_contains ((const gchar * const *) result->refs, refs[2])); + } + else if (g_strcmp0 (result->uri, "http://ref1") == 0) + { + g_assert_cmpuint (g_strv_length (result->refs), ==, 1); + g_assert_true (g_strv_contains ((const gchar * const *) result->refs, refs[1])); + } + else + { + g_assert_not_reached (); + } + } + + g_main_context_pop_thread_default (context); +} + +int main (int argc, char **argv) +{ + setlocale (LC_ALL, ""); + g_test_init (&argc, &argv, NULL); + + g_test_add_func ("/repo-finder-config/init", test_repo_finder_config_init); + g_test_add ("/repo-finder-config/no-configs", Fixture, NULL, setup, + test_repo_finder_config_no_configs, teardown); + g_test_add ("/repo-finder-config/mixed-configs", Fixture, NULL, setup, + test_repo_finder_config_mixed_configs, teardown); + + return g_test_run(); +} diff --git a/tests/test-repo-finder-mount.c b/tests/test-repo-finder-mount.c new file mode 100644 index 0000000000..898c473e25 --- /dev/null +++ b/tests/test-repo-finder-mount.c @@ -0,0 +1,371 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright © 2017 Endless Mobile, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + * Authors: + * - Philip Withnall + */ + +#include "config.h" + +#include +#include +#include +#include + +#include "ostree-repo-finder.h" +#include "ostree-repo-finder-mount.h" +#include "test-mock-gio.h" + +/* Test fixture. Creates a temporary directory. */ +typedef struct +{ + GFile *tmp_dir; +} Fixture; + +static void +setup (Fixture *fixture, + gconstpointer test_data) +{ + g_autofree gchar *tmp_dir_path = NULL; + g_autoptr(GError) error = NULL; + + tmp_dir_path = g_dir_make_tmp ("test-repo-finder-mount-XXXXXX", &error); + g_assert_no_error (error); + + fixture->tmp_dir = g_file_new_for_path (tmp_dir_path); +} + +/* Recursively delete @file and its children. @file may be a file or a directory. */ +static gboolean +rm_rf (GFile *file, + GCancellable *cancellable, + GError **error) +{ + g_autoptr(GFileEnumerator) enumerator = NULL; + + enumerator = g_file_enumerate_children (file, G_FILE_ATTRIBUTE_STANDARD_NAME, + G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, + cancellable, NULL); + + while (enumerator != NULL) + { + GFile *child; + + if (!g_file_enumerator_iterate (enumerator, NULL, &child, cancellable, error)) + return FALSE; + if (child == NULL) + break; + if (!rm_rf (child, cancellable, error)) + return FALSE; + } + + return g_file_delete (file, cancellable, error); +} + +static void +teardown (Fixture *fixture, + gconstpointer test_data) +{ + /* Recursively remove the temporary directory. */ + rm_rf (fixture->tmp_dir, NULL, NULL); +} + +/* Test the object constructor works at a basic level. */ +static void +test_repo_finder_mount_init (void) +{ + g_autoptr(OstreeRepoFinderMount) finder = NULL; + g_autoptr(GVolumeMonitor) monitor = NULL; + + /* Default #GVolumeMonitor. */ + finder = ostree_repo_finder_mount_new (NULL); + g_clear_object (&finder); + + /* Explicit #GVolumeMonitor. */ + monitor = ostree_mock_volume_monitor_new (NULL); + finder = ostree_repo_finder_mount_new (monitor); + g_clear_object (&finder); +} + +static void +result_cb (GObject *source_object, + GAsyncResult *result, + gpointer user_data) +{ + GAsyncResult **result_out = user_data; + *result_out = g_object_ref (result); +} + +/* Test that no remotes are found if the #GVolumeMonitor returns no mounts. */ +static void +test_repo_finder_mount_no_mounts (void) +{ + g_autoptr(OstreeRepoFinderMount) finder = NULL; + g_autoptr(GVolumeMonitor) monitor = NULL; + g_autoptr(GMainContext) context = NULL; + g_autoptr(GAsyncResult) result = NULL; + g_autoptr(GPtrArray) results = NULL; /* (element-type OstreeRepoFinderResult) */ + g_autoptr(GError) error = NULL; + const gchar * const refs[] = + { + "exampleos/x86_64/standard", + "exampleos/x86_64/buildmaster/standard", + NULL + }; + + context = g_main_context_new (); + g_main_context_push_thread_default (context); + + monitor = ostree_mock_volume_monitor_new (NULL); + finder = ostree_repo_finder_mount_new (monitor); + + ostree_repo_finder_resolve_async (OSTREE_REPO_FINDER (finder), refs, + NULL, result_cb, &result); + + while (result == NULL) + g_main_context_iteration (context, TRUE); + + results = ostree_repo_finder_resolve_finish (OSTREE_REPO_FINDER (finder), + result, &error); + g_assert_no_error (error); + g_assert_nonnull (results); + g_assert_cmpuint (results->len, ==, 0); + + g_main_context_pop_thread_default (context); +} + +/* Create a .ostree/repos directory under the given @mount_root, or abort. */ +static GFile * +assert_create_repos_dir (GFile *mount_root) +{ + g_autoptr(GFile) ostree = NULL, repos = NULL; + g_autoptr(GError) error = NULL; + + ostree = g_file_get_child (mount_root, ".ostree"); + repos = g_file_get_child (ostree, "repos"); + g_file_make_directory_with_parents (repos, NULL, &error); + g_assert_no_error (error); + + return g_steal_pointer (&repos); +} + +/* Create a @ref_name directory under the given @repos_dir, or abort. */ +static GFile * +assert_create_repo_dir (GFile *repos_dir, + const gchar *ref_name) +{ + g_autoptr(GFile) repo_dir = NULL; + g_autoptr(GError) error = NULL; + + repo_dir = g_file_get_child (repos_dir, ref_name); + g_file_make_directory_with_parents (repo_dir, NULL, &error); + g_assert_no_error (error); + + return g_steal_pointer (&repo_dir); +} + +/* Create a @ref_name symlink under the given @repos_dir, pointing to + * @symlink_target, or abort. */ +static GFile * +assert_create_repo_symlink (GFile *repos_dir, + const gchar *ref_name, + GFile *symlink_target) +{ + g_autoptr(GFile) repo_dir = NULL; + g_autoptr(GFile) repo_parent_dir = NULL; + g_autofree gchar *symlink_target_path = NULL; + g_autoptr(GError) error = NULL; + + /* The @repo_parent_dir is not necessarily @repos_dir, since @ref_name may + * contain slashes. */ + repo_dir = g_file_get_child (repos_dir, ref_name); + repo_parent_dir = g_file_get_parent (repo_dir); + symlink_target_path = g_file_get_path (symlink_target); + g_file_make_directory_with_parents (repo_parent_dir, NULL, &error); + if (g_error_matches (error, G_IO_ERROR, G_IO_ERROR_EXISTS)) + g_clear_error (&error); + g_assert_no_error (error); + g_file_make_symbolic_link (repo_dir, symlink_target_path, NULL, &error); + g_assert_no_error (error); + + return g_object_ref (symlink_target); +} + +/* Test resolving the refs against a collection of mock volumes, some of which + * are mounted, some of which are removable, some of which contain valid or + * invalid repo information on the file system, etc. */ +static void +test_repo_finder_mount_mixed_mounts (Fixture *fixture, + gconstpointer test_data) +{ + g_autoptr(OstreeRepoFinderMount) finder = NULL; + g_autoptr(GVolumeMonitor) monitor = NULL; + g_autoptr(GMainContext) context = NULL; + g_autoptr(GAsyncResult) result = NULL; + g_autoptr(GPtrArray) results = NULL; /* (element-type OstreeRepoFinderResult) */ + g_autoptr(GError) error = NULL; + g_autoptr(GList) volumes = NULL; /* (element-type OstreeMockVolume) */ + g_autoptr(OstreeMockVolume) volume_no_drive = NULL; + g_autoptr(OstreeMockVolume) volume_no_mount = NULL; + g_autoptr(OstreeMockVolume) volume_drive_not_removable = NULL; + g_autoptr(OstreeMockVolume) volume_no_repos = NULL; + g_autoptr(OstreeMockVolume) volume_repo1 = NULL; + g_autoptr(OstreeMockVolume) volume_repo2 = NULL; + g_autoptr(OstreeMockDrive) non_removable_drive = NULL; + g_autoptr(OstreeMockDrive) removable_drive = NULL; + g_autoptr(OstreeMockMount) non_removable_mount = NULL; + g_autoptr(OstreeMockMount) no_repos_mount = NULL; + g_autoptr(OstreeMockMount) repo1_mount = NULL; + g_autoptr(OstreeMockMount) repo2_mount = NULL; + g_autoptr(GFile) non_removable_root = NULL; + g_autoptr(GFile) no_repos_root = NULL; + g_autoptr(GFile) repo1_root = NULL; + g_autoptr(GFile) repo2_root = NULL; + g_autoptr(GFile) repo1_repos = NULL; + g_autoptr(GFile) repo2_repos = NULL; + g_autoptr(GFile) repo1_repo_a = NULL, repo1_repo_b = NULL, repo1_repo_c = NULL; + g_autoptr(GFile) repo2_repo_a = NULL, repo2_repo_b = NULL; + g_autoptr(GFile) repo2_repo_c = NULL, repo2_repo_d = NULL; + g_autofree gchar *repo1_repo_a_uri = NULL, *repo1_repo_b_uri = NULL; + g_autofree gchar *repo2_repo_a_uri = NULL; + gsize i; + const gchar * const refs[] = + { + "exampleos/x86_64/ref0", + "exampleos/x86_64/ref1", + "exampleos/x86_64/ref2", + "exampleos/x86_64/ref3", + NULL + }; + + context = g_main_context_new (); + g_main_context_push_thread_default (context); + + non_removable_drive = ostree_mock_drive_new (FALSE); + removable_drive = ostree_mock_drive_new (TRUE); + + /* Build the various mock drives/volumes/mounts, and some repositories with + * refs within them. */ + non_removable_root = g_file_get_child (fixture->tmp_dir, "non-removable"); + non_removable_mount = ostree_mock_mount_new (non_removable_root); + + no_repos_root = g_file_get_child (fixture->tmp_dir, "no-repos"); + assert_create_repos_dir (no_repos_root); + no_repos_mount = ostree_mock_mount_new (no_repos_root); + + repo1_root = g_file_get_child (fixture->tmp_dir, "repo1"); + repo1_repos = assert_create_repos_dir (repo1_root); + repo1_repo_a = assert_create_repo_dir (repo1_repos, refs[0]); + repo1_repo_b = assert_create_repo_dir (repo1_repos, refs[1]); + repo1_repo_c = assert_create_repo_symlink (repo1_repos, refs[2], repo1_repo_a); + + repo1_mount = ostree_mock_mount_new (repo1_root); + + repo2_root = g_file_get_child (fixture->tmp_dir, "repo2"); + repo2_repos = assert_create_repos_dir (repo2_root); + repo2_mount = ostree_mock_mount_new (repo2_root); + repo2_repo_a = assert_create_repo_dir (repo2_repos, refs[0]); + repo2_repo_b = assert_create_repo_symlink (repo2_repos, refs[1], repo2_repo_a); + repo2_repo_c = assert_create_repo_symlink (repo2_repos, refs[2], repo2_repo_b); + repo2_repo_d = assert_create_repo_symlink (repo2_repos, refs[3], repo1_repo_a); + + repo1_repo_a_uri = g_file_get_uri (repo1_repo_a); + repo1_repo_b_uri = g_file_get_uri (repo1_repo_b); + repo2_repo_a_uri = g_file_get_uri (repo2_repo_a); + + volume_no_drive = ostree_mock_volume_new ("no-drive", NULL, G_MOUNT (non_removable_mount)); + volume_no_mount = ostree_mock_volume_new ("no-mount", G_DRIVE (non_removable_drive), NULL); + volume_drive_not_removable = ostree_mock_volume_new ("drive-not-removable", + G_DRIVE (non_removable_drive), + G_MOUNT (non_removable_mount)); + volume_no_repos = ostree_mock_volume_new ("no-repos", G_DRIVE (removable_drive), + G_MOUNT (no_repos_mount)); + volume_repo1 = ostree_mock_volume_new ("repo1", G_DRIVE (removable_drive), + G_MOUNT (repo1_mount)); + volume_repo2 = ostree_mock_volume_new ("repo2", G_DRIVE (removable_drive), + G_MOUNT (repo2_mount)); + + volumes = g_list_prepend (volumes, volume_no_drive); + volumes = g_list_prepend (volumes, volume_no_mount); + volumes = g_list_prepend (volumes, volume_drive_not_removable); + volumes = g_list_prepend (volumes, volume_no_repos); + volumes = g_list_prepend (volumes, volume_repo1); + volumes = g_list_prepend (volumes, volume_repo2); + + monitor = ostree_mock_volume_monitor_new (volumes); + finder = ostree_repo_finder_mount_new (monitor); + + /* Resolve the refs. */ + ostree_repo_finder_resolve_async (OSTREE_REPO_FINDER (finder), refs, + NULL, result_cb, &result); + + while (result == NULL) + g_main_context_iteration (context, TRUE); + + results = ostree_repo_finder_resolve_finish (OSTREE_REPO_FINDER (finder), + result, &error); + g_assert_no_error (error); + g_assert_nonnull (results); + g_assert_cmpuint (results->len, ==, 3); + + /* Check that the results are correct: the invalid refs should have been + * ignored, and the valid results canonicalised and deduplicated. */ + for (i = 0; i < results->len; i++) + { + const OstreeRepoFinderResult *result = g_ptr_array_index (results, i); + + if (g_strcmp0 (result->uri, repo1_repo_a_uri) == 0) + { + g_assert_cmpuint (g_strv_length (result->refs), ==, 2); + g_assert_true (g_strv_contains ((const gchar * const *) result->refs, refs[0])); + g_assert_true (g_strv_contains ((const gchar * const *) result->refs, refs[2])); + } + else if (g_strcmp0 (result->uri, repo1_repo_b_uri) == 0) + { + g_assert_cmpuint (g_strv_length (result->refs), ==, 1); + g_assert_true (g_strv_contains ((const gchar * const *) result->refs, refs[1])); + } + else if (g_strcmp0 (result->uri, repo2_repo_a_uri) == 0) + { + g_assert_cmpuint (g_strv_length (result->refs), ==, 3); + g_assert_true (g_strv_contains ((const gchar * const *) result->refs, refs[0])); + g_assert_true (g_strv_contains ((const gchar * const *) result->refs, refs[1])); + g_assert_true (g_strv_contains ((const gchar * const *) result->refs, refs[2])); + } + else + { + g_assert_not_reached (); + } + } + + g_main_context_pop_thread_default (context); +} + +int main (int argc, char **argv) +{ + setlocale (LC_ALL, ""); + g_test_init (&argc, &argv, NULL); + + g_test_add_func ("/repo-finder-mount/init", test_repo_finder_mount_init); + g_test_add_func ("/repo-finder-mount/no-mounts", test_repo_finder_mount_no_mounts); + g_test_add ("/repo-finder-mount/mixed-mounts", Fixture, NULL, setup, + test_repo_finder_mount_mixed_mounts, teardown); + + return g_test_run(); +}