diff --git a/.redhat-ci.yml b/.redhat-ci.yml index 830320efd1..11e5a9d6c7 100644 --- a/.redhat-ci.yml +++ b/.redhat-ci.yml @@ -71,5 +71,27 @@ env: tests: - make check TESTS=tests/test-rollsum +--- + +inherit: true +required: true + +context: curl + +packages: + - pkgconfig(libcurl) + +build: + config-opts: > + --prefix=/usr + --libdir=/usr/lib64 + --enable-installed-tests + --enable-gtk-doc + --with-curl + +tests: + - make check + - gnome-desktop-testing-runner -p 0 ostree + artifacts: - test-suite.log diff --git a/.travis.yml b/.travis.yml index 27f8492194..a021592c40 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,7 @@ sudo: required env: - ci_distro=ubuntu ci_suite=trusty ci_test=no # TODO: use libcurl on this - ci_docker=debian:jessie-slim ci_distro=debian ci_suite=jessie + - ci_docker=debian:jessie-slim ci_distro=debian ci_suite=jessie ci_configopts="--with-curl" - ci_docker=debian:stretch-slim ci_distro=debian ci_suite=stretch ci_test=no # TODO gpgme flake https://github.com/ostreedev/ostree/pull/664#issuecomment-276033383 - ci_docker=ubuntu:xenial ci_distro=ubuntu ci_suite=xenial diff --git a/Makefile-libostree.am b/Makefile-libostree.am index d40196d4ee..79fddec74e 100644 --- a/Makefile-libostree.am +++ b/Makefile-libostree.am @@ -167,18 +167,31 @@ libostree_1_la_CFLAGS += $(LIBSYSTEMD_CFLAGS) libostree_1_la_LIBADD += $(LIBSYSTEMD_LIBS) endif -if USE_LIBSOUP +if USE_CURL_OR_SOUP libostree_1_la_SOURCES += \ src/libostree/ostree-fetcher.h \ - src/libostree/ostree-fetcher.c \ src/libostree/ostree-fetcher-util.h \ src/libostree/ostree-fetcher-util.c \ + src/libostree/ostree-fetcher-uri.c \ src/libostree/ostree-metalink.h \ src/libostree/ostree-metalink.c \ $(NULL) +endif + +if USE_CURL +libostree_1_la_SOURCES += src/libostree/ostree-fetcher-curl.c \ + src/libostree/ostree-soup-uri.h src/libostree/ostree-soup-uri.c \ + src/libostree/ostree-soup-form.c \ + $(NULL) +libostree_1_la_CFLAGS += $(OT_DEP_CURL_CFLAGS) +libostree_1_la_LIBADD += $(OT_DEP_CURL_LIBS) +else +if USE_LIBSOUP +libostree_1_la_SOURCES += src/libostree/ostree-fetcher-soup.c libostree_1_la_CFLAGS += $(OT_INTERNAL_SOUP_CFLAGS) libostree_1_la_LIBADD += $(OT_INTERNAL_SOUP_LIBS) endif +endif if USE_LIBMOUNT libostree_1_la_CFLAGS += $(OT_DEP_LIBMOUNT_CFLAGS) diff --git a/Makefile-ostree.am b/Makefile-ostree.am index 05fec15520..0b520c68ea 100644 --- a/Makefile-ostree.am +++ b/Makefile-ostree.am @@ -88,12 +88,14 @@ ostree_SOURCES += \ src/ostree/ot-remote-builtin-summary.c \ $(NULL) -if USE_LIBSOUP -ostree_SOURCES += \ - src/ostree/ot-remote-builtin-add-cookie.c \ - src/ostree/ot-remote-builtin-delete-cookie.c \ - src/ostree/ot-remote-builtin-list-cookies.c \ - $(NULL) + +if USE_CURL_OR_SOUP +ostree_SOURCES += src/ostree/ot-remote-builtin-add-cookie.c \ + src/ostree/ot-remote-builtin-delete-cookie.c \ + src/ostree/ot-remote-builtin-list-cookies.c \ + src/ostree/ot-remote-cookie-util.h \ + src/ostree/ot-remote-cookie-util.c \ + $(NULL) endif src/ostree/parse-datetime.c: src/ostree/parse-datetime.y Makefile @@ -112,15 +114,23 @@ ostree_CFLAGS = $(ostree_bin_shared_cflags) ostree_LDADD = $(ostree_bin_shared_ldadd) libbsdiff.la libostree-kernel-args.la $(LIBSYSTEMD_LIBS) -if USE_LIBSOUP -ostree_SOURCES += src/ostree/ot-builtin-pull.c src/ostree/ot-builtin-trivial-httpd.c -ostree_CFLAGS += $(OT_INTERNAL_SOUP_CFLAGS) -ostree_LDADD += $(OT_INTERNAL_SOUP_LIBS) +if USE_CURL_OR_SOUP +ostree_SOURCES += src/ostree/ot-builtin-pull.c +endif +if USE_LIBSOUP +# Eventually once we stop things from using this, we should support disabling this +ostree_SOURCES += src/ostree/ot-builtin-trivial-httpd.c pkglibexec_PROGRAMS += ostree-trivial-httpd ostree_trivial_httpd_SOURCES = src/ostree/ostree-trivial-httpd.c ostree_trivial_httpd_CFLAGS = $(ostree_bin_shared_cflags) $(OT_INTERNAL_SOUP_CFLAGS) ostree_trivial_httpd_LDADD = $(ostree_bin_shared_ldadd) $(OT_INTERNAL_SOUP_LIBS) + +if !USE_CURL +# This is necessary for the cookie jar bits +ostree_CFLAGS += $(OT_INTERNAL_SOUP_CFLAGS) +ostree_LDADD += $(OT_INTERNAL_SOUP_LIBS) +endif endif if USE_LIBARCHIVE diff --git a/configure.ac b/configure.ac index 88e6ea1b87..596ee04068 100644 --- a/configure.ac +++ b/configure.ac @@ -79,14 +79,29 @@ PKG_CHECK_MODULES(OT_DEP_ZLIB, zlib) dnl We're not actually linking to this, just using the header PKG_CHECK_MODULES(OT_DEP_E2P, e2p) +dnl Arbitrary version that's in CentOS7.2 now +CURL_DEPENDENCY=7.29.0 +AC_ARG_WITH(curl, + AS_HELP_STRING([--with-curl], [Use libcurl @<:@default=no@:>@]), + [], [with_curl=no]) +AS_IF([test x$with_curl != xno ], [ + PKG_CHECK_MODULES(OT_DEP_CURL, libcurl >= $CURL_DEPENDENCY) + with_curl=yes + AC_DEFINE([HAVE_LIBCURL], 1, [Define if we have libcurl.pc]) + dnl Currently using libcurl requires soup for trivial-httpd for tests + with_soup_default=yes +], [with_soup_default=check]) +AM_CONDITIONAL(USE_CURL, test x$with_curl != xno) +if test x$with_curl = xyes; then OSTREE_FEATURES="$OSTREE_FEATURES +libcurl"; fi + dnl When bumping the libsoup-2.4 dependency, remember to bump dnl SOUP_VERSION_MIN_REQUIRED and SOUP_VERSION_MAX_ALLOWED in dnl Makefile.am SOUP_DEPENDENCY="libsoup-2.4 >= 2.39.1" AC_ARG_WITH(soup, AS_HELP_STRING([--with-soup], [Use libsoup @<:@default=yes@:>@]), - [], [with_soup=check]) -AS_IF([test x$with_soup != xno ], [ + [], [with_soup=$with_soup_default]) +AS_IF([test x$with_soup != xno], [ AC_ARG_ENABLE(libsoup_client_certs, AS_HELP_STRING([--enable-libsoup-client-certs], [Require availability of new enough libsoup TLS client cert API (default: auto)]),, @@ -120,6 +135,14 @@ if test x$with_soup != xno; then OSTREE_FEATURES="$OSTREE_FEATURES +libsoup"; fi AM_CONDITIONAL(USE_LIBSOUP, test x$with_soup != xno) AM_CONDITIONAL(HAVE_LIBSOUP_CLIENT_CERTS, test x$have_libsoup_client_certs = xyes) +AS_IF([test x$with_curl = xyes && test x$with_soup = xno], [ + AC_MSG_ERROR([Curl enabled, but libsoup is not; libsoup is needed for tests]) +]) +AM_CONDITIONAL(USE_CURL_OR_SOUP, test x$with_curl != xno || test x$with_soup != xno) +AS_IF([test x$with_curl != xno || test x$with_soup != xno], + [AC_DEFINE([HAVE_LIBCURL_OR_LIBSOUP], 1, [Define if we have soup or curl])]) +AS_IF([test x$with_curl = xyes], [fetcher_backend=curl], [test x$with_soup = xyes], [fetcher_backend=libsoup], [fetcher_backend=none]) + m4_ifdef([GOBJECT_INTROSPECTION_CHECK], [ GOBJECT_INTROSPECTION_CHECK([1.34.0]) ]) @@ -374,8 +397,7 @@ echo " introspection: $found_introspection Rust (internal oxidation): $rust_debug_release rofiles-fuse: $enable_rofiles_fuse - libsoup (retrieve remote HTTP repositories): $with_soup - libsoup TLS client certs: $have_libsoup_client_certs + HTTP backend: $fetcher_backend SELinux: $with_selinux systemd: $have_libsystemd libmount: $with_libmount diff --git a/src/libostree/ostree-fetcher-curl.c b/src/libostree/ostree-fetcher-curl.c new file mode 100644 index 0000000000..be3250fbc1 --- /dev/null +++ b/src/libostree/ostree-fetcher-curl.c @@ -0,0 +1,922 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright (C) 2016 Colin Walters + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + */ + +#include "config.h" + +#include +#include +#include +#include + +/* These macros came from 7.43.0, but we want to check + * for versions a bit earlier than that (to work on CentOS 7), + * so define them here if we're using an older version. + */ +#ifndef CURL_VERSION_BITS +#define CURL_VERSION_BITS(x,y,z) ((x)<<16|(y)<<8|z) +#endif +#ifndef CURL_AT_LEAST_VERSION +#define CURL_AT_LEAST_VERSION(x,y,z) (LIBCURL_VERSION_NUM >= CURL_VERSION_BITS(x, y, z)) +#endif + +#include "ostree-fetcher.h" +#include "ostree-enumtypes.h" +#include "ostree-repo-private.h" +#include "otutil.h" + +#include "ostree-soup-uri.h" + +typedef struct FetcherRequest FetcherRequest; +typedef struct SockInfo SockInfo; + +static int sock_cb (CURL *e, curl_socket_t s, int what, void *cbp, void *sockp); +static gboolean timer_cb (gpointer data); +static void sock_unref (SockInfo *f); +static int update_timeout_cb (CURLM *multi, long timeout_ms, void *userp); +static void request_unref (FetcherRequest *req); +static void initiate_next_curl_request (FetcherRequest *req, GTask *task); +static void destroy_and_unref_source (GSource *source); + +struct OstreeFetcher +{ + GObject parent_instance; + + OstreeFetcherConfigFlags config_flags; + char *tls_ca_db_path; + char *tls_client_cert_path; + char *tls_client_key_path; + char *cookie_jar_path; + char *proxy; + struct curl_slist *extra_headers; + int tmpdir_dfd; + + GMainContext *mainctx; + CURLM *multi; + GSource *timer_event; + int curl_running; + GHashTable *outstanding_requests; /* Set */ + GHashTable *sockets; /* Set */ + + guint64 bytes_transferred; +}; + +/* Information associated with a request */ +struct FetcherRequest { + guint refcount; + GPtrArray *mirrorlist; + guint idx; + + char *filename; + guint64 current_size; + guint64 max_size; + OstreeFetcherRequestFlags flags; + gboolean is_membuf; + GError *caught_write_error; + char *out_tmpfile; + int out_tmpfile_fd; + GString *output_buf; + + CURL *easy; + char error[CURL_ERROR_SIZE]; + + OstreeFetcher *fetcher; +}; + +/* Information associated with a specific socket */ +struct SockInfo { + guint refcount; + curl_socket_t sockfd; + int action; + long timeout; + GSource *ch; + OstreeFetcher *fetcher; +}; + +enum { + PROP_0, + PROP_CONFIG_FLAGS +}; + +G_DEFINE_TYPE (OstreeFetcher, _ostree_fetcher, G_TYPE_OBJECT) + +static void +_ostree_fetcher_set_property (GObject *object, + guint prop_id, + const GValue *value, + GParamSpec *pspec) +{ + OstreeFetcher *self = OSTREE_FETCHER (object); + + switch (prop_id) + { + case PROP_CONFIG_FLAGS: + self->config_flags = g_value_get_flags (value); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +_ostree_fetcher_get_property (GObject *object, + guint prop_id, + GValue *value, + GParamSpec *pspec) +{ + OstreeFetcher *self = OSTREE_FETCHER (object); + + switch (prop_id) + { + case PROP_CONFIG_FLAGS: + g_value_set_flags (value, self->config_flags); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +_ostree_fetcher_finalize (GObject *object) +{ + OstreeFetcher *self = OSTREE_FETCHER (object); + + g_free (self->cookie_jar_path); + g_free (self->proxy); + g_assert_cmpint (g_hash_table_size (self->outstanding_requests), ==, 0); + g_clear_pointer (&self->extra_headers, (GDestroyNotify)curl_slist_free_all); + g_hash_table_unref (self->outstanding_requests); + g_hash_table_unref (self->sockets); + g_clear_pointer (&self->timer_event, (GDestroyNotify)destroy_and_unref_source); + if (self->mainctx) + g_main_context_unref (self->mainctx); + curl_multi_cleanup (self->multi); + + G_OBJECT_CLASS (_ostree_fetcher_parent_class)->finalize (object); +} + +static void +_ostree_fetcher_constructed (GObject *object) +{ + // OstreeFetcher *self = OSTREE_FETCHER (object); + + G_OBJECT_CLASS (_ostree_fetcher_parent_class)->constructed (object); +} + +static void +_ostree_fetcher_class_init (OstreeFetcherClass *klass) +{ + GObjectClass *gobject_class = G_OBJECT_CLASS (klass); + + gobject_class->set_property = _ostree_fetcher_set_property; + gobject_class->get_property = _ostree_fetcher_get_property; + gobject_class->finalize = _ostree_fetcher_finalize; + gobject_class->constructed = _ostree_fetcher_constructed; + + g_object_class_install_property (gobject_class, + PROP_CONFIG_FLAGS, + g_param_spec_flags ("config-flags", + "", + "", + OSTREE_TYPE_FETCHER_CONFIG_FLAGS, + OSTREE_FETCHER_FLAGS_NONE, + G_PARAM_READWRITE | + G_PARAM_CONSTRUCT_ONLY | + G_PARAM_STATIC_STRINGS)); +} + +static void +_ostree_fetcher_init (OstreeFetcher *self) +{ + self->multi = curl_multi_init(); + self->outstanding_requests = g_hash_table_new_full (NULL, NULL, (GDestroyNotify)g_object_unref, NULL); + self->sockets = g_hash_table_new_full (NULL, NULL, (GDestroyNotify)sock_unref, NULL); + curl_multi_setopt (self->multi, CURLMOPT_SOCKETFUNCTION, sock_cb); + curl_multi_setopt (self->multi, CURLMOPT_SOCKETDATA, self); + curl_multi_setopt (self->multi, CURLMOPT_TIMERFUNCTION, update_timeout_cb); + curl_multi_setopt (self->multi, CURLMOPT_TIMERDATA, self); +#if CURL_AT_LEAST_VERSION(7, 30, 0) + /* Let's do something reasonable here. */ + curl_multi_setopt (self->multi, CURLMOPT_MAX_TOTAL_CONNECTIONS, 8); +#endif +} + + +OstreeFetcher * +_ostree_fetcher_new (int tmpdir_dfd, + OstreeFetcherConfigFlags flags) +{ + OstreeFetcher *fetcher = g_object_new (OSTREE_TYPE_FETCHER, "config-flags", flags, NULL); + fetcher->tmpdir_dfd = tmpdir_dfd; + return fetcher; +} + +static void +destroy_and_unref_source (GSource *source) +{ + g_source_destroy (source); + g_source_unref (source); +} + +static char * +request_get_uri (FetcherRequest *req, guint idx) +{ + SoupURI *baseuri = req->mirrorlist->pdata[idx]; + if (!req->filename) + return soup_uri_to_string (baseuri, FALSE); + { g_autofree char *uristr = soup_uri_to_string (baseuri, FALSE); + return g_build_filename (uristr, req->filename, NULL); + } +} + +static gboolean +ensure_tmpfile (FetcherRequest *req, GError **error) +{ + if (req->out_tmpfile_fd == -1) + { + if (!glnx_open_tmpfile_linkable_at (req->fetcher->tmpdir_dfd, ".", + O_WRONLY, &req->out_tmpfile_fd, + &req->out_tmpfile, + error)) + return FALSE; + } + return TRUE; +} +/* Check for completed transfers, and remove their easy handles */ +static void +check_multi_info (OstreeFetcher *fetcher) +{ + CURLMsg *msg; + int msgs_left; + + while ((msg = curl_multi_info_read (fetcher->multi, &msgs_left)) != NULL) + { + long response; + CURL *easy = msg->easy_handle; + CURLcode curlres = msg->data.result; + GTask *task; + FetcherRequest *req; + const char *eff_url; + gboolean is_file; + gboolean continued_request = FALSE; + + if (msg->msg != CURLMSG_DONE) + continue; + + curl_easy_getinfo (easy, CURLINFO_PRIVATE, &task); + curl_easy_getinfo (easy, CURLINFO_EFFECTIVE_URL, &eff_url); + /* We should have limited the protocols; this is what + * curl's tool_operate.c does. + */ + is_file = g_str_has_prefix (eff_url, "file:"); + g_assert (is_file || g_str_has_prefix (eff_url, "http")); + + req = g_task_get_task_data (task); + + if (req->caught_write_error) + g_task_return_error (task, g_steal_pointer (&req->caught_write_error)); + else if (curlres != CURLE_OK) + { + if (is_file && curlres == CURLE_FILE_COULDNT_READ_FILE) + { + /* Handle file not found */ + g_task_return_new_error (task, G_IO_ERROR, G_IO_ERROR_NOT_FOUND, + "%s", + curl_easy_strerror (curlres)); + } + else + g_task_return_new_error (task, G_IO_ERROR, G_IO_ERROR_FAILED, "[%u] %s", + curlres, + curl_easy_strerror (curlres)); + } + else + { + curl_easy_getinfo (easy, CURLINFO_RESPONSE_CODE, &response); + if (!is_file && !(response >= 200 && response < 300)) + { + GIOErrorEnum giocode; + + /* TODO - share with soup */ + switch (response) + { + case 404: + case 403: + case 410: + giocode = G_IO_ERROR_NOT_FOUND; + break; + default: + giocode = G_IO_ERROR_FAILED; + } + + if (req->idx + 1 == req->mirrorlist->len) + { + g_task_return_new_error (task, G_IO_ERROR, giocode, + "Server returned HTTP %lu", response); + } + else + { + continued_request = TRUE; + } + } + else if (req->is_membuf) + { + GBytes *ret; + if ((req->flags & OSTREE_FETCHER_REQUEST_NUL_TERMINATION) > 0) + g_string_append_c (req->output_buf, '\0'); + ret = g_string_free_to_bytes (req->output_buf); + req->output_buf = NULL; + g_task_return_pointer (task, ret, (GDestroyNotify)g_bytes_unref); + } + else + { + g_autoptr(GError) local_error = NULL; + GError **error = &local_error; + + /* TODO - share file naming with soup, and fix it */ + g_autofree char *tmpfile_path = + g_compute_checksum_for_string (G_CHECKSUM_SHA256, + eff_url, strlen (eff_url)); + if (!ensure_tmpfile (req, error)) + { + g_task_return_error (task, g_steal_pointer (&local_error)); + } + else if (fchmod (req->out_tmpfile_fd, 0644) < 0) + { + glnx_set_error_from_errno (error); + g_task_return_error (task, g_steal_pointer (&local_error)); + } + else if (!glnx_link_tmpfile_at (fetcher->tmpdir_dfd, + GLNX_LINK_TMPFILE_REPLACE, + req->out_tmpfile_fd, + req->out_tmpfile, + fetcher->tmpdir_dfd, + tmpfile_path, + error)) + g_task_return_error (task, g_steal_pointer (&local_error)); + else + { + g_task_return_pointer (task, g_steal_pointer (&tmpfile_path), g_free); + } + } + } + + curl_multi_remove_handle (fetcher->multi, easy); + if (continued_request) + { + req->idx++; + initiate_next_curl_request (req, task); + } + else + { + g_hash_table_remove (fetcher->outstanding_requests, task); + if (g_hash_table_size (fetcher->outstanding_requests) == 0) + { + g_clear_pointer (&fetcher->mainctx, g_main_context_unref); + } + } + } +} + +/* Called by glib when our timeout expires */ +static gboolean +timer_cb (gpointer data) +{ + OstreeFetcher *fetcher = data; + CURLMcode rc; + + fetcher->timer_event = NULL; + rc = curl_multi_socket_action (fetcher->multi, CURL_SOCKET_TIMEOUT, 0, &fetcher->curl_running); + g_assert (rc == CURLM_OK); + check_multi_info (fetcher); + + return FALSE; +} + +/* Update the event timer after curl_multi library calls */ +static int +update_timeout_cb (CURLM *multi, long timeout_ms, void *userp) +{ + OstreeFetcher *fetcher = userp; + + g_clear_pointer (&fetcher->timer_event, (GDestroyNotify)destroy_and_unref_source); + + if (timeout_ms != -1) + { + fetcher->timer_event = g_timeout_source_new (timeout_ms); + g_source_set_callback (fetcher->timer_event, timer_cb, fetcher, NULL); + g_source_attach (fetcher->timer_event, fetcher->mainctx); + } + + return 0; +} + +/* Called by glib when we get action on a multi socket */ +static gboolean +event_cb (int fd, GIOCondition condition, gpointer data) +{ + OstreeFetcher *fetcher = data; + CURLMcode rc; + + int action = + (condition & G_IO_IN ? CURL_CSELECT_IN : 0) | + (condition & G_IO_OUT ? CURL_CSELECT_OUT : 0); + + rc = curl_multi_socket_action (fetcher->multi, fd, action, &fetcher->curl_running); + g_assert (rc == CURLM_OK); + + check_multi_info (fetcher); + if (fetcher->curl_running > 0) + { + return TRUE; + } + else + { + return FALSE; + } +} + +/* Clean up the SockInfo structure */ +static void +sock_unref (SockInfo *f) +{ + if (!f) + return; + if (--f->refcount != 0) + return; + g_clear_pointer (&f->ch, (GDestroyNotify)destroy_and_unref_source); + g_free (f); +} + +/* Assign information to a SockInfo structure */ +static void +setsock (SockInfo*f, curl_socket_t s, int act, OstreeFetcher *fetcher) +{ + GIOCondition kind = + (act&CURL_POLL_IN?G_IO_IN:0)|(act&CURL_POLL_OUT?G_IO_OUT:0); + + f->sockfd = s; + f->action = act; + g_clear_pointer (&f->ch, (GDestroyNotify)destroy_and_unref_source); + /* TODO - investigate new g_source_modify_unix_fd() so changing the poll + * flags involves less allocation. + */ + f->ch = g_unix_fd_source_new (f->sockfd, kind); + g_source_set_callback (f->ch, (GSourceFunc) event_cb, fetcher, NULL); + g_source_attach (f->ch, fetcher->mainctx); +} + +/* Initialize a new SockInfo structure */ +static void +addsock (curl_socket_t s, CURL *easy, int action, OstreeFetcher *fetcher) +{ + SockInfo *fdp = g_new0 (SockInfo, 1); + + fdp->refcount = 1; + fdp->fetcher = fetcher; + setsock (fdp, s, action, fetcher); + curl_multi_assign (fetcher->multi, s, fdp); + g_hash_table_add (fetcher->sockets, fdp); +} + +/* CURLMOPT_SOCKETFUNCTION */ +static int +sock_cb (CURL *easy, curl_socket_t s, int what, void *cbp, void *sockp) +{ + OstreeFetcher *fetcher = cbp; + SockInfo *fdp = (SockInfo*) sockp; + + if (what == CURL_POLL_REMOVE) + { + if (!g_hash_table_remove (fetcher->sockets, fdp)) + g_assert_not_reached (); + } + else + { + if (!fdp) + { + addsock (s, easy, what, fetcher); + } + else + { + setsock (fdp, s, what, fetcher); + } + } + return 0; +} + +/* CURLOPT_WRITEFUNCTION */ +static size_t +write_cb (void *ptr, size_t size, size_t nmemb, void *data) +{ + const size_t realsize = size * nmemb; + GTask *task = data; + FetcherRequest *req; + + req = g_task_get_task_data (task); + + if (req->caught_write_error) + return -1; + + if (req->max_size > 0) + { + if (realsize > req->max_size || + (realsize + req->current_size) > req->max_size) + { + const char *eff_url; + curl_easy_getinfo (req->easy, CURLINFO_EFFECTIVE_URL, &eff_url); + req->caught_write_error = g_error_new (G_IO_ERROR, G_IO_ERROR_FAILED, + "URI %s exceeded maximum size of %" G_GUINT64_FORMAT " bytes", + eff_url, req->max_size); + return -1; + } + } + + if (req->is_membuf) + g_string_append_len (req->output_buf, ptr, realsize); + else + { + if (!ensure_tmpfile (req, &req->caught_write_error)) + return -1; + g_assert (req->out_tmpfile_fd >= 0); + if (glnx_loop_write (req->out_tmpfile_fd, ptr, realsize) < 0) + { + glnx_set_error_from_errno (&req->caught_write_error); + return -1; + } + } + + req->current_size += realsize; + req->fetcher->bytes_transferred += realsize; + + return realsize; +} + +/* CURLOPT_PROGRESSFUNCTION */ +static int +prog_cb (void *p, double dltotal, double dlnow, double ult, double uln) +{ + GTask *task = p; + FetcherRequest *req; + char *eff_url; + req = g_task_get_task_data (task); + curl_easy_getinfo (req->easy, CURLINFO_EFFECTIVE_URL, &eff_url); + g_printerr ("Progress: %s (%g/%g)\n", eff_url, dlnow, dltotal); + return 0; +} + +static void +request_unref (FetcherRequest *req) +{ + if (--req->refcount != 0) + return; + + g_ptr_array_unref (req->mirrorlist); + g_free (req->filename); + g_clear_error (&req->caught_write_error); + if (req->out_tmpfile_fd != -1) + (void) close (req->out_tmpfile_fd); + g_free (req->out_tmpfile); + if (req->output_buf) + g_string_free (req->output_buf, TRUE); + curl_easy_cleanup (req->easy); + + g_free (req); +} + +int +_ostree_fetcher_get_dfd (OstreeFetcher *fetcher) +{ + return fetcher->tmpdir_dfd; +} + +void +_ostree_fetcher_set_proxy (OstreeFetcher *self, + const char *http_proxy) +{ + g_free (self->proxy); + self->proxy = g_strdup (http_proxy); +} + +void +_ostree_fetcher_set_cookie_jar (OstreeFetcher *self, + const char *jar_path) +{ + g_free (self->cookie_jar_path); + self->cookie_jar_path = g_strdup (jar_path); +} + +void +_ostree_fetcher_set_client_cert (OstreeFetcher *self, + const char *cert_path, + const char *key_path) +{ + g_assert ((cert_path == NULL && key_path == NULL) + || (cert_path != NULL && key_path != NULL)); + g_free (self->tls_client_cert_path); + self->tls_client_cert_path = g_strdup (cert_path); + g_free (self->tls_client_key_path); + self->tls_client_key_path = g_strdup (key_path); +} + +void +_ostree_fetcher_set_tls_database (OstreeFetcher *self, + const char *dbpath) +{ + g_free (self->tls_ca_db_path); + self->tls_ca_db_path = g_strdup (dbpath); +} + +void +_ostree_fetcher_set_extra_headers (OstreeFetcher *self, + GVariant *extra_headers) +{ + GVariantIter viter; + const char *key; + const char *value; + + g_clear_pointer (&self->extra_headers, (GDestroyNotify)curl_slist_free_all); + + g_variant_iter_init (&viter, extra_headers); + while (g_variant_iter_loop (&viter, "(&s&s)", &key, &value)) + { + g_autofree char *header = g_strdup_printf ("%s: %s", key, value); + self->extra_headers = curl_slist_append (self->extra_headers, header); + } +} + +/* Re-bind all of the outstanding curl items to our new main context */ +static void +adopt_steal_mainctx (OstreeFetcher *self, + GMainContext *mainctx) +{ + GHashTableIter hiter; + gpointer key, value; + + g_assert (self->mainctx == NULL); + self->mainctx = mainctx; /* Transfer */ + + if (self->timer_event != NULL) + { + guint64 readytime = g_source_get_ready_time (self->timer_event); + guint64 curtime = g_source_get_time (self->timer_event); + guint64 timeout_micros = curtime - readytime; + if (timeout_micros < 0) + timeout_micros = 0; + update_timeout_cb (self->multi, timeout_micros / 1000, self); + } + + g_hash_table_iter_init (&hiter, self->sockets); + while (g_hash_table_iter_next (&hiter, &key, &value)) + { + SockInfo *fdp = key; + setsock (fdp, fdp->sockfd, fdp->action, self); + } +} + +static void +initiate_next_curl_request (FetcherRequest *req, + GTask *task) +{ + CURLMcode rc; + OstreeFetcher *self = req->fetcher; + + req->easy = curl_easy_init (); + g_assert (req->easy); + + g_assert_cmpint (req->idx, <, req->mirrorlist->len); + + { g_autofree char *uri = request_get_uri (req, req->idx); + curl_easy_setopt (req->easy, CURLOPT_URL, uri); + } + + curl_easy_setopt (req->easy, CURLOPT_USERAGENT, "ostree "); + if (self->extra_headers) + curl_easy_setopt (req->easy, CURLOPT_HTTPHEADER, self->extra_headers); + + if (self->cookie_jar_path) + { + rc = curl_easy_setopt (req->easy, CURLOPT_COOKIEFILE, self->cookie_jar_path); + g_assert_cmpint (rc, ==, CURLM_OK); + rc = curl_easy_setopt (req->easy, CURLOPT_COOKIELIST, "RELOAD"); + g_assert_cmpint (rc, ==, CURLM_OK); + } + + if (self->proxy) + { + rc = curl_easy_setopt (req->easy, CURLOPT_PROXY, self->proxy); + g_assert_cmpint (rc, ==, CURLM_OK); + } + + if (self->tls_ca_db_path) + curl_easy_setopt (req->easy, CURLOPT_CAINFO, self->tls_ca_db_path); + + if ((self->config_flags & OSTREE_FETCHER_FLAGS_TLS_PERMISSIVE) > 0) + curl_easy_setopt (req->easy, CURLOPT_SSL_VERIFYPEER, 0L); + + if (self->tls_client_cert_path) + { + curl_easy_setopt (req->easy, CURLOPT_SSLCERT, self->tls_client_cert_path); + curl_easy_setopt (req->easy, CURLOPT_SSLKEY, self->tls_client_key_path); + } + + /* We should only speak HTTP; TODO: only enable file if specified */ + curl_easy_setopt (req->easy, CURLOPT_PROTOCOLS, (long)(CURLPROTO_HTTP | CURLPROTO_HTTPS | CURLPROTO_FILE)); + /* Picked the current version in F25 as of 20170127, since + * there are numerous HTTP/2 fixes since the original version in + * libcurl 7.43.0. + */ +#if CURL_AT_LEAST_VERSION(7, 51, 0) + curl_easy_setopt (req->easy, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0); +#endif + curl_easy_setopt (req->easy, CURLOPT_WRITEFUNCTION, write_cb); + if (g_getenv ("OSTREE_DEBUG_HTTP")) + curl_easy_setopt (req->easy, CURLOPT_VERBOSE, 1L); + curl_easy_setopt (req->easy, CURLOPT_ERRORBUFFER, req->error); + /* Note that the "easy" object's privdata is the task */ + curl_easy_setopt (req->easy, CURLOPT_NOPROGRESS, 1L); + curl_easy_setopt (req->easy, CURLOPT_PROGRESSFUNCTION, prog_cb); + curl_easy_setopt (req->easy, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt (req->easy, CURLOPT_CONNECTTIMEOUT, 30L); + curl_easy_setopt (req->easy, CURLOPT_LOW_SPEED_LIMIT, 1L); + curl_easy_setopt (req->easy, CURLOPT_LOW_SPEED_TIME, 30L); + + /* closure bindings -> task */ + curl_easy_setopt (req->easy, CURLOPT_PRIVATE, task); + curl_easy_setopt (req->easy, CURLOPT_WRITEDATA, task); + curl_easy_setopt (req->easy, CURLOPT_PROGRESSDATA, task); + + rc = curl_multi_add_handle (self->multi, req->easy); + g_assert (rc == CURLM_OK); +} + +static void +_ostree_fetcher_request_async (OstreeFetcher *self, + GPtrArray *mirrorlist, + const char *filename, + OstreeFetcherRequestFlags flags, + gboolean is_membuf, + guint64 max_size, + int priority, + GCancellable *cancellable, + GAsyncReadyCallback callback, + gpointer user_data) +{ + g_autoptr(GTask) task = NULL; + FetcherRequest *req; + g_autoptr(GMainContext) mainctx = g_main_context_ref_thread_default (); + + /* We don't support multiple concurrent main contexts; take + * a ref to the first one, and require that later invocations + * share it. + */ + if (g_hash_table_size (self->outstanding_requests) == 0 + && mainctx != self->mainctx) + { + adopt_steal_mainctx (self, g_steal_pointer (&mainctx)); + } + else + { + g_assert (self->mainctx == mainctx); + } + + req = g_new0 (FetcherRequest, 1); + req->refcount = 1; + req->error[0]='\0'; + req->fetcher = self; + req->mirrorlist = g_ptr_array_ref (mirrorlist); + req->filename = g_strdup (filename); + req->max_size = max_size; + req->flags = flags; + req->is_membuf = is_membuf; + /* We'll allocate the tmpfile on demand, so we handle + * file I/O errors just in the write func. + */ + req->out_tmpfile_fd = -1; + if (req->is_membuf) + req->output_buf = g_string_new (""); + + task = g_task_new (self, cancellable, callback, user_data); + /* We'll use the GTask priority for our own priority queue. */ + g_task_set_priority (task, priority); + g_task_set_source_tag (task, _ostree_fetcher_request_async); + g_task_set_task_data (task, req, (GDestroyNotify) request_unref); + + initiate_next_curl_request (req, task); + + g_hash_table_add (self->outstanding_requests, g_steal_pointer (&task)); + + /* Sanity check, I added * 2 just so we don't abort if something odd happens, + * but we do want to abort if we're asked to do obviously too many requests. + */ + g_assert_cmpint (g_hash_table_size (self->outstanding_requests), <, + _OSTREE_MAX_OUTSTANDING_FETCHER_REQUESTS * 2); +} + +void +_ostree_fetcher_request_to_tmpfile (OstreeFetcher *self, + GPtrArray *mirrorlist, + const char *filename, + guint64 max_size, + int priority, + GCancellable *cancellable, + GAsyncReadyCallback callback, + gpointer user_data) +{ + _ostree_fetcher_request_async (self, mirrorlist, filename, 0, FALSE, + max_size, priority, cancellable, + callback, user_data); +} + +gboolean +_ostree_fetcher_request_to_tmpfile_finish (OstreeFetcher *self, + GAsyncResult *result, + char **out_filename, + GError **error) +{ + GTask *task; + FetcherRequest *req; + gpointer ret; + + g_return_val_if_fail (g_task_is_valid (result, self), FALSE); + g_return_val_if_fail (g_async_result_is_tagged (result, _ostree_fetcher_request_async), FALSE); + + task = (GTask*)result; + req = g_task_get_task_data (task); + + ret = g_task_propagate_pointer (task, error); + if (!ret) + return FALSE; + + g_assert (!req->is_membuf); + g_assert (out_filename); + *out_filename = ret; + + return TRUE; +} + +void +_ostree_fetcher_request_to_membuf (OstreeFetcher *self, + GPtrArray *mirrorlist, + const char *filename, + OstreeFetcherRequestFlags flags, + guint64 max_size, + int priority, + GCancellable *cancellable, + GAsyncReadyCallback callback, + gpointer user_data) +{ + _ostree_fetcher_request_async (self, mirrorlist, filename, flags, TRUE, + max_size, priority, cancellable, + callback, user_data); +} + +gboolean +_ostree_fetcher_request_to_membuf_finish (OstreeFetcher *self, + GAsyncResult *result, + GBytes **out_buf, + GError **error) +{ + GTask *task; + FetcherRequest *req; + gpointer ret; + + g_return_val_if_fail (g_task_is_valid (result, self), FALSE); + g_return_val_if_fail (g_async_result_is_tagged (result, _ostree_fetcher_request_async), FALSE); + + task = (GTask*)result; + req = g_task_get_task_data (task); + + ret = g_task_propagate_pointer (task, error); + if (!ret) + return FALSE; + + g_assert (req->is_membuf); + g_assert (out_buf); + *out_buf = ret; + + return TRUE; +} + +guint64 +_ostree_fetcher_bytes_transferred (OstreeFetcher *self) +{ + return self->bytes_transferred; +} diff --git a/src/libostree/ostree-fetcher.c b/src/libostree/ostree-fetcher-soup.c similarity index 95% rename from src/libostree/ostree-fetcher.c rename to src/libostree/ostree-fetcher-soup.c index bb98023dc6..fcdf8e0e7c 100644 --- a/src/libostree/ostree-fetcher.c +++ b/src/libostree/ostree-fetcher-soup.c @@ -1332,84 +1332,3 @@ _ostree_fetcher_bytes_transferred (OstreeFetcher *self) return ret; } - -void -_ostree_fetcher_uri_free (OstreeFetcherURI *uri) -{ - if (uri) - soup_uri_free ((SoupURI*)uri); -} - -OstreeFetcherURI * -_ostree_fetcher_uri_parse (const char *str, - GError **error) -{ - SoupURI *soupuri = soup_uri_new (str); - if (soupuri == NULL) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Failed to parse uri: %s", str); - return NULL; - } - return (OstreeFetcherURI*)soupuri; -} - -static OstreeFetcherURI * -_ostree_fetcher_uri_new_path_internal (OstreeFetcherURI *uri, - gboolean extend, - const char *path) -{ - SoupURI *newuri = soup_uri_copy ((SoupURI*)uri); - if (path) - { - if (extend) - { - const char *origpath = soup_uri_get_path ((SoupURI*)uri); - g_autofree char *newpath = g_build_filename (origpath, path, NULL); - soup_uri_set_path (newuri, newpath); - } - else - { - soup_uri_set_path (newuri, path); - } - } - return (OstreeFetcherURI*)newuri; -} - -OstreeFetcherURI * -_ostree_fetcher_uri_new_path (OstreeFetcherURI *uri, - const char *path) -{ - return _ostree_fetcher_uri_new_path_internal (uri, FALSE, path); -} - -OstreeFetcherURI * -_ostree_fetcher_uri_new_subpath (OstreeFetcherURI *uri, - const char *subpath) -{ - return _ostree_fetcher_uri_new_path_internal (uri, TRUE, subpath); -} - -OstreeFetcherURI * -_ostree_fetcher_uri_clone (OstreeFetcherURI *uri) -{ - return _ostree_fetcher_uri_new_subpath (uri, NULL); -} - -char * -_ostree_fetcher_uri_get_scheme (OstreeFetcherURI *uri) -{ - return g_strdup (soup_uri_get_scheme ((SoupURI*)uri)); -} - -char * -_ostree_fetcher_uri_get_path (OstreeFetcherURI *uri) -{ - return g_strdup (soup_uri_get_path ((SoupURI*)uri)); -} - -char * -_ostree_fetcher_uri_to_string (OstreeFetcherURI *uri) -{ - return soup_uri_to_string ((SoupURI*)uri, FALSE); -} diff --git a/src/libostree/ostree-fetcher-uri.c b/src/libostree/ostree-fetcher-uri.c new file mode 100644 index 0000000000..7ef42eca84 --- /dev/null +++ b/src/libostree/ostree-fetcher-uri.c @@ -0,0 +1,118 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright (C) 2011,2017 Colin Walters + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + * Author: Colin Walters + */ + +#include "config.h" + + +#ifdef HAVE_LIBCURL +#include "ostree-soup-uri.h" +#else +#define LIBSOUP_USE_UNSTABLE_REQUEST_API +#include +#include +#include +#endif + +#include "ostree-fetcher.h" + +#include "libglnx.h" + +void +_ostree_fetcher_uri_free (OstreeFetcherURI *uri) +{ + if (uri) + soup_uri_free ((SoupURI*)uri); +} + +OstreeFetcherURI * +_ostree_fetcher_uri_parse (const char *str, + GError **error) +{ + SoupURI *soupuri = soup_uri_new (str); + if (soupuri == NULL) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Failed to parse uri: %s", str); + return NULL; + } + return (OstreeFetcherURI*)soupuri; +} + +static OstreeFetcherURI * +_ostree_fetcher_uri_new_path_internal (OstreeFetcherURI *uri, + gboolean extend, + const char *path) +{ + SoupURI *newuri = soup_uri_copy ((SoupURI*)uri); + if (path) + { + if (extend) + { + const char *origpath = soup_uri_get_path ((SoupURI*)uri); + g_autofree char *newpath = g_build_filename (origpath, path, NULL); + soup_uri_set_path (newuri, newpath); + } + else + { + soup_uri_set_path (newuri, path); + } + } + return (OstreeFetcherURI*)newuri; +} + +OstreeFetcherURI * +_ostree_fetcher_uri_new_path (OstreeFetcherURI *uri, + const char *path) +{ + return _ostree_fetcher_uri_new_path_internal (uri, FALSE, path); +} + +OstreeFetcherURI * +_ostree_fetcher_uri_new_subpath (OstreeFetcherURI *uri, + const char *subpath) +{ + return _ostree_fetcher_uri_new_path_internal (uri, TRUE, subpath); +} + +OstreeFetcherURI * +_ostree_fetcher_uri_clone (OstreeFetcherURI *uri) +{ + return _ostree_fetcher_uri_new_subpath (uri, NULL); +} + +char * +_ostree_fetcher_uri_get_scheme (OstreeFetcherURI *uri) +{ + return g_strdup (soup_uri_get_scheme ((SoupURI*)uri)); +} + +char * +_ostree_fetcher_uri_get_path (OstreeFetcherURI *uri) +{ + return g_strdup (soup_uri_get_path ((SoupURI*)uri)); +} + +char * +_ostree_fetcher_uri_to_string (OstreeFetcherURI *uri) +{ + return soup_uri_to_string ((SoupURI*)uri, FALSE); +} diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 79da78095c..295973ec3c 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -26,7 +26,7 @@ #include "ostree.h" #include "otutil.h" -#ifdef HAVE_LIBSOUP +#ifdef HAVE_LIBCURL_OR_LIBSOUP #include "ostree-core-private.h" #include "ostree-repo-private.h" @@ -3405,7 +3405,7 @@ ostree_repo_remote_fetch_summary_with_options (OstreeRepo *self, return ret; } -#else /* HAVE_LIBSOUP */ +#else /* HAVE_LIBCURL_OR_LIBSOUP */ gboolean ostree_repo_pull_with_options (OstreeRepo *self, @@ -3434,4 +3434,4 @@ ostree_repo_remote_fetch_summary_with_options (OstreeRepo *self, return FALSE; } -#endif /* HAVE_LIBSOUP */ +#endif /* HAVE_LIBCURL_OR_LIBSOUP */ diff --git a/src/libostree/ostree-soup-form.c b/src/libostree/ostree-soup-form.c new file mode 100644 index 0000000000..74f9c7bb04 --- /dev/null +++ b/src/libostree/ostree-soup-form.c @@ -0,0 +1,140 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*- */ +/* soup-form.c : utility functions for HTML forms */ + +/* + * Copyright 2008 Red Hat, Inc. + */ + +/* This one is stripped down to only have soup_form_encode_hash() + * and soup_form_encode_valist() which are the only bits that soup-uri.c + * calls. + */ + +#include + +#include + +#include "ostree-soup-uri.h" + +/** + * SECTION:soup-form + * @short_description: HTML form handling + * @see_also: #SoupMultipart + * + * libsoup contains several help methods for processing HTML forms as + * defined by the + * HTML 4.01 specification. + **/ + +/** + * SOUP_FORM_MIME_TYPE_URLENCODED: + * + * A macro containing the value + * "application/x-www-form-urlencoded"; the default + * MIME type for POSTing HTML form data. + * + * Since: 2.26 + **/ + +/** + * SOUP_FORM_MIME_TYPE_MULTIPART: + * + * A macro containing the value + * "multipart/form-data"; the MIME type used for + * posting form data that contains files to be uploaded. + * + * Since: 2.26 + **/ + +#define XDIGIT(c) ((c) <= '9' ? (c) - '0' : ((c) & 0x4F) - 'A' + 10) +#define HEXCHAR(s) ((XDIGIT (s[1]) << 4) + XDIGIT (s[2])) + +static void +append_form_encoded (GString *str, const char *in) +{ + const unsigned char *s = (const unsigned char *)in; + + while (*s) { + if (*s == ' ') { + g_string_append_c (str, '+'); + s++; + } else if (!g_ascii_isalnum (*s) && (*s != '-') && (*s != '_') + && (*s != '.')) + g_string_append_printf (str, "%%%02X", (int)*s++); + else + g_string_append_c (str, *s++); + } +} + +static void +encode_pair (GString *str, const char *name, const char *value) +{ + g_return_if_fail (name != NULL); + g_return_if_fail (value != NULL); + + if (str->len) + g_string_append_c (str, '&'); + append_form_encoded (str, name); + g_string_append_c (str, '='); + append_form_encoded (str, value); +} + +/** + * soup_form_encode_hash: + * @form_data_set: (element-type utf8 utf8): a hash table containing + * name/value pairs (as strings) + * + * Encodes @form_data_set into a value of type + * "application/x-www-form-urlencoded", as defined in the HTML 4.01 + * spec. + * + * Note that the HTML spec states that "The control names/values are + * listed in the order they appear in the document." Since this method + * takes a hash table, it cannot enforce that; if you care about the + * ordering of the form fields, use soup_form_encode_datalist(). + * + * Return value: the encoded form + **/ +char * +soup_form_encode_hash (GHashTable *form_data_set) +{ + GString *str = g_string_new (NULL); + GHashTableIter iter; + gpointer name, value; + + g_hash_table_iter_init (&iter, form_data_set); + while (g_hash_table_iter_next (&iter, &name, &value)) + encode_pair (str, name, value); + return g_string_free (str, FALSE); +} + +/** + * soup_form_encode_valist: + * @first_field: name of the first form field + * @args: pointer to additional values, as in soup_form_encode() + * + * See soup_form_encode(). This is mostly an internal method, used by + * various other methods such as soup_uri_set_query_from_fields() and + * soup_form_request_new(). + * + * Return value: the encoded form + **/ +char * +soup_form_encode_valist (const char *first_field, va_list args) +{ + GString *str = g_string_new (NULL); + const char *name, *value; + + name = first_field; + value = va_arg (args, const char *); + while (name && value) { + encode_pair (str, name, value); + + name = va_arg (args, const char *); + if (name) + value = va_arg (args, const char *); + } + + return g_string_free (str, FALSE); +} diff --git a/src/libostree/ostree-soup-uri.c b/src/libostree/ostree-soup-uri.c new file mode 100644 index 0000000000..97f74636b6 --- /dev/null +++ b/src/libostree/ostree-soup-uri.c @@ -0,0 +1,1483 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*- */ +/* soup-uri.c : utility functions to parse URLs */ + +/* + * Copyright 1999-2003 Ximian, Inc. + */ + +#include "config.h" + +#include +#include + +#include "ostree-soup-uri.h" + +/* OSTREECHANGE: definitions from soup-misc-private.h */ +char *soup_uri_decoded_copy (const char *str, int length, int *decoded_length); +char *soup_uri_to_string_internal (SoupURI *uri, gboolean just_path_and_query, + gboolean force_port); +gboolean soup_uri_is_http (SoupURI *uri, char **aliases); +gboolean soup_uri_is_https (SoupURI *uri, char **aliases); + +/* OSTREECHANGE: import soup-misc's char helpers */ +#define SOUP_CHAR_URI_PERCENT_ENCODED 0x01 +#define SOUP_CHAR_URI_GEN_DELIMS 0x02 +#define SOUP_CHAR_URI_SUB_DELIMS 0x04 +#define SOUP_CHAR_HTTP_SEPARATOR 0x08 +#define SOUP_CHAR_HTTP_CTL 0x10 + +/* 00 URI_UNRESERVED + * 01 URI_PCT_ENCODED + * 02 URI_GEN_DELIMS + * 04 URI_SUB_DELIMS + * 08 HTTP_SEPARATOR + * 10 HTTP_CTL + */ +const char soup_char_attributes[] = { + /* 0x00 - 0x07 */ + 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, + /* 0x08 - 0x0f */ + 0x11, 0x19, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, + /* 0x10 - 0x17 */ + 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, + /* 0x18 - 0x1f */ + 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, + /* !"#$%&' */ + 0x09, 0x04, 0x09, 0x02, 0x04, 0x01, 0x04, 0x04, + /* ()*+,-./ */ + 0x0c, 0x0c, 0x04, 0x04, 0x0c, 0x00, 0x00, 0x0a, + /* 01234567 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + /* 89:;<=>? */ + 0x00, 0x00, 0x0a, 0x0c, 0x09, 0x0a, 0x09, 0x0a, + /* @ABCDEFG */ + 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + /* HIJKLMNO */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + /* PQRSTUVW */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + /* XYZ[\]^_ */ + 0x00, 0x00, 0x00, 0x0a, 0x09, 0x0a, 0x01, 0x00, + /* `abcdefg */ + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + /* hijklmno */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + /* pqrstuvw */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + /* xyz{|}~ */ + 0x00, 0x00, 0x00, 0x09, 0x01, 0x09, 0x00, 0x11, + /* 0x80 - 0xFF */ + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01 +}; + +#define soup_char_is_uri_percent_encoded(ch) (soup_char_attributes[(guchar)ch] & SOUP_CHAR_URI_PERCENT_ENCODED) +#define soup_char_is_uri_gen_delims(ch) (soup_char_attributes[(guchar)ch] & SOUP_CHAR_URI_GEN_DELIMS) +#define soup_char_is_uri_sub_delims(ch) (soup_char_attributes[(guchar)ch] & SOUP_CHAR_URI_SUB_DELIMS) +#define soup_char_is_uri_unreserved(ch) (!(soup_char_attributes[(guchar)ch] & (SOUP_CHAR_URI_PERCENT_ENCODED | SOUP_CHAR_URI_GEN_DELIMS | SOUP_CHAR_URI_SUB_DELIMS))) +#define soup_char_is_token(ch) (!(soup_char_attributes[(guchar)ch] & (SOUP_CHAR_HTTP_SEPARATOR | SOUP_CHAR_HTTP_CTL))) + +/** + * soup_str_case_hash: + * @key: ASCII string to hash + * + * Hashes @key in a case-insensitive manner. + * + * Return value: the hash code. + **/ +static guint +soup_str_case_hash (gconstpointer key) +{ + const char *p = key; + guint h = g_ascii_toupper(*p); + + if (h) + for (p += 1; *p != '\0'; p++) + h = (h << 5) - h + g_ascii_toupper(*p); + + return h; +} + +/** + * SECTION:soup-uri + * @short_description: URIs + * + * A #SoupURI represents a (parsed) URI. + * + * Many applications will not need to use #SoupURI directly at all; on + * the client side, soup_message_new() takes a stringified URI, and on + * the server side, the path and query components are provided for you + * in the server callback. + **/ + +/** + * SoupURI: + * @scheme: the URI scheme (eg, "http") + * @user: a username, or %NULL + * @password: a password, or %NULL + * @host: the hostname or IP address + * @port: the port number on @host + * @path: the path on @host + * @query: a query for @path, or %NULL + * @fragment: a fragment identifier within @path, or %NULL + * + * A #SoupURI represents a (parsed) URI. #SoupURI supports RFC 3986 + * (URI Generic Syntax), and can parse any valid URI. However, libsoup + * only uses "http" and "https" URIs internally; You can use + * SOUP_URI_VALID_FOR_HTTP() to test if a #SoupURI is a valid HTTP + * URI. + * + * @scheme will always be set in any URI. It is an interned string and + * is always all lowercase. (If you parse a URI with a non-lowercase + * scheme, it will be converted to lowercase.) The macros + * %SOUP_URI_SCHEME_HTTP and %SOUP_URI_SCHEME_HTTPS provide the + * interned values for "http" and "https" and can be compared against + * URI @scheme values. + * + * @user and @password are parsed as defined in the older URI specs + * (ie, separated by a colon; RFC 3986 only talks about a single + * "userinfo" field). Note that @password is not included in the + * output of soup_uri_to_string(). libsoup does not normally use these + * fields; authentication is handled via #SoupSession signals. + * + * @host contains the hostname, and @port the port specified in the + * URI. If the URI doesn't contain a hostname, @host will be %NULL, + * and if it doesn't specify a port, @port may be 0. However, for + * "http" and "https" URIs, @host is guaranteed to be non-%NULL + * (trying to parse an http URI with no @host will return %NULL), and + * @port will always be non-0 (because libsoup knows the default value + * to use when it is not specified in the URI). + * + * @path is always non-%NULL. For http/https URIs, @path will never be + * an empty string either; if the input URI has no path, the parsed + * #SoupURI will have a @path of "/". + * + * @query and @fragment are optional for all URI types. + * soup_form_decode() may be useful for parsing @query. + * + * Note that @path, @query, and @fragment may contain + * %-encoded characters. soup_uri_new() calls + * soup_uri_normalize() on them, but not soup_uri_decode(). This is + * necessary to ensure that soup_uri_to_string() will generate a URI + * that has exactly the same meaning as the original. (In theory, + * #SoupURI should leave @user, @password, and @host partially-encoded + * as well, but this would be more annoying than useful.) + **/ + +/** + * SOUP_URI_IS_VALID: + * @uri: a #SoupURI + * + * Tests whether @uri is a valid #SoupURI; that is, that it is non-%NULL + * and its @scheme and @path members are also non-%NULL. + * + * This macro does not check whether http and https URIs have a non-%NULL + * @host member. + * + * Return value: %TRUE if @uri is valid for use. + * + * Since: 2.38 + **/ + +/** + * SOUP_URI_VALID_FOR_HTTP: + * @uri: a #SoupURI + * + * Tests if @uri is a valid #SoupURI for HTTP communication; that is, if + * it can be used to construct a #SoupMessage. + * + * Return value: %TRUE if @uri is a valid "http" or "https" URI. + * + * Since: 2.24 + **/ + +/** + * SOUP_URI_SCHEME_HTTP: + * + * "http" as an interned string; you can compare this directly to a + * #SoupURI's scheme field using + * ==. + */ +/** + * SOUP_URI_SCHEME_HTTPS: + * + * "https" as an interned string; you can compare this directly to a + * #SoupURI's scheme field using + * ==. + */ +/** + * SOUP_URI_SCHEME_FTP: + * + * "ftp" as an interned string; you can compare this directly to a + * #SoupURI's scheme field using + * ==. + * + * Since: 2.30 + */ +/** + * SOUP_URI_SCHEME_FILE: + * + * "file" as an interned string; you can compare this directly to a + * #SoupURI's scheme field using + * ==. + * + * Since: 2.30 + */ +/** + * SOUP_URI_SCHEME_DATA: + * + * "data" as an interned string; you can compare this directly to a + * #SoupURI's scheme field using + * ==. + * + * Since: 2.30 + */ +/** + * SOUP_URI_SCHEME_RESOURCE: + * + * "data" as an interned string; you can compare this directly to a + * #SoupURI's scheme field using + * ==. + * + * Since: 2.42 + */ +/** + * SOUP_URI_SCHEME_WS: + * + * "ws" (WebSocket) as an interned string; you can compare this + * directly to a #SoupURI's scheme field using + * ==. + * + * Since: 2.50 + */ +/** + * SOUP_URI_SCHEME_WSS: + * + * "wss" (WebSocket over TLS) as an interned string; you can compare + * this directly to a #SoupURI's scheme field using + * ==. + * + * Since: 2.50 + */ + +struct _SoupURI { + const char *scheme; + + char *user; + char *password; + + char *host; + guint port; + + char *path; + char *query; + + char *fragment; +}; + +static void append_uri_encoded (GString *str, const char *in, const char *extra_enc_chars); +static char *uri_normalized_copy (const char *str, int length, const char *unescape_extra); + +gpointer _SOUP_URI_SCHEME_HTTP, _SOUP_URI_SCHEME_HTTPS; +gpointer _SOUP_URI_SCHEME_WS, _SOUP_URI_SCHEME_WSS; +gpointer _SOUP_URI_SCHEME_FTP; +gpointer _SOUP_URI_SCHEME_FILE, _SOUP_URI_SCHEME_DATA, _SOUP_URI_SCHEME_RESOURCE; + +static inline const char * +soup_uri_parse_scheme (const char *scheme, int len) +{ + if (len == 4 && !g_ascii_strncasecmp (scheme, "http", len)) { + return SOUP_URI_SCHEME_HTTP; + } else if (len == 5 && !g_ascii_strncasecmp (scheme, "https", len)) { + return SOUP_URI_SCHEME_HTTPS; + } else if (len == 8 && !g_ascii_strncasecmp (scheme, "resource", len)) { + return SOUP_URI_SCHEME_RESOURCE; + } else if (len == 2 && !g_ascii_strncasecmp (scheme, "ws", len)) { + return SOUP_URI_SCHEME_WS; + } else if (len == 3 && !g_ascii_strncasecmp (scheme, "wss", len)) { + return SOUP_URI_SCHEME_WSS; + } else { + char *lower_scheme; + + lower_scheme = g_ascii_strdown (scheme, len); + scheme = g_intern_static_string (lower_scheme); + if (scheme != (const char *)lower_scheme) + g_free (lower_scheme); + return scheme; + } +} + +static inline guint +soup_scheme_default_port (const char *scheme) +{ + if (scheme == SOUP_URI_SCHEME_HTTP || scheme == SOUP_URI_SCHEME_WS) + return 80; + else if (scheme == SOUP_URI_SCHEME_HTTPS || scheme == SOUP_URI_SCHEME_WSS) + return 443; + else if (scheme == SOUP_URI_SCHEME_FTP) + return 21; + else + return 0; +} + +/** + * soup_uri_new_with_base: + * @base: a base URI + * @uri_string: the URI + * + * Parses @uri_string relative to @base. + * + * Return value: a parsed #SoupURI. + **/ +SoupURI * +soup_uri_new_with_base (SoupURI *base, const char *uri_string) +{ + SoupURI *uri, fixed_base; + const char *end, *hash, *colon, *at, *path, *question; + const char *p, *hostend; + gboolean remove_dot_segments = TRUE; + int len; + + g_return_val_if_fail (uri_string != NULL, NULL); + + /* Allow a %NULL path in @base, for compatibility */ + if (base && base->scheme && !base->path) { + g_warn_if_fail (SOUP_URI_IS_VALID (base)); + + memcpy (&fixed_base, base, sizeof (SoupURI)); + fixed_base.path = ""; + base = &fixed_base; + } + + g_return_val_if_fail (base == NULL || SOUP_URI_IS_VALID (base), NULL); + + /* First some cleanup steps (which are supposed to all be no-ops, + * but...). Skip initial whitespace, strip out internal tabs and + * line breaks, and ignore trailing whitespace. + */ + while (g_ascii_isspace (*uri_string)) + uri_string++; + + len = strcspn (uri_string, "\t\n\r"); + if (uri_string[len]) { + char *clean = g_malloc (strlen (uri_string) + 1), *d; + const char *s; + + for (s = uri_string, d = clean; *s; s++) { + if (*s != '\t' && *s != '\n' && *s != '\r') + *d++ = *s; + } + *d = '\0'; + + uri = soup_uri_new_with_base (base, clean); + g_free (clean); + return uri; + } + end = uri_string + len; + while (end > uri_string && g_ascii_isspace (end[-1])) + end--; + + uri = g_slice_new0 (SoupURI); + + /* Find fragment. */ + hash = strchr (uri_string, '#'); + if (hash) { + uri->fragment = uri_normalized_copy (hash + 1, end - hash + 1, + NULL); + end = hash; + } + + /* Find scheme */ + p = uri_string; + while (p < end && (g_ascii_isalpha (*p) || + (p > uri_string && (g_ascii_isdigit (*p) || + *p == '.' || + *p == '+' || + *p == '-')))) + p++; + + if (p > uri_string && *p == ':') { + uri->scheme = soup_uri_parse_scheme (uri_string, p - uri_string); + uri_string = p + 1; + } + + if (uri_string == end && !base && !uri->fragment) { + uri->path = g_strdup (""); + return uri; + } + + /* Check for authority */ + if (strncmp (uri_string, "//", 2) == 0) { + uri_string += 2; + + path = uri_string + strcspn (uri_string, "/?#"); + if (path > end) + path = end; + at = strchr (uri_string, '@'); + if (at && at < path) { + colon = strchr (uri_string, ':'); + if (colon && colon < at) { + uri->password = soup_uri_decoded_copy (colon + 1, + at - colon - 1, NULL); + } else { + uri->password = NULL; + colon = at; + } + + uri->user = soup_uri_decoded_copy (uri_string, + colon - uri_string, NULL); + uri_string = at + 1; + } else + uri->user = uri->password = NULL; + + /* Find host and port. */ + if (*uri_string == '[') { + const char *pct; + + uri_string++; + hostend = strchr (uri_string, ']'); + if (!hostend || hostend > path) { + soup_uri_free (uri); + return NULL; + } + if (*(hostend + 1) == ':') + colon = hostend + 1; + else + colon = NULL; + + pct = memchr (uri_string, '%', hostend - uri_string); + if (!pct || (pct[1] == '2' && pct[2] == '5')) { + uri->host = soup_uri_decoded_copy (uri_string, + hostend - uri_string, NULL); + } else + uri->host = g_strndup (uri_string, hostend - uri_string); + } else { + colon = memchr (uri_string, ':', path - uri_string); + hostend = colon ? colon : path; + uri->host = soup_uri_decoded_copy (uri_string, + hostend - uri_string, NULL); + } + + if (colon && colon != path - 1) { + char *portend; + uri->port = strtoul (colon + 1, &portend, 10); + if (portend != (char *)path) { + soup_uri_free (uri); + return NULL; + } + } + + uri_string = path; + } + + /* Find query */ + question = memchr (uri_string, '?', end - uri_string); + if (question) { + uri->query = uri_normalized_copy (question + 1, + end - (question + 1), + NULL); + end = question; + } + + if (end != uri_string) { + uri->path = uri_normalized_copy (uri_string, end - uri_string, + NULL); + } + + /* Apply base URI. This is spelled out in RFC 3986. */ + if (base && !uri->scheme && uri->host) + uri->scheme = base->scheme; + else if (base && !uri->scheme) { + uri->scheme = base->scheme; + uri->user = g_strdup (base->user); + uri->password = g_strdup (base->password); + uri->host = g_strdup (base->host); + uri->port = base->port; + + if (!uri->path) { + uri->path = g_strdup (base->path); + if (!uri->query) + uri->query = g_strdup (base->query); + remove_dot_segments = FALSE; + } else if (*uri->path != '/') { + char *newpath, *last; + + last = strrchr (base->path, '/'); + if (last) { + newpath = g_strdup_printf ("%.*s%s", + (int)(last + 1 - base->path), + base->path, + uri->path); + } else + newpath = g_strdup_printf ("/%s", uri->path); + + g_free (uri->path); + uri->path = newpath; + } + } + + if (remove_dot_segments && uri->path && *uri->path) { + char *p, *q; + + /* Remove "./" where "." is a complete segment. */ + for (p = uri->path + 1; *p; ) { + if (*(p - 1) == '/' && + *p == '.' && *(p + 1) == '/') + memmove (p, p + 2, strlen (p + 2) + 1); + else + p++; + } + /* Remove "." at end. */ + if (p > uri->path + 2 && + *(p - 1) == '.' && *(p - 2) == '/') + *(p - 1) = '\0'; + + /* Remove "/../" where != ".." */ + for (p = uri->path + 1; *p; ) { + if (!strncmp (p, "../", 3)) { + p += 3; + continue; + } + q = strchr (p + 1, '/'); + if (!q) + break; + if (strncmp (q, "/../", 4) != 0) { + p = q + 1; + continue; + } + memmove (p, q + 4, strlen (q + 4) + 1); + p = uri->path + 1; + } + /* Remove "/.." at end where != ".." */ + q = strrchr (uri->path, '/'); + if (q && !strcmp (q, "/..")) { + p = q - 1; + while (p > uri->path && *p != '/') + p--; + if (strncmp (p, "/../", 4) != 0) + *(p + 1) = 0; + } + + /* Remove extraneous initial "/.."s */ + while (!strncmp (uri->path, "/../", 4)) + memmove (uri->path, uri->path + 3, strlen (uri->path) - 2); + if (!strcmp (uri->path, "/..")) + uri->path[1] = '\0'; + } + + /* HTTP-specific stuff */ + if (uri->scheme == SOUP_URI_SCHEME_HTTP || + uri->scheme == SOUP_URI_SCHEME_HTTPS) { + if (!uri->path) + uri->path = g_strdup ("/"); + if (!SOUP_URI_VALID_FOR_HTTP (uri)) { + soup_uri_free (uri); + return NULL; + } + } + + if (uri->scheme == SOUP_URI_SCHEME_FTP) { + if (!uri->host) { + soup_uri_free (uri); + return NULL; + } + } + + if (!uri->port) + uri->port = soup_scheme_default_port (uri->scheme); + if (!uri->path) + uri->path = g_strdup (""); + + return uri; +} + +/** + * soup_uri_new: + * @uri_string: (allow-none): a URI + * + * Parses an absolute URI. + * + * You can also pass %NULL for @uri_string if you want to get back an + * "empty" #SoupURI that you can fill in by hand. (You will need to + * call at least soup_uri_set_scheme() and soup_uri_set_path(), since + * those fields are required.) + * + * Return value: (nullable): a #SoupURI, or %NULL if the given string + * was found to be invalid. + **/ +SoupURI * +soup_uri_new (const char *uri_string) +{ + SoupURI *uri; + + if (!uri_string) + return g_slice_new0 (SoupURI); + + uri = soup_uri_new_with_base (NULL, uri_string); + if (!uri) + return NULL; + if (!SOUP_URI_IS_VALID (uri)) { + soup_uri_free (uri); + return NULL; + } + + return uri; +} + + +char * +soup_uri_to_string_internal (SoupURI *uri, gboolean just_path_and_query, + gboolean force_port) +{ + GString *str; + char *return_result; + + g_return_val_if_fail (uri != NULL, NULL); + g_warn_if_fail (SOUP_URI_IS_VALID (uri)); + + str = g_string_sized_new (40); + + if (uri->scheme && !just_path_and_query) + g_string_append_printf (str, "%s:", uri->scheme); + if (uri->host && !just_path_and_query) { + g_string_append (str, "//"); + if (uri->user) { + append_uri_encoded (str, uri->user, ":;@?/"); + g_string_append_c (str, '@'); + } + if (strchr (uri->host, ':')) { + const char *pct; + + g_string_append_c (str, '['); + pct = strchr (uri->host, '%'); + if (pct) { + g_string_append_printf (str, "%.*s%%25%s", + (int) (pct - uri->host), + uri->host, pct + 1); + } else + g_string_append (str, uri->host); + g_string_append_c (str, ']'); + } else + append_uri_encoded (str, uri->host, ":/"); + if (uri->port && (force_port || uri->port != soup_scheme_default_port (uri->scheme))) + g_string_append_printf (str, ":%u", uri->port); + if (!uri->path && (uri->query || uri->fragment)) + g_string_append_c (str, '/'); + else if ((!uri->path || !*uri->path) && + (uri->scheme == SOUP_URI_SCHEME_HTTP || + uri->scheme == SOUP_URI_SCHEME_HTTPS)) + g_string_append_c (str, '/'); + } + + if (uri->path && *uri->path) + g_string_append (str, uri->path); + else if (just_path_and_query) + g_string_append_c (str, '/'); + + if (uri->query) { + g_string_append_c (str, '?'); + g_string_append (str, uri->query); + } + if (uri->fragment && !just_path_and_query) { + g_string_append_c (str, '#'); + g_string_append (str, uri->fragment); + } + + return_result = str->str; + g_string_free (str, FALSE); + + return return_result; +} + +/** + * soup_uri_to_string: + * @uri: a #SoupURI + * @just_path_and_query: if %TRUE, output just the path and query portions + * + * Returns a string representing @uri. + * + * If @just_path_and_query is %TRUE, this concatenates the path and query + * together. That is, it constructs the string that would be needed in + * the Request-Line of an HTTP request for @uri. + * + * Note that the output will never contain a password, even if @uri + * does. + * + * Return value: a string representing @uri, which the caller must free. + **/ +char * +soup_uri_to_string (SoupURI *uri, gboolean just_path_and_query) +{ + return soup_uri_to_string_internal (uri, just_path_and_query, FALSE); +} + +/** + * soup_uri_copy: + * @uri: a #SoupURI + * + * Copies @uri + * + * Return value: a copy of @uri, which must be freed with soup_uri_free() + **/ +SoupURI * +soup_uri_copy (SoupURI *uri) +{ + SoupURI *dup; + + g_return_val_if_fail (uri != NULL, NULL); + g_warn_if_fail (SOUP_URI_IS_VALID (uri)); + + dup = g_slice_new0 (SoupURI); + dup->scheme = uri->scheme; + dup->user = g_strdup (uri->user); + dup->password = g_strdup (uri->password); + dup->host = g_strdup (uri->host); + dup->port = uri->port; + dup->path = g_strdup (uri->path); + dup->query = g_strdup (uri->query); + dup->fragment = g_strdup (uri->fragment); + + return dup; +} + +static inline gboolean +parts_equal (const char *one, const char *two, gboolean insensitive) +{ + if (!one && !two) + return TRUE; + if (!one || !two) + return FALSE; + return insensitive ? !g_ascii_strcasecmp (one, two) : !strcmp (one, two); +} + +/** + * soup_uri_equal: + * @uri1: a #SoupURI + * @uri2: another #SoupURI + * + * Tests whether or not @uri1 and @uri2 are equal in all parts + * + * Return value: %TRUE or %FALSE + **/ +gboolean +soup_uri_equal (SoupURI *uri1, SoupURI *uri2) +{ + g_return_val_if_fail (uri1 != NULL, FALSE); + g_return_val_if_fail (uri2 != NULL, FALSE); + g_warn_if_fail (SOUP_URI_IS_VALID (uri1)); + g_warn_if_fail (SOUP_URI_IS_VALID (uri2)); + + if (uri1->scheme != uri2->scheme || + uri1->port != uri2->port || + !parts_equal (uri1->user, uri2->user, FALSE) || + !parts_equal (uri1->password, uri2->password, FALSE) || + !parts_equal (uri1->host, uri2->host, TRUE) || + !parts_equal (uri1->path, uri2->path, FALSE) || + !parts_equal (uri1->query, uri2->query, FALSE) || + !parts_equal (uri1->fragment, uri2->fragment, FALSE)) + return FALSE; + + return TRUE; +} + +/** + * soup_uri_free: + * @uri: a #SoupURI + * + * Frees @uri. + **/ +void +soup_uri_free (SoupURI *uri) +{ + g_return_if_fail (uri != NULL); + + g_free (uri->user); + g_free (uri->password); + g_free (uri->host); + g_free (uri->path); + g_free (uri->query); + g_free (uri->fragment); + + g_slice_free (SoupURI, uri); +} + +static void +append_uri_encoded (GString *str, const char *in, const char *extra_enc_chars) +{ + const unsigned char *s = (const unsigned char *)in; + + while (*s) { + if (soup_char_is_uri_percent_encoded (*s) || + soup_char_is_uri_gen_delims (*s) || + (extra_enc_chars && strchr (extra_enc_chars, *s))) + g_string_append_printf (str, "%%%02X", (int)*s++); + else + g_string_append_c (str, *s++); + } +} + +/** + * soup_uri_encode: + * @part: a URI part + * @escape_extra: (allow-none): additional reserved characters to + * escape (or %NULL) + * + * This %-encodes the given URI part and returns the escaped + * version in allocated memory, which the caller must free when it is + * done. + * + * Return value: the encoded URI part + **/ +char * +soup_uri_encode (const char *part, const char *escape_extra) +{ + GString *str; + char *encoded; + + g_return_val_if_fail (part != NULL, NULL); + + str = g_string_new (NULL); + append_uri_encoded (str, part, escape_extra); + encoded = str->str; + g_string_free (str, FALSE); + + return encoded; +} + +#define XDIGIT(c) ((c) <= '9' ? (c) - '0' : ((c) & 0x4F) - 'A' + 10) +#define HEXCHAR(s) ((XDIGIT (s[1]) << 4) + XDIGIT (s[2])) + +char * +soup_uri_decoded_copy (const char *part, int length, int *decoded_length) +{ + unsigned char *s, *d; + char *decoded; + + g_return_val_if_fail (part != NULL, NULL); + + decoded = g_strndup (part, length); + s = d = (unsigned char *)decoded; + do { + if (*s == '%') { + if (!g_ascii_isxdigit (s[1]) || + !g_ascii_isxdigit (s[2])) { + *d++ = *s; + continue; + } + *d++ = HEXCHAR (s); + s += 2; + } else + *d++ = *s; + } while (*s++); + + if (decoded_length) + *decoded_length = d - (unsigned char *)decoded - 1; + + return decoded; +} + +/** + * soup_uri_decode: + * @part: a URI part + * + * Fully %-decodes @part. + * + * In the past, this would return %NULL if @part contained invalid + * percent-encoding, but now it just ignores the problem (as + * soup_uri_new() already did). + * + * Return value: the decoded URI part. + */ +char * +soup_uri_decode (const char *part) +{ + g_return_val_if_fail (part != NULL, NULL); + + return soup_uri_decoded_copy (part, strlen (part), NULL); +} + +static char * +uri_normalized_copy (const char *part, int length, + const char *unescape_extra) +{ + unsigned char *s, *d, c; + char *normalized = g_strndup (part, length); + gboolean need_fixup = FALSE; + + if (!unescape_extra) + unescape_extra = ""; + + s = d = (unsigned char *)normalized; + while (*s) { + if (*s == '%') { + if (!g_ascii_isxdigit (s[1]) || + !g_ascii_isxdigit (s[2])) { + *d++ = *s++; + continue; + } + + c = HEXCHAR (s); + if (soup_char_is_uri_unreserved (c) || + (c && strchr (unescape_extra, c))) { + *d++ = c; + s += 3; + } else { + /* We leave it unchanged. We used to uppercase percent-encoded + * triplets but we do not do it any more as RFC3986 Section 6.2.2.1 + * says that they only SHOULD be case normalized. + */ + *d++ = *s++; + *d++ = *s++; + *d++ = *s++; + } + } else { + if (!g_ascii_isgraph (*s) && + !strchr (unescape_extra, *s)) + need_fixup = TRUE; + *d++ = *s++; + } + } + *d = '\0'; + + if (need_fixup) { + GString *fixed; + + fixed = g_string_new (NULL); + s = (guchar *)normalized; + while (*s) { + if (g_ascii_isgraph (*s) || + strchr (unescape_extra, *s)) + g_string_append_c (fixed, *s); + else + g_string_append_printf (fixed, "%%%02X", (int)*s); + s++; + } + g_free (normalized); + normalized = g_string_free (fixed, FALSE); + } + + return normalized; +} + +/** + * soup_uri_normalize: + * @part: a URI part + * @unescape_extra: (allow-none): reserved characters to unescape (or %NULL) + * + * %-decodes any "unreserved" characters (or characters in + * @unescape_extra) in @part, and %-encodes any non-ASCII + * characters, spaces, and non-printing characters in @part. + * + * "Unreserved" characters are those that are not allowed to be used + * for punctuation according to the URI spec. For example, letters are + * unreserved, so soup_uri_normalize() will turn + * http://example.com/foo/b%61r into + * http://example.com/foo/bar, which is guaranteed + * to mean the same thing. However, "/" is "reserved", so + * http://example.com/foo%2Fbar would not + * be changed, because it might mean something different to the + * server. + * + * In the past, this would return %NULL if @part contained invalid + * percent-encoding, but now it just ignores the problem (as + * soup_uri_new() already did). + * + * Return value: the normalized URI part + */ +char * +soup_uri_normalize (const char *part, const char *unescape_extra) +{ + g_return_val_if_fail (part != NULL, NULL); + + return uri_normalized_copy (part, strlen (part), unescape_extra); +} + + +/** + * soup_uri_uses_default_port: + * @uri: a #SoupURI + * + * Tests if @uri uses the default port for its scheme. (Eg, 80 for + * http.) (This only works for http, https and ftp; libsoup does not know + * the default ports of other protocols.) + * + * Return value: %TRUE or %FALSE + **/ +gboolean +soup_uri_uses_default_port (SoupURI *uri) +{ + g_return_val_if_fail (uri != NULL, FALSE); + g_warn_if_fail (SOUP_URI_IS_VALID (uri)); + + return uri->port == soup_scheme_default_port (uri->scheme); +} + +/** + * soup_uri_get_scheme: + * @uri: a #SoupURI + * + * Gets @uri's scheme. + * + * Return value: @uri's scheme. + * + * Since: 2.32 + **/ +const char * +soup_uri_get_scheme (SoupURI *uri) +{ + g_return_val_if_fail (uri != NULL, NULL); + + return uri->scheme; +} + +/** + * soup_uri_set_scheme: + * @uri: a #SoupURI + * @scheme: the URI scheme + * + * Sets @uri's scheme to @scheme. This will also set @uri's port to + * the default port for @scheme, if known. + **/ +void +soup_uri_set_scheme (SoupURI *uri, const char *scheme) +{ + g_return_if_fail (uri != NULL); + g_return_if_fail (scheme != NULL); + + uri->scheme = soup_uri_parse_scheme (scheme, strlen (scheme)); + uri->port = soup_scheme_default_port (uri->scheme); +} + +/** + * soup_uri_get_user: + * @uri: a #SoupURI + * + * Gets @uri's user. + * + * Return value: @uri's user. + * + * Since: 2.32 + **/ +const char * +soup_uri_get_user (SoupURI *uri) +{ + g_return_val_if_fail (uri != NULL, NULL); + + return uri->user; +} + +/** + * soup_uri_set_user: + * @uri: a #SoupURI + * @user: (allow-none): the username, or %NULL + * + * Sets @uri's user to @user. + **/ +void +soup_uri_set_user (SoupURI *uri, const char *user) +{ + g_return_if_fail (uri != NULL); + + g_free (uri->user); + uri->user = g_strdup (user); +} + +/** + * soup_uri_get_password: + * @uri: a #SoupURI + * + * Gets @uri's password. + * + * Return value: @uri's password. + * + * Since: 2.32 + **/ +const char * +soup_uri_get_password (SoupURI *uri) +{ + g_return_val_if_fail (uri != NULL, NULL); + + return uri->password; +} + +/** + * soup_uri_set_password: + * @uri: a #SoupURI + * @password: (allow-none): the password, or %NULL + * + * Sets @uri's password to @password. + **/ +void +soup_uri_set_password (SoupURI *uri, const char *password) +{ + g_return_if_fail (uri != NULL); + + g_free (uri->password); + uri->password = g_strdup (password); +} + +/** + * soup_uri_get_host: + * @uri: a #SoupURI + * + * Gets @uri's host. + * + * Return value: @uri's host. + * + * Since: 2.32 + **/ +const char * +soup_uri_get_host (SoupURI *uri) +{ + g_return_val_if_fail (uri != NULL, NULL); + + return uri->host; +} + +/** + * soup_uri_set_host: + * @uri: a #SoupURI + * @host: (allow-none): the hostname or IP address, or %NULL + * + * Sets @uri's host to @host. + * + * If @host is an IPv6 IP address, it should not include the brackets + * required by the URI syntax; they will be added automatically when + * converting @uri to a string. + * + * http and https URIs should not have a %NULL @host. + **/ +void +soup_uri_set_host (SoupURI *uri, const char *host) +{ + g_return_if_fail (uri != NULL); + + g_free (uri->host); + uri->host = g_strdup (host); +} + +/** + * soup_uri_get_port: + * @uri: a #SoupURI + * + * Gets @uri's port. + * + * Return value: @uri's port. + * + * Since: 2.32 + **/ +guint +soup_uri_get_port (SoupURI *uri) +{ + g_return_val_if_fail (uri != NULL, 0); + + return uri->port; +} + +/** + * soup_uri_set_port: + * @uri: a #SoupURI + * @port: the port, or 0 + * + * Sets @uri's port to @port. If @port is 0, @uri will not have an + * explicitly-specified port. + **/ +void +soup_uri_set_port (SoupURI *uri, guint port) +{ + g_return_if_fail (uri != NULL); + + uri->port = port; +} + +/** + * soup_uri_get_path: + * @uri: a #SoupURI + * + * Gets @uri's path. + * + * Return value: @uri's path. + * + * Since: 2.32 + **/ +const char * +soup_uri_get_path (SoupURI *uri) +{ + g_return_val_if_fail (uri != NULL, NULL); + + return uri->path; +} + +/** + * soup_uri_set_path: + * @uri: a #SoupURI + * @path: the non-%NULL path + * + * Sets @uri's path to @path. + **/ +void +soup_uri_set_path (SoupURI *uri, const char *path) +{ + g_return_if_fail (uri != NULL); + + /* We allow a NULL path for compatibility, but warn about it. */ + if (!path) { + g_warn_if_fail (path != NULL); + path = ""; + } + + g_free (uri->path); + uri->path = g_strdup (path); +} + +/** + * soup_uri_get_query: + * @uri: a #SoupURI + * + * Gets @uri's query. + * + * Return value: @uri's query. + * + * Since: 2.32 + **/ +const char * +soup_uri_get_query (SoupURI *uri) +{ + g_return_val_if_fail (uri != NULL, NULL); + + return uri->query; +} + +/** + * soup_uri_set_query: + * @uri: a #SoupURI + * @query: (allow-none): the query + * + * Sets @uri's query to @query. + **/ +void +soup_uri_set_query (SoupURI *uri, const char *query) +{ + g_return_if_fail (uri != NULL); + + g_free (uri->query); + uri->query = g_strdup (query); +} + +/** + * soup_uri_set_query_from_form: + * @uri: a #SoupURI + * @form: (element-type utf8 utf8): a #GHashTable containing HTML form + * information + * + * Sets @uri's query to the result of encoding @form according to the + * HTML form rules. See soup_form_encode_hash() for more information. + **/ +void +soup_uri_set_query_from_form (SoupURI *uri, GHashTable *form) +{ + g_return_if_fail (uri != NULL); + + g_free (uri->query); + uri->query = soup_form_encode_hash (form); +} + +/** + * soup_uri_set_query_from_fields: + * @uri: a #SoupURI + * @first_field: name of the first form field to encode into query + * @...: value of @first_field, followed by additional field names + * and values, terminated by %NULL. + * + * Sets @uri's query to the result of encoding the given form fields + * and values according to the * HTML form rules. See + * soup_form_encode() for more information. + **/ +void +soup_uri_set_query_from_fields (SoupURI *uri, + const char *first_field, + ...) +{ + va_list args; + + g_return_if_fail (uri != NULL); + + g_free (uri->query); + va_start (args, first_field); + uri->query = soup_form_encode_valist (first_field, args); + va_end (args); +} + +/** + * soup_uri_get_fragment: + * @uri: a #SoupURI + * + * Gets @uri's fragment. + * + * Return value: @uri's fragment. + * + * Since: 2.32 + **/ +const char * +soup_uri_get_fragment (SoupURI *uri) +{ + g_return_val_if_fail (uri != NULL, NULL); + + return uri->fragment; +} + +/** + * soup_uri_set_fragment: + * @uri: a #SoupURI + * @fragment: (allow-none): the fragment + * + * Sets @uri's fragment to @fragment. + **/ +void +soup_uri_set_fragment (SoupURI *uri, const char *fragment) +{ + g_return_if_fail (uri != NULL); + + g_free (uri->fragment); + uri->fragment = g_strdup (fragment); +} + +/** + * soup_uri_copy_host: + * @uri: a #SoupURI + * + * Makes a copy of @uri, considering only the protocol, host, and port + * + * Return value: the new #SoupURI + * + * Since: 2.28 + **/ +SoupURI * +soup_uri_copy_host (SoupURI *uri) +{ + SoupURI *dup; + + g_return_val_if_fail (uri != NULL, NULL); + g_warn_if_fail (SOUP_URI_IS_VALID (uri)); + + dup = soup_uri_new (NULL); + dup->scheme = uri->scheme; + dup->host = g_strdup (uri->host); + dup->port = uri->port; + dup->path = g_strdup (""); + + return dup; +} + +/** + * soup_uri_host_hash: + * @key: (type Soup.URI): a #SoupURI with a non-%NULL @host member + * + * Hashes @key, considering only the scheme, host, and port. + * + * Return value: a hash + * + * Since: 2.28 + **/ +guint +soup_uri_host_hash (gconstpointer key) +{ + const SoupURI *uri = key; + + g_return_val_if_fail (uri != NULL && uri->host != NULL, 0); + g_warn_if_fail (SOUP_URI_IS_VALID (uri)); + + return GPOINTER_TO_UINT (uri->scheme) + uri->port + + soup_str_case_hash (uri->host); +} + +/** + * soup_uri_host_equal: + * @v1: (type Soup.URI): a #SoupURI with a non-%NULL @host member + * @v2: (type Soup.URI): a #SoupURI with a non-%NULL @host member + * + * Compares @v1 and @v2, considering only the scheme, host, and port. + * + * Return value: whether or not the URIs are equal in scheme, host, + * and port. + * + * Since: 2.28 + **/ +gboolean +soup_uri_host_equal (gconstpointer v1, gconstpointer v2) +{ + const SoupURI *one = v1; + const SoupURI *two = v2; + + g_return_val_if_fail (one != NULL && two != NULL, one == two); + g_return_val_if_fail (one->host != NULL && two->host != NULL, one->host == two->host); + g_warn_if_fail (SOUP_URI_IS_VALID (one)); + g_warn_if_fail (SOUP_URI_IS_VALID (two)); + + if (one->scheme != two->scheme) + return FALSE; + if (one->port != two->port) + return FALSE; + + return g_ascii_strcasecmp (one->host, two->host) == 0; +} + +gboolean +soup_uri_is_http (SoupURI *uri, char **aliases) +{ + int i; + + if (uri->scheme == SOUP_URI_SCHEME_HTTP) + return TRUE; + else if (uri->scheme == SOUP_URI_SCHEME_HTTPS) + return FALSE; + else if (!aliases) + return FALSE; + + for (i = 0; aliases[i]; i++) { + if (uri->scheme == aliases[i]) + return TRUE; + } + + if (!aliases[1] && !strcmp (aliases[0], "*")) + return TRUE; + else + return FALSE; +} + +gboolean +soup_uri_is_https (SoupURI *uri, char **aliases) +{ + int i; + + if (uri->scheme == SOUP_URI_SCHEME_HTTPS) + return TRUE; + else if (uri->scheme == SOUP_URI_SCHEME_HTTP) + return FALSE; + else if (!aliases) + return FALSE; + + for (i = 0; aliases[i]; i++) { + if (uri->scheme == aliases[i]) + return TRUE; + } + + return FALSE; +} + +/* OSTREECHANGE: drop boxed type definition */ +/* G_DEFINE_BOXED_TYPE (SoupURI, soup_uri, soup_uri_copy, soup_uri_free) */ diff --git a/src/libostree/ostree-soup-uri.h b/src/libostree/ostree-soup-uri.h new file mode 100644 index 0000000000..650b7efcd3 --- /dev/null +++ b/src/libostree/ostree-soup-uri.h @@ -0,0 +1,147 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*- */ + +/* + * Copyright 1999-2002 Ximian, Inc. + */ + +/* NOTE - taken from the libsoup codebase for use by the ostree curl backend + * (yes, ironically enough). + * + * Please watch for future changes in libsoup. + */ + + +#ifndef SOUP_URI_H +#define SOUP_URI_H 1 + +/* OSTREECHANGE: make struct private + * Only include gio, and skip available definitions. + */ +#include +#define SOUP_AVAILABLE_IN_2_4 +#define SOUP_AVAILABLE_IN_2_28 +#define SOUP_AVAILABLE_IN_2_32 + +G_BEGIN_DECLS + +/* OSTREECHANGE: make struct private */ +typedef struct _SoupURI SoupURI; + +/* OSTREECHANGE: import soup-misc's interning */ +#define SOUP_VAR extern +#define _SOUP_ATOMIC_INTERN_STRING(variable, value) ((const char *)(g_atomic_pointer_get (&(variable)) ? (variable) : (g_atomic_pointer_set (&(variable), (gpointer)g_intern_static_string (value)), (variable)))) +#define SOUP_URI_SCHEME_HTTP _SOUP_ATOMIC_INTERN_STRING (_SOUP_URI_SCHEME_HTTP, "http") +#define SOUP_URI_SCHEME_HTTPS _SOUP_ATOMIC_INTERN_STRING (_SOUP_URI_SCHEME_HTTPS, "https") +#define SOUP_URI_SCHEME_FTP _SOUP_ATOMIC_INTERN_STRING (_SOUP_URI_SCHEME_FTP, "ftp") +#define SOUP_URI_SCHEME_FILE _SOUP_ATOMIC_INTERN_STRING (_SOUP_URI_SCHEME_FILE, "file") +#define SOUP_URI_SCHEME_DATA _SOUP_ATOMIC_INTERN_STRING (_SOUP_URI_SCHEME_DATA, "data") +#define SOUP_URI_SCHEME_RESOURCE _SOUP_ATOMIC_INTERN_STRING (_SOUP_URI_SCHEME_RESOURCE, "resource") +#define SOUP_URI_SCHEME_WS _SOUP_ATOMIC_INTERN_STRING (_SOUP_URI_SCHEME_WS, "ws") +#define SOUP_URI_SCHEME_WSS _SOUP_ATOMIC_INTERN_STRING (_SOUP_URI_SCHEME_WSS, "wss") + +/* OSTREECHANGE: import soup-form bits */ +SOUP_AVAILABLE_IN_2_4 +char *soup_form_encode_hash (GHashTable *form_data_set); +SOUP_AVAILABLE_IN_2_4 +char *soup_form_encode_valist (const char *first_field, + va_list args); + +SOUP_VAR gpointer _SOUP_URI_SCHEME_HTTP, _SOUP_URI_SCHEME_HTTPS; +SOUP_VAR gpointer _SOUP_URI_SCHEME_FTP; +SOUP_VAR gpointer _SOUP_URI_SCHEME_FILE, _SOUP_URI_SCHEME_DATA, _SOUP_URI_SCHEME_RESOURCE; +SOUP_VAR gpointer _SOUP_URI_SCHEME_WS, _SOUP_URI_SCHEME_WSS; + +SOUP_AVAILABLE_IN_2_4 +SoupURI *soup_uri_new_with_base (SoupURI *base, + const char *uri_string); +SOUP_AVAILABLE_IN_2_4 +SoupURI *soup_uri_new (const char *uri_string); + +SOUP_AVAILABLE_IN_2_4 +char *soup_uri_to_string (SoupURI *uri, + gboolean just_path_and_query); + +SOUP_AVAILABLE_IN_2_4 +SoupURI *soup_uri_copy (SoupURI *uri); + +SOUP_AVAILABLE_IN_2_4 +gboolean soup_uri_equal (SoupURI *uri1, + SoupURI *uri2); + +SOUP_AVAILABLE_IN_2_4 +void soup_uri_free (SoupURI *uri); + +SOUP_AVAILABLE_IN_2_4 +char *soup_uri_encode (const char *part, + const char *escape_extra); +SOUP_AVAILABLE_IN_2_4 +char *soup_uri_decode (const char *part); +SOUP_AVAILABLE_IN_2_4 +char *soup_uri_normalize (const char *part, + const char *unescape_extra); + +SOUP_AVAILABLE_IN_2_4 +gboolean soup_uri_uses_default_port (SoupURI *uri); + +SOUP_AVAILABLE_IN_2_32 +const char *soup_uri_get_scheme (SoupURI *uri); +SOUP_AVAILABLE_IN_2_4 +void soup_uri_set_scheme (SoupURI *uri, + const char *scheme); +SOUP_AVAILABLE_IN_2_32 +const char *soup_uri_get_user (SoupURI *uri); +SOUP_AVAILABLE_IN_2_4 +void soup_uri_set_user (SoupURI *uri, + const char *user); +SOUP_AVAILABLE_IN_2_32 +const char *soup_uri_get_password (SoupURI *uri); +SOUP_AVAILABLE_IN_2_4 +void soup_uri_set_password (SoupURI *uri, + const char *password); +SOUP_AVAILABLE_IN_2_32 +const char *soup_uri_get_host (SoupURI *uri); +SOUP_AVAILABLE_IN_2_4 +void soup_uri_set_host (SoupURI *uri, + const char *host); +SOUP_AVAILABLE_IN_2_32 +guint soup_uri_get_port (SoupURI *uri); +SOUP_AVAILABLE_IN_2_4 +void soup_uri_set_port (SoupURI *uri, + guint port); +SOUP_AVAILABLE_IN_2_32 +const char *soup_uri_get_path (SoupURI *uri); +SOUP_AVAILABLE_IN_2_4 +void soup_uri_set_path (SoupURI *uri, + const char *path); +SOUP_AVAILABLE_IN_2_32 +const char *soup_uri_get_query (SoupURI *uri); +SOUP_AVAILABLE_IN_2_4 +void soup_uri_set_query (SoupURI *uri, + const char *query); +SOUP_AVAILABLE_IN_2_4 +void soup_uri_set_query_from_form (SoupURI *uri, + GHashTable *form); +SOUP_AVAILABLE_IN_2_4 +void soup_uri_set_query_from_fields (SoupURI *uri, + const char *first_field, + ...) G_GNUC_NULL_TERMINATED; +SOUP_AVAILABLE_IN_2_32 +const char *soup_uri_get_fragment (SoupURI *uri); +SOUP_AVAILABLE_IN_2_4 +void soup_uri_set_fragment (SoupURI *uri, + const char *fragment); + +SOUP_AVAILABLE_IN_2_28 +SoupURI *soup_uri_copy_host (SoupURI *uri); +SOUP_AVAILABLE_IN_2_28 +guint soup_uri_host_hash (gconstpointer key); +SOUP_AVAILABLE_IN_2_28 +gboolean soup_uri_host_equal (gconstpointer v1, + gconstpointer v2); + +#define SOUP_URI_IS_VALID(uri) ((uri) && (uri)->scheme && (uri)->path) +#define SOUP_URI_VALID_FOR_HTTP(uri) ((uri) && ((uri)->scheme == SOUP_URI_SCHEME_HTTP || (uri)->scheme == SOUP_URI_SCHEME_HTTPS) && (uri)->host && (uri)->path) + +G_END_DECLS + +#endif /*SOUP_URI_H*/ diff --git a/src/ostree/ot-remote-builtin-add-cookie.c b/src/ostree/ot-remote-builtin-add-cookie.c index 509c9c7a5b..e415617266 100644 --- a/src/ostree/ot-remote-builtin-add-cookie.c +++ b/src/ostree/ot-remote-builtin-add-cookie.c @@ -21,13 +21,12 @@ #include "config.h" -#include - #include "otutil.h" #include "ot-main.h" #include "ot-remote-builtins.h" #include "ostree-repo-private.h" +#include "ot-remote-cookie-util.h" static GOptionEntry option_entries[] = { @@ -46,8 +45,6 @@ ot_remote_builtin_add_cookie (int argc, char **argv, GCancellable *cancellable, const char *value; g_autofree char *jar_path = NULL; g_autofree char *cookie_file = NULL; - glnx_unref_object SoupCookieJar *jar = NULL; - SoupCookie *cookie; context = g_option_context_new ("NAME DOMAIN PATH COOKIE_NAME VALUE - Add a cookie to remote"); @@ -70,15 +67,8 @@ ot_remote_builtin_add_cookie (int argc, char **argv, GCancellable *cancellable, cookie_file = g_strdup_printf ("%s.cookies.txt", remote_name); jar_path = g_build_filename (gs_file_get_path_cached (repo->repodir), cookie_file, NULL); - jar = soup_cookie_jar_text_new (jar_path, FALSE); - - /* Pick a silly long expire time, we're just storing the cookies in the - * jar and on pull the jar is read-only so expiry has little actual value */ - cookie = soup_cookie_new (cookie_name, value, domain, path, - SOUP_COOKIE_MAX_AGE_ONE_YEAR * 25); - - /* jar takes ownership of cookie */ - soup_cookie_jar_add_cookie (jar, cookie); + if (!ot_add_cookie_at (AT_FDCWD, jar_path, domain, path, cookie_name, value, error)) + return FALSE; return TRUE; } diff --git a/src/ostree/ot-remote-builtin-delete-cookie.c b/src/ostree/ot-remote-builtin-delete-cookie.c index d974dd8da5..6d1b85ad8b 100644 --- a/src/ostree/ot-remote-builtin-delete-cookie.c +++ b/src/ostree/ot-remote-builtin-delete-cookie.c @@ -21,14 +21,13 @@ #include "config.h" -#include - #include "otutil.h" +#include #include "ot-main.h" #include "ot-remote-builtins.h" #include "ostree-repo-private.h" - +#include "ot-remote-cookie-util.h" static GOptionEntry option_entries[] = { { NULL } @@ -45,9 +44,6 @@ ot_remote_builtin_delete_cookie (int argc, char **argv, GCancellable *cancellabl const char *cookie_name; g_autofree char *jar_path = NULL; g_autofree char *cookie_file = NULL; - glnx_unref_object SoupCookieJar *jar = NULL; - GSList *cookies; - gboolean found = FALSE; context = g_option_context_new ("NAME DOMAIN PATH COOKIE_NAME- Remote one cookie from remote"); @@ -69,28 +65,8 @@ ot_remote_builtin_delete_cookie (int argc, char **argv, GCancellable *cancellabl cookie_file = g_strdup_printf ("%s.cookies.txt", remote_name); jar_path = g_build_filename (gs_file_get_path_cached (repo->repodir), cookie_file, NULL); - jar = soup_cookie_jar_text_new (jar_path, FALSE); - cookies = soup_cookie_jar_all_cookies (jar); - - while (cookies != NULL) - { - SoupCookie *cookie = cookies->data; - - if (!strcmp (domain, soup_cookie_get_domain (cookie)) && - !strcmp (path, soup_cookie_get_path (cookie)) && - !strcmp (cookie_name, soup_cookie_get_name (cookie))) - { - soup_cookie_jar_delete_cookie (jar, cookie); - - found = TRUE; - } - - soup_cookie_free (cookie); - cookies = g_slist_delete_link (cookies, cookies); - } - - if (!found) - g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, "Cookie not found in jar"); + if (!ot_delete_cookie_at (AT_FDCWD, jar_path, domain, path, cookie_name, error)) + return FALSE; - return found; + return TRUE; } diff --git a/src/ostree/ot-remote-builtin-list-cookies.c b/src/ostree/ot-remote-builtin-list-cookies.c index 1865fb0761..1c3924af1d 100644 --- a/src/ostree/ot-remote-builtin-list-cookies.c +++ b/src/ostree/ot-remote-builtin-list-cookies.c @@ -21,14 +21,12 @@ #include "config.h" -#include - #include "otutil.h" #include "ot-main.h" #include "ot-remote-builtins.h" #include "ostree-repo-private.h" - +#include "ot-remote-cookie-util.h" static GOptionEntry option_entries[] = { { NULL } @@ -42,8 +40,6 @@ ot_remote_builtin_list_cookies (int argc, char **argv, GCancellable *cancellable const char *remote_name; g_autofree char *jar_path = NULL; g_autofree char *cookie_file = NULL; - glnx_unref_object SoupCookieJar *jar = NULL; - GSList *cookies; context = g_option_context_new ("NAME - Show remote repository cookies"); @@ -62,25 +58,8 @@ ot_remote_builtin_list_cookies (int argc, char **argv, GCancellable *cancellable cookie_file = g_strdup_printf ("%s.cookies.txt", remote_name); jar_path = g_build_filename (g_file_get_path (repo->repodir), cookie_file, NULL); - jar = soup_cookie_jar_text_new (jar_path, TRUE); - cookies = soup_cookie_jar_all_cookies (jar); - - while (cookies != NULL) - { - SoupCookie *cookie = cookies->data; - SoupDate *expiry = soup_cookie_get_expires (cookie); - - g_print ("--\n"); - g_print ("Domain: %s\n", soup_cookie_get_domain (cookie)); - g_print ("Path: %s\n", soup_cookie_get_path (cookie)); - g_print ("Name: %s\n", soup_cookie_get_name (cookie)); - g_print ("Secure: %s\n", soup_cookie_get_secure (cookie) ? "yes" : "no"); - g_print ("Expires: %s\n", soup_date_to_string (expiry, SOUP_DATE_COOKIE)); - g_print ("Value: %s\n", soup_cookie_get_value (cookie)); - - soup_cookie_free (cookie); - cookies = g_slist_delete_link (cookies, cookies); - } + if (!ot_list_cookies_at (AT_FDCWD, jar_path, error)) + return FALSE; return TRUE; } diff --git a/src/ostree/ot-remote-cookie-util.c b/src/ostree/ot-remote-cookie-util.c new file mode 100644 index 0000000000..a96038aa74 --- /dev/null +++ b/src/ostree/ot-remote-cookie-util.c @@ -0,0 +1,333 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright (C) 2015 Red Hat, Inc. + * Copyright (C) 2016 Sjoerd Simons + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + */ + +#include "config.h" + +#include "ot-remote-cookie-util.h" + +#ifndef HAVE_LIBCURL +#include +#endif + +#include "otutil.h" +#include "ot-main.h" +#include "ot-remote-builtins.h" +#include "ostree-repo-private.h" + +typedef struct OtCookieParser OtCookieParser; +struct OtCookieParser { + char *buf; + char *iter; + + char *line; + char *domain; + char *flag; + char *path; + char *secure; + long long unsigned int expiration; + char *name; + char *value; +}; +void ot_cookie_parser_free (OtCookieParser *parser); +G_DEFINE_AUTOPTR_CLEANUP_FUNC(OtCookieParser, ot_cookie_parser_free) + +gboolean +ot_parse_cookies_at (int dfd, const char *path, + OtCookieParser **out_parser, + GCancellable *cancellable, + GError **error); +gboolean +ot_parse_cookies_next (OtCookieParser *parser); + +static void +ot_cookie_parser_clear (OtCookieParser *parser) +{ + g_clear_pointer (&parser->domain, (GDestroyNotify)g_free); + g_clear_pointer (&parser->flag, (GDestroyNotify)g_free); + g_clear_pointer (&parser->path, (GDestroyNotify)g_free); + g_clear_pointer (&parser->secure, (GDestroyNotify)g_free); + g_clear_pointer (&parser->name, (GDestroyNotify)g_free); + g_clear_pointer (&parser->value, (GDestroyNotify)g_free); +} + +void +ot_cookie_parser_free (OtCookieParser *parser) +{ + ot_cookie_parser_clear (parser); + g_free (parser->buf); + g_free (parser); +} + +gboolean +ot_parse_cookies_at (int dfd, const char *path, + OtCookieParser **out_parser, + GCancellable *cancellable, + GError **error) +{ + OtCookieParser *parser; + g_autofree char *cookies_content = NULL; + glnx_fd_close int infd = -1; + + infd = openat (dfd, path, O_RDONLY | O_CLOEXEC); + if (infd < 0) + { + if (errno != ENOENT) + { + glnx_set_error_from_errno (error); + return FALSE; + } + } + else + { + cookies_content = glnx_fd_readall_utf8 (infd, NULL, cancellable, error); + if (!cookies_content) + return FALSE; + } + + parser = *out_parser = g_new0 (OtCookieParser, 1); + parser->buf = g_steal_pointer (&cookies_content); + parser->iter = parser->buf; + return TRUE; +} + +gboolean +ot_parse_cookies_next (OtCookieParser *parser) +{ + while (parser->iter) + { + char *iter = parser->iter; + char *next = strchr (iter, '\n'); + + if (next) + { + *next = '\0'; + parser->iter = next + 1; + } + else + parser->iter = NULL; + + ot_cookie_parser_clear (parser); + if (sscanf (iter, "%ms\t%ms\t%ms\t%ms\t%llu\t%ms\t%ms", + &parser->domain, + &parser->flag, + &parser->path, + &parser->secure, + &parser->expiration, + &parser->name, + &parser->value) != 7) + continue; + + parser->line = iter; + return TRUE; + } + + return FALSE; +} + +gboolean +ot_add_cookie_at (int dfd, const char *jar_path, + const char *domain, const char *path, + const char *name, const char *value, + GError **error) +{ +#ifdef HAVE_LIBCURL + glnx_fd_close int fd = openat (AT_FDCWD, jar_path, O_WRONLY | O_APPEND | O_CREAT, 0644); + g_autofree char *buf = NULL; + g_autoptr(GDateTime) now = NULL; + g_autoptr(GDateTime) expires = NULL; + + if (fd < 0) + { + glnx_set_error_from_errno (error); + return FALSE; + } + + now = g_date_time_new_now_utc (); + expires = g_date_time_add_years (now, 25); + + /* Adapted from soup-cookie-jar-text.c:write_cookie() */ + buf = g_strdup_printf ("%s\t%s\t%s\t%s\t%llu\t%s\t%s\n", + domain, + *domain == '.' ? "TRUE" : "FALSE", + path, + "FALSE", + (long long unsigned)g_date_time_to_unix (expires), + name, + value); + if (glnx_loop_write (fd, buf, strlen (buf)) < 0) + { + glnx_set_error_from_errno (error); + return FALSE; + } +#else + glnx_unref_object SoupCookieJar *jar = NULL; + SoupCookie *cookie; + + jar = soup_cookie_jar_text_new (jar_path, FALSE); + + /* Pick a silly long expire time, we're just storing the cookies in the + * jar and on pull the jar is read-only so expiry has little actual value */ + cookie = soup_cookie_new (name, value, domain, path, + SOUP_COOKIE_MAX_AGE_ONE_YEAR * 25); + + /* jar takes ownership of cookie */ + soup_cookie_jar_add_cookie (jar, cookie); +#endif + return TRUE; +} + +gboolean +ot_delete_cookie_at (int dfd, const char *jar_path, + const char *domain, const char *path, + const char *name, + GError **error) +{ + gboolean found = FALSE; +#ifdef HAVE_LIBCURL + glnx_fd_close int tempfile_fd = -1; + g_autofree char *tempfile_path = NULL; + g_autofree char *dnbuf = NULL; + const char *dn = NULL; + g_autoptr(OtCookieParser) parser = NULL; + + if (!ot_parse_cookies_at (dfd, jar_path, &parser, NULL, error)) + return FALSE; + + dnbuf = g_strdup (jar_path); + dn = dirname (dnbuf); + if (!glnx_open_tmpfile_linkable_at (AT_FDCWD, dn, O_WRONLY | O_CLOEXEC, + &tempfile_fd, &tempfile_path, + error)) + return FALSE; + + while (ot_parse_cookies_next (parser)) + { + if (strcmp (domain, parser->domain) == 0 && + strcmp (path, parser->path) == 0 && + strcmp (name, parser->name) == 0) + { + found = TRUE; + /* Match, skip writing this one */ + continue; + } + + if (glnx_loop_write (tempfile_fd, parser->line, strlen (parser->line)) < 0 || + glnx_loop_write (tempfile_fd, "\n", 1) < 0) + { + glnx_set_error_from_errno (error); + return FALSE; + } + } + + if (!glnx_link_tmpfile_at (AT_FDCWD, GLNX_LINK_TMPFILE_REPLACE, + tempfile_fd, + tempfile_path, + AT_FDCWD, jar_path, + error)) + return FALSE; +#else + GSList *cookies; + glnx_unref_object SoupCookieJar *jar = NULL; + + jar = soup_cookie_jar_text_new (jar_path, FALSE); + cookies = soup_cookie_jar_all_cookies (jar); + + while (cookies != NULL) + { + SoupCookie *cookie = cookies->data; + + if (!strcmp (domain, soup_cookie_get_domain (cookie)) && + !strcmp (path, soup_cookie_get_path (cookie)) && + !strcmp (name, soup_cookie_get_name (cookie))) + { + soup_cookie_jar_delete_cookie (jar, cookie); + + found = TRUE; + } + + soup_cookie_free (cookie); + cookies = g_slist_delete_link (cookies, cookies); + } +#endif + + if (!found) + g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, "Cookie not found in jar"); + + return TRUE; +} + + +gboolean +ot_list_cookies_at (int dfd, const char *jar_path, GError **error) +{ +#ifdef HAVE_LIBCURL + glnx_fd_close int tempfile_fd = -1; + g_autofree char *tempfile_path = NULL; + g_autofree char *dnbuf = NULL; + const char *dn = NULL; + g_autoptr(OtCookieParser) parser = NULL; + + if (!ot_parse_cookies_at (AT_FDCWD, jar_path, &parser, NULL, error)) + return FALSE; + + dnbuf = dirname (g_strdup (jar_path)); + dn = dnbuf; + if (!glnx_open_tmpfile_linkable_at (AT_FDCWD, dn, O_WRONLY | O_CLOEXEC, + &tempfile_fd, &tempfile_path, + error)) + return FALSE; + + while (ot_parse_cookies_next (parser)) + { + g_autoptr(GDateTime) expires = g_date_time_new_from_unix_utc (parser->expiration); + g_autofree char *expires_str = g_date_time_format (expires, "%Y-%m-%d %H:%M:%S +0000"); + + g_print ("--\n"); + g_print ("Domain: %s\n", parser->domain); + g_print ("Path: %s\n", parser->path); + g_print ("Name: %s\n", parser->name); + g_print ("Secure: %s\n", parser->secure); + g_print ("Expires: %s\n", expires_str); + g_print ("Value: %s\n", parser->value); + } +#else + glnx_unref_object SoupCookieJar *jar = soup_cookie_jar_text_new (jar_path, TRUE); + GSList *cookies = soup_cookie_jar_all_cookies (jar); + + while (cookies != NULL) + { + SoupCookie *cookie = cookies->data; + SoupDate *expiry = soup_cookie_get_expires (cookie); + + g_print ("--\n"); + g_print ("Domain: %s\n", soup_cookie_get_domain (cookie)); + g_print ("Path: %s\n", soup_cookie_get_path (cookie)); + g_print ("Name: %s\n", soup_cookie_get_name (cookie)); + g_print ("Secure: %s\n", soup_cookie_get_secure (cookie) ? "yes" : "no"); + g_print ("Expires: %s\n", soup_date_to_string (expiry, SOUP_DATE_COOKIE)); + g_print ("Value: %s\n", soup_cookie_get_value (cookie)); + + soup_cookie_free (cookie); + cookies = g_slist_delete_link (cookies, cookies); + } +#endif + return TRUE; +} diff --git a/src/ostree/ot-remote-cookie-util.h b/src/ostree/ot-remote-cookie-util.h new file mode 100644 index 0000000000..1bcc0e87fe --- /dev/null +++ b/src/ostree/ot-remote-cookie-util.h @@ -0,0 +1,42 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright (C) 2017 Colin Walters + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + */ + +#pragma once + +#include "libglnx.h" + +G_BEGIN_DECLS + +gboolean +ot_add_cookie_at (int dfd, const char *jar_path, + const char *domain, const char *path, + const char *name, const char *value, + GError **error); + +gboolean +ot_delete_cookie_at (int dfd, const char *jar_path, + const char *domain, const char *path, + const char *name, + GError **error); + +gboolean +ot_list_cookies_at (int dfd, const char *jar_path, GError **error); + +G_END_DECLS diff --git a/tests/ci-build.sh b/tests/ci-build.sh index 4b736a3cff..23eacf074f 100755 --- a/tests/ci-build.sh +++ b/tests/ci-build.sh @@ -52,6 +52,10 @@ NULL= # If yes, test failures break the build; if no, they are reported but ignored : "${ci_test_fatal:=yes}" +# ci_configopts: +# Additional args for configure +: "${ci_configopts:=}" + if [ -n "$ci_docker" ]; then exec docker run \ --env=ci_docker="" \ @@ -59,6 +63,7 @@ if [ -n "$ci_docker" ]; then --env=ci_sudo=yes \ --env=ci_test="${ci_test}" \ --env=ci_test_fatal="${ci_test_fatal}" \ + --env=ci_configopts="${ci_configopts}" \ --privileged \ ci-image \ tests/ci-build.sh @@ -81,6 +86,7 @@ make="make -j${ci_parallel} V=1 VERBOSE=1" ../configure \ --enable-always-build-tests \ --enable-installed-tests \ + ${ci_configopts} "$@" ${make} diff --git a/tests/ci-install.sh b/tests/ci-install.sh index dbc86c697b..92f802d906 100755 --- a/tests/ci-install.sh +++ b/tests/ci-install.sh @@ -50,6 +50,9 @@ NULL= # Typical values for ci_distro=fedora might be 25, rawhide : "${ci_suite:=jessie}" +# ci_configopts: Additional arguments for configure +: "${ci_configopts:=}" + if [ $(id -u) = 0 ]; then sudo= else @@ -104,6 +107,7 @@ case "$ci_distro" in libmount-dev \ libselinux1-dev \ libsoup2.4-dev \ + libcurl4-openssl-dev \ procps \ zlib1g-dev \ ${NULL} diff --git a/tests/test-remote-cookies.sh b/tests/test-remote-cookies.sh index 11c201f117..ab2bf26345 100755 --- a/tests/test-remote-cookies.sh +++ b/tests/test-remote-cookies.sh @@ -28,12 +28,9 @@ setup_fake_remote_repo1 "archive-z2" "" \ "--expected-cookies foo=bar --expected-cookies baz=badger" assert_fail (){ - set +e - $@ - if [ $? = 0 ] ; then - echo 1>&2 "$@ did not fail"; exit 1 + if $@; then + (echo 1>&2 "$@ did not fail"; exit 1) fi - set -euo pipefail } cd ${test_tmpdir} @@ -50,12 +47,16 @@ echo "ok, setup done" # Add 2 cookies, pull should succeed now ${CMD_PREFIX} ostree --repo=repo remote add-cookie origin 127.0.0.1 / foo bar ${CMD_PREFIX} ostree --repo=repo remote add-cookie origin 127.0.0.1 / baz badger +assert_file_has_content repo/origin.cookies.txt foo.*bar +assert_file_has_content repo/origin.cookies.txt baz.*badger ${CMD_PREFIX} ostree --repo=repo pull origin main echo "ok, initial cookie pull succeeded" # Delete one cookie, if successful pulls will fail again ${CMD_PREFIX} ostree --repo=repo remote delete-cookie origin 127.0.0.1 / baz badger +assert_file_has_content repo/origin.cookies.txt foo.*bar +assert_not_file_has_content repo/origin.cookies.txt baz.*badger assert_fail ${CMD_PREFIX} ostree --repo=repo pull origin main echo "ok, delete succeeded" @@ -63,6 +64,8 @@ echo "ok, delete succeeded" # Re-add the removed cooking and things succeed again, verified the removal # removed exactly one cookie ${CMD_PREFIX} ostree --repo=repo remote add-cookie origin 127.0.0.1 / baz badger +assert_file_has_content repo/origin.cookies.txt foo.*bar +assert_file_has_content repo/origin.cookies.txt baz.*badger ${CMD_PREFIX} ostree --repo=repo pull origin main echo "ok, second cookie pull succeeded"