From c91cc4fdddda153479d22f4429a02e9e7528f1f1 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 7 Dec 2016 21:02:30 -0500 Subject: [PATCH 1/4] libcurl backend For rpm-ostree, we already link to libcurl indirectly via librepo, and only having one HTTP library in process makes sense. Further, libcurl is (I think) more popular in the embedded space. It also supports HTTP/2.0 today, which is a *very* nice to have for OSTree. This seems to be working fairly well for me in my local testing, but it's obviously brand new nontrivial code, so it's going to need some soak time. The ugliest part of this is having to vendor in the soup-url code. With Oxidation we could follow the path of Firefox and use the [Servo URL parser](https://github.com/servo/rust-url). Having to redo cookie parsing also sucked, and that would also be a good oxidation target. But that's for the future. --- .redhat-ci.yml | 22 + .travis.yml | 1 + Makefile-libostree.am | 17 +- Makefile-ostree.am | 30 +- configure.ac | 30 +- src/libostree/ostree-fetcher-curl.c | 922 ++++++++++ ...ostree-fetcher.c => ostree-fetcher-soup.c} | 81 - src/libostree/ostree-fetcher-uri.c | 118 ++ src/libostree/ostree-repo-pull.c | 6 +- src/libostree/ostree-soup-form.c | 140 ++ src/libostree/ostree-soup-uri.c | 1483 +++++++++++++++++ src/libostree/ostree-soup-uri.h | 147 ++ src/ostree/ot-remote-builtin-add-cookie.c | 58 +- src/ostree/ot-remote-builtin-delete-cookie.c | 55 +- src/ostree/ot-remote-builtin-list-cookies.c | 76 +- src/ostree/ot-remote-cookie-util.c | 115 ++ src/ostree/ot-remote-cookie-util.h | 52 + tests/ci-build.sh | 6 + tests/ci-install.sh | 4 + tests/test-remote-cookies.sh | 13 +- 20 files changed, 3236 insertions(+), 140 deletions(-) create mode 100644 src/libostree/ostree-fetcher-curl.c rename src/libostree/{ostree-fetcher.c => ostree-fetcher-soup.c} (95%) create mode 100644 src/libostree/ostree-fetcher-uri.c create mode 100644 src/libostree/ostree-soup-form.c create mode 100644 src/libostree/ostree-soup-uri.c create mode 100644 src/libostree/ostree-soup-uri.h create mode 100644 src/ostree/ot-remote-cookie-util.c create mode 100644 src/ostree/ot-remote-cookie-util.h diff --git a/.redhat-ci.yml b/.redhat-ci.yml index 830320efd1..0e1d686427 100644 --- a/.redhat-ci.yml +++ b/.redhat-ci.yml @@ -71,5 +71,27 @@ env: tests: - make check TESTS=tests/test-rollsum +--- + +inherit: true +required: true + +context: curl + +packages: + - pkgconfig(libcurl) + +build: + config-opts: > + --prefix=/usr + --libdir=/usr/lib64 + --enable-installed-tests + --enable-gtk-doc + --enable-curl + +tests: + - make check + - gnome-desktop-testing-runner -p 0 ostree + artifacts: - test-suite.log diff --git a/.travis.yml b/.travis.yml index 27f8492194..a021592c40 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,7 @@ sudo: required env: - ci_distro=ubuntu ci_suite=trusty ci_test=no # TODO: use libcurl on this - ci_docker=debian:jessie-slim ci_distro=debian ci_suite=jessie + - ci_docker=debian:jessie-slim ci_distro=debian ci_suite=jessie ci_configopts="--with-curl" - ci_docker=debian:stretch-slim ci_distro=debian ci_suite=stretch ci_test=no # TODO gpgme flake https://github.com/ostreedev/ostree/pull/664#issuecomment-276033383 - ci_docker=ubuntu:xenial ci_distro=ubuntu ci_suite=xenial diff --git a/Makefile-libostree.am b/Makefile-libostree.am index d40196d4ee..79fddec74e 100644 --- a/Makefile-libostree.am +++ b/Makefile-libostree.am @@ -167,18 +167,31 @@ libostree_1_la_CFLAGS += $(LIBSYSTEMD_CFLAGS) libostree_1_la_LIBADD += $(LIBSYSTEMD_LIBS) endif -if USE_LIBSOUP +if USE_CURL_OR_SOUP libostree_1_la_SOURCES += \ src/libostree/ostree-fetcher.h \ - src/libostree/ostree-fetcher.c \ src/libostree/ostree-fetcher-util.h \ src/libostree/ostree-fetcher-util.c \ + src/libostree/ostree-fetcher-uri.c \ src/libostree/ostree-metalink.h \ src/libostree/ostree-metalink.c \ $(NULL) +endif + +if USE_CURL +libostree_1_la_SOURCES += src/libostree/ostree-fetcher-curl.c \ + src/libostree/ostree-soup-uri.h src/libostree/ostree-soup-uri.c \ + src/libostree/ostree-soup-form.c \ + $(NULL) +libostree_1_la_CFLAGS += $(OT_DEP_CURL_CFLAGS) +libostree_1_la_LIBADD += $(OT_DEP_CURL_LIBS) +else +if USE_LIBSOUP +libostree_1_la_SOURCES += src/libostree/ostree-fetcher-soup.c libostree_1_la_CFLAGS += $(OT_INTERNAL_SOUP_CFLAGS) libostree_1_la_LIBADD += $(OT_INTERNAL_SOUP_LIBS) endif +endif if USE_LIBMOUNT libostree_1_la_CFLAGS += $(OT_DEP_LIBMOUNT_CFLAGS) diff --git a/Makefile-ostree.am b/Makefile-ostree.am index 05fec15520..0b520c68ea 100644 --- a/Makefile-ostree.am +++ b/Makefile-ostree.am @@ -88,12 +88,14 @@ ostree_SOURCES += \ src/ostree/ot-remote-builtin-summary.c \ $(NULL) -if USE_LIBSOUP -ostree_SOURCES += \ - src/ostree/ot-remote-builtin-add-cookie.c \ - src/ostree/ot-remote-builtin-delete-cookie.c \ - src/ostree/ot-remote-builtin-list-cookies.c \ - $(NULL) + +if USE_CURL_OR_SOUP +ostree_SOURCES += src/ostree/ot-remote-builtin-add-cookie.c \ + src/ostree/ot-remote-builtin-delete-cookie.c \ + src/ostree/ot-remote-builtin-list-cookies.c \ + src/ostree/ot-remote-cookie-util.h \ + src/ostree/ot-remote-cookie-util.c \ + $(NULL) endif src/ostree/parse-datetime.c: src/ostree/parse-datetime.y Makefile @@ -112,15 +114,23 @@ ostree_CFLAGS = $(ostree_bin_shared_cflags) ostree_LDADD = $(ostree_bin_shared_ldadd) libbsdiff.la libostree-kernel-args.la $(LIBSYSTEMD_LIBS) -if USE_LIBSOUP -ostree_SOURCES += src/ostree/ot-builtin-pull.c src/ostree/ot-builtin-trivial-httpd.c -ostree_CFLAGS += $(OT_INTERNAL_SOUP_CFLAGS) -ostree_LDADD += $(OT_INTERNAL_SOUP_LIBS) +if USE_CURL_OR_SOUP +ostree_SOURCES += src/ostree/ot-builtin-pull.c +endif +if USE_LIBSOUP +# Eventually once we stop things from using this, we should support disabling this +ostree_SOURCES += src/ostree/ot-builtin-trivial-httpd.c pkglibexec_PROGRAMS += ostree-trivial-httpd ostree_trivial_httpd_SOURCES = src/ostree/ostree-trivial-httpd.c ostree_trivial_httpd_CFLAGS = $(ostree_bin_shared_cflags) $(OT_INTERNAL_SOUP_CFLAGS) ostree_trivial_httpd_LDADD = $(ostree_bin_shared_ldadd) $(OT_INTERNAL_SOUP_LIBS) + +if !USE_CURL +# This is necessary for the cookie jar bits +ostree_CFLAGS += $(OT_INTERNAL_SOUP_CFLAGS) +ostree_LDADD += $(OT_INTERNAL_SOUP_LIBS) +endif endif if USE_LIBARCHIVE diff --git a/configure.ac b/configure.ac index 88e6ea1b87..596ee04068 100644 --- a/configure.ac +++ b/configure.ac @@ -79,14 +79,29 @@ PKG_CHECK_MODULES(OT_DEP_ZLIB, zlib) dnl We're not actually linking to this, just using the header PKG_CHECK_MODULES(OT_DEP_E2P, e2p) +dnl Arbitrary version that's in CentOS7.2 now +CURL_DEPENDENCY=7.29.0 +AC_ARG_WITH(curl, + AS_HELP_STRING([--with-curl], [Use libcurl @<:@default=no@:>@]), + [], [with_curl=no]) +AS_IF([test x$with_curl != xno ], [ + PKG_CHECK_MODULES(OT_DEP_CURL, libcurl >= $CURL_DEPENDENCY) + with_curl=yes + AC_DEFINE([HAVE_LIBCURL], 1, [Define if we have libcurl.pc]) + dnl Currently using libcurl requires soup for trivial-httpd for tests + with_soup_default=yes +], [with_soup_default=check]) +AM_CONDITIONAL(USE_CURL, test x$with_curl != xno) +if test x$with_curl = xyes; then OSTREE_FEATURES="$OSTREE_FEATURES +libcurl"; fi + dnl When bumping the libsoup-2.4 dependency, remember to bump dnl SOUP_VERSION_MIN_REQUIRED and SOUP_VERSION_MAX_ALLOWED in dnl Makefile.am SOUP_DEPENDENCY="libsoup-2.4 >= 2.39.1" AC_ARG_WITH(soup, AS_HELP_STRING([--with-soup], [Use libsoup @<:@default=yes@:>@]), - [], [with_soup=check]) -AS_IF([test x$with_soup != xno ], [ + [], [with_soup=$with_soup_default]) +AS_IF([test x$with_soup != xno], [ AC_ARG_ENABLE(libsoup_client_certs, AS_HELP_STRING([--enable-libsoup-client-certs], [Require availability of new enough libsoup TLS client cert API (default: auto)]),, @@ -120,6 +135,14 @@ if test x$with_soup != xno; then OSTREE_FEATURES="$OSTREE_FEATURES +libsoup"; fi AM_CONDITIONAL(USE_LIBSOUP, test x$with_soup != xno) AM_CONDITIONAL(HAVE_LIBSOUP_CLIENT_CERTS, test x$have_libsoup_client_certs = xyes) +AS_IF([test x$with_curl = xyes && test x$with_soup = xno], [ + AC_MSG_ERROR([Curl enabled, but libsoup is not; libsoup is needed for tests]) +]) +AM_CONDITIONAL(USE_CURL_OR_SOUP, test x$with_curl != xno || test x$with_soup != xno) +AS_IF([test x$with_curl != xno || test x$with_soup != xno], + [AC_DEFINE([HAVE_LIBCURL_OR_LIBSOUP], 1, [Define if we have soup or curl])]) +AS_IF([test x$with_curl = xyes], [fetcher_backend=curl], [test x$with_soup = xyes], [fetcher_backend=libsoup], [fetcher_backend=none]) + m4_ifdef([GOBJECT_INTROSPECTION_CHECK], [ GOBJECT_INTROSPECTION_CHECK([1.34.0]) ]) @@ -374,8 +397,7 @@ echo " introspection: $found_introspection Rust (internal oxidation): $rust_debug_release rofiles-fuse: $enable_rofiles_fuse - libsoup (retrieve remote HTTP repositories): $with_soup - libsoup TLS client certs: $have_libsoup_client_certs + HTTP backend: $fetcher_backend SELinux: $with_selinux systemd: $have_libsystemd libmount: $with_libmount diff --git a/src/libostree/ostree-fetcher-curl.c b/src/libostree/ostree-fetcher-curl.c new file mode 100644 index 0000000000..be3250fbc1 --- /dev/null +++ b/src/libostree/ostree-fetcher-curl.c @@ -0,0 +1,922 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright (C) 2016 Colin Walters + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + */ + +#include "config.h" + +#include +#include +#include +#include + +/* These macros came from 7.43.0, but we want to check + * for versions a bit earlier than that (to work on CentOS 7), + * so define them here if we're using an older version. + */ +#ifndef CURL_VERSION_BITS +#define CURL_VERSION_BITS(x,y,z) ((x)<<16|(y)<<8|z) +#endif +#ifndef CURL_AT_LEAST_VERSION +#define CURL_AT_LEAST_VERSION(x,y,z) (LIBCURL_VERSION_NUM >= CURL_VERSION_BITS(x, y, z)) +#endif + +#include "ostree-fetcher.h" +#include "ostree-enumtypes.h" +#include "ostree-repo-private.h" +#include "otutil.h" + +#include "ostree-soup-uri.h" + +typedef struct FetcherRequest FetcherRequest; +typedef struct SockInfo SockInfo; + +static int sock_cb (CURL *e, curl_socket_t s, int what, void *cbp, void *sockp); +static gboolean timer_cb (gpointer data); +static void sock_unref (SockInfo *f); +static int update_timeout_cb (CURLM *multi, long timeout_ms, void *userp); +static void request_unref (FetcherRequest *req); +static void initiate_next_curl_request (FetcherRequest *req, GTask *task); +static void destroy_and_unref_source (GSource *source); + +struct OstreeFetcher +{ + GObject parent_instance; + + OstreeFetcherConfigFlags config_flags; + char *tls_ca_db_path; + char *tls_client_cert_path; + char *tls_client_key_path; + char *cookie_jar_path; + char *proxy; + struct curl_slist *extra_headers; + int tmpdir_dfd; + + GMainContext *mainctx; + CURLM *multi; + GSource *timer_event; + int curl_running; + GHashTable *outstanding_requests; /* Set */ + GHashTable *sockets; /* Set */ + + guint64 bytes_transferred; +}; + +/* Information associated with a request */ +struct FetcherRequest { + guint refcount; + GPtrArray *mirrorlist; + guint idx; + + char *filename; + guint64 current_size; + guint64 max_size; + OstreeFetcherRequestFlags flags; + gboolean is_membuf; + GError *caught_write_error; + char *out_tmpfile; + int out_tmpfile_fd; + GString *output_buf; + + CURL *easy; + char error[CURL_ERROR_SIZE]; + + OstreeFetcher *fetcher; +}; + +/* Information associated with a specific socket */ +struct SockInfo { + guint refcount; + curl_socket_t sockfd; + int action; + long timeout; + GSource *ch; + OstreeFetcher *fetcher; +}; + +enum { + PROP_0, + PROP_CONFIG_FLAGS +}; + +G_DEFINE_TYPE (OstreeFetcher, _ostree_fetcher, G_TYPE_OBJECT) + +static void +_ostree_fetcher_set_property (GObject *object, + guint prop_id, + const GValue *value, + GParamSpec *pspec) +{ + OstreeFetcher *self = OSTREE_FETCHER (object); + + switch (prop_id) + { + case PROP_CONFIG_FLAGS: + self->config_flags = g_value_get_flags (value); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +_ostree_fetcher_get_property (GObject *object, + guint prop_id, + GValue *value, + GParamSpec *pspec) +{ + OstreeFetcher *self = OSTREE_FETCHER (object); + + switch (prop_id) + { + case PROP_CONFIG_FLAGS: + g_value_set_flags (value, self->config_flags); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +_ostree_fetcher_finalize (GObject *object) +{ + OstreeFetcher *self = OSTREE_FETCHER (object); + + g_free (self->cookie_jar_path); + g_free (self->proxy); + g_assert_cmpint (g_hash_table_size (self->outstanding_requests), ==, 0); + g_clear_pointer (&self->extra_headers, (GDestroyNotify)curl_slist_free_all); + g_hash_table_unref (self->outstanding_requests); + g_hash_table_unref (self->sockets); + g_clear_pointer (&self->timer_event, (GDestroyNotify)destroy_and_unref_source); + if (self->mainctx) + g_main_context_unref (self->mainctx); + curl_multi_cleanup (self->multi); + + G_OBJECT_CLASS (_ostree_fetcher_parent_class)->finalize (object); +} + +static void +_ostree_fetcher_constructed (GObject *object) +{ + // OstreeFetcher *self = OSTREE_FETCHER (object); + + G_OBJECT_CLASS (_ostree_fetcher_parent_class)->constructed (object); +} + +static void +_ostree_fetcher_class_init (OstreeFetcherClass *klass) +{ + GObjectClass *gobject_class = G_OBJECT_CLASS (klass); + + gobject_class->set_property = _ostree_fetcher_set_property; + gobject_class->get_property = _ostree_fetcher_get_property; + gobject_class->finalize = _ostree_fetcher_finalize; + gobject_class->constructed = _ostree_fetcher_constructed; + + g_object_class_install_property (gobject_class, + PROP_CONFIG_FLAGS, + g_param_spec_flags ("config-flags", + "", + "", + OSTREE_TYPE_FETCHER_CONFIG_FLAGS, + OSTREE_FETCHER_FLAGS_NONE, + G_PARAM_READWRITE | + G_PARAM_CONSTRUCT_ONLY | + G_PARAM_STATIC_STRINGS)); +} + +static void +_ostree_fetcher_init (OstreeFetcher *self) +{ + self->multi = curl_multi_init(); + self->outstanding_requests = g_hash_table_new_full (NULL, NULL, (GDestroyNotify)g_object_unref, NULL); + self->sockets = g_hash_table_new_full (NULL, NULL, (GDestroyNotify)sock_unref, NULL); + curl_multi_setopt (self->multi, CURLMOPT_SOCKETFUNCTION, sock_cb); + curl_multi_setopt (self->multi, CURLMOPT_SOCKETDATA, self); + curl_multi_setopt (self->multi, CURLMOPT_TIMERFUNCTION, update_timeout_cb); + curl_multi_setopt (self->multi, CURLMOPT_TIMERDATA, self); +#if CURL_AT_LEAST_VERSION(7, 30, 0) + /* Let's do something reasonable here. */ + curl_multi_setopt (self->multi, CURLMOPT_MAX_TOTAL_CONNECTIONS, 8); +#endif +} + + +OstreeFetcher * +_ostree_fetcher_new (int tmpdir_dfd, + OstreeFetcherConfigFlags flags) +{ + OstreeFetcher *fetcher = g_object_new (OSTREE_TYPE_FETCHER, "config-flags", flags, NULL); + fetcher->tmpdir_dfd = tmpdir_dfd; + return fetcher; +} + +static void +destroy_and_unref_source (GSource *source) +{ + g_source_destroy (source); + g_source_unref (source); +} + +static char * +request_get_uri (FetcherRequest *req, guint idx) +{ + SoupURI *baseuri = req->mirrorlist->pdata[idx]; + if (!req->filename) + return soup_uri_to_string (baseuri, FALSE); + { g_autofree char *uristr = soup_uri_to_string (baseuri, FALSE); + return g_build_filename (uristr, req->filename, NULL); + } +} + +static gboolean +ensure_tmpfile (FetcherRequest *req, GError **error) +{ + if (req->out_tmpfile_fd == -1) + { + if (!glnx_open_tmpfile_linkable_at (req->fetcher->tmpdir_dfd, ".", + O_WRONLY, &req->out_tmpfile_fd, + &req->out_tmpfile, + error)) + return FALSE; + } + return TRUE; +} +/* Check for completed transfers, and remove their easy handles */ +static void +check_multi_info (OstreeFetcher *fetcher) +{ + CURLMsg *msg; + int msgs_left; + + while ((msg = curl_multi_info_read (fetcher->multi, &msgs_left)) != NULL) + { + long response; + CURL *easy = msg->easy_handle; + CURLcode curlres = msg->data.result; + GTask *task; + FetcherRequest *req; + const char *eff_url; + gboolean is_file; + gboolean continued_request = FALSE; + + if (msg->msg != CURLMSG_DONE) + continue; + + curl_easy_getinfo (easy, CURLINFO_PRIVATE, &task); + curl_easy_getinfo (easy, CURLINFO_EFFECTIVE_URL, &eff_url); + /* We should have limited the protocols; this is what + * curl's tool_operate.c does. + */ + is_file = g_str_has_prefix (eff_url, "file:"); + g_assert (is_file || g_str_has_prefix (eff_url, "http")); + + req = g_task_get_task_data (task); + + if (req->caught_write_error) + g_task_return_error (task, g_steal_pointer (&req->caught_write_error)); + else if (curlres != CURLE_OK) + { + if (is_file && curlres == CURLE_FILE_COULDNT_READ_FILE) + { + /* Handle file not found */ + g_task_return_new_error (task, G_IO_ERROR, G_IO_ERROR_NOT_FOUND, + "%s", + curl_easy_strerror (curlres)); + } + else + g_task_return_new_error (task, G_IO_ERROR, G_IO_ERROR_FAILED, "[%u] %s", + curlres, + curl_easy_strerror (curlres)); + } + else + { + curl_easy_getinfo (easy, CURLINFO_RESPONSE_CODE, &response); + if (!is_file && !(response >= 200 && response < 300)) + { + GIOErrorEnum giocode; + + /* TODO - share with soup */ + switch (response) + { + case 404: + case 403: + case 410: + giocode = G_IO_ERROR_NOT_FOUND; + break; + default: + giocode = G_IO_ERROR_FAILED; + } + + if (req->idx + 1 == req->mirrorlist->len) + { + g_task_return_new_error (task, G_IO_ERROR, giocode, + "Server returned HTTP %lu", response); + } + else + { + continued_request = TRUE; + } + } + else if (req->is_membuf) + { + GBytes *ret; + if ((req->flags & OSTREE_FETCHER_REQUEST_NUL_TERMINATION) > 0) + g_string_append_c (req->output_buf, '\0'); + ret = g_string_free_to_bytes (req->output_buf); + req->output_buf = NULL; + g_task_return_pointer (task, ret, (GDestroyNotify)g_bytes_unref); + } + else + { + g_autoptr(GError) local_error = NULL; + GError **error = &local_error; + + /* TODO - share file naming with soup, and fix it */ + g_autofree char *tmpfile_path = + g_compute_checksum_for_string (G_CHECKSUM_SHA256, + eff_url, strlen (eff_url)); + if (!ensure_tmpfile (req, error)) + { + g_task_return_error (task, g_steal_pointer (&local_error)); + } + else if (fchmod (req->out_tmpfile_fd, 0644) < 0) + { + glnx_set_error_from_errno (error); + g_task_return_error (task, g_steal_pointer (&local_error)); + } + else if (!glnx_link_tmpfile_at (fetcher->tmpdir_dfd, + GLNX_LINK_TMPFILE_REPLACE, + req->out_tmpfile_fd, + req->out_tmpfile, + fetcher->tmpdir_dfd, + tmpfile_path, + error)) + g_task_return_error (task, g_steal_pointer (&local_error)); + else + { + g_task_return_pointer (task, g_steal_pointer (&tmpfile_path), g_free); + } + } + } + + curl_multi_remove_handle (fetcher->multi, easy); + if (continued_request) + { + req->idx++; + initiate_next_curl_request (req, task); + } + else + { + g_hash_table_remove (fetcher->outstanding_requests, task); + if (g_hash_table_size (fetcher->outstanding_requests) == 0) + { + g_clear_pointer (&fetcher->mainctx, g_main_context_unref); + } + } + } +} + +/* Called by glib when our timeout expires */ +static gboolean +timer_cb (gpointer data) +{ + OstreeFetcher *fetcher = data; + CURLMcode rc; + + fetcher->timer_event = NULL; + rc = curl_multi_socket_action (fetcher->multi, CURL_SOCKET_TIMEOUT, 0, &fetcher->curl_running); + g_assert (rc == CURLM_OK); + check_multi_info (fetcher); + + return FALSE; +} + +/* Update the event timer after curl_multi library calls */ +static int +update_timeout_cb (CURLM *multi, long timeout_ms, void *userp) +{ + OstreeFetcher *fetcher = userp; + + g_clear_pointer (&fetcher->timer_event, (GDestroyNotify)destroy_and_unref_source); + + if (timeout_ms != -1) + { + fetcher->timer_event = g_timeout_source_new (timeout_ms); + g_source_set_callback (fetcher->timer_event, timer_cb, fetcher, NULL); + g_source_attach (fetcher->timer_event, fetcher->mainctx); + } + + return 0; +} + +/* Called by glib when we get action on a multi socket */ +static gboolean +event_cb (int fd, GIOCondition condition, gpointer data) +{ + OstreeFetcher *fetcher = data; + CURLMcode rc; + + int action = + (condition & G_IO_IN ? CURL_CSELECT_IN : 0) | + (condition & G_IO_OUT ? CURL_CSELECT_OUT : 0); + + rc = curl_multi_socket_action (fetcher->multi, fd, action, &fetcher->curl_running); + g_assert (rc == CURLM_OK); + + check_multi_info (fetcher); + if (fetcher->curl_running > 0) + { + return TRUE; + } + else + { + return FALSE; + } +} + +/* Clean up the SockInfo structure */ +static void +sock_unref (SockInfo *f) +{ + if (!f) + return; + if (--f->refcount != 0) + return; + g_clear_pointer (&f->ch, (GDestroyNotify)destroy_and_unref_source); + g_free (f); +} + +/* Assign information to a SockInfo structure */ +static void +setsock (SockInfo*f, curl_socket_t s, int act, OstreeFetcher *fetcher) +{ + GIOCondition kind = + (act&CURL_POLL_IN?G_IO_IN:0)|(act&CURL_POLL_OUT?G_IO_OUT:0); + + f->sockfd = s; + f->action = act; + g_clear_pointer (&f->ch, (GDestroyNotify)destroy_and_unref_source); + /* TODO - investigate new g_source_modify_unix_fd() so changing the poll + * flags involves less allocation. + */ + f->ch = g_unix_fd_source_new (f->sockfd, kind); + g_source_set_callback (f->ch, (GSourceFunc) event_cb, fetcher, NULL); + g_source_attach (f->ch, fetcher->mainctx); +} + +/* Initialize a new SockInfo structure */ +static void +addsock (curl_socket_t s, CURL *easy, int action, OstreeFetcher *fetcher) +{ + SockInfo *fdp = g_new0 (SockInfo, 1); + + fdp->refcount = 1; + fdp->fetcher = fetcher; + setsock (fdp, s, action, fetcher); + curl_multi_assign (fetcher->multi, s, fdp); + g_hash_table_add (fetcher->sockets, fdp); +} + +/* CURLMOPT_SOCKETFUNCTION */ +static int +sock_cb (CURL *easy, curl_socket_t s, int what, void *cbp, void *sockp) +{ + OstreeFetcher *fetcher = cbp; + SockInfo *fdp = (SockInfo*) sockp; + + if (what == CURL_POLL_REMOVE) + { + if (!g_hash_table_remove (fetcher->sockets, fdp)) + g_assert_not_reached (); + } + else + { + if (!fdp) + { + addsock (s, easy, what, fetcher); + } + else + { + setsock (fdp, s, what, fetcher); + } + } + return 0; +} + +/* CURLOPT_WRITEFUNCTION */ +static size_t +write_cb (void *ptr, size_t size, size_t nmemb, void *data) +{ + const size_t realsize = size * nmemb; + GTask *task = data; + FetcherRequest *req; + + req = g_task_get_task_data (task); + + if (req->caught_write_error) + return -1; + + if (req->max_size > 0) + { + if (realsize > req->max_size || + (realsize + req->current_size) > req->max_size) + { + const char *eff_url; + curl_easy_getinfo (req->easy, CURLINFO_EFFECTIVE_URL, &eff_url); + req->caught_write_error = g_error_new (G_IO_ERROR, G_IO_ERROR_FAILED, + "URI %s exceeded maximum size of %" G_GUINT64_FORMAT " bytes", + eff_url, req->max_size); + return -1; + } + } + + if (req->is_membuf) + g_string_append_len (req->output_buf, ptr, realsize); + else + { + if (!ensure_tmpfile (req, &req->caught_write_error)) + return -1; + g_assert (req->out_tmpfile_fd >= 0); + if (glnx_loop_write (req->out_tmpfile_fd, ptr, realsize) < 0) + { + glnx_set_error_from_errno (&req->caught_write_error); + return -1; + } + } + + req->current_size += realsize; + req->fetcher->bytes_transferred += realsize; + + return realsize; +} + +/* CURLOPT_PROGRESSFUNCTION */ +static int +prog_cb (void *p, double dltotal, double dlnow, double ult, double uln) +{ + GTask *task = p; + FetcherRequest *req; + char *eff_url; + req = g_task_get_task_data (task); + curl_easy_getinfo (req->easy, CURLINFO_EFFECTIVE_URL, &eff_url); + g_printerr ("Progress: %s (%g/%g)\n", eff_url, dlnow, dltotal); + return 0; +} + +static void +request_unref (FetcherRequest *req) +{ + if (--req->refcount != 0) + return; + + g_ptr_array_unref (req->mirrorlist); + g_free (req->filename); + g_clear_error (&req->caught_write_error); + if (req->out_tmpfile_fd != -1) + (void) close (req->out_tmpfile_fd); + g_free (req->out_tmpfile); + if (req->output_buf) + g_string_free (req->output_buf, TRUE); + curl_easy_cleanup (req->easy); + + g_free (req); +} + +int +_ostree_fetcher_get_dfd (OstreeFetcher *fetcher) +{ + return fetcher->tmpdir_dfd; +} + +void +_ostree_fetcher_set_proxy (OstreeFetcher *self, + const char *http_proxy) +{ + g_free (self->proxy); + self->proxy = g_strdup (http_proxy); +} + +void +_ostree_fetcher_set_cookie_jar (OstreeFetcher *self, + const char *jar_path) +{ + g_free (self->cookie_jar_path); + self->cookie_jar_path = g_strdup (jar_path); +} + +void +_ostree_fetcher_set_client_cert (OstreeFetcher *self, + const char *cert_path, + const char *key_path) +{ + g_assert ((cert_path == NULL && key_path == NULL) + || (cert_path != NULL && key_path != NULL)); + g_free (self->tls_client_cert_path); + self->tls_client_cert_path = g_strdup (cert_path); + g_free (self->tls_client_key_path); + self->tls_client_key_path = g_strdup (key_path); +} + +void +_ostree_fetcher_set_tls_database (OstreeFetcher *self, + const char *dbpath) +{ + g_free (self->tls_ca_db_path); + self->tls_ca_db_path = g_strdup (dbpath); +} + +void +_ostree_fetcher_set_extra_headers (OstreeFetcher *self, + GVariant *extra_headers) +{ + GVariantIter viter; + const char *key; + const char *value; + + g_clear_pointer (&self->extra_headers, (GDestroyNotify)curl_slist_free_all); + + g_variant_iter_init (&viter, extra_headers); + while (g_variant_iter_loop (&viter, "(&s&s)", &key, &value)) + { + g_autofree char *header = g_strdup_printf ("%s: %s", key, value); + self->extra_headers = curl_slist_append (self->extra_headers, header); + } +} + +/* Re-bind all of the outstanding curl items to our new main context */ +static void +adopt_steal_mainctx (OstreeFetcher *self, + GMainContext *mainctx) +{ + GHashTableIter hiter; + gpointer key, value; + + g_assert (self->mainctx == NULL); + self->mainctx = mainctx; /* Transfer */ + + if (self->timer_event != NULL) + { + guint64 readytime = g_source_get_ready_time (self->timer_event); + guint64 curtime = g_source_get_time (self->timer_event); + guint64 timeout_micros = curtime - readytime; + if (timeout_micros < 0) + timeout_micros = 0; + update_timeout_cb (self->multi, timeout_micros / 1000, self); + } + + g_hash_table_iter_init (&hiter, self->sockets); + while (g_hash_table_iter_next (&hiter, &key, &value)) + { + SockInfo *fdp = key; + setsock (fdp, fdp->sockfd, fdp->action, self); + } +} + +static void +initiate_next_curl_request (FetcherRequest *req, + GTask *task) +{ + CURLMcode rc; + OstreeFetcher *self = req->fetcher; + + req->easy = curl_easy_init (); + g_assert (req->easy); + + g_assert_cmpint (req->idx, <, req->mirrorlist->len); + + { g_autofree char *uri = request_get_uri (req, req->idx); + curl_easy_setopt (req->easy, CURLOPT_URL, uri); + } + + curl_easy_setopt (req->easy, CURLOPT_USERAGENT, "ostree "); + if (self->extra_headers) + curl_easy_setopt (req->easy, CURLOPT_HTTPHEADER, self->extra_headers); + + if (self->cookie_jar_path) + { + rc = curl_easy_setopt (req->easy, CURLOPT_COOKIEFILE, self->cookie_jar_path); + g_assert_cmpint (rc, ==, CURLM_OK); + rc = curl_easy_setopt (req->easy, CURLOPT_COOKIELIST, "RELOAD"); + g_assert_cmpint (rc, ==, CURLM_OK); + } + + if (self->proxy) + { + rc = curl_easy_setopt (req->easy, CURLOPT_PROXY, self->proxy); + g_assert_cmpint (rc, ==, CURLM_OK); + } + + if (self->tls_ca_db_path) + curl_easy_setopt (req->easy, CURLOPT_CAINFO, self->tls_ca_db_path); + + if ((self->config_flags & OSTREE_FETCHER_FLAGS_TLS_PERMISSIVE) > 0) + curl_easy_setopt (req->easy, CURLOPT_SSL_VERIFYPEER, 0L); + + if (self->tls_client_cert_path) + { + curl_easy_setopt (req->easy, CURLOPT_SSLCERT, self->tls_client_cert_path); + curl_easy_setopt (req->easy, CURLOPT_SSLKEY, self->tls_client_key_path); + } + + /* We should only speak HTTP; TODO: only enable file if specified */ + curl_easy_setopt (req->easy, CURLOPT_PROTOCOLS, (long)(CURLPROTO_HTTP | CURLPROTO_HTTPS | CURLPROTO_FILE)); + /* Picked the current version in F25 as of 20170127, since + * there are numerous HTTP/2 fixes since the original version in + * libcurl 7.43.0. + */ +#if CURL_AT_LEAST_VERSION(7, 51, 0) + curl_easy_setopt (req->easy, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0); +#endif + curl_easy_setopt (req->easy, CURLOPT_WRITEFUNCTION, write_cb); + if (g_getenv ("OSTREE_DEBUG_HTTP")) + curl_easy_setopt (req->easy, CURLOPT_VERBOSE, 1L); + curl_easy_setopt (req->easy, CURLOPT_ERRORBUFFER, req->error); + /* Note that the "easy" object's privdata is the task */ + curl_easy_setopt (req->easy, CURLOPT_NOPROGRESS, 1L); + curl_easy_setopt (req->easy, CURLOPT_PROGRESSFUNCTION, prog_cb); + curl_easy_setopt (req->easy, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt (req->easy, CURLOPT_CONNECTTIMEOUT, 30L); + curl_easy_setopt (req->easy, CURLOPT_LOW_SPEED_LIMIT, 1L); + curl_easy_setopt (req->easy, CURLOPT_LOW_SPEED_TIME, 30L); + + /* closure bindings -> task */ + curl_easy_setopt (req->easy, CURLOPT_PRIVATE, task); + curl_easy_setopt (req->easy, CURLOPT_WRITEDATA, task); + curl_easy_setopt (req->easy, CURLOPT_PROGRESSDATA, task); + + rc = curl_multi_add_handle (self->multi, req->easy); + g_assert (rc == CURLM_OK); +} + +static void +_ostree_fetcher_request_async (OstreeFetcher *self, + GPtrArray *mirrorlist, + const char *filename, + OstreeFetcherRequestFlags flags, + gboolean is_membuf, + guint64 max_size, + int priority, + GCancellable *cancellable, + GAsyncReadyCallback callback, + gpointer user_data) +{ + g_autoptr(GTask) task = NULL; + FetcherRequest *req; + g_autoptr(GMainContext) mainctx = g_main_context_ref_thread_default (); + + /* We don't support multiple concurrent main contexts; take + * a ref to the first one, and require that later invocations + * share it. + */ + if (g_hash_table_size (self->outstanding_requests) == 0 + && mainctx != self->mainctx) + { + adopt_steal_mainctx (self, g_steal_pointer (&mainctx)); + } + else + { + g_assert (self->mainctx == mainctx); + } + + req = g_new0 (FetcherRequest, 1); + req->refcount = 1; + req->error[0]='\0'; + req->fetcher = self; + req->mirrorlist = g_ptr_array_ref (mirrorlist); + req->filename = g_strdup (filename); + req->max_size = max_size; + req->flags = flags; + req->is_membuf = is_membuf; + /* We'll allocate the tmpfile on demand, so we handle + * file I/O errors just in the write func. + */ + req->out_tmpfile_fd = -1; + if (req->is_membuf) + req->output_buf = g_string_new (""); + + task = g_task_new (self, cancellable, callback, user_data); + /* We'll use the GTask priority for our own priority queue. */ + g_task_set_priority (task, priority); + g_task_set_source_tag (task, _ostree_fetcher_request_async); + g_task_set_task_data (task, req, (GDestroyNotify) request_unref); + + initiate_next_curl_request (req, task); + + g_hash_table_add (self->outstanding_requests, g_steal_pointer (&task)); + + /* Sanity check, I added * 2 just so we don't abort if something odd happens, + * but we do want to abort if we're asked to do obviously too many requests. + */ + g_assert_cmpint (g_hash_table_size (self->outstanding_requests), <, + _OSTREE_MAX_OUTSTANDING_FETCHER_REQUESTS * 2); +} + +void +_ostree_fetcher_request_to_tmpfile (OstreeFetcher *self, + GPtrArray *mirrorlist, + const char *filename, + guint64 max_size, + int priority, + GCancellable *cancellable, + GAsyncReadyCallback callback, + gpointer user_data) +{ + _ostree_fetcher_request_async (self, mirrorlist, filename, 0, FALSE, + max_size, priority, cancellable, + callback, user_data); +} + +gboolean +_ostree_fetcher_request_to_tmpfile_finish (OstreeFetcher *self, + GAsyncResult *result, + char **out_filename, + GError **error) +{ + GTask *task; + FetcherRequest *req; + gpointer ret; + + g_return_val_if_fail (g_task_is_valid (result, self), FALSE); + g_return_val_if_fail (g_async_result_is_tagged (result, _ostree_fetcher_request_async), FALSE); + + task = (GTask*)result; + req = g_task_get_task_data (task); + + ret = g_task_propagate_pointer (task, error); + if (!ret) + return FALSE; + + g_assert (!req->is_membuf); + g_assert (out_filename); + *out_filename = ret; + + return TRUE; +} + +void +_ostree_fetcher_request_to_membuf (OstreeFetcher *self, + GPtrArray *mirrorlist, + const char *filename, + OstreeFetcherRequestFlags flags, + guint64 max_size, + int priority, + GCancellable *cancellable, + GAsyncReadyCallback callback, + gpointer user_data) +{ + _ostree_fetcher_request_async (self, mirrorlist, filename, flags, TRUE, + max_size, priority, cancellable, + callback, user_data); +} + +gboolean +_ostree_fetcher_request_to_membuf_finish (OstreeFetcher *self, + GAsyncResult *result, + GBytes **out_buf, + GError **error) +{ + GTask *task; + FetcherRequest *req; + gpointer ret; + + g_return_val_if_fail (g_task_is_valid (result, self), FALSE); + g_return_val_if_fail (g_async_result_is_tagged (result, _ostree_fetcher_request_async), FALSE); + + task = (GTask*)result; + req = g_task_get_task_data (task); + + ret = g_task_propagate_pointer (task, error); + if (!ret) + return FALSE; + + g_assert (req->is_membuf); + g_assert (out_buf); + *out_buf = ret; + + return TRUE; +} + +guint64 +_ostree_fetcher_bytes_transferred (OstreeFetcher *self) +{ + return self->bytes_transferred; +} diff --git a/src/libostree/ostree-fetcher.c b/src/libostree/ostree-fetcher-soup.c similarity index 95% rename from src/libostree/ostree-fetcher.c rename to src/libostree/ostree-fetcher-soup.c index bb98023dc6..fcdf8e0e7c 100644 --- a/src/libostree/ostree-fetcher.c +++ b/src/libostree/ostree-fetcher-soup.c @@ -1332,84 +1332,3 @@ _ostree_fetcher_bytes_transferred (OstreeFetcher *self) return ret; } - -void -_ostree_fetcher_uri_free (OstreeFetcherURI *uri) -{ - if (uri) - soup_uri_free ((SoupURI*)uri); -} - -OstreeFetcherURI * -_ostree_fetcher_uri_parse (const char *str, - GError **error) -{ - SoupURI *soupuri = soup_uri_new (str); - if (soupuri == NULL) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Failed to parse uri: %s", str); - return NULL; - } - return (OstreeFetcherURI*)soupuri; -} - -static OstreeFetcherURI * -_ostree_fetcher_uri_new_path_internal (OstreeFetcherURI *uri, - gboolean extend, - const char *path) -{ - SoupURI *newuri = soup_uri_copy ((SoupURI*)uri); - if (path) - { - if (extend) - { - const char *origpath = soup_uri_get_path ((SoupURI*)uri); - g_autofree char *newpath = g_build_filename (origpath, path, NULL); - soup_uri_set_path (newuri, newpath); - } - else - { - soup_uri_set_path (newuri, path); - } - } - return (OstreeFetcherURI*)newuri; -} - -OstreeFetcherURI * -_ostree_fetcher_uri_new_path (OstreeFetcherURI *uri, - const char *path) -{ - return _ostree_fetcher_uri_new_path_internal (uri, FALSE, path); -} - -OstreeFetcherURI * -_ostree_fetcher_uri_new_subpath (OstreeFetcherURI *uri, - const char *subpath) -{ - return _ostree_fetcher_uri_new_path_internal (uri, TRUE, subpath); -} - -OstreeFetcherURI * -_ostree_fetcher_uri_clone (OstreeFetcherURI *uri) -{ - return _ostree_fetcher_uri_new_subpath (uri, NULL); -} - -char * -_ostree_fetcher_uri_get_scheme (OstreeFetcherURI *uri) -{ - return g_strdup (soup_uri_get_scheme ((SoupURI*)uri)); -} - -char * -_ostree_fetcher_uri_get_path (OstreeFetcherURI *uri) -{ - return g_strdup (soup_uri_get_path ((SoupURI*)uri)); -} - -char * -_ostree_fetcher_uri_to_string (OstreeFetcherURI *uri) -{ - return soup_uri_to_string ((SoupURI*)uri, FALSE); -} diff --git a/src/libostree/ostree-fetcher-uri.c b/src/libostree/ostree-fetcher-uri.c new file mode 100644 index 0000000000..7ef42eca84 --- /dev/null +++ b/src/libostree/ostree-fetcher-uri.c @@ -0,0 +1,118 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright (C) 2011,2017 Colin Walters + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + * Author: Colin Walters + */ + +#include "config.h" + + +#ifdef HAVE_LIBCURL +#include "ostree-soup-uri.h" +#else +#define LIBSOUP_USE_UNSTABLE_REQUEST_API +#include +#include +#include +#endif + +#include "ostree-fetcher.h" + +#include "libglnx.h" + +void +_ostree_fetcher_uri_free (OstreeFetcherURI *uri) +{ + if (uri) + soup_uri_free ((SoupURI*)uri); +} + +OstreeFetcherURI * +_ostree_fetcher_uri_parse (const char *str, + GError **error) +{ + SoupURI *soupuri = soup_uri_new (str); + if (soupuri == NULL) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Failed to parse uri: %s", str); + return NULL; + } + return (OstreeFetcherURI*)soupuri; +} + +static OstreeFetcherURI * +_ostree_fetcher_uri_new_path_internal (OstreeFetcherURI *uri, + gboolean extend, + const char *path) +{ + SoupURI *newuri = soup_uri_copy ((SoupURI*)uri); + if (path) + { + if (extend) + { + const char *origpath = soup_uri_get_path ((SoupURI*)uri); + g_autofree char *newpath = g_build_filename (origpath, path, NULL); + soup_uri_set_path (newuri, newpath); + } + else + { + soup_uri_set_path (newuri, path); + } + } + return (OstreeFetcherURI*)newuri; +} + +OstreeFetcherURI * +_ostree_fetcher_uri_new_path (OstreeFetcherURI *uri, + const char *path) +{ + return _ostree_fetcher_uri_new_path_internal (uri, FALSE, path); +} + +OstreeFetcherURI * +_ostree_fetcher_uri_new_subpath (OstreeFetcherURI *uri, + const char *subpath) +{ + return _ostree_fetcher_uri_new_path_internal (uri, TRUE, subpath); +} + +OstreeFetcherURI * +_ostree_fetcher_uri_clone (OstreeFetcherURI *uri) +{ + return _ostree_fetcher_uri_new_subpath (uri, NULL); +} + +char * +_ostree_fetcher_uri_get_scheme (OstreeFetcherURI *uri) +{ + return g_strdup (soup_uri_get_scheme ((SoupURI*)uri)); +} + +char * +_ostree_fetcher_uri_get_path (OstreeFetcherURI *uri) +{ + return g_strdup (soup_uri_get_path ((SoupURI*)uri)); +} + +char * +_ostree_fetcher_uri_to_string (OstreeFetcherURI *uri) +{ + return soup_uri_to_string ((SoupURI*)uri, FALSE); +} diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 79da78095c..295973ec3c 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -26,7 +26,7 @@ #include "ostree.h" #include "otutil.h" -#ifdef HAVE_LIBSOUP +#ifdef HAVE_LIBCURL_OR_LIBSOUP #include "ostree-core-private.h" #include "ostree-repo-private.h" @@ -3405,7 +3405,7 @@ ostree_repo_remote_fetch_summary_with_options (OstreeRepo *self, return ret; } -#else /* HAVE_LIBSOUP */ +#else /* HAVE_LIBCURL_OR_LIBSOUP */ gboolean ostree_repo_pull_with_options (OstreeRepo *self, @@ -3434,4 +3434,4 @@ ostree_repo_remote_fetch_summary_with_options (OstreeRepo *self, return FALSE; } -#endif /* HAVE_LIBSOUP */ +#endif /* HAVE_LIBCURL_OR_LIBSOUP */ diff --git a/src/libostree/ostree-soup-form.c b/src/libostree/ostree-soup-form.c new file mode 100644 index 0000000000..74f9c7bb04 --- /dev/null +++ b/src/libostree/ostree-soup-form.c @@ -0,0 +1,140 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*- */ +/* soup-form.c : utility functions for HTML forms */ + +/* + * Copyright 2008 Red Hat, Inc. + */ + +/* This one is stripped down to only have soup_form_encode_hash() + * and soup_form_encode_valist() which are the only bits that soup-uri.c + * calls. + */ + +#include + +#include + +#include "ostree-soup-uri.h" + +/** + * SECTION:soup-form + * @short_description: HTML form handling + * @see_also: #SoupMultipart + * + * libsoup contains several help methods for processing HTML forms as + * defined by the + * HTML 4.01 specification. + **/ + +/** + * SOUP_FORM_MIME_TYPE_URLENCODED: + * + * A macro containing the value + * "application/x-www-form-urlencoded"; the default + * MIME type for POSTing HTML form data. + * + * Since: 2.26 + **/ + +/** + * SOUP_FORM_MIME_TYPE_MULTIPART: + * + * A macro containing the value + * "multipart/form-data"; the MIME type used for + * posting form data that contains files to be uploaded. + * + * Since: 2.26 + **/ + +#define XDIGIT(c) ((c) <= '9' ? (c) - '0' : ((c) & 0x4F) - 'A' + 10) +#define HEXCHAR(s) ((XDIGIT (s[1]) << 4) + XDIGIT (s[2])) + +static void +append_form_encoded (GString *str, const char *in) +{ + const unsigned char *s = (const unsigned char *)in; + + while (*s) { + if (*s == ' ') { + g_string_append_c (str, '+'); + s++; + } else if (!g_ascii_isalnum (*s) && (*s != '-') && (*s != '_') + && (*s != '.')) + g_string_append_printf (str, "%%%02X", (int)*s++); + else + g_string_append_c (str, *s++); + } +} + +static void +encode_pair (GString *str, const char *name, const char *value) +{ + g_return_if_fail (name != NULL); + g_return_if_fail (value != NULL); + + if (str->len) + g_string_append_c (str, '&'); + append_form_encoded (str, name); + g_string_append_c (str, '='); + append_form_encoded (str, value); +} + +/** + * soup_form_encode_hash: + * @form_data_set: (element-type utf8 utf8): a hash table containing + * name/value pairs (as strings) + * + * Encodes @form_data_set into a value of type + * "application/x-www-form-urlencoded", as defined in the HTML 4.01 + * spec. + * + * Note that the HTML spec states that "The control names/values are + * listed in the order they appear in the document." Since this method + * takes a hash table, it cannot enforce that; if you care about the + * ordering of the form fields, use soup_form_encode_datalist(). + * + * Return value: the encoded form + **/ +char * +soup_form_encode_hash (GHashTable *form_data_set) +{ + GString *str = g_string_new (NULL); + GHashTableIter iter; + gpointer name, value; + + g_hash_table_iter_init (&iter, form_data_set); + while (g_hash_table_iter_next (&iter, &name, &value)) + encode_pair (str, name, value); + return g_string_free (str, FALSE); +} + +/** + * soup_form_encode_valist: + * @first_field: name of the first form field + * @args: pointer to additional values, as in soup_form_encode() + * + * See soup_form_encode(). This is mostly an internal method, used by + * various other methods such as soup_uri_set_query_from_fields() and + * soup_form_request_new(). + * + * Return value: the encoded form + **/ +char * +soup_form_encode_valist (const char *first_field, va_list args) +{ + GString *str = g_string_new (NULL); + const char *name, *value; + + name = first_field; + value = va_arg (args, const char *); + while (name && value) { + encode_pair (str, name, value); + + name = va_arg (args, const char *); + if (name) + value = va_arg (args, const char *); + } + + return g_string_free (str, FALSE); +} diff --git a/src/libostree/ostree-soup-uri.c b/src/libostree/ostree-soup-uri.c new file mode 100644 index 0000000000..97f74636b6 --- /dev/null +++ b/src/libostree/ostree-soup-uri.c @@ -0,0 +1,1483 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*- */ +/* soup-uri.c : utility functions to parse URLs */ + +/* + * Copyright 1999-2003 Ximian, Inc. + */ + +#include "config.h" + +#include +#include + +#include "ostree-soup-uri.h" + +/* OSTREECHANGE: definitions from soup-misc-private.h */ +char *soup_uri_decoded_copy (const char *str, int length, int *decoded_length); +char *soup_uri_to_string_internal (SoupURI *uri, gboolean just_path_and_query, + gboolean force_port); +gboolean soup_uri_is_http (SoupURI *uri, char **aliases); +gboolean soup_uri_is_https (SoupURI *uri, char **aliases); + +/* OSTREECHANGE: import soup-misc's char helpers */ +#define SOUP_CHAR_URI_PERCENT_ENCODED 0x01 +#define SOUP_CHAR_URI_GEN_DELIMS 0x02 +#define SOUP_CHAR_URI_SUB_DELIMS 0x04 +#define SOUP_CHAR_HTTP_SEPARATOR 0x08 +#define SOUP_CHAR_HTTP_CTL 0x10 + +/* 00 URI_UNRESERVED + * 01 URI_PCT_ENCODED + * 02 URI_GEN_DELIMS + * 04 URI_SUB_DELIMS + * 08 HTTP_SEPARATOR + * 10 HTTP_CTL + */ +const char soup_char_attributes[] = { + /* 0x00 - 0x07 */ + 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, + /* 0x08 - 0x0f */ + 0x11, 0x19, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, + /* 0x10 - 0x17 */ + 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, + /* 0x18 - 0x1f */ + 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, + /* !"#$%&' */ + 0x09, 0x04, 0x09, 0x02, 0x04, 0x01, 0x04, 0x04, + /* ()*+,-./ */ + 0x0c, 0x0c, 0x04, 0x04, 0x0c, 0x00, 0x00, 0x0a, + /* 01234567 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + /* 89:;<=>? */ + 0x00, 0x00, 0x0a, 0x0c, 0x09, 0x0a, 0x09, 0x0a, + /* @ABCDEFG */ + 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + /* HIJKLMNO */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + /* PQRSTUVW */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + /* XYZ[\]^_ */ + 0x00, 0x00, 0x00, 0x0a, 0x09, 0x0a, 0x01, 0x00, + /* `abcdefg */ + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + /* hijklmno */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + /* pqrstuvw */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + /* xyz{|}~ */ + 0x00, 0x00, 0x00, 0x09, 0x01, 0x09, 0x00, 0x11, + /* 0x80 - 0xFF */ + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01 +}; + +#define soup_char_is_uri_percent_encoded(ch) (soup_char_attributes[(guchar)ch] & SOUP_CHAR_URI_PERCENT_ENCODED) +#define soup_char_is_uri_gen_delims(ch) (soup_char_attributes[(guchar)ch] & SOUP_CHAR_URI_GEN_DELIMS) +#define soup_char_is_uri_sub_delims(ch) (soup_char_attributes[(guchar)ch] & SOUP_CHAR_URI_SUB_DELIMS) +#define soup_char_is_uri_unreserved(ch) (!(soup_char_attributes[(guchar)ch] & (SOUP_CHAR_URI_PERCENT_ENCODED | SOUP_CHAR_URI_GEN_DELIMS | SOUP_CHAR_URI_SUB_DELIMS))) +#define soup_char_is_token(ch) (!(soup_char_attributes[(guchar)ch] & (SOUP_CHAR_HTTP_SEPARATOR | SOUP_CHAR_HTTP_CTL))) + +/** + * soup_str_case_hash: + * @key: ASCII string to hash + * + * Hashes @key in a case-insensitive manner. + * + * Return value: the hash code. + **/ +static guint +soup_str_case_hash (gconstpointer key) +{ + const char *p = key; + guint h = g_ascii_toupper(*p); + + if (h) + for (p += 1; *p != '\0'; p++) + h = (h << 5) - h + g_ascii_toupper(*p); + + return h; +} + +/** + * SECTION:soup-uri + * @short_description: URIs + * + * A #SoupURI represents a (parsed) URI. + * + * Many applications will not need to use #SoupURI directly at all; on + * the client side, soup_message_new() takes a stringified URI, and on + * the server side, the path and query components are provided for you + * in the server callback. + **/ + +/** + * SoupURI: + * @scheme: the URI scheme (eg, "http") + * @user: a username, or %NULL + * @password: a password, or %NULL + * @host: the hostname or IP address + * @port: the port number on @host + * @path: the path on @host + * @query: a query for @path, or %NULL + * @fragment: a fragment identifier within @path, or %NULL + * + * A #SoupURI represents a (parsed) URI. #SoupURI supports RFC 3986 + * (URI Generic Syntax), and can parse any valid URI. However, libsoup + * only uses "http" and "https" URIs internally; You can use + * SOUP_URI_VALID_FOR_HTTP() to test if a #SoupURI is a valid HTTP + * URI. + * + * @scheme will always be set in any URI. It is an interned string and + * is always all lowercase. (If you parse a URI with a non-lowercase + * scheme, it will be converted to lowercase.) The macros + * %SOUP_URI_SCHEME_HTTP and %SOUP_URI_SCHEME_HTTPS provide the + * interned values for "http" and "https" and can be compared against + * URI @scheme values. + * + * @user and @password are parsed as defined in the older URI specs + * (ie, separated by a colon; RFC 3986 only talks about a single + * "userinfo" field). Note that @password is not included in the + * output of soup_uri_to_string(). libsoup does not normally use these + * fields; authentication is handled via #SoupSession signals. + * + * @host contains the hostname, and @port the port specified in the + * URI. If the URI doesn't contain a hostname, @host will be %NULL, + * and if it doesn't specify a port, @port may be 0. However, for + * "http" and "https" URIs, @host is guaranteed to be non-%NULL + * (trying to parse an http URI with no @host will return %NULL), and + * @port will always be non-0 (because libsoup knows the default value + * to use when it is not specified in the URI). + * + * @path is always non-%NULL. For http/https URIs, @path will never be + * an empty string either; if the input URI has no path, the parsed + * #SoupURI will have a @path of "/". + * + * @query and @fragment are optional for all URI types. + * soup_form_decode() may be useful for parsing @query. + * + * Note that @path, @query, and @fragment may contain + * %-encoded characters. soup_uri_new() calls + * soup_uri_normalize() on them, but not soup_uri_decode(). This is + * necessary to ensure that soup_uri_to_string() will generate a URI + * that has exactly the same meaning as the original. (In theory, + * #SoupURI should leave @user, @password, and @host partially-encoded + * as well, but this would be more annoying than useful.) + **/ + +/** + * SOUP_URI_IS_VALID: + * @uri: a #SoupURI + * + * Tests whether @uri is a valid #SoupURI; that is, that it is non-%NULL + * and its @scheme and @path members are also non-%NULL. + * + * This macro does not check whether http and https URIs have a non-%NULL + * @host member. + * + * Return value: %TRUE if @uri is valid for use. + * + * Since: 2.38 + **/ + +/** + * SOUP_URI_VALID_FOR_HTTP: + * @uri: a #SoupURI + * + * Tests if @uri is a valid #SoupURI for HTTP communication; that is, if + * it can be used to construct a #SoupMessage. + * + * Return value: %TRUE if @uri is a valid "http" or "https" URI. + * + * Since: 2.24 + **/ + +/** + * SOUP_URI_SCHEME_HTTP: + * + * "http" as an interned string; you can compare this directly to a + * #SoupURI's scheme field using + * ==. + */ +/** + * SOUP_URI_SCHEME_HTTPS: + * + * "https" as an interned string; you can compare this directly to a + * #SoupURI's scheme field using + * ==. + */ +/** + * SOUP_URI_SCHEME_FTP: + * + * "ftp" as an interned string; you can compare this directly to a + * #SoupURI's scheme field using + * ==. + * + * Since: 2.30 + */ +/** + * SOUP_URI_SCHEME_FILE: + * + * "file" as an interned string; you can compare this directly to a + * #SoupURI's scheme field using + * ==. + * + * Since: 2.30 + */ +/** + * SOUP_URI_SCHEME_DATA: + * + * "data" as an interned string; you can compare this directly to a + * #SoupURI's scheme field using + * ==. + * + * Since: 2.30 + */ +/** + * SOUP_URI_SCHEME_RESOURCE: + * + * "data" as an interned string; you can compare this directly to a + * #SoupURI's scheme field using + * ==. + * + * Since: 2.42 + */ +/** + * SOUP_URI_SCHEME_WS: + * + * "ws" (WebSocket) as an interned string; you can compare this + * directly to a #SoupURI's scheme field using + * ==. + * + * Since: 2.50 + */ +/** + * SOUP_URI_SCHEME_WSS: + * + * "wss" (WebSocket over TLS) as an interned string; you can compare + * this directly to a #SoupURI's scheme field using + * ==. + * + * Since: 2.50 + */ + +struct _SoupURI { + const char *scheme; + + char *user; + char *password; + + char *host; + guint port; + + char *path; + char *query; + + char *fragment; +}; + +static void append_uri_encoded (GString *str, const char *in, const char *extra_enc_chars); +static char *uri_normalized_copy (const char *str, int length, const char *unescape_extra); + +gpointer _SOUP_URI_SCHEME_HTTP, _SOUP_URI_SCHEME_HTTPS; +gpointer _SOUP_URI_SCHEME_WS, _SOUP_URI_SCHEME_WSS; +gpointer _SOUP_URI_SCHEME_FTP; +gpointer _SOUP_URI_SCHEME_FILE, _SOUP_URI_SCHEME_DATA, _SOUP_URI_SCHEME_RESOURCE; + +static inline const char * +soup_uri_parse_scheme (const char *scheme, int len) +{ + if (len == 4 && !g_ascii_strncasecmp (scheme, "http", len)) { + return SOUP_URI_SCHEME_HTTP; + } else if (len == 5 && !g_ascii_strncasecmp (scheme, "https", len)) { + return SOUP_URI_SCHEME_HTTPS; + } else if (len == 8 && !g_ascii_strncasecmp (scheme, "resource", len)) { + return SOUP_URI_SCHEME_RESOURCE; + } else if (len == 2 && !g_ascii_strncasecmp (scheme, "ws", len)) { + return SOUP_URI_SCHEME_WS; + } else if (len == 3 && !g_ascii_strncasecmp (scheme, "wss", len)) { + return SOUP_URI_SCHEME_WSS; + } else { + char *lower_scheme; + + lower_scheme = g_ascii_strdown (scheme, len); + scheme = g_intern_static_string (lower_scheme); + if (scheme != (const char *)lower_scheme) + g_free (lower_scheme); + return scheme; + } +} + +static inline guint +soup_scheme_default_port (const char *scheme) +{ + if (scheme == SOUP_URI_SCHEME_HTTP || scheme == SOUP_URI_SCHEME_WS) + return 80; + else if (scheme == SOUP_URI_SCHEME_HTTPS || scheme == SOUP_URI_SCHEME_WSS) + return 443; + else if (scheme == SOUP_URI_SCHEME_FTP) + return 21; + else + return 0; +} + +/** + * soup_uri_new_with_base: + * @base: a base URI + * @uri_string: the URI + * + * Parses @uri_string relative to @base. + * + * Return value: a parsed #SoupURI. + **/ +SoupURI * +soup_uri_new_with_base (SoupURI *base, const char *uri_string) +{ + SoupURI *uri, fixed_base; + const char *end, *hash, *colon, *at, *path, *question; + const char *p, *hostend; + gboolean remove_dot_segments = TRUE; + int len; + + g_return_val_if_fail (uri_string != NULL, NULL); + + /* Allow a %NULL path in @base, for compatibility */ + if (base && base->scheme && !base->path) { + g_warn_if_fail (SOUP_URI_IS_VALID (base)); + + memcpy (&fixed_base, base, sizeof (SoupURI)); + fixed_base.path = ""; + base = &fixed_base; + } + + g_return_val_if_fail (base == NULL || SOUP_URI_IS_VALID (base), NULL); + + /* First some cleanup steps (which are supposed to all be no-ops, + * but...). Skip initial whitespace, strip out internal tabs and + * line breaks, and ignore trailing whitespace. + */ + while (g_ascii_isspace (*uri_string)) + uri_string++; + + len = strcspn (uri_string, "\t\n\r"); + if (uri_string[len]) { + char *clean = g_malloc (strlen (uri_string) + 1), *d; + const char *s; + + for (s = uri_string, d = clean; *s; s++) { + if (*s != '\t' && *s != '\n' && *s != '\r') + *d++ = *s; + } + *d = '\0'; + + uri = soup_uri_new_with_base (base, clean); + g_free (clean); + return uri; + } + end = uri_string + len; + while (end > uri_string && g_ascii_isspace (end[-1])) + end--; + + uri = g_slice_new0 (SoupURI); + + /* Find fragment. */ + hash = strchr (uri_string, '#'); + if (hash) { + uri->fragment = uri_normalized_copy (hash + 1, end - hash + 1, + NULL); + end = hash; + } + + /* Find scheme */ + p = uri_string; + while (p < end && (g_ascii_isalpha (*p) || + (p > uri_string && (g_ascii_isdigit (*p) || + *p == '.' || + *p == '+' || + *p == '-')))) + p++; + + if (p > uri_string && *p == ':') { + uri->scheme = soup_uri_parse_scheme (uri_string, p - uri_string); + uri_string = p + 1; + } + + if (uri_string == end && !base && !uri->fragment) { + uri->path = g_strdup (""); + return uri; + } + + /* Check for authority */ + if (strncmp (uri_string, "//", 2) == 0) { + uri_string += 2; + + path = uri_string + strcspn (uri_string, "/?#"); + if (path > end) + path = end; + at = strchr (uri_string, '@'); + if (at && at < path) { + colon = strchr (uri_string, ':'); + if (colon && colon < at) { + uri->password = soup_uri_decoded_copy (colon + 1, + at - colon - 1, NULL); + } else { + uri->password = NULL; + colon = at; + } + + uri->user = soup_uri_decoded_copy (uri_string, + colon - uri_string, NULL); + uri_string = at + 1; + } else + uri->user = uri->password = NULL; + + /* Find host and port. */ + if (*uri_string == '[') { + const char *pct; + + uri_string++; + hostend = strchr (uri_string, ']'); + if (!hostend || hostend > path) { + soup_uri_free (uri); + return NULL; + } + if (*(hostend + 1) == ':') + colon = hostend + 1; + else + colon = NULL; + + pct = memchr (uri_string, '%', hostend - uri_string); + if (!pct || (pct[1] == '2' && pct[2] == '5')) { + uri->host = soup_uri_decoded_copy (uri_string, + hostend - uri_string, NULL); + } else + uri->host = g_strndup (uri_string, hostend - uri_string); + } else { + colon = memchr (uri_string, ':', path - uri_string); + hostend = colon ? colon : path; + uri->host = soup_uri_decoded_copy (uri_string, + hostend - uri_string, NULL); + } + + if (colon && colon != path - 1) { + char *portend; + uri->port = strtoul (colon + 1, &portend, 10); + if (portend != (char *)path) { + soup_uri_free (uri); + return NULL; + } + } + + uri_string = path; + } + + /* Find query */ + question = memchr (uri_string, '?', end - uri_string); + if (question) { + uri->query = uri_normalized_copy (question + 1, + end - (question + 1), + NULL); + end = question; + } + + if (end != uri_string) { + uri->path = uri_normalized_copy (uri_string, end - uri_string, + NULL); + } + + /* Apply base URI. This is spelled out in RFC 3986. */ + if (base && !uri->scheme && uri->host) + uri->scheme = base->scheme; + else if (base && !uri->scheme) { + uri->scheme = base->scheme; + uri->user = g_strdup (base->user); + uri->password = g_strdup (base->password); + uri->host = g_strdup (base->host); + uri->port = base->port; + + if (!uri->path) { + uri->path = g_strdup (base->path); + if (!uri->query) + uri->query = g_strdup (base->query); + remove_dot_segments = FALSE; + } else if (*uri->path != '/') { + char *newpath, *last; + + last = strrchr (base->path, '/'); + if (last) { + newpath = g_strdup_printf ("%.*s%s", + (int)(last + 1 - base->path), + base->path, + uri->path); + } else + newpath = g_strdup_printf ("/%s", uri->path); + + g_free (uri->path); + uri->path = newpath; + } + } + + if (remove_dot_segments && uri->path && *uri->path) { + char *p, *q; + + /* Remove "./" where "." is a complete segment. */ + for (p = uri->path + 1; *p; ) { + if (*(p - 1) == '/' && + *p == '.' && *(p + 1) == '/') + memmove (p, p + 2, strlen (p + 2) + 1); + else + p++; + } + /* Remove "." at end. */ + if (p > uri->path + 2 && + *(p - 1) == '.' && *(p - 2) == '/') + *(p - 1) = '\0'; + + /* Remove "/../" where != ".." */ + for (p = uri->path + 1; *p; ) { + if (!strncmp (p, "../", 3)) { + p += 3; + continue; + } + q = strchr (p + 1, '/'); + if (!q) + break; + if (strncmp (q, "/../", 4) != 0) { + p = q + 1; + continue; + } + memmove (p, q + 4, strlen (q + 4) + 1); + p = uri->path + 1; + } + /* Remove "/.." at end where != ".." */ + q = strrchr (uri->path, '/'); + if (q && !strcmp (q, "/..")) { + p = q - 1; + while (p > uri->path && *p != '/') + p--; + if (strncmp (p, "/../", 4) != 0) + *(p + 1) = 0; + } + + /* Remove extraneous initial "/.."s */ + while (!strncmp (uri->path, "/../", 4)) + memmove (uri->path, uri->path + 3, strlen (uri->path) - 2); + if (!strcmp (uri->path, "/..")) + uri->path[1] = '\0'; + } + + /* HTTP-specific stuff */ + if (uri->scheme == SOUP_URI_SCHEME_HTTP || + uri->scheme == SOUP_URI_SCHEME_HTTPS) { + if (!uri->path) + uri->path = g_strdup ("/"); + if (!SOUP_URI_VALID_FOR_HTTP (uri)) { + soup_uri_free (uri); + return NULL; + } + } + + if (uri->scheme == SOUP_URI_SCHEME_FTP) { + if (!uri->host) { + soup_uri_free (uri); + return NULL; + } + } + + if (!uri->port) + uri->port = soup_scheme_default_port (uri->scheme); + if (!uri->path) + uri->path = g_strdup (""); + + return uri; +} + +/** + * soup_uri_new: + * @uri_string: (allow-none): a URI + * + * Parses an absolute URI. + * + * You can also pass %NULL for @uri_string if you want to get back an + * "empty" #SoupURI that you can fill in by hand. (You will need to + * call at least soup_uri_set_scheme() and soup_uri_set_path(), since + * those fields are required.) + * + * Return value: (nullable): a #SoupURI, or %NULL if the given string + * was found to be invalid. + **/ +SoupURI * +soup_uri_new (const char *uri_string) +{ + SoupURI *uri; + + if (!uri_string) + return g_slice_new0 (SoupURI); + + uri = soup_uri_new_with_base (NULL, uri_string); + if (!uri) + return NULL; + if (!SOUP_URI_IS_VALID (uri)) { + soup_uri_free (uri); + return NULL; + } + + return uri; +} + + +char * +soup_uri_to_string_internal (SoupURI *uri, gboolean just_path_and_query, + gboolean force_port) +{ + GString *str; + char *return_result; + + g_return_val_if_fail (uri != NULL, NULL); + g_warn_if_fail (SOUP_URI_IS_VALID (uri)); + + str = g_string_sized_new (40); + + if (uri->scheme && !just_path_and_query) + g_string_append_printf (str, "%s:", uri->scheme); + if (uri->host && !just_path_and_query) { + g_string_append (str, "//"); + if (uri->user) { + append_uri_encoded (str, uri->user, ":;@?/"); + g_string_append_c (str, '@'); + } + if (strchr (uri->host, ':')) { + const char *pct; + + g_string_append_c (str, '['); + pct = strchr (uri->host, '%'); + if (pct) { + g_string_append_printf (str, "%.*s%%25%s", + (int) (pct - uri->host), + uri->host, pct + 1); + } else + g_string_append (str, uri->host); + g_string_append_c (str, ']'); + } else + append_uri_encoded (str, uri->host, ":/"); + if (uri->port && (force_port || uri->port != soup_scheme_default_port (uri->scheme))) + g_string_append_printf (str, ":%u", uri->port); + if (!uri->path && (uri->query || uri->fragment)) + g_string_append_c (str, '/'); + else if ((!uri->path || !*uri->path) && + (uri->scheme == SOUP_URI_SCHEME_HTTP || + uri->scheme == SOUP_URI_SCHEME_HTTPS)) + g_string_append_c (str, '/'); + } + + if (uri->path && *uri->path) + g_string_append (str, uri->path); + else if (just_path_and_query) + g_string_append_c (str, '/'); + + if (uri->query) { + g_string_append_c (str, '?'); + g_string_append (str, uri->query); + } + if (uri->fragment && !just_path_and_query) { + g_string_append_c (str, '#'); + g_string_append (str, uri->fragment); + } + + return_result = str->str; + g_string_free (str, FALSE); + + return return_result; +} + +/** + * soup_uri_to_string: + * @uri: a #SoupURI + * @just_path_and_query: if %TRUE, output just the path and query portions + * + * Returns a string representing @uri. + * + * If @just_path_and_query is %TRUE, this concatenates the path and query + * together. That is, it constructs the string that would be needed in + * the Request-Line of an HTTP request for @uri. + * + * Note that the output will never contain a password, even if @uri + * does. + * + * Return value: a string representing @uri, which the caller must free. + **/ +char * +soup_uri_to_string (SoupURI *uri, gboolean just_path_and_query) +{ + return soup_uri_to_string_internal (uri, just_path_and_query, FALSE); +} + +/** + * soup_uri_copy: + * @uri: a #SoupURI + * + * Copies @uri + * + * Return value: a copy of @uri, which must be freed with soup_uri_free() + **/ +SoupURI * +soup_uri_copy (SoupURI *uri) +{ + SoupURI *dup; + + g_return_val_if_fail (uri != NULL, NULL); + g_warn_if_fail (SOUP_URI_IS_VALID (uri)); + + dup = g_slice_new0 (SoupURI); + dup->scheme = uri->scheme; + dup->user = g_strdup (uri->user); + dup->password = g_strdup (uri->password); + dup->host = g_strdup (uri->host); + dup->port = uri->port; + dup->path = g_strdup (uri->path); + dup->query = g_strdup (uri->query); + dup->fragment = g_strdup (uri->fragment); + + return dup; +} + +static inline gboolean +parts_equal (const char *one, const char *two, gboolean insensitive) +{ + if (!one && !two) + return TRUE; + if (!one || !two) + return FALSE; + return insensitive ? !g_ascii_strcasecmp (one, two) : !strcmp (one, two); +} + +/** + * soup_uri_equal: + * @uri1: a #SoupURI + * @uri2: another #SoupURI + * + * Tests whether or not @uri1 and @uri2 are equal in all parts + * + * Return value: %TRUE or %FALSE + **/ +gboolean +soup_uri_equal (SoupURI *uri1, SoupURI *uri2) +{ + g_return_val_if_fail (uri1 != NULL, FALSE); + g_return_val_if_fail (uri2 != NULL, FALSE); + g_warn_if_fail (SOUP_URI_IS_VALID (uri1)); + g_warn_if_fail (SOUP_URI_IS_VALID (uri2)); + + if (uri1->scheme != uri2->scheme || + uri1->port != uri2->port || + !parts_equal (uri1->user, uri2->user, FALSE) || + !parts_equal (uri1->password, uri2->password, FALSE) || + !parts_equal (uri1->host, uri2->host, TRUE) || + !parts_equal (uri1->path, uri2->path, FALSE) || + !parts_equal (uri1->query, uri2->query, FALSE) || + !parts_equal (uri1->fragment, uri2->fragment, FALSE)) + return FALSE; + + return TRUE; +} + +/** + * soup_uri_free: + * @uri: a #SoupURI + * + * Frees @uri. + **/ +void +soup_uri_free (SoupURI *uri) +{ + g_return_if_fail (uri != NULL); + + g_free (uri->user); + g_free (uri->password); + g_free (uri->host); + g_free (uri->path); + g_free (uri->query); + g_free (uri->fragment); + + g_slice_free (SoupURI, uri); +} + +static void +append_uri_encoded (GString *str, const char *in, const char *extra_enc_chars) +{ + const unsigned char *s = (const unsigned char *)in; + + while (*s) { + if (soup_char_is_uri_percent_encoded (*s) || + soup_char_is_uri_gen_delims (*s) || + (extra_enc_chars && strchr (extra_enc_chars, *s))) + g_string_append_printf (str, "%%%02X", (int)*s++); + else + g_string_append_c (str, *s++); + } +} + +/** + * soup_uri_encode: + * @part: a URI part + * @escape_extra: (allow-none): additional reserved characters to + * escape (or %NULL) + * + * This %-encodes the given URI part and returns the escaped + * version in allocated memory, which the caller must free when it is + * done. + * + * Return value: the encoded URI part + **/ +char * +soup_uri_encode (const char *part, const char *escape_extra) +{ + GString *str; + char *encoded; + + g_return_val_if_fail (part != NULL, NULL); + + str = g_string_new (NULL); + append_uri_encoded (str, part, escape_extra); + encoded = str->str; + g_string_free (str, FALSE); + + return encoded; +} + +#define XDIGIT(c) ((c) <= '9' ? (c) - '0' : ((c) & 0x4F) - 'A' + 10) +#define HEXCHAR(s) ((XDIGIT (s[1]) << 4) + XDIGIT (s[2])) + +char * +soup_uri_decoded_copy (const char *part, int length, int *decoded_length) +{ + unsigned char *s, *d; + char *decoded; + + g_return_val_if_fail (part != NULL, NULL); + + decoded = g_strndup (part, length); + s = d = (unsigned char *)decoded; + do { + if (*s == '%') { + if (!g_ascii_isxdigit (s[1]) || + !g_ascii_isxdigit (s[2])) { + *d++ = *s; + continue; + } + *d++ = HEXCHAR (s); + s += 2; + } else + *d++ = *s; + } while (*s++); + + if (decoded_length) + *decoded_length = d - (unsigned char *)decoded - 1; + + return decoded; +} + +/** + * soup_uri_decode: + * @part: a URI part + * + * Fully %-decodes @part. + * + * In the past, this would return %NULL if @part contained invalid + * percent-encoding, but now it just ignores the problem (as + * soup_uri_new() already did). + * + * Return value: the decoded URI part. + */ +char * +soup_uri_decode (const char *part) +{ + g_return_val_if_fail (part != NULL, NULL); + + return soup_uri_decoded_copy (part, strlen (part), NULL); +} + +static char * +uri_normalized_copy (const char *part, int length, + const char *unescape_extra) +{ + unsigned char *s, *d, c; + char *normalized = g_strndup (part, length); + gboolean need_fixup = FALSE; + + if (!unescape_extra) + unescape_extra = ""; + + s = d = (unsigned char *)normalized; + while (*s) { + if (*s == '%') { + if (!g_ascii_isxdigit (s[1]) || + !g_ascii_isxdigit (s[2])) { + *d++ = *s++; + continue; + } + + c = HEXCHAR (s); + if (soup_char_is_uri_unreserved (c) || + (c && strchr (unescape_extra, c))) { + *d++ = c; + s += 3; + } else { + /* We leave it unchanged. We used to uppercase percent-encoded + * triplets but we do not do it any more as RFC3986 Section 6.2.2.1 + * says that they only SHOULD be case normalized. + */ + *d++ = *s++; + *d++ = *s++; + *d++ = *s++; + } + } else { + if (!g_ascii_isgraph (*s) && + !strchr (unescape_extra, *s)) + need_fixup = TRUE; + *d++ = *s++; + } + } + *d = '\0'; + + if (need_fixup) { + GString *fixed; + + fixed = g_string_new (NULL); + s = (guchar *)normalized; + while (*s) { + if (g_ascii_isgraph (*s) || + strchr (unescape_extra, *s)) + g_string_append_c (fixed, *s); + else + g_string_append_printf (fixed, "%%%02X", (int)*s); + s++; + } + g_free (normalized); + normalized = g_string_free (fixed, FALSE); + } + + return normalized; +} + +/** + * soup_uri_normalize: + * @part: a URI part + * @unescape_extra: (allow-none): reserved characters to unescape (or %NULL) + * + * %-decodes any "unreserved" characters (or characters in + * @unescape_extra) in @part, and %-encodes any non-ASCII + * characters, spaces, and non-printing characters in @part. + * + * "Unreserved" characters are those that are not allowed to be used + * for punctuation according to the URI spec. For example, letters are + * unreserved, so soup_uri_normalize() will turn + * http://example.com/foo/b%61r into + * http://example.com/foo/bar, which is guaranteed + * to mean the same thing. However, "/" is "reserved", so + * http://example.com/foo%2Fbar would not + * be changed, because it might mean something different to the + * server. + * + * In the past, this would return %NULL if @part contained invalid + * percent-encoding, but now it just ignores the problem (as + * soup_uri_new() already did). + * + * Return value: the normalized URI part + */ +char * +soup_uri_normalize (const char *part, const char *unescape_extra) +{ + g_return_val_if_fail (part != NULL, NULL); + + return uri_normalized_copy (part, strlen (part), unescape_extra); +} + + +/** + * soup_uri_uses_default_port: + * @uri: a #SoupURI + * + * Tests if @uri uses the default port for its scheme. (Eg, 80 for + * http.) (This only works for http, https and ftp; libsoup does not know + * the default ports of other protocols.) + * + * Return value: %TRUE or %FALSE + **/ +gboolean +soup_uri_uses_default_port (SoupURI *uri) +{ + g_return_val_if_fail (uri != NULL, FALSE); + g_warn_if_fail (SOUP_URI_IS_VALID (uri)); + + return uri->port == soup_scheme_default_port (uri->scheme); +} + +/** + * soup_uri_get_scheme: + * @uri: a #SoupURI + * + * Gets @uri's scheme. + * + * Return value: @uri's scheme. + * + * Since: 2.32 + **/ +const char * +soup_uri_get_scheme (SoupURI *uri) +{ + g_return_val_if_fail (uri != NULL, NULL); + + return uri->scheme; +} + +/** + * soup_uri_set_scheme: + * @uri: a #SoupURI + * @scheme: the URI scheme + * + * Sets @uri's scheme to @scheme. This will also set @uri's port to + * the default port for @scheme, if known. + **/ +void +soup_uri_set_scheme (SoupURI *uri, const char *scheme) +{ + g_return_if_fail (uri != NULL); + g_return_if_fail (scheme != NULL); + + uri->scheme = soup_uri_parse_scheme (scheme, strlen (scheme)); + uri->port = soup_scheme_default_port (uri->scheme); +} + +/** + * soup_uri_get_user: + * @uri: a #SoupURI + * + * Gets @uri's user. + * + * Return value: @uri's user. + * + * Since: 2.32 + **/ +const char * +soup_uri_get_user (SoupURI *uri) +{ + g_return_val_if_fail (uri != NULL, NULL); + + return uri->user; +} + +/** + * soup_uri_set_user: + * @uri: a #SoupURI + * @user: (allow-none): the username, or %NULL + * + * Sets @uri's user to @user. + **/ +void +soup_uri_set_user (SoupURI *uri, const char *user) +{ + g_return_if_fail (uri != NULL); + + g_free (uri->user); + uri->user = g_strdup (user); +} + +/** + * soup_uri_get_password: + * @uri: a #SoupURI + * + * Gets @uri's password. + * + * Return value: @uri's password. + * + * Since: 2.32 + **/ +const char * +soup_uri_get_password (SoupURI *uri) +{ + g_return_val_if_fail (uri != NULL, NULL); + + return uri->password; +} + +/** + * soup_uri_set_password: + * @uri: a #SoupURI + * @password: (allow-none): the password, or %NULL + * + * Sets @uri's password to @password. + **/ +void +soup_uri_set_password (SoupURI *uri, const char *password) +{ + g_return_if_fail (uri != NULL); + + g_free (uri->password); + uri->password = g_strdup (password); +} + +/** + * soup_uri_get_host: + * @uri: a #SoupURI + * + * Gets @uri's host. + * + * Return value: @uri's host. + * + * Since: 2.32 + **/ +const char * +soup_uri_get_host (SoupURI *uri) +{ + g_return_val_if_fail (uri != NULL, NULL); + + return uri->host; +} + +/** + * soup_uri_set_host: + * @uri: a #SoupURI + * @host: (allow-none): the hostname or IP address, or %NULL + * + * Sets @uri's host to @host. + * + * If @host is an IPv6 IP address, it should not include the brackets + * required by the URI syntax; they will be added automatically when + * converting @uri to a string. + * + * http and https URIs should not have a %NULL @host. + **/ +void +soup_uri_set_host (SoupURI *uri, const char *host) +{ + g_return_if_fail (uri != NULL); + + g_free (uri->host); + uri->host = g_strdup (host); +} + +/** + * soup_uri_get_port: + * @uri: a #SoupURI + * + * Gets @uri's port. + * + * Return value: @uri's port. + * + * Since: 2.32 + **/ +guint +soup_uri_get_port (SoupURI *uri) +{ + g_return_val_if_fail (uri != NULL, 0); + + return uri->port; +} + +/** + * soup_uri_set_port: + * @uri: a #SoupURI + * @port: the port, or 0 + * + * Sets @uri's port to @port. If @port is 0, @uri will not have an + * explicitly-specified port. + **/ +void +soup_uri_set_port (SoupURI *uri, guint port) +{ + g_return_if_fail (uri != NULL); + + uri->port = port; +} + +/** + * soup_uri_get_path: + * @uri: a #SoupURI + * + * Gets @uri's path. + * + * Return value: @uri's path. + * + * Since: 2.32 + **/ +const char * +soup_uri_get_path (SoupURI *uri) +{ + g_return_val_if_fail (uri != NULL, NULL); + + return uri->path; +} + +/** + * soup_uri_set_path: + * @uri: a #SoupURI + * @path: the non-%NULL path + * + * Sets @uri's path to @path. + **/ +void +soup_uri_set_path (SoupURI *uri, const char *path) +{ + g_return_if_fail (uri != NULL); + + /* We allow a NULL path for compatibility, but warn about it. */ + if (!path) { + g_warn_if_fail (path != NULL); + path = ""; + } + + g_free (uri->path); + uri->path = g_strdup (path); +} + +/** + * soup_uri_get_query: + * @uri: a #SoupURI + * + * Gets @uri's query. + * + * Return value: @uri's query. + * + * Since: 2.32 + **/ +const char * +soup_uri_get_query (SoupURI *uri) +{ + g_return_val_if_fail (uri != NULL, NULL); + + return uri->query; +} + +/** + * soup_uri_set_query: + * @uri: a #SoupURI + * @query: (allow-none): the query + * + * Sets @uri's query to @query. + **/ +void +soup_uri_set_query (SoupURI *uri, const char *query) +{ + g_return_if_fail (uri != NULL); + + g_free (uri->query); + uri->query = g_strdup (query); +} + +/** + * soup_uri_set_query_from_form: + * @uri: a #SoupURI + * @form: (element-type utf8 utf8): a #GHashTable containing HTML form + * information + * + * Sets @uri's query to the result of encoding @form according to the + * HTML form rules. See soup_form_encode_hash() for more information. + **/ +void +soup_uri_set_query_from_form (SoupURI *uri, GHashTable *form) +{ + g_return_if_fail (uri != NULL); + + g_free (uri->query); + uri->query = soup_form_encode_hash (form); +} + +/** + * soup_uri_set_query_from_fields: + * @uri: a #SoupURI + * @first_field: name of the first form field to encode into query + * @...: value of @first_field, followed by additional field names + * and values, terminated by %NULL. + * + * Sets @uri's query to the result of encoding the given form fields + * and values according to the * HTML form rules. See + * soup_form_encode() for more information. + **/ +void +soup_uri_set_query_from_fields (SoupURI *uri, + const char *first_field, + ...) +{ + va_list args; + + g_return_if_fail (uri != NULL); + + g_free (uri->query); + va_start (args, first_field); + uri->query = soup_form_encode_valist (first_field, args); + va_end (args); +} + +/** + * soup_uri_get_fragment: + * @uri: a #SoupURI + * + * Gets @uri's fragment. + * + * Return value: @uri's fragment. + * + * Since: 2.32 + **/ +const char * +soup_uri_get_fragment (SoupURI *uri) +{ + g_return_val_if_fail (uri != NULL, NULL); + + return uri->fragment; +} + +/** + * soup_uri_set_fragment: + * @uri: a #SoupURI + * @fragment: (allow-none): the fragment + * + * Sets @uri's fragment to @fragment. + **/ +void +soup_uri_set_fragment (SoupURI *uri, const char *fragment) +{ + g_return_if_fail (uri != NULL); + + g_free (uri->fragment); + uri->fragment = g_strdup (fragment); +} + +/** + * soup_uri_copy_host: + * @uri: a #SoupURI + * + * Makes a copy of @uri, considering only the protocol, host, and port + * + * Return value: the new #SoupURI + * + * Since: 2.28 + **/ +SoupURI * +soup_uri_copy_host (SoupURI *uri) +{ + SoupURI *dup; + + g_return_val_if_fail (uri != NULL, NULL); + g_warn_if_fail (SOUP_URI_IS_VALID (uri)); + + dup = soup_uri_new (NULL); + dup->scheme = uri->scheme; + dup->host = g_strdup (uri->host); + dup->port = uri->port; + dup->path = g_strdup (""); + + return dup; +} + +/** + * soup_uri_host_hash: + * @key: (type Soup.URI): a #SoupURI with a non-%NULL @host member + * + * Hashes @key, considering only the scheme, host, and port. + * + * Return value: a hash + * + * Since: 2.28 + **/ +guint +soup_uri_host_hash (gconstpointer key) +{ + const SoupURI *uri = key; + + g_return_val_if_fail (uri != NULL && uri->host != NULL, 0); + g_warn_if_fail (SOUP_URI_IS_VALID (uri)); + + return GPOINTER_TO_UINT (uri->scheme) + uri->port + + soup_str_case_hash (uri->host); +} + +/** + * soup_uri_host_equal: + * @v1: (type Soup.URI): a #SoupURI with a non-%NULL @host member + * @v2: (type Soup.URI): a #SoupURI with a non-%NULL @host member + * + * Compares @v1 and @v2, considering only the scheme, host, and port. + * + * Return value: whether or not the URIs are equal in scheme, host, + * and port. + * + * Since: 2.28 + **/ +gboolean +soup_uri_host_equal (gconstpointer v1, gconstpointer v2) +{ + const SoupURI *one = v1; + const SoupURI *two = v2; + + g_return_val_if_fail (one != NULL && two != NULL, one == two); + g_return_val_if_fail (one->host != NULL && two->host != NULL, one->host == two->host); + g_warn_if_fail (SOUP_URI_IS_VALID (one)); + g_warn_if_fail (SOUP_URI_IS_VALID (two)); + + if (one->scheme != two->scheme) + return FALSE; + if (one->port != two->port) + return FALSE; + + return g_ascii_strcasecmp (one->host, two->host) == 0; +} + +gboolean +soup_uri_is_http (SoupURI *uri, char **aliases) +{ + int i; + + if (uri->scheme == SOUP_URI_SCHEME_HTTP) + return TRUE; + else if (uri->scheme == SOUP_URI_SCHEME_HTTPS) + return FALSE; + else if (!aliases) + return FALSE; + + for (i = 0; aliases[i]; i++) { + if (uri->scheme == aliases[i]) + return TRUE; + } + + if (!aliases[1] && !strcmp (aliases[0], "*")) + return TRUE; + else + return FALSE; +} + +gboolean +soup_uri_is_https (SoupURI *uri, char **aliases) +{ + int i; + + if (uri->scheme == SOUP_URI_SCHEME_HTTPS) + return TRUE; + else if (uri->scheme == SOUP_URI_SCHEME_HTTP) + return FALSE; + else if (!aliases) + return FALSE; + + for (i = 0; aliases[i]; i++) { + if (uri->scheme == aliases[i]) + return TRUE; + } + + return FALSE; +} + +/* OSTREECHANGE: drop boxed type definition */ +/* G_DEFINE_BOXED_TYPE (SoupURI, soup_uri, soup_uri_copy, soup_uri_free) */ diff --git a/src/libostree/ostree-soup-uri.h b/src/libostree/ostree-soup-uri.h new file mode 100644 index 0000000000..650b7efcd3 --- /dev/null +++ b/src/libostree/ostree-soup-uri.h @@ -0,0 +1,147 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*- */ + +/* + * Copyright 1999-2002 Ximian, Inc. + */ + +/* NOTE - taken from the libsoup codebase for use by the ostree curl backend + * (yes, ironically enough). + * + * Please watch for future changes in libsoup. + */ + + +#ifndef SOUP_URI_H +#define SOUP_URI_H 1 + +/* OSTREECHANGE: make struct private + * Only include gio, and skip available definitions. + */ +#include +#define SOUP_AVAILABLE_IN_2_4 +#define SOUP_AVAILABLE_IN_2_28 +#define SOUP_AVAILABLE_IN_2_32 + +G_BEGIN_DECLS + +/* OSTREECHANGE: make struct private */ +typedef struct _SoupURI SoupURI; + +/* OSTREECHANGE: import soup-misc's interning */ +#define SOUP_VAR extern +#define _SOUP_ATOMIC_INTERN_STRING(variable, value) ((const char *)(g_atomic_pointer_get (&(variable)) ? (variable) : (g_atomic_pointer_set (&(variable), (gpointer)g_intern_static_string (value)), (variable)))) +#define SOUP_URI_SCHEME_HTTP _SOUP_ATOMIC_INTERN_STRING (_SOUP_URI_SCHEME_HTTP, "http") +#define SOUP_URI_SCHEME_HTTPS _SOUP_ATOMIC_INTERN_STRING (_SOUP_URI_SCHEME_HTTPS, "https") +#define SOUP_URI_SCHEME_FTP _SOUP_ATOMIC_INTERN_STRING (_SOUP_URI_SCHEME_FTP, "ftp") +#define SOUP_URI_SCHEME_FILE _SOUP_ATOMIC_INTERN_STRING (_SOUP_URI_SCHEME_FILE, "file") +#define SOUP_URI_SCHEME_DATA _SOUP_ATOMIC_INTERN_STRING (_SOUP_URI_SCHEME_DATA, "data") +#define SOUP_URI_SCHEME_RESOURCE _SOUP_ATOMIC_INTERN_STRING (_SOUP_URI_SCHEME_RESOURCE, "resource") +#define SOUP_URI_SCHEME_WS _SOUP_ATOMIC_INTERN_STRING (_SOUP_URI_SCHEME_WS, "ws") +#define SOUP_URI_SCHEME_WSS _SOUP_ATOMIC_INTERN_STRING (_SOUP_URI_SCHEME_WSS, "wss") + +/* OSTREECHANGE: import soup-form bits */ +SOUP_AVAILABLE_IN_2_4 +char *soup_form_encode_hash (GHashTable *form_data_set); +SOUP_AVAILABLE_IN_2_4 +char *soup_form_encode_valist (const char *first_field, + va_list args); + +SOUP_VAR gpointer _SOUP_URI_SCHEME_HTTP, _SOUP_URI_SCHEME_HTTPS; +SOUP_VAR gpointer _SOUP_URI_SCHEME_FTP; +SOUP_VAR gpointer _SOUP_URI_SCHEME_FILE, _SOUP_URI_SCHEME_DATA, _SOUP_URI_SCHEME_RESOURCE; +SOUP_VAR gpointer _SOUP_URI_SCHEME_WS, _SOUP_URI_SCHEME_WSS; + +SOUP_AVAILABLE_IN_2_4 +SoupURI *soup_uri_new_with_base (SoupURI *base, + const char *uri_string); +SOUP_AVAILABLE_IN_2_4 +SoupURI *soup_uri_new (const char *uri_string); + +SOUP_AVAILABLE_IN_2_4 +char *soup_uri_to_string (SoupURI *uri, + gboolean just_path_and_query); + +SOUP_AVAILABLE_IN_2_4 +SoupURI *soup_uri_copy (SoupURI *uri); + +SOUP_AVAILABLE_IN_2_4 +gboolean soup_uri_equal (SoupURI *uri1, + SoupURI *uri2); + +SOUP_AVAILABLE_IN_2_4 +void soup_uri_free (SoupURI *uri); + +SOUP_AVAILABLE_IN_2_4 +char *soup_uri_encode (const char *part, + const char *escape_extra); +SOUP_AVAILABLE_IN_2_4 +char *soup_uri_decode (const char *part); +SOUP_AVAILABLE_IN_2_4 +char *soup_uri_normalize (const char *part, + const char *unescape_extra); + +SOUP_AVAILABLE_IN_2_4 +gboolean soup_uri_uses_default_port (SoupURI *uri); + +SOUP_AVAILABLE_IN_2_32 +const char *soup_uri_get_scheme (SoupURI *uri); +SOUP_AVAILABLE_IN_2_4 +void soup_uri_set_scheme (SoupURI *uri, + const char *scheme); +SOUP_AVAILABLE_IN_2_32 +const char *soup_uri_get_user (SoupURI *uri); +SOUP_AVAILABLE_IN_2_4 +void soup_uri_set_user (SoupURI *uri, + const char *user); +SOUP_AVAILABLE_IN_2_32 +const char *soup_uri_get_password (SoupURI *uri); +SOUP_AVAILABLE_IN_2_4 +void soup_uri_set_password (SoupURI *uri, + const char *password); +SOUP_AVAILABLE_IN_2_32 +const char *soup_uri_get_host (SoupURI *uri); +SOUP_AVAILABLE_IN_2_4 +void soup_uri_set_host (SoupURI *uri, + const char *host); +SOUP_AVAILABLE_IN_2_32 +guint soup_uri_get_port (SoupURI *uri); +SOUP_AVAILABLE_IN_2_4 +void soup_uri_set_port (SoupURI *uri, + guint port); +SOUP_AVAILABLE_IN_2_32 +const char *soup_uri_get_path (SoupURI *uri); +SOUP_AVAILABLE_IN_2_4 +void soup_uri_set_path (SoupURI *uri, + const char *path); +SOUP_AVAILABLE_IN_2_32 +const char *soup_uri_get_query (SoupURI *uri); +SOUP_AVAILABLE_IN_2_4 +void soup_uri_set_query (SoupURI *uri, + const char *query); +SOUP_AVAILABLE_IN_2_4 +void soup_uri_set_query_from_form (SoupURI *uri, + GHashTable *form); +SOUP_AVAILABLE_IN_2_4 +void soup_uri_set_query_from_fields (SoupURI *uri, + const char *first_field, + ...) G_GNUC_NULL_TERMINATED; +SOUP_AVAILABLE_IN_2_32 +const char *soup_uri_get_fragment (SoupURI *uri); +SOUP_AVAILABLE_IN_2_4 +void soup_uri_set_fragment (SoupURI *uri, + const char *fragment); + +SOUP_AVAILABLE_IN_2_28 +SoupURI *soup_uri_copy_host (SoupURI *uri); +SOUP_AVAILABLE_IN_2_28 +guint soup_uri_host_hash (gconstpointer key); +SOUP_AVAILABLE_IN_2_28 +gboolean soup_uri_host_equal (gconstpointer v1, + gconstpointer v2); + +#define SOUP_URI_IS_VALID(uri) ((uri) && (uri)->scheme && (uri)->path) +#define SOUP_URI_VALID_FOR_HTTP(uri) ((uri) && ((uri)->scheme == SOUP_URI_SCHEME_HTTP || (uri)->scheme == SOUP_URI_SCHEME_HTTPS) && (uri)->host && (uri)->path) + +G_END_DECLS + +#endif /*SOUP_URI_H*/ diff --git a/src/ostree/ot-remote-builtin-add-cookie.c b/src/ostree/ot-remote-builtin-add-cookie.c index 509c9c7a5b..1c73fec6d6 100644 --- a/src/ostree/ot-remote-builtin-add-cookie.c +++ b/src/ostree/ot-remote-builtin-add-cookie.c @@ -21,7 +21,9 @@ #include "config.h" +#ifndef HAVE_LIBCURL #include +#endif #include "otutil.h" @@ -46,8 +48,6 @@ ot_remote_builtin_add_cookie (int argc, char **argv, GCancellable *cancellable, const char *value; g_autofree char *jar_path = NULL; g_autofree char *cookie_file = NULL; - glnx_unref_object SoupCookieJar *jar = NULL; - SoupCookie *cookie; context = g_option_context_new ("NAME DOMAIN PATH COOKIE_NAME VALUE - Add a cookie to remote"); @@ -70,15 +70,51 @@ ot_remote_builtin_add_cookie (int argc, char **argv, GCancellable *cancellable, cookie_file = g_strdup_printf ("%s.cookies.txt", remote_name); jar_path = g_build_filename (gs_file_get_path_cached (repo->repodir), cookie_file, NULL); - jar = soup_cookie_jar_text_new (jar_path, FALSE); - - /* Pick a silly long expire time, we're just storing the cookies in the - * jar and on pull the jar is read-only so expiry has little actual value */ - cookie = soup_cookie_new (cookie_name, value, domain, path, - SOUP_COOKIE_MAX_AGE_ONE_YEAR * 25); - - /* jar takes ownership of cookie */ - soup_cookie_jar_add_cookie (jar, cookie); +#ifdef HAVE_LIBCURL + { glnx_fd_close int fd = openat (AT_FDCWD, jar_path, O_WRONLY | O_APPEND | O_CREAT, 0644); + g_autofree char *buf = NULL; + g_autoptr(GDateTime) now = NULL; + g_autoptr(GDateTime) expires = NULL; + + if (fd < 0) + { + glnx_set_error_from_errno (error); + return FALSE; + } + + now = g_date_time_new_now_utc (); + expires = g_date_time_add_years (now, 25); + + /* Adapted from soup-cookie-jar-text.c:write_cookie() */ + buf = g_strdup_printf ("%s\t%s\t%s\t%s\t%llu\t%s\t%s\n", + domain, + *domain == '.' ? "TRUE" : "FALSE", + path, + "FALSE", + (long long unsigned)g_date_time_to_unix (expires), + cookie_name, + value); + if (glnx_loop_write (fd, buf, strlen (buf)) < 0) + { + glnx_set_error_from_errno (error); + return FALSE; + } + } +#else + { glnx_unref_object SoupCookieJar *jar = NULL; + SoupCookie *cookie; + + jar = soup_cookie_jar_text_new (jar_path, FALSE); + + /* Pick a silly long expire time, we're just storing the cookies in the + * jar and on pull the jar is read-only so expiry has little actual value */ + cookie = soup_cookie_new (cookie_name, value, domain, path, + SOUP_COOKIE_MAX_AGE_ONE_YEAR * 25); + + /* jar takes ownership of cookie */ + soup_cookie_jar_add_cookie (jar, cookie); + } +#endif return TRUE; } diff --git a/src/ostree/ot-remote-builtin-delete-cookie.c b/src/ostree/ot-remote-builtin-delete-cookie.c index d974dd8da5..c5f8bf7757 100644 --- a/src/ostree/ot-remote-builtin-delete-cookie.c +++ b/src/ostree/ot-remote-builtin-delete-cookie.c @@ -21,14 +21,17 @@ #include "config.h" +#ifndef HAVE_LIBCURL #include +#endif #include "otutil.h" +#include #include "ot-main.h" #include "ot-remote-builtins.h" #include "ostree-repo-private.h" - +#include "ot-remote-cookie-util.h" static GOptionEntry option_entries[] = { { NULL } @@ -45,8 +48,6 @@ ot_remote_builtin_delete_cookie (int argc, char **argv, GCancellable *cancellabl const char *cookie_name; g_autofree char *jar_path = NULL; g_autofree char *cookie_file = NULL; - glnx_unref_object SoupCookieJar *jar = NULL; - GSList *cookies; gboolean found = FALSE; context = g_option_context_new ("NAME DOMAIN PATH COOKIE_NAME- Remote one cookie from remote"); @@ -69,6 +70,52 @@ ot_remote_builtin_delete_cookie (int argc, char **argv, GCancellable *cancellabl cookie_file = g_strdup_printf ("%s.cookies.txt", remote_name); jar_path = g_build_filename (gs_file_get_path_cached (repo->repodir), cookie_file, NULL); +#ifdef HAVE_LIBCURL + { glnx_fd_close int tempfile_fd = -1; + g_autofree char *tempfile_path = NULL; + g_autofree char *dnbuf = NULL; + const char *dn = NULL; + g_autoptr(OtCookieParser) parser = NULL; + + if (!ot_parse_cookies_at (AT_FDCWD, jar_path, &parser, cancellable, error)) + return FALSE; + + dnbuf = g_strdup (jar_path); + dn = dirname (dnbuf); + if (!glnx_open_tmpfile_linkable_at (AT_FDCWD, dn, O_WRONLY | O_CLOEXEC, + &tempfile_fd, &tempfile_path, + error)) + return FALSE; + + while (ot_parse_cookies_next (parser)) + { + if (strcmp (domain, parser->domain) == 0 && + strcmp (path, parser->path) == 0 && + strcmp (cookie_name, parser->name) == 0) + { + found = TRUE; + /* Match, skip writing this one */ + continue; + } + + if (glnx_loop_write (tempfile_fd, parser->line, strlen (parser->line)) < 0 || + glnx_loop_write (tempfile_fd, "\n", 1) < 0) + { + glnx_set_error_from_errno (error); + return FALSE; + } + } + + if (!glnx_link_tmpfile_at (AT_FDCWD, GLNX_LINK_TMPFILE_REPLACE, + tempfile_fd, + tempfile_path, + AT_FDCWD, jar_path, + error)) + return FALSE; + } +#else + { GSList *cookies; + glnx_unref_object SoupCookieJar *jar = NULL; jar = soup_cookie_jar_text_new (jar_path, FALSE); cookies = soup_cookie_jar_all_cookies (jar); @@ -88,6 +135,8 @@ ot_remote_builtin_delete_cookie (int argc, char **argv, GCancellable *cancellabl soup_cookie_free (cookie); cookies = g_slist_delete_link (cookies, cookies); } + } +#endif if (!found) g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, "Cookie not found in jar"); diff --git a/src/ostree/ot-remote-builtin-list-cookies.c b/src/ostree/ot-remote-builtin-list-cookies.c index 1865fb0761..98be0ab8ba 100644 --- a/src/ostree/ot-remote-builtin-list-cookies.c +++ b/src/ostree/ot-remote-builtin-list-cookies.c @@ -21,14 +21,16 @@ #include "config.h" +#ifndef HAVE_LIBCURL #include +#endif #include "otutil.h" #include "ot-main.h" #include "ot-remote-builtins.h" #include "ostree-repo-private.h" - +#include "ot-remote-cookie-util.h" static GOptionEntry option_entries[] = { { NULL } @@ -42,8 +44,6 @@ ot_remote_builtin_list_cookies (int argc, char **argv, GCancellable *cancellable const char *remote_name; g_autofree char *jar_path = NULL; g_autofree char *cookie_file = NULL; - glnx_unref_object SoupCookieJar *jar = NULL; - GSList *cookies; context = g_option_context_new ("NAME - Show remote repository cookies"); @@ -62,25 +62,59 @@ ot_remote_builtin_list_cookies (int argc, char **argv, GCancellable *cancellable cookie_file = g_strdup_printf ("%s.cookies.txt", remote_name); jar_path = g_build_filename (g_file_get_path (repo->repodir), cookie_file, NULL); - jar = soup_cookie_jar_text_new (jar_path, TRUE); - cookies = soup_cookie_jar_all_cookies (jar); +#ifdef HAVE_LIBCURL + { glnx_fd_close int tempfile_fd = -1; + g_autofree char *tempfile_path = NULL; + g_autofree char *dnbuf = NULL; + const char *dn = NULL; + g_autoptr(OtCookieParser) parser = NULL; - while (cookies != NULL) - { - SoupCookie *cookie = cookies->data; - SoupDate *expiry = soup_cookie_get_expires (cookie); - - g_print ("--\n"); - g_print ("Domain: %s\n", soup_cookie_get_domain (cookie)); - g_print ("Path: %s\n", soup_cookie_get_path (cookie)); - g_print ("Name: %s\n", soup_cookie_get_name (cookie)); - g_print ("Secure: %s\n", soup_cookie_get_secure (cookie) ? "yes" : "no"); - g_print ("Expires: %s\n", soup_date_to_string (expiry, SOUP_DATE_COOKIE)); - g_print ("Value: %s\n", soup_cookie_get_value (cookie)); - - soup_cookie_free (cookie); - cookies = g_slist_delete_link (cookies, cookies); - } + if (!ot_parse_cookies_at (AT_FDCWD, jar_path, &parser, cancellable, error)) + return FALSE; + + dnbuf = dirname (g_strdup (jar_path)); + dn = dnbuf; + if (!glnx_open_tmpfile_linkable_at (AT_FDCWD, dn, O_WRONLY | O_CLOEXEC, + &tempfile_fd, &tempfile_path, + error)) + return FALSE; + + while (ot_parse_cookies_next (parser)) + { + g_autoptr(GDateTime) expires = g_date_time_new_from_unix_utc (parser->expiration); + g_autofree char *expires_str = g_date_time_format (expires, "%Y-%m-%d %H:%M:%S +0000"); + + g_print ("--\n"); + g_print ("Domain: %s\n", parser->domain); + g_print ("Path: %s\n", parser->path); + g_print ("Name: %s\n", parser->name); + g_print ("Secure: %s\n", parser->secure); + g_print ("Expires: %s\n", expires_str); + g_print ("Value: %s\n", parser->value); + } + } +#else + { glnx_unref_object SoupCookieJar *jar = soup_cookie_jar_text_new (jar_path, TRUE); + GSList *cookies = soup_cookie_jar_all_cookies (jar); + + while (cookies != NULL) + { + SoupCookie *cookie = cookies->data; + SoupDate *expiry = soup_cookie_get_expires (cookie); + + g_print ("--\n"); + g_print ("Domain: %s\n", soup_cookie_get_domain (cookie)); + g_print ("Path: %s\n", soup_cookie_get_path (cookie)); + g_print ("Name: %s\n", soup_cookie_get_name (cookie)); + g_print ("Secure: %s\n", soup_cookie_get_secure (cookie) ? "yes" : "no"); + g_print ("Expires: %s\n", soup_date_to_string (expiry, SOUP_DATE_COOKIE)); + g_print ("Value: %s\n", soup_cookie_get_value (cookie)); + + soup_cookie_free (cookie); + cookies = g_slist_delete_link (cookies, cookies); + } + } +#endif return TRUE; } diff --git a/src/ostree/ot-remote-cookie-util.c b/src/ostree/ot-remote-cookie-util.c new file mode 100644 index 0000000000..2221dd5210 --- /dev/null +++ b/src/ostree/ot-remote-cookie-util.c @@ -0,0 +1,115 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright (C) 2015 Red Hat, Inc. + * Copyright (C) 2016 Sjoerd Simons + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + */ + +#include "config.h" + +#include "ot-remote-cookie-util.h" + +#include "otutil.h" +#include "ot-main.h" +#include "ot-remote-builtins.h" +#include "ostree-repo-private.h" + +static void +ot_cookie_parser_clear (OtCookieParser *parser) +{ + g_clear_pointer (&parser->domain, (GDestroyNotify)g_free); + g_clear_pointer (&parser->flag, (GDestroyNotify)g_free); + g_clear_pointer (&parser->path, (GDestroyNotify)g_free); + g_clear_pointer (&parser->secure, (GDestroyNotify)g_free); + g_clear_pointer (&parser->name, (GDestroyNotify)g_free); + g_clear_pointer (&parser->value, (GDestroyNotify)g_free); +} + +void +ot_cookie_parser_free (OtCookieParser *parser) +{ + ot_cookie_parser_clear (parser); + g_free (parser->buf); + g_free (parser); +} + +gboolean +ot_parse_cookies_at (int dfd, const char *path, + OtCookieParser **out_parser, + GCancellable *cancellable, + GError **error) +{ + OtCookieParser *parser; + g_autofree char *cookies_content = NULL; + glnx_fd_close int infd = -1; + + infd = openat (dfd, path, O_RDONLY | O_CLOEXEC); + if (infd < 0) + { + if (errno != ENOENT) + { + glnx_set_error_from_errno (error); + return FALSE; + } + } + else + { + cookies_content = glnx_fd_readall_utf8 (infd, NULL, cancellable, error); + if (!cookies_content) + return FALSE; + } + + parser = *out_parser = g_new0 (OtCookieParser, 1); + parser->buf = g_steal_pointer (&cookies_content); + parser->iter = parser->buf; + return TRUE; +} + +gboolean +ot_parse_cookies_next (OtCookieParser *parser) +{ + while (parser->iter) + { + char *iter = parser->iter; + char *next = strchr (iter, '\n'); + + if (next) + { + *next = '\0'; + parser->iter = next + 1; + } + else + parser->iter = NULL; + + ot_cookie_parser_clear (parser); + if (sscanf (iter, "%ms\t%ms\t%ms\t%ms\t%llu\t%ms\t%ms", + &parser->domain, + &parser->flag, + &parser->path, + &parser->secure, + &parser->expiration, + &parser->name, + &parser->value) != 7) + continue; + + parser->line = iter; + return TRUE; + } + + return FALSE; + +} diff --git a/src/ostree/ot-remote-cookie-util.h b/src/ostree/ot-remote-cookie-util.h new file mode 100644 index 0000000000..f258ccfb87 --- /dev/null +++ b/src/ostree/ot-remote-cookie-util.h @@ -0,0 +1,52 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright (C) 2017 Colin Walters + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + */ + +#pragma once + +#include "libglnx.h" + +G_BEGIN_DECLS + +typedef struct OtCookieParser OtCookieParser; +struct OtCookieParser { + char *buf; + char *iter; + + char *line; + char *domain; + char *flag; + char *path; + char *secure; + long long unsigned int expiration; + char *name; + char *value; +}; +void ot_cookie_parser_free (OtCookieParser *parser); +G_DEFINE_AUTOPTR_CLEANUP_FUNC(OtCookieParser, ot_cookie_parser_free) + +gboolean +ot_parse_cookies_at (int dfd, const char *path, + OtCookieParser **out_parser, + GCancellable *cancellable, + GError **error); +gboolean +ot_parse_cookies_next (OtCookieParser *parser); + +G_END_DECLS diff --git a/tests/ci-build.sh b/tests/ci-build.sh index 4b736a3cff..23eacf074f 100755 --- a/tests/ci-build.sh +++ b/tests/ci-build.sh @@ -52,6 +52,10 @@ NULL= # If yes, test failures break the build; if no, they are reported but ignored : "${ci_test_fatal:=yes}" +# ci_configopts: +# Additional args for configure +: "${ci_configopts:=}" + if [ -n "$ci_docker" ]; then exec docker run \ --env=ci_docker="" \ @@ -59,6 +63,7 @@ if [ -n "$ci_docker" ]; then --env=ci_sudo=yes \ --env=ci_test="${ci_test}" \ --env=ci_test_fatal="${ci_test_fatal}" \ + --env=ci_configopts="${ci_configopts}" \ --privileged \ ci-image \ tests/ci-build.sh @@ -81,6 +86,7 @@ make="make -j${ci_parallel} V=1 VERBOSE=1" ../configure \ --enable-always-build-tests \ --enable-installed-tests \ + ${ci_configopts} "$@" ${make} diff --git a/tests/ci-install.sh b/tests/ci-install.sh index dbc86c697b..92f802d906 100755 --- a/tests/ci-install.sh +++ b/tests/ci-install.sh @@ -50,6 +50,9 @@ NULL= # Typical values for ci_distro=fedora might be 25, rawhide : "${ci_suite:=jessie}" +# ci_configopts: Additional arguments for configure +: "${ci_configopts:=}" + if [ $(id -u) = 0 ]; then sudo= else @@ -104,6 +107,7 @@ case "$ci_distro" in libmount-dev \ libselinux1-dev \ libsoup2.4-dev \ + libcurl4-openssl-dev \ procps \ zlib1g-dev \ ${NULL} diff --git a/tests/test-remote-cookies.sh b/tests/test-remote-cookies.sh index 11c201f117..ab2bf26345 100755 --- a/tests/test-remote-cookies.sh +++ b/tests/test-remote-cookies.sh @@ -28,12 +28,9 @@ setup_fake_remote_repo1 "archive-z2" "" \ "--expected-cookies foo=bar --expected-cookies baz=badger" assert_fail (){ - set +e - $@ - if [ $? = 0 ] ; then - echo 1>&2 "$@ did not fail"; exit 1 + if $@; then + (echo 1>&2 "$@ did not fail"; exit 1) fi - set -euo pipefail } cd ${test_tmpdir} @@ -50,12 +47,16 @@ echo "ok, setup done" # Add 2 cookies, pull should succeed now ${CMD_PREFIX} ostree --repo=repo remote add-cookie origin 127.0.0.1 / foo bar ${CMD_PREFIX} ostree --repo=repo remote add-cookie origin 127.0.0.1 / baz badger +assert_file_has_content repo/origin.cookies.txt foo.*bar +assert_file_has_content repo/origin.cookies.txt baz.*badger ${CMD_PREFIX} ostree --repo=repo pull origin main echo "ok, initial cookie pull succeeded" # Delete one cookie, if successful pulls will fail again ${CMD_PREFIX} ostree --repo=repo remote delete-cookie origin 127.0.0.1 / baz badger +assert_file_has_content repo/origin.cookies.txt foo.*bar +assert_not_file_has_content repo/origin.cookies.txt baz.*badger assert_fail ${CMD_PREFIX} ostree --repo=repo pull origin main echo "ok, delete succeeded" @@ -63,6 +64,8 @@ echo "ok, delete succeeded" # Re-add the removed cooking and things succeed again, verified the removal # removed exactly one cookie ${CMD_PREFIX} ostree --repo=repo remote add-cookie origin 127.0.0.1 / baz badger +assert_file_has_content repo/origin.cookies.txt foo.*bar +assert_file_has_content repo/origin.cookies.txt baz.*badger ${CMD_PREFIX} ostree --repo=repo pull origin main echo "ok, second cookie pull succeeded" From b2a264ffc1bf1d5a4f8a31086bd04aefec833f66 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 8 Feb 2017 13:31:14 -0500 Subject: [PATCH 2/4] fixup! libcurl backend --- .redhat-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.redhat-ci.yml b/.redhat-ci.yml index 0e1d686427..11e5a9d6c7 100644 --- a/.redhat-ci.yml +++ b/.redhat-ci.yml @@ -87,7 +87,7 @@ build: --libdir=/usr/lib64 --enable-installed-tests --enable-gtk-doc - --enable-curl + --with-curl tests: - make check From cadda2a0ec04b8c1490c5d9bc1e942096881bef8 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 8 Feb 2017 15:19:48 -0500 Subject: [PATCH 3/4] fixup! libcurl backend --- src/ostree/ot-remote-builtin-add-cookie.c | 52 +------ src/ostree/ot-remote-builtin-delete-cookie.c | 79 +---------- src/ostree/ot-remote-builtin-list-cookies.c | 4 - src/ostree/ot-remote-cookie-util.c | 138 ++++++++++++++++++- src/ostree/ot-remote-cookie-util.h | 13 ++ 5 files changed, 156 insertions(+), 130 deletions(-) diff --git a/src/ostree/ot-remote-builtin-add-cookie.c b/src/ostree/ot-remote-builtin-add-cookie.c index 1c73fec6d6..e415617266 100644 --- a/src/ostree/ot-remote-builtin-add-cookie.c +++ b/src/ostree/ot-remote-builtin-add-cookie.c @@ -21,15 +21,12 @@ #include "config.h" -#ifndef HAVE_LIBCURL -#include -#endif - #include "otutil.h" #include "ot-main.h" #include "ot-remote-builtins.h" #include "ostree-repo-private.h" +#include "ot-remote-cookie-util.h" static GOptionEntry option_entries[] = { @@ -70,51 +67,8 @@ ot_remote_builtin_add_cookie (int argc, char **argv, GCancellable *cancellable, cookie_file = g_strdup_printf ("%s.cookies.txt", remote_name); jar_path = g_build_filename (gs_file_get_path_cached (repo->repodir), cookie_file, NULL); -#ifdef HAVE_LIBCURL - { glnx_fd_close int fd = openat (AT_FDCWD, jar_path, O_WRONLY | O_APPEND | O_CREAT, 0644); - g_autofree char *buf = NULL; - g_autoptr(GDateTime) now = NULL; - g_autoptr(GDateTime) expires = NULL; - - if (fd < 0) - { - glnx_set_error_from_errno (error); - return FALSE; - } - - now = g_date_time_new_now_utc (); - expires = g_date_time_add_years (now, 25); - - /* Adapted from soup-cookie-jar-text.c:write_cookie() */ - buf = g_strdup_printf ("%s\t%s\t%s\t%s\t%llu\t%s\t%s\n", - domain, - *domain == '.' ? "TRUE" : "FALSE", - path, - "FALSE", - (long long unsigned)g_date_time_to_unix (expires), - cookie_name, - value); - if (glnx_loop_write (fd, buf, strlen (buf)) < 0) - { - glnx_set_error_from_errno (error); - return FALSE; - } - } -#else - { glnx_unref_object SoupCookieJar *jar = NULL; - SoupCookie *cookie; - - jar = soup_cookie_jar_text_new (jar_path, FALSE); - - /* Pick a silly long expire time, we're just storing the cookies in the - * jar and on pull the jar is read-only so expiry has little actual value */ - cookie = soup_cookie_new (cookie_name, value, domain, path, - SOUP_COOKIE_MAX_AGE_ONE_YEAR * 25); - - /* jar takes ownership of cookie */ - soup_cookie_jar_add_cookie (jar, cookie); - } -#endif + if (!ot_add_cookie_at (AT_FDCWD, jar_path, domain, path, cookie_name, value, error)) + return FALSE; return TRUE; } diff --git a/src/ostree/ot-remote-builtin-delete-cookie.c b/src/ostree/ot-remote-builtin-delete-cookie.c index c5f8bf7757..6d1b85ad8b 100644 --- a/src/ostree/ot-remote-builtin-delete-cookie.c +++ b/src/ostree/ot-remote-builtin-delete-cookie.c @@ -21,10 +21,6 @@ #include "config.h" -#ifndef HAVE_LIBCURL -#include -#endif - #include "otutil.h" #include @@ -48,7 +44,6 @@ ot_remote_builtin_delete_cookie (int argc, char **argv, GCancellable *cancellabl const char *cookie_name; g_autofree char *jar_path = NULL; g_autofree char *cookie_file = NULL; - gboolean found = FALSE; context = g_option_context_new ("NAME DOMAIN PATH COOKIE_NAME- Remote one cookie from remote"); @@ -70,76 +65,8 @@ ot_remote_builtin_delete_cookie (int argc, char **argv, GCancellable *cancellabl cookie_file = g_strdup_printf ("%s.cookies.txt", remote_name); jar_path = g_build_filename (gs_file_get_path_cached (repo->repodir), cookie_file, NULL); -#ifdef HAVE_LIBCURL - { glnx_fd_close int tempfile_fd = -1; - g_autofree char *tempfile_path = NULL; - g_autofree char *dnbuf = NULL; - const char *dn = NULL; - g_autoptr(OtCookieParser) parser = NULL; - - if (!ot_parse_cookies_at (AT_FDCWD, jar_path, &parser, cancellable, error)) - return FALSE; - - dnbuf = g_strdup (jar_path); - dn = dirname (dnbuf); - if (!glnx_open_tmpfile_linkable_at (AT_FDCWD, dn, O_WRONLY | O_CLOEXEC, - &tempfile_fd, &tempfile_path, - error)) - return FALSE; - - while (ot_parse_cookies_next (parser)) - { - if (strcmp (domain, parser->domain) == 0 && - strcmp (path, parser->path) == 0 && - strcmp (cookie_name, parser->name) == 0) - { - found = TRUE; - /* Match, skip writing this one */ - continue; - } - - if (glnx_loop_write (tempfile_fd, parser->line, strlen (parser->line)) < 0 || - glnx_loop_write (tempfile_fd, "\n", 1) < 0) - { - glnx_set_error_from_errno (error); - return FALSE; - } - } - - if (!glnx_link_tmpfile_at (AT_FDCWD, GLNX_LINK_TMPFILE_REPLACE, - tempfile_fd, - tempfile_path, - AT_FDCWD, jar_path, - error)) - return FALSE; - } -#else - { GSList *cookies; - glnx_unref_object SoupCookieJar *jar = NULL; - jar = soup_cookie_jar_text_new (jar_path, FALSE); - cookies = soup_cookie_jar_all_cookies (jar); - - while (cookies != NULL) - { - SoupCookie *cookie = cookies->data; - - if (!strcmp (domain, soup_cookie_get_domain (cookie)) && - !strcmp (path, soup_cookie_get_path (cookie)) && - !strcmp (cookie_name, soup_cookie_get_name (cookie))) - { - soup_cookie_jar_delete_cookie (jar, cookie); - - found = TRUE; - } - - soup_cookie_free (cookie); - cookies = g_slist_delete_link (cookies, cookies); - } - } -#endif - - if (!found) - g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, "Cookie not found in jar"); + if (!ot_delete_cookie_at (AT_FDCWD, jar_path, domain, path, cookie_name, error)) + return FALSE; - return found; + return TRUE; } diff --git a/src/ostree/ot-remote-builtin-list-cookies.c b/src/ostree/ot-remote-builtin-list-cookies.c index 98be0ab8ba..50a21d4179 100644 --- a/src/ostree/ot-remote-builtin-list-cookies.c +++ b/src/ostree/ot-remote-builtin-list-cookies.c @@ -21,10 +21,6 @@ #include "config.h" -#ifndef HAVE_LIBCURL -#include -#endif - #include "otutil.h" #include "ot-main.h" diff --git a/src/ostree/ot-remote-cookie-util.c b/src/ostree/ot-remote-cookie-util.c index 2221dd5210..8565aa0413 100644 --- a/src/ostree/ot-remote-cookie-util.c +++ b/src/ostree/ot-remote-cookie-util.c @@ -23,6 +23,10 @@ #include "ot-remote-cookie-util.h" +#ifndef HAVE_LIBCURL +#include +#endif + #include "otutil.h" #include "ot-main.h" #include "ot-remote-builtins.h" @@ -111,5 +115,137 @@ ot_parse_cookies_next (OtCookieParser *parser) } return FALSE; - } + +gboolean +ot_add_cookie_at (int dfd, const char *jar_path, + const char *domain, const char *path, + const char *name, const char *value, + GError **error) +{ +#ifdef HAVE_LIBCURL + glnx_fd_close int fd = openat (AT_FDCWD, jar_path, O_WRONLY | O_APPEND | O_CREAT, 0644); + g_autofree char *buf = NULL; + g_autoptr(GDateTime) now = NULL; + g_autoptr(GDateTime) expires = NULL; + + if (fd < 0) + { + glnx_set_error_from_errno (error); + return FALSE; + } + + now = g_date_time_new_now_utc (); + expires = g_date_time_add_years (now, 25); + + /* Adapted from soup-cookie-jar-text.c:write_cookie() */ + buf = g_strdup_printf ("%s\t%s\t%s\t%s\t%llu\t%s\t%s\n", + domain, + *domain == '.' ? "TRUE" : "FALSE", + path, + "FALSE", + (long long unsigned)g_date_time_to_unix (expires), + name, + value); + if (glnx_loop_write (fd, buf, strlen (buf)) < 0) + { + glnx_set_error_from_errno (error); + return FALSE; + } +#else + glnx_unref_object SoupCookieJar *jar = NULL; + SoupCookie *cookie; + + jar = soup_cookie_jar_text_new (jar_path, FALSE); + + /* Pick a silly long expire time, we're just storing the cookies in the + * jar and on pull the jar is read-only so expiry has little actual value */ + cookie = soup_cookie_new (cookie_name, value, domain, path, + SOUP_COOKIE_MAX_AGE_ONE_YEAR * 25); + + /* jar takes ownership of cookie */ + soup_cookie_jar_add_cookie (jar, cookie); +#endif + return TRUE; +} + +gboolean +ot_delete_cookie_at (int dfd, const char *jar_path, + const char *domain, const char *path, + const char *name, + GError **error) +{ + gboolean found = FALSE; +#ifdef HAVE_LIBCURL + glnx_fd_close int tempfile_fd = -1; + g_autofree char *tempfile_path = NULL; + g_autofree char *dnbuf = NULL; + const char *dn = NULL; + g_autoptr(OtCookieParser) parser = NULL; + + if (!ot_parse_cookies_at (dfd, jar_path, &parser, NULL, error)) + return FALSE; + + dnbuf = g_strdup (jar_path); + dn = dirname (dnbuf); + if (!glnx_open_tmpfile_linkable_at (AT_FDCWD, dn, O_WRONLY | O_CLOEXEC, + &tempfile_fd, &tempfile_path, + error)) + return FALSE; + + while (ot_parse_cookies_next (parser)) + { + if (strcmp (domain, parser->domain) == 0 && + strcmp (path, parser->path) == 0 && + strcmp (name, parser->name) == 0) + { + found = TRUE; + /* Match, skip writing this one */ + continue; + } + + if (glnx_loop_write (tempfile_fd, parser->line, strlen (parser->line)) < 0 || + glnx_loop_write (tempfile_fd, "\n", 1) < 0) + { + glnx_set_error_from_errno (error); + return FALSE; + } + } + + if (!glnx_link_tmpfile_at (AT_FDCWD, GLNX_LINK_TMPFILE_REPLACE, + tempfile_fd, + tempfile_path, + AT_FDCWD, jar_path, + error)) + return FALSE; +#else + GSList *cookies; + glnx_unref_object SoupCookieJar *jar = NULL; + + jar = soup_cookie_jar_text_new (jar_path, FALSE); + cookies = soup_cookie_jar_all_cookies (jar); + + while (cookies != NULL) + { + SoupCookie *cookie = cookies->data; + + if (!strcmp (domain, soup_cookie_get_domain (cookie)) && + !strcmp (path, soup_cookie_get_path (cookie)) && + !strcmp (cookie_name, soup_cookie_get_name (cookie))) + { + soup_cookie_jar_delete_cookie (jar, cookie); + + found = TRUE; + } + + soup_cookie_free (cookie); + cookies = g_slist_delete_link (cookies, cookies); + } +#endif + + if (!found) + g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, "Cookie not found in jar"); + + return TRUE; +} + diff --git a/src/ostree/ot-remote-cookie-util.h b/src/ostree/ot-remote-cookie-util.h index f258ccfb87..4f35bb7e5f 100644 --- a/src/ostree/ot-remote-cookie-util.h +++ b/src/ostree/ot-remote-cookie-util.h @@ -49,4 +49,17 @@ ot_parse_cookies_at (int dfd, const char *path, gboolean ot_parse_cookies_next (OtCookieParser *parser); +gboolean +ot_add_cookie_at (int dfd, const char *jar_path, + const char *domain, const char *path, + const char *name, const char *value, + GError **error); + +gboolean +ot_delete_cookie_at (int dfd, const char *jar_path, + const char *domain, const char *path, + const char *name, + GError **error); + + G_END_DECLS From a29e2a0078cd686411c719731678462efd49f4dd Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 8 Feb 2017 15:26:38 -0500 Subject: [PATCH 4/4] fixup! libcurl backend --- src/ostree/ot-remote-builtin-list-cookies.c | 55 +------------ src/ostree/ot-remote-cookie-util.c | 86 ++++++++++++++++++++- src/ostree/ot-remote-cookie-util.h | 27 +------ 3 files changed, 88 insertions(+), 80 deletions(-) diff --git a/src/ostree/ot-remote-builtin-list-cookies.c b/src/ostree/ot-remote-builtin-list-cookies.c index 50a21d4179..1c3924af1d 100644 --- a/src/ostree/ot-remote-builtin-list-cookies.c +++ b/src/ostree/ot-remote-builtin-list-cookies.c @@ -58,59 +58,8 @@ ot_remote_builtin_list_cookies (int argc, char **argv, GCancellable *cancellable cookie_file = g_strdup_printf ("%s.cookies.txt", remote_name); jar_path = g_build_filename (g_file_get_path (repo->repodir), cookie_file, NULL); -#ifdef HAVE_LIBCURL - { glnx_fd_close int tempfile_fd = -1; - g_autofree char *tempfile_path = NULL; - g_autofree char *dnbuf = NULL; - const char *dn = NULL; - g_autoptr(OtCookieParser) parser = NULL; - - if (!ot_parse_cookies_at (AT_FDCWD, jar_path, &parser, cancellable, error)) - return FALSE; - - dnbuf = dirname (g_strdup (jar_path)); - dn = dnbuf; - if (!glnx_open_tmpfile_linkable_at (AT_FDCWD, dn, O_WRONLY | O_CLOEXEC, - &tempfile_fd, &tempfile_path, - error)) - return FALSE; - - while (ot_parse_cookies_next (parser)) - { - g_autoptr(GDateTime) expires = g_date_time_new_from_unix_utc (parser->expiration); - g_autofree char *expires_str = g_date_time_format (expires, "%Y-%m-%d %H:%M:%S +0000"); - - g_print ("--\n"); - g_print ("Domain: %s\n", parser->domain); - g_print ("Path: %s\n", parser->path); - g_print ("Name: %s\n", parser->name); - g_print ("Secure: %s\n", parser->secure); - g_print ("Expires: %s\n", expires_str); - g_print ("Value: %s\n", parser->value); - } - } -#else - { glnx_unref_object SoupCookieJar *jar = soup_cookie_jar_text_new (jar_path, TRUE); - GSList *cookies = soup_cookie_jar_all_cookies (jar); - - while (cookies != NULL) - { - SoupCookie *cookie = cookies->data; - SoupDate *expiry = soup_cookie_get_expires (cookie); - - g_print ("--\n"); - g_print ("Domain: %s\n", soup_cookie_get_domain (cookie)); - g_print ("Path: %s\n", soup_cookie_get_path (cookie)); - g_print ("Name: %s\n", soup_cookie_get_name (cookie)); - g_print ("Secure: %s\n", soup_cookie_get_secure (cookie) ? "yes" : "no"); - g_print ("Expires: %s\n", soup_date_to_string (expiry, SOUP_DATE_COOKIE)); - g_print ("Value: %s\n", soup_cookie_get_value (cookie)); - - soup_cookie_free (cookie); - cookies = g_slist_delete_link (cookies, cookies); - } - } -#endif + if (!ot_list_cookies_at (AT_FDCWD, jar_path, error)) + return FALSE; return TRUE; } diff --git a/src/ostree/ot-remote-cookie-util.c b/src/ostree/ot-remote-cookie-util.c index 8565aa0413..a96038aa74 100644 --- a/src/ostree/ot-remote-cookie-util.c +++ b/src/ostree/ot-remote-cookie-util.c @@ -32,6 +32,31 @@ #include "ot-remote-builtins.h" #include "ostree-repo-private.h" +typedef struct OtCookieParser OtCookieParser; +struct OtCookieParser { + char *buf; + char *iter; + + char *line; + char *domain; + char *flag; + char *path; + char *secure; + long long unsigned int expiration; + char *name; + char *value; +}; +void ot_cookie_parser_free (OtCookieParser *parser); +G_DEFINE_AUTOPTR_CLEANUP_FUNC(OtCookieParser, ot_cookie_parser_free) + +gboolean +ot_parse_cookies_at (int dfd, const char *path, + OtCookieParser **out_parser, + GCancellable *cancellable, + GError **error); +gboolean +ot_parse_cookies_next (OtCookieParser *parser); + static void ot_cookie_parser_clear (OtCookieParser *parser) { @@ -160,7 +185,7 @@ ot_add_cookie_at (int dfd, const char *jar_path, /* Pick a silly long expire time, we're just storing the cookies in the * jar and on pull the jar is read-only so expiry has little actual value */ - cookie = soup_cookie_new (cookie_name, value, domain, path, + cookie = soup_cookie_new (name, value, domain, path, SOUP_COOKIE_MAX_AGE_ONE_YEAR * 25); /* jar takes ownership of cookie */ @@ -231,7 +256,7 @@ ot_delete_cookie_at (int dfd, const char *jar_path, if (!strcmp (domain, soup_cookie_get_domain (cookie)) && !strcmp (path, soup_cookie_get_path (cookie)) && - !strcmp (cookie_name, soup_cookie_get_name (cookie))) + !strcmp (name, soup_cookie_get_name (cookie))) { soup_cookie_jar_delete_cookie (jar, cookie); @@ -249,3 +274,60 @@ ot_delete_cookie_at (int dfd, const char *jar_path, return TRUE; } + +gboolean +ot_list_cookies_at (int dfd, const char *jar_path, GError **error) +{ +#ifdef HAVE_LIBCURL + glnx_fd_close int tempfile_fd = -1; + g_autofree char *tempfile_path = NULL; + g_autofree char *dnbuf = NULL; + const char *dn = NULL; + g_autoptr(OtCookieParser) parser = NULL; + + if (!ot_parse_cookies_at (AT_FDCWD, jar_path, &parser, NULL, error)) + return FALSE; + + dnbuf = dirname (g_strdup (jar_path)); + dn = dnbuf; + if (!glnx_open_tmpfile_linkable_at (AT_FDCWD, dn, O_WRONLY | O_CLOEXEC, + &tempfile_fd, &tempfile_path, + error)) + return FALSE; + + while (ot_parse_cookies_next (parser)) + { + g_autoptr(GDateTime) expires = g_date_time_new_from_unix_utc (parser->expiration); + g_autofree char *expires_str = g_date_time_format (expires, "%Y-%m-%d %H:%M:%S +0000"); + + g_print ("--\n"); + g_print ("Domain: %s\n", parser->domain); + g_print ("Path: %s\n", parser->path); + g_print ("Name: %s\n", parser->name); + g_print ("Secure: %s\n", parser->secure); + g_print ("Expires: %s\n", expires_str); + g_print ("Value: %s\n", parser->value); + } +#else + glnx_unref_object SoupCookieJar *jar = soup_cookie_jar_text_new (jar_path, TRUE); + GSList *cookies = soup_cookie_jar_all_cookies (jar); + + while (cookies != NULL) + { + SoupCookie *cookie = cookies->data; + SoupDate *expiry = soup_cookie_get_expires (cookie); + + g_print ("--\n"); + g_print ("Domain: %s\n", soup_cookie_get_domain (cookie)); + g_print ("Path: %s\n", soup_cookie_get_path (cookie)); + g_print ("Name: %s\n", soup_cookie_get_name (cookie)); + g_print ("Secure: %s\n", soup_cookie_get_secure (cookie) ? "yes" : "no"); + g_print ("Expires: %s\n", soup_date_to_string (expiry, SOUP_DATE_COOKIE)); + g_print ("Value: %s\n", soup_cookie_get_value (cookie)); + + soup_cookie_free (cookie); + cookies = g_slist_delete_link (cookies, cookies); + } +#endif + return TRUE; +} diff --git a/src/ostree/ot-remote-cookie-util.h b/src/ostree/ot-remote-cookie-util.h index 4f35bb7e5f..1bcc0e87fe 100644 --- a/src/ostree/ot-remote-cookie-util.h +++ b/src/ostree/ot-remote-cookie-util.h @@ -24,31 +24,6 @@ G_BEGIN_DECLS -typedef struct OtCookieParser OtCookieParser; -struct OtCookieParser { - char *buf; - char *iter; - - char *line; - char *domain; - char *flag; - char *path; - char *secure; - long long unsigned int expiration; - char *name; - char *value; -}; -void ot_cookie_parser_free (OtCookieParser *parser); -G_DEFINE_AUTOPTR_CLEANUP_FUNC(OtCookieParser, ot_cookie_parser_free) - -gboolean -ot_parse_cookies_at (int dfd, const char *path, - OtCookieParser **out_parser, - GCancellable *cancellable, - GError **error); -gboolean -ot_parse_cookies_next (OtCookieParser *parser); - gboolean ot_add_cookie_at (int dfd, const char *jar_path, const char *domain, const char *path, @@ -61,5 +36,7 @@ ot_delete_cookie_at (int dfd, const char *jar_path, const char *name, GError **error); +gboolean +ot_list_cookies_at (int dfd, const char *jar_path, GError **error); G_END_DECLS