From 79e38de8400ea384d24f8557bd16139cf16d13c3 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Sun, 26 Aug 2018 17:04:46 -0600 Subject: [PATCH] Add the ability to set some additional curlopt values Add the ability to set some additional curlopt values via .daprc (aka .dodsrc). This effects both DAP2 and DAP4 protocols. Related issues: [1] re: esupport: KOZ-821332 [2] re: github issue https://github.com/Unidata/netcdf4-python/issues/836 [3] re: github issue https://github.com/Unidata/netcdf-c/issues/1074 1. CURLOPT_BUFFERSIZE: Relevant to [1]. Allow user to set the read/write buffersizes used by curl. This is done by adding the following to .daprc (aka .dodsrc): HTTP.READ.BUFFERSIZE=n where n is the buffersize in bytes. There is a built-in (to curl) limit of 512k for this value. 2. CURLOPT_TCP_KEEPALIVE (and CURLOPT_TCP_KEEPIDLE and CURLOPT_TCP_KEEPINTVL): Relevant (maybe) to [2] and [3]. Allow the user to turn on KEEPALIVE This is done by adding the following to .daprc (aka .dodsrc): HTTP.KEEPALIVE=on|n/m If the value is "on", then simply enable default KEEPALIVE. If the value is n/m, then enable KEEPALIVE and set KEEPIDLE to n and KEEPINTVL to m. --- CMakeLists.txt | 12 ++++++ RELEASE_NOTES.md | 3 ++ cf | 2 +- config.h.cmake.in | 6 +++ configure.ac | 23 +++++++++++ dap4_test/Makefile.am | 2 +- dap4_test/findtestserver4.c | 2 + dap4_test/tst_curlopt.sh | 33 +++++++++++++++ docs/OPeNDAP.dox | 11 +++++ docs/guide.dox | 3 ++ include/ncrc.h | 1 + include/netcdf.h | 4 ++ libdap4/CMakeLists.txt | 2 +- libdap4/Makefile.am | 3 +- libdap4/d4curlflags.c | 8 +++- libdap4/d4curlfunctions.c | 82 +++++++++++++++++++++++++++++++++++-- libdap4/d4curlfunctions.h | 1 + libdap4/d4file.c | 2 + libdap4/ncd4dispatch.c | 3 ++ libdap4/ncd4types.h | 6 +++ libdispatch/drc.c | 29 +++++++++++++ ncdap_test/Makefile.am | 4 +- ncdap_test/tst_curlopt.sh | 33 +++++++++++++++ ncdump/ocprint.c | 2 + oc2/CMakeLists.txt | 2 +- oc2/Makefile.am | 1 - oc2/oc.c | 2 + oc2/occurlflags.c | 6 +++ oc2/occurlfunctions.c | 53 ++++++++++++++---------- oc2/ocinternal.c | 63 +++++++++++++++++++++++++++- oc2/ocinternal.h | 9 +++- 31 files changed, 378 insertions(+), 35 deletions(-) create mode 100644 dap4_test/tst_curlopt.sh create mode 100644 ncdap_test/tst_curlopt.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index 8d1a327cc3..1123fb6826 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -812,6 +812,18 @@ IF(ENABLE_DAP) #include int main() {int x = CURLINFO_HTTP_CONNECTCODE;}" HAVE_CURLINFO_HTTP_CONNECTCODE) + # Check to see if CURLOPT_BUFFERSIZE is defined. + # It is present starting version 7.59 + CHECK_C_SOURCE_COMPILES(" + #include + int main() {int x = CURLOPT_BUFFERSIZE;}" HAVE_CURLOPT_BUFFERSIZE) + + # Check to see if CURLOPT_TCP_KEEPALIVE is defined. + # It is present starting version 7.25 + CHECK_C_SOURCE_COMPILES(" + #include + int main() {int x = CURLOPT_TCP_KEEPALIVE;}" HAVE_CURLOPT_KEEPALIVE) + ELSE() SET(ENABLE_DAP2 OFF) SET(ENABLE_DAP4 OFF) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 0360b16368..e6058f8671 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -7,6 +7,9 @@ This file contains a high-level description of this package's evolution. Release ## 4.7.0 - TBD + +* [Enhancement] Allow user to set http read buffersize for DAP2 and DAP4 using the tag HTTP.READ.BUFFERSIZE in the .daprc file. +* [Enhancement] Allow user to set http keepalive for DAP2 and DAP4 using the tag HTTP.KEEPALIVE in the .daprc file (see the OPeNDAP documentation for details). * [Enhancement] Support DAP4 remote tests using a new remote test server locatedon the Unidata JetStream project. * [Enhancement] Improved the performance of the nc_get/put_vars operations by using the equivalent slab capabilities of hdf5. Result is a significant speedup of these operations. See [GitHub #1001](https://github.com/Unidata/netcdf-c/pull/1001) for more information. * [Enhancement] Expanded the capabilities of `NC_INMEMORY` to support writing and accessing the final modified memory. See [GitHub #879](https://github.com/Unidata/netcdf-c/pull/879) for more information. diff --git a/cf b/cf index 48edb83e3e..0b24d06cb5 100644 --- a/cf +++ b/cf @@ -200,7 +200,7 @@ DISTCHECK_CONFIGURE_FLAGS="$FLAGS" export DISTCHECK_CONFIGURE_FLAGS if test "x$NB" != x -o "x$FAST" = x ; then -${MAKE} maintainer-clean >/dev/null 2>&1 +${MAKE} distclean >/dev/null 2>&1 fi if test -z "$NB" ; then if autoreconf -i --force ; then ok=1; else exit ; fi diff --git a/config.h.cmake.in b/config.h.cmake.in index 143474b626..dd3c77f7f3 100644 --- a/config.h.cmake.in +++ b/config.h.cmake.in @@ -153,6 +153,12 @@ are set when opening a binary file on Windows. */ /* Is CURLINFO_HTTP_CODE defined */ #cmakedefine HAVE_CURLINFO_HTTP_CONNECTCODE 1 +/* Is CURLOPT_BUFFERSIZE defined */ +#cmakedefine HAVE_CURLOPT_BUFFERSIZE 1 + +/* Is CURLOPT_TCP_KEEPALIVE defined */ +#cmakedefine HAVE_CURLOPT_KEEPALIVE 1 + /* Is CURLOPT_KEYPASSWD defined */ #cmakedefine HAVE_CURLOPT_KEYPASSWD 1 diff --git a/configure.ac b/configure.ac index 48f88a944f..1555e5b503 100644 --- a/configure.ac +++ b/configure.ac @@ -688,6 +688,7 @@ AC_C_CONST # CURLOPT_KEYPASSWD is not defined until curl version 7.16.4 # CURLINFO_RESPONSE_CODE is not defined until curl version 7.10.7 # CURLOPT_CHUNK_BGN_FUNCTION is not defined until curl version 7.21.0 +# CURL_MAX_READ_SIZE is not defined until 7.59 # Save/restore CFLAGS SAVECFLAGS="$CFLAGS" @@ -737,6 +738,28 @@ if test $haveresponsecode = yes; then AC_DEFINE([HAVE_CURLINFO_RESPONSE_CODE],[1],[Is CURLINFO_RESPONSE_CODE defined]) fi +AC_COMPILE_IFELSE([AC_LANG_PROGRAM( +[#include "curl/curl.h"], +[[int x = CURLOPT_BUFFERSIZE;]])], + [havecurloption=yes], + [havecurloption=no]) +AC_MSG_CHECKING([whether CURLOPT_BUFFERSIZE is defined]) +AC_MSG_RESULT([${havecurloption}]) +if test $havecurloption = yes; then + AC_DEFINE([HAVE_CURLOPT_BUFFERSIZE],[1],[Is CURLOPT_BUFFERSIZE defined]) +fi + +AC_COMPILE_IFELSE([AC_LANG_PROGRAM( +[#include "curl/curl.h"], +[[int x = CURLOPT_TCP_KEEPALIVE;]])], + [havecurloption=yes], + [havecurloption=no]) +AC_MSG_CHECKING([whether CURLOPT_TCP_KEEPALIVE is defined]) +AC_MSG_RESULT([${havecurloption}]) +if test $havecurloption = yes; then + AC_DEFINE([HAVE_CURLOPT_KEEPALIVE],[1],[Is CURLOPT_TCP_KEEPALIVE defined]) +fi + CFLAGS="$SAVECFLAGS" # Set up libtool. diff --git a/dap4_test/Makefile.am b/dap4_test/Makefile.am index 8e09e847d2..3718b9d543 100644 --- a/dap4_test/Makefile.am +++ b/dap4_test/Makefile.am @@ -54,7 +54,7 @@ endif #ENABLE_DAP4 EXTRA_DIST = test_parse.sh test_meta.sh test_data.sh \ test_raw.sh test_remote.sh test_hyrax.sh \ - d4test_common.sh \ + tst_curlopt.sh d4test_common.sh \ daptestfiles dmrtestfiles cdltestfiles nctestfiles \ baseline baselineraw baselineremote CMakeLists.txt diff --git a/dap4_test/findtestserver4.c b/dap4_test/findtestserver4.c index 9ad5e697ef..ee4538f795 100644 --- a/dap4_test/findtestserver4.c +++ b/dap4_test/findtestserver4.c @@ -21,6 +21,8 @@ url for the server plus the path. If serverlist is present, then is should be a comma separated list of servers (host+port) to try. It defaults to REMOTETESTSERVERS. +Note that if accessing the standard Unidata test servers, +then the suffix argument will be either "dts" or "d4ts". */ static void diff --git a/dap4_test/tst_curlopt.sh b/dap4_test/tst_curlopt.sh new file mode 100644 index 0000000000..613864aebc --- /dev/null +++ b/dap4_test/tst_curlopt.sh @@ -0,0 +1,33 @@ +#!/bin/sh + +# These tests are intended to be run only manually. +# The idea is to gdb ncdump and check that the CURLOPT flags +# is being processed correctly. +# As a rule, you will need to set the breakpoint in +# NCD4_get_rcproperties + +if test "x$srcdir" = x ; then srcdir=`pwd`; fi +. ../test_common.sh + +# Figure our server; if none, then just stop +SVC=`${execdir}/findtestserver4 dap4 d4ts` +TESTCASE=test_atomic_types.nc + +if test "x$SVC" = "x" ; then +echo "WARNING: Cannot locate test server" +exit +fi + +URL="[log][dap4][show=fetch]${SVC}/testfiles/${TESTCASE}" + +# Create the .daprc file +rm -f ./.daprc +echo '# tst_curlopt.sh' > ./.daprc +echo 'HTTP.READ.BUFFERSIZE=max' >> ./.daprc +echo 'HTTP.KEEPALIVE=60/60' >> ./.daprc +gdb --args ${NCDUMP} "${URL}" + +# cleanup +rm -f ./.daprc + +exit diff --git a/docs/OPeNDAP.dox b/docs/OPeNDAP.dox index c8ad97cec9..c55713c56c 100644 --- a/docs/OPeNDAP.dox +++ b/docs/OPeNDAP.dox @@ -691,6 +691,17 @@ follows. Type: String representing url to access the proxy: (e.g.http://[username:password@]host[:port]) Description: Specify the needed information for accessing a proxy. Related CURL Flags: CURLOPT_PROXY, CURLOPT_PROXYHOST, CURLOPT_PROXYUSERPWD + HTTP.READ.BUFFERSIZE + Type: String ("dddddd") + Description: Specify the the internal buffer size for curl reads. + Related CURL Flags: CURLOPT_BUFFERSIZE, CURL_MAX_WRITE_SIZE (16kB), + CURL_MAX_READ_SIZE (512kB). + + HTTP.KEEPALIVE + Type: String ("on|n/m") + Description: Specify that TCP KEEPALIVE should be enabled and that the associated idle wait time is n and that the associated repeat interval is m. If the value is of the form is the string "on", then turn on keepalive, but do not set idle or interval. + Related CURL Flags: CURLOPT_TCP_KEEPALIVE, CURLOPT_TCP_KEEPIDLE, + CURLOPT_TCP_KEEPINTVL. The related curl flags line indicates the curl flags modified by this diff --git a/docs/guide.dox b/docs/guide.dox index 2de8484c72..796812a0eb 100644 --- a/docs/guide.dox +++ b/docs/guide.dox @@ -1982,6 +1982,9 @@ netCDF file or other external representation, the characters are UTF-8 encoded (note that ASCII is a subset of UTF-8). Libraries may use different internal representations, for example the Java library uses UTF-16 encoding. +Note especially that Microsoft Windows does not support UTF-8 +encoding, only ASCII and UTF-16. So using netcdf on Windows may +cause some problems with respect to objects like file paths. The netCDF char type contains uninterpreted characters, one character per byte. Typically these contain 7-bit ASCII characters, but the diff --git a/include/ncrc.h b/include/ncrc.h index b1b4065bdb..08720c9fcf 100644 --- a/include/ncrc.h +++ b/include/ncrc.h @@ -46,6 +46,7 @@ extern int NC_rcload(void); extern char* NC_rclookup(const char* key, const char* hostport); extern void NC_rcclear(NCRCinfo* info); extern int NC_set_rcfile(const char* rcfile); +extern int NC_rcfile_insert(const char* key, const char* value, const char* hostport); /* From dutil.c (Might later move to e.g. nc.h */ extern int NC__testurl(const char* path, char** basenamep); diff --git a/include/netcdf.h b/include/netcdf.h index a4b62e009c..4a1b019f14 100644 --- a/include/netcdf.h +++ b/include/netcdf.h @@ -1961,6 +1961,10 @@ ncrecget(int ncid, long recnum, void **datap); EXTERNL int ncrecput(int ncid, long recnum, void *const *datap); +/* This function may be called to force the library to + cleanup global memory so that memory checkers will not + report errors. It is not required, however. +*/ EXTERNL int nc_finalize(void); #if defined(__cplusplus) diff --git a/libdap4/CMakeLists.txt b/libdap4/CMakeLists.txt index 28cdb02912..ce586a498f 100644 --- a/libdap4/CMakeLists.txt +++ b/libdap4/CMakeLists.txt @@ -1,4 +1,4 @@ -SET(dap4_SOURCES d4crc32.c d4curlfunctions.c d4curlflags.c d4fix.c d4data.c d4file.c d4parser.c d4meta.c d4varx.c d4dump.c d4swap.c d4chunk.c d4printer.c d4read.c d4http.c d4util.c d4odom.c d4cvt.c d4debug.c ncd4dispatch.c ezxml_extra.c ezxml.c) +SET(dap4_SOURCES d4crc32.c d4curlfunctions.c d4fix.c d4data.c d4file.c d4parser.c d4meta.c d4varx.c d4dump.c d4swap.c d4chunk.c d4printer.c d4read.c d4http.c d4util.c d4odom.c d4cvt.c d4debug.c ncd4dispatch.c ezxml_extra.c ezxml.c) add_library(dap4 OBJECT ${dap4_SOURCES}) diff --git a/libdap4/Makefile.am b/libdap4/Makefile.am index 5c1c9e555d..3dd113090e 100644 --- a/libdap4/Makefile.am +++ b/libdap4/Makefile.am @@ -15,10 +15,11 @@ EXTRA_DIST = CMakeLists.txt LDADD= +#d4curlflags.c + SRC= \ d4crc32.c \ d4curlfunctions.c \ -d4curlflags.c \ d4fix.c \ d4data.c \ d4file.c \ diff --git a/libdap4/d4curlflags.c b/libdap4/d4curlflags.c index 641fa51e2d..89e09aa10a 100644 --- a/libdap4/d4curlflags.c +++ b/libdap4/d4curlflags.c @@ -8,6 +8,9 @@ /* Define supported curl flags */ struct CURLFLAG curlopts[] = { +#ifdef HAVE_CURLOPT_BUFFERSIZE +{"CURLOPT_BUFFERSIZE",CURLOPT_BUFFERSIZE,98,CF_LONG}, +#endif {"CURLOPT_PROXYUSERPWD",CURLOPT_PROXYUSERPWD,10006,CF_STRING}, {"CURLOPT_SSLCERT",CURLOPT_SSLCERT,10025,CF_STRING}, {"CURLOPT_SSLKEY",CURLOPT_SSLKEY,10087,CF_STRING}, @@ -16,11 +19,14 @@ struct CURLFLAG curlopts[] = { #endif {"CURLOPT_SSL_VERIFYHOST",CURLOPT_SSL_VERIFYHOST,81,CF_LONG}, {"CURLOPT_SSL_VERIFYPEER",CURLOPT_SSL_VERIFYPEER,64,CF_LONG}, +#ifdef HAVE_CURLOPT_KEEPALIVE +{"CURLOPT_TCP_KEEPALIVE",CURLOPT_TCP_KEEPALIVE,213,CF_LONG}, +#endif {"CURLOPT_TIMEOUT",CURLOPT_TIMEOUT,13,CF_LONG}, {"CURLOPT_USERAGENT",CURLOPT_USERAGENT,10018,CF_STRING}, {"CURLOPT_USERPWD",CURLOPT_USERPWD,10005,CF_STRING}, -{"CURLOPT_VERBOSE",CURLOPT_VERBOSE,41,CF_LONG}, {"CURLOPT_USE_SSL",CURLOPT_USE_SSL,119,CF_LONG}, +{"CURLOPT_VERBOSE",CURLOPT_VERBOSE,41,CF_LONG}, {NULL,0} }; diff --git a/libdap4/d4curlfunctions.c b/libdap4/d4curlfunctions.c index f4eafbaf26..7279d7289e 100644 --- a/libdap4/d4curlfunctions.c +++ b/libdap4/d4curlfunctions.c @@ -17,15 +17,20 @@ #define CURLOPT_KEYPASSWD CURLOPT_SSLKEYPASSWD #endif -#define NETRCFILETAG "HTTP.NETRC" +#define D4BUFFERSIZE "HTTP.READ.BUFFERSIZE" +#define D4KEEPALIVE "HTTP.KEEPALIVE" + +#ifdef HAVE_CURLOPT_BUFFERSIZE +#ifndef CURL_MAX_READ_SIZE +#define CURL_MAX_READ_SIZE (512*1024) +#endif +#endif #define CHECK(state,flag,value) {if(check(state,flag,(void*)value) != NC_NOERR) {goto done;}} /* forward */ static int set_curlflag(NCD4INFO*, int flag); static int set_curlopt(NCD4INFO*, int flag, void* value); -static int set_curl_options(NCD4INFO*); -static void* cvt(char* value, enum CURLFLAGTYPE type); static int check(NCD4INFO* info, int flag, void* value) @@ -140,6 +145,23 @@ set_curlflag(NCD4INFO* state, int flag) } break; +#ifdef HAVE_CURLOPT_BUFFERSIZE + case CURLOPT_BUFFERSIZE: + CHECK(state, CURLOPT_BUFFERSIZE, (OPTARG)state->curl->buffersize); + break; +#endif + +#ifdef HAVE_CURLOPT_KEEPALIVE + case CURLOPT_TCP_KEEPALIVE: + if(state->curl->keepalive.active != 0) + CHECK(state, CURLOPT_TCP_KEEPALIVE, (OPTARG)1L); + if(state->curl->keepalive.idle > 0) + CHECK(state, CURLOPT_TCP_KEEPIDLE, (OPTARG)state->curl->keepalive.idle); + if(state->curl->keepalive.interval > 0) + CHECK(state, CURLOPT_TCP_KEEPINTVL, (OPTARG)state->curl->keepalive.interval); + break; +#endif + default: nclog(NCLOGWARN,"Attempt to update unexpected curl flag: %d",flag); break; @@ -177,11 +199,24 @@ NCD4_set_flags_perlink(NCD4INFO* state) if(ret == NC_NOERR) ret = set_curlflag(state, CURLOPT_MAXREDIRS); if(ret == NC_NOERR) ret = set_curlflag(state, CURLOPT_ERRORBUFFER); + /* Optional */ +#ifdef HAVE_CURLOPT_BUFFERSIZE + if(ret == NC_NOERR && state->curl->buffersize > 0) + ret = set_curlflag(state, CURLOPT_BUFFERSIZE); +#endif +#ifdef HAVE_CURLOPT_KEEPALIVE + if(ret == NC_NOERR && state->curl->keepalive.active != 0) + ret = set_curlflag(state, CURLOPT_TCP_KEEPALIVE); +#endif + +#if 0 /* Set the CURL. options */ if(ret == NC_NOERR) ret = set_curl_options(state); +#endif return THROW(ret); } +#if 0 /** Directly set any options starting with 'CURL.' */ @@ -240,6 +275,7 @@ cvt(char* value, enum CURLFLAGTYPE type) } return NULL; } +#endif void NCD4_curl_debug(NCD4INFO* state) @@ -268,6 +304,46 @@ NCD4_curl_protocols(NCD4INFO* state) #endif } +/* + Extract state values from .rc file +*/ +ncerror +NCD4_get_rcproperties(NCD4INFO* state) +{ + ncerror err = NC_NOERR; + char* option = NULL; +#ifdef HAVE_CURLOPT_BUFFERSIZE + option = NC_rclookup(D4BUFFERSIZE,state->uri->uri); + if(option != NULL && strlen(option) != 0) { + long bufsize; + if(strcasecmp(option,"max")==0) + bufsize = CURL_MAX_READ_SIZE; + else if(sscanf(option,"%ld",&bufsize) != 1 || bufsize <= 0) + fprintf(stderr,"Illegal %s size\n",D4BUFFERSIZE); + state->curl->buffersize = bufsize; + } +#endif +#ifdef HAVE_CURLOPT_KEEPALIVE + option = NC_rclookup(D4KEEPALIVE,state->uri->uri); + if(option != NULL && strlen(option) != 0) { + /* The keepalive value is of the form 0 or n/m, + where n is the idle time and m is the interval time; + setting either to zero will prevent that field being set.*/ + if(strcasecmp(option,"on")==0) { + state->curl->keepalive.active = 1; + } else { + unsigned long idle=0; + unsigned long interval=0; + if(sscanf(option,"%lu/%lu",&idle,&interval) != 2) + fprintf(stderr,"Illegal KEEPALIVE VALUE: %s\n",option); + state->curl->keepalive.idle = idle; + state->curl->keepalive.interval = interval; + state->curl->keepalive.active = 1; + } + } +#endif + return err; +} #if 0 /* diff --git a/libdap4/d4curlfunctions.h b/libdap4/d4curlfunctions.h index 42c47fe193..52fe1b5d26 100644 --- a/libdap4/d4curlfunctions.h +++ b/libdap4/d4curlfunctions.h @@ -33,5 +33,6 @@ extern void NCD4_curl_debug(NCD4INFO* state); extern struct CURLFLAG* NCD4_curlflagbyname(const char* name); extern void NCD4_curl_protocols(NCD4INFO*); +extern ncerror NCD4_get_rcproperties(NCD4INFO* state); #endif /*D4CURLFUNCTIONS_H*/ diff --git a/libdap4/d4file.c b/libdap4/d4file.c index 13e61726db..c1d04a4604 100644 --- a/libdap4/d4file.c +++ b/libdap4/d4file.c @@ -146,6 +146,8 @@ NCD4_open(const char * path, int mode, /* create the connection */ if((ret=NCD4_curlopen(&curl))!= NC_NOERR) goto done; d4info->curl->curl = curl; + /* Load misc rc properties */ + NCD4_get_rcproperties(d4info); if((ret=set_curl_properties(d4info))!= NC_NOERR) goto done; /* Set the one-time curl flags */ if((ret=NCD4_set_flags_perlink(d4info))!= NC_NOERR) goto done; diff --git a/libdap4/ncd4dispatch.c b/libdap4/ncd4dispatch.c index 140d0e2268..8aaf82aeeb 100644 --- a/libdap4/ncd4dispatch.c +++ b/libdap4/ncd4dispatch.c @@ -54,6 +54,7 @@ NCD4_initialize(void) globalinit(); /* Load rc file */ NC_rcload(); + return THROW(NC_NOERR); } @@ -798,6 +799,8 @@ globalinit(void) if(cstat != CURLE_OK) fprintf(stderr,"curl_global_init failed!\n"); } + + return stat; } diff --git a/libdap4/ncd4types.h b/libdap4/ncd4types.h index fa74110f46..7379f13f7d 100644 --- a/libdap4/ncd4types.h +++ b/libdap4/ncd4types.h @@ -273,6 +273,12 @@ struct NCD4curl { long httpcode; char errorbuf[CURL_ERROR_SIZE]; /* CURLOPT_ERRORBUFFER*/ } errdata; + struct { + int active; /* Activate keepalive? */ + long idle; /* KEEPIDLE value */ + long interval; /* KEEPINTVL value */ + } keepalive; /* keepalive info */ + long buffersize; /* read buffer size */ }; /**************************************************/ diff --git a/libdispatch/drc.c b/libdispatch/drc.c index 644c83df8d..82fc07a1e2 100644 --- a/libdispatch/drc.c +++ b/libdispatch/drc.c @@ -423,6 +423,35 @@ rcsearch(const char* prefix, const char* rcname, char** pathp) return (ret); } +int +NC_rcfile_insert(const char* key, const char* value, const char* hostport) +{ + int ret = NC_NOERR; + /* See if this key already defined */ + struct NCTriple* triple = NULL; + NClist* rc = ncrc_globalstate.rcinfo.triples; + + if(rc == NULL) { + rc = nclistnew(); + if(rc == NULL) {ret = NC_ENOMEM; goto done;} + } + triple = rclocate(key,hostport); + if(triple == NULL) { + triple = (NCTriple*)calloc(1,sizeof(NCTriple)); + if(triple == NULL) {ret = NC_ENOMEM; goto done;} + triple->key = strdup(key); + triple->value = NULL; + rctrim(triple->key); + triple->host = (hostport == NULL ? NULL : strdup(hostport)); + nclistpush(rc,triple); + } + if(triple->value != NULL) free(triple->value); + triple->value = strdup(value); + rctrim(triple->value); +done: + return ret; +} + #ifdef D4DEBUG static void storedump(char* msg, NClist* triples) diff --git a/ncdap_test/Makefile.am b/ncdap_test/Makefile.am index 9f64f0ac1e..32d7a80a46 100644 --- a/ncdap_test/Makefile.am +++ b/ncdap_test/Makefile.am @@ -83,7 +83,7 @@ EXTRA_DIST = tst_ncdap3.sh \ tst_longremote3.sh \ tst_filelists.sh tst_urls.sh tst_utils.sh \ t_dap.c CMakeLists.txt tst_formatx.sh testauth.sh testurl.sh \ - t_ncf330.c tst_ber.sh + t_ncf330.c tst_ber.sh tst_curlopt.sh CLEANFILES = test_varm3 test_cvt3 file_results/* remote_results/* datadds* t_dap3a test_nstride_cached *.exe # This should only be left behind if using parallel io @@ -100,6 +100,8 @@ BUILT_SOURCES = .dodsrc .dodsrc: echo "#DODSRC" >.dodsrc + echo "HTTP.READ.BUFFERSIZE=max" >>.dodsrc + echo "HTTP.KEEPALIVE=60/60" >>.dodsrc clean-local: clean-local-check diff --git a/ncdap_test/tst_curlopt.sh b/ncdap_test/tst_curlopt.sh new file mode 100644 index 0000000000..4c1b36a215 --- /dev/null +++ b/ncdap_test/tst_curlopt.sh @@ -0,0 +1,33 @@ +#!/bin/sh + +# These tests are intended to be run only manually. +# The idea is to gdb ncdump and check that the CURLOPT flags +# is being processed correctly. +# As a rule, you will need to set the breakpoint in +# ocget_rcproperties + +if test "x$srcdir" = x ; then srcdir=`pwd`; fi +. ../test_common.sh + +# Figure our server; if none, then just stop +SVC=`${execdir}/findtestserver dap2 dts` +TESTCASE=test.01 + +if test "x$SVC" = "x" ; then +echo "WARNING: Cannot locate test server" +exit +fi + +URL="[log]${SVC}/${TESTCASE}" + +# Create the .daprc file +rm -f ./.daprc +echo '# tst_curlopt.sh' > ./.daprc +echo 'HTTP.READ.BUFFERSIZE=max' >> ./.daprc +echo 'HTTP.KEEPALIVE=60/60' >> ./.daprc +gdb --args ${NCDUMP} "${URL}" + +# cleanup +rm -f ./.daprc + +exit diff --git a/ncdump/ocprint.c b/ncdump/ocprint.c index 215ef43293..dd4fde3382 100644 --- a/ncdump/ocprint.c +++ b/ncdump/ocprint.c @@ -432,8 +432,10 @@ processdata(OCflags flags) if(ocopt.netrc) oc_set_netrc(link,ocopt.netrc); +#if 0 if(ocopt.selfsigned) oc_set_curlopt(link,"CURLOPT_VERIFYPEER", (void*)0L); +#endif if(ocopt.optdas) { ocstat = oc_fetch(link,ocopt.url->query,OCDAS,0,&dasroot); diff --git a/oc2/CMakeLists.txt b/oc2/CMakeLists.txt index c0d6233c11..92523aa54b 100644 --- a/oc2/CMakeLists.txt +++ b/oc2/CMakeLists.txt @@ -1,4 +1,4 @@ -SET(oc_SOURCES oc.c daplex.c dapparse.c dapy.c occompile.c occurlfunctions.c ocdata.c ocdebug.c ocdump.c ocinternal.c ocnode.c ochttp.c ocread.c ocutil.c occurlflags.c xxdr.c) +SET(oc_SOURCES oc.c daplex.c dapparse.c dapy.c occompile.c occurlfunctions.c ocdata.c ocdebug.c ocdump.c ocinternal.c ocnode.c ochttp.c ocread.c ocutil.c xxdr.c) add_library(oc2 OBJECT ${oc_SOURCES}) diff --git a/oc2/Makefile.am b/oc2/Makefile.am index 5a38362f9d..b22889b6b8 100644 --- a/oc2/Makefile.am +++ b/oc2/Makefile.am @@ -18,7 +18,6 @@ ocdata.c ocdebug.c ocdump.c \ ocinternal.c ocnode.c \ ochttp.c \ ocread.c ocutil.c \ -occurlflags.c \ xxdr.c HDRS=oc.h ocx.h \ diff --git a/oc2/oc.c b/oc2/oc.c index 5525aa8a5e..821093849f 100644 --- a/oc2/oc.c +++ b/oc2/oc.c @@ -1962,6 +1962,7 @@ oc_dds_free(OCobject link, OCobject dds0) /**************************************************/ /* Curl specific options */ +#if 0 /*!\defgroup Curl Curl-specifi Procedures @{*/ @@ -1992,6 +1993,7 @@ oc_set_curlopt(OClink link, const char* option, void* value) return OCTHROW(OC_ECURL); return OCTHROW(ocset_curlopt(state,f->flag,value)); } +#endif /*! Set the absolute path to use for the .netrc file diff --git a/oc2/occurlflags.c b/oc2/occurlflags.c index 89d49b9e34..d530fc4d56 100644 --- a/oc2/occurlflags.c +++ b/oc2/occurlflags.c @@ -15,6 +15,9 @@ static struct OCCURLFLAG** flagindices = NULL; /* for radix access */ /* Define supported curl flags */ static struct OCCURLFLAG oc_curlflags[] = { +#ifdef HAVE_CURLOPT_BUFFERSIZE +{"CURLOPT_BUFFERSIZE",CURLOPT_BUFFERSIZE,98,CF_LONG}, +#endif {"CURLOPT_CAINFO",CURLOPT_CAINFO,10065,CF_STRING}, {"CURLOPT_CAPATH",CURLOPT_CAPATH,10097,CF_STRING}, {"CURLOPT_COOKIEFILE",CURLOPT_COOKIEFILE,10031,CF_STRING}, @@ -38,6 +41,9 @@ static struct OCCURLFLAG oc_curlflags[] = { {"CURLOPT_SSLKEY",CURLOPT_SSLKEY,10087,CF_STRING}, {"CURLOPT_SSL_VERIFYHOST",CURLOPT_SSL_VERIFYHOST,81,CF_LONG}, {"CURLOPT_SSL_VERIFYPEER",CURLOPT_SSL_VERIFYPEER,64,CF_LONG}, +#ifdef HAVE_CURLOPT_KEEPALIVE +{"CURLOPT_TCP_KEEPALIVE",CURLOPT_TCP_KEEPALIVE,213,CF_LONG}, +#endif {"CURLOPT_TIMEOUT",CURLOPT_TIMEOUT,13,CF_LONG}, {"CURLOPT_USERAGENT",CURLOPT_USERAGENT,10018,CF_STRING}, {"CURLOPT_USERPWD",CURLOPT_USERPWD,10005,CF_STRING}, diff --git a/oc2/occurlfunctions.c b/oc2/occurlfunctions.c index 53cc0274ca..0832f782df 100644 --- a/oc2/occurlfunctions.c +++ b/oc2/occurlfunctions.c @@ -12,27 +12,16 @@ /* Mnemonic */ #define OPTARG void* +/* Define some .rc file entries of interest*/ #define NETRCFILETAG "HTTP.NETRC" +/* Check return value */ #define CHECK(state,flag,value) {if(check(state,flag,(void*)value) != OC_NOERR) {goto done;}} static OCerror check(OCstate* state, int flag, void* value) { OCerror stat = ocset_curlopt(state,flag,value); -#ifdef OCDEBUG - long l = (long)value; - const char* name = occurlflagbyflag(flag)->name; - if(l <= 1000) { - OCDBG2("%s=%ld",name,l); - } else { - char show[65]; - char* s = (char*)value; - strncpy(show,s,sizeof(show)-1); - show[sizeof(show)-1] = '\0'; - OCDBG2("%s=%s",name,show); - } -#endif return stat; } @@ -153,12 +142,26 @@ ocset_curlflag(OCstate* state, int flag) } break; - default: { - struct OCCURLFLAG* f = occurlflagbyflag(flag); - if(f != NULL) - nclog(NCLOGWARN,"Attempt to update unexpected curl flag: %s", - f->name); - } break; +#ifdef HAVE_CURLOPT_BUFFERSIZE + case CURLOPT_BUFFERSIZE: + CHECK(state, CURLOPT_BUFFERSIZE, (OPTARG)state->curlbuffersize); + break; +#endif + +#ifdef HAVE_CURLOPT_KEEPALIVE + case CURLOPT_TCP_KEEPALIVE: + if(state->curlkeepalive.active != 0) + CHECK(state, CURLOPT_TCP_KEEPALIVE, (OPTARG)1L); + if(state->curlkeepalive.idle > 0) + CHECK(state, CURLOPT_TCP_KEEPIDLE, (OPTARG)state->curlkeepalive.idle); + if(state->curlkeepalive.interval > 0) + CHECK(state, CURLOPT_TCP_KEEPINTVL, (OPTARG)state->curlkeepalive.interval); + break; +#endif + + default: + nclog(NCLOGWARN,"Attempt to update unexpected curl flag: %d",flag); + break; } done: return stat; @@ -194,6 +197,16 @@ ocset_flags_perlink(OCstate* state) if(stat == OC_NOERR) stat = ocset_curlflag(state, CURLOPT_FOLLOWLOCATION); if(stat == OC_NOERR) stat = ocset_curlflag(state, CURLOPT_MAXREDIRS); if(stat == OC_NOERR) stat = ocset_curlflag(state, CURLOPT_ERRORBUFFER); + +#ifdef HAVE_CURLOPT_BUFFERSIZE + /* Optional */ + if(stat == OC_NOERR && state->curlbuffersize > 0) + stat = ocset_curlflag(state, CURLOPT_BUFFERSIZE); +#endif +#ifdef HAVE_CURLOPT_KEEPALIVE + if(stat == NC_NOERR && state->curlkeepalive.active != 0) + stat = ocset_curlflag(state, CURLOPT_TCP_KEEPALIVE); +#endif return stat; } @@ -232,5 +245,3 @@ oc_curl_protocols(OCstate* state) state->auth.curlflags.proto_https=1; } } - - diff --git a/oc2/ocinternal.c b/oc2/ocinternal.c index e7305edf0f..947d17c3cd 100644 --- a/oc2/ocinternal.c +++ b/oc2/ocinternal.c @@ -36,6 +36,15 @@ #define CLBRACE '{' #define CRBRACE '}' +#define OCBUFFERSIZE "HTTP.READ.BUFFERSIZE" +#define OCKEEPALIVE "HTTP.KEEPALIVE" + +#ifdef HAVE_CURLOPT_BUFFERSIZE +#ifndef CURL_MAX_READ_SIZE +#define CURL_MAX_READ_SIZE (512*1024) +#endif +#endif + /*Forward*/ static OCerror ocextractddsinmemory(OCstate*,OCtree*,int); static OCerror ocextractddsinfile(OCstate*,OCtree*,int); @@ -45,6 +54,7 @@ static int dataError(XXDR* xdrs, OCstate*); static void ocremovefile(const char* path); static OCerror ocset_curlproperties(OCstate*); +static OCerror ocget_rcproperties(OCstate*); extern OCnode* makeunlimiteddimension(void); @@ -76,6 +86,9 @@ ocinternalinitialize(void) /* Compute some xdr related flags */ xxdr_init(); + /* Make sure that the rc file has been loaded */ + (void)NC_rcload(); + return OCTHROW(stat); } @@ -113,7 +126,11 @@ ocopen(OCstate** statep, const char* url) /* Initialize auth info from rc file */ stat = NC_authsetup(&state->auth, state->uri); - /* capture curl properties for this link from rc file1*/ + /* Initialize misc info from rc file */ + stat = ocget_rcproperties(state); + + /* Apply curl properties for this link; + assumes state has been initialized */ stat = ocset_curlproperties(state); if(stat != OC_NOERR) goto fail; @@ -491,7 +508,49 @@ ocupdatelastmodifieddata(OCstate* state) } /* - Set curl properties for link based on rc files etc. + Extract state values from .rc file +*/ +static OCerror +ocget_rcproperties(OCstate* state) +{ + OCerror ocerr = OC_NOERR; + char* option = NULL; +#ifdef HAVE_CURLOPT_BUFFERSIZE + option = NC_rclookup(OCBUFFERSIZE,state->uri->uri); + if(option != NULL && strlen(option) != 0) { + long bufsize; + if(strcasecmp(option,"max")==0) + bufsize = CURL_MAX_READ_SIZE; + else if(sscanf(option,"%ld",&bufsize) != 1 || bufsize <= 0) + fprintf(stderr,"Illegal %s size\n",OCBUFFERSIZE); + state->curlbuffersize = bufsize; + } +#endif +#ifdef HAVE_CURLOPT_KEEPALIVE + option = NC_rclookup(OCKEEPALIVE,state->uri->uri); + if(option != NULL && strlen(option) != 0) { + /* The keepalive value is of the form 0 or n/m, + where n is the idle time and m is the interval time; + setting either to zero will prevent that field being set. */ + if(strcasecmp(option,"on")==0) { + state->curlkeepalive.active = 1; + } else { + unsigned long idle=0; + unsigned long interval=0; + if(sscanf(option,"%lu/%lu",&idle,&interval) != 2) + fprintf(stderr,"Illegal KEEPALIVE VALUE: %s\n",option); + state->curlkeepalive.idle = idle; + state->curlkeepalive.interval = interval; + state->curlkeepalive.active = 1; + } + } +#endif + return ocerr; +} + + +/* + Set curl properties for link based on fields in the state. */ static OCerror ocset_curlproperties(OCstate* state) diff --git a/oc2/ocinternal.h b/oc2/ocinternal.h index 9cef372f83..f0e4642236 100644 --- a/oc2/ocinternal.h +++ b/oc2/ocinternal.h @@ -139,8 +139,8 @@ typedef struct OCheader { #define DFALTUSERAGENT "oc" #endif +#if 0 /* Hold known curl flags */ - enum OCCURLFLAGTYPE {CF_UNKNOWN=0,CF_OTHER=1,CF_STRING=2,CF_LONG=3}; struct OCCURLFLAG { const char* name; @@ -148,6 +148,7 @@ struct OCCURLFLAG { int value; enum OCCURLFLAGTYPE type; }; +#endif struct OCTriplestore { int ntriples; @@ -176,6 +177,12 @@ struct OCstate { NCauth auth; /* curl auth data */ long ddslastmodified; long datalastmodified; + long curlbuffersize; + struct { + int active; /* Activate keepalive? */ + long idle; /* KEEPIDLE value */ + long interval; /* KEEPINTVL value */ + } curlkeepalive; /* keepalive info */ }; /*! OCtree holds extra state info about trees */