Add the ability to set some additional curlopt values

Add the ability to set some additional curlopt values via .daprc (aka .dodsrc).
This effects both DAP2 and DAP4 protocols.

Related issues:
[1] re: esupport: KOZ-821332
[2] re: github issue Unidata/netcdf4-python#836
[3] re: github issue #1074

1. CURLOPT_BUFFERSIZE: Relevant to [1]. Allow user to set the read/write
buffersizes used by curl.
This is done by adding the following to .daprc (aka .dodsrc):
	HTTP.READ.BUFFERSIZE=n
where n is the buffersize in bytes. There is a built-in (to curl)
limit of 512k for this value.

2. CURLOPT_TCP_KEEPALIVE (and CURLOPT_TCP_KEEPIDLE and CURLOPT_TCP_KEEPINTVL):
Relevant (maybe) to [2] and [3]. Allow the user to turn on KEEPALIVE
This is done by adding the following to .daprc (aka .dodsrc):
	HTTP.KEEPALIVE=on|n/m
If the value is "on", then simply enable default KEEPALIVE. If the value
is n/m, then enable KEEPALIVE and set KEEPIDLE to n and KEEPINTVL to m.

CMakeLists.txt

@@ -812,6 +812,18 @@ IF(ENABLE_DAP)
   #include <curl/curl.h>
   int main() {int x = CURLINFO_HTTP_CONNECTCODE;}" HAVE_CURLINFO_HTTP_CONNECTCODE)
 
+  # Check to see if CURLOPT_BUFFERSIZE is defined.
+  # It is present starting version 7.59
+  CHECK_C_SOURCE_COMPILES("
+  #include <curl/curl.h>
+  int main() {int x = CURLOPT_BUFFERSIZE;}" HAVE_CURLOPT_BUFFERSIZE)
+
+  # Check to see if CURLOPT_TCP_KEEPALIVE is defined.
+  # It is present starting version 7.25
+  CHECK_C_SOURCE_COMPILES("
+  #include <curl/curl.h>
+  int main() {int x = CURLOPT_TCP_KEEPALIVE;}" HAVE_CURLOPT_KEEPALIVE)
+
 ELSE()
   SET(ENABLE_DAP2 OFF)
   SET(ENABLE_DAP4 OFF)

RELEASE_NOTES.md

@@ -7,6 +7,9 @@ This file contains a high-level description of this package's evolution. Release
 
 ## 4.7.0 - TBD
 
+
+* [Enhancement] Allow user to set http read buffersize for DAP2 and  DAP4 using the tag HTTP.READ.BUFFERSIZE in the .daprc file.
+* [Enhancement] Allow user to set http keepalive for DAP2 and  DAP4 using the tag HTTP.KEEPALIVE in the .daprc file (see the OPeNDAP documentation for details).
 * [Enhancement] Support DAP4 remote tests using a new remote test server locatedon the Unidata JetStream project.
 * [Enhancement] Improved the performance of the nc_get/put_vars operations by using the equivalent slab capabilities of hdf5. Result is a significant speedup of these operations.  See [GitHub #1001](https://github.com/Unidata/netcdf-c/pull/1001) for more information.
 * [Enhancement] Expanded the capabilities of `NC_INMEMORY` to support writing and accessing the final modified memory.  See [GitHub #879](https://github.com/Unidata/netcdf-c/pull/879) for more information.

cf

@@ -200,7 +200,7 @@ DISTCHECK_CONFIGURE_FLAGS="$FLAGS"
 export DISTCHECK_CONFIGURE_FLAGS
 
 if test "x$NB" != x -o "x$FAST" = x ; then
-${MAKE} maintainer-clean >/dev/null 2>&1
+${MAKE} distclean >/dev/null 2>&1
 fi
 if test -z "$NB" ; then
   if autoreconf -i --force ; then ok=1; else exit ; fi

config.h.cmake.in

@@ -153,6 +153,12 @@ are set when opening a binary file on Windows. */
 /* Is CURLINFO_HTTP_CODE defined */
 #cmakedefine HAVE_CURLINFO_HTTP_CONNECTCODE 1
 
+/* Is CURLOPT_BUFFERSIZE defined */
+#cmakedefine HAVE_CURLOPT_BUFFERSIZE 1
+
+/* Is CURLOPT_TCP_KEEPALIVE defined */
+#cmakedefine HAVE_CURLOPT_KEEPALIVE 1
+
 /* Is CURLOPT_KEYPASSWD defined */
 #cmakedefine HAVE_CURLOPT_KEYPASSWD 1
 

configure.ac

@@ -688,6 +688,7 @@ AC_C_CONST
 # CURLOPT_KEYPASSWD is not defined until curl version 7.16.4
 # CURLINFO_RESPONSE_CODE is not defined until curl version 7.10.7
 # CURLOPT_CHUNK_BGN_FUNCTION is not defined until curl version 7.21.0
+# CURL_MAX_READ_SIZE is not defined until 7.59
 
 # Save/restore CFLAGS
 SAVECFLAGS="$CFLAGS"
@@ -737,6 +738,28 @@ if test $haveresponsecode = yes; then
   AC_DEFINE([HAVE_CURLINFO_RESPONSE_CODE],[1],[Is CURLINFO_RESPONSE_CODE defined])
 fi
 
+AC_COMPILE_IFELSE([AC_LANG_PROGRAM(
+[#include "curl/curl.h"],
+[[int x = CURLOPT_BUFFERSIZE;]])],
+                   [havecurloption=yes],
+                   [havecurloption=no])
+AC_MSG_CHECKING([whether CURLOPT_BUFFERSIZE is defined])
+AC_MSG_RESULT([${havecurloption}])
+if test $havecurloption = yes; then
+  AC_DEFINE([HAVE_CURLOPT_BUFFERSIZE],[1],[Is CURLOPT_BUFFERSIZE defined])
+fi
+
+AC_COMPILE_IFELSE([AC_LANG_PROGRAM(
+[#include "curl/curl.h"],
+[[int x = CURLOPT_TCP_KEEPALIVE;]])],
+                   [havecurloption=yes],
+                   [havecurloption=no])
+AC_MSG_CHECKING([whether CURLOPT_TCP_KEEPALIVE is defined])
+AC_MSG_RESULT([${havecurloption}])
+if test $havecurloption = yes; then
+  AC_DEFINE([HAVE_CURLOPT_KEEPALIVE],[1],[Is CURLOPT_TCP_KEEPALIVE defined])
+fi
+
 CFLAGS="$SAVECFLAGS"
 
 # Set up libtool.

dap4_test/Makefile.am

@@ -54,7 +54,7 @@ endif #ENABLE_DAP4
 
 EXTRA_DIST = test_parse.sh test_meta.sh test_data.sh \
              test_raw.sh test_remote.sh test_hyrax.sh \
-             d4test_common.sh \
+             tst_curlopt.sh d4test_common.sh \
 	     daptestfiles dmrtestfiles cdltestfiles nctestfiles \
 	     baseline baselineraw baselineremote CMakeLists.txt
 

dap4_test/findtestserver4.c

@@ -21,6 +21,8 @@ url for the server plus the path.
 If serverlist is present, then is should be a comma
 separated list of servers (host+port) to try.
 It defaults to REMOTETESTSERVERS.
+Note that if accessing the standard Unidata test servers,
+then the suffix argument will be either "dts" or "d4ts".
 */
 
 static void

dap4_test/tst_curlopt.sh

@@ -0,0 +1,33 @@
+#!/bin/sh
+
+# These tests are intended to be run only manually.
+# The idea is to gdb ncdump and check that the CURLOPT flags
+# is being processed correctly.
+# As a rule, you will need to set the breakpoint in
+# NCD4_get_rcproperties
+
+if test "x$srcdir" = x ; then srcdir=`pwd`; fi
+. ../test_common.sh
+
+# Figure our server; if none, then just stop
+SVC=`${execdir}/findtestserver4 dap4 d4ts`
+TESTCASE=test_atomic_types.nc
+
+if test "x$SVC" = "x" ; then
+echo "WARNING: Cannot locate test server"
+exit
+fi
+
+URL="[log][dap4][show=fetch]${SVC}/testfiles/${TESTCASE}"
+
+# Create the .daprc file
+rm -f ./.daprc
+echo '# tst_curlopt.sh' > ./.daprc
+echo 'HTTP.READ.BUFFERSIZE=max' >> ./.daprc
+echo 'HTTP.KEEPALIVE=60/60' >> ./.daprc
+gdb --args ${NCDUMP} "${URL}"
+
+# cleanup
+rm -f ./.daprc
+
+exit

docs/OPeNDAP.dox

@@ -691,6 +691,17 @@ follows.
         Type: String representing url to access the proxy: (e.g.http://[username:password@]host[:port])
         Description: Specify the needed information for accessing a proxy.
         Related CURL Flags: CURLOPT_PROXY, CURLOPT_PROXYHOST, CURLOPT_PROXYUSERPWD
+    HTTP.READ.BUFFERSIZE
+        Type: String ("dddddd")
+        Description: Specify the the internal buffer size for curl reads.
+        Related CURL Flags: CURLOPT_BUFFERSIZE, CURL_MAX_WRITE_SIZE (16kB),
+                            CURL_MAX_READ_SIZE (512kB).
+
+    HTTP.KEEPALIVE
+        Type: String ("on|n/m")
+        Description: Specify that TCP KEEPALIVE should be enabled and that the associated idle wait time is n and that the associated repeat interval is m. If the value is of the form is the string "on", then turn on keepalive, but do not set idle or interval.
+        Related CURL Flags: CURLOPT_TCP_KEEPALIVE, CURLOPT_TCP_KEEPIDLE,
+                            CURLOPT_TCP_KEEPINTVL.
 </pre>
 
 The related curl flags line indicates the curl flags modified by this

docs/guide.dox

@@ -1982,6 +1982,9 @@ netCDF file or other external representation, the characters are UTF-8
 encoded <http://en.wikipedia.org/wiki/UTF-8> (note that ASCII is a
 subset of UTF-8). Libraries may use different internal
 representations, for example the Java library uses UTF-16 encoding.
+Note especially that Microsoft Windows does not support UTF-8
+encoding, only ASCII and UTF-16. So using netcdf on Windows may
+cause some problems with respect to objects like file paths.
 
 The netCDF char type contains uninterpreted characters, one character
 per byte.  Typically these contain 7-bit ASCII characters, but the

include/ncrc.h

@@ -46,6 +46,7 @@ extern int NC_rcload(void);
 extern char* NC_rclookup(const char* key, const char* hostport);
 extern void NC_rcclear(NCRCinfo* info);
 extern int NC_set_rcfile(const char* rcfile);
+extern int NC_rcfile_insert(const char* key, const char* value, const char* hostport);
 
 /* From dutil.c (Might later move to e.g. nc.h */
 extern int NC__testurl(const char* path, char** basenamep);

include/netcdf.h

@@ -1961,6 +1961,10 @@ ncrecget(int ncid, long recnum, void **datap);
 EXTERNL int
 ncrecput(int ncid, long recnum, void *const *datap);
 
+/* This function may be called to force the library to
+   cleanup global memory so that memory checkers will not
+   report errors. It is not required, however.
+*/
 EXTERNL int nc_finalize(void);
 
 #if defined(__cplusplus)

libdap4/CMakeLists.txt

@@ -1,4 +1,4 @@
-SET(dap4_SOURCES d4crc32.c d4curlfunctions.c d4curlflags.c d4fix.c d4data.c d4file.c d4parser.c d4meta.c d4varx.c d4dump.c d4swap.c d4chunk.c d4printer.c d4read.c d4http.c d4util.c d4odom.c d4cvt.c d4debug.c ncd4dispatch.c ezxml_extra.c ezxml.c)
+SET(dap4_SOURCES d4crc32.c d4curlfunctions.c d4fix.c d4data.c d4file.c d4parser.c d4meta.c d4varx.c d4dump.c d4swap.c d4chunk.c d4printer.c d4read.c d4http.c d4util.c d4odom.c d4cvt.c d4debug.c ncd4dispatch.c ezxml_extra.c ezxml.c)
 
 add_library(dap4 OBJECT ${dap4_SOURCES})
 

libdap4/Makefile.am

@@ -15,10 +15,11 @@ EXTRA_DIST = CMakeLists.txt
 
 LDADD=
 
+#d4curlflags.c
+
 SRC= \
 d4crc32.c \
 d4curlfunctions.c \
-d4curlflags.c \
 d4fix.c \
 d4data.c \
 d4file.c \

libdap4/d4curlflags.c

@@ -8,6 +8,9 @@
 
 /* Define supported curl flags */
 struct CURLFLAG curlopts[] = { 
+#ifdef HAVE_CURLOPT_BUFFERSIZE
+{"CURLOPT_BUFFERSIZE",CURLOPT_BUFFERSIZE,98,CF_LONG},
+#endif
 {"CURLOPT_PROXYUSERPWD",CURLOPT_PROXYUSERPWD,10006,CF_STRING},
 {"CURLOPT_SSLCERT",CURLOPT_SSLCERT,10025,CF_STRING},
 {"CURLOPT_SSLKEY",CURLOPT_SSLKEY,10087,CF_STRING},
@@ -16,11 +19,14 @@ struct CURLFLAG curlopts[] = {
 #endif
 {"CURLOPT_SSL_VERIFYHOST",CURLOPT_SSL_VERIFYHOST,81,CF_LONG},
 {"CURLOPT_SSL_VERIFYPEER",CURLOPT_SSL_VERIFYPEER,64,CF_LONG},
+#ifdef HAVE_CURLOPT_KEEPALIVE
+{"CURLOPT_TCP_KEEPALIVE",CURLOPT_TCP_KEEPALIVE,213,CF_LONG},
+#endif
 {"CURLOPT_TIMEOUT",CURLOPT_TIMEOUT,13,CF_LONG},
 {"CURLOPT_USERAGENT",CURLOPT_USERAGENT,10018,CF_STRING},
 {"CURLOPT_USERPWD",CURLOPT_USERPWD,10005,CF_STRING},
-{"CURLOPT_VERBOSE",CURLOPT_VERBOSE,41,CF_LONG},
 {"CURLOPT_USE_SSL",CURLOPT_USE_SSL,119,CF_LONG},
+{"CURLOPT_VERBOSE",CURLOPT_VERBOSE,41,CF_LONG},
 {NULL,0}
 };
 

libdap4/d4curlfunctions.c

@@ -17,15 +17,20 @@
 #define CURLOPT_KEYPASSWD CURLOPT_SSLKEYPASSWD
 #endif
 
-#define NETRCFILETAG "HTTP.NETRC"
+#define D4BUFFERSIZE "HTTP.READ.BUFFERSIZE"
+#define D4KEEPALIVE "HTTP.KEEPALIVE"
+
+#ifdef HAVE_CURLOPT_BUFFERSIZE
+#ifndef CURL_MAX_READ_SIZE
+#define CURL_MAX_READ_SIZE  (512*1024)
+#endif
+#endif
 
 #define CHECK(state,flag,value) {if(check(state,flag,(void*)value) != NC_NOERR) {goto done;}}
 
 /* forward */
 static int set_curlflag(NCD4INFO*, int flag);
 static int set_curlopt(NCD4INFO*, int flag, void* value);
-static int set_curl_options(NCD4INFO*);
-static void* cvt(char* value, enum CURLFLAGTYPE type);
 
 static int
 check(NCD4INFO* info, int flag, void* value)
@@ -140,6 +145,23 @@ set_curlflag(NCD4INFO* state, int flag)
     }
     break;
 
+#ifdef HAVE_CURLOPT_BUFFERSIZE
+    case CURLOPT_BUFFERSIZE:
+	CHECK(state, CURLOPT_BUFFERSIZE, (OPTARG)state->curl->buffersize);
+	break;
+#endif
+
+#ifdef HAVE_CURLOPT_KEEPALIVE
+    case CURLOPT_TCP_KEEPALIVE:
+	if(state->curl->keepalive.active != 0)
+	    CHECK(state, CURLOPT_TCP_KEEPALIVE, (OPTARG)1L);
+	if(state->curl->keepalive.idle > 0)
+	    CHECK(state, CURLOPT_TCP_KEEPIDLE, (OPTARG)state->curl->keepalive.idle);
+	if(state->curl->keepalive.interval > 0)
+	    CHECK(state, CURLOPT_TCP_KEEPINTVL, (OPTARG)state->curl->keepalive.interval);
+	break;
+#endif
+
     default:
         nclog(NCLOGWARN,"Attempt to update unexpected curl flag: %d",flag);
 	break;
@@ -177,11 +199,24 @@ NCD4_set_flags_perlink(NCD4INFO* state)
     if(ret == NC_NOERR) ret = set_curlflag(state, CURLOPT_MAXREDIRS);
     if(ret == NC_NOERR) ret = set_curlflag(state, CURLOPT_ERRORBUFFER);
 
+    /* Optional */
+#ifdef HAVE_CURLOPT_BUFFERSIZE
+    if(ret == NC_NOERR && state->curl->buffersize > 0)
+        ret = set_curlflag(state, CURLOPT_BUFFERSIZE);
+#endif
+#ifdef HAVE_CURLOPT_KEEPALIVE
+    if(ret == NC_NOERR && state->curl->keepalive.active != 0)
+        ret = set_curlflag(state, CURLOPT_TCP_KEEPALIVE);
+#endif
+
+#if 0
     /* Set the CURL. options */
     if(ret == NC_NOERR) ret = set_curl_options(state);
+#endif
     return THROW(ret);
 }
 
+#if 0
 /**
 Directly set any options starting with 'CURL.'
 */
@@ -240,6 +275,7 @@ cvt(char* value, enum CURLFLAGTYPE type)
     }
     return NULL;
 }
+#endif
 
 void
 NCD4_curl_debug(NCD4INFO* state)
@@ -268,6 +304,46 @@ NCD4_curl_protocols(NCD4INFO* state)
 #endif
 }
 
+/*
+    Extract state values from .rc file
+*/
+ncerror
+NCD4_get_rcproperties(NCD4INFO* state)
+{
+    ncerror err = NC_NOERR;
+    char* option = NULL;
+#ifdef HAVE_CURLOPT_BUFFERSIZE
+    option = NC_rclookup(D4BUFFERSIZE,state->uri->uri);
+    if(option != NULL && strlen(option) != 0) {
+	long bufsize;
+	if(strcasecmp(option,"max")==0) 
+	    bufsize = CURL_MAX_READ_SIZE;
+	else if(sscanf(option,"%ld",&bufsize) != 1 || bufsize <= 0)
+	    fprintf(stderr,"Illegal %s size\n",D4BUFFERSIZE);
+        state->curl->buffersize = bufsize;
+    }
+#endif
+#ifdef HAVE_CURLOPT_KEEPALIVE
+    option = NC_rclookup(D4KEEPALIVE,state->uri->uri);
+    if(option != NULL && strlen(option) != 0) {
+	/* The keepalive value is of the form 0 or n/m,
+           where n is the idle time and m is the interval time;
+           setting either to zero will prevent that field being set.*/
+	if(strcasecmp(option,"on")==0) {
+	    state->curl->keepalive.active = 1;
+	} else {
+	    unsigned long idle=0;
+	    unsigned long interval=0;
+	    if(sscanf(option,"%lu/%lu",&idle,&interval) != 2)
+	        fprintf(stderr,"Illegal KEEPALIVE VALUE: %s\n",option);
+	    state->curl->keepalive.idle = idle;
+	    state->curl->keepalive.interval = interval;
+	    state->curl->keepalive.active = 1;
+	}
+    }
+#endif
+    return err;
+}
 
 #if 0
 /*

libdap4/d4curlfunctions.h

@@ -33,5 +33,6 @@ extern void NCD4_curl_debug(NCD4INFO* state);
 
 extern struct CURLFLAG* NCD4_curlflagbyname(const char* name);
 extern void NCD4_curl_protocols(NCD4INFO*);
+extern ncerror NCD4_get_rcproperties(NCD4INFO* state);
 
 #endif /*D4CURLFUNCTIONS_H*/

libdap4/d4file.c

@@ -146,6 +146,8 @@ NCD4_open(const char * path, int mode,
 	/* create the connection */
         if((ret=NCD4_curlopen(&curl))!= NC_NOERR) goto done;
 	d4info->curl->curl = curl;
+        /* Load misc rc properties */
+        NCD4_get_rcproperties(d4info);
         if((ret=set_curl_properties(d4info))!= NC_NOERR) goto done;	
         /* Set the one-time curl flags */
         if((ret=NCD4_set_flags_perlink(d4info))!= NC_NOERR) goto done;

libdap4/ncd4dispatch.c

@@ -54,6 +54,7 @@ NCD4_initialize(void)
     globalinit();
     /* Load rc file */
     NC_rcload();    
+
     return THROW(NC_NOERR);
 }
 
@@ -798,6 +799,8 @@ globalinit(void)
 	if(cstat != CURLE_OK)
 	    fprintf(stderr,"curl_global_init failed!\n");
     }
+
+
     return stat;
 }