Update opts.py to allow for 'None' timeout-value in client_options (#…

…1541)

Rally sets a default timeout (60s) for every http api call to elasticsearch.

For long running queries this timeout introduces some problems, because rally will fail with a client 
timeout (which is only traceable via the logs). To work around this timeout issues, one must set the timeout to an 
arbitrary high value to account for any possible timeouts.

Rally actually supports the following three options for values set in the --client-options parameter: int, float or boolean. 
For the timeout value an additional None is required to allow not specifing a timeout at all.
This is due to the underlying urllib3 which requires either an int, float or None value as a timeout as shown by the 
following error message: .../urllib3/util/timeout.py": ValueError: Timeout value connect was None, but it must be an int, 
float or None. So far only int and float are usable from the commandline because Nonecannot be passed via the 
--client-options parameter.

This pull request fixes this issue by added the needed functionality to the opts.py module. 
A to_none() function is added which parses the String "None" to the pythonic None and is called in the kv_to_map() 
function which parses the --client-options parameters.

docs/command_line_reference.rst

@@ -670,8 +670,9 @@ We support the following data types:
 * Strings: Have to be enclosed in single quotes. Example: ``ca_certs:'/path/to/CA_certs'``
 * Numbers: There is nothing special about numbers. Example: ``sniffer_timeout:60``
 * Booleans: Specify either ``true`` or ``false``. Example: ``use_ssl:true``
+* None: Specify ``None`` without quotes.``'None'`` will be treated as string. Example: ``timeout:None``
 
-Default value: ``timeout:60`` (applies any time ``timeout`` is not specified)
+Default value: ``timeout:60`` (applies any time ``timeout`` is not specified. Set ``timeout:None`` to omit any timeouts)
 
 Rally recognizes the following client options in addition:
 

esrally/utils/opts.py

@@ -53,6 +53,15 @@ def to_bool(v):
         raise ValueError("Could not convert value '%s'" % v)
 
 
+def to_none(v):
+    if v is None:
+        return None
+    elif v.lower() == "none":
+        return None
+    else:
+        raise ValueError("Could not convert value '%s'" % v)
+
+
 def kv_to_map(kvs):
     def convert(v):
         # string (specified explicitly)
@@ -76,6 +85,12 @@ def convert(v):
             return to_bool(v)
         except ValueError:
             pass
+
+        try:
+            return to_none(v)
+        except ValueError:
+            pass
+
         # treat it as string by default
         return v
 
@@ -197,7 +212,8 @@ def parse_options(self):
         default_client_map = kv_to_map([ClientOptions.DEFAULT_CLIENT_OPTIONS])
         if self.argvalue == ClientOptions.DEFAULT_CLIENT_OPTIONS and self.target_hosts is not None:
             # --client-options unset but multi-clusters used in --target-hosts? apply options defaults for all cluster names.
-            self.parsed_options = {cluster_name: default_client_map for cluster_name in self.target_hosts.all_hosts.keys()}
+            self.parsed_options = {cluster_name: default_client_map for cluster_name in
+                                   self.target_hosts.all_hosts.keys()}
         else:
             self.parsed_options = to_dict(self.argvalue, default_parser=ClientOptions.normalize_to_dict)
 

tests/utils/opts_test.py

@@ -58,12 +58,20 @@ def test_kv_to_map(self):
         assert opts.kv_to_map(["k:3"]) == {"k": 3}
         # implicit treatment as string
         assert opts.kv_to_map(["k:v"]) == {"k": "v"}
-        assert opts.kv_to_map(["k:'v'", "size:4", "empty:false", "temperature:0.5"]) == {
+        assert opts.kv_to_map(["k:'v'", "size:4", "empty:false", "temperature:0.5", "timeout:None"]) == {
             "k": "v",
             "size": 4,
             "empty": False,
             "temperature": 0.5,
+            "timeout": None,
         }
+        # When passing 'None' with quotes, treat it as string
+        assert opts.kv_to_map(["k:'None'"]) == {"k": "None"}
+        # When passing None without quotes, treat it as pythonic None
+        assert opts.kv_to_map(["k:None"]) == {"k": None}
+        assert opts.kv_to_map(["k:none"]) == {"k": None}
+        assert opts.kv_to_map(["k:NONE"]) == {"k": None}
+
 
     def test_to_dict_with_inline_json(self):
         assert opts.to_dict('{"default": ["127.0.0.1:9200","10.17.0.5:19200"]}') == {"default": ["127.0.0.1:9200", "10.17.0.5:19200"]}