Merge pull request #231 from aerospike/3.5.0-in-progress

3.5.0 in progress

.travis/aerospike.conf

@@ -20,7 +20,6 @@ logging {
 }
 
 mod-lua {
-  system-path ${home}/share/udf/lua
   user-path ${home}/var/udf/lua
 }
 

VERSION

@@ -1 +1 @@
-3.4.2
+3.5.0

aerospike_helpers/operations/map_operations.py

@@ -24,7 +24,7 @@ def map_set_policy(bin_name, policy):
 
     Args:
         bin_name (str): The name of the bin containing the map.
-        policy (dict): The map policy dictionary
+        policy (dict): The map policy dictionary. See: See :ref:`aerospike_map_policies`
     Returns:
         A dictionary usable in operate or operate_ordered. The format of the dictionary
         should be considered an internal detail, and subject to change.
@@ -36,7 +36,7 @@ def map_set_policy(bin_name, policy):
     }
 
 
-def map_put(bin_name, key, value):
+def map_put(bin_name, key, value, map_policy=None):
     """Creates a map_put operation to be used with operate or operate_ordered
 
     The operation allows a user to set the value of an item in the map stored
@@ -46,38 +46,50 @@ def map_put(bin_name, key, value):
         bin_name (str): The name of the bin containing the map.
         key: The key for the map.
         value: The item to store in the map with the corresponding key.
+        map_policy (dict, optional):  Optional map_policy dictionary dictates the type of map to create when it does not exist.
+            The map policy also specifies the mode used when writing items to the map. Defaults to `None`. See :ref:`aerospike_map_policies`
     Returns:
         A dictionary usable in operate or operate_ordered. The format of the dictionary
         should be considered an internal detail, and subject to change.
     """
-    return {
+    op_dict = {
         OP_KEY: aerospike.OP_MAP_PUT,
         BIN_KEY: bin_name,
         KEY_KEY: key,
         VALUE_KEY: value
     }
+    if map_policy is not None:
+        op_dict[POLICY_KEY] = map_policy
+
+    return op_dict
 
 
-def map_put_items(bin_name, item_dict):
+def map_put_items(bin_name, item_dict, map_policy=None):
     """Creates a map_put_items operation to be used with operate or operate_ordered
 
     The operation allows a user to add or update items in the map stored on the server.
 
     Args:
         bin_name (str): The name of the bin containing the map.
         item_dict (dict): A dictionary of key value pairs to be added to the map on the server.
+        map_policy (dict, optional):  Optional map_policy dictionary dictates the type of map to create when it does not exist.
+            The map policy also specifies the mode used when writing items to the map. Defaults to `None`. See :ref:`aerospike_map_policies`
     Returns:
         A dictionary usable in operate or operate_ordered. The format of the dictionary
         should be considered an internal detail, and subject to change.
     """
-    return {
+    op_dict = {
         OP_KEY: aerospike.OP_MAP_PUT_ITEMS,
         BIN_KEY: bin_name,
         VALUE_KEY: item_dict
     }
 
+    if map_policy is not None:
+        op_dict[POLICY_KEY] = map_policy
+
+    return op_dict
 
-def map_increment(bin_name, key, amount):
+def map_increment(bin_name, key, amount, map_policy=None):
     """Creates a map_increment operation to be used with operate or operate_ordered
 
     The operation allows a user to increment the value of a value stored in the map on the server.
@@ -86,19 +98,26 @@ def map_increment(bin_name, key, amount):
         bin_name (str): The name of the bin containing the map.
         key: The key for the value to be incremented.
         amount: The amount by which to increment the value stored in map[key]
+        map_policy (dict, optional):  Optional map_policy dictionary dictates the type of map to create when it does not exist.
+            The map policy also specifies the mode used when writing items to the map. Defaults to `None`. See See :ref:`aerospike_map_policies`
     Returns:
         A dictionary usable in operate or operate_ordered. The format of the dictionary
         should be considered an internal detail, and subject to change.
     """
-    return {
+    op_dict =  {
         OP_KEY: aerospike.OP_MAP_INCREMENT,
         BIN_KEY: bin_name,
         KEY_KEY: key,
         VALUE_KEY: amount
     }
 
+    if map_policy is not None:
+        op_dict[POLICY_KEY] = map_policy
 
-def map_decrement(bin_name, key, amount):
+    return op_dict
+
+
+def map_decrement(bin_name, key, amount, map_policy=None):
     """Creates a map_decrement operation to be used with operate or operate_ordered
 
     The operation allows a user to decrement the value of a value stored in the map on the server.
@@ -107,17 +126,23 @@ def map_decrement(bin_name, key, amount):
         bin_name (str): The name of the bin containing the map.
         key: The key for the value to be decremented.
         amount: The amount by which to decrement the value stored in map[key]
+        map_policy (dict, optional):  Optional map_policy dictionary dictates the type of map to create when it does not exist.
+            The map policy also specifies the mode used when writing items to the map. Defaults to `None`. See See :ref:`aerospike_map_policies`
     Returns:
         A dictionary usable in operate or operate_ordered. The format of the dictionary
         should be considered an internal detail, and subject to change.
     """
-    return {
+    op_dict = {
         OP_KEY: aerospike.OP_MAP_DECREMENT,
         BIN_KEY: bin_name,
         KEY_KEY: key,
         VALUE_KEY: amount
     }
 
+    if map_policy is not None:
+        op_dict[POLICY_KEY] = map_policy
+
+    return op_dict
 
 def map_size(bin_name):
     """Creates a map_size operation to be used with operate or operate_ordered

doc/aerospike.rst

@@ -99,7 +99,6 @@ in an in-memory primary index.
 			that has already been reaped by the server.
 			Default: 0 seconds (disabled) for non-TLS connections, 55 seconds for TLS connections.
             * **max_conns_per_node** maximum number of pipeline connections allowed for each node 
-            * **batch_direct** whether to use the batch-direct protocol (default: ``False``, so will use batch-index if available) (**Deprecated**: set 'use_batch_direct' in the batch policy dictionary)
             * **tend_interval** polling interval in milliseconds for tending the cluster (default: 1000)
             * **compression_threshold** compress data for transmission if the object size is greater than a given number of bytes (default: 0, meaning 'never compress') (**Deprecated**, set this in the 'write' policy dictionary)
             * **cluster_name** only server nodes matching this name will be used when determining the cluster
@@ -174,6 +173,56 @@ in an in-memory primary index.
     .. versionadded:: 2.0.1
 
 
+.. py:function:: CDTWildcard()
+
+    A type representing a wildcard object. This type may only be used as a comparison value in operations.
+    It may not be stored in the database.
+
+    :return: a type representing a wildcard value.
+
+    .. code-block:: python
+
+        import aerospike
+        from aerospike_helpers.operations import list_operations as list_ops
+
+        client = aerospike.client({'hosts': [('localhost', 3000)]}).connect()
+        key = 'test', 'demo', 1
+
+        #  get all values of the form [1, ...] from a list of lists.
+        #  For example if list is [[1, 2, 3], [2, 3, 4], [1, 'a']], this operation will match
+        #  [1, 2, 3] and [1, 'a']
+        operations = [list_ops.list_get_by_value('list_bin', [1, aerospike.CDTWildcard()], aerospike.LIST_RETURN_VALUE)]
+        _, _, bins = client.operate(key, operations)
+
+    .. versionadded:: 3.5.0
+    .. note:: This requires Aerospike Server 4.3.1.3 or greater
+
+
+.. py:function:: CDTInfinite()
+
+    A type representing an infinte value. This type may only be used as a comparison value in operations.
+    It may not be stored in the database.
+
+    :return: a type representing an infinite value.
+
+    .. code-block:: python
+
+        import aerospike
+        from aerospike_helpers.operations import list_operations as list_ops
+
+        client = aerospike.client({'hosts': [('localhost', 3000)]}).connect()
+        key = 'test', 'demo', 1
+
+        #  get all values of the form [1, ...] from a list of lists.
+        #  For example if list is [[1, 2, 3], [2, 3, 4], [1, 'a']], this operation will match
+        #  [1, 2, 3] and [1, 'a']
+        operations = [list_ops.list_get_by_value_range('list_bin', aerospike.LIST_RETURN_VALUE, [1],  [1, aerospike.CDTInfinite()])]
+        _, _, bins = client.operate(key, operations)
+
+    .. versionadded:: 3.5.0
+    .. note:: This requires Aerospike Server 4.3.1.3 or greater
+
+
 .. py:function:: calc_digest(ns, set, key) -> bytearray
 
     Calculate the digest of a particular key. See: :ref:`aerospike_key_tuple`.

doc/client.rst

@@ -2950,17 +2950,39 @@ Map Policies
 
 .. object:: policy
 
-    A :class:`dict` of optional map policies, which are applicable to map operations.
+    A :class:`dict` of optional map policies, which are applicable to map operations. Only one of ``map_write_mode`` or ``map_write_flags`` should
+    be provided. ``map_write_mode`` should be used for Aerospike Server versions < `4.3.0` and ``map_write_flags`` should be used for Aerospike server versions
+    greater than or equal to `4.3.0` .
 
     .. hlist::
         :columns: 1
 
-        * **map_write_mode** write mode for the map. Valid values: ``aerospike.MAP_UPDATE``, ``aerospike.MAP_UPDATE_ONLY``, ``aerospike.MAP_CREATE_ONLY``,
-            ``aerospike.MAP_WRITE_PARTIAL`` and ``aerospike.MAP_WRITE_NO_FAIL``.
-            ``aerospike.MAP_WRITE_PARTIAL`` and ``aerospike.MAP_WRITE_NO_FAIL``. require server version 4.3.0 or greater. The values may be or'd together:
-            ``aerospike.MAP_UPDATE_ONLY | aerospike.MAP_WRITE_NO_FAIL``
+        * **map_write_mode** write mode for the map. This should only be used for Server version < 4.3.0 Valid values: ``aerospike.MAP_UPDATE``, ``aerospike.MAP_UPDATE_ONLY``, ``aerospike.MAP_CREATE_ONLY``,
+        * **map_write_flags** Flags to apply to the map operation. This is only valid for Aerospike Server versions >= 4.3.0:
+
+            | Possible values are  ``aerospike.MAP_WRITE_FLAGS_DEFAULT``, ``aerospike.MAP_WRITE_FLAGS_CREATE_ONLY``, ``aerospike.MAP_WRITE_FLAGS_UPDATE_ONLY``,
+            | ``aerospike.MAP_WRITE_FLAGS_PARTIAL`` and ``aerospike.MAP_WRITE_FLAGS_NO_FAIL``. 
+            | The values may be or'd together:
+            | ``aerospike.MAP_WRITE_FLAGS_UPDATE_ONLY | aerospike.MAP_WRITE_FLAGS_NO_FAIL``
+            | `New in version 3.5.0`
+
         * **map_order** ordering to maintain for the map entries. Valid values: ``aerospike.MAP_UNORDERED``, ``aerospike.MAP_KEY_ORDERED``, ``aerospike.MAP_KEY_VALUE_ORDERED``
 
+    Example:
+
+    .. code-block:: python
+
+        # Server >= 4.3.0
+        map_policy = {
+            'map_order': aerospike.MAP_UNORDERED,
+            'map_write_flags': aerospike.MAP_WRITE_FLAGS_CREATE_ONLY
+        }
+
+        # Server < 4.3.0
+        map_policy = {
+            'map_order': aerospike.MAP_UNORDERED,
+            'map_write_mode': aerospike.MAP_CREATE_ONLY
+        }
 
 .. _aerospike_privilege_dict:
 

doc/query.rst

@@ -261,6 +261,23 @@ Query Class --- :class:`Query`
             big_records = query.results()
             client.close()
 
+    .. method:: execute_background([, policy])
+
+        Execute a record UDF on records found by the query in the background. This method returns before the query has completed.
+        A UDF must have been added to the query with :meth:`Query.apply` .
+
+        :param dict policy: optional :ref:`aerospike_write_policies`.
+
+        :return: a job ID that can be used with :meth:`aerospike.Client.job_info` to track the status of the ``aerospike.JOB_QUERY`` , as it runs in the background.
+
+        .. code-block:: python
+
+            import aerospike
+            query = client.query('test', 'demo')
+            query.apply('myudfs', 'myfunction', ['a', 1])
+            # This id can be used to monitor the progress of the background query
+            query_id = query.execute_background()
+
 .. _aerospike_query_policies:
 
 Query Policies

setup.py

@@ -30,7 +30,7 @@
 os.environ['ARCHFLAGS'] = '-arch x86_64'
 AEROSPIKE_C_VERSION = os.getenv('AEROSPIKE_C_VERSION')
 if not AEROSPIKE_C_VERSION:
-    AEROSPIKE_C_VERSION = '4.3.17'
+    AEROSPIKE_C_VERSION = '4.3.20'
 DOWNLOAD_C_CLIENT = os.getenv('DOWNLOAD_C_CLIENT')
 AEROSPIKE_C_HOME = os.getenv('AEROSPIKE_C_HOME')
 PREFIX = None
@@ -272,6 +272,7 @@ def resolve_c_client():
                 'src/main/query/results.c',
                 'src/main/query/select.c',
                 'src/main/query/where.c',
+                'src/main/query/execute_background.c',
                 'src/main/scan/type.c',
                 'src/main/scan/foreach.c',
                 'src/main/scan/results.c',
@@ -289,6 +290,7 @@ def resolve_c_client():
                 'src/main/tls_config.c',
                 'src/main/global_hosts/type.c',
                 'src/main/nullobject/type.c',
+                'src/main/cdt_types/type.c',
             ],
 
             # Compile

src/include/cdt_types.h

@@ -0,0 +1,23 @@
+/*******************************************************************************
+ * Copyright 2013-2017 Aerospike, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+
+#pragma once
+#define AS_CDT_WILDCARD_NAME "aerospike.CDTWildcard"
+#define AS_CDT_INFINITE_NAME "aerospike.CDTInfinite"
+#include <Python.h>
+
+PyTypeObject * AerospikeWildcardObject_Ready();
+PyTypeObject * AerospikeInfiniteObject_Ready();

src/include/macros.h

@@ -49,3 +49,6 @@
 			ob = Py_InitModule3(name, methods, doc);
 	#define MOD_SUCCESS_VAL(val)
 #endif
+
+// pyval is a PyObject* classname is a string
+#define AS_Matches_Classname(pyval, classname) (strcmp((pyval)->ob_type->tp_name, (classname)) == 0)

src/include/query.h

@@ -94,6 +94,12 @@ PyObject * AerospikeQuery_Foreach(AerospikeQuery * self, PyObject * args, PyObje
  */
 PyObject * AerospikeQuery_Results(AerospikeQuery * self, PyObject * args, PyObject * kwds);
 
+/**
+ * Execute a UDF in the background. Returns the query id to allow status of the query to be monitored
+ * */
+
+PyObject * AerospikeQuery_ExecuteBackground(AerospikeQuery * self, PyObject * args, PyObject * kwds);
+
 /**
  * Store the Unicode -> UTF8 string converted PyObject into 
  * a pool of PyObjects. So that, they will be decref'ed at later stages

src/include/types.h

@@ -37,6 +37,14 @@ typedef struct {
 	PyObject_HEAD
 } AerospikeNullObject;
 
+typedef struct {
+	PyObject_HEAD
+} AerospikeCDTWildcardObject;
+
+typedef struct {
+	PyObject_HEAD
+} AerospikeCDTInfObject;
+
 typedef struct {
 	PyObject_HEAD
 	aerospike * as;

src/main/aerospike.c

@@ -31,6 +31,7 @@
 #include "serializer.h"
 #include "module_functions.h"
 #include "nullobject.h"
+#include "cdt_types.h"
 
 PyObject *py_global_hosts;
 int counter = 0xA7000000;
@@ -93,7 +94,7 @@ AerospikeConstants operator_constants[] = {
 MOD_INIT(aerospike)
 {
 
-	const char version[8] = "3.4.2";
+	const char version[8] = "3.5.0";
 	// Makes things "thread-safe"
 	PyEval_InitThreads();
 	int i = 0;
@@ -151,5 +152,13 @@ MOD_INIT(aerospike)
 	Py_INCREF(null_object);
 	PyModule_AddObject(aerospike, "null", (PyObject *) null_object);
 
+	PyTypeObject * wildcard_object = AerospikeWildcardObject_Ready();
+	Py_INCREF(wildcard_object);
+	PyModule_AddObject(aerospike, "CDTWildcard", (PyObject *) wildcard_object);
+
+	PyTypeObject * infinite_object = AerospikeInfiniteObject_Ready();
+	Py_INCREF(infinite_object);
+	PyModule_AddObject(aerospike, "CDTInfinite", (PyObject *) infinite_object);
+
 	return MOD_SUCCESS_VAL(aerospike);
 }