[CLIENT-2604] Add option to persist map indexes using map policy (#534)

- Remove "map_write_mode" option in map policy in favor of "map_write_flags".
- Added "map_create" operation

aerospike-stubs/aerospike.pyi

@@ -93,7 +93,6 @@ LOG_LEVEL_INFO: Literal[2]
 LOG_LEVEL_OFF: Literal[-1]
 LOG_LEVEL_TRACE: Literal[4]
 LOG_LEVEL_WARN: Literal[1]
-MAP_CREATE_ONLY: Literal[2]
 MAP_KEY_ORDERED: Literal[1]
 MAP_KEY_VALUE_ORDERED: Literal[3]
 MAP_RETURN_COUNT: Literal[5]
@@ -109,8 +108,6 @@ MAP_RETURN_VALUE: Literal[7]
 MAP_RETURN_ORDERED_MAP: Literal[17]
 MAP_RETURN_UNORDERED_MAP: Literal[16]
 MAP_UNORDERED: Literal[0]
-MAP_UPDATE: Literal[0]
-MAP_UPDATE_ONLY: Literal[1]
 MAP_WRITE_FLAGS_CREATE_ONLY: Literal[1]
 MAP_WRITE_FLAGS_DEFAULT: Literal[0]
 MAP_WRITE_FLAGS_NO_FAIL: Literal[4]
@@ -238,6 +235,7 @@ OP_MAP_REMOVE_BY_VALUE_RANK_RANGE_REL: Literal[1128]
 OP_MAP_REMOVE_BY_VALUE_REL_INDEX_RANGE: Literal[1138]
 OP_MAP_REMOVE_BY_VALUE_REL_RANK_RANGE: Literal[1139]
 OP_MAP_REMOVE_BY_VALUE_REL_RANK_RANGE_TO_END: Literal[1133]
+OP_MAP_CREATE: Literal[1144]
 OP_MAP_SET_POLICY: Literal[1101]
 OP_MAP_SIZE: Literal[1106]
 POLICY_COMMIT_LEVEL_ALL: Literal[0]

aerospike_helpers/operations/bitwise_operations.py

@@ -96,7 +96,7 @@
         client.remove(key)
 
     bit_policy = {
-        'map_write_mode': aerospike.BIT_WRITE_DEFAULT,
+        'bit_write_flags': aerospike.BIT_WRITE_DEFAULT,
     }
     client.put(key, {five_one_bin: five_one_blob})
 

aerospike_helpers/operations/map_operations.py

@@ -41,6 +41,8 @@
 COUNT_KEY = "count"
 RANK_KEY = "rank"
 CTX_KEY = "ctx"
+MAP_ORDER_KEY = "map_order"
+PERSIST_INDEX_KEY = "persist_index"
 
 
 def map_set_policy(bin_name: str, policy, ctx: Optional[list] = None):
@@ -65,6 +67,34 @@ def map_set_policy(bin_name: str, policy, ctx: Optional[list] = None):
     return op_dict
 
 
+def map_create(bin_name: str, map_order: int, persist_index: bool, ctx: Optional[list] = None):
+    """
+    Create map create operation.
+
+    Server creates map at given context level.
+
+    Args:
+        bin_name (str):	Bin name.
+        map_order (int): See :ref:`aerospike_map_order` for possible values.
+        persist_index (bool): If :py:obj:`True`, persist map index. A map index improves lookup performance,
+            but requires more storage. A map index can be created for a top-level
+            ordered map only. Nested and unordered map indexes are not supported.
+        ctx (Optional[dict]): An optional list of nested CDT :class:`cdt_ctx <aerospike_helpers.cdt_ctx>`
+            specifying the path to nested map. If not defined, the top-level map is used.
+    """
+    op_dict = {
+        OP_KEY: aerospike.OP_MAP_CREATE,
+        BIN_KEY: bin_name,
+        MAP_ORDER_KEY: map_order,
+        PERSIST_INDEX_KEY: persist_index
+    }
+
+    if ctx is not None:
+        op_dict[CTX_KEY] = ctx
+
+    return op_dict
+
+
 def map_put(bin_name: str, key, value, map_policy: Optional[dict] = None, ctx: Optional[list] = None):
     """Creates a map_put operation.
 

doc/aerospike.rst

@@ -1085,27 +1085,6 @@ Flags used by map write flag.
 
     Allow other valid map items to be committed if a map item is denied due to write flag constraints.
 
-.. _aerospike_map_write_mode:
-
-Map Write Mode
-^^^^^^^^^^^^^^
-
-Flags used by map *write mode*.
-
-.. note:: This should only be used for Server version < 4.3.0
-
-.. data:: MAP_UPDATE
-
-    Default. Allow create or update.
-
-.. data:: MAP_CREATE_ONLY
-
-    If the key already exists, the item will be denied. If the key does not exist, a new item will be created.
-
-.. data:: MAP_UPDATE_ONLY
-
-    If the key already exists, the item will be overwritten. If the key does not exist, the item will be denied.
-
 .. _aerospike_map_order:
 
 Map Order

doc/client.rst

@@ -2264,21 +2264,11 @@ Map Policies
 
 .. object:: policy
 
-    A :class:`dict` of optional map policies, which are applicable to map operations. Only one of ``map_write_mode`` or ``map_write_flags`` should
-    be provided. ``map_write_mode`` should be used for Aerospike Server versions < `4.3.0` and ``map_write_flags`` should be used for Aerospike server versions
-    greater than or equal to `4.3.0` .
+    A :class:`dict` of optional map policies, which are applicable to map operations.
 
     .. hlist::
         :columns: 1
 
-        * **map_write_mode**
-            | Write mode for the map operation.
-            | One of the :ref:`aerospike_map_write_mode` values such as :data:`aerospike.MAP_UPDATE`
-            |
-            | Default: :data:`aerospike.MAP_UPDATE`
-
-            .. note:: This should only be used for Server version < 4.3.0.
-
         * **map_write_flags**
             | Write flags for the map operation.
             | One of the :ref:`aerospike_map_write_flag` values such as :data:`aerospike.MAP_WRITE_FLAGS_DEFAULT`
@@ -2296,6 +2286,11 @@ Map Policies
             |
             | Default: :data:`aerospike.MAP_UNORDERED`
 
+        * **persist_index** (:class:`bool`)
+            | If :py:obj:`True`, persist map index. A map index improves lookup performance,
+            | but requires more storage. A map index can be created for a top-level
+            | ordered map only. Nested and unordered map indexes are not supported.
+
     Example:
 
     .. code-block:: python
@@ -2306,12 +2301,6 @@ Map Policies
             'map_write_flags': aerospike.MAP_WRITE_FLAGS_CREATE_ONLY
         }
 
-        # Server < 4.3.0
-        map_policy = {
-            'map_order': aerospike.MAP_UNORDERED,
-            'map_write_mode': aerospike.MAP_CREATE_ONLY
-        }
-
 .. _aerospike_bit_policies:
 
 Bit Policies

src/include/policy.h

@@ -136,7 +136,8 @@ enum Aerospike_map_operations {
     OP_MAP_GET_BY_KEY_REL_INDEX_RANGE,
     OP_MAP_GET_BY_VALUE_RANK_RANGE_REL_TO_END,
     OP_MAP_GET_BY_INDEX_RANGE_TO_END,
-    OP_MAP_GET_BY_RANK_RANGE_TO_END
+    OP_MAP_GET_BY_RANK_RANGE_TO_END,
+    OP_MAP_CREATE
 };
 
 enum aerospike_bitwise_operations {

src/main/client/operate.c

@@ -283,7 +283,8 @@ bool opRequiresValue(int op)
             op != OP_MAP_REMOVE_BY_KEY && op != OP_MAP_REMOVE_BY_INDEX &&
             op != OP_MAP_REMOVE_BY_RANK && op != OP_MAP_GET_BY_KEY &&
             op != OP_MAP_GET_BY_INDEX && op != OP_MAP_GET_BY_KEY_RANGE &&
-            op != OP_MAP_GET_BY_RANK && op != AS_OPERATOR_DELETE);
+            op != OP_MAP_GET_BY_RANK && op != AS_OPERATOR_DELETE &&
+            op != OP_MAP_CREATE);
 }
 
 bool opRequiresRange(int op)
@@ -347,6 +348,10 @@ as_status add_op(AerospikeClient *self, as_error *err, PyObject *py_val,
     PyObject *py_range = NULL;
     PyObject *py_map_policy = NULL;
     PyObject *py_return_type = NULL;
+    // For map_create operation
+    PyObject *py_map_order = NULL;
+    PyObject *py_persist_index = NULL;
+
     Py_ssize_t pos = 0;
 
     if (get_operation(err, py_val, &operation) != AEROSPIKE_OK) {
@@ -419,6 +424,12 @@ as_status add_op(AerospikeClient *self, as_error *err, PyObject *py_val,
                 CONVERT_PY_CTX_TO_AS_CTX();
                 ctx_ref = (ctx_in_use ? &ctx : NULL);
             }
+            else if (strcmp(name, "map_order") == 0) {
+                py_map_order = value;
+            }
+            else if (strcmp(name, "persist_index") == 0) {
+                py_persist_index = value;
+            }
             else {
                 as_error_update(
                     err, AEROSPIKE_ERR_PARAM,
@@ -653,6 +664,11 @@ as_status add_op(AerospikeClient *self, as_error *err, PyObject *py_val,
     case OP_MAP_SET_POLICY:
         as_operations_map_set_policy(ops, bin, ctx_ref, &map_policy);
         break;
+    case OP_MAP_CREATE:;
+        as_map_order order = (as_map_order)PyLong_AsLong(py_map_order);
+        bool persist_index = PyObject_IsTrue(py_persist_index);
+        as_operations_map_create_all(ops, bin, ctx_ref, order, persist_index);
+        break;
     case OP_MAP_PUT:
         CONVERT_VAL_TO_AS_VAL();
         CONVERT_KEY_TO_AS_VAL();

src/main/policy.c

@@ -187,6 +187,7 @@ static AerospikeConstants aerospike_constants[] = {
     {OP_LIST_INCREMENT, "OP_LIST_INCREMENT"},
 
     {OP_MAP_SET_POLICY, "OP_MAP_SET_POLICY"},
+    {OP_MAP_CREATE, "OP_MAP_CREATE"},
     {OP_MAP_PUT, "OP_MAP_PUT"},
     {OP_MAP_PUT_ITEMS, "OP_MAP_PUT_ITEMS"},
     {OP_MAP_INCREMENT, "OP_MAP_INCREMENT"},
@@ -218,10 +219,6 @@ static AerospikeConstants aerospike_constants[] = {
     {AS_MAP_KEY_ORDERED, "MAP_KEY_ORDERED"},
     {AS_MAP_KEY_VALUE_ORDERED, "MAP_KEY_VALUE_ORDERED"},
 
-    {AS_MAP_UPDATE, "MAP_UPDATE"},
-    {AS_MAP_UPDATE_ONLY, "MAP_UPDATE_ONLY"},
-    {AS_MAP_CREATE_ONLY, "MAP_CREATE_ONLY"},
-
     {AS_MAP_RETURN_NONE, "MAP_RETURN_NONE"},
     {AS_MAP_RETURN_INDEX, "MAP_RETURN_INDEX"},
     {AS_MAP_RETURN_REVERSE_INDEX, "MAP_RETURN_REVERSE_INDEX"},
@@ -1121,33 +1118,28 @@ as_status pyobject_to_map_policy(as_error *err, PyObject *py_policy,
     // Initialize Policy
     POLICY_INIT(as_map_policy);
 
+    // Defaults
     long map_order = AS_MAP_UNORDERED;
-    long map_write_mode = AS_MAP_UPDATE;
     uint32_t map_write_flags = AS_MAP_WRITE_DEFAULT;
+    bool persist_index = false;
 
     MAP_POLICY_SET_FIELD(map_order);
-    PyObject *mode_or_flags =
-        PyDict_GetItemString(py_policy, MAP_WRITE_FLAGS_KEY);
-
-    /*
-	This only works for client >= 3.5.0 and server >= 4.3.0
-	If py_policy["map_write_flags"] is set, we use it
-	otherwise we use py_policy["map_write_mode"]
-	*/
-    if (mode_or_flags) {
-        if (PyLong_Check(mode_or_flags)) {
-            map_write_flags = (uint32_t)PyLong_AsLong(mode_or_flags);
-            as_map_policy_set_flags(policy, map_order, map_write_flags);
+    MAP_POLICY_SET_FIELD(map_write_flags);
+
+    PyObject *py_persist_index =
+        PyDict_GetItemString(py_policy, "persist_index");
+    if (py_persist_index) {
+        if (PyBool_Check(py_persist_index)) {
+            persist_index = (bool)PyObject_IsTrue(py_persist_index);
         }
         else {
-            as_error_update(err, AEROSPIKE_ERR_PARAM,
-                            "map write flags must be an integer");
+            // persist_index value must be valid if it is set
+            return as_error_update(err, AEROSPIKE_ERR_PARAM,
+                                   "persist_index is not a boolean");
         }
-        return err->code;
     }
 
-    MAP_POLICY_SET_FIELD(map_write_mode);
-    as_map_policy_set(policy, map_order, map_write_mode);
+    as_map_policy_set_all(policy, map_order, map_write_flags, persist_index);
 
     return err->code;
 }

test/new_tests/test_batch_get_ops.py

@@ -122,7 +122,7 @@ def test_batch_result_output_format(self):
         # pp = pprint.PrettyPrinter(2, 80)
         policy = {"key": aerospike.POLICY_KEY_SEND}
         map_policy = {
-            "map_write_mode": aerospike.MAP_UPDATE,
+            "map_write_flags": aerospike.MAP_WRITE_FLAGS_UPDATE_ONLY,
             "map_order": aerospike.MAP_KEY_ORDERED,
         }
 

test/new_tests/test_inverted_map_operations.py

@@ -39,7 +39,7 @@ def maps_have_same_values(map1, map2):
 
 
 def sort_map(client, test_key, test_bin):
-    map_policy = {"map_write_mode": aerospike.MAP_CREATE_ONLY, "map_order": aerospike.MAP_KEY_ORDERED}
+    map_policy = {"map_write_flags": aerospike.MAP_WRITE_FLAGS_CREATE_ONLY, "map_order": aerospike.MAP_KEY_ORDERED}
     operations = [map_ops.map_set_policy(test_bin, map_policy)]
     client.operate(test_key, operations)
 

test/new_tests/test_map_operation_helpers.py

@@ -39,7 +39,7 @@ def maps_have_same_values(map1, map2):
 
 
 def sort_map(client, test_key, test_bin):
-    map_policy = {"map_write_mode": aerospike.MAP_CREATE_ONLY, "map_order": aerospike.MAP_KEY_ORDERED}
+    map_policy = {"map_write_flags": aerospike.MAP_WRITE_FLAGS_CREATE_ONLY, "map_order": aerospike.MAP_KEY_ORDERED}
     operations = [map_ops.map_set_policy(test_bin, map_policy)]
     client.operate(test_key, operations)
 
@@ -79,11 +79,27 @@ def test_map_set_policy(self):
         """
         Test setting map policy with an operation
         """
-        map_policy = {"map_write_mode": aerospike.MAP_CREATE_ONLY, "map_order": aerospike.MAP_KEY_VALUE_ORDERED}
+        map_policy = {
+            "map_write_flags": aerospike.MAP_WRITE_FLAGS_CREATE_ONLY,
+            "map_order": aerospike.MAP_KEY_VALUE_ORDERED,
+            "persist_index": True
+        }
         operations = [map_ops.map_set_policy(self.test_bin, map_policy)]
 
         self.as_connection.operate(self.test_key, operations)
 
+    def test_map_policy_invalid_persist_index(self):
+        map_policy = {
+            "persist_index": 1
+        }
+        operations = [map_ops.map_set_policy(self.test_bin, map_policy)]
+
+        with pytest.raises(e.ParamError):
+            self.as_connection.operate(self.test_key, operations)
+
+    # Default persist index value should be tested automatically
+    # from other tests that don't set the persist index option
+
     def test_map_put(self):
         operations = [map_ops.map_put(self.test_bin, "new", "map_put")]
         self.as_connection.operate(self.test_key, operations)
@@ -394,3 +410,14 @@ def test_map_get_exists_by_rank_range(self):
         operations = [map_ops.map_get_by_rank_range(self.test_bin, 1, 2, return_type=aerospike.MAP_RETURN_EXISTS)]
         ret_vals = get_map_result_from_operation(self.as_connection, self.test_key, operations, self.test_bin)
         assert ret_vals is True
+
+    def test_map_create(self):
+        # This should create an empty dictionary
+        # map_create only works if a map does not already exist at the given bin and context path
+        operations = [
+            map_ops.map_create(bin_name="new_map", map_order=aerospike.MAP_KEY_ORDERED, persist_index=False, ctx=None)
+        ]
+        get_map_result_from_operation(self.as_connection, self.test_key, operations, "new_map")
+
+        res_map = self.as_connection.get(self.test_key)[2]["new_map"]
+        assert res_map == {}

test/new_tests/test_map_write_flags.py

@@ -90,7 +90,7 @@ def test_update_only_does_not_allow_create(self):
             map_ops.map_put("map", "new", "new", map_policy=map_policy),
         ]
 
-        with pytest.raises(e.AerospikeError):
+        with pytest.raises(e.ElementNotFoundError):
             self.as_connection.operate(key, ops)
 
         _, _, bins = self.as_connection.get(key)