[CLIENT-2586] Add support for secondary indexes on blob bins (#527)

aerospike-stubs/aerospike.pyi

@@ -57,6 +57,7 @@ HLL_WRITE_CREATE_ONLY: Literal[1]
 HLL_WRITE_DEFAULT: Literal[0]
 HLL_WRITE_NO_FAIL: Literal[4]
 HLL_WRITE_UPDATE_ONLY: Literal[2]
+INDEX_BLOB: Literal[3]
 INDEX_GEO2DSPHERE: Literal[2]
 INDEX_NUMERIC: Literal[1]
 INDEX_STRING: Literal[0]
@@ -343,6 +344,7 @@ class Client:
     def index_map_values_create(self, ns: str, set: str, bin: str, index_datatype, name: str, policy: dict = ...) -> None: ...
     def index_remove(self, ns, name: str, policy: dict = ...) -> None: ...
     def index_string_create(self, ns: str, set: str, bin: str, name: str, policy: dict = ...) -> None: ...
+    def index_blob_create(self, ns: str, set: str, bin: str, name: str, policy: dict = ...) -> None: ...
     def info_all(self, command: str, policy: dict = ...) -> dict: ...
     def info_random_node(self, command: str, policy: dict = ...) -> str: ...
     def info_single_node(self, command: str, host: str, policy: dict = ...) -> str: ...

doc/aerospike.rst

@@ -1417,6 +1417,10 @@ Miscellaneous
 
     An index whose values are of the aerospike integer data type.
 
+.. data:: INDEX_BLOB
+
+    An index whose values are of the aerospike blob data type.
+
 .. data:: INDEX_GEO2DSPHERE
 
     An index whose values are of the aerospike GetJSON data type.

doc/client.rst

@@ -907,6 +907,18 @@ Index Operations
         :param dict policy: optional :ref:`aerospike_info_policies`.
         :raises: a subclass of :exc:`~aerospike.exception.AerospikeError`.
 
+    .. method:: index_blob_create(ns, set, bin, name[, policy])
+
+        Create an blob index with index name *name* on the *bin* in the specified \
+        *ns*, *set*.
+
+        :param str ns: the namespace in the aerospike cluster.
+        :param str set: the set name.
+        :param str bin: the name of the bin the secondary index is built on.
+        :param str name: the name of the index.
+        :param dict policy: optional :ref:`aerospike_info_policies`.
+        :raises: a subclass of :exc:`~aerospike.exception.AerospikeError`.
+
     .. method:: index_list_create(ns, set, bin, index_datatype, name[, policy: dict])
 
         Create an index named *name* for numeric, string or GeoJSON values \
@@ -916,7 +928,7 @@ Index Operations
         :param str ns: the namespace in the aerospike cluster.
         :param str set: the set name.
         :param str bin: the name of bin the secondary index is built on.
-        :param index_datatype: Possible values are ``aerospike.INDEX_STRING``, ``aerospike.INDEX_NUMERIC`` and ``aerospike.INDEX_GEO2DSPHERE``.
+        :param index_datatype: Possible values are ``aerospike.INDEX_STRING``, ``aerospike.INDEX_NUMERIC``, ``aerospike.INDEX_BLOB``, and ``aerospike.INDEX_GEO2DSPHERE``.
         :param str name: the name of the index.
         :param dict policy: optional :ref:`aerospike_info_policies`.
         :raises: a subclass of :exc:`~aerospike.exception.AerospikeError`.
@@ -932,7 +944,7 @@ Index Operations
         :param str ns: the namespace in the aerospike cluster.
         :param str set: the set name.
         :param str bin: the name of bin the secondary index is built on.
-        :param index_datatype: Possible values are ``aerospike.INDEX_STRING``, ``aerospike.INDEX_NUMERIC`` and ``aerospike.INDEX_GEO2DSPHERE``.
+        :param index_datatype: Possible values are ``aerospike.INDEX_STRING``, ``aerospike.INDEX_NUMERIC``, ``aerospike.INDEX_BLOB``, and ``aerospike.INDEX_GEO2DSPHERE``.
         :param str name: the name of the index.
         :param dict policy: optional :ref:`aerospike_info_policies`.
         :raises: a subclass of :exc:`~aerospike.exception.AerospikeError`.
@@ -948,7 +960,7 @@ Index Operations
         :param str ns: the namespace in the aerospike cluster.
         :param str set: the set name.
         :param str bin: the name of bin the secondary index is built on.
-        :param index_datatype: Possible values are ``aerospike.INDEX_STRING``, ``aerospike.INDEX_NUMERIC`` and ``aerospike.INDEX_GEO2DSPHERE``.
+        :param index_datatype: Possible values are ``aerospike.INDEX_STRING``, ``aerospike.INDEX_NUMERIC``, ``aerospike.INDEX_BLOB``, and ``aerospike.INDEX_GEO2DSPHERE``.
         :param str name: the name of the index.
         :param dict policy: optional :ref:`aerospike_info_policies`.
         :raises: a subclass of :exc:`~aerospike.exception.AerospikeError`.

src/include/client.h

@@ -353,6 +353,15 @@ PyObject *AerospikeClient_Index_Integer_Create(AerospikeClient *self,
 PyObject *AerospikeClient_Index_String_Create(AerospikeClient *self,
                                               PyObject *args, PyObject *kwds);
 
+/**
+ * Create secondary string index
+ *
+ *		client.index_blob_create(namespace, set, bin, index_name, policy)
+ *
+ */
+PyObject *AerospikeClient_Index_Blob_Create(AerospikeClient *self,
+                                            PyObject *args, PyObject *kwds);
+
 /**
  * Create secondary cdt index
  *

src/main/client/sec_index.c

@@ -127,6 +127,31 @@ PyObject *AerospikeClient_Index_String_Create(AerospikeClient *self,
         AS_INDEX_STRING, NULL);
 }
 
+PyObject *AerospikeClient_Index_Blob_Create(AerospikeClient *self,
+                                            PyObject *args, PyObject *kwds)
+{
+    // Python Function Arguments
+    PyObject *py_policy = NULL;
+    PyObject *py_ns = NULL;
+    PyObject *py_set = NULL;
+    PyObject *py_bin = NULL;
+    PyObject *py_name = NULL;
+
+    // Python Function Keyword Arguments
+    static char *kwlist[] = {"ns", "set", "bin", "name", "policy", NULL};
+
+    // Python Function Argument Parsing
+    if (PyArg_ParseTupleAndKeywords(args, kwds, "OOOO|O:index_blob_create",
+                                    kwlist, &py_ns, &py_set, &py_bin, &py_name,
+                                    &py_policy) == false) {
+        return NULL;
+    }
+
+    return createIndexWithDataAndCollectionType(
+        self, py_policy, py_ns, py_set, py_bin, py_name, AS_INDEX_TYPE_DEFAULT,
+        AS_INDEX_BLOB, NULL);
+}
+
 /**
  *******************************************************************************************************
  * Creates a cdt index for a bin in the Aerospike DB.

src/main/client/type.c

@@ -214,6 +214,11 @@ PyDoc_STRVAR(index_string_create_doc,
 \n\
 Create a string index with index_name on the bin in the specified ns, set.");
 
+PyDoc_STRVAR(index_blob_create_doc,
+             "index_blob_create(ns, set, bin, index_name[, policy])\n\
+\n\
+Create a blob index with index_name on the bin in the specified ns, set.");
+
 PyDoc_STRVAR(
     index_cdt_create_doc,
     "index_cdt_create(ns, set, bin,  index_type, index_datatype, index_name, ctx, [, policy])\n\
@@ -472,6 +477,8 @@ static PyMethodDef AerospikeClient_Type_Methods[] = {
      METH_VARARGS | METH_KEYWORDS, index_integer_create_doc},
     {"index_string_create", (PyCFunction)AerospikeClient_Index_String_Create,
      METH_VARARGS | METH_KEYWORDS, index_string_create_doc},
+    {"index_blob_create", (PyCFunction)AerospikeClient_Index_Blob_Create,
+     METH_VARARGS | METH_KEYWORDS, index_blob_create_doc},
     {"index_cdt_create", (PyCFunction)AerospikeClient_Index_Cdt_Create,
      METH_VARARGS | METH_KEYWORDS, index_cdt_create_doc},
     {"get_cdtctx_base64", (PyCFunction)AerospikeClient_GetCDTCTXBase64,

src/main/policy.c

@@ -155,6 +155,7 @@ static AerospikeConstants aerospike_constants[] = {
     {AS_INDEX_STRING, "INDEX_STRING"},
     {AS_INDEX_NUMERIC, "INDEX_NUMERIC"},
     {AS_INDEX_GEO2DSPHERE, "INDEX_GEO2DSPHERE"},
+    {AS_INDEX_BLOB, "INDEX_BLOB"},
     {AS_INDEX_TYPE_DEFAULT, "INDEX_TYPE_DEFAULT"},
     {AS_INDEX_TYPE_LIST, "INDEX_TYPE_LIST"},
     {AS_INDEX_TYPE_MAPKEYS, "INDEX_TYPE_MAPKEYS"},

src/main/predicates.c

@@ -45,6 +45,10 @@ static PyObject *AerospikePredicates_Equals(PyObject *self, PyObject *args)
         return Py_BuildValue("iiOO", AS_PREDICATE_EQUAL, AS_INDEX_STRING,
                              py_bin, py_val);
     }
+    else if (PyBytes_Check(py_val) || PyByteArray_Check(py_val)) {
+        return Py_BuildValue("iiOO", AS_PREDICATE_EQUAL, AS_INDEX_BLOB, py_bin,
+                             py_val);
+    }
 
     Py_INCREF(Py_None);
     return Py_None;
@@ -77,6 +81,10 @@ static PyObject *AerospikePredicates_Contains(PyObject *self, PyObject *args)
         return Py_BuildValue("iiOOOi", AS_PREDICATE_EQUAL, AS_INDEX_STRING,
                              py_bin, py_val, Py_None, index_type);
     }
+    else if (PyBytes_Check(py_val) || PyByteArray_Check(py_val)) {
+        return Py_BuildValue("iiOOOi", AS_PREDICATE_EQUAL, AS_INDEX_BLOB,
+                             py_bin, py_val, Py_None, index_type);
+    }
 
 exit:
     Py_INCREF(Py_None);

src/main/query/where.c

@@ -161,6 +161,75 @@ static int AerospikeQuery_Where_Add(AerospikeQuery *self, PyObject *py_ctx,
                 py_ubin = NULL;
             }
         }
+        else if (in_datatype == AS_INDEX_BLOB) {
+            // TODO: Some of this code can be shared by all the other index data types
+            if (PyUnicode_Check(py_bin)) {
+                const char *py_bin_buffer = PyUnicode_AsUTF8(py_bin);
+                // bin points to the internal buffer of the Python string
+                // so we need to make a copy of the bin string in case the Python string gets garbage collected
+                bin = strdup(py_bin_buffer);
+            }
+            else {
+                rc = 1;
+                break;
+            }
+            uint8_t *val = NULL;
+            Py_ssize_t bytes_size;
+
+            if (PyBytes_Check(py_val1)) {
+                val = (uint8_t *)PyBytes_AsString(py_val1);
+                bytes_size = PyBytes_Size(py_val1);
+            }
+            else if (PyByteArray_Check(py_val1)) {
+                val = (uint8_t *)PyByteArray_AsString(py_val1);
+                bytes_size = PyByteArray_Size(py_val1);
+            }
+            else {
+                rc = 1;
+                free(bin);
+                break;
+            }
+
+            uint8_t *bytes_buffer =
+                (uint8_t *)malloc(sizeof(uint8_t) * bytes_size);
+            memcpy(bytes_buffer, val, sizeof(uint8_t) * bytes_size);
+            val = bytes_buffer;
+
+            as_query_where_init(&self->query, 1);
+            if (index_type == AS_INDEX_TYPE_DEFAULT) {
+                as_query_where_with_ctx(&self->query, bin, pctx,
+                                        as_blob_equals(val, bytes_size, true));
+            }
+            else if (index_type == AS_INDEX_TYPE_LIST) {
+                as_query_where_with_ctx(
+                    &self->query, bin, pctx,
+                    as_blob_contains(LIST, val, bytes_size, true));
+            }
+            else if (index_type == AS_INDEX_TYPE_MAPKEYS) {
+                as_query_where_with_ctx(
+                    &self->query, bin, pctx,
+                    as_blob_contains(MAPKEYS, val, bytes_size, true));
+            }
+            else if (index_type == AS_INDEX_TYPE_MAPVALUES) {
+                as_query_where_with_ctx(
+                    &self->query, bin, pctx,
+                    as_blob_contains(MAPVALUES, val, bytes_size, true));
+            }
+            else {
+                rc = 1;
+                free(bin);
+                break;
+            }
+            if (py_ubin) {
+                Py_DECREF(py_ubin);
+                py_ubin = NULL;
+            }
+
+            self->query.where.entries[0].value.blob_val._free = true;
+
+            // Cleanup
+            free(bin);
+        }
         else {
             // If it ain't expected, raise and error
             as_error_update(
@@ -212,6 +281,7 @@ static int AerospikeQuery_Where_Add(AerospikeQuery *self, PyObject *py_ctx,
                 rc = 1;
                 break;
             }
+
             if (py_ubin) {
                 Py_DECREF(py_ubin);
                 py_ubin = NULL;

test/new_tests/test_index.py

@@ -14,7 +14,7 @@ class TestIndex(object):
     def setup(self, request, as_connection):
         for i in range(5):
             key = ("test", "demo", i)
-            rec = {"name": "name%s" % (str(i)), "addr": "name%s" % (str(i)), "age": i, "no": i}
+            rec = {"name": "name%s" % (str(i)), "addr": "name%s" % (str(i)), "age": i, "no": i, "bytes": b'123'}
             as_connection.put(key, rec)
 
         def teardown():
@@ -210,6 +210,14 @@ def test_create_integer_index_with_policy(self):
         ensure_dropped_index(self.as_connection, "test", "age_index")
         assert retobj == AerospikeStatus.AEROSPIKE_OK
 
+    def test_create_blob_index(self):
+        if self.server_version < [7, 0]:
+            pytest.skip("Blob secondary indexes are only supported in server 7.0+")
+
+        self.as_connection.index_blob_create(ns="test", set="demo", bin="bytes", name="bytes_index", policy={})
+
+        ensure_dropped_index(self.as_connection, "test", "bytes_index")
+
     def test_create_string_index_positive(self):
         """
         Invoke create string index() with correct arguments