From 88a93856d6f5046e45c2fe094140d9e2ad1e7034 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Mon, 6 Nov 2023 13:16:08 -0800 Subject: [PATCH 01/25] WIP adding ttl option to client config policies --- aerospike-client-c | 2 +- doc/client.rst | 9 +++++++++ doc/scan.rst | 4 ++++ src/main/client/batch_operate.c | 5 +++++ src/main/conversions.c | 3 +++ src/main/policy.c | 3 +++ src/main/policy_config.c | 27 ++++++++++++++++++++++++++ src/main/scan/type.c | 34 +++++++++++++++++++-------------- 8 files changed, 72 insertions(+), 15 deletions(-) diff --git a/aerospike-client-c b/aerospike-client-c index 1ac97cb72..600d2d2e7 160000 --- a/aerospike-client-c +++ b/aerospike-client-c @@ -1 +1 @@ -Subproject commit 1ac97cb72b413061c510026578a9e5cfe1216a6d +Subproject commit 600d2d2e73118a8b7bc1056777bdf2ad6e7ea3c7 diff --git a/doc/client.rst b/doc/client.rst index 62d53785f..0123d500d 100755 --- a/doc/client.rst +++ b/doc/client.rst @@ -1575,6 +1575,9 @@ Write Policies | One of the :ref:`POLICY_EXISTS` values such as :data:`aerospike.POLICY_EXISTS_CREATE` | | Default: :data:`aerospike.POLICY_EXISTS_IGNORE` + * **ttl** + | The default time-to-live (expiration) of the record in seconds. This field will only be used if a write + | transaction's ``ttl`` option in the metadata dictionary is not set. * **gen** | One of the :ref:`POLICY_GEN` values such as :data:`aerospike.POLICY_GEN_IGNORE` | @@ -1740,6 +1743,9 @@ Operate Policies | One of the :ref:`POLICY_GEN` values such as :data:`aerospike.POLICY_GEN_IGNORE` | | Default: :data:`aerospike.POLICY_GEN_IGNORE` + * **ttl** + | The default time-to-live (expiration) of the record in seconds. This field will only be used if an operate + | transaction's ``ttl`` option in the metadata dictionary is not set. * **replica** | One of the :ref:`POLICY_REPLICA` values such as :data:`aerospike.POLICY_REPLICA_MASTER` | @@ -1843,6 +1849,9 @@ Apply Policies | One of the :ref:`POLICY_COMMIT_LEVEL` values such as :data:`aerospike.POLICY_COMMIT_LEVEL_ALL` | | Default: :data:`aerospike.POLICY_COMMIT_LEVEL_ALL` + * **ttl** + | The default time-to-live (expiration) of the record in seconds. This field will only be used if an apply + | transaction's ``ttl`` option in the apply policy is not set. * **durable_delete** (:class:`bool`) | Perform durable delete | diff --git a/doc/scan.rst b/doc/scan.rst index d76a581ed..70aff0538 100755 --- a/doc/scan.rst +++ b/doc/scan.rst @@ -556,6 +556,10 @@ Policies | One of the :ref:`POLICY_REPLICA` values such as :data:`aerospike.POLICY_REPLICA_MASTER` | | Default: ``aerospike.POLICY_REPLICA_SEQUENCE`` + * **ttl** + | The default time-to-live (expiration) of the record in seconds. This field will only be + | used on background scan writes if the ``ttl`` parameter in :meth:`Scan.execute_background` is not + | set. .. _aerospike_scan_options: diff --git a/src/main/client/batch_operate.c b/src/main/client/batch_operate.c index 787fdd635..c294bee67 100644 --- a/src/main/client/batch_operate.c +++ b/src/main/client/batch_operate.c @@ -239,6 +239,11 @@ static PyObject *AerospikeClient_Batch_Operate_Invoke( ops.ttl = ttl; } } + else { + // If ttl in this transaction's batch write policy isn't set, use the client config's default batch write + // policy ttl + ops.ttl = AS_RECORD_CLIENT_DEFAULT_TTL; + } Py_XDECREF(py_ttl); } diff --git a/src/main/conversions.c b/src/main/conversions.c index a79b4d1f9..1ad27b34e 100644 --- a/src/main/conversions.c +++ b/src/main/conversions.c @@ -2205,6 +2205,9 @@ as_status check_and_set_meta(PyObject *py_meta, as_operations *ops, return as_error_update(err, AEROSPIKE_ERR_PARAM, "Metadata should be of type dictionary"); } + else if (py_meta == NULL) { + ops->ttl = AS_RECORD_CLIENT_DEFAULT_TTL; + } return err->code; } diff --git a/src/main/policy.c b/src/main/policy.c index 78b63e41d..398efdea0 100644 --- a/src/main/policy.c +++ b/src/main/policy.c @@ -236,6 +236,7 @@ static AerospikeConstants aerospike_constants[] = { {AS_RECORD_DEFAULT_TTL, "TTL_NAMESPACE_DEFAULT"}, {AS_RECORD_NO_EXPIRE_TTL, "TTL_NEVER_EXPIRE"}, {AS_RECORD_NO_CHANGE_TTL, "TTL_DONT_UPDATE"}, + {AS_RECORD_CLIENT_DEFAULT_TTL, "TTL_CLIENT_DEFAULT"}, {AS_AUTH_INTERNAL, "AUTH_INTERNAL"}, {AS_AUTH_EXTERNAL, "AUTH_EXTERNAL"}, {AS_AUTH_EXTERNAL_INSECURE, "AUTH_EXTERNAL_INSECURE"}, @@ -628,6 +629,7 @@ as_status pyobject_to_policy_apply(AerospikeClient *self, as_error *err, //POLICY_SET_FIELD(gen, as_policy_gen); removed POLICY_SET_FIELD(commit_level, as_policy_commit_level); POLICY_SET_FIELD(durable_delete, bool); + POLICY_SET_FIELD(ttl, uint32_t); // C client 5.0 new expressions POLICY_SET_EXPRESSIONS_BASE_FIELD(); @@ -882,6 +884,7 @@ as_status pyobject_to_policy_write(AerospikeClient *self, as_error *err, POLICY_SET_FIELD(durable_delete, bool); POLICY_SET_FIELD(replica, as_policy_replica); POLICY_SET_FIELD(compression_threshold, uint32_t); + POLICY_SET_FIELD(ttl, uint32_t); // C client 5.0 new expressions POLICY_SET_EXPRESSIONS_BASE_FIELD(); diff --git a/src/main/policy_config.c b/src/main/policy_config.c index 6cfaea6c3..701bfc4f8 100644 --- a/src/main/policy_config.c +++ b/src/main/policy_config.c @@ -229,6 +229,11 @@ as_status set_write_policy(as_policy_write *write_policy, PyObject *py_policy) return status; } + status = set_optional_uint32_property(&write_policy->ttl, py_policy, "ttl"); + if (status != AEROSPIKE_OK) { + return status; + } + status = set_optional_uint32_property( (uint32_t *)&write_policy->compression_threshold, py_policy, "compression_threshold"); @@ -273,6 +278,11 @@ as_status set_apply_policy(as_policy_apply *apply_policy, PyObject *py_policy) return status; } + status = set_optional_uint32_property(&apply_policy->ttl, py_policy, "ttl"); + if (status != AEROSPIKE_OK) { + return status; + } + status = set_optional_commit_level(&apply_policy->commit_level, py_policy, "commit_level"); if (status != AEROSPIKE_OK) { @@ -393,6 +403,11 @@ as_status set_scan_policy(as_policy_scan *scan_policy, PyObject *py_policy) return status; } + status = set_optional_uint32_property(&scan_policy->ttl, py_policy, "ttl"); + if (status != AEROSPIKE_OK) { + return status; + } + status = set_optional_replica(&scan_policy->replica, py_policy, "replica"); if (status != AEROSPIKE_OK) { return status; @@ -437,6 +452,12 @@ as_status set_operate_policy(as_policy_operate *operate_policy, return status; } + status = + set_optional_uint32_property(&operate_policy->ttl, py_policy, "ttl"); + if (status != AEROSPIKE_OK) { + return status; + } + status = set_optional_gen(&operate_policy->gen, py_policy, "gen"); if (status != AEROSPIKE_OK) { return status; @@ -637,6 +658,12 @@ as_status set_batch_write_policy(as_policy_batch_write *batch_write_policy, return status; } + status = set_optional_uint32_property(&batch_write_policy->ttl, py_policy, + "ttl"); + if (status != AEROSPIKE_OK) { + return status; + } + status = set_optional_gen(&batch_write_policy->gen, py_policy, "gen"); if (status != AEROSPIKE_OK) { return status; diff --git a/src/main/scan/type.c b/src/main/scan/type.c index 44b356c91..b35b686f7 100644 --- a/src/main/scan/type.c +++ b/src/main/scan/type.c @@ -99,6 +99,12 @@ static PyMethodDef AerospikeScan_Type_Methods[] = { {NULL}}; +static PyMemberDef AerospikeScan_Type_custom_members[] = { + {"ttl", T_UINT, offsetof(AerospikeScan, scan) + offsetof(as_query, ttl), 0, + "The time-to-live (expiration) of the record in seconds."}, + {NULL} /* Sentinel */ +}; + /******************************************************************************* * PYTHON TYPE HOOKS ******************************************************************************/ @@ -207,20 +213,20 @@ static PyTypeObject AerospikeScan_Type = { "operation. To create a new instance of the Scan class, call the\n" "scan() method on an instance of a Client class.\n", // tp_doc - 0, // tp_traverse - 0, // tp_clear - 0, // tp_richcompare - 0, // tp_weaklistoffset - 0, // tp_iter - 0, // tp_iternext - AerospikeScan_Type_Methods, // tp_methods - 0, // tp_members - 0, // tp_getset - 0, // tp_base - 0, // tp_dict - 0, // tp_descr_get - 0, // tp_descr_set - 0, // tp_dictoffset + 0, // tp_traverse + 0, // tp_clear + 0, // tp_richcompare + 0, // tp_weaklistoffset + 0, // tp_iter + 0, // tp_iternext + AerospikeScan_Type_Methods, // tp_methods + AerospikeScan_Type_custom_members, // tp_members + 0, // tp_getset + 0, // tp_base + 0, // tp_dict + 0, // tp_descr_get + 0, // tp_descr_set + 0, // tp_dictoffset (initproc)AerospikeScan_Type_Init, // tp_init 0, // tp_alloc From 09f0a51e7293685875f251406e6cd3baf43547aa Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Mon, 6 Nov 2023 13:32:26 -0800 Subject: [PATCH 02/25] Fix and add documentation --- doc/scan.rst | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/doc/scan.rst b/doc/scan.rst index 70aff0538..703ff8fc4 100755 --- a/doc/scan.rst +++ b/doc/scan.rst @@ -22,6 +22,22 @@ bins returned can be filtered using :meth:`select`. `Scans `_ and \ `Managing Scans `_. +Fields +====== + +.. class:: Scan + + ttl (:class:`int`) + The time-to-live (expiration) of the record in seconds. Note that ttl + is only used on background scan writes. This value must be a valid 32-bit unsigned integer. + + See :ref:`TTL_CONSTANTS` for special values that can be set in the record ttl. + + Default: ``0`` (no limit) + + .. note:: + Requires server version >= 6.0.0 + Methods ======= @@ -558,8 +574,7 @@ Policies | Default: ``aerospike.POLICY_REPLICA_SEQUENCE`` * **ttl** | The default time-to-live (expiration) of the record in seconds. This field will only be - | used on background scan writes if the ``ttl`` parameter in :meth:`Scan.execute_background` is not - | set. + | used on background scan writes if :py:attr:`aerospike.Scan.ttl` is set to :data:`aerospike.TTL_CLIENT_DEFAULT`. .. _aerospike_scan_options: From 4bb4583752a05300cf0c6d051aba014db5364a6c Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Mon, 6 Nov 2023 13:41:41 -0800 Subject: [PATCH 03/25] Add more docs, fix write default ttl behavior --- doc/aerospike.rst | 4 ++++ src/main/conversions.c | 3 +++ src/main/policy.c | 1 - 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/doc/aerospike.rst b/doc/aerospike.rst index 9bdddbcb9..36e240c15 100644 --- a/doc/aerospike.rst +++ b/doc/aerospike.rst @@ -858,6 +858,10 @@ Specifies the TTL constants. Do not change the current TTL of the record. +.. data:: TTL_CLIENT_DEFAULT + + For scans, use the client config's ``ttl`` value in the default scan policy. + .. _auth_mode: Auth Mode Constants diff --git a/src/main/conversions.c b/src/main/conversions.c index 1ad27b34e..389e5606a 100644 --- a/src/main/conversions.c +++ b/src/main/conversions.c @@ -1137,6 +1137,9 @@ as_status pyobject_to_record(AerospikeClient *self, as_error *err, } } } + else { + rec->ttl = AS_RECORD_CLIENT_DEFAULT_TTL; + } if (err->code != AEROSPIKE_OK) { as_record_destroy(rec); diff --git a/src/main/policy.c b/src/main/policy.c index 398efdea0..411ffac81 100644 --- a/src/main/policy.c +++ b/src/main/policy.c @@ -884,7 +884,6 @@ as_status pyobject_to_policy_write(AerospikeClient *self, as_error *err, POLICY_SET_FIELD(durable_delete, bool); POLICY_SET_FIELD(replica, as_policy_replica); POLICY_SET_FIELD(compression_threshold, uint32_t); - POLICY_SET_FIELD(ttl, uint32_t); // C client 5.0 new expressions POLICY_SET_EXPRESSIONS_BASE_FIELD(); From 6692183308f8a2f8d3a30eefa1fd1fb611ec267e Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Mon, 6 Nov 2023 13:47:20 -0800 Subject: [PATCH 04/25] fix docs warning --- doc/scan.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/scan.rst b/doc/scan.rst index 703ff8fc4..04c1dacfa 100755 --- a/doc/scan.rst +++ b/doc/scan.rst @@ -42,6 +42,7 @@ Methods ======= .. class:: Scan + :noindex: .. deprecated:: 7.0.0 :class:`aerospike.Query` should be used instead. From 82661ed88521d0569eabecdf5de8c67bc18f40a0 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Mon, 6 Nov 2023 15:10:21 -0800 Subject: [PATCH 05/25] WIP tests --- src/main/client/batch_write.c | 6 +-- src/main/client/operate.c | 12 ++--- test/new_tests/test_new_constructor.py | 64 ++++++++++++++++++++++++++ 3 files changed, 70 insertions(+), 12 deletions(-) diff --git a/src/main/client/batch_write.c b/src/main/client/batch_write.c index dc885178b..5408be4ea 100644 --- a/src/main/client/batch_write.c +++ b/src/main/client/batch_write.c @@ -277,10 +277,8 @@ static PyObject *AerospikeClient_BatchWriteInvoke(AerospikeClient *self, ops = as_operations_new(py_ops_size); garb->ops_to_free = ops; - if (py_meta) { - if (check_and_set_meta(py_meta, ops, err) != AEROSPIKE_OK) { - goto CLEANUP0; - } + if (check_and_set_meta(py_meta, ops, err) != AEROSPIKE_OK) { + goto CLEANUP0; } for (Py_ssize_t i = 0; i < py_ops_size; i++) { diff --git a/src/main/client/operate.c b/src/main/client/operate.c index 79d776e6d..b3b58b128 100644 --- a/src/main/client/operate.c +++ b/src/main/client/operate.c @@ -868,10 +868,8 @@ static PyObject *AerospikeClient_Operate_Invoke(AerospikeClient *self, memset(&static_pool, 0, sizeof(static_pool)); CHECK_CONNECTED(err); - if (py_meta) { - if (check_and_set_meta(py_meta, &ops, err) != AEROSPIKE_OK) { - goto CLEANUP; - } + if (check_and_set_meta(py_meta, &ops, err) != AEROSPIKE_OK) { + goto CLEANUP; } for (i = 0; i < size; i++) { @@ -1041,10 +1039,8 @@ AerospikeClient_OperateOrdered_Invoke(AerospikeClient *self, as_error *err, } } - if (py_meta) { - if (check_and_set_meta(py_meta, &ops, err) != AEROSPIKE_OK) { - goto CLEANUP; - } + if (check_and_set_meta(py_meta, &ops, err) != AEROSPIKE_OK) { + goto CLEANUP; } for (Py_ssize_t i = 0; i < ops_list_size; i++) { diff --git a/test/new_tests/test_new_constructor.py b/test/new_tests/test_new_constructor.py index b1830580a..6c69c49a2 100644 --- a/test/new_tests/test_new_constructor.py +++ b/test/new_tests/test_new_constructor.py @@ -4,6 +4,8 @@ from .test_base_class import TestBaseClass import aerospike from aerospike import exception as e +from aerospike_helpers.operations import operations +from aerospike_helpers.batch.records import Write, BatchRecords import copy gconfig = {} @@ -201,3 +203,65 @@ def test_setting_batch_policies(): for policy in policies: config["policies"][policy] = {} aerospike.client(config) + + +class TestConfigTTL: + @pytest.fixture + def config_ttl_setup(self, policy_name: str): + config = copy.deepcopy(gconfig) + self.new_ttl = 9000 + config["policies"][policy_name] = { + "ttl": self.new_ttl + } + self.client = aerospike.client(config) + self.key = ("test", "demo", 0) + + yield + + # Teardown + if policy_name == "apply": + self.client.udf_remove("test_record_udf.lua") + + def check_ttl(self): + _, meta = self.client.exists(self.key) + clock_skew_tolerance_secs = 50 + assert meta["ttl"] in range(self.new_ttl - clock_skew_tolerance_secs, self.new_ttl + clock_skew_tolerance_secs) + + @pytest.mark.parametrize("policy_name", ["write"]) + def test_setting_write_ttl(self, config_ttl_setup): + # Call without setting the ttl in the transaction metadata dict + self.client.put(self.key, bins={"a": 1}) + self.check_ttl() + + @pytest.mark.parametrize("policy_name", ["operate"]) + def test_setting_operate_ttl(self, config_ttl_setup): + # Call without setting the ttl in the transaction metadata dict + ops = [ + operations.write("a", 1) + ] + self.client.operate(self.key, ops) + self.check_ttl() + + @pytest.mark.parametrize("policy_name", ["apply"]) + def test_setting_apply_ttl(self, config_ttl_setup): + # Setup + self.client.udf_put("test_record_udf.lua") + self.client.put(self.key, {"bin": "a"}) + + # Call without setting the ttl in the transaction's apply policy + # Args: bin name, str + self.client.apply(self.key, module="test_record_udf", function="bin_udf_operation_string", args=["bin", "a"]) + self.check_ttl() + + @pytest.mark.parametrize("policy_name", ["batch_write"]) + def test_setting_batch_write_ttl(self, config_ttl_setup): + # Call without setting the ttl in the Write BatchRecord's metadata dict + ops = [ + operations.write("bin", 1) + ] + batch_records = BatchRecords([ + Write(self.key, ops=ops) + ]) + self.client.batch_write(batch_records) + + self.check_ttl() From e7bcb9d1235493d67735040b205922356088c86f Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Mon, 6 Nov 2023 16:01:08 -0800 Subject: [PATCH 06/25] WIP, scan test failing --- src/main/conversions.c | 3 ++- src/main/scan/type.c | 2 +- test/new_tests/test_new_constructor.py | 37 ++++++++++++++++++++++++-- 3 files changed, 38 insertions(+), 4 deletions(-) diff --git a/src/main/conversions.c b/src/main/conversions.c index 389e5606a..eab328bb6 100644 --- a/src/main/conversions.c +++ b/src/main/conversions.c @@ -2208,7 +2208,8 @@ as_status check_and_set_meta(PyObject *py_meta, as_operations *ops, return as_error_update(err, AEROSPIKE_ERR_PARAM, "Metadata should be of type dictionary"); } - else if (py_meta == NULL) { + else { + // Metadata dict was not set by user ops->ttl = AS_RECORD_CLIENT_DEFAULT_TTL; } return err->code; diff --git a/src/main/scan/type.c b/src/main/scan/type.c index b35b686f7..62084e319 100644 --- a/src/main/scan/type.c +++ b/src/main/scan/type.c @@ -100,7 +100,7 @@ static PyMethodDef AerospikeScan_Type_Methods[] = { {NULL}}; static PyMemberDef AerospikeScan_Type_custom_members[] = { - {"ttl", T_UINT, offsetof(AerospikeScan, scan) + offsetof(as_query, ttl), 0, + {"ttl", T_UINT, offsetof(AerospikeScan, scan) + offsetof(as_scan, ttl), 0, "The time-to-live (expiration) of the record in seconds."}, {NULL} /* Sentinel */ }; diff --git a/test/new_tests/test_new_constructor.py b/test/new_tests/test_new_constructor.py index 6c69c49a2..f08dfb9ef 100644 --- a/test/new_tests/test_new_constructor.py +++ b/test/new_tests/test_new_constructor.py @@ -6,6 +6,7 @@ from aerospike import exception as e from aerospike_helpers.operations import operations from aerospike_helpers.batch.records import Write, BatchRecords +from .test_scan_execute_background import wait_for_job_completion import copy gconfig = {} @@ -216,10 +217,13 @@ def config_ttl_setup(self, policy_name: str): self.client = aerospike.client(config) self.key = ("test", "demo", 0) + if "apply" in policy_name: + self.client.udf_put("test_record_udf.lua") + yield # Teardown - if policy_name == "apply": + if "apply" in policy_name: self.client.udf_remove("test_record_udf.lua") def check_ttl(self): @@ -245,7 +249,6 @@ def test_setting_operate_ttl(self, config_ttl_setup): @pytest.mark.parametrize("policy_name", ["apply"]) def test_setting_apply_ttl(self, config_ttl_setup): # Setup - self.client.udf_put("test_record_udf.lua") self.client.put(self.key, {"bin": "a"}) # Call without setting the ttl in the transaction's apply policy @@ -265,3 +268,33 @@ def test_setting_batch_write_ttl(self, config_ttl_setup): self.client.batch_write(batch_records) self.check_ttl() + + @pytest.mark.parametrize("policy_name", ["batch_apply"]) + def test_setting_batch_apply_ttl(self, config_ttl_setup): + # Setup + self.client.put(self.key, {"bin": "a"}) + + # Call without setting the ttl in batch_apply()'s batch apply policy + keys = [ + self.key + ] + self.client.batch_apply(keys, module="test_record_udf", function="bin_udf_operation_string", args=["bin", "a"]) + self.check_ttl() + + @pytest.mark.parametrize("policy_name", ["scan"]) + def test_setting_scan_ttl(self, config_ttl_setup): + # Setup + self.client.put(self.key, {"bin": "a"}) + + # Tell scan to use client config's scan policy ttl + scan = self.client.scan("test", "demo") + scan.ttl = aerospike.TTL_CLIENT_DEFAULT + ops = [ + operations.append("bin", "a") + ] + scan.add_ops(ops) + job_id = scan.execute_background() + + wait_for_job_completion(self.client, job_id) + + self.check_ttl() From e92222f790263328481d6c01d4b4f4e9023df56e Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Mon, 6 Nov 2023 16:27:31 -0800 Subject: [PATCH 07/25] WIP test --- test/new_tests/test_new_constructor.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/test/new_tests/test_new_constructor.py b/test/new_tests/test_new_constructor.py index f08dfb9ef..cda0743f3 100644 --- a/test/new_tests/test_new_constructor.py +++ b/test/new_tests/test_new_constructor.py @@ -223,8 +223,18 @@ def config_ttl_setup(self, policy_name: str): yield # Teardown + if "apply" in policy_name: - self.client.udf_remove("test_record_udf.lua") + try: + self.client.udf_remove("test_record_udf.lua") + except e.UDFError: + # In case UDF module does not exist + pass + + try: + self.client.remove(self.key) + except e.RecordNotFound: + pass def check_ttl(self): _, meta = self.client.exists(self.key) @@ -258,6 +268,7 @@ def test_setting_apply_ttl(self, config_ttl_setup): @pytest.mark.parametrize("policy_name", ["batch_write"]) def test_setting_batch_write_ttl(self, config_ttl_setup): + # self.client.put(self.key, {"bin": "a"}) # Call without setting the ttl in the Write BatchRecord's metadata dict ops = [ operations.write("bin", 1) @@ -265,7 +276,10 @@ def test_setting_batch_write_ttl(self, config_ttl_setup): batch_records = BatchRecords([ Write(self.key, ops=ops) ]) - self.client.batch_write(batch_records) + brs = self.client.batch_write(batch_records) + # assert brs.result == 0 + for br in brs.batch_records: + assert br.result == 0 self.check_ttl() From 42287003520882fb1a2a3840a8d6eec1f3e26d6b Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Mon, 6 Nov 2023 17:58:01 -0800 Subject: [PATCH 08/25] update type stubs --- aerospike-stubs/aerospike.pyi | 2 ++ 1 file changed, 2 insertions(+) diff --git a/aerospike-stubs/aerospike.pyi b/aerospike-stubs/aerospike.pyi index e5155ef54..87cfef35a 100644 --- a/aerospike-stubs/aerospike.pyi +++ b/aerospike-stubs/aerospike.pyi @@ -284,6 +284,7 @@ SERIALIZER_USER: Literal[3] TTL_DONT_UPDATE: Literal[0xFFFFFFFE] TTL_NAMESPACE_DEFAULT: Literal[0] TTL_NEVER_EXPIRE: Literal[0xFFFFFFFF] +TTL_CLIENT_DEFAULT: Literal[0xFFFFFFFD] UDF_TYPE_LUA: Literal[0] @final @@ -442,6 +443,7 @@ class Query: def where(self, predicate: tuple, ctx: list = ...) -> None: ... class Scan: + ttl: int def __init__(self, *args, **kwargs) -> None: ... def add_ops(self, ops: list) -> None: ... def apply(self, module: str, function: str, arguments: list = ...) -> Any: ... From f2d3bebe70952a81ce1b95aaab2ae8e25701f8aa Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Tue, 7 Nov 2023 09:21:25 -0800 Subject: [PATCH 09/25] Update C client to fix bug where batch write ttl client policy isn't applied --- aerospike-client-c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aerospike-client-c b/aerospike-client-c index 600d2d2e7..29860294a 160000 --- a/aerospike-client-c +++ b/aerospike-client-c @@ -1 +1 @@ -Subproject commit 600d2d2e73118a8b7bc1056777bdf2ad6e7ea3c7 +Subproject commit 29860294a714d254c71d4325d8242b66f7a773eb From bb1692477fde83056f0099b8e2a4ff2980a1feb1 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Tue, 7 Nov 2023 14:07:29 -0800 Subject: [PATCH 10/25] Add more tests, clarify docs --- doc/aerospike.rst | 4 -- doc/client.rst | 41 ++++++++++-------- doc/query.rst | 16 ++------ doc/scan.rst | 2 + test/new_tests/test_new_constructor.py | 57 ++++++++++++++++++++------ 5 files changed, 73 insertions(+), 47 deletions(-) diff --git a/doc/aerospike.rst b/doc/aerospike.rst index 36e240c15..9bdddbcb9 100644 --- a/doc/aerospike.rst +++ b/doc/aerospike.rst @@ -858,10 +858,6 @@ Specifies the TTL constants. Do not change the current TTL of the record. -.. data:: TTL_CLIENT_DEFAULT - - For scans, use the client config's ``ttl`` value in the default scan policy. - .. _auth_mode: Auth Mode Constants diff --git a/doc/client.rst b/doc/client.rst index 0123d500d..738948e8c 100755 --- a/doc/client.rst +++ b/doc/client.rst @@ -1508,7 +1508,7 @@ Metadata Dictionary The metadata dictionary has the following key-value pairs: - * ``"ttl"`` (:class:`int`): record time to live in seconds. See :ref:`TTL_CONSTANTS`. + * ``"ttl"`` (:class:`int`): record time to live in seconds. See :ref:`TTL_CONSTANTS` for possible special values. * ``"gen"`` (:class:`int`): record generation .. _aerospike_policies: @@ -1576,8 +1576,12 @@ Write Policies | | Default: :data:`aerospike.POLICY_EXISTS_IGNORE` * **ttl** - | The default time-to-live (expiration) of the record in seconds. This field will only be used if a write - | transaction's ``ttl`` option in the metadata dictionary is not set. + | The default time-to-live (expiration) of the record in seconds. This field will only be used if the write + | transaction: + | 1. Doesn't contain a metadata dictionary with a ``ttl`` value. + | 2. Contains a metadata dictionary with a ``ttl`` value set to :data:`aerospike.TTL_CLIENT_DEFAULT`. + | + | There are also special values that can be set for this option. See :ref:`TTL_CONSTANTS`. * **gen** | One of the :ref:`POLICY_GEN` values such as :data:`aerospike.POLICY_GEN_IGNORE` | @@ -1745,7 +1749,11 @@ Operate Policies | Default: :data:`aerospike.POLICY_GEN_IGNORE` * **ttl** | The default time-to-live (expiration) of the record in seconds. This field will only be used if an operate - | transaction's ``ttl`` option in the metadata dictionary is not set. + | transaction: + | 1. Doesn't contain a metadata dictionary with a ``ttl`` value. + | 2. Contains a metadata dictionary with a ``ttl`` value set to :data:`aerospike.TTL_CLIENT_DEFAULT`. + | + | There are also special values that can be set for this option. See :ref:`TTL_CONSTANTS`. * **replica** | One of the :ref:`POLICY_REPLICA` values such as :data:`aerospike.POLICY_REPLICA_MASTER` | @@ -1851,7 +1859,9 @@ Apply Policies | Default: :data:`aerospike.POLICY_COMMIT_LEVEL_ALL` * **ttl** | The default time-to-live (expiration) of the record in seconds. This field will only be used if an apply - | transaction's ``ttl`` option in the apply policy is not set. + | transaction doesn't have an apply policy with a ``ttl`` value that overrides this field. + | + | There are also special values that can be set for this field. See :ref:`TTL_CONSTANTS`. * **durable_delete** (:class:`bool`) | Perform durable delete | @@ -2095,10 +2105,16 @@ Batch Write Policies | | Default: None * **ttl** :class:`int` - | The time-to-live (expiration) in seconds to apply to every record in the batch. + | The time-to-live (expiration) in seconds to apply to every record in the batch. This field will only be + | used if a :meth:`~aerospike.Client.batch_write` call contains a :class:`~aerospike_helpers.batch.records.Write` that: + | + | 1. Doesn't contain a metadata dictionary with a ``ttl`` value. + | 2. Contains a metadata dictionary with a ``ttl`` value set to :data:`aerospike.TTL_CLIENT_DEFAULT`. | | The ttl must be a 32-bit unsigned integer, or a :exc:`~aerospike.exception.ParamError` will be raised. | + | There are also special values that can be set for this field. See :ref:`TTL_CONSTANTS`. + | | Default: ``0`` .. _aerospike_batch_apply_policies: @@ -2124,20 +2140,11 @@ Batch Apply Policies * **ttl** int | Time to live (expiration) of the record in seconds. | - | 0 which means that the - | record will adopt the default TTL value from the namespace. + | See :ref:`TTL_CONSTANTS` for possible special values. | - | 0xFFFFFFFF (also, -1 in a signed 32 bit int) - | which means that the record - | will get an internal "void_time" of zero, and thus will never expire. - | - | 0xFFFFFFFE (also, -2 in a signed 32 bit int) - | which means that the record - | - | ttl will not change when the record is updated. | Note that the TTL value will be employed ONLY on write/update calls. | - | Default: 0 + | Default: ``0`` * **durable_delete** :class:`bool` | If the transaction results in a record deletion, leave a tombstone for the record. This prevents deleted records from reappearing after node failures. Valid for Aerospike Server Enterprise Edition only. | diff --git a/doc/query.rst b/doc/query.rst index 2f76d3719..1d21dd369 100755 --- a/doc/query.rst +++ b/doc/query.rst @@ -80,20 +80,10 @@ Fields Default: ``0`` (no limit) ttl (:class:`int`) - The time-to-live (expiration) of the record in seconds. + The time-to-live (expiration) of the record in seconds. If set to :data:`aerospike.TTL_CLIENT_DEFAULT`, use the + client's default write policy ttl. - There are also special values that can be set in the record TTL: - - ``0`` (``TTL_NAMESPACE_DEFAULT``) - Which means that the record will adopt the default TTL value from the namespace. - - ``0xFFFFFFFF`` (``TTL_NEVER_EXPIRE``) - (also, ``-1`` in a signed 32 bit int) Which means that the record will never expire. - - ``0xFFFFFFFE`` (``TTL_DONT_UPDATE``) - (also, ``-2`` in a signed 32 bit int) - Which means that the record ttl will not change when the record is - updated. + See :ref:`TTL_CONSTANTS` for more possible special values. .. note:: Note that the TTL value will be employed ONLY on background query writes. diff --git a/doc/scan.rst b/doc/scan.rst index 04c1dacfa..5b79d77ba 100755 --- a/doc/scan.rst +++ b/doc/scan.rst @@ -576,6 +576,8 @@ Policies * **ttl** | The default time-to-live (expiration) of the record in seconds. This field will only be | used on background scan writes if :py:attr:`aerospike.Scan.ttl` is set to :data:`aerospike.TTL_CLIENT_DEFAULT`. + | + | There are also special values that can be set for this field. See :ref:`TTL_CONSTANTS`. .. _aerospike_scan_options: diff --git a/test/new_tests/test_new_constructor.py b/test/new_tests/test_new_constructor.py index cda0743f3..6767c25e1 100644 --- a/test/new_tests/test_new_constructor.py +++ b/test/new_tests/test_new_constructor.py @@ -207,12 +207,13 @@ def test_setting_batch_policies(): class TestConfigTTL: + NEW_TTL = 9000 + @pytest.fixture def config_ttl_setup(self, policy_name: str): config = copy.deepcopy(gconfig) - self.new_ttl = 9000 config["policies"][policy_name] = { - "ttl": self.new_ttl + "ttl": self.NEW_TTL } self.client = aerospike.client(config) self.key = ("test", "demo", 0) @@ -239,21 +240,30 @@ def config_ttl_setup(self, policy_name: str): def check_ttl(self): _, meta = self.client.exists(self.key) clock_skew_tolerance_secs = 50 - assert meta["ttl"] in range(self.new_ttl - clock_skew_tolerance_secs, self.new_ttl + clock_skew_tolerance_secs) + assert meta["ttl"] in range(self.NEW_TTL - clock_skew_tolerance_secs, self.NEW_TTL + clock_skew_tolerance_secs) @pytest.mark.parametrize("policy_name", ["write"]) - def test_setting_write_ttl(self, config_ttl_setup): - # Call without setting the ttl in the transaction metadata dict - self.client.put(self.key, bins={"a": 1}) + # The client's write policy ttl should be applied with no policy or a policy with the client default special value + @pytest.mark.parametrize( + "meta", + [None, {"ttl": aerospike.TTL_CLIENT_DEFAULT}], + ids=["no metadata", "metadata with special value"] + ) + def test_setting_write_ttl(self, config_ttl_setup, meta): + self.client.put(self.key, bins={"a": 1}, meta=meta) self.check_ttl() @pytest.mark.parametrize("policy_name", ["operate"]) - def test_setting_operate_ttl(self, config_ttl_setup): - # Call without setting the ttl in the transaction metadata dict + @pytest.mark.parametrize( + "meta", + [None, {"ttl": aerospike.TTL_CLIENT_DEFAULT}], + ids=["no metadata", "metadata with special value"] + ) + def test_setting_operate_ttl(self, config_ttl_setup, meta): ops = [ operations.write("a", 1) ] - self.client.operate(self.key, ops) + self.client.operate(self.key, ops, meta=meta) self.check_ttl() @pytest.mark.parametrize("policy_name", ["apply"]) @@ -267,14 +277,17 @@ def test_setting_apply_ttl(self, config_ttl_setup): self.check_ttl() @pytest.mark.parametrize("policy_name", ["batch_write"]) - def test_setting_batch_write_ttl(self, config_ttl_setup): - # self.client.put(self.key, {"bin": "a"}) - # Call without setting the ttl in the Write BatchRecord's metadata dict + @pytest.mark.parametrize( + "meta", + [None, {"ttl": aerospike.TTL_CLIENT_DEFAULT}], + ids=["no metadata", "metadata with special value"] + ) + def test_setting_batch_write_ttl(self, config_ttl_setup, meta): ops = [ operations.write("bin", 1) ] batch_records = BatchRecords([ - Write(self.key, ops=ops) + Write(self.key, ops=ops, meta=meta) ]) brs = self.client.batch_write(batch_records) # assert brs.result == 0 @@ -312,3 +325,21 @@ def test_setting_scan_ttl(self, config_ttl_setup): wait_for_job_completion(self.client, job_id) self.check_ttl() + + @pytest.mark.parametrize("policy_name", ["write"]) + def test_query_client_default_ttl(self, config_ttl_setup): + # Setup + self.client.put(self.key, {"bin": "a"}, meta={"ttl": 90}) + + # Tell scan to use client config's write policy ttl + query = self.client.query("test", "demo") + query.ttl = aerospike.TTL_CLIENT_DEFAULT + ops = [ + operations.append("bin", "a") + ] + query.add_ops(ops) + job_id = query.execute_background() + + wait_for_job_completion(self.client, job_id) + + self.check_ttl() From 14d1115e74915fafbc8e04f1cb756a36f9699980 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Tue, 7 Nov 2023 14:56:52 -0800 Subject: [PATCH 11/25] batch_operate(): take in ttl as parameter instead of through batch write policy --- doc/client.rst | 2 -- src/main/client/batch_operate.c | 53 ++++++++++++---------------- test/new_tests/test_batch_operate.py | 29 ++++++--------- 3 files changed, 33 insertions(+), 51 deletions(-) diff --git a/doc/client.rst b/doc/client.rst index 738948e8c..6c670309f 100755 --- a/doc/client.rst +++ b/doc/client.rst @@ -2111,8 +2111,6 @@ Batch Write Policies | 1. Doesn't contain a metadata dictionary with a ``ttl`` value. | 2. Contains a metadata dictionary with a ``ttl`` value set to :data:`aerospike.TTL_CLIENT_DEFAULT`. | - | The ttl must be a 32-bit unsigned integer, or a :exc:`~aerospike.exception.ParamError` will be raised. - | | There are also special values that can be set for this field. See :ref:`TTL_CONSTANTS`. | | Default: ``0`` diff --git a/src/main/client/batch_operate.c b/src/main/client/batch_operate.c index c294bee67..e2c73bfea 100644 --- a/src/main/client/batch_operate.c +++ b/src/main/client/batch_operate.c @@ -108,11 +108,13 @@ static bool batch_operate_cb(const as_batch_result *results, uint32_t n, * @param py_ops The list containing op dictionaries. * @param py_policy_batch Python dict used to populate policy_batch. * @param py_policy_batch_write Python dict used to populate policy_batch_write. + * @param py_ttl TTL value to set for each record. ******************************************************************************************************* */ static PyObject *AerospikeClient_Batch_Operate_Invoke( AerospikeClient *self, as_error *err, PyObject *py_keys, PyObject *py_ops, - PyObject *py_policy_batch, PyObject *py_policy_batch_write) + PyObject *py_policy_batch, PyObject *py_policy_batch_write, + PyObject *py_ttl) { long operation; long return_type = -1; @@ -220,31 +222,15 @@ static PyObject *AerospikeClient_Batch_Operate_Invoke( &batch_write_exp_list_p) != AEROSPIKE_OK) { goto CLEANUP; } + } - // The C client's batch write policy doesn't have a ttl option - // The correct way is to set the ttl inside the as_operations object - PyObject *py_ttl = PyDict_GetItemString(py_policy_batch_write, "ttl"); - Py_XINCREF(py_ttl); - // Default ttl - if (py_ttl != NULL) { - if (PyLong_Check(py_ttl)) { - long ttl = PyLong_AsLong(py_ttl); - if (ttl > UINT32_MAX || ttl < 0) { - as_error_update(err, AEROSPIKE_ERR_PARAM, - "ttl is out of range. It must be a 32 bit " - "unsigned integer."); - Py_DECREF(py_ttl); - goto CLEANUP; - } - ops.ttl = ttl; - } - } - else { - // If ttl in this transaction's batch write policy isn't set, use the client config's default batch write - // policy ttl - ops.ttl = AS_RECORD_CLIENT_DEFAULT_TTL; - } - Py_XDECREF(py_ttl); + if (py_ttl == NULL || py_ttl == Py_None) { + // If ttl in this transaction's batch write policy isn't set, use the client config's default batch write + // policy ttl + ops.ttl = AS_RECORD_CLIENT_DEFAULT_TTL; + } + else { + ops.ttl = (uint32_t)PyLong_AsLong(py_ttl); } // import batch_records helper @@ -360,15 +346,16 @@ PyObject *AerospikeClient_Batch_Operate(AerospikeClient *self, PyObject *args, PyObject *py_keys = NULL; PyObject *py_ops = NULL; PyObject *py_results = NULL; + PyObject *py_ttl = NULL; as_error_init(&err); // Python Function Keyword Arguments - static char *kwlist[] = {"keys", "ops", "policy_batch", - "policy_batch_write", NULL}; - if (PyArg_ParseTupleAndKeywords(args, kwds, "OO|OO:batch_Operate", kwlist, + static char *kwlist[] = { + "keys", "ops", "policy_batch", "policy_batch_write", "ttl", NULL}; + if (PyArg_ParseTupleAndKeywords(args, kwds, "OO|OOO:batch_Operate", kwlist, &py_keys, &py_ops, &py_policy_batch, - &py_policy_batch_write) == false) { + &py_policy_batch_write, &py_ttl) == false) { return NULL; } @@ -386,8 +373,14 @@ PyObject *AerospikeClient_Batch_Operate(AerospikeClient *self, PyObject *args, goto ERROR; } + if (py_ttl && py_ttl != Py_None && !PyLong_Check(py_ttl)) { + as_error_update(&err, AEROSPIKE_ERR_PARAM, "ttl should be an integer"); + goto ERROR; + } + py_results = AerospikeClient_Batch_Operate_Invoke( - self, &err, py_keys, py_ops, py_policy_batch, py_policy_batch_write); + self, &err, py_keys, py_ops, py_policy_batch, py_policy_batch_write, + py_ttl); return py_results; diff --git a/test/new_tests/test_batch_operate.py b/test/new_tests/test_batch_operate.py index 757f3b955..00cfb089a 100644 --- a/test/new_tests/test_batch_operate.py +++ b/test/new_tests/test_batch_operate.py @@ -58,7 +58,7 @@ def teardown(): request.addfinalizer(teardown) @pytest.mark.parametrize( - "name, keys, ops, policy_batch, policy_batch_write, exp_res, exp_rec", + "name, keys, ops, policy_batch, policy_batch_write, ttl, exp_res, exp_rec", [ ( "simple-write", @@ -66,6 +66,7 @@ def teardown(): [op.write("count", 2), op.read("count")], None, None, + None, [AerospikeStatus.AEROSPIKE_OK], [{"count": 2}], ), @@ -75,6 +76,7 @@ def teardown(): [op.write("count", 3), op.read("count")], {}, {}, + None, [AerospikeStatus.AEROSPIKE_OK], [{"count": 3}], ), @@ -94,6 +96,7 @@ def teardown(): ).compile(), }, {}, + None, [AerospikeStatus.AEROSPIKE_OK], [{"count": 7}], ), @@ -110,6 +113,7 @@ def teardown(): "durable_delete": False, "expressions": exp.Eq(exp.IntBin("count"), 0).compile(), }, + None, [AerospikeStatus.AEROSPIKE_OK], [{"count": 7}], ), @@ -121,9 +125,8 @@ def teardown(): op.read("count") ], {}, - { - "ttl": 200 - }, + {}, + 200, [AerospikeStatus.AEROSPIKE_OK], [{"count": 7}], ), @@ -150,17 +153,18 @@ def teardown(): "durable_delete": False, "expressions": exp.Eq(exp.IntBin("count"), 0).compile(), # this expression takes precedence }, + None, [AerospikeStatus.AEROSPIKE_OK], [{"count": 7}], ), ], ) - def test_batch_operate_pos(self, name, keys, ops, policy_batch, policy_batch_write, exp_res, exp_rec): + def test_batch_operate_pos(self, name, keys, ops, policy_batch, policy_batch_write, ttl, exp_res, exp_rec): """ Test batch_operate positive. """ - res = self.as_connection.batch_operate(keys, ops, policy_batch, policy_batch_write) + res = self.as_connection.batch_operate(keys, ops, policy_batch, policy_batch_write, ttl) for i, batch_rec in enumerate(res.batch_records): assert batch_rec.result == exp_res[i] @@ -254,19 +258,6 @@ def test_batch_operate_many_pos(self): ["bad-batch-write-policy"], e.ParamError, ), - ( - "bad-batch-write-policy-ttl", - [("test", "demo", 1)], - [ - op.write("count", 2), - ], - {}, - { - # Out of bounds - "ttl": 2**32 - }, - e.ParamError, - ), ], ) def test_batch_operate_neg(self, name, keys, ops, policy_batch, policy_batch_write, exp_res): From c55e09ae5c8ab3752a287c498dcbf0e36bc591e8 Mon Sep 17 00:00:00 2001 From: juliannguyen4 <109386615+juliannguyen4@users.noreply.github.com> Date: Tue, 7 Nov 2023 15:42:43 -0800 Subject: [PATCH 12/25] Remove confusing constraint for Scan.ttl --- doc/scan.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/scan.rst b/doc/scan.rst index 5b79d77ba..91a7dd980 100755 --- a/doc/scan.rst +++ b/doc/scan.rst @@ -29,7 +29,7 @@ Fields ttl (:class:`int`) The time-to-live (expiration) of the record in seconds. Note that ttl - is only used on background scan writes. This value must be a valid 32-bit unsigned integer. + is only used on background scan writes. See :ref:`TTL_CONSTANTS` for special values that can be set in the record ttl. From 17c6970b070fe5df57f2141f00b409febd8771c4 Mon Sep 17 00:00:00 2001 From: juliannguyen4 <109386615+juliannguyen4@users.noreply.github.com> Date: Tue, 7 Nov 2023 15:44:45 -0800 Subject: [PATCH 13/25] Add missing info for scan.ttl --- doc/scan.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/scan.rst b/doc/scan.rst index 91a7dd980..1e0234c97 100755 --- a/doc/scan.rst +++ b/doc/scan.rst @@ -31,6 +31,9 @@ Fields The time-to-live (expiration) of the record in seconds. Note that ttl is only used on background scan writes. + If this is set to :data:`aerospike.TTL_CLIENT_DEFAULT`, the scan will use the + client's default write policy ttl. + See :ref:`TTL_CONSTANTS` for special values that can be set in the record ttl. Default: ``0`` (no limit) From 5c95c37fc4662a2e35e567a9c122a049bb5cf6ad Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Tue, 7 Nov 2023 15:50:49 -0800 Subject: [PATCH 14/25] check ttl param is set in batch_operate() --- test/new_tests/test_batch_operate.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/new_tests/test_batch_operate.py b/test/new_tests/test_batch_operate.py index 00cfb089a..a0a5dacd4 100644 --- a/test/new_tests/test_batch_operate.py +++ b/test/new_tests/test_batch_operate.py @@ -126,7 +126,7 @@ def teardown(): ], {}, {}, - 200, + 9000, [AerospikeStatus.AEROSPIKE_OK], [{"count": 7}], ), @@ -172,6 +172,11 @@ def test_batch_operate_pos(self, name, keys, ops, policy_batch, policy_batch_wri assert batch_rec.key[:3] == keys[i] # checking key assert batch_rec.record[0][:3] == keys[i] # checking key in record + if ttl is not None: + for key in keys: + _, meta = self.as_connection.exists(key) + assert meta["ttl"] in range(9000 - 50, 9000 + 50) + def test_batch_operate_many_pos(self): """ Test batch operate with many keys. From 5d96388e8b52756171183cbe4ad01c5222e9b35d Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Tue, 7 Nov 2023 15:59:36 -0800 Subject: [PATCH 15/25] Add another test for client default batch write policy using batch_operate() --- aerospike-stubs/aerospike.pyi | 2 +- test/new_tests/test_new_constructor.py | 19 ++++++++++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/aerospike-stubs/aerospike.pyi b/aerospike-stubs/aerospike.pyi index 87cfef35a..edd248098 100644 --- a/aerospike-stubs/aerospike.pyi +++ b/aerospike-stubs/aerospike.pyi @@ -321,7 +321,7 @@ class Client: def apply(self, key: tuple, module: str, function: str, args: list, policy: dict = ...) -> Union[str, int, float, bytearray, list, dict]: ... def batch_apply(self, keys: list, module: str, function: str, args: list, policy_batch: dict = ..., policy_batch_apply: dict = ...) -> BatchRecords: ... def batch_get_ops(self, keys: list, ops: list, policy: dict) -> list: ... - def batch_operate(self, keys: list, ops: list, policy_batch: dict = ..., policy_batch_write: dict = ...) -> BatchRecords: ... + def batch_operate(self, keys: list, ops: list, policy_batch: dict = ..., policy_batch_write: dict = ..., ttl: int = ...) -> BatchRecords: ... def batch_remove(self, keys: list, policy_batch: dict = ..., policy_batch_remove: dict = ...) -> BatchRecords: ... def batch_read(self, keys: list, bins: list[str] = ..., policy_batch: dict = ...) -> BatchRecords: ... def batch_write(self, batch_records: BatchRecords, policy_batch: dict = ...) -> BatchRecords: ... diff --git a/test/new_tests/test_new_constructor.py b/test/new_tests/test_new_constructor.py index 6767c25e1..19a24f2b7 100644 --- a/test/new_tests/test_new_constructor.py +++ b/test/new_tests/test_new_constructor.py @@ -282,7 +282,7 @@ def test_setting_apply_ttl(self, config_ttl_setup): [None, {"ttl": aerospike.TTL_CLIENT_DEFAULT}], ids=["no metadata", "metadata with special value"] ) - def test_setting_batch_write_ttl(self, config_ttl_setup, meta): + def test_setting_batch_write_ttl_with_batch_write(self, config_ttl_setup, meta): ops = [ operations.write("bin", 1) ] @@ -296,6 +296,23 @@ def test_setting_batch_write_ttl(self, config_ttl_setup, meta): self.check_ttl() + @pytest.mark.parametrize("policy_name", ["batch_write"]) + @pytest.mark.parametrize( + "ttl", + [None, aerospike.TTL_CLIENT_DEFAULT], + ) + def test_setting_batch_write_ttl_with_batch_operate(self, ttl): + ops = [ + operations.write("bin", 1) + ] + keys = [self.key] + brs = self.client.batch_operate(ops, keys, ttl=ttl) + # assert brs.result == 0 + for br in brs.batch_records: + assert br.result == 0 + + self.check_ttl() + @pytest.mark.parametrize("policy_name", ["batch_apply"]) def test_setting_batch_apply_ttl(self, config_ttl_setup): # Setup From ec20b3988868f700c9259a17e50a72c69a2ba5b1 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Tue, 7 Nov 2023 16:16:30 -0800 Subject: [PATCH 16/25] fix batch operate test, WIP on fixing potential metadata bug --- src/main/conversions.c | 4 ++++ test/new_tests/test_new_constructor.py | 12 ++++++------ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/main/conversions.c b/src/main/conversions.c index eab328bb6..df91bf6e6 100644 --- a/src/main/conversions.c +++ b/src/main/conversions.c @@ -2186,6 +2186,10 @@ as_status check_and_set_meta(PyObject *py_meta, as_operations *ops, } ops->ttl = ttl; } + else { + // Metadata dict was present, but ttl field did not exist + ops->ttl = AS_RECORD_CLIENT_DEFAULT_TTL; + } if (py_gen) { if (PyLong_Check(py_gen)) { diff --git a/test/new_tests/test_new_constructor.py b/test/new_tests/test_new_constructor.py index 19a24f2b7..03bcdf238 100644 --- a/test/new_tests/test_new_constructor.py +++ b/test/new_tests/test_new_constructor.py @@ -246,8 +246,8 @@ def check_ttl(self): # The client's write policy ttl should be applied with no policy or a policy with the client default special value @pytest.mark.parametrize( "meta", - [None, {"ttl": aerospike.TTL_CLIENT_DEFAULT}], - ids=["no metadata", "metadata with special value"] + [None, {"ttl": aerospike.TTL_CLIENT_DEFAULT}, {"gen": 10}], + ids=["no metadata", "metadata with special ttl value", "metadata without ttl"] ) def test_setting_write_ttl(self, config_ttl_setup, meta): self.client.put(self.key, bins={"a": 1}, meta=meta) @@ -257,7 +257,7 @@ def test_setting_write_ttl(self, config_ttl_setup, meta): @pytest.mark.parametrize( "meta", [None, {"ttl": aerospike.TTL_CLIENT_DEFAULT}], - ids=["no metadata", "metadata with special value"] + ids=["no metadata", "metadata with special ttl value"] ) def test_setting_operate_ttl(self, config_ttl_setup, meta): ops = [ @@ -280,7 +280,7 @@ def test_setting_apply_ttl(self, config_ttl_setup): @pytest.mark.parametrize( "meta", [None, {"ttl": aerospike.TTL_CLIENT_DEFAULT}], - ids=["no metadata", "metadata with special value"] + ids=["no metadata", "metadata with special ttl value"] ) def test_setting_batch_write_ttl_with_batch_write(self, config_ttl_setup, meta): ops = [ @@ -301,12 +301,12 @@ def test_setting_batch_write_ttl_with_batch_write(self, config_ttl_setup, meta): "ttl", [None, aerospike.TTL_CLIENT_DEFAULT], ) - def test_setting_batch_write_ttl_with_batch_operate(self, ttl): + def test_setting_batch_write_ttl_with_batch_operate(self, config_ttl_setup, ttl): ops = [ operations.write("bin", 1) ] keys = [self.key] - brs = self.client.batch_operate(ops, keys, ttl=ttl) + brs = self.client.batch_operate(keys, ops, ttl=ttl) # assert brs.result == 0 for br in brs.batch_records: assert br.result == 0 From b14d3245c4b5f5f3e5df678118302892c3736a26 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Tue, 7 Nov 2023 16:34:50 -0800 Subject: [PATCH 17/25] Fix bug where meta without ttl doesn't apply default policy for put() --- src/main/conversions.c | 3 +++ test/new_tests/test_new_constructor.py | 7 +++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/main/conversions.c b/src/main/conversions.c index df91bf6e6..8a152a946 100644 --- a/src/main/conversions.c +++ b/src/main/conversions.c @@ -1118,6 +1118,9 @@ as_status pyobject_to_record(AerospikeClient *self, as_error *err, "TTL should be an int or long"); } } + else { + rec->ttl = AS_RECORD_CLIENT_DEFAULT_TTL; + } if (py_gen) { if (PyLong_Check(py_gen)) { diff --git a/test/new_tests/test_new_constructor.py b/test/new_tests/test_new_constructor.py index 03bcdf238..2f0397867 100644 --- a/test/new_tests/test_new_constructor.py +++ b/test/new_tests/test_new_constructor.py @@ -256,8 +256,11 @@ def test_setting_write_ttl(self, config_ttl_setup, meta): @pytest.mark.parametrize("policy_name", ["operate"]) @pytest.mark.parametrize( "meta", - [None, {"ttl": aerospike.TTL_CLIENT_DEFAULT}], - ids=["no metadata", "metadata with special ttl value"] + # The reason we also test a metadata dict without ttl for operate() + # is the codepath that handles the metadata dict for operate() is different + # from that for put() + [None, {"ttl": aerospike.TTL_CLIENT_DEFAULT}, {"gen": 10}], + ids=["no metadata", "metadata with special ttl value", "metadata without ttl"] ) def test_setting_operate_ttl(self, config_ttl_setup, meta): ops = [ From 8ef02fe918766d7e6f64e5f0db03b824b01ba858 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Tue, 7 Nov 2023 16:39:05 -0800 Subject: [PATCH 18/25] Update batch_operate() documentation --- doc/client.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/client.rst b/doc/client.rst index 6c670309f..75c9a0709 100755 --- a/doc/client.rst +++ b/doc/client.rst @@ -358,7 +358,7 @@ Batch Operations .. note:: Requires server version >= 6.0.0. - .. method:: batch_operate(keys: list, ops: list, [policy_batch: dict], [policy_batch_write: dict]) -> BatchRecords + .. method:: batch_operate(keys: list, ops: list, [policy_batch: dict], [policy_batch_write: dict], []) -> BatchRecords Perform the same read/write transactions on multiple keys. @@ -366,6 +366,7 @@ Batch Operations :param list ops: List of operations to apply. :param dict policy_batch: See :ref:`aerospike_batch_policies`. :param dict policy_batch_write: See :ref:`aerospike_batch_write_policies`. + :param int ttl: The time-to-live (expiration) of each record in seconds. :return: an instance of :class:`BatchRecords `. From 381cb6abbde47b4a20f346e0ddcb82fb94f2635d Mon Sep 17 00:00:00 2001 From: juliannguyen4 <109386615+juliannguyen4@users.noreply.github.com> Date: Tue, 7 Nov 2023 16:59:10 -0800 Subject: [PATCH 19/25] Remove confusing comment --- test/new_tests/test_new_constructor.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/new_tests/test_new_constructor.py b/test/new_tests/test_new_constructor.py index 2f0397867..335ec9dc2 100644 --- a/test/new_tests/test_new_constructor.py +++ b/test/new_tests/test_new_constructor.py @@ -243,7 +243,6 @@ def check_ttl(self): assert meta["ttl"] in range(self.NEW_TTL - clock_skew_tolerance_secs, self.NEW_TTL + clock_skew_tolerance_secs) @pytest.mark.parametrize("policy_name", ["write"]) - # The client's write policy ttl should be applied with no policy or a policy with the client default special value @pytest.mark.parametrize( "meta", [None, {"ttl": aerospike.TTL_CLIENT_DEFAULT}, {"gen": 10}], From a593bc4c1bd4125c4f2ea7a7b3a013fdfd1ae80d Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Tue, 7 Nov 2023 17:17:55 -0800 Subject: [PATCH 20/25] Fix documentation formatting, add classes --- doc/client.rst | 43 ++++++++++++++++++++++--------------------- doc/scan.rst | 2 +- 2 files changed, 23 insertions(+), 22 deletions(-) diff --git a/doc/client.rst b/doc/client.rst index 75c9a0709..9ae9e789a 100755 --- a/doc/client.rst +++ b/doc/client.rst @@ -1748,13 +1748,14 @@ Operate Policies | One of the :ref:`POLICY_GEN` values such as :data:`aerospike.POLICY_GEN_IGNORE` | | Default: :data:`aerospike.POLICY_GEN_IGNORE` - * **ttl** - | The default time-to-live (expiration) of the record in seconds. This field will only be used if an operate - | transaction: - | 1. Doesn't contain a metadata dictionary with a ``ttl`` value. - | 2. Contains a metadata dictionary with a ``ttl`` value set to :data:`aerospike.TTL_CLIENT_DEFAULT`. - | - | There are also special values that can be set for this option. See :ref:`TTL_CONSTANTS`. + * **ttl** (:class:`int`) + The default time-to-live (expiration) of the record in seconds. This field will only be used if an operate + transaction: + + 1. Doesn't contain a metadata dictionary with a ``ttl`` value. + 2. Contains a metadata dictionary with a ``ttl`` value set to :data:`aerospike.TTL_CLIENT_DEFAULT`. + + There are also special values that can be set for this option. See :ref:`TTL_CONSTANTS`. * **replica** | One of the :ref:`POLICY_REPLICA` values such as :data:`aerospike.POLICY_REPLICA_MASTER` | @@ -1858,11 +1859,11 @@ Apply Policies | One of the :ref:`POLICY_COMMIT_LEVEL` values such as :data:`aerospike.POLICY_COMMIT_LEVEL_ALL` | | Default: :data:`aerospike.POLICY_COMMIT_LEVEL_ALL` - * **ttl** - | The default time-to-live (expiration) of the record in seconds. This field will only be used if an apply - | transaction doesn't have an apply policy with a ``ttl`` value that overrides this field. - | - | There are also special values that can be set for this field. See :ref:`TTL_CONSTANTS`. + * **ttl** (:class:`int`) + The default time-to-live (expiration) of the record in seconds. This field will only be used if an apply + transaction doesn't have an apply policy with a ``ttl`` value that overrides this field. + + There are also special values that can be set for this field. See :ref:`TTL_CONSTANTS`. * **durable_delete** (:class:`bool`) | Perform durable delete | @@ -2106,15 +2107,15 @@ Batch Write Policies | | Default: None * **ttl** :class:`int` - | The time-to-live (expiration) in seconds to apply to every record in the batch. This field will only be - | used if a :meth:`~aerospike.Client.batch_write` call contains a :class:`~aerospike_helpers.batch.records.Write` that: - | - | 1. Doesn't contain a metadata dictionary with a ``ttl`` value. - | 2. Contains a metadata dictionary with a ``ttl`` value set to :data:`aerospike.TTL_CLIENT_DEFAULT`. - | - | There are also special values that can be set for this field. See :ref:`TTL_CONSTANTS`. - | - | Default: ``0`` + The time-to-live (expiration) in seconds to apply to every record in the batch. This field will only be + used if a :meth:`~aerospike.Client.batch_write` call contains a :class:`~aerospike_helpers.batch.records.Write` that: + + 1. Doesn't contain a metadata dictionary with a ``ttl`` value. + 2. Contains a metadata dictionary with a ``ttl`` value set to :data:`aerospike.TTL_CLIENT_DEFAULT`. + + There are also special values that can be set for this field. See :ref:`TTL_CONSTANTS`. + + Default: ``0`` .. _aerospike_batch_apply_policies: diff --git a/doc/scan.rst b/doc/scan.rst index 1e0234c97..8c2c25546 100755 --- a/doc/scan.rst +++ b/doc/scan.rst @@ -576,7 +576,7 @@ Policies | One of the :ref:`POLICY_REPLICA` values such as :data:`aerospike.POLICY_REPLICA_MASTER` | | Default: ``aerospike.POLICY_REPLICA_SEQUENCE`` - * **ttl** + * **ttl** (:class:`int`) | The default time-to-live (expiration) of the record in seconds. This field will only be | used on background scan writes if :py:attr:`aerospike.Scan.ttl` is set to :data:`aerospike.TTL_CLIENT_DEFAULT`. | From 75df23c92ec6ea49de0abc6a9dfc82fd7fe1b9ad Mon Sep 17 00:00:00 2001 From: juliannguyen4 <109386615+juliannguyen4@users.noreply.github.com> Date: Wed, 8 Nov 2023 09:42:05 -0800 Subject: [PATCH 21/25] Docs: fix missing ttl parameter in batch_operate() Co-authored-by: dwelch-spike <53876192+dwelch-spike@users.noreply.github.com> --- doc/client.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/client.rst b/doc/client.rst index 9ae9e789a..ffd9b68ed 100755 --- a/doc/client.rst +++ b/doc/client.rst @@ -358,7 +358,7 @@ Batch Operations .. note:: Requires server version >= 6.0.0. - .. method:: batch_operate(keys: list, ops: list, [policy_batch: dict], [policy_batch_write: dict], []) -> BatchRecords + .. method:: batch_operate(keys: list, ops: list, [policy_batch: dict], [policy_batch_write: dict], [ttl: int]) -> BatchRecords Perform the same read/write transactions on multiple keys. From 127418d2dbb2de58fba92087996cd96c363fd422 Mon Sep 17 00:00:00 2001 From: juliannguyen4 <109386615+juliannguyen4@users.noreply.github.com> Date: Wed, 8 Nov 2023 10:03:59 -0800 Subject: [PATCH 22/25] Improve formatting and clarify batch write policy ttl behavior --- doc/client.rst | 29 ++++++++++++++++++----------- doc/scan.rst | 9 +++++---- 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/doc/client.rst b/doc/client.rst index ffd9b68ed..13cd4b4dd 100755 --- a/doc/client.rst +++ b/doc/client.rst @@ -1577,12 +1577,13 @@ Write Policies | | Default: :data:`aerospike.POLICY_EXISTS_IGNORE` * **ttl** - | The default time-to-live (expiration) of the record in seconds. This field will only be used if the write - | transaction: - | 1. Doesn't contain a metadata dictionary with a ``ttl`` value. - | 2. Contains a metadata dictionary with a ``ttl`` value set to :data:`aerospike.TTL_CLIENT_DEFAULT`. - | - | There are also special values that can be set for this option. See :ref:`TTL_CONSTANTS`. + The default time-to-live (expiration) of the record in seconds. This field will only be used if + the write transaction: + + 1. Doesn't contain a metadata dictionary with a ``ttl`` value. + 2. Contains a metadata dictionary with a ``ttl`` value set to :data:`aerospike.TTL_CLIENT_DEFAULT`. + + There are also special values that can be set for this option. See :ref:`TTL_CONSTANTS`. * **gen** | One of the :ref:`POLICY_GEN` values such as :data:`aerospike.POLICY_GEN_IGNORE` | @@ -1749,8 +1750,8 @@ Operate Policies | | Default: :data:`aerospike.POLICY_GEN_IGNORE` * **ttl** (:class:`int`) - The default time-to-live (expiration) of the record in seconds. This field will only be used if an operate - transaction: + The default time-to-live (expiration) of the record in seconds. This field will only be used if an + operate transaction: 1. Doesn't contain a metadata dictionary with a ``ttl`` value. 2. Contains a metadata dictionary with a ``ttl`` value set to :data:`aerospike.TTL_CLIENT_DEFAULT`. @@ -2108,10 +2109,16 @@ Batch Write Policies | Default: None * **ttl** :class:`int` The time-to-live (expiration) in seconds to apply to every record in the batch. This field will only be - used if a :meth:`~aerospike.Client.batch_write` call contains a :class:`~aerospike_helpers.batch.records.Write` that: + used if: + 1. A :meth:`~aerospike.Client.batch_write` call contains a :class:`~aerospike_helpers.batch.records.Write` that: - 1. Doesn't contain a metadata dictionary with a ``ttl`` value. - 2. Contains a metadata dictionary with a ``ttl`` value set to :data:`aerospike.TTL_CLIENT_DEFAULT`. + a. Doesn't contain a metadata dictionary with a ``ttl`` value. + b. Contains a metadata dictionary with a ``ttl`` value set to :data:`aerospike.TTL_CLIENT_DEFAULT`. + + 2. A :meth:`~aerospike.Client.batch_operate` call: + + a. Doesn't pass in a `ttl` argument. + b. Passes in `aerospike.TTL_CLIENT_DEFAULT` to the `ttl` parameter. There are also special values that can be set for this field. See :ref:`TTL_CONSTANTS`. diff --git a/doc/scan.rst b/doc/scan.rst index 8c2c25546..71186c3d7 100755 --- a/doc/scan.rst +++ b/doc/scan.rst @@ -577,10 +577,11 @@ Policies | | Default: ``aerospike.POLICY_REPLICA_SEQUENCE`` * **ttl** (:class:`int`) - | The default time-to-live (expiration) of the record in seconds. This field will only be - | used on background scan writes if :py:attr:`aerospike.Scan.ttl` is set to :data:`aerospike.TTL_CLIENT_DEFAULT`. - | - | There are also special values that can be set for this field. See :ref:`TTL_CONSTANTS`. + The default time-to-live (expiration) of the record in seconds. This field will only be used on + background scan writes if :py:attr:`aerospike.Scan.ttl` is set to + :data:`aerospike.TTL_CLIENT_DEFAULT`. + + There are also special values that can be set for this field. See :ref:`TTL_CONSTANTS`. .. _aerospike_scan_options: From d7ae1138f16ad081ec953105da91299caa922034 Mon Sep 17 00:00:00 2001 From: juliannguyen4 <109386615+juliannguyen4@users.noreply.github.com> Date: Wed, 8 Nov 2023 10:22:36 -0800 Subject: [PATCH 23/25] Docs: Correct Scan.ttl behavior --- doc/scan.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/scan.rst b/doc/scan.rst index 71186c3d7..cccc04a91 100755 --- a/doc/scan.rst +++ b/doc/scan.rst @@ -32,7 +32,7 @@ Fields is only used on background scan writes. If this is set to :data:`aerospike.TTL_CLIENT_DEFAULT`, the scan will use the - client's default write policy ttl. + client's default scan policy ttl. See :ref:`TTL_CONSTANTS` for special values that can be set in the record ttl. From d6dd722a26990044b777ce809b3664848e242ad1 Mon Sep 17 00:00:00 2001 From: Julian Nguyen <109386615+juliannguyen4@users.noreply.github.com> Date: Wed, 8 Nov 2023 12:00:28 -0800 Subject: [PATCH 24/25] Add documentation for aerospike.TTL_CLIENT_DEFAULT --- doc/aerospike.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/doc/aerospike.rst b/doc/aerospike.rst index 9bdddbcb9..1db5681fe 100644 --- a/doc/aerospike.rst +++ b/doc/aerospike.rst @@ -858,6 +858,13 @@ Specifies the TTL constants. Do not change the current TTL of the record. +.. data:: TTL_CLIENT_DEFAULT + + NOTE: only applies to the policies mentioned below. + + Use the applicable client policy ttl in write, operate, batch write, and scan policies. + If the policy is not defined for the transaction, use the default client-level policy's ttl. + .. _auth_mode: Auth Mode Constants From e45a0ffd609272f1f6d7a88e5b17f7852490f2cf Mon Sep 17 00:00:00 2001 From: juliannguyen4 <109386615+juliannguyen4@users.noreply.github.com> Date: Wed, 8 Nov 2023 13:27:22 -0800 Subject: [PATCH 25/25] Remove confusing wording --- doc/aerospike.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/aerospike.rst b/doc/aerospike.rst index 1db5681fe..de3b2c7fe 100644 --- a/doc/aerospike.rst +++ b/doc/aerospike.rst @@ -862,7 +862,7 @@ Specifies the TTL constants. NOTE: only applies to the policies mentioned below. - Use the applicable client policy ttl in write, operate, batch write, and scan policies. + Use the applicable policy ttl in write, operate, batch write, and scan policies. If the policy is not defined for the transaction, use the default client-level policy's ttl. .. _auth_mode: