From 65753a9183fda11f1cf5ecc67b5b6e59a0989262 Mon Sep 17 00:00:00 2001
From: rik prohaska <prohaska7@gmail.com>
Date: Wed, 8 Apr 2020 16:53:56 -0400
Subject: [PATCH 01/29] glibc 2.30 defines the gettid system call, which
 conflicts with the tokuft test version.  rename tokuft test version to avoid
 conflict.

---
 portability/tests/test-xid.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/portability/tests/test-xid.cc b/portability/tests/test-xid.cc
index 9ee68906b..6143d3b58 100644
--- a/portability/tests/test-xid.cc
+++ b/portability/tests/test-xid.cc
@@ -54,7 +54,7 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved.
 
 // since we implement the same thing here as in toku_os_gettid, this test
 // is pretty pointless
-static int gettid(void) {
+static int test_gettid(void) {
 #if defined(__NR_gettid)
     return syscall(__NR_gettid);
 #elif defined(SYS_gettid)
@@ -68,6 +68,6 @@ static int gettid(void) {
 
 int main(void) {
     assert(toku_os_getpid() == getpid());
-    assert(toku_os_gettid() == gettid());
+    assert(toku_os_gettid() == test_gettid());
     return 0;
 }

From 1bbdec19277391c550e2897dc7ce7d0fccf6317a Mon Sep 17 00:00:00 2001
From: rik prohaska <prohaska7@gmail.com>
Date: Wed, 8 Apr 2020 20:57:46 -0400
Subject: [PATCH 02/29] Remove ctest timeout property so that we can use the
 ctest --timeout parameter instead.

---
 ft/tests/CMakeLists.txt   | 18 -----------
 src/tests/CMakeLists.txt  | 63 +--------------------------------------
 util/tests/CMakeLists.txt |  4 ---
 3 files changed, 1 insertion(+), 84 deletions(-)

diff --git a/ft/tests/CMakeLists.txt b/ft/tests/CMakeLists.txt
index 270ec9766..73930dda6 100644
--- a/ft/tests/CMakeLists.txt
+++ b/ft/tests/CMakeLists.txt
@@ -123,22 +123,4 @@ if(BUILD_TESTING OR BUILD_FT_TESTS)
     get_filename_component(test_basename "${test}" NAME)
     add_ft_test_aux(test-${test_basename} test-upgrade-recovery-logs ${test})
   endforeach(test)
-
-  ## give some tests, that time out normally, 1 hour to complete
-  set(long_tests
-    ft/ftloader-test-extractor-3a
-    ft/log-test7
-    ft/recovery-bad-last-entry
-    ft/subblock-test-compression
-    ft/upgrade_test_simple
-    )
-  set_tests_properties(${long_tests} PROPERTIES TIMEOUT 3600)
-  ## some take even longer, with valgrind
-  set(extra_long_tests
-    ft/benchmark-test
-    ft/benchmark-test_256
-    ft/is_empty
-    ft/subblock-test-checksum
-    )
-  set_tests_properties(${extra_long_tests} PROPERTIES TIMEOUT 7200)
 endif(BUILD_TESTING OR BUILD_FT_TESTS)
diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt
index c01a8f0d6..1a3d3e0d8 100644
--- a/src/tests/CMakeLists.txt
+++ b/src/tests/CMakeLists.txt
@@ -197,7 +197,6 @@ if(BUILD_TESTING OR BUILD_SRC_TESTS)
       endif ()
 
       add_ydb_drd_test_aux(drd_tiny_${test} ${test} --num_seconds 5 --num_elements 150 --join_timeout 3000)
-      set_tests_properties(ydb/drd_tiny_${test} PROPERTIES TIMEOUT 3600)
 
       add_test(ydb/drd_mid_${test}/prepare ${test} --only_create --num_elements 10000)
       setup_toku_test_properties(ydb/drd_mid_${test}/prepare drd_mid_${test})
@@ -205,7 +204,6 @@ if(BUILD_TESTING OR BUILD_SRC_TESTS)
       set_tests_properties(ydb/drd_mid_${test} PROPERTIES
         DEPENDS ydb/drd_mid_${test}/prepare
         REQUIRED_FILES "drd_mid_${test}.ctest-data"
-        TIMEOUT 15000
         )
 
       add_test(ydb/drd_large_${test}/prepare ${test} --only_create --num_elements 150000)
@@ -214,7 +212,6 @@ if(BUILD_TESTING OR BUILD_SRC_TESTS)
       set_tests_properties(ydb/drd_large_${test} PROPERTIES
         DEPENDS ydb/drd_large_${test}/prepare
         REQUIRED_FILES "drd_large_${test}.ctest-data"
-        TIMEOUT 30000
         )
     endif()
   endforeach(src)
@@ -252,8 +249,7 @@ if(BUILD_TESTING OR BUILD_SRC_TESTS)
               setup_toku_test_properties(${testnamebase} "${envdirbase}")
               set_tests_properties(${testnamebase} PROPERTIES
                 DEPENDS ${testnamebase}/copy
-                REQUIRED_FILES "${envdir}"
-                TIMEOUT 10800)
+                REQUIRED_FILES "${envdir}")
             endforeach(size)
           endif ()
         endforeach(p_or_s)
@@ -433,61 +429,4 @@ if(BUILD_TESTING OR BUILD_SRC_TESTS)
   string(REGEX REPLACE ";" ";ydb/" tdb_tests_that_should_fail "${tdb_tests_that_should_fail}")
   set_tests_properties(${tdb_tests_that_should_fail} PROPERTIES WILL_FAIL TRUE)
 
-  ## give some tests, that time out normally, 1 hour to complete
-  set(long_tests
-    ydb/drd_test_groupcommit_count.tdb
-    ydb/env-put-multiple.tdb
-    ydb/filesize.tdb
-    ydb/loader-cleanup-test0.tdb
-    ydb/loader-cleanup-test0z.tdb
-    ydb/manyfiles.tdb
-    ydb/recover-loader-test.abortrecover
-    ydb/recovery_fileops_stress.tdb
-    ydb/root_fifo_1.tdb
-    ydb/root_fifo_2.tdb
-    ydb/root_fifo_31.tdb
-    ydb/root_fifo_32.tdb
-    ydb/shutdown-3344.tdb
-    ydb/stat64-create-modify-times.tdb
-    ydb/test1572.tdb
-    ydb/test_abort4_19_0.tdb
-    ydb/test_abort4_19_1.tdb
-    ydb/test_abort5.tdb
-    ydb/test_archive1.tdb
-    ydb/test_logmax.tdb
-    ydb/test_query.tdb
-    ydb/test_txn_abort5.tdb
-    ydb/test_txn_abort5a.tdb
-    ydb/test_txn_abort6.tdb
-    ydb/test_txn_nested2.tdb
-    ydb/test_txn_nested4.tdb
-    ydb/test_txn_nested5.tdb
-    ydb/test_update_broadcast_stress.tdb
-    )
-  set_tests_properties(${long_tests} PROPERTIES TIMEOUT 3600)
-  ## some take even longer, with valgrind
-  set(extra_long_tests
-    ydb/drd_test_4015.tdb
-    ydb/hotindexer-with-queries.tdb
-    ydb/hot-optimize-table-tests.tdb
-    ydb/loader-cleanup-test2.tdb
-    ydb/loader-cleanup-test2z.tdb
-    ydb/loader-dup-test0.tdb
-    ydb/loader-stress-del.nop.loader
-    ydb/loader-stress-del.p.loader
-    ydb/loader-stress-del.comp.loader
-    ydb/test3039.tdb
-    ydb/test_update_stress.tdb
-    )
-  set_tests_properties(${extra_long_tests} PROPERTIES TIMEOUT 7200)
-  ## these really take a long time with valgrind
-  set(phenomenally_long_tests
-    ydb/checkpoint_stress.tdb
-    ydb/loader-stress-test4.tdb
-    ydb/loader-stress-test4z.tdb
-    ydb/recover_stress.tdb
-    ydb/test3529.tdb
-    ydb/test_insert_unique.tdb
-    )
-  set_tests_properties(${phenomenally_long_tests} PROPERTIES TIMEOUT 14400)
 endif(BUILD_TESTING OR BUILD_SRC_TESTS)
diff --git a/util/tests/CMakeLists.txt b/util/tests/CMakeLists.txt
index 8d53dd89a..780bf8d66 100644
--- a/util/tests/CMakeLists.txt
+++ b/util/tests/CMakeLists.txt
@@ -17,8 +17,4 @@ if(BUILD_TESTING)
     add_test(util/${test} ${test})
   endforeach(test)
 
-  set(long_tests
-    util/helgrind_test_partitioned_counter
-    )
-  set_tests_properties(${long_tests} PROPERTIES TIMEOUT 3600)
 endif(BUILD_TESTING)

From 9be7dda9da3366624a502da2f9d9356d9f1f89b3 Mon Sep 17 00:00:00 2001
From: rik prohaska <prohaska7@gmail.com>
Date: Thu, 9 Apr 2020 08:11:48 -0400
Subject: [PATCH 03/29] Fix ydb tests that fail due to loose file permissions. 
 Note that file permissions were changed in commit 5aca29f.

---
 src/tests/test_memcmp_magic.cc        | 2 +-
 src/tests/xa-bigtxn-discard-abort.cc  | 4 ++--
 src/tests/xa-bigtxn-discard-commit.cc | 4 ++--
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/tests/test_memcmp_magic.cc b/src/tests/test_memcmp_magic.cc
index 8b56e716a..21e131bc0 100644
--- a/src/tests/test_memcmp_magic.cc
+++ b/src/tests/test_memcmp_magic.cc
@@ -103,7 +103,7 @@ static void test_memcmp_magic_sort_order(void) {
     DB_ENV *env;
     r = db_env_create(&env, 0); CKERR(r);
     r = env->set_default_bt_compare(env, comparison_function_unused); CKERR(r);
-    r = env->open(env, TOKU_TEST_FILENAME, DB_CREATE+DB_PRIVATE+DB_INIT_MPOOL+DB_INIT_TXN, 0); CKERR(r);
+    r = env->open(env, TOKU_TEST_FILENAME, DB_CREATE+DB_PRIVATE+DB_INIT_MPOOL+DB_INIT_TXN, S_IRUSR+S_IWUSR); CKERR(r);
 
     const int magic = 49;
 
diff --git a/src/tests/xa-bigtxn-discard-abort.cc b/src/tests/xa-bigtxn-discard-abort.cc
index 9f1b904df..4163d9918 100644
--- a/src/tests/xa-bigtxn-discard-abort.cc
+++ b/src/tests/xa-bigtxn-discard-abort.cc
@@ -62,7 +62,7 @@ static void populate_foo(DB_ENV *env, DB_TXN *txn) {
     DB *db = nullptr;
     r = db_create(&db, env, 0);
     CKERR(r);
-    r = db->open(db, txn, "foo.db", 0, DB_BTREE, 0, 0);
+    r = db->open(db, txn, "foo.db", 0, DB_BTREE, 0, S_IRWXU);
     CKERR(r);
 
     for (int i = 0; i < test_nrows; i++) {
@@ -81,7 +81,7 @@ static void check_foo(DB_ENV *env, DB_TXN *txn) {
     DB *db;
     r = db_create(&db, env, 0);
     CKERR(r);
-    r = db->open(db, txn, "foo.db", 0, DB_BTREE, 0, 0);
+    r = db->open(db, txn, "foo.db", 0, DB_BTREE, 0, S_IRWXU);
     CKERR(r);
 
     DBC *c = nullptr;
diff --git a/src/tests/xa-bigtxn-discard-commit.cc b/src/tests/xa-bigtxn-discard-commit.cc
index ecbfa18bd..733df036e 100644
--- a/src/tests/xa-bigtxn-discard-commit.cc
+++ b/src/tests/xa-bigtxn-discard-commit.cc
@@ -59,7 +59,7 @@ static void populate_foo(DB_ENV *env, DB_TXN *txn) {
     DB *db = nullptr;
     r = db_create(&db, env, 0);
     CKERR(r);
-    r = db->open(db, txn, "foo.db", 0, DB_BTREE, 0, 0);
+    r = db->open(db, txn, "foo.db", 0, DB_BTREE, 0, S_IRWXU);
     CKERR(r);
 
     for (int i = 0; i < test_nrows; i++) {
@@ -78,7 +78,7 @@ static void check_foo(DB_ENV *env, DB_TXN *txn) {
     DB *db;
     r = db_create(&db, env, 0);
     CKERR(r);
-    r = db->open(db, txn, "foo.db", 0, DB_BTREE, 0, 0);
+    r = db->open(db, txn, "foo.db", 0, DB_BTREE, 0, S_IRWXU);
     CKERR(r);
 
     DBC *c = nullptr;

From 9af3125fd91a9c26ba0351cf67f30e8805611152 Mon Sep 17 00:00:00 2001
From: rik prohaska <prohaska7@gmail.com>
Date: Thu, 9 Apr 2020 08:38:43 -0400
Subject: [PATCH 04/29] Remove obsolete debug code that was left in the tests
 by accident.

---
 locktree/lock_request.cc                               | 4 ----
 locktree/lock_request.h                                | 1 -
 locktree/tests/lock_request_start_retry_race.cc        | 1 -
 locktree/tests/lock_request_start_retry_race_3.cc      | 1 -
 locktree/tests/lock_request_start_retry_wait_race_2.cc | 1 -
 5 files changed, 8 deletions(-)

diff --git a/locktree/lock_request.cc b/locktree/lock_request.cc
index 19ec146a3..b99eaff3d 100644
--- a/locktree/lock_request.cc
+++ b/locktree/lock_request.cc
@@ -93,10 +93,6 @@ void lock_request::destroy(void) {
     toku_cond_destroy(&m_wait_cond);
 }
 
-void lock_request::clearmem(char c) {
-     memset(this, c, sizeof(* this));
-}
-
 // set the lock request parameters. this API allows a lock request to be reused.
 void lock_request::set(locktree *lt, TXNID txnid, const DBT *left_key, const DBT *right_key, lock_request::type lock_type, bool big_txn, void *extra) {
     invariant(m_state != state::PENDING);
diff --git a/locktree/lock_request.h b/locktree/lock_request.h
index 36c3fd260..76ac953ba 100644
--- a/locktree/lock_request.h
+++ b/locktree/lock_request.h
@@ -89,7 +89,6 @@ class lock_request {
 
     // effect: Destroys a lock request.
     void destroy(void);
-    void clearmem(char c);
 
     // effect: Resets the lock request parameters, allowing it to be reused.
     // requires: Lock request was already created at some point
diff --git a/locktree/tests/lock_request_start_retry_race.cc b/locktree/tests/lock_request_start_retry_race.cc
index 83436a651..374bc8625 100644
--- a/locktree/tests/lock_request_start_retry_race.cc
+++ b/locktree/tests/lock_request_start_retry_race.cc
@@ -83,7 +83,6 @@ namespace toku {
             }
 
             request.destroy();
-            request.clearmem(0xab);
 
             toku_pthread_yield();
             if ((i % 10) == 0)
diff --git a/locktree/tests/lock_request_start_retry_race_3.cc b/locktree/tests/lock_request_start_retry_race_3.cc
index 288cb0855..4c900765f 100644
--- a/locktree/tests/lock_request_start_retry_race_3.cc
+++ b/locktree/tests/lock_request_start_retry_race_3.cc
@@ -96,7 +96,6 @@ namespace toku {
             }
 
             request.destroy();
-            request.clearmem(0xab);
 
             toku_pthread_yield();
             if ((i % 10) == 0)
diff --git a/locktree/tests/lock_request_start_retry_wait_race_2.cc b/locktree/tests/lock_request_start_retry_wait_race_2.cc
index cd3dc7b37..ce74dc3ab 100644
--- a/locktree/tests/lock_request_start_retry_wait_race_2.cc
+++ b/locktree/tests/lock_request_start_retry_wait_race_2.cc
@@ -98,7 +98,6 @@ namespace toku {
             }
 
             request.destroy();
-            request.clearmem(0xab);
 
             toku_pthread_yield();
             if ((i % 10) == 0)

From 64b819c89307cb8d10f5a24e1b5fac11f90e6c7c Mon Sep 17 00:00:00 2001
From: rik prohaska <prohaska7@gmail.com>
Date: Thu, 9 Apr 2020 08:42:14 -0400
Subject: [PATCH 05/29] Add locktree and ydb tests cases that expose the lock
 request retry bug when the lock memory use is over the limit.  The ydb tests
 show that DB->put normally returns TOKUDB_OUT_OF_LOCKS when the lock memory
 use is over the limit.

---
 locktree/tests/lock_request_create_set.cc     |   4 +-
 locktree/tests/lock_request_get_set_keys.cc   |   4 +-
 locktree/tests/lock_request_killed.cc         |   4 +-
 locktree/tests/lock_request_not_killed.cc     |   4 +-
 .../tests/lock_request_retry_out_of_locks.cc  | 117 +++++++++++++
 locktree/tests/lock_request_start_deadlock.cc |   4 +-
 locktree/tests/lock_request_start_pending.cc  |   4 +-
 locktree/tests/lock_request_unit_test.h       |  18 +-
 .../tests/lock_request_wait_out_of_locks.cc   | 114 ++++++++++++
 .../tests/lock_request_wait_time_callback.cc  |   4 +-
 locktree/tests/test.h                         |   4 +-
 src/tests/put-wait-retry-out-of-locks.cc      | 164 ++++++++++++++++++
 src/tests/simple-put-out-of-locks.cc          | 125 +++++++++++++
 13 files changed, 537 insertions(+), 33 deletions(-)
 create mode 100644 locktree/tests/lock_request_retry_out_of_locks.cc
 create mode 100644 locktree/tests/lock_request_wait_out_of_locks.cc
 create mode 100644 src/tests/put-wait-retry-out-of-locks.cc
 create mode 100644 src/tests/simple-put-out-of-locks.cc

diff --git a/locktree/tests/lock_request_create_set.cc b/locktree/tests/lock_request_create_set.cc
index 8ae685b98..cb0d9aa7d 100644
--- a/locktree/tests/lock_request_create_set.cc
+++ b/locktree/tests/lock_request_create_set.cc
@@ -41,7 +41,7 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved.
 namespace toku {
 
 // create and set the object's internals, destroy should not crash.
-void lock_request_unit_test::test_create_destroy(void) {
+void lock_request_unit_test::run(void) {
     lock_request request;
     request.create();
 
@@ -66,7 +66,7 @@ void lock_request_unit_test::test_create_destroy(void) {
 
 int main(void) {
     toku::lock_request_unit_test test;
-    test.test_create_destroy();
+    test.run();
     return 0;
 }
 
diff --git a/locktree/tests/lock_request_get_set_keys.cc b/locktree/tests/lock_request_get_set_keys.cc
index fd57b70f5..56b785dfe 100644
--- a/locktree/tests/lock_request_get_set_keys.cc
+++ b/locktree/tests/lock_request_get_set_keys.cc
@@ -43,7 +43,7 @@ namespace toku {
 // make setting keys and getting them back works properly.
 // at a high level, we want to make sure keys are copied
 // when appropriate and plays nice with +/- infinity.
-void lock_request_unit_test::test_get_set_keys(void) {
+void lock_request_unit_test::run(void) {
     lock_request request;
     request.create();
 
@@ -82,7 +82,7 @@ void lock_request_unit_test::test_get_set_keys(void) {
 
 int main(void) {
     toku::lock_request_unit_test test;
-    test.test_get_set_keys();
+    test.run();
     return 0;
 }
 
diff --git a/locktree/tests/lock_request_killed.cc b/locktree/tests/lock_request_killed.cc
index ec4644442..404550620 100644
--- a/locktree/tests/lock_request_killed.cc
+++ b/locktree/tests/lock_request_killed.cc
@@ -63,7 +63,7 @@ static int my_killed_callback(void) {
 }
 
 // make sure deadlocks are detected when a lock request starts
-void lock_request_unit_test::test_wait_time_callback(void) {
+void lock_request_unit_test::run(void) {
     int r;
     locktree lt;
 
@@ -118,7 +118,7 @@ void lock_request_unit_test::test_wait_time_callback(void) {
 
 int main(void) {
     toku::lock_request_unit_test test;
-    test.test_wait_time_callback();
+    test.run();
     return 0;
 }
 
diff --git a/locktree/tests/lock_request_not_killed.cc b/locktree/tests/lock_request_not_killed.cc
index 647b4d3c4..9ec9a57fe 100644
--- a/locktree/tests/lock_request_not_killed.cc
+++ b/locktree/tests/lock_request_not_killed.cc
@@ -58,7 +58,7 @@ static int my_killed_callback(void) {
 }
 
 // make sure deadlocks are detected when a lock request starts
-void lock_request_unit_test::test_wait_time_callback(void) {
+void lock_request_unit_test::run(void) {
     int r;
     locktree lt;
 
@@ -112,7 +112,7 @@ void lock_request_unit_test::test_wait_time_callback(void) {
 
 int main(void) {
     toku::lock_request_unit_test test;
-    test.test_wait_time_callback();
+    test.run();
     return 0;
 }
 
diff --git a/locktree/tests/lock_request_retry_out_of_locks.cc b/locktree/tests/lock_request_retry_out_of_locks.cc
new file mode 100644
index 000000000..d98417858
--- /dev/null
+++ b/locktree/tests/lock_request_retry_out_of_locks.cc
@@ -0,0 +1,117 @@
+/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
+
+// Verify that lock request retry returns TOKUDB_OUT_OF_LOCKS when
+// all of the locktree memory is used.
+
+#include "lock_request_unit_test.h"
+
+namespace toku {
+
+    static void locktree_release_lock(locktree *lt,
+                                      TXNID txn_id,
+                                      const DBT *left,
+                                      const DBT *right) {
+        range_buffer buffer;
+        buffer.create();
+        buffer.append(left, right);
+        lt->release_locks(txn_id, &buffer);
+        buffer.destroy();
+    }
+
+    void lock_request_unit_test::run(void) {
+        int r;
+
+        locktree_manager mgr;
+        mgr.create(nullptr, nullptr, nullptr, nullptr);
+
+        DICTIONARY_ID dict_id = {1};
+        locktree *lt = mgr.get_lt(dict_id, dbt_comparator, nullptr);
+
+        // set max lock memory small so that we can test the limit
+        // with just 2 locks
+        mgr.set_max_lock_memory(300);
+
+        // create a small key
+        DBT small_dbt;
+        int64_t small_key = 1;
+        toku_fill_dbt(&small_dbt, &small_key, sizeof small_key);
+        small_dbt.flags = DB_DBT_USERMEM;
+        const DBT *small_ptr = &small_dbt;
+
+        // create a large key
+        DBT large_dbt;
+        union { int64_t n; char c[64]; } large_key;
+        memset(&large_key, 0, sizeof large_key);
+        large_key.n = 2;
+        toku_fill_dbt(&large_dbt, &large_key, sizeof large_key);
+        large_dbt.flags = DB_DBT_USERMEM;
+        const DBT *large_dbt_ptr = &large_dbt;
+
+        TXNID txn_a = { 1 };
+        TXNID txn_b = { 2 };
+
+        // a locks small key
+        lock_request a;
+        a.create();
+        a.set(lt, txn_a, small_ptr, small_ptr, lock_request::type::WRITE, false);
+        r = a.start();
+        assert(r == 0);
+        assert(a.m_state == lock_request::state::COMPLETE);
+
+        // b tries to lock small key, fails since it is already locked
+        lock_request b;
+        b.create();
+        b.set(lt, txn_b, small_ptr, small_ptr, lock_request::type::WRITE, false);
+        r = b.start();
+        assert(r == DB_LOCK_NOTGRANTED);
+        assert(b.m_state == lock_request::state::PENDING);
+
+        // a locks large key. lock memory is over the limit
+        lock_request c;
+        c.create();
+        c.set(lt, txn_a, large_dbt_ptr, large_dbt_ptr, lock_request::type::WRITE, false);
+        r = c.start();
+        assert(r == 0);
+        assert(c.m_state == lock_request::state::COMPLETE);
+
+        // a releases small key, lock memory is still over the limit
+        locktree_release_lock(lt, txn_a, small_ptr, small_ptr);
+
+        // retry all lock requests, should complete lock request
+        // b with a TOKUDB_OUT_OF_LOCKS result
+        lock_request::retry_all_lock_requests(lt);
+
+        assert(b.m_state == lock_request::state::COMPLETE);
+        assert(b.m_complete_r == TOKUDB_OUT_OF_LOCKS);
+
+        // b waits for small key, gets out of locks
+        r = b.wait(0);
+        assert(r == TOKUDB_OUT_OF_LOCKS);
+        assert(b.m_state == lock_request::state::COMPLETE);
+        assert(b.m_complete_r == TOKUDB_OUT_OF_LOCKS);
+
+        // a releases large key
+        locktree_release_lock(lt, txn_a, large_dbt_ptr, large_dbt_ptr);
+
+        // b locks small key, gets its
+        r = b.start();
+        assert(r == 0);
+
+        // b releases lock so we can exit cleanly
+        locktree_release_lock(lt, txn_b, small_ptr, small_ptr);
+
+        a.destroy();
+        b.destroy();
+
+        mgr.release_lt(lt);
+        mgr.destroy();
+    }
+
+} /* namespace toku */
+
+int main(void) {
+    toku::lock_request_unit_test test;
+    test.run();
+    return 0;
+}
diff --git a/locktree/tests/lock_request_start_deadlock.cc b/locktree/tests/lock_request_start_deadlock.cc
index 343becfc7..c13d5b085 100644
--- a/locktree/tests/lock_request_start_deadlock.cc
+++ b/locktree/tests/lock_request_start_deadlock.cc
@@ -41,7 +41,7 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved.
 namespace toku {
 
 // make sure deadlocks are detected when a lock request starts
-void lock_request_unit_test::test_start_deadlock(void) {
+void lock_request_unit_test::run(void) {
     int r;
     locktree lt;
 
@@ -114,7 +114,7 @@ void lock_request_unit_test::test_start_deadlock(void) {
 
 int main(void) {
     toku::lock_request_unit_test test;
-    test.test_start_deadlock();
+    test.run();
     return 0;
 }
 
diff --git a/locktree/tests/lock_request_start_pending.cc b/locktree/tests/lock_request_start_pending.cc
index ce6510354..6c133b7a0 100644
--- a/locktree/tests/lock_request_start_pending.cc
+++ b/locktree/tests/lock_request_start_pending.cc
@@ -42,7 +42,7 @@ namespace toku {
 
 // starting a lock request without immediate success should get
 // stored in the lock request set as pending.
-void lock_request_unit_test::test_start_pending(void) {
+void lock_request_unit_test::run(void) {
     int r;
     locktree lt;
     lock_request request;
@@ -100,7 +100,7 @@ void lock_request_unit_test::test_start_pending(void) {
 
 int main(void) {
     toku::lock_request_unit_test test;
-    test.test_start_pending();
+    test.run();
     return 0;
 }
 
diff --git a/locktree/tests/lock_request_unit_test.h b/locktree/tests/lock_request_unit_test.h
index 81e6db25b..c5ad7aae4 100644
--- a/locktree/tests/lock_request_unit_test.h
+++ b/locktree/tests/lock_request_unit_test.h
@@ -47,23 +47,7 @@ namespace toku {
 
 class lock_request_unit_test {
 public:
-    // create and set the object's internals, destroy should not crash.
-    void test_create_destroy(void);
-
-    // make setting keys and getting them back works properly.
-    // at a high level, we want to make sure keys are copied
-    // when appropriate and plays nice with +/- infinity.
-    void test_get_set_keys(void);
-
-    // starting a lock request without immediate success should get
-    // stored in the lock request set as pending.
-    void test_start_pending(void);
-
-    // make sure deadlocks are detected when a lock request starts
-    void test_start_deadlock(void);
-
-    // test that the get_wait_time callback works
-    void test_wait_time_callback(void);
+    void run(void);
 
 private:
     // releases a single range lock and retries all lock requests.
diff --git a/locktree/tests/lock_request_wait_out_of_locks.cc b/locktree/tests/lock_request_wait_out_of_locks.cc
new file mode 100644
index 000000000..1c3750065
--- /dev/null
+++ b/locktree/tests/lock_request_wait_out_of_locks.cc
@@ -0,0 +1,114 @@
+/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
+
+// Verify that lock request wait returns TOKUDB_OUT_OF_LOCKS when
+// all of the locktree memory is used.
+
+#include "lock_request_unit_test.h"
+
+namespace toku {
+
+    static void locktree_release_lock(locktree *lt,
+                                      TXNID txn_id,
+                                      const DBT *left,
+                                      const DBT *right) {
+        range_buffer buffer;
+        buffer.create();
+        buffer.append(left, right);
+        lt->release_locks(txn_id, &buffer);
+        buffer.destroy();
+    }
+
+    void lock_request_unit_test::run(void) {
+        int r;
+
+        locktree_manager mgr;
+        mgr.create(nullptr, nullptr, nullptr, nullptr);
+
+        DICTIONARY_ID dict_id = {1};
+        locktree *lt = mgr.get_lt(dict_id, dbt_comparator, nullptr);
+
+        // set max lock memory small so that we can test the limit
+        // with just 2 locks
+        mgr.set_max_lock_memory(300);
+
+        // create a small key
+        DBT small_dbt;
+        int64_t small_key = 1;
+        toku_fill_dbt(&small_dbt, &small_key, sizeof small_key);
+        small_dbt.flags = DB_DBT_USERMEM;
+        const DBT *small_dbt_ptr = &small_dbt;
+
+        // create a large key
+        DBT large_dbt;
+        union { int64_t n; char c[64]; } large_key;
+        memset(&large_key, 0, sizeof large_key);
+        large_key.n = 2;
+        toku_fill_dbt(&large_dbt, &large_key, sizeof large_key);
+        large_dbt.flags = DB_DBT_USERMEM;
+        const DBT *large_dbt_ptr = &large_dbt;
+
+        TXNID txn_a = { 1 };
+        TXNID txn_b = { 2 };
+
+        // a locks small key
+        lock_request a;
+        a.create();
+        a.set(lt, txn_a, small_dbt_ptr, small_dbt_ptr, lock_request::type::WRITE, false);
+        r = a.start();
+        assert(r == 0);
+        assert(a.m_state == lock_request::state::COMPLETE);
+
+        // b tries to lock small key, fails since small key already locked
+        lock_request b;
+        b.create();
+        b.set(lt, txn_b, small_dbt_ptr, small_dbt_ptr, lock_request::type::WRITE, false);
+        r = b.start();
+        assert(r == DB_LOCK_NOTGRANTED);
+        assert(b.m_state == lock_request::state::PENDING);
+
+        // a locks large key. this uses all of the lock memory
+        lock_request c;
+        c.create();
+        c.set(lt, txn_a, large_dbt_ptr, large_dbt_ptr, lock_request::type::WRITE, false);
+        r = c.start();
+        assert(r == 0);
+        assert(c.m_state == lock_request::state::COMPLETE);
+
+        // a releases small key. the lock memory is still over the limit
+        locktree_release_lock(lt, txn_a, small_dbt_ptr, small_dbt_ptr);
+
+        // b waits for small key, gets out of locks since lock memory is over the limit
+        assert(b.m_state == lock_request::state::PENDING);
+        r = b.wait(0);
+        assert(r == TOKUDB_OUT_OF_LOCKS);
+        assert(b.m_state == lock_request::state::COMPLETE);
+
+        // retry pending lock requests
+        lock_request::retry_all_lock_requests(lt);
+
+        // a releases large key
+        locktree_release_lock(lt, txn_a, large_dbt_ptr, large_dbt_ptr);
+
+        // b locks small key, gets it
+        assert(b.m_state == lock_request::state::COMPLETE);
+        r = b.start();
+        assert(r == 0);
+
+        // b releases small key so we can exit cleanly
+        locktree_release_lock(lt, txn_b, small_dbt_ptr, small_dbt_ptr);
+
+        a.destroy();
+        b.destroy();
+
+        mgr.release_lt(lt);
+        mgr.destroy();
+    }
+
+} /* namespace toku */
+
+int main(void) {
+    toku::lock_request_unit_test test;
+    test.run();
+    return 0;
+}
diff --git a/locktree/tests/lock_request_wait_time_callback.cc b/locktree/tests/lock_request_wait_time_callback.cc
index 1647cee1d..e1c9a1d95 100644
--- a/locktree/tests/lock_request_wait_time_callback.cc
+++ b/locktree/tests/lock_request_wait_time_callback.cc
@@ -43,7 +43,7 @@ namespace toku {
 static const uint64_t my_lock_wait_time = 10 * 1000; // 10 sec
 
 // make sure deadlocks are detected when a lock request starts
-void lock_request_unit_test::test_wait_time_callback(void) {
+void lock_request_unit_test::run(void) {
     int r;
     locktree lt;
 
@@ -90,7 +90,7 @@ void lock_request_unit_test::test_wait_time_callback(void) {
 
 int main(void) {
     toku::lock_request_unit_test test;
-    test.test_wait_time_callback();
+    test.run();
     return 0;
 }
 
diff --git a/locktree/tests/test.h b/locktree/tests/test.h
index 921f2468b..8f051e708 100644
--- a/locktree/tests/test.h
+++ b/locktree/tests/test.h
@@ -93,8 +93,8 @@ namespace toku {
         if (toku_dbt_is_infinite(key1) || toku_dbt_is_infinite(key2)) {
             return toku_dbt_infinite_compare(key1, key2);
         } else {
-            invariant(key1->size == sizeof(int64_t));
-            invariant(key2->size == sizeof(int64_t));
+            invariant(key1->size >= sizeof(int64_t));
+            invariant(key2->size >= sizeof(int64_t));
             int64_t a = *(int64_t*) key1->data;
             int64_t b = *(int64_t*) key2->data;
             if (a < b) {
diff --git a/src/tests/put-wait-retry-out-of-locks.cc b/src/tests/put-wait-retry-out-of-locks.cc
new file mode 100644
index 000000000..24f3d1280
--- /dev/null
+++ b/src/tests/put-wait-retry-out-of-locks.cc
@@ -0,0 +1,164 @@
+/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
+#ident "$Id$"
+/*======
+This file is part of PerconaFT.
+
+
+Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved.
+
+    PerconaFT is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License, version 2,
+    as published by the Free Software Foundation.
+
+    PerconaFT is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with PerconaFT.  If not, see <http://www.gnu.org/licenses/>.
+
+----------------------------------------
+
+    PerconaFT is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License, version 3,
+    as published by the Free Software Foundation.
+
+    PerconaFT is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with PerconaFT.  If not, see <http://www.gnu.org/licenses/>.
+======= */
+
+#ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved."
+
+// Verify that a DB put that is waiting on a previously locked key get TOKUDB_OUT_OF_LOCKS
+// result when another key is released AND the lock memory used is still over the limit.
+
+#include "test.h"
+#include "toku_pthread.h"
+
+static int put_small_key(DB *db, DB_TXN *txn) {
+    char k[8] = {};
+    DBT key = { .data = &k, .size = sizeof k};
+    DBT val = {};
+    int r = db->put(db, txn, &key, &val, 0);
+    return r;
+}
+
+static int put_large_key(DB *db, DB_TXN *txn) {
+    char k[200] = {};
+    DBT key = { .data = &k, .size = sizeof k};
+    DBT val = {};
+    int r = db->put(db, txn, &key, &val, 0);
+    return r;
+}
+
+struct test_c_args {
+    DB *db;
+    DB_TXN *txn;
+};
+
+static void *test_c(void *arg) {
+    struct test_c_args *a = (struct test_c_args *) arg;
+    int r = put_small_key(a->db, a->txn);
+    assert(r == TOKUDB_OUT_OF_LOCKS);
+    return arg;
+}
+
+int test_main(int argc, char * const argv[]) {
+    const char *db_env_dir = TOKU_TEST_FILENAME;
+    const char *db_filename = "test.db";
+    int db_env_open_flags = DB_CREATE | DB_PRIVATE | DB_INIT_MPOOL | DB_INIT_TXN | DB_INIT_LOCK | DB_INIT_LOG | DB_THREAD;
+
+    // parse_args(argc, argv);
+    for (int i = 1; i < argc; i++) {
+        if (strcmp(argv[i], "-v") == 0 || strcmp(argv[i], "--verbose") == 0) {
+            verbose++;
+            continue;
+        }
+        if (strcmp(argv[i], "-q") == 0 || strcmp(argv[i], "--quiet") == 0) {
+            if (verbose > 0)
+                verbose--;
+            continue;
+        }
+        assert(0);
+    }
+
+    // setup the test environment
+    int r;
+    char rm_cmd[strlen(db_env_dir) + strlen("rm -rf ") + 1];
+    snprintf(rm_cmd, sizeof(rm_cmd), "rm -rf %s", db_env_dir);
+    r = system(rm_cmd); assert(r == 0);
+
+    r = toku_os_mkdir(db_env_dir, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH); assert(r == 0);
+
+    DB_ENV *db_env = nullptr;
+    r = db_env_create(&db_env, 0); assert(r == 0);
+
+    // Set a small lock memory limit
+    const uint64_t lock_memory_wanted = 300;
+    r = db_env->set_lk_max_memory(db_env, lock_memory_wanted); assert(r == 0);
+    uint64_t lock_memory_limit;
+    r = db_env->get_lk_max_memory(db_env, &lock_memory_limit); assert(r == 0 && lock_memory_limit == lock_memory_wanted);
+
+    r = db_env->open(db_env, db_env_dir, db_env_open_flags, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); assert(r == 0);
+    r = db_env->set_lock_timeout(db_env, 30 * 1000, nullptr); assert(r == 0);
+
+    // create the db
+    DB *db = nullptr;
+    r = db_create(&db, db_env, 0); assert(r == 0);
+    r = db->open(db, nullptr, db_filename, nullptr, DB_BTREE, DB_CREATE|DB_AUTO_COMMIT|DB_THREAD, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); assert(r == 0);
+
+    // create the txn's
+    DB_TXN *txn_a = nullptr;
+    r = db_env->txn_begin(db_env, nullptr, &txn_a, 0); assert(r == 0);
+
+    DB_TXN *txn_b = nullptr;
+    r = db_env->txn_begin(db_env, nullptr, &txn_b, 0); assert(r == 0);
+
+    DB_TXN *txn_c = nullptr;
+    r = db_env->txn_begin(db_env, nullptr, &txn_c, 0); assert(r == 0);
+
+    // Put a small key into the DB.
+    // Before: lock memory used is 0.
+    // After: lock memory used is under the limit.
+    r = put_small_key(db, txn_a);
+    assert(r == 0);
+
+    // Create a thread that will attempt to lock the same key as txn_a.
+    // Effect: this thread will be blocking on the lock request for this
+    // key
+    toku_pthread_t tid_c;
+    test_c_args a = { db, txn_c };
+    r = toku_pthread_create(toku_uninstrumented, &tid_c, nullptr, test_c, &a);
+    assert(r == 0);
+
+    // give thread c some time to get blocked
+    sleep(1);
+
+    // Put a large key into the DB, which should succeed.
+    // Before: lock memory used is under the limit
+    // After: lock memory used is over the limit due to the addition of the large key
+    r = put_large_key(db, txn_b);
+    assert(r == 0);
+
+    // abort txn a, should release lock on the small key but lock memory
+    // is still over the limit, so test c put lock retry should get
+    // TOKUDB_OUT_OF_LOCKS
+    r = txn_a->abort(txn_a); assert(r == 0);
+
+    // cleanup
+    void *ret;
+    r = toku_pthread_join(tid_c, &ret); assert(r == 0);
+    r = txn_b->abort(txn_b); assert(r == 0);
+    r = txn_c->abort(txn_c); assert(r == 0);
+    r = db->close(db, 0); assert(r == 0); db = nullptr;
+    r = db_env->close(db_env, 0); assert(r == 0); db_env = nullptr;
+
+    return 0;
+}
diff --git a/src/tests/simple-put-out-of-locks.cc b/src/tests/simple-put-out-of-locks.cc
new file mode 100644
index 000000000..40165a0c8
--- /dev/null
+++ b/src/tests/simple-put-out-of-locks.cc
@@ -0,0 +1,125 @@
+/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
+#ident "$Id$"
+/*======
+This file is part of PerconaFT.
+
+
+Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved.
+
+    PerconaFT is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License, version 2,
+    as published by the Free Software Foundation.
+
+    PerconaFT is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with PerconaFT.  If not, see <http://www.gnu.org/licenses/>.
+
+----------------------------------------
+
+    PerconaFT is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License, version 3,
+    as published by the Free Software Foundation.
+
+    PerconaFT is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with PerconaFT.  If not, see <http://www.gnu.org/licenses/>.
+======= */
+
+#ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved."
+
+// Verify that a DB put fails with TOKUDB_OUT_OF_LOCKS result when the lock memory
+// use is over the limit.
+
+#include "test.h"
+#include "toku_pthread.h"
+
+static int put_small_key(DB *db, DB_TXN *txn) {
+    char k[8] = {};
+    DBT key = { .data = &k, .size = sizeof k};
+    DBT val = {};
+    int r = db->put(db, txn, &key, &val, 0);
+    return r;
+}
+
+static int put_large_key(DB *db, DB_TXN *txn) {
+    char k[200] = {};
+    DBT key = { .data = &k, .size = sizeof k};
+    DBT val = {};
+    int r = db->put(db, txn, &key, &val, 0);
+    return r;
+}
+
+int test_main(int argc, char * const argv[]) {
+    const char *db_env_dir = TOKU_TEST_FILENAME;
+    const char *db_filename = "test.db";
+    int db_env_open_flags = DB_CREATE | DB_PRIVATE | DB_INIT_MPOOL | DB_INIT_TXN | DB_INIT_LOCK | DB_INIT_LOG | DB_THREAD;
+
+    // parse_args(argc, argv);
+    for (int i = 1; i < argc; i++) {
+        if (strcmp(argv[i], "-v") == 0 || strcmp(argv[i], "--verbose") == 0) {
+            verbose++;
+            continue;
+        }
+        if (strcmp(argv[i], "-q") == 0 || strcmp(argv[i], "--quiet") == 0) {
+            if (verbose > 0)
+                verbose--;
+            continue;
+        }
+        assert(0);
+    }
+
+    // setup the test
+    int r;
+    char rm_cmd[strlen(db_env_dir) + strlen("rm -rf ") + 1];
+    snprintf(rm_cmd, sizeof(rm_cmd), "rm -rf %s", db_env_dir);
+    r = system(rm_cmd); assert(r == 0);
+    r = toku_os_mkdir(db_env_dir, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH); assert(r == 0);
+
+    // create the env
+    DB_ENV *db_env = nullptr;
+    r = db_env_create(&db_env, 0); assert(r == 0);
+
+    // Setup a small lock memory limit
+    const uint64_t lock_memory_wanted = 300;
+    r = db_env->set_lk_max_memory(db_env, lock_memory_wanted); assert(r == 0);
+    uint64_t lock_memory_limit;
+    r = db_env->get_lk_max_memory(db_env, &lock_memory_limit); assert(r == 0 && lock_memory_limit == lock_memory_wanted);
+
+    r = db_env->open(db_env, db_env_dir, db_env_open_flags, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); assert(r == 0);
+    r = db_env->set_lock_timeout(db_env, 30 * 1000, nullptr); assert(r == 0);
+
+    // create the db
+    DB *db = nullptr;
+    r = db_create(&db, db_env, 0); assert(r == 0);
+    r = db->open(db, nullptr, db_filename, nullptr, DB_BTREE, DB_CREATE|DB_AUTO_COMMIT|DB_THREAD, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); assert(r == 0);
+
+    DB_TXN *txn_a = nullptr;
+    r = db_env->txn_begin(db_env, nullptr, &txn_a, 0); assert(r == 0);
+
+    // Put a large key into the DB.
+    // Before: lock memory use is 0.
+    // After: lock memory is over the limit
+    r = put_large_key(db, txn_a);
+    assert(r == 0);
+
+    // Try to put a small key into the DB.
+    // Should get TOKUDB_OUT_OF_LOCKS since lock memory is over the limit.
+    r = put_small_key(db, txn_a);
+    assert(r == TOKUDB_OUT_OF_LOCKS);
+
+    // cleanup
+    r = txn_a->abort(txn_a); assert(r == 0);
+    r = db->close(db, 0); assert(r == 0); db = nullptr;
+    r = db_env->close(db_env, 0); assert(r == 0); db_env = nullptr;
+
+    return 0;
+}

From ca0df3a33ca4a07849358800c6288f875b763fdb Mon Sep 17 00:00:00 2001
From: rik prohaska <prohaska7@gmail.com>
Date: Thu, 9 Apr 2020 08:57:13 -0400
Subject: [PATCH 06/29] When a lock is released, pending lock requests are
 retried.  If the retry occurs and the lock memory use is over the limit, then
 pending lock requests should be completed with TOKUDB_OUT_OF_LOCKS result.
 The current code does not handle this case and crashes.  This commit fixes
 this case and is sufficient to fix PS-4328.

---
 locktree/lock_request.cc | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/locktree/lock_request.cc b/locktree/lock_request.cc
index b99eaff3d..64e53821b 100644
--- a/locktree/lock_request.cc
+++ b/locktree/lock_request.cc
@@ -317,16 +317,20 @@ int lock_request::retry(void) {
             m_txnid, m_left_key, m_right_key, &conflicts, m_big_txn);
     }
 
-    // if the acquisition succeeded then remove ourselves from the
-    // set of lock requests, complete, and signal the waiting thread.
-    if (r == 0) {
+    // if the acquisition succeeded or if out of locks
+    // then remove ourselves from the set of lock requests, complete
+    // the lock request, and signal the waiting threads.
+    if (r == 0 || r == TOKUDB_OUT_OF_LOCKS) {
         remove_from_lock_requests();
         complete(r);
         if (m_retry_test_callback)
             m_retry_test_callback();  // test callback
         toku_cond_broadcast(&m_wait_cond);
-    } else {
+    } else if (r == DB_LOCK_NOTGRANTED) {
+        // get the conflicting txnid and remain pending
         m_conflicting_txnid = conflicts.get(0);
+    } else {
+        invariant(0);
     }
     conflicts.destroy();
 

From ea536c222ed5765234dbbf5cd6475db90e42e00b Mon Sep 17 00:00:00 2001
From: rik prohaska <prohaska7@gmail.com>
Date: Thu, 9 Apr 2020 13:50:49 -0400
Subject: [PATCH 07/29] Speed up helgrind_test_partitioned_counter test by
 running helgrind and drd scheduler with fairness (--fair-sched=try).  This
 should reduce execution time from minutes to seconds.

---
 cmake_modules/TokuSetupCTest.cmake | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cmake_modules/TokuSetupCTest.cmake b/cmake_modules/TokuSetupCTest.cmake
index 5b6882cc4..6bb164cea 100644
--- a/cmake_modules/TokuSetupCTest.cmake
+++ b/cmake_modules/TokuSetupCTest.cmake
@@ -126,7 +126,7 @@ if (BUILD_TESTING OR BUILD_FT_TESTS OR BUILD_SRC_TESTS)
   endmacro(add_toku_test)
 
   ## setup a function to write tests that will run with helgrind
-  set(CMAKE_HELGRIND_COMMAND_STRING "valgrind --quiet --tool=helgrind --error-exitcode=1 --soname-synonyms=somalloc=*tokuportability* --suppressions=${TokuDB_SOURCE_DIR}/src/tests/helgrind.suppressions --trace-children=yes --trace-children-skip=sh,*/sh,basename,*/basename,dirname,*/dirname,rm,*/rm,cp,*/cp,mv,*/mv,cat,*/cat,diff,*/diff,grep,*/grep,date,*/date,test,*/tokudb_dump* --trace-children-skip-by-arg=--only_create,--test,--no-shutdown,novalgrind")
+  set(CMAKE_HELGRIND_COMMAND_STRING "valgrind --quiet --tool=helgrind --error-exitcode=1 --soname-synonyms=somalloc=*tokuportability* --suppressions=${TokuDB_SOURCE_DIR}/src/tests/helgrind.suppressions --trace-children=yes --trace-children-skip=sh,*/sh,basename,*/basename,dirname,*/dirname,rm,*/rm,cp,*/cp,mv,*/mv,cat,*/cat,diff,*/diff,grep,*/grep,date,*/date,test,*/tokudb_dump* --trace-children-skip-by-arg=--only_create,--test,--no-shutdown,novalgrind --fair-sched=try")
   function(add_helgrind_test pfx name)
     separate_arguments(CMAKE_HELGRIND_COMMAND_STRING)
     add_test(
@@ -137,7 +137,7 @@ if (BUILD_TESTING OR BUILD_FT_TESTS OR BUILD_SRC_TESTS)
   endfunction(add_helgrind_test)
 
   ## setup a function to write tests that will run with drd
-  set(CMAKE_DRD_COMMAND_STRING "valgrind --quiet --tool=drd --error-exitcode=1 --soname-synonyms=somalloc=*tokuportability* --suppressions=${TokuDB_SOURCE_DIR}/src/tests/drd.suppressions --trace-children=yes --trace-children-skip=sh,*/sh,basename,*/basename,dirname,*/dirname,rm,*/rm,cp,*/cp,mv,*/mv,cat,*/cat,diff,*/diff,grep,*/grep,date,*/date,test,*/tokudb_dump* --trace-children-skip-by-arg=--only_create,--test,--no-shutdown,novalgrind")
+  set(CMAKE_DRD_COMMAND_STRING "valgrind --quiet --tool=drd --error-exitcode=1 --soname-synonyms=somalloc=*tokuportability* --suppressions=${TokuDB_SOURCE_DIR}/src/tests/drd.suppressions --trace-children=yes --trace-children-skip=sh,*/sh,basename,*/basename,dirname,*/dirname,rm,*/rm,cp,*/cp,mv,*/mv,cat,*/cat,diff,*/diff,grep,*/grep,date,*/date,test,*/tokudb_dump* --trace-children-skip-by-arg=--only_create,--test,--no-shutdown,novalgrind --fair-sched=try")
   function(add_drd_test pfx name)
     separate_arguments(CMAKE_DRD_COMMAND_STRING)
     add_test(

From fddf161813931bb9322217948d6f60274442f0f2 Mon Sep 17 00:00:00 2001
From: rik prohaska <prohaska7@gmail.com>
Date: Thu, 9 Apr 2020 13:57:20 -0400
Subject: [PATCH 08/29] Fix a test that crashed in put_callback due to a null
 deref.  The test should intentially abort instead.

---
 src/tests/recover-del-multiple-abort.cc | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/tests/recover-del-multiple-abort.cc b/src/tests/recover-del-multiple-abort.cc
index 425c12e1a..3ffd63919 100644
--- a/src/tests/recover-del-multiple-abort.cc
+++ b/src/tests/recover-del-multiple-abort.cc
@@ -57,9 +57,12 @@ get_data(int *v, int i, int ndbs) {
 static int
 put_callback(DB *dest_db, DB *src_db, DBT_ARRAY *dest_keys, DBT_ARRAY *dest_vals, const DBT *src_key, const DBT *src_val) {
     toku_dbt_array_resize(dest_keys, 1);
-    toku_dbt_array_resize(dest_vals, 1);
     DBT *dest_key = &dest_keys->dbts[0];
-    DBT *dest_val = &dest_vals->dbts[0];
+    DBT *dest_val = NULL;
+    if (dest_vals) {
+        toku_dbt_array_resize(dest_vals, 1);
+        dest_val = &dest_vals->dbts[0];
+    }
     (void) dest_db; (void) src_db; (void) dest_key; (void) dest_val; (void) src_key; (void) src_val;
     assert(src_db == NULL);
 

From a3c530c3e402e1372cb063094c3f77c19cf82c04 Mon Sep 17 00:00:00 2001
From: rik prohaska <prohaska7@gmail.com>
Date: Thu, 9 Apr 2020 17:28:26 -0400
Subject: [PATCH 09/29] Ignore mallopt error return in memory_init when running
 the address sanitizer.  This allows the tests to run.

---
 portability/memory.cc | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/portability/memory.cc b/portability/memory.cc
index f4888dccc..ddfca5f32 100644
--- a/portability/memory.cc
+++ b/portability/memory.cc
@@ -100,6 +100,12 @@ toku_memory_startup(void) {
     } else {
         result = EINVAL;
     }
+#if defined(__has_feature)
+#if __has_feature(address_sanitizer)
+    result = 0; // the clang 8 address sanitizer fails mallopt, ignore it
+    status.mmap_threshold = mmap_threshold;
+#endif
+#endif
     assert(result == 0);
 #else
     // just a guess

From 54c2ac861790af98ebe205cd3cd3e9234cda3310 Mon Sep 17 00:00:00 2001
From: rik prohaska <prohaska7@gmail.com>
Date: Fri, 10 Apr 2020 08:16:39 -0400
Subject: [PATCH 10/29] Suppress pthread_mutex_destroy race with
 pthread_mutex_unlock again. It causes helgrind to report false positive data
 races on tests, specifically the helgrind_helgrind tests.

4101: ==17113== Possible data race during read of size 1 at 0x52FA980 by thread #1
4101: ==17113== Locks held: 1, at address 0x4C126C0
4101: ==17113==    at 0x483E99C: ??? (in /usr/lib/x86_64-linux-gnu/valgrind/vgpreload_helgrind-amd64-linux.so)
4101: ==17113==    by 0x483EC4A: ??? (in /usr/lib/x86_64-linux-gnu/valgrind/vgpreload_helgrind-amd64-linux.so)
4101: ==17113==    by 0x4A95654: toku_mutex_destroy(toku_mutex_t*) (toku_pthread.h:427)
4101: ==17113==    by 0x4B6CF5C: bjm_destroy(background_job_manager_struct*) (background_job_manager.cc:67)
4101: ==17113==    by 0x4ABE119: write_dirty_pairs_for_close(cachetable*, cachefile*) (cachetable.cc:2406)
4101: ==17113==    by 0x4ABE308: cachetable_flush_cachefile(cachetable*, cachefile*, bool) (cachetable.cc:2493)
4101: ==17113==    by 0x4AB99BE: toku_cachefile_close(cachefile**, bool, __toku_lsn) (cachetable.cc:529)
4101: ==17113==    by 0x4ACD7C2: toku_ft_evict_from_memory(ft*, bool, __toku_lsn) (ft.cc:499)
4101: ==17113==    by 0x4ACEADD: toku_ft_remove_reference(ft*, bool, __toku_lsn, void (*)(ft*, void*), void*) (ft.cc:949)
4101: ==17113==    by 0x4AE9B21: ft_handle_close(ft_handle*, bool, __toku_lsn) (ft-ops.cc:3272)
4101: ==17113==    by 0x4AE9B54: toku_ft_handle_close(ft_handle*) (ft-ops.cc:3280)
4101: ==17113==    by 0x4A8CD68: toku_db_close(__toku_db*) (ydb_db.cc:212)
4101: ==17113==
4101: ==17113== This conflicts with a previous write of size 4 by thread #10
4101: ==17113== Locks held: none
4101: ==17113==    at 0x4C557A1: __pthread_mutex_unlock_usercnt (pthread_mutex_unlock.c:51)
4101: ==17113==    by 0x4C557A1: pthread_mutex_unlock (pthread_mutex_unlock.c:357)
4101: ==17113==    by 0x483F458: ??? (in /usr/lib/x86_64-linux-gnu/valgrind/vgpreload_helgrind-amd64-linux.so)
4101: ==17113==    by 0x4A84508: toku_mutex_unlock(toku_mutex_t*) (toku_pthread.h:256)
4101: ==17113==    by 0x4B6D0EC: bjm_remove_background_job(background_job_manager_struct*) (background_job_manager.cc:98)
4101: ==17113==    by 0x4ABDCDD: cachetable_flush_pair_for_close(void*) (cachetable.cc:2327)
4101: ==17113==    by 0x4B7D785: work_on_kibbutz(void*) (kibbutz.cc:154)
4101: ==17113==    by 0x4841B1A: ??? (in /usr/lib/x86_64-linux-gnu/valgrind/vgpreload_helgrind-amd64-linux.so)
4101: ==17113==    by 0x4C51668: start_thread (pthread_create.c:479)
---
 src/tests/helgrind.suppressions | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/tests/helgrind.suppressions b/src/tests/helgrind.suppressions
index 4729c1d14..c6bdda640 100644
--- a/src/tests/helgrind.suppressions
+++ b/src/tests/helgrind.suppressions
@@ -156,3 +156,10 @@
     ...
     fun:pthread_mutex_destroy
 }
+{
+    pthread_mutex_destroy/pthread_mutex_unlock race glibc 2.30 gcc-9.2.1 valgrind-3.15.0
+    Helgrind:Race
+    obj:/usr/lib*/valgrind/vgpreload_helgrind-amd64-linux.so*
+    obj:/usr/lib*/valgrind/vgpreload_helgrind-amd64-linux.so*
+    fun:*toku_mutex_destroy*
+}

From e0dcade5ab6db559919b5cbf184cdb45164da371 Mon Sep 17 00:00:00 2001
From: rik prohaska <prohaska7@gmail.com>
Date: Fri, 10 Apr 2020 08:19:38 -0400
Subject: [PATCH 11/29] Suppress memcheck uninitialized data jump in
 lz_encoder_prepare. See
 https://prohaska7.blogspot.com/2015/11/uninitialized-data-problem-in-lzma.html
 for details.

---
 cmake_modules/TokuSetupCTest.cmake | 2 ++
 third_party/xz.suppressions        | 5 +++++
 2 files changed, 7 insertions(+)
 create mode 100644 third_party/xz.suppressions

diff --git a/cmake_modules/TokuSetupCTest.cmake b/cmake_modules/TokuSetupCTest.cmake
index 6bb164cea..66d3497fc 100644
--- a/cmake_modules/TokuSetupCTest.cmake
+++ b/cmake_modules/TokuSetupCTest.cmake
@@ -94,6 +94,8 @@ if (BUILD_TESTING OR BUILD_FT_TESTS OR BUILD_SRC_TESTS)
   ## set up full valgrind suppressions file (concatenate the suppressions files)
   file(READ ft/valgrind.suppressions valgrind_suppressions)
   file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/valgrind.suppressions" "${valgrind_suppressions}")
+  file(READ third_party/xz.suppressions xz_suppressions)
+  file(APPEND "${CMAKE_CURRENT_BINARY_DIR}/valgrind.suppressions" "${xz_suppressions}")
   file(READ bash.suppressions bash_suppressions)
   file(APPEND "${CMAKE_CURRENT_BINARY_DIR}/valgrind.suppressions" "${bash_suppressions}")
 
diff --git a/third_party/xz.suppressions b/third_party/xz.suppressions
new file mode 100644
index 000000000..a6b87c302
--- /dev/null
+++ b/third_party/xz.suppressions
@@ -0,0 +1,5 @@
+{
+    lzma: Conditional jump or move depends on uninitialised value (https://prohaska7.blogspot.com/2015/11/uninitialized-data-problem-in-lzma.html)
+    Memcheck:Cond
+    fun:lz_encoder_prepare
+}

From bfd3a72c0dbf2c97b38f9592a6ea800eed4e8fe1 Mon Sep 17 00:00:00 2001
From: rik prohaska <prohaska7@gmail.com>
Date: Fri, 10 Apr 2020 17:07:23 -0400
Subject: [PATCH 12/29] Some ydb tests leak memory for pfs keys. Fix the ydb
 library init and destroy functions.

---
 src/ydb.cc | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/src/ydb.cc b/src/ydb.cc
index 3e21ad0f8..d88d7a26e 100644
--- a/src/ydb.cc
+++ b/src/ydb.cc
@@ -179,23 +179,26 @@ static int toku_maybe_get_engine_status_text (char* buff, int buffsize);  // for
 static int toku_maybe_err_engine_status (void);
 static void toku_maybe_set_env_panic(int code, const char * msg);               // for use by toku_assert
 
-int 
-toku_ydb_init(void) {
+static bool ydb_initialized = false;
+int toku_ydb_init(void) {
     int r = 0;
-    //Lower level must be initialized first.
-    r = toku_ft_layer_init();
+    if (!ydb_initialized) {
+        //Lower level must be initialized first.
+        r = toku_ft_layer_init();
+        if (r == 0)
+            ydb_initialized = true;
+    }
     return r;
 }
 
 // Do not clean up resources if env is panicked, just exit ugly
-void 
-toku_ydb_destroy(void) {
-    if (!ydb_layer_status.initialized)
+void toku_ydb_destroy(void) {
+    if (!ydb_initialized)
         return;
     if (env_is_panicked == 0) {
         toku_ft_layer_destroy();
     }
-    ydb_layer_status.initialized = false;
+    ydb_initialized = false;
 }
 
 static int

From 875a17399dcd9330fd2f099c9135a00070e9f4df Mon Sep 17 00:00:00 2001
From: rik prohaska <prohaska7@gmail.com>
Date: Sat, 11 Apr 2020 07:17:00 -0400
Subject: [PATCH 13/29] Suppress memory leak in libjson on ubuntu 19.10 when
 running cmake.

---
 cmake_modules/TokuSetupCTest.cmake                 |  4 +++-
 bash.suppressions => third_party/bash.suppressions |  0
 third_party/cmake.suppressions                     | 11 +++++++++++
 3 files changed, 14 insertions(+), 1 deletion(-)
 rename bash.suppressions => third_party/bash.suppressions (100%)
 create mode 100644 third_party/cmake.suppressions

diff --git a/cmake_modules/TokuSetupCTest.cmake b/cmake_modules/TokuSetupCTest.cmake
index 66d3497fc..9540d30b9 100644
--- a/cmake_modules/TokuSetupCTest.cmake
+++ b/cmake_modules/TokuSetupCTest.cmake
@@ -96,7 +96,9 @@ if (BUILD_TESTING OR BUILD_FT_TESTS OR BUILD_SRC_TESTS)
   file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/valgrind.suppressions" "${valgrind_suppressions}")
   file(READ third_party/xz.suppressions xz_suppressions)
   file(APPEND "${CMAKE_CURRENT_BINARY_DIR}/valgrind.suppressions" "${xz_suppressions}")
-  file(READ bash.suppressions bash_suppressions)
+  file(READ third_party/cmake.suppressions cmake_suppressions)
+  file(APPEND "${CMAKE_CURRENT_BINARY_DIR}/valgrind.suppressions" "${cmake_suppressions}")
+  file(READ third_party/bash.suppressions bash_suppressions)
   file(APPEND "${CMAKE_CURRENT_BINARY_DIR}/valgrind.suppressions" "${bash_suppressions}")
 
   include(CMakeDependentOption)
diff --git a/bash.suppressions b/third_party/bash.suppressions
similarity index 100%
rename from bash.suppressions
rename to third_party/bash.suppressions
diff --git a/third_party/cmake.suppressions b/third_party/cmake.suppressions
new file mode 100644
index 000000000..4b6168ffe
--- /dev/null
+++ b/third_party/cmake.suppressions
@@ -0,0 +1,11 @@
+{
+   cmake 3.13.4 on ubuntu 19.04
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   fun:_ZN4Json5Value13nullSingletonEv
+   obj:/usr/*lib*/libjsoncpp*
+   ...
+   fun:call_init
+   fun:_dl_init
+}

From 0b201fda1bb05fb09e68247968be2d1e52d2f2be Mon Sep 17 00:00:00 2001
From: rik prohaska <prohaska7@gmail.com>
Date: Sat, 11 Apr 2020 07:18:33 -0400
Subject: [PATCH 14/29] Run try-leak-reachable with RelWithDebInfo build type
 by avoiding gcc optimization.

---
 portability/tests/try-leak-reachable.cc | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/portability/tests/try-leak-reachable.cc b/portability/tests/try-leak-reachable.cc
index f93d10087..dd0b1ad03 100644
--- a/portability/tests/try-leak-reachable.cc
+++ b/portability/tests/try-leak-reachable.cc
@@ -36,11 +36,14 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved.
 
 #ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved."
 
+#include <stdio.h>
 #include <stdlib.h>
 
 static void *vp;
 
 int main(void) {
     vp = malloc(42);
+    // GCC has gotten smart enough to optimize this away unless we use it.
+    printf("%p\n", vp);
     return 0;
 }

From d4f11d23ff28fb4c0192b354b20056a9daa36d58 Mon Sep 17 00:00:00 2001
From: rik prohaska <prohaska7@gmail.com>
Date: Wed, 8 Apr 2020 16:53:56 -0400
Subject: [PATCH 15/29] glibc 2.30 defines the gettid system call, which
 conflicts with the tokuft test version.  rename tokuft test version to avoid
 conflict.

---
 portability/tests/test-xid.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/portability/tests/test-xid.cc b/portability/tests/test-xid.cc
index 9ee68906b..6143d3b58 100644
--- a/portability/tests/test-xid.cc
+++ b/portability/tests/test-xid.cc
@@ -54,7 +54,7 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved.
 
 // since we implement the same thing here as in toku_os_gettid, this test
 // is pretty pointless
-static int gettid(void) {
+static int test_gettid(void) {
 #if defined(__NR_gettid)
     return syscall(__NR_gettid);
 #elif defined(SYS_gettid)
@@ -68,6 +68,6 @@ static int gettid(void) {
 
 int main(void) {
     assert(toku_os_getpid() == getpid());
-    assert(toku_os_gettid() == gettid());
+    assert(toku_os_gettid() == test_gettid());
     return 0;
 }

From e1ed3f1bed8b0307de06e96e3041554706d119db Mon Sep 17 00:00:00 2001
From: rik prohaska <prohaska7@gmail.com>
Date: Wed, 8 Apr 2020 20:57:46 -0400
Subject: [PATCH 16/29] Remove ctest timeout property so that we can use the
 ctest --timeout parameter instead.

---
 ft/tests/CMakeLists.txt   | 18 -----------
 src/tests/CMakeLists.txt  | 63 +--------------------------------------
 util/tests/CMakeLists.txt |  4 ---
 3 files changed, 1 insertion(+), 84 deletions(-)

diff --git a/ft/tests/CMakeLists.txt b/ft/tests/CMakeLists.txt
index 270ec9766..73930dda6 100644
--- a/ft/tests/CMakeLists.txt
+++ b/ft/tests/CMakeLists.txt
@@ -123,22 +123,4 @@ if(BUILD_TESTING OR BUILD_FT_TESTS)
     get_filename_component(test_basename "${test}" NAME)
     add_ft_test_aux(test-${test_basename} test-upgrade-recovery-logs ${test})
   endforeach(test)
-
-  ## give some tests, that time out normally, 1 hour to complete
-  set(long_tests
-    ft/ftloader-test-extractor-3a
-    ft/log-test7
-    ft/recovery-bad-last-entry
-    ft/subblock-test-compression
-    ft/upgrade_test_simple
-    )
-  set_tests_properties(${long_tests} PROPERTIES TIMEOUT 3600)
-  ## some take even longer, with valgrind
-  set(extra_long_tests
-    ft/benchmark-test
-    ft/benchmark-test_256
-    ft/is_empty
-    ft/subblock-test-checksum
-    )
-  set_tests_properties(${extra_long_tests} PROPERTIES TIMEOUT 7200)
 endif(BUILD_TESTING OR BUILD_FT_TESTS)
diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt
index c01a8f0d6..1a3d3e0d8 100644
--- a/src/tests/CMakeLists.txt
+++ b/src/tests/CMakeLists.txt
@@ -197,7 +197,6 @@ if(BUILD_TESTING OR BUILD_SRC_TESTS)
       endif ()
 
       add_ydb_drd_test_aux(drd_tiny_${test} ${test} --num_seconds 5 --num_elements 150 --join_timeout 3000)
-      set_tests_properties(ydb/drd_tiny_${test} PROPERTIES TIMEOUT 3600)
 
       add_test(ydb/drd_mid_${test}/prepare ${test} --only_create --num_elements 10000)
       setup_toku_test_properties(ydb/drd_mid_${test}/prepare drd_mid_${test})
@@ -205,7 +204,6 @@ if(BUILD_TESTING OR BUILD_SRC_TESTS)
       set_tests_properties(ydb/drd_mid_${test} PROPERTIES
         DEPENDS ydb/drd_mid_${test}/prepare
         REQUIRED_FILES "drd_mid_${test}.ctest-data"
-        TIMEOUT 15000
         )
 
       add_test(ydb/drd_large_${test}/prepare ${test} --only_create --num_elements 150000)
@@ -214,7 +212,6 @@ if(BUILD_TESTING OR BUILD_SRC_TESTS)
       set_tests_properties(ydb/drd_large_${test} PROPERTIES
         DEPENDS ydb/drd_large_${test}/prepare
         REQUIRED_FILES "drd_large_${test}.ctest-data"
-        TIMEOUT 30000
         )
     endif()
   endforeach(src)
@@ -252,8 +249,7 @@ if(BUILD_TESTING OR BUILD_SRC_TESTS)
               setup_toku_test_properties(${testnamebase} "${envdirbase}")
               set_tests_properties(${testnamebase} PROPERTIES
                 DEPENDS ${testnamebase}/copy
-                REQUIRED_FILES "${envdir}"
-                TIMEOUT 10800)
+                REQUIRED_FILES "${envdir}")
             endforeach(size)
           endif ()
         endforeach(p_or_s)
@@ -433,61 +429,4 @@ if(BUILD_TESTING OR BUILD_SRC_TESTS)
   string(REGEX REPLACE ";" ";ydb/" tdb_tests_that_should_fail "${tdb_tests_that_should_fail}")
   set_tests_properties(${tdb_tests_that_should_fail} PROPERTIES WILL_FAIL TRUE)
 
-  ## give some tests, that time out normally, 1 hour to complete
-  set(long_tests
-    ydb/drd_test_groupcommit_count.tdb
-    ydb/env-put-multiple.tdb
-    ydb/filesize.tdb
-    ydb/loader-cleanup-test0.tdb
-    ydb/loader-cleanup-test0z.tdb
-    ydb/manyfiles.tdb
-    ydb/recover-loader-test.abortrecover
-    ydb/recovery_fileops_stress.tdb
-    ydb/root_fifo_1.tdb
-    ydb/root_fifo_2.tdb
-    ydb/root_fifo_31.tdb
-    ydb/root_fifo_32.tdb
-    ydb/shutdown-3344.tdb
-    ydb/stat64-create-modify-times.tdb
-    ydb/test1572.tdb
-    ydb/test_abort4_19_0.tdb
-    ydb/test_abort4_19_1.tdb
-    ydb/test_abort5.tdb
-    ydb/test_archive1.tdb
-    ydb/test_logmax.tdb
-    ydb/test_query.tdb
-    ydb/test_txn_abort5.tdb
-    ydb/test_txn_abort5a.tdb
-    ydb/test_txn_abort6.tdb
-    ydb/test_txn_nested2.tdb
-    ydb/test_txn_nested4.tdb
-    ydb/test_txn_nested5.tdb
-    ydb/test_update_broadcast_stress.tdb
-    )
-  set_tests_properties(${long_tests} PROPERTIES TIMEOUT 3600)
-  ## some take even longer, with valgrind
-  set(extra_long_tests
-    ydb/drd_test_4015.tdb
-    ydb/hotindexer-with-queries.tdb
-    ydb/hot-optimize-table-tests.tdb
-    ydb/loader-cleanup-test2.tdb
-    ydb/loader-cleanup-test2z.tdb
-    ydb/loader-dup-test0.tdb
-    ydb/loader-stress-del.nop.loader
-    ydb/loader-stress-del.p.loader
-    ydb/loader-stress-del.comp.loader
-    ydb/test3039.tdb
-    ydb/test_update_stress.tdb
-    )
-  set_tests_properties(${extra_long_tests} PROPERTIES TIMEOUT 7200)
-  ## these really take a long time with valgrind
-  set(phenomenally_long_tests
-    ydb/checkpoint_stress.tdb
-    ydb/loader-stress-test4.tdb
-    ydb/loader-stress-test4z.tdb
-    ydb/recover_stress.tdb
-    ydb/test3529.tdb
-    ydb/test_insert_unique.tdb
-    )
-  set_tests_properties(${phenomenally_long_tests} PROPERTIES TIMEOUT 14400)
 endif(BUILD_TESTING OR BUILD_SRC_TESTS)
diff --git a/util/tests/CMakeLists.txt b/util/tests/CMakeLists.txt
index 8d53dd89a..780bf8d66 100644
--- a/util/tests/CMakeLists.txt
+++ b/util/tests/CMakeLists.txt
@@ -17,8 +17,4 @@ if(BUILD_TESTING)
     add_test(util/${test} ${test})
   endforeach(test)
 
-  set(long_tests
-    util/helgrind_test_partitioned_counter
-    )
-  set_tests_properties(${long_tests} PROPERTIES TIMEOUT 3600)
 endif(BUILD_TESTING)

From 5a5c2533c2e15704e19a356a7df2d34fb095eb0e Mon Sep 17 00:00:00 2001
From: rik prohaska <prohaska7@gmail.com>
Date: Thu, 9 Apr 2020 08:11:48 -0400
Subject: [PATCH 17/29] Fix ydb tests that fail due to loose file permissions. 
 Note that file permissions were changed in commit 5aca29f.

---
 src/tests/test_memcmp_magic.cc        | 2 +-
 src/tests/xa-bigtxn-discard-abort.cc  | 4 ++--
 src/tests/xa-bigtxn-discard-commit.cc | 4 ++--
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/tests/test_memcmp_magic.cc b/src/tests/test_memcmp_magic.cc
index 8b56e716a..21e131bc0 100644
--- a/src/tests/test_memcmp_magic.cc
+++ b/src/tests/test_memcmp_magic.cc
@@ -103,7 +103,7 @@ static void test_memcmp_magic_sort_order(void) {
     DB_ENV *env;
     r = db_env_create(&env, 0); CKERR(r);
     r = env->set_default_bt_compare(env, comparison_function_unused); CKERR(r);
-    r = env->open(env, TOKU_TEST_FILENAME, DB_CREATE+DB_PRIVATE+DB_INIT_MPOOL+DB_INIT_TXN, 0); CKERR(r);
+    r = env->open(env, TOKU_TEST_FILENAME, DB_CREATE+DB_PRIVATE+DB_INIT_MPOOL+DB_INIT_TXN, S_IRUSR+S_IWUSR); CKERR(r);
 
     const int magic = 49;
 
diff --git a/src/tests/xa-bigtxn-discard-abort.cc b/src/tests/xa-bigtxn-discard-abort.cc
index 9f1b904df..4163d9918 100644
--- a/src/tests/xa-bigtxn-discard-abort.cc
+++ b/src/tests/xa-bigtxn-discard-abort.cc
@@ -62,7 +62,7 @@ static void populate_foo(DB_ENV *env, DB_TXN *txn) {
     DB *db = nullptr;
     r = db_create(&db, env, 0);
     CKERR(r);
-    r = db->open(db, txn, "foo.db", 0, DB_BTREE, 0, 0);
+    r = db->open(db, txn, "foo.db", 0, DB_BTREE, 0, S_IRWXU);
     CKERR(r);
 
     for (int i = 0; i < test_nrows; i++) {
@@ -81,7 +81,7 @@ static void check_foo(DB_ENV *env, DB_TXN *txn) {
     DB *db;
     r = db_create(&db, env, 0);
     CKERR(r);
-    r = db->open(db, txn, "foo.db", 0, DB_BTREE, 0, 0);
+    r = db->open(db, txn, "foo.db", 0, DB_BTREE, 0, S_IRWXU);
     CKERR(r);
 
     DBC *c = nullptr;
diff --git a/src/tests/xa-bigtxn-discard-commit.cc b/src/tests/xa-bigtxn-discard-commit.cc
index ecbfa18bd..733df036e 100644
--- a/src/tests/xa-bigtxn-discard-commit.cc
+++ b/src/tests/xa-bigtxn-discard-commit.cc
@@ -59,7 +59,7 @@ static void populate_foo(DB_ENV *env, DB_TXN *txn) {
     DB *db = nullptr;
     r = db_create(&db, env, 0);
     CKERR(r);
-    r = db->open(db, txn, "foo.db", 0, DB_BTREE, 0, 0);
+    r = db->open(db, txn, "foo.db", 0, DB_BTREE, 0, S_IRWXU);
     CKERR(r);
 
     for (int i = 0; i < test_nrows; i++) {
@@ -78,7 +78,7 @@ static void check_foo(DB_ENV *env, DB_TXN *txn) {
     DB *db;
     r = db_create(&db, env, 0);
     CKERR(r);
-    r = db->open(db, txn, "foo.db", 0, DB_BTREE, 0, 0);
+    r = db->open(db, txn, "foo.db", 0, DB_BTREE, 0, S_IRWXU);
     CKERR(r);
 
     DBC *c = nullptr;

From d3832213775918551360183523402beda72b98de Mon Sep 17 00:00:00 2001
From: rik prohaska <prohaska7@gmail.com>
Date: Thu, 9 Apr 2020 08:38:43 -0400
Subject: [PATCH 18/29] Remove obsolete debug code that was left in the tests
 by accident.

---
 locktree/lock_request.cc                               | 4 ----
 locktree/lock_request.h                                | 1 -
 locktree/tests/lock_request_start_retry_race.cc        | 1 -
 locktree/tests/lock_request_start_retry_race_3.cc      | 1 -
 locktree/tests/lock_request_start_retry_wait_race_2.cc | 1 -
 5 files changed, 8 deletions(-)

diff --git a/locktree/lock_request.cc b/locktree/lock_request.cc
index 19ec146a3..b99eaff3d 100644
--- a/locktree/lock_request.cc
+++ b/locktree/lock_request.cc
@@ -93,10 +93,6 @@ void lock_request::destroy(void) {
     toku_cond_destroy(&m_wait_cond);
 }
 
-void lock_request::clearmem(char c) {
-     memset(this, c, sizeof(* this));
-}
-
 // set the lock request parameters. this API allows a lock request to be reused.
 void lock_request::set(locktree *lt, TXNID txnid, const DBT *left_key, const DBT *right_key, lock_request::type lock_type, bool big_txn, void *extra) {
     invariant(m_state != state::PENDING);
diff --git a/locktree/lock_request.h b/locktree/lock_request.h
index 36c3fd260..76ac953ba 100644
--- a/locktree/lock_request.h
+++ b/locktree/lock_request.h
@@ -89,7 +89,6 @@ class lock_request {
 
     // effect: Destroys a lock request.
     void destroy(void);
-    void clearmem(char c);
 
     // effect: Resets the lock request parameters, allowing it to be reused.
     // requires: Lock request was already created at some point
diff --git a/locktree/tests/lock_request_start_retry_race.cc b/locktree/tests/lock_request_start_retry_race.cc
index 83436a651..374bc8625 100644
--- a/locktree/tests/lock_request_start_retry_race.cc
+++ b/locktree/tests/lock_request_start_retry_race.cc
@@ -83,7 +83,6 @@ namespace toku {
             }
 
             request.destroy();
-            request.clearmem(0xab);
 
             toku_pthread_yield();
             if ((i % 10) == 0)
diff --git a/locktree/tests/lock_request_start_retry_race_3.cc b/locktree/tests/lock_request_start_retry_race_3.cc
index 288cb0855..4c900765f 100644
--- a/locktree/tests/lock_request_start_retry_race_3.cc
+++ b/locktree/tests/lock_request_start_retry_race_3.cc
@@ -96,7 +96,6 @@ namespace toku {
             }
 
             request.destroy();
-            request.clearmem(0xab);
 
             toku_pthread_yield();
             if ((i % 10) == 0)
diff --git a/locktree/tests/lock_request_start_retry_wait_race_2.cc b/locktree/tests/lock_request_start_retry_wait_race_2.cc
index cd3dc7b37..ce74dc3ab 100644
--- a/locktree/tests/lock_request_start_retry_wait_race_2.cc
+++ b/locktree/tests/lock_request_start_retry_wait_race_2.cc
@@ -98,7 +98,6 @@ namespace toku {
             }
 
             request.destroy();
-            request.clearmem(0xab);
 
             toku_pthread_yield();
             if ((i % 10) == 0)

From 6a38ee403cdd713c6df94a86b091719039f47e9d Mon Sep 17 00:00:00 2001
From: rik prohaska <prohaska7@gmail.com>
Date: Thu, 9 Apr 2020 08:42:14 -0400
Subject: [PATCH 19/29] Add locktree and ydb tests cases that expose the lock
 request retry bug when the lock memory use is over the limit.  The ydb tests
 show that DB->put normally returns TOKUDB_OUT_OF_LOCKS when the lock memory
 use is over the limit.

---
 locktree/tests/lock_request_create_set.cc     |   4 +-
 locktree/tests/lock_request_get_set_keys.cc   |   4 +-
 locktree/tests/lock_request_killed.cc         |   4 +-
 locktree/tests/lock_request_not_killed.cc     |   4 +-
 .../tests/lock_request_retry_out_of_locks.cc  | 117 +++++++++++++
 locktree/tests/lock_request_start_deadlock.cc |   4 +-
 locktree/tests/lock_request_start_pending.cc  |   4 +-
 locktree/tests/lock_request_unit_test.h       |  18 +-
 .../tests/lock_request_wait_out_of_locks.cc   | 114 ++++++++++++
 .../tests/lock_request_wait_time_callback.cc  |   4 +-
 locktree/tests/test.h                         |   4 +-
 src/tests/put-wait-retry-out-of-locks.cc      | 164 ++++++++++++++++++
 src/tests/simple-put-out-of-locks.cc          | 125 +++++++++++++
 13 files changed, 537 insertions(+), 33 deletions(-)
 create mode 100644 locktree/tests/lock_request_retry_out_of_locks.cc
 create mode 100644 locktree/tests/lock_request_wait_out_of_locks.cc
 create mode 100644 src/tests/put-wait-retry-out-of-locks.cc
 create mode 100644 src/tests/simple-put-out-of-locks.cc

diff --git a/locktree/tests/lock_request_create_set.cc b/locktree/tests/lock_request_create_set.cc
index 8ae685b98..cb0d9aa7d 100644
--- a/locktree/tests/lock_request_create_set.cc
+++ b/locktree/tests/lock_request_create_set.cc
@@ -41,7 +41,7 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved.
 namespace toku {
 
 // create and set the object's internals, destroy should not crash.
-void lock_request_unit_test::test_create_destroy(void) {
+void lock_request_unit_test::run(void) {
     lock_request request;
     request.create();
 
@@ -66,7 +66,7 @@ void lock_request_unit_test::test_create_destroy(void) {
 
 int main(void) {
     toku::lock_request_unit_test test;
-    test.test_create_destroy();
+    test.run();
     return 0;
 }
 
diff --git a/locktree/tests/lock_request_get_set_keys.cc b/locktree/tests/lock_request_get_set_keys.cc
index fd57b70f5..56b785dfe 100644
--- a/locktree/tests/lock_request_get_set_keys.cc
+++ b/locktree/tests/lock_request_get_set_keys.cc
@@ -43,7 +43,7 @@ namespace toku {
 // make setting keys and getting them back works properly.
 // at a high level, we want to make sure keys are copied
 // when appropriate and plays nice with +/- infinity.
-void lock_request_unit_test::test_get_set_keys(void) {
+void lock_request_unit_test::run(void) {
     lock_request request;
     request.create();
 
@@ -82,7 +82,7 @@ void lock_request_unit_test::test_get_set_keys(void) {
 
 int main(void) {
     toku::lock_request_unit_test test;
-    test.test_get_set_keys();
+    test.run();
     return 0;
 }
 
diff --git a/locktree/tests/lock_request_killed.cc b/locktree/tests/lock_request_killed.cc
index ec4644442..404550620 100644
--- a/locktree/tests/lock_request_killed.cc
+++ b/locktree/tests/lock_request_killed.cc
@@ -63,7 +63,7 @@ static int my_killed_callback(void) {
 }
 
 // make sure deadlocks are detected when a lock request starts
-void lock_request_unit_test::test_wait_time_callback(void) {
+void lock_request_unit_test::run(void) {
     int r;
     locktree lt;
 
@@ -118,7 +118,7 @@ void lock_request_unit_test::test_wait_time_callback(void) {
 
 int main(void) {
     toku::lock_request_unit_test test;
-    test.test_wait_time_callback();
+    test.run();
     return 0;
 }
 
diff --git a/locktree/tests/lock_request_not_killed.cc b/locktree/tests/lock_request_not_killed.cc
index 647b4d3c4..9ec9a57fe 100644
--- a/locktree/tests/lock_request_not_killed.cc
+++ b/locktree/tests/lock_request_not_killed.cc
@@ -58,7 +58,7 @@ static int my_killed_callback(void) {
 }
 
 // make sure deadlocks are detected when a lock request starts
-void lock_request_unit_test::test_wait_time_callback(void) {
+void lock_request_unit_test::run(void) {
     int r;
     locktree lt;
 
@@ -112,7 +112,7 @@ void lock_request_unit_test::test_wait_time_callback(void) {
 
 int main(void) {
     toku::lock_request_unit_test test;
-    test.test_wait_time_callback();
+    test.run();
     return 0;
 }
 
diff --git a/locktree/tests/lock_request_retry_out_of_locks.cc b/locktree/tests/lock_request_retry_out_of_locks.cc
new file mode 100644
index 000000000..d98417858
--- /dev/null
+++ b/locktree/tests/lock_request_retry_out_of_locks.cc
@@ -0,0 +1,117 @@
+/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
+
+// Verify that lock request retry returns TOKUDB_OUT_OF_LOCKS when
+// all of the locktree memory is used.
+
+#include "lock_request_unit_test.h"
+
+namespace toku {
+
+    static void locktree_release_lock(locktree *lt,
+                                      TXNID txn_id,
+                                      const DBT *left,
+                                      const DBT *right) {
+        range_buffer buffer;
+        buffer.create();
+        buffer.append(left, right);
+        lt->release_locks(txn_id, &buffer);
+        buffer.destroy();
+    }
+
+    void lock_request_unit_test::run(void) {
+        int r;
+
+        locktree_manager mgr;
+        mgr.create(nullptr, nullptr, nullptr, nullptr);
+
+        DICTIONARY_ID dict_id = {1};
+        locktree *lt = mgr.get_lt(dict_id, dbt_comparator, nullptr);
+
+        // set max lock memory small so that we can test the limit
+        // with just 2 locks
+        mgr.set_max_lock_memory(300);
+
+        // create a small key
+        DBT small_dbt;
+        int64_t small_key = 1;
+        toku_fill_dbt(&small_dbt, &small_key, sizeof small_key);
+        small_dbt.flags = DB_DBT_USERMEM;
+        const DBT *small_ptr = &small_dbt;
+
+        // create a large key
+        DBT large_dbt;
+        union { int64_t n; char c[64]; } large_key;
+        memset(&large_key, 0, sizeof large_key);
+        large_key.n = 2;
+        toku_fill_dbt(&large_dbt, &large_key, sizeof large_key);
+        large_dbt.flags = DB_DBT_USERMEM;
+        const DBT *large_dbt_ptr = &large_dbt;
+
+        TXNID txn_a = { 1 };
+        TXNID txn_b = { 2 };
+
+        // a locks small key
+        lock_request a;
+        a.create();
+        a.set(lt, txn_a, small_ptr, small_ptr, lock_request::type::WRITE, false);
+        r = a.start();
+        assert(r == 0);
+        assert(a.m_state == lock_request::state::COMPLETE);
+
+        // b tries to lock small key, fails since it is already locked
+        lock_request b;
+        b.create();
+        b.set(lt, txn_b, small_ptr, small_ptr, lock_request::type::WRITE, false);
+        r = b.start();
+        assert(r == DB_LOCK_NOTGRANTED);
+        assert(b.m_state == lock_request::state::PENDING);
+
+        // a locks large key. lock memory is over the limit
+        lock_request c;
+        c.create();
+        c.set(lt, txn_a, large_dbt_ptr, large_dbt_ptr, lock_request::type::WRITE, false);
+        r = c.start();
+        assert(r == 0);
+        assert(c.m_state == lock_request::state::COMPLETE);
+
+        // a releases small key, lock memory is still over the limit
+        locktree_release_lock(lt, txn_a, small_ptr, small_ptr);
+
+        // retry all lock requests, should complete lock request
+        // b with a TOKUDB_OUT_OF_LOCKS result
+        lock_request::retry_all_lock_requests(lt);
+
+        assert(b.m_state == lock_request::state::COMPLETE);
+        assert(b.m_complete_r == TOKUDB_OUT_OF_LOCKS);
+
+        // b waits for small key, gets out of locks
+        r = b.wait(0);
+        assert(r == TOKUDB_OUT_OF_LOCKS);
+        assert(b.m_state == lock_request::state::COMPLETE);
+        assert(b.m_complete_r == TOKUDB_OUT_OF_LOCKS);
+
+        // a releases large key
+        locktree_release_lock(lt, txn_a, large_dbt_ptr, large_dbt_ptr);
+
+        // b locks small key, gets its
+        r = b.start();
+        assert(r == 0);
+
+        // b releases lock so we can exit cleanly
+        locktree_release_lock(lt, txn_b, small_ptr, small_ptr);
+
+        a.destroy();
+        b.destroy();
+
+        mgr.release_lt(lt);
+        mgr.destroy();
+    }
+
+} /* namespace toku */
+
+int main(void) {
+    toku::lock_request_unit_test test;
+    test.run();
+    return 0;
+}
diff --git a/locktree/tests/lock_request_start_deadlock.cc b/locktree/tests/lock_request_start_deadlock.cc
index 343becfc7..c13d5b085 100644
--- a/locktree/tests/lock_request_start_deadlock.cc
+++ b/locktree/tests/lock_request_start_deadlock.cc
@@ -41,7 +41,7 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved.
 namespace toku {
 
 // make sure deadlocks are detected when a lock request starts
-void lock_request_unit_test::test_start_deadlock(void) {
+void lock_request_unit_test::run(void) {
     int r;
     locktree lt;
 
@@ -114,7 +114,7 @@ void lock_request_unit_test::test_start_deadlock(void) {
 
 int main(void) {
     toku::lock_request_unit_test test;
-    test.test_start_deadlock();
+    test.run();
     return 0;
 }
 
diff --git a/locktree/tests/lock_request_start_pending.cc b/locktree/tests/lock_request_start_pending.cc
index ce6510354..6c133b7a0 100644
--- a/locktree/tests/lock_request_start_pending.cc
+++ b/locktree/tests/lock_request_start_pending.cc
@@ -42,7 +42,7 @@ namespace toku {
 
 // starting a lock request without immediate success should get
 // stored in the lock request set as pending.
-void lock_request_unit_test::test_start_pending(void) {
+void lock_request_unit_test::run(void) {
     int r;
     locktree lt;
     lock_request request;
@@ -100,7 +100,7 @@ void lock_request_unit_test::test_start_pending(void) {
 
 int main(void) {
     toku::lock_request_unit_test test;
-    test.test_start_pending();
+    test.run();
     return 0;
 }
 
diff --git a/locktree/tests/lock_request_unit_test.h b/locktree/tests/lock_request_unit_test.h
index 81e6db25b..c5ad7aae4 100644
--- a/locktree/tests/lock_request_unit_test.h
+++ b/locktree/tests/lock_request_unit_test.h
@@ -47,23 +47,7 @@ namespace toku {
 
 class lock_request_unit_test {
 public:
-    // create and set the object's internals, destroy should not crash.
-    void test_create_destroy(void);
-
-    // make setting keys and getting them back works properly.
-    // at a high level, we want to make sure keys are copied
-    // when appropriate and plays nice with +/- infinity.
-    void test_get_set_keys(void);
-
-    // starting a lock request without immediate success should get
-    // stored in the lock request set as pending.
-    void test_start_pending(void);
-
-    // make sure deadlocks are detected when a lock request starts
-    void test_start_deadlock(void);
-
-    // test that the get_wait_time callback works
-    void test_wait_time_callback(void);
+    void run(void);
 
 private:
     // releases a single range lock and retries all lock requests.
diff --git a/locktree/tests/lock_request_wait_out_of_locks.cc b/locktree/tests/lock_request_wait_out_of_locks.cc
new file mode 100644
index 000000000..1c3750065
--- /dev/null
+++ b/locktree/tests/lock_request_wait_out_of_locks.cc
@@ -0,0 +1,114 @@
+/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
+
+// Verify that lock request wait returns TOKUDB_OUT_OF_LOCKS when
+// all of the locktree memory is used.
+
+#include "lock_request_unit_test.h"
+
+namespace toku {
+
+    static void locktree_release_lock(locktree *lt,
+                                      TXNID txn_id,
+                                      const DBT *left,
+                                      const DBT *right) {
+        range_buffer buffer;
+        buffer.create();
+        buffer.append(left, right);
+        lt->release_locks(txn_id, &buffer);
+        buffer.destroy();
+    }
+
+    void lock_request_unit_test::run(void) {
+        int r;
+
+        locktree_manager mgr;
+        mgr.create(nullptr, nullptr, nullptr, nullptr);
+
+        DICTIONARY_ID dict_id = {1};
+        locktree *lt = mgr.get_lt(dict_id, dbt_comparator, nullptr);
+
+        // set max lock memory small so that we can test the limit
+        // with just 2 locks
+        mgr.set_max_lock_memory(300);
+
+        // create a small key
+        DBT small_dbt;
+        int64_t small_key = 1;
+        toku_fill_dbt(&small_dbt, &small_key, sizeof small_key);
+        small_dbt.flags = DB_DBT_USERMEM;
+        const DBT *small_dbt_ptr = &small_dbt;
+
+        // create a large key
+        DBT large_dbt;
+        union { int64_t n; char c[64]; } large_key;
+        memset(&large_key, 0, sizeof large_key);
+        large_key.n = 2;
+        toku_fill_dbt(&large_dbt, &large_key, sizeof large_key);
+        large_dbt.flags = DB_DBT_USERMEM;
+        const DBT *large_dbt_ptr = &large_dbt;
+
+        TXNID txn_a = { 1 };
+        TXNID txn_b = { 2 };
+
+        // a locks small key
+        lock_request a;
+        a.create();
+        a.set(lt, txn_a, small_dbt_ptr, small_dbt_ptr, lock_request::type::WRITE, false);
+        r = a.start();
+        assert(r == 0);
+        assert(a.m_state == lock_request::state::COMPLETE);
+
+        // b tries to lock small key, fails since small key already locked
+        lock_request b;
+        b.create();
+        b.set(lt, txn_b, small_dbt_ptr, small_dbt_ptr, lock_request::type::WRITE, false);
+        r = b.start();
+        assert(r == DB_LOCK_NOTGRANTED);
+        assert(b.m_state == lock_request::state::PENDING);
+
+        // a locks large key. this uses all of the lock memory
+        lock_request c;
+        c.create();
+        c.set(lt, txn_a, large_dbt_ptr, large_dbt_ptr, lock_request::type::WRITE, false);
+        r = c.start();
+        assert(r == 0);
+        assert(c.m_state == lock_request::state::COMPLETE);
+
+        // a releases small key. the lock memory is still over the limit
+        locktree_release_lock(lt, txn_a, small_dbt_ptr, small_dbt_ptr);
+
+        // b waits for small key, gets out of locks since lock memory is over the limit
+        assert(b.m_state == lock_request::state::PENDING);
+        r = b.wait(0);
+        assert(r == TOKUDB_OUT_OF_LOCKS);
+        assert(b.m_state == lock_request::state::COMPLETE);
+
+        // retry pending lock requests
+        lock_request::retry_all_lock_requests(lt);
+
+        // a releases large key
+        locktree_release_lock(lt, txn_a, large_dbt_ptr, large_dbt_ptr);
+
+        // b locks small key, gets it
+        assert(b.m_state == lock_request::state::COMPLETE);
+        r = b.start();
+        assert(r == 0);
+
+        // b releases small key so we can exit cleanly
+        locktree_release_lock(lt, txn_b, small_dbt_ptr, small_dbt_ptr);
+
+        a.destroy();
+        b.destroy();
+
+        mgr.release_lt(lt);
+        mgr.destroy();
+    }
+
+} /* namespace toku */
+
+int main(void) {
+    toku::lock_request_unit_test test;
+    test.run();
+    return 0;
+}
diff --git a/locktree/tests/lock_request_wait_time_callback.cc b/locktree/tests/lock_request_wait_time_callback.cc
index 1647cee1d..e1c9a1d95 100644
--- a/locktree/tests/lock_request_wait_time_callback.cc
+++ b/locktree/tests/lock_request_wait_time_callback.cc
@@ -43,7 +43,7 @@ namespace toku {
 static const uint64_t my_lock_wait_time = 10 * 1000; // 10 sec
 
 // make sure deadlocks are detected when a lock request starts
-void lock_request_unit_test::test_wait_time_callback(void) {
+void lock_request_unit_test::run(void) {
     int r;
     locktree lt;
 
@@ -90,7 +90,7 @@ void lock_request_unit_test::test_wait_time_callback(void) {
 
 int main(void) {
     toku::lock_request_unit_test test;
-    test.test_wait_time_callback();
+    test.run();
     return 0;
 }
 
diff --git a/locktree/tests/test.h b/locktree/tests/test.h
index 921f2468b..8f051e708 100644
--- a/locktree/tests/test.h
+++ b/locktree/tests/test.h
@@ -93,8 +93,8 @@ namespace toku {
         if (toku_dbt_is_infinite(key1) || toku_dbt_is_infinite(key2)) {
             return toku_dbt_infinite_compare(key1, key2);
         } else {
-            invariant(key1->size == sizeof(int64_t));
-            invariant(key2->size == sizeof(int64_t));
+            invariant(key1->size >= sizeof(int64_t));
+            invariant(key2->size >= sizeof(int64_t));
             int64_t a = *(int64_t*) key1->data;
             int64_t b = *(int64_t*) key2->data;
             if (a < b) {
diff --git a/src/tests/put-wait-retry-out-of-locks.cc b/src/tests/put-wait-retry-out-of-locks.cc
new file mode 100644
index 000000000..24f3d1280
--- /dev/null
+++ b/src/tests/put-wait-retry-out-of-locks.cc
@@ -0,0 +1,164 @@
+/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
+#ident "$Id$"
+/*======
+This file is part of PerconaFT.
+
+
+Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved.
+
+    PerconaFT is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License, version 2,
+    as published by the Free Software Foundation.
+
+    PerconaFT is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with PerconaFT.  If not, see <http://www.gnu.org/licenses/>.
+
+----------------------------------------
+
+    PerconaFT is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License, version 3,
+    as published by the Free Software Foundation.
+
+    PerconaFT is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with PerconaFT.  If not, see <http://www.gnu.org/licenses/>.
+======= */
+
+#ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved."
+
+// Verify that a DB put that is waiting on a previously locked key get TOKUDB_OUT_OF_LOCKS
+// result when another key is released AND the lock memory used is still over the limit.
+
+#include "test.h"
+#include "toku_pthread.h"
+
+static int put_small_key(DB *db, DB_TXN *txn) {
+    char k[8] = {};
+    DBT key = { .data = &k, .size = sizeof k};
+    DBT val = {};
+    int r = db->put(db, txn, &key, &val, 0);
+    return r;
+}
+
+static int put_large_key(DB *db, DB_TXN *txn) {
+    char k[200] = {};
+    DBT key = { .data = &k, .size = sizeof k};
+    DBT val = {};
+    int r = db->put(db, txn, &key, &val, 0);
+    return r;
+}
+
+struct test_c_args {
+    DB *db;
+    DB_TXN *txn;
+};
+
+static void *test_c(void *arg) {
+    struct test_c_args *a = (struct test_c_args *) arg;
+    int r = put_small_key(a->db, a->txn);
+    assert(r == TOKUDB_OUT_OF_LOCKS);
+    return arg;
+}
+
+int test_main(int argc, char * const argv[]) {
+    const char *db_env_dir = TOKU_TEST_FILENAME;
+    const char *db_filename = "test.db";
+    int db_env_open_flags = DB_CREATE | DB_PRIVATE | DB_INIT_MPOOL | DB_INIT_TXN | DB_INIT_LOCK | DB_INIT_LOG | DB_THREAD;
+
+    // parse_args(argc, argv);
+    for (int i = 1; i < argc; i++) {
+        if (strcmp(argv[i], "-v") == 0 || strcmp(argv[i], "--verbose") == 0) {
+            verbose++;
+            continue;
+        }
+        if (strcmp(argv[i], "-q") == 0 || strcmp(argv[i], "--quiet") == 0) {
+            if (verbose > 0)
+                verbose--;
+            continue;
+        }
+        assert(0);
+    }
+
+    // setup the test environment
+    int r;
+    char rm_cmd[strlen(db_env_dir) + strlen("rm -rf ") + 1];
+    snprintf(rm_cmd, sizeof(rm_cmd), "rm -rf %s", db_env_dir);
+    r = system(rm_cmd); assert(r == 0);
+
+    r = toku_os_mkdir(db_env_dir, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH); assert(r == 0);
+
+    DB_ENV *db_env = nullptr;
+    r = db_env_create(&db_env, 0); assert(r == 0);
+
+    // Set a small lock memory limit
+    const uint64_t lock_memory_wanted = 300;
+    r = db_env->set_lk_max_memory(db_env, lock_memory_wanted); assert(r == 0);
+    uint64_t lock_memory_limit;
+    r = db_env->get_lk_max_memory(db_env, &lock_memory_limit); assert(r == 0 && lock_memory_limit == lock_memory_wanted);
+
+    r = db_env->open(db_env, db_env_dir, db_env_open_flags, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); assert(r == 0);
+    r = db_env->set_lock_timeout(db_env, 30 * 1000, nullptr); assert(r == 0);
+
+    // create the db
+    DB *db = nullptr;
+    r = db_create(&db, db_env, 0); assert(r == 0);
+    r = db->open(db, nullptr, db_filename, nullptr, DB_BTREE, DB_CREATE|DB_AUTO_COMMIT|DB_THREAD, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); assert(r == 0);
+
+    // create the txn's
+    DB_TXN *txn_a = nullptr;
+    r = db_env->txn_begin(db_env, nullptr, &txn_a, 0); assert(r == 0);
+
+    DB_TXN *txn_b = nullptr;
+    r = db_env->txn_begin(db_env, nullptr, &txn_b, 0); assert(r == 0);
+
+    DB_TXN *txn_c = nullptr;
+    r = db_env->txn_begin(db_env, nullptr, &txn_c, 0); assert(r == 0);
+
+    // Put a small key into the DB.
+    // Before: lock memory used is 0.
+    // After: lock memory used is under the limit.
+    r = put_small_key(db, txn_a);
+    assert(r == 0);
+
+    // Create a thread that will attempt to lock the same key as txn_a.
+    // Effect: this thread will be blocking on the lock request for this
+    // key
+    toku_pthread_t tid_c;
+    test_c_args a = { db, txn_c };
+    r = toku_pthread_create(toku_uninstrumented, &tid_c, nullptr, test_c, &a);
+    assert(r == 0);
+
+    // give thread c some time to get blocked
+    sleep(1);
+
+    // Put a large key into the DB, which should succeed.
+    // Before: lock memory used is under the limit
+    // After: lock memory used is over the limit due to the addition of the large key
+    r = put_large_key(db, txn_b);
+    assert(r == 0);
+
+    // abort txn a, should release lock on the small key but lock memory
+    // is still over the limit, so test c put lock retry should get
+    // TOKUDB_OUT_OF_LOCKS
+    r = txn_a->abort(txn_a); assert(r == 0);
+
+    // cleanup
+    void *ret;
+    r = toku_pthread_join(tid_c, &ret); assert(r == 0);
+    r = txn_b->abort(txn_b); assert(r == 0);
+    r = txn_c->abort(txn_c); assert(r == 0);
+    r = db->close(db, 0); assert(r == 0); db = nullptr;
+    r = db_env->close(db_env, 0); assert(r == 0); db_env = nullptr;
+
+    return 0;
+}
diff --git a/src/tests/simple-put-out-of-locks.cc b/src/tests/simple-put-out-of-locks.cc
new file mode 100644
index 000000000..40165a0c8
--- /dev/null
+++ b/src/tests/simple-put-out-of-locks.cc
@@ -0,0 +1,125 @@
+/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
+#ident "$Id$"
+/*======
+This file is part of PerconaFT.
+
+
+Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved.
+
+    PerconaFT is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License, version 2,
+    as published by the Free Software Foundation.
+
+    PerconaFT is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with PerconaFT.  If not, see <http://www.gnu.org/licenses/>.
+
+----------------------------------------
+
+    PerconaFT is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License, version 3,
+    as published by the Free Software Foundation.
+
+    PerconaFT is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with PerconaFT.  If not, see <http://www.gnu.org/licenses/>.
+======= */
+
+#ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved."
+
+// Verify that a DB put fails with TOKUDB_OUT_OF_LOCKS result when the lock memory
+// use is over the limit.
+
+#include "test.h"
+#include "toku_pthread.h"
+
+static int put_small_key(DB *db, DB_TXN *txn) {
+    char k[8] = {};
+    DBT key = { .data = &k, .size = sizeof k};
+    DBT val = {};
+    int r = db->put(db, txn, &key, &val, 0);
+    return r;
+}
+
+static int put_large_key(DB *db, DB_TXN *txn) {
+    char k[200] = {};
+    DBT key = { .data = &k, .size = sizeof k};
+    DBT val = {};
+    int r = db->put(db, txn, &key, &val, 0);
+    return r;
+}
+
+int test_main(int argc, char * const argv[]) {
+    const char *db_env_dir = TOKU_TEST_FILENAME;
+    const char *db_filename = "test.db";
+    int db_env_open_flags = DB_CREATE | DB_PRIVATE | DB_INIT_MPOOL | DB_INIT_TXN | DB_INIT_LOCK | DB_INIT_LOG | DB_THREAD;
+
+    // parse_args(argc, argv);
+    for (int i = 1; i < argc; i++) {
+        if (strcmp(argv[i], "-v") == 0 || strcmp(argv[i], "--verbose") == 0) {
+            verbose++;
+            continue;
+        }
+        if (strcmp(argv[i], "-q") == 0 || strcmp(argv[i], "--quiet") == 0) {
+            if (verbose > 0)
+                verbose--;
+            continue;
+        }
+        assert(0);
+    }
+
+    // setup the test
+    int r;
+    char rm_cmd[strlen(db_env_dir) + strlen("rm -rf ") + 1];
+    snprintf(rm_cmd, sizeof(rm_cmd), "rm -rf %s", db_env_dir);
+    r = system(rm_cmd); assert(r == 0);
+    r = toku_os_mkdir(db_env_dir, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH); assert(r == 0);
+
+    // create the env
+    DB_ENV *db_env = nullptr;
+    r = db_env_create(&db_env, 0); assert(r == 0);
+
+    // Setup a small lock memory limit
+    const uint64_t lock_memory_wanted = 300;
+    r = db_env->set_lk_max_memory(db_env, lock_memory_wanted); assert(r == 0);
+    uint64_t lock_memory_limit;
+    r = db_env->get_lk_max_memory(db_env, &lock_memory_limit); assert(r == 0 && lock_memory_limit == lock_memory_wanted);
+
+    r = db_env->open(db_env, db_env_dir, db_env_open_flags, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); assert(r == 0);
+    r = db_env->set_lock_timeout(db_env, 30 * 1000, nullptr); assert(r == 0);
+
+    // create the db
+    DB *db = nullptr;
+    r = db_create(&db, db_env, 0); assert(r == 0);
+    r = db->open(db, nullptr, db_filename, nullptr, DB_BTREE, DB_CREATE|DB_AUTO_COMMIT|DB_THREAD, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); assert(r == 0);
+
+    DB_TXN *txn_a = nullptr;
+    r = db_env->txn_begin(db_env, nullptr, &txn_a, 0); assert(r == 0);
+
+    // Put a large key into the DB.
+    // Before: lock memory use is 0.
+    // After: lock memory is over the limit
+    r = put_large_key(db, txn_a);
+    assert(r == 0);
+
+    // Try to put a small key into the DB.
+    // Should get TOKUDB_OUT_OF_LOCKS since lock memory is over the limit.
+    r = put_small_key(db, txn_a);
+    assert(r == TOKUDB_OUT_OF_LOCKS);
+
+    // cleanup
+    r = txn_a->abort(txn_a); assert(r == 0);
+    r = db->close(db, 0); assert(r == 0); db = nullptr;
+    r = db_env->close(db_env, 0); assert(r == 0); db_env = nullptr;
+
+    return 0;
+}

From 1fede892c2ae7cb8320a4279d8368b6411f8414e Mon Sep 17 00:00:00 2001
From: rik prohaska <prohaska7@gmail.com>
Date: Thu, 9 Apr 2020 08:57:13 -0400
Subject: [PATCH 20/29] When a lock is released, pending lock requests are
 retried.  If the retry occurs and the lock memory use is over the limit, then
 pending lock requests should be completed with TOKUDB_OUT_OF_LOCKS result.
 The current code does not handle this case and crashes.  This commit fixes
 this case and is sufficient to fix PS-4328.

---
 locktree/lock_request.cc | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/locktree/lock_request.cc b/locktree/lock_request.cc
index b99eaff3d..64e53821b 100644
--- a/locktree/lock_request.cc
+++ b/locktree/lock_request.cc
@@ -317,16 +317,20 @@ int lock_request::retry(void) {
             m_txnid, m_left_key, m_right_key, &conflicts, m_big_txn);
     }
 
-    // if the acquisition succeeded then remove ourselves from the
-    // set of lock requests, complete, and signal the waiting thread.
-    if (r == 0) {
+    // if the acquisition succeeded or if out of locks
+    // then remove ourselves from the set of lock requests, complete
+    // the lock request, and signal the waiting threads.
+    if (r == 0 || r == TOKUDB_OUT_OF_LOCKS) {
         remove_from_lock_requests();
         complete(r);
         if (m_retry_test_callback)
             m_retry_test_callback();  // test callback
         toku_cond_broadcast(&m_wait_cond);
-    } else {
+    } else if (r == DB_LOCK_NOTGRANTED) {
+        // get the conflicting txnid and remain pending
         m_conflicting_txnid = conflicts.get(0);
+    } else {
+        invariant(0);
     }
     conflicts.destroy();
 

From 3c3f6a0c5badec3f02fd7a64e888c5a316a34524 Mon Sep 17 00:00:00 2001
From: rik prohaska <prohaska7@gmail.com>
Date: Thu, 9 Apr 2020 13:50:49 -0400
Subject: [PATCH 21/29] Speed up helgrind_test_partitioned_counter test by
 running helgrind and drd scheduler with fairness (--fair-sched=try).  This
 should reduce execution time from minutes to seconds.

---
 cmake_modules/TokuSetupCTest.cmake | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cmake_modules/TokuSetupCTest.cmake b/cmake_modules/TokuSetupCTest.cmake
index 5b6882cc4..6bb164cea 100644
--- a/cmake_modules/TokuSetupCTest.cmake
+++ b/cmake_modules/TokuSetupCTest.cmake
@@ -126,7 +126,7 @@ if (BUILD_TESTING OR BUILD_FT_TESTS OR BUILD_SRC_TESTS)
   endmacro(add_toku_test)
 
   ## setup a function to write tests that will run with helgrind
-  set(CMAKE_HELGRIND_COMMAND_STRING "valgrind --quiet --tool=helgrind --error-exitcode=1 --soname-synonyms=somalloc=*tokuportability* --suppressions=${TokuDB_SOURCE_DIR}/src/tests/helgrind.suppressions --trace-children=yes --trace-children-skip=sh,*/sh,basename,*/basename,dirname,*/dirname,rm,*/rm,cp,*/cp,mv,*/mv,cat,*/cat,diff,*/diff,grep,*/grep,date,*/date,test,*/tokudb_dump* --trace-children-skip-by-arg=--only_create,--test,--no-shutdown,novalgrind")
+  set(CMAKE_HELGRIND_COMMAND_STRING "valgrind --quiet --tool=helgrind --error-exitcode=1 --soname-synonyms=somalloc=*tokuportability* --suppressions=${TokuDB_SOURCE_DIR}/src/tests/helgrind.suppressions --trace-children=yes --trace-children-skip=sh,*/sh,basename,*/basename,dirname,*/dirname,rm,*/rm,cp,*/cp,mv,*/mv,cat,*/cat,diff,*/diff,grep,*/grep,date,*/date,test,*/tokudb_dump* --trace-children-skip-by-arg=--only_create,--test,--no-shutdown,novalgrind --fair-sched=try")
   function(add_helgrind_test pfx name)
     separate_arguments(CMAKE_HELGRIND_COMMAND_STRING)
     add_test(
@@ -137,7 +137,7 @@ if (BUILD_TESTING OR BUILD_FT_TESTS OR BUILD_SRC_TESTS)
   endfunction(add_helgrind_test)
 
   ## setup a function to write tests that will run with drd
-  set(CMAKE_DRD_COMMAND_STRING "valgrind --quiet --tool=drd --error-exitcode=1 --soname-synonyms=somalloc=*tokuportability* --suppressions=${TokuDB_SOURCE_DIR}/src/tests/drd.suppressions --trace-children=yes --trace-children-skip=sh,*/sh,basename,*/basename,dirname,*/dirname,rm,*/rm,cp,*/cp,mv,*/mv,cat,*/cat,diff,*/diff,grep,*/grep,date,*/date,test,*/tokudb_dump* --trace-children-skip-by-arg=--only_create,--test,--no-shutdown,novalgrind")
+  set(CMAKE_DRD_COMMAND_STRING "valgrind --quiet --tool=drd --error-exitcode=1 --soname-synonyms=somalloc=*tokuportability* --suppressions=${TokuDB_SOURCE_DIR}/src/tests/drd.suppressions --trace-children=yes --trace-children-skip=sh,*/sh,basename,*/basename,dirname,*/dirname,rm,*/rm,cp,*/cp,mv,*/mv,cat,*/cat,diff,*/diff,grep,*/grep,date,*/date,test,*/tokudb_dump* --trace-children-skip-by-arg=--only_create,--test,--no-shutdown,novalgrind --fair-sched=try")
   function(add_drd_test pfx name)
     separate_arguments(CMAKE_DRD_COMMAND_STRING)
     add_test(

From 853be0be80f728f4917bb9f257ea6432b9906779 Mon Sep 17 00:00:00 2001
From: rik prohaska <prohaska7@gmail.com>
Date: Thu, 9 Apr 2020 13:57:20 -0400
Subject: [PATCH 22/29] Fix a test that crashed in put_callback due to a null
 deref.  The test should intentially abort instead.

---
 src/tests/recover-del-multiple-abort.cc | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/tests/recover-del-multiple-abort.cc b/src/tests/recover-del-multiple-abort.cc
index 425c12e1a..3ffd63919 100644
--- a/src/tests/recover-del-multiple-abort.cc
+++ b/src/tests/recover-del-multiple-abort.cc
@@ -57,9 +57,12 @@ get_data(int *v, int i, int ndbs) {
 static int
 put_callback(DB *dest_db, DB *src_db, DBT_ARRAY *dest_keys, DBT_ARRAY *dest_vals, const DBT *src_key, const DBT *src_val) {
     toku_dbt_array_resize(dest_keys, 1);
-    toku_dbt_array_resize(dest_vals, 1);
     DBT *dest_key = &dest_keys->dbts[0];
-    DBT *dest_val = &dest_vals->dbts[0];
+    DBT *dest_val = NULL;
+    if (dest_vals) {
+        toku_dbt_array_resize(dest_vals, 1);
+        dest_val = &dest_vals->dbts[0];
+    }
     (void) dest_db; (void) src_db; (void) dest_key; (void) dest_val; (void) src_key; (void) src_val;
     assert(src_db == NULL);
 

From 477b5165ec3f193a849738723a133006b2947c1a Mon Sep 17 00:00:00 2001
From: rik prohaska <prohaska7@gmail.com>
Date: Thu, 9 Apr 2020 17:28:26 -0400
Subject: [PATCH 23/29] Ignore mallopt error return in memory_init when running
 the address sanitizer.  This allows the tests to run.

---
 portability/memory.cc | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/portability/memory.cc b/portability/memory.cc
index f4888dccc..ddfca5f32 100644
--- a/portability/memory.cc
+++ b/portability/memory.cc
@@ -100,6 +100,12 @@ toku_memory_startup(void) {
     } else {
         result = EINVAL;
     }
+#if defined(__has_feature)
+#if __has_feature(address_sanitizer)
+    result = 0; // the clang 8 address sanitizer fails mallopt, ignore it
+    status.mmap_threshold = mmap_threshold;
+#endif
+#endif
     assert(result == 0);
 #else
     // just a guess

From 5bee63ff01aa1ca68aed2b370f6e514bb65707ee Mon Sep 17 00:00:00 2001
From: rik prohaska <prohaska7@gmail.com>
Date: Fri, 10 Apr 2020 08:16:39 -0400
Subject: [PATCH 24/29] Suppress pthread_mutex_destroy race with
 pthread_mutex_unlock again. It causes helgrind to report false positive data
 races on tests, specifically the helgrind_helgrind tests.

4101: ==17113== Possible data race during read of size 1 at 0x52FA980 by thread #1
4101: ==17113== Locks held: 1, at address 0x4C126C0
4101: ==17113==    at 0x483E99C: ??? (in /usr/lib/x86_64-linux-gnu/valgrind/vgpreload_helgrind-amd64-linux.so)
4101: ==17113==    by 0x483EC4A: ??? (in /usr/lib/x86_64-linux-gnu/valgrind/vgpreload_helgrind-amd64-linux.so)
4101: ==17113==    by 0x4A95654: toku_mutex_destroy(toku_mutex_t*) (toku_pthread.h:427)
4101: ==17113==    by 0x4B6CF5C: bjm_destroy(background_job_manager_struct*) (background_job_manager.cc:67)
4101: ==17113==    by 0x4ABE119: write_dirty_pairs_for_close(cachetable*, cachefile*) (cachetable.cc:2406)
4101: ==17113==    by 0x4ABE308: cachetable_flush_cachefile(cachetable*, cachefile*, bool) (cachetable.cc:2493)
4101: ==17113==    by 0x4AB99BE: toku_cachefile_close(cachefile**, bool, __toku_lsn) (cachetable.cc:529)
4101: ==17113==    by 0x4ACD7C2: toku_ft_evict_from_memory(ft*, bool, __toku_lsn) (ft.cc:499)
4101: ==17113==    by 0x4ACEADD: toku_ft_remove_reference(ft*, bool, __toku_lsn, void (*)(ft*, void*), void*) (ft.cc:949)
4101: ==17113==    by 0x4AE9B21: ft_handle_close(ft_handle*, bool, __toku_lsn) (ft-ops.cc:3272)
4101: ==17113==    by 0x4AE9B54: toku_ft_handle_close(ft_handle*) (ft-ops.cc:3280)
4101: ==17113==    by 0x4A8CD68: toku_db_close(__toku_db*) (ydb_db.cc:212)
4101: ==17113==
4101: ==17113== This conflicts with a previous write of size 4 by thread #10
4101: ==17113== Locks held: none
4101: ==17113==    at 0x4C557A1: __pthread_mutex_unlock_usercnt (pthread_mutex_unlock.c:51)
4101: ==17113==    by 0x4C557A1: pthread_mutex_unlock (pthread_mutex_unlock.c:357)
4101: ==17113==    by 0x483F458: ??? (in /usr/lib/x86_64-linux-gnu/valgrind/vgpreload_helgrind-amd64-linux.so)
4101: ==17113==    by 0x4A84508: toku_mutex_unlock(toku_mutex_t*) (toku_pthread.h:256)
4101: ==17113==    by 0x4B6D0EC: bjm_remove_background_job(background_job_manager_struct*) (background_job_manager.cc:98)
4101: ==17113==    by 0x4ABDCDD: cachetable_flush_pair_for_close(void*) (cachetable.cc:2327)
4101: ==17113==    by 0x4B7D785: work_on_kibbutz(void*) (kibbutz.cc:154)
4101: ==17113==    by 0x4841B1A: ??? (in /usr/lib/x86_64-linux-gnu/valgrind/vgpreload_helgrind-amd64-linux.so)
4101: ==17113==    by 0x4C51668: start_thread (pthread_create.c:479)
---
 src/tests/helgrind.suppressions | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/tests/helgrind.suppressions b/src/tests/helgrind.suppressions
index 4729c1d14..c6bdda640 100644
--- a/src/tests/helgrind.suppressions
+++ b/src/tests/helgrind.suppressions
@@ -156,3 +156,10 @@
     ...
     fun:pthread_mutex_destroy
 }
+{
+    pthread_mutex_destroy/pthread_mutex_unlock race glibc 2.30 gcc-9.2.1 valgrind-3.15.0
+    Helgrind:Race
+    obj:/usr/lib*/valgrind/vgpreload_helgrind-amd64-linux.so*
+    obj:/usr/lib*/valgrind/vgpreload_helgrind-amd64-linux.so*
+    fun:*toku_mutex_destroy*
+}

From 6dd081f78ef6a95b242b7b7c367043c0eb3e59b2 Mon Sep 17 00:00:00 2001
From: rik prohaska <prohaska7@gmail.com>
Date: Fri, 10 Apr 2020 08:19:38 -0400
Subject: [PATCH 25/29] Suppress memcheck uninitialized data jump in
 lz_encoder_prepare. See
 https://prohaska7.blogspot.com/2015/11/uninitialized-data-problem-in-lzma.html
 for details.

---
 cmake_modules/TokuSetupCTest.cmake | 2 ++
 third_party/xz.suppressions        | 5 +++++
 2 files changed, 7 insertions(+)
 create mode 100644 third_party/xz.suppressions

diff --git a/cmake_modules/TokuSetupCTest.cmake b/cmake_modules/TokuSetupCTest.cmake
index 6bb164cea..66d3497fc 100644
--- a/cmake_modules/TokuSetupCTest.cmake
+++ b/cmake_modules/TokuSetupCTest.cmake
@@ -94,6 +94,8 @@ if (BUILD_TESTING OR BUILD_FT_TESTS OR BUILD_SRC_TESTS)
   ## set up full valgrind suppressions file (concatenate the suppressions files)
   file(READ ft/valgrind.suppressions valgrind_suppressions)
   file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/valgrind.suppressions" "${valgrind_suppressions}")
+  file(READ third_party/xz.suppressions xz_suppressions)
+  file(APPEND "${CMAKE_CURRENT_BINARY_DIR}/valgrind.suppressions" "${xz_suppressions}")
   file(READ bash.suppressions bash_suppressions)
   file(APPEND "${CMAKE_CURRENT_BINARY_DIR}/valgrind.suppressions" "${bash_suppressions}")
 
diff --git a/third_party/xz.suppressions b/third_party/xz.suppressions
new file mode 100644
index 000000000..a6b87c302
--- /dev/null
+++ b/third_party/xz.suppressions
@@ -0,0 +1,5 @@
+{
+    lzma: Conditional jump or move depends on uninitialised value (https://prohaska7.blogspot.com/2015/11/uninitialized-data-problem-in-lzma.html)
+    Memcheck:Cond
+    fun:lz_encoder_prepare
+}

From 603a917627634a1fea7debe1bb9e981ec76cbe52 Mon Sep 17 00:00:00 2001
From: rik prohaska <prohaska7@gmail.com>
Date: Fri, 10 Apr 2020 17:07:23 -0400
Subject: [PATCH 26/29] Some ydb tests leak memory for pfs keys. Fix the ydb
 library init and destroy functions.

---
 src/ydb.cc | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/src/ydb.cc b/src/ydb.cc
index 3e21ad0f8..d88d7a26e 100644
--- a/src/ydb.cc
+++ b/src/ydb.cc
@@ -179,23 +179,26 @@ static int toku_maybe_get_engine_status_text (char* buff, int buffsize);  // for
 static int toku_maybe_err_engine_status (void);
 static void toku_maybe_set_env_panic(int code, const char * msg);               // for use by toku_assert
 
-int 
-toku_ydb_init(void) {
+static bool ydb_initialized = false;
+int toku_ydb_init(void) {
     int r = 0;
-    //Lower level must be initialized first.
-    r = toku_ft_layer_init();
+    if (!ydb_initialized) {
+        //Lower level must be initialized first.
+        r = toku_ft_layer_init();
+        if (r == 0)
+            ydb_initialized = true;
+    }
     return r;
 }
 
 // Do not clean up resources if env is panicked, just exit ugly
-void 
-toku_ydb_destroy(void) {
-    if (!ydb_layer_status.initialized)
+void toku_ydb_destroy(void) {
+    if (!ydb_initialized)
         return;
     if (env_is_panicked == 0) {
         toku_ft_layer_destroy();
     }
-    ydb_layer_status.initialized = false;
+    ydb_initialized = false;
 }
 
 static int

From 4cc6aafa2954d095dd950a35b0fbf92ba5367d68 Mon Sep 17 00:00:00 2001
From: rik prohaska <prohaska7@gmail.com>
Date: Sat, 11 Apr 2020 07:17:00 -0400
Subject: [PATCH 27/29] Suppress memory leak in libjson on ubuntu 19.10 when
 running cmake.

---
 cmake_modules/TokuSetupCTest.cmake                 |  4 +++-
 bash.suppressions => third_party/bash.suppressions |  0
 third_party/cmake.suppressions                     | 11 +++++++++++
 3 files changed, 14 insertions(+), 1 deletion(-)
 rename bash.suppressions => third_party/bash.suppressions (100%)
 create mode 100644 third_party/cmake.suppressions

diff --git a/cmake_modules/TokuSetupCTest.cmake b/cmake_modules/TokuSetupCTest.cmake
index 66d3497fc..9540d30b9 100644
--- a/cmake_modules/TokuSetupCTest.cmake
+++ b/cmake_modules/TokuSetupCTest.cmake
@@ -96,7 +96,9 @@ if (BUILD_TESTING OR BUILD_FT_TESTS OR BUILD_SRC_TESTS)
   file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/valgrind.suppressions" "${valgrind_suppressions}")
   file(READ third_party/xz.suppressions xz_suppressions)
   file(APPEND "${CMAKE_CURRENT_BINARY_DIR}/valgrind.suppressions" "${xz_suppressions}")
-  file(READ bash.suppressions bash_suppressions)
+  file(READ third_party/cmake.suppressions cmake_suppressions)
+  file(APPEND "${CMAKE_CURRENT_BINARY_DIR}/valgrind.suppressions" "${cmake_suppressions}")
+  file(READ third_party/bash.suppressions bash_suppressions)
   file(APPEND "${CMAKE_CURRENT_BINARY_DIR}/valgrind.suppressions" "${bash_suppressions}")
 
   include(CMakeDependentOption)
diff --git a/bash.suppressions b/third_party/bash.suppressions
similarity index 100%
rename from bash.suppressions
rename to third_party/bash.suppressions
diff --git a/third_party/cmake.suppressions b/third_party/cmake.suppressions
new file mode 100644
index 000000000..4b6168ffe
--- /dev/null
+++ b/third_party/cmake.suppressions
@@ -0,0 +1,11 @@
+{
+   cmake 3.13.4 on ubuntu 19.04
+   Memcheck:Leak
+   match-leak-kinds: reachable
+   fun:_Znwm
+   fun:_ZN4Json5Value13nullSingletonEv
+   obj:/usr/*lib*/libjsoncpp*
+   ...
+   fun:call_init
+   fun:_dl_init
+}

From d4426921f35985cb0e0fb2a8b4e7de32abd55170 Mon Sep 17 00:00:00 2001
From: rik prohaska <prohaska7@gmail.com>
Date: Sat, 11 Apr 2020 07:18:33 -0400
Subject: [PATCH 28/29] Run try-leak-reachable with RelWithDebInfo build type
 by avoiding gcc optimization.

---
 portability/tests/try-leak-reachable.cc | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/portability/tests/try-leak-reachable.cc b/portability/tests/try-leak-reachable.cc
index f93d10087..dd0b1ad03 100644
--- a/portability/tests/try-leak-reachable.cc
+++ b/portability/tests/try-leak-reachable.cc
@@ -36,11 +36,14 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved.
 
 #ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved."
 
+#include <stdio.h>
 #include <stdlib.h>
 
 static void *vp;
 
 int main(void) {
     vp = malloc(42);
+    // GCC has gotten smart enough to optimize this away unless we use it.
+    printf("%p\n", vp);
     return 0;
 }

From 95515e82719704bcc9cc04c29c737179eecc64f3 Mon Sep 17 00:00:00 2001
From: rik prohaska <prohaska7@gmail.com>
Date: Sun, 19 Apr 2020 16:16:45 -0400
Subject: [PATCH 29/29] Fix data race on the writing_rollback variable detected
 by the DRD and the thread sanitizer.

Multiple threads are allowed to increment and decrement this variable without
any atomicity.  The solution is to change the type of the writing_rollback
variable from an 'int' to an 'atomic_int'.

One may need to run the following test cases multiple times to stumble upon
the data race.

ctest -R ydb/drd_test_groupcommit_count --verbose

4104: ==21516== Thread 43:
4104: ==21516== Conflicting load by thread 43 at 0x04c22440 size 4
4104: ==21516==    at 0x4B4EB26: rollback_log_create(tokutxn*, blocknum_s, rollback_log_node**) (rollback.cc:160)
4104: ==21516==    by 0x4B4F4D9: toku_get_and_pin_rollback_log_for_new_entry(tokutxn*, rollback_log_node**) (rollback.cc:327)
4104: ==21516==    by 0x4B6F45F: toku_logger_save_rollback_cmdinsert(tokutxn*, FILENUM, BYTESTRING*) (log_code.cc:2007)
4104: ==21516==    by 0x4AF3117: ft_txn_log_insert(ft*, __toku_dbt*, __toku_dbt*, tokutxn*, bool, ft_msg_type) (ft-ops.cc:2406)
4104: ==21516==    by 0x4AF32B0: toku_ft_maybe_insert(ft_handle*, __toku_dbt*, __toku_dbt*, tokutxn*, bool, __toku_lsn, bool, ft_msg_type) (ft-ops.cc:2423)
4104: ==21516==    by 0x4AA2AA9: db_put(__toku_db*, __toku_db_txn*, __toku_dbt*, __toku_dbt*, int, bool) (ydb_write.cc:221)
4104: ==21516==    by 0x4AA2CB3: toku_db_put(__toku_db*, __toku_db_txn*, __toku_dbt*, __toku_dbt*, unsigned int, bool) (ydb_write.cc:248)
4104: ==21516==    by 0x4AA68FC: autotxn_db_put(__toku_db*, __toku_db_txn*, __toku_dbt*, __toku_dbt*, unsigned int) (ydb_write.cc:1078)
4104: ==21516==    by 0x10B099: start_a_thread(void*) (test_groupcommit_count.cc:64)
4104: ==21516==    by 0x48414BA: ??? (in /usr/lib/x86_64-linux-gnu/valgrind/vgpreload_drd-amd64-linux.so)
4104: ==21516==    by 0x4C5D668: start_thread (pthread_create.c:479)
4104: ==21516==    by 0x4D99322: clone (clone.S:95)
4104: ==21516== Allocation context: BSS section of /home/rfp/projects/tokuft-Debug-gcc-7/src/libtokufractaltree.so
4104: ==21516== Other segment start (thread 42)
4104: ==21516==    at 0x484587C: pthread_mutex_unlock (in /usr/lib/x86_64-linux-gnu/valgrind/vgpreload_drd-amd64-linux.so)
4104: ==21516==    by 0x4B8BE37: pc_unlock() (partitioned_counter.cc:151)
4104: ==21516==    by 0x4B8C583: get_or_alloc_thread_local_counter(partitioned_counter*) (partitioned_counter.cc:366)
4104: ==21516==    by 0x4B8C5C0: increment_partitioned_counter (partitioned_counter.cc:375)
4104: ==21516==    by 0x4ACD9B5: evictor::add_pair_attr(pair_attr_s) (cachetable.cc:3769)
4104: ==21516==    by 0x4AC684E: cachetable_insert_pair_at(cachetable*, ctpair*, pair_attr_s) (cachetable.cc:871)
4104: ==21516==    by 0x4AC689E: cachetable_put_internal(cachefile*, ctpair*, void*, pair_attr_s, void (*)(blocknum_s, void*, ctpair*)) (cachetable.cc:897)
4104: ==21516==    by 0x4AC7410: toku_cachetable_put(cachefile*, blocknum_s, unsigned int, void*, pair_attr_s, CACHETABLE_WRITE_CALLBACK, void (*)(blocknum_s, void*, ctpair*)) (cachetable.cc:1207)
4104: ==21516==    by 0x4B4EC6F: rollback_log_create(tokutxn*, blocknum_s, rollback_log_node**) (rollback.cc:170)
4104: ==21516==    by 0x4B4F4D9: toku_get_and_pin_rollback_log_for_new_entry(tokutxn*, rollback_log_node**) (rollback.cc:327)
4104: ==21516==    by 0x4B6F45F: toku_logger_save_rollback_cmdinsert(tokutxn*, FILENUM, BYTESTRING*) (log_code.cc:2007)
4104: ==21516==    by 0x4AF3117: ft_txn_log_insert(ft*, __toku_dbt*, __toku_dbt*, tokutxn*, bool, ft_msg_type) (ft-ops.cc:2406)
4104: ==21516== Other segment end (thread 42)
4104: ==21516==    at 0x484482C: pthread_mutex_lock (in /usr/lib/x86_64-linux-gnu/valgrind/vgpreload_drd-amd64-linux.so)
4104: ==21516==    by 0x4A90590: toku_mutex_lock_with_source_location(toku_mutex_t*, char const*, int) (toku_pthread.h:267)
4104: ==21516==    by 0x4AC49BB: pair_lock(ctpair*) (cachetable.cc:117)
4104: ==21516==    by 0x4AC8D38: cachetable_unpin_internal(cachefile*, ctpair*, cachetable_dirty, pair_attr_s, bool) (cachetable.cc:1899)
4104: ==21516==    by 0x4AC8F5B: toku_cachetable_unpin(cachefile*, ctpair*, cachetable_dirty, pair_attr_s) (cachetable.cc:1932)
4104: ==21516==    by 0x4B4ED32: toku_rollback_log_unpin(tokutxn*, rollback_log_node*) (rollback.cc:181)
4104: ==21516==    by 0x4B6F5CE: toku_logger_save_rollback_cmdinsert(tokutxn*, FILENUM, BYTESTRING*) (log_code.cc:2029)
4104: ==21516==    by 0x4AF3117: ft_txn_log_insert(ft*, __toku_dbt*, __toku_dbt*, tokutxn*, bool, ft_msg_type) (ft-ops.cc:2406)
4104: ==21516==    by 0x4AF32B0: toku_ft_maybe_insert(ft_handle*, __toku_dbt*, __toku_dbt*, tokutxn*, bool, __toku_lsn, bool, ft_msg_type) (ft-ops.cc:2423)
4104: ==21516==    by 0x4AA2AA9: db_put(__toku_db*, __toku_db_txn*, __toku_dbt*, __toku_dbt*, int, bool) (ydb_write.cc:221)
4104: ==21516==    by 0x4AA2CB3: toku_db_put(__toku_db*, __toku_db_txn*, __toku_dbt*, __toku_dbt*, unsigned int, bool) (ydb_write.cc:248)
4104: ==21516==    by 0x4AA68FC: autotxn_db_put(__toku_db*, __toku_db_txn*, __toku_dbt*, __toku_dbt*, unsigned int) (ydb_write.cc:1078)
4104: ==21516==
4104: ==21516== Conflicting store by thread 43 at 0x04c22440 size 4
4104: ==21516==    at 0x4B4EB32: rollback_log_create(tokutxn*, blocknum_s, rollback_log_node**) (rollback.cc:160)
4104: ==21516==    by 0x4B4F4D9: toku_get_and_pin_rollback_log_for_new_entry(tokutxn*, rollback_log_node**) (rollback.cc:327)
4104: ==21516==    by 0x4B6F45F: toku_logger_save_rollback_cmdinsert(tokutxn*, FILENUM, BYTESTRING*) (log_code.cc:2007)
4104: ==21516==    by 0x4AF3117: ft_txn_log_insert(ft*, __toku_dbt*, __toku_dbt*, tokutxn*, bool, ft_msg_type) (ft-ops.cc:2406)
4104: ==21516==    by 0x4AF32B0: toku_ft_maybe_insert(ft_handle*, __toku_dbt*, __toku_dbt*, tokutxn*, bool, __toku_lsn, bool, ft_msg_type) (ft-ops.cc:2423)
4104: ==21516==    by 0x4AA2AA9: db_put(__toku_db*, __toku_db_txn*, __toku_dbt*, __toku_dbt*, int, bool) (ydb_write.cc:221)
4104: ==21516==    by 0x4AA2CB3: toku_db_put(__toku_db*, __toku_db_txn*, __toku_dbt*, __toku_dbt*, unsigned int, bool) (ydb_write.cc:248)
4104: ==21516==    by 0x4AA68FC: autotxn_db_put(__toku_db*, __toku_db_txn*, __toku_dbt*, __toku_dbt*, unsigned int) (ydb_write.cc:1078)
4104: ==21516==    by 0x10B099: start_a_thread(void*) (test_groupcommit_count.cc:64)
4104: ==21516==    by 0x48414BA: ??? (in /usr/lib/x86_64-linux-gnu/valgrind/vgpreload_drd-amd64-linux.so)
4104: ==21516==    by 0x4C5D668: start_thread (pthread_create.c:479)
4104: ==21516==    by 0x4D99322: clone (clone.S:95)
4104: ==21516== Allocation context: BSS section of /home/rfp/projects/tokuft-Debug-gcc-7/src/libtokufractaltree.so
4104: ==21516== Other segment start (thread 42)
4104: ==21516==    at 0x484587C: pthread_mutex_unlock (in /usr/lib/x86_64-linux-gnu/valgrind/vgpreload_drd-amd64-linux.so)
4104: ==21516==    by 0x4B8BE37: pc_unlock() (partitioned_counter.cc:151)
4104: ==21516==    by 0x4B8C583: get_or_alloc_thread_local_counter(partitioned_counter*) (partitioned_counter.cc:366)
4104: ==21516==    by 0x4B8C5C0: increment_partitioned_counter (partitioned_counter.cc:375)
4104: ==21516==    by 0x4ACD9B5: evictor::add_pair_attr(pair_attr_s) (cachetable.cc:3769)
4104: ==21516==    by 0x4AC684E: cachetable_insert_pair_at(cachetable*, ctpair*, pair_attr_s) (cachetable.cc:871)
4104: ==21516==    by 0x4AC689E: cachetable_put_internal(cachefile*, ctpair*, void*, pair_attr_s, void (*)(blocknum_s, void*, ctpair*)) (cachetable.cc:897)
4104: ==21516==    by 0x4AC7410: toku_cachetable_put(cachefile*, blocknum_s, unsigned int, void*, pair_attr_s, CACHETABLE_WRITE_CALLBACK, void (*)(blocknum_s, void*, ctpair*)) (cachetable.cc:1207)
4104: ==21516==    by 0x4B4EC6F: rollback_log_create(tokutxn*, blocknum_s, rollback_log_node**) (rollback.cc:170)
4104: ==21516==    by 0x4B4F4D9: toku_get_and_pin_rollback_log_for_new_entry(tokutxn*, rollback_log_node**) (rollback.cc:327)
4104: ==21516==    by 0x4B6F45F: toku_logger_save_rollback_cmdinsert(tokutxn*, FILENUM, BYTESTRING*) (log_code.cc:2007)
4104: ==21516==    by 0x4AF3117: ft_txn_log_insert(ft*, __toku_dbt*, __toku_dbt*, tokutxn*, bool, ft_msg_type) (ft-ops.cc:2406)
4104: ==21516== Other segment end (thread 42)
4104: ==21516==    at 0x484482C: pthread_mutex_lock (in /usr/lib/x86_64-linux-gnu/valgrind/vgpreload_drd-amd64-linux.so)
4104: ==21516==    by 0x4A90590: toku_mutex_lock_with_source_location(toku_mutex_t*, char const*, int) (toku_pthread.h:267)
4104: ==21516==    by 0x4AC49BB: pair_lock(ctpair*) (cachetable.cc:117)
4104: ==21516==    by 0x4AC8D38: cachetable_unpin_internal(cachefile*, ctpair*, cachetable_dirty, pair_attr_s, bool) (cachetable.cc:1899)
4104: ==21516==    by 0x4AC8F5B: toku_cachetable_unpin(cachefile*, ctpair*, cachetable_dirty, pair_attr_s) (cachetable.cc:1932)
4104: ==21516==    by 0x4B4ED32: toku_rollback_log_unpin(tokutxn*, rollback_log_node*) (rollback.cc:181)
4104: ==21516==    by 0x4B6F5CE: toku_logger_save_rollback_cmdinsert(tokutxn*, FILENUM, BYTESTRING*) (log_code.cc:2029)
4104: ==21516==    by 0x4AF3117: ft_txn_log_insert(ft*, __toku_dbt*, __toku_dbt*, tokutxn*, bool, ft_msg_type) (ft-ops.cc:2406)
4104: ==21516==    by 0x4AF32B0: toku_ft_maybe_insert(ft_handle*, __toku_dbt*, __toku_dbt*, tokutxn*, bool, __toku_lsn, bool, ft_msg_type) (ft-ops.cc:2423)
4104: ==21516==    by 0x4AA2AA9: db_put(__toku_db*, __toku_db_txn*, __toku_dbt*, __toku_dbt*, int, bool) (ydb_write.cc:221)
4104: ==21516==    by 0x4AA2CB3: toku_db_put(__toku_db*, __toku_db_txn*, __toku_dbt*, __toku_dbt*, unsigned int, bool) (ydb_write.cc:248)
4104: ==21516==    by 0x4AA68FC: autotxn_db_put(__toku_db*, __toku_db_txn*, __toku_dbt*, __toku_dbt*, unsigned int) (ydb_write.cc:1078)

ctest -R ydb/drd_tiny_test_stress0 --verbose

4107: ==21390== Thread 52:
4107: ==21390== Conflicting load by thread 52 at 0x04c22440 size 4
4107: ==21390==    at 0x4B4EB26: rollback_log_create(tokutxn*, blocknum_s, rollback_log_node**) (rollback.cc:160)
4107: ==21390==    by 0x4B4F4D9: toku_get_and_pin_rollback_log_for_new_entry(tokutxn*, rollback_log_node**) (rollback.cc:327)
4107: ==21390==    by 0x4B6FF22: toku_logger_save_rollback_cmdupdate(tokutxn*, FILENUM, BYTESTRING*) (log_code.cc:2168)
4107: ==21390==    by 0x4AF3741: toku_ft_maybe_update(ft_handle*, __toku_dbt const*, __toku_dbt const*, tokutxn*, bool, __toku_lsn, bool) (ft-ops.cc:2485)
4107: ==21390==    by 0x4AA2FB8: toku_db_update(__toku_db*, __toku_db_txn*, __toku_dbt const*, __toku_dbt const*, unsigned int) (ydb_write.cc:290)
4107: ==21390==    by 0x4AA69C5: autotxn_db_update(__toku_db*, __toku_db_txn*, __toku_dbt const*, __toku_dbt const*, unsigned int) (ydb_write.cc:1094)
4107: ==21390==    by 0x113C8F: update_op_db(__toku_db*, __toku_db_txn*, arg*, void*, void*) (threaded_stress_test_helpers.h:1488)
4107: ==21390==    by 0x113D55: update_op(__toku_db_txn*, arg*, void*, void*) (threaded_stress_test_helpers.h:1508)
4107: ==21390==    by 0x10F64D: worker(void*) (threaded_stress_test_helpers.h:537)
4107: ==21390==    by 0x48414BA: ??? (in /usr/lib/x86_64-linux-gnu/valgrind/vgpreload_drd-amd64-linux.so)
4107: ==21390==    by 0x4C8B668: start_thread (pthread_create.c:479)
4107: ==21390==    by 0x4DC7322: clone (clone.S:95)
4107: ==21390== Allocation context: BSS section of /home/rfp/projects/tokuft-Debug-gcc-7/src/libtokufractaltree.so
4107: ==21390== Other segment start (thread 53)
4107: ==21390==    at 0x484E3FB: pthread_rwlock_unlock (in /usr/lib/x86_64-linux-gnu/valgrind/vgpreload_drd-amd64-linux.so)
4107: ==21390==    by 0x4A9095C: toku_pthread_rwlock_wrunlock(toku_pthread_rwlock_t*) (toku_pthread.h:496)
4107: ==21390==    by 0x4ACCE5D: pair_list::write_list_unlock() (cachetable.cc:3580)
4107: ==21390==    by 0x4AC742C: toku_cachetable_put(cachefile*, blocknum_s, unsigned int, void*, pair_attr_s, CACHETABLE_WRITE_CALLBACK, void (*)(blocknum_s, void*, ctpair*)) (cachetable.cc:1215)
4107: ==21390==    by 0x4B4EC6F: rollback_log_create(tokutxn*, blocknum_s, rollback_log_node**) (rollback.cc:170)
4107: ==21390==    by 0x4B4F4D9: toku_get_and_pin_rollback_log_for_new_entry(tokutxn*, rollback_log_node**) (rollback.cc:327)
4107: ==21390==    by 0x4B6FF22: toku_logger_save_rollback_cmdupdate(tokutxn*, FILENUM, BYTESTRING*) (log_code.cc:2168)
4107: ==21390==    by 0x4AF3741: toku_ft_maybe_update(ft_handle*, __toku_dbt const*, __toku_dbt const*, tokutxn*, bool, __toku_lsn, bool) (ft-ops.cc:2485)
4107: ==21390==    by 0x4AA2FB8: toku_db_update(__toku_db*, __toku_db_txn*, __toku_dbt const*, __toku_dbt const*, unsigned int) (ydb_write.cc:290)
4107: ==21390==    by 0x4AA69C5: autotxn_db_update(__toku_db*, __toku_db_txn*, __toku_dbt const*, __toku_dbt const*, unsigned int) (ydb_write.cc:1094)
4107: ==21390==    by 0x113C8F: update_op_db(__toku_db*, __toku_db_txn*, arg*, void*, void*) (threaded_stress_test_helpers.h:1488)
4107: ==21390==    by 0x113D55: update_op(__toku_db_txn*, arg*, void*, void*) (threaded_stress_test_helpers.h:1508)
4107: ==21390== Other segment end (thread 53)
4107: ==21390==    at 0x484482C: pthread_mutex_lock (in /usr/lib/x86_64-linux-gnu/valgrind/vgpreload_drd-amd64-linux.so)
4107: ==21390==    by 0x4A90590: toku_mutex_lock_with_source_location(toku_mutex_t*, char const*, int) (toku_pthread.h:267)
4107: ==21390==    by 0x4AC49BB: pair_lock(ctpair*) (cachetable.cc:117)
4107: ==21390==    by 0x4AC8D38: cachetable_unpin_internal(cachefile*, ctpair*, cachetable_dirty, pair_attr_s, bool) (cachetable.cc:1899)
4107: ==21390==    by 0x4AC8F5B: toku_cachetable_unpin(cachefile*, ctpair*, cachetable_dirty, pair_attr_s) (cachetable.cc:1932)
4107: ==21390==    by 0x4B4ED32: toku_rollback_log_unpin(tokutxn*, rollback_log_node*) (rollback.cc:181)
4107: ==21390==    by 0x4B70091: toku_logger_save_rollback_cmdupdate(tokutxn*, FILENUM, BYTESTRING*) (log_code.cc:2190)
4107: ==21390==    by 0x4AF3741: toku_ft_maybe_update(ft_handle*, __toku_dbt const*, __toku_dbt const*, tokutxn*, bool, __toku_lsn, bool) (ft-ops.cc:2485)
4107: ==21390==    by 0x4AA2FB8: toku_db_update(__toku_db*, __toku_db_txn*, __toku_dbt const*, __toku_dbt const*, unsigned int) (ydb_write.cc:290)
4107: ==21390==    by 0x4AA69C5: autotxn_db_update(__toku_db*, __toku_db_txn*, __toku_dbt const*, __toku_dbt const*, unsigned int) (ydb_write.cc:1094)
4107: ==21390==    by 0x113C8F: update_op_db(__toku_db*, __toku_db_txn*, arg*, void*, void*) (threaded_stress_test_helpers.h:1488)
4107: ==21390==    by 0x113D55: update_op(__toku_db_txn*, arg*, void*, void*) (threaded_stress_test_helpers.h:1508)
4107: ==21390== Other segment start (thread 53)
4107: ==21390==    at 0x484587C: pthread_mutex_unlock (in /usr/lib/x86_64-linux-gnu/valgrind/vgpreload_drd-amd64-linux.so)
4107: ==21390==    by 0x4A90508: toku_mutex_unlock(toku_mutex_t*) (toku_pthread.h:256)
4107: ==21390==    by 0x4B521CD: rollback_log_node_cache::get_rollback_log_node(tokutxn*, rollback_log_node**) (rollback_log_node_cache.cc:101)
4107: ==21390==    by 0x4B4F478: toku_get_and_pin_rollback_log_for_new_entry(tokutxn*, rollback_log_node**) (rollback.cc:316)
4107: ==21390==    by 0x4B6FF22: toku_logger_save_rollback_cmdupdate(tokutxn*, FILENUM, BYTESTRING*) (log_code.cc:2168)
4107: ==21390==    by 0x4AF3741: toku_ft_maybe_update(ft_handle*, __toku_dbt const*, __toku_dbt const*, tokutxn*, bool, __toku_lsn, bool) (ft-ops.cc:2485)
4107: ==21390==    by 0x4AA2FB8: toku_db_update(__toku_db*, __toku_db_txn*, __toku_dbt const*, __toku_dbt const*, unsigned int) (ydb_write.cc:290)
4107: ==21390==    by 0x4AA69C5: autotxn_db_update(__toku_db*, __toku_db_txn*, __toku_dbt const*, __toku_dbt const*, unsigned int) (ydb_write.cc:1094)
4107: ==21390==    by 0x113C8F: update_op_db(__toku_db*, __toku_db_txn*, arg*, void*, void*) (threaded_stress_test_helpers.h:1488)
4107: ==21390==    by 0x113D55: update_op(__toku_db_txn*, arg*, void*, void*) (threaded_stress_test_helpers.h:1508)
4107: ==21390==    by 0x10F64D: worker(void*) (threaded_stress_test_helpers.h:537)
4107: ==21390==    by 0x48414BA: ??? (in /usr/lib/x86_64-linux-gnu/valgrind/vgpreload_drd-amd64-linux.so)
4107: ==21390== Other segment end (thread 53)
4107: ==21390==    at 0x484482C: pthread_mutex_lock (in /usr/lib/x86_64-linux-gnu/valgrind/vgpreload_drd-amd64-linux.so)
4107: ==21390==    by 0x4A90590: toku_mutex_lock_with_source_location(toku_mutex_t*, char const*, int) (toku_pthread.h:267)
4107: ==21390==    by 0x4B3A2D1: block_table::_mutex_lock() (block_table.cc:72)
4107: ==21390==    by 0x4B3BE7D: block_table::allocate_blocknum(blocknum_s*, ft*) (block_table.cc:682)
4107: ==21390==    by 0x4B4EBBB: rollback_log_create(tokutxn*, blocknum_s, rollback_log_node**) (rollback.cc:167)
4107: ==21390==    by 0x4B4F4D9: toku_get_and_pin_rollback_log_for_new_entry(tokutxn*, rollback_log_node**) (rollback.cc:327)
4107: ==21390==    by 0x4B6FF22: toku_logger_save_rollback_cmdupdate(tokutxn*, FILENUM, BYTESTRING*) (log_code.cc:2168)
4107: ==21390==    by 0x4AF3741: toku_ft_maybe_update(ft_handle*, __toku_dbt const*, __toku_dbt const*, tokutxn*, bool, __toku_lsn, bool) (ft-ops.cc:2485)
4107: ==21390==    by 0x4AA2FB8: toku_db_update(__toku_db*, __toku_db_txn*, __toku_dbt const*, __toku_dbt const*, unsigned int) (ydb_write.cc:290)
4107: ==21390==    by 0x4AA69C5: autotxn_db_update(__toku_db*, __toku_db_txn*, __toku_dbt const*, __toku_dbt const*, unsigned int) (ydb_write.cc:1094)
4107: ==21390==    by 0x113C8F: update_op_db(__toku_db*, __toku_db_txn*, arg*, void*, void*) (threaded_stress_test_helpers.h:1488)
4107: ==21390==    by 0x113D55: update_op(__toku_db_txn*, arg*, void*, void*) (threaded_stress_test_helpers.h:1508)
4107: ==21390==
4107: ==21390== Conflicting store by thread 52 at 0x04c22440 size 4
4107: ==21390==    at 0x4B4EB32: rollback_log_create(tokutxn*, blocknum_s, rollback_log_node**) (rollback.cc:160)
4107: ==21390==    by 0x4B4F4D9: toku_get_and_pin_rollback_log_for_new_entry(tokutxn*, rollback_log_node**) (rollback.cc:327)
4107: ==21390==    by 0x4B6FF22: toku_logger_save_rollback_cmdupdate(tokutxn*, FILENUM, BYTESTRING*) (log_code.cc:2168)
4107: ==21390==    by 0x4AF3741: toku_ft_maybe_update(ft_handle*, __toku_dbt const*, __toku_dbt const*, tokutxn*, bool, __toku_lsn, bool) (ft-ops.cc:2485)
4107: ==21390==    by 0x4AA2FB8: toku_db_update(__toku_db*, __toku_db_txn*, __toku_dbt const*, __toku_dbt const*, unsigned int) (ydb_write.cc:290)
4107: ==21390==    by 0x4AA69C5: autotxn_db_update(__toku_db*, __toku_db_txn*, __toku_dbt const*, __toku_dbt const*, unsigned int) (ydb_write.cc:1094)
4107: ==21390==    by 0x113C8F: update_op_db(__toku_db*, __toku_db_txn*, arg*, void*, void*) (threaded_stress_test_helpers.h:1488)
4107: ==21390==    by 0x113D55: update_op(__toku_db_txn*, arg*, void*, void*) (threaded_stress_test_helpers.h:1508)
4107: ==21390==    by 0x10F64D: worker(void*) (threaded_stress_test_helpers.h:537)
4107: ==21390==    by 0x48414BA: ??? (in /usr/lib/x86_64-linux-gnu/valgrind/vgpreload_drd-amd64-linux.so)
4107: ==21390==    by 0x4C8B668: start_thread (pthread_create.c:479)
4107: ==21390==    by 0x4DC7322: clone (clone.S:95)
4107: ==21390== Allocation context: BSS section of /home/rfp/projects/tokuft-Debug-gcc-7/src/libtokufractaltree.so
4107: ==21390== Other segment start (thread 53)
4107: ==21390==    at 0x484E3FB: pthread_rwlock_unlock (in /usr/lib/x86_64-linux-gnu/valgrind/vgpreload_drd-amd64-linux.so)
4107: ==21390==    by 0x4A9095C: toku_pthread_rwlock_wrunlock(toku_pthread_rwlock_t*) (toku_pthread.h:496)
4107: ==21390==    by 0x4ACCE5D: pair_list::write_list_unlock() (cachetable.cc:3580)
4107: ==21390==    by 0x4AC742C: toku_cachetable_put(cachefile*, blocknum_s, unsigned int, void*, pair_attr_s, CACHETABLE_WRITE_CALLBACK, void (*)(blocknum_s, void*, ctpair*)) (cachetable.cc:1215)
4107: ==21390==    by 0x4B4EC6F: rollback_log_create(tokutxn*, blocknum_s, rollback_log_node**) (rollback.cc:170)
4107: ==21390==    by 0x4B4F4D9: toku_get_and_pin_rollback_log_for_new_entry(tokutxn*, rollback_log_node**) (rollback.cc:327)
4107: ==21390==    by 0x4B6FF22: toku_logger_save_rollback_cmdupdate(tokutxn*, FILENUM, BYTESTRING*) (log_code.cc:2168)
4107: ==21390==    by 0x4AF3741: toku_ft_maybe_update(ft_handle*, __toku_dbt const*, __toku_dbt const*, tokutxn*, bool, __toku_lsn, bool) (ft-ops.cc:2485)
4107: ==21390==    by 0x4AA2FB8: toku_db_update(__toku_db*, __toku_db_txn*, __toku_dbt const*, __toku_dbt const*, unsigned int) (ydb_write.cc:290)
4107: ==21390==    by 0x4AA69C5: autotxn_db_update(__toku_db*, __toku_db_txn*, __toku_dbt const*, __toku_dbt const*, unsigned int) (ydb_write.cc:1094)
4107: ==21390==    by 0x113C8F: update_op_db(__toku_db*, __toku_db_txn*, arg*, void*, void*) (threaded_stress_test_helpers.h:1488)
4107: ==21390==    by 0x113D55: update_op(__toku_db_txn*, arg*, void*, void*) (threaded_stress_test_helpers.h:1508)
4107: ==21390== Other segment end (thread 53)
4107: ==21390==    at 0x484482C: pthread_mutex_lock (in /usr/lib/x86_64-linux-gnu/valgrind/vgpreload_drd-amd64-linux.so)
4107: ==21390==    by 0x4A90590: toku_mutex_lock_with_source_location(toku_mutex_t*, char const*, int) (toku_pthread.h:267)
4107: ==21390==    by 0x4AC49BB: pair_lock(ctpair*) (cachetable.cc:117)
4107: ==21390==    by 0x4AC8D38: cachetable_unpin_internal(cachefile*, ctpair*, cachetable_dirty, pair_attr_s, bool) (cachetable.cc:1899)
4107: ==21390==    by 0x4AC8F5B: toku_cachetable_unpin(cachefile*, ctpair*, cachetable_dirty, pair_attr_s) (cachetable.cc:1932)
4107: ==21390==    by 0x4B4ED32: toku_rollback_log_unpin(tokutxn*, rollback_log_node*) (rollback.cc:181)
4107: ==21390==    by 0x4B70091: toku_logger_save_rollback_cmdupdate(tokutxn*, FILENUM, BYTESTRING*) (log_code.cc:2190)
4107: ==21390==    by 0x4AF3741: toku_ft_maybe_update(ft_handle*, __toku_dbt const*, __toku_dbt const*, tokutxn*, bool, __toku_lsn, bool) (ft-ops.cc:2485)
4107: ==21390==    by 0x4AA2FB8: toku_db_update(__toku_db*, __toku_db_txn*, __toku_dbt const*, __toku_dbt const*, unsigned int) (ydb_write.cc:290)
4107: ==21390==    by 0x4AA69C5: autotxn_db_update(__toku_db*, __toku_db_txn*, __toku_dbt const*, __toku_dbt const*, unsigned int) (ydb_write.cc:1094)
4107: ==21390==    by 0x113C8F: update_op_db(__toku_db*, __toku_db_txn*, arg*, void*, void*) (threaded_stress_test_helpers.h:1488)
4107: ==21390==    by 0x113D55: update_op(__toku_db_txn*, arg*, void*, void*) (threaded_stress_test_helpers.h:1508)
4107: ==21390== Other segment start (thread 53)
4107: ==21390==    at 0x484587C: pthread_mutex_unlock (in /usr/lib/x86_64-linux-gnu/valgrind/vgpreload_drd-amd64-linux.so)
4107: ==21390==    by 0x4A90508: toku_mutex_unlock(toku_mutex_t*) (toku_pthread.h:256)
4107: ==21390==    by 0x4B521CD: rollback_log_node_cache::get_rollback_log_node(tokutxn*, rollback_log_node**) (rollback_log_node_cache.cc:101)
4107: ==21390==    by 0x4B4F478: toku_get_and_pin_rollback_log_for_new_entry(tokutxn*, rollback_log_node**) (rollback.cc:316)
4107: ==21390==    by 0x4B6FF22: toku_logger_save_rollback_cmdupdate(tokutxn*, FILENUM, BYTESTRING*) (log_code.cc:2168)
4107: ==21390==    by 0x4AF3741: toku_ft_maybe_update(ft_handle*, __toku_dbt const*, __toku_dbt const*, tokutxn*, bool, __toku_lsn, bool) (ft-ops.cc:2485)
4107: ==21390==    by 0x4AA2FB8: toku_db_update(__toku_db*, __toku_db_txn*, __toku_dbt const*, __toku_dbt const*, unsigned int) (ydb_write.cc:290)
4107: ==21390==    by 0x4AA69C5: autotxn_db_update(__toku_db*, __toku_db_txn*, __toku_dbt const*, __toku_dbt const*, unsigned int) (ydb_write.cc:1094)
4107: ==21390==    by 0x113C8F: update_op_db(__toku_db*, __toku_db_txn*, arg*, void*, void*) (threaded_stress_test_helpers.h:1488)
4107: ==21390==    by 0x113D55: update_op(__toku_db_txn*, arg*, void*, void*) (threaded_stress_test_helpers.h:1508)
4107: ==21390==    by 0x10F64D: worker(void*) (threaded_stress_test_helpers.h:537)
4107: ==21390==    by 0x48414BA: ??? (in /usr/lib/x86_64-linux-gnu/valgrind/vgpreload_drd-amd64-linux.so)
4107: ==21390== Other segment end (thread 53)
4107: ==21390==    at 0x484482C: pthread_mutex_lock (in /usr/lib/x86_64-linux-gnu/valgrind/vgpreload_drd-amd64-linux.so)
4107: ==21390==    by 0x4A90590: toku_mutex_lock_with_source_location(toku_mutex_t*, char const*, int) (toku_pthread.h:267)
4107: ==21390==    by 0x4B3A2D1: block_table::_mutex_lock() (block_table.cc:72)
4107: ==21390==    by 0x4B3BE7D: block_table::allocate_blocknum(blocknum_s*, ft*) (block_table.cc:682)
4107: ==21390==    by 0x4B4EBBB: rollback_log_create(tokutxn*, blocknum_s, rollback_log_node**) (rollback.cc:167)
4107: ==21390==    by 0x4B4F4D9: toku_get_and_pin_rollback_log_for_new_entry(tokutxn*, rollback_log_node**) (rollback.cc:327)
4107: ==21390==    by 0x4B6FF22: toku_logger_save_rollback_cmdupdate(tokutxn*, FILENUM, BYTESTRING*) (log_code.cc:2168)
4107: ==21390==    by 0x4AF3741: toku_ft_maybe_update(ft_handle*, __toku_dbt const*, __toku_dbt const*, tokutxn*, bool, __toku_lsn, bool) (ft-ops.cc:2485)
4107: ==21390==    by 0x4AA2FB8: toku_db_update(__toku_db*, __toku_db_txn*, __toku_dbt const*, __toku_dbt const*, unsigned int) (ydb_write.cc:290)
4107: ==21390==    by 0x4AA69C5: autotxn_db_update(__toku_db*, __toku_db_txn*, __toku_dbt const*, __toku_dbt const*, unsigned int) (ydb_write.cc:1094)
4107: ==21390==    by 0x113C8F: update_op_db(__toku_db*, __toku_db_txn*, arg*, void*, void*) (threaded_stress_test_helpers.h:1488)
4107: ==21390==    by 0x113D55: update_op(__toku_db_txn*, arg*, void*, void*) (threaded_stress_test_helpers.h:1508)

Also observed by the thread sanitizer when running
ctest -R ydb/checkpoint_fairness --verbose

4793: WARNING: ThreadSanitizer: data race (pid=10298)
4793:   Write of size 4 at 0x7fd7194ac2b8 by thread T35 (mutexes: read M11, write M1048777):
4793:     #0 rollback_log_create(tokutxn*, blocknum_s, rollback_log_node**) /home/rfp/projects/tokuft/ft/txn/rollback.cc:160:21 (libtokufractaltree.so+0x18b990)
4793:     #1 toku_get_and_pin_rollback_log_for_new_entry(tokutxn*, rollback_log_node**) /home/rfp/projects/tokuft/ft/txn/rollback.cc:327:13 (libtokufractaltree.so+0x18b4c9)
4793:     #2 toku_logger_save_rollback_cmdinsert(tokutxn*, FILENUM, BYTESTRING*) /home/rfp/projects/tokuft-tsan/ft/log_code.cc:2007:3 (libtokufractaltree.so+0x1c7912)
4793:     #3 ft_txn_log_insert(ft*, __toku_dbt*, __toku_dbt*, tokutxn*, bool, ft_msg_type) /home/rfp/projects/tokuft/ft/ft-ops.cc:2406:9 (libtokufractaltree.so+0xd2c2c)
4793:     #4 toku_ft_maybe_insert(ft_handle*, __toku_dbt*, __toku_dbt*, tokutxn*, bool, __toku_lsn, bool, ft_msg_type) /home/rfp/projects/tokuft/ft/ft-ops.cc:2423:5 (libtokufractaltree.so+0xd3135)
4793:     #5 db_put(__toku_db*, __toku_db_txn*, __toku_dbt*, __toku_dbt*, int, bool) /home/rfp/projects/tokuft/src/ydb_write.cc:221:13 (libtokufractaltree.so+0x45d33)
4793:     #6 toku_db_put(__toku_db*, __toku_db_txn*, __toku_dbt*, __toku_dbt*, unsigned int, bool) /home/rfp/projects/tokuft/src/ydb_write.cc:248:13 (libtokufractaltree.so+0x456e3)
4793:     #7 autotxn_db_put(__toku_db*, __toku_db_txn*, __toku_dbt*, __toku_dbt*, unsigned int) /home/rfp/projects/tokuft/src/ydb_write.cc:1078:9 (libtokufractaltree.so+0x4ab47)
4793:     #8 start_txns(void*) /home/rfp/projects/tokuft/src/tests/checkpoint_fairness.cc:73:16 (checkpoint_fairness.tdb+0x4b6f5b)
4793:
4793:   Previous write of size 4 at 0x7fd7194ac2b8 by threa T34 (mutexes: read M11, write M1048756):
4793:     #0 rollback_log_create(tokutxn*, blocknum_s, rollback_log_node**) /home/rfp/projects/tokuft/ft/txn/rollback.cc:175:22 (libtokufractaltree.so+0x18bc94)
4793:     #1 toku_get_and_pin_rollback_log_for_new_entry(tokutxn*, rollback_log_node**) /home/rfp/projects/tokuft/ft/txn/rollback.cc:327:13 (libtokufractaltree.so+0x18b4c9)
4793:     #2 toku_logger_save_rollback_cmdinsert(tokutxn*, FILENUM, BYTESTRING*) /home/rfp/projects/tokuft-tsan/ft/log_code.cc:2007:3 (libtokufractaltree.so+0x1c7912)
4793:     #3 ft_txn_log_insert(ft*, __toku_dbt*, __toku_dbt*, tokutxn*, bool, ft_msg_type) /home/rfp/projects/tokuft/ft/ft-ops.cc:2406:9 (libtokufractaltree.so+0xd2c2c)
4793:     #4 toku_ft_maybe_insert(ft_handle*, __toku_dbt*, __toku_dbt*, tokutxn*, bool, __toku_lsn, bool, ft_msg_type) /home/rfp/projects/tokuft/ft/ft-ops.cc:2423:5 (libtokufractaltree.so+0xd3135)
4793:     #5 db_put(__toku_db*, __toku_db_txn*, __toku_dbt*, __toku_dbt*, int, bool) /home/rfp/projects/tokuft/src/ydb_write.cc:221:13 (libtokufractaltree.so+0x45d33)
4793:     #6 toku_db_put(__toku_db*, __toku_db_txn*, __toku_dbt*, __toku_dbt*, unsigned int, bool) /home/rfp/projects/tokuft/src/ydb_write.cc:248:13 (libtokufractaltree.so+0x456e3)
4793:     #7 autotxn_db_put(__toku_db*, __toku_db_txn*, __toku_dbt*, __toku_dbt*, unsigned int) /home/rfp/projects/tokuft/src/ydb_write.cc:1078:9 (libtokufractaltree.so+0x4ab47)
4793:     #8 start_txns(void*) /home/rfp/projects/tokuft/src/tests/checkpoint_fairness.cc:73:16 (checkpoint_fairness.tdb+0x4b6f5b)
4793:
4793:   Location is global 'writing_rollback' of size 4 at 0x7fd7194ac2b8 (libtokufractaltree.so+0x0000002972b8)
---
 ft/ft-internal.h          | 2 +-
 ft/logger/logger.cc       | 2 +-
 ft/node.h                 | 2 +-
 ft/txn/rollback.cc        | 2 +-
 portability/toku_atomic.h | 1 +
 src/ydb.cc                | 5 ++---
 6 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/ft/ft-internal.h b/ft/ft-internal.h
index 130d3c302..2379cdb44 100644
--- a/ft/ft-internal.h
+++ b/ft/ft-internal.h
@@ -80,7 +80,7 @@ extern "C" {
 extern uint force_recovery;
 }
 
-extern int writing_rollback;
+extern std::atomic_int writing_rollback;
 
 // The ft_header is not managed by the cachetable.  Instead, it hangs off the cachefile as userdata.
 struct ft_header {
diff --git a/ft/logger/logger.cc b/ft/logger/logger.cc
index 60280a940..8e9e26921 100644
--- a/ft/logger/logger.cc
+++ b/ft/logger/logger.cc
@@ -49,7 +49,7 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved.
 
 #include "util/status.h"
 
-int writing_rollback = 0;
+std::atomic_int writing_rollback = {0};
 extern "C" {
   uint force_recovery = 0;
 }
diff --git a/ft/node.h b/ft/node.h
index 61093f3ed..ef0f0d84d 100644
--- a/ft/node.h
+++ b/ft/node.h
@@ -155,7 +155,7 @@ class ftnode_pivot_keys {
     size_t _total_size;
 };
 
-extern int writing_rollback;
+extern std::atomic_int writing_rollback;
 
 extern "C" {
 extern uint force_recovery;
diff --git a/ft/txn/rollback.cc b/ft/txn/rollback.cc
index 105f980dc..6c7445eb0 100644
--- a/ft/txn/rollback.cc
+++ b/ft/txn/rollback.cc
@@ -43,7 +43,7 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved.
 #include "ft/logger/log-internal.h"
 #include "ft/txn/rollback-ct-callbacks.h"
 
-extern int writing_rollback;
+extern std::atomic_int writing_rollback;
 
 static void rollback_unpin_remove_callback(CACHEKEY* cachekey, bool for_checkpoint, void* extra) {
     FT CAST_FROM_VOIDP(ft, extra);
diff --git a/portability/toku_atomic.h b/portability/toku_atomic.h
index 88644f569..5dcdb2fd3 100644
--- a/portability/toku_atomic.h
+++ b/portability/toku_atomic.h
@@ -58,6 +58,7 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved.
 #include <stdbool.h>
 #include <stdint.h>
 #include <stddef.h>
+#include <atomic>
 
 __attribute__((const, always_inline))
 static inline intptr_t which_cache_line(intptr_t addr) {
diff --git a/src/ydb.cc b/src/ydb.cc
index d88d7a26e..27f935f7d 100644
--- a/src/ydb.cc
+++ b/src/ydb.cc
@@ -39,9 +39,6 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved.
 extern const char *toku_patent_string;
 const char *toku_copyright_string = "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved.";
 
-
-extern int writing_rollback;
-
 #include <db.h>
 #include <errno.h>
 #include <string.h>
@@ -90,6 +87,8 @@ extern int writing_rollback;
  int toku_close_trace_file (void) { return 0; } 
 #endif
 
+extern std::atomic_int writing_rollback;
+
 extern uint force_recovery;
 
 // Set when env is panicked, never cleared.