add unit test of whitelist & fix bug (#228)

include/dsn/dist/failure_detector/failure_detector.h

@@ -141,6 +141,8 @@ class failure_detector : public failure_detector_service,
 
     void set_allow_list(const std::vector<std::string> &replica_addrs);
 
+    std::string get_allow_list(const std::vector<std::string> &args) const;
+
     int worker_count() const { return static_cast<int>(_workers.size()); }
 
     int master_count() const { return static_cast<int>(_masters.size()); }
@@ -158,8 +160,6 @@ class failure_detector : public failure_detector_service,
 private:
     void check_all_records();
 
-    std::string get_allow_list(const std::vector<std::string> &args) const;
-
 private:
     class master_record
     {

include/dsn/tool-api/network.h

@@ -136,7 +136,6 @@ class network
     network_header_format unknown_msg_hdr_format() const { return _unknown_msg_header_format; }
     int message_buffer_block_size() const { return _message_buffer_block_size; }
 
-protected:
     DSN_API static uint32_t get_local_ipv4();
 
 protected:

src/core/tests/gtest.filter

@@ -1,3 +1,4 @@
 config-test.ini -core.corrupt_message:core.aio*:core.operation_failed:tools_hpc.*
-config-test-sim.ini -core.corrupt_message:core.aio*:core.operation_failed:tools_hpc.*
+config-test-sim.ini -core.corrupt_message:core.aio*:core.operation_failed:tools_hpc.*:tools_simulator.*
+config-test-sim.ini tools_simulator.*
 config-test-posix-aio.ini core.aio*:core.operation_failed

src/dist/failure_detector/failure_detector.cpp

@@ -67,22 +67,6 @@ void failure_detector::register_ctrl_commands()
 
 void failure_detector::unregister_ctrl_commands() { UNREGISTER_VALID_HANDLER(_get_allow_list); }
 
-std::string failure_detector::get_allow_list(const std::vector<std::string> &args) const
-{
-    if (!_is_started)
-        return "error: fd is not started";
-
-    std::stringstream oss;
-    dsn::zauto_lock l(_lock);
-    oss << "get ok: allow list " << (_use_allow_list ? "enabled. list: " : "disabled.");
-    for (auto iter = _allow_list.begin(); iter != _allow_list.end(); ++iter) {
-        if (iter != _allow_list.begin())
-            oss << ",";
-        oss << iter->to_string();
-    }
-    return oss.str();
-}
-
 error_code failure_detector::start(uint32_t check_interval_seconds,
                                    uint32_t beacon_interval_seconds,
                                    uint32_t lease_seconds,
@@ -321,7 +305,7 @@ bool failure_detector::remove_from_allow_list(::dsn::rpc_address node)
 
 void failure_detector::set_allow_list(const std::vector<std::string> &replica_addrs)
 {
-    dassert(_is_started, "FD is already started, the allow list should really not be modified");
+    dassert(!_is_started, "FD is already started, the allow list should really not be modified");
 
     std::vector<rpc_address> nodes;
     for (auto &addr : replica_addrs) {
@@ -338,6 +322,22 @@ void failure_detector::set_allow_list(const std::vector<std::string> &replica_ad
         add_allow_list(node);
 }
 
+std::string failure_detector::get_allow_list(const std::vector<std::string> &args) const
+{
+    if (!_is_started)
+        return "error: FD is not started";
+
+    std::stringstream oss;
+    dsn::zauto_lock l(_lock);
+    oss << "get ok: allow list " << (_use_allow_list ? "enabled. list: " : "disabled.");
+    for (auto iter = _allow_list.begin(); iter != _allow_list.end(); ++iter) {
+        if (iter != _allow_list.begin())
+            oss << ",";
+        oss << iter->to_string();
+    }
+    return oss.str();
+}
+
 void failure_detector::on_ping_internal(const beacon_msg &beacon, /*out*/ beacon_ack &ack)
 {
     ack.time = beacon.time;

src/tests/dsn/CMakeLists.txt

@@ -46,6 +46,9 @@ set(MY_BINPLACES
     "${CMAKE_CURRENT_SOURCE_DIR}/clear.sh"
     "${CMAKE_CURRENT_SOURCE_DIR}/clear.cmd"
     "${CMAKE_CURRENT_SOURCE_DIR}/config-test.ini"
+    "${CMAKE_CURRENT_SOURCE_DIR}/config-whitelist-test.ini"
+    "${CMAKE_CURRENT_SOURCE_DIR}/config-whitelist-test-failed.ini"
+    "${CMAKE_CURRENT_SOURCE_DIR}/gtest.filter"
 )
 
 dsn_add_test()

src/tests/dsn/config-whitelist-test-failed.ini

@@ -0,0 +1,123 @@
+[apps..default]
+run = true
+count = 1
+;network.client.RPC_CHANNEL_TCP = dsn::tools::sim_network_provider, 65536
+;network.client.RPC_CHANNEL_UDP = dsn::tools::sim_network_provider, 65536
+;network.server.0.RPC_CHANNEL_TCP = dsn::tools::sim_network_provider, 65536
+
+[apps.client]
+type = test
+arguments = localhost 20101
+run = true
+ports =
+count = 1
+delay_seconds = 1
+pools = THREAD_POOL_DEFAULT, THREAD_POOL_DLOCK, THREAD_POOL_REPLICATION, THREAD_POOL_REPLICATION_LONG, THREAD_POOL_FDS_SERVICE
+
+[apps.server]
+type = test
+arguments =
+ports = 20101
+run = false
+count = 0
+pools = THREAD_POOL_DEFAULT, THREAD_POOL_REPLICATION
+
+[apps.add_server]
+type = adder
+arguments =
+ports = 20201
+run = true
+count = 3
+pools = THREAD_POOL_DEFAULT, THREAD_POOL_DLOCK
+
+[apps.test_worker]
+type = worker
+arguments =
+ports = 40001
+run = true
+count = 1
+pools = THREAD_POOL_DEFAULT, THREAD_POOL_FD
+
+[apps.test_master]
+type = master
+#whitelist port,port,...
+arguments = whitelist 22222
+ports = 30001
+run = true
+count = 3
+pools = THREAD_POOL_DEFAULT, THREAD_POOL_FD
+
+[core]
+;tool = simulator
+tool = nativerun
+
+;toollets = tracer, profiler
+;fault_injector
+pause_on_start = false
+
+logging_start_level = LOG_LEVEL_INFORMATION
+logging_factory_name = dsn::tools::simple_logger
+
+;io_mode = IOE_PER_NODE
+io_mode = IOE_PER_QUEUE
+io_worker_count = 1
+
+[tools.simple_logger]
+fast_flush = true
+short_header = false
+stderr_start_level = LOG_LEVEL_WARNING
+
+[tools.simulator]
+random_seed = 0
+
+[tools.screen_logger]
+short_header = false
+
+[network]
+; how many network threads for network library (used by asio)
+io_service_worker_count = 2
+
+[task..default]
+is_trace = true
+is_profile = true
+allow_inline = false
+rpc_call_channel = RPC_CHANNEL_TCP
+rpc_message_header_format = dsn
+rpc_timeout_milliseconds = 5000
+
+[task.LPC_AIO_IMMEDIATE_CALLBACK]
+is_trace = false
+is_profile = false
+allow_inline = false
+
+[task.LPC_RPC_TIMEOUT]
+is_trace = false
+is_profile = false
+
+; specification for each thread pool
+[threadpool..default]
+worker_count = 2
+
+[threadpool.THREAD_POOL_DEFAULT]
+partitioned = false
+worker_priority = THREAD_xPRIORITY_NORMAL
+
+[threadpool.THREAD_POOL_TEST_SERVER]
+partitioned = false
+
+[threadpool.THREAD_POOL_FDS_SERVICE]
+worker_count = 8
+
+[threadpool.THREAD_POOL_DLOCK]
+partitioned = true
+
+[zookeeper]
+hosts_list = localhost:12181
+timeout_ms = 30000
+logfile = zoolog.log
+
+[fds_concurrent_test]
+total_files = 64
+min_size = 100
+max_size = 150
+

src/tests/dsn/config-whitelist-test.ini

@@ -0,0 +1,123 @@
+[apps..default]
+run = true
+count = 1
+;network.client.RPC_CHANNEL_TCP = dsn::tools::sim_network_provider, 65536
+;network.client.RPC_CHANNEL_UDP = dsn::tools::sim_network_provider, 65536
+;network.server.0.RPC_CHANNEL_TCP = dsn::tools::sim_network_provider, 65536
+
+[apps.client]
+type = test
+arguments = localhost 20101
+run = true
+ports =
+count = 1
+delay_seconds = 1
+pools = THREAD_POOL_DEFAULT, THREAD_POOL_DLOCK, THREAD_POOL_REPLICATION, THREAD_POOL_REPLICATION_LONG, THREAD_POOL_FDS_SERVICE
+
+[apps.server]
+type = test
+arguments =
+ports = 20101
+run = false
+count = 0
+pools = THREAD_POOL_DEFAULT, THREAD_POOL_REPLICATION
+
+[apps.add_server]
+type = adder
+arguments =
+ports = 20201
+run = true
+count = 3
+pools = THREAD_POOL_DEFAULT, THREAD_POOL_DLOCK
+
+[apps.test_worker]
+type = worker
+arguments =
+ports = 40001
+run = true
+count = 1
+pools = THREAD_POOL_DEFAULT, THREAD_POOL_FD
+
+[apps.test_master]
+type = master
+#whitelist port,port,...
+arguments = whitelist 40001
+ports = 30001
+run = true
+count = 3
+pools = THREAD_POOL_DEFAULT, THREAD_POOL_FD
+
+[core]
+;tool = simulator
+tool = nativerun
+
+;toollets = tracer, profiler
+;fault_injector
+pause_on_start = false
+
+logging_start_level = LOG_LEVEL_INFORMATION
+logging_factory_name = dsn::tools::simple_logger
+
+;io_mode = IOE_PER_NODE
+io_mode = IOE_PER_QUEUE
+io_worker_count = 1
+
+[tools.simple_logger]
+fast_flush = true
+short_header = false
+stderr_start_level = LOG_LEVEL_WARNING
+
+[tools.simulator]
+random_seed = 0
+
+[tools.screen_logger]
+short_header = false
+
+[network]
+; how many network threads for network library (used by asio)
+io_service_worker_count = 2
+
+[task..default]
+is_trace = true
+is_profile = true
+allow_inline = false
+rpc_call_channel = RPC_CHANNEL_TCP
+rpc_message_header_format = dsn
+rpc_timeout_milliseconds = 5000
+
+[task.LPC_AIO_IMMEDIATE_CALLBACK]
+is_trace = false
+is_profile = false
+allow_inline = false
+
+[task.LPC_RPC_TIMEOUT]
+is_trace = false
+is_profile = false
+
+; specification for each thread pool
+[threadpool..default]
+worker_count = 2
+
+[threadpool.THREAD_POOL_DEFAULT]
+partitioned = false
+worker_priority = THREAD_xPRIORITY_NORMAL
+
+[threadpool.THREAD_POOL_TEST_SERVER]
+partitioned = false
+
+[threadpool.THREAD_POOL_FDS_SERVICE]
+worker_count = 8
+
+[threadpool.THREAD_POOL_DLOCK]
+partitioned = true
+
+[zookeeper]
+hosts_list = localhost:12181
+timeout_ms = 30000
+logfile = zoolog.log
+
+[fds_concurrent_test]
+total_files = 64
+min_size = 100
+max_size = 150
+

src/tests/dsn/failure_detector.cpp

@@ -169,7 +169,20 @@ class test_master : public service_app
 
         _master_fd = new master_fd_test();
         _master_fd->set_options(&_opts);
-        _master_fd->start(1, 1, 4, 5);
+        bool use_allow_list = false;
+        if (args.size() >= 3 && args[1] == "whitelist") {
+            std::vector<std::string> ports;
+            utils::split_args(args[2].c_str(), ports, ',');
+            for (auto &port : ports) {
+                rpc_address addr;
+                addr.assign_ipv4(network::get_local_ipv4(), std::stoi(port));
+                _master_fd->add_allow_list(addr);
+            }
+            use_allow_list = true;
+        }
+
+        _master_fd->start(1, 1, 4, 5, use_allow_list);
+        dinfo("%s", _master_fd->get_allow_list(std::vector<std::string>{}).c_str());
         ++started_apps;
 
         return ERR_OK;
@@ -660,3 +673,24 @@ TEST(fd, update_stability)
     ASSERT_EQ(msg.start_time, ws.last_start_time_ms);
     ASSERT_EQ(0, ws.unstable_restart_count);
 }
+
+TEST(fd, not_in_whitelist)
+{
+    test_worker *worker;
+    std::vector<test_master *> masters;
+    ASSERT_TRUE(get_worker_and_master(worker, masters));
+
+    clear(worker, masters);
+    // set master with smallest index as the leader
+    master_group_set_leader(masters, 0);
+    // set the worker contact leader
+    worker_set_leader(worker, 0);
+
+    std::atomic_int wait_count;
+    wait_count.store(1);
+    auto cb = [&wait_count](rpc_address) mutable { --wait_count; };
+    worker->fd()->when_connected(cb);
+    worker->fd()->toggle_send_ping(true);
+
+    ASSERT_TRUE(spin_wait_condition([&wait_count] { return wait_count == 1; }, 20));
+}

src/tests/dsn/gtest.filter

@@ -0,0 +1,4 @@
+config-test.ini distributed_lock_service_zookeeper.simple_lock_unlock
+config-test.ini -distributed_lock_service_zookeeper.simple_lock_unlock:fd.not_in_whitelist
+config-whitelist-test.ini -distributed_lock_service_zookeeper.simple_lock_unlock:fd.not_in_whitelist
+config-whitelist-test-failed.ini fd.not_in_whitelist