fix: worker not exited when executing quit or reload command (#9909)

apisix/core/config_etcd.lua

@@ -55,6 +55,9 @@ local constants    = require("apisix.constants")
 local health_check = require("resty.etcd.health_check")
 local semaphore    = require("ngx.semaphore")
 local tablex       = require("pl.tablex")
+local ngx_thread_spawn = ngx.thread.spawn
+local ngx_thread_kill = ngx.thread.kill
+local ngx_thread_wait = ngx.thread.wait
 
 
 local is_http = ngx.config.subsystem == "http"
@@ -124,7 +127,7 @@ local function produce_res(res, err)
 end
 
 
-local function run_watch(premature)
+local function do_run_watch(premature)
     if premature then
         return
     end
@@ -257,6 +260,30 @@ local function run_watch(premature)
 end
 
 
+local function run_watch(premature)
+    local run_watch_th = ngx_thread_spawn(do_run_watch, premature)
+
+    ::restart::
+    local check_worker_th = ngx_thread_spawn(function ()
+        while not exiting() do
+            ngx_sleep(0.1)
+        end
+    end)
+
+    local ok, err = ngx_thread_wait(check_worker_th)
+
+    if not ok then
+        log.error("check_worker thread terminates failed, retart checker, error: " .. err)
+        ngx_thread_kill(check_worker_th)
+        goto restart
+    end
+
+    ngx_thread_kill(run_watch_th)
+    -- notify child watchers
+    produce_res(nil, "worker exited")
+end
+
+
 local function init_watch_ctx(key)
     if not watch_ctx then
         watch_ctx = {
@@ -851,6 +878,9 @@ local function _automatic_fetch(premature, self)
                                        .. backoff_duration .. "s")
                         end
                     end
+                elseif err == "worker exited" then
+                    log.info("worker exited.")
+                    return
                 elseif err ~= "timeout" and err ~= "Key not found"
                     and self.last_err ~= err then
                     log.error("failed to fetch data from etcd: ", err, ", ",

t/cli/test_cmd.sh

@@ -198,3 +198,41 @@ fi
 
 rm conf/customized_config.yaml
 echo "passed: test customized config successful"
+
+# test quit command
+bin/apisix start
+
+if ! ps -ef | grep "apisix" | grep "master process" | grep -v "grep"; then
+    echo "apisix not started"
+    exit 1
+fi
+
+bin/apisix quit
+
+sleep 0.5
+
+if ps -ef | grep "worker process is shutting down" | grep -v "grep"; then
+    echo "all workers should exited"
+    exit 1
+fi
+
+echo "passed: test quit command successful"
+
+# test reload command
+bin/apisix start
+
+if ! ps -ef | grep "apisix" | grep "master process" | grep -v "grep"; then
+    echo "apisix not started"
+    exit 1
+fi
+
+bin/apisix reload
+
+sleep 0.5
+
+if ps -ef | grep "worker process is shutting down" | grep -v "grep"; then
+    echo "old workers should exited"
+    exit 1
+fi
+
+echo "passed: test reload command successful"

t/node/healthcheck-stop-checker.t

@@ -165,15 +165,6 @@ create new checker: table: 0x
         content_by_lua_block {
             local t = require("lib.test_admin").test
 
-            -- release the clean handler of previous test
-            local code, _, body = t('/apisix/admin/routes/1', "DELETE")
-
-            if code > 300 then
-                ngx.status = code
-                ngx.say(body)
-                return
-            end
-
             local code, _, body = t('/apisix/admin/upstreams/stopchecker',
                 "PUT",
                 [[{"type":"roundrobin","nodes":{"127.0.0.1:1980":1,"127.0.0.1:1981":1},"checks":{"active":{"http_path":"/status","healthy":{"interval":1,"successes":1},"unhealthy":{"interval":1,"http_failures":2}}}}]]
@@ -256,7 +247,6 @@ ok
 --- grep_error_log eval
 qr/create new checker: table: 0x|try to release checker: table: 0x/
 --- grep_error_log_out
-try to release checker: table: 0x
 create new checker: table: 0x
 try to release checker: table: 0x
 create new checker: table: 0x