feat(log-rorate): log-rotate plugin support max_size

apisix/plugins/log-rotate.lua

@@ -21,6 +21,7 @@ local plugin = require("apisix.plugin")
 local process = require("ngx.process")
 local signal = require("resty.signal")
 local shell = require("resty.shell")
+local ipairs = ipairs
 local ngx = ngx
 local ngx_time = ngx.time
 local ngx_update_time = ngx.update_time
@@ -43,6 +44,7 @@ local local_conf
 local plugin_name = "log-rotate"
 local INTERVAL = 60 * 60    -- rotate interval (unit: second)
 local MAX_KEPT = 24 * 7     -- max number of log files will be kept
+local MAX_SIZE = -1         -- max size of file will be rotated
 local COMPRESSION_FILE_SUFFIX = ".tar.gz" -- compression file suffix
 local rotate_time
 local default_logs
@@ -123,34 +125,22 @@ local function tab_sort_comp(a, b)
 end
 
 
-local function scan_log_folder()
-    local t = {
-        access = {},
-        error = {},
-    }
+local function scan_log_folder(log_file_name)
+    local t = {}
 
-    local log_dir, access_name = get_log_path_info("access.log")
-    local _, error_name = get_log_path_info("error.log")
-
-    if enable_compression then
-        access_name = access_name .. COMPRESSION_FILE_SUFFIX
-        error_name = error_name .. COMPRESSION_FILE_SUFFIX
-    end
+    local log_dir, _ = get_log_path_info(log_file_name)
 
     for file in lfs.dir(log_dir) do
         local n = get_last_index(file, "__")
         if n ~= nil then
             local log_type = file:sub(n + 2)
-            if log_type == access_name then
-                tab_insert(t.access, file)
-            elseif log_type == error_name then
-                tab_insert(t.error, file)
+            if log_type == log_file_name then
+                tab_insert(t, file)
             end
         end
     end
 
-    tab_sort(t.access, tab_sort_comp)
-    tab_sort(t.error, tab_sort_comp)
+    tab_sort(t, tab_sort_comp)
     return t, log_dir
 end
 
@@ -219,18 +209,67 @@ local function init_default_logs(logs_info, log_type)
 end
 
 
+local function file_size(file)
+    local attr = lfs.attributes(file)
+    if attr then
+        return attr.size
+    end
+    return 0
+end
+
+
+local function rotate_file(files, now_time, max_kept)
+    for _, file in ipairs(files) do
+        local now_date = os_date("%Y-%m-%d_%H-%M-%S", now_time)
+        local new_file = rename_file(default_logs[file], now_date)
+        if not new_file then
+            return
+        end
+
+        local pid = process.get_master_pid()
+        local sig_user1 = signal.signum("USR1")
+
+        core.log.warn("send USR1 signal to master process [", pid, "] for reopening log file")
+
+        if (pid and sig_user1) then
+            local ok, err = signal.kill(pid, sig_user1)
+            if not ok then
+                core.log.error("failed to send USR1 signal for reopening log file: ", err)
+            end
+        end
+
+        if enable_compression then
+            compression_file(new_file)
+        end
+
+        -- clean the oldest file
+        local log_list, log_dir = scan_log_folder(file)
+        for i = max_kept + 1, #log_list do
+            local path = log_dir .. log_list[i]
+            local ok, err = os_remove(path)
+            if err then
+               core.log.error("remove old log file: ", path, " log: ", err, "  res:", ok)
+            end
+        end
+    end
+end
+
+
 local function rotate()
     local interval = INTERVAL
     local max_kept = MAX_KEPT
+    local max_size = MAX_SIZE
     local attr = plugin.plugin_attr(plugin_name)
     if attr then
         interval = attr.interval or interval
         max_kept = attr.max_kept or max_kept
+        max_size = attr.max_size or max_size
         enable_compression = attr.enable_compression or enable_compression
     end
 
     core.log.info("rotate interval:", interval)
     core.log.info("rotate max keep:", max_kept)
+    core.log.info("rotate max size:", max_size)
 
     if not default_logs then
         -- first init default log filepath and filename
@@ -248,53 +287,22 @@ local function rotate()
         return
     end
 
-    if now_time < rotate_time then
-        -- did not reach the rotate time
-        core.log.info("rotate time: ", rotate_time, " now time: ", now_time)
-        return
-    end
+    if now_time >= rotate_time then
+        local files = {DEFAULT_ACCESS_LOG_FILENAME, DEFAULT_ERROR_LOG_FILENAME}
+        rotate_file(files, now_time, max_kept)
 
-    local now_date = os_date("%Y-%m-%d_%H-%M-%S", now_time)
-    local access_new_file = rename_file(default_logs[DEFAULT_ACCESS_LOG_FILENAME], now_date)
-    local error_new_file = rename_file(default_logs[DEFAULT_ERROR_LOG_FILENAME], now_date)
-    if not access_new_file and not error_new_file then
         -- reset rotate time
         rotate_time = rotate_time + interval
-        return
-    end
-
-    core.log.warn("send USR1 signal to master process [",
-                  process.get_master_pid(), "] for reopening log file")
-    local ok, err = signal.kill(process.get_master_pid(), signal.signum("USR1"))
-    if not ok then
-        core.log.error("failed to send USR1 signal for reopening log file: ", err)
-    end
-
-    if enable_compression then
-        compression_file(access_new_file)
-        compression_file(error_new_file)
-    end
-
-    -- clean the oldest file
-    local log_list, log_dir = scan_log_folder()
-    for i = max_kept + 1, #log_list.error do
-        local path = log_dir .. log_list.error[i]
-        ok, err = os_remove(path)
-        if err then
-           core.log.error("remove old error file: ", path, " err: ", err, "  res:", ok)
+    elseif max_size > 0 then
+        local access_log_file_size = file_size(default_logs[DEFAULT_ACCESS_LOG_FILENAME].file)
+        local error_log_file_size = file_size(default_logs[DEFAULT_ERROR_LOG_FILENAME].file)
+        if access_log_file_size >= max_size then
+            rotate_file({DEFAULT_ACCESS_LOG_FILENAME}, now_time, max_kept)
         end
-    end
-
-    for i = max_kept + 1, #log_list.access do
-        local path = log_dir .. log_list.access[i]
-        ok, err = os_remove(path)
-        if err then
-           core.log.error("remove old error file: ", path, " err: ", err, "  res:", ok)
+        if error_log_file_size >= max_size then
+            rotate_file({DEFAULT_ERROR_LOG_FILENAME}, now_time, max_kept)
         end
     end
-
-    -- reset rotate time
-    rotate_time = rotate_time + interval
 end
 
 

conf/config-default.yaml

@@ -493,6 +493,7 @@ plugin_attr:
   log-rotate:
     interval: 3600    # rotate interval (unit: second)
     max_kept: 168     # max number of log files will be kept
+    max_size: -1      # max size bytes of log files to be rotated, size check would be skipped with a value less than 0
     enable_compression: false    # enable log file compression(gzip) or not, default false
   skywalking:
     service_name: APISIX

docs/en/latest/plugins/log-rotate.md

@@ -39,6 +39,7 @@ You can configure how often the logs are rotated and how many logs to keep. When
 |--------------------|---------|----------|---------|------------------------------------------------------------------------------------------------|
 | interval           | integer | True     | 60 * 60 | Time in seconds specifying how often to rotate the logs.                                       |
 | max_kept           | integer | True     | 24 * 7  | Maximum number of historical logs to keep. If this number is exceeded, older logs are deleted. |
+| max_size           | integer | False    | -1      | Max size(Bytes) of log files to be rotated, size check would be skipped with a value less than 0 or time is up specified by interval. |
 | enable_compression | boolean | False    | false   | When set to `true`, compresses the log file (gzip). Requires `tar` to be installed.            |
 
 ## Enabling the Plugin
@@ -51,9 +52,10 @@ plugins:
 
 plugin_attr:
     log-rotate:
-        interval: 3600
-        max_kept: 168
-        enable_compression: false
+        interval: 3600    # rotate interval (unit: second)
+        max_kept: 168     # max number of log files will be kept
+        max_size: -1      # max size of log files will be kept
+        enable_compression: false    # enable log file compression(gzip) or not, default false
 ```
 
 ## Example usage

docs/zh/latest/plugins/log-rotate.md

@@ -39,6 +39,7 @@ description: 云原生 API 网关 Apache APISIX log-rotate 插件用于定期切
 | ------------------ | ------- | ------ | ------- | ------------- | ---------------------------------------------------------------------------- |
 | interval           | integer | 是     | 60 * 60 |               | 每间隔多长时间切分一次日志，以秒为单位。                                        |
 | max_kept           | integer | 是     | 24 * 7  |               | 最多保留多少份历史日志，超过指定数量后，自动删除老文件。                         |
+| max_size           | integer | 否     | -1      |               | 日志文件超过指定大小时进行切分，单位为 Byte 。如果 `max_size` 小于 0 或者根据 `interval` 计算的时间到达时，将不会根据 `max_size` 切分日志。 |
 | enable_compression | boolean | 否     | false   | [false, true] | 当设置为 `true` 时，启用日志文件压缩。该功能需要在系统中安装 `tar` 。     |
 
 开启该插件后，就会按照参数自动切分日志文件了。比如以下示例是根据 `interval: 10` 和 `max_kept: 10` 得到的样本。
@@ -92,6 +93,7 @@ plugin_attr:
     log-rotate:
         interval: 3600    # rotate interval (unit: second)
         max_kept: 168     # max number of log files will be kept
+        max_size: -1      # max size of log files will be kept
         enable_compression: false    # enable log file compression(gzip) or not, default false
 ```
 

t/plugin/log-rotate3.t

@@ -0,0 +1,141 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+use t::APISIX 'no_plan';
+
+repeat_each(1);
+no_long_string();
+no_shuffle();
+no_root_location();
+
+add_block_preprocessor(sub {
+    my ($block) = @_;
+
+    if (!defined $block->yaml_config) {
+        my $yaml_config = <<_EOC_;
+apisix:
+  node_listen: 1984
+  admin_key: ~
+plugins:
+  - log-rotate
+plugin_attr:
+  log-rotate:
+    interval: 86400
+    max_size: 9
+    max_kept: 3
+    enable_compression: false
+_EOC_
+
+        $block->set_value("yaml_config", $yaml_config);
+    }
+
+    if ((!defined $block->error_log) && (!defined $block->no_error_log)) {
+        $block->set_value("no_error_log", "[error]");
+    }
+
+    if (!defined $block->request) {
+        $block->set_value("request", "GET /t");
+    }
+
+});
+
+run_tests;
+
+__DATA__
+
+=== TEST 1: log rotate by max_size
+--- config
+    location /t {
+        content_by_lua_block {
+            ngx.log(ngx.ERR, "start xxxxxx")
+            ngx.sleep(2)
+            local has_split_access_file = false
+            local has_split_error_file = false
+            local lfs = require("lfs")
+            for file_name in lfs.dir(ngx.config.prefix() .. "/logs/") do
+                if string.match(file_name, "__access.log$") then
+                    has_split_access_file = true
+                end
+
+                if string.match(file_name, "__error.log$") then
+                    has_split_error_file = true
+                end
+            end
+
+            if not has_split_access_file and has_split_error_file then
+               ngx.status = 200
+            else
+               ngx.status = 500
+            end
+        }
+    }
+
+
+
+=== TEST 2: in current log
+--- config
+    location /t {
+        content_by_lua_block {
+            ngx.sleep(0.1)
+            ngx.log(ngx.WARN, "start xxxxxx")
+            ngx.say("done")
+        }
+    }
+--- response_body
+done
+--- error_log
+start xxxxxx
+
+
+
+=== TEST 3: check file changes
+--- config
+    location /t {
+        content_by_lua_block {
+            ngx.sleep(1)
+
+            local default_logs = {}
+            for file_name in lfs.dir(ngx.config.prefix() .. "/logs/") do
+                if string.match(file_name, "__error.log$") or string.match(file_name, "__access.log$") then
+                    local filepath = ngx.config.prefix() .. "/logs/" .. file_name
+                    local attr = lfs.attributes(filepath)
+                    if attr then
+                        default_logs[filepath] = { change = attr.change, size = attr.size }
+                    end
+                end
+            end
+
+            ngx.sleep(1)
+
+            local passed = false
+            for filepath, origin_attr in pairs(default_logs) do
+                local check_attr = lfs.attributes(filepath)
+                if check_attr.change == origin_attr.change and check_attr.size == origin_attr.size then
+                    passed = true
+                else
+                    passed = false
+                    break
+                end
+            end
+
+            if passed then
+                ngx.say("passed")
+            end
+        }
+    }
+--- response_body
+passed