diff --git a/apisix/plugins/log-rotate.lua b/apisix/plugins/log-rotate.lua index 79459371702e..571fc9b52525 100644 --- a/apisix/plugins/log-rotate.lua +++ b/apisix/plugins/log-rotate.lua @@ -21,6 +21,7 @@ local plugin = require("apisix.plugin") local process = require("ngx.process") local signal = require("resty.signal") local shell = require("resty.shell") +local ipairs = ipairs local ngx = ngx local ngx_time = ngx.time local ngx_update_time = ngx.update_time @@ -43,6 +44,7 @@ local local_conf local plugin_name = "log-rotate" local INTERVAL = 60 * 60 -- rotate interval (unit: second) local MAX_KEPT = 24 * 7 -- max number of log files will be kept +local MAX_SIZE = -1 -- max size of file will be rotated local COMPRESSION_FILE_SUFFIX = ".tar.gz" -- compression file suffix local rotate_time local default_logs @@ -123,34 +125,22 @@ local function tab_sort_comp(a, b) end -local function scan_log_folder() - local t = { - access = {}, - error = {}, - } +local function scan_log_folder(log_file_name) + local t = {} - local log_dir, access_name = get_log_path_info("access.log") - local _, error_name = get_log_path_info("error.log") - - if enable_compression then - access_name = access_name .. COMPRESSION_FILE_SUFFIX - error_name = error_name .. COMPRESSION_FILE_SUFFIX - end + local log_dir, _ = get_log_path_info(log_file_name) for file in lfs.dir(log_dir) do local n = get_last_index(file, "__") if n ~= nil then local log_type = file:sub(n + 2) - if log_type == access_name then - tab_insert(t.access, file) - elseif log_type == error_name then - tab_insert(t.error, file) + if log_type == log_file_name then + tab_insert(t, file) end end end - tab_sort(t.access, tab_sort_comp) - tab_sort(t.error, tab_sort_comp) + tab_sort(t, tab_sort_comp) return t, log_dir end @@ -219,18 +209,62 @@ local function init_default_logs(logs_info, log_type) end +local function file_size(file) + local attr = lfs.attributes(file) + if attr then + return attr.size + end + return 0 +end + + +local function rotate_file(files, now_time, max_kept) + for _, file in ipairs(files) do + local now_date = os_date("%Y-%m-%d_%H-%M-%S", now_time) + local new_file = rename_file(default_logs[file], now_date) + if not new_file then + return + end + + local pid = process.get_master_pid() + core.log.warn("send USR1 signal to master process [", pid, "] for reopening log file") + local ok, err = signal.kill(pid, signal.signum("USR1")) + if not ok then + core.log.error("failed to send USR1 signal for reopening log file: ", err) + end + + if enable_compression then + compression_file(new_file) + end + + -- clean the oldest file + local log_list, log_dir = scan_log_folder(file) + for i = max_kept + 1, #log_list do + local path = log_dir .. log_list[i] + local ok, err = os_remove(path) + if err then + core.log.error("remove old log file: ", path, " err: ", err, " res:", ok) + end + end + end +end + + local function rotate() local interval = INTERVAL local max_kept = MAX_KEPT + local max_size = MAX_SIZE local attr = plugin.plugin_attr(plugin_name) if attr then interval = attr.interval or interval max_kept = attr.max_kept or max_kept + max_size = attr.max_size or max_size enable_compression = attr.enable_compression or enable_compression end core.log.info("rotate interval:", interval) core.log.info("rotate max keep:", max_kept) + core.log.info("rotate max size:", max_size) if not default_logs then -- first init default log filepath and filename @@ -248,53 +282,22 @@ local function rotate() return end - if now_time < rotate_time then - -- did not reach the rotate time - core.log.info("rotate time: ", rotate_time, " now time: ", now_time) - return - end + if now_time >= rotate_time then + local files = {DEFAULT_ACCESS_LOG_FILENAME, DEFAULT_ERROR_LOG_FILENAME} + rotate_file(files, now_time, max_kept) - local now_date = os_date("%Y-%m-%d_%H-%M-%S", now_time) - local access_new_file = rename_file(default_logs[DEFAULT_ACCESS_LOG_FILENAME], now_date) - local error_new_file = rename_file(default_logs[DEFAULT_ERROR_LOG_FILENAME], now_date) - if not access_new_file and not error_new_file then -- reset rotate time rotate_time = rotate_time + interval - return - end - - core.log.warn("send USR1 signal to master process [", - process.get_master_pid(), "] for reopening log file") - local ok, err = signal.kill(process.get_master_pid(), signal.signum("USR1")) - if not ok then - core.log.error("failed to send USR1 signal for reopening log file: ", err) - end - - if enable_compression then - compression_file(access_new_file) - compression_file(error_new_file) - end - - -- clean the oldest file - local log_list, log_dir = scan_log_folder() - for i = max_kept + 1, #log_list.error do - local path = log_dir .. log_list.error[i] - ok, err = os_remove(path) - if err then - core.log.error("remove old error file: ", path, " err: ", err, " res:", ok) + elseif max_size > 0 then + local access_log_file_size = file_size(default_logs[DEFAULT_ACCESS_LOG_FILENAME].file) + local error_log_file_size = file_size(default_logs[DEFAULT_ERROR_LOG_FILENAME].file) + if access_log_file_size >= max_size then + rotate_file({DEFAULT_ACCESS_LOG_FILENAME}, now_time, max_kept) end - end - - for i = max_kept + 1, #log_list.access do - local path = log_dir .. log_list.access[i] - ok, err = os_remove(path) - if err then - core.log.error("remove old error file: ", path, " err: ", err, " res:", ok) + if error_log_file_size >= max_size then + rotate_file({DEFAULT_ERROR_LOG_FILENAME}, now_time, max_kept) end end - - -- reset rotate time - rotate_time = rotate_time + interval end diff --git a/conf/config-default.yaml b/conf/config-default.yaml index 9e42add2bcdf..fa73fc3b5325 100755 --- a/conf/config-default.yaml +++ b/conf/config-default.yaml @@ -493,6 +493,7 @@ plugin_attr: log-rotate: interval: 3600 # rotate interval (unit: second) max_kept: 168 # max number of log files will be kept + max_size: -1 # max size bytes of log files to be rotated, size check would be skipped with a value less than 0 enable_compression: false # enable log file compression(gzip) or not, default false skywalking: service_name: APISIX diff --git a/docs/en/latest/plugins/log-rotate.md b/docs/en/latest/plugins/log-rotate.md index 3c4cc7ecca26..3aee947fe50b 100644 --- a/docs/en/latest/plugins/log-rotate.md +++ b/docs/en/latest/plugins/log-rotate.md @@ -39,6 +39,7 @@ You can configure how often the logs are rotated and how many logs to keep. When |--------------------|---------|----------|---------|------------------------------------------------------------------------------------------------| | interval | integer | True | 60 * 60 | Time in seconds specifying how often to rotate the logs. | | max_kept | integer | True | 24 * 7 | Maximum number of historical logs to keep. If this number is exceeded, older logs are deleted. | +| max_size | integer | False | -1 | Max size(Bytes) of log files to be rotated, size check would be skipped with a value less than 0 or time is up specified by interval. | | enable_compression | boolean | False | false | When set to `true`, compresses the log file (gzip). Requires `tar` to be installed. | ## Enabling the Plugin @@ -51,9 +52,10 @@ plugins: plugin_attr: log-rotate: - interval: 3600 - max_kept: 168 - enable_compression: false + interval: 3600 # rotate interval (unit: second) + max_kept: 168 # max number of log files will be kept + max_size: -1 # max size of log files will be kept + enable_compression: false # enable log file compression(gzip) or not, default false ``` ## Example usage diff --git a/docs/zh/latest/plugins/log-rotate.md b/docs/zh/latest/plugins/log-rotate.md index 58b675946d5c..14f01c8379b4 100644 --- a/docs/zh/latest/plugins/log-rotate.md +++ b/docs/zh/latest/plugins/log-rotate.md @@ -39,6 +39,7 @@ description: 云原生 API 网关 Apache APISIX log-rotate 插件用于定期切 | ------------------ | ------- | ------ | ------- | ------------- | ---------------------------------------------------------------------------- | | interval | integer | 是 | 60 * 60 | | 每间隔多长时间切分一次日志，以秒为单位。 | | max_kept | integer | 是 | 24 * 7 | | 最多保留多少份历史日志，超过指定数量后，自动删除老文件。 | +| max_size | integer | 否 | -1 | | 日志文件超过指定大小时进行切分，单位为 Byte 。如果 `max_size` 小于 0 或者根据 `interval` 计算的时间到达时，将不会根据 `max_size` 切分日志。 | | enable_compression | boolean | 否 | false | [false, true] | 当设置为 `true` 时，启用日志文件压缩。该功能需要在系统中安装 `tar` 。 | 开启该插件后，就会按照参数自动切分日志文件了。比如以下示例是根据 `interval: 10` 和 `max_kept: 10` 得到的样本。 @@ -92,6 +93,7 @@ plugin_attr: log-rotate: interval: 3600 # rotate interval (unit: second) max_kept: 168 # max number of log files will be kept + max_size: -1 # max size of log files will be kept enable_compression: false # enable log file compression(gzip) or not, default false ``` diff --git a/t/plugin/log-rotate3.t b/t/plugin/log-rotate3.t new file mode 100644 index 000000000000..bfab0f9b63e9 --- /dev/null +++ b/t/plugin/log-rotate3.t @@ -0,0 +1,141 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +use t::APISIX 'no_plan'; + +repeat_each(1); +no_long_string(); +no_shuffle(); +no_root_location(); + +add_block_preprocessor(sub { + my ($block) = @_; + + if (!defined $block->yaml_config) { + my $yaml_config = <<_EOC_; +apisix: + node_listen: 1984 + admin_key: ~ +plugins: + - log-rotate +plugin_attr: + log-rotate: + interval: 86400 + max_size: 9 + max_kept: 3 + enable_compression: false +_EOC_ + + $block->set_value("yaml_config", $yaml_config); + } + + if ((!defined $block->error_log) && (!defined $block->no_error_log)) { + $block->set_value("no_error_log", "[error]"); + } + + if (!defined $block->request) { + $block->set_value("request", "GET /t"); + } + +}); + +run_tests; + +__DATA__ + +=== TEST 1: log rotate by max_size +--- config + location /t { + content_by_lua_block { + ngx.log(ngx.ERR, "start xxxxxx") + ngx.sleep(2) + local has_split_access_file = false + local has_split_error_file = false + local lfs = require("lfs") + for file_name in lfs.dir(ngx.config.prefix() .. "/logs/") do + if string.match(file_name, "__access.log$") then + has_split_access_file = true + end + + if string.match(file_name, "__error.log$") then + has_split_error_file = true + end + end + + if not has_split_access_file and has_split_error_file then + ngx.status = 200 + else + ngx.status = 500 + end + } + } + + + +=== TEST 2: in current log +--- config + location /t { + content_by_lua_block { + ngx.sleep(0.1) + ngx.log(ngx.WARN, "start xxxxxx") + ngx.say("done") + } + } +--- response_body +done +--- error_log +start xxxxxx + + + +=== TEST 3: check file changes +--- config + location /t { + content_by_lua_block { + ngx.sleep(1) + + local default_logs = {} + for file_name in lfs.dir(ngx.config.prefix() .. "/logs/") do + if string.match(file_name, "__error.log$") or string.match(file_name, "__access.log$") then + local filepath = ngx.config.prefix() .. "/logs/" .. file_name + local attr = lfs.attributes(filepath) + if attr then + default_logs[filepath] = { change = attr.change, size = attr.size } + end + end + end + + ngx.sleep(1) + + local passed = false + for filepath, origin_attr in pairs(default_logs) do + local check_attr = lfs.attributes(filepath) + if check_attr.change == origin_attr.change and check_attr.size == origin_attr.size then + passed = true + else + passed = false + break + end + end + + if passed then + ngx.say("passed") + end + } + } +--- response_body +passed