diff --git a/src/log-manager/deploy/log-manager.yaml.template b/src/log-manager/deploy/log-manager.yaml.template index 294dcc64fe..cfc7c9cd05 100644 --- a/src/log-manager/deploy/log-manager.yaml.template +++ b/src/log-manager/deploy/log-manager.yaml.template @@ -37,8 +37,8 @@ spec: image: {{ cluster_cfg["cluster"]["docker-registry"]["prefix"] }}log-manager-logrotate:{{ cluster_cfg["cluster"]["docker-registry"]["tag"] }} imagePullPolicy: Always env: - - name: LOGROTATE_INTERVAL - value: "hourly" + - name: LOGROTATE_CRONSCHEDULE + value: "*/10 * * * *" volumeMounts: - name: pai-log mountPath: /usr/local/pai/logs diff --git a/src/log-manager/src/logrotate/docker-entrypoint.sh b/src/log-manager/src/logrotate/docker-entrypoint.sh index 03166e8741..09f55dccf7 100755 --- a/src/log-manager/src/logrotate/docker-entrypoint.sh +++ b/src/log-manager/src/logrotate/docker-entrypoint.sh @@ -81,8 +81,13 @@ fi logrotate_cron_timetable="/usr/sbin/logrotate ${logrotate_parameters} --state=${logrotate_logstatus} /usr/bin/logrotate.d/logrotate.conf ${logrotate_cronlog}" -# ----- Cron Start ------ +log_exist_time=30 # 30 day +if [ -n "${LOG_EXIST_TIME}" ]; then + log_exist_time=${LOG_EXIST_TIME} +fi +# ----- Cron Start ------ +exec /usr/bin/go-cron '@daily' /bin/bash -c "find /usr/local/pai/logs/*/*/*/*/* -mtime +${log_exist_time} -type f -exec rm -fv {} \;"& if [ "$1" = 'cron' ]; then exec /usr/bin/go-cron "${logrotate_croninterval}" /bin/bash -c "${logrotate_cron_timetable}" fi diff --git a/src/log-manager/src/logrotate/logrotate.conf b/src/log-manager/src/logrotate/logrotate.conf index 26631d5c31..fc6e36183d 100644 --- a/src/log-manager/src/logrotate/logrotate.conf +++ b/src/log-manager/src/logrotate/logrotate.conf @@ -4,12 +4,11 @@ /usr/local/pai/logs/*/*/*/*/*.all { nomail - weekly - rotate 4 - compress + rotate 1 + nocompress missingok notifempty copytruncate - maxsize 300M + size 256M maxage 30 } diff --git a/src/webportal/src/app/job/job-view/fabric/job-detail/conn.js b/src/webportal/src/app/job/job-view/fabric/job-detail/conn.js index f15c4f9bcb..55a1c23d91 100644 --- a/src/webportal/src/app/job/job-view/fabric/job-detail/conn.js +++ b/src/webportal/src/app/job/job-view/fabric/job-detail/conn.js @@ -176,7 +176,7 @@ export async function getContainerLog(logUrl) { text: null, }; const res = await fetch(logUrl); - const text = await res.text(); + var text = await res.text(); if (!res.ok) { throw new Error(res.statusText); } @@ -222,6 +222,22 @@ export async function getContainerLog(logUrl) { throw new Error(`Log not available`); } } else if (config.logType === 'log-manager') { + // Try to get roated log if currently log content is less than 15KB + if (text.length <= 15 * 1024) { + const fullLogUrl = logUrl.replace('/tail/', '/full/'); + const rotatedLogUrl = logUrl + '.1'; + const rotatedLogRes = await fetch(rotatedLogUrl); + const fullLogRes = await fetch(fullLogUrl); + const rotatedText = await rotatedLogRes.text(); + const fullLog = await fullLogRes.text(); + if (rotatedLogRes.ok && rotatedText.trim() !== 'No such file!') { + text = rotatedText + .concat('\n--------log is rotated, may be lost during this--------\n') + .concat(fullLog); + } + // get last 16KB + text = text.slice(-16 * 1024); + } ret.text = text; ret.fullLogLink = logUrl.replace('/tail/', '/full/'); return ret;