From eb1ee577c539eea534eeb014059b5f7255639f46 Mon Sep 17 00:00:00 2001 From: Jianjun Shen Date: Wed, 30 Sep 2020 22:44:28 -0700 Subject: [PATCH] Use logrotate to rotate OVS log files Install logrotate in the OVS Docker image. Enhance start_ovs to run logrotate against the OVS log files in /var/log/openvswitch/ every hour, with two flags: --log_file_max_num and --log_file_max_size to specify the maximum number and maximum size of log files respectively. Update the Antrea deployment YAMLs to set the default value of --log_file_max_num to 4, and the default value of --log_file_max_size to 100MB. --- build/images/ovs/Dockerfile | 7 ++- build/images/scripts/start_ovs | 82 ++++++++++++++++++++++++++++------ build/yamls/antrea-aks.yml | 5 ++- build/yamls/antrea-eks.yml | 5 ++- build/yamls/antrea-gke.yml | 5 ++- build/yamls/antrea-ipsec.yml | 5 ++- build/yamls/antrea.yml | 5 ++- build/yamls/base/agent.yml | 1 + 8 files changed, 95 insertions(+), 20 deletions(-) diff --git a/build/images/ovs/Dockerfile b/build/images/ovs/Dockerfile index 9288c827995..7b31b902ebf 100644 --- a/build/images/ovs/Dockerfile +++ b/build/images/ovs/Dockerfile @@ -32,11 +32,14 @@ LABEL description="A Docker image based on Ubuntu 20.04 which includes Open vSwi COPY --from=ovs-debs /tmp/ovs-debs/* /tmp/ovs-debs/ COPY charon-logging.conf /tmp +# Install OVS debs, iptables, logrotate, and strongSwan; update the OVS +# logrotate config file; update the strongSwan logging config. # We clean-up apt cache after installing packages to reduce the size of the -# final image +# final image. RUN apt-get update && \ - apt-get install -y --no-install-recommends iptables libstrongswan-standard-plugins && \ + apt-get install -y --no-install-recommends iptables logrotate libstrongswan-standard-plugins && \ (dpkg -i /tmp/ovs-debs/*.deb || apt-get -f -y --no-install-recommends install) && \ rm -rf /var/cache/apt/* /var/lib/apt/lists/* && \ + sed -i "/rotate /a\ #size 100M" /etc/logrotate.d/openvswitch-switch && \ sed -i "/^.*filelog.*{/r /tmp/charon-logging.conf" /etc/strongswan.d/charon-logging.conf && \ rm -rf /tmp/* diff --git a/build/images/scripts/start_ovs b/build/images/scripts/start_ovs index 63a5441bac1..738ddda84b9 100755 --- a/build/images/scripts/start_ovs +++ b/build/images/scripts/start_ovs @@ -7,8 +7,57 @@ source /usr/share/openvswitch/scripts/ovs-lib CONTAINER_NAME="antrea-ovs" OVS_RUN_DIR="/var/run/openvswitch" OVS_DB_FILE="${OVS_RUN_DIR}/conf.db" +OVS_LOGROTATE_CONF="/etc/logrotate.d/openvswitch-switch" + +hw_offload="false" +log_file_max_num=0 +log_file_max_size=0 + +function usage { + echo "start_ovs" + echo -e " -h|--help\t\t \tPrint help message" + echo -e " --hw-offload\t\t \tEnable OVS hardware offload" + echo -e " --log_file_max_num= \tMaximum number of log files to be kept for an OVS daemon. Value 0 means keeping the current value" + echo -e " --log_file_max_size= \tMaximum size (in megabytes) of an OVS log file. Value 0 means keeping the current value" +} -set -euo pipefail +while (( "$#" )); do + case "$1" in + -h|--help) + usage + exit 0 + ;; + --hw-offload) + hw_offload="true" + ;; + --log_file_max_num=*) + log_file_max_num=$1 + log_file_max_num=${log_file_max_num#"--log_file_max_num="} + ;; + --log_file_max_size=*) + log_file_max_size=$1 + log_file_max_size=${log_file_max_size#"--log_file_max_size="} + ;; + -*|--*) # unsupported flags + echo "Error: unsupported flag $1" >&2 + exit 128 + ;; + *) # standalone arguments are not supported + echo "Error: unsupported argument $1" >&2 + exit 128 + ;; + esac + shift +done + +function update_logrotate_config_file { + if [ $log_file_max_num -gt 0 ]; then + sed -i "s/.*rotate .*/ rotate $log_file_max_num/" $OVS_LOGROTATE_CONF + fi + if [ $log_file_max_size -gt 0 ]; then + sed -i "s/.*size .*/ size $log_file_max_size\M/" $OVS_LOGROTATE_CONF + fi +} # We once (issue #870) observed that ovsdb-server failed to restart with error: # "ovsdb-server: /var/run/openvswitch/ovsdb-server.pid: pidfile check failed @@ -21,8 +70,6 @@ function cleanup_ovs_run_files { rm -rf ${OVS_RUN_DIR}/.conf.db.*~lock~ } -# start_ovs takes one parameter: a boolean value which indicates whether OVS -# kernel HW offload should be enabled. function start_ovs { offload=$1 if daemon_is_running ovsdb-server; then @@ -42,7 +89,7 @@ function start_ovs { # restoring flows. ovs-vsctl --no-wait set open_vswitch . other_config:flow-restore-wait="true" log_info $CONTAINER_NAME "ovs-vswitchd set hw-offload to $offload" - ovs-vsctl --no-wait set open_vswitch . other_config:hw-offload="$offload" + ovs-vsctl --no-wait set open_vswitch . other_config:hw-offload=$offload /usr/share/openvswitch/scripts/ovs-ctl --no-ovsdb-server --system-id=random start --db-file=$OVS_DB_FILE log_info $CONTAINER_NAME "Started ovs-vswitchd" fi @@ -63,29 +110,34 @@ function quit { exit 0 } +set -euo pipefail + # Do not trap EXIT as it would then ignore the "exit 0" statement in quit and # exit with code 128 + SIGNAL trap "quit" INT TERM -cleanup_ovs_run_files +update_logrotate_config_file -offload=false -if [ $# == 1 ] && [ -n "$1" ] && [ "$1" == "--hw-offload" ]; then - offload=true -fi +cleanup_ovs_run_files -start_ovs $offload +start_ovs $hw_offload # Restrict read permissions for "others" # See discussion in https://github.com/vmware-tanzu/antrea/issues/1292 chmod 0640 $OVS_DB_FILE -log_info $CONTAINER_NAME "Started the loop that checks OVS status every 30 seconds" +# Check OVS status every 30 seconds +CHECK_OVS_INTERVAL=30 +# Run logrotate every hour +LOG_ROTATE_INTERVAL=60*60 +counter=0 + +log_info $CONTAINER_NAME "Started the loop that checks OVS status every $CHECK_OVS_INTERVAL seconds" while true; do # we run sleep in the background so that we can immediately exit when we # receive SIGINT / SIGTERM # see https://stackoverflow.com/questions/32041674/linux-how-to-kill-sleep - sleep 30 & + sleep $CHECK_OVS_INTERVAL & SLEEP_PID=$! wait $SLEEP_PID @@ -93,6 +145,10 @@ while true; do # OVS was stopped in the container. log_warning $CONTAINER_NAME "OVS was stopped. Starting it again" - start_ovs $offload + start_ovs $hw_offload + fi + + if [ $((++counter % (LOG_ROTATE_INTERVAL / CHECK_OVS_INTERVAL))) == 0 ]; then + logrotate $OVS_LOGROTATE_CONF fi done diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml index ae849a0ab86..66cd54ed7a4 100644 --- a/build/yamls/antrea-aks.yml +++ b/build/yamls/antrea-aks.yml @@ -1439,7 +1439,10 @@ spec: readOnly: true - mountPath: /run/xtables.lock name: xtables-lock - - command: + - args: + - --log_file_max_size=100 + - --log_file_max_num=4 + command: - start_ovs image: antrea/antrea-ubuntu:latest imagePullPolicy: IfNotPresent diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml index 3398ed8f635..2234707c352 100644 --- a/build/yamls/antrea-eks.yml +++ b/build/yamls/antrea-eks.yml @@ -1441,7 +1441,10 @@ spec: readOnly: true - mountPath: /run/xtables.lock name: xtables-lock - - command: + - args: + - --log_file_max_size=100 + - --log_file_max_num=4 + command: - start_ovs image: antrea/antrea-ubuntu:latest imagePullPolicy: IfNotPresent diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml index e59532d9c71..53be82082d9 100644 --- a/build/yamls/antrea-gke.yml +++ b/build/yamls/antrea-gke.yml @@ -1439,7 +1439,10 @@ spec: readOnly: true - mountPath: /run/xtables.lock name: xtables-lock - - command: + - args: + - --log_file_max_size=100 + - --log_file_max_num=4 + command: - start_ovs image: antrea/antrea-ubuntu:latest imagePullPolicy: IfNotPresent diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml index edea0314951..d517bc92324 100644 --- a/build/yamls/antrea-ipsec.yml +++ b/build/yamls/antrea-ipsec.yml @@ -1458,7 +1458,10 @@ spec: readOnly: true - mountPath: /run/xtables.lock name: xtables-lock - - command: + - args: + - --log_file_max_size=100 + - --log_file_max_num=4 + command: - start_ovs image: antrea/antrea-ubuntu:latest imagePullPolicy: IfNotPresent diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml index 36667630410..f8f5a868b4a 100644 --- a/build/yamls/antrea.yml +++ b/build/yamls/antrea.yml @@ -1444,7 +1444,10 @@ spec: readOnly: true - mountPath: /run/xtables.lock name: xtables-lock - - command: + - args: + - --log_file_max_size=100 + - --log_file_max_num=4 + command: - start_ovs image: antrea/antrea-ubuntu:latest imagePullPolicy: IfNotPresent diff --git a/build/yamls/base/agent.yml b/build/yamls/base/agent.yml index d9d5c9b75aa..ad7798a7034 100644 --- a/build/yamls/base/agent.yml +++ b/build/yamls/base/agent.yml @@ -149,6 +149,7 @@ spec: requests: cpu: "200m" command: ["start_ovs"] + args: ["--log_file_max_size=100", "--log_file_max_num=4"] securityContext: # capabilities required by OVS daemons capabilities: