Use logrotate to rotate OVS log files (#1329)

Install logrotate in the OVS Docker image.
Enhance start_ovs to run logrotate against the OVS log files in
/var/log/openvswitch/ every hour, with two flags:
--log_file_max_num and --log_file_max_size to specify the maximum
number and maximum size of log files respectively.
Update the Antrea deployment YAMLs to set the default value of
--log_file_max_num to 4, and the default value of --log_file_max_size
to 100MB.

build/images/ovs/Dockerfile

@@ -32,11 +32,14 @@ LABEL description="A Docker image based on Ubuntu 20.04 which includes Open vSwi
 COPY --from=ovs-debs /tmp/ovs-debs/* /tmp/ovs-debs/
 COPY charon-logging.conf /tmp
 
+# Install OVS debs, iptables, logrotate, and strongSwan; update the OVS
+# logrotate config file; update the strongSwan logging config.
 # We clean-up apt cache after installing packages to reduce the size of the
-# final image
+# final image.
 RUN apt-get update && \
-    apt-get install -y --no-install-recommends iptables libstrongswan-standard-plugins && \
+    apt-get install -y --no-install-recommends iptables logrotate libstrongswan-standard-plugins && \
     (dpkg -i /tmp/ovs-debs/*.deb || apt-get -f -y --no-install-recommends install) && \
     rm -rf /var/cache/apt/* /var/lib/apt/lists/* && \
+    sed -i "/rotate /a\    #size 100M" /etc/logrotate.d/openvswitch-switch && \
     sed -i "/^.*filelog.*{/r /tmp/charon-logging.conf" /etc/strongswan.d/charon-logging.conf && \
     rm -rf /tmp/*

build/images/scripts/start_ovs

@@ -7,8 +7,57 @@ source /usr/share/openvswitch/scripts/ovs-lib
 CONTAINER_NAME="antrea-ovs"
 OVS_RUN_DIR="/var/run/openvswitch"
 OVS_DB_FILE="${OVS_RUN_DIR}/conf.db"
+OVS_LOGROTATE_CONF="/etc/logrotate.d/openvswitch-switch"
+
+hw_offload="false"
+log_file_max_num=0
+log_file_max_size=0
+
+function usage {
+    echo "start_ovs"
+    echo -e "  -h|--help\t\t \tPrint help message"
+    echo -e "  --hw-offload\t\t \tEnable OVS hardware offload"
+    echo -e "  --log_file_max_num=<uint> \tMaximum number of log files to be kept for an OVS daemon. Value 0 means keeping the current value"
+    echo -e "  --log_file_max_size=<uint> \tMaximum size (in megabytes) of an OVS log file. Value 0 means keeping the current value"
+}
 
-set -euo pipefail
+while (( "$#" )); do
+  case "$1" in
+    -h|--help)
+      usage
+      exit 0
+      ;;
+    --hw-offload)
+      hw_offload="true"
+      ;;
+    --log_file_max_num=*)
+      log_file_max_num=$1
+      log_file_max_num=${log_file_max_num#"--log_file_max_num="}
+      ;;
+    --log_file_max_size=*)
+      log_file_max_size=$1
+      log_file_max_size=${log_file_max_size#"--log_file_max_size="}
+      ;;
+    -*|--*) # unsupported flags
+      echo "Error: unsupported flag $1" >&2
+      exit 1
+      ;;
+    *) # standalone arguments are not supported
+      echo "Error: unsupported argument $1" >&2
+      exit 1
+      ;;
+  esac
+  shift
+done
+
+function update_logrotate_config_file {
+    if [ $log_file_max_num -gt 0 ]; then
+        sed -i "s/.*rotate .*/    rotate $log_file_max_num/" $OVS_LOGROTATE_CONF
+    fi
+    if [ $log_file_max_size -gt 0 ]; then
+        sed -i "s/.*size .*/    size $log_file_max_size\M/" $OVS_LOGROTATE_CONF
+    fi
+}
 
 # We once (issue #870) observed that ovsdb-server failed to restart with error:
 # "ovsdb-server: /var/run/openvswitch/ovsdb-server.pid: pidfile check failed
@@ -21,8 +70,6 @@ function cleanup_ovs_run_files {
     rm -rf ${OVS_RUN_DIR}/.conf.db.*~lock~
 }
 
-# start_ovs takes one parameter: a boolean value which indicates whether OVS
-# kernel HW offload should be enabled.
 function start_ovs {
     offload=$1
     if daemon_is_running ovsdb-server; then
@@ -42,7 +89,7 @@ function start_ovs {
         # restoring flows.
         ovs-vsctl --no-wait set open_vswitch . other_config:flow-restore-wait="true"
         log_info $CONTAINER_NAME "ovs-vswitchd set hw-offload to $offload"
-        ovs-vsctl --no-wait set open_vswitch . other_config:hw-offload="$offload"
+        ovs-vsctl --no-wait set open_vswitch . other_config:hw-offload=$offload
         /usr/share/openvswitch/scripts/ovs-ctl --no-ovsdb-server --system-id=random start --db-file=$OVS_DB_FILE
         log_info $CONTAINER_NAME "Started ovs-vswitchd"
     fi
@@ -63,36 +110,45 @@ function quit {
     exit 0
 }
 
+set -euo pipefail
+
 # Do not trap EXIT as it would then ignore the "exit 0" statement in quit and
 # exit with code 128 + SIGNAL
 trap "quit" INT TERM
 
-cleanup_ovs_run_files
+update_logrotate_config_file
 
-offload=false
-if [ $# == 1 ] && [ -n "$1" ] && [ "$1" == "--hw-offload" ]; then
-    offload=true
-fi
+cleanup_ovs_run_files
 
-start_ovs $offload
+start_ovs $hw_offload
 
 # Restrict read permissions for "others"
 # See discussion in https://github.com/vmware-tanzu/antrea/issues/1292
 chmod 0640 $OVS_DB_FILE
 
-log_info $CONTAINER_NAME "Started the loop that checks OVS status every 30 seconds"
+# Check OVS status every 30 seconds
+CHECK_OVS_INTERVAL=30
+# Run logrotate every hour
+LOG_ROTATE_INTERVAL=60*60
+counter=0
+
+log_info $CONTAINER_NAME "Started the loop that checks OVS status every $CHECK_OVS_INTERVAL seconds"
 while true; do
     # we run sleep in the background so that we can immediately exit when we
     # receive SIGINT / SIGTERM
     # see https://stackoverflow.com/questions/32041674/linux-how-to-kill-sleep
-    sleep 30 &
+    sleep $CHECK_OVS_INTERVAL &
     SLEEP_PID=$!
     wait $SLEEP_PID
 
     if ! check_ovs_status > /dev/null ; then
         # OVS was stopped in the container.
         log_warning $CONTAINER_NAME "OVS was stopped. Starting it again"
 
-        start_ovs $offload
+        start_ovs $hw_offload
+    fi
+
+    if [ $((++counter % (LOG_ROTATE_INTERVAL / CHECK_OVS_INTERVAL))) == 0 ]; then
+       logrotate $OVS_LOGROTATE_CONF
     fi
 done

build/yamls/antrea-aks.yml

@@ -1439,7 +1439,10 @@ spec:
           readOnly: true
         - mountPath: /run/xtables.lock
           name: xtables-lock
-      - command:
+      - args:
+        - --log_file_max_size=100
+        - --log_file_max_num=4
+        command:
         - start_ovs
         image: antrea/antrea-ubuntu:latest
         imagePullPolicy: IfNotPresent

build/yamls/antrea-eks.yml

@@ -1441,7 +1441,10 @@ spec:
           readOnly: true
         - mountPath: /run/xtables.lock
           name: xtables-lock
-      - command:
+      - args:
+        - --log_file_max_size=100
+        - --log_file_max_num=4
+        command:
         - start_ovs
         image: antrea/antrea-ubuntu:latest
         imagePullPolicy: IfNotPresent

build/yamls/antrea-gke.yml

@@ -1439,7 +1439,10 @@ spec:
           readOnly: true
         - mountPath: /run/xtables.lock
           name: xtables-lock
-      - command:
+      - args:
+        - --log_file_max_size=100
+        - --log_file_max_num=4
+        command:
         - start_ovs
         image: antrea/antrea-ubuntu:latest
         imagePullPolicy: IfNotPresent

build/yamls/antrea-ipsec.yml

@@ -1458,7 +1458,10 @@ spec:
           readOnly: true
         - mountPath: /run/xtables.lock
           name: xtables-lock
-      - command:
+      - args:
+        - --log_file_max_size=100
+        - --log_file_max_num=4
+        command:
         - start_ovs
         image: antrea/antrea-ubuntu:latest
         imagePullPolicy: IfNotPresent

build/yamls/antrea.yml

@@ -1444,7 +1444,10 @@ spec:
           readOnly: true
         - mountPath: /run/xtables.lock
           name: xtables-lock
-      - command:
+      - args:
+        - --log_file_max_size=100
+        - --log_file_max_num=4
+        command:
         - start_ovs
         image: antrea/antrea-ubuntu:latest
         imagePullPolicy: IfNotPresent

build/yamls/base/agent.yml

@@ -149,6 +149,7 @@ spec:
             requests:
               cpu: "200m"
           command: ["start_ovs"]
+          args: ["--log_file_max_size=100", "--log_file_max_num=4"]
           securityContext:
             # capabilities required by OVS daemons
             capabilities: