Merge pull request #873 from grahamwhaley/20180123_ksm_auto

ci: metrics: improve density tests interaction with KSM

.ci/run_metrics_ci.sh

@@ -24,7 +24,16 @@ source "${CURRENTDIR}/../metrics/lib/common.bash"
 
 REPORT_CMDS=("checkmetrics" "emailreport")
 
-KSM_ENABLE_FILE="/sys/kernel/mm/ksm/run"
+KSM_BASE="/sys/kernel/mm/ksm"
+KSM_ENABLE_FILE="${KSM_BASE}/run"
+KSM_PAGES_FILE="${KSM_BASE}/pages_to_scan"
+KSM_SLEEP_FILE="${KSM_BASE}/sleep_millisecs"
+
+# The settings we use for an 'aggresive' KSM setup
+# Scan 1000 pages every 50ms - 20,000 pages/s
+KSM_AGGRESIVE_PAGES=1000
+KSM_AGGRESIVE_SLEEP=50
+
 GITHUB_URL="https://github.com"
 RESULTS_BACKUP_PATH="/var/local/localCI/backup"
 RESULTS_DIR="results"
@@ -40,6 +49,38 @@ for cmd in "${REPORT_CMDS[@]}"; do
 	popd
 done
 
+# Save the current KSM settings so we can restore them later
+save_ksm_settings(){
+	echo "saving KSM settings"
+	ksm_stored_run=$(cat ${KSM_ENABLE_FILE})
+	ksm_stored_pages=$(cat ${KSM_ENABLE_FILE})
+	ksm_stored_sleep=$(cat ${KSM_ENABLE_FILE})
+}
+
+set_ksm_aggressive(){
+	echo "setting KSM to aggressive mode"
+	# Flip the run off/on to ensure a restart/rescan
+	sudo bash -c "echo 0 > ${KSM_ENABLE_FILE}"
+	sudo bash -c "echo ${KSM_AGGRESIVE_PAGES} > ${KSM_PAGES_FILE}"
+	sudo bash -c "echo ${KSM_AGGRESIVE_SLEEP} > ${KSM_SLEEP_FILE}"
+	sudo bash -c "echo 1 > ${KSM_ENABLE_FILE}"
+}
+
+restore_ksm_settings(){
+	echo "restoring KSM settings"
+	# First turn off the run to ensure if we are then re-enabling
+	# that any changes take effect
+	sudo bash -c "echo 0 > ${KSM_ENABLE_FILE}"
+	sudo bash -c "echo ${ksm_stored_pages} > ${KSM_PAGES_FILE}"
+	sudo bash -c "echo ${ksm_stored_sleep} > ${KSM_SLEEP_FILE}"
+	sudo bash -c "echo ${ksm_stored_run} > ${KSM_ENABLE_FILE}"
+}
+
+disable_ksm(){
+	echo "disabling KSM"
+	sudo bash -c "echo 0 > ${KSM_ENABLE_FILE}"
+}
+
 # Execute metrics scripts, save the results and report them
 # by email.
 pushd "$CURRENTDIR/../metrics"
@@ -49,6 +90,10 @@ pushd "$CURRENTDIR/../metrics"
 	# and then turn it off for the rest of the tests, as KSM may introduce
 	# some extra noise in the results by stealing CPU time for instance
 	if [[ -f ${KSM_ENABLE_FILE} ]]; then
+		save_ksm_settings
+		trap restore_ksm_settings EXIT QUIT KILL
+		set_ksm_aggressive
+
 		# Ensure KSM is enabled
 		sudo bash -c "echo 1 > ${KSM_ENABLE_FILE}"
 
@@ -59,10 +104,11 @@ pushd "$CURRENTDIR/../metrics"
 		# Run the memory footprint test. With default Ubuntu 16.04
 		# settings, and 20 containers, it takes ~200s to 'settle' to
 		# a steady memory footprint
-		bash density/docker_memory_usage.sh 20 300
+		bash density/docker_memory_usage.sh 20 300 auto
 
 		# And now ensure KSM is turned off for the rest of the tests
 		sudo bash -c "echo 0 > ${KSM_ENABLE_FILE}"
+		disable_ksm
 	fi
 
 	# Run the time tests

metrics/density/README.md

@@ -7,7 +7,20 @@ This number of containers could be working under different configurations/condit
 - `docker_memory_usage.sh`: measures the Proportional Set Size (PSS) memory average
    for a number of containers launched in idle mode. This test uses the `sleep` command
    to allow any memory optimizations to 'settle' (e.g. KSM execution) for a configurable
-   period of time.
+   period of time. It also has an optional 'auto' mode which will try to determine when KSM
+   has settled down by examining the KSM `full_scans` and `pages_shared` values. auto mode
+   detection of KSM settling over-rides the timeout value for taking the measurements and
+   completing the test.
+- `docker_memory_usage.sh`: measures the Proportional Set Size (PSS) memory average
+   for a number of containers launched in idle mode. 
+   The test has two methods that can be utilized to wait for `KSM` to 'settle'.
+    - The test sleeps for the amount of time specified by the 'timeout', which is
+      in seconds, before taking measurements.
+    - If `auto` mode is enabled, the test terminates `timeout` early
+      under the condition that 'KSM' has already settled, which
+      is determined by exmaining the `KSM` `full_scans` and `pages_shared`
+      information.
+
 - `footprint_data.sh`: Sequentially runs a number of identical containers and takes a
    number of memory related measurements after each launch. Generally not used in a CI
    type environment, but more for hand run/analysis. Refer to the

metrics/density/docker_memory_usage.sh

@@ -37,6 +37,7 @@ IMAGE='busybox'
 CMD='sh'
 NUM_CONTAINERS="$1"
 WAIT_TIME="$2"
+AUTO_MODE="$3"
 TEST_NAME="memory footprint"
 TEST_ARGS="workload=${IMAGE} units=kb"
 SMEM_BIN="smem"
@@ -45,11 +46,13 @@ KSM_ENABLE_FILE="/sys/kernel/mm/ksm/run"
 # Show help about this script
 function help(){
 cat << EOF
-Usage: $0 <count> <wait_time>
+Usage: $0 <count> <wait_time> [auto]
    Description:
         <count>      : Number of Clear Containers to run.
         <wait_time>  : Time in seconds to wait before taking
                        metrics.
+        [auto]       : Optional 'auto KSM settle' mode
+                       waits for ksm pages_shared to settle down
 EOF
 }
 
@@ -120,6 +123,58 @@ function get_pss_memory(){
 	echo "$avg"
 }
 
+# Wait for KSM to settle down, or timeout waiting
+# The basic algorithm is to look at the pages_shared value
+# at the end of every 'full scan', and if the value
+# has changed very little, then we are done (because we presume
+# a full scan has managed to do few new merges)
+#
+# arg1 - timeout in seconds
+wait_ksm_settle(){
+	local t pcnt
+	local oldscan=-1 newscan
+	local oldpages=-1 newpages
+
+	oldscan=$(cat /sys/kernel/mm/ksm/full_scans)
+
+	# Go around the loop until either we see a small % change
+	# between two full_scans, or we timeout
+	for ((t=0; t<$1; t++)); do
+
+		newscan=$(cat /sys/kernel/mm/ksm/full_scans)
+		if (( newscan != oldscan )); then
+			echo -e "\nnew full_scan ($oldscan to $newscan)"
+
+			newpages=$(cat /sys/kernel/mm/ksm/pages_shared)
+			# Do we have a previous scan to compare with
+			echo "check pages $oldpages to $newpages"
+			if (( oldpages != -1 )); then
+				# avoid divide by zero problems
+				if (( $oldpages > 0 )); then
+					pcnt=$(( 100 - ((newpages * 100) / oldpages) ))
+					# abs()
+					pcnt=$(( $pcnt * -1 ))
+
+					echo "$oldpages to $newpages is ${pcnt}%"
+
+					if (( $pcnt <= 5 )); then
+						echo "KSM stabilised at ${t}s"
+						return
+					fi
+				else
+					echo "$oldpages KSM pages... waiting"
+				fi
+			fi
+			oldscan=$newscan
+			oldpages=$newpages
+		else
+			echo -n "."
+		fi
+		sleep 1
+	done
+	echo "Timed out after ${1}s waiting for KSM to settle"
+}
+
 # It calculates the memory footprint
 # of a CC.
 function get_docker_memory_usage(){
@@ -136,9 +191,19 @@ function get_docker_memory_usage(){
 		${DOCKER_EXE} run --runtime "$RUNTIME" --name ${containers[-1]} -tid $IMAGE $CMD
 	done
 
-	# If KSM is enabled, then you normally want to sleep long enough to
-	# let it do its work and for the numbers to 'settle'.
-	sleep "$WAIT_TIME"
+	if [ "$AUTO_MODE" == "auto" ]; then
+		if (( ksm_on != 1 )); then
+			die "KSM not enabled, cannot use auto mode"
+		fi
+
+		echo "Entering KSM settle auto detect mode..."
+		wait_ksm_settle $WAIT_TIME
+	else
+		# If KSM is enabled, then you normally want to sleep long enough to
+		# let it do its work and for the numbers to 'settle'.
+		echo "napping $WAIT_TIME s"
+		sleep "$WAIT_TIME"
+	fi
 
 	# Check the runtime in order in order to determine which process will
 	# be measured about PSS
@@ -186,7 +251,7 @@ function get_docker_memory_usage(){
 }
 
 # Verify enough arguments
-if [[ $# != 2 ]];then
+if [ $# != 2 ] && [ $# != 3 ];then
 	echo >&2 "error: Not enough arguments [$@]"
 	help
 	exit 1