From 74d165293e7aecd20e8dfa3c89a09ebcfc5d76f8 Mon Sep 17 00:00:00 2001 From: Adnan Choudhury Date: Fri, 16 Jun 2017 14:23:18 -0400 Subject: [PATCH] [FAB-4764] Rearrange Ledger test parameter setup Increase LTE tool usability by rearranging the Ledger performance test parameter setup so that the parameter file has more control on the test run. Also, make the test driver take the parameter file as a (required) option instead of being hardcoded. Modifies the README.md and the python test wrapper to reflect these changes. Change-Id: I2cbdb0d082b3b8f9f143d2f346742c84556e3c3a Signed-off-by: Adnan Choudhury --- test/regression/daily/ledger_lte.py | 34 +++++---- test/tools/LTE/README.md | 20 +++-- test/tools/LTE/scripts/parameters_daily_CI.sh | 26 ++++++- test/tools/LTE/scripts/runbenchmarks.sh | 74 ++++++++++++++----- 4 files changed, 111 insertions(+), 43 deletions(-) diff --git a/test/regression/daily/ledger_lte.py b/test/regression/daily/ledger_lte.py index f7b06fd29b1..6a2244b259b 100644 --- a/test/regression/daily/ledger_lte.py +++ b/test/regression/daily/ledger_lte.py @@ -6,6 +6,8 @@ import unittest import subprocess +tool_directory = '../../tools/LTE/scripts' + class perf_goleveldb(unittest.TestCase): def test_FAB_3790_VaryNumParallelTxPerChain(self): @@ -20,9 +22,9 @@ def test_FAB_3790_VaryNumParallelTxPerChain(self): ''' logfile = open("output_VaryNumParallelTxPerChain.log", "w") returncode = subprocess.call( - "./runbenchmarks.sh varyNumParallelTxPerChain", + "./runbenchmarks.sh -f parameters_daily_CI.sh varyNumParallelTxPerChain", shell=True, stderr=subprocess.STDOUT, stdout=logfile, - cwd='../../tools/LTE/scripts') + cwd=tool_directory) logfile.close() self.assertEqual(returncode, 0, msg="VaryNumParallelTxPerChain " "performance test failed. \nPlease check the logfile " @@ -40,9 +42,9 @@ def test_FAB_3795_VaryNumChains(self): ''' logfile = open("output_VaryNumChains.log", "w") returncode = subprocess.call( - "./runbenchmarks.sh varyNumChains", + "./runbenchmarks.sh -f parameters_daily_CI.sh varyNumChains", shell=True, stderr=subprocess.STDOUT, stdout=logfile, - cwd='../../tools/LTE/scripts') + cwd=tool_directory) logfile.close() self.assertEqual(returncode, 0, msg="VaryNumChains performance test" " failed. \nPlease check the logfile "+logfile.name+" for more " @@ -60,9 +62,9 @@ def test_FAB_3798_VaryNumParallelTxWithSingleChain(self): ''' logfile = open("output_VaryNumParallelTxWithSingleChain.log", "w") returncode = subprocess.call( - "./runbenchmarks.sh varyNumParallelTxWithSingleChain", + "./runbenchmarks.sh -f parameters_daily_CI.sh varyNumParallelTxWithSingleChain", shell=True, stderr=subprocess.STDOUT, stdout=logfile, - cwd='../../tools/LTE/scripts') + cwd=tool_directory) logfile.close() self.assertEqual(returncode, 0, msg="VaryNumParallelTxWithSingleChain " "performance test failed. \nPlease check the logfile " @@ -80,9 +82,9 @@ def test_FAB_3799_VaryNumChainsWithNoParallelism(self): ''' logfile = open("output_VaryNumChainsWithNoParallelism.log", "w") returncode = subprocess.call( - "./runbenchmarks.sh varyNumChainsWithNoParallelism", + "./runbenchmarks.sh -f parameters_daily_CI.sh varyNumChainsWithNoParallelism", shell=True, stderr=subprocess.STDOUT, stdout=logfile, - cwd='../../tools/LTE/scripts') + cwd=tool_directory) logfile.close() self.assertEqual(returncode, 0, msg="varyNumChainsWithNoParallelism " "performance test failed. \nPlease check the logfile " @@ -99,9 +101,9 @@ def test_FAB_3801_VaryKVSize(self): ''' logfile = open("output_VaryKVSize.log", "w") returncode = subprocess.call( - "./runbenchmarks.sh varyKVSize", + "./runbenchmarks.sh -f parameters_daily_CI.sh varyKVSize", shell=True, stderr=subprocess.STDOUT, stdout=logfile, - cwd='../../tools/LTE/scripts') + cwd=tool_directory) logfile.close() self.assertEqual(returncode, 0, msg="varyKVSize performance test" " failed. \nPlease check the logfile "+logfile.name+" for more " @@ -119,9 +121,9 @@ def test_FAB_3802_VaryBatchSize(self): ''' logfile = open("output_VaryBatchSize.log", "w") returncode = subprocess.call( - "./runbenchmarks.sh varyBatchSize", + "./runbenchmarks.sh -f parameters_daily_CI.sh varyBatchSize", shell=True, stderr=subprocess.STDOUT, stdout=logfile, - cwd='../../tools/LTE/scripts') + cwd=tool_directory) logfile.close() self.assertEqual(returncode, 0, msg="varyBatchSize performance test" " failed. \nPlease check the logfile "+logfile.name+" for more " @@ -139,9 +141,9 @@ def test_FAB_3800_VaryNumKeysInEachTx(self): ''' logfile = open("output_VaryNumKeysInEachTx.log", "w") returncode = subprocess.call( - "./runbenchmarks.sh varyNumKeysInEachTx", + "./runbenchmarks.sh -f parameters_daily_CI.sh varyNumKeysInEachTx", shell=True, stderr=subprocess.STDOUT, stdout=logfile, - cwd='../../tools/LTE/scripts') + cwd=tool_directory) logfile.close() self.assertEqual(returncode, 0, msg="varyNumKeysInEachTx performance " "test failed. \nPlease check the logfile "+logfile.name @@ -159,9 +161,9 @@ def test_FAB_3803_VaryNumTxs(self): ''' logfile = open("output_VaryNumTxs.log", "w") returncode = subprocess.call( - "./runbenchmarks.sh varyNumTxs", + "./runbenchmarks.sh -f parameters_daily_CI.sh varyNumTxs", shell=True, stderr=subprocess.STDOUT, stdout=logfile, - cwd='../../tools/LTE/scripts') + cwd=tool_directory) logfile.close() self.assertEqual(returncode, 0, msg="varyNumTxs performance test" " failed. \nPlease check the logfile "+logfile.name+" for more " diff --git a/test/tools/LTE/README.md b/test/tools/LTE/README.md index 7931a89587a..070f37eaf09 100644 --- a/test/tools/LTE/README.md +++ b/test/tools/LTE/README.md @@ -21,12 +21,15 @@ considered to constitute a single test-run. ## How to Run The tests -To run all the available tests with default parameters, run: +To run all the available tests, run: ``` cd fabric/test/tools/LTE/scripts -./runbenchmark.sh all +./runbenchmark.sh -f parameters_daily_CI.sh all ``` -you can run individual tests without running all the available tests by giving +where the file `parameters_daily_CI.sh` has all the necessary test parameters. + + +You can run individual tests without running all the available tests by giving the name of the test as parameter, instead of `all`. You can get the available test names by: ``` @@ -35,7 +38,7 @@ test names by: ### What the Tests Do -Each test reads test parameters from the file `parameters_daily_CI.sh` and +Each test reads test parameters from the provided parameter file and conducts several test-runs by varying one or two of the parameters. The parameters are: * number of chains (ledger), @@ -95,8 +98,15 @@ transactions on chain_2 in parallel In each of the transactions executed by any client, the transaction expects and modifies any key(s) between Key_1 to key_50 (because, total keys are to be 100 across two chains). +### Running with Custom Parameters + +The tests can be run with user-defined parameters by creating a new file that +has all the necessary parameters to run and using that file as the input (see +the section on how to run the tests) . The names of necessary parameters can be +found in the file `parameters_daily_CI.sh`. + ## How to View the Test Results The test results can be viewed as in the stdout where it shows how long each single operation took to complete in a test. These results are also saved in a -.csv file in the following directory: `/tmp/fabric/test/tools/LTE/results` +.csv file in the following directory: `/tmp/experiments` diff --git a/test/tools/LTE/scripts/parameters_daily_CI.sh b/test/tools/LTE/scripts/parameters_daily_CI.sh index c15774dc554..9192145fd6a 100644 --- a/test/tools/LTE/scripts/parameters_daily_CI.sh +++ b/test/tools/LTE/scripts/parameters_daily_CI.sh @@ -7,8 +7,30 @@ DataDir="/tmp/fabric/test/tools/LTE/data" NumChains=10 NumParallelTxPerChain=10 -NumKVs=1000000 -NumTotalTx=1000000 +NumKVs=100000 +NumTotalTx=100000 NumKeysInEachTx=4 BatchSize=50 KVSize=200 + +# Each test consists of several test-runs, where one single parameter is varied +# between the test-runs and rest of the parameters remain same. Each array below +# holds the values of that parameter for each corresponding test. For example, +# the "ArrayNumChains" will be used in the test titled "varyNumChains", and the +# 8 test-runs that exist for testing the varying number of chains will use these +# values: +# NumChains = (1 5 10 20 50 100 500 2000) +# NumParallelTxPerChain=10 +# NumKVs=100000 +# NumTotalTx=100000 +# NumKeysInEachTx=4 +# BatchSize=50 +# KVSize=200 +ArrayNumParallelTxPerChain=(1 5 10 20 50 100 500 2000) +ArrayNumChains=(1 5 10 20 50 100 500 2000) +ArrayNumKeysInEachTx=(1 2 5 10 20) +ArrayKVSize=(100 200 500 1000 2000) +ArrayBatchSize=(10 20 100 500) +ArrayNumParallelTxWithSingleChain=(1 5 10 20 50 100 500 2000) +ArrayNumChainsWithNoParallelism=(1 5 10 20 50 100 500 2000) +ArrayNumTxs=(100000 200000 500000 1000000) diff --git a/test/tools/LTE/scripts/runbenchmarks.sh b/test/tools/LTE/scripts/runbenchmarks.sh index ea569a98561..f3d5103a662 100755 --- a/test/tools/LTE/scripts/runbenchmarks.sh +++ b/test/tools/LTE/scripts/runbenchmarks.sh @@ -6,43 +6,46 @@ # source ./benchmarks.sh -source ./parameters_daily_CI.sh ######################################################################################################## # This shell script contains a series of benchmark tests -# The test parameters are imported from "parameters_daily_CI.sh" ######################################################################################################## function varyNumParallelTxPerChain { - for v in 1 5 10 20 50 100 500 2000; do + for v in "${ArrayNumParallelTxPerChain[@]}" + do NumParallelTxPerChain=$v rm -rf $DataDir;runInsertTxs;runReadWriteTxs done } function varyNumChains { - for v in 1 5 10 20 50 100 500 2000; do + for v in "${ArrayNumChains[@]}" + do NumChains=$v rm -rf $DataDir;runInsertTxs;runReadWriteTxs done } function varyNumKeysInEachTx { - for v in 1 2 5 10 20; do + for v in "${ArrayNumKeysInEachTx[@]}" + do NumKeysInEachTx=$v rm -rf $DataDir;runInsertTxs;runReadWriteTxs done } function varyKVSize { - for v in 100 200 500 1000 2000; do + for v in "${ArrayKVSize[@]}" + do KVSize=$v rm -rf $DataDir;runInsertTxs;runReadWriteTxs done } function varyBatchSize { - for v in 10 20 100 500; do + for v in "${ArrayBatchSize[@]}" + do BatchSize=$v rm -rf $DataDir;runInsertTxs;runReadWriteTxs done @@ -50,7 +53,8 @@ function varyBatchSize { function varyNumParallelTxWithSingleChain { NumChains=1 - for v in 1 5 10 20 50 100 500 2000; do + for v in "${ArrayNumParallelTxWithSingleChain[@]}" + do NumParallelTxPerChain=$v rm -rf $DataDir;runInsertTxs;runReadWriteTxs done @@ -58,14 +62,16 @@ function varyNumParallelTxWithSingleChain { function varyNumChainsWithNoParallelism { NumParallelTxPerChain=1 - for v in 1 5 10 20 50 100 500 2000; do + for v in "${ArrayNumChainsWithNoParallelism[@]}" + do NumChains=$v rm -rf $DataDir;runInsertTxs;runReadWriteTxs done } function varyNumTxs { - for v in 1000000 2000000 5000000 10000000; do + for v in "${ArrayNumTxs[@]}" + do NumTotalTx=$v rm -rf $DataDir;runInsertTxs;runReadWriteTxs done @@ -77,6 +83,42 @@ function runLargeDataExperiment { rm -rf $DataDir;runInsertTxs;runReadWriteTxs } +function usage () { + printf "Usage: ./runbenchmarks.sh [-f parameter_file_name] [test_name]\nAvailable tests (use \"all\" to run all tests): +varyNumParallelTxPerChain +varyNumChains +varyNumParallelTxWithSingleChain +varyNumChainsWithNoParallelism +varyNumKeysInEachTx +varyKVSize +varyBatchSize +varyNumTxs +runLargeDataExperiment\n" +} + +PARAM_FILE="" + +while getopts ":f:" opt; do + case $opt in + f) + printf "Parameter file: $OPTARG" + PARAM_FILE=$OPTARG;; + \?) + printf "Error: invalid parameter -$OPTARG!" >> /dev/stderr + usage + exit 1;; + esac +done + +if [ ! $PARAM_FILE ] +then + printf "Error: No Parameter file given!" >> /dev/stderr + usage + exit 1 +else + source $PARAM_FILE +fi + shift $(expr $OPTIND - 1 ) case $1 in @@ -99,16 +141,7 @@ case $1 in runLargeDataExperiment) runLargeDataExperiment ;; help) - printf "Usage: ./runbenchmarks.sh [test_name]\nAvailable tests (use \"all\" to run all tests): -varyNumParallelTxPerChain -varyNumChains -varyNumParallelTxWithSingleChain -varyNumChainsWithNoParallelism -varyNumKeysInEachTx -varyKVSize -varyBatchSize -varyNumTxs -runLargeDataExperiment\n" + usage ;; all) varyNumParallelTxPerChain @@ -122,5 +155,6 @@ runLargeDataExperiment\n" runLargeDataExperiment ;; *) printf "Error: test name empty/incorrect!\n" >> /dev/stderr + usage exit 1 ;; esac