-
Notifications
You must be signed in to change notification settings - Fork 70
/
bench.py
116 lines (92 loc) · 4.1 KB
/
bench.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
#! /usr/bin/env python
"""\
%prog [options] <pdf>
Helper script to compare the perfomance of code changes for a PDF.
Can be run in two modes:
-stash: Compare stats of current changes and stashes them to build
a baseline.
-commits: Compare stats of two commit ids. Specify id's with -c and -b
options.
"""
import sys
import os
import shutil
from subprocess import call
from optparse import OptionParser
class TestOptions(OptionParser):
def __init__(self, **kwargs):
OptionParser.__init__(self, **kwargs)
self.add_option("-m", "--mode", dest="mode", type="string",
help="stash|commits [%default]", default="stash")
self.add_option("-b", "--baseline", dest="baseline", type="string",
help="Baseline commit id for commit mode.", default=None)
self.add_option("-c", "--current", dest="current", type="string",
help="Current commit id for commit mode.", default=None)
self.add_option("-l", "--last_page", dest="last_page", type="int",
help="Last page to run. [%default]", default=1)
self.add_option("-r", "--rounds", dest="rounds", type="int",
help="Rounds to run. [%default]", default=20)
self.add_option("-t", "--test_path", dest="test_path", type="string",
help="Path to the test directory. [%default]", default=".")
self.set_usage(__doc__)
def verify_options(self, options, args):
if len(args) < 1:
self.print_help()
exit(-1)
if not os.path.isfile(args[0]):
self.error("test file not found '" + args[0] + "'")
args[0] = os.path.abspath(args[0])
os.chdir(options.test_path)
if not os.path.exists('test.py'):
self.error("test.py not found in directory '" + os.getcwd() + "'")
if options.mode == "stash":
if options.baseline is not None or options.current is not None:
self.error("Invalid options for stash mode.")
elif options.mode == "commits":
if options.baseline is None or options.current is None:
self.error("Baseline and Current options required for commits mode.")
else:
self.error("Unrecognized mode.")
return options
def create_dir(dir):
try:
os.makedirs(dir)
except OSError, e:
if e.errno != 17: # file exists
print >>sys.stderr, 'Creating', dir, 'failed!'
def call_die(command):
if call(command, shell=True) != 0:
print >>sys.stderr, 'Running command failed!'
sys.exit(code);
def main():
option_parser = TestOptions()
options, args = option_parser.parse_args()
options = option_parser.verify_options(options, args)
parts = args[0].split('/')
pdf_name = parts[-1]
current_path = os.getcwd() + '/';
path = current_path + 'stats/results/' + pdf_name + '/'
print 'Creating dir: ' + path
create_dir(path)
shutil.copy(args[0], path)
manifest_path = path + 'manifest.json'
f = open(manifest_path, 'w')
f.write('[{ "id": "' + pdf_name + '", "file": "/test/stats/results/' + pdf_name + '/' + pdf_name + '", "md5": "09a41b9a759d60c698228224ab85b46d", "rounds": ' + str(options.rounds) + ', "lastPage": ' + str(options.last_page) + ',"type": "load"}]')
f.close()
# Kick off testing
run_test_py = 'python test.py --browserManifestFile=resources/browser_manifests/browser_manifest.json --manifestFile=stats/results/' + pdf_name + '/manifest.json --statsFile=' + path
if (options.mode == "commits"):
call_die('git checkout ' + options.current)
call_die(run_test_py + 'current.json')
if (options.mode == "commits"):
call_die('git checkout ' + options.baseline)
elif (options.mode == "stash"):
call_die('git stash save')
call_die(run_test_py + 'baseline.json')
if (options.mode == "stash"):
call_die('git stash pop')
call_die('python stats/statcmp.py ' + path + 'baseline.json ' + path + 'current.json')
# Cleanup
os.remove(path + pdf_name);
if __name__ == '__main__':
main()