Skip to content

Commit

Permalink
Make sure all swarming test command use the same ADB
Browse files Browse the repository at this point in the history 
Make sure all commands used in swarming test scripts use the same ADB
command. This can help avoid flakes due to AGI detecting a different
ADB than the one that comes by default on swarming bots path.

Bug: b/183624452
Test: manual, on all swarming scripts
  • Loading branch information
@hevrard
hevrard committed Apr 1, 2021
1 parent 7eb7cc0 commit b76c39b
Show file tree
Hide file tree
Showing 5 changed files with 163 additions and 124 deletions.
43 changes: 24 additions & 19 deletions test/swarming/bot-harness.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import subprocess
import sys
import time
from shutil import which

# Load our own botutil
sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'bot-scripts'))
Expand All @@ -48,6 +49,10 @@ def main():
assert os.path.isdir('agi')
agi_dir = os.path.abspath('agi')

#### Create BotUtil object with adb path
adb_path = os.path.abspath(which('adb'))
bu = botutil.BotUtil(adb_path)

#### Print AGI build properties (AGI version, build commit SHA)
cmd = ['cat', os.path.join(agi_dir, 'build.properties')]
botutil.runcmd(cmd)
Expand All @@ -72,42 +77,42 @@ def main():

#### Check Android device access
# This first adb command may take a while if the adb deamon has to launch
p = botutil.adb(['shell', 'true'], timeout=10)
p = bu.adb(['shell', 'true'], timeout=10)
if p.returncode != 0:
print('Error: zero or more than one device connected')
return 1
# Print device fingerprint
p = botutil.adb(['shell', 'getprop', 'ro.build.fingerprint'])
p = bu.adb(['shell', 'getprop', 'ro.build.fingerprint'])
print('Device fingerprint: ' + p.stdout)

#### Prepare device
# Wake up (224) and unlock (82) screen, sleep to pass any kind of animation
# The screen wakeup (224) call sometimes takes more than a second to return,
# hence the extended timeout.
botutil.adb(['shell', 'input', 'keyevent', '224'], timeout=2)
bu.adb(['shell', 'input', 'keyevent', '224'], timeout=2)
time.sleep(2)
# TODO(b/157444640): Temporary workaround: touch the screen before unlocking it to bypass a possible "Android preview" notification
botutil.adb(['shell', 'input', 'touchscreen', 'tap', '100', '100'])
bu.adb(['shell', 'input', 'touchscreen', 'tap', '100', '100'])
time.sleep(1)
botutil.adb(['shell', 'input', 'keyevent', '82'])
bu.adb(['shell', 'input', 'keyevent', '82'])
time.sleep(1)
# Turn brightness to a minimum, to prevent device to get too hot
botutil.adb(['shell', 'settings', 'put', 'system', 'screen_brightness', '0'])
bu.adb(['shell', 'settings', 'put', 'system', 'screen_brightness', '0'])
# Make sure to have the screen "stay awake" during the test, we turn off the screen ourselves at the end
botutil.adb(['shell', 'settings', 'put', 'global', 'stay_on_while_plugged_in', '7'])
bu.adb(['shell', 'settings', 'put', 'global', 'stay_on_while_plugged_in', '7'])
# Avoid "Viewing full screen" notifications that makes app loose focus
botutil.adb(['shell', 'settings', 'put', 'secure', 'immersive_mode_confirmations', 'confirmed'])
bu.adb(['shell', 'settings', 'put', 'secure', 'immersive_mode_confirmations', 'confirmed'])
# Remove any implicit vulkan layers
botutil.adb(['shell', 'settings', 'delete', 'global', 'enable_gpu_debug_layers'])
botutil.adb(['shell', 'settings', 'delete', 'global', 'gpu_debug_app'])
botutil.adb(['shell', 'settings', 'delete', 'global', 'gpu_debug_layers'])
botutil.adb(['shell', 'settings', 'delete', 'global', 'gpu_debug_layer_app'])
bu.adb(['shell', 'settings', 'delete', 'global', 'enable_gpu_debug_layers'])
bu.adb(['shell', 'settings', 'delete', 'global', 'gpu_debug_app'])
bu.adb(['shell', 'settings', 'delete', 'global', 'gpu_debug_layers'])
bu.adb(['shell', 'settings', 'delete', 'global', 'gpu_debug_layer_app'])
# Clean up logcat, can take a few seconds
botutil.adb(['logcat', '-c'], timeout=5)
bu.adb(['logcat', '-c'], timeout=5)

#### Launch test script
print('Start test script "{}" with timeout of {} seconds'.format(test_script, test_timeout))
cmd = [test_script, agi_dir, out_dir]
cmd = [test_script, adb_path, agi_dir, out_dir]
test_returncode = None
stdout_filename = os.path.abspath(os.path.join(out_dir, 'stdout.txt'))
stderr_filename = os.path.abspath(os.path.join(out_dir, 'stderr.txt'))
Expand All @@ -123,7 +128,7 @@ def main():
#### Dump the logcat
logcat_file = os.path.join(out_dir, 'logcat.txt')
with open(logcat_file, 'w') as f:
cmd = ['adb', 'logcat', '-d']
cmd = [adb_path, 'logcat', '-d']
p = subprocess.run(cmd, timeout=5, check=True, stdout=f)

#### Dump test outputs
Expand All @@ -141,16 +146,16 @@ def main():
# have the screen on with key "wake up" (224), then press "power" (26).
# The screen wakeup (224) call sometimes takes more than a second to return,
# hence the extended timeout.
botutil.adb(['shell', 'input', 'keyevent', '224'], timeout=2)
bu.adb(['shell', 'input', 'keyevent', '224'], timeout=2)
# Wait a bit to let any kind of device wake up animation terminate
time.sleep(2)
botutil.adb(['shell', 'input', 'keyevent', '26'])
bu.adb(['shell', 'input', 'keyevent', '26'])

#### Force-stop AGI and test app
for abi in ['armeabiv7a', 'arm64v8a']:
botutil.adb(['shell', 'am', 'force-stop', 'com.google.android.gapid.' + abi])
bu.adb(['shell', 'am', 'force-stop', 'com.google.android.gapid.' + abi])
if 'package' in test_params.keys():
botutil.adb(['shell', 'am', 'force-stop', test_params['package']])
bu.adb(['shell', 'am', 'force-stop', test_params['package']])

#### Test may fail halfway through, salvage any gfxtrace
gfxtraces = glob.glob(os.path.join(test_dir, '*.gfxtrace'))
Expand Down
144 changes: 83 additions & 61 deletions test/swarming/bot-scripts/botutil.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import subprocess
import sys
import time
from shutil import which

def log(msg):
'''Log the message, making sure to force flushing to stdout'''
Expand All @@ -34,13 +35,6 @@ def runcmd(cmd):
return subprocess.run(cmd, stdout=sys.stdout, stderr=sys.stderr)


def adb(args, timeout=1):
'''Log and run an ADB command, returning a subprocess.CompletedProcess with output captured'''
cmd = ['adb'] + args
print('ADB command: ' + ' '.join(cmd), flush=True)
return subprocess.run(cmd, timeout=timeout, check=True, capture_output=True, text=True)


def load_params(test_params, params_file='params.json', required_keys=[]):
'''Load the JSON params_file into test_params.
Expand All @@ -56,60 +50,6 @@ def load_params(test_params, params_file='params.json', required_keys=[]):
test_params[k] = j[k]


def is_package_installed(package):
'''Check if package is installed on the device.'''
line_to_match = 'package:' + package
cmd = ['adb', 'shell', 'pm', 'list', 'packages']
with tempfile.TemporaryFile(mode='w+') as tmp:
subprocess.run(cmd, timeout=2, check=True, stdout=tmp)
tmp.seek(0)
for line in tmp.readlines():
line = line.rstrip()
if line == line_to_match:
return True
return False


def install_apk(test_params):
'''Install the test APK
test_params is a dict where:
{
"apk": "foobar.apk", # APK file
"package": "com.example.foobar", # Package name
"force_install": true|false, # (Optional): force APK installation,
# even if the package is already found
# on the device
"install_flags": ["-g", "-t"], # (Opriotnal) list of flags to pass
# to adb install
...
}
'''
force = False
if 'force_install' in test_params.keys():
force = test_params['force_install']
# -g: grant all needed permissions, -t: accept test APK
install_flags = ['-g', '-t']
if 'install_flags' in test_params.keys():
install_flags = test_params['install_flags']
if force and is_package_installed(test_params['package']):
cmd = ['adb', 'uninstall', test_params['package']]
log('Force install, start by uninstalling: ' + ' '.join(cmd))
subprocess.run(cmd, timeout=20, check=True, stdout=sys.stdout, stderr=sys.stderr)
if force or not is_package_installed(test_params['package']):
cmd = ['adb', 'install']
cmd += install_flags
cmd += [test_params['apk']]
log('Install APK with command: ' + ' '.join(cmd))
# Installing big APKs can take more than a minute, but get also get
# stuck, so give a big timeout to this command.
subprocess.run(cmd, timeout=120, check=True, stdout=sys.stdout, stderr=sys.stderr)
# Sleep a bit, as the app may not be listed right after install
time.sleep(1)
else:
log('Skip install of {} because package {} is already installed.'.format(test_params['apk'], test_params['package']))


def is_valid_json(filename):
'''Return true if filename contains valid JSON, false otherwise'''
with open(filename, 'r') as f:
Expand All @@ -119,3 +59,85 @@ def is_valid_json(filename):
log('Invalid JSON: {}'.format(err))
return False
return True


class BotUtil:
'''Various utilities that rely on ADB. Since using different ADB commands
can lead to loosing device connection, this class takes a path to ADB in its
constructor, and makes sure to use this ADB across all commands.'''

def __init__(self, adb_path):
assert(os.path.isfile(adb_path))
self.adb_path = adb_path
self.gapit_path = ''

def adb(self, args, timeout=1):
'''Log and run an ADB command, r_patheturning a subprocess.CompletedProcess with output captured'''
cmd = [self.adb_path] + args
print('ADB command: ' + ' '.join(cmd), flush=True)
return subprocess.run(cmd, timeout=timeout, check=True, capture_output=True, text=True)

def set_gapit_path(self, gapit_path):
'''Set path to gapit, must be called once before gapit() can be used.'''
self.gapit_path = gapit_path

def gapit(self, verb, args, stdout=sys.stdout, stderr=sys.stderr):
'''Build and run gapit command. Requires gapit path to be set.'''
assert(self.gapit_path != '')
cmd = [self.gapit_path, verb]
cmd += ['-gapis-args=-adb ' + self.adb_path]
cmd += args
print('GAPIT command: ' + ' '.join(cmd), flush=True)
return subprocess.run(cmd, stdout=stdout, stderr=stderr)

def is_package_installed(self, package):
'''Check if package is installed on the device.'''
line_to_match = 'package:' + package
cmd = [self.adb_path, 'shell', 'pm', 'list', 'packages']
with tempfile.TemporaryFile(mode='w+') as tmp:
subprocess.run(cmd, timeout=2, check=True, stdout=tmp)
tmp.seek(0)
for line in tmp.readlines():
line = line.rstrip()
if line == line_to_match:
return True
return False

def install_apk(self, test_params):
'''Install the test APK
test_params is a dict where:
{
"apk": "foobar.apk", # APK file
"package": "com.example.foobar", # Package name
"force_install": true|false, # (Optional): force APK installation,
# even if the package is already found
# on the device
"install_flags": ["-g", "-t"], # (Opriotnal) list of flags to pass
# to adb install
...
}
'''
force = False
if 'force_install' in test_params.keys():
force = test_params['force_install']
# -g: grant all needed permissions, -t: accept test APK
install_flags = ['-g', '-t']
if 'install_flags' in test_params.keys():
install_flags = test_params['install_flags']
if force and self.is_package_installed(test_params['package']):
cmd = [self.adb_path, 'uninstall', test_params['package']]
log('Force install, start by uninstalling: ' + ' '.join(cmd))
subprocess.run(cmd, timeout=20, check=True, stdout=sys.stdout, stderr=sys.stderr)
if force or not self.is_package_installed(test_params['package']):
cmd = [self.adb_path, 'install']
cmd += install_flags
cmd += [test_params['apk']]
log('Install APK with command: ' + ' '.join(cmd))
# Installing big APKs can take more than a minute, but get also get
# stuck, so give a big timeout to this command.
subprocess.run(cmd, timeout=120, check=True, stdout=sys.stdout, stderr=sys.stderr)
# Sleep a bit, as the app may not be listed right after install
time.sleep(1)
else:
log('Skip install of {} because package {} is already installed.'.format(test_params['apk'], test_params['package']))
20 changes: 13 additions & 7 deletions test/swarming/bot-scripts/frame_profile.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,15 +25,23 @@

def main():
parser = argparse.ArgumentParser()
parser.add_argument('adb_path', help='Path to adb command')
parser.add_argument('agi_dir', help='Path to AGI build')
parser.add_argument('out_dir', help='Path to output directory')
args = parser.parse_args()

#### Early checks and sanitization
assert os.path.isfile(args.adb_path)
adb_path = os.path.abspath(args.adb_path)
assert os.path.isdir(args.agi_dir)
agi_dir = os.path.normpath(args.agi_dir)
agi_dir = os.path.abspath(args.agi_dir)
assert os.path.isdir(args.out_dir)
out_dir = os.path.normpath(args.out_dir)
out_dir = os.path.abspath(args.out_dir)
gapit_path = os.path.join(agi_dir, 'gapit')

#### Create BotUtil with relevant adb and gapit paths
bu = botutil.BotUtil(adb_path)
bu.set_gapit_path(gapit_path)

#### Test parameters
test_params = {}
Expand All @@ -42,16 +50,14 @@ def main():
assert os.path.isfile(test_params['gfxtrace'])

#### Profile
gapit = os.path.join(agi_dir, 'gapit')
cmd = [
gapit, 'profile',
gapit_args = [
'-gapir-os', 'android',
'-gapir-nofallback',
test_params['gfxtrace']
]
print(cmd)

with open(os.path.join(out_dir, 'profile.stdout'), 'w') as f:
p = subprocess.run(cmd, stdout=f, stderr=sys.stderr)
p = bu.gapit('profile', gapit_args, stdout=f)
return p.returncode


Expand Down
Loading

0 comments on commit b76c39b

Please sign in to comment.