1#!/usr/bin/env python 2 3# Copyright (C) 2016 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17'''This script will run one specific test.''' 18from __future__ import print_function, absolute_import 19 20import os 21import sys 22import atexit 23import inspect 24import logging 25import argparse 26import warnings 27 28import harness 29from harness import util_constants 30from harness import util_log 31from harness import util_warnings 32from harness.util_functions import load_py_module 33from harness.util_lldb import UtilLLDB 34from harness.exception import DisconnectedException 35from harness.exception import TestSuiteException, TestIgnoredException 36from harness.util_timer import Timer 37 38 39class TestState(object): 40 '''Simple mutable mapping (like namedtuple)''' 41 def __init__(self, **kwargs): 42 for key, val in kwargs.items(): 43 setattr(self, key, val) 44 45 46def _test_pre_run(state): 47 '''This function is called before a test is executed (setup). 48 49 Args: 50 state: Test suite state collection, instance of TestState. 51 52 Returns: 53 True if the pre_run step completed without error. Currently the pre-run 54 will launch the target test binary on the device and attach an 55 lldb-server to it in platform mode. 56 57 Raises: 58 AssertionError: If an assertion fails. 59 TestSuiteException: Previous processes of this apk required for this 60 test could not be killed. 61 ''' 62 assert state.test 63 assert state.bundle 64 65 log = util_log.get_logger() 66 log.info('running: {0}'.format(state.name)) 67 68 # Remove any cached NDK scripts between tests 69 state.bundle.delete_ndk_cache() 70 71 # query our test case for the remote target app it needs 72 # First try the legacy behaviour 73 try: 74 target_name = state.test.get_bundle_target() 75 warnings.warn("get_bundle_target() is deprecated and will be removed soon" 76 " - use the `bundle_target` dictionary attribute instead") 77 except AttributeError: 78 try: 79 target_name = state.test.bundle_target[state.bundle_type] 80 except KeyError: 81 raise TestIgnoredException() 82 83 if target_name is None: 84 # test case doesn't require a remote process to debug 85 return True 86 else: 87 # find the pid of our remote test process 88 state.pid = state.bundle.launch(target_name) 89 if not state.pid: 90 log.error('unable to get pid of target') 91 return False 92 state.android.kill_servers() 93 # spawn lldb platform on the target device 94 state.android.launch_lldb_platform(state.device_port) 95 return True 96 97 98def _test_post_run(state): 99 '''This function is called after a test is executed (cleanup). 100 101 Args: 102 state: Test suite state collection, instance of TestState. 103 104 Raises: 105 AssertionError: If an assertion fails. 106 ''' 107 assert state.test 108 assert state.bundle 109 110 try: 111 target_name = state.test.get_bundle_target() 112 warnings.warn("get_bundle_target() is deprecated and will be removed soon" 113 " - use the `bundle_target` dictionary attribute instead") 114 except AttributeError: 115 try: 116 target_name = state.test.bundle_target[state.bundle_type] 117 except KeyError: 118 raise TestIgnoredException() 119 120 121 if target_name: 122 if state.bundle.is_apk(target_name): 123 state.android.stop_app(state.bundle.get_package(target_name)) 124 else: 125 state.android.kill_process(target_name) 126 127 128def _test_run(state): 129 '''Execute a single test suite. 130 131 Args: 132 state: test suite state collection, instance of TestState. 133 134 Returns: 135 True: if the test case ran successfully and passed. 136 False: if the test case failed or suffered an error. 137 138 Raises: 139 AssertionError: If an assertion fails. 140 ''' 141 assert state.lldb 142 assert state.lldb_module 143 assert state.test 144 145 test_failures = state.test.run(state.lldb, state.pid, state.lldb_module) 146 147 if test_failures: 148 log = util_log.get_logger() 149 for test, err in test_failures: 150 log.error('test %s:%s failed: %r' % (state.name, test, err)) 151 152 return False 153 154 return True 155 156 157def _initialise_timer(android, interval): 158 '''Start a 'timeout' timer, to catch stalled execution. 159 160 This function will start a timer that will act as a timeout killing this 161 test session if a test becomes un-responsive. 162 163 Args: 164 android: current instance of harness.UtilAndroid 165 interval: the interval for the timeout, in seconds 166 167 Returns: 168 The instance of the Timer class that was created. 169 ''' 170 171 def on_timeout(): 172 '''This is a callback function that will fire if a test takes longer 173 then a threshold time to complete.''' 174 # Clean up the android properties 175 android.reset_all_props() 176 # pylint: disable=protected-access 177 sys.stdout.flush() 178 # hard exit to force kill all threads that may block our exit 179 os._exit(util_constants.RC_TEST_TIMEOUT) 180 181 timer = Timer(interval, on_timeout) 182 timer.start() 183 atexit.register(Timer.stop, timer) 184 return timer 185 186 187def _quit_test(num, timer): 188 '''This function will exit making sure the timeout thread is killed. 189 190 Args: 191 num: An integer specifying the exit status, 0 meaning "successful 192 termination". 193 timer: The current Timer instance. 194 ''' 195 if timer: 196 timer.stop() 197 sys.stdout.flush() 198 sys.exit(num) 199 200 201def _execute_test(state): 202 '''Execute a test suite. 203 204 Args: 205 state: The current TestState object. 206 ''' 207 log = util_log.get_logger() 208 209 state.test.setup(state.android) 210 try: 211 if not _test_pre_run(state): 212 raise TestSuiteException('test_pre_run() failed') 213 if not _test_run(state): 214 raise TestSuiteException('test_run() failed') 215 _test_post_run(state) 216 log.info('Test passed') 217 218 finally: 219 state.test.post_run() 220 state.test.teardown(state.android) 221 222 223def _get_test_case_class(module): 224 '''Inspect a test case module and return the test case class. 225 226 Args: 227 module: A loaded test case module. 228 ''' 229 # We consider only subclasses of TestCase that have `test_` methods` 230 log = util_log.get_logger() 231 log.debug("loading test suites from %r", module) 232 for name, klass in inspect.getmembers(module, inspect.isclass): 233 for attr in dir(klass): 234 if attr.startswith('test_'): 235 log.info("Found test class %r", name) 236 return klass 237 else: 238 log.debug("class %r has no test_ methods", name) 239 return None 240 241 242def get_test_dir(test_name): 243 ''' Get the directory that contains a test with a given name. 244 245 Returns: 246 A string that is the directory containing the test. 247 248 Raises: 249 TestSuiteException: If a test with this name does not exist. 250 ''' 251 tests_dir = os.path.dirname(os.path.realpath(__file__)) 252 for sub_dir in os.listdir(tests_dir): 253 current_test_dir = os.path.join(tests_dir, sub_dir) 254 if (os.path.isdir(current_test_dir) and 255 test_name in os.listdir(current_test_dir)): 256 return current_test_dir 257 258 raise TestSuiteException( 259 'unable to find test: {0}'.format(test_name)) 260 261 262def main(): 263 '''Test runner entry point.''' 264 265 # re-open stdout with no buffering 266 sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0) 267 268 android = None 269 timer = None 270 log = None 271 272 # parse the command line (positional arguments only) 273 truthy = lambda x: x.lower() in ('true', '1') 274 parser = argparse.ArgumentParser("Run a single RenderScript TestSuite against lldb") 275 for name, formatter in ( 276 ('test_name', str), 277 ('log_file_path', str), 278 ('adb_path', str), 279 ('lldb_server_path_device', str), 280 ('aosp_product_path', str), 281 ('device_port', int), 282 ('device', str), 283 ('print_to_stdout', truthy), 284 ('verbose', truthy), 285 ('wimpy', truthy), 286 ('timeout', int), 287 ('bundle_type', str), 288 ): 289 parser.add_argument(name, type=formatter) 290 291 args = parser.parse_args() 292 293 try: 294 # create utility classes 295 harness.util_log.initialise( 296 '%s(%s)' % (args.test_name, args.bundle_type), 297 print_to_stdout=args.print_to_stdout, 298 level=logging.INFO if not args.verbose else logging.DEBUG, 299 file_path=args.log_file_path, 300 file_mode='a' 301 ) 302 log = util_log.get_logger() 303 log.debug('Logger initialised') 304 305 android = harness.UtilAndroid(args.adb_path, 306 args.lldb_server_path_device, 307 args.device) 308 309 # start the timeout counter 310 timer = _initialise_timer(android, args.timeout) 311 312 # startup lldb and register teardown handler 313 atexit.register(UtilLLDB.stop) 314 UtilLLDB.start() 315 316 current_test_dir = get_test_dir(args.test_name) 317 318 # load a test case module 319 test_module = load_py_module(os.path.join(current_test_dir, 320 args.test_name)) 321 322 323 # inspect the test module and locate our test case class 324 test_class = _get_test_case_class(test_module) 325 326 # if our test inherits from TestBaseRemote, check we have a valid device 327 if (hasattr(test_module, "TestBaseRemote") and 328 issubclass(test_class, test_module.TestBaseRemote)): 329 android.validate_device() 330 331 # create an instance of our test case 332 test_inst = test_class( 333 args.device_port, 334 args.device, 335 timer, 336 args.bundle_type, 337 wimpy=args.wimpy 338 ) 339 340 # instantiate a test target bundle 341 bundle = harness.UtilBundle(android, args.aosp_product_path) 342 343 # execute the test case 344 try: 345 for _ in range(2): 346 try: 347 # create an lldb instance 348 lldb = UtilLLDB.create_debugger() 349 350 # create state object to encapsulate instances 351 352 state = TestState( 353 android=android, 354 bundle=bundle, 355 lldb=lldb, 356 lldb_module=UtilLLDB.get_module(), 357 test=test_inst, 358 pid=None, 359 name=args.test_name, 360 device_port=args.device_port, 361 bundle_type=args.bundle_type 362 ) 363 364 util_warnings.redirect_warnings() 365 366 _execute_test(state) 367 368 # tear down the lldb instance 369 UtilLLDB.destroy_debugger(lldb) 370 break 371 except DisconnectedException as error: 372 log.warning(error) 373 log.warning('Trying again.') 374 else: 375 log.fatal('Not trying again, maximum retries exceeded.') 376 raise TestSuiteException('Lost connection to lldb-server') 377 378 finally: 379 util_warnings.restore_warnings() 380 381 _quit_test(util_constants.RC_TEST_OK, timer) 382 383 except AssertionError: 384 if log: 385 log.critical('Internal test suite error', exc_info=1) 386 print('Internal test suite error', file=sys.stderr) 387 _quit_test(util_constants.RC_TEST_FATAL, timer) 388 389 except TestIgnoredException: 390 if log: 391 log.warn("test ignored") 392 _quit_test(util_constants.RC_TEST_IGNORED, timer) 393 394 except TestSuiteException as error: 395 if log: 396 log.exception(str(error)) 397 else: 398 print(error, file=sys.stderr) 399 _quit_test(util_constants.RC_TEST_FAIL, timer) 400 401 # use a global exception handler to be sure that we will 402 # exit safely and correctly 403 except Exception: 404 if log: 405 log.exception('INTERNAL ERROR') 406 else: 407 import traceback 408 print('Exception {0}'.format(traceback.format_exc()), 409 file=sys.stderr) 410 _quit_test(util_constants.RC_TEST_FATAL, timer) 411 412 finally: 413 if android: 414 android.reset_all_props() 415 if timer: 416 timer.stop() 417 418 419# execution trampoline 420if __name__ == '__main__': 421 print(' '.join(sys.argv)) 422 main() 423