1#!/usr/bin/python
2import hashlib
3import optparse
4import os
5import re
6import shlex
7import subprocess
8import sys
9import threading
10import time
11
12TASK_COMPILATION = 'compile'
13TASK_DISABLE_OVERLAYS = 'disable overlays'
14TASK_ENABLE_MULTIPLE_OVERLAYS = 'enable multiple overlays'
15TASK_ENABLE_SINGLE_OVERLAY = 'enable single overlay'
16TASK_ENABLE_FILTERED_OVERLAYS = 'enable filtered overlays'
17TASK_FILE_EXISTS_TEST = 'test (file exists)'
18TASK_GREP_IDMAP_TEST = 'test (grep idmap)'
19TASK_MD5_TEST = 'test (md5)'
20TASK_IDMAP_PATH = 'idmap --path'
21TASK_IDMAP_SCAN = 'idmap --scan'
22TASK_INSTRUMENTATION = 'instrumentation'
23TASK_INSTRUMENTATION_TEST = 'test (instrumentation)'
24TASK_MKDIR = 'mkdir'
25TASK_PUSH = 'push'
26TASK_ROOT = 'root'
27TASK_REMOUNT = 'remount'
28TASK_RM = 'rm'
29TASK_SETPROP = 'setprop'
30TASK_SETUP_IDMAP_PATH = 'setup idmap --path'
31TASK_SETUP_IDMAP_SCAN = 'setup idmap --scan'
32TASK_START = 'start'
33TASK_STOP = 'stop'
34
35adb = 'adb'
36
37def _adb_shell(cmd):
38    argv = shlex.split(adb + " shell '" + cmd + "; echo $?'")
39    proc = subprocess.Popen(argv, bufsize=1, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
40    (stdout, stderr) = proc.communicate()
41    (stdout, stderr) = (stdout.replace('\r', ''), stderr.replace('\r', ''))
42    tmp = stdout.rsplit('\n', 2)
43    if len(tmp) == 2:
44        stdout == ''
45        returncode = int(tmp[0])
46    else:
47        stdout = tmp[0] + '\n'
48        returncode = int(tmp[1])
49    return returncode, stdout, stderr
50
51class VerbosePrinter:
52    class Ticker(threading.Thread):
53        def _print(self):
54            s = '\r' + self.text + '[' + '.' * self.i + ' ' * (4 - self.i) + ']'
55            sys.stdout.write(s)
56            sys.stdout.flush()
57            self.i = (self.i + 1) % 5
58
59        def __init__(self, cond_var, text):
60            threading.Thread.__init__(self)
61            self.text = text
62            self.setDaemon(True)
63            self.cond_var = cond_var
64            self.running = False
65            self.i = 0
66            self._print()
67            self.running = True
68
69        def run(self):
70            self.cond_var.acquire()
71            while True:
72                self.cond_var.wait(0.25)
73                running = self.running
74                if not running:
75                    break
76                self._print()
77            self.cond_var.release()
78
79        def stop(self):
80            self.cond_var.acquire()
81            self.running = False
82            self.cond_var.notify_all()
83            self.cond_var.release()
84
85    def _start_ticker(self):
86        self.ticker = VerbosePrinter.Ticker(self.cond_var, self.text)
87        self.ticker.start()
88
89    def _stop_ticker(self):
90        self.ticker.stop()
91        self.ticker.join()
92        self.ticker = None
93
94    def _format_begin(self, type, name):
95        N = self.width - len(type) - len(' [    ] ')
96        fmt = '%%s %%-%ds ' % N
97        return fmt % (type, name)
98
99    def __init__(self, use_color):
100        self.cond_var = threading.Condition()
101        self.ticker = None
102        if use_color:
103            self.color_RED = '\033[1;31m'
104            self.color_red = '\033[0;31m'
105            self.color_reset = '\033[0;37m'
106        else:
107            self.color_RED = ''
108            self.color_red = ''
109            self.color_reset = ''
110
111        argv = shlex.split('stty size') # get terminal width
112        proc = subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
113        (stdout, stderr) = proc.communicate()
114        if proc.returncode == 0:
115            (h, w) = stdout.split()
116            self.width = int(w)
117        else:
118            self.width = 72 # conservative guesstimate
119
120    def begin(self, type, name):
121        self.text = self._format_begin(type, name)
122        sys.stdout.write(self.text + '[    ]')
123        sys.stdout.flush()
124        self._start_ticker()
125
126    def end_pass(self, type, name):
127        self._stop_ticker()
128        sys.stdout.write('\r' + self.text + '[ OK ]\n')
129        sys.stdout.flush()
130
131    def end_fail(self, type, name, msg):
132        self._stop_ticker()
133        sys.stdout.write('\r' + self.color_RED + self.text + '[FAIL]\n')
134        sys.stdout.write(self.color_red)
135        sys.stdout.write(msg)
136        sys.stdout.write(self.color_reset)
137        sys.stdout.flush()
138
139class QuietPrinter:
140    def begin(self, type, name):
141        pass
142
143    def end_pass(self, type, name):
144        sys.stdout.write('PASS ' + type + ' ' + name + '\n')
145        sys.stdout.flush()
146
147    def end_fail(self, type, name, msg):
148        sys.stdout.write('FAIL ' + type + ' ' + name + '\n')
149        sys.stdout.flush()
150
151class CompilationTask:
152    def __init__(self, makefile):
153        self.makefile = makefile
154
155    def get_type(self):
156        return TASK_COMPILATION
157
158    def get_name(self):
159        return self.makefile
160
161    def execute(self):
162        os.putenv('ONE_SHOT_MAKEFILE', os.getcwd() + "/" + self.makefile)
163        argv = shlex.split('make -C "../../../../../" files')
164        proc = subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
165        (stdout, stderr) = proc.communicate()
166        return proc.returncode, stdout, stderr
167
168class InstrumentationTask:
169    def __init__(self, instrumentation_class):
170        self.instrumentation_class = instrumentation_class
171
172    def get_type(self):
173        return TASK_INSTRUMENTATION
174
175    def get_name(self):
176        return self.instrumentation_class
177
178    def execute(self):
179        return _adb_shell('am instrument -r -w -e class %s com.android.overlaytest/android.test.InstrumentationTestRunner' % self.instrumentation_class)
180
181class PushTask:
182    def __init__(self, src, dest):
183        self.src = src
184        self.dest = dest
185
186    def get_type(self):
187        return TASK_PUSH
188
189    def get_name(self):
190        return "%s -> %s" % (self.src, self.dest)
191
192    def execute(self):
193        src = os.getenv('OUT')
194        if (src is None):
195          return 1, "", "Unable to proceed - $OUT environment var not set\n"
196        src += "/" + self.src
197        argv = shlex.split(adb + ' push %s %s' % (src, self.dest))
198        proc = subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
199        (stdout, stderr) = proc.communicate()
200        return proc.returncode, stdout, stderr
201
202class MkdirTask:
203    def __init__(self, path):
204        self.path = path
205
206    def get_type(self):
207        return TASK_MKDIR
208
209    def get_name(self):
210        return self.path
211
212    def execute(self):
213        return _adb_shell('mkdir -p %s' % self.path)
214
215class RmTask:
216    def __init__(self, path):
217        self.path = path
218
219    def get_type(self):
220        return TASK_RM
221
222    def get_name(self):
223        return self.path
224
225    def execute(self):
226        returncode, stdout, stderr = _adb_shell('ls %s' % self.path)
227        if returncode != 0 and stderr.endswith(': No such file or directory\n'):
228            return 0, "", ""
229        return _adb_shell('rm -r %s' % self.path)
230
231class SetPropTask:
232    def __init__(self, prop, value):
233        self.prop = prop
234        self.value = value
235
236    def get_type(self):
237        return TASK_SETPROP
238
239    def get_name(self):
240        return self.prop
241
242    def execute(self):
243      return _adb_shell('setprop %s %s' % (self.prop, self.value))
244
245class IdmapPathTask:
246    def __init__(self, path_target_apk, path_overlay_apk, path_idmap):
247        self.path_target_apk = path_target_apk
248        self.path_overlay_apk = path_overlay_apk
249        self.path_idmap = path_idmap
250
251    def get_type(self):
252        return TASK_IDMAP_PATH
253
254    def get_name(self):
255        return self.path_idmap
256
257    def execute(self):
258        return _adb_shell('su system idmap --scan "%s" "%s" "%s" "%s"' % (self.target_pkg_name, self.target_pkg, self.idmap_dir, self.overlay_dir))
259
260class IdmapScanTask:
261    def __init__(self, overlay_dir, target_pkg_name, target_pkg, idmap_dir, symlink_dir):
262        self.overlay_dir = overlay_dir
263        self.target_pkg_name = target_pkg_name
264        self.target_pkg = target_pkg
265        self.idmap_dir = idmap_dir
266        self.symlink_dir = symlink_dir
267
268    def get_type(self):
269        return TASK_IDMAP_SCAN
270
271    def get_name(self):
272        return self.target_pkg_name
273
274    def execute(self):
275        return _adb_shell('su system idmap --scan "%s" "%s" "%s" "%s"' % (self.overlay_dir, self.target_pkg_name, self.target_pkg, self.idmap_dir))
276
277class FileExistsTest:
278    def __init__(self, path):
279        self.path = path
280
281    def get_type(self):
282        return TASK_FILE_EXISTS_TEST
283
284    def get_name(self):
285        return self.path
286
287    def execute(self):
288        return _adb_shell('ls %s' % self.path)
289
290class GrepIdmapTest:
291    def __init__(self, path_idmap, pattern, expected_n):
292        self.path_idmap = path_idmap
293        self.pattern = pattern
294        self.expected_n = expected_n
295
296    def get_type(self):
297        return TASK_GREP_IDMAP_TEST
298
299    def get_name(self):
300        return self.pattern
301
302    def execute(self):
303        returncode, stdout, stderr = _adb_shell('idmap --inspect %s' % self.path_idmap)
304        if returncode != 0:
305            return returncode, stdout, stderr
306        all_matches = re.findall('\s' + self.pattern + '$', stdout, flags=re.MULTILINE)
307        if len(all_matches) != self.expected_n:
308            return 1, 'pattern=%s idmap=%s expected=%d found=%d\n' % (self.pattern, self.path_idmap, self.expected_n, len(all_matches)), ''
309        return 0, "", ""
310
311class Md5Test:
312    def __init__(self, path, expected_content):
313        self.path = path
314        self.expected_md5 = hashlib.md5(expected_content).hexdigest()
315
316    def get_type(self):
317        return TASK_MD5_TEST
318
319    def get_name(self):
320        return self.path
321
322    def execute(self):
323        returncode, stdout, stderr = _adb_shell('md5sum %s' % self.path)
324        if returncode != 0:
325            return returncode, stdout, stderr
326        actual_md5 = stdout.split()[0]
327        if actual_md5 != self.expected_md5:
328            return 1, 'expected %s, got %s\n' % (self.expected_md5, actual_md5), ''
329        return 0, "", ""
330
331class StartTask:
332    def get_type(self):
333        return TASK_START
334
335    def get_name(self):
336        return ""
337
338    def execute(self):
339        (returncode, stdout, stderr) = _adb_shell('start')
340        if returncode != 0:
341            return returncode, stdout, stderr
342
343        while True:
344            (returncode, stdout, stderr) = _adb_shell('getprop dev.bootcomplete')
345            if returncode != 0:
346                return returncode, stdout, stderr
347            if stdout.strip() == "1":
348                break
349            time.sleep(0.5)
350
351        return 0, "", ""
352
353class StopTask:
354    def get_type(self):
355        return TASK_STOP
356
357    def get_name(self):
358        return ""
359
360    def execute(self):
361        (returncode, stdout, stderr) = _adb_shell('stop')
362        if returncode != 0:
363            return returncode, stdout, stderr
364        return _adb_shell('setprop dev.bootcomplete 0')
365
366class RootTask:
367    def get_type(self):
368        return TASK_ROOT
369
370    def get_name(self):
371        return ""
372
373    def execute(self):
374        (returncode, stdout, stderr) = _adb_shell('getprop service.adb.root 0')
375        if returncode != 0:
376            return returncode, stdout, stderr
377        if stdout.strip() == '1': # already root
378            return 0, "", ""
379
380        argv = shlex.split(adb + ' root')
381        proc = subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
382        (stdout, stderr) = proc.communicate()
383        if proc.returncode != 0:
384            return proc.returncode, stdout, stderr
385
386        argv = shlex.split(adb + ' wait-for-device')
387        proc = subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
388        (stdout, stderr) = proc.communicate()
389        return proc.returncode, stdout, stderr
390
391class RemountTask:
392    def get_type(self):
393        return TASK_REMOUNT
394
395    def get_name(self):
396        return ""
397
398    def execute(self):
399        argv = shlex.split(adb + ' remount')
400        proc = subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
401        (stdout, stderr) = proc.communicate()
402        # adb remount returns 0 even if the operation failed, so check stdout
403        if stdout.startswith('remount failed:'):
404            return 1, stdout, stderr
405        return proc.returncode, stdout, stderr
406
407class CompoundTask:
408    def __init__(self, type, tasks):
409        self.type = type
410        self.tasks = tasks
411
412    def get_type(self):
413        return self.type
414
415    def get_name(self):
416        return ""
417
418    def execute(self):
419        for t in self.tasks:
420            (returncode, stdout, stderr) = t.execute()
421            if returncode != 0:
422                return returncode, stdout, stderr
423        return 0, "", ""
424
425def _create_disable_overlays_task():
426    tasks = [
427        RmTask("/vendor/overlay/framework_a.apk"),
428        RmTask("/vendor/overlay/framework_b.apk"),
429        RmTask("/data/resource-cache/vendor@overlay@framework_a.apk@idmap"),
430        RmTask("/data/resource-cache/vendor@overlay@framework_b.apk@idmap"),
431        RmTask("/vendor/overlay/app_a.apk"),
432        RmTask("/vendor/overlay/app_b.apk"),
433        RmTask("/vendor/overlay/app_c.apk"),
434        RmTask("/data/resource-cache/vendor@overlay@app_a.apk@idmap"),
435        RmTask("/data/resource-cache/vendor@overlay@app_b.apk@idmap"),
436        RmTask("/data/resource-cache/vendor@overlay@app_c.apk@idmap"),
437        SetPropTask('persist.oem.overlay.test', '""'),
438        RmTask("/data/property/persist.oem.overlay.test"),
439    ]
440    return CompoundTask(TASK_DISABLE_OVERLAYS, tasks)
441
442def _create_enable_single_overlay_task():
443    tasks = [
444        _create_disable_overlays_task(),
445        MkdirTask('/system/vendor'),
446        MkdirTask('/vendor/overlay'),
447        PushTask('/data/app/com.android.overlaytest.overlay/com.android.overlaytest.overlay.apk', '/vendor/overlay/framework_a.apk'),
448        PushTask('/data/app/com.android.overlaytest.first_app_overlay/com.android.overlaytest.first_app_overlay.apk', '/vendor/overlay/app_a.apk'),
449    ]
450    return CompoundTask(TASK_ENABLE_SINGLE_OVERLAY, tasks)
451
452def _create_enable_multiple_overlays_task():
453    tasks = [
454        _create_disable_overlays_task(),
455        MkdirTask('/system/vendor'),
456        MkdirTask('/vendor/overlay'),
457
458        PushTask('/data/app/com.android.overlaytest.overlay/com.android.overlaytest.overlay.apk', '/vendor/overlay/framework_b.apk'),
459        PushTask('/data/app/com.android.overlaytest.first_app_overlay/com.android.overlaytest.first_app_overlay.apk', '/vendor/overlay/app_a.apk'),
460        PushTask('/data/app/com.android.overlaytest.second_app_overlay/com.android.overlaytest.second_app_overlay.apk', '/vendor/overlay/app_b.apk'),
461        PushTask('/data/app/com.android.overlaytest.filtered_app_overlay/com.android.overlaytest.filtered_app_overlay.apk', '/vendor/overlay/app_c.apk'),
462    ]
463    return CompoundTask(TASK_ENABLE_MULTIPLE_OVERLAYS, tasks)
464
465def _create_enable_filtered_overlays_task():
466      tasks = [
467        _create_disable_overlays_task(),
468        SetPropTask('persist.oem.overlay.test', 'foo'),
469        MkdirTask('/system/vendor'),
470        MkdirTask('/vendor/overlay'),
471        PushTask('/data/app/com.android.overlaytest.overlay/com.android.overlaytest.overlay.apk', '/vendor/overlay/framework_b.apk'),
472        PushTask('/data/app/com.android.overlaytest.first_app_overlay/com.android.overlaytest.first_app_overlay.apk', '/vendor/overlay/app_a.apk'),
473        PushTask('/data/app/com.android.overlaytest.second_app_overlay/com.android.overlaytest.second_app_overlay.apk', '/vendor/overlay/app_b.apk'),
474        PushTask('/data/app/com.android.overlaytest.filtered_app_overlay/com.android.overlaytest.filtered_app_overlay.apk', '/vendor/overlay/app_c.apk'),
475      ]
476      return CompoundTask(TASK_ENABLE_FILTERED_OVERLAYS, tasks)
477
478def _create_setup_idmap_path_task(idmaps, symlinks):
479    tasks = [
480        _create_enable_single_overlay_task(),
481        RmTask(symlinks),
482        RmTask(idmaps),
483        MkdirTask(idmaps),
484        MkdirTask(symlinks),
485    ]
486    return CompoundTask(TASK_SETUP_IDMAP_PATH, tasks)
487
488def _create_setup_idmap_scan_task(idmaps, symlinks):
489    tasks = [
490        _create_enable_filtered_overlays_task(),
491        RmTask(symlinks),
492        RmTask(idmaps),
493        MkdirTask(idmaps),
494        MkdirTask(symlinks),
495    ]
496    return CompoundTask(TASK_SETUP_IDMAP_SCAN, tasks)
497
498def _handle_instrumentation_task_output(stdout, printer):
499    regex_status_code = re.compile(r'^INSTRUMENTATION_STATUS_CODE: -?(\d+)')
500    regex_name = re.compile(r'^INSTRUMENTATION_STATUS: test=(.*)')
501    regex_begin_stack = re.compile(r'^INSTRUMENTATION_STATUS: stack=(.*)')
502    regex_end_stack = re.compile(r'^$')
503
504    failed_tests = 0
505    current_test = None
506    current_stack = []
507    mode_stack = False
508    for line in stdout.split("\n"):
509        line = line.rstrip() # strip \r from adb output
510        m = regex_status_code.match(line)
511        if m:
512            c = int(m.group(1))
513            if c == 1:
514                printer.begin(TASK_INSTRUMENTATION_TEST, current_test)
515            elif c == 0:
516                printer.end_pass(TASK_INSTRUMENTATION_TEST, current_test)
517            else:
518                failed_tests += 1
519                current_stack.append("\n")
520                msg = "\n".join(current_stack)
521                printer.end_fail(TASK_INSTRUMENTATION_TEST, current_test, msg.rstrip() + '\n')
522            continue
523
524        m = regex_name.match(line)
525        if m:
526            current_test = m.group(1)
527            continue
528
529        m = regex_begin_stack.match(line)
530        if m:
531            mode_stack = True
532            current_stack = []
533            current_stack.append("  " + m.group(1))
534            continue
535
536        m = regex_end_stack.match(line)
537        if m:
538            mode_stack = False
539            continue
540
541        if mode_stack:
542            current_stack.append("    " + line.strip())
543
544    return failed_tests
545
546def _set_adb_device(option, opt, value, parser):
547    global adb
548    if opt == '-d' or opt == '--device':
549        adb = 'adb -d'
550    if opt == '-e' or opt == '--emulator':
551        adb = 'adb -e'
552    if opt == '-s' or opt == '--serial':
553        adb = 'adb -s ' + value
554
555def _create_opt_parser():
556    parser = optparse.OptionParser()
557    parser.add_option('-d', '--device', action='callback', callback=_set_adb_device,
558            help='pass -d to adb')
559    parser.add_option('-e', '--emulator', action='callback', callback=_set_adb_device,
560            help='pass -e to adb')
561    parser.add_option('-s', '--serial', type="str", action='callback', callback=_set_adb_device,
562            help='pass -s <serical> to adb')
563    parser.add_option('-C', '--no-color', action='store_false',
564            dest='use_color', default=True,
565            help='disable color escape sequences in output')
566    parser.add_option('-q', '--quiet', action='store_true',
567            dest='quiet_mode', default=False,
568            help='quiet mode, output only results')
569    parser.add_option('-b', '--no-build', action='store_false',
570            dest='do_build', default=True,
571            help='do not rebuild test projects')
572    parser.add_option('-k', '--continue', action='store_true',
573            dest='do_continue', default=False,
574            help='do not rebuild test projects')
575    parser.add_option('-i', '--test-idmap', action='store_true',
576            dest='test_idmap', default=False,
577            help='run tests for idmap')
578    parser.add_option('-0', '--test-no-overlay', action='store_true',
579            dest='test_no_overlay', default=False,
580            help='run tests without any overlay')
581    parser.add_option('-1', '--test-single-overlay', action='store_true',
582            dest='test_single_overlay', default=False,
583            help='run tests for single overlay')
584    parser.add_option('-2', '--test-multiple-overlays', action='store_true',
585            dest='test_multiple_overlays', default=False,
586            help='run tests for multiple overlays')
587    parser.add_option('-3', '--test-filtered-overlays', action='store_true',
588            dest='test_filtered_overlays', default=False,
589            help='run tests for filtered (sys prop) overlays')
590    return parser
591
592if __name__ == '__main__':
593    opt_parser = _create_opt_parser()
594    opts, args = opt_parser.parse_args(sys.argv[1:])
595    if not opts.test_idmap and not opts.test_no_overlay and not opts.test_single_overlay and not opts.test_multiple_overlays and not opts.test_filtered_overlays:
596        opts.test_idmap = True
597        opts.test_no_overlay = True
598        opts.test_single_overlay = True
599        opts.test_multiple_overlays = True
600        opts.test_filtered_overlays = True
601
602    if len(args) > 0:
603        opt_parser.error("unexpected arguments: %s" % " ".join(args))
604        # will never reach this: opt_parser.error will call sys.exit
605
606    if opts.quiet_mode:
607        printer = QuietPrinter()
608    else:
609        printer = VerbosePrinter(opts.use_color)
610    tasks = []
611
612    # must be in the same directory as this script for compilation tasks to work
613    script = sys.argv[0]
614    dirname = os.path.dirname(script)
615    wd = os.path.realpath(dirname)
616    os.chdir(wd)
617
618    # build test cases
619    if opts.do_build:
620        tasks.append(CompilationTask('OverlayTest/Android.mk'))
621        tasks.append(CompilationTask('OverlayTestOverlay/Android.mk'))
622        tasks.append(CompilationTask('OverlayAppFirst/Android.mk'))
623        tasks.append(CompilationTask('OverlayAppSecond/Android.mk'))
624        tasks.append(CompilationTask('OverlayAppFiltered/Android.mk'))
625
626    # remount filesystem, install test project
627    tasks.append(RootTask())
628    tasks.append(RemountTask())
629    tasks.append(PushTask('/system/app/OverlayTest/OverlayTest.apk', '/system/app/OverlayTest.apk'))
630
631    # test idmap
632    if opts.test_idmap:
633        idmaps='/data/local/tmp/idmaps'
634        symlinks='/data/local/tmp/symlinks'
635
636        # idmap --path
637        tasks.append(StopTask())
638        tasks.append(_create_setup_idmap_path_task(idmaps, symlinks))
639        tasks.append(StartTask())
640        tasks.append(IdmapPathTask('/vendor/overlay/framework_a.apk', '/system/framework/framework-res.apk', idmaps + '/a.idmap'))
641        tasks.append(FileExistsTest(idmaps + '/a.idmap'))
642        tasks.append(GrepIdmapTest(idmaps + '/a.idmap', 'bool/config_annoy_dianne', 1))
643
644        # idmap --scan
645        tasks.append(StopTask())
646        tasks.append(_create_setup_idmap_scan_task(idmaps, symlinks))
647        tasks.append(StartTask())
648        tasks.append(IdmapScanTask('/vendor/overlay', 'android', '/system/framework/framework-res.apk', idmaps, symlinks))
649        tasks.append(FileExistsTest(idmaps + '/vendor@overlay@framework_b.apk@idmap'))
650        tasks.append(GrepIdmapTest(idmaps + '/vendor@overlay@framework_b.apk@idmap', 'bool/config_annoy_dianne', 1))
651
652
653        # overlays.list
654        overlays_list_path = idmaps + '/overlays.list'
655        expected_content = '''\
656/vendor/overlay/framework_b.apk /data/local/tmp/idmaps/vendor@overlay@framework_b.apk@idmap
657'''
658        tasks.append(FileExistsTest(overlays_list_path))
659        tasks.append(Md5Test(overlays_list_path, expected_content))
660
661        # idmap cleanup
662        tasks.append(RmTask(symlinks))
663        tasks.append(RmTask(idmaps))
664
665    # test no overlay: all overlays cleared
666    if opts.test_no_overlay:
667        tasks.append(StopTask())
668        tasks.append(_create_disable_overlays_task())
669        tasks.append(StartTask())
670        tasks.append(InstrumentationTask('com.android.overlaytest.WithoutOverlayTest'))
671
672    # test single overlay: one overlay (a)
673    if opts.test_single_overlay:
674        tasks.append(StopTask())
675        tasks.append(_create_enable_single_overlay_task())
676        tasks.append(StartTask())
677        tasks.append(InstrumentationTask('com.android.overlaytest.WithOverlayTest'))
678
679    # test multiple overlays: all overlays - including 'disabled' filtered
680    # overlay (system property unset) so expect 'b[p=2]' overrides 'a[p=1]' but
681    # 'c[p=3]' should be ignored
682    if opts.test_multiple_overlays:
683        tasks.append(StopTask())
684        tasks.append(_create_enable_multiple_overlays_task())
685        tasks.append(StartTask())
686        tasks.append(InstrumentationTask('com.android.overlaytest.WithMultipleOverlaysTest'))
687
688    # test filtered overlays: all overlays - including 'enabled' filtered
689    # overlay (system property set/matched) so expect c[p=3] to override both a
690    # & b where applicable
691    if opts.test_filtered_overlays:
692        tasks.append(StopTask())
693        tasks.append(_create_enable_filtered_overlays_task())
694        tasks.append(StartTask())
695        tasks.append(InstrumentationTask('com.android.overlaytest.WithFilteredOverlaysTest'))
696
697    ignored_errors = 0
698    for t in tasks:
699        type = t.get_type()
700        name = t.get_name()
701        if type == TASK_INSTRUMENTATION:
702            # InstrumentationTask will run several tests, but we want it
703            # to appear as if each test was run individually. Calling
704            # "am instrument" with a single test method is prohibitively
705            # expensive, so let's instead post-process the output to
706            # emulate individual calls.
707            retcode, stdout, stderr = t.execute()
708            if retcode != 0:
709                printer.begin(TASK_INSTRUMENTATION, name)
710                printer.end_fail(TASK_INSTRUMENTATION, name, stderr)
711                sys.exit(retcode)
712            retcode = _handle_instrumentation_task_output(stdout, printer)
713            if retcode != 0:
714                if not opts.do_continue:
715                    sys.exit(retcode)
716                else:
717                    ignored_errors += retcode
718        else:
719            printer.begin(type, name)
720            retcode, stdout, stderr = t.execute()
721            if retcode == 0:
722                printer.end_pass(type, name)
723            if retcode != 0:
724                if len(stderr) == 0:
725                    # hope for output from stdout instead (true for eg adb shell rm)
726                    stderr = stdout
727                printer.end_fail(type, name, stderr)
728                if not opts.do_continue:
729                    sys.exit(retcode)
730                else:
731                    ignored_errors += retcode
732    sys.exit(ignored_errors)
733